Merge pull request #2893 from blockstack/develop

Release 2.0.11.4.0

.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests

@@ -1,36 +1,7 @@
-FROM rust:bullseye
-
-WORKDIR /src/
-
-COPY . .
-
-WORKDIR /src/testnet/stacks-node
-RUN cargo test --no-run
-
-RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
-RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
-
-RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
+FROM stacks-node:integrations
 
+ARG test_name
 ENV BITCOIND_TEST 1
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::microblock_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_check_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::cost_voting_integration
-RUN cargo test -- --test-threads 1 --ignored tests::integrations::integration_test_get_info
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::liquid_ustx_integration
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::stx_transfer_btc_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_forking_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::should_fix_2771
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::pox_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::bitcoin_regtest::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::should_succeed_handling_malformed_and_valid_txs
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::antientropy_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::filter_low_fee_tx_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::filter_long_runtime_tx_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::mining_transactions_is_fair
+
+RUN cargo test -- --test-threads 1 --ignored "$test_name"
 

.github/actions/bitcoin-int-tests/Dockerfile.atlas-test → .github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests

@@ -1,17 +1,15 @@
 FROM rust:bullseye
 
-WORKDIR /src
+WORKDIR /src/
 
 COPY . .
 
-RUN cargo test --no-run --workspace
+WORKDIR /src/testnet/stacks-node
+RUN cargo test --no-run
 
 RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 
 RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
 
 ENV BITCOIND_TEST 1
-WORKDIR /src/testnet/stacks-node
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::atlas_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::atlas_stress_integration_test

.github/workflows/bitcoin-tests.yml

@@ -6,22 +6,86 @@ on:
   pull_request:
 
 jobs:
-  # Run sampled genesis tests
+  build-integration-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build bitcoin integration testing image
+        env:
+          DOCKER_BUILDKIT: 1
+        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests -t stacks-node:integrations .
+      - name: Export docker image as tarball
+        run: docker save -o integration-image.tar stacks-node:integrations
+      - name: Upload built docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: integration-image.tar
+          path: integration-image.tar
+  # Run integration tests using sampled genesis block
   sampled-genesis:
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      fail-fast: false
+      matrix:
+        test-name:
+          - tests::neon_integrations::microblock_integration_test
+          - tests::neon_integrations::size_check_integration_test
+          - tests::neon_integrations::cost_voting_integration
+          - tests::integrations::integration_test_get_info
+          - tests::neon_integrations::bitcoind_integration_test
+          - tests::neon_integrations::liquid_ustx_integration
+          - tests::neon_integrations::stx_transfer_btc_integration_test
+          - tests::neon_integrations::bitcoind_forking_test
+          - tests::neon_integrations::should_fix_2771
+          - tests::neon_integrations::pox_integration_test
+          - tests::bitcoin_regtest::bitcoind_integration_test
+          - tests::should_succeed_handling_malformed_and_valid_txs
+          - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
+          - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::antientropy_integration_test
+          - tests::neon_integrations::filter_low_fee_tx_integration_test
+          - tests::neon_integrations::filter_long_runtime_tx_integration_test
+          - tests::neon_integrations::mining_transactions_is_fair
     steps:
       - uses: actions/checkout@v2
+      - name: Download docker image
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
       - name: All integration tests with sampled genesis
+        timeout-minutes: 30
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
   atlas-test:
-    # disable this job/test for now, as the atlas endpoints are currently disabled.
     if: ${{ true }}
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      fail-fast: false
+      matrix:
+        test-name:
+          - tests::neon_integrations::atlas_integration_test
+          - tests::neon_integrations::atlas_stress_integration_test
     steps:
       - uses: actions/checkout@v2
-      - name: All integration tests with sampled genesis
+      - name: Download docker image
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
+      - name: Atlas integration tests
+        timeout-minutes: 40
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.atlas-test .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .

CHANGELOG.md

@@ -5,7 +5,28 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
-## [Not Yet Released]
+## [2.0.11.4.0]
+
+This software update is a point-release to change the transaction
+selection logic in the default miner to prioritize by an estimated fee
+rate instead of raw fee. This release's chainstate directory is
+compatible with chainstate directories from 2.0.11.3.0.
+
+## Added
+
+- FeeEstimator and CostEstimator interfaces. These can be controlled
+  via node configuration options. See the `README.md` for more
+  information on the configuration.
+- New fee rate estimation endpoint `/v2/fees/transaction` (#2872). See
+  `docs/rpc/openapi.yaml` for more information.
+
+## Changed
+
+- Prioritize transaction inclusion in blocks by estimated fee rates (#2859).
+- MARF sqlite connections will now use `mmap`'ed connections with up to 256MB
+  space (#2869).
+
+## [2.0.11.3.0]
 
 This software update is a point-release to change the transaction selection
 logic in the default miner to prioritize by fee instead of nonce sequence.  This

Cargo.toml

@@ -64,6 +64,15 @@ slog-json = { version = "2.3.0", optional = true }
 chrono = "0.4.19"
 libc = "0.2.82"
 
+[target.'cfg(unix)'.dependencies]
+nix = "0.23"
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["consoleapi", "handleapi", "synchapi", "winbase"] }
+
+[target.'cfg(windows)'.dev-dependencies]
+winapi = { version = "0.3", features = ["fileapi", "processenv", "winnt"] }
+
 [dependencies.serde_json]
 version = "1.0"
 features = ["arbitrary_precision", "unbounded_depth"]

README.md

@@ -354,6 +354,30 @@ INFO [1630127492.031042] [testnet/stacks-node/src/run_loop/neon.rs:146] [main] M
 INFO [1630127492.062652] [testnet/stacks-node/src/run_loop/neon.rs:164] [main] UTXOs found - will run as a Miner node
 ```
 
+### Configuring Cost and Fee Estimation
+
+Fee and cost estimators can be configure via the config section `[fee_estimation]`:
+
+```
+[fee_estimation]
+cost_estimator = naive_pessimistic
+fee_estimator = scalar_fee_rate
+cost_metric = proportion_dot_product
+log_error = true
+enabled = true
+```
+
+Fee and cost estimators observe transactions on the network and use the
+observed costs of those transactions to build estimates for viable fee rates
+and expected execution costs for transactions. Estimators and metrics can be
+selected using the configuration fields above, though the default values are
+the only options currently. `log_error` controls whether or not the INFO logger
+will display information about the cost estimator accuracy as new costs are
+observed. Setting `enabled = false` turns off the cost estimators. Cost estimators
+are **not** consensus-critical components, but rather can be used by miners to
+rank transactions in the mempool or client to determine appropriate fee rates
+for transactions before broadcasting them.
+
 ## Non-Consensus Breaking Release Process
 
 For non-consensus breaking releases, this project uses the following release process:

clarity/Cargo.toml

@@ -41,6 +41,15 @@ slog-json = { version = "2.3.0", optional = true }
 chrono = "0.4.19"
 libc = "0.2.82"
 
+[target.'cfg(unix)'.dependencies]
+nix = "0.23"
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["consoleapi", "handleapi", "synchapi", "winbase"] }
+
+[target.'cfg(windows)'.dev-dependencies]
+winapi = { version = "0.3", features = ["fileapi", "processenv", "winnt"] }
+
 [dependencies.serde_json]
 version = "1.0"
 features = ["arbitrary_precision", "unbounded_depth"]

docs/rpc-endpoints.md

@@ -28,6 +28,9 @@ Possible values for the "reason" field and "reason_data" field are:
 * `Deserialization`
    * The `reason_data` field will be an object containing a `message`
      string detailing the deserialization error
+* `EstimatorError`
+   * The `reason_data` field will be an object containing a `message`
+     string detailing the error
 * `SignatureValidation`
    * The `reason_data` field will be an object containing a `message`
      string detailing the signature validation error
@@ -343,4 +346,4 @@ object of the following form:
 
 Determine whether a given trait is implemented within the specified contract (either explicitly or implicitly).
 
-See OpenAPI [spec](./rpc/openapi.yaml) for details.
+See OpenAPI [spec](./rpc/openapi.yaml) for details.