Bonsai archive feature

CHANGELOG.md

@@ -39,6 +39,7 @@
 - Add support for `chainId` in `CallParameters` [#7720](https://github.com/hyperledger/besu/pull/7720)
 - Add `--ephemery` network support for Ephemery Testnet [#7563](https://github.com/hyperledger/besu/pull/7563) thanks to [@gconnect](https://github.com/gconnect)
 - Add configuration of Consolidation Request Contract Address via genesis configuration [#7647](https://github.com/hyperledger/besu/pull/7647)
+- Experimental Bonsai Archive support [#7475](https://github.com/hyperledger/besu/pull/7475)
 
 ### Bug fixes
 - Fix mounted data path directory permissions for besu user [#7575](https://github.com/hyperledger/besu/pull/7575)

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -479,7 +479,9 @@ public BesuNode createQbftNode(
             .dataStorageConfiguration(
                 storageFormat == DataStorageFormat.FOREST
                     ? DataStorageConfiguration.DEFAULT_FOREST_CONFIG
-                    : DataStorageConfiguration.DEFAULT_BONSAI_CONFIG)
+                    : storageFormat == DataStorageFormat.BONSAI
+                        ? DataStorageConfiguration.DEFAULT_BONSAI_CONFIG
+                        : DataStorageConfiguration.DEFAULT_BONSAI_ARCHIVE_CONFIG)
             .genesisConfigProvider(GenesisConfigurationFactory::createQbftGenesisConfig);
     if (fixedPort) {
       builder.p2pPort(

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftAcceptanceTestParameterization.java

@@ -74,6 +74,11 @@ public BesuNode createBonsaiNodeFixedPort(BesuNodeFactory factory, String name)
     return creatorFn.create(factory, name, true, DataStorageFormat.BONSAI);
   }
 
+  public BesuNode createBonsaiArchiveNodeFixedPort(BesuNodeFactory factory, String name)
+      throws Exception {
+    return creatorFn.create(factory, name, true, DataStorageFormat.X_BONSAI_ARCHIVE);
+  }
+
   public BesuNode createForestNodeFixedPort(BesuNodeFactory factory, String name) throws Exception {
     return creatorFn.create(factory, name, true, DataStorageFormat.FOREST);
   }

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bftsoak/BftMiningSoakTest.java

@@ -32,9 +32,15 @@
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.web3j.protocol.core.DefaultBlockParameter;
+import org.web3j.tx.exceptions.ContractCallException;
 
 public class BftMiningSoakTest extends ParameterizedBftTestBase {
 
+  private static final Logger LOG = LoggerFactory.getLogger(BftMiningSoakTest.class);
+
   private final int NUM_STEPS = 5;
 
   private final int MIN_TEST_TIME_MINS = 60;
@@ -62,7 +68,7 @@ public void shouldBeStableDuringLongTest(
 
     // Create a mix of Bonsai and Forest DB nodes
     final BesuNode minerNode1 = nodeFactory.createBonsaiNodeFixedPort(besu, "miner1");
-    final BesuNode minerNode2 = nodeFactory.createForestNodeFixedPort(besu, "miner2");
+    final BesuNode minerNode2 = nodeFactory.createBonsaiArchiveNodeFixedPort(besu, "miner2");
     final BesuNode minerNode3 = nodeFactory.createBonsaiNodeFixedPort(besu, "miner3");
     final BesuNode minerNode4 = nodeFactory.createForestNodeFixedPort(besu, "miner4");
 
@@ -82,6 +88,13 @@ public void shouldBeStableDuringLongTest(
     SimpleStorage simpleStorageContract =
         minerNode1.execute(contractTransactions.createSmartContract(SimpleStorage.class));
 
+    // Create another instance of the contract referencing the same contract address but on the
+    // archive node. This contract instance should be able to query state from the beginning of
+    // the test
+    SimpleStorage simpleStorageArchive =
+        minerNode2.execute(contractTransactions.createSmartContract(SimpleStorage.class));
+    simpleStorageArchive.setContractAddress(simpleStorageContract.getContractAddress());
+
     // Check the contract address is as expected for this sender & nonce
     contractVerifier
         .validTransactionReceipt("0x42699a7612a82f1d9c36148af9c77354759b210b")
@@ -104,6 +117,9 @@ public void shouldBeStableDuringLongTest(
     // Set to something new
     simpleStorageContract.set(BigInteger.valueOf(101)).send();
 
+    // Save this block height to check on the archive node at the end of the test
+    BigInteger archiveChainHeight = minerNode1.execute(ethTransactions.blockNumber());
+
     // Check the state of the contract has updated correctly. We'll set & get this several times
     // during the test
     assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(101));
@@ -132,6 +148,7 @@ public void shouldBeStableDuringLongTest(
     // Step 2
     // Stop one of the nodes, check that the chain continues mining
     // blocks
+    LOG.info("Stopping node 4 to check the chain continues mining (albeit more slowly)");
     stopNode(minerNode4);
 
     nextStepEndTime =
@@ -151,6 +168,7 @@ public void shouldBeStableDuringLongTest(
     // Step 3
     // Stop another one of the nodes, check that the chain now stops
     // mining blocks
+    LOG.info("Stopping node 3 to check that the chain stalls");
     stopNode(minerNode3);
 
     chainHeight = minerNode1.execute(ethTransactions.blockNumber());
@@ -170,8 +188,8 @@ public void shouldBeStableDuringLongTest(
     // Step 4
     // Restart both of the stopped nodes. Check that the chain resumes
     // mining blocks
+    LOG.info("Starting node 3 and node 4 to ensure the chain resumes mining new blocks");
     startNode(minerNode3);
-
     startNode(minerNode4);
 
     previousStepEndTime = Instant.now();
@@ -203,11 +221,16 @@ public void shouldBeStableDuringLongTest(
       lastChainHeight = chainHeight;
     }
 
+    LOG.info("Updating the value in the smart contract from 101 to 201");
     // Update our smart contract before upgrading from berlin to london
     assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(101));
     simpleStorageContract.set(BigInteger.valueOf(201)).send();
     assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(201));
 
+    LOG.info(
+        "Upgrading the entire chain to the London fork one node at a time. The genesis for each node will be updated to londonBlock = "
+            + lastChainHeight.intValue()
+            + 120);
     // Upgrade the chain from berlin to london in 120 blocks time
     upgradeToLondon(
         minerNode1, minerNode2, minerNode3, minerNode4, lastChainHeight.intValue() + 120);
@@ -219,6 +242,9 @@ public void shouldBeStableDuringLongTest(
         previousStepEndTime.plus(getTestDurationMins() / NUM_STEPS, ChronoUnit.MINUTES);
     lastChainHeight = chainHeight;
 
+    // Allow the chain to restart mining blocks
+    Thread.sleep(THREE_MINUTES);
+
     while (System.currentTimeMillis() < nextStepEndTime.toEpochMilli()) {
       Thread.sleep(ONE_MINUTE);
       chainHeight = minerNode1.execute(ethTransactions.blockNumber());
@@ -229,6 +255,8 @@ public void shouldBeStableDuringLongTest(
       lastChainHeight = chainHeight;
     }
 
+    LOG.info(
+        "Chain has successfully upgraded to the London fork. Checking the contract state is correct");
     // Check that the state of our smart contract is still correct
     assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(201));
 
@@ -237,18 +265,42 @@ public void shouldBeStableDuringLongTest(
     assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(301));
 
     // Upgrade the chain to shanghai in 120 seconds. Then try to deploy a shanghai contract
+    LOG.info(
+        "Upgrading the entire chain to the Shanghai fork one node at a time. The genesis for each node will be updated to shanghaiTime = "
+            + Instant.now().getEpochSecond()
+            + 120);
     upgradeToShanghai(
         minerNode1, minerNode2, minerNode3, minerNode4, Instant.now().getEpochSecond() + 120);
 
+    // Allow the chain to restart mining blocks
     Thread.sleep(THREE_MINUTES);
 
+    LOG.info(
+        "Deploying a smart contract that should only work if the chain is running on the shanghai fork");
     SimpleStorageShanghai simpleStorageContractShanghai =
         minerNode1.execute(contractTransactions.createSmartContract(SimpleStorageShanghai.class));
 
     // Check the contract address is as expected for this sender & nonce
     contractVerifier
-        .validTransactionReceipt("0x05d91b9031a655d08e654177336d08543ac4b711")
+        .validTransactionReceipt("0xfeae27388a65ee984f452f86effed42aabd438fd")
         .verify(simpleStorageContractShanghai);
+
+    // Archive node test. Check the state of the contract when it was first updated in the test
+    LOG.info(
+        "Checking that the archive node shows us the original smart contract value if we set a historic block number");
+    simpleStorageArchive.setDefaultBlockParameter(
+        DefaultBlockParameter.valueOf(archiveChainHeight));
+    assertThat(simpleStorageArchive.get().send()).isEqualTo(BigInteger.valueOf(101));
+
+    try {
+      simpleStorageContract.setDefaultBlockParameter(
+          DefaultBlockParameter.valueOf(archiveChainHeight));
+      // Should throw ContractCallException because a non-archive not can't satisfy this request
+      simpleStorageContract.get().send();
+      Assertions.fail("Request for historic state from non-archive node should have failed");
+    } catch (ContractCallException e) {
+      // Ignore
+    }
   }
 
   private static void updateGenesisConfigToLondon(
@@ -284,21 +336,26 @@ private void upgradeToLondon(
       final int londonBlockNumber)
       throws InterruptedException {
     // Node 1
+
+    LOG.info("Upgrading node 1 to london fork");
     stopNode(minerNode1);
     updateGenesisConfigToLondon(minerNode1, true, londonBlockNumber);
     startNode(minerNode1);
 
     // Node 2
+    LOG.info("Upgrading node 2 to london fork");
     stopNode(minerNode2);
     updateGenesisConfigToLondon(minerNode2, true, londonBlockNumber);
     startNode(minerNode2);
 
     // Node 3
+    LOG.info("Upgrading node 3 to london fork");
     stopNode(minerNode3);
     updateGenesisConfigToLondon(minerNode3, true, londonBlockNumber);
     startNode(minerNode3);
 
     // Node 4
+    LOG.info("Upgrading node 4 to london fork");
     stopNode(minerNode4);
     updateGenesisConfigToLondon(minerNode4, true, londonBlockNumber);
     startNode(minerNode4);
@@ -312,21 +369,25 @@ private void upgradeToShanghai(
       final long shanghaiTime)
       throws InterruptedException {
     // Node 1
+    LOG.info("Upgrading node 1 to shanghai fork");
     stopNode(minerNode1);
     updateGenesisConfigToShanghai(minerNode1, shanghaiTime);
     startNode(minerNode1);
 
     // Node 2
+    LOG.info("Upgrading node 2 to shanghai fork");
     stopNode(minerNode2);
     updateGenesisConfigToShanghai(minerNode2, shanghaiTime);
     startNode(minerNode2);
 
     // Node 3
+    LOG.info("Upgrading node 3 to shanghai fork");
     stopNode(minerNode3);
     updateGenesisConfigToShanghai(minerNode3, shanghaiTime);
     startNode(minerNode3);
 
     // Node 4
+    LOG.info("Upgrading node 4 to shanghai fork");
     stopNode(minerNode4);
     updateGenesisConfigToShanghai(minerNode4, shanghaiTime);
     startNode(minerNode4);

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -1933,9 +1933,14 @@ private PrivacyParameters privacyParameters() {
         throw new ParameterException(
             commandLine, String.format("%s %s", "Checkpoint sync", errorSuffix));
       }
-      if (getDataStorageConfiguration().getDataStorageFormat().equals(DataStorageFormat.BONSAI)) {
+      if (getDataStorageConfiguration().getDataStorageFormat() == DataStorageFormat.BONSAI) {
         throw new ParameterException(commandLine, String.format("%s %s", "Bonsai", errorSuffix));
       }
+      if (getDataStorageConfiguration().getDataStorageFormat()
+          == DataStorageFormat.X_BONSAI_ARCHIVE) {
+        throw new ParameterException(
+            commandLine, String.format("%s %s", "Bonsai archive", errorSuffix));
+      }
 
       if (Boolean.TRUE.equals(privacyOptionGroup.isPrivacyMultiTenancyEnabled)
           && Boolean.FALSE.equals(jsonRpcConfiguration.isAuthenticationEnabled())

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RevertMetadataSubCommand.java

@@ -123,6 +123,7 @@ public void run() {
             switch (dataStorageFormat) {
               case FOREST -> 1;
               case BONSAI -> 2;
+              case X_BONSAI_ARCHIVE -> 3;
             };
 
         @JsonSerialize

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java

@@ -30,7 +30,6 @@
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
-import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -328,8 +327,8 @@ private static TrieLogContext getTrieLogContext() {
     BesuController besuController = createBesuController();
     final DataStorageConfiguration config = besuController.getDataStorageConfiguration();
     checkArgument(
-        DataStorageFormat.BONSAI.equals(config.getDataStorageFormat()),
-        "Subcommand only works with data-storage-format=BONSAI");
+        config.getDataStorageFormat().isBonsaiFormat(),
+        "Subcommand only works with data-storage-format=BONSAI or X_BONSAI_ARCHIVE");
 
     final StorageProvider storageProvider = besuController.getStorageProvider();
     final BonsaiWorldStateKeyValueStorage rootWorldStateStorage =

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -87,6 +87,7 @@
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider;
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader;
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiArchiver;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
 import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive;
@@ -762,6 +763,19 @@ public BesuController build() {
       }
     }
 
+    if (DataStorageFormat.X_BONSAI_ARCHIVE.equals(
+        dataStorageConfiguration.getDataStorageFormat())) {
+      final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
+          worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class);
+      final BonsaiArchiver archiver =
+          createBonsaiArchiver(
+              worldStateKeyValueStorage,
+              blockchain,
+              scheduler,
+              ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager());
+      blockchain.observeBlockAdded(archiver);
+    }
+
     final List<Closeable> closeables = new ArrayList<>();
     closeables.add(protocolContext.getWorldStateArchive());
     closeables.add(storageProvider);
@@ -829,6 +843,24 @@ private TrieLogPruner createTrieLogPruner(
     return trieLogPruner;
   }
 
+  private BonsaiArchiver createBonsaiArchiver(
+      final WorldStateKeyValueStorage worldStateStorage,
+      final Blockchain blockchain,
+      final EthScheduler scheduler,
+      final TrieLogManager trieLogManager) {
+    final BonsaiArchiver archiver =
+        new BonsaiArchiver(
+            (BonsaiWorldStateKeyValueStorage) worldStateStorage,
+            blockchain,
+            scheduler::executeServiceTask,
+            trieLogManager,
+            metricsSystem);
+
+    archiver.initialize();
+    LOG.info("Bonsai archiver initialised");
+    return archiver;
+  }
+
   /**
    * Create synchronizer synchronizer.
    *
@@ -1118,6 +1150,25 @@ yield new BonsaiWorldStateProvider(
             besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
             evmConfiguration);
       }
+      case X_BONSAI_ARCHIVE -> {
+        final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
+            worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class);
+
+        worldStateKeyValueStorage
+            .getFlatDbStrategy()
+            .updateBlockContext(blockchain.getChainHeadHeader());
+
+        yield new BonsaiWorldStateProvider(
+            worldStateKeyValueStorage,
+            blockchain,
+            Optional.of(
+                dataStorageConfiguration
+                    .getDiffBasedSubStorageConfiguration()
+                    .getMaxLayersToLoad()),
+            bonsaiCachedMerkleTrieLoader,
+            besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
+            evmConfiguration);
+      }
       case FOREST -> {
         final WorldStatePreimageStorage preimageStorage =
             storageProvider.createWorldStatePreimageStorage();

besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java

@@ -214,6 +214,16 @@ public void privacyWithBonsaiExplicitMustError() {
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
   }
 
+  @Test
+  public void privacyWithBonsaiArchiveExplicitMustError() {
+    // bypass overridden parseCommand method which specifies bonsai
+    super.parseCommand("--privacy-enabled", "--data-storage-format", "X_BONSAI_ARCHIVE");
+
+    assertThat(commandErrorOutput.toString(UTF_8))
+        .contains("Bonsai archive cannot be enabled with privacy.");
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+  }
+
   @Test
   public void privacyWithoutPrivacyPublicKeyFails() {
     parseCommand("--privacy-enabled", "--privacy-url", ENCLAVE_URI);