Improving backwards sync (#3638)

Refactor Backwards sync to use a rocks db. Signed-off-by: Jiri Peinlich <[email protected]>
hyperledger · Apr 14, 2022 · 998ea64 · 998ea64
1 parent 6c8602d
commit 998ea64
Show file tree

Hide file tree

Showing 25 changed files with 760 additions and 1,298 deletions.
diff --git a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java
@@ -28,11 +28,12 @@
 import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager;
 import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator;
 import org.hyperledger.besu.ethereum.eth.peervalidation.RequiredBlocksPeerValidator;
+import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardChain;
 import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncContext;
-import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncLookupService;
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
+import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 
 import java.util.List;
@@ -68,9 +69,8 @@ protected MiningCoordinator createMiningCoordinator(
             metricsSystem,
             ethProtocolManager.ethContext(),
             syncState,
-            new BackwardSyncLookupService(
-                protocolSchedule, ethProtocolManager.ethContext(), metricsSystem, protocolContext),
-            storageProvider));
+            BackwardChain.from(
+                storageProvider, ScheduleBasedBlockHeaderFunctions.create(protocolSchedule))));
   }
 
   protected MiningCoordinator createTransitionMiningCoordinator(

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java
@@ -34,7 +34,6 @@
 import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
 import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncContext;
-import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncLookupService;
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
@@ -98,11 +97,6 @@ protected MiningCoordinator createMiningCoordinator(
             metricsSystem,
             ethProtocolManager.ethContext(),
             syncState,
-            new BackwardSyncLookupService(
-                transitionProtocolSchedule,
-                ethProtocolManager.ethContext(),
-                metricsSystem,
-                protocolContext),
             storageProvider);
 
     final TransitionCoordinator composedCoordinator =

diff --git a/besu/src/test/java/org/hyperledger/besu/controller/TransitionControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/TransitionControllerBuilderTest.java
@@ -30,12 +30,14 @@
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
 import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture;
 import org.hyperledger.besu.ethereum.core.Difficulty;
+import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager;
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
+import org.hyperledger.besu.ethereum.storage.StorageProvider;
 
 import java.util.Optional;
 
@@ -61,6 +63,7 @@ public class TransitionControllerBuilderTest {
   @Mock SyncState syncState;
   @Mock EthProtocolManager ethProtocolManager;
   @Mock PostMergeContext mergeContext;
+  StorageProvider storageProvider = new InMemoryKeyValueStorageProvider();
 
   @Spy CliqueBesuControllerBuilder cliqueBuilder = new CliqueBesuControllerBuilder();
   @Spy BesuControllerBuilder powBuilder = new MainnetBesuControllerBuilder();
@@ -76,6 +79,7 @@ public void setup() {
             new TransitionProtocolSchedule(
                 preMergeProtocolSchedule, postMergeProtocolSchedule, mergeContext));
     cliqueBuilder.nodeKey(NodeKeyUtils.generate());
+    postMergeBuilder.storageProvider(storageProvider);
     when(protocolContext.getBlockchain()).thenReturn(mockBlockchain);
     when(transitionProtocolSchedule.getPostMergeSchedule()).thenReturn(postMergeProtocolSchedule);
     when(transitionProtocolSchedule.getPreMergeSchedule()).thenReturn(preMergeProtocolSchedule);
@@ -152,6 +156,7 @@ public void assertPreMergeScheduleForBelowTerminalBlockWhenPostMergeIfNotFinaliz
   TransitionCoordinator buildTransitionCoordinator(
       final BesuControllerBuilder preMerge, final MergeBesuControllerBuilder postMerge) {
     var builder = new TransitionBesuControllerBuilder(preMerge, postMerge);
+    builder.storageProvider(storageProvider);
     var coordinator =
         builder.createMiningCoordinator(
             transitionProtocolSchedule,

diff --git a/...rge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionBackwardSyncContext.java b/...rge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionBackwardSyncContext.java
@@ -18,9 +18,10 @@
 import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.eth.manager.EthContext;
+import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardChain;
 import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncContext;
-import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncLookupService;
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
+import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 
@@ -34,16 +35,15 @@ public TransitionBackwardSyncContext(
       final MetricsSystem metricsSystem,
       final EthContext ethContext,
       final SyncState syncState,
-      final BackwardSyncLookupService backwardSyncLookupService,
       final StorageProvider storageProvider) {
     super(
         protocolContext,
         transitionProtocolSchedule,
         metricsSystem,
         ethContext,
         syncState,
-        backwardSyncLookupService,
-        storageProvider);
+        BackwardChain.from(
+            storageProvider, ScheduleBasedBlockHeaderFunctions.create(transitionProtocolSchedule)));
     this.transitionProtocolSchedule = transitionProtocolSchedule;
   }
 

diff --git a/...ge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/...ge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java
@@ -16,7 +16,6 @@
 
 import static org.hyperledger.besu.consensus.merge.TransitionUtils.isTerminalProofOfWorkBlock;
 import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda;
-import static org.hyperledger.besu.util.Slf4jLambdaHelper.infoLambda;
 
 import org.hyperledger.besu.consensus.merge.MergeContext;
 import org.hyperledger.besu.datatypes.Address;
@@ -215,7 +214,7 @@ public Optional<BlockHeader> getOrSyncHeaderByHash(final Hash blockhash) {
     if (optHeader.isPresent()) {
       debugLambda(LOG, "BlockHeader {} is already present", () -> optHeader.get().toLogString());
     } else {
-      infoLambda(LOG, "appending block hash {} to backward sync", blockhash::toHexString);
+      debugLambda(LOG, "appending block hash {} to backward sync", blockhash::toHexString);
       backwardSyncContext.syncBackwardsUntil(blockhash);
     }
     return optHeader;

diff --git a/...c/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java b/...c/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java
@@ -32,7 +32,8 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
   GOQUORUM_PRIVATE_WORLD_STATE(new byte[] {11}),
   GOQUORUM_PRIVATE_STORAGE(new byte[] {12}),
   BACKWARD_SYNC_HEADERS(new byte[] {13}),
-  BACKWARD_SYNC_BLOCKS(new byte[] {14});
+  BACKWARD_SYNC_BLOCKS(new byte[] {14}),
+  BACKWARD_SYNC_CHAIN(new byte[] {15});
 
   private final byte[] id;
   private final int[] versionList;