Merge branch 'layered-txpool-tx-selection-metrics' into layered-txpoo…

…l-skip-sender-opt

# Conflicts:
#	ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java
#	ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java
#	plugin-api/build.gradle
#	plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionSelectionResult.java

.circleci/config.yml

@@ -240,7 +240,7 @@ jobs:
       - capture_test_results
       - capture_test_logs
 
-  acceptanceTestsNonMainnet:
+  acceptanceTestsCliqueBft:
     parallelism: 6
     executor: xl_machine_executor
     steps:
@@ -258,7 +258,51 @@ jobs:
               | circleci tests split --split-by=timings --timings-type=classname)
             # Format the arguments to "./gradlew test"
             GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
-            ./gradlew --no-daemon acceptanceTestNonMainnet $GRADLE_ARGS
+            ./gradlew --no-daemon acceptanceTestCliqueBft $GRADLE_ARGS
+      - capture_test_results
+      - capture_test_logs
+
+  acceptanceTestsPrivacy:
+    parallelism: 6
+    executor: xl_machine_executor
+    steps:
+      - prepare
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: AcceptanceTests (Non-Mainnet)
+          no_output_timeout: 20m
+          command: |
+            CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
+              | sed 's@.*/src/test/java/@@' \
+              | sed 's@/@.@g' \
+              | sed 's/.\{5\}$//' \
+              | circleci tests split --split-by=timings --timings-type=classname)
+            # Format the arguments to "./gradlew test"
+            GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
+            ./gradlew --no-daemon acceptanceTestPrivacy $GRADLE_ARGS
+      - capture_test_results
+      - capture_test_logs
+
+  acceptanceTestsPermissioning:
+    parallelism: 6
+    executor: xl_machine_executor
+    steps:
+      - prepare
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: AcceptanceTests (Non-Mainnet)
+          no_output_timeout: 20m
+          command: |
+            CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
+              | sed 's@.*/src/test/java/@@' \
+              | sed 's@/@.@g' \
+              | sed 's/.\{5\}$//' \
+              | circleci tests split --split-by=timings --timings-type=classname)
+            # Format the arguments to "./gradlew test"
+            GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
+            ./gradlew --no-daemon acceptanceTestPermissioning $GRADLE_ARGS
       - capture_test_results
       - capture_test_logs
 
@@ -401,6 +445,15 @@ workflows:
       - acceptanceTests:
           requires:
             - assemble
+      - acceptanceTestsCliqueBft:
+          requires:
+            - assemble
+      - acceptanceTestsPrivacy:
+          requires:
+            - assemble
+      - acceptanceTestsPermissioning:
+          requires:
+            - assemble
       - buildDocker:
           requires:
             - assemble
@@ -472,4 +525,3 @@ workflows:
     jobs:
       - assemble
       - dockerScan
-      - acceptanceTestsNonMainnet

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 23.4.2
+
+### Breaking Changes
+
+### Additions and Improvements
+
+### Bug Fixes
+
+### Download Links
+
+---
+
 ## 23.4.1
 
 ### Breaking Changes
@@ -14,7 +26,8 @@
 - New optional feature to save the txpool content to file on shutdown and reloading it on startup [#5434](https://github.com/hyperledger/besu/pull/5434)
 - New option to send SNI header in TLS ClientHello message [#5439](https://github.com/hyperledger/besu/pull/5439)
 - Early access - layered transaction pool implementation [#5290](https://github.com/hyperledger/besu/pull/5290)
-- New RPC method `debug_getRawReceipts`
+- New RPC method `debug_getRawReceipts` [#5476](https://github.com/hyperledger/besu/pull/5476)
+- Add TrieLogFactory plugin support [#5440](https://github.com/hyperledger/besu/pull/5440)
 - Ignore `min-block-occupancy-ratio` option when on PoS networks, since in some cases, it prevents to have full blocks even if enough transactions are present [#5491](https://github.com/hyperledger/besu/pull/5491) 
 
 ### Bug Fixes
@@ -23,6 +36,9 @@
 
 ### Download Links
 
+https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.4.1/besu-23.4.1.tar.gz / sha256: 49d3a7a069cae307497093d834f873ce7804a46dd59207d5e8321459532d318e
+https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.4.1/besu-23.4.1.zip / sha256: 1d82ed83a816968aa9366d9310b275ca6438100f5d3eb1ec03d3474b2a5f5e76
+
 ## 23.4.0
 
 ### Breaking Changes

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java

@@ -386,7 +386,7 @@ public String getHostName() {
     return LOCALHOST;
   }
 
-  public NodeRequests nodeRequests() {
+  private NodeRequests nodeRequests() {
     Optional<WebSocketService> websocketService = Optional.empty();
     if (nodeRequests == null) {
       final Web3jService web3jService;

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -45,9 +45,7 @@
 import org.hyperledger.besu.plugin.services.PicoCLIOptions;
 import org.hyperledger.besu.plugin.services.SecurityModuleService;
 import org.hyperledger.besu.plugin.services.StorageService;
-import org.hyperledger.besu.plugin.services.TransactionSelectionService;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin;
-import org.hyperledger.besu.plugin.services.txselection.TransactionSelectorFactory;
 import org.hyperledger.besu.services.BesuConfigurationImpl;
 import org.hyperledger.besu.services.BesuEventsImpl;
 import org.hyperledger.besu.services.BesuPluginContextImpl;
@@ -56,7 +54,6 @@
 import org.hyperledger.besu.services.RpcEndpointServiceImpl;
 import org.hyperledger.besu.services.SecurityModuleServiceImpl;
 import org.hyperledger.besu.services.StorageServiceImpl;
-import org.hyperledger.besu.services.TransactionSelectionServiceImpl;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -66,7 +63,6 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
@@ -95,22 +91,14 @@ private BesuPluginContextImpl buildPluginContext(
     besuPluginContext.addService(StorageService.class, storageService);
     besuPluginContext.addService(SecurityModuleService.class, securityModuleService);
     besuPluginContext.addService(PicoCLIOptions.class, new PicoCLIOptionsImpl(commandLine));
-    besuPluginContext.addService(
-        TransactionSelectionService.class, new TransactionSelectionServiceImpl());
-
-    final Path pluginsPath;
-    final String pluginDirEnv = System.getenv("besu.plugins.dir");
-    if (pluginDirEnv == null || pluginDirEnv.isEmpty()) {
-      pluginsPath = node.homeDirectory().resolve("plugins");
-      final File pluginsDirFile = pluginsPath.toFile();
-      if (!pluginsDirFile.isDirectory()) {
-        pluginsDirFile.mkdirs();
-        pluginsDirFile.deleteOnExit();
-      }
-    } else {
-      pluginsPath = Path.of(pluginDirEnv);
-      System.setProperty("besu.plugins.dir", pluginsPath.toString());
+
+    final Path pluginsPath = node.homeDirectory().resolve("plugins");
+    final File pluginsDirFile = pluginsPath.toFile();
+    if (!pluginsDirFile.isDirectory()) {
+      pluginsDirFile.mkdirs();
+      pluginsDirFile.deleteOnExit();
     }
+    System.setProperty("besu.plugins.dir", pluginsPath.toString());
     besuPluginContext.registerPlugins(pluginsPath);
 
     commandLine.parseArgs(node.getConfiguration().getExtraCLIOptions().toArray(new String[0]));
@@ -181,9 +169,6 @@ public void startNode(final BesuNode node) {
 
     final int maxPeers = 25;
 
-    final Optional<TransactionSelectorFactory> transactionSelectorFactory =
-        getTransactionSelectorFactory(besuPluginContext);
-
     builder
         .synchronizerConfiguration(new SynchronizerConfiguration.Builder().build())
         .dataDirectory(node.homeDirectory())
@@ -205,8 +190,7 @@ public void startNode(final BesuNode node) {
         .lowerBoundPeers(maxPeers)
         .maxRemotelyInitiatedPeers(15)
         .networkConfiguration(node.getNetworkingConfiguration())
-        .randomPeerPriority(false)
-        .transactionSelectorFactory(transactionSelectorFactory);
+        .randomPeerPriority(false);
 
     node.getGenesisConfig()
         .map(GenesisConfigFile::fromConfig)
@@ -315,11 +299,4 @@ public void startConsoleCapture() {
   public String getConsoleContents() {
     throw new RuntimeException("Console contents can only be captured in process execution");
   }
-
-  private Optional<TransactionSelectorFactory> getTransactionSelectorFactory(
-      final BesuPluginContextImpl besuPluginContext) {
-    final Optional<TransactionSelectionService> txSelectionService =
-        besuPluginContext.getService(TransactionSelectionService.class);
-    return txSelectionService.isPresent() ? txSelectionService.get().get() : Optional.empty();
-  }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -107,33 +107,10 @@ public BesuNode createMinerNode(
     return create(config);
   }
 
-  public BesuNode createMinerNodeWithExtraCliOptions(
-      final String name,
-      final UnaryOperator<BesuNodeConfigurationBuilder> configModifier,
-      final List<String> extraCliOptions)
-      throws IOException {
-    BesuNodeConfigurationBuilder builder =
-        new BesuNodeConfigurationBuilder()
-            .name(name)
-            .miningEnabled()
-            .jsonRpcEnabled()
-            .webSocketEnabled()
-            .extraCLIOptions(extraCliOptions);
-    builder = configModifier.apply(builder);
-    final BesuNodeConfiguration config = builder.build();
-
-    return create(config);
-  }
-
   public BesuNode createMinerNode(final String name) throws IOException {
     return createMinerNode(name, UnaryOperator.identity());
   }
 
-  public BesuNode createMinerNodeWithExtraCliOptions(
-      final String name, final List<String> extraCliOptions) throws IOException {
-    return createMinerNodeWithExtraCliOptions(name, UnaryOperator.identity(), extraCliOptions);
-  }
-
   public BesuNode createMinerNodeWithRevertReasonEnabled(final String name) throws IOException {
     return createMinerNode(name, BesuNodeConfigurationBuilder::revertReasonEnabled);
   }

acceptance-tests/tests/build.gradle

@@ -155,11 +155,36 @@ task acceptanceTestMainnet(type: Test) {
   doFirst { mkdir "${buildDir}/jvmErrorLogs" }
 }
 
-task acceptanceTestNonMainnet(type: Test) {
+task acceptanceTestCliqueBft(type: Test) {
   inputs.property "integration.date", LocalTime.now() // so it runs at every invocation
   include '**/bft/**'
   include '**/clique/**'
-  include '**/permissioning/**'
+
+  useJUnitPlatform {}
+
+  dependsOn(rootProject.installDist)
+  setSystemProperties(test.getSystemProperties())
+  systemProperty 'acctests.runBesuAsProcess', 'true'
+  systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
+  mustRunAfter rootProject.subprojects*.test
+  description = 'Runs Clique and BFT Besu acceptance tests.'
+  group = 'verification'
+
+  jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"
+
+  testLogging {
+    exceptionFormat = 'full'
+    showStackTraces = true
+    showStandardStreams = Boolean.getBoolean('acctests.showStandardStreams')
+    showExceptions = true
+    showCauses = true
+  }
+
+  doFirst { mkdir "${buildDir}/jvmErrorLogs" }
+}
+
+task acceptanceTestPrivacy(type: Test) {
+  inputs.property "integration.date", LocalTime.now() // so it runs at every invocation
   include '**/privacy/**'
 
   useJUnitPlatform {}
@@ -169,7 +194,34 @@ task acceptanceTestNonMainnet(type: Test) {
   systemProperty 'acctests.runBesuAsProcess', 'true'
   systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
   mustRunAfter rootProject.subprojects*.test
-  description = 'Runs NON-MAINNET Besu acceptance tests for permissioning and privacy (excluding mainnet features).'
+  description = 'Runs Privacy Besu acceptance tests.'
+  group = 'verification'
+
+  jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"
+
+  testLogging {
+    exceptionFormat = 'full'
+    showStackTraces = true
+    showStandardStreams = Boolean.getBoolean('acctests.showStandardStreams')
+    showExceptions = true
+    showCauses = true
+  }
+
+  doFirst { mkdir "${buildDir}/jvmErrorLogs" }
+}
+
+task acceptanceTestPermissioning(type: Test) {
+  inputs.property "integration.date", LocalTime.now() // so it runs at every invocation
+  include '**/permissioning/**'
+
+  useJUnitPlatform {}
+
+  dependsOn(rootProject.installDist)
+  setSystemProperties(test.getSystemProperties())
+  systemProperty 'acctests.runBesuAsProcess', 'true'
+  systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
+  mustRunAfter rootProject.subprojects*.test
+  description = 'Runs Permissioning Besu acceptance tests.'
   group = 'verification'
 
   jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineAcceptanceTest.java → acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineParisAcceptanceTest.java

@@ -24,13 +24,13 @@
 import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
-public class ExecutionEngineAcceptanceTest extends AbstractJsonRpcTest {
-  private static final String GENESIS_FILE = "/jsonrpc/engine/genesis.json";
-  private static final String TEST_CASE_PATH = "/jsonrpc/engine/test-cases/";
+public class ExecutionEngineParisAcceptanceTest extends AbstractJsonRpcTest {
+  private static final String GENESIS_FILE = "/jsonrpc/engine/paris/genesis.json";
+  private static final String TEST_CASE_PATH = "/jsonrpc/engine/paris/test-cases/";
 
   private static JsonRpcTestsContext testsContext;
 
-  public ExecutionEngineAcceptanceTest(final String ignored, final URI testCaseFileURI) {
+  public ExecutionEngineParisAcceptanceTest(final String ignored, final URI testCaseFileURI) {
     super(ignored, testsContext, testCaseFileURI);
   }
 

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineShanghaiAcceptanceTest.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright contributors to Hyperledger Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance.jsonrpc;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class ExecutionEngineShanghaiAcceptanceTest extends AbstractJsonRpcTest {
+  private static final String GENESIS_FILE = "/jsonrpc/engine/shanghai/genesis.json";
+  private static final String TEST_CASE_PATH = "/jsonrpc/engine/shanghai/test-cases/";
+
+  private static JsonRpcTestsContext testsContext;
+
+  public ExecutionEngineShanghaiAcceptanceTest(final String ignored, final URI testCaseFileURI) {
+    super(ignored, testsContext, testCaseFileURI);
+  }
+
+  @BeforeClass
+  public static void init() throws IOException {
+    testsContext = new JsonRpcTestsContext(GENESIS_FILE);
+  }
+
+  @Parameterized.Parameters(name = "{0}")
+  public static Iterable<Object[]> testCases() throws URISyntaxException {
+    return testCases(TEST_CASE_PATH);
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    testsContext.cluster.close();
+  }
+}