merge branch master

.circleci/config.yml

@@ -1,11 +1,11 @@
 version: 2.1
 orbs:
   slack: circleci/[email protected]
-  win: circleci/windows@2.4.1
+  win: circleci/windows@5.0.0
 executors:
   small_executor:
     docker:
-      - image: cimg/openjdk:11.0.13
+      - image: cimg/openjdk:17.0.7
         auth: &docker-auth
           # Don't panic, throw away account to avoid Docker rate limits when downloading.
           # Second reason we're doing this is so that forked PRs from external contributors works ie env vars aren't visible to forked PRs from within contexts
@@ -27,7 +27,7 @@ executors:
 
   medium_executor:
     docker:
-      - image: cimg/openjdk:11.0.13
+      - image: cimg/openjdk:17.0.7
         auth:
           <<: *docker-auth
     resource_class: medium
@@ -39,7 +39,7 @@ executors:
 
   medium_plus_executor:
     docker:
-      - image: cimg/openjdk:11.0.13
+      - image: cimg/openjdk:17.0.7
         auth:
           <<: *docker-auth
     resource_class: "medium+"
@@ -50,7 +50,7 @@ executors:
 
   large_executor:
     docker:
-      - image: cimg/openjdk:11.0.13
+      - image: cimg/openjdk:17.0.7
         auth:
           <<: *docker-auth
     resource_class: large
@@ -61,7 +61,7 @@ executors:
 
   machine_executor_amd64:
     machine:
-      image: ubuntu-2004:202201-02 # Ubuntu 20.04, Docker v20.10.12, Docker Compose v1.29.2
+      image: ubuntu-2204:2023.04.2 # https://circleci.com/developer/machine/image/ubuntu-2204
       docker_layer_caching: true
     working_directory: ~/project
     environment:
@@ -70,15 +70,15 @@ executors:
 
   machine_executor_arm64:
     machine:
-      image: ubuntu-2004:202201-02 # Ubuntu 20.04, Docker v20.10.12, Docker Compose v1.29.2
+      image: ubuntu-2204:2023.04.2 # https://circleci.com/developer/machine/image/ubuntu-2204
     resource_class: arm.medium
     environment:
       architecture: "arm64"
       platform: "linux/arm64"
 
   node_executor:
     docker:
-      - image: circleci/node:14-buster
+      - image: circleci/node:16-bullseye
         auth:
           <<: *docker-auth
 
@@ -139,7 +139,7 @@ commands:
             mkdir -p $HOME/.docker/trust/private
             echo $DCT_KEY | base64 --decode > $HOME/.docker/trust/private/$DCT_HASH.key
             chmod 600 $HOME/.docker/trust/private/$DCT_HASH.key
-            docker trust key load $HOME/.docker/trust/private/$DCT_HASH.key --name opsquorum
+            docker trust key load $HOME/.docker/trust/private/$DCT_HASH.key --name ecosystem
 
   docker_publish_images:
     description: "Upload the docker images"

.github/workflows/codeql.yml

@@ -40,6 +40,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2
@@ -70,4 +76,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v2

.openapidoc/publish.js

@@ -50,7 +50,9 @@ async function main() {
  * @param {string} dirPath
  */
 function prepareDistDir(dirPath) {
-  fs.rmdirSync(dirPath, {recursive: true});
+  if (fs.existsSync(dirPath)) {
+    fs.rmdirSync(dirPath, {recursive: true});
+  }
   fs.mkdirSync(dirPath, {recursive: true});
 }
 

CHANGELOG.md

@@ -2,8 +2,6 @@
 
 ## Upcoming Breaking Changes
 
-- Upgrading the minimum Java version to Java 17, which means users will need to upgrade their Java installation to at least `Java 17`.  The docker versions relying on `jdk16` will no longer be published, so docker users explicitly referencing the `jdk16` build need to update their package to reference `jdk17`, as tags `develop-jdk16`, `develop-jdk16-arm64`, `latest-jdk16` will no longer be updated.
-
 ## Current Releases
 
 For information on changes in released versions of Teku, see the [releases page](https://github.com/ConsenSys/teku/releases).
@@ -12,14 +10,10 @@ For information on changes in released versions of Teku, see the [releases page]
 
 ### Breaking Changes
 
-- `--exchange-capabilities-enabled` option has been removed since it is no longer applicable because of [execution-apis](https://github.com/ethereum/execution-apis/pull/418) spec change.
-- Removed kiln and ropsten as `--network` options
+- Upgrading the minimum Java version to Java 17, which means users will need to upgrade their Java installation to at least `Java 17`.  The docker versions relying on `jdk16` will no longer be published, so docker users explicitly referencing the `jdk16` build need to update their package to reference `jdk17`, as tags `develop-jdk16`, `develop-jdk16-arm64`, `latest-jdk16` will no longer be updated.
 
 ### Additions and Improvements
 
-- `--validators-external-signer-public-keys` parameter now accepts `external-signer` value. It will enable public key retrieval from external signer standard API, making sure that configured keystore and trustStore will be used, if any.
-- Stopped calling `engine_exchangeTransitionConfigurationV1` Engine API method, since the method will be deprecated in the future.
+Introduce `--exchange-capabilities-monitoring-enabled` parameter. If enabled, EL will be queried periodically for the Engine API methods it supports. If incompatibility is detected, there will be a warning raised in the logs. The default is `true`.
 
 ### Bug Fixes
-
-- Fix Get Attestation Rewards API to fetch beacon state instead of block and beacon state (fixes #7338)

README.md

@@ -50,9 +50,9 @@ Release notifications are available via:
 
 ### Install Prerequisites
 
-* Java 11+
+* Java 17+
 
-Note: Official builds of Teku are performed with Java 11.
+Note: Official builds of Teku are performed with Java 17.
 Building on a more recent version of Java is supported, but the resulting build will not work on earlier versions of Java.
 
 

acceptance-tests/build.gradle

@@ -23,7 +23,6 @@ dependencies {
   testFixturesImplementation project(':infrastructure:crypto')
   testFixturesImplementation project(':infrastructure:time')
   testFixturesImplementation project(':infrastructure:bls-keystore')
-  testFixturesImplementation 'com.google.code.gson:gson'
   testFixturesImplementation 'com.squareup.okhttp3:okhttp'
   testFixturesImplementation 'io.libp2p:jvm-libp2p-minimal'
   testFixturesImplementation 'org.apache.commons:commons-lang3'

beacon/pow/build.gradle

@@ -16,8 +16,7 @@ dependencies {
   implementation project(':storage:api')
 
   api 'org.web3j:core'
-
-  implementation 'com.google.code.gson:gson'
+
   implementation 'org.apache.tuweni:tuweni-units'
 
   testImplementation testFixtures(project(':infrastructure:async'))

build.gradle

@@ -130,8 +130,8 @@ allprojects {
     jar.preserveFileTimestamps = false
   }
 
-  sourceCompatibility = '11'
-  targetCompatibility = '11'
+  sourceCompatibility = '17'
+  targetCompatibility = '17'
 
   repositories {
     mavenLocal()
@@ -494,7 +494,7 @@ tasks.register("dockerDistUntar") {
 }
 
 def dockerImage = "consensys/teku"
-def dockerJdkVariants = [ "jdk17", "jdk16", ]
+def dockerJdkVariants = [ "jdk17" ]
 def dockerBuildDir = "build/docker-teku/"
 
 task distDocker  {
@@ -773,6 +773,12 @@ subprojects {
     acceptanceTestRuntimeOnly.extendsFrom testRuntimeOnly
     propertyTestRuntimeOnly.extendsFrom testRuntimeOnly
     referenceTestRuntimeOnly.extendsFrom testRuntimeOnly
+
+    // Details at https://github.com/Hakky54/log-captor#using-log-captor-alongside-with-other-logging-libraries
+    testImplementation {
+      exclude(group: "org.apache.logging.log4j", module: "log4j-slf4j-impl")
+      exclude(group: "org.apache.logging.log4j", module: "log4j-slf4j2-impl")
+    }
   }
 
   def jarName = project.name

data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetStateByBlockRoot.java

@@ -69,7 +69,7 @@ public void handleRequest(RestApiRequest request) throws JsonProcessingException
     request.header(Header.CACHE_CONTROL, CACHE_NONE);
 
     final String blockId = request.getPathParameter(PARAMETER_BLOCK_ID);
-    SafeFuture<Optional<BeaconState>> future = chainDataProvider.getBeaconStateByBlockRoot(blockId);
+    SafeFuture<Optional<BeaconState>> future = chainDataProvider.getBeaconStateByBlockId(blockId);
 
     request.respondAsync(
         future.thenApply(

data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecars.java

@@ -38,8 +38,6 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
-import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
-import tech.pegasys.teku.spec.datastructures.metadata.ObjectAndMetaData;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionCache;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
 
@@ -76,25 +74,14 @@ private static EndpointMetadata createEndpointMetadata(final SchemaDefinitionCac
 
   @Override
   public void handleRequest(RestApiRequest request) throws JsonProcessingException {
-    final SafeFuture<Optional<ObjectAndMetaData<SignedBeaconBlock>>> blockFuture =
-        chainDataProvider.getBlock(request.getPathParameter(PARAMETER_BLOCK_ID));
     final List<UInt64> indices = request.getQueryParameterList(BLOB_INDICES_PARAMETER);
-    final SafeFuture<Optional<List<BlobSidecar>>> resultFuture =
-        blockFuture.thenCompose(
-            maybeSignedBlock ->
-                maybeSignedBlock
-                    .map(
-                        signedBlock ->
-                            chainDataProvider
-                                .getBlobSidecars(
-                                    signedBlock.getData().getSlotAndBlockRoot(), indices)
-                                .thenApply(Optional::of))
-                    .orElse(SafeFuture.completedFuture(Optional.empty())));
+    final SafeFuture<Optional<List<BlobSidecar>>> future =
+        chainDataProvider.getBlobSidecars(request.getPathParameter(PARAMETER_BLOCK_ID), indices);
 
     request.respondAsync(
-        resultFuture.thenApply(
-            blobSidecarList ->
-                blobSidecarList
+        future.thenApply(
+            blobSidecars ->
+                blobSidecars
                     .map(AsyncApiResponse::respondOk)
                     .orElse(AsyncApiResponse.respondNotFound())));
   }
@@ -113,9 +100,7 @@ private static SerializableTypeDefinition<List<BlobSidecar>> getResponseType(
 
   private static ResponseContentTypeDefinition<List<BlobSidecar>> getSszResponseType() {
     final OctetStreamResponseContentTypeDefinition.OctetStreamSerializer<List<BlobSidecar>>
-        serializer =
-            (data, out) ->
-                data.stream().forEachOrdered(blobSidecar -> blobSidecar.sszSerialize(out));
+        serializer = (data, out) -> data.forEach(blobSidecar -> blobSidecar.sszSerialize(out));
 
     return new OctetStreamResponseContentTypeDefinition<>(serializer, __ -> Collections.emptyMap());
   }

data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetStateRandao.java

@@ -85,10 +85,9 @@ private static EndpointMetadata createMetadata() {
 
   @Override
   public void handleRequest(final RestApiRequest request) throws JsonProcessingException {
-    final String stateIdParam = request.getPathParameter(PARAMETER_STATE_ID);
     final Optional<UInt64> epoch = request.getOptionalQueryParameter(EPOCH_PARAMETER);
     final SafeFuture<Optional<ObjectAndMetaData<Optional<Bytes32>>>> future =
-        chainDataProvider.getRandaoAtEpoch(stateIdParam, epoch);
+        chainDataProvider.getRandaoAtEpoch(request.getPathParameter(PARAMETER_STATE_ID), epoch);
 
     request.respondAsync(
         future.thenApply(

data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetStateByBlockRootTest.java

@@ -42,10 +42,10 @@ void setup() {
   }
 
   @Test
-  public void shouldReturnStateByBlockRoot() throws JsonProcessingException {
+  public void shouldReturnStateByBlockId() throws JsonProcessingException {
     final BeaconState state = dataStructureUtil.randomBeaconState();
     request.setPathParameter(PARAM_BLOCK_ID, "head");
-    when(chainDataProvider.getBeaconStateByBlockRoot(eq("head")))
+    when(chainDataProvider.getBeaconStateByBlockId(eq("head")))
         .thenReturn(SafeFuture.completedFuture(Optional.of(state)));
 
     handler.handleRequest(request);

data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecarsTest.java

@@ -53,7 +53,7 @@ void shouldReturnBlobSidecars()
     final ObjectAndMetaData<SignedBeaconBlock> blockAndMetaData =
         chainDataProvider.getBlock("head").get().orElseThrow();
     final List<BlobSidecar> blobSidecars =
-        chainDataProvider
+        combinedChainDataClient
             .getBlobSidecars(
                 blockAndMetaData.getData().getSlotAndBlockRoot(), Collections.emptyList())
             .get();

data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetStateValidatorsTest.java

@@ -147,7 +147,7 @@ public void shouldGetBadRequestForInvalidState() {
 
     assertThatThrownBy(() -> handler.handleRequest(request))
         .isInstanceOf(BadRequestException.class)
-        .hasMessageContaining("Invalid state");
+        .hasMessageContaining("Invalid state ID: invalid");
   }
 
   @Test

data/provider/build.gradle

@@ -15,7 +15,6 @@ dependencies {
     implementation project(':beacon:sync')
     implementation project(':validator:api')
 
-    implementation 'com.google.code.gson:gson'
     implementation 'org.apache.tuweni:tuweni-units'
 
     testImplementation testFixtures(project(':ethereum:spec'))