merge branch master

.gitignore

@@ -29,7 +29,6 @@ target/
 tmp/
 build/
 out/
-ganache-cli/
 pow/src/main/resources/keys.json
 !validator_test_data.json
 /interopDepositsAndKeys.json

CHANGELOG.md

@@ -14,6 +14,6 @@ For information on changes in released versions of Teku, see the [releases page]
 
 ### Additions and Improvements
 
-Introduce `--exchange-capabilities-monitoring-enabled` parameter. If enabled, EL will be queried periodically for the Engine API methods it supports. If incompatibility is detected, there will be a warning raised in the logs. The default is `true`.
+- Introduce `--exchange-capabilities-monitoring-enabled` parameter. If enabled, EL will be queried periodically for the Engine API methods it supports. If incompatibility is detected, a warning is raised in the logs. The default is `true`.
 
 ### Bug Fixes

data/provider/src/main/java/tech/pegasys/teku/api/AbstractSelectorFactory.java

@@ -93,17 +93,13 @@ private boolean isHexString(final String identifier) {
   }
 
   private T createSelectorForKeywordOrSlot(final String identifier) {
-    switch (identifier) {
-      case HEAD:
-        return headSelector();
-      case GENESIS:
-        return genesisSelector();
-      case FINALIZED:
-        return finalizedSelector();
-      case JUSTIFIED:
-        return justifiedSelector();
-    }
-    return slotSelector(UInt64.valueOf(identifier));
+    return switch (identifier) {
+      case HEAD -> headSelector();
+      case GENESIS -> genesisSelector();
+      case FINALIZED -> finalizedSelector();
+      case JUSTIFIED -> justifiedSelector();
+      default -> slotSelector(UInt64.valueOf(identifier));
+    };
   }
 
   private BadRequestException badRequestException(final String type, final String identifier) {

gradle/versions.gradle

@@ -87,7 +87,6 @@ dependencyManagement {
     dependency 'org.apache.commons:commons-text:1.10.0'
     dependency 'org.apache.commons:commons-lang3:3.12.0'
     dependency 'commons-io:commons-io:2.13.0'
-    dependency 'org.apache.commons:commons-compress:1.21'
     dependency 'org.commonjava.mimeparse:mimeparse:0.1.3.3'
 
     dependencySet(group: 'org.apache.logging.log4j', version: '2.20.0') {

infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/primitive/SszUInt64.java

@@ -19,13 +19,24 @@
 
 public class SszUInt64 extends AbstractSszPrimitive<UInt64, SszUInt64> {
 
-  public static final SszUInt64 ZERO = SszUInt64.of(UInt64.ZERO);
+  public static final SszUInt64 ZERO = new SszUInt64(UInt64.ZERO);
+  public static final SszUInt64 THIRTY_TWO_ETH = new SszUInt64(UInt64.THIRTY_TWO_ETH);
+  public static final SszUInt64 MAX_VALUE = new SszUInt64(UInt64.MAX_VALUE);
 
-  public static SszUInt64 of(UInt64 val) {
+  public static SszUInt64 of(final UInt64 val) {
+    if (val.isZero()) {
+      return ZERO;
+    }
+    if (val.isThirtyTwoEth()) {
+      return THIRTY_TWO_ETH;
+    }
+    if (val.isMaxValue()) {
+      return MAX_VALUE;
+    }
     return new SszUInt64(val);
   }
 
-  private SszUInt64(UInt64 val) {
+  private SszUInt64(final UInt64 val) {
     super(val, SszPrimitiveSchemas.UINT64_SCHEMA);
   }
 

infrastructure/unsigned/src/main/java/tech/pegasys/teku/infrastructure/unsigned/UInt64.java

@@ -28,6 +28,7 @@ public final class UInt64 implements Comparable<UInt64> {
 
   public static final UInt64 ZERO = new UInt64(0);
   public static final UInt64 ONE = new UInt64(1);
+  public static final UInt64 THIRTY_TWO_ETH = new UInt64(32_000_000_000L);
   public static final UInt64 MAX_VALUE = new UInt64(-1L);
 
   private final long value;
@@ -82,7 +83,16 @@ public static UInt64 valueOf(final BigInteger value) {
    * @return the created UInt64.
    */
   public static UInt64 fromLongBits(final long value) {
-    return value == 0 ? ZERO : new UInt64(value);
+    if (value == 0L) {
+      return ZERO;
+    }
+    if (value == 32_000_000_000L) {
+      return THIRTY_TWO_ETH;
+    }
+    if (value == -1L) {
+      return MAX_VALUE;
+    }
+    return new UInt64(value);
   }
 
   /**
@@ -365,6 +375,19 @@ public boolean isZero() {
     return value == 0;
   }
 
+  /**
+   * Returns true if the value is 32_000_000_000L
+   *
+   * @return true if the value is 32_000_000_000L
+   */
+  public boolean isThirtyTwoEth() {
+    return value == 32_000_000_000L;
+  }
+
+  public boolean isMaxValue() {
+    return value == -1L;
+  }
+
   /**
    * Returns true if this value is strictly greater than the specified value.
    *

infrastructure/unsigned/src/test/java/tech/pegasys/teku/infrastructure/unsigned/UInt64Test.java

@@ -290,6 +290,7 @@ void lessThanOrEqualTo_shouldReturnTrueWhenNumberIsLessThanOrEqual(
   @Test
   void constants_shouldHaveExpectedValues() {
     assertThat(UInt64.ZERO).isEqualTo(UInt64.valueOf(0));
+    assertThat(UInt64.THIRTY_TWO_ETH).isEqualTo(UInt64.valueOf(32_000_000_000L));
     assertThat(UInt64.ONE).isEqualTo(UInt64.valueOf(1));
     assertThat(UInt64.MAX_VALUE).isEqualTo(UInt64.fromLongBits(-1));
   }

services/chainstorage/build.gradle

@@ -5,6 +5,7 @@ dependencies {
   implementation project(':infrastructure:async')
   implementation project(':infrastructure:exceptions')
   implementation project(':infrastructure:logging')
+  implementation project(':infrastructure:metrics')
   implementation project(':infrastructure:serviceutils')
   implementation project(':storage')
   implementation project(':storage:api')

services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java

@@ -22,6 +22,8 @@
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.async.eventthread.AsyncRunnerEventThread;
 import tech.pegasys.teku.infrastructure.events.EventChannels;
+import tech.pegasys.teku.infrastructure.metrics.SettableLabelledGauge;
+import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory;
 import tech.pegasys.teku.service.serviceutils.Service;
 import tech.pegasys.teku.service.serviceutils.ServiceConfig;
 import tech.pegasys.teku.spec.SpecMilestone;
@@ -80,14 +82,33 @@ protected SafeFuture<?> doStart() {
 
               database.migrate();
 
+              final SettableLabelledGauge pruningTimingsLabelledGauge =
+                  SettableLabelledGauge.create(
+                      serviceConfig.getMetricsSystem(),
+                      TekuMetricCategory.STORAGE,
+                      "pruning_time",
+                      "Tracks last pruning duration in milliseconds",
+                      "type");
+
+              final SettableLabelledGauge pruningActiveLabelledGauge =
+                  SettableLabelledGauge.create(
+                      serviceConfig.getMetricsSystem(),
+                      TekuMetricCategory.STORAGE,
+                      "pruning_active",
+                      "Tracks when pruner is active",
+                      "type");
+
               if (!config.getDataStorageMode().storesAllBlocks()) {
                 blockPruner =
                     Optional.of(
                         new BlockPruner(
                             config.getSpec(),
                             database,
                             storagePrunerAsyncRunner,
-                            config.getBlockPruningInterval()));
+                            config.getBlockPruningInterval(),
+                            "block",
+                            pruningTimingsLabelledGauge,
+                            pruningActiveLabelledGauge));
               }
               if (config.getSpec().isMilestoneSupported(SpecMilestone.DENEB)) {
                 blobsPruner =
@@ -101,6 +122,9 @@ protected SafeFuture<?> doStart() {
                             config.getBlobsPruningInterval(),
                             config.getBlobsPruningLimit(),
                             blobSidecarsStorageCountersEnabled,
+                            "blob_sidecar",
+                            pruningTimingsLabelledGauge,
+                            pruningActiveLabelledGauge,
                             config.isStoreNonCanonicalBlocksEnabled()));
               }
               final EventChannels eventChannels = serviceConfig.getEventChannels();

storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java

@@ -24,6 +24,7 @@
 import tech.pegasys.teku.infrastructure.async.AsyncRunner;
 import tech.pegasys.teku.infrastructure.async.Cancellable;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
+import tech.pegasys.teku.infrastructure.metrics.SettableLabelledGauge;
 import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory;
 import tech.pegasys.teku.infrastructure.time.TimeProvider;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
@@ -44,6 +45,9 @@ public class BlobSidecarPruner extends Service {
   private final int pruneLimit;
   private final TimeProvider timeProvider;
   private final boolean blobSidecarsStorageCountersEnabled;
+  private final String pruningMetricsType;
+  private final SettableLabelledGauge pruningTimingsLabelledGauge;
+  private final SettableLabelledGauge pruningActiveLabelledGauge;
 
   private Optional<Cancellable> scheduledPruner = Optional.empty();
   private Optional<UInt64> genesisTime = Optional.empty();
@@ -61,6 +65,9 @@ public BlobSidecarPruner(
       final Duration pruneInterval,
       final int pruneLimit,
       final boolean blobSidecarsStorageCountersEnabled,
+      final String pruningMetricsType,
+      final SettableLabelledGauge pruningTimingsLabelledGauge,
+      final SettableLabelledGauge pruningActiveLabelledGauge,
       final boolean storeNonCanonicalBlobSidecars) {
     this.spec = spec;
     this.database = database;
@@ -69,6 +76,9 @@ public BlobSidecarPruner(
     this.pruneLimit = pruneLimit;
     this.timeProvider = timeProvider;
     this.blobSidecarsStorageCountersEnabled = blobSidecarsStorageCountersEnabled;
+    this.pruningMetricsType = pruningMetricsType;
+    this.pruningTimingsLabelledGauge = pruningTimingsLabelledGauge;
+    this.pruningActiveLabelledGauge = pruningActiveLabelledGauge;
     this.storeNonCanonicalBlobSidecars = storeNonCanonicalBlobSidecars;
 
     if (blobSidecarsStorageCountersEnabled) {
@@ -103,8 +113,14 @@ protected synchronized SafeFuture<?> doStop() {
   }
 
   private void pruneBlobs() {
+    pruningActiveLabelledGauge.set(1, pruningMetricsType);
+    final long start = System.currentTimeMillis();
+
     pruneBlobsPriorToAvailabilityWindow();
 
+    pruningTimingsLabelledGauge.set(System.currentTimeMillis() - start, pruningMetricsType);
+    pruningActiveLabelledGauge.set(0, pruningMetricsType);
+
     if (blobSidecarsStorageCountersEnabled) {
       blobColumnSize.set(database.getBlobSidecarColumnCount());
       earliestBlobSidecarSlot.set(
@@ -147,6 +163,8 @@ private void pruneBlobsPriorToAvailabilityWindow() {
             nonCanonicalBlobsPruningStart,
             nonCanonicalBlobsLimitReached);
       }
+      final boolean limitReached = database.pruneOldestBlobSidecars(latestPrunableSlot, pruneLimit);
+      LOG.debug("Blobs pruning finished. Limit reached: {}", () -> limitReached);
     } catch (ShuttingDownException | RejectedExecutionException ex) {
       LOG.debug("Shutting down", ex);
     }

storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlockPruner.java

@@ -21,6 +21,7 @@
 import tech.pegasys.teku.infrastructure.async.AsyncRunner;
 import tech.pegasys.teku.infrastructure.async.Cancellable;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
+import tech.pegasys.teku.infrastructure.metrics.SettableLabelledGauge;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.service.serviceutils.Service;
 import tech.pegasys.teku.spec.Spec;
@@ -35,26 +36,42 @@ public class BlockPruner extends Service {
   private final Database database;
   private final AsyncRunner asyncRunner;
   private final Duration pruneInterval;
+  private final SettableLabelledGauge pruningTimingsLabelledGauge;
+  private final SettableLabelledGauge pruningActiveLabelledGauge;
+  private final String pruningMetricsType;
 
   private Optional<Cancellable> scheduledPruner = Optional.empty();
 
   public BlockPruner(
       final Spec spec,
       final Database database,
       final AsyncRunner asyncRunner,
-      final Duration pruneInterval) {
+      final Duration pruneInterval,
+      final String pruningMetricsType,
+      final SettableLabelledGauge pruningTimingsLabelledGauge,
+      final SettableLabelledGauge pruningActiveLabelledGauge) {
     this.spec = spec;
     this.database = database;
     this.asyncRunner = asyncRunner;
     this.pruneInterval = pruneInterval;
+    this.pruningMetricsType = pruningMetricsType;
+    this.pruningTimingsLabelledGauge = pruningTimingsLabelledGauge;
+    this.pruningActiveLabelledGauge = pruningActiveLabelledGauge;
   }
 
   @Override
   protected synchronized SafeFuture<?> doStart() {
     scheduledPruner =
         Optional.of(
             asyncRunner.runWithFixedDelay(
-                this::pruneBlocks,
+                () -> {
+                  pruningActiveLabelledGauge.set(1, pruningMetricsType);
+                  final long start = System.currentTimeMillis();
+                  pruneBlocks();
+                  pruningTimingsLabelledGauge.set(
+                      System.currentTimeMillis() - start, pruningMetricsType);
+                  pruningActiveLabelledGauge.set(0, pruningMetricsType);
+                },
                 Duration.ZERO,
                 pruneInterval,
                 error -> LOG.error("Failed to prune old blocks", error)));

storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java

@@ -28,6 +28,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.infrastructure.async.StubAsyncRunner;
+import tech.pegasys.teku.infrastructure.metrics.SettableLabelledGauge;
 import tech.pegasys.teku.infrastructure.metrics.StubMetricsSystem;
 import tech.pegasys.teku.infrastructure.time.StubTimeProvider;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
@@ -64,6 +65,9 @@ public class BlobSidecarPrunerTest {
           PRUNE_INTERVAL,
           PRUNE_LIMIT,
           false,
+          "test",
+          mock(SettableLabelledGauge.class),
+          mock(SettableLabelledGauge.class),
           false);
 
   @BeforeEach
@@ -149,6 +153,9 @@ void shouldUseEpochsStoreBlobs() {
             PRUNE_INTERVAL,
             PRUNE_LIMIT,
             false,
+            "test",
+            mock(SettableLabelledGauge.class),
+            mock(SettableLabelledGauge.class),
             false);
     when(databaseOverride.getGenesisTime()).thenReturn(Optional.of(genesisTime));
     assertThat(blobsPrunerOverride.start()).isCompleted();
@@ -201,6 +208,9 @@ void shouldNotPruneWhenEpochsStoreBlobsIsMax() {
             PRUNE_INTERVAL,
             PRUNE_LIMIT,
             false,
+            "test",
+            mock(SettableLabelledGauge.class),
+            mock(SettableLabelledGauge.class),
             false);
     when(databaseOverride.getGenesisTime()).thenReturn(Optional.of(genesisTime));
     assertThat(blobsPrunerOverride.start()).isCompleted();

storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlockPrunerTest.java

@@ -25,6 +25,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.infrastructure.async.StubAsyncRunner;
+import tech.pegasys.teku.infrastructure.metrics.SettableLabelledGauge;
 import tech.pegasys.teku.infrastructure.time.StubTimeProvider;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
@@ -51,7 +52,15 @@ class BlockPrunerTest {
   private final StubAsyncRunner asyncRunner = new StubAsyncRunner(timeProvider);
   private final Database database = mock(Database.class);
 
-  private final BlockPruner pruner = new BlockPruner(spec, database, asyncRunner, PRUNE_INTERVAL);
+  private final BlockPruner pruner =
+      new BlockPruner(
+          spec,
+          database,
+          asyncRunner,
+          PRUNE_INTERVAL,
+          "test",
+          mock(SettableLabelledGauge.class),
+          mock(SettableLabelledGauge.class));
 
   @BeforeEach
   void setUp() {