HDDS-9192. Update Ratis to 3.0.0. (#5205)

apache · Nov 22, 2023 · 904a916 · 904a916
1 parent 429d876
commit 904a916
Show file tree

Hide file tree

Showing 13 changed files with 136 additions and 55 deletions.
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
@@ -106,16 +106,23 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-reload4j</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>io.dropwizard.metrics</groupId>
-          <artifactId>metrics-core</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>org.bouncycastle</groupId>
           <artifactId>bcprov-jdk15on</artifactId>
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <artifactId>ratis-metrics-dropwizard3</artifactId>
+      <groupId>org.apache.ratis</groupId>
+      <exclusions>
+        <exclusion>
+          <groupId>io.dropwizard.metrics</groupId>
+          <artifactId>metrics-core</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
     <dependency>
       <artifactId>ratis-netty</artifactId>
       <groupId>org.apache.ratis</groupId>

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -99,7 +99,7 @@ private RatisHelper() {
   }
 
   public static JvmPauseMonitor newJvmPauseMonitor(String name) {
-    return new JvmPauseMonitor(name, extraSleep -> { });
+    return JvmPauseMonitor.newBuilder().setName(name).build();
   }
 
   private static String toRaftPeerIdString(DatanodeDetails id) {

diff --git a/...a/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/...a/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -102,6 +102,7 @@
 import org.apache.ratis.server.RaftServerConfigKeys;
 import org.apache.ratis.server.RaftServerRpc;
 import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.server.storage.RaftStorage;
 import org.apache.ratis.util.SizeInBytes;
 import org.apache.ratis.util.TimeDuration;
 import org.apache.ratis.util.TraditionalBinaryPrefix;
@@ -176,12 +177,13 @@ private XceiverServerRatis(DatanodeDetails dd,
         conf.getObject(DatanodeRatisServerConfig.class)
             .shouldDeleteRatisLogDirectory();
 
-    RaftServer.Builder builder =
+    this.server =
         RaftServer.newBuilder().setServerId(raftPeerId)
             .setProperties(serverProperties)
             .setStateMachineRegistry(this::getStateMachine)
-            .setParameters(parameters);
-    this.server = builder.build();
+            .setParameters(parameters)
+            .setOption(RaftStorage.StartupOption.RECOVER)
+            .build();
     this.requestTimeout = conf.getTimeDuration(
         HddsConfigKeys.HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT,
         HddsConfigKeys.HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT,

diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml
@@ -112,16 +112,22 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-reload4j</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>io.dropwizard.metrics</groupId>
-          <artifactId>metrics-core</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>org.bouncycastle</groupId>
           <artifactId>bcprov-jdk15on</artifactId>
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <artifactId>ratis-metrics-dropwizard3</artifactId>
+      <groupId>org.apache.ratis</groupId>
+      <exclusions>
+        <exclusion>
+          <groupId>io.dropwizard.metrics</groupId>
+          <artifactId>metrics-core</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>io.prometheus</groupId>
       <artifactId>simpleclient_dropwizard</artifactId>

diff --git a/...ds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java b/...ds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java
@@ -27,8 +27,9 @@
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import org.apache.ratis.metrics.MetricRegistries;
-import org.apache.ratis.metrics.MetricsReporting;
+import org.apache.ratis.metrics.MetricRegistryInfo;
 import org.apache.ratis.metrics.RatisMetricRegistry;
+import org.apache.ratis.metrics.dropwizard3.RatisMetricsUtils;
 
 import java.util.Map;
 
@@ -52,8 +53,9 @@ public static List<MetricReporter> registerRatisMetricReporters(
     //All the Ratis metrics (registered from now) will be published via JMX and
     //via the prometheus exporter (used by the /prom servlet
     List<MetricReporter> ratisReporterList = new ArrayList<>();
-    ratisReporterList.add(new MetricReporter(MetricsReporting.jmxReporter(),
-        MetricsReporting.stopJmxReporter()));
+    ratisReporterList.add(new MetricReporter(
+        RatisMetricsUtils.jmxReporter(),
+        RatisMetricsUtils.stopJmxReporter()));
     Consumer<RatisMetricRegistry> reporter
         = r1 -> registerDropwizard(r1, ratisMetricsMap, checkStopped);
     Consumer<RatisMetricRegistry> stopper
@@ -88,6 +90,12 @@ public static void clear(
     MetricRegistries.global().clear();
   }
 
+  static String getName(MetricRegistryInfo info) {
+    return MetricRegistry.name(info.getApplicationName(),
+        info.getMetricsComponentName(),
+        info.getPrefix());
+  }
+
   private static void registerDropwizard(RatisMetricRegistry registry,
       Map<String, RatisDropwizardExports> ratisMetricsMap,
       BooleanSupplier checkStopped) {
@@ -96,8 +104,8 @@ private static void registerDropwizard(RatisMetricRegistry registry,
     }
 
     RatisDropwizardExports rde = new RatisDropwizardExports(
-        registry.getDropWizardMetricRegistry());
-    String name = registry.getMetricRegistryInfo().getName();
+        RatisMetricsUtils.getDropWizardMetricRegistry(registry));
+    final String name = getName(registry.getMetricRegistryInfo());
     if (null == ratisMetricsMap.putIfAbsent(name, rde)) {
       // new rde is added for the name, so need register
       CollectorRegistry.defaultRegistry.register(rde);
@@ -106,7 +114,7 @@ private static void registerDropwizard(RatisMetricRegistry registry,
 
   private static void deregisterDropwizard(RatisMetricRegistry registry,
       Map<String, RatisDropwizardExports> ratisMetricsMap) {
-    String name = registry.getMetricRegistryInfo().getName();
+    final String name = getName(registry.getMetricRegistryInfo());
     Collector c = ratisMetricsMap.remove(name);
     if (c != null) {
       CollectorRegistry.defaultRegistry.unregister(c);

diff --git a/...-hdds/framework/src/main/java/org/apache/ratis/metrics/dropwizard3/RatisMetricsUtils.java b/...-hdds/framework/src/main/java/org/apache/ratis/metrics/dropwizard3/RatisMetricsUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.metrics.dropwizard3;
+
+import com.codahale.metrics.MetricRegistry;
+import org.apache.ratis.metrics.RatisMetricRegistry;
+
+import java.util.function.Consumer;
+
+/**
+ * Utilities for ratis metrics dropwizard3.
+ */
+public interface RatisMetricsUtils {
+
+  static MetricRegistry getDropWizardMetricRegistry(RatisMetricRegistry r) {
+    return ((Dm3RatisMetricRegistryImpl) r).getDropWizardMetricRegistry();
+  }
+
+  static Consumer<RatisMetricRegistry> jmxReporter() {
+    return Dm3MetricsReporting.jmxReporter();
+  }
+
+  static Consumer<RatisMetricRegistry> stopJmxReporter() {
+    return Dm3MetricsReporting.stopJmxReporter();
+  }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/ratis/metrics/dropwizard3/package-info.java b/hadoop-hdds/framework/src/main/java/org/apache/ratis/metrics/dropwizard3/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for ratis metrics dropwizard3.
+ */
+package org.apache.ratis.metrics.dropwizard3;
diff --git a/...ramework/src/test/java/org/apache/hadoop/hdds/server/http/TestRatisDropwizardExports.java b/...ramework/src/test/java/org/apache/hadoop/hdds/server/http/TestRatisDropwizardExports.java
@@ -19,32 +19,46 @@
 
 import java.io.IOException;
 import java.io.StringWriter;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
 import io.prometheus.client.CollectorRegistry;
 import io.prometheus.client.exporter.common.TextFormat;
+import org.apache.ratis.metrics.dropwizard3.RatisMetricsUtils;
 import org.apache.ratis.protocol.RaftGroupId;
 import org.apache.ratis.protocol.RaftGroupMemberId;
 import org.apache.ratis.protocol.RaftPeerId;
 import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
+import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RAFT_LOG_SYNC_TIME;
+
 /**
  * Test RatisDropwizardRexporter.
  */
 public class TestRatisDropwizardExports {
+  static Timer findTimer(String name, MetricRegistry registry) {
+    for (Map.Entry<String, Timer> e : registry.getTimers().entrySet()) {
+      if (e.getKey().contains(name)) {
+        return e.getValue();
+      }
+    }
+    throw new IllegalStateException(name + " not found");
+  }
 
   @Test
   public void export() throws IOException {
     //create Ratis metrics
     SegmentedRaftLogMetrics instance = new SegmentedRaftLogMetrics(
         RaftGroupMemberId.valueOf(
             RaftPeerId.valueOf("peerId"), RaftGroupId.randomId()));
-    instance.getRaftLogSyncTimer().update(10, TimeUnit.MILLISECONDS);
     MetricRegistry dropWizardMetricRegistry =
-        instance.getRegistry().getDropWizardMetricRegistry();
+        RatisMetricsUtils.getDropWizardMetricRegistry(instance.getRegistry());
+    findTimer(RAFT_LOG_SYNC_TIME, dropWizardMetricRegistry)
+        .update(10, TimeUnit.MILLISECONDS);
 
     //create and register prometheus collector
     RatisDropwizardExports exports =

diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
@@ -208,37 +208,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         <scope>test</scope>
       </dependency>
 
-      <dependency>
-        <groupId>org.apache.ratis</groupId>
-        <artifactId>ratis-proto-shaded</artifactId>
-        <version>${ratis.version}</version>
-      </dependency>
-      <dependency>
-        <artifactId>ratis-common</artifactId>
-        <groupId>org.apache.ratis</groupId>
-        <version>${ratis.version}</version>
-      </dependency>
-      <dependency>
-        <artifactId>ratis-client</artifactId>
-        <groupId>org.apache.ratis</groupId>
-        <version>${ratis.version}</version>
-      </dependency>
-      <dependency>
-        <artifactId>ratis-server</artifactId>
-        <groupId>org.apache.ratis</groupId>
-        <version>${ratis.version}</version>
-      </dependency>
-      <dependency>
-        <artifactId>ratis-netty</artifactId>
-        <groupId>org.apache.ratis</groupId>
-        <version>${ratis.version}</version>
-      </dependency>
-      <dependency>
-        <artifactId>ratis-grpc</artifactId>
-        <groupId>org.apache.ratis</groupId>
-        <version>${ratis.version}</version>
-      </dependency>
-
       <dependency>
         <groupId>org.bouncycastle</groupId>
         <artifactId>bcprov-jdk15on</artifactId>

diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
@@ -56,6 +56,7 @@
 import org.apache.ratis.protocol.exceptions.NotLeaderException;
 import org.apache.ratis.protocol.SetConfigurationRequest;
 import org.apache.ratis.server.RaftServer;
+import org.apache.ratis.server.storage.RaftStorage;
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -112,6 +113,7 @@ public class SCMRatisServerImpl implements SCMRatisServer {
 
     this.server = newRaftServer(scm.getScmId(), conf)
         .setStateMachineRegistry((gId) -> new SCMStateMachine(scm, buffer))
+        .setOption(RaftStorage.StartupOption.RECOVER)
         .setGroup(RaftGroup.valueOf(groupId))
         .setParameters(parameters).build();
 
@@ -129,6 +131,7 @@ public static void initialize(String clusterId, String scmId,
     try {
       server = newRaftServer(scmId, conf).setGroup(group)
               .setStateMachineRegistry((groupId -> new SCMStateMachine()))
+              .setOption(RaftStorage.StartupOption.RECOVER)
               .build();
       server.start();
       waitForLeaderToBeReady(server, conf, group);

diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -49,6 +49,7 @@ share/ozone/lib/grpc-netty.jar
 share/ozone/lib/grpc-protobuf.jar
 share/ozone/lib/grpc-protobuf-lite.jar
 share/ozone/lib/grpc-stub.jar
+share/ozone/lib/grpc-util.jar
 share/ozone/lib/gson.jar
 share/ozone/lib/guava-jre.jar
 share/ozone/lib/guice-assistedinject.jar
@@ -239,7 +240,8 @@ share/ozone/lib/ranger-plugins-cred.jar
 share/ozone/lib/ratis-client.jar
 share/ozone/lib/ratis-common.jar
 share/ozone/lib/ratis-grpc.jar
-share/ozone/lib/ratis-metrics.jar
+share/ozone/lib/ratis-metrics-api.jar
+share/ozone/lib/ratis-metrics-dropwizard3.jar
 share/ozone/lib/ratis-netty.jar
 share/ozone/lib/ratis-proto.jar
 share/ozone/lib/ratis-server-api.jar

diff --git a/...ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/...ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
@@ -85,6 +85,7 @@
 import org.apache.ratis.server.RaftServer;
 import org.apache.ratis.server.RaftServerConfigKeys;
 import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.server.storage.RaftStorage;
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 import org.apache.ratis.util.LifeCycle;
 import org.apache.ratis.util.SizeInBytes;
@@ -176,6 +177,7 @@ private OzoneManagerRatisServer(ConfigurationSource conf, OzoneManager om,
         .setProperties(serverProperties)
         .setParameters(parameters)
         .setStateMachine(omStateMachine)
+        .setOption(RaftStorage.StartupOption.RECOVER)
         .build();
     this.perfMetrics = om.getPerfMetrics();
   }

diff --git a/pom.xml b/pom.xml
@@ -75,10 +75,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <!-- HDDS Rocks Native dependency version-->
     <hdds.rocks.native.version>${hdds.version}</hdds.rocks.native.version>
     <!-- Apache Ratis version -->
-    <ratis.version>2.5.1</ratis.version>
+    <ratis.version>3.0.0</ratis.version>
 
     <!-- Apache Ratis thirdparty version -->
-    <ratis.thirdparty.version>1.0.4</ratis.thirdparty.version>
+    <ratis.thirdparty.version>1.0.5</ratis.thirdparty.version>
 
     <!-- Apache Ranger plugin version -->
     <ranger.version>2.3.0</ranger.version>
@@ -237,7 +237,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <!-- Use netty version known to work with grpc-netty. See table: -->
     <!-- https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty -->
     <netty.version>4.1.100.Final</netty.version>
-    <io.grpc.version>1.51.1</io.grpc.version>
+    <io.grpc.version>1.58.0</io.grpc.version>
 
     <rocksdb.version>7.7.3</rocksdb.version>
     <sqlite.version>3.41.2.2</sqlite.version>
@@ -1043,6 +1043,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <groupId>org.apache.ratis</groupId>
         <version>${ratis.version}</version>
       </dependency>
+      <dependency>
+        <artifactId>ratis-metrics-dropwizard3</artifactId>
+        <groupId>org.apache.ratis</groupId>
+        <version>${ratis.version}</version>
+      </dependency>
       <dependency>
         <artifactId>ratis-netty</artifactId>
         <groupId>org.apache.ratis</groupId>