HDDS-11585. Add DN Ratis log purge parameters to close pipeline with …

…slow follower
apache · Oct 16, 2024 · e0de584 · e0de584
1 parent c579d06
commit e0de584
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 1 deletion.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -105,11 +105,19 @@ public final class ScmConfigKeys {
       "hdds.container.ratis.log.appender.queue.byte-limit";
   public static final String
       HDDS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT = "32MB";
+
   public static final String HDDS_CONTAINER_RATIS_LOG_PURGE_GAP =
       "hdds.container.ratis.log.purge.gap";
   // TODO: Set to 1024 once RATIS issue around purge is fixed.
   public static final int HDDS_CONTAINER_RATIS_LOG_PURGE_GAP_DEFAULT =
       1000000;
+  public static final String HDDS_RATIS_LOG_PURGE_UPTO_INDEX_KEY =
+      "hdds.ratis.log.purge.upto.snapshot.index";
+  public static final boolean HDDS_RATIS_LOG_PURGE_UPTO_INDEX_DEFAULT = true;
+  public static final String HDDS_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_KEY =
+      "hdds.ratis.log.purge.preservation.log.num";
+  public static final long HDDS_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_DEFAULT = 1000000;
+
   public static final String HDDS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT =
       "hdds.container.ratis.leader.pending.bytes.limit";
   public static final String

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -432,6 +432,14 @@ public final class OzoneConfigKeys {
       ScmConfigKeys.HDDS_RATIS_SNAPSHOT_THRESHOLD_KEY;
   public static final long HDDS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT =
       ScmConfigKeys.HDDS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT;
+  public static final String HDDS_RATIS_LOG_PURGE_UPTO_INDEX_KEY =
+      ScmConfigKeys.HDDS_RATIS_LOG_PURGE_UPTO_INDEX_KEY;
+  public static final boolean HDDS_RATIS_LOG_PURGE_UPTO_INDEX_DEFAULT =
+      ScmConfigKeys.HDDS_RATIS_LOG_PURGE_UPTO_INDEX_DEFAULT;
+  public static final String HDDS_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_KEY =
+      ScmConfigKeys.HDDS_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_KEY;
+  public static final long HDDS_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_DEFAULT =
+      ScmConfigKeys.HDDS_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_DEFAULT;
 
   public static final String HDDS_DATANODE_PLUGINS_KEY =
       "hdds.datanode.plugins";

diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -279,12 +279,32 @@
   </property>
   <property>
     <name>hdds.ratis.snapshot.threshold</name>
-    <value>10000</value>
+    <value>100000</value>
     <tag>OZONE, RATIS</tag>
     <description>Number of transactions after which a ratis snapshot should be
       taken.
     </description>
   </property>
+  <property>
+    <name>hdds.ratis.log.purge.upto.snapshot.index</name>
+    <value>true</value>
+    <tag>OZONE, RATIS</tag>
+    <description>
+      Enable/disable Raft server to purge its log up to the snapshot index
+      after taking snapshot.
+    </description>
+  </property>
+  <property>
+    <name>hdds.ratis.log.purge.preservation.log.num</name>
+    <value>1000000</value>
+    <tag>OZONE, RATIS</tag>
+    <description>
+      The number of latest Raft logs to not be purged after taking snapshot.
+      In Ratis write pipeline, when hdds.ratis.log.purge.upto.snapshot.index is set in datanodes,
+      this configuration set how far behind the "slow" follower log index compared to the leader
+      before the pipeline is eligible be closed due to the notifyInstallSnapshot mechanism.
+    </description>
+  </property>
   <property>
     <name>hdds.container.ratis.statemachine.max.pending.apply-transactions</name>
     <value>10000</value>

diff --git a/...a/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/...a/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -357,6 +357,16 @@ public RaftProperties newRaftProperties() {
         OzoneConfigKeys.HDDS_CONTAINER_RATIS_LOG_PURGE_GAP,
         OzoneConfigKeys.HDDS_CONTAINER_RATIS_LOG_PURGE_GAP_DEFAULT);
     RaftServerConfigKeys.Log.setPurgeGap(properties, purgeGap);
+    boolean purgeUpToSnapshotIndex =
+        conf.getBoolean(OzoneConfigKeys.HDDS_RATIS_LOG_PURGE_UPTO_INDEX_KEY,
+            OzoneConfigKeys.HDDS_RATIS_LOG_PURGE_UPTO_INDEX_DEFAULT);
+    long purgePreservationLogNum =
+        conf.getLong(OzoneConfigKeys.HDDS_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_KEY,
+            OzoneConfigKeys.HDDS_RATIS_LOG_PURGE_PRESERVATION_LOG_NUM_DEFAULT);
+    RaftServerConfigKeys.Log.setPurgeUptoSnapshotIndex(
+        properties, purgeUpToSnapshotIndex);
+    RaftServerConfigKeys.Log.setPurgePreservationLogNum(
+        properties, purgePreservationLogNum);
 
     //Set the number of Snapshots Retained.
     RatisServerConfiguration ratisServerConfiguration =