Merge remote-tracking branch 'upstream/2.x' into decommission-2.x

Signed-off-by: Rishab Nahata <[email protected]>
opensearch-project · Oct 28, 2022 · 8c5377d · 8c5377d
2 parents 55f8b33 + 213b966
commit 8c5377d
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -85,7 +85,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fix a bug on handling an invalid array value for point type field #4900([#4900](https://github.com/opensearch-project/OpenSearch/pull/4900))
 - Fix decommission status update to non leader nodes ([4800](https://github.com/opensearch-project/OpenSearch/pull/4800))
 - Fix bug in AwarenessAttributeDecommissionIT([4822](https://github.com/opensearch-project/OpenSearch/pull/4822))
-
+- Fix for failing checkExtraction, checkLicense and checkNotice tasks for windows gradle check ([#4941](https://github.com/opensearch-project/OpenSearch/pull/4941))
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))
 
@@ -128,6 +128,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Disable merge on refresh in DiskThresholdDeciderIT ([#4828](https://github.com/opensearch-project/OpenSearch/pull/4828))
 - Better plural stemmer than minimal_english ([#4738](https://github.com/opensearch-project/OpenSearch/pull/4738))
 - Fix AbstractStringFieldDataTestCase tests to account for TotalHits lower bound ([4867](https://github.com/opensearch-project/OpenSearch/pull/4867))
+- [Segment Replication] Fix bug of replica shard's translog not purging on index flush when segment replication is enabled ([4975](https://github.com/opensearch-project/OpenSearch/pull/4975))
+-
 
 ### Security
 

diff --git a/.../src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java b/.../src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java
@@ -73,12 +73,14 @@ public void apply(Project project) {
             .create("distributionArchiveCheck", DistributionArchiveCheckExtension.class);
 
         File archiveExtractionDir = calculateArchiveExtractionDir(project);
-
         // sanity checks if archives can be extracted
         TaskProvider<Copy> checkExtraction = registerCheckExtractionTask(project, buildDistTask, archiveExtractionDir);
+        checkExtraction.configure(InternalDistributionArchiveSetupPlugin.configure(buildTaskName));
         TaskProvider<Task> checkLicense = registerCheckLicenseTask(project, checkExtraction);
+        checkLicense.configure(InternalDistributionArchiveSetupPlugin.configure(buildTaskName));
 
         TaskProvider<Task> checkNotice = registerCheckNoticeTask(project, checkExtraction);
+        checkNotice.configure(InternalDistributionArchiveSetupPlugin.configure(buildTaskName));
         TaskProvider<Task> checkTask = project.getTasks().named("check");
         checkTask.configure(task -> {
             task.dependsOn(checkExtraction);
@@ -118,7 +120,7 @@ public void execute(Task task) {
     }
 
     private TaskProvider<Task> registerCheckLicenseTask(Project project, TaskProvider<Copy> checkExtraction) {
-        TaskProvider<Task> checkLicense = project.getTasks().register("checkLicense", task -> {
+        return project.getTasks().register("checkLicense", task -> {
             task.dependsOn(checkExtraction);
             task.doLast(new Action<Task>() {
                 @Override
@@ -138,7 +140,6 @@ public void execute(Task task) {
                 }
             });
         });
-        return checkLicense;
     }
 
     private TaskProvider<Copy> registerCheckExtractionTask(Project project, TaskProvider<Task> buildDistTask, File archiveExtractionDir) {

diff --git a/.../src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/.../src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
@@ -87,7 +87,7 @@ public void apply(Project project) {
         configureTarDefaults(project);
     }
 
-    private Action<Task> configure(String name) {
+    static Action<Task> configure(String name) {
         return (Task task) -> task.onlyIf(s -> {
             if (OperatingSystem.current().isWindows()) {
                 // On Windows, include only Windows distributions and integTestZip

diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
@@ -129,6 +129,7 @@ public synchronized void updateSegments(final SegmentInfos infos, long seqNo) th
         // lower/higher gens are possible from a new primary that was just elected.
         if (incomingGeneration != lastReceivedGen) {
             commitSegmentInfos();
+            translogManager.getDeletionPolicy().setLocalCheckpointOfSafeCommit(seqNo);
             translogManager.rollTranslogGeneration();
         }
         lastReceivedGen = incomingGeneration;

diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
@@ -246,16 +246,36 @@ public void testReplicaReceivesGenIncrease() throws Exception {
             final IndexShard replica = shards.getReplicas().get(0);
             final int numDocs = randomIntBetween(10, 100);
             shards.indexDocs(numDocs);
+            assertEquals(numDocs, primary.translogStats().estimatedNumberOfOperations());
+            assertEquals(numDocs, replica.translogStats().estimatedNumberOfOperations());
+            assertEquals(numDocs, primary.translogStats().getUncommittedOperations());
+            assertEquals(numDocs, replica.translogStats().getUncommittedOperations());
             flushShard(primary, true);
             replicateSegments(primary, shards.getReplicas());
+            assertEquals(0, primary.translogStats().estimatedNumberOfOperations());
+            assertEquals(0, replica.translogStats().estimatedNumberOfOperations());
+            assertEquals(0, primary.translogStats().getUncommittedOperations());
+            assertEquals(0, replica.translogStats().getUncommittedOperations());
 
-            final int totalDocs = numDocs + shards.indexDocs(randomIntBetween(numDocs + 1, numDocs + 10));
-            flushShard(primary);
+            final int additionalDocs = shards.indexDocs(randomIntBetween(numDocs + 1, numDocs + 10));
+
+            final int totalDocs = numDocs + additionalDocs;
+            primary.refresh("test");
+            replicateSegments(primary, shards.getReplicas());
+            assertEquals(additionalDocs, primary.translogStats().estimatedNumberOfOperations());
+            assertEquals(additionalDocs, replica.translogStats().estimatedNumberOfOperations());
+            assertEquals(additionalDocs, primary.translogStats().getUncommittedOperations());
+            assertEquals(additionalDocs, replica.translogStats().getUncommittedOperations());
+            flushShard(primary, true);
             replicateSegments(primary, shards.getReplicas());
 
             assertEqualCommittedSegments(primary, replica);
             assertDocCount(primary, totalDocs);
             assertDocCount(replica, totalDocs);
+            assertEquals(0, primary.translogStats().estimatedNumberOfOperations());
+            assertEquals(0, replica.translogStats().estimatedNumberOfOperations());
+            assertEquals(0, primary.translogStats().getUncommittedOperations());
+            assertEquals(0, replica.translogStats().getUncommittedOperations());
         }
     }
 
@@ -388,16 +408,20 @@ public void testNRTReplicaPromotedAsPrimary() throws Exception {
             // 2. Create ops that are in the replica's xlog, not in the index.
             // index some more into both but don't replicate. replica will have only numDocs searchable, but should have totalDocs
             // persisted.
-            final int totalDocs = numDocs + shards.indexDocs(randomInt(10));
+            final int additonalDocs = shards.indexDocs(randomInt(10));
+            final int totalDocs = numDocs + additonalDocs;
 
             assertDocCounts(oldPrimary, totalDocs, totalDocs);
             for (IndexShard shard : shards.getReplicas()) {
                 assertDocCounts(shard, totalDocs, numDocs);
             }
+            assertEquals(additonalDocs, nextPrimary.translogStats().estimatedNumberOfOperations());
+            assertEquals(additonalDocs, replica.translogStats().estimatedNumberOfOperations());
+            assertEquals(additonalDocs, nextPrimary.translogStats().getUncommittedOperations());
+            assertEquals(additonalDocs, replica.translogStats().getUncommittedOperations());
 
             // promote the replica
             shards.syncGlobalCheckpoint();
-            assertEquals(totalDocs, nextPrimary.translogStats().estimatedNumberOfOperations());
             shards.promoteReplicaToPrimary(nextPrimary);
 
             // close and start the oldPrimary as a replica.