Fix overlap between tsfile is not correctly marked

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionValidationFailedException.java

@@ -21,22 +21,24 @@
 
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 
+import java.util.List;
+
 public class CompactionValidationFailedException extends RuntimeException {
-  private TsFileResource overlappedTsFileResource = null;
+  private List<TsFileResource> overlappedTsFileResources = null;
 
   public CompactionValidationFailedException(String msg) {
     super(msg);
   }
 
-  public CompactionValidationFailedException(TsFileResource overlappedTsFileResource) {
+  public CompactionValidationFailedException(List<TsFileResource> overlappedTsFileResources) {
     super(
         "Failed to pass compaction validation, sequence files has overlap, file is "
-            + overlappedTsFileResource);
-    this.overlappedTsFileResource = overlappedTsFileResource;
+            + overlappedTsFileResources);
+    this.overlappedTsFileResources = overlappedTsFileResources;
   }
 
-  public TsFileResource getOverlappedTsFileResource() {
-    return overlappedTsFileResource;
+  public List<TsFileResource> getOverlappedTsFileResources() {
+    return overlappedTsFileResources;
   }
 
   @Override

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java

@@ -33,6 +33,7 @@
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionTaskStage;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.TsFileIdentifier;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairDataFileScanUtil;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
 import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
@@ -149,10 +150,10 @@ protected void handleException(Logger logger, Exception e) {
       } else {
         CompactionValidationFailedException validationException =
             (CompactionValidationFailedException) e;
-        TsFileResource overlappedTsFileResource = validationException.getOverlappedTsFileResource();
-        if (overlappedTsFileResource != null) {
-          unsortedTsFileResources.add(overlappedTsFileResource);
-        }
+        List<TsFileResource> overlappedTsFileResource =
+            validationException.getOverlappedTsFileResources();
+        unsortedTsFileResources =
+            overlappedTsFileResource == null ? unsortedTsFileResources : overlappedTsFileResource;
       }
       // these exceptions generally caused by unsorted data, mark all source files as NEED_TO_REPAIR
       for (TsFileResource resource : unsortedTsFileResources) {
@@ -472,15 +473,15 @@ protected void validateCompactionResult(
                     Long.parseLong(f2.getTsFile().getName().split("-")[1]))
                 : timeDiff;
           });
-      if (!validator.validateTsFilesIsHasNoOverlap(timePartitionSeqFiles)) {
+      List<TsFileResource> overlapFilesInTimePartition =
+          RepairDataFileScanUtil.checkTimePartitionHasOverlap(timePartitionSeqFiles);
+      if (!overlapFilesInTimePartition.isEmpty()) {
         LOGGER.error(
             "Failed to pass compaction validation, source seq files: {}, source unseq files: {}, target files: {}",
             sourceSeqFiles,
             sourceUnseqFiles,
             targetFiles);
-        throw new CompactionValidationFailedException(
-            "Failed to pass compaction validation, sequence files has overlap, time partition id is "
-                + timePartition);
+        throw new CompactionValidationFailedException(overlapFilesInTimePartition);
       }
     }
     if (needToValidateTsFileCorrectness && !validator.validateTsFiles(validTargetFiles)) {

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java

@@ -63,24 +63,22 @@ public void endMeasurement(int subTaskId) throws IOException {
         mergeTimeValuePair(timeValuePair, previousTimeValuePair);
         continue;
       }
-      writeDataPoint(
-          previousTimeValuePair.getTimestamp(),
-          previousTimeValuePair.getValue(),
-          chunkWriters[subTaskId]);
-      chunkPointNumArray[subTaskId]++;
+      writeToChunkWriter(previousTimeValuePair, subTaskId);
       previousTimeValuePair = timeValuePair;
     }
     // write last time value pair
-    writeDataPoint(
-        previousTimeValuePair.getTimestamp(),
-        previousTimeValuePair.getValue(),
-        chunkWriters[subTaskId]);
-    chunkPointNumArray[subTaskId]++;
+    writeToChunkWriter(previousTimeValuePair, subTaskId);
 
     dataOfCurrentSeriesArr[subTaskId] = null;
     super.endMeasurement(subTaskId);
   }
 
+  private void writeToChunkWriter(TimeValuePair timeValuePair, int subTaskId) throws IOException {
+    writeDataPoint(timeValuePair.getTimestamp(), timeValuePair.getValue(), chunkWriters[subTaskId]);
+    chunkPointNumArray[subTaskId]++;
+    checkChunkSizeAndMayOpenANewChunk(fileWriter, chunkWriters[subTaskId], subTaskId);
+  }
+
   private void mergeTimeValuePair(TimeValuePair from, TimeValuePair to) {
     if (!isAlign) {
       // ignore not aligned TimeValuePair with same timestamp

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java

@@ -995,4 +995,29 @@ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException {
       }
     }
   }
+
+  @Test
+  public void testSplitChunk() throws IOException {
+    TsFileResource resource = createEmptyFileAndResource(true);
+    try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) {
+      writer.startChunkGroup("d1");
+      writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue(
+          Arrays.asList("s1", "s2", "s3"),
+          new TimeRange[][] {new TimeRange[] {new TimeRange(100000, 300000)}},
+          TSEncoding.PLAIN,
+          CompressionType.LZ4,
+          Arrays.asList(true, false, false));
+      writer.endChunkGroup();
+      writer.endFile();
+    }
+    RepairUnsortedFileCompactionTask task =
+        new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, true, 0);
+    Assert.assertTrue(task.start());
+    TsFileResource target = tsFileManager.getTsFileList(false).get(0);
+    try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) {
+      List<AlignedChunkMetadata> chunkMetadataList =
+          reader.getAlignedChunkMetadata(new PlainDeviceID("root.testsg.d1"));
+      Assert.assertEquals(3, chunkMetadataList.size());
+    }
+  }
 }