Incremental restore: fix the issue that backfill data is not covered …

…by newTS (#54430) (#54505)

close #54426

br/pkg/task/restore.go

@@ -844,10 +844,6 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 		return errors.Trace(err)
 	}
 
-	var newTS uint64
-	if client.IsIncremental() {
-		newTS = restoreTS
-	}
 	ddlJobs := restore.FilterDDLJobs(client.GetDDLJobs(), tables)
 	ddlJobs = restore.FilterDDLJobByRules(ddlJobs, restore.DDLJobBlockListRule)
 
@@ -906,6 +902,17 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 		return errors.Trace(err)
 	}
 
+	var newTS uint64
+	if client.IsIncremental() {
+		// we need to get the new ts after execDDL
+		// or backfilled data in upstream may not be covered by
+		// the new ts.
+		// see https://github.com/pingcap/tidb/issues/54426
+		newTS, err = client.GetTSWithRetry(ctx)
+		if err != nil {
+			return errors.Trace(err)
+		}
+	}
 	// We make bigger errCh so we won't block on multi-part failed.
 	errCh := make(chan error, 32)
 

br/tests/br_incremental/run.sh

@@ -27,8 +27,9 @@ go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_P
 row_count_ori_full=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}')
 
 run_sql "CREATE TABLE IF NOT EXISTS ${DB}.${AUTO_ID_TABLE} (c1 INT);"
+run_sql "create table ${DB}.t (pk bigint primary key, val int not null);"
+run_sql "insert into ${DB}.t values (1, 11), (2, 22), (3, 33), (4, 44);"
 
-# full backup
 echo "full backup start..."
 run_br --pd $PD_ADDR backup db -s "local://$TEST_DIR/$DB/full" --db $DB
 
@@ -38,6 +39,9 @@ for i in $(seq $ROW_COUNT); do
     run_sql "INSERT INTO ${DB}.${AUTO_ID_TABLE}(c1) VALUES ($i);"
 done
 
+run_sql "create index index_val on ${DB}.t (val);"
+run_sql "update ${DB}.t set val = 66 - val;"
+
 # incremental backup
 echo "incremental backup start..."
 last_backup_ts=$(run_br validate decode --field="end-version" -s "local://$TEST_DIR/$DB/full" | grep -oE "^[0-9]+")
@@ -70,4 +74,15 @@ for i in $(seq $ROW_COUNT); do
     run_sql "INSERT INTO ${DB}.${AUTO_ID_TABLE}(c1) VALUES ($i);"
 done
 
+if run_sql "admin check table ${DB}.t;" | grep -q 'inconsistency'; then
+    echo "TEST: [$TEST_NAME] incremental restore fail on database $DB.t"
+    exit 1
+fi
+
+index_count=$(run_sql "select count(*) from $DB.t use index (index_val);" | awk '/count/{print $2}')
+if [ "${index_count}" != "4" ];then
+    echo "TEST: [$TEST_NAME] index check fail on database $DB.t"
+    exit 1
+fi
+
 run_sql "DROP DATABASE $DB;"