lightning: skip restore table when source files are empty (#28189) (#…

…32389) close #28144
pingcap · Feb 18, 2022 · 0063773 · 0063773
1 parent 164ffbe
commit 0063773
Show file tree

Hide file tree

Showing 7 changed files with 60 additions and 63 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -1,5 +1,5 @@
 run:
-  timeout: 7m
+  timeout: 10m
 linters:
   disable-all: true
   enable:

diff --git a/br/pkg/lightning/restore/restore.go b/br/pkg/lightning/restore/restore.go
@@ -1464,6 +1464,9 @@ func (rc *Controller) restoreTables(ctx context.Context) error {
 			if err != nil {
 				return errors.Trace(err)
 			}
+			if cp.Status < checkpoints.CheckpointStatusAllWritten && len(tableMeta.DataFiles) == 0 {
+				continue
+			}
 			igCols, err := rc.cfg.Mydumper.IgnoreColumns.GetIgnoreColumns(dbInfo.Name, tableInfo.Name, rc.cfg.Mydumper.CaseSensitive)
 			if err != nil {
 				return errors.Trace(err)
@@ -1529,7 +1532,6 @@ func (tr *TableRestore) restoreTable(
 	cp *checkpoints.TableCheckpoint,
 ) (bool, error) {
 	// 1. Load the table info.
-
 	select {
 	case <-ctx.Done():
 		return false, ctx.Err()

diff --git a/br/pkg/lightning/restore/table_restore.go b/br/pkg/lightning/restore/table_restore.go
@@ -196,17 +196,6 @@ func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp
 		tr.logger.Error("fail to restoreEngines because indexengine is nil")
 		return errors.Errorf("table %v index engine checkpoint not found", tr.tableName)
 	}
-	// If there is an index engine only, it indicates no data needs to restore.
-	// So we can change status to imported directly and avoid opening engine.
-	if len(cp.Engines) == 1 {
-		if err := rc.saveStatusCheckpoint(pCtx, tr.tableName, indexEngineID, nil, checkpoints.CheckpointStatusImported); err != nil {
-			return errors.Trace(err)
-		}
-		if err := rc.saveStatusCheckpoint(pCtx, tr.tableName, checkpoints.WholeTableEngineID, nil, checkpoints.CheckpointStatusIndexImported); err != nil {
-			return errors.Trace(err)
-		}
-		return nil
-	}
 
 	ctx, cancel := context.WithCancel(pCtx)
 	defer cancel()
@@ -678,10 +667,7 @@ func (tr *TableRestore) postProcess(
 	forcePostProcess bool,
 	metaMgr tableMetaMgr,
 ) (bool, error) {
-	// there are no data in this table, no need to do post process
-	// this is important for tables that are just the dump table of views
-	// because at this stage, the table was already deleted and replaced by the related view
-	if !rc.backend.ShouldPostProcess() || len(cp.Engines) == 1 {
+	if !rc.backend.ShouldPostProcess() {
 		return false, nil
 	}
 

diff --git a/br/tests/lightning_incremental/data/incr.empty_table-schema.sql b/br/tests/lightning_incremental/data/incr.empty_table-schema.sql
@@ -0,0 +1 @@
+CREATE TABLE `empty_table` (id int primary key);
diff --git a/br/tests/lightning_incremental/data1/incr.empty_table-schema.sql b/br/tests/lightning_incremental/data1/incr.empty_table-schema.sql
@@ -0,0 +1 @@
+CREATE TABLE `empty_table` (id int primary key);
diff --git a/br/tests/lightning_incremental/data1/incr.empty_table2-schema.sql b/br/tests/lightning_incremental/data1/incr.empty_table2-schema.sql
@@ -0,0 +1 @@
+CREATE TABLE `empty_table2` (id int primary key, s varchar(16));
diff --git a/br/tests/lightning_incremental/run.sh b/br/tests/lightning_incremental/run.sh
@@ -18,60 +18,66 @@ set -eu
 
 check_cluster_version 4 0 0 "incremental restore" || exit 0
 
-DB_NAME=incr
+run_lightning_and_check_meta() {
+  run_lightning --backend local "$@"
+  # check metadata table is not exist
+  run_sql "SHOW DATABASES like 'lightning_metadata';"
+  check_not_contains "Database: lightning_metadata"
+}
 
-for backend in importer local; do
-  run_sql "DROP DATABASE IF EXISTS incr;"
-  run_lightning --backend $backend
+DB_NAME=incr
 
-  for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
-    run_sql "SELECT count(*) from incr.$tbl"
-    check_contains "count(*): 3"
-  done
+run_sql "DROP DATABASE IF EXISTS incr;"
+run_sql "DROP DATABASE IF EXISTS lightning_metadata;"
+run_lightning_and_check_meta
 
-  for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
-    if [ "$tbl" = "auto_random" ]; then
-      run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl"
-    else
-      run_sql "SELECT id as inc FROM incr.$tbl"
-    fi
-    check_contains 'inc: 1'
-    check_contains 'inc: 2'
-    check_contains 'inc: 3'
-  done
+for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
+  run_sql "SELECT count(*) from incr.$tbl"
+  check_contains "count(*): 3"
+done
 
-  for tbl in pk_auto_inc rowid_uk_inc; do
-    run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';"
-    check_contains "group_concat(v): a,b,c"
-  done
+for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
+  if [ "$tbl" = "auto_random" ]; then
+    run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl"
+  else
+    run_sql "SELECT id as inc FROM incr.$tbl"
+  fi
+  check_contains 'inc: 1'
+  check_contains 'inc: 2'
+  check_contains 'inc: 3'
+done
 
-  run_sql "SELECT sum(pk) from incr.uk_auto_inc;"
-  check_contains "sum(pk): 6"
+for tbl in pk_auto_inc rowid_uk_inc; do
+  run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';"
+  check_contains "group_concat(v): a,b,c"
+done
 
-  # incrementally import all data in data1
-  run_lightning --backend $backend -d "tests/$TEST_NAME/data1"
+run_sql "SELECT sum(pk) from incr.uk_auto_inc;"
+check_contains "sum(pk): 6"
 
-  for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
-    run_sql "SELECT count(*) from incr.$tbl"
-    check_contains "count(*): 6"
-  done
+# incrementally import all data in data1
+run_lightning_and_check_meta -d "tests/$TEST_NAME/data1"
 
-  for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
-    if [ "$tbl" = "auto_random" ]; then
-      run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl"
-    else
-      run_sql "SELECT id as inc FROM incr.$tbl"
-    fi
-    check_contains 'inc: 4'
-    check_contains 'inc: 5'
-    check_contains 'inc: 6'
-  done
+for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
+  run_sql "SELECT count(*) from incr.$tbl"
+  check_contains "count(*): 6"
+done
 
-  for tbl in pk_auto_inc rowid_uk_inc; do
-    run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';"
-    check_contains "group_concat(v): a,b,c,d,e,f"
-  done
+for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
+  if [ "$tbl" = "auto_random" ]; then
+    run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl"
+  else
+    run_sql "SELECT id as inc FROM incr.$tbl"
+  fi
+  check_contains 'inc: 4'
+  check_contains 'inc: 5'
+  check_contains 'inc: 6'
+done
 
-  run_sql "SELECT sum(pk) from incr.uk_auto_inc;"
-  check_contains "sum(pk): 21"
+for tbl in pk_auto_inc rowid_uk_inc; do
+  run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';"
+  check_contains "group_concat(v): a,b,c,d,e,f"
 done
+
+run_sql "SELECT sum(pk) from incr.uk_auto_inc;"
+check_contains "sum(pk): 21"