lightning: skip restore table when source files are empty

br/pkg/lightning/restore/restore.go

@@ -1467,6 +1467,9 @@ func (rc *Controller) restoreTables(ctx context.Context) (finalErr error) {
 			if err != nil {
 				return errors.Trace(err)
 			}
+			if cp.Status < checkpoints.CheckpointStatusAllWritten && len(tableMeta.DataFiles) == 0 {
+				continue
+			}
 			igCols, err := rc.cfg.Mydumper.IgnoreColumns.GetIgnoreColumns(dbInfo.Name, tableInfo.Name, rc.cfg.Mydumper.CaseSensitive)
 			if err != nil {
 				return errors.Trace(err)
@@ -1526,7 +1529,6 @@ func (tr *TableRestore) restoreTable(
 	cp *checkpoints.TableCheckpoint,
 ) (bool, error) {
 	// 1. Load the table info.
-
 	select {
 	case <-ctx.Done():
 		return false, ctx.Err()

br/pkg/lightning/restore/table_restore.go

@@ -202,17 +202,6 @@ func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp
 		tr.logger.Error("fail to restoreEngines because indexengine is nil")
 		return errors.Errorf("table %v index engine checkpoint not found", tr.tableName)
 	}
-	// If there is an index engine only, it indicates no data needs to restore.
-	// So we can change status to imported directly and avoid opening engine.
-	if len(cp.Engines) == 1 {
-		if err := rc.saveStatusCheckpoint(pCtx, tr.tableName, indexEngineID, nil, checkpoints.CheckpointStatusImported); err != nil {
-			return errors.Trace(err)
-		}
-		if err := rc.saveStatusCheckpoint(pCtx, tr.tableName, checkpoints.WholeTableEngineID, nil, checkpoints.CheckpointStatusIndexImported); err != nil {
-			return errors.Trace(err)
-		}
-		return nil
-	}
 
 	ctx, cancel := context.WithCancel(pCtx)
 	defer cancel()
@@ -683,10 +672,7 @@ func (tr *TableRestore) postProcess(
 	forcePostProcess bool,
 	metaMgr tableMetaMgr,
 ) (bool, error) {
-	// there are no data in this table, no need to do post process
-	// this is important for tables that are just the dump table of views
-	// because at this stage, the table was already deleted and replaced by the related view
-	if !rc.backend.ShouldPostProcess() || len(cp.Engines) == 1 {
+	if !rc.backend.ShouldPostProcess() {
 		return false, nil
 	}
 

br/tests/lightning_incremental/data/incr.empty_table-schema.sql

@@ -0,0 +1 @@
+CREATE TABLE `empty_table` (id int primary key);

br/tests/lightning_incremental/data1/incr.empty_table-schema.sql

@@ -0,0 +1 @@
+CREATE TABLE `empty_table` (id int primary key);

br/tests/lightning_incremental/data1/incr.empty_table2-schema.sql

@@ -0,0 +1 @@
+CREATE TABLE `empty_table2` (id int primary key, s varchar(16));

br/tests/lightning_incremental/run.sh

@@ -18,60 +18,66 @@ set -eu
 
 check_cluster_version 4 0 0 "incremental restore" || exit 0
 
-DB_NAME=incr
+run_lightning_and_check_meta() {
+  run_lightning --backend local "$@"
+  # check metadata table is not exist
+  run_sql "SHOW DATABASES like 'lightning_metadata';"
+  check_not_contains "Database: lightning_metadata"
+}
 
-for backend in importer local; do
-  run_sql "DROP DATABASE IF EXISTS incr;"
-  run_lightning --backend $backend
+DB_NAME=incr
 
-  for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
-    run_sql "SELECT count(*) from incr.$tbl"
-    check_contains "count(*): 3"
-  done
+run_sql "DROP DATABASE IF EXISTS incr;"
+run_sql "DROP DATABASE IF EXISTS lightning_metadata;"
+run_lightning_and_check_meta
 
-  for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
-    if [ "$tbl" = "auto_random" ]; then
-      run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl"
-    else
-      run_sql "SELECT id as inc FROM incr.$tbl"
-    fi
-    check_contains 'inc: 1'
-    check_contains 'inc: 2'
-    check_contains 'inc: 3'
-  done
+for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
+  run_sql "SELECT count(*) from incr.$tbl"
+  check_contains "count(*): 3"
+done
 
-  for tbl in pk_auto_inc rowid_uk_inc; do
-    run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';"
-    check_contains "group_concat(v): a,b,c"
-  done
+for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
+  if [ "$tbl" = "auto_random" ]; then
+    run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl"
+  else
+    run_sql "SELECT id as inc FROM incr.$tbl"
+  fi
+  check_contains 'inc: 1'
+  check_contains 'inc: 2'
+  check_contains 'inc: 3'
+done
 
-  run_sql "SELECT sum(pk) from incr.uk_auto_inc;"
-  check_contains "sum(pk): 6"
+for tbl in pk_auto_inc rowid_uk_inc; do
+  run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';"
+  check_contains "group_concat(v): a,b,c"
+done
 
-  # incrementally import all data in data1
-  run_lightning --backend $backend -d "tests/$TEST_NAME/data1"
+run_sql "SELECT sum(pk) from incr.uk_auto_inc;"
+check_contains "sum(pk): 6"
 
-  for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
-    run_sql "SELECT count(*) from incr.$tbl"
-    check_contains "count(*): 6"
-  done
+# incrementally import all data in data1
+run_lightning_and_check_meta -d "tests/$TEST_NAME/data1"
 
-  for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
-    if [ "$tbl" = "auto_random" ]; then
-      run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl"
-    else
-      run_sql "SELECT id as inc FROM incr.$tbl"
-    fi
-    check_contains 'inc: 4'
-    check_contains 'inc: 5'
-    check_contains 'inc: 6'
-  done
+for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
+  run_sql "SELECT count(*) from incr.$tbl"
+  check_contains "count(*): 6"
+done
 
-  for tbl in pk_auto_inc rowid_uk_inc; do
-    run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';"
-    check_contains "group_concat(v): a,b,c,d,e,f"
-  done
+for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do
+  if [ "$tbl" = "auto_random" ]; then
+    run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl"
+  else
+    run_sql "SELECT id as inc FROM incr.$tbl"
+  fi
+  check_contains 'inc: 4'
+  check_contains 'inc: 5'
+  check_contains 'inc: 6'
+done
 
-  run_sql "SELECT sum(pk) from incr.uk_auto_inc;"
-  check_contains "sum(pk): 21"
+for tbl in pk_auto_inc rowid_uk_inc; do
+  run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';"
+  check_contains "group_concat(v): a,b,c,d,e,f"
 done
+
+run_sql "SELECT sum(pk) from incr.uk_auto_inc;"
+check_contains "sum(pk): 21"