diff --git a/statistics/handle/gc.go b/statistics/handle/gc.go
index 6a7500d663528..2d3238ed8f922 100644
--- a/statistics/handle/gc.go
+++ b/statistics/handle/gc.go
@@ -167,6 +167,10 @@ func (h *Handle) deleteHistStatsFromKV(physicalID int64, histID int64, isIndex i
 	if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_buckets where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
 		return err
 	}
+	// delete all fm sketch
+	if _, err := exec.ExecuteInternal(ctx, "delete from mysql.stats_fm_sketch where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
+		return err
+	}
 	return nil
 }
 
diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go
index 6e8b3b3d3f607..6d12067f8e2a8 100644
--- a/statistics/handle/handle.go
+++ b/statistics/handle/handle.go
@@ -996,6 +996,9 @@ func (h *Handle) SaveStatsToStorage(tableID int64, count int64, isIndex int, hg
 			}
 		}
 	}
+	if _, err := exec.ExecuteInternal(ctx, "delete from mysql.stats_fm_sketch where table_id = %? and is_index = %? and hist_id = %?", tableID, isIndex, hg.ID); err != nil {
+		return err
+	}
 	if fmSketch != nil {
 		if _, err = exec.ExecuteInternal(ctx, "insert into mysql.stats_fm_sketch (table_id, is_index, hist_id, value) values (%?, %?, %?, %?)", tableID, isIndex, hg.ID, fmSketch); err != nil {
 			return err
diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go
index a497371202490..fababfaa58a76 100644
--- a/statistics/handle/handle_test.go
+++ b/statistics/handle/handle_test.go
@@ -2116,3 +2116,20 @@ func (s *testStatsSuite) TestHideExtendedStatsSwitch(c *C) {
 	}
 	tk.MustQuery("show variables like 'tidb_enable_extended_stats'").Check(testkit.Rows())
 }
+
+func (s *testStatsSuite) TestDuplicateFMSketch(c *C) {
+	defer cleanEnv(c, s.store, s.do)
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("create table t(a int, b int, c int)")
+	tk.MustExec("insert into t values (1, 1, 1)")
+	tk.MustExec("analyze table t")
+	tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("3"))
+	tk.MustExec("analyze table t")
+	tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("3"))
+
+	tk.MustExec("alter table t drop column a")
+	s.do.StatsHandle().SetLastUpdateVersion(s.do.StatsHandle().LastUpdateVersion() + 1)
+	c.Assert(s.do.StatsHandle().GCStats(s.do.InfoSchema(), time.Duration(0)), IsNil)
+	tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("2"))
+}