diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go
index ea4b57bd417a7..210931c1db70c 100644
--- a/planner/core/rule_column_pruning.go
+++ b/planner/core/rule_column_pruning.go
@@ -130,11 +130,13 @@ func (ls *LogicalSort) PruneColumns(parentUsedCols []*expression.Column) {
 // PruneColumns implements LogicalPlan interface.
 func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column) {
 	used := getUsedList(parentUsedCols, p.schema)
+	hasBeenUsed := false
 	for i := range used {
-		if used[i] {
-			continue
-		}
-		parentUsedCols = append(parentUsedCols, p.schema.Columns[i])
+		hasBeenUsed = hasBeenUsed || used[i]
+	}
+	if !hasBeenUsed {
+		parentUsedCols = make([]*expression.Column, len(p.schema.Columns))
+		copy(parentUsedCols, p.schema.Columns)
 	}
 	for _, child := range p.Children() {
 		child.PruneColumns(parentUsedCols)