Foreign key on update action with non literal values

go/test/endtoend/vtgate/foreignkey/fk_test.go

@@ -27,6 +27,7 @@ import (
 
 	"vitess.io/vitess/go/test/endtoend/cluster"
 	"vitess.io/vitess/go/test/endtoend/utils"
+	"vitess.io/vitess/go/vt/log"
 	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
 	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/vtgate/vtgateconn"
@@ -775,6 +776,181 @@ func TestFkScenarios(t *testing.T) {
 	}
 }
 
+// TestFkQueries is for testing a specific set of queries one after the other.
+func TestFkQueries(t *testing.T) {
+	// Wait for schema-tracking to be complete.
+	waitForSchemaTrackingForFkTables(t)
+	// Remove all the foreign key constraints for all the replicas.
+	// We can then verify that the replica, and the primary have the same data, to ensure
+	// that none of the queries ever lead to cascades/updates on MySQL level.
+	for _, ks := range []string{shardedKs, unshardedKs} {
+		replicas := getReplicaTablets(ks)
+		for _, replica := range replicas {
+			removeAllForeignKeyConstraints(t, replica, ks)
+		}
+	}
+
+	testcases := []struct {
+		name    string
+		queries []string
+	}{
+		{
+			name: "Non-literal update",
+			queries: []string{
+				"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"update fk_t10 set col = id + 3",
+			},
+		}, {
+			name: "Non-literal update with order by",
+			queries: []string{
+				"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"update fk_t10 set col = id + 3 order by id desc",
+			},
+		}, {
+			name: "Non-literal update with order by that require parent and child foreign keys verification - success",
+			queries: []string{
+				"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8)",
+				"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t13 (id, col) values (1,1),(2,2)",
+				"update fk_t11 set col = id + 3 where id >= 3",
+			},
+		}, {
+			name: "Non-literal update with order by that require parent and child foreign keys verification - parent fails",
+			queries: []string{
+				"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"update fk_t11 set col = id + 3",
+			},
+		}, {
+			name: "Non-literal update with order by that require parent and child foreign keys verification - child fails",
+			queries: []string{
+				"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8)",
+				"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t13 (id, col) values (1,1),(2,2)",
+				"update fk_t11 set col = id + 3",
+			},
+		}, {
+			name: "Single column update in a multi-col table - success",
+			queries: []string{
+				"insert into fk_multicol_t1 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
+				"insert into fk_multicol_t2 (id, cola, colb) values (1, 1, 1)",
+				"update fk_multicol_t1 set colb = 4 + (colb) where id = 2",
+			},
+		}, {
+			name: "Single column update in a multi-col table - restrict failure",
+			queries: []string{
+				"insert into fk_multicol_t1 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
+				"insert into fk_multicol_t2 (id, cola, colb) values (1, 1, 1)",
+				"update fk_multicol_t1 set colb = 4 + (colb) where id = 1",
+			},
+		}, {
+			name: "Single column update in multi-col table - cascade and set null",
+			queries: []string{
+				"insert into fk_multicol_t15 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
+				"insert into fk_multicol_t16 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
+				"insert into fk_multicol_t17 (id, cola, colb) values (1, 1, 1), (2, 2, 2)",
+				"update fk_multicol_t15 set colb = 4 + (colb) where id = 1",
+			},
+		}, {
+			name: "Non literal update that evaluates to NULL - restricted",
+			queries: []string{
+				"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t13 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"update fk_t10 set col = id + null where id = 1",
+			},
+		}, {
+			name: "Non literal update that evaluates to NULL - success",
+			queries: []string{
+				"insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)",
+				"update fk_t10 set col = id + null where id = 1",
+			},
+		}, {
+			name: "Multi column foreign key update with one literal and one non-literal update",
+			queries: []string{
+				"insert into fk_multicol_t15 (id, cola, colb) values (1,1,1),(2,2,2)",
+				"insert into fk_multicol_t16 (id, cola, colb) values (1,1,1),(2,2,2)",
+				"update fk_multicol_t15 set cola = 3, colb = (id * 2) - 2",
+			},
+		},
+	}
+
+	for _, testcase := range testcases {
+		t.Run(testcase.name, func(t *testing.T) {
+			mcmp, closer := start(t)
+			defer closer()
+			_ = utils.Exec(t, mcmp.VtConn, "use `uks`")
+
+			// Ensure that the Vitess database is originally empty
+			ensureDatabaseState(t, mcmp.VtConn, true)
+			ensureDatabaseState(t, mcmp.MySQLConn, true)
+
+			for _, query := range testcase.queries {
+				_, _ = mcmp.ExecAllowAndCompareError(query)
+				if t.Failed() {
+					break
+				}
+			}
+
+			// ensure Vitess database has some data. This ensures not all the commands failed.
+			ensureDatabaseState(t, mcmp.VtConn, false)
+			// Verify the consistency of the data.
+			verifyDataIsCorrect(t, mcmp, 1)
+		})
+	}
+}
+
+// TestFkOneCase is for testing a specific set of queries. On the CI this test won't run since we'll keep the queries empty.
+func TestFkOneCase(t *testing.T) {
+	queries := []string{}
+	if len(queries) == 0 {
+		t.Skip("No queries to test")
+	}
+	// Wait for schema-tracking to be complete.
+	waitForSchemaTrackingForFkTables(t)
+	// Remove all the foreign key constraints for all the replicas.
+	// We can then verify that the replica, and the primary have the same data, to ensure
+	// that none of the queries ever lead to cascades/updates on MySQL level.
+	for _, ks := range []string{shardedKs, unshardedKs} {
+		replicas := getReplicaTablets(ks)
+		for _, replica := range replicas {
+			removeAllForeignKeyConstraints(t, replica, ks)
+		}
+	}
+
+	mcmp, closer := start(t)
+	defer closer()
+	_ = utils.Exec(t, mcmp.VtConn, "use `uks`")
+
+	// Ensure that the Vitess database is originally empty
+	ensureDatabaseState(t, mcmp.VtConn, true)
+	ensureDatabaseState(t, mcmp.MySQLConn, true)
+
+	for _, query := range queries {
+		_, _ = mcmp.ExecAllowAndCompareError(query)
+		if t.Failed() {
+			log.Errorf("Query failed - %v", query)
+			break
+		}
+	}
+	vitessData := collectFkTablesState(mcmp.VtConn)
+	for idx, table := range fkTables {
+		log.Errorf("Vitess data for %v -\n%v", table, vitessData[idx].Rows)
+	}
+
+	// ensure Vitess database has some data. This ensures not all the commands failed.
+	ensureDatabaseState(t, mcmp.VtConn, false)
+	// Verify the consistency of the data.
+	verifyDataIsCorrect(t, mcmp, 1)
+}
+
 func TestCyclicFks(t *testing.T) {
 	mcmp, closer := start(t)
 	defer closer()

go/test/endtoend/vtgate/foreignkey/utils_test.go

@@ -19,15 +19,21 @@ package foreignkey
 import (
 	"database/sql"
 	"fmt"
+	"math/rand"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
+	"vitess.io/vitess/go/mysql"
+	"vitess.io/vitess/go/sqltypes"
 	"vitess.io/vitess/go/test/endtoend/cluster"
 	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
+var supportedOpps = []string{"*", "+", "-"}
+
 // getTestName prepends whether the test is for a sharded keyspace or not to the test name.
 func getTestName(testName string, testSharded bool) string {
 	if testSharded {
@@ -41,6 +47,32 @@ func isMultiColFkTable(tableName string) bool {
 	return strings.Contains(tableName, "multicol")
 }
 
+func (fz *fuzzer) generateExpression(length int, cols ...string) string {
+	expr := fz.getColOrInt(cols...)
+	if length == 1 {
+		return expr
+	}
+	rhsExpr := fz.generateExpression(length-1, cols...)
+	op := supportedOpps[rand.Intn(len(supportedOpps))]
+	return fmt.Sprintf("%v %s (%v)", expr, op, rhsExpr)
+}
+
+// getColOrInt gets a column or an integer/NULL literal with equal probability.
+func (fz *fuzzer) getColOrInt(cols ...string) string {
+	if len(cols) == 0 || rand.Intn(2) == 0 {
+		return convertIntValueToString(rand.Intn(1 + fz.maxValForCol))
+	}
+	return cols[rand.Intn(len(cols))]
+}
+
+// convertIntValueToString converts the given value to a string
+func convertIntValueToString(value int) string {
+	if value == 0 {
+		return "NULL"
+	}
+	return fmt.Sprintf("%d", value)
+}
+
 // waitForSchemaTrackingForFkTables waits for schema tracking to have run and seen the tables used
 // for foreign key tests.
 func waitForSchemaTrackingForFkTables(t *testing.T) {
@@ -142,3 +174,85 @@ func compareVitessAndMySQLErrors(t *testing.T, vtErr, mysqlErr error) {
 	out := fmt.Sprintf("Vitess and MySQL are not erroring the same way.\nVitess error: %v\nMySQL error: %v", vtErr, mysqlErr)
 	t.Error(out)
 }
+
+// ensureDatabaseState ensures that the database is either empty or not.
+func ensureDatabaseState(t *testing.T, vtconn *mysql.Conn, empty bool) {
+	results := collectFkTablesState(vtconn)
+	isEmpty := true
+	for _, res := range results {
+		if len(res.Rows) > 0 {
+			isEmpty = false
+		}
+	}
+	require.Equal(t, isEmpty, empty)
+}
+
+// verifyDataIsCorrect verifies that the data in MySQL database matches the data in the Vitess database.
+func verifyDataIsCorrect(t *testing.T, mcmp utils.MySQLCompare, concurrency int) {
+	// For single concurrent thread, we run all the queries on both MySQL and Vitess, so we can verify correctness
+	// by just checking if the data in MySQL and Vitess match.
+	if concurrency == 1 {
+		for _, table := range fkTables {
+			query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table)
+			mcmp.Exec(query)
+		}
+	} else {
+		// For higher concurrency, we don't have MySQL data to verify everything is fine,
+		// so we'll have to do something different.
+		// We run LEFT JOIN queries on all the parent and child tables linked by foreign keys
+		// to make sure that nothing is broken in the database.
+		for _, reference := range fkReferences {
+			query := fmt.Sprintf("select %v.id from %v left join %v on (%v.col = %v.col) where %v.col is null and %v.col is not null", reference.childTable, reference.childTable, reference.parentTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable)
+			if isMultiColFkTable(reference.childTable) {
+				query = fmt.Sprintf("select %v.id from %v left join %v on (%v.cola = %v.cola and %v.colb = %v.colb) where %v.cola is null and %v.cola is not null and %v.colb is not null", reference.childTable, reference.childTable, reference.parentTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable, reference.childTable)
+			}
+			res, err := mcmp.VtConn.ExecuteFetch(query, 1000, false)
+			require.NoError(t, err)
+			require.Zerof(t, len(res.Rows), "Query %v gave non-empty results", query)
+		}
+	}
+	// We also verify that the results in Primary and Replica table match as is.
+	for _, keyspace := range clusterInstance.Keyspaces {
+		for _, shard := range keyspace.Shards {
+			var primaryTab, replicaTab *cluster.Vttablet
+			for _, vttablet := range shard.Vttablets {
+				if vttablet.Type == "primary" {
+					primaryTab = vttablet
+				} else {
+					replicaTab = vttablet
+				}
+			}
+			require.NotNil(t, primaryTab)
+			require.NotNil(t, replicaTab)
+			checkReplicationHealthy(t, replicaTab)
+			cluster.WaitForReplicationPos(t, primaryTab, replicaTab, true, 1*time.Minute)
+			primaryConn, err := utils.GetMySQLConn(primaryTab, fmt.Sprintf("vt_%v", keyspace.Name))
+			require.NoError(t, err)
+			replicaConn, err := utils.GetMySQLConn(replicaTab, fmt.Sprintf("vt_%v", keyspace.Name))
+			require.NoError(t, err)
+			primaryRes := collectFkTablesState(primaryConn)
+			replicaRes := collectFkTablesState(replicaConn)
+			verifyDataMatches(t, primaryRes, replicaRes)
+		}
+	}
+}
+
+// verifyDataMatches verifies that the two list of results are the same.
+func verifyDataMatches(t *testing.T, resOne []*sqltypes.Result, resTwo []*sqltypes.Result) {
+	require.EqualValues(t, len(resTwo), len(resOne), "Res 1 - %v, Res 2 - %v", resOne, resTwo)
+	for idx, resultOne := range resOne {
+		resultTwo := resTwo[idx]
+		require.True(t, resultOne.Equal(resultTwo), "Data for %v doesn't match\nRows 1\n%v\nRows 2\n%v", fkTables[idx], resultOne.Rows, resultTwo.Rows)
+	}
+}
+
+// collectFkTablesState collects the data stored in the foreign key tables for the given connection.
+func collectFkTablesState(conn *mysql.Conn) []*sqltypes.Result {
+	var tablesData []*sqltypes.Result
+	for _, table := range fkTables {
+		query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table)
+		res, _ := conn.ExecuteFetch(query, 10000, true)
+		tablesData = append(tablesData, res)
+	}
+	return tablesData
+}