Improving random query generation for endtoend testing (#13460)

Co-authored-by: Andres Taylor <[email protected]>

go/test/endtoend/vtgate/queries/random/query_gen_test.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package random
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/vt/sqlparser"
+)
+
+// TestSeed makes sure that the seed is deterministic
+func TestSeed(t *testing.T) {
+	// specify the schema (that is defined in schema.sql)
+	schemaTables := []tableT{
+		{tableExpr: sqlparser.NewTableName("emp")},
+		{tableExpr: sqlparser.NewTableName("dept")},
+	}
+	schemaTables[0].addColumns([]column{
+		{name: "empno", typ: "bigint"},
+		{name: "ename", typ: "varchar"},
+		{name: "job", typ: "varchar"},
+		{name: "mgr", typ: "bigint"},
+		{name: "hiredate", typ: "date"},
+		{name: "sal", typ: "bigint"},
+		{name: "comm", typ: "bigint"},
+		{name: "deptno", typ: "bigint"},
+	}...)
+	schemaTables[1].addColumns([]column{
+		{name: "deptno", typ: "bigint"},
+		{name: "dname", typ: "varchar"},
+		{name: "loc", typ: "varchar"},
+	}...)
+
+	seed := int64(1689757943775102000)
+	genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
+	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
+	qg.randomQuery()
+	query1 := sqlparser.String(qg.stmt)
+	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
+	qg.randomQuery()
+	query2 := sqlparser.String(qg.stmt)
+	fmt.Println(query1)
+	require.Equal(t, query1, query2)
+}

go/test/endtoend/vtgate/queries/random/random_expr_test.go

@@ -17,18 +17,20 @@ limitations under the License.
 package random
 
 import (
+	"math/rand"
 	"testing"
 	"time"
 
 	"vitess.io/vitess/go/slice"
 	"vitess.io/vitess/go/vt/sqlparser"
 )
 
-// This test tests that generating a random expression with a schema does not panic
+// This test tests that generating random expressions with a schema does not panic
 func TestRandomExprWithTables(t *testing.T) {
+	// specify the schema (that is defined in schema.sql)
 	schemaTables := []tableT{
-		{name: sqlparser.NewTableName("emp")},
-		{name: sqlparser.NewTableName("dept")},
+		{tableExpr: sqlparser.NewTableName("emp")},
+		{tableExpr: sqlparser.NewTableName("dept")},
 	}
 	schemaTables[0].addColumns([]column{
 		{name: "empno", typ: "bigint"},
@@ -46,7 +48,12 @@ func TestRandomExprWithTables(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	seed := time.Now().UnixNano()
-	g := sqlparser.NewGenerator(seed, 3, slice.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
-	g.Expression()
+	for i := 0; i < 100; i++ {
+
+		seed := time.Now().UnixNano()
+		r := rand.New(rand.NewSource(seed))
+		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CanAggregate, "", 0, false)
+		g := sqlparser.NewGenerator(r, 3, slice.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
+		g.Expression(genConfig)
+	}
 }

go/test/endtoend/vtgate/queries/random/simplifier_test.go

@@ -0,0 +1,116 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package random
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"vitess.io/vitess/go/test/vschemawrapper"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/test/endtoend/utils"
+	"vitess.io/vitess/go/vt/sqlparser"
+	"vitess.io/vitess/go/vt/vtgate/planbuilder"
+	"vitess.io/vitess/go/vt/vtgate/simplifier"
+	"vitess.io/vitess/go/vt/vtgate/vindexes"
+)
+
+func TestSimplifyResultsMismatchedQuery(t *testing.T) {
+	t.Skip("Skip CI")
+
+	var queries []string
+	queries = append(queries, "select /*vt+ PLANNER=Gen4 */ (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1",
+		"select /*vt+ PLANNER=Gen4 */ distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'",
+		"select /*vt+ PLANNER=Gen4 */ distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1 where tbl0.deptno and tbl1.comm in (12, tbl0.deptno, case false when 67 then -17 when -78 then -35 end, -76 >> -68)",
+		"select /*vt+ PLANNER=Gen4 */ count(*) + 1 from emp as tbl0 order by count(*) desc",
+		"select /*vt+ PLANNER=Gen4 */ count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32",
+		"select /*vt+ PLANNER=Gen4 */ sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1",
+		"select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc",
+		"select /*vt+ PLANNER=Gen4 */ tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) = sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8",
+		"select /*vt+ PLANNER=Gen4 */ distinct count(*) as caggr1 from dept as tbl0, emp as tbl1 group by tbl1.sal having max(tbl1.comm) != true",
+		"select /*vt+ PLANNER=Gen4 */ distinct sum(tbl1.loc) as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(tbl1.dname) <= 1",
+		"select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm",
+		"select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 1 = 0",
+		"select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 'octopus'",
+		"select /*vt+ PLANNER=Gen4 */ distinct 'octopus' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.empno having count(*) = count(*)",
+		"select /*vt+ PLANNER=Gen4 */ max(tbl0.deptno) from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno",
+		"select /*vt+ PLANNER=Gen4 */ count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal")
+
+	for _, query := range queries {
+		var simplified string
+		t.Run("simplification "+query, func(t *testing.T) {
+			simplified = simplifyResultsMismatchedQuery(t, query)
+		})
+
+		t.Run("simplified "+query, func(t *testing.T) {
+			mcmp, closer := start(t)
+			defer closer()
+
+			mcmp.ExecAllowAndCompareError(simplified)
+		})
+
+		fmt.Printf("final simplified query: %s\n", simplified)
+	}
+}
+
+// given a query that errors with results mismatched, simplifyResultsMismatchedQuery returns a simpler version with the same error
+func simplifyResultsMismatchedQuery(t *testing.T, query string) string {
+	t.Helper()
+	mcmp, closer := start(t)
+	defer closer()
+
+	_, err := mcmp.ExecAllowAndCompareError(query)
+	if err == nil {
+		t.Fatalf("query (%s) does not error", query)
+	} else if !strings.Contains(err.Error(), "mismatched") {
+		t.Fatalf("query (%s) does not error with results mismatched\nError: %v", query, err)
+	}
+
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
+
+	formal, err := vindexes.LoadFormal("svschema.json")
+	require.NoError(t, err)
+	vSchema := vindexes.BuildVSchema(formal)
+	vSchemaWrapper := &vschemawrapper.VSchemaWrapper{
+		V:       vSchema,
+		Version: planbuilder.Gen4,
+	}
+
+	stmt, err := sqlparser.Parse(query)
+	require.NoError(t, err)
+
+	simplified := simplifier.SimplifyStatement(
+		stmt.(sqlparser.SelectStatement),
+		vSchemaWrapper.CurrentDb(),
+		vSchemaWrapper,
+		func(statement sqlparser.SelectStatement) bool {
+			q := sqlparser.String(statement)
+			_, newErr := mcmp.ExecAllowAndCompareError(q)
+			if newErr == nil {
+				return false
+			} else {
+				return strings.Contains(newErr.Error(), "mismatched")
+			}
+		},
+	)
+
+	return sqlparser.String(simplified)
+}

go/test/endtoend/vtgate/queries/random/svschema.json

@@ -0,0 +1,6 @@
+{
+  "keyspaces": {
+    "ks_random": {
+    }
+  }
+}