From 3d10df52804784fd00bbf23a30cd210fbfa95d27 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Thu, 29 Jun 2023 21:27:34 -0700
Subject: [PATCH 01/27] added flag for if the randomly generated query can have
 aggregation

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 84 ++++++++++++-------
 .../vtgate/queries/random/random_test.go      | 78 ++++++-----------
 2 files changed, 82 insertions(+), 80 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index d673a904189..974b17807e6 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -136,21 +136,43 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 	// create both tables and join at the same time since both occupy the from clause
 	tables, isJoin := createTablesAndJoin(schemaTables, sel)
 
+	// canAggregate determines if the query will have
+	// aggregate columns, group by, and having
+	canAggregate := rand.Intn(4) > 0
+
 	var (
 		groupBy          sqlparser.GroupBy
 		groupSelectExprs sqlparser.SelectExprs
 		grouping         []column
+		aggrExprs        sqlparser.SelectExprs
+		aggregates       []column
 	)
 	// TODO: distinct makes vitess think there is grouping on aggregation columns
-	if testFailingQueries || !isDistinct {
-		groupBy, groupSelectExprs, grouping = createGroupBy(tables, maxGroupBy)
-		sel.AddSelectExprs(groupSelectExprs)
-		sel.GroupBy = groupBy
+	if canAggregate {
+		if testFailingQueries || !isDistinct {
+			// group by
+			groupBy, groupSelectExprs, grouping = createGroupBy(tables, maxGroupBy)
+			sel.AddSelectExprs(groupSelectExprs)
+			sel.GroupBy = groupBy
+		}
 
-	}
+		// aggregate columns
+		aggrExprs, aggregates = createAggregations(tables, maxAggrs)
+		sel.AddSelectExprs(aggrExprs)
+
+		// having
+		isHaving := rand.Intn(2) < 1
+		if isHaving {
+			sel.AddHaving(sqlparser.AndExpressions(createHavingPredicates(tables)...))
+			if rand.Intn(2) < 1 && testFailingQueries {
+				// TODO: having can only contain aggregate or grouping columns in mysql, works fine in vitess
+				// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
+				sel.AddHaving(sqlparser.AndExpressions(createWherePredicates(tables, false)...))
+			}
+		}
+		// TODO: use sqlparser.ExprGenerator to generate a random expression with aggregation functions
 
-	aggrExprs, aggregates := createAggregations(tables, maxAggrs)
-	sel.AddSelectExprs(aggrExprs)
+	}
 
 	// can add both aggregate and grouping columns to order by
 	// TODO: order fails with distinct and outer joins
@@ -171,42 +193,44 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 		sel.AddWhere(predRandomExpr)
 	}
 
-	// having
-	isHaving := rand.Intn(2) < 1
-	if isHaving {
-		sel.AddHaving(sqlparser.AndExpressions(createHavingPredicates(tables)...))
-		if rand.Intn(2) < 1 && testFailingQueries {
-			// TODO: having can only contain aggregate or grouping columns in mysql, works fine in vitess
-			// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
-			sel.AddHaving(sqlparser.AndExpressions(createWherePredicates(tables, false)...))
-		}
-	}
-	// TODO: use sqlparser.ExprGenerator to generate a random expression with aggregation functions
-
 	// only add a limit if the grouping columns are ordered
 	// TODO: limit fails a lot
 	if rand.Intn(2) < 1 && (isOrdered || len(groupBy) == 0) && testFailingQueries {
 		sel.Limit = createLimit()
 	}
 
-	var newTable tableT
+	var (
+		newTable tableT
+		f        func() sqlparser.Expr
+		typ      string
+	)
 	// add random expression to select
 	// TODO: random expressions cause a lot of failures
 	isRandomExpr := rand.Intn(2) < 1 && testFailingQueries
-	var (
-		randomExpr sqlparser.Expr
-		typ        string
-	)
+
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
 	if testFailingQueries {
-		randomExpr = getRandomExpr(tables)
+		f = func() sqlparser.Expr { return getRandomExpr(tables) }
 	} else {
-		randomExpr = getRandomExpr(nil)
+		f = func() sqlparser.Expr { return getRandomExpr(nil) }
 	}
 
 	// make sure we have at least one select expression
-	if isRandomExpr || len(sel.SelectExprs) == 0 {
+	for isRandomExpr || len(sel.SelectExprs) == 0 {
+		// TODO: if the random expression is an int literal,
+		// TODO: and if the query is (potentially) an aggregate query,
+		// TODO: then we must group by the random expression,
+		// TODO: but we cannot do this for int literals,
+		// TODO: so we loop until we get a non-int-literal random expression
+		// TODO: this is necessary because grouping by the alias (crandom0) currently fails on vitess
+		randomExpr := f()
+		literal, ok := randomExpr.(*sqlparser.Literal)
+		isIntLiteral := ok && literal.Type == sqlparser.IntVal
+		if isIntLiteral && canAggregate {
+			continue
+		}
+
 		// TODO: select distinct [literal] fails
 		sel.Distinct = false
 
@@ -217,7 +241,11 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 		})
 
 		// make sure to add the random expression to group by for only_full_group_by
-		sel.AddGroupBy(randomExpr)
+		if canAggregate {
+			sel.AddGroupBy(randomExpr)
+		}
+
+		break
 	}
 
 	// add them to newTable
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index a51b919e0dc..440fedd7e79 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -78,15 +78,11 @@ func helperTest(t *testing.T, query string) {
 }
 
 func TestMustFix(t *testing.T) {
-	t.Skip("Skip CI")
+	//t.Skip("Skip CI")
 
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
-	// mismatched results
-	// sum values returned as int64 instead of decimal
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ sum(tbl1.sal) as caggr1 from emp as tbl0, emp as tbl1 group by tbl1.ename order by tbl1.ename asc")
-
 	// mismatched results
 	// limit >= 9 works
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) = sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8")
@@ -110,10 +106,6 @@ func TestMustFix(t *testing.T) {
 	// mismatched results
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct 'octopus' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.empno having count(*) = count(*)")
 
-	// mismatched results
-	// previously failing, then succeeding query, now failing again
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(tbl0.deptno) from dept as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job limit 3")
-
 	// mismatched results (group by + right join)
 	// left instead of right works
 	// swapping tables and predicates and changing to left fails
@@ -132,7 +124,7 @@ func TestMustFix(t *testing.T) {
 }
 
 func TestKnownFailures(t *testing.T) {
-	t.Skip("Skip CI")
+	//t.Skip("Skip CI")
 
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
@@ -228,6 +220,7 @@ func TestRandom(t *testing.T) {
 	endBy := time.Now().Add(1 * time.Second)
 
 	var queryCount int
+	// continue testing after an error if and only if testFailingQueries is true
 	for time.Now().Before(endBy) && (!t.Failed() || testFailingQueries) {
 		query := sqlparser.String(randomQuery(schemaTables, 3, 3))
 		_, vtErr := mcmp.ExecAllowAndCompareError(query)
@@ -257,6 +250,7 @@ func TestRandom(t *testing.T) {
 	fmt.Printf("Queries successfully executed: %d\n", queryCount)
 }
 
+// these queries were previously failing and have now been fixed
 func TestBuggyQueries(t *testing.T) {
 	mcmp, closer := start(t)
 	defer closer()
@@ -264,47 +258,27 @@ func TestBuggyQueries(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*), count(*), count(*) from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.deptno group by tbl1.empno order by tbl1.empno",
-		`[[INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)]]`)
-	//mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(tbl0.deptno) from dept as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job limit 3",
-	//	`[[INT64(8)] [INT64(16)] [INT64(12)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*), count(*) from emp as tbl0 group by tbl0.empno order by tbl0.empno",
-		`[[INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ distinct count(*), tbl0.loc from dept as tbl0 group by tbl0.loc",
-		`[[INT64(1) VARCHAR("BOSTON")] [INT64(1) VARCHAR("CHICAGO")] [INT64(1) VARCHAR("DALLAS")] [INT64(1) VARCHAR("NEW YORK")]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ distinct count(*) from dept as tbl0 group by tbl0.loc",
-		`[[INT64(1)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ sum(tbl1.comm) from emp as tbl0, emp as tbl1",
-		`[[DECIMAL(30800)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl1.mgr, tbl1.mgr, count(*) from emp as tbl1 group by tbl1.mgr",
-		`[[NULL NULL INT64(1)] [INT64(7566) INT64(7566) INT64(2)] [INT64(7698) INT64(7698) INT64(5)] [INT64(7782) INT64(7782) INT64(1)] [INT64(7788) INT64(7788) INT64(1)] [INT64(7839) INT64(7839) INT64(3)] [INT64(7902) INT64(7902) INT64(1)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl1.mgr, tbl1.mgr, count(*) from emp as tbl0, emp as tbl1 group by tbl1.mgr",
-		`[[NULL NULL INT64(14)] [INT64(7566) INT64(7566) INT64(28)] [INT64(7698) INT64(7698) INT64(70)] [INT64(7782) INT64(7782) INT64(14)] [INT64(7788) INT64(7788) INT64(14)] [INT64(7839) INT64(7839) INT64(42)] [INT64(7902) INT64(7902) INT64(14)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*), count(*), count(tbl0.comm) from emp as tbl0, emp as tbl1 join dept as tbl2",
-		`[[INT64(784) INT64(784) INT64(224)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0, dept as tbl1",
-		`[[INT64(16) INT64(16)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0",
-		`[[INT64(4)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ min(tbl0.loc) from dept as tbl0",
-		`[[VARCHAR("BOSTON")]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl1.empno, max(tbl1.job) from dept as tbl0, emp as tbl1 group by tbl1.empno",
-		`[[INT64(7369) VARCHAR("CLERK")] [INT64(7499) VARCHAR("SALESMAN")] [INT64(7521) VARCHAR("SALESMAN")] [INT64(7566) VARCHAR("MANAGER")] [INT64(7654) VARCHAR("SALESMAN")] [INT64(7698) VARCHAR("MANAGER")] [INT64(7782) VARCHAR("MANAGER")] [INT64(7788) VARCHAR("ANALYST")] [INT64(7839) VARCHAR("PRESIDENT")] [INT64(7844) VARCHAR("SALESMAN")] [INT64(7876) VARCHAR("CLERK")] [INT64(7900) VARCHAR("CLERK")] [INT64(7902) VARCHAR("ANALYST")] [INT64(7934) VARCHAR("CLERK")]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl1.ename, max(tbl0.comm) from emp as tbl0, emp as tbl1 group by tbl1.ename",
-		`[[VARCHAR("ADAMS") INT64(1400)] [VARCHAR("ALLEN") INT64(1400)] [VARCHAR("BLAKE") INT64(1400)] [VARCHAR("CLARK") INT64(1400)] [VARCHAR("FORD") INT64(1400)] [VARCHAR("JAMES") INT64(1400)] [VARCHAR("JONES") INT64(1400)] [VARCHAR("KING") INT64(1400)] [VARCHAR("MARTIN") INT64(1400)] [VARCHAR("MILLER") INT64(1400)] [VARCHAR("SCOTT") INT64(1400)] [VARCHAR("SMITH") INT64(1400)] [VARCHAR("TURNER") INT64(1400)] [VARCHAR("WARD") INT64(1400)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl0.dname, tbl0.dname, min(tbl0.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl0.dname",
-		`[[VARCHAR("ACCOUNTING") VARCHAR("ACCOUNTING") INT64(10)] [VARCHAR("OPERATIONS") VARCHAR("OPERATIONS") INT64(40)] [VARCHAR("RESEARCH") VARCHAR("RESEARCH") INT64(20)] [VARCHAR("SALES") VARCHAR("SALES") INT64(30)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl0.dname, min(tbl1.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl1.dname",
-		`[[VARCHAR("ACCOUNTING") INT64(10)] [VARCHAR("ACCOUNTING") INT64(40)] [VARCHAR("ACCOUNTING") INT64(20)] [VARCHAR("ACCOUNTING") INT64(30)] [VARCHAR("OPERATIONS") INT64(10)] [VARCHAR("OPERATIONS") INT64(40)] [VARCHAR("OPERATIONS") INT64(20)] [VARCHAR("OPERATIONS") INT64(30)] [VARCHAR("RESEARCH") INT64(10)] [VARCHAR("RESEARCH") INT64(40)] [VARCHAR("RESEARCH") INT64(20)] [VARCHAR("RESEARCH") INT64(30)] [VARCHAR("SALES") INT64(10)] [VARCHAR("SALES") INT64(40)] [VARCHAR("SALES") INT64(20)] [VARCHAR("SALES") INT64(30)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ max(tbl0.hiredate) from emp as tbl0",
-		`[[DATE("1983-01-12")]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0, count(*) as caggr1 from dept as tbl0 left join dept as tbl1 on tbl1.loc = tbl1.dname",
-		`[[INT64(10) INT64(4)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(tbl1.loc) as caggr0 from dept as tbl1 left join dept as tbl2 on tbl1.loc = tbl2.loc where (tbl2.deptno)",
-		`[[INT64(4)]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ sum(tbl1.ename), min(tbl0.empno) from emp as tbl0, emp as tbl1 left join dept as tbl2 on tbl1.job = tbl2.loc and tbl1.comm = tbl2.deptno where ('trout') and tbl0.deptno = tbl1.comm",
-		`[[NULL NULL]]`)
-	mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ distinct max(tbl0.deptno), count(tbl0.job) from emp as tbl0, dept as tbl1 left join dept as tbl2 on tbl1.dname = tbl2.loc and tbl1.dname = tbl2.loc where (tbl2.loc) and tbl0.deptno = tbl1.deptno",
-		`[[NULL INT64(0)]]`)
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ sum(tbl1.sal) as caggr1 from emp as tbl0, emp as tbl1 group by tbl1.ename order by tbl1.ename asc")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(*), count(*), count(*) from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.deptno group by tbl1.empno order by tbl1.empno")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(tbl0.deptno) from dept as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job limit 3")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(*), count(*) from emp as tbl0 group by tbl0.empno order by tbl0.empno")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ distinct count(*), tbl0.loc from dept as tbl0 group by tbl0.loc")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ distinct count(*) from dept as tbl0 group by tbl0.loc")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ sum(tbl1.comm) from emp as tbl0, emp as tbl1")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ tbl1.mgr, tbl1.mgr, count(*) from emp as tbl1 group by tbl1.mgr")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ tbl1.mgr, tbl1.mgr, count(*) from emp as tbl0, emp as tbl1 group by tbl1.mgr")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(*), count(*), count(tbl0.comm) from emp as tbl0, emp as tbl1 join dept as tbl2")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0, dept as tbl1")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ min(tbl0.loc) from dept as tbl0")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ tbl1.empno, max(tbl1.job) from dept as tbl0, emp as tbl1 group by tbl1.empno")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ tbl1.ename, max(tbl0.comm) from emp as tbl0, emp as tbl1 group by tbl1.ename")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ tbl0.dname, tbl0.dname, min(tbl0.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl0.dname")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ tbl0.dname, min(tbl1.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl1.dname")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ max(tbl0.hiredate) from emp as tbl0")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0, count(*) as caggr1 from dept as tbl0 left join dept as tbl1 on tbl1.loc = tbl1.dname")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(tbl1.loc) as caggr0 from dept as tbl1 left join dept as tbl2 on tbl1.loc = tbl2.loc where (tbl2.deptno)")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ sum(tbl1.ename), min(tbl0.empno) from emp as tbl0, emp as tbl1 left join dept as tbl2 on tbl1.job = tbl2.loc and tbl1.comm = tbl2.deptno where ('trout') and tbl0.deptno = tbl1.comm")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ distinct max(tbl0.deptno), count(tbl0.job) from emp as tbl0, dept as tbl1 left join dept as tbl2 on tbl1.dname = tbl2.loc and tbl1.dname = tbl2.loc where (tbl2.loc) and tbl0.deptno = tbl1.deptno")
 
 }

From b40e319f4dce29c7ee7883a5522f9545ad276585 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Tue, 4 Jul 2023 01:42:41 -0700
Subject: [PATCH 02/27] added ExprGeneratorConfig to random expression
 generation

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        |  22 +-
 .../vtgate/queries/random/random_expr_test.go |   7 +-
 .../vtgate/queries/random/random_test.go      |  12 +-
 go/vt/sqlparser/precedence_test.go            |   2 +-
 go/vt/sqlparser/random_expr.go                | 202 ++++++++++--------
 go/vt/sqlparser/rewriter_test.go              |   2 +-
 go/vt/sqlparser/walker_test.go                |   4 +-
 7 files changed, 146 insertions(+), 105 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 974b17807e6..5e686f21cc3 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -31,7 +31,7 @@ import (
 // this file contains the structs and functions to generate random queries
 
 // if true then known failing query types are still generated by randomQuery()
-const testFailingQueries = false
+const testFailingQueries = true
 
 type (
 	column struct {
@@ -51,13 +51,13 @@ type (
 
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
 
-func (t *tableT) typeExpr(typ string) sqlparser.Expr {
+func (t *tableT) Generate(config sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	tableCopy := t.clone()
 
 	for len(tableCopy.cols) > 0 {
 		idx := rand.Intn(len(tableCopy.cols))
 		randCol := tableCopy.cols[idx]
-		if randCol.typ == typ {
+		if randCol.typ == config.Type {
 			newTableName := ""
 			if tName, ok := tableCopy.name.(sqlparser.TableName); ok {
 				newTableName = sqlparser.String(tName.Name)
@@ -73,15 +73,6 @@ func (t *tableT) typeExpr(typ string) sqlparser.Expr {
 	return nil
 }
 
-func (t *tableT) IntExpr() sqlparser.Expr {
-	// TODO: better way to check if int type?
-	return t.typeExpr("bigint")
-}
-
-func (t *tableT) StringExpr() sqlparser.Expr {
-	return t.typeExpr("varchar")
-}
-
 // setName sets the alias for t, as well as setting the tableName for all columns in cols
 func (t *tableT) setName(newName string) {
 	t.name = sqlparser.NewTableName(newName)
@@ -240,9 +231,10 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 			typ:  typ,
 		})
 
-		// make sure to add the random expression to group by for only_full_group_by
+		// make sure to add the random expression to group by and order by for only_full_group_by
 		if canAggregate {
 			sel.AddGroupBy(randomExpr)
+			sel.AddOrder(sqlparser.NewOrder(randomExpr, getRandomOrderDirection()))
 		}
 
 		break
@@ -456,7 +448,7 @@ func createWherePredicates(tables []tableT, isJoin bool) (predicates sqlparser.E
 
 // creates predicates for the having clause comparing a column to a random expression
 func createHavingPredicates(tables []tableT) (havingPredicates sqlparser.Exprs) {
-	aggrSelectExprs, _ := createAggregations(tables, 2)
+	aggrSelectExprs, _ := createAggregations(tables, 2) // createAggregations(generators)
 	for i := range aggrSelectExprs {
 		if lhs, ok := aggrSelectExprs[i].(*sqlparser.AliasedExpr); ok {
 			// TODO: HAVING can only contain aggregate or grouping columns in mysql, works fine in vitess
@@ -488,7 +480,7 @@ func createLimit() *sqlparser.Limit {
 func getRandomExpr(tables []tableT) sqlparser.Expr {
 	seed := time.Now().UnixNano()
 	g := sqlparser.NewGenerator(seed, 2, slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
-	return g.Expression()
+	return g.Expression(sqlparser.ExprGeneratorConfig{})
 }
 
 func newAliasedTable(tbl tableT, alias string) *sqlparser.AliasedTableExpr {
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index b4e1cfb9c2d..7622e3dfabd 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package random
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -26,6 +27,7 @@ import (
 
 // This test tests that generating a random expression with a schema does not panic
 func TestRandomExprWithTables(t *testing.T) {
+
 	schemaTables := []tableT{
 		{name: sqlparser.NewTableName("emp")},
 		{name: sqlparser.NewTableName("dept")},
@@ -48,5 +50,8 @@ func TestRandomExprWithTables(t *testing.T) {
 
 	seed := time.Now().UnixNano()
 	g := sqlparser.NewGenerator(seed, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
-	g.Expression()
+	for i := 0; i < 100; i++ {
+		expr := g.Expression(sqlparser.ExprGeneratorConfig{})
+		fmt.Println(sqlparser.String(expr))
+	}
 }
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 440fedd7e79..9daee61cd84 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -83,6 +83,12 @@ func TestMustFix(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
+	// mismatched results
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
+
+	// EOF
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1, tbl1.deptno as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
+
 	// mismatched results
 	// limit >= 9 works
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) = sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8")
@@ -132,6 +138,10 @@ func TestKnownFailures(t *testing.T) {
 	// logs more stuff
 	//clusterInstance.EnableGeneralLog()
 
+	// vitess error: vttablet: rpc error: code = InvalidArgument desc = BIGINT UNSIGNED value is out of range in '(-(273) + (-(15) & 124))'
+	// mysql error: <nil>
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ -273 + (-15 & 124) as crandom0 from emp as tbl0, emp as tbl1 where tbl1.sal >= tbl1.mgr")
+
 	// cannot compare strings, collation is unknown or unsupported (collation ID: 0)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ max(tbl1.dname) as caggr1 from dept as tbl0, dept as tbl1 group by tbl1.dname order by tbl1.dname asc")
 
@@ -217,7 +227,7 @@ func TestRandom(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	endBy := time.Now().Add(1 * time.Second)
+	endBy := time.Now().Add(10 * time.Second)
 
 	var queryCount int
 	// continue testing after an error if and only if testFailingQueries is true
diff --git a/go/vt/sqlparser/precedence_test.go b/go/vt/sqlparser/precedence_test.go
index cb8c1f23805..4f2c1cbe319 100644
--- a/go/vt/sqlparser/precedence_test.go
+++ b/go/vt/sqlparser/precedence_test.go
@@ -224,7 +224,7 @@ func TestRandom(t *testing.T) {
 			break
 		}
 		// Given a random expression
-		randomExpr := g.Expression()
+		randomExpr := g.Expression(ExprGeneratorConfig{})
 		inputQ := "select " + String(randomExpr) + " from t"
 
 		// When it's parsed and unparsed
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 9b3b711c87f..66cf8bd516d 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -25,11 +25,34 @@ import (
 
 type (
 	ExprGenerator interface {
-		IntExpr() Expr
-		StringExpr() Expr
+		Generate(config ExprGeneratorConfig) Expr
+	}
+
+	ExprGeneratorConfig struct {
+		Type string
 	}
 )
 
+func (genConfig ExprGeneratorConfig) boolTypeConfig() ExprGeneratorConfig {
+	genConfig.Type = "tinyint"
+	return genConfig
+}
+
+func (genConfig ExprGeneratorConfig) intTypeConfig() ExprGeneratorConfig {
+	genConfig.Type = "bigint"
+	return genConfig
+}
+
+func (genConfig ExprGeneratorConfig) stringTypeConfig() ExprGeneratorConfig {
+	genConfig.Type = "varchar"
+	return genConfig
+}
+
+func (genConfig ExprGeneratorConfig) anyTypeConfig() ExprGeneratorConfig {
+	genConfig.Type = ""
+	return genConfig
+}
+
 func NewGenerator(seed int64, maxDepth int, exprGenerators ...ExprGenerator) *Generator {
 	g := Generator{
 		seed:          seed,
@@ -78,56 +101,65 @@ func (g *Generator) atMaxDepth() bool {
 Note: It's important to update this method so that it produces all expressions that need precedence checking.
 It's currently missing function calls and string operators
 */
-func (g *Generator) Expression() Expr {
-	if g.randomBool() {
-		return g.booleanExpr()
+func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
+	switch genConfig.Type {
+	case "bigint":
+		return g.intExpr(genConfig)
+	case "varchar":
+		return g.stringExpr(genConfig)
+	case "tinyint":
+		return g.booleanExpr(genConfig)
 	}
 
 	options := []exprF{
-		func() Expr { return g.intExpr() },
-		func() Expr { return g.stringExpr() },
-		func() Expr { return g.booleanExpr() },
+		func() Expr { return g.intExpr(genConfig) },
+		func() Expr { return g.stringExpr(genConfig) },
+		func() Expr { return g.booleanExpr(genConfig) },
 	}
 
 	return g.randomOf(options)
 }
 
-func (g *Generator) booleanExpr() Expr {
+func (g *Generator) booleanExpr(genConfig ExprGeneratorConfig) Expr {
 	if g.atMaxDepth() {
 		return g.booleanLiteral()
 	}
 
+	genConfig.Type = "tinyint"
+
 	options := []exprF{
-		func() Expr { return g.andExpr() },
-		func() Expr { return g.xorExpr() },
-		func() Expr { return g.orExpr() },
-		func() Expr { return g.comparison(g.intExpr) },
-		func() Expr { return g.comparison(g.stringExpr) },
-		//func() Expr { return g.comparison(g.booleanExpr) }, // this is not accepted by the parser
-		func() Expr { return g.inExpr() },
-		func() Expr { return g.between() },
-		func() Expr { return g.isExpr() },
-		func() Expr { return g.notExpr() },
-		func() Expr { return g.likeExpr() },
+		func() Expr { return g.andExpr(genConfig) },
+		func() Expr { return g.xorExpr(genConfig) },
+		func() Expr { return g.orExpr(genConfig) },
+		func() Expr { return g.comparison(genConfig.intTypeConfig()) },
+		func() Expr { return g.comparison(genConfig.stringTypeConfig()) },
+		//func() Expr { return g.comparison(genConfig) }, // this is not accepted by the parser
+		func() Expr { return g.inExpr(genConfig.intTypeConfig()) },
+		func() Expr { return g.between(genConfig.intTypeConfig()) },
+		func() Expr { return g.isExpr(genConfig) },
+		func() Expr { return g.notExpr(genConfig) },
+		func() Expr { return g.likeExpr(genConfig.stringTypeConfig()) },
 	}
 
 	return g.randomOf(options)
 }
 
-func (g *Generator) intExpr() Expr {
+func (g *Generator) intExpr(genConfig ExprGeneratorConfig) Expr {
 	if g.atMaxDepth() {
 		return g.intLiteral()
 	}
 
+	genConfig.Type = "bigint"
+
 	options := []exprF{
-		func() Expr { return g.arithmetic() },
+		func() Expr { return g.arithmetic(genConfig) },
 		func() Expr { return g.intLiteral() },
-		func() Expr { return g.caseExpr(g.intExpr) },
+		func() Expr { return g.caseExpr(genConfig) },
 	}
 
 	for _, generator := range g.exprGenerator {
 		options = append(options, func() Expr {
-			expr := generator.IntExpr()
+			expr := generator.Generate(genConfig)
 			if expr == nil {
 				return g.intLiteral()
 			}
@@ -138,39 +170,21 @@ func (g *Generator) intExpr() Expr {
 	return g.randomOf(options)
 }
 
-func (g *Generator) booleanLiteral() Expr {
-	return BoolVal(g.randomBool())
-}
-
-func (g *Generator) randomBool() bool {
-	return g.r.Float32() < 0.5
-}
-
-func (g *Generator) intLiteral() Expr {
-	t := fmt.Sprintf("%d", g.r.Intn(1000)-g.r.Intn(1000))
-
-	return NewIntLiteral(t)
-}
-
-var words = []string{"ox", "ant", "ape", "asp", "bat", "bee", "boa", "bug", "cat", "cod", "cow", "cub", "doe", "dog", "eel", "eft", "elf", "elk", "emu", "ewe", "fly", "fox", "gar", "gnu", "hen", "hog", "imp", "jay", "kid", "kit", "koi", "lab", "man", "owl", "pig", "pug", "pup", "ram", "rat", "ray", "yak", "bass", "bear", "bird", "boar", "buck", "bull", "calf", "chow", "clam", "colt", "crab", "crow", "dane", "deer", "dodo", "dory", "dove", "drum", "duck", "fawn", "fish", "flea", "foal", "fowl", "frog", "gnat", "goat", "grub", "gull", "hare", "hawk", "ibex", "joey", "kite", "kiwi", "lamb", "lark", "lion", "loon", "lynx", "mako", "mink", "mite", "mole", "moth", "mule", "mutt", "newt", "orca", "oryx", "pika", "pony", "puma", "seal", "shad", "slug", "sole", "stag", "stud", "swan", "tahr", "teal", "tick", "toad", "tuna", "wasp", "wolf", "worm", "wren", "yeti", "adder", "akita", "alien", "aphid", "bison", "boxer", "bream", "bunny", "burro", "camel", "chimp", "civet", "cobra", "coral", "corgi", "crane", "dingo", "drake", "eagle", "egret", "filly", "finch", "gator", "gecko", "ghost", "ghoul", "goose", "guppy", "heron", "hippo", "horse", "hound", "husky", "hyena", "koala", "krill", "leech", "lemur", "liger", "llama", "louse", "macaw", "midge", "molly", "moose", "moray", "mouse", "panda", "perch", "prawn", "quail", "racer", "raven", "rhino", "robin", "satyr", "shark", "sheep", "shrew", "skink", "skunk", "sloth", "snail", "snake", "snipe", "squid", "stork", "swift", "swine", "tapir", "tetra", "tiger", "troll", "trout", "viper", "wahoo", "whale", "zebra", "alpaca", "amoeba", "baboon", "badger", "beagle", "bedbug", "beetle", "bengal", "bobcat", "caiman", "cattle", "cicada", "collie", "condor", "cougar", "coyote", "dassie", "donkey", "dragon", "earwig", "falcon", "feline", "ferret", "gannet", "gibbon", "glider", "goblin", "gopher", "grouse", "guinea", "hermit", "hornet", "iguana", "impala", "insect", "jackal", "jaguar", "jennet", "kitten", "kodiak", "lizard", "locust", "maggot", "magpie", "mammal", "mantis", "marlin", "marmot", "marten", "martin", "mayfly", "minnow", "monkey", "mullet", "muskox", "ocelot", "oriole", "osprey", "oyster", "parrot", "pigeon", "piglet", "poodle", "possum", "python", "quagga", "rabbit", "raptor", "rodent", "roughy", "salmon", "sawfly", "serval", "shiner", "shrimp", "spider", "sponge", "tarpon", "thrush", "tomcat", "toucan", "turkey", "turtle", "urchin", "vervet", "walrus", "weasel", "weevil", "wombat", "anchovy", "anemone", "bluejay", "buffalo", "bulldog", "buzzard", "caribou", "catfish", "chamois", "cheetah", "chicken", "chigger", "cowbird", "crappie", "crawdad", "cricket", "dogfish", "dolphin", "firefly", "garfish", "gazelle", "gelding", "giraffe", "gobbler", "gorilla", "goshawk", "grackle", "griffon", "grizzly", "grouper", "haddock", "hagfish", "halibut", "hamster", "herring", "jackass", "javelin", "jawfish", "jaybird", "katydid", "ladybug", "lamprey", "lemming", "leopard", "lioness", "lobster", "macaque", "mallard", "mammoth", "manatee", "mastiff", "meerkat", "mollusk", "monarch", "mongrel", "monitor", "monster", "mudfish", "muskrat", "mustang", "narwhal", "oarfish", "octopus", "opossum", "ostrich", "panther", "peacock", "pegasus", "pelican", "penguin", "phoenix", "piranha", "polecat", "primate", "quetzal", "raccoon", "rattler", "redbird", "redfish", "reptile", "rooster", "sawfish", "sculpin", "seagull", "skylark", "snapper", "spaniel", "sparrow", "sunbeam", "sunbird", "sunfish", "tadpole", "termite", "terrier", "unicorn", "vulture", "wallaby", "walleye", "warthog", "whippet", "wildcat", "aardvark", "airedale", "albacore", "anteater", "antelope", "arachnid", "barnacle", "basilisk", "blowfish", "bluebird", "bluegill", "bonefish", "bullfrog", "cardinal", "chipmunk", "cockatoo", "crayfish", "dinosaur", "doberman", "duckling", "elephant", "escargot", "flamingo", "flounder", "foxhound", "glowworm", "goldfish", "grubworm", "hedgehog", "honeybee", "hookworm", "humpback", "kangaroo", "killdeer", "kingfish", "labrador", "lacewing", "ladybird", "lionfish", "longhorn", "mackerel", "malamute", "marmoset", "mastodon", "moccasin", "mongoose", "monkfish", "mosquito", "pangolin", "parakeet", "pheasant", "pipefish", "platypus", "polliwog", "porpoise", "reindeer", "ringtail", "sailfish", "scorpion", "seahorse", "seasnail", "sheepdog", "shepherd", "silkworm", "squirrel", "stallion", "starfish", "starling", "stingray", "stinkbug", "sturgeon", "terrapin", "titmouse", "tortoise", "treefrog", "werewolf", "woodcock"}
-
-func (g *Generator) stringLiteral() Expr {
-	return NewStrLiteral(g.randomOfS(words))
-}
-
-func (g *Generator) stringExpr() Expr {
+func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 	if g.atMaxDepth() {
 		return g.stringLiteral()
 	}
 
+	genConfig.Type = "varchar"
+
 	options := []exprF{
 		func() Expr { return g.stringLiteral() },
-		func() Expr { return g.caseExpr(g.stringExpr) },
+		func() Expr { return g.caseExpr(genConfig) },
 	}
 
 	for _, generator := range g.exprGenerator {
 		options = append(options, func() Expr {
-			expr := generator.StringExpr()
+			expr := generator.Generate(genConfig)
 			if expr == nil {
 				return g.stringLiteral()
 			}
@@ -181,54 +195,74 @@ func (g *Generator) stringExpr() Expr {
 	return g.randomOf(options)
 }
 
-func (g *Generator) likeExpr() Expr {
+func (g *Generator) booleanLiteral() Expr {
+	return BoolVal(g.randomBool())
+}
+
+func (g *Generator) randomBool() bool {
+	return g.r.Float32() < 0.5
+}
+
+func (g *Generator) intLiteral() Expr {
+	t := fmt.Sprintf("%d", g.r.Intn(1000)-g.r.Intn(1000))
+
+	return NewIntLiteral(t)
+}
+
+var words = []string{"ox", "ant", "ape", "asp", "bat", "bee", "boa", "bug", "cat", "cod", "cow", "cub", "doe", "dog", "eel", "eft", "elf", "elk", "emu", "ewe", "fly", "fox", "gar", "gnu", "hen", "hog", "imp", "jay", "kid", "kit", "koi", "lab", "man", "owl", "pig", "pug", "pup", "ram", "rat", "ray", "yak", "bass", "bear", "bird", "boar", "buck", "bull", "calf", "chow", "clam", "colt", "crab", "crow", "dane", "deer", "dodo", "dory", "dove", "drum", "duck", "fawn", "fish", "flea", "foal", "fowl", "frog", "gnat", "goat", "grub", "gull", "hare", "hawk", "ibex", "joey", "kite", "kiwi", "lamb", "lark", "lion", "loon", "lynx", "mako", "mink", "mite", "mole", "moth", "mule", "mutt", "newt", "orca", "oryx", "pika", "pony", "puma", "seal", "shad", "slug", "sole", "stag", "stud", "swan", "tahr", "teal", "tick", "toad", "tuna", "wasp", "wolf", "worm", "wren", "yeti", "adder", "akita", "alien", "aphid", "bison", "boxer", "bream", "bunny", "burro", "camel", "chimp", "civet", "cobra", "coral", "corgi", "crane", "dingo", "drake", "eagle", "egret", "filly", "finch", "gator", "gecko", "ghost", "ghoul", "goose", "guppy", "heron", "hippo", "horse", "hound", "husky", "hyena", "koala", "krill", "leech", "lemur", "liger", "llama", "louse", "macaw", "midge", "molly", "moose", "moray", "mouse", "panda", "perch", "prawn", "quail", "racer", "raven", "rhino", "robin", "satyr", "shark", "sheep", "shrew", "skink", "skunk", "sloth", "snail", "snake", "snipe", "squid", "stork", "swift", "swine", "tapir", "tetra", "tiger", "troll", "trout", "viper", "wahoo", "whale", "zebra", "alpaca", "amoeba", "baboon", "badger", "beagle", "bedbug", "beetle", "bengal", "bobcat", "caiman", "cattle", "cicada", "collie", "condor", "cougar", "coyote", "dassie", "donkey", "dragon", "earwig", "falcon", "feline", "ferret", "gannet", "gibbon", "glider", "goblin", "gopher", "grouse", "guinea", "hermit", "hornet", "iguana", "impala", "insect", "jackal", "jaguar", "jennet", "kitten", "kodiak", "lizard", "locust", "maggot", "magpie", "mammal", "mantis", "marlin", "marmot", "marten", "martin", "mayfly", "minnow", "monkey", "mullet", "muskox", "ocelot", "oriole", "osprey", "oyster", "parrot", "pigeon", "piglet", "poodle", "possum", "python", "quagga", "rabbit", "raptor", "rodent", "roughy", "salmon", "sawfly", "serval", "shiner", "shrimp", "spider", "sponge", "tarpon", "thrush", "tomcat", "toucan", "turkey", "turtle", "urchin", "vervet", "walrus", "weasel", "weevil", "wombat", "anchovy", "anemone", "bluejay", "buffalo", "bulldog", "buzzard", "caribou", "catfish", "chamois", "cheetah", "chicken", "chigger", "cowbird", "crappie", "crawdad", "cricket", "dogfish", "dolphin", "firefly", "garfish", "gazelle", "gelding", "giraffe", "gobbler", "gorilla", "goshawk", "grackle", "griffon", "grizzly", "grouper", "haddock", "hagfish", "halibut", "hamster", "herring", "jackass", "javelin", "jawfish", "jaybird", "katydid", "ladybug", "lamprey", "lemming", "leopard", "lioness", "lobster", "macaque", "mallard", "mammoth", "manatee", "mastiff", "meerkat", "mollusk", "monarch", "mongrel", "monitor", "monster", "mudfish", "muskrat", "mustang", "narwhal", "oarfish", "octopus", "opossum", "ostrich", "panther", "peacock", "pegasus", "pelican", "penguin", "phoenix", "piranha", "polecat", "primate", "quetzal", "raccoon", "rattler", "redbird", "redfish", "reptile", "rooster", "sawfish", "sculpin", "seagull", "skylark", "snapper", "spaniel", "sparrow", "sunbeam", "sunbird", "sunfish", "tadpole", "termite", "terrier", "unicorn", "vulture", "wallaby", "walleye", "warthog", "whippet", "wildcat", "aardvark", "airedale", "albacore", "anteater", "antelope", "arachnid", "barnacle", "basilisk", "blowfish", "bluebird", "bluegill", "bonefish", "bullfrog", "cardinal", "chipmunk", "cockatoo", "crayfish", "dinosaur", "doberman", "duckling", "elephant", "escargot", "flamingo", "flounder", "foxhound", "glowworm", "goldfish", "grubworm", "hedgehog", "honeybee", "hookworm", "humpback", "kangaroo", "killdeer", "kingfish", "labrador", "lacewing", "ladybird", "lionfish", "longhorn", "mackerel", "malamute", "marmoset", "mastodon", "moccasin", "mongoose", "monkfish", "mosquito", "pangolin", "parakeet", "pheasant", "pipefish", "platypus", "polliwog", "porpoise", "reindeer", "ringtail", "sailfish", "scorpion", "seahorse", "seasnail", "sheepdog", "shepherd", "silkworm", "squirrel", "stallion", "starfish", "starling", "stingray", "stinkbug", "sturgeon", "terrapin", "titmouse", "tortoise", "treefrog", "werewolf", "woodcock"}
+
+func (g *Generator) stringLiteral() Expr {
+	return NewStrLiteral(g.randomOfS(words))
+}
+
+func (g *Generator) likeExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 	return &ComparisonExpr{
 		Operator: LikeOp,
-		Left:     g.stringExpr(),
-		Right:    g.stringExpr(),
+		Left:     g.Expression(genConfig),
+		Right:    g.Expression(genConfig),
 	}
 }
 
 var comparisonOps = []ComparisonExprOperator{EqualOp, LessThanOp, GreaterThanOp, LessEqualOp, GreaterEqualOp, NotEqualOp, NullSafeEqualOp}
 
-func (g *Generator) comparison(f func() Expr) Expr {
+func (g *Generator) comparison(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 
 	cmp := &ComparisonExpr{
 		Operator: comparisonOps[g.r.Intn(len(comparisonOps))],
-		Left:     f(),
-		Right:    f(),
+		Left:     g.Expression(genConfig),
+		Right:    g.Expression(genConfig),
 	}
 	return cmp
 }
 
-func (g *Generator) caseExpr(valueF func() Expr) Expr {
+func (g *Generator) caseExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 
 	var exp Expr
 	var elseExpr Expr
 	if g.randomBool() {
-		exp = valueF()
+		exp = g.Expression(genConfig.anyTypeConfig())
 	}
 	if g.randomBool() {
-		elseExpr = valueF()
+		elseExpr = g.Expression(genConfig)
 	}
 
-	size := g.r.Intn(5) + 2
+	size := g.r.Intn(2) + 2
 	var whens []*When
 	for i := 0; i < size; i++ {
 		var cond Expr
 		if exp == nil {
-			cond = g.booleanExpr()
+			cond = g.Expression(genConfig.boolTypeConfig())
 		} else {
-			cond = g.Expression()
+			cond = g.Expression(genConfig.anyTypeConfig())
 		}
 
-		val := g.Expression()
+		val := g.Expression(genConfig)
 		whens = append(whens, &When{
 			Cond: cond,
 			Val:  val,
@@ -244,7 +278,7 @@ func (g *Generator) caseExpr(valueF func() Expr) Expr {
 
 var arithmeticOps = []BinaryExprOperator{BitAndOp, BitOrOp, BitXorOp, PlusOp, MinusOp, MultOp, DivOp, IntDivOp, ModOp, ShiftRightOp, ShiftLeftOp}
 
-func (g *Generator) arithmetic() Expr {
+func (g *Generator) arithmetic(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 
@@ -252,8 +286,8 @@ func (g *Generator) arithmetic() Expr {
 
 	return &BinaryExpr{
 		Operator: op,
-		Left:     g.intExpr(),
-		Right:    g.intExpr(),
+		Left:     g.Expression(genConfig),
+		Right:    g.Expression(genConfig),
 	}
 }
 
@@ -267,48 +301,48 @@ func (g *Generator) randomOfS(options []string) string {
 	return options[g.r.Intn(len(options))]
 }
 
-func (g *Generator) andExpr() Expr {
+func (g *Generator) andExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 	return &AndExpr{
-		Left:  g.booleanExpr(),
-		Right: g.booleanExpr(),
+		Left:  g.Expression(genConfig),
+		Right: g.Expression(genConfig),
 	}
 }
 
-func (g *Generator) orExpr() Expr {
+func (g *Generator) orExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 	return &OrExpr{
-		Left:  g.booleanExpr(),
-		Right: g.booleanExpr(),
+		Left:  g.Expression(genConfig),
+		Right: g.Expression(genConfig),
 	}
 }
 
-func (g *Generator) xorExpr() Expr {
+func (g *Generator) xorExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 	return &XorExpr{
-		Left:  g.booleanExpr(),
-		Right: g.booleanExpr(),
+		Left:  g.Expression(genConfig),
+		Right: g.Expression(genConfig),
 	}
 }
 
-func (g *Generator) notExpr() Expr {
+func (g *Generator) notExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
-	return &NotExpr{g.booleanExpr()}
+	return &NotExpr{g.Expression(genConfig)}
 }
 
-func (g *Generator) inExpr() Expr {
+func (g *Generator) inExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 
-	expr := g.intExpr()
-	size := g.r.Intn(5) + 2
+	expr := g.Expression(genConfig)
+	size := g.r.Intn(3) + 2
 	tuples := ValTuple{}
 	for i := 0; i < size; i++ {
-		tuples = append(tuples, g.intExpr())
+		tuples = append(tuples, g.Expression(genConfig))
 	}
 	op := InOp
 	if g.randomBool() {
@@ -322,7 +356,7 @@ func (g *Generator) inExpr() Expr {
 	}
 }
 
-func (g *Generator) between() Expr {
+func (g *Generator) between(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 
@@ -335,13 +369,13 @@ func (g *Generator) between() Expr {
 
 	return &BetweenExpr{
 		IsBetween: IsBetween,
-		Left:      g.intExpr(),
-		From:      g.intExpr(),
-		To:        g.intExpr(),
+		Left:      g.Expression(genConfig),
+		From:      g.Expression(genConfig),
+		To:        g.Expression(genConfig),
 	}
 }
 
-func (g *Generator) isExpr() Expr {
+func (g *Generator) isExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 
@@ -349,6 +383,6 @@ func (g *Generator) isExpr() Expr {
 
 	return &IsExpr{
 		Right: ops[g.r.Intn(len(ops))],
-		Left:  g.booleanExpr(),
+		Left:  g.Expression(genConfig),
 	}
 }
diff --git a/go/vt/sqlparser/rewriter_test.go b/go/vt/sqlparser/rewriter_test.go
index 9adae1b4a81..3f4f9823ae2 100644
--- a/go/vt/sqlparser/rewriter_test.go
+++ b/go/vt/sqlparser/rewriter_test.go
@@ -26,7 +26,7 @@ import (
 
 func BenchmarkVisitLargeExpression(b *testing.B) {
 	gen := NewGenerator(1, 5)
-	exp := gen.Expression()
+	exp := gen.Expression(ExprGeneratorConfig{})
 
 	depth := 0
 	for i := 0; i < b.N; i++ {
diff --git a/go/vt/sqlparser/walker_test.go b/go/vt/sqlparser/walker_test.go
index 5359235afa5..70432591b08 100644
--- a/go/vt/sqlparser/walker_test.go
+++ b/go/vt/sqlparser/walker_test.go
@@ -26,7 +26,7 @@ import (
 func BenchmarkWalkLargeExpression(b *testing.B) {
 	for i := 0; i < 10; i++ {
 		b.Run(fmt.Sprintf("%d", i), func(b *testing.B) {
-			exp := NewGenerator(int64(i*100), 5).Expression()
+			exp := NewGenerator(int64(i*100), 5).Expression(ExprGeneratorConfig{})
 			count := 0
 			for i := 0; i < b.N; i++ {
 				err := Walk(func(node SQLNode) (kontinue bool, err error) {
@@ -42,7 +42,7 @@ func BenchmarkWalkLargeExpression(b *testing.B) {
 func BenchmarkRewriteLargeExpression(b *testing.B) {
 	for i := 1; i < 7; i++ {
 		b.Run(fmt.Sprintf("%d", i), func(b *testing.B) {
-			exp := NewGenerator(int64(i*100), i).Expression()
+			exp := NewGenerator(int64(i*100), i).Expression(ExprGeneratorConfig{})
 			count := 0
 			for i := 0; i < b.N; i++ {
 				_ = Rewrite(exp, func(_ *Cursor) bool {

From 54368c63d3e7e579a81c2ef47c7ceae092c050d4 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Tue, 4 Jul 2023 03:01:39 -0700
Subject: [PATCH 03/27] refactor tableT struct

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 89 +++++++++----------
 .../vtgate/queries/random/random_expr_test.go |  4 +-
 .../vtgate/queries/random/random_test.go      |  4 +-
 go/vt/sqlparser/random_expr.go                | 28 +++---
 4 files changed, 64 insertions(+), 61 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 5e686f21cc3..eed7d8de7f8 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -41,41 +41,55 @@ type (
 	}
 	tableT struct {
 		// the tableT struct can be used to represent the schema of a table or a derived table
-		// in the former case name will be a sqlparser.TableName, in the latter a sqlparser.DerivedTable
-		// in order to create a query with a derived table, its AST form is retrieved from name
-		// once the derived table is aliased, name is replaced by a sqlparser.TableName with that alias
-		name sqlparser.SimpleTableExpr
-		cols []column
+		// in the former case tableExpr will be a sqlparser.TableName, in the latter a sqlparser.DerivedTable
+		// in order to create a query with a derived table, its AST form is retrieved from tableExpr
+		// once the derived table is aliased, alias is updated
+		tableExpr sqlparser.SimpleTableExpr
+		alias     string
+		cols      []column
 	}
 )
 
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
 
+// var _ sqlparser.ExprGenerator = column{}
+
 func (t *tableT) Generate(config sqlparser.ExprGeneratorConfig) sqlparser.Expr {
-	tableCopy := t.clone()
+	colsCopy := slices.Clone(t.cols)
 
-	for len(tableCopy.cols) > 0 {
-		idx := rand.Intn(len(tableCopy.cols))
-		randCol := tableCopy.cols[idx]
+	for len(colsCopy) > 0 {
+		idx := rand.Intn(len(colsCopy))
+		randCol := colsCopy[idx]
 		if randCol.typ == config.Type {
-			newTableName := ""
-			if tName, ok := tableCopy.name.(sqlparser.TableName); ok {
-				newTableName = sqlparser.String(tName.Name)
-			}
+			newTableName := t.getName()
 			return sqlparser.NewColNameWithQualifier(randCol.name, sqlparser.NewTableName(newTableName))
 		}
 
-		// delete randCol from table.columns
-		tableCopy.cols[idx] = tableCopy.cols[len(tableCopy.cols)-1]
-		tableCopy.cols = tableCopy.cols[:len(tableCopy.cols)-1]
+		// delete randCol from colsCopy
+		colsCopy[idx] = colsCopy[len(colsCopy)-1]
+		colsCopy = colsCopy[:len(colsCopy)-1]
 	}
 
 	return nil
 }
 
+// getName returns the alias if it is nonempty
+// if the alias is nonempty and tableExpr is of type sqlparser.TableName,
+// then getName returns Name from tableExpr
+// otherwise getName returns an empty string
+func (t *tableT) getName() string {
+	if t.alias != "" {
+		return t.alias
+	} else if tName, ok := t.tableExpr.(*sqlparser.TableName); ok {
+		return tName.Name.String()
+	}
+
+	return ""
+}
+
 // setName sets the alias for t, as well as setting the tableName for all columns in cols
 func (t *tableT) setName(newName string) {
-	t.name = sqlparser.NewTableName(newName)
+	t.alias = newName
 	for i := range t.cols {
 		t.cols[i].tableName = newName
 	}
@@ -88,27 +102,15 @@ func (t *tableT) setColumns(col ...column) {
 	t.addColumns(col...)
 }
 
-// addColumns adds columns to t, and automatically assigns tableName
-// this makes it unnatural (but still possible as cols is exportable) to modify tableName
+// addColumns adds columns to t, and automatically assigns each column.tableName
+// this makes it unnatural to modify tableName
 func (t *tableT) addColumns(col ...column) {
 	for i := range col {
-		// only change the Col's tableName if t is of type tableName
-		if tName, ok := t.name.(sqlparser.TableName); ok {
-			col[i].tableName = sqlparser.String(tName.Name)
-		}
-
+		col[i].tableName = t.getName()
 		t.cols = append(t.cols, col[i])
 	}
 }
 
-// clone returns a deep copy of t
-func (t *tableT) clone() *tableT {
-	return &tableT{
-		name: t.name,
-		cols: slices.Clone(t.cols),
-	}
-}
-
 // getColumnName returns tableName.name
 func (c *column) getColumnName() string {
 	return fmt.Sprintf("%s.%s", c.tableName, c.name)
@@ -245,7 +247,7 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 	newTable.addColumns(aggregates...)
 
 	// add new table to schemaTables
-	newTable.name = sqlparser.NewDerivedTable(false, sel)
+	newTable.tableExpr = sqlparser.NewDerivedTable(false, sel)
 	schemaTables = append(schemaTables, newTable)
 
 	// derived tables (partially unsupported)
@@ -275,18 +277,11 @@ func createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT
 	// TODO: outer joins produce mismatched results
 	isJoin := rand.Intn(2) < 1 && testFailingQueries
 	if isJoin {
+		// TODO: do nested joins
 		newTable := randomEl(schemaTables)
 		tables = append(tables, newTable)
 
-		// create the join before aliasing
-		newJoinTableExpr := createJoin(tables, sel)
-
-		// alias
-		tables[numTables+1].setName(fmt.Sprintf("tbl%d", numTables+1))
-
-		// create the condition after aliasing
-		newJoinTableExpr.Condition = sqlparser.NewJoinCondition(sqlparser.AndExpressions(createWherePredicates(tables, true)...), nil)
-		sel.From[numTables] = newJoinTableExpr
+		createJoin(tables, sel)
 	}
 
 	return tables, isJoin
@@ -294,13 +289,17 @@ func createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT
 
 // creates a left join (without the condition) between the last table in sel and newTable
 // tables should have one more table than sel
-func createJoin(tables []tableT, sel *sqlparser.Select) *sqlparser.JoinTableExpr {
+func createJoin(tables []tableT, sel *sqlparser.Select) {
 	n := len(sel.From)
 	if len(tables) != n+1 {
 		log.Fatalf("sel has %d tables and tables has %d tables", len(sel.From), n)
 	}
 
-	return sqlparser.NewJoinTableExpr(sel.From[n-1], sqlparser.LeftJoinType, newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n)), nil)
+	// alias
+	tables[n].setName(fmt.Sprintf("tbl%d", n))
+
+	newJoinCondition := sqlparser.NewJoinCondition(sqlparser.AndExpressions(createWherePredicates(tables, true)...), nil)
+	sel.From[n-1] = sqlparser.NewJoinTableExpr(sel.From[n-1], sqlparser.LeftJoinType, newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n)), newJoinCondition)
 }
 
 // returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
@@ -484,7 +483,7 @@ func getRandomExpr(tables []tableT) sqlparser.Expr {
 }
 
 func newAliasedTable(tbl tableT, alias string) *sqlparser.AliasedTableExpr {
-	return sqlparser.NewAliasedTableExpr(tbl.name, alias)
+	return sqlparser.NewAliasedTableExpr(tbl.tableExpr, alias)
 }
 
 func newAliasedColumn(col column, alias string) *sqlparser.AliasedExpr {
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index 7622e3dfabd..7c498e1a269 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -29,8 +29,8 @@ import (
 func TestRandomExprWithTables(t *testing.T) {
 
 	schemaTables := []tableT{
-		{name: sqlparser.NewTableName("emp")},
-		{name: sqlparser.NewTableName("dept")},
+		{tableExpr: sqlparser.NewTableName("emp")},
+		{tableExpr: sqlparser.NewTableName("dept")},
 	}
 	schemaTables[0].addColumns([]column{
 		{name: "empno", typ: "bigint"},
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 9daee61cd84..f1d370bc32f 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -208,8 +208,8 @@ func TestRandom(t *testing.T) {
 
 	// specify the schema (that is defined in schema.sql)
 	schemaTables := []tableT{
-		{name: sqlparser.NewTableName("emp")},
-		{name: sqlparser.NewTableName("dept")},
+		{tableExpr: sqlparser.NewTableName("emp")},
+		{tableExpr: sqlparser.NewTableName("dept")},
 	}
 	schemaTables[0].addColumns([]column{
 		{name: "empno", typ: "bigint"},
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 66cf8bd516d..2d634a3269f 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -55,20 +55,20 @@ func (genConfig ExprGeneratorConfig) anyTypeConfig() ExprGeneratorConfig {
 
 func NewGenerator(seed int64, maxDepth int, exprGenerators ...ExprGenerator) *Generator {
 	g := Generator{
-		seed:          seed,
-		r:             rand.New(rand.NewSource(seed)),
-		maxDepth:      maxDepth,
-		exprGenerator: exprGenerators,
+		seed:           seed,
+		r:              rand.New(rand.NewSource(seed)),
+		maxDepth:       maxDepth,
+		exprGenerators: exprGenerators,
 	}
 	return &g
 }
 
 type Generator struct {
-	seed          int64
-	r             *rand.Rand
-	depth         int
-	maxDepth      int
-	exprGenerator []ExprGenerator
+	seed           int64
+	r              *rand.Rand
+	depth          int
+	maxDepth       int
+	exprGenerators []ExprGenerator
 }
 
 // enter should be called whenever we are producing an intermediate node. it should be followed by a `defer g.exit()`
@@ -157,15 +157,19 @@ func (g *Generator) intExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.caseExpr(genConfig) },
 	}
 
-	for _, generator := range g.exprGenerator {
+	// TODO: this adds two identical functions to options, both of which use the last element in g.exprGenerators
+	for i := range g.exprGenerators {
 		options = append(options, func() Expr {
-			expr := generator.Generate(genConfig)
+			//fmt.Printf("index: %d", i)
+			expr := g.exprGenerators[i].Generate(genConfig)
 			if expr == nil {
 				return g.intLiteral()
 			}
 			return expr
 		})
 	}
+	//fmt.Printf("options: %v\n", options)
+	//fmt.Printf("expr generators: %v\n", g.exprGenerators)
 
 	return g.randomOf(options)
 }
@@ -182,7 +186,7 @@ func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.caseExpr(genConfig) },
 	}
 
-	for _, generator := range g.exprGenerator {
+	for _, generator := range g.exprGenerators {
 		options = append(options, func() Expr {
 			expr := generator.Generate(genConfig)
 			if expr == nil {

From 0b07313b17a3ad8ff45f46344f7282afae1e7851 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Thu, 6 Jul 2023 02:20:24 -0700
Subject: [PATCH 04/27] simplified random predicate generation

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 138 ++++++++----------
 .../vtgate/queries/random/random_test.go      |   5 +-
 go/vt/sqlparser/random_expr.go                |  18 ++-
 3 files changed, 74 insertions(+), 87 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index eed7d8de7f8..28ee4ddfc89 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -24,6 +24,7 @@ import (
 	"golang.org/x/exp/slices"
 
 	"vitess.io/vitess/go/slices2"
+
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/sqlparser"
 )
@@ -31,7 +32,7 @@ import (
 // this file contains the structs and functions to generate random queries
 
 // if true then known failing query types are still generated by randomQuery()
-const testFailingQueries = true
+const testFailingQueries = false
 
 type (
 	column struct {
@@ -51,16 +52,17 @@ type (
 )
 
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
+var _ sqlparser.ExprGenerator = (*column)(nil)
 
 // var _ sqlparser.ExprGenerator = column{}
 
-func (t *tableT) Generate(config sqlparser.ExprGeneratorConfig) sqlparser.Expr {
+func (t *tableT) Generate(genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	colsCopy := slices.Clone(t.cols)
 
 	for len(colsCopy) > 0 {
 		idx := rand.Intn(len(colsCopy))
 		randCol := colsCopy[idx]
-		if randCol.typ == config.Type {
+		if randCol.typ == genConfig.Type {
 			newTableName := t.getName()
 			return sqlparser.NewColNameWithQualifier(randCol.name, sqlparser.NewTableName(newTableName))
 		}
@@ -80,8 +82,8 @@ func (t *tableT) Generate(config sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 func (t *tableT) getName() string {
 	if t.alias != "" {
 		return t.alias
-	} else if tName, ok := t.tableExpr.(*sqlparser.TableName); ok {
-		return tName.Name.String()
+	} else if tName, ok := t.tableExpr.(sqlparser.TableName); ok {
+		return sqlparser.String(tName.Name)
 	}
 
 	return ""
@@ -95,13 +97,6 @@ func (t *tableT) setName(newName string) {
 	}
 }
 
-// setColumns sets the columns of t, and automatically assigns tableName
-// this makes it unnatural (but still possible as cols is exportable) to modify tableName
-func (t *tableT) setColumns(col ...column) {
-	t.cols = nil
-	t.addColumns(col...)
-}
-
 // addColumns adds columns to t, and automatically assigns each column.tableName
 // this makes it unnatural to modify tableName
 func (t *tableT) addColumns(col ...column) {
@@ -116,6 +111,14 @@ func (c *column) getColumnName() string {
 	return fmt.Sprintf("%s.%s", c.tableName, c.name)
 }
 
+func (c *column) Generate(genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
+	if c.typ == genConfig.Type {
+		return sqlparser.NewColNameWithQualifier(c.name, sqlparser.NewTableName(c.tableName))
+	}
+
+	return nil
+}
+
 func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Select {
 	sel := &sqlparser.Select{}
 	sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"})
@@ -160,7 +163,7 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 			if rand.Intn(2) < 1 && testFailingQueries {
 				// TODO: having can only contain aggregate or grouping columns in mysql, works fine in vitess
 				// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
-				sel.AddHaving(sqlparser.AndExpressions(createWherePredicates(tables, false)...))
+				sel.AddHaving(sqlparser.AndExpressions(createWherePredicates(tables)...))
 			}
 		}
 		// TODO: use sqlparser.ExprGenerator to generate a random expression with aggregation functions
@@ -176,15 +179,9 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 		sel.OrderBy = createOrderBy(groupBy, aggrExprs)
 	}
 
-	// where
-	sel.AddWhere(sqlparser.AndExpressions(createWherePredicates(tables, false)...))
-
-	// random predicate expression
 	// TODO: random expressions cause a lot of failures
-	if rand.Intn(2) < 1 && testFailingQueries {
-		predRandomExpr := getRandomExpr(tables)
-		sel.AddWhere(predRandomExpr)
-	}
+	// where
+	sel.AddWhere(sqlparser.AndExpressions(createWherePredicates(tables)...))
 
 	// only add a limit if the grouping columns are ordered
 	// TODO: limit fails a lot
@@ -204,9 +201,14 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
 	if testFailingQueries {
-		f = func() sqlparser.Expr { return getRandomExpr(tables) }
+		// TODO: ugly
+		f = func() sqlparser.Expr {
+			return getRandomExpr(sqlparser.ExprGeneratorConfig{}, slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
+				return &t
+			})...)
+		}
 	} else {
-		f = func() sqlparser.Expr { return getRandomExpr(nil) }
+		f = func() sqlparser.Expr { return getRandomExpr(sqlparser.ExprGeneratorConfig{}) }
 	}
 
 	// make sure we have at least one select expression
@@ -279,6 +281,7 @@ func createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT
 	if isJoin {
 		// TODO: do nested joins
 		newTable := randomEl(schemaTables)
+		newTable.setName(fmt.Sprintf("tbl%d", numTables))
 		tables = append(tables, newTable)
 
 		createJoin(tables, sel)
@@ -295,11 +298,20 @@ func createJoin(tables []tableT, sel *sqlparser.Select) {
 		log.Fatalf("sel has %d tables and tables has %d tables", len(sel.From), n)
 	}
 
-	// alias
-	tables[n].setName(fmt.Sprintf("tbl%d", n))
+	joinPredicate := sqlparser.AndExpressions(createJoinPredicates(tables)...)
+	joinCondition := sqlparser.NewJoinCondition(joinPredicate, nil)
+	newTable := newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n))
+	sel.From[n-1] = sqlparser.NewJoinTableExpr(sel.From[n-1], sqlparser.LeftJoinType, newTable, joinCondition)
+}
 
-	newJoinCondition := sqlparser.NewJoinCondition(sqlparser.AndExpressions(createWherePredicates(tables, true)...), nil)
-	sel.From[n-1] = sqlparser.NewJoinTableExpr(sel.From[n-1], sqlparser.LeftJoinType, newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n)), newJoinCondition)
+// returns 1-3 random expressions based on the last two elements of tables
+// tables should have at least two elements
+func createJoinPredicates(tables []tableT) sqlparser.Exprs {
+	if len(tables) < 2 {
+		log.Fatalf("tables has %d elements, needs at least 2", len(tables))
+	}
+
+	return createPredicates(1, &tables[len(tables)-2], &tables[len(tables)-1])
 }
 
 // returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
@@ -394,52 +406,16 @@ func createOrderBy(groupBy sqlparser.GroupBy, aggrExprs sqlparser.SelectExprs) (
 	return
 }
 
-// compares two random columns (usually of the same type)
-// returns a random expression if there are no other predicates and isJoin is true
-// returns the predicates as a sqlparser.Exprs (slice of sqlparser.Expr's)
-func createWherePredicates(tables []tableT, isJoin bool) (predicates sqlparser.Exprs) {
-	// if creating predicates for a join,
-	// then make sure predicates are created for the last two tables (which are being joined)
-	incr := 0
-	if isJoin && len(tables) > 2 {
-		incr += len(tables) - 2
-	}
-
-	for idx1 := range tables {
-		for idx2 := range tables {
-			// fmt.Printf("predicate tables:\n%v\n idx1: %d idx2: %d, incr: %d", tables, idx1, idx2, incr)
-			if idx1 >= idx2 || idx1 < incr || idx2 < incr {
-				continue
-			}
-			noOfPredicates := rand.Intn(2)
-			if isJoin {
-				noOfPredicates++
-			}
-
-			for i := 0; noOfPredicates > 0; i++ {
-				col1 := randomEl(tables[idx1].cols)
-				col2 := randomEl(tables[idx2].cols)
-
-				// prevent infinite loops
-				if i > 50 {
-					predicates = append(predicates, sqlparser.NewComparisonExpr(getRandomComparisonExprOperator(), newColumn(col1), newColumn(col2), nil))
-					break
-				}
-
-				if col1.typ != col2.typ {
-					continue
-				}
-
-				predicates = append(predicates, sqlparser.NewComparisonExpr(getRandomComparisonExprOperator(), newColumn(col1), newColumn(col2), nil))
-				noOfPredicates--
-			}
-		}
-	}
+// returns 0-2 random expressions based on tables
+func createWherePredicates(tables []tableT) sqlparser.Exprs {
+	return createPredicates(0, slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
+}
 
-	// make sure the join predicate is never empty
-	if len(predicates) == 0 && isJoin {
-		predRandomExpr := getRandomExpr(tables)
-		predicates = append(predicates, predRandomExpr)
+// returns between minPredicates and minPredicates + 2 random expressions using generators
+func createPredicates(minPredicates int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
+	numPredicates := rand.Intn(3) + minPredicates
+	for i := 0; i < numPredicates; i++ {
+		predicates = append(predicates, getRandomExpr(sqlparser.ExprGeneratorConfig{}, generators...))
 	}
 
 	return
@@ -454,7 +430,9 @@ func createHavingPredicates(tables []tableT) (havingPredicates sqlparser.Exprs)
 			// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
 			// TODO: but random expressions without the columns also fails
 			if testFailingQueries {
-				predRandomExpr := getRandomExpr(tables)
+				predRandomExpr := getRandomExpr(sqlparser.ExprGeneratorConfig{}, slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
+					return &t
+				})...)
 				havingPredicates = append(havingPredicates, sqlparser.NewComparisonExpr(getRandomComparisonExprOperator(), lhs.Expr, predRandomExpr, nil))
 			} else if rhs, ok1 := randomEl(aggrSelectExprs).(*sqlparser.AliasedExpr); ok1 {
 				havingPredicates = append(havingPredicates, sqlparser.NewComparisonExpr(getRandomComparisonExprOperator(), lhs.Expr, rhs.Expr, nil))
@@ -475,13 +453,6 @@ func createLimit() *sqlparser.Limit {
 	return sqlparser.NewLimitWithoutOffset(limitNum)
 }
 
-// returns a random expression and its type
-func getRandomExpr(tables []tableT) sqlparser.Expr {
-	seed := time.Now().UnixNano()
-	g := sqlparser.NewGenerator(seed, 2, slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
-	return g.Expression(sqlparser.ExprGeneratorConfig{})
-}
-
 func newAliasedTable(tbl tableT, alias string) *sqlparser.AliasedTableExpr {
 	return sqlparser.NewAliasedTableExpr(tbl.tableExpr, alias)
 }
@@ -504,6 +475,13 @@ func getRandomOrderDirection() sqlparser.OrderDirection {
 	return randomEl([]sqlparser.OrderDirection{0, 1})
 }
 
+// getRandomExpr returns a random expression
+func getRandomExpr(genConfig sqlparser.ExprGeneratorConfig, generators ...sqlparser.ExprGenerator) sqlparser.Expr {
+	seed := time.Now().UnixNano()
+	g := sqlparser.NewGenerator(seed, 2, generators...)
+	return g.Expression(genConfig)
+}
+
 func randomEl[K any](in []K) K {
 	return in[rand.Intn(len(in))]
 }
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index f1d370bc32f..dc9f2853723 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -99,6 +99,9 @@ func TestMustFix(t *testing.T) {
 	// mismatched results
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(tbl1.loc) as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(tbl1.dname) <= 1")
 
+	// mismatched results
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm")
+
 	// mismatched results
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct max(tbl0.dname) as caggr0, 'cattle' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno != tbl1.sal group by tbl1.comm")
 
@@ -227,7 +230,7 @@ func TestRandom(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	endBy := time.Now().Add(10 * time.Second)
+	endBy := time.Now().Add(1 * time.Second)
 
 	var queryCount int
 	// continue testing after an error if and only if testFailingQueries is true
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 2d634a3269f..1847c8dfdbd 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -157,19 +157,20 @@ func (g *Generator) intExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.caseExpr(genConfig) },
 	}
 
-	// TODO: this adds two identical functions to options, both of which use the last element in g.exprGenerators
 	for i := range g.exprGenerators {
+		generator := g.exprGenerators[i]
+		if generator == nil {
+			continue
+		}
+
 		options = append(options, func() Expr {
-			//fmt.Printf("index: %d", i)
-			expr := g.exprGenerators[i].Generate(genConfig)
+			expr := generator.Generate(genConfig)
 			if expr == nil {
 				return g.intLiteral()
 			}
 			return expr
 		})
 	}
-	//fmt.Printf("options: %v\n", options)
-	//fmt.Printf("expr generators: %v\n", g.exprGenerators)
 
 	return g.randomOf(options)
 }
@@ -186,7 +187,12 @@ func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.caseExpr(genConfig) },
 	}
 
-	for _, generator := range g.exprGenerators {
+	for i := range g.exprGenerators {
+		generator := g.exprGenerators[i]
+		if generator == nil {
+			continue
+		}
+
 		options = append(options, func() Expr {
 			expr := generator.Generate(genConfig)
 			if expr == nil {

From f14943d00577e55fabcbe97d4751b2b2adb1890d Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Fri, 7 Jul 2023 23:44:45 -0700
Subject: [PATCH 05/27] refactor column struct

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 73 +++++++++----------
 .../vtgate/queries/random/random_expr_test.go | 22 +++---
 .../vtgate/queries/random/random_test.go      | 26 +++----
 go/vt/sqlparser/random_expr.go                | 14 +++-
 4 files changed, 69 insertions(+), 66 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 28ee4ddfc89..183cb6df056 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -36,10 +36,10 @@ const testFailingQueries = false
 
 type (
 	column struct {
-		tableName string
-		name      string
-		typ       string
+		name *sqlparser.ColName
+		typ  string
 	}
+
 	tableT struct {
 		// the tableT struct can be used to represent the schema of a table or a derived table
 		// in the former case tableExpr will be a sqlparser.TableName, in the latter a sqlparser.DerivedTable
@@ -54,7 +54,18 @@ type (
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
 var _ sqlparser.ExprGenerator = (*column)(nil)
 
-// var _ sqlparser.ExprGenerator = column{}
+// getColumnName returns tableName.name
+func (c *column) getColumnName() string {
+	return sqlparser.String(c.name)
+}
+
+func (c *column) Generate(genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
+	if c.typ == genConfig.Type {
+		return c.name
+	}
+
+	return nil
+}
 
 func (t *tableT) Generate(genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	colsCopy := slices.Clone(t.cols)
@@ -63,8 +74,7 @@ func (t *tableT) Generate(genConfig sqlparser.ExprGeneratorConfig) sqlparser.Exp
 		idx := rand.Intn(len(colsCopy))
 		randCol := colsCopy[idx]
 		if randCol.typ == genConfig.Type {
-			newTableName := t.getName()
-			return sqlparser.NewColNameWithQualifier(randCol.name, sqlparser.NewTableName(newTableName))
+			return randCol.name
 		}
 
 		// delete randCol from colsCopy
@@ -89,11 +99,11 @@ func (t *tableT) getName() string {
 	return ""
 }
 
-// setName sets the alias for t, as well as setting the tableName for all columns in cols
-func (t *tableT) setName(newName string) {
+// setAlias sets the alias for t, as well as setting the tableName for all columns in cols
+func (t *tableT) setAlias(newName string) {
 	t.alias = newName
 	for i := range t.cols {
-		t.cols[i].tableName = newName
+		t.cols[i].name.Qualifier = sqlparser.NewTableName(newName)
 	}
 }
 
@@ -101,24 +111,11 @@ func (t *tableT) setName(newName string) {
 // this makes it unnatural to modify tableName
 func (t *tableT) addColumns(col ...column) {
 	for i := range col {
-		col[i].tableName = t.getName()
+		col[i].name.Qualifier = sqlparser.NewTableName(t.getName())
 		t.cols = append(t.cols, col[i])
 	}
 }
 
-// getColumnName returns tableName.name
-func (c *column) getColumnName() string {
-	return fmt.Sprintf("%s.%s", c.tableName, c.name)
-}
-
-func (c *column) Generate(genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
-	if c.typ == genConfig.Type {
-		return sqlparser.NewColNameWithQualifier(c.name, sqlparser.NewTableName(c.tableName))
-	}
-
-	return nil
-}
-
 func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Select {
 	sel := &sqlparser.Select{}
 	sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"})
@@ -231,7 +228,7 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 
 		sel.SelectExprs = append(sel.SelectExprs, sqlparser.NewAliasedExpr(randomExpr, "crandom0"))
 		newTable.addColumns(column{
-			name: "crandom0",
+			name: sqlparser.NewColName("crandom0"),
 			typ:  typ,
 		})
 
@@ -267,13 +264,13 @@ func createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT
 	tables = append(tables, schemaTables[rand.Intn(2)])
 
 	sel.From = append(sel.From, newAliasedTable(tables[0], "tbl0"))
-	tables[0].setName("tbl0")
+	tables[0].setAlias("tbl0")
 
 	numTables := rand.Intn(len(schemaTables))
 	for i := 0; i < numTables; i++ {
 		tables = append(tables, randomEl(schemaTables))
 		sel.From = append(sel.From, newAliasedTable(tables[i+1], fmt.Sprintf("tbl%d", i+1)))
-		tables[i+1].setName(fmt.Sprintf("tbl%d", i+1))
+		tables[i+1].setAlias(fmt.Sprintf("tbl%d", i+1))
 	}
 
 	// TODO: outer joins produce mismatched results
@@ -281,7 +278,7 @@ func createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT
 	if isJoin {
 		// TODO: do nested joins
 		newTable := randomEl(schemaTables)
-		newTable.setName(fmt.Sprintf("tbl%d", numTables))
+		newTable.setAlias(fmt.Sprintf("tbl%d", numTables))
 		tables = append(tables, newTable)
 
 		createJoin(tables, sel)
@@ -324,13 +321,13 @@ func createGroupBy(tables []tableT, maxGB int) (groupBy sqlparser.GroupBy, group
 		if col.typ == "date" && !testFailingQueries {
 			continue
 		}
-		groupBy = append(groupBy, newColumn(col))
+		groupBy = append(groupBy, col.name)
 
 		// add to select
 		if rand.Intn(2) < 1 {
 			groupSelectExprs = append(groupSelectExprs, newAliasedColumn(col, fmt.Sprintf("cgroup%d", i)))
 			// TODO: alias in a separate function to properly generate the having clause
-			col.name = fmt.Sprintf("cgroup%d", i)
+			col.name = sqlparser.NewColName(fmt.Sprintf("cgroup%d", i))
 			grouping = append(grouping, col)
 		}
 	}
@@ -342,11 +339,11 @@ func createGroupBy(tables []tableT, maxGB int) (groupBy sqlparser.GroupBy, group
 func createAggregations(tables []tableT, maxAggrs int) (aggrExprs sqlparser.SelectExprs, aggregates []column) {
 	aggregations := []func(col column) sqlparser.Expr{
 		func(_ column) sqlparser.Expr { return &sqlparser.CountStar{} },
-		func(col column) sqlparser.Expr { return &sqlparser.Count{Args: sqlparser.Exprs{newColumn(col)}} },
-		func(col column) sqlparser.Expr { return &sqlparser.Sum{Arg: newColumn(col)} },
-		// func(col column) sqlparser.Expr { return &sqlparser.Avg{Arg: newAggregateExpr(col)} },
-		func(col column) sqlparser.Expr { return &sqlparser.Min{Arg: newColumn(col)} },
-		func(col column) sqlparser.Expr { return &sqlparser.Max{Arg: newColumn(col)} },
+		func(col column) sqlparser.Expr { return &sqlparser.Count{Args: sqlparser.Exprs{col.name}} },
+		func(col column) sqlparser.Expr { return &sqlparser.Sum{Arg: col.name} },
+		// func(col column) sqlparser.Expr { return &sqlparser.Avg{Arg: col.name} },
+		func(col column) sqlparser.Expr { return &sqlparser.Min{Arg: col.name} },
+		func(col column) sqlparser.Expr { return &sqlparser.Max{Arg: col.name} },
 	}
 
 	numAggrs := rand.Intn(maxAggrs)
@@ -383,7 +380,7 @@ func createAggregations(tables []tableT, maxAggrs int) (aggrExprs sqlparser.Sele
 			col.typ = "decimal"
 		}
 
-		col.name = fmt.Sprintf("caggr%d", i)
+		col.name = sqlparser.NewColName(fmt.Sprintf("caggr%d", i))
 		aggregates = append(aggregates, col)
 	}
 	return
@@ -458,11 +455,7 @@ func newAliasedTable(tbl tableT, alias string) *sqlparser.AliasedTableExpr {
 }
 
 func newAliasedColumn(col column, alias string) *sqlparser.AliasedExpr {
-	return sqlparser.NewAliasedExpr(newColumn(col), alias)
-}
-
-func newColumn(col column) *sqlparser.ColName {
-	return sqlparser.NewColNameWithQualifier(col.name, sqlparser.NewTableName(col.tableName))
+	return sqlparser.NewAliasedExpr(col.name, alias)
 }
 
 func getRandomComparisonExprOperator() sqlparser.ComparisonExprOperator {
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index 7c498e1a269..65833c655b8 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -33,19 +33,19 @@ func TestRandomExprWithTables(t *testing.T) {
 		{tableExpr: sqlparser.NewTableName("dept")},
 	}
 	schemaTables[0].addColumns([]column{
-		{name: "empno", typ: "bigint"},
-		{name: "ename", typ: "varchar"},
-		{name: "job", typ: "varchar"},
-		{name: "mgr", typ: "bigint"},
-		{name: "hiredate", typ: "date"},
-		{name: "sal", typ: "bigint"},
-		{name: "comm", typ: "bigint"},
-		{name: "deptno", typ: "bigint"},
+		{name: sqlparser.NewColName("empno"), typ: "bigint"},
+		{name: sqlparser.NewColName("ename"), typ: "varchar"},
+		{name: sqlparser.NewColName("job"), typ: "varchar"},
+		{name: sqlparser.NewColName("mgr"), typ: "bigint"},
+		{name: sqlparser.NewColName("hiredate"), typ: "date"},
+		{name: sqlparser.NewColName("sal"), typ: "bigint"},
+		{name: sqlparser.NewColName("comm"), typ: "bigint"},
+		{name: sqlparser.NewColName("deptno"), typ: "bigint"},
 	}...)
 	schemaTables[1].addColumns([]column{
-		{name: "deptno", typ: "bigint"},
-		{name: "dname", typ: "varchar"},
-		{name: "loc", typ: "varchar"},
+		{name: sqlparser.NewColName("deptno"), typ: "bigint"},
+		{name: sqlparser.NewColName("dname"), typ: "varchar"},
+		{name: sqlparser.NewColName("loc"), typ: "varchar"},
 	}...)
 
 	seed := time.Now().UnixNano()
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index dc9f2853723..6452fccc5c0 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -127,9 +127,6 @@ func TestMustFix(t *testing.T) {
 
 	// EOF
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) from dept as tbl0, (select count(*) from emp as tbl0, emp as tbl1 limit 18) as tbl1")
-
-	// EOF
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0")
 }
 
 func TestKnownFailures(t *testing.T) {
@@ -215,19 +212,19 @@ func TestRandom(t *testing.T) {
 		{tableExpr: sqlparser.NewTableName("dept")},
 	}
 	schemaTables[0].addColumns([]column{
-		{name: "empno", typ: "bigint"},
-		{name: "ename", typ: "varchar"},
-		{name: "job", typ: "varchar"},
-		{name: "mgr", typ: "bigint"},
-		{name: "hiredate", typ: "date"},
-		{name: "sal", typ: "bigint"},
-		{name: "comm", typ: "bigint"},
-		{name: "deptno", typ: "bigint"},
+		{name: sqlparser.NewColName("empno"), typ: "bigint"},
+		{name: sqlparser.NewColName("ename"), typ: "varchar"},
+		{name: sqlparser.NewColName("job"), typ: "varchar"},
+		{name: sqlparser.NewColName("mgr"), typ: "bigint"},
+		{name: sqlparser.NewColName("hiredate"), typ: "date"},
+		{name: sqlparser.NewColName("sal"), typ: "bigint"},
+		{name: sqlparser.NewColName("comm"), typ: "bigint"},
+		{name: sqlparser.NewColName("deptno"), typ: "bigint"},
 	}...)
 	schemaTables[1].addColumns([]column{
-		{name: "deptno", typ: "bigint"},
-		{name: "dname", typ: "varchar"},
-		{name: "loc", typ: "varchar"},
+		{name: sqlparser.NewColName("deptno"), typ: "bigint"},
+		{name: sqlparser.NewColName("dname"), typ: "varchar"},
+		{name: sqlparser.NewColName("loc"), typ: "varchar"},
 	}...)
 
 	endBy := time.Now().Add(1 * time.Second)
@@ -293,5 +290,6 @@ func TestBuggyQueries(t *testing.T) {
 	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(tbl1.loc) as caggr0 from dept as tbl1 left join dept as tbl2 on tbl1.loc = tbl2.loc where (tbl2.deptno)")
 	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ sum(tbl1.ename), min(tbl0.empno) from emp as tbl0, emp as tbl1 left join dept as tbl2 on tbl1.job = tbl2.loc and tbl1.comm = tbl2.deptno where ('trout') and tbl0.deptno = tbl1.comm")
 	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ distinct max(tbl0.deptno), count(tbl0.job) from emp as tbl0, dept as tbl1 left join dept as tbl2 on tbl1.dname = tbl2.loc and tbl1.dname = tbl2.loc where (tbl2.loc) and tbl0.deptno = tbl1.deptno")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0")
 
 }
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 1847c8dfdbd..82b7e034e01 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -29,7 +29,9 @@ type (
 	}
 
 	ExprGeneratorConfig struct {
-		Type string
+		// IsAggregate determines if the random expression is an aggregation expression
+		IsAggregate bool
+		Type        string
 	}
 )
 
@@ -53,6 +55,16 @@ func (genConfig ExprGeneratorConfig) anyTypeConfig() ExprGeneratorConfig {
 	return genConfig
 }
 
+func (genConfig ExprGeneratorConfig) aggregateConfig() ExprGeneratorConfig {
+	genConfig.IsAggregate = true
+	return genConfig
+}
+
+func (genConfig ExprGeneratorConfig) notAggregateConfig() ExprGeneratorConfig {
+	genConfig.IsAggregate = false
+	return genConfig
+}
+
 func NewGenerator(seed int64, maxDepth int, exprGenerators ...ExprGenerator) *Generator {
 	g := Generator{
 		seed:           seed,

From e005e7878225caa4056c8674502fbdf9a2389daf Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Mon, 10 Jul 2023 01:27:18 -0700
Subject: [PATCH 06/27] added aggregation to random expressions and changed
 random query generation to use a single seed

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 315 ++++++++----------
 .../vtgate/queries/random/query_gen_test.go   |  57 ++++
 .../vtgate/queries/random/random_expr_test.go |  28 +-
 .../vtgate/queries/random/random_test.go      |  43 ++-
 go/vt/sqlparser/ast_funcs.go                  |  25 --
 go/vt/sqlparser/ast_test.go                   |  17 -
 go/vt/sqlparser/precedence_test.go            |   4 +-
 go/vt/sqlparser/random_expr.go                |  87 +++--
 go/vt/sqlparser/rewriter_test.go              |   3 +-
 go/vt/sqlparser/walker_test.go                |   5 +-
 10 files changed, 303 insertions(+), 281 deletions(-)
 create mode 100644 go/test/endtoend/vtgate/queries/random/query_gen_test.go

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 183cb6df056..b75743636c8 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -19,7 +19,6 @@ package random
 import (
 	"fmt"
 	"math/rand"
-	"time"
 
 	"golang.org/x/exp/slices"
 
@@ -36,10 +35,11 @@ const testFailingQueries = false
 
 type (
 	column struct {
-		name *sqlparser.ColName
-		typ  string
+		name string
+		// TODO: perhaps remove tableName and always pass columns through a tableT
+		tableName string
+		typ       string
 	}
-
 	tableT struct {
 		// the tableT struct can be used to represent the schema of a table or a derived table
 		// in the former case tableExpr will be a sqlparser.TableName, in the latter a sqlparser.DerivedTable
@@ -54,35 +54,19 @@ type (
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
 var _ sqlparser.ExprGenerator = (*column)(nil)
 
-// getColumnName returns tableName.name
+// getColumnName returns tableName.name (if tableName is nonempty), otherwise name
 func (c *column) getColumnName() string {
-	return sqlparser.String(c.name)
-}
-
-func (c *column) Generate(genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
-	if c.typ == genConfig.Type {
-		return c.name
+	var columnName string
+	if c.tableName != "" {
+		columnName += c.tableName + "."
 	}
 
-	return nil
+	return columnName + c.name
 }
 
-func (t *tableT) Generate(genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
-	colsCopy := slices.Clone(t.cols)
-
-	for len(colsCopy) > 0 {
-		idx := rand.Intn(len(colsCopy))
-		randCol := colsCopy[idx]
-		if randCol.typ == genConfig.Type {
-			return randCol.name
-		}
-
-		// delete randCol from colsCopy
-		colsCopy[idx] = colsCopy[len(colsCopy)-1]
-		colsCopy = colsCopy[:len(colsCopy)-1]
-	}
-
-	return nil
+// getASTExpr returns the AST representation of a column
+func (c *column) getASTExpr() sqlparser.Expr {
+	return sqlparser.NewColNameWithQualifier(c.name, sqlparser.NewTableName(c.tableName))
 }
 
 // getName returns the alias if it is nonempty
@@ -103,7 +87,7 @@ func (t *tableT) getName() string {
 func (t *tableT) setAlias(newName string) {
 	t.alias = newName
 	for i := range t.cols {
-		t.cols[i].name.Qualifier = sqlparser.NewTableName(newName)
+		t.cols[i].tableName = newName
 	}
 }
 
@@ -111,56 +95,83 @@ func (t *tableT) setAlias(newName string) {
 // this makes it unnatural to modify tableName
 func (t *tableT) addColumns(col ...column) {
 	for i := range col {
-		col[i].name.Qualifier = sqlparser.NewTableName(t.getName())
+		col[i].tableName = t.getName()
 		t.cols = append(t.cols, col[i])
 	}
 }
 
-func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Select {
+func (c *column) Generate(_ *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
+	if c.typ == genConfig.Type {
+		return c.getASTExpr()
+	}
+
+	return nil
+}
+
+func (t *tableT) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
+	colsCopy := slices.Clone(t.cols)
+
+	for len(colsCopy) > 0 {
+		idx := r.Intn(len(colsCopy))
+		randCol := colsCopy[idx]
+		if randCol.typ == genConfig.Type {
+			return randCol.getASTExpr()
+		}
+
+		// delete randCol from colsCopy
+		colsCopy[idx] = colsCopy[len(colsCopy)-1]
+		colsCopy = colsCopy[:len(colsCopy)-1]
+	}
+
+	return nil
+}
+
+func randomQuery(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, schemaTables []tableT) *sqlparser.Select {
 	sel := &sqlparser.Select{}
 	sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"})
 
 	// select distinct (fails with group by bigint)
-	isDistinct := rand.Intn(2) < 1
+	isDistinct := r.Intn(2) < 1
 	if isDistinct {
 		sel.MakeDistinct()
 	}
 
 	// create both tables and join at the same time since both occupy the from clause
-	tables, isJoin := createTablesAndJoin(schemaTables, sel)
+	tables, isJoin := createTablesAndJoin(r, genConfig, schemaTables, sel)
 
 	// canAggregate determines if the query will have
 	// aggregate columns, group by, and having
-	canAggregate := rand.Intn(4) > 0
+	canAggregate := r.Intn(4) > 0
 
 	var (
 		groupBy          sqlparser.GroupBy
 		groupSelectExprs sqlparser.SelectExprs
 		grouping         []column
-		aggrExprs        sqlparser.SelectExprs
+		aggrSelectExprs  sqlparser.SelectExprs
 		aggregates       []column
 	)
 	// TODO: distinct makes vitess think there is grouping on aggregation columns
 	if canAggregate {
 		if testFailingQueries || !isDistinct {
 			// group by
-			groupBy, groupSelectExprs, grouping = createGroupBy(tables, maxGroupBy)
+			groupBy, groupSelectExprs, grouping = createGroupBy(r, tables)
 			sel.AddSelectExprs(groupSelectExprs)
 			sel.GroupBy = groupBy
 		}
 
 		// aggregate columns
-		aggrExprs, aggregates = createAggregations(tables, maxAggrs)
-		sel.AddSelectExprs(aggrExprs)
+		aggrSelectExprs, aggregates = createAggregations(r, genConfig, tables)
+		sel.AddSelectExprs(aggrSelectExprs)
+		// sel.GroupBy = append(sel.GroupBy, aggrExprs...)
 
 		// having
-		isHaving := rand.Intn(2) < 1
+		isHaving := r.Intn(2) < 1
 		if isHaving {
-			sel.AddHaving(sqlparser.AndExpressions(createHavingPredicates(tables)...))
-			if rand.Intn(2) < 1 && testFailingQueries {
+			sel.AddHaving(sqlparser.AndExpressions(createHavingPredicates(r, genConfig, grouping)...))
+			if r.Intn(2) < 1 && testFailingQueries {
 				// TODO: having can only contain aggregate or grouping columns in mysql, works fine in vitess
 				// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
-				sel.AddHaving(sqlparser.AndExpressions(createWherePredicates(tables)...))
+				sel.AddHaving(sqlparser.AndExpressions(createWherePredicates(r, genConfig, tables)...))
 			}
 		}
 		// TODO: use sqlparser.ExprGenerator to generate a random expression with aggregation functions
@@ -169,21 +180,21 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 
 	// can add both aggregate and grouping columns to order by
 	// TODO: order fails with distinct and outer joins
-	isOrdered := rand.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries) && testFailingQueries
+	isOrdered := r.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries) && testFailingQueries
 	// TODO: order by fails a lot; probably related to the previously passing query
 	// TODO: should be fixed soon
 	if isOrdered {
-		sel.OrderBy = createOrderBy(groupBy, aggrExprs)
+		sel.OrderBy = createOrderBy(r, groupBy, aggrSelectExprs)
 	}
 
 	// TODO: random expressions cause a lot of failures
 	// where
-	sel.AddWhere(sqlparser.AndExpressions(createWherePredicates(tables)...))
+	sel.AddWhere(sqlparser.AndExpressions(createWherePredicates(r, genConfig, tables)...))
 
 	// only add a limit if the grouping columns are ordered
 	// TODO: limit fails a lot
-	if rand.Intn(2) < 1 && (isOrdered || len(groupBy) == 0) && testFailingQueries {
-		sel.Limit = createLimit()
+	if r.Intn(2) < 1 && (isOrdered || len(groupBy) == 0) && testFailingQueries {
+		sel.Limit = createLimit(r)
 	}
 
 	var (
@@ -193,19 +204,20 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 	)
 	// add random expression to select
 	// TODO: random expressions cause a lot of failures
-	isRandomExpr := rand.Intn(2) < 1 && testFailingQueries
+	isRandomExpr := r.Intn(2) < 1 && testFailingQueries
 
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
 	if testFailingQueries {
 		// TODO: ugly
 		f = func() sqlparser.Expr {
-			return getRandomExpr(sqlparser.ExprGeneratorConfig{}, slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
-				return &t
-			})...)
+			return getRandomExpr(r, sqlparser.ExprGeneratorConfig{},
+				slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
+					return &t
+				})...)
 		}
 	} else {
-		f = func() sqlparser.Expr { return getRandomExpr(sqlparser.ExprGeneratorConfig{}) }
+		f = func() sqlparser.Expr { return getRandomExpr(r, sqlparser.ExprGeneratorConfig{}) }
 	}
 
 	// make sure we have at least one select expression
@@ -228,19 +240,25 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 
 		sel.SelectExprs = append(sel.SelectExprs, sqlparser.NewAliasedExpr(randomExpr, "crandom0"))
 		newTable.addColumns(column{
-			name: sqlparser.NewColName("crandom0"),
+			name: "crandom0",
 			typ:  typ,
 		})
 
 		// make sure to add the random expression to group by and order by for only_full_group_by
 		if canAggregate {
 			sel.AddGroupBy(randomExpr)
-			sel.AddOrder(sqlparser.NewOrder(randomExpr, getRandomOrderDirection()))
+			sel.AddOrder(sqlparser.NewOrder(randomExpr, getRandomOrderDirection(r)))
 		}
 
 		break
 	}
 
+	// alias the grouping columns
+	for i, col := range grouping {
+		alias := fmt.Sprintf("cgroup%d", i)
+		col.name = alias
+	}
+
 	// add them to newTable
 	newTable.addColumns(grouping...)
 	newTable.addColumns(aggregates...)
@@ -251,37 +269,39 @@ func randomQuery(schemaTables []tableT, maxAggrs, maxGroupBy int) *sqlparser.Sel
 
 	// derived tables (partially unsupported)
 	// TODO: derived tables fails a lot
-	if rand.Intn(10) < 1 && testFailingQueries {
-		sel = randomQuery(schemaTables, 3, 3)
+	if r.Intn(10) < 1 && testFailingQueries {
+		sel = randomQuery(r, genConfig, schemaTables)
 	}
 
 	return sel
 }
 
-func createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT, bool) {
+func createTablesAndJoin(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, schemaTables []tableT, sel *sqlparser.Select) ([]tableT, bool) {
 	var tables []tableT
 	// add at least one of original emp/dept tables for now because derived tables have nil columns
-	tables = append(tables, schemaTables[rand.Intn(2)])
+	tables = append(tables, schemaTables[r.Intn(2)])
 
-	sel.From = append(sel.From, newAliasedTable(tables[0], "tbl0"))
 	tables[0].setAlias("tbl0")
+	sel.From = append(sel.From, newAliasedTable(tables[0], "tbl0"))
 
-	numTables := rand.Intn(len(schemaTables))
+	numTables := r.Intn(len(schemaTables))
 	for i := 0; i < numTables; i++ {
-		tables = append(tables, randomEl(schemaTables))
-		sel.From = append(sel.From, newAliasedTable(tables[i+1], fmt.Sprintf("tbl%d", i+1)))
-		tables[i+1].setAlias(fmt.Sprintf("tbl%d", i+1))
+		tables = append(tables, randomEl(r, schemaTables))
+		alias := fmt.Sprintf("tbl%d", i+1)
+		sel.From = append(sel.From, newAliasedTable(tables[i+1], alias))
+		tables[i+1].setAlias(alias)
 	}
 
 	// TODO: outer joins produce mismatched results
-	isJoin := rand.Intn(2) < 1 && testFailingQueries
+	isJoin := r.Intn(2) < 1 && testFailingQueries
 	if isJoin {
 		// TODO: do nested joins
-		newTable := randomEl(schemaTables)
-		newTable.setAlias(fmt.Sprintf("tbl%d", numTables))
+		newTable := randomEl(r, schemaTables)
+		alias := fmt.Sprintf("tbl%d", numTables)
+		newTable.setAlias(alias)
 		tables = append(tables, newTable)
 
-		createJoin(tables, sel)
+		createJoin(r, genConfig, tables, sel)
 	}
 
 	return tables, isJoin
@@ -289,13 +309,13 @@ func createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT
 
 // creates a left join (without the condition) between the last table in sel and newTable
 // tables should have one more table than sel
-func createJoin(tables []tableT, sel *sqlparser.Select) {
+func createJoin(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []tableT, sel *sqlparser.Select) {
 	n := len(sel.From)
 	if len(tables) != n+1 {
 		log.Fatalf("sel has %d tables and tables has %d tables", len(sel.From), n)
 	}
 
-	joinPredicate := sqlparser.AndExpressions(createJoinPredicates(tables)...)
+	joinPredicate := sqlparser.AndExpressions(createJoinPredicates(r, genConfig, tables)...)
 	joinCondition := sqlparser.NewJoinCondition(joinPredicate, nil)
 	newTable := newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n))
 	sel.From[n-1] = sqlparser.NewJoinTableExpr(sel.From[n-1], sqlparser.LeftJoinType, newTable, joinCondition)
@@ -303,31 +323,32 @@ func createJoin(tables []tableT, sel *sqlparser.Select) {
 
 // returns 1-3 random expressions based on the last two elements of tables
 // tables should have at least two elements
-func createJoinPredicates(tables []tableT) sqlparser.Exprs {
+func createJoinPredicates(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []tableT) sqlparser.Exprs {
 	if len(tables) < 2 {
 		log.Fatalf("tables has %d elements, needs at least 2", len(tables))
 	}
 
-	return createPredicates(1, &tables[len(tables)-2], &tables[len(tables)-1])
+	return createRandomExprs(r, genConfig, 1, &tables[len(tables)-2], &tables[len(tables)-1])
 }
 
 // returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
-func createGroupBy(tables []tableT, maxGB int) (groupBy sqlparser.GroupBy, groupSelectExprs sqlparser.SelectExprs, grouping []column) {
-	numGBs := rand.Intn(maxGB)
+func createGroupBy(r *rand.Rand, tables []tableT) (groupBy sqlparser.GroupBy, groupSelectExprs sqlparser.SelectExprs, grouping []column) {
+	numGBs := r.Intn(3)
 	for i := 0; i < numGBs; i++ {
-		tblIdx := rand.Intn(len(tables))
-		col := randomEl(tables[tblIdx].cols)
+		tblIdx := r.Intn(len(tables))
+		col := randomEl(r, tables[tblIdx].cols)
 		// TODO: grouping by a date column sometimes errors
 		if col.typ == "date" && !testFailingQueries {
 			continue
 		}
-		groupBy = append(groupBy, col.name)
+		groupBy = append(groupBy, col.getASTExpr())
 
 		// add to select
-		if rand.Intn(2) < 1 {
-			groupSelectExprs = append(groupSelectExprs, newAliasedColumn(col, fmt.Sprintf("cgroup%d", i)))
+		if r.Intn(2) < 1 {
+			alias := fmt.Sprintf("cgroup%d", len(grouping))
+			groupSelectExprs = append(groupSelectExprs, newAliasedColumn(col, alias))
 			// TODO: alias in a separate function to properly generate the having clause
-			col.name = sqlparser.NewColName(fmt.Sprintf("cgroup%d", i))
+			//col.name = sqlparser.NewColName(alias)
 			grouping = append(grouping, col)
 		}
 	}
@@ -336,67 +357,32 @@ func createGroupBy(tables []tableT, maxGB int) (groupBy sqlparser.GroupBy, group
 }
 
 // returns the aggregation columns as three types: sqlparser.SelectExprs, []column
-func createAggregations(tables []tableT, maxAggrs int) (aggrExprs sqlparser.SelectExprs, aggregates []column) {
-	aggregations := []func(col column) sqlparser.Expr{
-		func(_ column) sqlparser.Expr { return &sqlparser.CountStar{} },
-		func(col column) sqlparser.Expr { return &sqlparser.Count{Args: sqlparser.Exprs{col.name}} },
-		func(col column) sqlparser.Expr { return &sqlparser.Sum{Arg: col.name} },
-		// func(col column) sqlparser.Expr { return &sqlparser.Avg{Arg: col.name} },
-		func(col column) sqlparser.Expr { return &sqlparser.Min{Arg: col.name} },
-		func(col column) sqlparser.Expr { return &sqlparser.Max{Arg: col.name} },
+func createAggregations(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []tableT) (aggrSelectExprs sqlparser.SelectExprs, aggregates []column) {
+	aggrExprs := createRandomExprs(r, genConfig, 0,
+		slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
+			return &t
+		})...)
+	for i := range aggrExprs {
+		expr := sqlparser.RandomAggregate(r, aggrExprs[i])
+		alias := fmt.Sprintf("caggr%d", i)
+		aggrSelectExprs = append(aggrSelectExprs, sqlparser.NewAliasedExpr(expr, alias))
+		aggregates = append(aggregates, column{name: alias})
 	}
 
-	numAggrs := rand.Intn(maxAggrs)
-	for i := 0; i < numAggrs; i++ {
-		tblIdx, aggrIdx := rand.Intn(len(tables)), rand.Intn(len(aggregations))
-		col := randomEl(tables[tblIdx].cols)
-		// TODO: aggregating on a date column sometimes errors
-		if col.typ == "date" && !testFailingQueries {
-			i--
-			continue
-		}
-
-		newAggregate := aggregations[aggrIdx](col)
-		// TODO: collating on strings sometimes errors
-		if col.typ == "varchar" && !testFailingQueries {
-			switch newAggregate.(type) {
-			case *sqlparser.Min, *sqlparser.Max:
-				i--
-				continue
-			}
-		}
-
-		// TODO: type of sum() is incorrect (int64 vs decimal) in certain queries
-		if _, ok := newAggregate.(*sqlparser.Sum); ok && !testFailingQueries {
-			i--
-			continue
-		}
-
-		aggrExprs = append(aggrExprs, sqlparser.NewAliasedExpr(newAggregate, fmt.Sprintf("caggr%d", i)))
-
-		if aggrIdx <= 1 /* CountStar and Count */ {
-			col.typ = "bigint"
-		} else if _, ok := newAggregate.(*sqlparser.Avg); ok && col.getColumnName() == "bigint" {
-			col.typ = "decimal"
-		}
-
-		col.name = sqlparser.NewColName(fmt.Sprintf("caggr%d", i))
-		aggregates = append(aggregates, col)
-	}
 	return
 }
 
 // orders on all non-aggregate SelectExprs and independently at random on all aggregate SelectExprs of sel
-func createOrderBy(groupBy sqlparser.GroupBy, aggrExprs sqlparser.SelectExprs) (orderBy sqlparser.OrderBy) {
+func createOrderBy(r *rand.Rand, groupBy sqlparser.GroupBy, aggrExprs sqlparser.SelectExprs) (orderBy sqlparser.OrderBy) {
 	// always order on grouping columns
 	for i := range groupBy {
-		orderBy = append(orderBy, sqlparser.NewOrder(groupBy[i], getRandomOrderDirection()))
+		orderBy = append(orderBy, sqlparser.NewOrder(groupBy[i], getRandomOrderDirection(r)))
 	}
 
 	// randomly order on aggregation columns
 	for i := range aggrExprs {
-		if aliasedExpr, ok := aggrExprs[i].(*sqlparser.AliasedExpr); ok && rand.Intn(2) < 1 {
-			orderBy = append(orderBy, sqlparser.NewOrder(aliasedExpr.Expr, getRandomOrderDirection()))
+		if aliasedExpr, ok := aggrExprs[i].(*sqlparser.AliasedExpr); ok && r.Intn(2) < 1 {
+			orderBy = append(orderBy, sqlparser.NewOrder(aliasedExpr.Expr, getRandomOrderDirection(r)))
 		}
 	}
 
@@ -404,46 +390,39 @@ func createOrderBy(groupBy sqlparser.GroupBy, aggrExprs sqlparser.SelectExprs) (
 }
 
 // returns 0-2 random expressions based on tables
-func createWherePredicates(tables []tableT) sqlparser.Exprs {
-	return createPredicates(0, slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
+func createWherePredicates(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []tableT) sqlparser.Exprs {
+	// TODO: create gen config outside
+	return createRandomExprs(r, sqlparser.ExprGeneratorConfig{}, 0,
+		slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 }
 
-// returns between minPredicates and minPredicates + 2 random expressions using generators
-func createPredicates(minPredicates int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
-	numPredicates := rand.Intn(3) + minPredicates
+// creates predicates for the having clause comparing a column to a random expression
+func createHavingPredicates(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, grouping []column) sqlparser.Exprs {
+	return createRandomExprs(r, sqlparser.ExprGeneratorConfig{CanAggregate: true}, 0,
+		slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })...)
+}
+
+// returns between minExprs and minExprs + 2 random expressions using generators
+func createRandomExprs(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, minExprs int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
+	numPredicates := r.Intn(3) + minExprs
 	for i := 0; i < numPredicates; i++ {
-		predicates = append(predicates, getRandomExpr(sqlparser.ExprGeneratorConfig{}, generators...))
+		predicates = append(predicates, getRandomExpr(r, genConfig, generators...))
 	}
 
 	return
 }
 
-// creates predicates for the having clause comparing a column to a random expression
-func createHavingPredicates(tables []tableT) (havingPredicates sqlparser.Exprs) {
-	aggrSelectExprs, _ := createAggregations(tables, 2) // createAggregations(generators)
-	for i := range aggrSelectExprs {
-		if lhs, ok := aggrSelectExprs[i].(*sqlparser.AliasedExpr); ok {
-			// TODO: HAVING can only contain aggregate or grouping columns in mysql, works fine in vitess
-			// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
-			// TODO: but random expressions without the columns also fails
-			if testFailingQueries {
-				predRandomExpr := getRandomExpr(sqlparser.ExprGeneratorConfig{}, slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
-					return &t
-				})...)
-				havingPredicates = append(havingPredicates, sqlparser.NewComparisonExpr(getRandomComparisonExprOperator(), lhs.Expr, predRandomExpr, nil))
-			} else if rhs, ok1 := randomEl(aggrSelectExprs).(*sqlparser.AliasedExpr); ok1 {
-				havingPredicates = append(havingPredicates, sqlparser.NewComparisonExpr(getRandomComparisonExprOperator(), lhs.Expr, rhs.Expr, nil))
-			}
-		}
-	}
-	return
+// getRandomExpr returns a random expression
+func getRandomExpr(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, generators ...sqlparser.ExprGenerator) sqlparser.Expr {
+	g := sqlparser.NewGenerator(r, 2, generators...)
+	return g.Expression(genConfig)
 }
 
 // creates sel.Limit
-func createLimit() *sqlparser.Limit {
-	limitNum := rand.Intn(10)
-	if rand.Intn(2) < 1 {
-		offset := rand.Intn(10)
+func createLimit(r *rand.Rand) *sqlparser.Limit {
+	limitNum := r.Intn(10)
+	if r.Intn(2) < 1 {
+		offset := r.Intn(10)
 		return sqlparser.NewLimit(offset, limitNum)
 	}
 
@@ -455,26 +434,14 @@ func newAliasedTable(tbl tableT, alias string) *sqlparser.AliasedTableExpr {
 }
 
 func newAliasedColumn(col column, alias string) *sqlparser.AliasedExpr {
-	return sqlparser.NewAliasedExpr(col.name, alias)
-}
-
-func getRandomComparisonExprOperator() sqlparser.ComparisonExprOperator {
-	// =, <, >, <=, >=, !=, <=>
-	return randomEl([]sqlparser.ComparisonExprOperator{0, 1, 2, 3, 4, 5, 6})
+	return sqlparser.NewAliasedExpr(col.getASTExpr(), alias)
 }
 
-func getRandomOrderDirection() sqlparser.OrderDirection {
+func getRandomOrderDirection(r *rand.Rand) sqlparser.OrderDirection {
 	// asc, desc
-	return randomEl([]sqlparser.OrderDirection{0, 1})
-}
-
-// getRandomExpr returns a random expression
-func getRandomExpr(genConfig sqlparser.ExprGeneratorConfig, generators ...sqlparser.ExprGenerator) sqlparser.Expr {
-	seed := time.Now().UnixNano()
-	g := sqlparser.NewGenerator(seed, 2, generators...)
-	return g.Expression(genConfig)
+	return randomEl(r, []sqlparser.OrderDirection{0, 1})
 }
 
-func randomEl[K any](in []K) K {
-	return in[rand.Intn(len(in))]
+func randomEl[K any](r *rand.Rand, in []K) K {
+	return in[r.Intn(len(in))]
 }
diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
new file mode 100644
index 00000000000..e56a2207b1b
--- /dev/null
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -0,0 +1,57 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package random
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/vt/sqlparser"
+)
+
+// TestSeed makes sure that the seed is deterministic
+func TestSeed(t *testing.T) {
+	// specify the schema (that is defined in schema.sql)
+	schemaTables := []tableT{
+		{tableExpr: sqlparser.NewTableName("emp")},
+		{tableExpr: sqlparser.NewTableName("dept")},
+	}
+	schemaTables[0].addColumns([]column{
+		{name: "empno", typ: "bigint"},
+		{name: "ename", typ: "varchar"},
+		{name: "job", typ: "varchar"},
+		{name: "mgr", typ: "bigint"},
+		{name: "hiredate", typ: "date"},
+		{name: "sal", typ: "bigint"},
+		{name: "comm", typ: "bigint"},
+		{name: "deptno", typ: "bigint"},
+	}...)
+	schemaTables[1].addColumns([]column{
+		{name: "deptno", typ: "bigint"},
+		{name: "dname", typ: "varchar"},
+		{name: "loc", typ: "varchar"},
+	}...)
+
+	seed := int64(1688972682389909000)
+	query1 := sqlparser.String(randomQuery(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, schemaTables))
+	query2 := sqlparser.String(randomQuery(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, schemaTables))
+	fmt.Println(query1)
+	require.Equal(t, query1, query2)
+}
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index 65833c655b8..e69c843e374 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -18,6 +18,7 @@ package random
 
 import (
 	"fmt"
+	"math/rand"
 	"testing"
 	"time"
 
@@ -33,25 +34,26 @@ func TestRandomExprWithTables(t *testing.T) {
 		{tableExpr: sqlparser.NewTableName("dept")},
 	}
 	schemaTables[0].addColumns([]column{
-		{name: sqlparser.NewColName("empno"), typ: "bigint"},
-		{name: sqlparser.NewColName("ename"), typ: "varchar"},
-		{name: sqlparser.NewColName("job"), typ: "varchar"},
-		{name: sqlparser.NewColName("mgr"), typ: "bigint"},
-		{name: sqlparser.NewColName("hiredate"), typ: "date"},
-		{name: sqlparser.NewColName("sal"), typ: "bigint"},
-		{name: sqlparser.NewColName("comm"), typ: "bigint"},
-		{name: sqlparser.NewColName("deptno"), typ: "bigint"},
+		{name: "empno", typ: "bigint"},
+		{name: "ename", typ: "varchar"},
+		{name: "job", typ: "varchar"},
+		{name: "mgr", typ: "bigint"},
+		{name: "hiredate", typ: "date"},
+		{name: "sal", typ: "bigint"},
+		{name: "comm", typ: "bigint"},
+		{name: "deptno", typ: "bigint"},
 	}...)
 	schemaTables[1].addColumns([]column{
-		{name: sqlparser.NewColName("deptno"), typ: "bigint"},
-		{name: sqlparser.NewColName("dname"), typ: "varchar"},
-		{name: sqlparser.NewColName("loc"), typ: "varchar"},
+		{name: "deptno", typ: "bigint"},
+		{name: "dname", typ: "varchar"},
+		{name: "loc", typ: "varchar"},
 	}...)
 
 	seed := time.Now().UnixNano()
-	g := sqlparser.NewGenerator(seed, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
+	r := rand.New(rand.NewSource(seed))
+	g := sqlparser.NewGenerator(r, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 	for i := 0; i < 100; i++ {
-		expr := g.Expression(sqlparser.ExprGeneratorConfig{})
+		expr := g.Expression(sqlparser.ExprGeneratorConfig{CanAggregate: true})
 		fmt.Println(sqlparser.String(expr))
 	}
 }
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 6452fccc5c0..764da5b2566 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -18,6 +18,7 @@ package random
 
 import (
 	"fmt"
+	"math/rand"
 	"strings"
 	"testing"
 	"time"
@@ -34,7 +35,7 @@ import (
 // this test uses the AST defined in the sqlparser package to randomly generate queries
 
 // if true then execution will always stop on a "must fix" error: a mismatched results or EOF
-const stopOnMustFixError = true
+const stopOnMustFixError = false
 
 func start(t *testing.T) (utils.MySQLCompare, func()) {
 	mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams)
@@ -83,6 +84,9 @@ func TestMustFix(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
+	// mismatched results
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1")
+
 	// mismatched results
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
 
@@ -138,6 +142,10 @@ func TestKnownFailures(t *testing.T) {
 	// logs more stuff
 	//clusterInstance.EnableGeneralLog()
 
+	// vitess error: unsupported: min/max on types that are not comparable is not supported
+	// mysql error: <nil>
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ max(case true when false then 'gnu' when true then 'meerkat' end) as caggr0 from dept as tbl0")
+
 	// vitess error: vttablet: rpc error: code = InvalidArgument desc = BIGINT UNSIGNED value is out of range in '(-(273) + (-(15) & 124))'
 	// mysql error: <nil>
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ -273 + (-15 & 124) as crandom0 from emp as tbl0, emp as tbl1 where tbl1.sal >= tbl1.mgr")
@@ -212,32 +220,35 @@ func TestRandom(t *testing.T) {
 		{tableExpr: sqlparser.NewTableName("dept")},
 	}
 	schemaTables[0].addColumns([]column{
-		{name: sqlparser.NewColName("empno"), typ: "bigint"},
-		{name: sqlparser.NewColName("ename"), typ: "varchar"},
-		{name: sqlparser.NewColName("job"), typ: "varchar"},
-		{name: sqlparser.NewColName("mgr"), typ: "bigint"},
-		{name: sqlparser.NewColName("hiredate"), typ: "date"},
-		{name: sqlparser.NewColName("sal"), typ: "bigint"},
-		{name: sqlparser.NewColName("comm"), typ: "bigint"},
-		{name: sqlparser.NewColName("deptno"), typ: "bigint"},
+		{name: "empno", typ: "bigint"},
+		{name: "ename", typ: "varchar"},
+		{name: "job", typ: "varchar"},
+		{name: "mgr", typ: "bigint"},
+		{name: "hiredate", typ: "date"},
+		{name: "sal", typ: "bigint"},
+		{name: "comm", typ: "bigint"},
+		{name: "deptno", typ: "bigint"},
 	}...)
 	schemaTables[1].addColumns([]column{
-		{name: sqlparser.NewColName("deptno"), typ: "bigint"},
-		{name: sqlparser.NewColName("dname"), typ: "varchar"},
-		{name: sqlparser.NewColName("loc"), typ: "varchar"},
+		{name: "deptno", typ: "bigint"},
+		{name: "dname", typ: "varchar"},
+		{name: "loc", typ: "varchar"},
 	}...)
 
 	endBy := time.Now().Add(1 * time.Second)
 
 	var queryCount int
 	// continue testing after an error if and only if testFailingQueries is true
-	for time.Now().Before(endBy) && (!t.Failed() || testFailingQueries) {
-		query := sqlparser.String(randomQuery(schemaTables, 3, 3))
+	for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) {
+		seed := time.Now().UnixNano()
+		fmt.Printf("seed: %d\n", seed)
+		r := rand.New(rand.NewSource(seed))
+		query := sqlparser.String(randomQuery(r, sqlparser.ExprGeneratorConfig{}, schemaTables))
 		_, vtErr := mcmp.ExecAllowAndCompareError(query)
-
+		fmt.Println(query)
 		// this assumes all queries are valid mysql queries
 		if vtErr != nil {
-			fmt.Println(query)
+
 			fmt.Println(vtErr)
 
 			if stopOnMustFixError {
diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go
index 0b7a0ea4541..e4213fe702c 100644
--- a/go/vt/sqlparser/ast_funcs.go
+++ b/go/vt/sqlparser/ast_funcs.go
@@ -601,31 +601,6 @@ func (node *ColName) Equal(c *ColName) bool {
 	return node.Name.Equal(c.Name) && node.Qualifier == c.Qualifier
 }
 
-// Aggregates is a map of all aggregate functions.
-var Aggregates = map[string]bool{
-	"avg":          true,
-	"bit_and":      true,
-	"bit_or":       true,
-	"bit_xor":      true,
-	"count":        true,
-	"group_concat": true,
-	"max":          true,
-	"min":          true,
-	"std":          true,
-	"stddev_pop":   true,
-	"stddev_samp":  true,
-	"stddev":       true,
-	"sum":          true,
-	"var_pop":      true,
-	"var_samp":     true,
-	"variance":     true,
-}
-
-// IsAggregate returns true if the function is an aggregate.
-func (node *FuncExpr) IsAggregate() bool {
-	return Aggregates[node.Name.Lowered()]
-}
-
 // NewIdentifierCI makes a new IdentifierCI.
 func NewIdentifierCI(str string) IdentifierCI {
 	return IdentifierCI{
diff --git a/go/vt/sqlparser/ast_test.go b/go/vt/sqlparser/ast_test.go
index a4254ad7989..2ec067e95aa 100644
--- a/go/vt/sqlparser/ast_test.go
+++ b/go/vt/sqlparser/ast_test.go
@@ -362,23 +362,6 @@ func TestWhere(t *testing.T) {
 	}
 }
 
-func TestIsAggregate(t *testing.T) {
-	f := FuncExpr{Name: NewIdentifierCI("avg")}
-	if !f.IsAggregate() {
-		t.Error("IsAggregate: false, want true")
-	}
-
-	f = FuncExpr{Name: NewIdentifierCI("Avg")}
-	if !f.IsAggregate() {
-		t.Error("IsAggregate: false, want true")
-	}
-
-	f = FuncExpr{Name: NewIdentifierCI("foo")}
-	if f.IsAggregate() {
-		t.Error("IsAggregate: true, want false")
-	}
-}
-
 func TestIsImpossible(t *testing.T) {
 	f := ComparisonExpr{
 		Operator: NotEqualOp,
diff --git a/go/vt/sqlparser/precedence_test.go b/go/vt/sqlparser/precedence_test.go
index 4f2c1cbe319..ebab6bbd698 100644
--- a/go/vt/sqlparser/precedence_test.go
+++ b/go/vt/sqlparser/precedence_test.go
@@ -18,6 +18,7 @@ package sqlparser
 
 import (
 	"fmt"
+	"math/rand"
 	"testing"
 	"time"
 
@@ -215,8 +216,9 @@ func TestRandom(t *testing.T) {
 	// The purpose of this test is to find discrepancies between Format and parsing. If for example our precedence rules are not consistent between the two, this test should find it.
 	// The idea is to generate random queries, and pass them through the parser and then the unparser, and one more time. The result of the first unparse should be the same as the second result.
 	seed := time.Now().UnixNano()
+	r := rand.New(rand.NewSource(seed))
 	fmt.Println(fmt.Sprintf("seed is %d", seed)) // nolint
-	g := NewGenerator(seed, 5)
+	g := NewGenerator(r, 5)
 	endBy := time.Now().Add(1 * time.Second)
 
 	for {
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 82b7e034e01..2bf4f7a8915 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -25,50 +25,49 @@ import (
 
 type (
 	ExprGenerator interface {
-		Generate(config ExprGeneratorConfig) Expr
+		Generate(r *rand.Rand, config ExprGeneratorConfig) Expr
 	}
 
 	ExprGeneratorConfig struct {
 		// IsAggregate determines if the random expression is an aggregation expression
-		IsAggregate bool
-		Type        string
+		CanAggregate bool
+		Type         string
 	}
 )
 
-func (genConfig ExprGeneratorConfig) boolTypeConfig() ExprGeneratorConfig {
-	genConfig.Type = "tinyint"
-	return genConfig
+func (egc ExprGeneratorConfig) boolTypeConfig() ExprGeneratorConfig {
+	egc.Type = "tinyint"
+	return egc
 }
 
-func (genConfig ExprGeneratorConfig) intTypeConfig() ExprGeneratorConfig {
-	genConfig.Type = "bigint"
-	return genConfig
+func (egc ExprGeneratorConfig) intTypeConfig() ExprGeneratorConfig {
+	egc.Type = "bigint"
+	return egc
 }
 
-func (genConfig ExprGeneratorConfig) stringTypeConfig() ExprGeneratorConfig {
-	genConfig.Type = "varchar"
-	return genConfig
+func (egc ExprGeneratorConfig) stringTypeConfig() ExprGeneratorConfig {
+	egc.Type = "varchar"
+	return egc
 }
 
-func (genConfig ExprGeneratorConfig) anyTypeConfig() ExprGeneratorConfig {
-	genConfig.Type = ""
-	return genConfig
+func (egc ExprGeneratorConfig) anyTypeConfig() ExprGeneratorConfig {
+	egc.Type = ""
+	return egc
 }
 
-func (genConfig ExprGeneratorConfig) aggregateConfig() ExprGeneratorConfig {
-	genConfig.IsAggregate = true
-	return genConfig
+func (egc ExprGeneratorConfig) CanAggregateConfig() ExprGeneratorConfig {
+	egc.CanAggregate = true
+	return egc
 }
 
-func (genConfig ExprGeneratorConfig) notAggregateConfig() ExprGeneratorConfig {
-	genConfig.IsAggregate = false
-	return genConfig
+func (egc ExprGeneratorConfig) cannotAggregateConfig() ExprGeneratorConfig {
+	egc.CanAggregate = false
+	return egc
 }
 
-func NewGenerator(seed int64, maxDepth int, exprGenerators ...ExprGenerator) *Generator {
+func NewGenerator(r *rand.Rand, maxDepth int, exprGenerators ...ExprGenerator) *Generator {
 	g := Generator{
-		seed:           seed,
-		r:              rand.New(rand.NewSource(seed)),
+		r:              r,
 		maxDepth:       maxDepth,
 		exprGenerators: exprGenerators,
 	}
@@ -76,7 +75,6 @@ func NewGenerator(seed int64, maxDepth int, exprGenerators ...ExprGenerator) *Ge
 }
 
 type Generator struct {
-	seed           int64
 	r              *rand.Rand
 	depth          int
 	maxDepth       int
@@ -129,15 +127,40 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.booleanExpr(genConfig) },
 	}
 
+	if genConfig.CanAggregate {
+		genConfig = genConfig.cannotAggregateConfig()
+		options = append(options, func() Expr { return RandomAggregate(g.r, g.Expression(genConfig)) })
+	}
+
 	return g.randomOf(options)
 }
 
+var aggregates = []string{"count(*)", "count", "sum" /*, "min", "max" */}
+
+func RandomAggregate(r *rand.Rand, expr Expr) Expr {
+	aggr := aggregates[r.Intn(len(aggregates))]
+	switch aggr {
+	case "count(*)":
+		return &CountStar{}
+	case "count":
+		return &Count{Args: Exprs{expr}}
+	case "sum":
+		return &Sum{Arg: expr}
+	case "min":
+		return &Min{Arg: expr}
+	case "max":
+		return &Max{Arg: expr}
+	}
+
+	return expr
+}
+
 func (g *Generator) booleanExpr(genConfig ExprGeneratorConfig) Expr {
 	if g.atMaxDepth() {
 		return g.booleanLiteral()
 	}
 
-	genConfig.Type = "tinyint"
+	genConfig = genConfig.boolTypeConfig()
 
 	options := []exprF{
 		func() Expr { return g.andExpr(genConfig) },
@@ -161,7 +184,7 @@ func (g *Generator) intExpr(genConfig ExprGeneratorConfig) Expr {
 		return g.intLiteral()
 	}
 
-	genConfig.Type = "bigint"
+	genConfig = genConfig.intTypeConfig()
 
 	options := []exprF{
 		func() Expr { return g.arithmetic(genConfig) },
@@ -176,7 +199,7 @@ func (g *Generator) intExpr(genConfig ExprGeneratorConfig) Expr {
 		}
 
 		options = append(options, func() Expr {
-			expr := generator.Generate(genConfig)
+			expr := generator.Generate(g.r, genConfig)
 			if expr == nil {
 				return g.intLiteral()
 			}
@@ -192,7 +215,7 @@ func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 		return g.stringLiteral()
 	}
 
-	genConfig.Type = "varchar"
+	genConfig = genConfig.stringTypeConfig()
 
 	options := []exprF{
 		func() Expr { return g.stringLiteral() },
@@ -206,7 +229,7 @@ func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 		}
 
 		options = append(options, func() Expr {
-			expr := generator.Generate(genConfig)
+			expr := generator.Generate(g.r, genConfig)
 			if expr == nil {
 				return g.stringLiteral()
 			}
@@ -226,7 +249,7 @@ func (g *Generator) randomBool() bool {
 }
 
 func (g *Generator) intLiteral() Expr {
-	t := fmt.Sprintf("%d", g.r.Intn(1000)-g.r.Intn(1000))
+	t := fmt.Sprintf("%d", g.r.Intn(100)-g.r.Intn(100))
 
 	return NewIntLiteral(t)
 }
@@ -274,7 +297,7 @@ func (g *Generator) caseExpr(genConfig ExprGeneratorConfig) Expr {
 		elseExpr = g.Expression(genConfig)
 	}
 
-	size := g.r.Intn(2) + 2
+	size := g.r.Intn(2) + 1
 	var whens []*When
 	for i := 0; i < size; i++ {
 		var cond Expr
diff --git a/go/vt/sqlparser/rewriter_test.go b/go/vt/sqlparser/rewriter_test.go
index 3f4f9823ae2..3044e04f8b0 100644
--- a/go/vt/sqlparser/rewriter_test.go
+++ b/go/vt/sqlparser/rewriter_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package sqlparser
 
 import (
+	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -25,7 +26,7 @@ import (
 )
 
 func BenchmarkVisitLargeExpression(b *testing.B) {
-	gen := NewGenerator(1, 5)
+	gen := NewGenerator(rand.New(rand.NewSource(1)), 5)
 	exp := gen.Expression(ExprGeneratorConfig{})
 
 	depth := 0
diff --git a/go/vt/sqlparser/walker_test.go b/go/vt/sqlparser/walker_test.go
index 70432591b08..560ed2ff470 100644
--- a/go/vt/sqlparser/walker_test.go
+++ b/go/vt/sqlparser/walker_test.go
@@ -18,6 +18,7 @@ package sqlparser
 
 import (
 	"fmt"
+	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -26,7 +27,7 @@ import (
 func BenchmarkWalkLargeExpression(b *testing.B) {
 	for i := 0; i < 10; i++ {
 		b.Run(fmt.Sprintf("%d", i), func(b *testing.B) {
-			exp := NewGenerator(int64(i*100), 5).Expression(ExprGeneratorConfig{})
+			exp := NewGenerator(rand.New(rand.NewSource(int64(i*100))), 5).Expression(ExprGeneratorConfig{})
 			count := 0
 			for i := 0; i < b.N; i++ {
 				err := Walk(func(node SQLNode) (kontinue bool, err error) {
@@ -42,7 +43,7 @@ func BenchmarkWalkLargeExpression(b *testing.B) {
 func BenchmarkRewriteLargeExpression(b *testing.B) {
 	for i := 1; i < 7; i++ {
 		b.Run(fmt.Sprintf("%d", i), func(b *testing.B) {
-			exp := NewGenerator(int64(i*100), i).Expression(ExprGeneratorConfig{})
+			exp := NewGenerator(rand.New(rand.NewSource(int64(i*100))), i).Expression(ExprGeneratorConfig{})
 			count := 0
 			for i := 0; i < b.N; i++ {
 				_ = Rewrite(exp, func(_ *Cursor) bool {

From 8e28e6011acf8fd4881d4783431062f600734114 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Tue, 11 Jul 2023 00:57:24 -0700
Subject: [PATCH 07/27] added queryGen struct to pass along Rand and
 ExprGeneratorConfig

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 173 ++++++++++--------
 .../vtgate/queries/random/query_gen_test.go   |   8 +-
 .../vtgate/queries/random/random_expr_test.go |   4 +-
 .../vtgate/queries/random/random_test.go      |   9 +-
 go/vt/sqlparser/random_expr.go                |  14 +-
 5 files changed, 116 insertions(+), 92 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index b75743636c8..19b4df3d093 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -34,6 +34,10 @@ import (
 const testFailingQueries = false
 
 type (
+	queryGen struct {
+		r         *rand.Rand
+		genConfig sqlparser.ExprGeneratorConfig
+	}
 	column struct {
 		name string
 		// TODO: perhaps remove tableName and always pass columns through a tableT
@@ -54,6 +58,13 @@ type (
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
 var _ sqlparser.ExprGenerator = (*column)(nil)
 
+func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) queryGen {
+	return queryGen{
+		r:         r,
+		genConfig: genConfig,
+	}
+}
+
 // getColumnName returns tableName.name (if tableName is nonempty), otherwise name
 func (c *column) getColumnName() string {
 	var columnName string
@@ -126,22 +137,25 @@ func (t *tableT) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig)
 	return nil
 }
 
-func randomQuery(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, schemaTables []tableT) *sqlparser.Select {
+func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
+	// make sure the random expressions can generally not contain aggregates; change appropriately
+	qg.genConfig = qg.genConfig.CannotAggregateConfig()
+
 	sel := &sqlparser.Select{}
 	sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"})
 
 	// select distinct (fails with group by bigint)
-	isDistinct := r.Intn(2) < 1
+	isDistinct := qg.r.Intn(2) < 1
 	if isDistinct {
 		sel.MakeDistinct()
 	}
 
 	// create both tables and join at the same time since both occupy the from clause
-	tables, isJoin := createTablesAndJoin(r, genConfig, schemaTables, sel)
+	tables, isJoin := qg.createTablesAndJoin(schemaTables, sel)
 
 	// canAggregate determines if the query will have
 	// aggregate columns, group by, and having
-	canAggregate := r.Intn(4) > 0
+	canAggregate := qg.r.Intn(4) > 0
 
 	var (
 		groupBy          sqlparser.GroupBy
@@ -154,47 +168,38 @@ func randomQuery(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, schemaTa
 	if canAggregate {
 		if testFailingQueries || !isDistinct {
 			// group by
-			groupBy, groupSelectExprs, grouping = createGroupBy(r, tables)
+			groupBy, groupSelectExprs, grouping = qg.createGroupBy(tables)
 			sel.AddSelectExprs(groupSelectExprs)
 			sel.GroupBy = groupBy
 		}
 
 		// aggregate columns
-		aggrSelectExprs, aggregates = createAggregations(r, genConfig, tables)
+		aggrSelectExprs, aggregates = qg.createAggregations(tables)
 		sel.AddSelectExprs(aggrSelectExprs)
 		// sel.GroupBy = append(sel.GroupBy, aggrExprs...)
 
 		// having
-		isHaving := r.Intn(2) < 1
+		isHaving := qg.r.Intn(2) < 1
 		if isHaving {
-			sel.AddHaving(sqlparser.AndExpressions(createHavingPredicates(r, genConfig, grouping)...))
-			if r.Intn(2) < 1 && testFailingQueries {
-				// TODO: having can only contain aggregate or grouping columns in mysql, works fine in vitess
-				// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
-				sel.AddHaving(sqlparser.AndExpressions(createWherePredicates(r, genConfig, tables)...))
-			}
+			sel.AddHaving(sqlparser.AndExpressions(qg.createHavingPredicates(grouping)...))
 		}
-		// TODO: use sqlparser.ExprGenerator to generate a random expression with aggregation functions
-
 	}
 
 	// can add both aggregate and grouping columns to order by
 	// TODO: order fails with distinct and outer joins
-	isOrdered := r.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries) && testFailingQueries
-	// TODO: order by fails a lot; probably related to the previously passing query
-	// TODO: should be fixed soon
+	isOrdered := qg.r.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries)
 	if isOrdered {
-		sel.OrderBy = createOrderBy(r, groupBy, aggrSelectExprs)
+		sel.OrderBy = qg.createOrderBy(groupBy, aggrSelectExprs)
 	}
 
-	// TODO: random expressions cause a lot of failures
 	// where
-	sel.AddWhere(sqlparser.AndExpressions(createWherePredicates(r, genConfig, tables)...))
+	sel.AddWhere(sqlparser.AndExpressions(qg.createWherePredicates(tables)...))
 
 	// only add a limit if the grouping columns are ordered
 	// TODO: limit fails a lot
-	if r.Intn(2) < 1 && (isOrdered || len(groupBy) == 0) && testFailingQueries {
-		sel.Limit = createLimit(r)
+	isLimit := qg.r.Intn(2) < 1 && (isOrdered || len(groupBy) == 0) && testFailingQueries
+	if isLimit {
+		sel.Limit = qg.createLimit()
 	}
 
 	var (
@@ -204,20 +209,17 @@ func randomQuery(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, schemaTa
 	)
 	// add random expression to select
 	// TODO: random expressions cause a lot of failures
-	isRandomExpr := r.Intn(2) < 1 && testFailingQueries
+	isRandomExpr := qg.r.Intn(2) < 1 && testFailingQueries
 
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
 	if testFailingQueries {
 		// TODO: ugly
 		f = func() sqlparser.Expr {
-			return getRandomExpr(r, sqlparser.ExprGeneratorConfig{},
-				slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
-					return &t
-				})...)
+			return qg.getRandomExpr(slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 		}
 	} else {
-		f = func() sqlparser.Expr { return getRandomExpr(r, sqlparser.ExprGeneratorConfig{}) }
+		f = func() sqlparser.Expr { return qg.getRandomExpr() }
 	}
 
 	// make sure we have at least one select expression
@@ -247,7 +249,9 @@ func randomQuery(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, schemaTa
 		// make sure to add the random expression to group by and order by for only_full_group_by
 		if canAggregate {
 			sel.AddGroupBy(randomExpr)
-			sel.AddOrder(sqlparser.NewOrder(randomExpr, getRandomOrderDirection(r)))
+			if isOrdered || isLimit {
+				sel.AddOrder(sqlparser.NewOrder(randomExpr, qg.getRandomOrderDirection()))
+			}
 		}
 
 		break
@@ -269,39 +273,39 @@ func randomQuery(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, schemaTa
 
 	// derived tables (partially unsupported)
 	// TODO: derived tables fails a lot
-	if r.Intn(10) < 1 && testFailingQueries {
-		sel = randomQuery(r, genConfig, schemaTables)
+	if qg.r.Intn(10) < 1 && testFailingQueries {
+		sel = qg.randomQuery(schemaTables)
 	}
 
 	return sel
 }
 
-func createTablesAndJoin(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, schemaTables []tableT, sel *sqlparser.Select) ([]tableT, bool) {
+func (qg queryGen) createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT, bool) {
 	var tables []tableT
 	// add at least one of original emp/dept tables for now because derived tables have nil columns
-	tables = append(tables, schemaTables[r.Intn(2)])
+	tables = append(tables, schemaTables[qg.r.Intn(2)])
 
 	tables[0].setAlias("tbl0")
 	sel.From = append(sel.From, newAliasedTable(tables[0], "tbl0"))
 
-	numTables := r.Intn(len(schemaTables))
+	numTables := qg.r.Intn(len(schemaTables))
 	for i := 0; i < numTables; i++ {
-		tables = append(tables, randomEl(r, schemaTables))
+		tables = append(tables, randomEl(qg.r, schemaTables))
 		alias := fmt.Sprintf("tbl%d", i+1)
 		sel.From = append(sel.From, newAliasedTable(tables[i+1], alias))
 		tables[i+1].setAlias(alias)
 	}
 
 	// TODO: outer joins produce mismatched results
-	isJoin := r.Intn(2) < 1 && testFailingQueries
+	isJoin := qg.r.Intn(2) < 1 && testFailingQueries
 	if isJoin {
 		// TODO: do nested joins
-		newTable := randomEl(r, schemaTables)
+		newTable := randomEl(qg.r, schemaTables)
 		alias := fmt.Sprintf("tbl%d", numTables)
 		newTable.setAlias(alias)
 		tables = append(tables, newTable)
 
-		createJoin(r, genConfig, tables, sel)
+		qg.createJoin(tables, sel)
 	}
 
 	return tables, isJoin
@@ -309,13 +313,13 @@ func createTablesAndJoin(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig,
 
 // creates a left join (without the condition) between the last table in sel and newTable
 // tables should have one more table than sel
-func createJoin(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []tableT, sel *sqlparser.Select) {
+func (qg queryGen) createJoin(tables []tableT, sel *sqlparser.Select) {
 	n := len(sel.From)
 	if len(tables) != n+1 {
 		log.Fatalf("sel has %d tables and tables has %d tables", len(sel.From), n)
 	}
 
-	joinPredicate := sqlparser.AndExpressions(createJoinPredicates(r, genConfig, tables)...)
+	joinPredicate := sqlparser.AndExpressions(qg.createJoinPredicates(tables)...)
 	joinCondition := sqlparser.NewJoinCondition(joinPredicate, nil)
 	newTable := newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n))
 	sel.From[n-1] = sqlparser.NewJoinTableExpr(sel.From[n-1], sqlparser.LeftJoinType, newTable, joinCondition)
@@ -323,20 +327,20 @@ func createJoin(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []
 
 // returns 1-3 random expressions based on the last two elements of tables
 // tables should have at least two elements
-func createJoinPredicates(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []tableT) sqlparser.Exprs {
+func (qg queryGen) createJoinPredicates(tables []tableT) sqlparser.Exprs {
 	if len(tables) < 2 {
 		log.Fatalf("tables has %d elements, needs at least 2", len(tables))
 	}
 
-	return createRandomExprs(r, genConfig, 1, &tables[len(tables)-2], &tables[len(tables)-1])
+	return qg.createRandomExprs(1, &tables[len(tables)-2], &tables[len(tables)-1])
 }
 
 // returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
-func createGroupBy(r *rand.Rand, tables []tableT) (groupBy sqlparser.GroupBy, groupSelectExprs sqlparser.SelectExprs, grouping []column) {
-	numGBs := r.Intn(3)
+func (qg queryGen) createGroupBy(tables []tableT) (groupBy sqlparser.GroupBy, groupSelectExprs sqlparser.SelectExprs, grouping []column) {
+	numGBs := qg.r.Intn(3)
 	for i := 0; i < numGBs; i++ {
-		tblIdx := r.Intn(len(tables))
-		col := randomEl(r, tables[tblIdx].cols)
+		tblIdx := qg.r.Intn(len(tables))
+		col := randomEl(qg.r, tables[tblIdx].cols)
 		// TODO: grouping by a date column sometimes errors
 		if col.typ == "date" && !testFailingQueries {
 			continue
@@ -344,7 +348,7 @@ func createGroupBy(r *rand.Rand, tables []tableT) (groupBy sqlparser.GroupBy, gr
 		groupBy = append(groupBy, col.getASTExpr())
 
 		// add to select
-		if r.Intn(2) < 1 {
+		if qg.r.Intn(2) < 1 {
 			alias := fmt.Sprintf("cgroup%d", len(grouping))
 			groupSelectExprs = append(groupSelectExprs, newAliasedColumn(col, alias))
 			// TODO: alias in a separate function to properly generate the having clause
@@ -357,13 +361,13 @@ func createGroupBy(r *rand.Rand, tables []tableT) (groupBy sqlparser.GroupBy, gr
 }
 
 // returns the aggregation columns as three types: sqlparser.SelectExprs, []column
-func createAggregations(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []tableT) (aggrSelectExprs sqlparser.SelectExprs, aggregates []column) {
-	aggrExprs := createRandomExprs(r, genConfig, 0,
+func (qg queryGen) createAggregations(tables []tableT) (aggrSelectExprs sqlparser.SelectExprs, aggregates []column) {
+	aggrExprs := qg.createRandomExprs(0,
 		slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
 			return &t
 		})...)
 	for i := range aggrExprs {
-		expr := sqlparser.RandomAggregate(r, aggrExprs[i])
+		expr := sqlparser.RandomAggregate(qg.r, aggrExprs[i])
 		alias := fmt.Sprintf("caggr%d", i)
 		aggrSelectExprs = append(aggrSelectExprs, sqlparser.NewAliasedExpr(expr, alias))
 		aggregates = append(aggregates, column{name: alias})
@@ -373,16 +377,16 @@ func createAggregations(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, t
 }
 
 // orders on all non-aggregate SelectExprs and independently at random on all aggregate SelectExprs of sel
-func createOrderBy(r *rand.Rand, groupBy sqlparser.GroupBy, aggrExprs sqlparser.SelectExprs) (orderBy sqlparser.OrderBy) {
+func (qg queryGen) createOrderBy(groupBy sqlparser.GroupBy, aggrExprs sqlparser.SelectExprs) (orderBy sqlparser.OrderBy) {
 	// always order on grouping columns
 	for i := range groupBy {
-		orderBy = append(orderBy, sqlparser.NewOrder(groupBy[i], getRandomOrderDirection(r)))
+		orderBy = append(orderBy, sqlparser.NewOrder(groupBy[i], qg.getRandomOrderDirection()))
 	}
 
 	// randomly order on aggregation columns
 	for i := range aggrExprs {
-		if aliasedExpr, ok := aggrExprs[i].(*sqlparser.AliasedExpr); ok && r.Intn(2) < 1 {
-			orderBy = append(orderBy, sqlparser.NewOrder(aliasedExpr.Expr, getRandomOrderDirection(r)))
+		if aliasedExpr, ok := aggrExprs[i].(*sqlparser.AliasedExpr); ok && qg.r.Intn(2) < 1 {
+			orderBy = append(orderBy, sqlparser.NewOrder(aliasedExpr.Expr, qg.getRandomOrderDirection()))
 		}
 	}
 
@@ -390,58 +394,69 @@ func createOrderBy(r *rand.Rand, groupBy sqlparser.GroupBy, aggrExprs sqlparser.
 }
 
 // returns 0-2 random expressions based on tables
-func createWherePredicates(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, tables []tableT) sqlparser.Exprs {
+func (qg queryGen) createWherePredicates(tables []tableT) sqlparser.Exprs {
 	// TODO: create gen config outside
-	return createRandomExprs(r, sqlparser.ExprGeneratorConfig{}, 0,
+	return qg.createRandomExprs(0,
 		slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 }
 
 // creates predicates for the having clause comparing a column to a random expression
-func createHavingPredicates(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, grouping []column) sqlparser.Exprs {
-	return createRandomExprs(r, sqlparser.ExprGeneratorConfig{CanAggregate: true}, 0,
-		slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })...)
+func (qg queryGen) createHavingPredicates(grouping []column) sqlparser.Exprs {
+	predicates := sqlparser.Exprs{}
+	if qg.r.Intn(2) < 1 && testFailingQueries {
+		// TODO: having can only contain aggregate or grouping columns in mysql, works fine in vitess
+		// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
+		predicates = append(predicates, qg.createRandomExprs(0,
+			slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })...)...)
+	}
+
+	qg.genConfig = qg.genConfig.CanAggregateConfig()
+	predicates = append(predicates, qg.createRandomExprs(0,
+		slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })...)...)
+
+	return predicates
 }
 
 // returns between minExprs and minExprs + 2 random expressions using generators
-func createRandomExprs(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, minExprs int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
-	numPredicates := r.Intn(3) + minExprs
+func (qg queryGen) createRandomExprs(minExprs int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
+	numPredicates := qg.r.Intn(3) + minExprs
 	for i := 0; i < numPredicates; i++ {
-		predicates = append(predicates, getRandomExpr(r, genConfig, generators...))
+		predicates = append(predicates, qg.getRandomExpr(generators...))
 	}
 
 	return
 }
 
 // getRandomExpr returns a random expression
-func getRandomExpr(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, generators ...sqlparser.ExprGenerator) sqlparser.Expr {
-	g := sqlparser.NewGenerator(r, 2, generators...)
-	return g.Expression(genConfig)
+func (qg queryGen) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlparser.Expr {
+	g := sqlparser.NewGenerator(qg.r, 2, generators...)
+	return g.Expression(qg.genConfig)
 }
 
 // creates sel.Limit
-func createLimit(r *rand.Rand) *sqlparser.Limit {
-	limitNum := r.Intn(10)
-	if r.Intn(2) < 1 {
-		offset := r.Intn(10)
+func (qg queryGen) createLimit() *sqlparser.Limit {
+	limitNum := qg.r.Intn(10)
+	if qg.r.Intn(2) < 1 {
+		offset := qg.r.Intn(10)
 		return sqlparser.NewLimit(offset, limitNum)
 	}
 
 	return sqlparser.NewLimitWithoutOffset(limitNum)
 }
 
-func newAliasedTable(tbl tableT, alias string) *sqlparser.AliasedTableExpr {
-	return sqlparser.NewAliasedTableExpr(tbl.tableExpr, alias)
+func (qg queryGen) getRandomOrderDirection() sqlparser.OrderDirection {
+	// asc, desc
+	return randomEl(qg.r, []sqlparser.OrderDirection{0, 1})
 }
 
-func newAliasedColumn(col column, alias string) *sqlparser.AliasedExpr {
-	return sqlparser.NewAliasedExpr(col.getASTExpr(), alias)
+func randomEl[K any](r *rand.Rand, in []K) K {
+	return in[r.Intn(len(in))]
 }
 
-func getRandomOrderDirection(r *rand.Rand) sqlparser.OrderDirection {
-	// asc, desc
-	return randomEl(r, []sqlparser.OrderDirection{0, 1})
+func newAliasedTable(tbl tableT, alias string) *sqlparser.AliasedTableExpr {
+	return sqlparser.NewAliasedTableExpr(tbl.tableExpr, alias)
 }
 
-func randomEl[K any](r *rand.Rand, in []K) K {
-	return in[r.Intn(len(in))]
+func newAliasedColumn(col column, alias string) *sqlparser.AliasedExpr {
+	return sqlparser.NewAliasedExpr(col.getASTExpr(), alias)
 }
diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index e56a2207b1b..900eb83983e 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -49,9 +49,11 @@ func TestSeed(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	seed := int64(1688972682389909000)
-	query1 := sqlparser.String(randomQuery(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, schemaTables))
-	query2 := sqlparser.String(randomQuery(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, schemaTables))
+	seed := int64(1)
+	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
+	query1 := sqlparser.String(qg.randomQuery(schemaTables))
+	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
+	query2 := sqlparser.String(qg.randomQuery(schemaTables))
 	fmt.Println(query1)
 	require.Equal(t, query1, query2)
 }
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index e69c843e374..e5e3a3bcf86 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -26,9 +26,9 @@ import (
 	"vitess.io/vitess/go/vt/sqlparser"
 )
 
-// This test tests that generating a random expression with a schema does not panic
+// This test tests that generating random expressions with a schema does not panic
 func TestRandomExprWithTables(t *testing.T) {
-
+	// specify the schema (that is defined in schema.sql)
 	schemaTables := []tableT{
 		{tableExpr: sqlparser.NewTableName("emp")},
 		{tableExpr: sqlparser.NewTableName("dept")},
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 764da5b2566..4603893150a 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -84,6 +84,11 @@ func TestMustFix(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
+	// panics instead of normally logging the error
+	// Received unexpected error:
+	// VT12001: unsupported: only one DISTINCT aggregation is allowed in a SELECT: sum(distinct 1) as caggr1
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(distinct tbl0.comm) as caggr0, sum(distinct 1) as caggr1 from emp as tbl0, emp as tbl1 having 'redfish' < 'blowfish'")
+
 	// mismatched results
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1")
 
@@ -242,8 +247,8 @@ func TestRandom(t *testing.T) {
 	for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) {
 		seed := time.Now().UnixNano()
 		fmt.Printf("seed: %d\n", seed)
-		r := rand.New(rand.NewSource(seed))
-		query := sqlparser.String(randomQuery(r, sqlparser.ExprGeneratorConfig{}, schemaTables))
+		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
+		query := sqlparser.String(qg.randomQuery(schemaTables))
 		_, vtErr := mcmp.ExecAllowAndCompareError(query)
 		fmt.Println(query)
 		// this assumes all queries are valid mysql queries
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 2bf4f7a8915..d973dbefcdb 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -60,7 +60,7 @@ func (egc ExprGeneratorConfig) CanAggregateConfig() ExprGeneratorConfig {
 	return egc
 }
 
-func (egc ExprGeneratorConfig) cannotAggregateConfig() ExprGeneratorConfig {
+func (egc ExprGeneratorConfig) CannotAggregateConfig() ExprGeneratorConfig {
 	egc.CanAggregate = false
 	return egc
 }
@@ -128,7 +128,7 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 	}
 
 	if genConfig.CanAggregate {
-		genConfig = genConfig.cannotAggregateConfig()
+		genConfig = genConfig.CannotAggregateConfig()
 		options = append(options, func() Expr { return RandomAggregate(g.r, g.Expression(genConfig)) })
 	}
 
@@ -138,18 +138,20 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 var aggregates = []string{"count(*)", "count", "sum" /*, "min", "max" */}
 
 func RandomAggregate(r *rand.Rand, expr Expr) Expr {
+	isDisinct := r.Intn(2) < 1
+
 	aggr := aggregates[r.Intn(len(aggregates))]
 	switch aggr {
 	case "count(*)":
 		return &CountStar{}
 	case "count":
-		return &Count{Args: Exprs{expr}}
+		return &Count{Args: Exprs{expr}, Distinct: isDisinct}
 	case "sum":
-		return &Sum{Arg: expr}
+		return &Sum{Arg: expr, Distinct: isDisinct}
 	case "min":
-		return &Min{Arg: expr}
+		return &Min{Arg: expr, Distinct: isDisinct}
 	case "max":
-		return &Max{Arg: expr}
+		return &Max{Arg: expr, Distinct: isDisinct}
 	}
 
 	return expr

From f32f5dd8a469f46ab672f00f3d7435d5ff138e7c Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Wed, 12 Jul 2023 22:20:08 -0700
Subject: [PATCH 08/27] added random column aliases

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 49 ++++++++++++-----
 .../vtgate/queries/random/query_gen_test.go   |  2 +-
 .../vtgate/queries/random/random_test.go      | 53 +++++++++++++------
 go/vt/sqlparser/random_expr.go                |  2 +-
 4 files changed, 75 insertions(+), 31 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 19b4df3d093..f7e50ff134f 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -37,6 +37,7 @@ type (
 	queryGen struct {
 		r         *rand.Rand
 		genConfig sqlparser.ExprGeneratorConfig
+		maxCols   int
 	}
 	column struct {
 		name string
@@ -195,13 +196,6 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 	// where
 	sel.AddWhere(sqlparser.AndExpressions(qg.createWherePredicates(tables)...))
 
-	// only add a limit if the grouping columns are ordered
-	// TODO: limit fails a lot
-	isLimit := qg.r.Intn(2) < 1 && (isOrdered || len(groupBy) == 0) && testFailingQueries
-	if isLimit {
-		sel.Limit = qg.createLimit()
-	}
-
 	var (
 		newTable tableT
 		f        func() sqlparser.Expr
@@ -249,7 +243,7 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 		// make sure to add the random expression to group by and order by for only_full_group_by
 		if canAggregate {
 			sel.AddGroupBy(randomExpr)
-			if isOrdered || isLimit {
+			if isOrdered {
 				sel.AddOrder(sqlparser.NewOrder(randomExpr, qg.getRandomOrderDirection()))
 			}
 		}
@@ -257,10 +251,20 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 		break
 	}
 
-	// alias the grouping columns
-	for i, col := range grouping {
+	// only add a limit if there is an ordering
+	// TODO: limit fails a lot
+	isLimit := qg.r.Intn(2) < 1 && len(sel.OrderBy) > 0 && testFailingQueries
+	if isLimit {
+		sel.Limit = qg.createLimit()
+	}
+
+	// alias the grouping columns randomly
+	for i := range grouping {
+		if qg.r.Intn(2) < 1 {
+			continue
+		}
 		alias := fmt.Sprintf("cgroup%d", i)
-		col.name = alias
+		grouping[i].name = alias
 	}
 
 	// add them to newTable
@@ -301,7 +305,7 @@ func (qg queryGen) createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Sel
 	if isJoin {
 		// TODO: do nested joins
 		newTable := randomEl(qg.r, schemaTables)
-		alias := fmt.Sprintf("tbl%d", numTables)
+		alias := fmt.Sprintf("tbl%d", numTables+1)
 		newTable.setAlias(alias)
 		tables = append(tables, newTable)
 
@@ -368,9 +372,26 @@ func (qg queryGen) createAggregations(tables []tableT) (aggrSelectExprs sqlparse
 		})...)
 	for i := range aggrExprs {
 		expr := sqlparser.RandomAggregate(qg.r, aggrExprs[i])
-		alias := fmt.Sprintf("caggr%d", i)
+		// TODO: min/max often fails
+		for testFailingQueries {
+			switch expr.(type) {
+			case *sqlparser.Min, *sqlparser.Max:
+				expr = sqlparser.RandomAggregate(qg.r, aggrExprs[i])
+				continue
+			}
+
+			break
+		}
+
+		// randomly alias
+		var alias string
+		if qg.r.Intn(2) < 1 {
+			alias = fmt.Sprintf("caggr%d", i)
+			aggregates = append(aggregates, column{name: alias})
+		} else {
+			aggregates = append(aggregates, column{name: sqlparser.String(expr)})
+		}
 		aggrSelectExprs = append(aggrSelectExprs, sqlparser.NewAliasedExpr(expr, alias))
-		aggregates = append(aggregates, column{name: alias})
 	}
 
 	return
diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index 900eb83983e..1eab1217456 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -49,7 +49,7 @@ func TestSeed(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	seed := int64(1)
+	seed := int64(1689140137270728000)
 	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
 	query1 := sqlparser.String(qg.randomQuery(schemaTables))
 	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 4603893150a..eb41d6b8f85 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -84,6 +84,9 @@ func TestMustFix(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
+	// mismatched results (mismatched types)
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32")
+
 	// panics instead of normally logging the error
 	// Received unexpected error:
 	// VT12001: unsupported: only one DISTINCT aggregation is allowed in a SELECT: sum(distinct 1) as caggr1
@@ -134,6 +137,9 @@ func TestMustFix(t *testing.T) {
 	// swapping tables and predicates and changing to left fails
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal")
 
+	// EOF
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.deptno as cgroup0, tbl1.loc as cgroup1, count(distinct tbl1.loc) as caggr1, tbl1.loc as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.deptno, tbl1.loc")
+
 	// EOF
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) from dept as tbl0, (select count(*) from emp as tbl0, emp as tbl1 limit 18) as tbl1")
 }
@@ -147,15 +153,19 @@ func TestKnownFailures(t *testing.T) {
 	// logs more stuff
 	//clusterInstance.EnableGeneralLog()
 
-	// vitess error: unsupported: min/max on types that are not comparable is not supported
-	// mysql error: <nil>
+	// VT13001: [BUG] in scatter query: complex ORDER BY expression: :vtg1 /* VARCHAR */
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.job as cgroup0, sum(distinct 'mudfish'), tbl1.job as crandom0 from emp as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job desc, tbl1.job asc limit 8, 1")
+
+	// VT13001: [BUG] column should not be pushed to projection while doing a column lookup
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ -26 in (tbl2.mgr, -8, tbl0.deptno) as crandom0 from dept as tbl0, emp as tbl1 left join emp as tbl2 on tbl2.ename")
+
+	// unsupported: min/max on types that are not comparable is not supported
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ max(case true when false then 'gnu' when true then 'meerkat' end) as caggr0 from dept as tbl0")
 
-	// vitess error: vttablet: rpc error: code = InvalidArgument desc = BIGINT UNSIGNED value is out of range in '(-(273) + (-(15) & 124))'
-	// mysql error: <nil>
+	// vttablet: rpc error: code = InvalidArgument desc = BIGINT UNSIGNED value is out of range in '(-(273) + (-(15) & 124))'
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ -273 + (-15 & 124) as crandom0 from emp as tbl0, emp as tbl1 where tbl1.sal >= tbl1.mgr")
 
-	// cannot compare strings, collation is unknown or unsupported (collation ID: 0)
+	// vitess error: cannot compare strings, collation is unknown or unsupported (collation ID: 0)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ max(tbl1.dname) as caggr1 from dept as tbl0, dept as tbl1 group by tbl1.dname order by tbl1.dname asc")
 
 	// vitess error: <nil>
@@ -169,13 +179,17 @@ func TestKnownFailures(t *testing.T) {
 	// coercion should not try to coerce this value: DATE("1980-12-17")
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct tbl1.hiredate as cgroup0, count(tbl1.mgr) as caggr0 from emp as tbl1 group by tbl1.hiredate, tbl1.ename")
 
-	// only_full_group_by enabled (vitess sometimes (?) produces the correct result assuming only_full_group_by is disabled)
-	// vitess error: nil
+	// only_full_group_by enabled
+	// vitess error: In aggregated query without GROUP BY, expression #1 of SELECT list contains nonaggregated column 'ks_random.tbl0.EMPNO'; this is incompatible with sql_mode=only_full_group_by
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct tbl0.empno as cgroup0, count(distinct 56) as caggr0, min('flounder' = 'penguin') as caggr1 from emp as tbl0, (select /*vt+ PLANNER=Gen4 */ 'manatee' as crandom0 from dept as tbl0 where -26 limit 2) as tbl2 where 'anteater' like 'catfish' is null and -11 group by tbl0.empno order by tbl0.empno asc, count(distinct 56) asc, min('flounder' = 'penguin') desc")
+
+	// only_full_group_by enabled
+	// vitess error: <nil>
 	// mysql error: In aggregated query without GROUP BY, expression #1 of SELECT list contains nonaggregated column 'ks_random.tbl0.ENAME'; this is incompatible with sql_mode=only_full_group_by
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl0.ename, min(tbl0.comm) from emp as tbl0 left join emp as tbl1 on tbl0.empno = tbl1.comm and tbl0.empno = tbl1.empno")
 
 	// only_full_group_by enabled
-	// vitess error:  nil
+	// vitess error: <nil>
 	// mysql error: Expression #1 of ORDER BY clause is not in SELECT list, references column 'ks_random.tbl2.DNAME' which is not in SELECT list; this is incompatible with DISTINCT
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct count(*) as caggr0 from dept as tbl2 group by tbl2.dname order by tbl2.dname asc")
 
@@ -188,28 +202,37 @@ func TestKnownFailures(t *testing.T) {
 	// vttablet: rpc error: code = InvalidArgument desc = Can't group on 'count(*)' (errno 1056) (sqlstate 42000) (CallerID: userData1)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct count(*) from dept as tbl0 group by tbl0.deptno")
 
-	// [BUG] push projection does not yet support: *planbuilder.memorySort (errno 1815) (sqlstate HY000)
+	// unsupported
+	// VT12001: unsupported: only one DISTINCT aggregation is allowed in a SELECT: sum(distinct 1) as caggr1
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ sum(distinct tbl0.comm) as caggr0, sum(distinct 1) as caggr1 from emp as tbl0 having 'redfish' < 'blowfish'")
+
+	// unsupported
+	// VT12001: unsupported: aggregation on top of aggregation not supported
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) from dept as tbl1 join (select count(*) from emp as tbl0, dept as tbl1 group by tbl1.loc) as tbl2")
 
 	// unsupported
-	// unsupported: in scatter query: complex aggregate expression (errno 1235) (sqlstate 42000)
+	// VT12001: unsupported: in scatter query: complex aggregate expression
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ (select count(*) from emp as tbl0) from emp as tbl0")
 
 	// unsupported
-	// unsupported: using aggregation on top of a *planbuilder.filter plan
+	// VT12001: unsupported: using aggregation on top of a *planbuilder.filter plan
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(tbl1.dname) as caggr1 from dept as tbl0 left join dept as tbl1 on tbl1.dname > tbl1.loc where tbl1.loc <=> tbl1.dname group by tbl1.dname order by tbl1.dname asc")
 
 	// unsupported
-	// unsupported: using aggregation on top of a *planbuilder.orderedAggregate plan
+	// VT12001: unsupported: aggregation on top of aggregation not supported
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) from (select count(*) from dept as tbl0) as tbl0")
 
 	// unsupported
-	// unsupported: using aggregation on top of a *planbuilder.orderedAggregate plan
+	// VT12001: unsupported: aggregation on top of aggregation not supported
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*), count(*) from (select count(*) from dept as tbl0) as tbl0, dept as tbl1")
 
 	// unsupported
-	// unsupported: in scatter query: aggregation function 'avg(tbl0.deptno)'
+	// VT12001: unsupported: in scatter query: aggregation function 'avg(tbl0.deptno)'
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ avg(tbl0.deptno) from dept as tbl0")
+
+	// unsupported
+	// VT12001: unsupported: LEFT JOIN with derived tables
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ -1 as crandom0 from emp as tbl2 left join (select count(*) from dept as tbl1) as tbl3 on 6 != tbl2.deptno")
 }
 
 func TestRandom(t *testing.T) {
@@ -244,7 +267,7 @@ func TestRandom(t *testing.T) {
 
 	var queryCount int
 	// continue testing after an error if and only if testFailingQueries is true
-	for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) {
+	for time.Now().Before(endBy) && (!t.Failed() || testFailingQueries) {
 		seed := time.Now().UnixNano()
 		fmt.Printf("seed: %d\n", seed)
 		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index d973dbefcdb..3bba1373dc8 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -135,7 +135,7 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 	return g.randomOf(options)
 }
 
-var aggregates = []string{"count(*)", "count", "sum" /*, "min", "max" */}
+var aggregates = []string{"count(*)", "count", "sum", "min", "max"}
 
 func RandomAggregate(r *rand.Rand, expr Expr) Expr {
 	isDisinct := r.Intn(2) < 1

From e92b3c2d9a9af60fb0662eedaaf7ded4566a8b85 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Thu, 13 Jul 2023 01:41:17 -0700
Subject: [PATCH 09/27] added more complex aggregate expressions

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 32 ++++-----
 .../vtgate/queries/random/random_expr_test.go |  2 +-
 go/vt/sqlparser/random_expr.go                | 69 +++++++++++--------
 3 files changed, 57 insertions(+), 46 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index f7e50ff134f..5133c0f9eb5 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -244,7 +244,7 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 		if canAggregate {
 			sel.AddGroupBy(randomExpr)
 			if isOrdered {
-				sel.AddOrder(sqlparser.NewOrder(randomExpr, qg.getRandomOrderDirection()))
+				sel.AddOrder(sqlparser.NewOrder(randomExpr, getRandomOrderDirection(qg.r)))
 			}
 		}
 
@@ -326,7 +326,7 @@ func (qg queryGen) createJoin(tables []tableT, sel *sqlparser.Select) {
 	joinPredicate := sqlparser.AndExpressions(qg.createJoinPredicates(tables)...)
 	joinCondition := sqlparser.NewJoinCondition(joinPredicate, nil)
 	newTable := newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n))
-	sel.From[n-1] = sqlparser.NewJoinTableExpr(sel.From[n-1], sqlparser.LeftJoinType, newTable, joinCondition)
+	sel.From[n-1] = sqlparser.NewJoinTableExpr(sel.From[n-1], getRandomJoinType(qg.r), newTable, joinCondition)
 }
 
 // returns 1-3 random expressions based on the last two elements of tables
@@ -366,23 +366,14 @@ func (qg queryGen) createGroupBy(tables []tableT) (groupBy sqlparser.GroupBy, gr
 
 // returns the aggregation columns as three types: sqlparser.SelectExprs, []column
 func (qg queryGen) createAggregations(tables []tableT) (aggrSelectExprs sqlparser.SelectExprs, aggregates []column) {
+	qg.genConfig = qg.genConfig.IsAggregateConfig()
 	aggrExprs := qg.createRandomExprs(0,
 		slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
 			return &t
 		})...)
-	for i := range aggrExprs {
-		expr := sqlparser.RandomAggregate(qg.r, aggrExprs[i])
-		// TODO: min/max often fails
-		for testFailingQueries {
-			switch expr.(type) {
-			case *sqlparser.Min, *sqlparser.Max:
-				expr = sqlparser.RandomAggregate(qg.r, aggrExprs[i])
-				continue
-			}
-
-			break
-		}
 
+	for i, expr := range aggrExprs {
+		// TODO: min/max often fails
 		// randomly alias
 		var alias string
 		if qg.r.Intn(2) < 1 {
@@ -401,13 +392,13 @@ func (qg queryGen) createAggregations(tables []tableT) (aggrSelectExprs sqlparse
 func (qg queryGen) createOrderBy(groupBy sqlparser.GroupBy, aggrExprs sqlparser.SelectExprs) (orderBy sqlparser.OrderBy) {
 	// always order on grouping columns
 	for i := range groupBy {
-		orderBy = append(orderBy, sqlparser.NewOrder(groupBy[i], qg.getRandomOrderDirection()))
+		orderBy = append(orderBy, sqlparser.NewOrder(groupBy[i], getRandomOrderDirection(qg.r)))
 	}
 
 	// randomly order on aggregation columns
 	for i := range aggrExprs {
 		if aliasedExpr, ok := aggrExprs[i].(*sqlparser.AliasedExpr); ok && qg.r.Intn(2) < 1 {
-			orderBy = append(orderBy, sqlparser.NewOrder(aliasedExpr.Expr, qg.getRandomOrderDirection()))
+			orderBy = append(orderBy, sqlparser.NewOrder(aliasedExpr.Expr, getRandomOrderDirection(qg.r)))
 		}
 	}
 
@@ -465,9 +456,14 @@ func (qg queryGen) createLimit() *sqlparser.Limit {
 	return sqlparser.NewLimitWithoutOffset(limitNum)
 }
 
-func (qg queryGen) getRandomOrderDirection() sqlparser.OrderDirection {
+func getRandomOrderDirection(r *rand.Rand) sqlparser.OrderDirection {
 	// asc, desc
-	return randomEl(qg.r, []sqlparser.OrderDirection{0, 1})
+	return randomEl(r, []sqlparser.OrderDirection{0, 1})
+}
+
+func getRandomJoinType(r *rand.Rand) sqlparser.JoinType {
+	// normal, straight, left, right, natural, natural left, natural right
+	return randomEl(r, []sqlparser.JoinType{0, 1, 2, 3, 4, 5, 6})
 }
 
 func randomEl[K any](r *rand.Rand, in []K) K {
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index e5e3a3bcf86..fe1be048b35 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -53,7 +53,7 @@ func TestRandomExprWithTables(t *testing.T) {
 	r := rand.New(rand.NewSource(seed))
 	g := sqlparser.NewGenerator(r, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 	for i := 0; i < 100; i++ {
-		expr := g.Expression(sqlparser.ExprGeneratorConfig{CanAggregate: true})
+		expr := g.Expression(sqlparser.ExprGeneratorConfig{AggrRule: sqlparser.IsAggregate})
 		fmt.Println(sqlparser.String(expr))
 	}
 }
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 3bba1373dc8..a2d2a06dad4 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -23,15 +23,24 @@ import (
 
 // This file is used to generate random expressions to be used for testing
 
+// Constants for Enum Type - AggregateRule
+const (
+	CannotAggregate AggregateRule = iota
+	CanAggregate
+	IsAggregate
+)
+
 type (
 	ExprGenerator interface {
 		Generate(r *rand.Rand, config ExprGeneratorConfig) Expr
 	}
 
+	AggregateRule int8
+
 	ExprGeneratorConfig struct {
-		// IsAggregate determines if the random expression is an aggregation expression
-		CanAggregate bool
-		Type         string
+		// AggrRule determines if the random expression can, cannot, or must be an aggregatable expression
+		AggrRule AggregateRule
+		Type     string
 	}
 )
 
@@ -55,13 +64,18 @@ func (egc ExprGeneratorConfig) anyTypeConfig() ExprGeneratorConfig {
 	return egc
 }
 
+func (egc ExprGeneratorConfig) CannotAggregateConfig() ExprGeneratorConfig {
+	egc.AggrRule = CannotAggregate
+	return egc
+}
+
 func (egc ExprGeneratorConfig) CanAggregateConfig() ExprGeneratorConfig {
-	egc.CanAggregate = true
+	egc.AggrRule = CanAggregate
 	return egc
 }
 
-func (egc ExprGeneratorConfig) CannotAggregateConfig() ExprGeneratorConfig {
-	egc.CanAggregate = false
+func (egc ExprGeneratorConfig) IsAggregateConfig() ExprGeneratorConfig {
+	egc.AggrRule = IsAggregate
 	return egc
 }
 
@@ -127,34 +141,25 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.booleanExpr(genConfig) },
 	}
 
-	if genConfig.CanAggregate {
-		genConfig = genConfig.CannotAggregateConfig()
-		options = append(options, func() Expr { return RandomAggregate(g.r, g.Expression(genConfig)) })
+	if genConfig.AggrRule != CannotAggregate {
+		options = append(options, func() Expr { return g.randomAggregate(genConfig.CannotAggregateConfig()) })
 	}
 
 	return g.randomOf(options)
 }
 
-var aggregates = []string{"count(*)", "count", "sum", "min", "max"}
+func (g *Generator) randomAggregate(genConfig ExprGeneratorConfig) Expr {
+	isDistinct := g.r.Intn(2) < 1
 
-func RandomAggregate(r *rand.Rand, expr Expr) Expr {
-	isDisinct := r.Intn(2) < 1
-
-	aggr := aggregates[r.Intn(len(aggregates))]
-	switch aggr {
-	case "count(*)":
-		return &CountStar{}
-	case "count":
-		return &Count{Args: Exprs{expr}, Distinct: isDisinct}
-	case "sum":
-		return &Sum{Arg: expr, Distinct: isDisinct}
-	case "min":
-		return &Min{Arg: expr, Distinct: isDisinct}
-	case "max":
-		return &Max{Arg: expr, Distinct: isDisinct}
+	options := []exprF{
+		func() Expr { return &CountStar{} },
+		func() Expr { return &Count{Args: Exprs{g.Expression(genConfig.anyTypeConfig())}, Distinct: isDistinct} },
+		func() Expr { return &Sum{Arg: g.Expression(genConfig), Distinct: isDistinct} },
+		func() Expr { return &Min{Arg: g.Expression(genConfig), Distinct: isDistinct} },
+		func() Expr { return &Max{Arg: g.Expression(genConfig), Distinct: isDistinct} },
 	}
 
-	return expr
+	return g.randomOf(options)
 }
 
 func (g *Generator) booleanExpr(genConfig ExprGeneratorConfig) Expr {
@@ -194,6 +199,11 @@ func (g *Generator) intExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.caseExpr(genConfig) },
 	}
 
+	// if the expression must be aggregatable, then don't generate from the ExprGenerators at this depth
+	if genConfig.AggrRule == IsAggregate {
+		return g.randomOf(options)
+	}
+
 	for i := range g.exprGenerators {
 		generator := g.exprGenerators[i]
 		if generator == nil {
@@ -224,6 +234,11 @@ func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.caseExpr(genConfig) },
 	}
 
+	// if the expression must be aggregatable, then don't generate from the ExprGenerators at this depth
+	if genConfig.AggrRule == IsAggregate {
+		return g.randomOf(options)
+	}
+
 	for i := range g.exprGenerators {
 		generator := g.exprGenerators[i]
 		if generator == nil {
@@ -306,7 +321,7 @@ func (g *Generator) caseExpr(genConfig ExprGeneratorConfig) Expr {
 		if exp == nil {
 			cond = g.Expression(genConfig.boolTypeConfig())
 		} else {
-			cond = g.Expression(genConfig.anyTypeConfig())
+			cond = g.Expression(genConfig)
 		}
 
 		val := g.Expression(genConfig)

From 1953053a626666b00938c27fd51e5dd6f871731c Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Sun, 16 Jul 2023 04:01:25 -0700
Subject: [PATCH 10/27] added scalar subqueries to fuzzer

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 229 +++++++++++-------
 .../vtgate/queries/random/query_gen_test.go   |  12 +-
 .../vtgate/queries/random/random_test.go      |   7 +-
 go/vt/sqlparser/random_expr.go                |  22 +-
 4 files changed, 174 insertions(+), 96 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 5133c0f9eb5..77080a26429 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -37,7 +37,15 @@ type (
 	queryGen struct {
 		r         *rand.Rand
 		genConfig sqlparser.ExprGeneratorConfig
-		maxCols   int
+		maxTables int
+		maxAggrs  int
+		maxGBs    int
+		// maxCols = 0 indicates no limit
+		maxCols int
+		// maxRows = 0 indicates no limit
+		maxRows      int
+		schemaTables []tableT
+		sel          *sqlparser.Select
 	}
 	column struct {
 		name string
@@ -58,11 +66,22 @@ type (
 
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
 var _ sqlparser.ExprGenerator = (*column)(nil)
+var _ sqlparser.ExprGenerator = (*queryGen)(nil)
 
-func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) queryGen {
-	return queryGen{
-		r:         r,
-		genConfig: genConfig,
+func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, maxTables, maxAggrs, maxGBs int, schemaTables []tableT) *queryGen {
+	if maxTables <= 0 {
+		log.Fatalf("maxTables must be at least 1, currently %d\n", maxTables)
+	}
+
+	return &queryGen{
+		r:            r,
+		genConfig:    genConfig,
+		maxTables:    maxTables,
+		maxAggrs:     maxAggrs,
+		maxGBs:       maxGBs,
+		maxCols:      1,
+		maxRows:      1,
+		schemaTables: schemaTables,
 	}
 }
 
@@ -112,6 +131,14 @@ func (t *tableT) addColumns(col ...column) {
 	}
 }
 
+func (t *tableT) clone() *tableT {
+	return &tableT{
+		tableExpr: sqlparser.CloneSimpleTableExpr(t.tableExpr),
+		alias:     t.alias,
+		cols:      slices.Clone(t.cols),
+	}
+}
+
 func (c *column) Generate(_ *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	if c.typ == genConfig.Type {
 		return c.getASTExpr()
@@ -138,51 +165,60 @@ func (t *tableT) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig)
 	return nil
 }
 
-func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
+// Generate generates a scalar subquery based on qg
+func (qg *queryGen) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
+	var newSchemaTables []tableT
+	for _, tbl := range qg.schemaTables {
+		newSchemaTables = append(newSchemaTables, *tbl.clone())
+	}
+
+	newQG := newQueryGenerator(r, genConfig, qg.maxTables, qg.maxAggrs, qg.maxGBs, newSchemaTables)
+	newQG.maxRows = 1
+	newQG.maxCols = 1
+	newQG.randomQuery()
+
+	return &sqlparser.Subquery{Select: newQG.sel}
+}
+
+func (qg *queryGen) randomQuery() {
 	// make sure the random expressions can generally not contain aggregates; change appropriately
 	qg.genConfig = qg.genConfig.CannotAggregateConfig()
 
-	sel := &sqlparser.Select{}
-	sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"})
+	qg.sel = &sqlparser.Select{}
+	qg.sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"})
 
 	// select distinct (fails with group by bigint)
 	isDistinct := qg.r.Intn(2) < 1
 	if isDistinct {
-		sel.MakeDistinct()
+		qg.sel.MakeDistinct()
 	}
 
 	// create both tables and join at the same time since both occupy the from clause
-	tables, isJoin := qg.createTablesAndJoin(schemaTables, sel)
+	tables, isJoin := qg.createTablesAndJoin()
 
 	// canAggregate determines if the query will have
 	// aggregate columns, group by, and having
 	canAggregate := qg.r.Intn(4) > 0
 
 	var (
-		groupBy          sqlparser.GroupBy
-		groupSelectExprs sqlparser.SelectExprs
-		grouping         []column
-		aggrSelectExprs  sqlparser.SelectExprs
-		aggregates       []column
+		grouping, aggregates []column
 	)
 	// TODO: distinct makes vitess think there is grouping on aggregation columns
 	if canAggregate {
 		if testFailingQueries || !isDistinct {
 			// group by
-			groupBy, groupSelectExprs, grouping = qg.createGroupBy(tables)
-			sel.AddSelectExprs(groupSelectExprs)
-			sel.GroupBy = groupBy
+			if qg.maxRows == 0 {
+				grouping = qg.createGroupBy(tables)
+			}
 		}
 
 		// aggregate columns
-		aggrSelectExprs, aggregates = qg.createAggregations(tables)
-		sel.AddSelectExprs(aggrSelectExprs)
-		// sel.GroupBy = append(sel.GroupBy, aggrExprs...)
+		aggregates = qg.createAggregations(tables)
 
 		// having
 		isHaving := qg.r.Intn(2) < 1
 		if isHaving {
-			sel.AddHaving(sqlparser.AndExpressions(qg.createHavingPredicates(grouping)...))
+			qg.createHavingPredicates(grouping)
 		}
 	}
 
@@ -190,11 +226,11 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 	// TODO: order fails with distinct and outer joins
 	isOrdered := qg.r.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries)
 	if isOrdered {
-		sel.OrderBy = qg.createOrderBy(groupBy, aggrSelectExprs)
+		qg.createOrderBy()
 	}
 
 	// where
-	sel.AddWhere(sqlparser.AndExpressions(qg.createWherePredicates(tables)...))
+	qg.createWherePredicates(tables)
 
 	var (
 		newTable tableT
@@ -217,7 +253,7 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 	}
 
 	// make sure we have at least one select expression
-	for isRandomExpr || len(sel.SelectExprs) == 0 {
+	for isRandomExpr || len(qg.sel.SelectExprs) == 0 {
 		// TODO: if the random expression is an int literal,
 		// TODO: and if the query is (potentially) an aggregate query,
 		// TODO: then we must group by the random expression,
@@ -232,9 +268,9 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 		}
 
 		// TODO: select distinct [literal] fails
-		sel.Distinct = false
+		qg.sel.Distinct = false
 
-		sel.SelectExprs = append(sel.SelectExprs, sqlparser.NewAliasedExpr(randomExpr, "crandom0"))
+		qg.sel.SelectExprs = append(qg.sel.SelectExprs, sqlparser.NewAliasedExpr(randomExpr, "crandom0"))
 		newTable.addColumns(column{
 			name: "crandom0",
 			typ:  typ,
@@ -242,20 +278,30 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 
 		// make sure to add the random expression to group by and order by for only_full_group_by
 		if canAggregate {
-			sel.AddGroupBy(randomExpr)
+			qg.sel.AddGroupBy(randomExpr)
 			if isOrdered {
-				sel.AddOrder(sqlparser.NewOrder(randomExpr, getRandomOrderDirection(qg.r)))
+				qg.sel.AddOrder(sqlparser.NewOrder(randomExpr, getRandomOrderDirection(qg.r)))
 			}
 		}
 
 		break
 	}
 
+	if qg.maxCols > 0 {
+		// remove SelectExprs randomly until there are qg.maxCols amount
+		for len(qg.sel.SelectExprs) > qg.maxCols {
+			// select a random index and remove it from SelectExprs
+			idx := qg.r.Intn(len(qg.sel.SelectExprs))
+			qg.sel.SelectExprs[idx] = qg.sel.SelectExprs[len(qg.sel.SelectExprs)-1]
+			qg.sel.SelectExprs = qg.sel.SelectExprs[:len(qg.sel.SelectExprs)-1]
+		}
+	}
+
 	// only add a limit if there is an ordering
 	// TODO: limit fails a lot
-	isLimit := qg.r.Intn(2) < 1 && len(sel.OrderBy) > 0 && testFailingQueries
+	isLimit := qg.r.Intn(2) < 1 && len(qg.sel.OrderBy) > 0 && testFailingQueries
 	if isLimit {
-		sel.Limit = qg.createLimit()
+		qg.createLimit()
 	}
 
 	// alias the grouping columns randomly
@@ -272,31 +318,29 @@ func (qg queryGen) randomQuery(schemaTables []tableT) *sqlparser.Select {
 	newTable.addColumns(aggregates...)
 
 	// add new table to schemaTables
-	newTable.tableExpr = sqlparser.NewDerivedTable(false, sel)
-	schemaTables = append(schemaTables, newTable)
+	newTable.tableExpr = sqlparser.NewDerivedTable(false, qg.sel)
+	qg.schemaTables = append(qg.schemaTables, newTable)
 
 	// derived tables (partially unsupported)
 	// TODO: derived tables fails a lot
 	if qg.r.Intn(10) < 1 && testFailingQueries {
-		sel = qg.randomQuery(schemaTables)
+		qg.randomQuery()
 	}
-
-	return sel
 }
 
-func (qg queryGen) createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Select) ([]tableT, bool) {
+func (qg *queryGen) createTablesAndJoin() ([]tableT, bool) {
 	var tables []tableT
 	// add at least one of original emp/dept tables for now because derived tables have nil columns
-	tables = append(tables, schemaTables[qg.r.Intn(2)])
+	tables = append(tables, qg.schemaTables[qg.r.Intn(2)])
 
 	tables[0].setAlias("tbl0")
-	sel.From = append(sel.From, newAliasedTable(tables[0], "tbl0"))
+	qg.sel.From = append(qg.sel.From, newAliasedTable(tables[0], "tbl0"))
 
-	numTables := qg.r.Intn(len(schemaTables))
+	numTables := qg.r.Intn(qg.maxTables)
 	for i := 0; i < numTables; i++ {
-		tables = append(tables, randomEl(qg.r, schemaTables))
+		tables = append(tables, randomEl(qg.r, qg.schemaTables))
 		alias := fmt.Sprintf("tbl%d", i+1)
-		sel.From = append(sel.From, newAliasedTable(tables[i+1], alias))
+		qg.sel.From = append(qg.sel.From, newAliasedTable(tables[i+1], alias))
 		tables[i+1].setAlias(alias)
 	}
 
@@ -304,12 +348,12 @@ func (qg queryGen) createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Sel
 	isJoin := qg.r.Intn(2) < 1 && testFailingQueries
 	if isJoin {
 		// TODO: do nested joins
-		newTable := randomEl(qg.r, schemaTables)
+		newTable := randomEl(qg.r, qg.schemaTables)
 		alias := fmt.Sprintf("tbl%d", numTables+1)
 		newTable.setAlias(alias)
 		tables = append(tables, newTable)
 
-		qg.createJoin(tables, sel)
+		qg.createJoin(tables)
 	}
 
 	return tables, isJoin
@@ -317,31 +361,34 @@ func (qg queryGen) createTablesAndJoin(schemaTables []tableT, sel *sqlparser.Sel
 
 // creates a left join (without the condition) between the last table in sel and newTable
 // tables should have one more table than sel
-func (qg queryGen) createJoin(tables []tableT, sel *sqlparser.Select) {
-	n := len(sel.From)
+func (qg *queryGen) createJoin(tables []tableT) {
+	n := len(qg.sel.From)
 	if len(tables) != n+1 {
-		log.Fatalf("sel has %d tables and tables has %d tables", len(sel.From), n)
+		log.Fatalf("sel has %d tables and tables has %d tables", len(qg.sel.From), n)
 	}
 
 	joinPredicate := sqlparser.AndExpressions(qg.createJoinPredicates(tables)...)
 	joinCondition := sqlparser.NewJoinCondition(joinPredicate, nil)
 	newTable := newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n))
-	sel.From[n-1] = sqlparser.NewJoinTableExpr(sel.From[n-1], getRandomJoinType(qg.r), newTable, joinCondition)
+	qg.sel.From[n-1] = sqlparser.NewJoinTableExpr(qg.sel.From[n-1], getRandomJoinType(qg.r), newTable, joinCondition)
 }
 
 // returns 1-3 random expressions based on the last two elements of tables
 // tables should have at least two elements
-func (qg queryGen) createJoinPredicates(tables []tableT) sqlparser.Exprs {
+func (qg *queryGen) createJoinPredicates(tables []tableT) sqlparser.Exprs {
 	if len(tables) < 2 {
 		log.Fatalf("tables has %d elements, needs at least 2", len(tables))
 	}
 
-	return qg.createRandomExprs(1, &tables[len(tables)-2], &tables[len(tables)-1])
+	return qg.createRandomExprs(1, 3, &tables[len(tables)-2], &tables[len(tables)-1])
 }
 
 // returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
-func (qg queryGen) createGroupBy(tables []tableT) (groupBy sqlparser.GroupBy, groupSelectExprs sqlparser.SelectExprs, grouping []column) {
-	numGBs := qg.r.Intn(3)
+func (qg *queryGen) createGroupBy(tables []tableT) (grouping []column) {
+	if qg.maxGBs <= 0 {
+		return
+	}
+	numGBs := qg.r.Intn(qg.maxGBs + 1)
 	for i := 0; i < numGBs; i++ {
 		tblIdx := qg.r.Intn(len(tables))
 		col := randomEl(qg.r, tables[tblIdx].cols)
@@ -349,12 +396,12 @@ func (qg queryGen) createGroupBy(tables []tableT) (groupBy sqlparser.GroupBy, gr
 		if col.typ == "date" && !testFailingQueries {
 			continue
 		}
-		groupBy = append(groupBy, col.getASTExpr())
+		qg.sel.GroupBy = append(qg.sel.GroupBy, col.getASTExpr())
 
 		// add to select
 		if qg.r.Intn(2) < 1 {
 			alias := fmt.Sprintf("cgroup%d", len(grouping))
-			groupSelectExprs = append(groupSelectExprs, newAliasedColumn(col, alias))
+			qg.sel.SelectExprs = append(qg.sel.SelectExprs, newAliasedColumn(col, alias))
 			// TODO: alias in a separate function to properly generate the having clause
 			//col.name = sqlparser.NewColName(alias)
 			grouping = append(grouping, col)
@@ -365,12 +412,13 @@ func (qg queryGen) createGroupBy(tables []tableT) (groupBy sqlparser.GroupBy, gr
 }
 
 // returns the aggregation columns as three types: sqlparser.SelectExprs, []column
-func (qg queryGen) createAggregations(tables []tableT) (aggrSelectExprs sqlparser.SelectExprs, aggregates []column) {
+func (qg *queryGen) createAggregations(tables []tableT) (aggregates []column) {
 	qg.genConfig = qg.genConfig.IsAggregateConfig()
-	aggrExprs := qg.createRandomExprs(0,
+	aggrExprs := qg.createRandomExprs(0, qg.maxAggrs,
 		slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
 			return &t
 		})...)
+	qg.genConfig = qg.genConfig.CannotAggregateConfig()
 
 	for i, expr := range aggrExprs {
 		// TODO: min/max often fails
@@ -382,56 +430,69 @@ func (qg queryGen) createAggregations(tables []tableT) (aggrSelectExprs sqlparse
 		} else {
 			aggregates = append(aggregates, column{name: sqlparser.String(expr)})
 		}
-		aggrSelectExprs = append(aggrSelectExprs, sqlparser.NewAliasedExpr(expr, alias))
+		qg.sel.SelectExprs = append(qg.sel.SelectExprs, sqlparser.NewAliasedExpr(expr, alias))
 	}
 
 	return
 }
 
-// orders on all non-aggregate SelectExprs and independently at random on all aggregate SelectExprs of sel
-func (qg queryGen) createOrderBy(groupBy sqlparser.GroupBy, aggrExprs sqlparser.SelectExprs) (orderBy sqlparser.OrderBy) {
-	// always order on grouping columns
-	for i := range groupBy {
-		orderBy = append(orderBy, sqlparser.NewOrder(groupBy[i], getRandomOrderDirection(qg.r)))
+// orders on all grouping expressions and on random SelectExprs
+func (qg *queryGen) createOrderBy() {
+	// always order on grouping expressions
+	for _, expr := range qg.sel.GroupBy {
+		qg.sel.OrderBy = append(qg.sel.OrderBy, sqlparser.NewOrder(expr, getRandomOrderDirection(qg.r)))
 	}
 
-	// randomly order on aggregation columns
-	for i := range aggrExprs {
-		if aliasedExpr, ok := aggrExprs[i].(*sqlparser.AliasedExpr); ok && qg.r.Intn(2) < 1 {
-			orderBy = append(orderBy, sqlparser.NewOrder(aliasedExpr.Expr, getRandomOrderDirection(qg.r)))
+	// randomly order on SelectExprs
+	for _, selExpr := range qg.sel.SelectExprs {
+		if aliasedExpr, ok := selExpr.(*sqlparser.AliasedExpr); ok && qg.r.Intn(2) < 1 {
+			literal, ok := aliasedExpr.Expr.(*sqlparser.Literal)
+			isIntLiteral := ok && literal.Type == sqlparser.IntVal
+			if isIntLiteral {
+				continue
+			}
+			qg.sel.OrderBy = append(qg.sel.OrderBy, sqlparser.NewOrder(aliasedExpr.Expr, getRandomOrderDirection(qg.r)))
 		}
 	}
-
-	return
 }
 
 // returns 0-2 random expressions based on tables
-func (qg queryGen) createWherePredicates(tables []tableT) sqlparser.Exprs {
-	// TODO: create gen config outside
-	return qg.createRandomExprs(0,
-		slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
+func (qg *queryGen) createWherePredicates(tables []tableT) {
+	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
+	// TODO: when qg is added as an expression generator
+	if qg.r.Intn(5) < 1 {
+		exprGenerators = append(exprGenerators, qg)
+	}
+	predicates := qg.createRandomExprs(0, 2, exprGenerators...)
+	qg.sel.AddWhere(sqlparser.AndExpressions(predicates...))
 }
 
 // creates predicates for the having clause comparing a column to a random expression
-func (qg queryGen) createHavingPredicates(grouping []column) sqlparser.Exprs {
+func (qg *queryGen) createHavingPredicates(grouping []column) {
 	predicates := sqlparser.Exprs{}
 	if qg.r.Intn(2) < 1 && testFailingQueries {
 		// TODO: having can only contain aggregate or grouping columns in mysql, works fine in vitess
 		// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
-		predicates = append(predicates, qg.createRandomExprs(0,
+		predicates = append(predicates, qg.createRandomExprs(0, 2,
 			slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })...)...)
 	}
 
 	qg.genConfig = qg.genConfig.CanAggregateConfig()
-	predicates = append(predicates, qg.createRandomExprs(0,
+	predicates = append(predicates, qg.createRandomExprs(0, 2,
 		slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })...)...)
+	qg.genConfig = qg.genConfig.CannotAggregateConfig()
 
-	return predicates
+	qg.sel.AddHaving(sqlparser.AndExpressions(predicates...))
 }
 
-// returns between minExprs and minExprs + 2 random expressions using generators
-func (qg queryGen) createRandomExprs(minExprs int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
-	numPredicates := qg.r.Intn(3) + minExprs
+// returns between minExprs and maxExprs random expressions using generators
+func (qg *queryGen) createRandomExprs(minExprs, maxExprs int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
+	if minExprs > maxExprs {
+		log.Fatalf("minExprs is greater than maxExprs; minExprs: %d, maxExprs: %d\n", minExprs, maxExprs)
+	} else if maxExprs <= 0 {
+		return
+	}
+	numPredicates := qg.r.Intn(maxExprs-minExprs+1) + minExprs
 	for i := 0; i < numPredicates; i++ {
 		predicates = append(predicates, qg.getRandomExpr(generators...))
 	}
@@ -440,20 +501,20 @@ func (qg queryGen) createRandomExprs(minExprs int, generators ...sqlparser.ExprG
 }
 
 // getRandomExpr returns a random expression
-func (qg queryGen) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlparser.Expr {
+func (qg *queryGen) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlparser.Expr {
 	g := sqlparser.NewGenerator(qg.r, 2, generators...)
 	return g.Expression(qg.genConfig)
 }
 
 // creates sel.Limit
-func (qg queryGen) createLimit() *sqlparser.Limit {
+func (qg *queryGen) createLimit() {
 	limitNum := qg.r.Intn(10)
 	if qg.r.Intn(2) < 1 {
 		offset := qg.r.Intn(10)
-		return sqlparser.NewLimit(offset, limitNum)
+		qg.sel.Limit = sqlparser.NewLimit(offset, limitNum)
+	} else {
+		qg.sel.Limit = sqlparser.NewLimitWithoutOffset(limitNum)
 	}
-
-	return sqlparser.NewLimitWithoutOffset(limitNum)
 }
 
 func getRandomOrderDirection(r *rand.Rand) sqlparser.OrderDirection {
diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index 1eab1217456..714967f3c83 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -49,11 +49,13 @@ func TestSeed(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	seed := int64(1689140137270728000)
-	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
-	query1 := sqlparser.String(qg.randomQuery(schemaTables))
-	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
-	query2 := sqlparser.String(qg.randomQuery(schemaTables))
+	seed := int64(1689502508367394000)
+	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 3, 3, 3, schemaTables)
+	qg.randomQuery()
+	query1 := sqlparser.String(qg.sel)
+	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 3, 3, 3, schemaTables)
+	qg.randomQuery()
+	query2 := sqlparser.String(qg.sel)
 	fmt.Println(query1)
 	require.Equal(t, query1, query2)
 }
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index eb41d6b8f85..51d77321a40 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -267,11 +267,12 @@ func TestRandom(t *testing.T) {
 
 	var queryCount int
 	// continue testing after an error if and only if testFailingQueries is true
-	for time.Now().Before(endBy) && (!t.Failed() || testFailingQueries) {
+	for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) {
 		seed := time.Now().UnixNano()
 		fmt.Printf("seed: %d\n", seed)
-		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{})
-		query := sqlparser.String(qg.randomQuery(schemaTables))
+		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 3, 3, 3, schemaTables)
+		qg.randomQuery()
+		query := sqlparser.String(qg.sel)
 		_, vtErr := mcmp.ExecAllowAndCompareError(query)
 		fmt.Println(query)
 		// this assumes all queries are valid mysql queries
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index a2d2a06dad4..6aaf6bb8814 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -92,6 +92,7 @@ type Generator struct {
 	r              *rand.Rand
 	depth          int
 	maxDepth       int
+	isAggregate    bool
 	exprGenerators []ExprGenerator
 }
 
@@ -142,10 +143,24 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 	}
 
 	if genConfig.AggrRule != CannotAggregate {
-		options = append(options, func() Expr { return g.randomAggregate(genConfig.CannotAggregateConfig()) })
+		options = append(options, func() Expr {
+			g.isAggregate = true
+			return g.randomAggregate(genConfig.CannotAggregateConfig())
+		})
 	}
 
-	return g.randomOf(options)
+	expr := g.randomOf(options)
+	// if the generated expression must be an aggregate, and it is not,
+	// tack on an extra "+ count(*)" to make it aggregate
+	if genConfig.AggrRule == IsAggregate && !g.isAggregate && g.depth == 0 {
+		expr = &BinaryExpr{
+			Operator: BitAndOp,
+			Left:     expr,
+			Right:    &CountStar{},
+		}
+	}
+
+	return expr
 }
 
 func (g *Generator) randomAggregate(genConfig ExprGeneratorConfig) Expr {
@@ -173,8 +188,7 @@ func (g *Generator) booleanExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.andExpr(genConfig) },
 		func() Expr { return g.xorExpr(genConfig) },
 		func() Expr { return g.orExpr(genConfig) },
-		func() Expr { return g.comparison(genConfig.intTypeConfig()) },
-		func() Expr { return g.comparison(genConfig.stringTypeConfig()) },
+		func() Expr { return g.comparison(genConfig) },
 		//func() Expr { return g.comparison(genConfig) }, // this is not accepted by the parser
 		func() Expr { return g.inExpr(genConfig.intTypeConfig()) },
 		func() Expr { return g.between(genConfig.intTypeConfig()) },

From bf5380103a5ffa943e46b5beee30b5ad02d4e966 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Sun, 16 Jul 2023 22:57:54 -0700
Subject: [PATCH 11/27] added scalar subqueries to all random expression
 generation for the fuzzer

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 52 +++++++++++--------
 .../vtgate/queries/random/query_gen_test.go   |  6 +--
 .../vtgate/queries/random/random_test.go      |  7 ++-
 go/vt/sqlparser/random_expr.go                | 10 ++--
 4 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 77080a26429..71d1954463f 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -82,6 +82,7 @@ func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, ma
 		maxCols:      1,
 		maxRows:      1,
 		schemaTables: schemaTables,
+		sel:          &sqlparser.Select{},
 	}
 }
 
@@ -198,7 +199,7 @@ func (qg *queryGen) randomQuery() {
 
 	// canAggregate determines if the query will have
 	// aggregate columns, group by, and having
-	canAggregate := qg.r.Intn(4) > 0
+	canAggregate := qg.r.Intn(4) < 3
 
 	var (
 		grouping, aggregates []column
@@ -244,10 +245,13 @@ func (qg *queryGen) randomQuery() {
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
 	if testFailingQueries {
-		// TODO: ugly
-		f = func() sqlparser.Expr {
-			return qg.getRandomExpr(slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
+		exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
+		// add scalar subqueries
+		if qg.r.Intn(10) < 1 {
+			exprGenerators = append(exprGenerators, qg)
 		}
+
+		f = func() sqlparser.Expr { return qg.getRandomExpr(exprGenerators...) }
 	} else {
 		f = func() sqlparser.Expr { return qg.getRandomExpr() }
 	}
@@ -305,6 +309,7 @@ func (qg *queryGen) randomQuery() {
 	}
 
 	// alias the grouping columns randomly
+	// TODO: alias is done after HAVING because vitess does not properly recognize aliases in it
 	for i := range grouping {
 		if qg.r.Intn(2) < 1 {
 			continue
@@ -380,7 +385,13 @@ func (qg *queryGen) createJoinPredicates(tables []tableT) sqlparser.Exprs {
 		log.Fatalf("tables has %d elements, needs at least 2", len(tables))
 	}
 
-	return qg.createRandomExprs(1, 3, &tables[len(tables)-2], &tables[len(tables)-1])
+	exprGenerators := []sqlparser.ExprGenerator{&tables[len(tables)-2], &tables[len(tables)-1]}
+	// add scalar subqueries
+	if qg.r.Intn(10) < 1 {
+		exprGenerators = append(exprGenerators, qg)
+	}
+
+	return qg.createRandomExprs(1, 3, exprGenerators...)
 }
 
 // returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
@@ -402,8 +413,6 @@ func (qg *queryGen) createGroupBy(tables []tableT) (grouping []column) {
 		if qg.r.Intn(2) < 1 {
 			alias := fmt.Sprintf("cgroup%d", len(grouping))
 			qg.sel.SelectExprs = append(qg.sel.SelectExprs, newAliasedColumn(col, alias))
-			// TODO: alias in a separate function to properly generate the having clause
-			//col.name = sqlparser.NewColName(alias)
 			grouping = append(grouping, col)
 		}
 	}
@@ -413,11 +422,14 @@ func (qg *queryGen) createGroupBy(tables []tableT) (grouping []column) {
 
 // returns the aggregation columns as three types: sqlparser.SelectExprs, []column
 func (qg *queryGen) createAggregations(tables []tableT) (aggregates []column) {
+	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
+	// add scalar subqueries
+	if qg.r.Intn(10) < 1 {
+		exprGenerators = append(exprGenerators, qg)
+	}
+
 	qg.genConfig = qg.genConfig.IsAggregateConfig()
-	aggrExprs := qg.createRandomExprs(0, qg.maxAggrs,
-		slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator {
-			return &t
-		})...)
+	aggrExprs := qg.createRandomExprs(0, qg.maxAggrs, exprGenerators...)
 	qg.genConfig = qg.genConfig.CannotAggregateConfig()
 
 	for i, expr := range aggrExprs {
@@ -459,27 +471,25 @@ func (qg *queryGen) createOrderBy() {
 // returns 0-2 random expressions based on tables
 func (qg *queryGen) createWherePredicates(tables []tableT) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
-	// TODO: when qg is added as an expression generator
-	if qg.r.Intn(5) < 1 {
+	// add scalar subqueries
+	if qg.r.Intn(10) < 1 {
 		exprGenerators = append(exprGenerators, qg)
 	}
+
 	predicates := qg.createRandomExprs(0, 2, exprGenerators...)
 	qg.sel.AddWhere(sqlparser.AndExpressions(predicates...))
 }
 
 // creates predicates for the having clause comparing a column to a random expression
 func (qg *queryGen) createHavingPredicates(grouping []column) {
-	predicates := sqlparser.Exprs{}
-	if qg.r.Intn(2) < 1 && testFailingQueries {
-		// TODO: having can only contain aggregate or grouping columns in mysql, works fine in vitess
-		// TODO: Can fix this by putting only the table with the grouping and aggregates column (newTable)
-		predicates = append(predicates, qg.createRandomExprs(0, 2,
-			slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })...)...)
+	exprGenerators := slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })
+	// add scalar subqueries
+	if qg.r.Intn(10) < 1 {
+		exprGenerators = append(exprGenerators, qg)
 	}
 
 	qg.genConfig = qg.genConfig.CanAggregateConfig()
-	predicates = append(predicates, qg.createRandomExprs(0, 2,
-		slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })...)...)
+	predicates := qg.createRandomExprs(0, 2, exprGenerators...)
 	qg.genConfig = qg.genConfig.CannotAggregateConfig()
 
 	qg.sel.AddHaving(sqlparser.AndExpressions(predicates...))
diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index 714967f3c83..e051c71ab5d 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -49,11 +49,11 @@ func TestSeed(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	seed := int64(1689502508367394000)
-	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 3, 3, 3, schemaTables)
+	seed := int64(1689568235947103000)
+	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 2, 2, 2, schemaTables)
 	qg.randomQuery()
 	query1 := sqlparser.String(qg.sel)
-	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 3, 3, 3, schemaTables)
+	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 2, 2, 2, schemaTables)
 	qg.randomQuery()
 	query2 := sqlparser.String(qg.sel)
 	fmt.Println(query1)
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 51d77321a40..6ce986babf9 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -84,6 +84,9 @@ func TestMustFix(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
+	// mismatched number of columns
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) + 1 from emp as tbl0 order by count(*) desc")
+
 	// mismatched results (mismatched types)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32")
 
@@ -263,14 +266,14 @@ func TestRandom(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	endBy := time.Now().Add(1 * time.Second)
+	endBy := time.Now().Add(5 * time.Second)
 
 	var queryCount int
 	// continue testing after an error if and only if testFailingQueries is true
 	for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) {
 		seed := time.Now().UnixNano()
 		fmt.Printf("seed: %d\n", seed)
-		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 3, 3, 3, schemaTables)
+		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 2, 2, 2, schemaTables)
 		qg.randomQuery()
 		query := sqlparser.String(qg.sel)
 		_, vtErr := mcmp.ExecAllowAndCompareError(query)
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 6aaf6bb8814..03a322a8492 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -153,10 +153,9 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 	// if the generated expression must be an aggregate, and it is not,
 	// tack on an extra "+ count(*)" to make it aggregate
 	if genConfig.AggrRule == IsAggregate && !g.isAggregate && g.depth == 0 {
-		expr = &BinaryExpr{
-			Operator: BitAndOp,
-			Left:     expr,
-			Right:    &CountStar{},
+		expr = &AndExpr{
+			Left:  expr,
+			Right: &CountStar{},
 		}
 	}
 
@@ -188,7 +187,8 @@ func (g *Generator) booleanExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.andExpr(genConfig) },
 		func() Expr { return g.xorExpr(genConfig) },
 		func() Expr { return g.orExpr(genConfig) },
-		func() Expr { return g.comparison(genConfig) },
+		func() Expr { return g.comparison(genConfig.intTypeConfig()) },
+		func() Expr { return g.comparison(genConfig.stringTypeConfig()) },
 		//func() Expr { return g.comparison(genConfig) }, // this is not accepted by the parser
 		func() Expr { return g.inExpr(genConfig.intTypeConfig()) },
 		func() Expr { return g.between(genConfig.intTypeConfig()) },

From 98055b7a97248e454c0fd9d1633f09ce1c2d47ab Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Tue, 18 Jul 2023 02:47:17 -0700
Subject: [PATCH 12/27] added support for tuple random expressions

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 204 ++++++++++-------
 .../vtgate/queries/random/query_gen_test.go   |   7 +-
 .../vtgate/queries/random/random_test.go      |  39 +++-
 go/vt/sqlparser/ast_funcs.go                  |   4 +
 go/vt/sqlparser/parse_test.go                 |   2 +
 go/vt/sqlparser/random_expr.go                | 212 ++++++++++++------
 6 files changed, 296 insertions(+), 172 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 71d1954463f..213bfbc0b4f 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -35,15 +35,11 @@ const testFailingQueries = false
 
 type (
 	queryGen struct {
-		r         *rand.Rand
-		genConfig sqlparser.ExprGeneratorConfig
-		maxTables int
-		maxAggrs  int
-		maxGBs    int
-		// maxCols = 0 indicates no limit
-		maxCols int
-		// maxRows = 0 indicates no limit
-		maxRows      int
+		r            *rand.Rand
+		genConfig    sqlparser.ExprGeneratorConfig
+		maxTables    int
+		maxAggrs     int
+		maxGBs       int
 		schemaTables []tableT
 		sel          *sqlparser.Select
 	}
@@ -79,8 +75,6 @@ func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, ma
 		maxTables:    maxTables,
 		maxAggrs:     maxAggrs,
 		maxGBs:       maxGBs,
-		maxCols:      1,
-		maxRows:      1,
 		schemaTables: schemaTables,
 		sel:          &sqlparser.Select{},
 	}
@@ -141,7 +135,7 @@ func (t *tableT) clone() *tableT {
 }
 
 func (c *column) Generate(_ *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
-	if c.typ == genConfig.Type {
+	if c.typ == genConfig.Type || genConfig.Type == "" {
 		return c.getASTExpr()
 	}
 
@@ -154,7 +148,7 @@ func (t *tableT) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig)
 	for len(colsCopy) > 0 {
 		idx := r.Intn(len(colsCopy))
 		randCol := colsCopy[idx]
-		if randCol.typ == genConfig.Type {
+		if randCol.typ == genConfig.Type || genConfig.Type == "" {
 			return randCol.getASTExpr()
 		}
 
@@ -166,21 +160,21 @@ func (t *tableT) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig)
 	return nil
 }
 
-// Generate generates a scalar subquery based on qg
+// Generate generates a subquery based on qg
 func (qg *queryGen) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
-	var newSchemaTables []tableT
+	var schemaTablesCopy []tableT
 	for _, tbl := range qg.schemaTables {
-		newSchemaTables = append(newSchemaTables, *tbl.clone())
+		schemaTablesCopy = append(schemaTablesCopy, *tbl.clone())
 	}
 
-	newQG := newQueryGenerator(r, genConfig, qg.maxTables, qg.maxAggrs, qg.maxGBs, newSchemaTables)
-	newQG.maxRows = 1
-	newQG.maxCols = 1
+	newQG := newQueryGenerator(r, genConfig, qg.maxTables, qg.maxAggrs, qg.maxGBs, schemaTablesCopy)
 	newQG.randomQuery()
 
 	return &sqlparser.Subquery{Select: newQG.sel}
 }
 
+func (qg *queryGen) IQueryGenerator() {}
+
 func (qg *queryGen) randomQuery() {
 	// make sure the random expressions can generally not contain aggregates; change appropriately
 	qg.genConfig = qg.genConfig.CannotAggregateConfig()
@@ -203,51 +197,49 @@ func (qg *queryGen) randomQuery() {
 
 	var (
 		grouping, aggregates []column
+		newTable             tableT
 	)
 	// TODO: distinct makes vitess think there is grouping on aggregation columns
 	if canAggregate {
 		if testFailingQueries || !isDistinct {
 			// group by
-			if qg.maxRows == 0 {
+			if !qg.genConfig.SingleRow {
 				grouping = qg.createGroupBy(tables)
 			}
 		}
 
-		// aggregate columns
-		aggregates = qg.createAggregations(tables)
-
 		// having
 		isHaving := qg.r.Intn(2) < 1
-		if isHaving {
+		if isHaving && testFailingQueries {
 			qg.createHavingPredicates(grouping)
 		}
-	}
 
-	// can add both aggregate and grouping columns to order by
-	// TODO: order fails with distinct and outer joins
-	isOrdered := qg.r.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries)
-	if isOrdered {
-		qg.createOrderBy()
+		// alias the grouping columns
+		// TODO: alias the grouping columns after having because vitess doesn't recognize aliases in the HAVING
+		grouping = qg.aliasGroupingColumns(grouping)
+
+		// aggregation columns
+		aggregates = qg.createAggregations(tables)
+
+		// add the grouping and aggregation to newTable
+		newTable.addColumns(grouping...)
+		newTable.addColumns(aggregates...)
 	}
 
 	// where
 	qg.createWherePredicates(tables)
 
-	var (
-		newTable tableT
-		f        func() sqlparser.Expr
-		typ      string
-	)
+	var f func() sqlparser.Expr
 	// add random expression to select
 	// TODO: random expressions cause a lot of failures
 	isRandomExpr := qg.r.Intn(2) < 1 && testFailingQueries
 
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
-	if testFailingQueries {
+	if canAggregate && testFailingQueries {
 		exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 		// add scalar subqueries
-		if qg.r.Intn(10) < 1 {
+		if qg.r.Intn(10) < 1 && testFailingQueries {
 			exprGenerators = append(exprGenerators, qg)
 		}
 
@@ -274,53 +266,33 @@ func (qg *queryGen) randomQuery() {
 		// TODO: select distinct [literal] fails
 		qg.sel.Distinct = false
 
-		qg.sel.SelectExprs = append(qg.sel.SelectExprs, sqlparser.NewAliasedExpr(randomExpr, "crandom0"))
-		newTable.addColumns(column{
-			name: "crandom0",
-			typ:  typ,
-		})
+		// alias randomly
+		col := qg.randomlyAlias(randomExpr, "crandom0")
+		newTable.addColumns(col)
 
-		// make sure to add the random expression to group by and order by for only_full_group_by
+		// make sure to add the random expression to group by for only_full_group_by
 		if canAggregate {
 			qg.sel.AddGroupBy(randomExpr)
-			if isOrdered {
-				qg.sel.AddOrder(sqlparser.NewOrder(randomExpr, getRandomOrderDirection(qg.r)))
-			}
 		}
 
 		break
 	}
 
-	if qg.maxCols > 0 {
-		// remove SelectExprs randomly until there are qg.maxCols amount
-		for len(qg.sel.SelectExprs) > qg.maxCols {
-			// select a random index and remove it from SelectExprs
-			idx := qg.r.Intn(len(qg.sel.SelectExprs))
-			qg.sel.SelectExprs[idx] = qg.sel.SelectExprs[len(qg.sel.SelectExprs)-1]
-			qg.sel.SelectExprs = qg.sel.SelectExprs[:len(qg.sel.SelectExprs)-1]
-		}
+	// can add both aggregate and grouping columns to order by
+	// TODO: order fails with distinct and outer joins
+	isOrdered := qg.r.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries)
+	if isOrdered || (!canAggregate && qg.genConfig.SingleRow) /* TODO: might be redundant */ {
+		qg.createOrderBy()
 	}
 
 	// only add a limit if there is an ordering
 	// TODO: limit fails a lot
 	isLimit := qg.r.Intn(2) < 1 && len(qg.sel.OrderBy) > 0 && testFailingQueries
-	if isLimit {
+	if isLimit || (!canAggregate && qg.genConfig.SingleRow) /* TODO: might be redundant */ {
 		qg.createLimit()
 	}
 
-	// alias the grouping columns randomly
-	// TODO: alias is done after HAVING because vitess does not properly recognize aliases in it
-	for i := range grouping {
-		if qg.r.Intn(2) < 1 {
-			continue
-		}
-		alias := fmt.Sprintf("cgroup%d", i)
-		grouping[i].name = alias
-	}
-
-	// add them to newTable
-	newTable.addColumns(grouping...)
-	newTable.addColumns(aggregates...)
+	newTable = qg.matchNumCols(tables, newTable, canAggregate)
 
 	// add new table to schemaTables
 	newTable.tableExpr = sqlparser.NewDerivedTable(false, qg.sel)
@@ -328,7 +300,7 @@ func (qg *queryGen) randomQuery() {
 
 	// derived tables (partially unsupported)
 	// TODO: derived tables fails a lot
-	if qg.r.Intn(10) < 1 && testFailingQueries {
+	if qg.r.Intn(10) < 1 {
 		qg.randomQuery()
 	}
 }
@@ -387,7 +359,7 @@ func (qg *queryGen) createJoinPredicates(tables []tableT) sqlparser.Exprs {
 
 	exprGenerators := []sqlparser.ExprGenerator{&tables[len(tables)-2], &tables[len(tables)-1]}
 	// add scalar subqueries
-	if qg.r.Intn(10) < 1 {
+	if qg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, qg)
 	}
 
@@ -411,8 +383,7 @@ func (qg *queryGen) createGroupBy(tables []tableT) (grouping []column) {
 
 		// add to select
 		if qg.r.Intn(2) < 1 {
-			alias := fmt.Sprintf("cgroup%d", len(grouping))
-			qg.sel.SelectExprs = append(qg.sel.SelectExprs, newAliasedColumn(col, alias))
+			qg.sel.SelectExprs = append(qg.sel.SelectExprs, newAliasedColumn(col, ""))
 			grouping = append(grouping, col)
 		}
 	}
@@ -420,11 +391,30 @@ func (qg *queryGen) createGroupBy(tables []tableT) (grouping []column) {
 	return
 }
 
+// aliasGroupingColumns randomly aliases the grouping columns in the SelectExprs
+func (qg *queryGen) aliasGroupingColumns(grouping []column) []column {
+	if len(grouping) != len(qg.sel.SelectExprs) {
+		log.Fatalf("grouping (length: %d) and qg.sel.SelectExprs (length: %d) should have the same length at this point", len(grouping), len(qg.sel.SelectExprs))
+	}
+
+	for i := range grouping {
+		if qg.r.Intn(2) < 1 {
+			if aliasedExpr, ok := qg.sel.SelectExprs[i].(*sqlparser.AliasedExpr); ok {
+				alias := fmt.Sprintf("cgroup%d", i)
+				aliasedExpr.SetAlias(alias)
+				grouping[i].name = alias
+			}
+		}
+	}
+
+	return grouping
+}
+
 // returns the aggregation columns as three types: sqlparser.SelectExprs, []column
 func (qg *queryGen) createAggregations(tables []tableT) (aggregates []column) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 	// add scalar subqueries
-	if qg.r.Intn(10) < 1 {
+	if qg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, qg)
 	}
 
@@ -433,16 +423,8 @@ func (qg *queryGen) createAggregations(tables []tableT) (aggregates []column) {
 	qg.genConfig = qg.genConfig.CannotAggregateConfig()
 
 	for i, expr := range aggrExprs {
-		// TODO: min/max often fails
-		// randomly alias
-		var alias string
-		if qg.r.Intn(2) < 1 {
-			alias = fmt.Sprintf("caggr%d", i)
-			aggregates = append(aggregates, column{name: alias})
-		} else {
-			aggregates = append(aggregates, column{name: sqlparser.String(expr)})
-		}
-		qg.sel.SelectExprs = append(qg.sel.SelectExprs, sqlparser.NewAliasedExpr(expr, alias))
+		col := qg.randomlyAlias(expr, fmt.Sprintf("caggr%d", i))
+		aggregates = append(aggregates, col)
 	}
 
 	return
@@ -472,7 +454,7 @@ func (qg *queryGen) createOrderBy() {
 func (qg *queryGen) createWherePredicates(tables []tableT) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 	// add scalar subqueries
-	if qg.r.Intn(10) < 1 {
+	if qg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, qg)
 	}
 
@@ -484,7 +466,7 @@ func (qg *queryGen) createWherePredicates(tables []tableT) {
 func (qg *queryGen) createHavingPredicates(grouping []column) {
 	exprGenerators := slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })
 	// add scalar subqueries
-	if qg.r.Intn(10) < 1 {
+	if qg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, qg)
 	}
 
@@ -518,6 +500,11 @@ func (qg *queryGen) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlpars
 
 // creates sel.Limit
 func (qg *queryGen) createLimit() {
+	if qg.genConfig.SingleRow {
+		qg.sel.Limit = sqlparser.NewLimitWithoutOffset(1)
+		return
+	}
+
 	limitNum := qg.r.Intn(10)
 	if qg.r.Intn(2) < 1 {
 		offset := qg.r.Intn(10)
@@ -527,6 +514,53 @@ func (qg *queryGen) createLimit() {
 	}
 }
 
+// randomlyAlias randomly aliases expr with alias alias, adds it to sel.SelectExprs and returns the column created
+func (qg *queryGen) randomlyAlias(expr sqlparser.Expr, alias string) column {
+	var col column
+	if qg.r.Intn(2) < 1 {
+		alias = ""
+		col.name = sqlparser.String(expr)
+	} else {
+		col.name = alias
+	}
+	qg.sel.SelectExprs = append(qg.sel.SelectExprs, sqlparser.NewAliasedExpr(expr, alias))
+
+	return col
+}
+
+// matchNumCols makes sure qg.sel.SelectExprs and newTable both have the same number of cols: qg.genConfig.NumCols
+func (qg *queryGen) matchNumCols(tables []tableT, newTable tableT, canAggregate bool) tableT {
+	// remove SelectExprs and newTable.cols randomly until there are qg.genConfig.NumCols amount
+	for len(qg.sel.SelectExprs) > qg.genConfig.NumCols && qg.genConfig.NumCols > 0 {
+		// select a random index and remove it from SelectExprs and newTable
+		idx := qg.r.Intn(len(qg.sel.SelectExprs))
+
+		qg.sel.SelectExprs[idx] = qg.sel.SelectExprs[len(qg.sel.SelectExprs)-1]
+		qg.sel.SelectExprs = qg.sel.SelectExprs[:len(qg.sel.SelectExprs)-1]
+
+		newTable.cols[idx] = newTable.cols[len(newTable.cols)-1]
+		newTable.cols = newTable.cols[:len(newTable.cols)-1]
+	}
+
+	// alternatively, add random expressions until there are qg.genConfig.NumCols amount
+	if qg.genConfig.NumCols > len(qg.sel.SelectExprs) {
+		diff := qg.genConfig.NumCols - len(qg.sel.SelectExprs)
+		exprs := qg.createRandomExprs(diff, diff,
+			slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
+
+		for i, expr := range exprs {
+			col := qg.randomlyAlias(expr, fmt.Sprintf("crandom%d", i+1))
+			newTable.addColumns(col)
+
+			if canAggregate {
+				qg.sel.AddGroupBy(expr)
+			}
+		}
+	}
+
+	return newTable
+}
+
 func getRandomOrderDirection(r *rand.Rand) sqlparser.OrderDirection {
 	// asc, desc
 	return randomEl(r, []sqlparser.OrderDirection{0, 1})
diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index e051c71ab5d..556bf6cccac 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -49,11 +49,12 @@ func TestSeed(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	seed := int64(1689568235947103000)
-	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 2, 2, 2, schemaTables)
+	seed := int64(1689672658271399000)
+	genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
+	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
 	qg.randomQuery()
 	query1 := sqlparser.String(qg.sel)
-	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 2, 2, 2, schemaTables)
+	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
 	qg.randomQuery()
 	query2 := sqlparser.String(qg.sel)
 	fmt.Println(query1)
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 6ce986babf9..aecd15a676a 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -84,18 +84,25 @@ func TestMustFix(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
+	// mismatched results
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct case count(*) when -41 ^ 10 then -17 when 22 then -52 else 7 end from emp as tbl0, emp as tbl1 where tbl1.job < tbl1.job")
+
+	// mismatched results (maybe derived tables)
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1")
+
+	// mismatched results
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'")
+
+	// mismatched results
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1 where tbl0.deptno and tbl1.comm in (12, tbl0.deptno, case false when 67 then -17 when -78 then -35 end, -76 >> -68)")
+
 	// mismatched number of columns
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) + 1 from emp as tbl0 order by count(*) desc")
 
 	// mismatched results (mismatched types)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32")
 
-	// panics instead of normally logging the error
-	// Received unexpected error:
-	// VT12001: unsupported: only one DISTINCT aggregation is allowed in a SELECT: sum(distinct 1) as caggr1
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(distinct tbl0.comm) as caggr0, sum(distinct 1) as caggr1 from emp as tbl0, emp as tbl1 having 'redfish' < 'blowfish'")
-
-	// mismatched results
+	// mismatched results (decimals off by a little; evalengine problem)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1")
 
 	// mismatched results
@@ -140,6 +147,9 @@ func TestMustFix(t *testing.T) {
 	// swapping tables and predicates and changing to left fails
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal")
 
+	// EOF
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ 8 < -31 xor (-29, sum((tbl0.deptno, 'wren', 'ostrich')), max(distinct (tbl0.dname, -15, -8))) in ((sum(distinct (tbl0.dname, 'bengal', -10)), 'ant', true)) as caggr0 from dept as tbl0 where tbl0.deptno * (77 - 61)")
+
 	// EOF
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.deptno as cgroup0, tbl1.loc as cgroup1, count(distinct tbl1.loc) as caggr1, tbl1.loc as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.deptno, tbl1.loc")
 
@@ -156,8 +166,11 @@ func TestKnownFailures(t *testing.T) {
 	// logs more stuff
 	//clusterInstance.EnableGeneralLog()
 
+	// rhs of an In operation should be a tuple
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ (case when true then min(distinct tbl1.job) else 'bee' end, 'molly') not in (('dane', 0)) as caggr1 from emp as tbl0, emp as tbl1")
+
 	// VT13001: [BUG] in scatter query: complex ORDER BY expression: :vtg1 /* VARCHAR */
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.job as cgroup0, sum(distinct 'mudfish'), tbl1.job as crandom0 from emp as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job desc, tbl1.job asc limit 8, 1")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.job as cgroup0, sum(distinct 'mudfish'), tbl1.job as crandom0 from emp as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job asc limit 8, 1")
 
 	// VT13001: [BUG] column should not be pushed to projection while doing a column lookup
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ -26 in (tbl2.mgr, -8, tbl0.deptno) as crandom0 from dept as tbl0, emp as tbl1 left join emp as tbl2 on tbl2.ename")
@@ -266,22 +279,24 @@ func TestRandom(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	endBy := time.Now().Add(5 * time.Second)
+	endBy := time.Now().Add(1 * time.Second)
 
 	var queryCount int
 	// continue testing after an error if and only if testFailingQueries is true
 	for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) {
 		seed := time.Now().UnixNano()
-		fmt.Printf("seed: %d\n", seed)
-		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), sqlparser.ExprGeneratorConfig{}, 2, 2, 2, schemaTables)
+		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
+		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
 		qg.randomQuery()
 		query := sqlparser.String(qg.sel)
 		_, vtErr := mcmp.ExecAllowAndCompareError(query)
-		fmt.Println(query)
+
 		// this assumes all queries are valid mysql queries
 		if vtErr != nil {
-
+			fmt.Printf("seed: %d\n", seed)
+			fmt.Println(query)
 			fmt.Println(vtErr)
+			fmt.Printf("\n\n\n")
 
 			if stopOnMustFixError {
 				// EOF
diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go
index e4213fe702c..9b1b9381567 100644
--- a/go/vt/sqlparser/ast_funcs.go
+++ b/go/vt/sqlparser/ast_funcs.go
@@ -675,6 +675,10 @@ func NewAliasedExpr(expr Expr, alias string) *AliasedExpr {
 	}
 }
 
+func (ae *AliasedExpr) SetAlias(alias string) {
+	ae.As = NewIdentifierCI(alias)
+}
+
 // NewOrder makes a new Order
 func NewOrder(expr Expr, direction OrderDirection) *Order {
 	return &Order{
diff --git a/go/vt/sqlparser/parse_test.go b/go/vt/sqlparser/parse_test.go
index 24360c6b9c1..d7822addaeb 100644
--- a/go/vt/sqlparser/parse_test.go
+++ b/go/vt/sqlparser/parse_test.go
@@ -44,6 +44,8 @@ var (
 		partialDDL           bool
 		ignoreNormalizerTest bool
 	}{{
+		input: "select * from foo limit 5 + 5",
+	}, {
 		input:  "create table x(location GEOMETRYCOLLECTION DEFAULT (POINT(7.0, 3.0)))",
 		output: "create table x (\n\tlocation GEOMETRYCOLLECTION default (point(7.0, 3.0))\n)",
 	}, {
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 03a322a8492..74c82c64ca0 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -35,15 +35,57 @@ type (
 		Generate(r *rand.Rand, config ExprGeneratorConfig) Expr
 	}
 
+	QueryGenerator interface {
+		IQueryGenerator()
+		ExprGenerator
+	}
+
 	AggregateRule int8
 
 	ExprGeneratorConfig struct {
-		// AggrRule determines if the random expression can, cannot, or must be an aggregatable expression
+		// AggrRule determines if the random expression can, cannot, or must be an aggregation expression
 		AggrRule AggregateRule
-		Type     string
+		// TODO: get rid of ExprGeneratorConfig Type
+		Type string
+		// MaxCols = 0 indicates no limit
+		NumCols int
+		// SingleRow indicates that the query must have at most one row
+		SingleRow bool
+	}
+
+	Generator struct {
+		r              *rand.Rand
+		depth          int
+		maxDepth       int
+		isAggregate    bool
+		exprGenerators []ExprGenerator
 	}
 )
 
+func NewExprGeneratorConfig(aggrRule AggregateRule, typ string, numCols int, singleRow bool) ExprGeneratorConfig {
+	return ExprGeneratorConfig{
+		AggrRule:  aggrRule,
+		Type:      typ,
+		NumCols:   numCols,
+		SingleRow: singleRow,
+	}
+}
+
+func (egc ExprGeneratorConfig) singleRowConfig() ExprGeneratorConfig {
+	egc.SingleRow = true
+	return egc
+}
+
+func (egc ExprGeneratorConfig) multiRowConfig() ExprGeneratorConfig {
+	egc.SingleRow = false
+	return egc
+}
+
+func (egc ExprGeneratorConfig) setNumCols(numCols int) ExprGeneratorConfig {
+	egc.NumCols = numCols
+	return egc
+}
+
 func (egc ExprGeneratorConfig) boolTypeConfig() ExprGeneratorConfig {
 	egc.Type = "tinyint"
 	return egc
@@ -88,14 +130,6 @@ func NewGenerator(r *rand.Rand, maxDepth int, exprGenerators ...ExprGenerator) *
 	return &g
 }
 
-type Generator struct {
-	r              *rand.Rand
-	depth          int
-	maxDepth       int
-	isAggregate    bool
-	exprGenerators []ExprGenerator
-}
-
 // enter should be called whenever we are producing an intermediate node. it should be followed by a `defer g.exit()`
 func (g *Generator) enter() {
 	g.depth++
@@ -127,36 +161,88 @@ Note: It's important to update this method so that it produces all expressions t
 It's currently missing function calls and string operators
 */
 func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
+	var exprOptions, tupleOptions []exprF
+	// this will only be used for tuple expressions, everything else will need genConfig.NumCols = 1
+	numCols := genConfig.NumCols
+	genConfig = genConfig.setNumCols(1)
+
 	switch genConfig.Type {
 	case "bigint":
-		return g.intExpr(genConfig)
+		exprOptions = append(exprOptions, func() Expr { return g.intExpr(genConfig) })
 	case "varchar":
-		return g.stringExpr(genConfig)
+		exprOptions = append(exprOptions, func() Expr { return g.stringExpr(genConfig) })
 	case "tinyint":
-		return g.booleanExpr(genConfig)
+		exprOptions = append(exprOptions, func() Expr { return g.booleanExpr(genConfig) })
+	case "":
+		exprOptions = append(exprOptions, []exprF{
+			func() Expr { return g.intExpr(genConfig) },
+			func() Expr { return g.stringExpr(genConfig) },
+			func() Expr { return g.booleanExpr(genConfig) },
+		}...)
 	}
 
-	options := []exprF{
-		func() Expr { return g.intExpr(genConfig) },
-		func() Expr { return g.stringExpr(genConfig) },
-		func() Expr { return g.booleanExpr(genConfig) },
+	for i := range g.exprGenerators {
+		generator := g.exprGenerators[i]
+		if generator == nil {
+			continue
+		}
+
+		// Only generate subqueries with multiple rows when we require more than 1 column
+		if qg, ok := generator.(QueryGenerator); ok && genConfig.NumCols >= 2 {
+			tupleOptions = append(tupleOptions, func() Expr {
+				expr := qg.Generate(g.r, genConfig.setNumCols(numCols))
+				if expr == nil {
+					return g.randomTupleLiteral(numCols)
+				}
+				return expr
+			})
+			// don't create expressions from the expression exprGenerators if we haven't created an aggregation yet
+		} else if genConfig.AggrRule != IsAggregate {
+			exprOptions = append(exprOptions, func() Expr {
+				expr := generator.Generate(g.r, genConfig)
+				if expr == nil {
+					return g.randomLiteral()
+				}
+				return expr
+			})
+		}
 	}
 
 	if genConfig.AggrRule != CannotAggregate {
-		options = append(options, func() Expr {
+		exprOptions = append(exprOptions, func() Expr {
 			g.isAggregate = true
 			return g.randomAggregate(genConfig.CannotAggregateConfig())
 		})
 	}
 
-	expr := g.randomOf(options)
+	// tupleOptions will be nil in this case
+	if numCols <= 1 {
+		return g.makeAggregateIfNecessary(genConfig, g.randomOf(exprOptions))
+	}
+
+	// with 1/5 probability choose a tuple subquery
+	if g.randomBool(0.2) && len(tupleOptions) > 0 {
+		return g.randomOf(tupleOptions)
+	}
+
+	tuple := ValTuple{}
+	for i := 0; i < numCols; i++ {
+		tuple = append(tuple, g.makeAggregateIfNecessary(genConfig, g.randomOf(exprOptions)))
+	}
+
+	return tuple
+}
+
+// makeAggregateIfNecessary is a failsafe to make sure an IsAggregate expression is in fact an aggregation
+func (g *Generator) makeAggregateIfNecessary(genConfig ExprGeneratorConfig, expr Expr) Expr {
 	// if the generated expression must be an aggregate, and it is not,
-	// tack on an extra "+ count(*)" to make it aggregate
+	// tack on an extra "and count(*)" to make it aggregate
 	if genConfig.AggrRule == IsAggregate && !g.isAggregate && g.depth == 0 {
 		expr = &AndExpr{
 			Left:  expr,
 			Right: &CountStar{},
 		}
+		g.isAggregate = true
 	}
 
 	return expr
@@ -190,7 +276,7 @@ func (g *Generator) booleanExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.comparison(genConfig.intTypeConfig()) },
 		func() Expr { return g.comparison(genConfig.stringTypeConfig()) },
 		//func() Expr { return g.comparison(genConfig) }, // this is not accepted by the parser
-		func() Expr { return g.inExpr(genConfig.intTypeConfig()) },
+		func() Expr { return g.inExpr(genConfig) },
 		func() Expr { return g.between(genConfig.intTypeConfig()) },
 		func() Expr { return g.isExpr(genConfig) },
 		func() Expr { return g.notExpr(genConfig) },
@@ -208,31 +294,11 @@ func (g *Generator) intExpr(genConfig ExprGeneratorConfig) Expr {
 	genConfig = genConfig.intTypeConfig()
 
 	options := []exprF{
+		g.intLiteral,
 		func() Expr { return g.arithmetic(genConfig) },
-		func() Expr { return g.intLiteral() },
 		func() Expr { return g.caseExpr(genConfig) },
 	}
 
-	// if the expression must be aggregatable, then don't generate from the ExprGenerators at this depth
-	if genConfig.AggrRule == IsAggregate {
-		return g.randomOf(options)
-	}
-
-	for i := range g.exprGenerators {
-		generator := g.exprGenerators[i]
-		if generator == nil {
-			continue
-		}
-
-		options = append(options, func() Expr {
-			expr := generator.Generate(g.r, genConfig)
-			if expr == nil {
-				return g.intLiteral()
-			}
-			return expr
-		})
-	}
-
 	return g.randomOf(options)
 }
 
@@ -244,39 +310,42 @@ func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 	genConfig = genConfig.stringTypeConfig()
 
 	options := []exprF{
-		func() Expr { return g.stringLiteral() },
+		g.stringLiteral,
 		func() Expr { return g.caseExpr(genConfig) },
 	}
 
-	// if the expression must be aggregatable, then don't generate from the ExprGenerators at this depth
-	if genConfig.AggrRule == IsAggregate {
-		return g.randomOf(options)
+	return g.randomOf(options)
+}
+
+func (g *Generator) randomTupleLiteral(len int) Expr {
+	tuple := ValTuple{}
+	for i := 0; i < len; i++ {
+		tuple = append(tuple, g.randomLiteral())
 	}
 
-	for i := range g.exprGenerators {
-		generator := g.exprGenerators[i]
-		if generator == nil {
-			continue
-		}
+	return tuple
+}
 
-		options = append(options, func() Expr {
-			expr := generator.Generate(g.r, genConfig)
-			if expr == nil {
-				return g.stringLiteral()
-			}
-			return expr
-		})
+func (g *Generator) randomLiteral() Expr {
+	options := []exprF{
+		g.intLiteral,
+		g.stringLiteral,
+		g.booleanLiteral,
 	}
 
 	return g.randomOf(options)
 }
 
 func (g *Generator) booleanLiteral() Expr {
-	return BoolVal(g.randomBool())
+	return BoolVal(g.randomBool(0.5))
 }
 
-func (g *Generator) randomBool() bool {
-	return g.r.Float32() < 0.5
+// randomBool returns true with probability prob
+func (g *Generator) randomBool(prob float32) bool {
+	if prob < 0 || prob > 1 {
+		prob = 0.5
+	}
+	return g.r.Float32() < prob
 }
 
 func (g *Generator) intLiteral() Expr {
@@ -321,10 +390,10 @@ func (g *Generator) caseExpr(genConfig ExprGeneratorConfig) Expr {
 
 	var exp Expr
 	var elseExpr Expr
-	if g.randomBool() {
+	if g.randomBool(0.5) {
 		exp = g.Expression(genConfig.anyTypeConfig())
 	}
-	if g.randomBool() {
+	if g.randomBool(0.5) {
 		elseExpr = g.Expression(genConfig)
 	}
 
@@ -414,21 +483,20 @@ func (g *Generator) inExpr(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 
-	expr := g.Expression(genConfig)
 	size := g.r.Intn(3) + 2
-	tuples := ValTuple{}
-	for i := 0; i < size; i++ {
-		tuples = append(tuples, g.Expression(genConfig))
-	}
+	inExprGenConfig := NewExprGeneratorConfig(genConfig.AggrRule, "", size, true)
+	tuple1 := g.Expression(inExprGenConfig)
+	tuple2 := ValTuple{g.Expression(inExprGenConfig)}
+
 	op := InOp
-	if g.randomBool() {
+	if g.randomBool(0.5) {
 		op = NotInOp
 	}
 
 	return &ComparisonExpr{
 		Operator: op,
-		Left:     expr,
-		Right:    tuples,
+		Left:     tuple1,
+		Right:    tuple2,
 	}
 }
 
@@ -437,7 +505,7 @@ func (g *Generator) between(genConfig ExprGeneratorConfig) Expr {
 	defer g.exit()
 
 	var IsBetween bool
-	if g.randomBool() {
+	if g.randomBool(0.5) {
 		IsBetween = true
 	} else {
 		IsBetween = false

From dccd6ecb49d490871230219995e0e84da97e9d7f Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Wed, 19 Jul 2023 02:51:16 -0700
Subject: [PATCH 13/27] added support for random expressions with exists

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        |   6 +-
 .../vtgate/queries/random/query_gen_test.go   |   2 +-
 .../vtgate/queries/random/random_expr_test.go |  27 ++++-
 .../vtgate/queries/random/random_test.go      |  12 +-
 go/vt/sqlparser/ast_funcs.go                  |  31 +++--
 go/vt/sqlparser/random_expr.go                | 106 ++++++++++++------
 6 files changed, 134 insertions(+), 50 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 213bfbc0b4f..69575406d7b 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -62,7 +62,7 @@ type (
 
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
 var _ sqlparser.ExprGenerator = (*column)(nil)
-var _ sqlparser.ExprGenerator = (*queryGen)(nil)
+var _ sqlparser.QueryGenerator = (*queryGen)(nil)
 
 func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, maxTables, maxAggrs, maxGBs int, schemaTables []tableT) *queryGen {
 	if maxTables <= 0 {
@@ -173,7 +173,7 @@ func (qg *queryGen) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConf
 	return &sqlparser.Subquery{Select: newQG.sel}
 }
 
-func (qg *queryGen) IQueryGenerator() {}
+func (qg *queryGen) IsQueryGenerator() {}
 
 func (qg *queryGen) randomQuery() {
 	// make sure the random expressions can generally not contain aggregates; change appropriately
@@ -495,7 +495,7 @@ func (qg *queryGen) createRandomExprs(minExprs, maxExprs int, generators ...sqlp
 // getRandomExpr returns a random expression
 func (qg *queryGen) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlparser.Expr {
 	g := sqlparser.NewGenerator(qg.r, 2, generators...)
-	return g.Expression(qg.genConfig)
+	return g.Expression(qg.genConfig.SingleRowConfig().SetNumCols(1))
 }
 
 // creates sel.Limit
diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index 556bf6cccac..36e9aa3eada 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -49,7 +49,7 @@ func TestSeed(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	seed := int64(1689672658271399000)
+	seed := int64(1689757943775102000)
 	genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
 	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
 	qg.randomQuery()
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index fe1be048b35..b33871a62d8 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -22,12 +22,23 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/test/endtoend/utils"
+
 	"vitess.io/vitess/go/slices2"
 	"vitess.io/vitess/go/vt/sqlparser"
 )
 
 // This test tests that generating random expressions with a schema does not panic
 func TestRandomExprWithTables(t *testing.T) {
+	//t.Skip("Skip CI")
+	mcmp, closer := start(t)
+	defer closer()
+
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
+
 	// specify the schema (that is defined in schema.sql)
 	schemaTables := []tableT{
 		{tableExpr: sqlparser.NewTableName("emp")},
@@ -49,11 +60,19 @@ func TestRandomExprWithTables(t *testing.T) {
 		{name: "loc", typ: "varchar"},
 	}...)
 
-	seed := time.Now().UnixNano()
-	r := rand.New(rand.NewSource(seed))
-	g := sqlparser.NewGenerator(r, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 	for i := 0; i < 100; i++ {
-		expr := g.Expression(sqlparser.ExprGeneratorConfig{AggrRule: sqlparser.IsAggregate})
+
+		seed := time.Now().UnixNano()
+		fmt.Printf("seed: %d\n", seed)
+
+		r := rand.New(rand.NewSource(seed))
+		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.IsAggregate, "", 0, false)
+		g := sqlparser.NewGenerator(r, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
+		expr := g.Expression(genConfig)
 		fmt.Println(sqlparser.String(expr))
+
+		from := sqlparser.TableExprs{sqlparser.NewAliasedTableExpr(sqlparser.NewTableName("emp"), ""), sqlparser.NewAliasedTableExpr(sqlparser.NewTableName("dept"), "")}
+		query := sqlparser.NewSelect(nil, sqlparser.SelectExprs{sqlparser.NewAliasedExpr(expr, "")}, nil, nil, from, nil, nil, nil, nil)
+		mcmp.ExecAllowAndCompareError(sqlparser.String(query))
 	}
 }
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index aecd15a676a..2953389b0ae 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -147,7 +147,9 @@ func TestMustFix(t *testing.T) {
 	// swapping tables and predicates and changing to left fails
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal")
 
-	// EOF
+	// Passes with different errors
+	// vitess error: EOF
+	// mysql error: Operand should contain 1 column(s)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ 8 < -31 xor (-29, sum((tbl0.deptno, 'wren', 'ostrich')), max(distinct (tbl0.dname, -15, -8))) in ((sum(distinct (tbl0.dname, 'bengal', -10)), 'ant', true)) as caggr0 from dept as tbl0 where tbl0.deptno * (77 - 61)")
 
 	// EOF
@@ -166,6 +168,10 @@ func TestKnownFailures(t *testing.T) {
 	// logs more stuff
 	//clusterInstance.EnableGeneralLog()
 
+	// vitess error: <nil>
+	// mysql error: Operand should contain 1 column(s)
+	helperTest(t, "select (count('sheepdog') ^ (-71 % sum(emp.mgr) ^ count('koi')) and count(*), 'fly') from emp, dept")
+
 	// rhs of an In operation should be a tuple
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ (case when true then min(distinct tbl1.job) else 'bee' end, 'molly') not in (('dane', 0)) as caggr1 from emp as tbl0, emp as tbl1")
 
@@ -249,6 +255,10 @@ func TestKnownFailures(t *testing.T) {
 	// unsupported
 	// VT12001: unsupported: LEFT JOIN with derived tables
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ -1 as crandom0 from emp as tbl2 left join (select count(*) from dept as tbl1) as tbl3 on 6 != tbl2.deptno")
+
+	// unsupported
+	// VT12001: unsupported: subqueries in GROUP BY
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ exists (select 1) as crandom0 from dept as tbl0 group by exists (select 1)")
 }
 
 func TestRandom(t *testing.T) {
diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go
index 9b1b9381567..57c73348743 100644
--- a/go/vt/sqlparser/ast_funcs.go
+++ b/go/vt/sqlparser/ast_funcs.go
@@ -641,6 +641,19 @@ func NewTableNameWithQualifier(name, qualifier string) TableName {
 	}
 }
 
+// NewSubquery makes a new Subquery
+func NewSubquery(selectStatement SelectStatement) *Subquery {
+	return &Subquery{Select: selectStatement}
+}
+
+// NewDerivedTable makes a new DerivedTable
+func NewDerivedTable(lateral bool, selectStatement SelectStatement) *DerivedTable {
+	return &DerivedTable{
+		Lateral: lateral,
+		Select:  selectStatement,
+	}
+}
+
 // NewAliasedTableExpr makes a new AliasedTableExpr with an alias
 func NewAliasedTableExpr(simpleTableExpr SimpleTableExpr, alias string) *AliasedTableExpr {
 	return &AliasedTableExpr{
@@ -687,6 +700,11 @@ func NewOrder(expr Expr, direction OrderDirection) *Order {
 	}
 }
 
+// NewNotExpr makes a new NotExpr
+func NewNotExpr(expr Expr) *NotExpr {
+	return &NotExpr{Expr: expr}
+}
+
 // NewComparisonExpr makes a new ComparisonExpr
 func NewComparisonExpr(operator ComparisonExprOperator, left, right, escape Expr) *ComparisonExpr {
 	return &ComparisonExpr{
@@ -697,6 +715,11 @@ func NewComparisonExpr(operator ComparisonExprOperator, left, right, escape Expr
 	}
 }
 
+// NewExistsExpr makes a new ExistsExpr
+func NewExistsExpr(subquery *Subquery) *ExistsExpr {
+	return &ExistsExpr{Subquery: subquery}
+}
+
 // NewLimit makes a new Limit
 func NewLimit(offset, rowCount int) *Limit {
 	return &Limit{
@@ -722,14 +745,6 @@ func NewLimitWithoutOffset(rowCount int) *Limit {
 	}
 }
 
-// NewDerivedTable makes a new DerivedTable
-func NewDerivedTable(lateral bool, selectStatement SelectStatement) *DerivedTable {
-	return &DerivedTable{
-		Lateral: lateral,
-		Select:  selectStatement,
-	}
-}
-
 // NewSelect is used to create a select statement
 func NewSelect(comments Comments, exprs SelectExprs, selectOptions []string, into *SelectInto, from TableExprs, where *Where, groupBy GroupBy, having *Where, windows NamedWindows) *Select {
 	var cache *bool
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 74c82c64ca0..3dd7c57fb0a 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -36,17 +36,17 @@ type (
 	}
 
 	QueryGenerator interface {
-		IQueryGenerator()
+		IsQueryGenerator()
 		ExprGenerator
 	}
 
-	AggregateRule int8
+	AggregateRule  int8
+	AggregateState int8
 
 	ExprGeneratorConfig struct {
 		// AggrRule determines if the random expression can, cannot, or must be an aggregation expression
 		AggrRule AggregateRule
-		// TODO: get rid of ExprGeneratorConfig Type
-		Type string
+		Type     string
 		// MaxCols = 0 indicates no limit
 		NumCols int
 		// SingleRow indicates that the query must have at most one row
@@ -71,17 +71,17 @@ func NewExprGeneratorConfig(aggrRule AggregateRule, typ string, numCols int, sin
 	}
 }
 
-func (egc ExprGeneratorConfig) singleRowConfig() ExprGeneratorConfig {
+func (egc ExprGeneratorConfig) SingleRowConfig() ExprGeneratorConfig {
 	egc.SingleRow = true
 	return egc
 }
 
-func (egc ExprGeneratorConfig) multiRowConfig() ExprGeneratorConfig {
+func (egc ExprGeneratorConfig) MultiRowConfig() ExprGeneratorConfig {
 	egc.SingleRow = false
 	return egc
 }
 
-func (egc ExprGeneratorConfig) setNumCols(numCols int) ExprGeneratorConfig {
+func (egc ExprGeneratorConfig) SetNumCols(numCols int) ExprGeneratorConfig {
 	egc.NumCols = numCols
 	return egc
 }
@@ -151,6 +151,7 @@ func (g *Generator) atMaxDepth() bool {
 	    - AND/OR/NOT
 	    - string literals, numeric literals (-/+ 1000)
 		- columns of types bigint and varchar
+		- scalar and tuple subqueries
 	    - =, >, <, >=, <=, <=>, !=
 		- &, |, ^, +, -, *, /, div, %, <<, >>
 	    - IN, BETWEEN and CASE
@@ -161,20 +162,20 @@ Note: It's important to update this method so that it produces all expressions t
 It's currently missing function calls and string operators
 */
 func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
-	var exprOptions, tupleOptions []exprF
+	var options []exprF
 	// this will only be used for tuple expressions, everything else will need genConfig.NumCols = 1
 	numCols := genConfig.NumCols
-	genConfig = genConfig.setNumCols(1)
+	genConfig = genConfig.SetNumCols(1)
 
 	switch genConfig.Type {
 	case "bigint":
-		exprOptions = append(exprOptions, func() Expr { return g.intExpr(genConfig) })
+		options = append(options, func() Expr { return g.intExpr(genConfig) })
 	case "varchar":
-		exprOptions = append(exprOptions, func() Expr { return g.stringExpr(genConfig) })
+		options = append(options, func() Expr { return g.stringExpr(genConfig) })
 	case "tinyint":
-		exprOptions = append(exprOptions, func() Expr { return g.booleanExpr(genConfig) })
+		options = append(options, func() Expr { return g.booleanExpr(genConfig) })
 	case "":
-		exprOptions = append(exprOptions, []exprF{
+		options = append(options, []exprF{
 			func() Expr { return g.intExpr(genConfig) },
 			func() Expr { return g.stringExpr(genConfig) },
 			func() Expr { return g.booleanExpr(genConfig) },
@@ -187,18 +188,9 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 			continue
 		}
 
-		// Only generate subqueries with multiple rows when we require more than 1 column
-		if qg, ok := generator.(QueryGenerator); ok && genConfig.NumCols >= 2 {
-			tupleOptions = append(tupleOptions, func() Expr {
-				expr := qg.Generate(g.r, genConfig.setNumCols(numCols))
-				if expr == nil {
-					return g.randomTupleLiteral(numCols)
-				}
-				return expr
-			})
-			// don't create expressions from the expression exprGenerators if we haven't created an aggregation yet
-		} else if genConfig.AggrRule != IsAggregate {
-			exprOptions = append(exprOptions, func() Expr {
+		// don't create expressions from the expression exprGenerators if we haven't created an aggregation yet
+		if _, ok := generator.(QueryGenerator); ok || genConfig.AggrRule != IsAggregate {
+			options = append(options, func() Expr {
 				expr := generator.Generate(g.r, genConfig)
 				if expr == nil {
 					return g.randomLiteral()
@@ -209,25 +201,29 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 	}
 
 	if genConfig.AggrRule != CannotAggregate {
-		exprOptions = append(exprOptions, func() Expr {
+		options = append(options, func() Expr {
 			g.isAggregate = true
 			return g.randomAggregate(genConfig.CannotAggregateConfig())
 		})
 	}
 
-	// tupleOptions will be nil in this case
-	if numCols <= 1 {
-		return g.makeAggregateIfNecessary(genConfig, g.randomOf(exprOptions))
+	if numCols == 1 {
+		return g.makeAggregateIfNecessary(genConfig, g.randomOf(options))
 	}
 
 	// with 1/5 probability choose a tuple subquery
-	if g.randomBool(0.2) && len(tupleOptions) > 0 {
-		return g.randomOf(tupleOptions)
+	if g.randomBool(0.2) {
+		return g.subqueryExpr(genConfig.SetNumCols(numCols))
+	}
+
+	// if an arbitrary number of columns may be generated, randomly choose 1-3 columns
+	if numCols == 0 {
+		numCols = g.r.Intn(3) + 1
 	}
 
 	tuple := ValTuple{}
 	for i := 0; i < numCols; i++ {
-		tuple = append(tuple, g.makeAggregateIfNecessary(genConfig, g.randomOf(exprOptions)))
+		tuple = append(tuple, g.makeAggregateIfNecessary(genConfig, g.randomOf(options)))
 	}
 
 	return tuple
@@ -249,7 +245,7 @@ func (g *Generator) makeAggregateIfNecessary(genConfig ExprGeneratorConfig, expr
 }
 
 func (g *Generator) randomAggregate(genConfig ExprGeneratorConfig) Expr {
-	isDistinct := g.r.Intn(2) < 1
+	isDistinct := g.r.Intn(10) < 1
 
 	options := []exprF{
 		func() Expr { return &CountStar{} },
@@ -259,6 +255,7 @@ func (g *Generator) randomAggregate(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return &Max{Arg: g.Expression(genConfig), Distinct: isDistinct} },
 	}
 
+	g.isAggregate = true
 	return g.randomOf(options)
 }
 
@@ -277,6 +274,7 @@ func (g *Generator) booleanExpr(genConfig ExprGeneratorConfig) Expr {
 		func() Expr { return g.comparison(genConfig.stringTypeConfig()) },
 		//func() Expr { return g.comparison(genConfig) }, // this is not accepted by the parser
 		func() Expr { return g.inExpr(genConfig) },
+		func() Expr { return g.existsExpr(genConfig) },
 		func() Expr { return g.between(genConfig.intTypeConfig()) },
 		func() Expr { return g.isExpr(genConfig) },
 		func() Expr { return g.notExpr(genConfig) },
@@ -317,6 +315,28 @@ func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 	return g.randomOf(options)
 }
 
+func (g *Generator) subqueryExpr(genConfig ExprGeneratorConfig) Expr {
+	var options []exprF
+
+	for _, generator := range g.exprGenerators {
+		if qg, ok := generator.(QueryGenerator); ok {
+			options = append(options, func() Expr {
+				expr := qg.Generate(g.r, genConfig)
+				if expr == nil {
+					return g.randomTupleLiteral(genConfig.NumCols)
+				}
+				return expr
+			})
+		}
+	}
+
+	if len(options) == 0 {
+		return g.makeAggregateIfNecessary(genConfig, g.randomTupleLiteral(genConfig.NumCols))
+	}
+
+	return g.randomOf(options)
+}
+
 func (g *Generator) randomTupleLiteral(len int) Expr {
 	tuple := ValTuple{}
 	for i := 0; i < len; i++ {
@@ -530,3 +550,23 @@ func (g *Generator) isExpr(genConfig ExprGeneratorConfig) Expr {
 		Left:  g.Expression(genConfig),
 	}
 }
+
+func (g *Generator) existsExpr(genConfig ExprGeneratorConfig) Expr {
+	expr := g.subqueryExpr(genConfig.MultiRowConfig().SetNumCols(0))
+	if subquery, ok := expr.(*Subquery); ok {
+		expr = NewExistsExpr(subquery)
+	} else {
+		// if g.subqueryExpr doesn't return a valid subquery, replace with
+		// select 1
+		selectExprs := SelectExprs{NewAliasedExpr(NewIntLiteral("1"), "")}
+		from := TableExprs{NewAliasedTableExpr(NewTableName("dual"), "")}
+		expr = NewExistsExpr(NewSubquery(NewSelect(nil, selectExprs, nil, nil, from, nil, nil, nil, nil)))
+	}
+
+	// not exists
+	if g.randomBool(0.5) {
+		expr = NewNotExpr(expr)
+	}
+
+	return expr
+}

From 35e045b0bbd3c5edae0839ec748c538f71d894e1 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Thu, 20 Jul 2023 01:01:38 -0700
Subject: [PATCH 14/27] added union queries in fuzzer

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 313 ++++++++++--------
 .../vtgate/queries/random/query_gen_test.go   |   4 +-
 .../vtgate/queries/random/random_expr_test.go |   2 +-
 .../vtgate/queries/random/random_test.go      |   6 +-
 go/vt/sqlparser/random_expr.go                |  15 +-
 5 files changed, 197 insertions(+), 143 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 69575406d7b..f6810856fb8 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -34,7 +34,7 @@ import (
 const testFailingQueries = false
 
 type (
-	queryGen struct {
+	selectGenerator struct {
 		r            *rand.Rand
 		genConfig    sqlparser.ExprGeneratorConfig
 		maxTables    int
@@ -43,6 +43,10 @@ type (
 		schemaTables []tableT
 		sel          *sqlparser.Select
 	}
+	queryGenerator struct {
+		stmt   sqlparser.SelectStatement
+		selGen *selectGenerator
+	}
 	column struct {
 		name string
 		// TODO: perhaps remove tableName and always pass columns through a tableT
@@ -62,14 +66,21 @@ type (
 
 var _ sqlparser.ExprGenerator = (*tableT)(nil)
 var _ sqlparser.ExprGenerator = (*column)(nil)
-var _ sqlparser.QueryGenerator = (*queryGen)(nil)
+var _ sqlparser.QueryGenerator = (*selectGenerator)(nil)
+var _ sqlparser.QueryGenerator = (*queryGenerator)(nil)
+
+func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, maxTables, maxAggrs, maxGBs int, schemaTables []tableT) *queryGenerator {
+	return &queryGenerator{
+		selGen: newSelectGenerator(r, genConfig, maxTables, maxAggrs, maxGBs, schemaTables),
+	}
+}
 
-func newQueryGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, maxTables, maxAggrs, maxGBs int, schemaTables []tableT) *queryGen {
+func newSelectGenerator(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig, maxTables, maxAggrs, maxGBs int, schemaTables []tableT) *selectGenerator {
 	if maxTables <= 0 {
 		log.Fatalf("maxTables must be at least 1, currently %d\n", maxTables)
 	}
 
-	return &queryGen{
+	return &selectGenerator{
 		r:            r,
 		genConfig:    genConfig,
 		maxTables:    maxTables,
@@ -160,40 +171,82 @@ func (t *tableT) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig)
 	return nil
 }
 
+// Generate generates a subquery based on sg
+func (sg *selectGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
+	var schemaTablesCopy []tableT
+	for _, tbl := range sg.schemaTables {
+		schemaTablesCopy = append(schemaTablesCopy, *tbl.clone())
+	}
+
+	newSG := newQueryGenerator(r, genConfig, sg.maxTables, sg.maxAggrs, sg.maxGBs, schemaTablesCopy)
+	newSG.randomQuery()
+
+	return &sqlparser.Subquery{Select: newSG.selGen.sel}
+}
+
+func (sg *selectGenerator) IsQueryGenerator() {}
+
 // Generate generates a subquery based on qg
-func (qg *queryGen) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
+func (qg *queryGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	var schemaTablesCopy []tableT
-	for _, tbl := range qg.schemaTables {
+	for _, tbl := range qg.selGen.schemaTables {
 		schemaTablesCopy = append(schemaTablesCopy, *tbl.clone())
 	}
 
-	newQG := newQueryGenerator(r, genConfig, qg.maxTables, qg.maxAggrs, qg.maxGBs, schemaTablesCopy)
+	newQG := newQueryGenerator(r, genConfig, qg.selGen.maxTables, qg.selGen.maxAggrs, qg.selGen.maxGBs, schemaTablesCopy)
 	newQG.randomQuery()
 
-	return &sqlparser.Subquery{Select: newQG.sel}
+	return &sqlparser.Subquery{Select: newQG.stmt}
 }
 
-func (qg *queryGen) IsQueryGenerator() {}
+func (qg *queryGenerator) IsQueryGenerator() {}
+
+func (qg *queryGenerator) randomQuery() {
+	if qg.selGen.r.Intn(10) < 1 && testFailingQueries {
+		qg.createUnion()
+	} else {
+		qg.selGen.randomSelect()
+		qg.stmt = qg.selGen.sel
+	}
+}
+
+func (qg *queryGenerator) createUnion() {
+	union := &sqlparser.Union{}
+
+	if qg.selGen.r.Intn(2) < 1 {
+		union.Distinct = true
+	}
+
+	// specify between 1-4 columns
+	qg.selGen.genConfig.NumCols = qg.selGen.r.Intn(4) + 1
+
+	qg.randomQuery()
+	union.Left = qg.stmt
+	qg.randomQuery()
+	union.Right = qg.stmt
+
+	qg.stmt = union
+}
 
-func (qg *queryGen) randomQuery() {
+func (sg *selectGenerator) randomSelect() {
 	// make sure the random expressions can generally not contain aggregates; change appropriately
-	qg.genConfig = qg.genConfig.CannotAggregateConfig()
+	sg.genConfig = sg.genConfig.CannotAggregateConfig()
 
-	qg.sel = &sqlparser.Select{}
-	qg.sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"})
+	sg.sel = &sqlparser.Select{}
+	sg.sel.SetComments(sqlparser.Comments{"/*vt+ PLANNER=Gen4 */"})
 
 	// select distinct (fails with group by bigint)
-	isDistinct := qg.r.Intn(2) < 1
+	isDistinct := sg.r.Intn(2) < 1
 	if isDistinct {
-		qg.sel.MakeDistinct()
+		sg.sel.MakeDistinct()
 	}
 
 	// create both tables and join at the same time since both occupy the from clause
-	tables, isJoin := qg.createTablesAndJoin()
+	tables, isJoin := sg.createTablesAndJoin()
 
 	// canAggregate determines if the query will have
 	// aggregate columns, group by, and having
-	canAggregate := qg.r.Intn(4) < 3
+	canAggregate := sg.r.Intn(4) < 3
 
 	var (
 		grouping, aggregates []column
@@ -203,23 +256,23 @@ func (qg *queryGen) randomQuery() {
 	if canAggregate {
 		if testFailingQueries || !isDistinct {
 			// group by
-			if !qg.genConfig.SingleRow {
-				grouping = qg.createGroupBy(tables)
+			if !sg.genConfig.SingleRow {
+				grouping = sg.createGroupBy(tables)
 			}
 		}
 
 		// having
-		isHaving := qg.r.Intn(2) < 1
+		isHaving := sg.r.Intn(2) < 1
 		if isHaving && testFailingQueries {
-			qg.createHavingPredicates(grouping)
+			sg.createHavingPredicates(grouping)
 		}
 
 		// alias the grouping columns
 		// TODO: alias the grouping columns after having because vitess doesn't recognize aliases in the HAVING
-		grouping = qg.aliasGroupingColumns(grouping)
+		grouping = sg.aliasGroupingColumns(grouping)
 
 		// aggregation columns
-		aggregates = qg.createAggregations(tables)
+		aggregates = sg.createAggregations(tables)
 
 		// add the grouping and aggregation to newTable
 		newTable.addColumns(grouping...)
@@ -227,29 +280,29 @@ func (qg *queryGen) randomQuery() {
 	}
 
 	// where
-	qg.createWherePredicates(tables)
+	sg.createWherePredicates(tables)
 
 	var f func() sqlparser.Expr
 	// add random expression to select
 	// TODO: random expressions cause a lot of failures
-	isRandomExpr := qg.r.Intn(2) < 1 && testFailingQueries
+	isRandomExpr := sg.r.Intn(2) < 1 && testFailingQueries
 
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
 	if canAggregate && testFailingQueries {
 		exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 		// add scalar subqueries
-		if qg.r.Intn(10) < 1 && testFailingQueries {
-			exprGenerators = append(exprGenerators, qg)
+		if sg.r.Intn(10) < 1 && testFailingQueries {
+			exprGenerators = append(exprGenerators, sg)
 		}
 
-		f = func() sqlparser.Expr { return qg.getRandomExpr(exprGenerators...) }
+		f = func() sqlparser.Expr { return sg.getRandomExpr(exprGenerators...) }
 	} else {
-		f = func() sqlparser.Expr { return qg.getRandomExpr() }
+		f = func() sqlparser.Expr { return sg.getRandomExpr() }
 	}
 
 	// make sure we have at least one select expression
-	for isRandomExpr || len(qg.sel.SelectExprs) == 0 {
+	for isRandomExpr || len(sg.sel.SelectExprs) == 0 {
 		// TODO: if the random expression is an int literal,
 		// TODO: and if the query is (potentially) an aggregate query,
 		// TODO: then we must group by the random expression,
@@ -264,15 +317,15 @@ func (qg *queryGen) randomQuery() {
 		}
 
 		// TODO: select distinct [literal] fails
-		qg.sel.Distinct = false
+		sg.sel.Distinct = false
 
 		// alias randomly
-		col := qg.randomlyAlias(randomExpr, "crandom0")
+		col := sg.randomlyAlias(randomExpr, "crandom0")
 		newTable.addColumns(col)
 
 		// make sure to add the random expression to group by for only_full_group_by
 		if canAggregate {
-			qg.sel.AddGroupBy(randomExpr)
+			sg.sel.AddGroupBy(randomExpr)
 		}
 
 		break
@@ -280,57 +333,57 @@ func (qg *queryGen) randomQuery() {
 
 	// can add both aggregate and grouping columns to order by
 	// TODO: order fails with distinct and outer joins
-	isOrdered := qg.r.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries)
-	if isOrdered || (!canAggregate && qg.genConfig.SingleRow) /* TODO: might be redundant */ {
-		qg.createOrderBy()
+	isOrdered := sg.r.Intn(2) < 1 && (!isDistinct || testFailingQueries) && (!isJoin || testFailingQueries)
+	if isOrdered || (!canAggregate && sg.genConfig.SingleRow) /* TODO: might be redundant */ {
+		sg.createOrderBy()
 	}
 
 	// only add a limit if there is an ordering
 	// TODO: limit fails a lot
-	isLimit := qg.r.Intn(2) < 1 && len(qg.sel.OrderBy) > 0 && testFailingQueries
-	if isLimit || (!canAggregate && qg.genConfig.SingleRow) /* TODO: might be redundant */ {
-		qg.createLimit()
+	isLimit := sg.r.Intn(2) < 1 && len(sg.sel.OrderBy) > 0 && testFailingQueries
+	if isLimit || (!canAggregate && sg.genConfig.SingleRow) /* TODO: might be redundant */ {
+		sg.createLimit()
 	}
 
-	newTable = qg.matchNumCols(tables, newTable, canAggregate)
+	newTable = sg.matchNumCols(tables, newTable, canAggregate)
 
 	// add new table to schemaTables
-	newTable.tableExpr = sqlparser.NewDerivedTable(false, qg.sel)
-	qg.schemaTables = append(qg.schemaTables, newTable)
+	newTable.tableExpr = sqlparser.NewDerivedTable(false, sg.sel)
+	sg.schemaTables = append(sg.schemaTables, newTable)
 
 	// derived tables (partially unsupported)
 	// TODO: derived tables fails a lot
-	if qg.r.Intn(10) < 1 {
-		qg.randomQuery()
+	if sg.r.Intn(10) < 1 {
+		sg.randomSelect()
 	}
 }
 
-func (qg *queryGen) createTablesAndJoin() ([]tableT, bool) {
+func (sg *selectGenerator) createTablesAndJoin() ([]tableT, bool) {
 	var tables []tableT
 	// add at least one of original emp/dept tables for now because derived tables have nil columns
-	tables = append(tables, qg.schemaTables[qg.r.Intn(2)])
+	tables = append(tables, sg.schemaTables[sg.r.Intn(2)])
 
 	tables[0].setAlias("tbl0")
-	qg.sel.From = append(qg.sel.From, newAliasedTable(tables[0], "tbl0"))
+	sg.sel.From = append(sg.sel.From, newAliasedTable(tables[0], "tbl0"))
 
-	numTables := qg.r.Intn(qg.maxTables)
+	numTables := sg.r.Intn(sg.maxTables)
 	for i := 0; i < numTables; i++ {
-		tables = append(tables, randomEl(qg.r, qg.schemaTables))
+		tables = append(tables, randomEl(sg.r, sg.schemaTables))
 		alias := fmt.Sprintf("tbl%d", i+1)
-		qg.sel.From = append(qg.sel.From, newAliasedTable(tables[i+1], alias))
+		sg.sel.From = append(sg.sel.From, newAliasedTable(tables[i+1], alias))
 		tables[i+1].setAlias(alias)
 	}
 
 	// TODO: outer joins produce mismatched results
-	isJoin := qg.r.Intn(2) < 1 && testFailingQueries
+	isJoin := sg.r.Intn(2) < 1 && testFailingQueries
 	if isJoin {
 		// TODO: do nested joins
-		newTable := randomEl(qg.r, qg.schemaTables)
+		newTable := randomEl(sg.r, sg.schemaTables)
 		alias := fmt.Sprintf("tbl%d", numTables+1)
 		newTable.setAlias(alias)
 		tables = append(tables, newTable)
 
-		qg.createJoin(tables)
+		sg.createJoin(tables)
 	}
 
 	return tables, isJoin
@@ -338,52 +391,52 @@ func (qg *queryGen) createTablesAndJoin() ([]tableT, bool) {
 
 // creates a left join (without the condition) between the last table in sel and newTable
 // tables should have one more table than sel
-func (qg *queryGen) createJoin(tables []tableT) {
-	n := len(qg.sel.From)
+func (sg *selectGenerator) createJoin(tables []tableT) {
+	n := len(sg.sel.From)
 	if len(tables) != n+1 {
-		log.Fatalf("sel has %d tables and tables has %d tables", len(qg.sel.From), n)
+		log.Fatalf("sel has %d tables and tables has %d tables", len(sg.sel.From), n)
 	}
 
-	joinPredicate := sqlparser.AndExpressions(qg.createJoinPredicates(tables)...)
+	joinPredicate := sqlparser.AndExpressions(sg.createJoinPredicates(tables)...)
 	joinCondition := sqlparser.NewJoinCondition(joinPredicate, nil)
 	newTable := newAliasedTable(tables[n], fmt.Sprintf("tbl%d", n))
-	qg.sel.From[n-1] = sqlparser.NewJoinTableExpr(qg.sel.From[n-1], getRandomJoinType(qg.r), newTable, joinCondition)
+	sg.sel.From[n-1] = sqlparser.NewJoinTableExpr(sg.sel.From[n-1], getRandomJoinType(sg.r), newTable, joinCondition)
 }
 
 // returns 1-3 random expressions based on the last two elements of tables
 // tables should have at least two elements
-func (qg *queryGen) createJoinPredicates(tables []tableT) sqlparser.Exprs {
+func (sg *selectGenerator) createJoinPredicates(tables []tableT) sqlparser.Exprs {
 	if len(tables) < 2 {
 		log.Fatalf("tables has %d elements, needs at least 2", len(tables))
 	}
 
 	exprGenerators := []sqlparser.ExprGenerator{&tables[len(tables)-2], &tables[len(tables)-1]}
 	// add scalar subqueries
-	if qg.r.Intn(10) < 1 && testFailingQueries {
-		exprGenerators = append(exprGenerators, qg)
+	if sg.r.Intn(10) < 1 && testFailingQueries {
+		exprGenerators = append(exprGenerators, sg)
 	}
 
-	return qg.createRandomExprs(1, 3, exprGenerators...)
+	return sg.createRandomExprs(1, 3, exprGenerators...)
 }
 
 // returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
-func (qg *queryGen) createGroupBy(tables []tableT) (grouping []column) {
-	if qg.maxGBs <= 0 {
+func (sg *selectGenerator) createGroupBy(tables []tableT) (grouping []column) {
+	if sg.maxGBs <= 0 {
 		return
 	}
-	numGBs := qg.r.Intn(qg.maxGBs + 1)
+	numGBs := sg.r.Intn(sg.maxGBs + 1)
 	for i := 0; i < numGBs; i++ {
-		tblIdx := qg.r.Intn(len(tables))
-		col := randomEl(qg.r, tables[tblIdx].cols)
+		tblIdx := sg.r.Intn(len(tables))
+		col := randomEl(sg.r, tables[tblIdx].cols)
 		// TODO: grouping by a date column sometimes errors
 		if col.typ == "date" && !testFailingQueries {
 			continue
 		}
-		qg.sel.GroupBy = append(qg.sel.GroupBy, col.getASTExpr())
+		sg.sel.GroupBy = append(sg.sel.GroupBy, col.getASTExpr())
 
 		// add to select
-		if qg.r.Intn(2) < 1 {
-			qg.sel.SelectExprs = append(qg.sel.SelectExprs, newAliasedColumn(col, ""))
+		if sg.r.Intn(2) < 1 {
+			sg.sel.SelectExprs = append(sg.sel.SelectExprs, newAliasedColumn(col, ""))
 			grouping = append(grouping, col)
 		}
 	}
@@ -392,14 +445,14 @@ func (qg *queryGen) createGroupBy(tables []tableT) (grouping []column) {
 }
 
 // aliasGroupingColumns randomly aliases the grouping columns in the SelectExprs
-func (qg *queryGen) aliasGroupingColumns(grouping []column) []column {
-	if len(grouping) != len(qg.sel.SelectExprs) {
-		log.Fatalf("grouping (length: %d) and qg.sel.SelectExprs (length: %d) should have the same length at this point", len(grouping), len(qg.sel.SelectExprs))
+func (sg *selectGenerator) aliasGroupingColumns(grouping []column) []column {
+	if len(grouping) != len(sg.sel.SelectExprs) {
+		log.Fatalf("grouping (length: %d) and sg.sel.SelectExprs (length: %d) should have the same length at this point", len(grouping), len(sg.sel.SelectExprs))
 	}
 
 	for i := range grouping {
-		if qg.r.Intn(2) < 1 {
-			if aliasedExpr, ok := qg.sel.SelectExprs[i].(*sqlparser.AliasedExpr); ok {
+		if sg.r.Intn(2) < 1 {
+			if aliasedExpr, ok := sg.sel.SelectExprs[i].(*sqlparser.AliasedExpr); ok {
 				alias := fmt.Sprintf("cgroup%d", i)
 				aliasedExpr.SetAlias(alias)
 				grouping[i].name = alias
@@ -411,19 +464,19 @@ func (qg *queryGen) aliasGroupingColumns(grouping []column) []column {
 }
 
 // returns the aggregation columns as three types: sqlparser.SelectExprs, []column
-func (qg *queryGen) createAggregations(tables []tableT) (aggregates []column) {
+func (sg *selectGenerator) createAggregations(tables []tableT) (aggregates []column) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 	// add scalar subqueries
-	if qg.r.Intn(10) < 1 && testFailingQueries {
-		exprGenerators = append(exprGenerators, qg)
+	if sg.r.Intn(10) < 1 && testFailingQueries {
+		exprGenerators = append(exprGenerators, sg)
 	}
 
-	qg.genConfig = qg.genConfig.IsAggregateConfig()
-	aggrExprs := qg.createRandomExprs(0, qg.maxAggrs, exprGenerators...)
-	qg.genConfig = qg.genConfig.CannotAggregateConfig()
+	sg.genConfig = sg.genConfig.IsAggregateConfig()
+	aggrExprs := sg.createRandomExprs(0, sg.maxAggrs, exprGenerators...)
+	sg.genConfig = sg.genConfig.CannotAggregateConfig()
 
 	for i, expr := range aggrExprs {
-		col := qg.randomlyAlias(expr, fmt.Sprintf("caggr%d", i))
+		col := sg.randomlyAlias(expr, fmt.Sprintf("caggr%d", i))
 		aggregates = append(aggregates, col)
 	}
 
@@ -431,129 +484,129 @@ func (qg *queryGen) createAggregations(tables []tableT) (aggregates []column) {
 }
 
 // orders on all grouping expressions and on random SelectExprs
-func (qg *queryGen) createOrderBy() {
+func (sg *selectGenerator) createOrderBy() {
 	// always order on grouping expressions
-	for _, expr := range qg.sel.GroupBy {
-		qg.sel.OrderBy = append(qg.sel.OrderBy, sqlparser.NewOrder(expr, getRandomOrderDirection(qg.r)))
+	for _, expr := range sg.sel.GroupBy {
+		sg.sel.OrderBy = append(sg.sel.OrderBy, sqlparser.NewOrder(expr, getRandomOrderDirection(sg.r)))
 	}
 
 	// randomly order on SelectExprs
-	for _, selExpr := range qg.sel.SelectExprs {
-		if aliasedExpr, ok := selExpr.(*sqlparser.AliasedExpr); ok && qg.r.Intn(2) < 1 {
+	for _, selExpr := range sg.sel.SelectExprs {
+		if aliasedExpr, ok := selExpr.(*sqlparser.AliasedExpr); ok && sg.r.Intn(2) < 1 {
 			literal, ok := aliasedExpr.Expr.(*sqlparser.Literal)
 			isIntLiteral := ok && literal.Type == sqlparser.IntVal
 			if isIntLiteral {
 				continue
 			}
-			qg.sel.OrderBy = append(qg.sel.OrderBy, sqlparser.NewOrder(aliasedExpr.Expr, getRandomOrderDirection(qg.r)))
+			sg.sel.OrderBy = append(sg.sel.OrderBy, sqlparser.NewOrder(aliasedExpr.Expr, getRandomOrderDirection(sg.r)))
 		}
 	}
 }
 
 // returns 0-2 random expressions based on tables
-func (qg *queryGen) createWherePredicates(tables []tableT) {
+func (sg *selectGenerator) createWherePredicates(tables []tableT) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 	// add scalar subqueries
-	if qg.r.Intn(10) < 1 && testFailingQueries {
-		exprGenerators = append(exprGenerators, qg)
+	if sg.r.Intn(10) < 1 && testFailingQueries {
+		exprGenerators = append(exprGenerators, sg)
 	}
 
-	predicates := qg.createRandomExprs(0, 2, exprGenerators...)
-	qg.sel.AddWhere(sqlparser.AndExpressions(predicates...))
+	predicates := sg.createRandomExprs(0, 2, exprGenerators...)
+	sg.sel.AddWhere(sqlparser.AndExpressions(predicates...))
 }
 
 // creates predicates for the having clause comparing a column to a random expression
-func (qg *queryGen) createHavingPredicates(grouping []column) {
+func (sg *selectGenerator) createHavingPredicates(grouping []column) {
 	exprGenerators := slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })
 	// add scalar subqueries
-	if qg.r.Intn(10) < 1 && testFailingQueries {
-		exprGenerators = append(exprGenerators, qg)
+	if sg.r.Intn(10) < 1 && testFailingQueries {
+		exprGenerators = append(exprGenerators, sg)
 	}
 
-	qg.genConfig = qg.genConfig.CanAggregateConfig()
-	predicates := qg.createRandomExprs(0, 2, exprGenerators...)
-	qg.genConfig = qg.genConfig.CannotAggregateConfig()
+	sg.genConfig = sg.genConfig.CanAggregateConfig()
+	predicates := sg.createRandomExprs(0, 2, exprGenerators...)
+	sg.genConfig = sg.genConfig.CannotAggregateConfig()
 
-	qg.sel.AddHaving(sqlparser.AndExpressions(predicates...))
+	sg.sel.AddHaving(sqlparser.AndExpressions(predicates...))
 }
 
 // returns between minExprs and maxExprs random expressions using generators
-func (qg *queryGen) createRandomExprs(minExprs, maxExprs int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
+func (sg *selectGenerator) createRandomExprs(minExprs, maxExprs int, generators ...sqlparser.ExprGenerator) (predicates sqlparser.Exprs) {
 	if minExprs > maxExprs {
 		log.Fatalf("minExprs is greater than maxExprs; minExprs: %d, maxExprs: %d\n", minExprs, maxExprs)
 	} else if maxExprs <= 0 {
 		return
 	}
-	numPredicates := qg.r.Intn(maxExprs-minExprs+1) + minExprs
+	numPredicates := sg.r.Intn(maxExprs-minExprs+1) + minExprs
 	for i := 0; i < numPredicates; i++ {
-		predicates = append(predicates, qg.getRandomExpr(generators...))
+		predicates = append(predicates, sg.getRandomExpr(generators...))
 	}
 
 	return
 }
 
 // getRandomExpr returns a random expression
-func (qg *queryGen) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlparser.Expr {
-	g := sqlparser.NewGenerator(qg.r, 2, generators...)
-	return g.Expression(qg.genConfig.SingleRowConfig().SetNumCols(1))
+func (sg *selectGenerator) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlparser.Expr {
+	g := sqlparser.NewGenerator(sg.r, 2, generators...)
+	return g.Expression(sg.genConfig.SingleRowConfig().SetNumCols(1))
 }
 
 // creates sel.Limit
-func (qg *queryGen) createLimit() {
-	if qg.genConfig.SingleRow {
-		qg.sel.Limit = sqlparser.NewLimitWithoutOffset(1)
+func (sg *selectGenerator) createLimit() {
+	if sg.genConfig.SingleRow {
+		sg.sel.Limit = sqlparser.NewLimitWithoutOffset(1)
 		return
 	}
 
-	limitNum := qg.r.Intn(10)
-	if qg.r.Intn(2) < 1 {
-		offset := qg.r.Intn(10)
-		qg.sel.Limit = sqlparser.NewLimit(offset, limitNum)
+	limitNum := sg.r.Intn(10)
+	if sg.r.Intn(2) < 1 {
+		offset := sg.r.Intn(10)
+		sg.sel.Limit = sqlparser.NewLimit(offset, limitNum)
 	} else {
-		qg.sel.Limit = sqlparser.NewLimitWithoutOffset(limitNum)
+		sg.sel.Limit = sqlparser.NewLimitWithoutOffset(limitNum)
 	}
 }
 
 // randomlyAlias randomly aliases expr with alias alias, adds it to sel.SelectExprs and returns the column created
-func (qg *queryGen) randomlyAlias(expr sqlparser.Expr, alias string) column {
+func (sg *selectGenerator) randomlyAlias(expr sqlparser.Expr, alias string) column {
 	var col column
-	if qg.r.Intn(2) < 1 {
+	if sg.r.Intn(2) < 1 {
 		alias = ""
 		col.name = sqlparser.String(expr)
 	} else {
 		col.name = alias
 	}
-	qg.sel.SelectExprs = append(qg.sel.SelectExprs, sqlparser.NewAliasedExpr(expr, alias))
+	sg.sel.SelectExprs = append(sg.sel.SelectExprs, sqlparser.NewAliasedExpr(expr, alias))
 
 	return col
 }
 
-// matchNumCols makes sure qg.sel.SelectExprs and newTable both have the same number of cols: qg.genConfig.NumCols
-func (qg *queryGen) matchNumCols(tables []tableT, newTable tableT, canAggregate bool) tableT {
-	// remove SelectExprs and newTable.cols randomly until there are qg.genConfig.NumCols amount
-	for len(qg.sel.SelectExprs) > qg.genConfig.NumCols && qg.genConfig.NumCols > 0 {
+// matchNumCols makes sure sg.sel.SelectExprs and newTable both have the same number of cols: sg.genConfig.NumCols
+func (sg *selectGenerator) matchNumCols(tables []tableT, newTable tableT, canAggregate bool) tableT {
+	// remove SelectExprs and newTable.cols randomly until there are sg.genConfig.NumCols amount
+	for len(sg.sel.SelectExprs) > sg.genConfig.NumCols && sg.genConfig.NumCols > 0 {
 		// select a random index and remove it from SelectExprs and newTable
-		idx := qg.r.Intn(len(qg.sel.SelectExprs))
+		idx := sg.r.Intn(len(sg.sel.SelectExprs))
 
-		qg.sel.SelectExprs[idx] = qg.sel.SelectExprs[len(qg.sel.SelectExprs)-1]
-		qg.sel.SelectExprs = qg.sel.SelectExprs[:len(qg.sel.SelectExprs)-1]
+		sg.sel.SelectExprs[idx] = sg.sel.SelectExprs[len(sg.sel.SelectExprs)-1]
+		sg.sel.SelectExprs = sg.sel.SelectExprs[:len(sg.sel.SelectExprs)-1]
 
 		newTable.cols[idx] = newTable.cols[len(newTable.cols)-1]
 		newTable.cols = newTable.cols[:len(newTable.cols)-1]
 	}
 
-	// alternatively, add random expressions until there are qg.genConfig.NumCols amount
-	if qg.genConfig.NumCols > len(qg.sel.SelectExprs) {
-		diff := qg.genConfig.NumCols - len(qg.sel.SelectExprs)
-		exprs := qg.createRandomExprs(diff, diff,
+	// alternatively, add random expressions until there are sg.genConfig.NumCols amount
+	if sg.genConfig.NumCols > len(sg.sel.SelectExprs) {
+		diff := sg.genConfig.NumCols - len(sg.sel.SelectExprs)
+		exprs := sg.createRandomExprs(diff, diff,
 			slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 
 		for i, expr := range exprs {
-			col := qg.randomlyAlias(expr, fmt.Sprintf("crandom%d", i+1))
+			col := sg.randomlyAlias(expr, fmt.Sprintf("crandom%d", i+1))
 			newTable.addColumns(col)
 
 			if canAggregate {
-				qg.sel.AddGroupBy(expr)
+				sg.sel.AddGroupBy(expr)
 			}
 		}
 	}
diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index 36e9aa3eada..fe8aa6f6492 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -53,10 +53,10 @@ func TestSeed(t *testing.T) {
 	genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
 	qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
 	qg.randomQuery()
-	query1 := sqlparser.String(qg.sel)
+	query1 := sqlparser.String(qg.stmt)
 	qg = newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
 	qg.randomQuery()
-	query2 := sqlparser.String(qg.sel)
+	query2 := sqlparser.String(qg.stmt)
 	fmt.Println(query1)
 	require.Equal(t, query1, query2)
 }
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index b33871a62d8..4fede4928a3 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -66,7 +66,7 @@ func TestRandomExprWithTables(t *testing.T) {
 		fmt.Printf("seed: %d\n", seed)
 
 		r := rand.New(rand.NewSource(seed))
-		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.IsAggregate, "", 0, false)
+		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CanAggregate, "", 0, false)
 		g := sqlparser.NewGenerator(r, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 		expr := g.Expression(genConfig)
 		fmt.Println(sqlparser.String(expr))
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 2953389b0ae..a8f3ce3d6ce 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -124,9 +124,6 @@ func TestMustFix(t *testing.T) {
 	// mismatched results
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm")
 
-	// mismatched results
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct max(tbl0.dname) as caggr0, 'cattle' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno != tbl1.sal group by tbl1.comm")
-
 	// mismatched results
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 1 = 0")
 
@@ -298,7 +295,7 @@ func TestRandom(t *testing.T) {
 		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
 		qg := newQueryGenerator(rand.New(rand.NewSource(seed)), genConfig, 2, 2, 2, schemaTables)
 		qg.randomQuery()
-		query := sqlparser.String(qg.sel)
+		query := sqlparser.String(qg.stmt)
 		_, vtErr := mcmp.ExecAllowAndCompareError(query)
 
 		// this assumes all queries are valid mysql queries
@@ -359,5 +356,6 @@ func TestBuggyQueries(t *testing.T) {
 	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ sum(tbl1.ename), min(tbl0.empno) from emp as tbl0, emp as tbl1 left join dept as tbl2 on tbl1.job = tbl2.loc and tbl1.comm = tbl2.deptno where ('trout') and tbl0.deptno = tbl1.comm")
 	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ distinct max(tbl0.deptno), count(tbl0.job) from emp as tbl0, dept as tbl1 left join dept as tbl2 on tbl1.dname = tbl2.loc and tbl1.dname = tbl2.loc where (tbl2.loc) and tbl0.deptno = tbl1.deptno")
 	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0")
+	mcmp.Exec("select /*vt+ PLANNER=Gen4 */ distinct max(tbl0.dname) as caggr0, 'cattle' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno != tbl1.sal group by tbl1.comm")
 
 }
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 3dd7c57fb0a..0e506f8cadc 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -40,8 +40,7 @@ type (
 		ExprGenerator
 	}
 
-	AggregateRule  int8
-	AggregateState int8
+	AggregateRule int8
 
 	ExprGeneratorConfig struct {
 		// AggrRule determines if the random expression can, cannot, or must be an aggregation expression
@@ -323,7 +322,7 @@ func (g *Generator) subqueryExpr(genConfig ExprGeneratorConfig) Expr {
 			options = append(options, func() Expr {
 				expr := qg.Generate(g.r, genConfig)
 				if expr == nil {
-					return g.randomTupleLiteral(genConfig.NumCols)
+					return g.randomTupleLiteral(genConfig)
 				}
 				return expr
 			})
@@ -331,15 +330,19 @@ func (g *Generator) subqueryExpr(genConfig ExprGeneratorConfig) Expr {
 	}
 
 	if len(options) == 0 {
-		return g.makeAggregateIfNecessary(genConfig, g.randomTupleLiteral(genConfig.NumCols))
+		return g.makeAggregateIfNecessary(genConfig, g.randomTupleLiteral(genConfig))
 	}
 
 	return g.randomOf(options)
 }
 
-func (g *Generator) randomTupleLiteral(len int) Expr {
+func (g *Generator) randomTupleLiteral(genConfig ExprGeneratorConfig) Expr {
+	if genConfig.NumCols == 0 {
+		genConfig.NumCols = g.r.Intn(3) + 1
+	}
+
 	tuple := ValTuple{}
-	for i := 0; i < len; i++ {
+	for i := 0; i < genConfig.NumCols; i++ {
 		tuple = append(tuple, g.randomLiteral())
 	}
 

From 13c89ca9aca057dd233cc140194e6735ffe0b869 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Thu, 20 Jul 2023 22:36:04 -0700
Subject: [PATCH 15/27] added tuple comparisons to random expression generator

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen_test.go   |  3 +++
 .../vtgate/queries/random/random_expr_test.go |  2 +-
 go/vt/sqlparser/random_expr.go                | 23 ++++++++++++-------
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index fe8aa6f6492..1b7938ccb6a 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -28,6 +28,9 @@ import (
 
 // TestSeed makes sure that the seed is deterministic
 func TestSeed(t *testing.T) {
+	sel, err := sqlparser.Parse("select 1 < any (select empno from emp)")
+	fmt.Println(sel, err)
+
 	// specify the schema (that is defined in schema.sql)
 	schemaTables := []tableT{
 		{tableExpr: sqlparser.NewTableName("emp")},
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index 4fede4928a3..333c5df59e3 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -66,7 +66,7 @@ func TestRandomExprWithTables(t *testing.T) {
 		fmt.Printf("seed: %d\n", seed)
 
 		r := rand.New(rand.NewSource(seed))
-		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CanAggregate, "", 0, false)
+		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
 		g := sqlparser.NewGenerator(r, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
 		expr := g.Expression(genConfig)
 		fmt.Println(sqlparser.String(expr))
diff --git a/go/vt/sqlparser/random_expr.go b/go/vt/sqlparser/random_expr.go
index 0e506f8cadc..6eed8145ed2 100644
--- a/go/vt/sqlparser/random_expr.go
+++ b/go/vt/sqlparser/random_expr.go
@@ -206,6 +206,11 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 		})
 	}
 
+	// if an arbitrary number of columns may be generated, randomly choose 1-3 columns
+	if numCols == 0 {
+		numCols = g.r.Intn(3) + 1
+	}
+
 	if numCols == 1 {
 		return g.makeAggregateIfNecessary(genConfig, g.randomOf(options))
 	}
@@ -215,11 +220,6 @@ func (g *Generator) Expression(genConfig ExprGeneratorConfig) Expr {
 		return g.subqueryExpr(genConfig.SetNumCols(numCols))
 	}
 
-	// if an arbitrary number of columns may be generated, randomly choose 1-3 columns
-	if numCols == 0 {
-		numCols = g.r.Intn(3) + 1
-	}
-
 	tuple := ValTuple{}
 	for i := 0; i < numCols; i++ {
 		tuple = append(tuple, g.makeAggregateIfNecessary(genConfig, g.randomOf(options)))
@@ -315,6 +315,10 @@ func (g *Generator) stringExpr(genConfig ExprGeneratorConfig) Expr {
 }
 
 func (g *Generator) subqueryExpr(genConfig ExprGeneratorConfig) Expr {
+	if g.atMaxDepth() {
+		return g.makeAggregateIfNecessary(genConfig, g.randomTupleLiteral(genConfig))
+	}
+
 	var options []exprF
 
 	for _, generator := range g.exprGenerators {
@@ -330,7 +334,7 @@ func (g *Generator) subqueryExpr(genConfig ExprGeneratorConfig) Expr {
 	}
 
 	if len(options) == 0 {
-		return g.makeAggregateIfNecessary(genConfig, g.randomTupleLiteral(genConfig))
+		return g.Expression(genConfig)
 	}
 
 	return g.randomOf(options)
@@ -399,10 +403,13 @@ func (g *Generator) comparison(genConfig ExprGeneratorConfig) Expr {
 	g.enter()
 	defer g.exit()
 
+	// specifc 1-3 columns
+	numCols := g.r.Intn(3) + 1
+
 	cmp := &ComparisonExpr{
 		Operator: comparisonOps[g.r.Intn(len(comparisonOps))],
-		Left:     g.Expression(genConfig),
-		Right:    g.Expression(genConfig),
+		Left:     g.Expression(genConfig.SetNumCols(numCols)),
+		Right:    g.Expression(genConfig.SetNumCols(numCols)),
 	}
 	return cmp
 }

From 60ec73c83b9112f7f610c7353f921a368ecb2174 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Fri, 21 Jul 2023 00:24:17 -0700
Subject: [PATCH 16/27] exported vschemaWrapper to go/test/utils

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 go/test/utils/vschema_wrapper.go            | 316 +++++++++++++
 go/vt/vtgate/planbuilder/collations_test.go |  15 +-
 go/vt/vtgate/planbuilder/plan_test.go       | 467 ++++----------------
 go/vt/vtgate/planbuilder/show_test.go       |   5 +-
 go/vt/vtgate/planbuilder/simplifier_test.go |  39 +-
 5 files changed, 439 insertions(+), 403 deletions(-)
 create mode 100644 go/test/utils/vschema_wrapper.go

diff --git a/go/test/utils/vschema_wrapper.go b/go/test/utils/vschema_wrapper.go
new file mode 100644
index 00000000000..9708e0f009b
--- /dev/null
+++ b/go/test/utils/vschema_wrapper.go
@@ -0,0 +1,316 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"vitess.io/vitess/go/mysql/collations"
+	"vitess.io/vitess/go/sqltypes"
+	"vitess.io/vitess/go/vt/key"
+	querypb "vitess.io/vitess/go/vt/proto/query"
+	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
+	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
+	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
+	"vitess.io/vitess/go/vt/sqlparser"
+	"vitess.io/vitess/go/vt/topo/topoproto"
+	"vitess.io/vitess/go/vt/vterrors"
+	"vitess.io/vitess/go/vt/vtgate/engine"
+	"vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext"
+	"vitess.io/vitess/go/vt/vtgate/semantics"
+	"vitess.io/vitess/go/vt/vtgate/vindexes"
+)
+
+var _ plancontext.VSchema = (*VSchemaWrapper)(nil)
+
+type VSchemaWrapper struct {
+	V             *vindexes.VSchema
+	Keyspace      *vindexes.Keyspace
+	TabletType_   topodatapb.TabletType
+	Dest          key.Destination
+	SysVarEnabled bool
+	Version       plancontext.PlannerVersion
+	EnableViews   bool
+	TestBuilder   func(query string, vschema plancontext.VSchema, keyspace string) (*engine.Plan, error)
+}
+
+func (vw *VSchemaWrapper) GetPrepareData(stmtName string) *vtgatepb.PrepareData {
+	switch stmtName {
+	case "prep_one_param":
+		return &vtgatepb.PrepareData{
+			PrepareStatement: "select 1 from user where id = :v1",
+			ParamsCount:      1,
+		}
+	case "prep_in_param":
+		return &vtgatepb.PrepareData{
+			PrepareStatement: "select 1 from user where id in (:v1, :v2)",
+			ParamsCount:      2,
+		}
+	case "prep_no_param":
+		return &vtgatepb.PrepareData{
+			PrepareStatement: "select 1 from user",
+			ParamsCount:      0,
+		}
+	}
+	return nil
+}
+
+func (vw *VSchemaWrapper) PlanPrepareStatement(ctx context.Context, query string) (*engine.Plan, sqlparser.Statement, error) {
+	plan, err := vw.TestBuilder(query, vw, vw.CurrentDb())
+	if err != nil {
+		return nil, nil, err
+	}
+	stmt, _, err := sqlparser.Parse2(query)
+	if err != nil {
+		return nil, nil, err
+	}
+	return plan, stmt, nil
+}
+
+func (vw *VSchemaWrapper) ClearPrepareData(lowered string) {
+}
+
+func (vw *VSchemaWrapper) StorePrepareData(string, *vtgatepb.PrepareData) {}
+
+func (vw *VSchemaWrapper) GetUDV(name string) *querypb.BindVariable {
+	if strings.EqualFold(name, "prep_stmt") {
+		return sqltypes.StringBindVariable("select * from user where id in (?, ?, ?)")
+	}
+	return nil
+}
+
+func (vw *VSchemaWrapper) IsShardRoutingEnabled() bool {
+	return false
+}
+
+func (vw *VSchemaWrapper) GetVSchema() *vindexes.VSchema {
+	return vw.V
+}
+
+func (vw *VSchemaWrapper) GetSrvVschema() *vschemapb.SrvVSchema {
+	return &vschemapb.SrvVSchema{
+		Keyspaces: map[string]*vschemapb.Keyspace{
+			"user": {
+				Sharded:  true,
+				Vindexes: map[string]*vschemapb.Vindex{},
+				Tables: map[string]*vschemapb.Table{
+					"user": {},
+				},
+			},
+		},
+	}
+}
+
+func (vw *VSchemaWrapper) ConnCollation() collations.ID {
+	return collations.CollationUtf8mb3ID
+}
+
+func (vw *VSchemaWrapper) PlannerWarning(_ string) {
+}
+
+func (vw *VSchemaWrapper) ForeignKeyMode() string {
+	return "allow"
+}
+
+func (vw *VSchemaWrapper) AllKeyspace() ([]*vindexes.Keyspace, error) {
+	if vw.Keyspace == nil {
+		return nil, vterrors.VT13001("keyspace not available")
+	}
+	return []*vindexes.Keyspace{vw.Keyspace}, nil
+}
+
+// FindKeyspace implements the VSchema interface
+func (vw *VSchemaWrapper) FindKeyspace(keyspace string) (*vindexes.Keyspace, error) {
+	if vw.Keyspace == nil {
+		return nil, vterrors.VT13001("keyspace not available")
+	}
+	if vw.Keyspace.Name == keyspace {
+		return vw.Keyspace, nil
+	}
+	return nil, nil
+}
+
+func (vw *VSchemaWrapper) Planner() plancontext.PlannerVersion {
+	return vw.Version
+}
+
+// SetPlannerVersion implements the ContextVSchema interface
+func (vw *VSchemaWrapper) SetPlannerVersion(v plancontext.PlannerVersion) {
+	vw.Version = v
+}
+
+func (vw *VSchemaWrapper) GetSemTable() *semantics.SemTable {
+	return nil
+}
+
+func (vw *VSchemaWrapper) KeyspaceExists(keyspace string) bool {
+	if vw.Keyspace != nil {
+		return vw.Keyspace.Name == keyspace
+	}
+	return false
+}
+
+func (vw *VSchemaWrapper) SysVarSetEnabled() bool {
+	return vw.SysVarEnabled
+}
+
+func (vw *VSchemaWrapper) TargetDestination(qualifier string) (key.Destination, *vindexes.Keyspace, topodatapb.TabletType, error) {
+	var keyspaceName string
+	if vw.Keyspace != nil {
+		keyspaceName = vw.Keyspace.Name
+	}
+	if vw.Dest == nil && qualifier != "" {
+		keyspaceName = qualifier
+	}
+	if keyspaceName == "" {
+		return nil, nil, 0, vterrors.VT03007()
+	}
+	keyspace := vw.V.Keyspaces[keyspaceName]
+	if keyspace == nil {
+		return nil, nil, 0, vterrors.VT05003(keyspaceName)
+	}
+	return vw.Dest, keyspace.Keyspace, vw.TabletType_, nil
+
+}
+
+func (vw *VSchemaWrapper) TabletType() topodatapb.TabletType {
+	return vw.TabletType_
+}
+
+func (vw *VSchemaWrapper) Destination() key.Destination {
+	return vw.Dest
+}
+
+func (vw *VSchemaWrapper) FindTable(tab sqlparser.TableName) (*vindexes.Table, string, topodatapb.TabletType, key.Destination, error) {
+	destKeyspace, destTabletType, destTarget, err := topoproto.ParseDestination(tab.Qualifier.String(), topodatapb.TabletType_PRIMARY)
+	if err != nil {
+		return nil, destKeyspace, destTabletType, destTarget, err
+	}
+	table, err := vw.V.FindTable(destKeyspace, tab.Name.String())
+	if err != nil {
+		return nil, destKeyspace, destTabletType, destTarget, err
+	}
+	return table, destKeyspace, destTabletType, destTarget, nil
+}
+
+func (vw *VSchemaWrapper) FindView(tab sqlparser.TableName) sqlparser.SelectStatement {
+	destKeyspace, _, _, err := topoproto.ParseDestination(tab.Qualifier.String(), topodatapb.TabletType_PRIMARY)
+	if err != nil {
+		return nil
+	}
+	return vw.V.FindView(destKeyspace, tab.Name.String())
+}
+
+func (vw *VSchemaWrapper) FindTableOrVindex(tab sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error) {
+	if tab.Qualifier.IsEmpty() && tab.Name.String() == "dual" {
+		ksName := vw.getActualKeyspace()
+		var ks *vindexes.Keyspace
+		if ksName == "" {
+			ks = vw.getfirstKeyspace()
+			ksName = ks.Name
+		} else {
+			ks = vw.V.Keyspaces[ksName].Keyspace
+		}
+		tbl := &vindexes.Table{
+			Name:     sqlparser.NewIdentifierCS("dual"),
+			Keyspace: ks,
+			Type:     vindexes.TypeReference,
+		}
+		return tbl, nil, ksName, topodatapb.TabletType_PRIMARY, nil, nil
+	}
+	destKeyspace, destTabletType, destTarget, err := topoproto.ParseDestination(tab.Qualifier.String(), topodatapb.TabletType_PRIMARY)
+	if err != nil {
+		return nil, nil, destKeyspace, destTabletType, destTarget, err
+	}
+	if destKeyspace == "" {
+		destKeyspace = vw.getActualKeyspace()
+	}
+	table, vindex, err := vw.V.FindTableOrVindex(destKeyspace, tab.Name.String(), topodatapb.TabletType_PRIMARY)
+	if err != nil {
+		return nil, nil, destKeyspace, destTabletType, destTarget, err
+	}
+	return table, vindex, destKeyspace, destTabletType, destTarget, nil
+}
+
+func (vw *VSchemaWrapper) getfirstKeyspace() (ks *vindexes.Keyspace) {
+	var f string
+	for name, schema := range vw.V.Keyspaces {
+		if f == "" || f > name {
+			f = name
+			ks = schema.Keyspace
+		}
+	}
+	return
+}
+
+func (vw *VSchemaWrapper) getActualKeyspace() string {
+	if vw.Keyspace == nil {
+		return ""
+	}
+	if !sqlparser.SystemSchema(vw.Keyspace.Name) {
+		return vw.Keyspace.Name
+	}
+	ks, err := vw.AnyKeyspace()
+	if err != nil {
+		return ""
+	}
+	return ks.Name
+}
+
+func (vw *VSchemaWrapper) DefaultKeyspace() (*vindexes.Keyspace, error) {
+	return vw.V.Keyspaces["main"].Keyspace, nil
+}
+
+func (vw *VSchemaWrapper) AnyKeyspace() (*vindexes.Keyspace, error) {
+	return vw.DefaultKeyspace()
+}
+
+func (vw *VSchemaWrapper) FirstSortedKeyspace() (*vindexes.Keyspace, error) {
+	return vw.V.Keyspaces["main"].Keyspace, nil
+}
+
+func (vw *VSchemaWrapper) TargetString() string {
+	return "targetString"
+}
+
+func (vw *VSchemaWrapper) WarnUnshardedOnly(_ string, _ ...any) {
+
+}
+
+func (vw *VSchemaWrapper) ErrorIfShardedF(keyspace *vindexes.Keyspace, _, errFmt string, params ...any) error {
+	if keyspace.Sharded {
+		return fmt.Errorf(errFmt, params...)
+	}
+	return nil
+}
+
+func (vw *VSchemaWrapper) CurrentDb() string {
+	ksName := ""
+	if vw.Keyspace != nil {
+		ksName = vw.Keyspace.Name
+	}
+	return ksName
+}
+
+func (vw *VSchemaWrapper) FindRoutedShard(keyspace, shard string) (string, error) {
+	return "", nil
+}
+
+func (vw *VSchemaWrapper) IsViewsEnabled() bool {
+	return vw.EnableViews
+}
diff --git a/go/vt/vtgate/planbuilder/collations_test.go b/go/vt/vtgate/planbuilder/collations_test.go
index 2a7ffebf91c..534dbc7dda2 100644
--- a/go/vt/vtgate/planbuilder/collations_test.go
+++ b/go/vt/vtgate/planbuilder/collations_test.go
@@ -19,6 +19,7 @@ package planbuilder
 import (
 	"fmt"
 	"testing"
+	"vitess.io/vitess/go/test/utils"
 
 	"github.com/stretchr/testify/require"
 
@@ -39,21 +40,21 @@ type collationTestCase struct {
 }
 
 func (tc *collationTestCase) run(t *testing.T) {
-	vschemaWrapper := &vschemaWrapper{
-		v:             loadSchema(t, "vschemas/schema.json", false),
-		sysVarEnabled: true,
-		version:       Gen4,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:             loadSchema(t, "vschemas/schema.json", false),
+		SysVarEnabled: true,
+		Version:       Gen4,
 	}
 
 	tc.addCollationsToSchema(vschemaWrapper)
-	plan, err := TestBuilder(tc.query, vschemaWrapper, vschemaWrapper.currentDb())
+	plan, err := TestBuilder(tc.query, vschemaWrapper, vschemaWrapper.CurrentDb())
 	require.NoError(t, err)
 	tc.check(t, tc.collations, plan.Instructions)
 }
 
-func (tc *collationTestCase) addCollationsToSchema(vschema *vschemaWrapper) {
+func (tc *collationTestCase) addCollationsToSchema(vschema *utils.VSchemaWrapper) {
 	for _, collation := range tc.collations {
-		tbl := vschema.v.Keyspaces[collation.ks].Tables[collation.table]
+		tbl := vschema.V.Keyspaces[collation.ks].Tables[collation.table]
 		for i, c := range tbl.Columns {
 			if c.Name.EqualString(collation.colName) {
 				tbl.Columns[i].CollationName = collation.collationName
diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go
index 755a5fbf916..15f28ef8b1c 100644
--- a/go/vt/vtgate/planbuilder/plan_test.go
+++ b/go/vt/vtgate/planbuilder/plan_test.go
@@ -31,20 +31,14 @@ import (
 	"github.com/nsf/jsondiff"
 	"github.com/stretchr/testify/require"
 
-	"vitess.io/vitess/go/mysql/collations"
 	"vitess.io/vitess/go/sqltypes"
 	"vitess.io/vitess/go/test/utils"
 	"vitess.io/vitess/go/vt/key"
-	querypb "vitess.io/vitess/go/vt/proto/query"
 	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
-	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
-	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
 	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/sidecardb"
 	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/topo/memorytopo"
-	"vitess.io/vitess/go/vt/topo/topoproto"
-	"vitess.io/vitess/go/vt/vterrors"
 	"vitess.io/vitess/go/vt/vtgate/engine"
 	oprewriters "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/rewrite"
 	"vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext"
@@ -59,10 +53,11 @@ func makeTestOutput(t *testing.T) string {
 }
 
 func TestPlan(t *testing.T) {
-	vschemaWrapper := &vschemaWrapper{
-		v:             loadSchema(t, "vschemas/schema.json", true),
-		tabletType:    topodatapb.TabletType_PRIMARY,
-		sysVarEnabled: true,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:             loadSchema(t, "vschemas/schema.json", true),
+		TabletType_:   topodatapb.TabletType_PRIMARY,
+		SysVarEnabled: true,
+		TestBuilder:   TestBuilder,
 	}
 	testOutputTempDir := makeTestOutput(t)
 
@@ -104,24 +99,24 @@ func TestSystemTables57(t *testing.T) {
 	// first we move everything to use 5.7 logic
 	servenv.SetMySQLServerVersionForTest("5.7")
 	defer servenv.SetMySQLServerVersionForTest("")
-	vschemaWrapper := &vschemaWrapper{v: loadSchema(t, "vschemas/schema.json", true)}
+	vschemaWrapper := &utils.VSchemaWrapper{V: loadSchema(t, "vschemas/schema.json", true)}
 	testOutputTempDir := makeTestOutput(t)
 	testFile(t, "info_schema57_cases.json", testOutputTempDir, vschemaWrapper, false)
 }
 
 func TestSysVarSetDisabled(t *testing.T) {
-	vschemaWrapper := &vschemaWrapper{
-		v:             loadSchema(t, "vschemas/schema.json", true),
-		sysVarEnabled: false,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:             loadSchema(t, "vschemas/schema.json", true),
+		SysVarEnabled: false,
 	}
 
 	testFile(t, "set_sysvar_disabled_cases.json", makeTestOutput(t), vschemaWrapper, false)
 }
 
 func TestViews(t *testing.T) {
-	vschemaWrapper := &vschemaWrapper{
-		v:           loadSchema(t, "vschemas/schema.json", true),
-		enableViews: true,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:           loadSchema(t, "vschemas/schema.json", true),
+		EnableViews: true,
 	}
 
 	testFile(t, "view_cases.json", makeTestOutput(t), vschemaWrapper, false)
@@ -131,25 +126,25 @@ func TestOne(t *testing.T) {
 	reset := oprewriters.EnableDebugPrinting()
 	defer reset()
 
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
 	}
 
 	testFile(t, "onecase.json", "", vschema, false)
 }
 
 func TestOneTPCC(t *testing.T) {
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/tpcc_schema.json", true),
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/tpcc_schema.json", true),
 	}
 
 	testFile(t, "onecase.json", "", vschema, false)
 }
 
 func TestOneWithMainAsDefault(t *testing.T) {
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
 			Sharded: false,
 		},
@@ -159,9 +154,9 @@ func TestOneWithMainAsDefault(t *testing.T) {
 }
 
 func TestOneWithSecondUserAsDefault(t *testing.T) {
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "second_user",
 			Sharded: true,
 		},
@@ -171,9 +166,9 @@ func TestOneWithSecondUserAsDefault(t *testing.T) {
 }
 
 func TestOneWithUserAsDefault(t *testing.T) {
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "user",
 			Sharded: true,
 		},
@@ -183,9 +178,9 @@ func TestOneWithUserAsDefault(t *testing.T) {
 }
 
 func TestOneWithTPCHVSchema(t *testing.T) {
-	vschema := &vschemaWrapper{
-		v:             loadSchema(t, "vschemas/tpch_schema.json", true),
-		sysVarEnabled: true,
+	vschema := &utils.VSchemaWrapper{
+		V:             loadSchema(t, "vschemas/tpch_schema.json", true),
+		SysVarEnabled: true,
 	}
 
 	testFile(t, "onecase.json", "", vschema, false)
@@ -195,42 +190,42 @@ func TestOneWith57Version(t *testing.T) {
 	// first we move everything to use 5.7 logic
 	servenv.SetMySQLServerVersionForTest("5.7")
 	defer servenv.SetMySQLServerVersionForTest("")
-	vschema := &vschemaWrapper{v: loadSchema(t, "vschemas/schema.json", true)}
+	vschema := &utils.VSchemaWrapper{V: loadSchema(t, "vschemas/schema.json", true)}
 
 	testFile(t, "onecase.json", "", vschema, false)
 }
 
 func TestRubyOnRailsQueries(t *testing.T) {
-	vschemaWrapper := &vschemaWrapper{
-		v:             loadSchema(t, "vschemas/rails_schema.json", true),
-		sysVarEnabled: true,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:             loadSchema(t, "vschemas/rails_schema.json", true),
+		SysVarEnabled: true,
 	}
 
 	testFile(t, "rails_cases.json", makeTestOutput(t), vschemaWrapper, false)
 }
 
 func TestOLTP(t *testing.T) {
-	vschemaWrapper := &vschemaWrapper{
-		v:             loadSchema(t, "vschemas/oltp_schema.json", true),
-		sysVarEnabled: true,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:             loadSchema(t, "vschemas/oltp_schema.json", true),
+		SysVarEnabled: true,
 	}
 
 	testFile(t, "oltp_cases.json", makeTestOutput(t), vschemaWrapper, false)
 }
 
 func TestTPCC(t *testing.T) {
-	vschemaWrapper := &vschemaWrapper{
-		v:             loadSchema(t, "vschemas/tpcc_schema.json", true),
-		sysVarEnabled: true,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:             loadSchema(t, "vschemas/tpcc_schema.json", true),
+		SysVarEnabled: true,
 	}
 
 	testFile(t, "tpcc_cases.json", makeTestOutput(t), vschemaWrapper, false)
 }
 
 func TestTPCH(t *testing.T) {
-	vschemaWrapper := &vschemaWrapper{
-		v:             loadSchema(t, "vschemas/tpch_schema.json", true),
-		sysVarEnabled: true,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:             loadSchema(t, "vschemas/tpch_schema.json", true),
+		SysVarEnabled: true,
 	}
 
 	testFile(t, "tpch_cases.json", makeTestOutput(t), vschemaWrapper, false)
@@ -249,9 +244,9 @@ func BenchmarkTPCH(b *testing.B) {
 }
 
 func benchmarkWorkload(b *testing.B, name string) {
-	vschemaWrapper := &vschemaWrapper{
-		v:             loadSchema(b, "vschemas/"+name+"_schema.json", true),
-		sysVarEnabled: true,
+	vschemaWrapper := &utils.VSchemaWrapper{
+		V:             loadSchema(b, "vschemas/"+name+"_schema.json", true),
+		SysVarEnabled: true,
 	}
 
 	testCases := readJSONTests(name + "_cases.json")
@@ -264,14 +259,14 @@ func benchmarkWorkload(b *testing.B, name string) {
 }
 
 func TestBypassPlanningShardTargetFromFile(t *testing.T) {
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
 			Sharded: false,
 		},
-		tabletType: topodatapb.TabletType_PRIMARY,
-		dest:       key.DestinationShard("-80")}
+		TabletType_: topodatapb.TabletType_PRIMARY,
+		Dest:        key.DestinationShard("-80")}
 
 	testFile(t, "bypass_shard_cases.json", makeTestOutput(t), vschema, false)
 }
@@ -279,14 +274,14 @@ func TestBypassPlanningShardTargetFromFile(t *testing.T) {
 func TestBypassPlanningKeyrangeTargetFromFile(t *testing.T) {
 	keyRange, _ := key.ParseShardingSpec("-")
 
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
 			Sharded: false,
 		},
-		tabletType: topodatapb.TabletType_PRIMARY,
-		dest:       key.DestinationExactKeyRange{KeyRange: keyRange[0]},
+		TabletType_: topodatapb.TabletType_PRIMARY,
+		Dest:        key.DestinationExactKeyRange{KeyRange: keyRange[0]},
 	}
 
 	testFile(t, "bypass_keyrange_cases.json", makeTestOutput(t), vschema, false)
@@ -294,13 +289,13 @@ func TestBypassPlanningKeyrangeTargetFromFile(t *testing.T) {
 
 func TestWithDefaultKeyspaceFromFile(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
 			Sharded: false,
 		},
-		tabletType: topodatapb.TabletType_PRIMARY,
+		TabletType_: topodatapb.TabletType_PRIMARY,
 	}
 	ts := memorytopo.NewServer("cell1")
 	ts.CreateKeyspace(context.Background(), "main", &topodatapb.Keyspace{})
@@ -327,13 +322,13 @@ func TestWithDefaultKeyspaceFromFile(t *testing.T) {
 
 func TestWithDefaultKeyspaceFromFileSharded(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "second_user",
 			Sharded: true,
 		},
-		tabletType: topodatapb.TabletType_PRIMARY,
+		TabletType_: topodatapb.TabletType_PRIMARY,
 	}
 
 	testOutputTempDir := makeTestOutput(t)
@@ -342,13 +337,13 @@ func TestWithDefaultKeyspaceFromFileSharded(t *testing.T) {
 
 func TestWithUserDefaultKeyspaceFromFileSharded(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "user",
 			Sharded: true,
 		},
-		tabletType: topodatapb.TabletType_PRIMARY,
+		TabletType_: topodatapb.TabletType_PRIMARY,
 	}
 
 	testOutputTempDir := makeTestOutput(t)
@@ -357,10 +352,10 @@ func TestWithUserDefaultKeyspaceFromFileSharded(t *testing.T) {
 
 func TestWithSystemSchemaAsDefaultKeyspace(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &vschemaWrapper{
-		v:          loadSchema(t, "vschemas/schema.json", true),
-		keyspace:   &vindexes.Keyspace{Name: "information_schema"},
-		tabletType: topodatapb.TabletType_PRIMARY,
+	vschema := &utils.VSchemaWrapper{
+		V:           loadSchema(t, "vschemas/schema.json", true),
+		Keyspace:    &vindexes.Keyspace{Name: "information_schema"},
+		TabletType_: topodatapb.TabletType_PRIMARY,
 	}
 
 	testFile(t, "sysschema_default.json", makeTestOutput(t), vschema, false)
@@ -368,13 +363,13 @@ func TestWithSystemSchemaAsDefaultKeyspace(t *testing.T) {
 
 func TestOtherPlanningFromFile(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &vschemaWrapper{
-		v: loadSchema(t, "vschemas/schema.json", true),
-		keyspace: &vindexes.Keyspace{
+	vschema := &utils.VSchemaWrapper{
+		V: loadSchema(t, "vschemas/schema.json", true),
+		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
 			Sharded: false,
 		},
-		tabletType: topodatapb.TabletType_PRIMARY,
+		TabletType_: topodatapb.TabletType_PRIMARY,
 	}
 
 	testOutputTempDir := makeTestOutput(t)
@@ -421,284 +416,6 @@ func loadSchema(t testing.TB, filename string, setCollation bool) *vindexes.VSch
 	return vschema
 }
 
-var _ plancontext.VSchema = (*vschemaWrapper)(nil)
-
-type vschemaWrapper struct {
-	v             *vindexes.VSchema
-	keyspace      *vindexes.Keyspace
-	tabletType    topodatapb.TabletType
-	dest          key.Destination
-	sysVarEnabled bool
-	version       plancontext.PlannerVersion
-	enableViews   bool
-}
-
-func (vw *vschemaWrapper) GetPrepareData(stmtName string) *vtgatepb.PrepareData {
-	switch stmtName {
-	case "prep_one_param":
-		return &vtgatepb.PrepareData{
-			PrepareStatement: "select 1 from user where id = :v1",
-			ParamsCount:      1,
-		}
-	case "prep_in_param":
-		return &vtgatepb.PrepareData{
-			PrepareStatement: "select 1 from user where id in (:v1, :v2)",
-			ParamsCount:      2,
-		}
-	case "prep_no_param":
-		return &vtgatepb.PrepareData{
-			PrepareStatement: "select 1 from user",
-			ParamsCount:      0,
-		}
-	}
-	return nil
-}
-
-func (vw *vschemaWrapper) PlanPrepareStatement(ctx context.Context, query string) (*engine.Plan, sqlparser.Statement, error) {
-	plan, err := TestBuilder(query, vw, vw.currentDb())
-	if err != nil {
-		return nil, nil, err
-	}
-	stmt, _, err := sqlparser.Parse2(query)
-	if err != nil {
-		return nil, nil, err
-	}
-	return plan, stmt, nil
-}
-
-func (vw *vschemaWrapper) ClearPrepareData(lowered string) {
-}
-
-func (vw *vschemaWrapper) StorePrepareData(string, *vtgatepb.PrepareData) {}
-
-func (vw *vschemaWrapper) GetUDV(name string) *querypb.BindVariable {
-	if strings.EqualFold(name, "prep_stmt") {
-		return sqltypes.StringBindVariable("select * from user where id in (?, ?, ?)")
-	}
-	return nil
-}
-
-func (vw *vschemaWrapper) IsShardRoutingEnabled() bool {
-	return false
-}
-
-func (vw *vschemaWrapper) GetVSchema() *vindexes.VSchema {
-	return vw.v
-}
-
-func (vw *vschemaWrapper) GetSrvVschema() *vschemapb.SrvVSchema {
-	return &vschemapb.SrvVSchema{
-		Keyspaces: map[string]*vschemapb.Keyspace{
-			"user": {
-				Sharded:  true,
-				Vindexes: map[string]*vschemapb.Vindex{},
-				Tables: map[string]*vschemapb.Table{
-					"user": {},
-				},
-			},
-		},
-	}
-}
-
-func (vw *vschemaWrapper) ConnCollation() collations.ID {
-	return collations.Default()
-}
-
-func (vw *vschemaWrapper) PlannerWarning(_ string) {
-}
-
-func (vw *vschemaWrapper) ForeignKeyMode() string {
-	return "allow"
-}
-
-func (vw *vschemaWrapper) AllKeyspace() ([]*vindexes.Keyspace, error) {
-	if vw.keyspace == nil {
-		return nil, vterrors.VT13001("keyspace not available")
-	}
-	return []*vindexes.Keyspace{vw.keyspace}, nil
-}
-
-// FindKeyspace implements the VSchema interface
-func (vw *vschemaWrapper) FindKeyspace(keyspace string) (*vindexes.Keyspace, error) {
-	if vw.keyspace == nil {
-		return nil, vterrors.VT13001("keyspace not available")
-	}
-	if vw.keyspace.Name == keyspace {
-		return vw.keyspace, nil
-	}
-	return nil, nil
-}
-
-func (vw *vschemaWrapper) Planner() plancontext.PlannerVersion {
-	return vw.version
-}
-
-// SetPlannerVersion implements the ContextVSchema interface
-func (vw *vschemaWrapper) SetPlannerVersion(v plancontext.PlannerVersion) {
-	vw.version = v
-}
-
-func (vw *vschemaWrapper) GetSemTable() *semantics.SemTable {
-	return nil
-}
-
-func (vw *vschemaWrapper) KeyspaceExists(keyspace string) bool {
-	if vw.keyspace != nil {
-		return vw.keyspace.Name == keyspace
-	}
-	return false
-}
-
-func (vw *vschemaWrapper) SysVarSetEnabled() bool {
-	return vw.sysVarEnabled
-}
-
-func (vw *vschemaWrapper) TargetDestination(qualifier string) (key.Destination, *vindexes.Keyspace, topodatapb.TabletType, error) {
-	var keyspaceName string
-	if vw.keyspace != nil {
-		keyspaceName = vw.keyspace.Name
-	}
-	if vw.dest == nil && qualifier != "" {
-		keyspaceName = qualifier
-	}
-	if keyspaceName == "" {
-		return nil, nil, 0, vterrors.VT03007()
-	}
-	keyspace := vw.v.Keyspaces[keyspaceName]
-	if keyspace == nil {
-		return nil, nil, 0, vterrors.VT05003(keyspaceName)
-	}
-	return vw.dest, keyspace.Keyspace, vw.tabletType, nil
-
-}
-
-func (vw *vschemaWrapper) TabletType() topodatapb.TabletType {
-	return vw.tabletType
-}
-
-func (vw *vschemaWrapper) Destination() key.Destination {
-	return vw.dest
-}
-
-func (vw *vschemaWrapper) FindTable(tab sqlparser.TableName) (*vindexes.Table, string, topodatapb.TabletType, key.Destination, error) {
-	destKeyspace, destTabletType, destTarget, err := topoproto.ParseDestination(tab.Qualifier.String(), topodatapb.TabletType_PRIMARY)
-	if err != nil {
-		return nil, destKeyspace, destTabletType, destTarget, err
-	}
-	table, err := vw.v.FindTable(destKeyspace, tab.Name.String())
-	if err != nil {
-		return nil, destKeyspace, destTabletType, destTarget, err
-	}
-	return table, destKeyspace, destTabletType, destTarget, nil
-}
-
-func (vw *vschemaWrapper) FindView(tab sqlparser.TableName) sqlparser.SelectStatement {
-	destKeyspace, _, _, err := topoproto.ParseDestination(tab.Qualifier.String(), topodatapb.TabletType_PRIMARY)
-	if err != nil {
-		return nil
-	}
-	return vw.v.FindView(destKeyspace, tab.Name.String())
-}
-
-func (vw *vschemaWrapper) FindTableOrVindex(tab sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error) {
-	if tab.Qualifier.IsEmpty() && tab.Name.String() == "dual" {
-		ksName := vw.getActualKeyspace()
-		var ks *vindexes.Keyspace
-		if ksName == "" {
-			ks = vw.getfirstKeyspace()
-			ksName = ks.Name
-		} else {
-			ks = vw.v.Keyspaces[ksName].Keyspace
-		}
-		tbl := &vindexes.Table{
-			Name:     sqlparser.NewIdentifierCS("dual"),
-			Keyspace: ks,
-			Type:     vindexes.TypeReference,
-		}
-		return tbl, nil, ksName, topodatapb.TabletType_PRIMARY, nil, nil
-	}
-	destKeyspace, destTabletType, destTarget, err := topoproto.ParseDestination(tab.Qualifier.String(), topodatapb.TabletType_PRIMARY)
-	if err != nil {
-		return nil, nil, destKeyspace, destTabletType, destTarget, err
-	}
-	if destKeyspace == "" {
-		destKeyspace = vw.getActualKeyspace()
-	}
-	table, vindex, err := vw.v.FindTableOrVindex(destKeyspace, tab.Name.String(), topodatapb.TabletType_PRIMARY)
-	if err != nil {
-		return nil, nil, destKeyspace, destTabletType, destTarget, err
-	}
-	return table, vindex, destKeyspace, destTabletType, destTarget, nil
-}
-
-func (vw *vschemaWrapper) getfirstKeyspace() (ks *vindexes.Keyspace) {
-	var f string
-	for name, schema := range vw.v.Keyspaces {
-		if f == "" || f > name {
-			f = name
-			ks = schema.Keyspace
-		}
-	}
-	return
-}
-
-func (vw *vschemaWrapper) getActualKeyspace() string {
-	if vw.keyspace == nil {
-		return ""
-	}
-	if !sqlparser.SystemSchema(vw.keyspace.Name) {
-		return vw.keyspace.Name
-	}
-	ks, err := vw.AnyKeyspace()
-	if err != nil {
-		return ""
-	}
-	return ks.Name
-}
-
-func (vw *vschemaWrapper) DefaultKeyspace() (*vindexes.Keyspace, error) {
-	return vw.v.Keyspaces["main"].Keyspace, nil
-}
-
-func (vw *vschemaWrapper) AnyKeyspace() (*vindexes.Keyspace, error) {
-	return vw.DefaultKeyspace()
-}
-
-func (vw *vschemaWrapper) FirstSortedKeyspace() (*vindexes.Keyspace, error) {
-	return vw.v.Keyspaces["main"].Keyspace, nil
-}
-
-func (vw *vschemaWrapper) TargetString() string {
-	return "targetString"
-}
-
-func (vw *vschemaWrapper) WarnUnshardedOnly(_ string, _ ...any) {
-
-}
-
-func (vw *vschemaWrapper) ErrorIfShardedF(keyspace *vindexes.Keyspace, _, errFmt string, params ...any) error {
-	if keyspace.Sharded {
-		return fmt.Errorf(errFmt, params...)
-	}
-	return nil
-}
-
-func (vw *vschemaWrapper) currentDb() string {
-	ksName := ""
-	if vw.keyspace != nil {
-		ksName = vw.keyspace.Name
-	}
-	return ksName
-}
-
-func (vw *vschemaWrapper) FindRoutedShard(keyspace, shard string) (string, error) {
-	return "", nil
-}
-
-func (vw *vschemaWrapper) IsViewsEnabled() bool {
-	return vw.enableViews
-}
-
 type (
 	planTest struct {
 		Comment string          `json:"comment,omitempty"`
@@ -707,7 +424,7 @@ type (
 	}
 )
 
-func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper, render bool) {
+func testFile(t *testing.T, filename, tempDir string, vschema *utils.VSchemaWrapper, render bool) {
 	opts := jsondiff.DefaultConsoleOptions()
 
 	t.Run(filename, func(t *testing.T) {
@@ -724,7 +441,7 @@ func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper, r
 				Comment: testName,
 				Query:   tcase.Query,
 			}
-			vschema.version = Gen4
+			vschema.Version = Gen4
 			out := getPlanOutput(tcase, vschema, render)
 
 			// our expectation for the planner on the query is one of three
@@ -770,13 +487,13 @@ func readJSONTests(filename string) []planTest {
 	return output
 }
 
-func getPlanOutput(tcase planTest, vschema *vschemaWrapper, render bool) (out string) {
+func getPlanOutput(tcase planTest, vschema *utils.VSchemaWrapper, render bool) (out string) {
 	defer func() {
 		if r := recover(); r != nil {
 			out = fmt.Sprintf("panicked: %v\n%s", r, string(debug.Stack()))
 		}
 	}()
-	plan, err := TestBuilder(tcase.Query, vschema, vschema.currentDb())
+	plan, err := TestBuilder(tcase.Query, vschema, vschema.CurrentDb())
 	if render && plan != nil {
 		viz, err := engine.GraphViz(plan.Instructions)
 		if err == nil {
@@ -808,9 +525,9 @@ func locateFile(name string) string {
 var benchMarkFiles = []string{"from_cases.json", "filter_cases.json", "large_cases.json", "aggr_cases.json", "select_cases.json", "union_cases.json"}
 
 func BenchmarkPlanner(b *testing.B) {
-	vschema := &vschemaWrapper{
-		v:             loadSchema(b, "vschemas/schema.json", true),
-		sysVarEnabled: true,
+	vschema := &utils.VSchemaWrapper{
+		V:             loadSchema(b, "vschemas/schema.json", true),
+		SysVarEnabled: true,
 	}
 	for _, filename := range benchMarkFiles {
 		testCases := readJSONTests(filename)
@@ -824,15 +541,15 @@ func BenchmarkPlanner(b *testing.B) {
 }
 
 func BenchmarkSemAnalysis(b *testing.B) {
-	vschema := &vschemaWrapper{
-		v:             loadSchema(b, "vschemas/schema.json", true),
-		sysVarEnabled: true,
+	vschema := &utils.VSchemaWrapper{
+		V:             loadSchema(b, "vschemas/schema.json", true),
+		SysVarEnabled: true,
 	}
 
 	for i := 0; i < b.N; i++ {
 		for _, filename := range benchMarkFiles {
 			for _, tc := range readJSONTests(filename) {
-				exerciseAnalyzer(tc.Query, vschema.currentDb(), vschema)
+				exerciseAnalyzer(tc.Query, vschema.CurrentDb(), vschema)
 			}
 		}
 	}
@@ -857,10 +574,10 @@ func exerciseAnalyzer(query, database string, s semantics.SchemaInformation) {
 }
 
 func BenchmarkSelectVsDML(b *testing.B) {
-	vschema := &vschemaWrapper{
-		v:             loadSchema(b, "vschemas/schema.json", true),
-		sysVarEnabled: true,
-		version:       Gen4,
+	vschema := &utils.VSchemaWrapper{
+		V:             loadSchema(b, "vschemas/schema.json", true),
+		SysVarEnabled: true,
+		Version:       Gen4,
 	}
 
 	dmlCases := readJSONTests("dml_cases.json")
@@ -883,13 +600,13 @@ func BenchmarkSelectVsDML(b *testing.B) {
 	})
 }
 
-func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCases []planTest, vschema *vschemaWrapper) {
+func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCases []planTest, vschema *utils.VSchemaWrapper) {
 	b.ReportAllocs()
 	for n := 0; n < b.N; n++ {
 		for _, tcase := range testCases {
 			if len(tcase.Plan) > 0 {
-				vschema.version = version
-				_, _ = TestBuilder(tcase.Query, vschema, vschema.currentDb())
+				vschema.Version = version
+				_, _ = TestBuilder(tcase.Query, vschema, vschema.CurrentDb())
 			}
 		}
 	}
diff --git a/go/vt/vtgate/planbuilder/show_test.go b/go/vt/vtgate/planbuilder/show_test.go
index 3caae74bf27..9401db2c741 100644
--- a/go/vt/vtgate/planbuilder/show_test.go
+++ b/go/vt/vtgate/planbuilder/show_test.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"vitess.io/vitess/go/test/utils"
 
 	"github.com/stretchr/testify/require"
 
@@ -31,8 +32,8 @@ import (
 )
 
 func TestBuildDBPlan(t *testing.T) {
-	vschema := &vschemaWrapper{
-		keyspace: &vindexes.Keyspace{Name: "main"},
+	vschema := &utils.VSchemaWrapper{
+		Keyspace: &vindexes.Keyspace{Name: "main"},
 	}
 
 	testCases := []struct {
diff --git a/go/vt/vtgate/planbuilder/simplifier_test.go b/go/vt/vtgate/planbuilder/simplifier_test.go
index 509564225d7..9e2473bb3c1 100644
--- a/go/vt/vtgate/planbuilder/simplifier_test.go
+++ b/go/vt/vtgate/planbuilder/simplifier_test.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"vitess.io/vitess/go/test/utils"
 
 	"vitess.io/vitess/go/vt/vterrors"
 
@@ -38,18 +39,18 @@ import (
 // It will try to minimize the query to make it easier to understand and work with the bug.
 func TestSimplifyBuggyQuery(t *testing.T) {
 	query := "(select id from unsharded union select id from unsharded_auto) union (select id from user union select name from unsharded)"
-	vschema := &vschemaWrapper{
-		v:       loadSchema(t, "vschemas/schema.json", true),
-		version: Gen4,
+	vschema := &utils.VSchemaWrapper{
+		V:       loadSchema(t, "vschemas/schema.json", true),
+		Version: Gen4,
 	}
 	stmt, reserved, err := sqlparser.Parse2(query)
 	require.NoError(t, err)
-	rewritten, _ := sqlparser.RewriteAST(sqlparser.CloneStatement(stmt), vschema.currentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil)
+	rewritten, _ := sqlparser.RewriteAST(sqlparser.CloneStatement(stmt), vschema.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil)
 	reservedVars := sqlparser.NewReservedVars("vtg", reserved)
 
 	simplified := simplifier.SimplifyStatement(
 		stmt.(sqlparser.SelectStatement),
-		vschema.currentDb(),
+		vschema.CurrentDb(),
 		vschema,
 		keepSameError(query, reservedVars, vschema, rewritten.BindVarNeeds),
 	)
@@ -60,18 +61,18 @@ func TestSimplifyBuggyQuery(t *testing.T) {
 func TestSimplifyPanic(t *testing.T) {
 	t.Skip("not needed to run")
 	query := "(select id from unsharded union select id from unsharded_auto) union (select id from unsharded_auto union select name from unsharded)"
-	vschema := &vschemaWrapper{
-		v:       loadSchema(t, "vschemas/schema.json", true),
-		version: Gen4,
+	vschema := &utils.VSchemaWrapper{
+		V:       loadSchema(t, "vschemas/schema.json", true),
+		Version: Gen4,
 	}
 	stmt, reserved, err := sqlparser.Parse2(query)
 	require.NoError(t, err)
-	rewritten, _ := sqlparser.RewriteAST(sqlparser.CloneStatement(stmt), vschema.currentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil)
+	rewritten, _ := sqlparser.RewriteAST(sqlparser.CloneStatement(stmt), vschema.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil)
 	reservedVars := sqlparser.NewReservedVars("vtg", reserved)
 
 	simplified := simplifier.SimplifyStatement(
 		stmt.(sqlparser.SelectStatement),
-		vschema.currentDb(),
+		vschema.CurrentDb(),
 		vschema,
 		keepPanicking(query, reservedVars, vschema, rewritten.BindVarNeeds),
 	)
@@ -81,9 +82,9 @@ func TestSimplifyPanic(t *testing.T) {
 
 func TestUnsupportedFile(t *testing.T) {
 	t.Skip("run manually to see if any queries can be simplified")
-	vschema := &vschemaWrapper{
-		v:       loadSchema(t, "vschemas/schema.json", true),
-		version: Gen4,
+	vschema := &utils.VSchemaWrapper{
+		V:       loadSchema(t, "vschemas/schema.json", true),
+		Version: Gen4,
 	}
 	fmt.Println(vschema)
 	for _, tcase := range readJSONTests("unsupported_cases.txt") {
@@ -96,11 +97,11 @@ func TestUnsupportedFile(t *testing.T) {
 				t.Skip()
 				return
 			}
-			rewritten, err := sqlparser.RewriteAST(stmt, vschema.currentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil)
+			rewritten, err := sqlparser.RewriteAST(stmt, vschema.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil)
 			if err != nil {
 				t.Skip()
 			}
-			vschema.currentDb()
+			vschema.CurrentDb()
 
 			reservedVars := sqlparser.NewReservedVars("vtg", reserved)
 			ast := rewritten.AST
@@ -108,7 +109,7 @@ func TestUnsupportedFile(t *testing.T) {
 			stmt, _, _ = sqlparser.Parse2(tcase.Query)
 			simplified := simplifier.SimplifyStatement(
 				stmt.(sqlparser.SelectStatement),
-				vschema.currentDb(),
+				vschema.CurrentDb(),
 				vschema,
 				keepSameError(tcase.Query, reservedVars, vschema, rewritten.BindVarNeeds),
 			)
@@ -125,12 +126,12 @@ func TestUnsupportedFile(t *testing.T) {
 	}
 }
 
-func keepSameError(query string, reservedVars *sqlparser.ReservedVars, vschema *vschemaWrapper, needs *sqlparser.BindVarNeeds) func(statement sqlparser.SelectStatement) bool {
+func keepSameError(query string, reservedVars *sqlparser.ReservedVars, vschema *utils.VSchemaWrapper, needs *sqlparser.BindVarNeeds) func(statement sqlparser.SelectStatement) bool {
 	stmt, _, err := sqlparser.Parse2(query)
 	if err != nil {
 		panic(err)
 	}
-	rewritten, _ := sqlparser.RewriteAST(stmt, vschema.currentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil)
+	rewritten, _ := sqlparser.RewriteAST(stmt, vschema.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil)
 	ast := rewritten.AST
 	_, expected := BuildFromStmt(context.Background(), query, ast, reservedVars, vschema, rewritten.BindVarNeeds, true, true)
 	if expected == nil {
@@ -149,7 +150,7 @@ func keepSameError(query string, reservedVars *sqlparser.ReservedVars, vschema *
 	}
 }
 
-func keepPanicking(query string, reservedVars *sqlparser.ReservedVars, vschema *vschemaWrapper, needs *sqlparser.BindVarNeeds) func(statement sqlparser.SelectStatement) bool {
+func keepPanicking(query string, reservedVars *sqlparser.ReservedVars, vschema *utils.VSchemaWrapper, needs *sqlparser.BindVarNeeds) func(statement sqlparser.SelectStatement) bool {
 	cmp := func(statement sqlparser.SelectStatement) (res bool) {
 		defer func() {
 			r := recover()

From a0acb5a03f5f15b0d4c0ecd9f63b95dc7cadd4b9 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Sun, 23 Jul 2023 22:30:15 -0700
Subject: [PATCH 17/27] added query simplification to fuzzer

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/random_test.go      |  4 +-
 .../endtoend/vtgate/queries/random/schema.sql | 21 +++++
 .../vtgate/queries/random/simplifier_test.go  | 84 +++++++++++++++++++
 .../vtgate/queries/random/svschema.json       | 60 +++++++++++++
 .../vtgate/queries/random/vschema.json        | 16 ++++
 5 files changed, 184 insertions(+), 1 deletion(-)
 create mode 100644 go/test/endtoend/vtgate/queries/random/simplifier_test.go
 create mode 100644 go/test/endtoend/vtgate/queries/random/svschema.json

diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index a8f3ce3d6ce..a83dfb17b22 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -44,7 +44,7 @@ func start(t *testing.T) (utils.MySQLCompare, func()) {
 	deleteAll := func() {
 		_, _ = utils.ExecAllowError(t, mcmp.VtConn, "set workload = oltp")
 
-		tables := []string{"dept", "emp"}
+		tables := []string{"emp", "dept", "uemp", "udept"}
 		for _, table := range tables {
 			_, _ = mcmp.ExecAndIgnore("delete from " + table)
 		}
@@ -58,6 +58,8 @@ func start(t *testing.T) (utils.MySQLCompare, func()) {
 	// insert data
 	mcmp.Exec("INSERT INTO emp(empno, ename, job, mgr, hiredate, sal, comm, deptno) VALUES (7369,'SMITH','CLERK',7902,'1980-12-17',800,NULL,20), (7499,'ALLEN','SALESMAN',7698,'1981-02-20',1600,300,30), (7521,'WARD','SALESMAN',7698,'1981-02-22',1250,500,30), (7566,'JONES','MANAGER',7839,'1981-04-02',2975,NULL,20), (7654,'MARTIN','SALESMAN',7698,'1981-09-28',1250,1400,30), (7698,'BLAKE','MANAGER',7839,'1981-05-01',2850,NULL,30), (7782,'CLARK','MANAGER',7839,'1981-06-09',2450,NULL,10), (7788,'SCOTT','ANALYST',7566,'1982-12-09',3000,NULL,20), (7839,'KING','PRESIDENT',NULL,'1981-11-17',5000,NULL,10), (7844,'TURNER','SALESMAN',7698,'1981-09-08',1500,0,30), (7876,'ADAMS','CLERK',7788,'1983-01-12',1100,NULL,20), (7900,'JAMES','CLERK',7698,'1981-12-03',950,NULL,30), (7902,'FORD','ANALYST',7566,'1981-12-03',3000,NULL,20), (7934,'MILLER','CLERK',7782,'1982-01-23',1300,NULL,10)")
 	mcmp.Exec("INSERT INTO dept(deptno, dname, loc) VALUES ('10','ACCOUNTING','NEW YORK'), ('20','RESEARCH','DALLAS'), ('30','SALES','CHICAGO'), ('40','OPERATIONS','BOSTON')")
+	mcmp.Exec("INSERT INTO uemp(empno, ename, job, mgr, hiredate, sal, comm, deptno) VALUES (7369,'SMITH','CLERK',7902,'1980-12-17',800,NULL,20), (7499,'ALLEN','SALESMAN',7698,'1981-02-20',1600,300,30), (7521,'WARD','SALESMAN',7698,'1981-02-22',1250,500,30), (7566,'JONES','MANAGER',7839,'1981-04-02',2975,NULL,20), (7654,'MARTIN','SALESMAN',7698,'1981-09-28',1250,1400,30), (7698,'BLAKE','MANAGER',7839,'1981-05-01',2850,NULL,30), (7782,'CLARK','MANAGER',7839,'1981-06-09',2450,NULL,10), (7788,'SCOTT','ANALYST',7566,'1982-12-09',3000,NULL,20), (7839,'KING','PRESIDENT',NULL,'1981-11-17',5000,NULL,10), (7844,'TURNER','SALESMAN',7698,'1981-09-08',1500,0,30), (7876,'ADAMS','CLERK',7788,'1983-01-12',1100,NULL,20), (7900,'JAMES','CLERK',7698,'1981-12-03',950,NULL,30), (7902,'FORD','ANALYST',7566,'1981-12-03',3000,NULL,20), (7934,'MILLER','CLERK',7782,'1982-01-23',1300,NULL,10)")
+	mcmp.Exec("INSERT INTO udept(deptno, dname, loc) VALUES ('10','ACCOUNTING','NEW YORK'), ('20','RESEARCH','DALLAS'), ('30','SALES','CHICAGO'), ('40','OPERATIONS','BOSTON')")
 
 	return mcmp, func() {
 		deleteAll()
diff --git a/go/test/endtoend/vtgate/queries/random/schema.sql b/go/test/endtoend/vtgate/queries/random/schema.sql
index 7ef4721a381..ceff307ad90 100644
--- a/go/test/endtoend/vtgate/queries/random/schema.sql
+++ b/go/test/endtoend/vtgate/queries/random/schema.sql
@@ -16,5 +16,26 @@ CREATE TABLE dept (
  DNAME VARCHAR(14),
  LOC VARCHAR(13),
  PRIMARY KEY (DEPTNO)
+) Engine = InnoDB
+  COLLATE = utf8mb4_general_ci;
+
+CREATE TABLE uemp (
+ empno bigint NOT NULL,
+ ename VARCHAR(10),
+ job VARCHAR(9),
+ mgr bigint,
+ hiredate DATE,
+ sal bigint,
+ comm bigint,
+ deptno bigint,
+ PRIMARY KEY (empno)
+) Engine = InnoDB
+  COLLATE = utf8mb4_general_ci;
+
+CREATE TABLE udept (
+ deptno bigint,
+ dname VARCHAR(14),
+ loc VARCHAR(13),
+ PRIMARY KEY (deptno)
 ) Engine = InnoDB
   COLLATE = utf8mb4_general_ci;
\ No newline at end of file
diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
new file mode 100644
index 00000000000..74aed5e28ca
--- /dev/null
+++ b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
@@ -0,0 +1,84 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package random
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/test/endtoend/utils"
+	utils2 "vitess.io/vitess/go/test/utils"
+	"vitess.io/vitess/go/vt/sqlparser"
+	"vitess.io/vitess/go/vt/vtgate/planbuilder"
+	"vitess.io/vitess/go/vt/vtgate/simplifier"
+	"vitess.io/vitess/go/vt/vtgate/vindexes"
+)
+
+func TestSimplifyBuggyQuery(t *testing.T) {
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "uemp", clusterInstance.VtgateProcess.ReadVSchema))
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "udept", clusterInstance.VtgateProcess.ReadVSchema))
+
+	query := "select /*vt+ PLANNER=Gen4 */ distinct 'opossum' and count(*) as caggr0, case count('mustang') > -36 when min(18 * -41) then 22 - max(tbl0.sal) else 7 end as caggr1 from emp as tbl0 where case when false then tbl0.ename when 17 then 'gator' else 'mite' end and case false and true when 'worm' then tbl0.job when tbl0.ename then case when true then tbl0.ename when false then 'squirrel' end end"
+
+	var err error
+	t.Helper()
+	t.Run(query, func(t *testing.T) {
+		mcmp, closer := start(t)
+		defer closer()
+
+		mcmp.ExecAllowAndCompareError(query)
+	})
+
+	formal, err := vindexes.LoadFormal("svschema.json")
+	require.NoError(t, err)
+
+	vSchema := vindexes.BuildVSchema(formal)
+
+	vSchemaWrapper := &utils2.VSchemaWrapper{
+		V:       vSchema,
+		Version: planbuilder.Gen4,
+	}
+
+	stmt, err := sqlparser.Parse(query)
+	require.NoError(t, err)
+
+	simplified := simplifier.SimplifyStatement(
+		stmt.(sqlparser.SelectStatement),
+		"ks_random",
+		vSchemaWrapper,
+		func(statement sqlparser.SelectStatement) (sameError bool) {
+			q := sqlparser.String(statement)
+			t.Run(q, func(t *testing.T) {
+				mcmp, closer := start(t)
+				defer closer()
+
+				_, newErr := mcmp.ExecAllowAndCompareError(q)
+				if newErr == nil {
+					sameError = false
+				} else {
+					sameError = strings.Contains(newErr.Error(), "mismatched")
+				}
+			})
+			return
+		},
+	)
+
+	fmt.Printf("final simplified query: %s\n, ", sqlparser.String(simplified))
+}
diff --git a/go/test/endtoend/vtgate/queries/random/svschema.json b/go/test/endtoend/vtgate/queries/random/svschema.json
new file mode 100644
index 00000000000..cef93a34c84
--- /dev/null
+++ b/go/test/endtoend/vtgate/queries/random/svschema.json
@@ -0,0 +1,60 @@
+{
+  "keyspaces": {
+    "ks_random": {
+      "tables": {
+        "uemp": {
+          "columns": [
+            {
+              "name": "empno",
+              "type": "INT64"
+            },
+            {
+              "name": "ename",
+              "type": "VARCHAR"
+            },
+            {
+              "name": "job",
+              "type": "VARCHAR"
+            },
+            {
+              "name": "mgr",
+              "type": "INT64"
+            },
+            {
+              "name": "hiredate",
+              "type": "DATE"
+            },
+            {
+              "name": "sal",
+              "type": "INT64"
+            },
+            {
+              "name": "comm",
+              "type": "INT64"
+            },
+            {
+              "name": "deptno",
+              "type": "INT64"
+            }
+          ]
+        },
+        "udept": {
+          "columns": [
+            {
+              "name": "deptno",
+              "type": "INT64"
+            },
+            {
+              "name": "dname",
+              "type": "VARCHAR"
+            },
+            {
+              "name": "loc",
+              "type": "VARCHAR"
+            }
+          ]
+        }
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/go/test/endtoend/vtgate/queries/random/vschema.json b/go/test/endtoend/vtgate/queries/random/vschema.json
index 21e31d5618c..de8ad73eae0 100644
--- a/go/test/endtoend/vtgate/queries/random/vschema.json
+++ b/go/test/endtoend/vtgate/queries/random/vschema.json
@@ -21,6 +21,22 @@
           "name": "hash"
         }
       ]
+    },
+    "uemp": {
+      "column_vindexes": [
+        {
+          "column": "deptno",
+          "name": "hash"
+        }
+      ]
+    },
+    "udept": {
+      "column_vindexes": [
+        {
+          "column": "deptno",
+          "name": "hash"
+        }
+      ]
     }
   }
 }
\ No newline at end of file

From a28cd5fa6152fb7e1b523ba65a7f9c4d867492fe Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Sun, 23 Jul 2023 22:46:44 -0700
Subject: [PATCH 18/27] simplified vschema and schema for simplifying fuzzer
 queries

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/random_test.go      |  4 +-
 .../endtoend/vtgate/queries/random/schema.sql | 21 --------
 .../vtgate/queries/random/simplifier_test.go  |  4 +-
 .../vtgate/queries/random/svschema.json       | 54 -------------------
 .../vtgate/queries/random/vschema.json        | 16 ------
 5 files changed, 3 insertions(+), 96 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index a83dfb17b22..74be62a60be 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -44,7 +44,7 @@ func start(t *testing.T) (utils.MySQLCompare, func()) {
 	deleteAll := func() {
 		_, _ = utils.ExecAllowError(t, mcmp.VtConn, "set workload = oltp")
 
-		tables := []string{"emp", "dept", "uemp", "udept"}
+		tables := []string{"emp", "dept"}
 		for _, table := range tables {
 			_, _ = mcmp.ExecAndIgnore("delete from " + table)
 		}
@@ -58,8 +58,6 @@ func start(t *testing.T) (utils.MySQLCompare, func()) {
 	// insert data
 	mcmp.Exec("INSERT INTO emp(empno, ename, job, mgr, hiredate, sal, comm, deptno) VALUES (7369,'SMITH','CLERK',7902,'1980-12-17',800,NULL,20), (7499,'ALLEN','SALESMAN',7698,'1981-02-20',1600,300,30), (7521,'WARD','SALESMAN',7698,'1981-02-22',1250,500,30), (7566,'JONES','MANAGER',7839,'1981-04-02',2975,NULL,20), (7654,'MARTIN','SALESMAN',7698,'1981-09-28',1250,1400,30), (7698,'BLAKE','MANAGER',7839,'1981-05-01',2850,NULL,30), (7782,'CLARK','MANAGER',7839,'1981-06-09',2450,NULL,10), (7788,'SCOTT','ANALYST',7566,'1982-12-09',3000,NULL,20), (7839,'KING','PRESIDENT',NULL,'1981-11-17',5000,NULL,10), (7844,'TURNER','SALESMAN',7698,'1981-09-08',1500,0,30), (7876,'ADAMS','CLERK',7788,'1983-01-12',1100,NULL,20), (7900,'JAMES','CLERK',7698,'1981-12-03',950,NULL,30), (7902,'FORD','ANALYST',7566,'1981-12-03',3000,NULL,20), (7934,'MILLER','CLERK',7782,'1982-01-23',1300,NULL,10)")
 	mcmp.Exec("INSERT INTO dept(deptno, dname, loc) VALUES ('10','ACCOUNTING','NEW YORK'), ('20','RESEARCH','DALLAS'), ('30','SALES','CHICAGO'), ('40','OPERATIONS','BOSTON')")
-	mcmp.Exec("INSERT INTO uemp(empno, ename, job, mgr, hiredate, sal, comm, deptno) VALUES (7369,'SMITH','CLERK',7902,'1980-12-17',800,NULL,20), (7499,'ALLEN','SALESMAN',7698,'1981-02-20',1600,300,30), (7521,'WARD','SALESMAN',7698,'1981-02-22',1250,500,30), (7566,'JONES','MANAGER',7839,'1981-04-02',2975,NULL,20), (7654,'MARTIN','SALESMAN',7698,'1981-09-28',1250,1400,30), (7698,'BLAKE','MANAGER',7839,'1981-05-01',2850,NULL,30), (7782,'CLARK','MANAGER',7839,'1981-06-09',2450,NULL,10), (7788,'SCOTT','ANALYST',7566,'1982-12-09',3000,NULL,20), (7839,'KING','PRESIDENT',NULL,'1981-11-17',5000,NULL,10), (7844,'TURNER','SALESMAN',7698,'1981-09-08',1500,0,30), (7876,'ADAMS','CLERK',7788,'1983-01-12',1100,NULL,20), (7900,'JAMES','CLERK',7698,'1981-12-03',950,NULL,30), (7902,'FORD','ANALYST',7566,'1981-12-03',3000,NULL,20), (7934,'MILLER','CLERK',7782,'1982-01-23',1300,NULL,10)")
-	mcmp.Exec("INSERT INTO udept(deptno, dname, loc) VALUES ('10','ACCOUNTING','NEW YORK'), ('20','RESEARCH','DALLAS'), ('30','SALES','CHICAGO'), ('40','OPERATIONS','BOSTON')")
 
 	return mcmp, func() {
 		deleteAll()
diff --git a/go/test/endtoend/vtgate/queries/random/schema.sql b/go/test/endtoend/vtgate/queries/random/schema.sql
index ceff307ad90..7ef4721a381 100644
--- a/go/test/endtoend/vtgate/queries/random/schema.sql
+++ b/go/test/endtoend/vtgate/queries/random/schema.sql
@@ -16,26 +16,5 @@ CREATE TABLE dept (
  DNAME VARCHAR(14),
  LOC VARCHAR(13),
  PRIMARY KEY (DEPTNO)
-) Engine = InnoDB
-  COLLATE = utf8mb4_general_ci;
-
-CREATE TABLE uemp (
- empno bigint NOT NULL,
- ename VARCHAR(10),
- job VARCHAR(9),
- mgr bigint,
- hiredate DATE,
- sal bigint,
- comm bigint,
- deptno bigint,
- PRIMARY KEY (empno)
-) Engine = InnoDB
-  COLLATE = utf8mb4_general_ci;
-
-CREATE TABLE udept (
- deptno bigint,
- dname VARCHAR(14),
- loc VARCHAR(13),
- PRIMARY KEY (deptno)
 ) Engine = InnoDB
   COLLATE = utf8mb4_general_ci;
\ No newline at end of file
diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
index 74aed5e28ca..f1842a3fffc 100644
--- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go
+++ b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
@@ -32,8 +32,8 @@ import (
 )
 
 func TestSimplifyBuggyQuery(t *testing.T) {
-	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "uemp", clusterInstance.VtgateProcess.ReadVSchema))
-	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "udept", clusterInstance.VtgateProcess.ReadVSchema))
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
 	query := "select /*vt+ PLANNER=Gen4 */ distinct 'opossum' and count(*) as caggr0, case count('mustang') > -36 when min(18 * -41) then 22 - max(tbl0.sal) else 7 end as caggr1 from emp as tbl0 where case when false then tbl0.ename when 17 then 'gator' else 'mite' end and case false and true when 'worm' then tbl0.job when tbl0.ename then case when true then tbl0.ename when false then 'squirrel' end end"
 
diff --git a/go/test/endtoend/vtgate/queries/random/svschema.json b/go/test/endtoend/vtgate/queries/random/svschema.json
index cef93a34c84..ccbbc6ed3a6 100644
--- a/go/test/endtoend/vtgate/queries/random/svschema.json
+++ b/go/test/endtoend/vtgate/queries/random/svschema.json
@@ -1,60 +1,6 @@
 {
   "keyspaces": {
     "ks_random": {
-      "tables": {
-        "uemp": {
-          "columns": [
-            {
-              "name": "empno",
-              "type": "INT64"
-            },
-            {
-              "name": "ename",
-              "type": "VARCHAR"
-            },
-            {
-              "name": "job",
-              "type": "VARCHAR"
-            },
-            {
-              "name": "mgr",
-              "type": "INT64"
-            },
-            {
-              "name": "hiredate",
-              "type": "DATE"
-            },
-            {
-              "name": "sal",
-              "type": "INT64"
-            },
-            {
-              "name": "comm",
-              "type": "INT64"
-            },
-            {
-              "name": "deptno",
-              "type": "INT64"
-            }
-          ]
-        },
-        "udept": {
-          "columns": [
-            {
-              "name": "deptno",
-              "type": "INT64"
-            },
-            {
-              "name": "dname",
-              "type": "VARCHAR"
-            },
-            {
-              "name": "loc",
-              "type": "VARCHAR"
-            }
-          ]
-        }
-      }
     }
   }
 }
\ No newline at end of file
diff --git a/go/test/endtoend/vtgate/queries/random/vschema.json b/go/test/endtoend/vtgate/queries/random/vschema.json
index de8ad73eae0..21e31d5618c 100644
--- a/go/test/endtoend/vtgate/queries/random/vschema.json
+++ b/go/test/endtoend/vtgate/queries/random/vschema.json
@@ -21,22 +21,6 @@
           "name": "hash"
         }
       ]
-    },
-    "uemp": {
-      "column_vindexes": [
-        {
-          "column": "deptno",
-          "name": "hash"
-        }
-      ]
-    },
-    "udept": {
-      "column_vindexes": [
-        {
-          "column": "deptno",
-          "name": "hash"
-        }
-      ]
     }
   }
 }
\ No newline at end of file

From ce019af95a82e741be344e17feaeda9db5e47dce Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Mon, 24 Jul 2023 02:38:16 -0700
Subject: [PATCH 19/27] added simplification for case expressions

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        |  2 +-
 .../vtgate/queries/random/random_test.go      | 51 ++++++++++---------
 .../vtgate/queries/random/simplifier_test.go  | 39 ++++++++++----
 .../simplifier/expression_simplifier.go       | 16 ++++++
 go/vt/vtgate/simplifier/simplifier.go         | 25 +++++----
 go/vt/vtgate/simplifier/simplifier_test.go    |  4 +-
 6 files changed, 89 insertions(+), 48 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index f6810856fb8..49873a048c0 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -374,7 +374,7 @@ func (sg *selectGenerator) createTablesAndJoin() ([]tableT, bool) {
 		tables[i+1].setAlias(alias)
 	}
 
-	// TODO: outer joins produce mismatched results
+	// TODO: outer joins produce results mismatched
 	isJoin := sg.r.Intn(2) < 1 && testFailingQueries
 	if isJoin {
 		// TODO: do nested joins
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 74be62a60be..c1197c983ad 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -34,7 +34,7 @@ import (
 
 // this test uses the AST defined in the sqlparser package to randomly generate queries
 
-// if true then execution will always stop on a "must fix" error: a mismatched results or EOF
+// if true then execution will always stop on a "must fix" error: a results mismatched or EOF
 const stopOnMustFixError = false
 
 func start(t *testing.T) (utils.MySQLCompare, func()) {
@@ -84,62 +84,62 @@ func TestMustFix(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct case count(*) when -41 ^ 10 then -17 when 22 then -52 else 7 end from emp as tbl0, emp as tbl1 where tbl1.job < tbl1.job")
 
-	// mismatched results (maybe derived tables)
+	// results mismatched (maybe derived tables)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1")
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'")
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1 where tbl0.deptno and tbl1.comm in (12, tbl0.deptno, case false when 67 then -17 when -78 then -35 end, -76 >> -68)")
 
 	// mismatched number of columns
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) + 1 from emp as tbl0 order by count(*) desc")
 
-	// mismatched results (mismatched types)
+	// results mismatched (mismatched types)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32")
 
-	// mismatched results (decimals off by a little; evalengine problem)
+	// results mismatched (decimals off by a little; evalengine problem)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1")
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
 
 	// EOF
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1, tbl1.deptno as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
 
-	// mismatched results
+	// results mismatched
 	// limit >= 9 works
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) = sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8")
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct count(*) as caggr1 from dept as tbl0, emp as tbl1 group by tbl1.sal having max(tbl1.comm) != true")
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(tbl1.loc) as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(tbl1.dname) <= 1")
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm")
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 1 = 0")
 
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 'octopus'")
 
 	// similar to previous two
-	// mismatched results
+	// results mismatched
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct 'octopus' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.empno having count(*) = count(*)")
 
-	// mismatched results (group by + right join)
+	// results mismatched (group by + right join)
 	// left instead of right works
 	// swapping tables and predicates and changing to left fails
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ max(tbl0.deptno) from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno")
 
-	// mismatched results (count + right join)
+	// results mismatched (count + right join)
 	// left instead of right works
 	// swapping tables and predicates and changing to left fails
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal")
@@ -288,7 +288,7 @@ func TestRandom(t *testing.T) {
 
 	endBy := time.Now().Add(1 * time.Second)
 
-	var queryCount int
+	var queryCount, queryFailCount int
 	// continue testing after an error if and only if testFailingQueries is true
 	for time.Now().Before(endBy) && (!t.Failed() || !testFailingQueries) {
 		seed := time.Now().UnixNano()
@@ -303,15 +303,16 @@ func TestRandom(t *testing.T) {
 			fmt.Printf("seed: %d\n", seed)
 			fmt.Println(query)
 			fmt.Println(vtErr)
-			fmt.Printf("\n\n\n")
 
 			if stopOnMustFixError {
-				// EOF
-				if sqlError, ok := vtErr.(*mysql.SQLError); ok && strings.Contains(sqlError.Message, "EOF") {
+				// results mismatched
+				if strings.Contains(vtErr.Error(), "results mismatched") {
+					simplified := simplifyResultsMismatchedQuery(t, query)
+					fmt.Printf("final simplified query: %s\n", simplified)
 					break
 				}
-				// mismatched results
-				if strings.Contains(vtErr.Error(), "results mismatched") {
+				// EOF
+				if sqlError, ok := vtErr.(*mysql.SQLError); ok && strings.Contains(sqlError.Message, "EOF") {
 					break
 				}
 			}
@@ -319,10 +320,14 @@ func TestRandom(t *testing.T) {
 			// restart the mysql and vitess connections in case something bad happened
 			closer()
 			mcmp, closer = start(t)
+
+			fmt.Printf("\n\n\n")
+			queryFailCount++
 		}
 		queryCount++
 	}
 	fmt.Printf("Queries successfully executed: %d\n", queryCount)
+	fmt.Printf("Queries failed: %d\n", queryFailCount)
 }
 
 // these queries were previously failing and have now been fixed
diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
index f1842a3fffc..4f5f079e214 100644
--- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go
+++ b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
@@ -17,7 +17,6 @@ limitations under the License.
 package random
 
 import (
-	"fmt"
 	"strings"
 	"testing"
 
@@ -31,26 +30,47 @@ import (
 	"vitess.io/vitess/go/vt/vtgate/vindexes"
 )
 
-func TestSimplifyBuggyQuery(t *testing.T) {
+func TestSimplifyResultsMismatchedQuery(t *testing.T) {
+	// t.Skip("Skip CI")
+	query := "select /*vt+ PLANNER=Gen4 */ distinct 'opossum' and count(*) as caggr0, case count('mustang') > -36 when min(18 * -41) then 22 - max(tbl0.sal) else 7 end as caggr1 from emp as tbl0 where case when false then tbl0.ename when 17 then 'gator' else 'mite' end and case false and true when 'worm' then tbl0.job when tbl0.ename then case when true then tbl0.ename when false then 'squirrel' end end"
+	simplified := simplifyResultsMismatchedQuery(t, query)
+
+	var err error
+	t.Helper()
+	t.Run(simplified, func(t *testing.T) {
+		mcmp, closer := start(t)
+		defer closer()
+
+		_, err = mcmp.ExecAllowAndCompareError(simplified)
+	})
+
+	require.ErrorContains(t, err, "mismatched")
+
+}
+
+// TODO: suppress output from comparing intermediate simplified results
+// given a query that errors with results mismatched, simplifyResultsMismatchedQuery returns a simpler version with the same error
+func simplifyResultsMismatchedQuery(t *testing.T, query string) string {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
-	query := "select /*vt+ PLANNER=Gen4 */ distinct 'opossum' and count(*) as caggr0, case count('mustang') > -36 when min(18 * -41) then 22 - max(tbl0.sal) else 7 end as caggr1 from emp as tbl0 where case when false then tbl0.ename when 17 then 'gator' else 'mite' end and case false and true when 'worm' then tbl0.job when tbl0.ename then case when true then tbl0.ename when false then 'squirrel' end end"
-
 	var err error
 	t.Helper()
 	t.Run(query, func(t *testing.T) {
 		mcmp, closer := start(t)
 		defer closer()
 
-		mcmp.ExecAllowAndCompareError(query)
+		_, err = mcmp.ExecAllowAndCompareError(query)
+		if err == nil {
+			t.Fatalf("query (%s) does not error", query)
+		} else if !strings.Contains(err.Error(), "mismatched") {
+			t.Fatalf("query (%s) does not error with results mismatched\nError: %v", query, err)
+		}
 	})
 
 	formal, err := vindexes.LoadFormal("svschema.json")
 	require.NoError(t, err)
-
 	vSchema := vindexes.BuildVSchema(formal)
-
 	vSchemaWrapper := &utils2.VSchemaWrapper{
 		V:       vSchema,
 		Version: planbuilder.Gen4,
@@ -61,10 +81,11 @@ func TestSimplifyBuggyQuery(t *testing.T) {
 
 	simplified := simplifier.SimplifyStatement(
 		stmt.(sqlparser.SelectStatement),
-		"ks_random",
+		vSchemaWrapper.CurrentDb(),
 		vSchemaWrapper,
 		func(statement sqlparser.SelectStatement) (sameError bool) {
 			q := sqlparser.String(statement)
+			t.Helper()
 			t.Run(q, func(t *testing.T) {
 				mcmp, closer := start(t)
 				defer closer()
@@ -80,5 +101,5 @@ func TestSimplifyBuggyQuery(t *testing.T) {
 		},
 	)
 
-	fmt.Printf("final simplified query: %s\n, ", sqlparser.String(simplified))
+	return sqlparser.String(simplified)
 }
diff --git a/go/vt/vtgate/simplifier/expression_simplifier.go b/go/vt/vtgate/simplifier/expression_simplifier.go
index 279cb1ac7dd..10209d4fd85 100644
--- a/go/vt/vtgate/simplifier/expression_simplifier.go
+++ b/go/vt/vtgate/simplifier/expression_simplifier.go
@@ -142,6 +142,10 @@ func (s *shrinker) Next() sqlparser.Expr {
 func (s *shrinker) fillQueue() bool {
 	before := len(s.queue)
 	switch e := s.orig.(type) {
+	case *sqlparser.AndExpr:
+		s.queue = append(s.queue, e.Left, e.Right)
+	case *sqlparser.OrExpr:
+		s.queue = append(s.queue, e.Left, e.Right)
 	case *sqlparser.ComparisonExpr:
 		s.queue = append(s.queue, e.Left, e.Right)
 	case *sqlparser.BinaryExpr:
@@ -231,6 +235,18 @@ func (s *shrinker) fillQueue() bool {
 	case *sqlparser.ColName:
 		// we can try to replace the column with a literal value
 		s.queue = []sqlparser.Expr{sqlparser.NewIntLiteral("0")}
+	case *sqlparser.CaseExpr:
+		for i := range e.Whens {
+			whensCopy := sqlparser.CloneSliceOfRefOfWhen(e.Whens)
+			// replace ith element with last element, then truncate last element
+			whensCopy[i] = whensCopy[len(whensCopy)-1]
+			whensCopy = whensCopy[:len(whensCopy)-1]
+			s.queue = append(s.queue, &sqlparser.CaseExpr{
+				Expr:  e.Expr,
+				Whens: whensCopy,
+				Else:  e.Else,
+			})
+		}
 	default:
 		return false
 	}
diff --git a/go/vt/vtgate/simplifier/simplifier.go b/go/vt/vtgate/simplifier/simplifier.go
index ef7be4e30e5..21a4d8ca3c8 100644
--- a/go/vt/vtgate/simplifier/simplifier.go
+++ b/go/vt/vtgate/simplifier/simplifier.go
@@ -70,27 +70,26 @@ func trySimplifyExpressions(in sqlparser.SelectStatement, test func(sqlparser.Se
 			if test(in) {
 				log.Errorf("removed expression: %s", sqlparser.String(cursor.expr))
 				simplified = true
-				return false
+				// initially return false, but that made the rewriter prematurely abort, if it was the last selectExpr
+				return true
 			}
 			cursor.restore()
 		}
 
-		// ok, we seem to need this expression. let's see if we can find a simpler version
-		s := &shrinker{orig: cursor.expr}
-		newExpr := s.Next()
-		for newExpr != nil {
-			cursor.replace(newExpr)
+		newExpr := SimplifyExpr(cursor.expr, func(expr sqlparser.Expr) bool {
+			cursor.replace(expr)
 			if test(in) {
-				log.Errorf("simplified expression: %s -> %s", sqlparser.String(cursor.expr), sqlparser.String(newExpr))
+				log.Errorf("simplified expression: %s -> %s", sqlparser.String(cursor.expr), sqlparser.String(expr))
+				cursor.restore()
 				simplified = true
-				return false
+				return true
 			}
-			newExpr = s.Next()
-		}
 
-		// if we get here, we failed to simplify this expression,
-		// so we put back in the original expression
-		cursor.restore()
+			cursor.restore()
+			return false
+		})
+
+		cursor.replace(newExpr)
 		return true
 	})
 
diff --git a/go/vt/vtgate/simplifier/simplifier_test.go b/go/vt/vtgate/simplifier/simplifier_test.go
index 63f43a7febb..1eaede18838 100644
--- a/go/vt/vtgate/simplifier/simplifier_test.go
+++ b/go/vt/vtgate/simplifier/simplifier_test.go
@@ -54,11 +54,11 @@ limit 123 offset 456
 	require.NoError(t, err)
 	visitAllExpressionsInAST(ast.(sqlparser.SelectStatement), func(cursor expressionCursor) bool {
 		fmt.Printf(">> found expression: %s\n", sqlparser.String(cursor.expr))
-		cursor.replace(sqlparser.NewIntLiteral("1"))
+		cursor.remove()
 		fmt.Printf("remove: %s\n", sqlparser.String(ast))
 		cursor.restore()
 		fmt.Printf("restore: %s\n", sqlparser.String(ast))
-		cursor.remove()
+		cursor.replace(sqlparser.NewIntLiteral("1"))
 		fmt.Printf("replace it with literal: %s\n", sqlparser.String(ast))
 		cursor.restore()
 		fmt.Printf("restore: %s\n", sqlparser.String(ast))

From 10c20345a87cced4995df933c2049c7510b5c46c Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Mon, 24 Jul 2023 03:30:25 -0700
Subject: [PATCH 20/27] moved VSchemaWrapper to its own package

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/simplifier_test.go  |  3 +-
 .../vschema_wrapper.go                        |  2 +-
 go/vt/vtgate/planbuilder/collations_test.go   |  7 ++-
 go/vt/vtgate/planbuilder/plan_test.go         | 60 ++++++++++---------
 go/vt/vtgate/planbuilder/show_test.go         |  5 +-
 go/vt/vtgate/planbuilder/simplifier_test.go   | 13 ++--
 6 files changed, 48 insertions(+), 42 deletions(-)
 rename go/test/{utils => vschemawrapper}/vschema_wrapper.go (99%)

diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
index 4f5f079e214..f333339f4b8 100644
--- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go
+++ b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
@@ -20,10 +20,11 @@ import (
 	"strings"
 	"testing"
 
+	utils2 "vitess.io/vitess/go/test/vschemawrapper"
+
 	"github.com/stretchr/testify/require"
 
 	"vitess.io/vitess/go/test/endtoend/utils"
-	utils2 "vitess.io/vitess/go/test/utils"
 	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/vtgate/planbuilder"
 	"vitess.io/vitess/go/vt/vtgate/simplifier"
diff --git a/go/test/utils/vschema_wrapper.go b/go/test/vschemawrapper/vschema_wrapper.go
similarity index 99%
rename from go/test/utils/vschema_wrapper.go
rename to go/test/vschemawrapper/vschema_wrapper.go
index 9708e0f009b..af40e1174c8 100644
--- a/go/test/utils/vschema_wrapper.go
+++ b/go/test/vschemawrapper/vschema_wrapper.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package utils
+package vschemawrapper
 
 import (
 	"context"
diff --git a/go/vt/vtgate/planbuilder/collations_test.go b/go/vt/vtgate/planbuilder/collations_test.go
index 534dbc7dda2..bc88749204b 100644
--- a/go/vt/vtgate/planbuilder/collations_test.go
+++ b/go/vt/vtgate/planbuilder/collations_test.go
@@ -19,7 +19,8 @@ package planbuilder
 import (
 	"fmt"
 	"testing"
-	"vitess.io/vitess/go/test/utils"
+
+	"vitess.io/vitess/go/test/vschemawrapper"
 
 	"github.com/stretchr/testify/require"
 
@@ -40,7 +41,7 @@ type collationTestCase struct {
 }
 
 func (tc *collationTestCase) run(t *testing.T) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(t, "vschemas/schema.json", false),
 		SysVarEnabled: true,
 		Version:       Gen4,
@@ -52,7 +53,7 @@ func (tc *collationTestCase) run(t *testing.T) {
 	tc.check(t, tc.collations, plan.Instructions)
 }
 
-func (tc *collationTestCase) addCollationsToSchema(vschema *utils.VSchemaWrapper) {
+func (tc *collationTestCase) addCollationsToSchema(vschema *vschemawrapper.VSchemaWrapper) {
 	for _, collation := range tc.collations {
 		tbl := vschema.V.Keyspaces[collation.ks].Tables[collation.table]
 		for i, c := range tbl.Columns {
diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go
index 15f28ef8b1c..f0a76316405 100644
--- a/go/vt/vtgate/planbuilder/plan_test.go
+++ b/go/vt/vtgate/planbuilder/plan_test.go
@@ -28,6 +28,8 @@ import (
 	"strings"
 	"testing"
 
+	"vitess.io/vitess/go/test/vschemawrapper"
+
 	"github.com/nsf/jsondiff"
 	"github.com/stretchr/testify/require"
 
@@ -53,7 +55,7 @@ func makeTestOutput(t *testing.T) string {
 }
 
 func TestPlan(t *testing.T) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(t, "vschemas/schema.json", true),
 		TabletType_:   topodatapb.TabletType_PRIMARY,
 		SysVarEnabled: true,
@@ -99,13 +101,13 @@ func TestSystemTables57(t *testing.T) {
 	// first we move everything to use 5.7 logic
 	servenv.SetMySQLServerVersionForTest("5.7")
 	defer servenv.SetMySQLServerVersionForTest("")
-	vschemaWrapper := &utils.VSchemaWrapper{V: loadSchema(t, "vschemas/schema.json", true)}
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{V: loadSchema(t, "vschemas/schema.json", true)}
 	testOutputTempDir := makeTestOutput(t)
 	testFile(t, "info_schema57_cases.json", testOutputTempDir, vschemaWrapper, false)
 }
 
 func TestSysVarSetDisabled(t *testing.T) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(t, "vschemas/schema.json", true),
 		SysVarEnabled: false,
 	}
@@ -114,7 +116,7 @@ func TestSysVarSetDisabled(t *testing.T) {
 }
 
 func TestViews(t *testing.T) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:           loadSchema(t, "vschemas/schema.json", true),
 		EnableViews: true,
 	}
@@ -126,7 +128,7 @@ func TestOne(t *testing.T) {
 	reset := oprewriters.EnableDebugPrinting()
 	defer reset()
 
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 	}
 
@@ -134,7 +136,7 @@ func TestOne(t *testing.T) {
 }
 
 func TestOneTPCC(t *testing.T) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/tpcc_schema.json", true),
 	}
 
@@ -142,7 +144,7 @@ func TestOneTPCC(t *testing.T) {
 }
 
 func TestOneWithMainAsDefault(t *testing.T) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
@@ -154,7 +156,7 @@ func TestOneWithMainAsDefault(t *testing.T) {
 }
 
 func TestOneWithSecondUserAsDefault(t *testing.T) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "second_user",
@@ -166,7 +168,7 @@ func TestOneWithSecondUserAsDefault(t *testing.T) {
 }
 
 func TestOneWithUserAsDefault(t *testing.T) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "user",
@@ -178,7 +180,7 @@ func TestOneWithUserAsDefault(t *testing.T) {
 }
 
 func TestOneWithTPCHVSchema(t *testing.T) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(t, "vschemas/tpch_schema.json", true),
 		SysVarEnabled: true,
 	}
@@ -190,13 +192,13 @@ func TestOneWith57Version(t *testing.T) {
 	// first we move everything to use 5.7 logic
 	servenv.SetMySQLServerVersionForTest("5.7")
 	defer servenv.SetMySQLServerVersionForTest("")
-	vschema := &utils.VSchemaWrapper{V: loadSchema(t, "vschemas/schema.json", true)}
+	vschema := &vschemawrapper.VSchemaWrapper{V: loadSchema(t, "vschemas/schema.json", true)}
 
 	testFile(t, "onecase.json", "", vschema, false)
 }
 
 func TestRubyOnRailsQueries(t *testing.T) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(t, "vschemas/rails_schema.json", true),
 		SysVarEnabled: true,
 	}
@@ -205,7 +207,7 @@ func TestRubyOnRailsQueries(t *testing.T) {
 }
 
 func TestOLTP(t *testing.T) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(t, "vschemas/oltp_schema.json", true),
 		SysVarEnabled: true,
 	}
@@ -214,7 +216,7 @@ func TestOLTP(t *testing.T) {
 }
 
 func TestTPCC(t *testing.T) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(t, "vschemas/tpcc_schema.json", true),
 		SysVarEnabled: true,
 	}
@@ -223,7 +225,7 @@ func TestTPCC(t *testing.T) {
 }
 
 func TestTPCH(t *testing.T) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(t, "vschemas/tpch_schema.json", true),
 		SysVarEnabled: true,
 	}
@@ -244,7 +246,7 @@ func BenchmarkTPCH(b *testing.B) {
 }
 
 func benchmarkWorkload(b *testing.B, name string) {
-	vschemaWrapper := &utils.VSchemaWrapper{
+	vschemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(b, "vschemas/"+name+"_schema.json", true),
 		SysVarEnabled: true,
 	}
@@ -259,7 +261,7 @@ func benchmarkWorkload(b *testing.B, name string) {
 }
 
 func TestBypassPlanningShardTargetFromFile(t *testing.T) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
@@ -274,7 +276,7 @@ func TestBypassPlanningShardTargetFromFile(t *testing.T) {
 func TestBypassPlanningKeyrangeTargetFromFile(t *testing.T) {
 	keyRange, _ := key.ParseShardingSpec("-")
 
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
@@ -289,7 +291,7 @@ func TestBypassPlanningKeyrangeTargetFromFile(t *testing.T) {
 
 func TestWithDefaultKeyspaceFromFile(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
@@ -322,7 +324,7 @@ func TestWithDefaultKeyspaceFromFile(t *testing.T) {
 
 func TestWithDefaultKeyspaceFromFileSharded(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "second_user",
@@ -337,7 +339,7 @@ func TestWithDefaultKeyspaceFromFileSharded(t *testing.T) {
 
 func TestWithUserDefaultKeyspaceFromFileSharded(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "user",
@@ -352,7 +354,7 @@ func TestWithUserDefaultKeyspaceFromFileSharded(t *testing.T) {
 
 func TestWithSystemSchemaAsDefaultKeyspace(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V:           loadSchema(t, "vschemas/schema.json", true),
 		Keyspace:    &vindexes.Keyspace{Name: "information_schema"},
 		TabletType_: topodatapb.TabletType_PRIMARY,
@@ -363,7 +365,7 @@ func TestWithSystemSchemaAsDefaultKeyspace(t *testing.T) {
 
 func TestOtherPlanningFromFile(t *testing.T) {
 	// We are testing this separately so we can set a default keyspace
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V: loadSchema(t, "vschemas/schema.json", true),
 		Keyspace: &vindexes.Keyspace{
 			Name:    "main",
@@ -424,7 +426,7 @@ type (
 	}
 )
 
-func testFile(t *testing.T, filename, tempDir string, vschema *utils.VSchemaWrapper, render bool) {
+func testFile(t *testing.T, filename, tempDir string, vschema *vschemawrapper.VSchemaWrapper, render bool) {
 	opts := jsondiff.DefaultConsoleOptions()
 
 	t.Run(filename, func(t *testing.T) {
@@ -487,7 +489,7 @@ func readJSONTests(filename string) []planTest {
 	return output
 }
 
-func getPlanOutput(tcase planTest, vschema *utils.VSchemaWrapper, render bool) (out string) {
+func getPlanOutput(tcase planTest, vschema *vschemawrapper.VSchemaWrapper, render bool) (out string) {
 	defer func() {
 		if r := recover(); r != nil {
 			out = fmt.Sprintf("panicked: %v\n%s", r, string(debug.Stack()))
@@ -525,7 +527,7 @@ func locateFile(name string) string {
 var benchMarkFiles = []string{"from_cases.json", "filter_cases.json", "large_cases.json", "aggr_cases.json", "select_cases.json", "union_cases.json"}
 
 func BenchmarkPlanner(b *testing.B) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(b, "vschemas/schema.json", true),
 		SysVarEnabled: true,
 	}
@@ -541,7 +543,7 @@ func BenchmarkPlanner(b *testing.B) {
 }
 
 func BenchmarkSemAnalysis(b *testing.B) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(b, "vschemas/schema.json", true),
 		SysVarEnabled: true,
 	}
@@ -574,7 +576,7 @@ func exerciseAnalyzer(query, database string, s semantics.SchemaInformation) {
 }
 
 func BenchmarkSelectVsDML(b *testing.B) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V:             loadSchema(b, "vschemas/schema.json", true),
 		SysVarEnabled: true,
 		Version:       Gen4,
@@ -600,7 +602,7 @@ func BenchmarkSelectVsDML(b *testing.B) {
 	})
 }
 
-func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCases []planTest, vschema *utils.VSchemaWrapper) {
+func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCases []planTest, vschema *vschemawrapper.VSchemaWrapper) {
 	b.ReportAllocs()
 	for n := 0; n < b.N; n++ {
 		for _, tcase := range testCases {
diff --git a/go/vt/vtgate/planbuilder/show_test.go b/go/vt/vtgate/planbuilder/show_test.go
index 9401db2c741..04f8c571764 100644
--- a/go/vt/vtgate/planbuilder/show_test.go
+++ b/go/vt/vtgate/planbuilder/show_test.go
@@ -20,7 +20,8 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"vitess.io/vitess/go/test/utils"
+
+	"vitess.io/vitess/go/test/vschemawrapper"
 
 	"github.com/stretchr/testify/require"
 
@@ -32,7 +33,7 @@ import (
 )
 
 func TestBuildDBPlan(t *testing.T) {
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		Keyspace: &vindexes.Keyspace{Name: "main"},
 	}
 
diff --git a/go/vt/vtgate/planbuilder/simplifier_test.go b/go/vt/vtgate/planbuilder/simplifier_test.go
index 9e2473bb3c1..6830aaf1b63 100644
--- a/go/vt/vtgate/planbuilder/simplifier_test.go
+++ b/go/vt/vtgate/planbuilder/simplifier_test.go
@@ -20,7 +20,8 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"vitess.io/vitess/go/test/utils"
+
+	"vitess.io/vitess/go/test/vschemawrapper"
 
 	"vitess.io/vitess/go/vt/vterrors"
 
@@ -39,7 +40,7 @@ import (
 // It will try to minimize the query to make it easier to understand and work with the bug.
 func TestSimplifyBuggyQuery(t *testing.T) {
 	query := "(select id from unsharded union select id from unsharded_auto) union (select id from user union select name from unsharded)"
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V:       loadSchema(t, "vschemas/schema.json", true),
 		Version: Gen4,
 	}
@@ -61,7 +62,7 @@ func TestSimplifyBuggyQuery(t *testing.T) {
 func TestSimplifyPanic(t *testing.T) {
 	t.Skip("not needed to run")
 	query := "(select id from unsharded union select id from unsharded_auto) union (select id from unsharded_auto union select name from unsharded)"
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V:       loadSchema(t, "vschemas/schema.json", true),
 		Version: Gen4,
 	}
@@ -82,7 +83,7 @@ func TestSimplifyPanic(t *testing.T) {
 
 func TestUnsupportedFile(t *testing.T) {
 	t.Skip("run manually to see if any queries can be simplified")
-	vschema := &utils.VSchemaWrapper{
+	vschema := &vschemawrapper.VSchemaWrapper{
 		V:       loadSchema(t, "vschemas/schema.json", true),
 		Version: Gen4,
 	}
@@ -126,7 +127,7 @@ func TestUnsupportedFile(t *testing.T) {
 	}
 }
 
-func keepSameError(query string, reservedVars *sqlparser.ReservedVars, vschema *utils.VSchemaWrapper, needs *sqlparser.BindVarNeeds) func(statement sqlparser.SelectStatement) bool {
+func keepSameError(query string, reservedVars *sqlparser.ReservedVars, vschema *vschemawrapper.VSchemaWrapper, needs *sqlparser.BindVarNeeds) func(statement sqlparser.SelectStatement) bool {
 	stmt, _, err := sqlparser.Parse2(query)
 	if err != nil {
 		panic(err)
@@ -150,7 +151,7 @@ func keepSameError(query string, reservedVars *sqlparser.ReservedVars, vschema *
 	}
 }
 
-func keepPanicking(query string, reservedVars *sqlparser.ReservedVars, vschema *utils.VSchemaWrapper, needs *sqlparser.BindVarNeeds) func(statement sqlparser.SelectStatement) bool {
+func keepPanicking(query string, reservedVars *sqlparser.ReservedVars, vschema *vschemawrapper.VSchemaWrapper, needs *sqlparser.BindVarNeeds) func(statement sqlparser.SelectStatement) bool {
 	cmp := func(statement sqlparser.SelectStatement) (res bool) {
 		defer func() {
 			r := recover()

From a066e0981ea09fa25eed7c30ab17ac2c1f6b89d5 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Tue, 25 Jul 2023 01:06:00 -0700
Subject: [PATCH 21/27] updated caseExpr simplification and reverted reverted
 simplifier changes

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/simplifier_test.go  |  3 ++
 .../simplifier/expression_simplifier.go       | 38 ++++++++++++++-----
 2 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
index f333339f4b8..b9a7c6f51a0 100644
--- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go
+++ b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package random
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -34,6 +35,7 @@ import (
 func TestSimplifyResultsMismatchedQuery(t *testing.T) {
 	// t.Skip("Skip CI")
 	query := "select /*vt+ PLANNER=Gen4 */ distinct 'opossum' and count(*) as caggr0, case count('mustang') > -36 when min(18 * -41) then 22 - max(tbl0.sal) else 7 end as caggr1 from emp as tbl0 where case when false then tbl0.ename when 17 then 'gator' else 'mite' end and case false and true when 'worm' then tbl0.job when tbl0.ename then case when true then tbl0.ename when false then 'squirrel' end end"
+	// select /*vt+ PLANNER=Gen4 */ distinct case when min(-0) then 0 else 0 end as caggr1 from emp as tbl0 where false
 	simplified := simplifyResultsMismatchedQuery(t, query)
 
 	var err error
@@ -45,6 +47,7 @@ func TestSimplifyResultsMismatchedQuery(t *testing.T) {
 		_, err = mcmp.ExecAllowAndCompareError(simplified)
 	})
 
+	fmt.Printf("final simplified query: %s\n", simplified)
 	require.ErrorContains(t, err, "mismatched")
 
 }
diff --git a/go/vt/vtgate/simplifier/expression_simplifier.go b/go/vt/vtgate/simplifier/expression_simplifier.go
index 10209d4fd85..57ecc3ff67c 100644
--- a/go/vt/vtgate/simplifier/expression_simplifier.go
+++ b/go/vt/vtgate/simplifier/expression_simplifier.go
@@ -236,16 +236,34 @@ func (s *shrinker) fillQueue() bool {
 		// we can try to replace the column with a literal value
 		s.queue = []sqlparser.Expr{sqlparser.NewIntLiteral("0")}
 	case *sqlparser.CaseExpr:
-		for i := range e.Whens {
-			whensCopy := sqlparser.CloneSliceOfRefOfWhen(e.Whens)
-			// replace ith element with last element, then truncate last element
-			whensCopy[i] = whensCopy[len(whensCopy)-1]
-			whensCopy = whensCopy[:len(whensCopy)-1]
-			s.queue = append(s.queue, &sqlparser.CaseExpr{
-				Expr:  e.Expr,
-				Whens: whensCopy,
-				Else:  e.Else,
-			})
+		s.queue = append(s.queue, e.Expr, e.Else)
+		for _, when := range e.Whens {
+			s.queue = append(s.queue, when.Cond, when.Val)
+		}
+
+		if len(e.Whens) > 1 {
+			for i := range e.Whens {
+				whensCopy := sqlparser.CloneSliceOfRefOfWhen(e.Whens)
+				// replace ith element with last element, then truncate last element
+				whensCopy[i] = whensCopy[len(whensCopy)-1]
+				whensCopy = whensCopy[:len(whensCopy)-1]
+				s.queue = append(s.queue, &sqlparser.CaseExpr{
+					Expr:  e.Expr,
+					Whens: whensCopy,
+					Else:  e.Else,
+				})
+			}
+		}
+
+		if e.Else != nil {
+			klon := sqlparser.CloneRefOfCaseExpr(e)
+			klon.Else = nil
+			s.queue = append(s.queue, klon)
+		}
+		if e.Expr != nil {
+			klon := sqlparser.CloneRefOfCaseExpr(e)
+			klon.Expr = nil
+			s.queue = append(s.queue, klon)
 		}
 	default:
 		return false

From a2f7920f7f8be63ea8c3f17a9483a3454b2bf02d Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Tue, 25 Jul 2023 01:14:16 -0700
Subject: [PATCH 22/27] added sqlparser.NewCaseExpr

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 go/vt/sqlparser/ast_funcs.go                     |  9 +++++++++
 go/vt/vtgate/simplifier/expression_simplifier.go | 14 +++-----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go
index 57c73348743..cff9e22c6b7 100644
--- a/go/vt/sqlparser/ast_funcs.go
+++ b/go/vt/sqlparser/ast_funcs.go
@@ -720,6 +720,15 @@ func NewExistsExpr(subquery *Subquery) *ExistsExpr {
 	return &ExistsExpr{Subquery: subquery}
 }
 
+// NewCaseExpr makes a new CaseExpr
+func NewCaseExpr(expr Expr, whens []*When, elseExpr Expr) *CaseExpr {
+	return &CaseExpr{
+		Expr:  expr,
+		Whens: whens,
+		Else:  elseExpr,
+	}
+}
+
 // NewLimit makes a new Limit
 func NewLimit(offset, rowCount int) *Limit {
 	return &Limit{
diff --git a/go/vt/vtgate/simplifier/expression_simplifier.go b/go/vt/vtgate/simplifier/expression_simplifier.go
index 57ecc3ff67c..194e5422c04 100644
--- a/go/vt/vtgate/simplifier/expression_simplifier.go
+++ b/go/vt/vtgate/simplifier/expression_simplifier.go
@@ -247,23 +247,15 @@ func (s *shrinker) fillQueue() bool {
 				// replace ith element with last element, then truncate last element
 				whensCopy[i] = whensCopy[len(whensCopy)-1]
 				whensCopy = whensCopy[:len(whensCopy)-1]
-				s.queue = append(s.queue, &sqlparser.CaseExpr{
-					Expr:  e.Expr,
-					Whens: whensCopy,
-					Else:  e.Else,
-				})
+				s.queue = append(s.queue, sqlparser.NewCaseExpr(e.Expr, whensCopy, e.Else))
 			}
 		}
 
 		if e.Else != nil {
-			klon := sqlparser.CloneRefOfCaseExpr(e)
-			klon.Else = nil
-			s.queue = append(s.queue, klon)
+			s.queue = append(s.queue, sqlparser.NewCaseExpr(e.Expr, e.Whens, nil))
 		}
 		if e.Expr != nil {
-			klon := sqlparser.CloneRefOfCaseExpr(e)
-			klon.Expr = nil
-			s.queue = append(s.queue, klon)
+			s.queue = append(s.queue, sqlparser.NewCaseExpr(nil, e.Whens, e.Else))
 		}
 	default:
 		return false

From 1d2ab6c624a05a45f9236d815412387d4615469c Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Tue, 25 Jul 2023 22:39:35 -0700
Subject: [PATCH 23/27] removed separate testing runs from simplifier in query
 fuzzer

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/simplifier_test.go  | 65 ++++++++-----------
 1 file changed, 28 insertions(+), 37 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
index b9a7c6f51a0..556e40afda5 100644
--- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go
+++ b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
@@ -21,7 +21,7 @@ import (
 	"strings"
 	"testing"
 
-	utils2 "vitess.io/vitess/go/test/vschemawrapper"
+	"vitess.io/vitess/go/test/vschemawrapper"
 
 	"github.com/stretchr/testify/require"
 
@@ -35,47 +35,45 @@ import (
 func TestSimplifyResultsMismatchedQuery(t *testing.T) {
 	// t.Skip("Skip CI")
 	query := "select /*vt+ PLANNER=Gen4 */ distinct 'opossum' and count(*) as caggr0, case count('mustang') > -36 when min(18 * -41) then 22 - max(tbl0.sal) else 7 end as caggr1 from emp as tbl0 where case when false then tbl0.ename when 17 then 'gator' else 'mite' end and case false and true when 'worm' then tbl0.job when tbl0.ename then case when true then tbl0.ename when false then 'squirrel' end end"
-	// select /*vt+ PLANNER=Gen4 */ distinct case when min(-0) then 0 else 0 end as caggr1 from emp as tbl0 where false
-	simplified := simplifyResultsMismatchedQuery(t, query)
 
-	var err error
-	t.Helper()
-	t.Run(simplified, func(t *testing.T) {
+	var simplified string
+	t.Run("simplification", func(t *testing.T) {
+		simplified = simplifyResultsMismatchedQuery(t, query)
+	})
+
+	t.Run("final simplified query", func(t *testing.T) {
 		mcmp, closer := start(t)
 		defer closer()
 
-		_, err = mcmp.ExecAllowAndCompareError(simplified)
+		mcmp.ExecAllowAndCompareError(simplified)
 	})
 
+	// select /*vt+ PLANNER=Gen4 */ distinct case when min(-0) then 0 else 0 end as caggr1 from emp as tbl0 where false
 	fmt.Printf("final simplified query: %s\n", simplified)
-	require.ErrorContains(t, err, "mismatched")
 
 }
 
 // TODO: suppress output from comparing intermediate simplified results
 // given a query that errors with results mismatched, simplifyResultsMismatchedQuery returns a simpler version with the same error
 func simplifyResultsMismatchedQuery(t *testing.T, query string) string {
-	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
-	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
-
-	var err error
 	t.Helper()
-	t.Run(query, func(t *testing.T) {
-		mcmp, closer := start(t)
-		defer closer()
+	mcmp, closer := start(t)
+	defer closer()
+
+	_, err := mcmp.ExecAllowAndCompareError(query)
+	if err == nil {
+		t.Fatalf("query (%s) does not error", query)
+	} else if !strings.Contains(err.Error(), "mismatched") {
+		t.Fatalf("query (%s) does not error with results mismatched\nError: %v", query, err)
+	}
 
-		_, err = mcmp.ExecAllowAndCompareError(query)
-		if err == nil {
-			t.Fatalf("query (%s) does not error", query)
-		} else if !strings.Contains(err.Error(), "mismatched") {
-			t.Fatalf("query (%s) does not error with results mismatched\nError: %v", query, err)
-		}
-	})
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
+	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
 	formal, err := vindexes.LoadFormal("svschema.json")
 	require.NoError(t, err)
 	vSchema := vindexes.BuildVSchema(formal)
-	vSchemaWrapper := &utils2.VSchemaWrapper{
+	vSchemaWrapper := &vschemawrapper.VSchemaWrapper{
 		V:       vSchema,
 		Version: planbuilder.Gen4,
 	}
@@ -87,21 +85,14 @@ func simplifyResultsMismatchedQuery(t *testing.T, query string) string {
 		stmt.(sqlparser.SelectStatement),
 		vSchemaWrapper.CurrentDb(),
 		vSchemaWrapper,
-		func(statement sqlparser.SelectStatement) (sameError bool) {
+		func(statement sqlparser.SelectStatement) bool {
 			q := sqlparser.String(statement)
-			t.Helper()
-			t.Run(q, func(t *testing.T) {
-				mcmp, closer := start(t)
-				defer closer()
-
-				_, newErr := mcmp.ExecAllowAndCompareError(q)
-				if newErr == nil {
-					sameError = false
-				} else {
-					sameError = strings.Contains(newErr.Error(), "mismatched")
-				}
-			})
-			return
+			_, newErr := mcmp.ExecAllowAndCompareError(q)
+			if newErr == nil {
+				return false
+			} else {
+				return strings.Contains(newErr.Error(), "mismatched")
+			}
 		},
 	)
 

From 756ad95e16af384e2e34b3729c5c5586a59ce45b Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Tue, 25 Jul 2023 22:58:22 -0700
Subject: [PATCH 24/27] simplified existing results mismatched queries

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/random_test.go      | 34 ++++++-------
 .../vtgate/queries/random/simplifier_test.go  | 50 ++++++++++++-------
 2 files changed, 50 insertions(+), 34 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index c1197c983ad..89ec9a0e309 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -85,59 +85,56 @@ func TestMustFix(t *testing.T) {
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
 
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct case count(*) when -41 ^ 10 then -17 when 22 then -52 else 7 end from emp as tbl0, emp as tbl1 where tbl1.job < tbl1.job")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct case count(*) when 0 then -0 end from emp as tbl0, emp as tbl1 where 0")
 
 	// results mismatched (maybe derived tables)
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ 0 as crandom0 from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ distinct count(*) from emp as tbl1 where 0) as tbl1")
 
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct case count(distinct true) when 'b' then 't' end from emp as tbl1 where 's'")
 
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1 where tbl0.deptno and tbl1.comm in (12, tbl0.deptno, case false when 67 then -17 when -78 then -35 end, -76 >> -68)")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1")
 
 	// mismatched number of columns
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) + 1 from emp as tbl0 order by count(*) desc")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) + 0 from emp as tbl0 order by count(*) desc")
 
 	// results mismatched (mismatched types)
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(0 >> 0), sum(distinct tbl2.empno) from emp as tbl0 left join emp as tbl2 on -32")
 
 	// results mismatched (decimals off by a little; evalengine problem)
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1")
 
-	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
-
 	// EOF
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1, tbl1.deptno as crandom0 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
 
 	// results mismatched
 	// limit >= 9 works
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) = sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8")
 
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct count(*) as caggr1 from dept as tbl0, emp as tbl1 group by tbl1.sal having max(tbl1.comm) != true")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct count(*) as caggr1 from emp as tbl1 group by tbl1.sal having max(0) != true")
 
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct sum(tbl1.loc) as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(tbl1.dname) <= 1")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct 0 as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(0) <= 0")
 
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ min(0) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm")
 
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 1 = 0")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 0 as crandom0 from dept as tbl0, emp as tbl1 where 0")
 
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 'octopus'")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 0 as crandom0 from dept as tbl0, emp as tbl1 where 'o'")
 
 	// similar to previous two
 	// results mismatched
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct 'octopus' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.empno having count(*) = count(*)")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ distinct 'o' as crandom0 from dept as tbl0, emp as tbl1 where 0 having count(*) = count(*)")
 
 	// results mismatched (group by + right join)
 	// left instead of right works
 	// swapping tables and predicates and changing to left fails
-	helperTest(t, "select /*vt+ PLANNER=Gen4 */ max(tbl0.deptno) from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno")
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ 0 from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno")
 
 	// results mismatched (count + right join)
 	// left instead of right works
@@ -165,6 +162,9 @@ func TestKnownFailures(t *testing.T) {
 	// logs more stuff
 	//clusterInstance.EnableGeneralLog()
 
+	// VT13001: [BUG] failed to find the corresponding column
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
+
 	// vitess error: <nil>
 	// mysql error: Operand should contain 1 column(s)
 	helperTest(t, "select (count('sheepdog') ^ (-71 % sum(emp.mgr) ^ count('koi')) and count(*), 'fly') from emp, dept")
diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
index 556e40afda5..a2fec8791b2 100644
--- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go
+++ b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
@@ -34,23 +34,39 @@ import (
 
 func TestSimplifyResultsMismatchedQuery(t *testing.T) {
 	// t.Skip("Skip CI")
-	query := "select /*vt+ PLANNER=Gen4 */ distinct 'opossum' and count(*) as caggr0, case count('mustang') > -36 when min(18 * -41) then 22 - max(tbl0.sal) else 7 end as caggr1 from emp as tbl0 where case when false then tbl0.ename when 17 then 'gator' else 'mite' end and case false and true when 'worm' then tbl0.job when tbl0.ename then case when true then tbl0.ename when false then 'squirrel' end end"
-
-	var simplified string
-	t.Run("simplification", func(t *testing.T) {
-		simplified = simplifyResultsMismatchedQuery(t, query)
-	})
-
-	t.Run("final simplified query", func(t *testing.T) {
-		mcmp, closer := start(t)
-		defer closer()
-
-		mcmp.ExecAllowAndCompareError(simplified)
-	})
-
-	// select /*vt+ PLANNER=Gen4 */ distinct case when min(-0) then 0 else 0 end as caggr1 from emp as tbl0 where false
-	fmt.Printf("final simplified query: %s\n", simplified)
-
+	var queries []string
+	queries = append(queries, "select /*vt+ PLANNER=Gen4 */ (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1",
+		"select /*vt+ PLANNER=Gen4 */ distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'",
+		"select /*vt+ PLANNER=Gen4 */ distinct sum(distinct tbl1.deptno) from dept as tbl0, emp as tbl1 where tbl0.deptno and tbl1.comm in (12, tbl0.deptno, case false when 67 then -17 when -78 then -35 end, -76 >> -68)",
+		"select /*vt+ PLANNER=Gen4 */ count(*) + 1 from emp as tbl0 order by count(*) desc",
+		"select /*vt+ PLANNER=Gen4 */ count(2 >> tbl2.mgr), sum(distinct tbl2.empno <=> 15) from emp as tbl0 left join emp as tbl2 on -32",
+		"select /*vt+ PLANNER=Gen4 */ sum(case false when true then tbl1.deptno else -154 / 132 end) as caggr1 from emp as tbl0, dept as tbl1",
+		"select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc",
+		"select /*vt+ PLANNER=Gen4 */ tbl0.ename as cgroup1 from emp as tbl0 group by tbl0.job, tbl0.ename having sum(tbl0.mgr) = sum(tbl0.mgr) order by tbl0.job desc, tbl0.ename asc limit 8",
+		"select /*vt+ PLANNER=Gen4 */ distinct count(*) as caggr1 from dept as tbl0, emp as tbl1 group by tbl1.sal having max(tbl1.comm) != true",
+		"select /*vt+ PLANNER=Gen4 */ distinct sum(tbl1.loc) as caggr0 from dept as tbl0, dept as tbl1 group by tbl1.deptno having max(tbl1.dname) <= 1",
+		"select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0 from dept as tbl0, emp as tbl1 where case when false then tbl0.dname end group by tbl1.comm",
+		"select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 1 = 0",
+		"select /*vt+ PLANNER=Gen4 */ count(*) as caggr0, 1 as crandom0 from dept as tbl0, emp as tbl1 where 'octopus'",
+		"select /*vt+ PLANNER=Gen4 */ distinct 'octopus' as crandom0 from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.empno having count(*) = count(*)",
+		"select /*vt+ PLANNER=Gen4 */ max(tbl0.deptno) from dept as tbl0 right join emp as tbl1 on tbl0.deptno = tbl1.empno and tbl0.deptno = tbl1.deptno group by tbl0.deptno",
+		"select /*vt+ PLANNER=Gen4 */ count(tbl1.comm) from emp as tbl1 right join emp as tbl2 on tbl1.mgr = tbl2.sal")
+
+	for _, query := range queries {
+		var simplified string
+		t.Run("simplification "+query, func(t *testing.T) {
+			simplified = simplifyResultsMismatchedQuery(t, query)
+		})
+
+		t.Run("simplified "+query, func(t *testing.T) {
+			mcmp, closer := start(t)
+			defer closer()
+
+			mcmp.ExecAllowAndCompareError(simplified)
+		})
+
+		fmt.Printf("final simplified query: %s\n", simplified)
+	}
 }
 
 // TODO: suppress output from comparing intermediate simplified results

From 359189e9181682027cf626bd9d19419fe74aa8bd Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Thu, 27 Jul 2023 02:16:25 -0700
Subject: [PATCH 25/27] minor fixes/improvements

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen.go        | 48 ++++++++++++-------
 .../vtgate/queries/random/random_test.go      |  9 +++-
 .../vtgate/queries/random/simplifier_test.go  |  4 +-
 3 files changed, 40 insertions(+), 21 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen.go b/go/test/endtoend/vtgate/queries/random/query_gen.go
index 49873a048c0..a62e5972f10 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen.go
@@ -30,10 +30,13 @@ import (
 
 // this file contains the structs and functions to generate random queries
 
+// to test only a particular type of query, delete the corresponding testFailingQueries clause
+// there should be a comment indicating the type of query being disabled
 // if true then known failing query types are still generated by randomQuery()
 const testFailingQueries = false
 
 type (
+	// selectGenerator generates select statements
 	selectGenerator struct {
 		r            *rand.Rand
 		genConfig    sqlparser.ExprGeneratorConfig
@@ -43,16 +46,20 @@ type (
 		schemaTables []tableT
 		sel          *sqlparser.Select
 	}
+
+	// queryGenerator generates queries, which can either be unions or select statements
 	queryGenerator struct {
 		stmt   sqlparser.SelectStatement
 		selGen *selectGenerator
 	}
+
 	column struct {
 		name string
 		// TODO: perhaps remove tableName and always pass columns through a tableT
 		tableName string
 		typ       string
 	}
+
 	tableT struct {
 		// the tableT struct can be used to represent the schema of a table or a derived table
 		// in the former case tableExpr will be a sqlparser.TableName, in the latter a sqlparser.DerivedTable
@@ -172,6 +179,7 @@ func (t *tableT) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig)
 }
 
 // Generate generates a subquery based on sg
+// TODO: currently unused; generate random expressions with union
 func (sg *selectGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	var schemaTablesCopy []tableT
 	for _, tbl := range sg.schemaTables {
@@ -184,8 +192,6 @@ func (sg *selectGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGenera
 	return &sqlparser.Subquery{Select: newSG.selGen.sel}
 }
 
-func (sg *selectGenerator) IsQueryGenerator() {}
-
 // Generate generates a subquery based on qg
 func (qg *queryGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGeneratorConfig) sqlparser.Expr {
 	var schemaTablesCopy []tableT
@@ -199,7 +205,8 @@ func (qg *queryGenerator) Generate(r *rand.Rand, genConfig sqlparser.ExprGenerat
 	return &sqlparser.Subquery{Select: newQG.stmt}
 }
 
-func (qg *queryGenerator) IsQueryGenerator() {}
+func (sg *selectGenerator) IsQueryGenerator() {}
+func (qg *queryGenerator) IsQueryGenerator()  {}
 
 func (qg *queryGenerator) randomQuery() {
 	if qg.selGen.r.Intn(10) < 1 && testFailingQueries {
@@ -210,6 +217,7 @@ func (qg *queryGenerator) randomQuery() {
 	}
 }
 
+// createUnion creates a simple UNION or UNION ALL; no LIMIT or ORDER BY
 func (qg *queryGenerator) createUnion() {
 	union := &sqlparser.Union{}
 
@@ -263,12 +271,12 @@ func (sg *selectGenerator) randomSelect() {
 
 		// having
 		isHaving := sg.r.Intn(2) < 1
+		// TODO: having creates a lot of results mismatched
 		if isHaving && testFailingQueries {
 			sg.createHavingPredicates(grouping)
 		}
 
 		// alias the grouping columns
-		// TODO: alias the grouping columns after having because vitess doesn't recognize aliases in the HAVING
 		grouping = sg.aliasGroupingColumns(grouping)
 
 		// aggregation columns
@@ -282,23 +290,19 @@ func (sg *selectGenerator) randomSelect() {
 	// where
 	sg.createWherePredicates(tables)
 
-	var f func() sqlparser.Expr
 	// add random expression to select
 	// TODO: random expressions cause a lot of failures
 	isRandomExpr := sg.r.Intn(2) < 1 && testFailingQueries
 
 	// TODO: selecting a random expression potentially with columns creates
 	// TODO: only_full_group_by related errors in Vitess
+	var exprGenerators []sqlparser.ExprGenerator
 	if canAggregate && testFailingQueries {
-		exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
+		exprGenerators = slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 		// add scalar subqueries
-		if sg.r.Intn(10) < 1 && testFailingQueries {
+		if sg.r.Intn(10) < 1 {
 			exprGenerators = append(exprGenerators, sg)
 		}
-
-		f = func() sqlparser.Expr { return sg.getRandomExpr(exprGenerators...) }
-	} else {
-		f = func() sqlparser.Expr { return sg.getRandomExpr() }
 	}
 
 	// make sure we have at least one select expression
@@ -309,7 +313,7 @@ func (sg *selectGenerator) randomSelect() {
 		// TODO: but we cannot do this for int literals,
 		// TODO: so we loop until we get a non-int-literal random expression
 		// TODO: this is necessary because grouping by the alias (crandom0) currently fails on vitess
-		randomExpr := f()
+		randomExpr := sg.getRandomExpr(exprGenerators...)
 		literal, ok := randomExpr.(*sqlparser.Literal)
 		isIntLiteral := ok && literal.Type == sqlparser.IntVal
 		if isIntLiteral && canAggregate {
@@ -345,6 +349,7 @@ func (sg *selectGenerator) randomSelect() {
 		sg.createLimit()
 	}
 
+	// this makes sure the query generated has the correct number of columns (sg.selGen.genConfig.numCols)
 	newTable = sg.matchNumCols(tables, newTable, canAggregate)
 
 	// add new table to schemaTables
@@ -352,7 +357,6 @@ func (sg *selectGenerator) randomSelect() {
 	sg.schemaTables = append(sg.schemaTables, newTable)
 
 	// derived tables (partially unsupported)
-	// TODO: derived tables fails a lot
 	if sg.r.Intn(10) < 1 {
 		sg.randomSelect()
 	}
@@ -360,7 +364,7 @@ func (sg *selectGenerator) randomSelect() {
 
 func (sg *selectGenerator) createTablesAndJoin() ([]tableT, bool) {
 	var tables []tableT
-	// add at least one of original emp/dept tables for now because derived tables have nil columns
+	// add at least one of original emp/dept tables
 	tables = append(tables, sg.schemaTables[sg.r.Intn(2)])
 
 	tables[0].setAlias("tbl0")
@@ -412,6 +416,7 @@ func (sg *selectGenerator) createJoinPredicates(tables []tableT) sqlparser.Exprs
 
 	exprGenerators := []sqlparser.ExprGenerator{&tables[len(tables)-2], &tables[len(tables)-1]}
 	// add scalar subqueries
+	// TODO: subqueries fail
 	if sg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, sg)
 	}
@@ -419,7 +424,7 @@ func (sg *selectGenerator) createJoinPredicates(tables []tableT) sqlparser.Exprs
 	return sg.createRandomExprs(1, 3, exprGenerators...)
 }
 
-// returns the grouping columns as three types: sqlparser.GroupBy, sqlparser.SelectExprs, []column
+// returns the grouping columns as []column
 func (sg *selectGenerator) createGroupBy(tables []tableT) (grouping []column) {
 	if sg.maxGBs <= 0 {
 		return
@@ -467,6 +472,7 @@ func (sg *selectGenerator) aliasGroupingColumns(grouping []column) []column {
 func (sg *selectGenerator) createAggregations(tables []tableT) (aggregates []column) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 	// add scalar subqueries
+	// TODO: subqueries fail
 	if sg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, sg)
 	}
@@ -507,6 +513,7 @@ func (sg *selectGenerator) createOrderBy() {
 func (sg *selectGenerator) createWherePredicates(tables []tableT) {
 	exprGenerators := slices2.Map(tables, func(t tableT) sqlparser.ExprGenerator { return &t })
 	// add scalar subqueries
+	// TODO: subqueries fail
 	if sg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, sg)
 	}
@@ -519,6 +526,7 @@ func (sg *selectGenerator) createWherePredicates(tables []tableT) {
 func (sg *selectGenerator) createHavingPredicates(grouping []column) {
 	exprGenerators := slices2.Map(grouping, func(c column) sqlparser.ExprGenerator { return &c })
 	// add scalar subqueries
+	// TODO: subqueries fail
 	if sg.r.Intn(10) < 1 && testFailingQueries {
 		exprGenerators = append(exprGenerators, sg)
 	}
@@ -547,7 +555,13 @@ func (sg *selectGenerator) createRandomExprs(minExprs, maxExprs int, generators
 
 // getRandomExpr returns a random expression
 func (sg *selectGenerator) getRandomExpr(generators ...sqlparser.ExprGenerator) sqlparser.Expr {
-	g := sqlparser.NewGenerator(sg.r, 2, generators...)
+	var g *sqlparser.Generator
+	if generators == nil {
+		g = sqlparser.NewGenerator(sg.r, 2)
+	} else {
+		g = sqlparser.NewGenerator(sg.r, 2, generators...)
+	}
+
 	return g.Expression(sg.genConfig.SingleRowConfig().SetNumCols(1))
 }
 
@@ -567,7 +581,7 @@ func (sg *selectGenerator) createLimit() {
 	}
 }
 
-// randomlyAlias randomly aliases expr with alias alias, adds it to sel.SelectExprs and returns the column created
+// randomlyAlias randomly aliases expr with alias alias, adds it to sel.SelectExprs, and returns the column created
 func (sg *selectGenerator) randomlyAlias(expr sqlparser.Expr, alias string) column {
 	var col column
 	if sg.r.Intn(2) < 1 {
diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 89ec9a0e309..9050971fad8 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -79,7 +79,7 @@ func helperTest(t *testing.T, query string) {
 }
 
 func TestMustFix(t *testing.T) {
-	//t.Skip("Skip CI")
+	t.Skip("Skip CI")
 
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
@@ -154,7 +154,7 @@ func TestMustFix(t *testing.T) {
 }
 
 func TestKnownFailures(t *testing.T) {
-	//t.Skip("Skip CI")
+	t.Skip("Skip CI")
 
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
 	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
@@ -162,6 +162,9 @@ func TestKnownFailures(t *testing.T) {
 	// logs more stuff
 	//clusterInstance.EnableGeneralLog()
 
+	// column 'tbl1.`not exists (select 1 from dual)`' not found
+	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.`not exists (select 1 from dual)`, count(*) from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ not exists (select 1 from dual) from dept as tbl0 where tbl0.dname) as tbl1 group by tbl0.deptno, tbl1.`not exists (select 1 from dual)`")
+
 	// VT13001: [BUG] failed to find the corresponding column
 	helperTest(t, "select /*vt+ PLANNER=Gen4 */ tbl1.dname as cgroup0, tbl1.dname as cgroup1 from dept as tbl0, dept as tbl1 group by tbl1.dname, tbl1.deptno order by tbl1.deptno desc")
 
@@ -259,6 +262,8 @@ func TestKnownFailures(t *testing.T) {
 }
 
 func TestRandom(t *testing.T) {
+	// t.Skip("Skip CI; random expressions generate too many failures to properly limit")
+
 	mcmp, closer := start(t)
 	defer closer()
 
diff --git a/go/test/endtoend/vtgate/queries/random/simplifier_test.go b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
index a2fec8791b2..478ee355d34 100644
--- a/go/test/endtoend/vtgate/queries/random/simplifier_test.go
+++ b/go/test/endtoend/vtgate/queries/random/simplifier_test.go
@@ -33,7 +33,8 @@ import (
 )
 
 func TestSimplifyResultsMismatchedQuery(t *testing.T) {
-	// t.Skip("Skip CI")
+	t.Skip("Skip CI")
+
 	var queries []string
 	queries = append(queries, "select /*vt+ PLANNER=Gen4 */ (68 - -16) / case false when -45 then 3 when 28 then -43 else -62 end as crandom0 from dept as tbl0, (select /*vt+ PLANNER=Gen4 */ distinct not not false and count(*) from emp as tbl0, emp as tbl1 where tbl1.ename) as tbl1 limit 1",
 		"select /*vt+ PLANNER=Gen4 */ distinct case true when 'burro' then 'trout' else 'elf' end < case count(distinct true) when 'bobcat' then 'turkey' else 'penguin' end from dept as tbl0, emp as tbl1 where 'spider'",
@@ -69,7 +70,6 @@ func TestSimplifyResultsMismatchedQuery(t *testing.T) {
 	}
 }
 
-// TODO: suppress output from comparing intermediate simplified results
 // given a query that errors with results mismatched, simplifyResultsMismatchedQuery returns a simpler version with the same error
 func simplifyResultsMismatchedQuery(t *testing.T, query string) string {
 	t.Helper()

From a60a1c2860b2165233d86b0e44a5d780c874c2c9 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Thu, 27 Jul 2023 22:23:16 -0700
Subject: [PATCH 26/27] fixed goimports and failing endtoend tests

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 .../vtgate/queries/random/query_gen_test.go   |  3 ---
 .../vtgate/queries/random/random_expr_test.go | 23 ++-----------------
 go/test/vschemawrapper/vschema_wrapper.go     |  4 ++--
 3 files changed, 4 insertions(+), 26 deletions(-)

diff --git a/go/test/endtoend/vtgate/queries/random/query_gen_test.go b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
index 1b7938ccb6a..fe8aa6f6492 100644
--- a/go/test/endtoend/vtgate/queries/random/query_gen_test.go
+++ b/go/test/endtoend/vtgate/queries/random/query_gen_test.go
@@ -28,9 +28,6 @@ import (
 
 // TestSeed makes sure that the seed is deterministic
 func TestSeed(t *testing.T) {
-	sel, err := sqlparser.Parse("select 1 < any (select empno from emp)")
-	fmt.Println(sel, err)
-
 	// specify the schema (that is defined in schema.sql)
 	schemaTables := []tableT{
 		{tableExpr: sqlparser.NewTableName("emp")},
diff --git a/go/test/endtoend/vtgate/queries/random/random_expr_test.go b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
index 333c5df59e3..e192c86b0b5 100644
--- a/go/test/endtoend/vtgate/queries/random/random_expr_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_expr_test.go
@@ -17,28 +17,16 @@ limitations under the License.
 package random
 
 import (
-	"fmt"
 	"math/rand"
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/require"
-
-	"vitess.io/vitess/go/test/endtoend/utils"
-
 	"vitess.io/vitess/go/slices2"
 	"vitess.io/vitess/go/vt/sqlparser"
 )
 
 // This test tests that generating random expressions with a schema does not panic
 func TestRandomExprWithTables(t *testing.T) {
-	//t.Skip("Skip CI")
-	mcmp, closer := start(t)
-	defer closer()
-
-	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "emp", clusterInstance.VtgateProcess.ReadVSchema))
-	require.NoError(t, utils.WaitForAuthoritative(t, keyspaceName, "dept", clusterInstance.VtgateProcess.ReadVSchema))
-
 	// specify the schema (that is defined in schema.sql)
 	schemaTables := []tableT{
 		{tableExpr: sqlparser.NewTableName("emp")},
@@ -63,16 +51,9 @@ func TestRandomExprWithTables(t *testing.T) {
 	for i := 0; i < 100; i++ {
 
 		seed := time.Now().UnixNano()
-		fmt.Printf("seed: %d\n", seed)
-
 		r := rand.New(rand.NewSource(seed))
-		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CannotAggregate, "", 0, false)
+		genConfig := sqlparser.NewExprGeneratorConfig(sqlparser.CanAggregate, "", 0, false)
 		g := sqlparser.NewGenerator(r, 3, slices2.Map(schemaTables, func(t tableT) sqlparser.ExprGenerator { return &t })...)
-		expr := g.Expression(genConfig)
-		fmt.Println(sqlparser.String(expr))
-
-		from := sqlparser.TableExprs{sqlparser.NewAliasedTableExpr(sqlparser.NewTableName("emp"), ""), sqlparser.NewAliasedTableExpr(sqlparser.NewTableName("dept"), "")}
-		query := sqlparser.NewSelect(nil, sqlparser.SelectExprs{sqlparser.NewAliasedExpr(expr, "")}, nil, nil, from, nil, nil, nil, nil)
-		mcmp.ExecAllowAndCompareError(sqlparser.String(query))
+		g.Expression(genConfig)
 	}
 }
diff --git a/go/test/vschemawrapper/vschema_wrapper.go b/go/test/vschemawrapper/vschema_wrapper.go
index af40e1174c8..8f0375ba4e1 100644
--- a/go/test/vschemawrapper/vschema_wrapper.go
+++ b/go/test/vschemawrapper/vschema_wrapper.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+
 	"vitess.io/vitess/go/mysql/collations"
 	"vitess.io/vitess/go/sqltypes"
 	"vitess.io/vitess/go/vt/key"
@@ -82,8 +83,7 @@ func (vw *VSchemaWrapper) PlanPrepareStatement(ctx context.Context, query string
 	return plan, stmt, nil
 }
 
-func (vw *VSchemaWrapper) ClearPrepareData(lowered string) {
-}
+func (vw *VSchemaWrapper) ClearPrepareData(string) {}
 
 func (vw *VSchemaWrapper) StorePrepareData(string, *vtgatepb.PrepareData) {}
 

From 67367932ef245690611373e8202b26a7bcb0b542 Mon Sep 17 00:00:00 2001
From: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
Date: Thu, 27 Jul 2023 23:13:24 -0700
Subject: [PATCH 27/27] skip TestRandom

Signed-off-by: Arvind Murty <10248018+arvind-murty@users.noreply.github.com>
---
 go/test/endtoend/vtgate/queries/random/random_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go/test/endtoend/vtgate/queries/random/random_test.go b/go/test/endtoend/vtgate/queries/random/random_test.go
index 9050971fad8..4aeee1c64f3 100644
--- a/go/test/endtoend/vtgate/queries/random/random_test.go
+++ b/go/test/endtoend/vtgate/queries/random/random_test.go
@@ -262,7 +262,7 @@ func TestKnownFailures(t *testing.T) {
 }
 
 func TestRandom(t *testing.T) {
-	// t.Skip("Skip CI; random expressions generate too many failures to properly limit")
+	t.Skip("Skip CI; random expressions generate too many failures to properly limit")
 
 	mcmp, closer := start(t)
 	defer closer()