Merge pull request #103026 from e-mbrown/backport23.1-98147

release-23.1: sql: add plpgsql parser for telemetry purposes

BUILD.bazel

@@ -118,6 +118,9 @@ exports_files([
 # gazelle:exclude pkg/sql/lexbase/keywords.go
 # gazelle:exclude pkg/sql/lexbase/tokens.go
 # gazelle:exclude pkg/sql/lexbase/reserved_keywords.go
+# gazelle:exclude pkg/sql/plpgsql/parser/lexbase/keywords.go
+# gazelle:exclude pkg/sql/plpgsql/parser/lexbase/tokens.go
+# gazelle:exclude pkg/sql/plpgsql/parser/lexbase/reserved_keywords.go
 # gazelle:exclude pkg/sql/scanner/token_names_test.go
 # gazelle:exclude pkg/sql/schemachanger/scexec/mocks_generated_test.go
 # gazelle:exclude pkg/cmd/prereqs/testdata

build/bazelutil/unused_checker/unused_checker.go

@@ -132,6 +132,7 @@ func ignoreUnused(obj string) bool {
 		"github.com/cockroachdb/cockroach/pkg/geo/geographiclib.",
 		"github.com/cockroachdb/cockroach/pkg/util/goschedstats.",
 		"github.com/cockroachdb/cockroach/pkg/sql/parser.",
+		"github.com/cockroachdb/cockroach/pkg/sql/plpgsql/parser.",
 	}
 	for _, prefix := range prefixes {
 		if strings.HasPrefix(obj, prefix) {

pkg/BUILD.bazel

@@ -481,6 +481,7 @@ ALL_TESTS = [
     "//pkg/sql/pgwire:pgwire_test",
     "//pkg/sql/physicalplan/replicaoracle:replicaoracle_test",
     "//pkg/sql/physicalplan:physicalplan_test",
+    "//pkg/sql/plpgsql/parser:parser_test",
     "//pkg/sql/privilege:privilege_test",
     "//pkg/sql/protoreflect:protoreflect_test",
     "//pkg/sql/querycache:querycache_test",
@@ -1799,6 +1800,7 @@ GO_TARGETS = [
     "//pkg/sql/optionalnodeliveness:optionalnodeliveness",
     "//pkg/sql/paramparse:paramparse",
     "//pkg/sql/parser/fuzz:fuzz",
+    "//pkg/sql/parser/statements:statements",
     "//pkg/sql/parser:parser",
     "//pkg/sql/parser:parser_test",
     "//pkg/sql/pgwire/hba:hba",
@@ -1818,6 +1820,9 @@ GO_TARGETS = [
     "//pkg/sql/physicalplan/replicaoracle:replicaoracle_test",
     "//pkg/sql/physicalplan:physicalplan",
     "//pkg/sql/physicalplan:physicalplan_test",
+    "//pkg/sql/plpgsql/parser/lexbase:lexbase",
+    "//pkg/sql/plpgsql/parser:parser_test",
+    "//pkg/sql/plpgsql/parser:plpgparser",
     "//pkg/sql/privilege:privilege",
     "//pkg/sql/privilege:privilege_test",
     "//pkg/sql/protoreflect/test:test",
@@ -1924,6 +1929,7 @@ GO_TARGETS = [
     "//pkg/sql/sem/eval:eval_test",
     "//pkg/sql/sem/normalize:normalize",
     "//pkg/sql/sem/normalize:normalize_test",
+    "//pkg/sql/sem/plpgsqltree:plpgsqltree",
     "//pkg/sql/sem/semenumpb:semenumpb",
     "//pkg/sql/sem/transform:transform",
     "//pkg/sql/sem/tree/evalgen:evalgen",
@@ -3042,6 +3048,7 @@ GET_X_DATA_TARGETS = [
     "//pkg/sql/paramparse:get_x_data",
     "//pkg/sql/parser:get_x_data",
     "//pkg/sql/parser/fuzz:get_x_data",
+    "//pkg/sql/parser/statements:get_x_data",
     "//pkg/sql/pgwire:get_x_data",
     "//pkg/sql/pgwire/hba:get_x_data",
     "//pkg/sql/pgwire/identmap:get_x_data",
@@ -3052,6 +3059,8 @@ GET_X_DATA_TARGETS = [
     "//pkg/sql/pgwire/pgwirecancel:get_x_data",
     "//pkg/sql/physicalplan:get_x_data",
     "//pkg/sql/physicalplan/replicaoracle:get_x_data",
+    "//pkg/sql/plpgsql/parser:get_x_data",
+    "//pkg/sql/plpgsql/parser/lexbase:get_x_data",
     "//pkg/sql/privilege:get_x_data",
     "//pkg/sql/protoreflect:get_x_data",
     "//pkg/sql/protoreflect/test:get_x_data",
@@ -3120,6 +3129,7 @@ GET_X_DATA_TARGETS = [
     "//pkg/sql/sem/eval/cast_test:get_x_data",
     "//pkg/sql/sem/eval/eval_test:get_x_data",
     "//pkg/sql/sem/normalize:get_x_data",
+    "//pkg/sql/sem/plpgsqltree:get_x_data",
     "//pkg/sql/sem/semenumpb:get_x_data",
     "//pkg/sql/sem/transform:get_x_data",
     "//pkg/sql/sem/tree:get_x_data",

pkg/cli/BUILD.bazel

@@ -162,6 +162,7 @@ go_library(
         "//pkg/sql/execinfrapb",
         "//pkg/sql/lexbase",
         "//pkg/sql/parser",
+        "//pkg/sql/parser/statements",
         "//pkg/sql/pgwire/pgcode",
         "//pkg/sql/protoreflect",
         "//pkg/sql/row",

pkg/cli/clisqlshell/context.go

@@ -137,7 +137,7 @@ func (c *internalContext) addStatementWrapper(w statementWrapper) {
 func (c *internalContext) maybeWrapStatement(
 	ctx context.Context, statement string, state *cliState,
 ) (err error) {
-	var s scanner.Scanner
+	var s scanner.SQLScanner
 	for _, sw := range c.statementWrappers {
 		s.Init(statement)
 		if sw.Pattern.matches(s) {

pkg/cli/clisqlshell/parser.go

@@ -45,7 +45,7 @@ type parseNode struct {
 	defaultTransition *parseNode
 }
 
-func (t *parseNode) matches(stmts scanner.Scanner) bool {
+func (t *parseNode) matches(stmts scanner.SQLScanner) bool {
 	if t.abortMatch {
 		return false
 	}

pkg/cli/sqlfmt.go

@@ -16,6 +16,7 @@ import (
 	"os"
 
 	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
@@ -43,7 +44,7 @@ func runSQLFmt(cmd *cobra.Command, args []string) error {
 		return errors.Errorf("tab width must be > 0: %d", sqlfmtCtx.tabWidth)
 	}
 
-	var sl parser.Statements
+	var sl statements.Statements
 	if len(sqlfmtCtx.execStmts) != 0 {
 		for _, exec := range sqlfmtCtx.execStmts {
 			stmts, err := parser.Parse(exec)

pkg/cmd/reduce/reduce/reducesql/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
     deps = [
         "//pkg/cmd/reduce/reduce",
         "//pkg/sql/parser",
+        "//pkg/sql/parser/statements",
         "//pkg/sql/sem/builtins",
         "//pkg/sql/sem/tree",
     ],

pkg/cmd/reduce/reduce/reducesql/reducesql.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/reduce/reduce"
 	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	// Import builtins.
 	_ "github.com/cockroachdb/cockroach/pkg/sql/sem/builtins"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
@@ -399,7 +400,7 @@ func Pretty(s string) (string, error) {
 	return joinASTs(collectASTs(stmts)), nil
 }
 
-func collectASTs(stmts parser.Statements) []tree.NodeFormatter {
+func collectASTs(stmts statements.Statements) []tree.NodeFormatter {
 	asts := make([]tree.NodeFormatter, len(stmts))
 	for i, stmt := range stmts {
 		asts[i] = stmt.AST

pkg/gen/misc.bzl

@@ -9,6 +9,9 @@ MISC_SRCS = [
     "//pkg/roachprod/vm/aws:terraform/main.tf",
     "//pkg/spanconfig/spanconfigstore:entry_interval_btree.go",
     "//pkg/spanconfig/spanconfigstore:entry_interval_btree_test.go",
+    "//pkg/sql/plpgsql/parser/lexbase:keywords.go",
+    "//pkg/sql/plpgsql/parser/lexbase:tokens.go",
+    "//pkg/sql/plpgsql/parser:plpgsql.go",
     "//pkg/sql/scanner:token_names_test.go",
     "//pkg/sql:txnstatetransitions_diagram.gv",
     "//pkg/sql:txnstatetransitions_report.txt",

pkg/server/BUILD.bazel

@@ -225,6 +225,7 @@ go_library(
         "//pkg/sql/lexbase",
         "//pkg/sql/optionalnodeliveness",
         "//pkg/sql/parser",
+        "//pkg/sql/parser/statements",
         "//pkg/sql/pgwire",
         "//pkg/sql/pgwire/pgcode",
         "//pkg/sql/pgwire/pgerror",

pkg/server/api_v2_sql.go

@@ -22,6 +22,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo"
 	"github.com/cockroachdb/cockroach/pkg/sql/isql"
 	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
@@ -204,9 +205,9 @@ func (a *apiV2Server) execSQL(w http.ResponseWriter, r *http.Request) {
 		ApplicationName string `json:"application_name"`
 		Execute         bool   `json:"execute"`
 		Statements      []struct {
-			SQL       string           `json:"sql"`
-			stmt      parser.Statement `json:"-"`
-			Arguments []interface{}    `json:"arguments,omitempty"`
+			SQL       string                               `json:"sql"`
+			stmt      statements.Statement[tree.Statement] `json:"-"`
+			Arguments []interface{}                        `json:"arguments,omitempty"`
 		} `json:"statements"`
 	}
 	// Type for the result.

pkg/sql/BUILD.bazel

@@ -431,6 +431,7 @@ go_library(
         "//pkg/sql/optionalnodeliveness",
         "//pkg/sql/paramparse",
         "//pkg/sql/parser",
+        "//pkg/sql/parser/statements",
         "//pkg/sql/pgwire/pgcode",
         "//pkg/sql/pgwire/pgerror",
         "//pkg/sql/pgwire/pgnotice",
@@ -786,6 +787,7 @@ go_test(
         "//pkg/sql/mutations",
         "//pkg/sql/opt/exec/explain",
         "//pkg/sql/parser",
+        "//pkg/sql/parser/statements",
         "//pkg/sql/pgwire/pgcode",
         "//pkg/sql/pgwire/pgerror",
         "//pkg/sql/pgwire/pgwirebase",

pkg/sql/conn_executor.go

@@ -44,6 +44,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/idxrecommendations"
 	"github.com/cockroachdb/cockroach/pkg/sql/idxusage"
 	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirecancel"
@@ -2894,7 +2895,7 @@ func (ex *connExecutor) execCopyOut(
 	return nil, nil
 }
 
-func (ex *connExecutor) setCopyLoggingFields(stmt parser.Statement) {
+func (ex *connExecutor) setCopyLoggingFields(stmt statements.Statement[tree.Statement]) {
 	// These fields need to be set for logging purposes.
 	ex.planner.stmt = Statement{
 		Statement: stmt,

pkg/sql/conn_executor_exec.go

@@ -37,6 +37,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/opt/exec/explain"
 	"github.com/cockroachdb/cockroach/pkg/sql/paramparse"
 	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
 	"github.com/cockroachdb/cockroach/pkg/sql/physicalplan"
@@ -94,7 +95,7 @@ const numTxnRetryErrors = 3
 //	then the statement cannot have any placeholder.
 func (ex *connExecutor) execStmt(
 	ctx context.Context,
-	parserStmt parser.Statement,
+	parserStmt statements.Statement[tree.Statement],
 	portal *PreparedPortal,
 	pinfo *tree.PlaceholderInfo,
 	res RestrictedCommandResult,
@@ -274,7 +275,7 @@ func (ex *connExecutor) execPortal(
 // The returned event can be nil if no state transition is required.
 func (ex *connExecutor) execStmtInOpenState(
 	ctx context.Context,
-	parserStmt parser.Statement,
+	parserStmt statements.Statement[tree.Statement],
 	portal *PreparedPortal,
 	pinfo *tree.PlaceholderInfo,
 	res RestrictedCommandResult,
@@ -815,7 +816,7 @@ func (ex *connExecutor) execStmtInOpenState(
 			}
 		}
 		prepStmt := makeStatement(
-			parser.Statement{
+			statements.Statement[tree.Statement]{
 				// We need the SQL string just for the part that comes after
 				// "PREPARE ... AS",
 				// TODO(radu): it would be nice if the parser would figure out this
@@ -2652,7 +2653,7 @@ func (ex *connExecutor) enableTracing(modes []string) error {
 
 // addActiveQuery adds a running query to the list of running queries.
 func (ex *connExecutor) addActiveQuery(
-	stmt parser.Statement,
+	stmt statements.Statement[tree.Statement],
 	placeholders *tree.PlaceholderInfo,
 	queryID clusterunique.ID,
 	cancelQuery context.CancelFunc,

pkg/sql/conn_executor_internal_test.go

@@ -30,8 +30,10 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/execinfra"
 	"github.com/cockroachdb/cockroach/pkg/sql/flowinfra"
 	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase"
 	"github.com/cockroachdb/cockroach/pkg/sql/querycache"
+	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics"
 	"github.com/cockroachdb/cockroach/pkg/storage"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
@@ -236,7 +238,7 @@ func TestPortalsDestroyedOnTxnFinish(t *testing.T) {
 	}
 }
 
-func mustParseOne(s string) parser.Statement {
+func mustParseOne(s string) statements.Statement[tree.Statement] {
 	stmts, err := parser.Parse(s)
 	if err != nil {
 		log.Fatalf(context.Background(), "%v", err)

pkg/sql/conn_io.go

@@ -19,7 +19,7 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/col/coldata"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo"
-	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgnotice"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
@@ -127,7 +127,7 @@ type ExecStmt struct {
 	// Information returned from parsing: AST, SQL, NumPlaceholders.
 	// Note that AST can be nil, in which case executing it should produce an
 	// "empty query response" message.
-	parser.Statement
+	statements.Statement[tree.Statement]
 
 	// TimeReceived is the time at which the exec message was received
 	// from the client. Used to compute the service latency.
@@ -198,7 +198,7 @@ type PrepareStmt struct {
 	// Information returned from parsing: AST, SQL, NumPlaceholders.
 	// Note that AST can be nil, in which case executing it should produce an
 	// "empty query response" message.
-	parser.Statement
+	statements.Statement[tree.Statement]
 
 	TypeHints tree.PlaceholderTypes
 	// RawTypeHints is the representation of type hints exactly as specified by
@@ -330,7 +330,7 @@ var _ Command = Flush{}
 
 // CopyIn is the command for execution of the Copy-in pgwire subprotocol.
 type CopyIn struct {
-	ParsedStmt parser.Statement
+	ParsedStmt statements.Statement[tree.Statement]
 	Stmt       *tree.CopyFrom
 	// Conn is the network connection. Execution of the CopyFrom statement takes
 	// control of the connection.
@@ -362,7 +362,7 @@ var _ Command = CopyIn{}
 
 // CopyOut is the command for execution of the Copy-out pgwire subprotocol.
 type CopyOut struct {
-	ParsedStmt parser.Statement
+	ParsedStmt statements.Statement[tree.Statement]
 	Stmt       *tree.CopyTo
 	// TimeReceived is the time at which the message was received
 	// from the client. Used to compute the service latency.

pkg/sql/distsql_plan_changefeed.go

@@ -22,7 +22,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/execinfra"
 	"github.com/cockroachdb/cockroach/pkg/sql/flowinfra"
 	"github.com/cockroachdb/cockroach/pkg/sql/opt/cat"
-	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/catid"
@@ -82,7 +82,7 @@ func PlanCDCExpression(
 		opt.apply(&cfg)
 	}
 
-	p.stmt = makeStatement(parser.Statement{
+	p.stmt = makeStatement(statements.Statement[tree.Statement]{
 		AST: cdcExpr,
 		SQL: tree.AsString(cdcExpr),
 	}, clusterunique.ID{} /* queryID */)

pkg/sql/exec_util.go

@@ -76,6 +76,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/opt"
 	"github.com/cockroachdb/cockroach/pkg/sql/optionalnodeliveness"
 	"github.com/cockroachdb/cockroach/pkg/sql/parser"
+	"github.com/cockroachdb/cockroach/pkg/sql/parser/statements"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgnotice"
@@ -2073,7 +2074,7 @@ type queryMeta struct {
 	start time.Time
 
 	// The SQL statement being executed.
-	stmt parser.Statement
+	stmt statements.Statement[tree.Statement]
 
 	// The placeholders that the query was executed with if any.
 	placeholders *tree.PlaceholderInfo

pkg/sql/lexbase/BUILD.bazel

@@ -89,4 +89,11 @@ genrule(
     ],
 )
 
+exports_files(
+    [
+        "sql-gen.sh",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 get_x_data(name = "get_x_data")