Merge #73062 #75148 #75160 #75165 #75169

73062: physicalplan: add support for multi-stage execution of covar_pop. r=yuzefovich a=mneverov

physicalplan: add support for multi-stage execution of covar_pop.

See #58347.

Release note (performance improvement): `covar_pop` aggregate function is now
evaluated more efficiently in a distributed setting.

75148: sql/rowenc: avoid panic when encoding delete preserving index r=rhu713 a=stevendanna

It appears, we can occasionally be asked to encode empty KVs. Notably,
this seems to happen along the delete path when multiple column
families are in play.

The included test case mirrors the case that caught this bug on an
in-progress branch. Namely, a "temporary" index used by the new index
backfiller during a primary key change.

Release note: None

75160: build: add utility function to merge test xml's r=rail a=rickystewart

The `stress` utility runs tests many times to accumulate results, so
we'd like some code to merge multiple `test.xml` files into a single
one.

Release note: None

75165: kv: fix GenerateForcedRetryableError to return a bumped epoch r=lidorcarmel a=lidorcarmel

This is needed for PR #74563, where we change how txn is reset.
Without this change GenerateForcedRetryableError returns an
error with an inner txn that has an epoch 0. With this change
the epoch is copied from the original txn.

Release note: None

75169: vendor: pull in latest version of `stress` r=rail a=rickystewart

Pull in the latest version of `stress` including these changes:

```
43d99a9 Merge pull request #13 from cockroachdb/bazelsharding
01690a1 stress: add `-bazel` support, support for sharding artifacts
```

Release note: None

Co-authored-by: Max Neverov <[email protected]>
Co-authored-by: Steven Danna <[email protected]>
Co-authored-by: Ricky Stewart <[email protected]>
Co-authored-by: Lidor Carmel <[email protected]>

DEPS.bzl

@@ -1258,10 +1258,10 @@ def go_deps():
         name = "com_github_cockroachdb_stress",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/cockroachdb/stress",
-        sha256 = "2b7b584a4cafacd0d971adf1e150fe9c1f770eb2b56993af06a4ae7fa329a521",
-        strip_prefix = "github.com/cockroachdb/[email protected]20170808184505-29b5d31b4c3a",
+        sha256 = "bd0dea0ebea9ea71aa12e5918fa8b61a78a548d54e8e9c126aa285c1ae84d3e4",
+        strip_prefix = "github.com/cockroachdb/[email protected]20220119200057-43d99a9e6d7f",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20170808184505-29b5d31b4c3a.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220119200057-43d99a9e6d7f.zip",
         ],
     )
     go_repository(

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/cockroachdb/redact v1.1.3
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2
-	github.com/cockroachdb/stress v0.0.0-20170808184505-29b5d31b4c3a
+	github.com/cockroachdb/stress v0.0.0-20220119200057-43d99a9e6d7f
 	github.com/cockroachdb/tools v0.0.0-20211112185054-642e51449b40
 	github.com/cockroachdb/ttycolor v0.0.0-20210902133924-c7d7dcdde4e8
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd

go.sum

@@ -426,8 +426,8 @@ github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd h1:KFOt5I9
 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd/go.mod h1:AN708GD2FFeLgUHMbD58YPe4Nw8GG//3rwgyG4L9gR0=
 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM=
 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
-github.com/cockroachdb/stress v0.0.0-20170808184505-29b5d31b4c3a h1:7bnDlMxIJCg3o3vIILG2COsbNZpiYHgI4UkCYmeAWnQ=
-github.com/cockroachdb/stress v0.0.0-20170808184505-29b5d31b4c3a/go.mod h1:oapRqABg5KA/TaAikQH53fpP5nvCe9sftYmYV/F/yVE=
+github.com/cockroachdb/stress v0.0.0-20220119200057-43d99a9e6d7f h1:jXa4+uPllulalBUwZlptnW177YokSScljz6TpsMGld8=
+github.com/cockroachdb/stress v0.0.0-20220119200057-43d99a9e6d7f/go.mod h1:oapRqABg5KA/TaAikQH53fpP5nvCe9sftYmYV/F/yVE=
 github.com/cockroachdb/tablewriter v0.0.5-0.20200105123400-bd15540e8847 h1:c7yLgqcm/3c9lYtpWeVD9NYqA9cKsKHdpQM62PHtTUM=
 github.com/cockroachdb/tablewriter v0.0.5-0.20200105123400-bd15540e8847/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/cockroachdb/teamcity v0.0.0-20180905144921-8ca25c33eb11 h1:UAqRo5xPCyTtZznAJ9iPVpDUFxGI0a6QWtQ8E+zwJRg=

pkg/build/util/util.go

@@ -18,6 +18,7 @@ import (
 	"io"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 )
 
@@ -137,6 +138,51 @@ func MungeTestXML(srcContent []byte, outFile io.Writer) error {
 	return writeToFile(&suites.Suites[0], outFile)
 }
 
+// MergeTestXMLs merges the given list of test suites into a single test suite,
+// then writes the serialized XML to the given outFile. The prefix is passed
+// to xml.Unmarshal. Note that this function might modify the passed-in
+// TestSuites in-place.
+func MergeTestXMLs(suitesToMerge []TestSuites, outFile io.Writer) error {
+	if len(suitesToMerge) == 0 {
+		return fmt.Errorf("expected at least one test suite")
+	}
+	var resultSuites TestSuites
+	resultSuites.Suites = append(resultSuites.Suites, testSuite{})
+	resultSuite := &resultSuites.Suites[0]
+	resultSuite.Attrs = suitesToMerge[0].Suites[0].Attrs
+	cases := make(map[string]*testCase)
+	for _, suites := range suitesToMerge {
+		for _, testCase := range suites.Suites[0].TestCases {
+			oldCase, ok := cases[testCase.Name]
+			if !ok {
+				cases[testCase.Name] = testCase
+				continue
+			}
+			if testCase.Failure != nil {
+				if oldCase.Failure == nil {
+					oldCase.Failure = testCase.Failure
+				} else {
+					oldCase.Failure.Contents = oldCase.Failure.Contents + "\n" + testCase.Failure.Contents
+				}
+			}
+			if testCase.Error != nil {
+				if oldCase.Error == nil {
+					oldCase.Error = testCase.Error
+				} else {
+					oldCase.Error.Contents = oldCase.Error.Contents + "\n" + testCase.Error.Contents
+				}
+			}
+		}
+	}
+	for _, testCase := range cases {
+		resultSuite.TestCases = append(resultSuite.TestCases, testCase)
+	}
+	sort.Slice(resultSuite.TestCases, func(i, j int) bool {
+		return resultSuite.TestCases[i].Name < resultSuite.TestCases[j].Name
+	})
+	return writeToFile(&resultSuites, outFile)
+}
+
 func writeToFile(suite interface{}, outFile io.Writer) error {
 	bytes, err := xml.MarshalIndent(suite, "", "\t")
 	if err != nil {

pkg/build/util/util_test.go

@@ -11,6 +11,8 @@
 package util
 
 import (
+	"bytes"
+	"encoding/xml"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -75,3 +77,85 @@ func TestOutputsOfGenrule(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, out, expected)
 }
+
+func TestMergeXml(t *testing.T) {
+	const xml1 = `<testsuites>
+	<testsuite errors="0" failures="1" skipped="0" tests="17" time="0.029" name="github.com/cockroachdb/cockroach/pkg/cmd/dev">
+		<testcase classname="dev" name="TestDataDriven" time="0.010"></testcase>
+		<testcase classname="dev" name="TestDataDriven/bench.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/build.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/builder.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/generate.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/lint.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/logic.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/bench.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/build.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/builder.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/generate.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/lint.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/logic.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/test.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/test.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestSetupPath" time="0.000">
+			<failure message="Failed" type="">FAILED :(</failure>
+		</testcase>
+	</testsuite>
+</testsuites>`
+	const xml2 = `<testsuites>
+	<testsuite errors="0" failures="1" skipped="0" tests="17" time="0.029" name="github.com/cockroachdb/cockroach/pkg/cmd/dev">
+		<testcase classname="dev" name="TestDataDriven" time="0.010"></testcase>
+		<testcase classname="dev" name="TestDataDriven/bench.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/build.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/builder.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/generate.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/lint.txt" time="0.000">
+			<failure message="Failed" type="">ALSO FAILED :(</failure>
+		</testcase>
+		<testcase classname="dev" name="TestDataDriven/logic.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/bench.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/build.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/builder.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/generate.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/lint.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/logic.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/recording/test.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestDataDriven/test.txt" time="0.000"></testcase>
+		<testcase classname="dev" name="TestSetupPath" time="0.000"></testcase>
+	</testsuite>
+</testsuites>`
+	const expected = `<testsuites>
+	<testsuite errors="0" failures="1" skipped="0" tests="17" time="0.029" name="github.com/cockroachdb/cockroach/pkg/cmd/dev">
+		<testcase name="TestDataDriven" time="0.010"></testcase>
+		<testcase name="TestDataDriven/bench.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/build.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/builder.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/generate.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/lint.txt" time="0.000">
+			<failure message="Failed" type="">ALSO FAILED :(</failure>
+		</testcase>
+		<testcase name="TestDataDriven/logic.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/recording" time="0.000"></testcase>
+		<testcase name="TestDataDriven/recording/bench.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/recording/build.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/recording/builder.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/recording/generate.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/recording/lint.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/recording/logic.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/recording/test.txt" time="0.000"></testcase>
+		<testcase name="TestDataDriven/test.txt" time="0.000"></testcase>
+		<testcase name="TestSetupPath" time="0.000">
+			<failure message="Failed" type="">FAILED :(</failure>
+		</testcase>
+	</testsuite>
+</testsuites>
+`
+
+	var suite1, suite2 TestSuites
+	require.NoError(t, xml.Unmarshal([]byte(xml1), &suite1))
+	require.NoError(t, xml.Unmarshal([]byte(xml2), &suite2))
+	var buf bytes.Buffer
+	require.NoError(t, MergeTestXMLs([]TestSuites{suite1, suite2}, &buf))
+	require.Equal(t, expected, buf.String())
+}

pkg/kv/db_test.go

@@ -699,3 +699,19 @@ func TestDBDecommissionedOperations(t *testing.T) {
 		})
 	}
 }
+
+// TestGenerateForcedRetryableError verifies that GenerateForcedRetryableError
+// returns an error with a transaction that had the epoch bumped (and not epoch 0).
+func TestGenerateForcedRetryableError(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	ctx := context.Background()
+	s, db := setup(t)
+	defer s.Stopper().Stop(context.Background())
+	txn := db.NewTxn(ctx, "test: TestGenerateForcedRetryableError")
+	require.Equal(t, 0, int(txn.Epoch()))
+	err := txn.GenerateForcedRetryableError(ctx, "testing TestGenerateForcedRetryableError")
+	var retryErr *roachpb.TransactionRetryWithProtoRefreshError
+	require.True(t, errors.As(err, &retryErr))
+	require.Equal(t, 1, int(retryErr.Transaction.Epoch))
+}

pkg/kv/txn.go

@@ -1468,17 +1468,7 @@ func (txn *Txn) GenerateForcedRetryableError(ctx context.Context, msg string) er
 	now := txn.db.clock.NowAsClockTimestamp()
 	txn.mu.sender.ManualRestart(ctx, txn.mu.userPriority, now.ToTimestamp())
 	txn.resetDeadlineLocked()
-	return roachpb.NewTransactionRetryWithProtoRefreshError(
-		msg,
-		txn.mu.ID,
-		roachpb.MakeTransaction(
-			txn.debugNameLocked(),
-			nil, // baseKey
-			txn.mu.userPriority,
-			now.ToTimestamp(),
-			txn.db.clock.MaxOffset().Nanoseconds(),
-			int32(txn.db.ctx.NodeID.SQLInstanceID())),
-	)
+	return txn.mu.sender.PrepareRetryableError(ctx, msg)
 }
 
 // PrepareRetryableError returns a

pkg/sql/BUILD.bazel

@@ -617,6 +617,7 @@ go_test(
         "//pkg/sql/rowinfra",
         "//pkg/sql/scrub",
         "//pkg/sql/sem/builtins",
+        "//pkg/sql/sem/catid",
         "//pkg/sql/sem/tree",
         "//pkg/sql/sessiondata",
         "//pkg/sql/sessiondatapb",

pkg/sql/delete_preserving_index_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/rowenc"
 	"github.com/cockroachdb/cockroach/pkg/sql/rowenc/rowencpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/rowinfra"
+	"github.com/cockroachdb/cockroach/pkg/sql/sem/catid"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/tests"
 	"github.com/cockroachdb/cockroach/pkg/startupmigrations"
@@ -239,7 +240,71 @@ func TestDeletePreservingIndexEncoding(t *testing.T) {
 			}
 		})
 	}
+}
 
+// TestDeletePreservingIndexEncodingWithEmptyValues tests
+func TestDeletePreservingIndexEncodingWithEmptyValues(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	params, _ := tests.CreateTestServerParams()
+	server, sqlDB, kvDB := serverutils.StartServer(t, params)
+	defer server.Stopper().Stop(context.Background())
+	setupSQL := `
+CREATE DATABASE t;
+CREATE TABLE t.test (
+        x INT PRIMARY KEY, y INT NOT NULL, z INT, a INT,
+        FAMILY (x), FAMILY (y), FAMILY (a), FAMILY (z)
+);
+CREATE UNIQUE INDEX test_index_to_mutate ON t.test (y) STORING (z, a);
+`
+	_, err := sqlDB.Exec(setupSQL)
+	require.NoError(t, err)
+	codec := server.ExecutorConfig().(sql.ExecutorConfig).Codec
+	tableDesc := catalogkv.TestingGetMutableExistingTableDescriptor(kvDB, codec, "t", "test")
+	err = mutateIndexByName(kvDB, codec, tableDesc, "test_index_to_mutate", func(idx *descpb.IndexDescriptor) error {
+		// Here, we make this index look like the temporary
+		// index for a new primary index during the
+		// MVCC-compatible index backfilling process.
+		idx.UseDeletePreservingEncoding = true
+		idx.EncodingType = descpb.PrimaryIndexEncoding
+		idx.StoreColumnNames = []string{"x", "z", "a"}
+		idx.StoreColumnIDs = []catid.ColumnID{0x1, 0x3, 0x4}
+		idx.KeySuffixColumnIDs = nil
+		return nil
+	})
+	require.NoError(t, err)
+	_, err = sqlDB.Exec(`INSERT INTO t.test VALUES (1, 1, 1, 1); DELETE FROM t.test WHERE x = 1;`)
+	require.NoError(t, err)
+}
+
+func mutateIndexByName(
+	kvDB *kv.DB,
+	codec keys.SQLCodec,
+	tableDesc *tabledesc.Mutable,
+	index string,
+	fn func(*descpb.IndexDescriptor) error,
+) error {
+	idx, err := tableDesc.FindIndexWithName(index)
+	if err != nil {
+		return err
+	}
+	idxCopy := *idx.IndexDesc()
+	if err := fn(&idxCopy); err != nil {
+		return err
+	}
+
+	tableDesc.RemovePublicNonPrimaryIndex(idx.Ordinal())
+	m := descpb.DescriptorMutation{}
+	m.Descriptor_ = &descpb.DescriptorMutation_Index{Index: &idxCopy}
+	m.Direction = descpb.DescriptorMutation_ADD
+	m.State = descpb.DescriptorMutation_DELETE_AND_WRITE_ONLY
+	tableDesc.Mutations = append(tableDesc.Mutations, m)
+	tableDesc.Version++
+	return kvDB.Put(
+		context.Background(),
+		catalogkeys.MakeDescMetadataKey(codec, tableDesc.ID),
+		tableDesc.DescriptorProto(),
+	)
 }
 
 type WrappedVersionedValues struct {

pkg/sql/distsql/BUILD.bazel

@@ -34,7 +34,7 @@ go_library(
 
 go_test(
     name = "distsql_test",
-    size = "small",
+    size = "medium",
     srcs = [
         "columnar_operators_test.go",
         "columnar_utils_test.go",