From 49cf44d4d42742840fe998655193dd2eebac1f65 Mon Sep 17 00:00:00 2001 From: Jeremy Cole Date: Wed, 11 Jan 2023 16:09:21 -0800 Subject: [PATCH 1/3] Add new placement Vindex strategy Signed-off-by: Jeremy Cole --- go/vt/vtgate/vindexes/placement.go | 209 +++++++++++++++++ go/vt/vtgate/vindexes/placement_test.go | 290 ++++++++++++++++++++++++ 2 files changed, 499 insertions(+) create mode 100644 go/vt/vtgate/vindexes/placement.go create mode 100644 go/vt/vtgate/vindexes/placement_test.go diff --git a/go/vt/vtgate/vindexes/placement.go b/go/vt/vtgate/vindexes/placement.go new file mode 100644 index 00000000000..97fc75d210d --- /dev/null +++ b/go/vt/vtgate/vindexes/placement.go @@ -0,0 +1,209 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + +A Vindex which uses a mapping lookup table `placement_map` to set the first `placement_prefix_bytes` of the Keyspace ID +and another Vindex type `placement_sub_vindex_type` (which must support Hashing) as a sub-Vindex to set the rest. +This is suitable for regional sharding (like region_json or region_experimental) but does not require a mapping file, +and can support non-integer types for the sharding column. All parameters are prefixed with `placement_` so as to avoid +conflict, because the `params` map is passed to the sub-Vindex as well. + +*/ + +package vindexes + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "strconv" + "strings" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" +) + +var ( + _ MultiColumn = (*Placement)(nil) + + PlacementRequiredParams = []string{ + "placement_map", + "placement_prefix_bytes", + "placement_sub_vindex_type", + } +) + +func init() { + Register("placement", NewPlacement) +} + +type PlacementMap map[string]uint64 + +type Placement struct { + name string + placementMap PlacementMap + subVindex Vindex + subVindexType string + subVindexName string + prefixBytes int +} + +// Parse a string containing a list of delimited string:integer key-value pairs, e.g. "foo:1,bar:2". +func parsePlacementMap(s string) (*PlacementMap, error) { + placementMap := make(PlacementMap) + for _, entry := range strings.Split(s, ",") { + if entry == "" { + continue + } + + kv := strings.Split(entry, ":") + if len(kv) != 2 { + return nil, fmt.Errorf("entry: %v; expected key:value", entry) + } + if kv[0] == "" { + return nil, fmt.Errorf("entry: %v; unexpected empty key", entry) + } + if kv[1] == "" { + return nil, fmt.Errorf("entry: %v; unexpected empty value", entry) + } + + value, err := strconv.ParseUint(kv[1], 0, 64) + if err != nil { + return nil, fmt.Errorf("entry: %v; %v", entry, err) + } + placementMap[kv[0]] = value + } + return &placementMap, nil +} + +func NewPlacement(name string, params map[string]string) (Vindex, error) { + var missingParams []string + for _, param := range PlacementRequiredParams { + if params[param] == "" { + missingParams = append(missingParams, param) + } + } + + if len(missingParams) > 0 { + return nil, fmt.Errorf("missing params: %s", strings.Join(missingParams, ", ")) + } + + placementMap, parseError := parsePlacementMap(params["placement_map"]) + if parseError != nil { + return nil, fmt.Errorf("malformed placement_map; %v", parseError) + } + + prefixBytes, prefixError := strconv.Atoi(params["placement_prefix_bytes"]) + if prefixError != nil { + return nil, prefixError + } + + if prefixBytes < 1 || prefixBytes > 7 { + return nil, fmt.Errorf("invalid placement_prefix_bytes: %v; expected integer between 1 and 7", prefixBytes) + } + + subVindexType := params["placement_sub_vindex_type"] + subVindexName := fmt.Sprintf("%s_sub_vindex", name) + subVindex, createVindexError := CreateVindex(subVindexType, subVindexName, params) + if createVindexError != nil { + return nil, fmt.Errorf("invalid placement_sub_vindex_type: %v", createVindexError) + } + + // TODO: Should we support MultiColumn Vindex? + if _, subVindexSupportsHashing := subVindex.(Hashing); !subVindexSupportsHashing { + return nil, fmt.Errorf("invalid placement_sub_vindex_type: %v; does not support the Hashing interface", createVindexError) + } + + return &Placement{ + name: name, + placementMap: *placementMap, + subVindex: subVindex, + subVindexType: subVindexType, + subVindexName: subVindexName, + prefixBytes: prefixBytes, + }, nil +} + +func (p *Placement) String() string { + return p.name +} + +func (p *Placement) Cost() int { + return 1 +} + +func (p *Placement) IsUnique() bool { + return true +} + +func (p *Placement) NeedsVCursor() bool { + return false +} + +func (p *Placement) PartialVindex() bool { + return false +} + +func (p *Placement) Map(ctx context.Context, vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) { + destinations := make([]key.Destination, 0, len(rowsColValues)) + + for _, row := range rowsColValues { + if len(row) != 2 { + destinations = append(destinations, key.DestinationNone{}) + continue + } + + placementKey := row[0].ToString() + placementDestinationValue, placementMappingFound := p.placementMap[placementKey] + if !placementMappingFound { + destinations = append(destinations, key.DestinationNone{}) + continue + } + + placementDestinationPrefix := make([]byte, 8) + binary.BigEndian.PutUint64(placementDestinationPrefix, placementDestinationValue) + if p.prefixBytes < 8 { + // Shorten the prefix to the desired length. + placementDestinationPrefix = placementDestinationPrefix[(8 - p.prefixBytes):] + } + + subVindexValue, hashingError := p.subVindex.(Hashing).Hash(row[1]) + if hashingError != nil { + return nil, hashingError // TODO: Should we be less fatal here and use DestinationNone? + } + + // Concatenate and add to destinations. + rowDestination := append(placementDestinationPrefix, subVindexValue...) + destinations = append(destinations, key.DestinationKeyspaceID(rowDestination[0:8])) + } + + return destinations, nil +} + +func (p *Placement) Verify(ctx context.Context, vcursor VCursor, rowsColValues [][]sqltypes.Value, keyspaceIDs [][]byte) ([]bool, error) { + result := make([]bool, len(rowsColValues)) + destinations, _ := p.Map(ctx, vcursor, rowsColValues) + for i, destination := range destinations { + destinationKeyspaceID, ok := destination.(key.DestinationKeyspaceID) + if !ok { + continue + } + result[i] = bytes.Equal(destinationKeyspaceID, keyspaceIDs[i]) + } + return result, nil +} diff --git a/go/vt/vtgate/vindexes/placement_test.go b/go/vt/vtgate/vindexes/placement_test.go new file mode 100644 index 00000000000..22ab40adc08 --- /dev/null +++ b/go/vt/vtgate/vindexes/placement_test.go @@ -0,0 +1,290 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" +) + +func createBasicPlacementVindex(t *testing.T) (Vindex, error) { + return CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "1", + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "xxhash", + }) +} + +func TestPlacementName(t *testing.T) { + vindex, err := createBasicPlacementVindex(t) + require.NoError(t, err) + assert.Equal(t, "placement", vindex.String()) +} + +func TestPlacementCost(t *testing.T) { + vindex, err := createBasicPlacementVindex(t) + require.NoError(t, err) + assert.Equal(t, 1, vindex.Cost()) +} + +func TestPlacementIsUnique(t *testing.T) { + vindex, err := createBasicPlacementVindex(t) + require.NoError(t, err) + assert.True(t, vindex.IsUnique()) +} + +func TestPlacementNeedsVCursor(t *testing.T) { + vindex, err := createBasicPlacementVindex(t) + require.NoError(t, err) + assert.False(t, vindex.NeedsVCursor()) +} + +func TestPlacementNoParams(t *testing.T) { + _, err := CreateVindex("placement", "placement", nil) + assert.EqualError(t, err, "missing params: placement_map, placement_prefix_bytes, placement_sub_vindex_type") +} + +func TestPlacementPlacementMapMissing(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_prefix_bytes": "1", + "placement_sub_vindex_type": "hash", + }) + assert.EqualError(t, err, "missing params: placement_map") +} + +func TestPlacementPlacementMapMalformedMap(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "xyz", + "placement_prefix_bytes": "1", + "placement_sub_vindex_type": "hash", + }) + assert.EqualError(t, err, "malformed placement_map; entry: xyz; expected key:value") +} + +func TestPlacementPlacementMapMissingKey(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "abc:1,:2,ghi:3", + "placement_prefix_bytes": "1", + "placement_sub_vindex_type": "hash", + }) + assert.EqualError(t, err, "malformed placement_map; entry: :2; unexpected empty key") +} + +func TestPlacementPlacementMapMissingValue(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "abc:1,def:,ghi:3", + "placement_prefix_bytes": "1", + "placement_sub_vindex_type": "hash", + }) + assert.EqualError(t, err, "malformed placement_map; entry: def:; unexpected empty value") +} + +func TestPlacementPlacementMapMalformedValue(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "abc:xyz", + "placement_prefix_bytes": "1", + "placement_sub_vindex_type": "hash", + }) + assert.EqualError(t, err, "malformed placement_map; entry: abc:xyz; strconv.ParseUint: parsing \"xyz\": invalid syntax") +} + +func TestPlacementPrefixBytesMissing(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "hash", + }) + assert.EqualError(t, err, "missing params: placement_prefix_bytes") +} + +func TestPlacementPrefixBytesTooLow(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "foo:1,bar:2", + "placement_prefix_bytes": "0", + "placement_sub_vindex_type": "hash", + }) + assert.EqualError(t, err, "invalid placement_prefix_bytes: 0; expected integer between 1 and 7") +} + +func TestPlacementPrefixBytesTooHigh(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "foo:1,bar:2", + "placement_prefix_bytes": "17", + "placement_sub_vindex_type": "hash", + }) + assert.EqualError(t, err, "invalid placement_prefix_bytes: 17; expected integer between 1 and 7") +} + +func TestPlacementSubVindexTypeMissing(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "foo:1,bar:2", + "placement_prefix_bytes": "1", + }) + assert.EqualError(t, err, "missing params: placement_sub_vindex_type") +} + +func TestPlacementSubVindexTypeIncorrect(t *testing.T) { + _, err := CreateVindex("placement", "placement", map[string]string{ + "placement_map": "foo:1,bar:2", + "placement_prefix_bytes": "1", + "placement_sub_vindex_type": "doesnotexist", + }) + assert.EqualError(t, err, "invalid placement_sub_vindex_type: vindexType \"doesnotexist\" not found") +} + +func TestPlacementMapSqlTypeVarChar(t *testing.T) { + vindex, err := CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "1", + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "xxhash", + }) + assert.NoError(t, err) + actualDestinations, err := vindex.(MultiColumn).Map(context.Background(), nil, [][]sqltypes.Value{{ + sqltypes.NewVarChar("foo"), sqltypes.NewVarChar("hello world"), + }, { + sqltypes.NewVarChar("bar"), sqltypes.NewVarChar("hello world"), + }, { + sqltypes.NewVarChar("xyz"), sqltypes.NewVarChar("hello world"), + }}) + assert.NoError(t, err) + + expectedDestinations := []key.Destination{ + key.DestinationKeyspaceID{0x01, 0x68, 0x69, 0x1e, 0xb2, 0x34, 0x67, 0xab}, + key.DestinationKeyspaceID{0x02, 0x68, 0x69, 0x1e, 0xb2, 0x34, 0x67, 0xab}, + key.DestinationNone{}, + } + assert.Equal(t, expectedDestinations, actualDestinations) +} + +func TestPlacementMapSqlTypeInt64(t *testing.T) { + vindex, err := CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "1", + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "xxhash", + }) + assert.NoError(t, err) + actualDestinations, err := vindex.(MultiColumn).Map(context.Background(), nil, [][]sqltypes.Value{{ + sqltypes.NewVarChar("foo"), sqltypes.NewInt64(1), + }, { + sqltypes.NewVarChar("bar"), sqltypes.NewInt64(1), + }, { + sqltypes.NewVarChar("xyz"), sqltypes.NewInt64(1), + }}) + assert.NoError(t, err) + + expectedDestinations := []key.Destination{ + key.DestinationKeyspaceID{0x01, 0xd4, 0x64, 0x05, 0x36, 0x76, 0x12, 0xb4}, + key.DestinationKeyspaceID{0x02, 0xd4, 0x64, 0x05, 0x36, 0x76, 0x12, 0xb4}, + key.DestinationNone{}, + } + assert.Equal(t, expectedDestinations, actualDestinations) +} + +func TestPlacementMapWithHexValues(t *testing.T) { + vindex, err := CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "1", + "placement_map": "foo:0x01,bar:0x02", + "placement_sub_vindex_type": "xxhash", + }) + assert.NoError(t, err) + actualDestinations, err := vindex.(MultiColumn).Map(context.Background(), nil, [][]sqltypes.Value{{ + sqltypes.NewVarChar("foo"), sqltypes.NewInt64(1), + }, { + sqltypes.NewVarChar("bar"), sqltypes.NewInt64(1), + }, { + sqltypes.NewVarChar("xyz"), sqltypes.NewInt64(1), + }}) + assert.NoError(t, err) + + expectedDestinations := []key.Destination{ + key.DestinationKeyspaceID{0x01, 0xd4, 0x64, 0x05, 0x36, 0x76, 0x12, 0xb4}, + key.DestinationKeyspaceID{0x02, 0xd4, 0x64, 0x05, 0x36, 0x76, 0x12, 0xb4}, + key.DestinationNone{}, + } + assert.Equal(t, expectedDestinations, actualDestinations) +} + +func TestPlacementMapMultiplePrefixBytes(t *testing.T) { + vindex, err := CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "4", + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "xxhash", + }) + assert.NoError(t, err) + actualDestinations, err := vindex.(MultiColumn).Map(context.Background(), nil, [][]sqltypes.Value{{ + sqltypes.NewVarChar("foo"), sqltypes.NewInt64(1), + }, { + sqltypes.NewVarChar("bar"), sqltypes.NewInt64(1), + }, { + sqltypes.NewVarChar("xyz"), sqltypes.NewInt64(1), + }}) + assert.NoError(t, err) + + expectedDestinations := []key.Destination{ + key.DestinationKeyspaceID{0x00, 0x00, 0x00, 0x01, 0xd4, 0x64, 0x05, 0x36}, + key.DestinationKeyspaceID{0x00, 0x00, 0x00, 0x02, 0xd4, 0x64, 0x05, 0x36}, + key.DestinationNone{}, + } + assert.Equal(t, expectedDestinations, actualDestinations) +} + +func TestPlacementSubVindexNumeric(t *testing.T) { + vindex, err := CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "1", + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "numeric", + }) + assert.NoError(t, err) + actualDestinations, err := vindex.(MultiColumn).Map(context.Background(), nil, [][]sqltypes.Value{{ + sqltypes.NewVarChar("foo"), sqltypes.NewInt64(0x01234567deadbeef), + }, { + sqltypes.NewVarChar("bar"), sqltypes.NewInt64(0x01234567deadbeef), + }, { + sqltypes.NewVarChar("xyz"), sqltypes.NewInt64(0x01234567deadbeef), + }}) + assert.NoError(t, err) + + expectedDestinations := []key.Destination{ + key.DestinationKeyspaceID{0x01, 0x01, 0x23, 0x45, 0x67, 0xde, 0xad, 0xbe}, + key.DestinationKeyspaceID{0x02, 0x01, 0x23, 0x45, 0x67, 0xde, 0xad, 0xbe}, + key.DestinationNone{}, + } + assert.Equal(t, expectedDestinations, actualDestinations) +} From 2a9129de2be126bb19beae2d256e9dfc376ce487 Mon Sep 17 00:00:00 2001 From: Jeremy Cole Date: Tue, 7 Feb 2023 15:30:12 -0800 Subject: [PATCH 2/3] Sprinkle addPadding everywhere when comparing KeyRange Start/End values --- go/vt/key/key.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/go/vt/key/key.go b/go/vt/key/key.go index fc603554ecf..6617be18818 100644 --- a/go/vt/key/key.go +++ b/go/vt/key/key.go @@ -113,10 +113,10 @@ func KeyRangeAdd(first, second *topodatapb.KeyRange) (*topodatapb.KeyRange, bool if first == nil || second == nil { return nil, false } - if len(first.End) != 0 && bytes.Equal(first.End, second.Start) { + if len(first.End) != 0 && bytes.Equal(addPadding(first.End), addPadding(second.Start)) { return &topodatapb.KeyRange{Start: first.Start, End: second.End}, true } - if len(second.End) != 0 && bytes.Equal(second.End, first.Start) { + if len(second.End) != 0 && bytes.Equal(addPadding(second.End), addPadding(first.Start)) { return &topodatapb.KeyRange{Start: second.Start, End: first.End}, true } return nil, false @@ -127,8 +127,8 @@ func KeyRangeContains(kr *topodatapb.KeyRange, id []byte) bool { if kr == nil { return true } - return bytes.Compare(kr.Start, id) <= 0 && - (len(kr.End) == 0 || bytes.Compare(id, kr.End) < 0) + return bytes.Compare(addPadding(kr.Start), addPadding(id)) <= 0 && + (len(kr.End) == 0 || bytes.Compare(addPadding(id), addPadding(kr.End)) < 0) } // ParseKeyRangeParts parses a start and end hex values and build a proto KeyRange @@ -202,7 +202,7 @@ func KeyRangeStartSmaller(left, right *topodatapb.KeyRange) bool { if right == nil { return false } - return bytes.Compare(left.Start, right.Start) < 0 + return bytes.Compare(addPadding(left.Start), addPadding(right.Start)) < 0 } // KeyRangeStartEqual returns true if both key ranges have the same start @@ -250,8 +250,8 @@ func KeyRangesIntersect(first, second *topodatapb.KeyRange) bool { if first == nil || second == nil { return true } - return (len(first.End) == 0 || bytes.Compare(second.Start, first.End) < 0) && - (len(second.End) == 0 || bytes.Compare(first.Start, second.End) < 0) + return (len(first.End) == 0 || bytes.Compare(addPadding(second.Start), addPadding(first.End)) < 0) && + (len(second.End) == 0 || bytes.Compare(addPadding(first.Start), addPadding(second.End)) < 0) } // KeyRangesOverlap returns the overlap between two KeyRanges. @@ -270,14 +270,14 @@ func KeyRangesOverlap(first, second *topodatapb.KeyRange) (*topodatapb.KeyRange, // start with (a,b) result := proto.Clone(first).(*topodatapb.KeyRange) // if c > a, then use c - if bytes.Compare(second.Start, first.Start) > 0 { + if bytes.Compare(addPadding(second.Start), addPadding(first.Start)) > 0 { result.Start = second.Start } // if b is maxed out, or // (d is not maxed out and d < b) // ^ valid test as neither b nor d are max // then use d - if len(first.End) == 0 || (len(second.End) != 0 && bytes.Compare(second.End, first.End) < 0) { + if len(first.End) == 0 || (len(second.End) != 0 && bytes.Compare(addPadding(second.End), addPadding(first.End)) < 0) { result.End = second.End } return result, nil @@ -297,10 +297,10 @@ func KeyRangeIncludes(big, small *topodatapb.KeyRange) bool { return len(big.Start) == 0 && len(big.End) == 0 } // Now we check small.Start >= big.Start, and small.End <= big.End - if len(big.Start) != 0 && bytes.Compare(small.Start, big.Start) < 0 { + if len(big.Start) != 0 && bytes.Compare(addPadding(small.Start), addPadding(big.Start)) < 0 { return false } - if len(big.End) != 0 && (len(small.End) == 0 || bytes.Compare(small.End, big.End) > 0) { + if len(big.End) != 0 && (len(small.End) == 0 || bytes.Compare(addPadding(small.End), addPadding(big.End)) > 0) { return false } return true From 6faac58532a0ea55f992615c99c4e98deb14ea52 Mon Sep 17 00:00:00 2001 From: Jeremy Cole Date: Wed, 8 Feb 2023 21:21:00 -0800 Subject: [PATCH 3/3] Implement support for PartialVindex usage Signed-off-by: Jeremy Cole --- go/vt/vtgate/vindexes/placement.go | 57 ++++++++++++++-------- go/vt/vtgate/vindexes/placement_test.go | 64 +++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 21 deletions(-) diff --git a/go/vt/vtgate/vindexes/placement.go b/go/vt/vtgate/vindexes/placement.go index 97fc75d210d..4382bc95c83 100644 --- a/go/vt/vtgate/vindexes/placement.go +++ b/go/vt/vtgate/vindexes/placement.go @@ -34,6 +34,9 @@ import ( "strconv" "strings" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" ) @@ -156,18 +159,30 @@ func (p *Placement) NeedsVCursor() bool { } func (p *Placement) PartialVindex() bool { - return false + return true +} + +func makeDestinationPrefix(value uint64, prefixBytes int) []byte { + destinationPrefix := make([]byte, 8) + binary.BigEndian.PutUint64(destinationPrefix, value) + if prefixBytes < 8 { + // Shorten the prefix to the desired length. + destinationPrefix = destinationPrefix[(8 - prefixBytes):] + } + + return destinationPrefix } func (p *Placement) Map(ctx context.Context, vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) { destinations := make([]key.Destination, 0, len(rowsColValues)) for _, row := range rowsColValues { - if len(row) != 2 { - destinations = append(destinations, key.DestinationNone{}) - continue + if len(row) != 1 && len(row) != 2 { + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "wrong number of column values were passed: expected 1-2, got %d", len(row)) } + // Calculate the destination prefix from the placement key which will be the same whether this is a partial + // or full usage of the Vindex. placementKey := row[0].ToString() placementDestinationValue, placementMappingFound := p.placementMap[placementKey] if !placementMappingFound { @@ -175,21 +190,20 @@ func (p *Placement) Map(ctx context.Context, vcursor VCursor, rowsColValues [][] continue } - placementDestinationPrefix := make([]byte, 8) - binary.BigEndian.PutUint64(placementDestinationPrefix, placementDestinationValue) - if p.prefixBytes < 8 { - // Shorten the prefix to the desired length. - placementDestinationPrefix = placementDestinationPrefix[(8 - p.prefixBytes):] - } + placementDestinationPrefix := makeDestinationPrefix(placementDestinationValue, p.prefixBytes) - subVindexValue, hashingError := p.subVindex.(Hashing).Hash(row[1]) - if hashingError != nil { - return nil, hashingError // TODO: Should we be less fatal here and use DestinationNone? - } + if len(row) == 1 { // Partial Vindex usage with only the placement column provided. + destinations = append(destinations, NewKeyRangeFromPrefix(placementDestinationPrefix)) + } else if len(row) == 2 { // Full Vindex usage with the placement column and subVindex column provided. + subVindexValue, hashingError := p.subVindex.(Hashing).Hash(row[1]) + if hashingError != nil { + return nil, hashingError // TODO: Should we be less fatal here and use DestinationNone? + } - // Concatenate and add to destinations. - rowDestination := append(placementDestinationPrefix, subVindexValue...) - destinations = append(destinations, key.DestinationKeyspaceID(rowDestination[0:8])) + // Concatenate and add to destinations. + rowDestination := append(placementDestinationPrefix, subVindexValue...) + destinations = append(destinations, key.DestinationKeyspaceID(rowDestination[0:8])) + } } return destinations, nil @@ -199,11 +213,12 @@ func (p *Placement) Verify(ctx context.Context, vcursor VCursor, rowsColValues [ result := make([]bool, len(rowsColValues)) destinations, _ := p.Map(ctx, vcursor, rowsColValues) for i, destination := range destinations { - destinationKeyspaceID, ok := destination.(key.DestinationKeyspaceID) - if !ok { - continue + switch d := destination.(type) { + case key.DestinationKeyspaceID: + result[i] = bytes.Equal(d, keyspaceIDs[i]) + default: + result[i] = false } - result[i] = bytes.Equal(destinationKeyspaceID, keyspaceIDs[i]) } return result, nil } diff --git a/go/vt/vtgate/vindexes/placement_test.go b/go/vt/vtgate/vindexes/placement_test.go index 22ab40adc08..c590f1de3e8 100644 --- a/go/vt/vtgate/vindexes/placement_test.go +++ b/go/vt/vtgate/vindexes/placement_test.go @@ -288,3 +288,67 @@ func TestPlacementSubVindexNumeric(t *testing.T) { } assert.Equal(t, expectedDestinations, actualDestinations) } + +func TestPlacementMapPrefixOnly(t *testing.T) { + vindex, err := CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "1", + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "xxhash", + }) + assert.NoError(t, err) + actualDestinations, err := vindex.(MultiColumn).Map(context.Background(), nil, [][]sqltypes.Value{{ + sqltypes.NewVarChar("foo"), + }, { + sqltypes.NewVarChar("bar"), + }, { + sqltypes.NewVarChar("xyz"), + }}) + assert.NoError(t, err) + + expectedDestinations := []key.Destination{ + NewKeyRangeFromPrefix([]byte{0x01}), + NewKeyRangeFromPrefix([]byte{0x02}), + key.DestinationNone{}, + } + assert.Equal(t, expectedDestinations, actualDestinations) +} + +func TestPlacementMapTooManyColumns(t *testing.T) { + vindex, err := CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "1", + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "xxhash", + }) + assert.NoError(t, err) + actualDestinations, err := vindex.(MultiColumn).Map(context.Background(), nil, [][]sqltypes.Value{{ + // Too many columns; expecting two, providing three. + sqltypes.NewVarChar("a"), sqltypes.NewVarChar("b"), sqltypes.NewVarChar("c"), + }}) + assert.EqualError(t, err, "wrong number of column values were passed: expected 1-2, got 3") + + assert.Nil(t, actualDestinations) +} + +func TestPlacementMapNoColumns(t *testing.T) { + vindex, err := CreateVindex("placement", "placement", map[string]string{ + "table": "t", + "from": "f1,f2", + "to": "toc", + "placement_prefix_bytes": "1", + "placement_map": "foo:1,bar:2", + "placement_sub_vindex_type": "xxhash", + }) + assert.NoError(t, err) + actualDestinations, err := vindex.(MultiColumn).Map(context.Background(), nil, [][]sqltypes.Value{{ + // Empty column list. + }}) + assert.EqualError(t, err, "wrong number of column values were passed: expected 1-2, got 0") + + assert.Nil(t, actualDestinations) +}