Skip to content

Commit

Permalink
feat: [cherry-pick]The expression supports filling elements through t…
Browse files Browse the repository at this point in the history
…emplates (milvus-io#37058)

issue: milvus-io#36672

master pr: milvus-io#37033

milvus-proto pr: milvus-io/milvus-proto#332

Signed-off-by: Cai Zhang <[email protected]>
  • Loading branch information
xiaocai2333 committed Oct 28, 2024
1 parent fc69df4 commit 93fd146
Show file tree
Hide file tree
Showing 30 changed files with 2,650 additions and 1,146 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.17.9
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240930043709-0c23514e4c34
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7
github.com/minio/minio-go/v7 v7.0.73
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
github.com/prometheus/client_golang v1.14.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,8 @@ github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZz
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240930043709-0c23514e4c34 h1:Fwxpg98128gfWRbQ1A3PMP9o2IfYZk7RSEy8rcoCWDA=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240930043709-0c23514e4c34/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7 h1:HwAitQk+V59QdYUwwVVYHTujd4QZrebg2Cc2hmcjhAg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
Expand Down
11 changes: 7 additions & 4 deletions internal/parser/planparserv2/Plan.g4
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ expr:
| FloatingConstant # Floating
| BooleanConstant # Boolean
| StringLiteral # String
| Identifier # Identifier
| (Identifier|Meta) # Identifier
| JSONIdentifier # JSONIdentifier
| LBRACE Identifier RBRACE # TemplateVariable
| '(' expr ')' # Parens
| '[' expr (',' expr)* ','? ']' # Array
| EmptyArray # EmptyArray
Expand All @@ -19,7 +20,6 @@ expr:
| expr op = (ADD | SUB) expr # AddSub
| expr op = (SHL | SHR) expr # Shift
| expr op = NOT? IN expr # Term
// | EmptyTerm # EmptyTerm
| (JSONContains | ArrayContains)'('expr',' expr')' # JSONContains
| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')' # JSONContainsAll
| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')' # JSONContainsAny
Expand All @@ -45,6 +45,8 @@ expr:
// INT64: 'int64';
// FLOAT: 'float';
// DOUBLE: 'double';
LBRACE: '{';
RBRACE: '}';

LT: '<';
LE: '<=';
Expand Down Expand Up @@ -99,10 +101,11 @@ FloatingConstant:
DecimalFloatingConstant
| HexadecimalFloatingConstant;

Identifier: Nondigit (Nondigit | Digit)* | '$meta';
Identifier: Nondigit (Nondigit | Digit)*;
Meta: '$meta';

StringLiteral: EncodingPrefix? ('"' DoubleSCharSequence? '"' | '\'' SingleSCharSequence? '\'');
JSONIdentifier: Identifier('[' (StringLiteral | DecimalConstant) ']')+;
JSONIdentifier: (Identifier | Meta)('[' (StringLiteral | DecimalConstant) ']')+;

fragment EncodingPrefix: 'u8' | 'u' | 'U' | 'L';

Expand Down
4 changes: 2 additions & 2 deletions internal/parser/planparserv2/check_identical_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ func TestCheckIdentical(t *testing.T) {
exprStr1 := exprStr1Arr[i]
exprStr2 := exprStr2Arr[i]

expr1, err := ParseExpr(helper, exprStr1)
expr1, err := ParseExpr(helper, exprStr1, nil)
assert.NoError(t, err)
expr2, err := ParseExpr(helper, exprStr2)
expr2, err := ParseExpr(helper, exprStr2, nil)
assert.NoError(t, err)

assert.True(t, CheckPredicatesIdentical(expr1, expr1))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package planparserv2

import (
"fmt"

"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/proto/planpb"
)

func ConvertToGenericValue(templateName string, templateValue *schemapb.TemplateValue) (*planpb.GenericValue, error) {
if templateValue == nil {
return nil, fmt.Errorf("expression template variable value is nil, template name: {%s}", templateName)
}
switch templateValue.GetType() {
case schemapb.DataType_Bool:
return &planpb.GenericValue{
Val: &planpb.GenericValue_BoolVal{
BoolVal: templateValue.GetBoolVal(),
},
}, nil
case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32, schemapb.DataType_Int64:
return &planpb.GenericValue{
Val: &planpb.GenericValue_Int64Val{
Int64Val: templateValue.GetInt64Val(),
},
}, nil
case schemapb.DataType_Float, schemapb.DataType_Double:
return &planpb.GenericValue{
Val: &planpb.GenericValue_FloatVal{
FloatVal: templateValue.GetFloatVal(),
},
}, nil
case schemapb.DataType_String, schemapb.DataType_VarChar:
return &planpb.GenericValue{
Val: &planpb.GenericValue_StringVal{
StringVal: templateValue.GetStringVal(),
},
}, nil
case schemapb.DataType_Array:
elements := templateValue.GetArrayVal().GetArray()
arrayValues := make([]*planpb.GenericValue, len(elements))
for i, element := range elements {
arrayElement, err := ConvertToGenericValue(templateName, element)
if err != nil {
return nil, err
}
arrayValues[i] = arrayElement
}
return &planpb.GenericValue{
Val: &planpb.GenericValue_ArrayVal{
ArrayVal: &planpb.Array{
Array: arrayValues,
SameType: templateValue.GetArrayVal().GetSameType(),
ElementType: templateValue.GetArrayVal().GetElementType(),
},
},
}, nil
default:
return nil, fmt.Errorf("expression elements can only be scalars")
}
}

func UnmarshalExpressionValues(input map[string]*schemapb.TemplateValue) (map[string]*planpb.GenericValue, error) {
result := make(map[string]*planpb.GenericValue, len(input))
for name, value := range input {
rv, err := ConvertToGenericValue(name, value)
if err != nil {
return nil, err
}
result[name] = rv
}
return result, nil
}
Loading

0 comments on commit 93fd146

Please sign in to comment.