Skip to content

Commit

Permalink
Support template to parse expression
Browse files Browse the repository at this point in the history
Signed-off-by: Cai Zhang <[email protected]>
  • Loading branch information
xiaocai2333 committed Oct 25, 2024
1 parent ae4ce9b commit 3367309
Show file tree
Hide file tree
Showing 30 changed files with 2,742 additions and 1,369 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.17.9
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b
github.com/minio/minio-go/v7 v7.0.73
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
github.com/prometheus/client_golang v1.14.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,8 @@ github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZz
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13 h1:79fAk5rUFNl7A5RhYUlxcZTHAY31lygULC6QUvpBrUo=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b h1:Q4f3a6Y+HZFEadEx3pUXTZmBwrS3QRvkEHWNd8fjDtI=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70 h1:Z+sp64fmAOxAG7mU0dfVOXvAXlwRB0c8a96rIM5HevI=
github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
Expand Down
19 changes: 11 additions & 8 deletions internal/parser/planparserv2/Plan.g4
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,20 @@ expr:
| FloatingConstant # Floating
| BooleanConstant # Boolean
| StringLiteral # String
| Identifier # Identifier
| (Identifier|Meta) # Identifier
| JSONIdentifier # JSONIdentifier
| LBRACE Identifier RBRACE # TemplateVariable
| '(' expr ')' # Parens
| '[' expr (',' expr)* ','? ']' # Array
| EmptyArray # EmptyArray
| expr LIKE StringLiteral # Like
| expr POW expr # Power
| op = (ADD | SUB | BNOT | NOT) expr # Unary
// | '(' typeName ')' expr # Cast
| expr op = (MUL | DIV | MOD) expr # MulDivMod
| expr op = (ADD | SUB) expr # AddSub
| expr op = (SHL | SHR) expr # Shift
| expr op = (IN | NIN) ('[' expr (',' expr)* ','? ']') # Term
| expr op = (IN | NIN) EmptyTerm # EmptyTerm
| expr op = NOT? IN expr # Term
| (JSONContains | ArrayContains)'('expr',' expr')' # JSONContains
| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')' # JSONContainsAll
| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')' # JSONContainsAny
Expand All @@ -42,6 +43,8 @@ expr:
// INT64: 'int64';
// FLOAT: 'float';
// DOUBLE: 'double';
LBRACE: '{';
RBRACE: '}';

LT: '<';
LE: '<=';
Expand Down Expand Up @@ -71,9 +74,8 @@ OR: '||' | 'or';
BNOT: '~';
NOT: '!' | 'not';

IN: 'in';
NIN: 'not in';
EmptyTerm: '[' (Whitespace | Newline)* ']';
IN: 'in' | 'IN';
EmptyArray: '[' (Whitespace | Newline)* ']';

JSONContains: 'json_contains' | 'JSON_CONTAINS';
JSONContainsAll: 'json_contains_all' | 'JSON_CONTAINS_ALL';
Expand All @@ -96,10 +98,11 @@ FloatingConstant:
DecimalFloatingConstant
| HexadecimalFloatingConstant;

Identifier: Nondigit (Nondigit | Digit)* | '$meta';
Identifier: Nondigit (Nondigit | Digit)*;
Meta: '$meta';

StringLiteral: EncodingPrefix? ('"' DoubleSCharSequence? '"' | '\'' SingleSCharSequence? '\'');
JSONIdentifier: Identifier('[' (StringLiteral | DecimalConstant) ']')+;
JSONIdentifier: (Identifier | Meta)('[' (StringLiteral | DecimalConstant) ']')+;

fragment EncodingPrefix: 'u8' | 'u' | 'U' | 'L';

Expand Down
4 changes: 2 additions & 2 deletions internal/parser/planparserv2/check_identical_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ func TestCheckIdentical(t *testing.T) {
exprStr1 := `not (((Int64Field > 0) and (FloatField <= 20.0)) or ((Int32Field in [1, 2, 3]) and (VarCharField < "str")))`
exprStr2 := `Int32Field in [1, 2, 3]`

expr1, err := ParseExpr(helper, exprStr1)
expr1, err := ParseExpr(helper, exprStr1, nil)
assert.NoError(t, err)
expr2, err := ParseExpr(helper, exprStr2)
expr2, err := ParseExpr(helper, exprStr2, nil)
assert.NoError(t, err)

assert.True(t, CheckPredicatesIdentical(expr1, expr1))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package planparserv2

import (
"fmt"

"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/proto/planpb"
)

func ConvertToGenericValue(templateName string, templateValue *schemapb.TemplateValue) (*planpb.GenericValue, error) {
if templateValue == nil {
return nil, fmt.Errorf("expression template variable values is nil, template name: {%s}", templateName)
}
switch templateValue.GetType() {
case schemapb.DataType_Bool:
return &planpb.GenericValue{
Val: &planpb.GenericValue_BoolVal{
BoolVal: templateValue.GetBoolVal(),
},
}, nil
case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32, schemapb.DataType_Int64:
return &planpb.GenericValue{
Val: &planpb.GenericValue_Int64Val{
Int64Val: templateValue.GetInt64Val(),
},
}, nil
case schemapb.DataType_Float, schemapb.DataType_Double:
return &planpb.GenericValue{
Val: &planpb.GenericValue_FloatVal{
FloatVal: templateValue.GetFloatVal(),
},
}, nil
case schemapb.DataType_String, schemapb.DataType_VarChar:
return &planpb.GenericValue{
Val: &planpb.GenericValue_StringVal{
StringVal: templateValue.GetStringVal(),
},
}, nil
case schemapb.DataType_Array:
elements := templateValue.GetArrayVal().GetArray()
arrayValues := make([]*planpb.GenericValue, len(elements))
for i, element := range elements {
arrayElement, err := ConvertToGenericValue(templateName, element)
if err != nil {
return nil, err
}
arrayValues[i] = arrayElement
}
return &planpb.GenericValue{
Val: &planpb.GenericValue_ArrayVal{
ArrayVal: &planpb.Array{
Array: arrayValues,
SameType: templateValue.GetArrayVal().GetSameType(),
ElementType: templateValue.GetArrayVal().GetElementType(),
},
},
}, nil
default:
return nil, fmt.Errorf("expression elements can only be scalars")

}
}

func UnmarshalExpressionValues(input map[string]*schemapb.TemplateValue) (map[string]*planpb.GenericValue, error) {
result := make(map[string]*planpb.GenericValue, len(input))
for name, value := range input {
rv, err := ConvertToGenericValue(name, value)
if err != nil {
return nil, err
}
result[name] = rv
}
return result, nil
}
Loading

0 comments on commit 3367309

Please sign in to comment.