Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat!: unify json lib #579

Merged
merged 2 commits into from
Apr 10, 2023
Merged

feat!: unify json lib #579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9ba0e97
feat: remove gjson and use one same json lib
xdlbdy Mar 28, 2023
e91168c
fix: code review
xdlbdy Apr 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ require (
github.com/jedib0t/go-pretty/v6 v6.3.1
github.com/json-iterator/go v1.1.12
github.com/ncw/directio v1.0.5
github.com/ohler55/ojg v1.14.5
github.com/panjf2000/ants/v2 v2.7.1
github.com/pkg/errors v0.9.1
github.com/prashantv/gostub v1.1.0
Expand Down Expand Up @@ -91,6 +90,7 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
github.com/ohler55/ojg v1.18.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
Expand Down Expand Up @@ -119,11 +119,11 @@ require (
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.17.0 // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/net v0.4.0 // indirect
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
)

Expand Down
20 changes: 10 additions & 10 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,8 @@ github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/ohler55/ojg v1.14.5 h1:xCX2oyh/ZaoesbLH6fwVHStSJpk4o4eJs8ttXutzdg0=
github.com/ohler55/ojg v1.14.5/go.mod h1:7Ghirupn8NC8hSSDpI0gcjorPxj+vSVIONDWfliHR1k=
github.com/ohler55/ojg v1.18.1 h1:kNJHB1qIp9ev/I/+3E3ObImjsAlAGRR+2VMCuq9lQCY=
github.com/ohler55/ojg v1.18.1/go.mod h1:uHcD1ErbErC27Zhb5Df2jUjbseLLcmOCo6oxSr3jZxo=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
Expand Down Expand Up @@ -535,8 +535,8 @@ golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand All @@ -545,8 +545,8 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk=
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -607,8 +607,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -619,8 +619,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
123 changes: 115 additions & 8 deletions internal/primitive/cel/cel.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@ package cel

import (
"fmt"
"reflect"
"strconv"
"strings"

ce "github.com/cloudevents/sdk-go/v2"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/checker/decls"
"github.com/tidwall/gjson"
exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"

"github.com/vanus-labs/vanus/pkg/util"
)

var ErrInvalidExpression = fmt.Errorf("expression is invalid,format is: $json_path.(type)")
Expand Down Expand Up @@ -95,7 +98,7 @@ func parseExpression(expression string) (string, map[string]Variable, error) {

varMap[safeCELName] = Variable{
Name: safeCELName,
Path: expression[pos+1 : typeStartPos],
Path: "$." + expression[pos+1:typeStartPos],
Type: expression[typeStartPos+2 : typeEndPos],
}
expr += safeCELName
Expand Down Expand Up @@ -145,24 +148,128 @@ func newCelProgram(expr string, vars map[string]Variable) (cel.Program, error) {

func (e *Expression) Eval(event ce.Event) (bool, error) {
vars := make(map[string]interface{})

obj, err := util.ParseJSON(event.Data())
if err != nil {
return false, err
}
for _, v := range e.variables {
value, err := util.GetJSONValue(obj, v.Path)
if err != nil {
return false, err
}
var val interface{}

switch v.Type {
case "string":
vars[v.Name] = gjson.GetBytes(event.Data(), v.Path).String()
val, err = stringValue(value)
if err != nil {
return false, err
}
case "int64":
vars[v.Name] = gjson.GetBytes(event.Data(), v.Path).Int()
val, err = intValue(value)
if err != nil {
return false, err
}
case "uint64":
vars[v.Name] = gjson.GetBytes(event.Data(), v.Path).Uint()
val, err = uintValue(value)
if err != nil {
return false, err
}
case "bool":
vars[v.Name] = gjson.GetBytes(event.Data(), v.Path).Bool()
val, err = boolValue(value)
if err != nil {
return false, err
}
case "double":
vars[v.Name] = gjson.GetBytes(event.Data(), v.Path).Float()
val, err = floatValue(value)
if err != nil {
return false, err
}
}
vars[v.Name] = val
}
out, _, err := e.program.Eval(vars)
if err != nil {
return false, err
}
return out.Value().(bool), nil
}

func stringValue(value interface{}) (string, error) {
v, ok := value.(string)
if ok {
return v, nil
}
reflectValue := reflect.ValueOf(value)
switch reflectValue.Kind() {
case reflect.Float32, reflect.Float64, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, reflect.Bool:
return fmt.Sprintf("%v", value), nil
default:
return "", fmt.Errorf("%s can't convert to string", reflectValue.Kind().String())
}
}

func intValue(value interface{}) (int64, error) {
reflectValue := reflect.ValueOf(value)
switch {
case reflectValue.CanInt():
ifplusor marked this conversation as resolved.
Show resolved Hide resolved
return reflectValue.Int(), nil
case reflectValue.CanFloat():
return int64(reflectValue.Float()), nil
case reflectValue.CanUint():
return int64(reflectValue.Uint()), nil
case reflectValue.Kind() == reflect.String:
v, err := strconv.ParseInt(value.(string), 10, 64)
return v, err
default:
return 0, fmt.Errorf("%s can't convert to int64", reflectValue.Kind().String())
}
}

func uintValue(value interface{}) (uint64, error) {
reflectValue := reflect.ValueOf(value)
switch {
case reflectValue.CanUint():
return reflectValue.Uint(), nil
case reflectValue.CanFloat():
return uint64(reflectValue.Float()), nil
case reflectValue.CanInt():
return uint64(reflectValue.Int()), nil
case reflectValue.Kind() == reflect.String:
v, err := strconv.ParseUint(value.(string), 10, 64)
return v, err
default:
return 0, fmt.Errorf("%s can't convert to uint64", reflectValue.Kind().String())
}
}

func boolValue(value interface{}) (bool, error) {
reflectValue := reflect.ValueOf(value)
switch reflectValue.Kind() {
case reflect.Bool:
return reflectValue.Bool(), nil
case reflect.String:
b, err := strconv.ParseBool(reflectValue.String())
return b, err
default:
return false, fmt.Errorf("%s can't convert to bool", reflectValue.Kind().String())
}
}

func floatValue(value interface{}) (float64, error) {
reflectValue := reflect.ValueOf(value)
switch {
case reflectValue.CanFloat():
return reflectValue.Float(), nil
case reflectValue.CanInt():
return float64(reflectValue.Int()), nil
case reflectValue.CanUint():
return float64(reflectValue.Uint()), nil
case reflectValue.Kind() == reflect.String:
v, err := strconv.ParseFloat(value.(string), 64)
return v, err
default:
return 0, fmt.Errorf("%s can't convert to float64", reflectValue.Kind().String())
}
}
4 changes: 2 additions & 2 deletions internal/primitive/transform/arg/event.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@
package arg

import (
"errors"
"strings"

"github.com/vanus-labs/vanus/pkg/errors"
pkgUtil "github.com/vanus-labs/vanus/pkg/util"

"github.com/vanus-labs/vanus/internal/primitive/transform/context"
Expand Down Expand Up @@ -106,7 +106,7 @@ func (arg eventData) Original() string {
func (arg eventData) Evaluate(ceCtx *context.EventContext) (interface{}, error) {
v, err := util.LookupData(ceCtx.Data, EventArgPrefix+arg.path)
if err != nil {
if errors.Is(err, util.ErrKeyNotFound) {
if errors.Is(err, errors.ErrJSONPathNotExist) {
return nil, ErrArgValueNil
}
return nil, err
Expand Down
71 changes: 54 additions & 17 deletions internal/trigger/filter/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,20 @@
package filter

import (
"fmt"
"reflect"

ce "github.com/cloudevents/sdk-go/v2"
"github.com/cloudevents/sdk-go/v2/types"
"github.com/tidwall/gjson"
"github.com/vanus-labs/vanus/observability/log"

"github.com/vanus-labs/vanus/internal/trigger/util"
"github.com/vanus-labs/vanus/observability/log"
putil "github.com/vanus-labs/vanus/pkg/util"
)

type commonFilter struct {
attribute map[string]string
dataValue string
data map[string]string
meetCondition meetCondition
}
Expand All @@ -34,6 +38,7 @@ type meetCondition func(value, compareValue string) bool
func newCommonFilter(value map[string]string, meetCondition meetCondition) *commonFilter {
attribute := map[string]string{}
data := map[string]string{}
var dataValue string
for attr, v := range value {
if attr == "" || v == "" {
log.Info().Str("attr", attr).
Expand All @@ -43,9 +48,10 @@ func newCommonFilter(value map[string]string, meetCondition meetCondition) *comm
}
switch {
case attr == "data":
data[""] = v
dataValue = v
case len(attr) > 4 && attr[:5] == "data.":
attr = attr[5:]
attr = "$." + attr
data[attr] = v
default:
// event attribute.
Expand All @@ -55,6 +61,7 @@ func newCommonFilter(value map[string]string, meetCondition meetCondition) *comm
return &commonFilter{
attribute: attribute,
data: data,
dataValue: dataValue,
meetCondition: meetCondition,
}
}
Expand All @@ -65,34 +72,64 @@ func (filter *commonFilter) Filter(event ce.Event) Result {
if !ok {
return false
}
if !filter.meetCondition(attrValue2String(value), v) {
strValue, err := attrValue2String(value)
if err != nil {
log.Info().Str("attr", attr).Err(err).Msg("filter attr value to string failed")
return false
}
if !filter.meetCondition(strValue, v) {
return FailFilter
}
}
if filter.dataValue != "" && !filter.meetCondition(string(event.Data()), filter.dataValue) {
return FailFilter
}
if len(filter.data) == 0 {
return PassFilter
}
obj, err := putil.ParseJSON(event.Data())
if err != nil {
log.Info().Str("data", string(event.Data())).Err(err).Msg("filter parse data error")
return false
}
for attr, v := range filter.data {
if attr == "" {
// event data
if !filter.meetCondition(string(event.Data()), v) {
return FailFilter
}
continue
value, err := putil.GetJSONValue(obj, attr)
if err != nil {
log.Info().Str("path", attr).Err(err).Msg("filter parse json path error")
return false
}
result := gjson.GetBytes(event.Data(), attr)
if !result.Exists() {
strValue, err := dataValue2String(value)
if err != nil {
log.Info().Str("path", attr).Err(err).Msg("filter data value to string failed")
return false
}
if !filter.meetCondition(result.String(), v) {
if !filter.meetCondition(strValue, v) {
return FailFilter
}
}
return PassFilter
}

func attrValue2String(value interface{}) string {
func dataValue2String(value interface{}) (string, error) {
v, ok := value.(string)
if ok {
return v, nil
}
reflectValue := reflect.ValueOf(value)
switch reflectValue.Kind() {
case reflect.Float32, reflect.Float64, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, reflect.Bool:
return fmt.Sprintf("%v", value), nil
default:
return "", fmt.Errorf("filter value %s can't convert to string", reflectValue.Kind())
}
}

func attrValue2String(value interface{}) (string, error) {
v, ok := value.(string)
if ok {
return v
return v, nil
}
v, _ = types.Format(value)
return v
val, err := types.Format(value)
return val, err
}
Loading