From 2eed8935313fcb2aea3ddec478c81cc26cd798ee Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Sat, 30 Jul 2022 23:55:47 +0200 Subject: [PATCH] Add support for json[] via JSONArray Signed-off-by: Jakob Ackermann --- json_array.go | 546 +++++++++++++++++++++++++++++++++++++++++++++ json_array_test.go | 88 ++++++++ pgtype.go | 3 + typed_array_gen.sh | 1 + 4 files changed, 638 insertions(+) create mode 100644 json_array.go create mode 100644 json_array_test.go diff --git a/json_array.go b/json_array.go new file mode 100644 index 0000000..8d68882 --- /dev/null +++ b/json_array.go @@ -0,0 +1,546 @@ +// Code generated by erb. DO NOT EDIT. + +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "encoding/json" + "fmt" + "reflect" + + "github.com/jackc/pgio" +) + +type JSONArray struct { + Elements []JSON + Dimensions []ArrayDimension + Status Status +} + +func (dst *JSONArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = JSONArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + // Attempt to match to select common types: + switch value := src.(type) { + + case []string: + if value == nil { + *dst = JSONArray{Status: Null} + } else if len(value) == 0 { + *dst = JSONArray{Status: Present} + } else { + elements := make([]JSON, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = JSONArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case [][]byte: + if value == nil { + *dst = JSONArray{Status: Null} + } else if len(value) == 0 { + *dst = JSONArray{Status: Present} + } else { + elements := make([]JSON, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = JSONArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []json.RawMessage: + if value == nil { + *dst = JSONArray{Status: Null} + } else if len(value) == 0 { + *dst = JSONArray{Status: Present} + } else { + elements := make([]JSON, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = JSONArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []JSON: + if value == nil { + *dst = JSONArray{Status: Null} + } else if len(value) == 0 { + *dst = JSONArray{Status: Present} + } else { + *dst = JSONArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + // Fallback to reflection if an optimised match was not found. + // The reflection is necessary for arrays and multidimensional slices, + // but it comes with a 20-50% performance penalty for large arrays/slices + reflectedValue := reflect.ValueOf(src) + if !reflectedValue.IsValid() || reflectedValue.IsZero() { + *dst = JSONArray{Status: Null} + return nil + } + + dimensions, elementsLength, ok := findDimensionsFromValue(reflectedValue, nil, 0) + if !ok { + return fmt.Errorf("cannot find dimensions of %v for JSONArray", src) + } + if elementsLength == 0 { + *dst = JSONArray{Status: Present} + return nil + } + if len(dimensions) == 0 { + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return fmt.Errorf("cannot convert %v to JSONArray", src) + } + + *dst = JSONArray{ + Elements: make([]JSON, elementsLength), + Dimensions: dimensions, + Status: Present, + } + elementCount, err := dst.setRecursive(reflectedValue, 0, 0) + if err != nil { + // Maybe the target was one dimension too far, try again: + if len(dst.Dimensions) > 1 { + dst.Dimensions = dst.Dimensions[:len(dst.Dimensions)-1] + elementsLength = 0 + for _, dim := range dst.Dimensions { + if elementsLength == 0 { + elementsLength = int(dim.Length) + } else { + elementsLength *= int(dim.Length) + } + } + dst.Elements = make([]JSON, elementsLength) + elementCount, err = dst.setRecursive(reflectedValue, 0, 0) + if err != nil { + return err + } + } else { + return err + } + } + if elementCount != len(dst.Elements) { + return fmt.Errorf("cannot convert %v to JSONArray, expected %d dst.Elements, but got %d instead", src, len(dst.Elements), elementCount) + } + } + + return nil +} + +func (dst *JSONArray) setRecursive(value reflect.Value, index, dimension int) (int, error) { + switch value.Kind() { + case reflect.Array: + fallthrough + case reflect.Slice: + if len(dst.Dimensions) == dimension { + break + } + + valueLen := value.Len() + if int32(valueLen) != dst.Dimensions[dimension].Length { + return 0, fmt.Errorf("multidimensional arrays must have array expressions with matching dimensions") + } + for i := 0; i < valueLen; i++ { + var err error + index, err = dst.setRecursive(value.Index(i), index, dimension+1) + if err != nil { + return 0, err + } + } + + return index, nil + } + if !value.CanInterface() { + return 0, fmt.Errorf("cannot convert all values to JSONArray") + } + if err := dst.Elements[index].Set(value.Interface()); err != nil { + return 0, fmt.Errorf("%v in JSONArray", err) + } + index++ + + return index, nil +} + +func (dst JSONArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *JSONArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + if len(src.Dimensions) <= 1 { + // Attempt to match to select common types: + switch v := dst.(type) { + + case *[]string: + *v = make([]string, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[][]byte: + *v = make([][]byte, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]json.RawMessage: + *v = make([]json.RawMessage, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + } + } + + // Try to convert to something AssignTo can use directly. + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + + // Fallback to reflection if an optimised match was not found. + // The reflection is necessary for arrays and multidimensional slices, + // but it comes with a 20-50% performance penalty for large arrays/slices + value := reflect.ValueOf(dst) + if value.Kind() == reflect.Ptr { + value = value.Elem() + } + + switch value.Kind() { + case reflect.Array, reflect.Slice: + default: + return fmt.Errorf("cannot assign %T to %T", src, dst) + } + + if len(src.Elements) == 0 { + if value.Kind() == reflect.Slice { + value.Set(reflect.MakeSlice(value.Type(), 0, 0)) + return nil + } + } + + elementCount, err := src.assignToRecursive(value, 0, 0) + if err != nil { + return err + } + if elementCount != len(src.Elements) { + return fmt.Errorf("cannot assign %v, needed to assign %d elements, but only assigned %d", dst, len(src.Elements), elementCount) + } + + return nil + case Null: + return NullAssignTo(dst) + } + + return fmt.Errorf("cannot decode %#v into %T", src, dst) +} + +func (src *JSONArray) assignToRecursive(value reflect.Value, index, dimension int) (int, error) { + switch kind := value.Kind(); kind { + case reflect.Array: + fallthrough + case reflect.Slice: + if len(src.Dimensions) == dimension { + break + } + + length := int(src.Dimensions[dimension].Length) + if reflect.Array == kind { + typ := value.Type() + if typ.Len() != length { + return 0, fmt.Errorf("expected size %d array, but %s has size %d array", length, typ, typ.Len()) + } + value.Set(reflect.New(typ).Elem()) + } else { + value.Set(reflect.MakeSlice(value.Type(), length, length)) + } + + var err error + for i := 0; i < length; i++ { + index, err = src.assignToRecursive(value.Index(i), index, dimension+1) + if err != nil { + return 0, err + } + } + + return index, nil + } + if len(src.Dimensions) != dimension { + return 0, fmt.Errorf("incorrect dimensions, expected %d, found %d", len(src.Dimensions), dimension) + } + if !value.CanAddr() { + return 0, fmt.Errorf("cannot assign all values from JSONArray") + } + addr := value.Addr() + if !addr.CanInterface() { + return 0, fmt.Errorf("cannot assign all values from JSONArray") + } + if err := src.Elements[index].AssignTo(addr.Interface()); err != nil { + return 0, err + } + index++ + return index, nil +} + +func (dst *JSONArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = JSONArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []JSON + + if len(uta.Elements) > 0 { + elements = make([]JSON, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem JSON + var elemSrc []byte + if s != "NULL" || uta.Quoted[i] { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = JSONArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *JSONArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = JSONArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = JSONArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]JSON, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = JSONArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src JSONArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src JSONArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("json"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, fmt.Errorf("unable to find oid for type name %v", "json") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *JSONArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return fmt.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src JSONArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/json_array_test.go b/json_array_test.go new file mode 100644 index 0000000..7400b54 --- /dev/null +++ b/json_array_test.go @@ -0,0 +1,88 @@ +package pgtype_test + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/jackc/pgtype" + "github.com/jackc/pgtype/testutil" +) + +func TestJSONArrayTranscode(t *testing.T) { + testutil.TestSuccessfulTranscode(t, "json[]", []interface{}{ + &pgtype.JSONArray{ + Elements: nil, + Dimensions: nil, + Status: pgtype.Present, + }, + &pgtype.JSONArray{ + Elements: []pgtype.JSON{ + {Bytes: []byte(`"foo"`), Status: pgtype.Present}, + {Status: pgtype.Null}, + }, + Dimensions: []pgtype.ArrayDimension{{Length: 2, LowerBound: 1}}, + Status: pgtype.Present, + }, + &pgtype.JSONArray{Status: pgtype.Null}, + &pgtype.JSONArray{ + Elements: []pgtype.JSON{ + {Bytes: []byte(`"foo"`), Status: pgtype.Present}, + {Bytes: []byte("null"), Status: pgtype.Present}, + {Bytes: []byte("42"), Status: pgtype.Present}, + }, + Dimensions: []pgtype.ArrayDimension{{Length: 3, LowerBound: 1}}, + Status: pgtype.Present, + }, + }) +} + +func TestJSONArraySet(t *testing.T) { + successfulTests := []struct { + source interface{} + result pgtype.JSONArray + }{ + {source: []string{"{}"}, result: pgtype.JSONArray{ + Elements: []pgtype.JSON{{Bytes: []byte("{}"), Status: pgtype.Present}}, + Dimensions: []pgtype.ArrayDimension{{Length: 1, LowerBound: 1}}, + Status: pgtype.Present, + }}, + {source: [][]byte{[]byte("{}")}, result: pgtype.JSONArray{ + Elements: []pgtype.JSON{{Bytes: []byte("{}"), Status: pgtype.Present}}, + Dimensions: []pgtype.ArrayDimension{{Length: 1, LowerBound: 1}}, + Status: pgtype.Present, + }}, + {source: [][]byte{[]byte(`{"foo":1}`), []byte(`{"bar":2}`)}, result: pgtype.JSONArray{ + Elements: []pgtype.JSON{{Bytes: []byte(`{"foo":1}`), Status: pgtype.Present}, {Bytes: []byte(`{"bar":2}`), Status: pgtype.Present}}, + Dimensions: []pgtype.ArrayDimension{{Length: 2, LowerBound: 1}}, + Status: pgtype.Present, + }}, + {source: []json.RawMessage{json.RawMessage(`{"foo":1}`), json.RawMessage(`{"bar":2}`)}, result: pgtype.JSONArray{ + Elements: []pgtype.JSON{{Bytes: []byte(`{"foo":1}`), Status: pgtype.Present}, {Bytes: []byte(`{"bar":2}`), Status: pgtype.Present}}, + Dimensions: []pgtype.ArrayDimension{{Length: 2, LowerBound: 1}}, + Status: pgtype.Present, + }}, + {source: []json.RawMessage{json.RawMessage(`{"foo":12}`), json.RawMessage(`{"bar":2}`)}, result: pgtype.JSONArray{ + Elements: []pgtype.JSON{{Bytes: []byte(`{"foo":12}`), Status: pgtype.Present}, {Bytes: []byte(`{"bar":2}`), Status: pgtype.Present}}, + Dimensions: []pgtype.ArrayDimension{{Length: 2, LowerBound: 1}}, + Status: pgtype.Present, + }}, + {source: []json.RawMessage{json.RawMessage(`{"foo":1}`), json.RawMessage(`{"bar":{"x":2}}`)}, result: pgtype.JSONArray{ + Elements: []pgtype.JSON{{Bytes: []byte(`{"foo":1}`), Status: pgtype.Present}, {Bytes: []byte(`{"bar":{"x":2}}`), Status: pgtype.Present}}, + Dimensions: []pgtype.ArrayDimension{{Length: 2, LowerBound: 1}}, + Status: pgtype.Present, + }}, + } + + for i, tt := range successfulTests { + var d pgtype.JSONArray + err := d.Set(tt.source) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if !reflect.DeepEqual(d, tt.result) { + t.Errorf("%d: expected %+v to convert to %+v, but it was %+v", i, tt.source, tt.result, d) + } + } +} diff --git a/pgtype.go b/pgtype.go index 08e7395..addfc66 100644 --- a/pgtype.go +++ b/pgtype.go @@ -26,6 +26,7 @@ const ( XIDOID = 28 CIDOID = 29 JSONOID = 114 + JSONArrayOID = 199 PointOID = 600 LsegOID = 601 PathOID = 602 @@ -297,6 +298,7 @@ func NewConnInfo() *ConnInfo { ci.RegisterDataType(DataType{Value: &Int8multirange{}, Name: "int8multirange", OID: Int8multirangeOID}) ci.RegisterDataType(DataType{Value: &Interval{}, Name: "interval", OID: IntervalOID}) ci.RegisterDataType(DataType{Value: &JSON{}, Name: "json", OID: JSONOID}) + ci.RegisterDataType(DataType{Value: &JSONArray{}, Name: "_json", OID: JSONArrayOID}) ci.RegisterDataType(DataType{Value: &JSONB{}, Name: "jsonb", OID: JSONBOID}) ci.RegisterDataType(DataType{Value: &JSONBArray{}, Name: "_jsonb", OID: JSONBArrayOID}) ci.RegisterDataType(DataType{Value: &Line{}, Name: "line", OID: LineOID}) @@ -941,6 +943,7 @@ func init() { "_timestamptz": &TimestamptzArray{}, "_uuid": &UUIDArray{}, "_varchar": &VarcharArray{}, + "_json": &JSONArray{}, "_jsonb": &JSONBArray{}, "aclitem": &ACLItem{}, "bit": &Bit{}, diff --git a/typed_array_gen.sh b/typed_array_gen.sh index d922f1c..9ec768b 100755 --- a/typed_array_gen.sh +++ b/typed_array_gen.sh @@ -20,6 +20,7 @@ erb pgtype_array_type=ACLItemArray pgtype_element_type=ACLItem go_array_types=[] erb pgtype_array_type=HstoreArray pgtype_element_type=Hstore go_array_types=[]map[string]string element_type_name=hstore typed_array.go.erb > hstore_array.go erb pgtype_array_type=NumericArray pgtype_element_type=Numeric go_array_types=[]float32,[]*float32,[]float64,[]*float64,[]int64,[]*int64,[]uint64,[]*uint64 element_type_name=numeric typed_array.go.erb > numeric_array.go erb pgtype_array_type=UUIDArray pgtype_element_type=UUID go_array_types=[][16]byte,[][]byte,[]string,[]*string element_type_name=uuid typed_array.go.erb > uuid_array.go +erb pgtype_array_type=JSONArray pgtype_element_type=JSON go_array_types=[]string,[][]byte,[]json.RawMessage element_type_name=json typed_array.go.erb > json_array.go erb pgtype_array_type=JSONBArray pgtype_element_type=JSONB go_array_types=[]string,[][]byte,[]json.RawMessage element_type_name=jsonb typed_array.go.erb > jsonb_array.go # While the binary format is theoretically possible it is only practical to use the text format.