Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(encode): ensure quoting is maintained for scalars #115

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

fix(encode): ensure quoting is maintained for scalars #115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f404b76
fix(encode): ensure quoting is maintained for scalars
stormqueen1990 Aug 13, 2024
742f5e6
fix(encode): extra test cases and fixes
stormqueen1990 Aug 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions goyaml.v2/encode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding"
"fmt"
"io"
"math/big"
"reflect"
"regexp"
"sort"
Expand Down Expand Up @@ -122,6 +123,9 @@ func (e *encoder) marshal(tag string, in reflect.Value) {
// we don't want to treat it as a string for YAML
// purposes because YAML has special support for
// timestamps.
case *big.Int:
e.bigintv(tag, reflect.ValueOf(m.String()))
return
case Marshaler:
v, err := m.MarshalYAML()
if err != nil {
Expand Down Expand Up @@ -354,6 +358,12 @@ func (e *encoder) uintv(tag string, in reflect.Value) {
e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE)
}

func (e *encoder) bigintv(tag string, in reflect.Value) {
value := &big.Int{}
s, _ := value.SetString(in.String(), 0)
e.emitScalar(s.String(), "", tag, yaml_PLAIN_SCALAR_STYLE)
}

func (e *encoder) timev(tag string, in reflect.Value) {
t := in.Interface().(time.Time)
s := t.Format(time.RFC3339Nano)
Expand Down
36 changes: 35 additions & 1 deletion goyaml.v2/encode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"os"

. "gopkg.in/check.v1"
"sigs.k8s.io/yaml/goyaml.v2"
yaml "sigs.k8s.io/yaml/goyaml.v2"
)

type jsonNumberT string
Expand Down Expand Up @@ -641,6 +641,40 @@ func (s *S) TestSortedOutput(c *C) {
}
}

func (s *S) TestQuotingMaintained(c *C) {
var buf bytes.Buffer
var yamlValue map[string]interface{}
const originalYaml = `data:
A1: "0x0000000000000000000000010000000000000000"
A2: "0x000000000000000000000000FFFFFFFFFFFFFFFF"
A3: "1234"
A4: 0x0000000000000000000000010000000000000000
A5: 0x000000000000000000000000FFFFFFFFFFFFFFFF
A6: 1234
`
const outputYaml = `data:
A1: "0x0000000000000000000000010000000000000000"
A2: "0x000000000000000000000000FFFFFFFFFFFFFFFF"
A3: "1234"
A4: 18446744073709551616
A5: 18446744073709551615
A6: 1234
`

dec := yaml.NewDecoder(strings.NewReader(originalYaml))
errDec := dec.Decode(&yamlValue)
c.Assert(errDec, IsNil)

enc := yaml.NewEncoder(&buf)
errEnc := enc.Encode(yamlValue)
c.Assert(errEnc, IsNil)

errClose := enc.Close()
c.Assert(errClose, IsNil)

c.Assert(buf.String(), Equals, outputYaml)
}

func newTime(t time.Time) *time.Time {
return &t
}
23 changes: 19 additions & 4 deletions goyaml.v2/resolve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package yaml

import (
"encoding/base64"
"errors"
"math"
"math/big"
"regexp"
"strconv"
"strings"
Expand Down Expand Up @@ -148,16 +150,18 @@ func resolve(tag string, in string) (rtag string, out interface{}) {
}

plain := strings.Replace(in, "_", "", -1)
intv, err := strconv.ParseInt(plain, 0, 64)
if err == nil {

var intConvErr error
intv, intConvErr := strconv.ParseInt(plain, 0, 64)
if intConvErr == nil {
if intv == int64(int(intv)) {
return yaml_INT_TAG, int(intv)
} else {
return yaml_INT_TAG, intv
}
}
uintv, err := strconv.ParseUint(plain, 0, 64)
if err == nil {
uintv, intConvErr := strconv.ParseUint(plain, 0, 64)
if intConvErr == nil {
return yaml_INT_TAG, uintv
}
if yamlStyleFloat.MatchString(plain) {
Expand Down Expand Up @@ -189,6 +193,17 @@ func resolve(tag string, in string) (rtag string, out interface{}) {
}
}
}

// If number out of range and doesn't fit any of the other cases,
// check if it's valid for bigger than 64 bytes
if errors.Is(intConvErr, strconv.ErrRange) {
value := &big.Int{}

bigintv, ok := value.SetString(plain, 0)
if ok {
return yaml_INT_TAG, bigintv
}
}
default:
panic("resolveTable item not yet handled: " + string(rune(hint)) + " (with " + in + ")")
}
Expand Down
10 changes: 10 additions & 0 deletions goyaml.v3/encode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"encoding"
"fmt"
"io"
"math/big"
"reflect"
"regexp"
"sort"
Expand Down Expand Up @@ -136,6 +137,9 @@ func (e *encoder) marshal(tag string, in reflect.Value) {
case time.Duration:
e.stringv(tag, reflect.ValueOf(value.String()))
return
case *big.Int:
e.bigintv(tag, reflect.ValueOf(value.String()))
return
case Marshaler:
v, err := value.MarshalYAML()
if err != nil {
Expand Down Expand Up @@ -382,6 +386,12 @@ func (e *encoder) uintv(tag string, in reflect.Value) {
e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil)
}

func (e *encoder) bigintv(tag string, in reflect.Value) {
value := &big.Int{}
s, _ := value.SetString(in.String(), 0)
e.emitScalar(s.String(), "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil)
}

func (e *encoder) timev(tag string, in reflect.Value) {
t := in.Interface().(time.Time)
s := t.Format(time.RFC3339Nano)
Expand Down
36 changes: 35 additions & 1 deletion goyaml.v3/encode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
"os"

. "gopkg.in/check.v1"
"sigs.k8s.io/yaml/goyaml.v3"
yaml "sigs.k8s.io/yaml/goyaml.v3"
)

var marshalIntTest = 123
Expand Down Expand Up @@ -731,6 +731,40 @@ func (s *S) TestSortedOutput(c *C) {
}
}

func (s *S) TestQuotingMaintained(c *C) {
var buf bytes.Buffer
var yamlValue map[string]interface{}
const originalYaml = `data:
A1: "0x0000000000000000000000010000000000000000"
A2: "0x000000000000000000000000FFFFFFFFFFFFFFFF"
Comment on lines +738 to +739
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maintaining the quoting on A1/A2 is good ... do we also know that YAMLToJSON works properly here? that's what will be used when reading a yaml manifest and sending it to the kube-apiserver

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add a test case for YAMLToJSON

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

understanding all the paths that come through here and what they do with these values today (as well as how kube-apiserver treats submitted yaml values) is important so we make sure we don't break anyone that is currently "working"

I think the test cases would be something like:

unquoted yaml → yaml (round-trip)
unquoted yaml → json (conversion)
quoted yaml → yaml (round-trip)
quoted yaml → json (conversion)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@liggitt I added some test cases to the TestYAMLtoJSON function in yaml_test.go in my last commit 742f5e6, which I believe covers both of these scenarios since it first converts the YAML to JSON, and then converts it back, but please let me know if I misunderstood!

A3: "1234"
A4: 0x0000000000000000000000010000000000000000
A5: 0x000000000000000000000000FFFFFFFFFFFFFFFF
Comment on lines +741 to +742
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This repo is oriented at yaml which can be round-tripped to json and used in Kubernetes API objects

These bigints would be problematic for Kubernetes as they do not round-trip to json ... I'm on the fence that we should enable this in this library

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My general thought process was that unquoted values are already not round-trippable to JSON when bigger than 64 bytes, but if quoted those should be maintained. The current implementation assumes that if an integer value does not fit a 64-byte integer then it must be a string but that's not exactly true.

A6: 1234
`
const outputYaml = `data:
A1: "0x0000000000000000000000010000000000000000"
A2: "0x000000000000000000000000FFFFFFFFFFFFFFFF"
A3: "1234"
A4: 18446744073709551616
A5: 18446744073709551615
A6: 1234
`

dec := yaml.NewDecoder(strings.NewReader(originalYaml))
errDec := dec.Decode(&yamlValue)
c.Assert(errDec, IsNil)

enc := yaml.NewEncoder(&buf)
errEnc := enc.Encode(yamlValue)
c.Assert(errEnc, IsNil)

errClose := enc.Close()
c.Assert(errClose, IsNil)

c.Assert(buf.String(), Equals, outputYaml)
}

func newTime(t time.Time) *time.Time {
return &t
}
23 changes: 19 additions & 4 deletions goyaml.v3/resolve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ package yaml

import (
"encoding/base64"
"errors"
"math"
"math/big"
"regexp"
"strconv"
"strings"
Expand Down Expand Up @@ -189,16 +191,18 @@ func resolve(tag string, in string) (rtag string, out interface{}) {
}

plain := strings.Replace(in, "_", "", -1)
intv, err := strconv.ParseInt(plain, 0, 64)
if err == nil {

var intConvErr error
intv, intConvErr := strconv.ParseInt(plain, 0, 64)
if intConvErr == nil {
if intv == int64(int(intv)) {
return intTag, int(intv)
} else {
return intTag, intv
}
}
uintv, err := strconv.ParseUint(plain, 0, 64)
if err == nil {
uintv, intConvErr := strconv.ParseUint(plain, 0, 64)
if intConvErr == nil {
return intTag, uintv
}
if yamlStyleFloat.MatchString(plain) {
Expand Down Expand Up @@ -257,6 +261,17 @@ func resolve(tag string, in string) (rtag string, out interface{}) {
}
}
}

// If number out of range and doesn't fit any of the other cases,
// check if it's valid for bigger than 64 bytes
if errors.Is(intConvErr, strconv.ErrRange) {
value := &big.Int{}

bigintv, ok := value.SetString(plain, 0)
if ok {
return intTag, bigintv
}
}
default:
panic("internal error: missing handler for resolver table: " + string(rune(hint)) + " (with " + in + ")")
}
Expand Down
27 changes: 27 additions & 0 deletions yaml_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,31 @@ func TestYAMLToJSON(t *testing.T) {
json: `{"a":"\ufffd\ufffd\ufffd"}`,
yamlReverseOverwrite: strPtr("a: \ufffd\ufffd\ufffd\n"),
},
"small hex value": {
yaml: "key: 0x0001",
json: `{"key":1}`,
yamlReverseOverwrite: strPtr("key: 1\n"),
},
"hex value at the 64 byte mark": {
yaml: "key: 0x000000000000000000000000FFFFFFFFFFFFFFFF",
json: `{"key":18446744073709551615}`,
yamlReverseOverwrite: strPtr("key: 18446744073709551615\n"),
},
"hex value at the 64 byte mark in string format": {
yaml: "key: \"0x000000000000000000000000FFFFFFFFFFFFFFFF\"",
json: `{"key":"0x000000000000000000000000FFFFFFFFFFFFFFFF"}`,
yamlReverseOverwrite: strPtr("key: \"0x000000000000000000000000FFFFFFFFFFFFFFFF\"\n"),
},
"huge hex value bigger than 64 bytes": {
yaml: "key: 0x0000000000000000000000010000000000000000",
json: `{"key":18446744073709551616}`,
yamlReverseOverwrite: strPtr("key: 1.8446744073709552e+19\n"),
},
"huge hex value bigger than 64 bytes in string format": {
yaml: "key: \"0x0000000000000000000000010000000000000000\"",
json: `{"key":"0x0000000000000000000000010000000000000000"}`,
yamlReverseOverwrite: strPtr("key: \"0x0000000000000000000000010000000000000000\"\n"),
},

// Cases that should produce errors.
"~ key": {
Expand Down Expand Up @@ -809,6 +834,7 @@ func TestJSONObjectToYAMLObject(t *testing.T) {
"slice": []interface{}{"foo", "bar"},
"string": string("foo"),
"uint64 big": bigUint64,
"big hex int": "0x0000000000000000000000010000000000000000",
},
expected: yamlv2.MapSlice{
{Key: "nil slice"},
Expand All @@ -826,6 +852,7 @@ func TestJSONObjectToYAMLObject(t *testing.T) {
{Key: "slice", Value: []interface{}{"foo", "bar"}},
{Key: "string", Value: string("foo")},
{Key: "uint64 big", Value: bigUint64},
{Key: "big hex int", Value: "0x0000000000000000000000010000000000000000"},
},
},
}
Expand Down
Loading