From fca4c97c66f5897a6ea6a4e24ce23171a8548de9 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Mon, 22 Apr 2024 22:43:45 +0300 Subject: [PATCH 1/3] fix: get rid of code under BUSL license --- terraform/cliconfig/config.go | 3 +- terraform/cliconfig/marshal.go | 391 ++++++++++++++++++++++++++++ terraform/cliconfig/marshal_test.go | 38 +++ 3 files changed, 430 insertions(+), 2 deletions(-) create mode 100644 terraform/cliconfig/marshal.go create mode 100644 terraform/cliconfig/marshal_test.go diff --git a/terraform/cliconfig/config.go b/terraform/cliconfig/config.go index c408635bc..6c219e2f3 100644 --- a/terraform/cliconfig/config.go +++ b/terraform/cliconfig/config.go @@ -4,7 +4,6 @@ import ( "os" "regexp" - "github.com/genelet/determined/dethcl" "github.com/gruntwork-io/go-commons/errors" "github.com/hashicorp/terraform/command/cliconfig" ) @@ -69,7 +68,7 @@ func (cfg *Config) Save(configPath string) error { // Since `Config` structure already has `plugin_cache_dir`, remove it from the raw HCL config to prevent repeating in the saved file. rawHCL = configParamPluginCacheDirReg.ReplaceAll(rawHCL, []byte{}) - newHCL, err := dethcl.Marshal(cfg) + newHCL, err := Marshal(cfg) if err != nil { return errors.WithStackTrace(err) } diff --git a/terraform/cliconfig/marshal.go b/terraform/cliconfig/marshal.go new file mode 100644 index 000000000..6741cae59 --- /dev/null +++ b/terraform/cliconfig/marshal.go @@ -0,0 +1,391 @@ +// Copyright (c) 2022 Peter Bi + +package cliconfig + +import ( + "fmt" + "reflect" + "strings" + "unicode" + + "github.com/genelet/determined/utils" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclwrite" +) + +// Marshal marshals object into HCL string +func Marshal(current interface{}) ([]byte, error) { + if current == nil { + return nil, nil + } + return MarshalLevel(current, 0) +} + +func MarshalLevel(current interface{}, level int) ([]byte, error) { + rv := reflect.ValueOf(current) + if rv.IsValid() && rv.IsZero() { + return nil, nil + } + + switch rv.Kind() { + case reflect.Pointer, reflect.Struct: + return marshal(current, level) + default: + } + + return utils.Encoding(current, level) +} + +func marshal(current interface{}, level int, keyname ...string) ([]byte, error) { + if current == nil { + return nil, nil + } + leading := strings.Repeat(" ", level+1) + lessLeading := strings.Repeat(" ", level) + + t := reflect.TypeOf(current) + oriValue := reflect.ValueOf(current) + if t.Kind() == reflect.Pointer { + t = t.Elem() + oriValue = oriValue.Elem() + } + + switch t.Kind() { + case reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: + if oriValue.IsValid() { + return []byte(fmt.Sprintf("= %v", oriValue.Interface())), nil + } + return nil, nil + case reflect.String: + if oriValue.IsValid() { + return []byte(" = " + oriValue.String()), nil + } + return nil, nil + case reflect.Pointer: + return marshal(oriValue.Elem().Interface(), level, keyname...) + default: + } + + newFields, err := getFields(t, oriValue) + if err != nil { + return nil, err + } + + var plains []reflect.StructField + for _, mField := range newFields { + if !mField.out { + plains = append(plains, mField.field) + } + } + newType := reflect.StructOf(plains) + tmp := reflect.New(newType).Elem() + var outliers []*marshalOut + var labels []string + + k := 0 + for _, mField := range newFields { + field := mField.field + oriField := mField.value + if mField.out { + outlier, err := getOutlier(field, oriField, level) + if err != nil { + return nil, err + } + outliers = append(outliers, outlier...) + } else { + fieldTag := field.Tag + hcl := tag2(fieldTag) + if hcl[1] == "label" { + label := oriField.Interface().(string) + if keyname == nil || keyname[0] != label { + labels = append(labels, label) + } + k++ + continue + } + tmp.Field(k).Set(oriField) + k++ + } + } + + f := hclwrite.NewEmptyFile() + gohcl.EncodeIntoBody(tmp.Addr().Interface(), f.Body()) + bs := f.Bytes() + + str := string(bs) + str = leading + strings.ReplaceAll(str, "\n", "\n"+leading) + + var lines []string + for _, item := range outliers { + line := string(item.b0) + " " + if item.encode { + line += " = " + } + if item.b1 != nil { + line += `"` + strings.Join(item.b1, `" "`) + `" ` + } + line += string(item.b2) + lines = append(lines, line) + } + if len(lines) > 0 { + str += strings.Join(lines, "\n"+leading) + } + + str = strings.TrimRight(str, " \t\n\r") + if level > 0 { // not root + str = fmt.Sprintf("{\n%s\n%s}", str, lessLeading) + if labels != nil { + str = "\"" + strings.Join(labels, "\" \"") + "\" " + str + } + } + + return []byte(str), nil +} + +type marshalField struct { + field reflect.StructField + value reflect.Value + out bool +} + +func getFields(t reflect.Type, oriValue reflect.Value) ([]*marshalField, error) { + var newFields []*marshalField + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + typ := field.Type + if !unicode.IsUpper([]rune(field.Name)[0]) { + continue + } + if !oriValue.IsValid() || oriValue.IsZero() { + continue + } + oriField := oriValue.Field(i) + two := tag2(field.Tag) + tcontent := two[0] + if tcontent == `-` || (len(tcontent) >= 2 && tcontent[len(tcontent)-2:] == `,-`) { + continue + } + + if field.Anonymous && tcontent == "" { + switch typ.Kind() { + case reflect.Ptr: + mfs, err := getFields(typ.Elem(), oriField.Elem()) + if err != nil { + return nil, err + } + for _, v := range mfs { + newFields = append(newFields, v) + } + case reflect.Struct: + mfs, err := getFields(typ, oriField) + if err != nil { + return nil, err + } + for _, v := range mfs { + newFields = append(newFields, v) + } + default: + } + continue + } + + pass := false + switch typ.Kind() { + case reflect.Interface, reflect.Pointer, reflect.Struct: + pass = true + case reflect.Slice: + if oriField.Len() == 0 { + continue + } + switch oriField.Index(0).Kind() { + case reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice, reflect.Struct: + pass = true + default: + } + case reflect.Map: + if oriField.Len() == 0 { + continue + } + switch oriField.MapIndex(oriField.MapKeys()[0]).Kind() { + case reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice, reflect.Struct: + pass = true + default: + } + default: + if oriField.IsValid() && oriField.IsZero() { + continue + } + } + if tcontent == "" { + if pass { + field.Tag = reflect.StructTag(fmt.Sprintf(`hcl:"%s,block"`, strings.ToLower(field.Name))) + } else { + field.Tag = reflect.StructTag(fmt.Sprintf(`hcl:"%s,optional"`, strings.ToLower(field.Name))) + } + } + newFields = append(newFields, &marshalField{field, oriField, pass}) + } + return newFields, nil +} + +type marshalOut struct { + b0 []byte + b1 []string + b2 []byte + encode bool +} + +func getOutlier(field reflect.StructField, oriField reflect.Value, level int) ([]*marshalOut, error) { + var empty []*marshalOut + fieldTag := field.Tag + typ := field.Type + newlevel := level + 1 + + switch typ.Kind() { + case reflect.Interface, reflect.Pointer: + newCurrent := oriField.Interface() + bs, err := MarshalLevel(newCurrent, newlevel) + if err != nil { + return nil, err + } + if bs == nil { + return nil, nil + } + empty = append(empty, &marshalOut{hcltag(fieldTag), nil, bs, false}) + case reflect.Struct: + var newCurrent interface{} + if oriField.CanAddr() { + newCurrent = oriField.Addr().Interface() + } else { + newCurrent = oriField.Interface() + } + + bs, err := MarshalLevel(newCurrent, newlevel) + if err != nil { + return nil, err + } + if bs == nil { + return nil, nil + } + empty = append(empty, &marshalOut{hcltag(fieldTag), nil, bs, false}) + case reflect.Slice: + n := oriField.Len() + if n < 1 { + return nil, nil + } + first := oriField.Index(0) + var isLoop bool + switch first.Kind() { + case reflect.Pointer, reflect.Struct: + isLoop = true + case reflect.Interface: + if first.Elem().Kind() == reflect.Pointer || first.Elem().Kind() == reflect.Struct { + isLoop = true + } + default: + } + + if isLoop { + for i := 0; i < n; i++ { + item := oriField.Index(i) + bs, err := MarshalLevel(item.Interface(), newlevel) + if err != nil { + return nil, err + } + if bs == nil { + continue + } + empty = append(empty, &marshalOut{hcltag(fieldTag), nil, bs, false}) + } + } else { + bs, err := utils.Encoding(oriField.Interface(), newlevel) + if err != nil { + return nil, err + } + if bs == nil { + return nil, nil + } + empty = append(empty, &marshalOut{hcltag(fieldTag), nil, bs, true}) + } + case reflect.Map: + n := oriField.Len() + if n < 1 { + return nil, nil + } + first := oriField.MapIndex(oriField.MapKeys()[0]) + var isLoop bool + switch first.Kind() { + case reflect.Pointer, reflect.Struct: + isLoop = true + case reflect.Interface: + if first.Elem().Kind() == reflect.Pointer || first.Elem().Kind() == reflect.Struct { + isLoop = true + } + default: + } + + if isLoop { + iter := oriField.MapRange() + for iter.Next() { + k := iter.Key() + var arr []string + switch k.Kind() { + case reflect.Array, reflect.Slice: + for i := 0; i < k.Len(); i++ { + item := k.Index(i) + if !item.IsZero() { + arr = append(arr, item.String()) + } + } + default: + arr = append(arr, k.String()) + } + + v := iter.Value() + var bs []byte + var err error + bs, err = marshal(v.Interface(), newlevel, arr...) + if err != nil { + return empty, err + } + if bs == nil { + continue + } + empty = append(empty, &marshalOut{hcltag(fieldTag), arr, bs, false}) + } + } else { + bs, err := utils.Encoding(oriField.Interface(), newlevel) + if err != nil { + return nil, err + } + if bs == nil { + return nil, nil + } + empty = append(empty, &marshalOut{hcltag(fieldTag), nil, bs, true}) + } + default: + } + return empty, nil +} + +func tag2(old reflect.StructTag) [2]string { + for _, tag := range strings.Fields(string(old)) { + if len(tag) >= 5 && strings.ToLower(tag[:5]) == "hcl:\"" { + tag = tag[5 : len(tag)-1] + two := strings.SplitN(tag, ",", 2) + if len(two) == 2 { + return [2]string{two[0], two[1]} + } + return [2]string{two[0], ""} + } + } + return [2]string{} +} + +func hcltag(tag reflect.StructTag) []byte { + two := tag2(tag) + return []byte(two[0]) +} diff --git a/terraform/cliconfig/marshal_test.go b/terraform/cliconfig/marshal_test.go new file mode 100644 index 000000000..19232256a --- /dev/null +++ b/terraform/cliconfig/marshal_test.go @@ -0,0 +1,38 @@ +package cliconfig + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJsonHcl(t *testing.T) { + jsonData := `{"name": "peter", "parties":["one", "two", ["three", "four"], {"five":"51", "six":61}]}` + + expectedHCL := ` + name = "peter" + parties = [ + "one", + "two", + [ + "three", + "four" + ], + { + five = "51" + six = 61 + } + ] +` + + data := map[string]interface{}{} + err := json.Unmarshal([]byte(jsonData), &data) + require.NoError(t, err) + + hclBytes, err := Marshal(data) + require.NoError(t, err) + + assert.Equal(t, string(hclBytes), expectedHCL) +} From 1fc48643e51c5586eed89b786227e86f7b70f227 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Mon, 22 Apr 2024 22:55:48 +0300 Subject: [PATCH 2/3] fix: go linter --- terraform/cliconfig/marshal.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/terraform/cliconfig/marshal.go b/terraform/cliconfig/marshal.go index 6741cae59..4c47b8567 100644 --- a/terraform/cliconfig/marshal.go +++ b/terraform/cliconfig/marshal.go @@ -176,17 +176,13 @@ func getFields(t reflect.Type, oriValue reflect.Value) ([]*marshalField, error) if err != nil { return nil, err } - for _, v := range mfs { - newFields = append(newFields, v) - } + newFields = append(newFields, mfs...) case reflect.Struct: mfs, err := getFields(typ, oriField) if err != nil { return nil, err } - for _, v := range mfs { - newFields = append(newFields, v) - } + newFields = append(newFields, mfs...) default: } continue @@ -376,7 +372,8 @@ func tag2(old reflect.StructTag) [2]string { if len(tag) >= 5 && strings.ToLower(tag[:5]) == "hcl:\"" { tag = tag[5 : len(tag)-1] two := strings.SplitN(tag, ",", 2) - if len(two) == 2 { + count := 2 + if len(two) == count { return [2]string{two[0], two[1]} } return [2]string{two[0], ""} From 746edfd596ac50c91ba26f826a7a04acab0e5303 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Mon, 22 Apr 2024 23:06:58 +0300 Subject: [PATCH 3/3] fix: unit test --- terraform/cliconfig/marshal_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/terraform/cliconfig/marshal_test.go b/terraform/cliconfig/marshal_test.go index 19232256a..8a4d69f8d 100644 --- a/terraform/cliconfig/marshal_test.go +++ b/terraform/cliconfig/marshal_test.go @@ -9,7 +9,7 @@ import ( ) func TestJsonHcl(t *testing.T) { - jsonData := `{"name": "peter", "parties":["one", "two", ["three", "four"], {"five":"51", "six":61}]}` + jsonData := `{"name": "peter", "parties":["one", "two", ["four"], {"six":61}]}` expectedHCL := ` name = "peter" @@ -17,11 +17,9 @@ func TestJsonHcl(t *testing.T) { "one", "two", [ - "three", "four" ], { - five = "51" six = 61 } ]