JSON, TOML, YAML: support for nested config data (#112)

* internal: add TraverseMap

* gofumpt

* JSONParseConfig

* internal: TraverseMap: map[any]any

* fftoml: don't break the API

* internal: TraverseMap: test for map[any]any

* fftest, ffyaml: nested node tests

* Can't remove ParseError types

* Can't remove StringConversionError type

ffcli/examples/objectctl/pkg/objectapi/client.go

@@ -102,17 +102,17 @@ func (s *mockServer) list(token string) ([]Object, error) {
 }
 
 var defaultObjects = map[string]Object{
-	"apple": Object{
+	"apple": {
 		Key:    "apple",
 		Value:  "The fruit of any of certain other species of tree of the same genus.",
 		Access: mustParseTime(time.RFC3339, "2019-03-15T15:01:00Z"),
 	},
-	"beach": Object{
+	"beach": {
 		Key:    "beach",
 		Value:  "The shore of a body of water, especially when sandy or pebbly.",
 		Access: mustParseTime(time.RFC3339, "2019-04-20T12:21:30Z"),
 	},
-	"carillon": Object{
+	"carillon": {
 		Key:    "carillon",
 		Value:  "A stationary set of chromatically tuned bells in a tower.",
 		Access: mustParseTime(time.RFC3339, "2019-07-04T23:59:59Z"),

fftest/tempfile.go

@@ -16,7 +16,7 @@ func TempFile(t *testing.T, content string) string {
 
 	filename := filepath.Join(t.TempDir(), strconv.Itoa(rand.Int()))
 
-	if err := os.WriteFile(filename, []byte(content), 0600); err != nil {
+	if err := os.WriteFile(filename, []byte(content), 0o0600); err != nil {
 		t.Fatal(err)
 	}
 

fftest/vars.go

@@ -3,6 +3,7 @@ package fftest
 import (
 	"errors"
 	"flag"
+	"fmt"
 	"reflect"
 	"strings"
 	"testing"
@@ -44,6 +45,19 @@ func NonzeroDefaultVars(fs *flag.FlagSet) *Vars {
 	return &v
 }
 
+// NestedDefaultVars is similar to DefaultVars, but uses nested flag names.
+func NestedDefaultVars(delimiter string) func(fs *flag.FlagSet) *Vars {
+	return func(fs *flag.FlagSet) *Vars {
+		var v Vars
+		fs.StringVar(&v.S, fmt.Sprintf("foo%ss", delimiter), "", "string")
+		fs.IntVar(&v.I, fmt.Sprintf("bar%[1]snested%[1]si", delimiter), 0, "int")
+		fs.Float64Var(&v.F, fmt.Sprintf("bar%[1]snested%[1]sf", delimiter), 0., "float64")
+		fs.BoolVar(&v.B, fmt.Sprintf("foo%sb", delimiter), false, "bool")
+		fs.Var(&v.X, fmt.Sprintf("baz%[1]snested%[1]sx", delimiter), "collection of strings (repeatable)")
+		return &v
+	}
+}
+
 // Vars are a common set of variables used for testing.
 type Vars struct {
 	S string

fftoml/fftoml.go

@@ -4,10 +4,9 @@ package fftoml
 import (
 	"fmt"
 	"io"
-	"strconv"
 
 	"github.com/pelletier/go-toml"
-	"github.com/peterbourgon/ff/v3"
+	"github.com/peterbourgon/ff/v3/internal"
 )
 
 // Parser is a parser for TOML file format. Flags and their values are read
@@ -36,12 +35,16 @@ func New(opts ...Option) (c ConfigFileParser) {
 // Parse parses the provided io.Reader as a TOML file and uses the provided set function
 // to set flag names derived from the tables names and their key/value pairs.
 func (c ConfigFileParser) Parse(r io.Reader, set func(name, value string) error) error {
-	tree, err := toml.LoadReader(r)
-	if err != nil {
+	var m map[string]any
+	if err := toml.NewDecoder(r).Decode(&m); err != nil {
 		return ParseError{Inner: err}
 	}
 
-	return parseTree(tree, "", c.delimiter, set)
+	if err := internal.TraverseMap(m, c.delimiter, set); err != nil {
+		return ParseError{Inner: err}
+	}
+
+	return nil
 }
 
 // Option is a function which changes the behavior of the TOML config file parser.
@@ -65,69 +68,6 @@ func WithTableDelimiter(d string) Option {
 	}
 }
 
-func parseTree(tree *toml.Tree, parent, delimiter string, set func(name, value string) error) error {
-	for _, key := range tree.Keys() {
-		name := key
-		if parent != "" {
-			name = parent + delimiter + key
-		}
-		switch t := tree.Get(key).(type) {
-		case *toml.Tree:
-			if err := parseTree(t, name, delimiter, set); err != nil {
-				return err
-			}
-		case interface{}:
-			values, err := valsToStrs(t)
-			if err != nil {
-				return ParseError{Inner: err}
-			}
-			for _, value := range values {
-				if err = set(name, value); err != nil {
-					return err
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func valsToStrs(val interface{}) ([]string, error) {
-	if vals, ok := val.([]interface{}); ok {
-		ss := make([]string, len(vals))
-		for i := range vals {
-			s, err := valToStr(vals[i])
-			if err != nil {
-				return nil, err
-			}
-			ss[i] = s
-		}
-		return ss, nil
-	}
-	s, err := valToStr(val)
-	if err != nil {
-		return nil, err
-	}
-	return []string{s}, nil
-
-}
-
-func valToStr(val interface{}) (string, error) {
-	switch v := val.(type) {
-	case string:
-		return v, nil
-	case bool:
-		return strconv.FormatBool(v), nil
-	case uint64:
-		return strconv.FormatUint(v, 10), nil
-	case int64:
-		return strconv.FormatInt(v, 10), nil
-	case float64:
-		return strconv.FormatFloat(v, 'g', -1, 64), nil
-	default:
-		return "", ff.StringConversionError{Value: val}
-	}
-}
-
 // ParseError wraps all errors originating from the TOML parser.
 type ParseError struct {
 	Inner error

fftoml/fftoml_test.go

@@ -2,7 +2,9 @@ package fftoml_test
 
 import (
 	"flag"
+	"fmt"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -56,61 +58,48 @@ func TestParser(t *testing.T) {
 func TestParser_WithTables(t *testing.T) {
 	t.Parallel()
 
-	type fields struct {
-		String  string
-		Float   float64
-		Strings fftest.StringSlice
-	}
-
-	expected := fields{
-		String:  "a string",
-		Float:   1.23,
-		Strings: fftest.StringSlice{"one", "two", "three"},
-	}
-
-	for _, testcase := range []struct {
-		name string
-		opts []fftoml.Option
-		// expectations
-		stringKey  string
-		floatKey   string
-		stringsKey string
-	}{
-		{
-			name:       "defaults",
-			stringKey:  "string.key",
-			floatKey:   "float.nested.key",
-			stringsKey: "strings.nested.key",
-		},
-		{
-			name:       "defaults",
-			opts:       []fftoml.Option{fftoml.WithTableDelimiter("-")},
-			stringKey:  "string-key",
-			floatKey:   "float-nested-key",
-			stringsKey: "strings-nested-key",
-		},
+	for _, delim := range []string{
+		".",
+		"-",
 	} {
-		t.Run(testcase.name, func(t *testing.T) {
+		t.Run(fmt.Sprintf("delim=%q", delim), func(t *testing.T) {
 			var (
-				found fields
-				fs    = flag.NewFlagSet("fftest", flag.ContinueOnError)
+				skey = strings.Join([]string{"string", "key"}, delim)
+				fkey = strings.Join([]string{"float", "nested", "key"}, delim)
+				xkey = strings.Join([]string{"strings", "nested", "key"}, delim)
+
+				sval string
+				fval float64
+				xval fftest.StringSlice
 			)
 
-			fs.StringVar(&found.String, testcase.stringKey, "", "string")
-			fs.Float64Var(&found.Float, testcase.floatKey, 0, "float64")
-			fs.Var(&found.Strings, testcase.stringsKey, "string slice")
+			fs := flag.NewFlagSet("fftest", flag.ContinueOnError)
+			{
+				fs.StringVar(&sval, skey, "xxx", "string")
+				fs.Float64Var(&fval, fkey, 999, "float64")
+				fs.Var(&xval, xkey, "strings")
+			}
+
+			parseConfig := fftoml.New(fftoml.WithTableDelimiter(delim))
 
 			if err := ff.Parse(fs, []string{},
 				ff.WithConfigFile("testdata/table.toml"),
-				ff.WithConfigFileParser(fftoml.New(testcase.opts...).Parse),
+				ff.WithConfigFileParser(parseConfig.Parse),
 			); err != nil {
 				t.Fatal(err)
 			}
 
-			if !reflect.DeepEqual(expected, found) {
-				t.Errorf(`expected %v, to be %v`, found, expected)
+			if want, have := "a string", sval; want != have {
+				t.Errorf("string key: want %q, have %q", want, have)
+			}
+
+			if want, have := 1.23, fval; want != have {
+				t.Errorf("float nested key: want %v, have %v", want, have)
+			}
+
+			if want, have := (fftest.StringSlice{"one", "two", "three"}), xval; !reflect.DeepEqual(want, have) {
+				t.Errorf("strings nested key: want %v, have %v", want, have)
 			}
 		})
 	}
-
 }

ffyaml/ffyaml.go

@@ -2,77 +2,44 @@
 package ffyaml
 
 import (
+	"errors"
 	"fmt"
 	"io"
-	"strconv"
 
-	"github.com/peterbourgon/ff/v3"
+	"github.com/peterbourgon/ff/v3/internal"
 	"gopkg.in/yaml.v2"
 )
 
-// Parser is a parser for YAML file format. Flags and their values are read
-// from the key/value pairs defined in the config file.
+// Parser is a helper function that uses a default ParseConfig.
 func Parser(r io.Reader, set func(name, value string) error) error {
-	var m map[string]interface{}
-	d := yaml.NewDecoder(r)
-	if err := d.Decode(&m); err != nil && err != io.EOF {
-		return ParseError{err}
-	}
-	for key, val := range m {
-		values, err := valsToStrs(val)
-		if err != nil {
-			return ParseError{err}
-		}
-		for _, value := range values {
-			if err := set(key, value); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
+	return (&ParseConfig{}).Parse(r, set)
 }
 
-func valsToStrs(val interface{}) ([]string, error) {
-	if vals, ok := val.([]interface{}); ok {
-		ss := make([]string, len(vals))
-		for i := range vals {
-			s, err := valToStr(vals[i])
-			if err != nil {
-				return nil, err
-			}
-			ss[i] = s
-		}
-		return ss, nil
-	}
-	s, err := valToStr(val)
-	if err != nil {
-		return nil, err
+// ParseConfig collects parameters for the YAML config file parser.
+type ParseConfig struct {
+	// Delimiter is used when concatenating nested node keys into a flag name.
+	// The default delimiter is ".".
+	Delimiter string
+}
+
+// Parse a YAML document from the provided io.Reader, using the provided set
+// function to set flag values. Flag names are derived from the node names and
+// their key/value pairs.
+func (pc *ParseConfig) Parse(r io.Reader, set func(name, value string) error) error {
+	if pc.Delimiter == "" {
+		pc.Delimiter = "."
 	}
-	return []string{s}, nil
 
-}
+	var m map[string]interface{}
+	if err := yaml.NewDecoder(r).Decode(&m); err != nil && !errors.Is(err, io.EOF) {
+		return ParseError{Inner: err}
+	}
 
-func valToStr(val interface{}) (string, error) {
-	switch v := val.(type) {
-	case byte:
-		return string([]byte{v}), nil
-	case string:
-		return v, nil
-	case bool:
-		return strconv.FormatBool(v), nil
-	case uint64:
-		return strconv.FormatUint(v, 10), nil
-	case int:
-		return strconv.Itoa(v), nil
-	case int64:
-		return strconv.FormatInt(v, 10), nil
-	case float64:
-		return strconv.FormatFloat(v, 'g', -1, 64), nil
-	case nil:
-		return "", nil
-	default:
-		return "", ff.StringConversionError{Value: val}
+	if err := internal.TraverseMap(m, pc.Delimiter, set); err != nil {
+		return ParseError{Inner: err}
 	}
+
+	return nil
 }
 
 // ParseError wraps all errors originating from the YAML parser.

ffyaml/ffyaml_test.go

@@ -80,6 +80,12 @@ func TestParser(t *testing.T) {
 			miss: false,
 			want: fftest.Vars{WantParseErrorIs: os.ErrNotExist},
 		},
+		{
+			name: "nested nodes",
+			file: "testdata/nested.yaml",
+			vars: fftest.NestedDefaultVars("."),
+			want: fftest.Vars{S: "a string", B: true, I: 123, F: 1.23, X: []string{"one", "two", "three"}},
+		},
 	} {
 		t.Run(testcase.name, func(t *testing.T) {
 			if testcase.vars == nil {

ffyaml/testdata/nested.yaml

@@ -0,0 +1,13 @@
+foo:
+  s: a string
+  b: true
+bar:
+  nested:
+    i: 123
+    f: 1.23
+baz:
+  nested:
+    x:
+      - one
+      - two
+      - three

internal/doc.go

@@ -0,0 +1,2 @@
+// Package internal provides private helpers used by various module packages.
+package internal