Skip to content

Commit

Permalink
feat: add multiple config patches, patches from files, YAML support
Browse files Browse the repository at this point in the history
Include filename content if value begins with @ (see curl for example).

Add multiple config-path option on cmdline to apply them in order.

ex:

```
talosctl-linux-amd64 gen config talos1 https://127.0.0.1:6443 --config-patch-control-plan @cidrs.json --config-patch-worker @sysctls-workders.json --config-path @cluster-name.json
```

Load JSON patch from YAML.

This applies to all commands handling config patches.

Closes: #4764

Signed-off-by: Sébastien Bernard <[email protected]>
Signed-off-by: Andrey Smirnov <[email protected]>
  • Loading branch information
Bernard Sébastien authored and smira committed Jan 31, 2022
1 parent 202290b commit 7f0b3aa
Show file tree
Hide file tree
Showing 11 changed files with 308 additions and 137 deletions.
21 changes: 9 additions & 12 deletions cmd/talosctl/cmd/mgmt/cluster/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/talos-systems/talos/pkg/images"
clientconfig "github.com/talos-systems/talos/pkg/machinery/client/config"
"github.com/talos-systems/talos/pkg/machinery/config"
"github.com/talos-systems/talos/pkg/machinery/config/configpatcher"
"github.com/talos-systems/talos/pkg/machinery/config/encoder"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/bundle"
Expand Down Expand Up @@ -112,9 +113,9 @@ var (
useVIP bool
enableKubeSpan bool
enableClusterDiscovery bool
configPatch string
configPatchControlPlane string
configPatchWorker string
configPatch []string
configPatchControlPlane []string
configPatchWorker []string
badRTC bool
extraBootKernelArgs string
)
Expand Down Expand Up @@ -453,14 +454,10 @@ func create(ctx context.Context) (err error) {
)
}

addConfigPatch := func(configPatch string, configOpt func(jsonpatch.Patch) bundle.Option) error {
if configPatch == "" {
return nil
}

addConfigPatch := func(configPatches []string, configOpt func(jsonpatch.Patch) bundle.Option) error {
var jsonPatch jsonpatch.Patch

jsonPatch, err = jsonpatch.DecodePatch([]byte(configPatch))
jsonPatch, err = configpatcher.LoadPatches(configPatches)
if err != nil {
return fmt.Errorf("error parsing config JSON patch: %w", err)
}
Expand Down Expand Up @@ -867,9 +864,9 @@ func init() {
createCmd.Flags().BoolVar(&useVIP, "use-vip", false, "use a virtual IP for the controlplane endpoint instead of the loadbalancer")
createCmd.Flags().BoolVar(&enableClusterDiscovery, "with-cluster-discovery", true, "enable cluster discovery")
createCmd.Flags().BoolVar(&enableKubeSpan, "with-kubespan", false, "enable KubeSpan system")
createCmd.Flags().StringVar(&configPatch, "config-patch", "", "patch generated machineconfigs (applied to all node types)")
createCmd.Flags().StringVar(&configPatchControlPlane, "config-patch-control-plane", "", "patch generated machineconfigs (applied to 'init' and 'controlplane' types)")
createCmd.Flags().StringVar(&configPatchWorker, "config-patch-worker", "", "patch generated machineconfigs (applied to 'worker' type)")
createCmd.Flags().StringArrayVar(&configPatch, "config-patch", nil, "patch generated machineconfigs (applied to all node types), use @file to read a patch from file")
createCmd.Flags().StringArrayVar(&configPatchControlPlane, "config-patch-control-plane", nil, "patch generated machineconfigs (applied to 'init' and 'controlplane' types)")
createCmd.Flags().StringArrayVar(&configPatchWorker, "config-patch-worker", nil, "patch generated machineconfigs (applied to 'worker' type)")
createCmd.Flags().BoolVar(&badRTC, "bad-rtc", false, "launch VM with bad RTC state (QEMU only)")
createCmd.Flags().StringVar(&extraBootKernelArgs, "extra-boot-kernel-args", "", "add extra kernel args to the initial boot from vmlinuz and initramfs (QEMU only)")

Expand Down
27 changes: 12 additions & 15 deletions cmd/talosctl/cmd/mgmt/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/talos-systems/talos/cmd/talosctl/pkg/mgmt/helpers"
"github.com/talos-systems/talos/pkg/images"
"github.com/talos-systems/talos/pkg/machinery/config"
"github.com/talos-systems/talos/pkg/machinery/config/configpatcher"
"github.com/talos-systems/talos/pkg/machinery/config/encoder"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/bundle"
Expand All @@ -38,9 +39,9 @@ var genConfigCmdFlags struct {
installDisk string
installImage string
outputDir string
configPatch string
configPatchControlPlane string
configPatchWorker string
configPatch []string
configPatchControlPlane []string
configPatchWorker []string
registryMirrors []string
persistConfig bool
withExamples bool
Expand Down Expand Up @@ -120,9 +121,9 @@ func GenV1Alpha1Config(genOptions []generate.GenOption,
clusterName string,
endpoint string,
kubernetesVersion string,
configPatch string,
configPatchControlPlane string,
configPatchWorker string) (*v1alpha1.ConfigBundle, error) {
configPatch []string,
configPatchControlPlane []string,
configPatchWorker []string) (*v1alpha1.ConfigBundle, error) {
configBundleOpts := []bundle.Option{
bundle.WithInputOptions(
&bundle.InputOptions{
Expand All @@ -134,12 +135,8 @@ func GenV1Alpha1Config(genOptions []generate.GenOption,
),
}

addConfigPatch := func(configPatch string, configOpt func(jsonpatch.Patch) bundle.Option) error {
if configPatch == "" {
return nil
}

jsonPatch, err := jsonpatch.DecodePatch([]byte(configPatch))
addConfigPatch := func(configPatches []string, configOpt func(jsonpatch.Patch) bundle.Option) error {
jsonPatch, err := configpatcher.LoadPatches(configPatches)
if err != nil {
return fmt.Errorf("error parsing config JSON patch: %w", err)
}
Expand Down Expand Up @@ -277,9 +274,9 @@ func init() {
genConfigCmd.Flags().StringVar(&genConfigCmdFlags.talosVersion, "talos-version", "", "the desired Talos version to generate config for (backwards compatibility, e.g. v0.8)")
genConfigCmd.Flags().StringVar(&genConfigCmdFlags.kubernetesVersion, "kubernetes-version", "", "desired kubernetes version to run")
genConfigCmd.Flags().StringVarP(&genConfigCmdFlags.outputDir, "output-dir", "o", "", "destination to output generated files")
genConfigCmd.Flags().StringVar(&genConfigCmdFlags.configPatch, "config-patch", "", "patch generated machineconfigs (applied to all node types)")
genConfigCmd.Flags().StringVar(&genConfigCmdFlags.configPatchControlPlane, "config-patch-control-plane", "", "patch generated machineconfigs (applied to 'init' and 'controlplane' types)")
genConfigCmd.Flags().StringVar(&genConfigCmdFlags.configPatchWorker, "config-patch-worker", "", "patch generated machineconfigs (applied to 'worker' type)")
genConfigCmd.Flags().StringArrayVar(&genConfigCmdFlags.configPatch, "config-patch", nil, "patch generated machineconfigs (applied to all node types), use @file to read a patch from file")
genConfigCmd.Flags().StringArrayVar(&genConfigCmdFlags.configPatchControlPlane, "config-patch-control-plane", nil, "patch generated machineconfigs (applied to 'init' and 'controlplane' types)")
genConfigCmd.Flags().StringArrayVar(&genConfigCmdFlags.configPatchWorker, "config-patch-worker", nil, "patch generated machineconfigs (applied to 'worker' type)")
genConfigCmd.Flags().StringSliceVar(&genConfigCmdFlags.registryMirrors, "registry-mirror", []string{}, "list of registry mirrors to use in format: <registry host>=<mirror URL>")
genConfigCmd.Flags().BoolVarP(&genConfigCmdFlags.persistConfig, "persist", "p", true, "the desired persist value for configs")
genConfigCmd.Flags().BoolVarP(&genConfigCmdFlags.withExamples, "with-examples", "", true, "renders all machine configs with the commented examples")
Expand Down
30 changes: 7 additions & 23 deletions cmd/talosctl/cmd/talos/patch.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"strings"

jsonpatch "github.com/evanphx/json-patch"
Expand All @@ -27,7 +25,7 @@ import (
var patchCmdFlags struct {
helpers.Mode
namespace string
patch string
patch []string
patchFile string
}

Expand Down Expand Up @@ -86,29 +84,15 @@ var patchCmd = &cobra.Command{
Args: cobra.RangeArgs(1, 2),
RunE: func(cmd *cobra.Command, args []string) error {
return WithClient(func(ctx context.Context, c *client.Client) error {
var (
patch jsonpatch.Patch
patchData []byte
)

switch {
case patchCmdFlags.patch != "":
patchData = []byte(patchCmdFlags.patch)
case patchCmdFlags.patchFile != "":
f, err := os.Open(patchCmdFlags.patchFile)
if err != nil {
return err
}
if patchCmdFlags.patchFile != "" {
patchCmdFlags.patch = append(patchCmdFlags.patch, "@"+patchCmdFlags.patchFile)
}

patchData, err = ioutil.ReadAll(f)
if err != nil {
return err
}
default:
if len(patchCmdFlags.patch) == 0 {
return fmt.Errorf("either --patch or --patch-file should be defined")
}

patch, err := jsonpatch.DecodePatch(patchData)
patch, err := configpatcher.LoadPatches(patchCmdFlags.patch)
if err != nil {
return err
}
Expand All @@ -128,7 +112,7 @@ var patchCmd = &cobra.Command{
func init() {
patchCmd.Flags().StringVar(&patchCmdFlags.namespace, "namespace", "", "resource namespace (default is to use default namespace per resource)")
patchCmd.Flags().StringVar(&patchCmdFlags.patchFile, "patch-file", "", "a file containing a patch to be applied to the resource.")
patchCmd.Flags().StringVarP(&patchCmdFlags.patch, "patch", "p", "", "the patch to be applied to the resource file.")
patchCmd.Flags().StringArrayVarP(&patchCmdFlags.patch, "patch", "p", nil, "the patch to be applied to the resource file, use @file to read a patch from file.")
helpers.AddModeFlags(&patchCmdFlags.Mode, patchCmd)
addCommand(patchCmd)
}
7 changes: 7 additions & 0 deletions hack/release.toml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ even if Talos is booted in maintenance mode (without machine configuration is pl
title = "SBC Support"
description="""\
Talos now supports Jetson Nano SBC.
"""

[notes.patching]
title = "Machine Configuration Patching"
description="""\
`talosctl` commands which accept JSON patches (`gen config`, `cluster create`, `patch machineconfig`) now support multiple patches, loading patches
from files with `@file.json` syntax, and support loading from YAML format.
"""

[make_deps]
Expand Down
7 changes: 4 additions & 3 deletions pkg/machinery/config/configpatcher/configpatcher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//nolint:scopelint,testpackage
package configpatcher
package configpatcher_test

import (
"reflect"
"testing"

jsonpatch "github.com/evanphx/json-patch"

"github.com/talos-systems/talos/pkg/machinery/config/configpatcher"
)

const dummyConfig = `machine:
Expand Down Expand Up @@ -53,7 +54,7 @@ func TestJSON6902(t *testing.T) {
return
}

got, err := JSON6902(tt.args.talosMachineConfig, patch)
got, err := configpatcher.JSON6902(tt.args.talosMachineConfig, patch)
if (err != nil) != tt.wantErr {
t.Errorf("JSON6902 error: %v, but wanted: %v", err, tt.wantErr)

Expand Down
86 changes: 86 additions & 0 deletions pkg/machinery/config/configpatcher/load.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package configpatcher

import (
"encoding/json"
"os"
"strings"

jsonpatch "github.com/evanphx/json-patch"
"gopkg.in/yaml.v3"
)

type patch []map[string]interface{}

// LoadPatch loads the JSON patch either from JSON or YAML.
func LoadPatch(in []byte) (p jsonpatch.Patch, err error) {
var jsonErr error

// try JSON first
if p, jsonErr = jsonpatch.DecodePatch(in); jsonErr == nil {
return p, nil
}

// try YAML
var yamlPatch patch

if err = yaml.Unmarshal(in, &yamlPatch); err != nil {
// not YAML either, return JSON error
return p, jsonErr
}

p = make(jsonpatch.Patch, 0, len(yamlPatch))

for _, yp := range yamlPatch {
op := make(jsonpatch.Operation, len(yp))

for key, value := range yp {
m, err := json.Marshal(value)
if err != nil {
return p, err
}

op[key] = (*json.RawMessage)(&m)
}

p = append(p, op)
}

return p, nil
}

// LoadPatches loads the JSON patch either from value literal or from a file if the patch starts with '@'.
func LoadPatches(in []string) (jsonpatch.Patch, error) {
var result jsonpatch.Patch

for _, patchString := range in {
var (
p jsonpatch.Patch
contents []byte
err error
)

if strings.HasPrefix(patchString, "@") {
filename := patchString[1:]

contents, err = os.ReadFile(filename)
if err != nil {
return result, err
}
} else {
contents = []byte(patchString)
}

p, err = LoadPatch(contents)
if err != nil {
return result, err
}

result = append(result, p...)
}

return result, nil
}
68 changes: 68 additions & 0 deletions pkg/machinery/config/configpatcher/load_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package configpatcher_test

import (
_ "embed"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/talos-systems/talos/pkg/machinery/config/configpatcher"
)

//go:embed testdata/patch.json
var jsonPatch []byte

//go:embed testdata/patch.yaml
var yamlPatch []byte

func TestLoadJSON(t *testing.T) {
p, err := configpatcher.LoadPatch(jsonPatch)
require.NoError(t, err)

assert.Len(t, p, 1)
assert.Equal(t, p[0].Kind(), "add")

var path string
path, err = p[0].Path()

require.NoError(t, err)
assert.Equal(t, path, "/machine/certSANs")
}

func TestLoadYAML(t *testing.T) {
p, err := configpatcher.LoadPatch(yamlPatch)
require.NoError(t, err)

assert.Len(t, p, 1)
assert.Equal(t, p[0].Kind(), "add")

var path string
path, err = p[0].Path()

require.NoError(t, err)
assert.Equal(t, path, "/some/path")

var v interface{}
v, err = p[0].ValueInterface()
require.NoError(t, err)
assert.Equal(t, v, []interface{}{"a", "b", "c"})
}

func TestLoadPatches(t *testing.T) {
p, err := configpatcher.LoadPatches([]string{
"@testdata/patch.json",
"@testdata/patch.yaml",
`[{"op":"replace","path":"/some","value": []}]`,
})
require.NoError(t, err)

assert.Len(t, p, 3)
assert.Equal(t, p[0].Kind(), "add")
assert.Equal(t, p[1].Kind(), "add")
assert.Equal(t, p[2].Kind(), "replace")
}
7 changes: 7 additions & 0 deletions pkg/machinery/config/configpatcher/testdata/patch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"op": "add",
"path": "/machine/certSANs",
"value": ["foo.com"]
}
]
6 changes: 6 additions & 0 deletions pkg/machinery/config/configpatcher/testdata/patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
- op: add
path: /some/path
value:
- a
- b
- c
Loading

0 comments on commit 7f0b3aa

Please sign in to comment.