diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 265c873..1844087 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -56,18 +56,19 @@ jobs: with: path: dist key: dist-${{ github.run_id }} + - run: find dist # The upload-artifact action doesn't support multi upload 🤷‍♂️! - name: Upload Artifacts - AMD uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3 with: name: merger-${{ matrix.os.id }}-amd path: | - dist/kustomize-plugin-merger_${{ matrix.os.id }}_amd*/kustomize-plugin-merger* + dist/kustomize-plugin-merger_*_${{ matrix.os.id }}_amd*.{tar.gz,zip} dist/*checksums.txt* - name: Upload Artifacts - ARM uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3 with: name: merger-${{ matrix.os.id }}-arm path: | - dist/kustomize-plugin-merger_${{ matrix.os.id }}_arm*/kustomize-plugin-merger* + dist/kustomize-plugin-merger_*_${{ matrix.os.id }}_arm*.{tar.gz,zip} dist/*checksums.txt* diff --git a/Makefile b/Makefile index 734a3d5..868f896 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ CONTROLLER_GEN_VERSION := v0.13.0 # Schema targets. # -.PHONY: schema.install-tools +.PHONY: schema.install-tools schema.install-tools: mkdir -p $(CONTROLLER_GEN_HOME);\ export GOBIN=$(CONTROLLER_GEN_HOME);\ diff --git a/README.md b/README.md index eebca65..865f5b2 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,14 @@ A Kustomize generator plugin to merge YAML files seamlessly for real-world use c **Merger** provides schemaless merge with different merge strategies (StrategicMerge). - - [Why](#why) - [Features](#features) - [Options](#options) - [Common use cases](#common-use-cases) - [1. Generate multiple manifests from a single base](#1-generate-multiple-manifests-from-a-single-base) - - [2. Merge lists in manifests without schema or a unique identifier](#2-merge-lists-in-manifests-without-schema-or-a-unique-identifier) - - [3. Organize long manifests into smaller ones](#3-organize-long-manifests-into-smaller-ones) + - [2. Merge non-manifest files and store them into ConfigMap or Secret](#2-merge-non-manifest-files-and-store-them-into-configmap-or-secret) + - [3. Merge lists in manifests without schema or a unique identifier](#3-merge-lists-in-manifests-without-schema-or-a-unique-identifier) + - [4. Organize long manifests into smaller ones](#4-organize-long-manifests-into-smaller-ones) - [TO-DO](#to-do) - [Project status](#project-status) - [Contributing](#contributing) @@ -49,10 +49,10 @@ and for more details on the challenge of providing OpenAPI schema to merge files - Generate multiple resources/manifests from a single base without copying the resources multiple times. - Merge any manifests (even CustomResources) without needing their OpenAPI schema. +- Merge applications configuration YAML files into a ConfigMap or Secret. - Merge manifests with a list of maps without a unique identifier (when using `x-kubernetes-patch-merge-key` is not possible). - Merge YAML files with different merge strategies (StrategicMerge). -- Merge applications configuration YAML files into a ConfigMap or Secret (WIP). ## Options @@ -74,8 +74,8 @@ metadata: dst: /mnt # Exec KRM functions. # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger + # exec: + # path: kustomize-plugin-merger spec: resources: - name: example @@ -98,7 +98,7 @@ spec: # - Combine: Maps from source merged with destination, but the lists will be combined together. strategy: combine output: - # Available options: raw. + # Available options: raw,configmap,secret format: raw ``` @@ -112,29 +112,34 @@ This section shows a couple of use cases where Merger can help. In this case, you have multiple `CronJobs`, all of which share the same body, but each has a different command or other config. -[Use case full example](./examples/multiple-manifests-from-single-file/README.md). +[Read the full example](./examples/multiple-manifests-from-single-file/README.md). + +### 2. Merge non-manifest files and store them into ConfigMap or Secret + +TBA + -### 2. Merge lists in manifests without schema or a unique identifier +### 3. Merge lists in manifests without schema or a unique identifier Currently, in Kustomize, it's not possible to merge resources without a unique identifier, even with Open API schema. It's possible to do that using the merge strategy `append` in Merger (later on, `combineWithKey` will also be supported). -[Use case full example](./examples/manifest-lists-without-schema/README.md). +[Read the full example](./examples/manifest-lists-without-schema/README.md). -### 3. Organize long manifests into smaller ones +### 4. Organize long manifests into smaller ones In some use cases (e.g., [Crossplane Compositions](https://docs.crossplane.io/latest/concepts/compositions/)), you could have a really long YAML manifest, and it's hard to read. You can split that file and use the Merger `patch` input method to make it a single manifest again. -[Use case full example](./examples/long-omni-manifest/README.md). +[Read the full example](./examples/long-omni-manifest/README.md). ## TO-DO -- Support `ConfigMap` or `Secret` as an output. - Support `combine` merge strategy with an identifier key (similar to `x-kubernetes-patch-merge-key`). +- Configure the output indentation. - Provide better docs for Merger options. diff --git a/examples/krm-and-kustomize/merger.yaml b/examples/krm-and-kustomize/merger.yaml index 352dda7..c54339c 100644 --- a/examples/krm-and-kustomize/merger.yaml +++ b/examples/krm-and-kustomize/merger.yaml @@ -14,8 +14,8 @@ metadata: dst: /mnt # Exec KRM functions. # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger + # exec: + # path: kustomize-plugin-merger spec: resources: - name: my-envs diff --git a/examples/krm-and-kustomize/resourcelist.yaml b/examples/krm-and-kustomize/resourcelist.yaml index 792c916..52c4f1e 100644 --- a/examples/krm-and-kustomize/resourcelist.yaml +++ b/examples/krm-and-kustomize/resourcelist.yaml @@ -8,27 +8,48 @@ functionConfig: kind: Merger metadata: name: merge - annotations: - # Containerized KRM function. - config.kubernetes.io/function: | - container: - image: ghcr.io/aabouzaid/kustomize-generator-merger - mounts: - - type: bind - src: ./ - dst: /mnt - # Exec KRM functions. - # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger spec: resources: - - name: my-envs + - name: raw-output-patch + input: + method: patch + files: + sources: + - input/dev.yaml + - input/stage.yaml + destination: input/base.yaml + merge: + strategy: append + output: + format: raw + - name: raw-output-overlay + input: + method: overlay + files: + sources: + - input/dev.yaml + - input/stage.yaml + destination: input/base.yaml + merge: + strategy: append + output: + format: raw + - name: configmap-output + input: + method: overlay + files: + sources: + - input/dev.yaml + - input/stage.yaml + destination: input/base.yaml + merge: + strategy: append + output: + format: configmap + - name: secret-output input: method: overlay files: - # The same as in the KRM container above, omit it if Exec KRM is used. - root: /mnt sources: - input/dev.yaml - input/stage.yaml @@ -36,4 +57,4 @@ functionConfig: merge: strategy: append output: - format: raw \ No newline at end of file + format: secret diff --git a/examples/long-omni-manifest/README.md b/examples/long-omni-manifest/README.md index b5f99ba..c2d95ae 100644 --- a/examples/long-omni-manifest/README.md +++ b/examples/long-omni-manifest/README.md @@ -110,8 +110,8 @@ metadata: dst: /mnt # Exec KRM functions. # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger + # exec: + # path: kustomize-plugin-merger spec: resources: - name: my-eks-composition diff --git a/examples/long-omni-manifest/merger.yaml b/examples/long-omni-manifest/merger.yaml index d23cfbf..8f7c53e 100644 --- a/examples/long-omni-manifest/merger.yaml +++ b/examples/long-omni-manifest/merger.yaml @@ -14,8 +14,8 @@ metadata: dst: /mnt # Exec KRM functions. # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger + # exec: + # path: kustomize-plugin-merger spec: resources: - name: my-eks-composition diff --git a/examples/manifest-lists-without-schema/README.md b/examples/manifest-lists-without-schema/README.md index 4d14813..aa129e5 100644 --- a/examples/manifest-lists-without-schema/README.md +++ b/examples/manifest-lists-without-schema/README.md @@ -78,8 +78,8 @@ metadata: dst: /mnt # Exec KRM functions. # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger + # exec: + # path: kustomize-plugin-merger spec: resources: - name: my-iam-policy diff --git a/examples/manifest-lists-without-schema/merger.yaml b/examples/manifest-lists-without-schema/merger.yaml index 4ee2a14..56aec1e 100644 --- a/examples/manifest-lists-without-schema/merger.yaml +++ b/examples/manifest-lists-without-schema/merger.yaml @@ -14,8 +14,8 @@ metadata: dst: /mnt # Exec KRM functions. # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger + # exec: + # path: kustomize-plugin-merger spec: resources: - name: my-iam-policy diff --git a/examples/multiple-manifests-from-single-file/README.md b/examples/multiple-manifests-from-single-file/README.md index ff2ec59..21db4cb 100644 --- a/examples/multiple-manifests-from-single-file/README.md +++ b/examples/multiple-manifests-from-single-file/README.md @@ -126,8 +126,8 @@ metadata: dst: /mnt # Exec KRM functions. # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger + # exec: + # path: kustomize-plugin-merger spec: resources: - name: my-cronjobs diff --git a/examples/multiple-manifests-from-single-file/merger.yaml b/examples/multiple-manifests-from-single-file/merger.yaml index 1079569..aaaff18 100644 --- a/examples/multiple-manifests-from-single-file/merger.yaml +++ b/examples/multiple-manifests-from-single-file/merger.yaml @@ -14,8 +14,8 @@ metadata: dst: /mnt # Exec KRM functions. # config.kubernetes.io/function: | - # exec: - # path: kustomize-plugin-merger + # exec: + # path: kustomize-plugin-merger spec: resources: - name: my-cronjobs diff --git a/examples/non-manifest-into-configmap-or-secret/README.md b/examples/non-manifest-into-configmap-or-secret/README.md new file mode 100644 index 0000000..ba22151 --- /dev/null +++ b/examples/non-manifest-into-configmap-or-secret/README.md @@ -0,0 +1,124 @@ + +# Example - Merge non-manifest files and store them into ConfigMap or Secret + +- [Use case](#use-case) +- [Input](#input) +- [Manifest](#manifest) +- [Build](#build) +- [Output](#output) + +## Use case + +No plans from Kustomize to support non-manifest files merge and storing them into ConfigMap +or Secret. Using Merger you can merge any YAML files like application configuration. +In the example, two Prometheus config are merged together and stored as ConfigMap. + +## Input + +```yaml +# prometheus-base.yaml +global: + scrape_interval: 15s + evaluation_interval: 15s + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +# This will be added via Merger. +rule_files: + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] +``` + +```yaml +# prometheus.yaml +rule_files: + - "first_rules.yml" + - "second_rules.yml" +``` + +## Manifest + +```yaml +# kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +generators: +- merger.yaml +``` + +```yaml +# merger.yaml +apiVersion: generators.kustomize.aabouzaid.com/v1alpha1 +kind: Merger +metadata: + name: merge + annotations: + # Containerized KRM function. + config.kubernetes.io/function: | + container: + image: ghcr.io/aabouzaid/kustomize-generator-merger + mounts: + - type: bind + src: ./ + dst: /mnt + # Exec KRM functions. + # config.kubernetes.io/function: | + # exec: + # path: kustomize-plugin-merger +spec: + resources: + - name: my-prometheus + input: + method: overlay + files: + # The same as in the KRM container above, omit it if Exec KRM is used. + root: /mnt + sources: + - input/prometheus.yaml + destination: input/prometheus-base.yaml + merge: + strategy: combine + output: + format: configmap +``` + +## Build + +```shell +kustomize build --enable-alpha-plugins --as-current-user . +``` + +## Output + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-prometheus +data: + prometheus.yaml: | + alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + global: + evaluation_interval: 15s + scrape_interval: 15s + rule_files: + - first_rules.yml + - second_rules.yml + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 +``` \ No newline at end of file diff --git a/examples/non-manifest-into-configmap-or-secret/input/prometheus-base.yaml b/examples/non-manifest-into-configmap-or-secret/input/prometheus-base.yaml new file mode 100644 index 0000000..6eed798 --- /dev/null +++ b/examples/non-manifest-into-configmap-or-secret/input/prometheus-base.yaml @@ -0,0 +1,19 @@ +# Copied from: +# https://github.com/prometheus/prometheus/blob/main/documentation/examples/prometheus.yml +global: + scrape_interval: 15s + evaluation_interval: 15s + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +# This will be added via Merger. +rule_files: + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] diff --git a/examples/non-manifest-into-configmap-or-secret/input/prometheus.yaml b/examples/non-manifest-into-configmap-or-secret/input/prometheus.yaml new file mode 100644 index 0000000..be77d2b --- /dev/null +++ b/examples/non-manifest-into-configmap-or-secret/input/prometheus.yaml @@ -0,0 +1,3 @@ +rule_files: + - "first_rules.yml" + - "second_rules.yml" diff --git a/examples/non-manifest-into-configmap-or-secret/kustomization.yaml b/examples/non-manifest-into-configmap-or-secret/kustomization.yaml new file mode 100644 index 0000000..2880438 --- /dev/null +++ b/examples/non-manifest-into-configmap-or-secret/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +generators: +- merger.yaml diff --git a/examples/non-manifest-into-configmap-or-secret/merger.yaml b/examples/non-manifest-into-configmap-or-secret/merger.yaml new file mode 100644 index 0000000..6caf0bd --- /dev/null +++ b/examples/non-manifest-into-configmap-or-secret/merger.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: generators.kustomize.aabouzaid.com/v1alpha1 +kind: Merger +metadata: + name: merge + annotations: + # Containerized KRM function. + config.kubernetes.io/function: | + container: + image: ghcr.io/aabouzaid/kustomize-generator-merger + mounts: + - type: bind + src: ./ + dst: /mnt + # Exec KRM functions. + # config.kubernetes.io/function: | + # exec: + # path: kustomize-plugin-merger +spec: + resources: + - name: prometheus + input: + method: overlay + files: + # The same as in the KRM container above, omit it if Exec KRM is used. + root: /mnt + sources: + - input/prometheus.yaml + destination: input/prometheus-base.yaml + merge: + strategy: combine + output: + format: configmap diff --git a/pkg/merger/fn.go b/pkg/merger/fn.go index f2eda53..f9834f5 100644 --- a/pkg/merger/fn.go +++ b/pkg/merger/fn.go @@ -25,12 +25,12 @@ func (m *Merger) Schema() (*spec.Schema, error) { // Default sets default values for Merger resources. func (m *Merger) Default() error { for index := range m.Spec.Resources { - // Defaults input files. - if err := m.Spec.Resources[index].setInputFiles(); err != nil { - return err - } + // + m.Spec.Resources[index].setInputFilesRoot() // Defaults merge strategy. m.Spec.Resources[index].setMergeStrategy() + // Create empty map for staged data. + m.Spec.Resources[index].Output.items = make(map[string]string) } return nil } @@ -43,8 +43,8 @@ func (m *Merger) Validate() error { // Filter performs the merging of configuration files for Merger resources. func (m *Merger) Filter(rlItems []*yaml.RNode) ([]*yaml.RNode, error) { for _, resource := range m.Spec.Resources { - resource.merge(resource.Input.items) - rlItems = append(rlItems, resource.Output.rlItems...) + resource.merge() + rlItems = append(rlItems, resource.export()...) } return rlItems, nil } diff --git a/pkg/merger/merge.go b/pkg/merger/merge.go index 529d2ca..243e10c 100644 --- a/pkg/merger/merge.go +++ b/pkg/merger/merge.go @@ -3,6 +3,7 @@ package merger import ( "log" + "path/filepath" "strings" "dario.cat/mergo" @@ -32,64 +33,82 @@ func (r *mergerResource) setInputFilesRoot() { } } -func (r *mergerResource) setInputFilesOverlay() { - r.setInputFilesRoot() - for _, inputFileSource := range r.Input.Files.Sources { - r.Input.items = append(r.Input.items, - resourceInputFiles{ - Sources: []string{r.Input.Files.Root + inputFileSource}, - Destination: r.Input.Files.Root + r.Input.Files.Destination, - }) +func (r *mergerResource) loadDestinationFile() *koanf.Koanf { + k := koanf.New(".") + dstFile := r.Input.Files.Root + r.Input.Files.Destination + if err := k.Load(koanfFile.Provider(dstFile), koanfYaml.Parser()); err != nil { + log.Fatalf("Error loading config: %v", err) } -} - -func (r *mergerResource) setInputFilesPatch() { - r.setInputFilesRoot() - r.Input.Files.Destination = r.Input.Files.Root + r.Input.Files.Destination - for index, inputFileSource := range r.Input.Files.Sources { - r.Input.Files.Sources[index] = r.Input.Files.Root + inputFileSource - } - r.Input.items = append(r.Input.items, r.Input.Files) -} - -// setInputFiles determines the input file sources based on the input method (Overlay or Patch). -func (r *mergerResource) setInputFiles() error { - switch r.Input.Method { - case Overlay: - r.setInputFilesOverlay() - case Patch: - r.setInputFilesPatch() - } - return nil + return k } // merge performs the actual merging of configuration files from resourceInputFiles sources. -func (r *mergerResource) merge(rfs []resourceInputFiles) { - for _, rf := range rfs { - k := koanf.New(".") +func (r *mergerResource) merge() { + // TODO: Simplify/split the logic in merge method. + k := r.loadDestinationFile() + fileKey := r.Name - if err := k.Load(koanfFile.Provider(rf.Destination), koanfYaml.Parser()); err != nil { - log.Fatalf("Error loading config: %v", err) - } + for _, srcFile := range r.Input.Files.Sources { + srcFile = r.Input.Files.Root + srcFile - for _, srcFile := range rf.Sources { - err := k.Load(koanfFile.Provider(srcFile), koanfYaml.Parser(), - koanf.WithMergeFunc(func(src, dst map[string]interface{}) error { - if err := mergo.Merge(&dst, src, mergo.WithOverride, r.Merge.config); err != nil { - log.Fatalf("Error merging config: %v", err) - } - return nil - })) - if err != nil { - log.Fatalf("Error loading config: %v", err) - } + if r.Input.Method == Overlay { + k = r.loadDestinationFile() + fileKey = filepath.Base(srcFile) } + err := k.Load(koanfFile.Provider(srcFile), koanfYaml.Parser(), + koanf.WithMergeFunc(func(src, dst map[string]interface{}) error { + if err := mergo.Merge(&dst, src, mergo.WithOverride, r.Merge.config); err != nil { + log.Fatalf("Error merging config: %v", err) + } + return nil + })) + if err != nil { + log.Fatalf("Error loading config: %v", err) + } + // mergedData, err := k.Marshal(koanfYaml.Parser()) if err != nil { log.Fatalf("Error marshaling yaml: %v", err) } - rNode := yaml.MustParse(string(mergedData)) - r.Output.rlItems = append(r.Output.rlItems, rNode) + r.Output.items[fileKey] = string(mergedData) + } +} + +func (r *mergerResource) export() []*yaml.RNode { + rlItems := []*yaml.RNode{} + switch r.Output.Format { + case Raw: + for _, value := range r.Output.items { + rlItems = append(rlItems, yaml.MustParse(value)) + } + + case ConfigMap: + rNode, _ := yaml.FromMap(map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]string{ + "name": r.Name, + }, + }) + if err := rNode.LoadMapIntoConfigMapData(r.Output.items); err != nil { + log.Fatalf("Error creating ConfigMap data: %v", err) + } + rlItems = append(rlItems, rNode) + + case Secret: + rNode, _ := yaml.FromMap(map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]string{ + "name": r.Name, + }, + }) + if err := rNode.LoadMapIntoSecretData(r.Output.items); err != nil { + log.Fatalf("Error creating Secret data: %v", err) + } + rlItems = append(rlItems, rNode) } + + return rlItems } diff --git a/pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml b/pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml index 0de139d..07c4ac7 100644 --- a/pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml +++ b/pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml @@ -82,6 +82,8 @@ spec: format: enum: - raw + - configmap + - secret type: string required: - format diff --git a/pkg/merger/types.go b/pkg/merger/types.go index 67cbcfd..b6baa3a 100644 --- a/pkg/merger/types.go +++ b/pkg/merger/types.go @@ -10,7 +10,6 @@ package merger import ( "dario.cat/mergo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/kustomize/kyaml/yaml" ) // Merger manifest configuration. @@ -68,7 +67,6 @@ const ( type resourceInput struct { Method resourceInputMethod `yaml:"method" json:"method"` Files resourceInputFiles `yaml:"files" json:"files"` - items []resourceInputFiles } // @@ -80,8 +78,8 @@ type resourceInput struct { type resourceMergeStrategy string // Merger resource merge strategy available options. -// TODO: Support combine lists by named key. const ( + // TODO: Support combine lists by named key. Append resourceMergeStrategy = "append" Combine resourceMergeStrategy = "combine" Replace resourceMergeStrategy = "replace" @@ -97,15 +95,17 @@ type resourceMerge struct { // // +enum -// +kubebuilder:validation:Enum=raw +// +kubebuilder:validation:Enum=raw;configmap;secret type resourceOutputFormat string -// TODO: Support ConfigMap and Secret. +// Merger resource output available options. const ( - Raw resourceOutputFormat = "raw" + Raw resourceOutputFormat = "raw" + ConfigMap resourceOutputFormat = "configmap" + Secret resourceOutputFormat = "secret" ) type resourceOutput struct { - Format resourceOutputFormat `yaml:"format" json:"format"` - rlItems []*yaml.RNode + Format resourceOutputFormat `yaml:"format" json:"format"` + items map[string]string }