feat: support go templates in config sources

fixes #37

This allows the config sources, like ConfigMaps to be `go` templates.
All the functions from sprig lib are supported,
there's also a k8sLookup function to get values from k8s objects,
and you can access environment variables from the template under .Env
object.

config-reloader/datasource/fake.go

@@ -11,7 +11,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-var template = `
+var logzTemplate = `
 <match **>
   @type logzio_buffered
   endpoint_url https://listener.logz.io:8071?token=secret
@@ -36,7 +36,7 @@ func NewFakeDatasource(ctx context.Context) Datasource {
 }
 
 func makeFakeConfig(namespace string) string {
-	contents := template
+	contents := logzTemplate
 	contents = strings.ReplaceAll(contents, "$ns$", namespace)
 	contents = strings.ReplaceAll(contents, "$ts$", time.Now().String())
 

config-reloader/datasource/kube_informer.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/vmware/kube-fluentd-operator/config-reloader/fluentd"
+	"github.com/vmware/kube-fluentd-operator/config-reloader/template"
 	"github.com/vmware/kube-fluentd-operator/config-reloader/util"
 
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -107,7 +108,7 @@ func NewKubernetesInformerDatasource(ctx context.Context, cfg *config.Config, up
 		factory.Core().V1().Pods().Informer().HasSynced,
 		factory.Core().V1().ConfigMaps().Informer().HasSynced,
 		kubeds.IsReady) {
-		return nil, fmt.Errorf("Failed to sync local informer with upstream Kubernetes API")
+		return nil, fmt.Errorf("failed to sync local informer with upstream Kubernetes API")
 	}
 	logrus.Infof("Synced local informer with upstream Kubernetes API")
 
@@ -301,7 +302,7 @@ func (d *kubeInformerConnection) discoverNamespaces(ctx context.Context) ([]stri
 			// Find the configmaps that exist on this cluster to find namespaces:
 			confMapsList, err := d.cmlist.List(labels.NewSelector())
 			if err != nil {
-				return nil, fmt.Errorf("Failed to list all configmaps in cluster: %v", err)
+				return nil, fmt.Errorf("failed to list all configmaps in cluster: %v", err)
 			}
 			// If default configmap name is defined get all namespaces for those configmaps:
 			if d.cfg.DefaultConfigmapName != "" {
@@ -328,11 +329,11 @@ func (d *kubeInformerConnection) discoverNamespaces(ctx context.Context) ([]stri
 				// get all namespaces and iterrate through them like before:
 				nses, err := d.nslist.List(labels.NewSelector())
 				if err != nil {
-					return nil, fmt.Errorf("Failed to list all namespaces in cluster: %v", err)
+					return nil, fmt.Errorf("failed to list all namespaces in cluster: %v", err)
 				}
 				namespaces = make([]string, 0)
 				for _, ns := range nses {
-					namespaces = append(namespaces, ns.ObjectMeta.Name)
+					namespaces = append(namespaces, ns.Name)
 				}
 			}
 		}
@@ -355,6 +356,10 @@ func (d *kubeInformerConnection) handlePodChange(ctx context.Context, obj interf
 	mObj := obj.(*core.Pod)
 	logrus.Tracef("Detected pod change %s in namespace: %s", mObj.GetName(), mObj.GetNamespace())
 	configdata, err := d.kubeds.GetFluentdConfig(ctx, mObj.GetNamespace())
+	buf := new(strings.Builder)
+	if err := template.Render(buf, configdata, nil); err == nil {
+		configdata = buf.String()
+	}
 	nsConfigStr := fmt.Sprintf("%#v", configdata)
 
 	if err == nil {
@@ -386,11 +391,11 @@ func matchAny(contLabels map[string]string, mountedLabelsInNs []map[string]strin
 
 func (d *kubeInformerConnection) discoverFluentdConfigNamespaces() ([]string, error) {
 	if d.fdlist == nil {
-		return nil, fmt.Errorf("Failed to initialize the fluentdconfig crd client, d.fclient = nil")
+		return nil, fmt.Errorf("failed to initialize the fluentdconfig crd client, d.fclient = nil")
 	}
 	fcList, err := d.fdlist.List(labels.NewSelector())
 	if err != nil {
-		return nil, fmt.Errorf("Failed to list all fluentdconfig crds in cluster: %v", err)
+		return nil, fmt.Errorf("failed to list all fluentdconfig crds in cluster: %v", err)
 	}
 	nsList := make([]string, 0)
 	for _, crd := range fcList {

config-reloader/datasource/kubedatasource/configmap.go

@@ -97,7 +97,7 @@ func (c *ConfigMapDS) fetchConfigMaps(ctx context.Context, ns string) ([]*core.C
 		// Get all configmaps which match a specified label, but only if we have a selector
 		mapslist, err := nsmaps.List(c.cfg.ParsedLabelSelector.AsSelector())
 		if err != nil {
-			return nil, fmt.Errorf("Failed to list configmaps in namespace '%s': %v", ns, err)
+			return nil, fmt.Errorf("failed to list configmaps in namespace '%s': %v", ns, err)
 		}
 		confMapByName := make(map[string]*core.ConfigMap)
 		sortedConfMaps := make([]string, 0, len(mapslist))

config-reloader/fluentd/validator.go

@@ -7,7 +7,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 	"time"
@@ -54,7 +53,7 @@ func (v *validatorState) ValidateConfigExtremely(config string, namespace string
 		return nil
 	}
 
-	tmpfile, err := ioutil.TempFile("", "validate-ext-"+namespace)
+	tmpfile, err := os.CreateTemp("", "validate-ext-"+namespace)
 	if err != nil {
 		logrus.Errorf("error creating temporary file for namespace %s: %s", namespace, err.Error())
 		return err
@@ -98,7 +97,7 @@ func (v *validatorState) ValidateConfig(config string, namespace string) error {
 		return nil
 	}
 
-	tmpfile, err := ioutil.TempFile("", "validate-"+namespace)
+	tmpfile, err := os.CreateTemp("", "validate-"+namespace)
 	if err != nil {
 		logrus.Errorf("error creating temporary file for namespace %s: %s", namespace, err.Error())
 		return err

config-reloader/generator/generator.go

@@ -10,14 +10,14 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"text/template"
 	"time"
 
 	"github.com/vmware/kube-fluentd-operator/config-reloader/config"
 	"github.com/vmware/kube-fluentd-operator/config-reloader/datasource"
 	"github.com/vmware/kube-fluentd-operator/config-reloader/fluentd"
 	"github.com/vmware/kube-fluentd-operator/config-reloader/metrics"
 	"github.com/vmware/kube-fluentd-operator/config-reloader/processors"
+	"github.com/vmware/kube-fluentd-operator/config-reloader/template"
 	"github.com/vmware/kube-fluentd-operator/config-reloader/util"
 
 	"github.com/sirupsen/logrus"

config-reloader/go.mod

@@ -3,6 +3,7 @@ module github.com/vmware/kube-fluentd-operator/config-reloader
 go 1.19
 
 require (
+	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alecthomas/kingpin v2.2.6+incompatible
 	github.com/prometheus/client_golang v1.15.1
 	github.com/sirupsen/logrus v1.9.0
@@ -11,16 +12,21 @@ require (
 	k8s.io/apiextensions-apiserver v0.27.0
 	k8s.io/apimachinery v0.27.0
 	k8s.io/client-go v0.27.0
+	sigs.k8s.io/controller-runtime v0.14.2
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.1 // indirect
@@ -31,11 +37,14 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/huandu/xstrings v1.3.3 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mitchellh/copystructure v1.0.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -44,13 +53,16 @@ require (
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -61,5 +73,4 @@ require (
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )