Add support for reading Auth data from a file on disk

cmd/reloader/main.go

@@ -7,12 +7,26 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 
-	"github.com/nats-io/nats-operator/pkg/reloader"
+	natsreloader "github.com/nats-io/nats-operator/pkg/reloader"
 	"github.com/nats-io/nats-operator/version"
 )
 
+// StringSet is a wrapper for []string to allow using it with the flags package.
+type StringSet []string
+
+func (s *StringSet) String() string {
+	return strings.Join([]string(*s), ", ")
+}
+
+// Set appends the value provided to the list of strings.
+func (s *StringSet) Set(val string) error {
+	*s = append(*s, val)
+	return nil
+}
+
 func main() {
 	fs := flag.NewFlagSet("nats-server-config-reloader", flag.ExitOnError)
 	flag.Usage = func() {
@@ -25,22 +39,29 @@ func main() {
 	var (
 		showHelp    bool
 		showVersion bool
+		fileSet     StringSet
 	)
+
+	nconfig := &natsreloader.Config{}
 	fs.BoolVar(&showHelp, "h", false, "Show help")
 	fs.BoolVar(&showHelp, "help", false, "Show help")
 	fs.BoolVar(&showVersion, "v", false, "Show version")
 	fs.BoolVar(&showVersion, "version", false, "Show version")
 
-	nconfig := &natsreloader.Config{}
 	fs.StringVar(&nconfig.PidFile, "P", "/var/run/nats/gnatsd.pid", "NATS Server Pid File")
 	fs.StringVar(&nconfig.PidFile, "pid", "/var/run/nats/gnatsd.pid", "NATS Server Pid File")
-	fs.StringVar(&nconfig.ConfigFile, "c", "/etc/nats/gnatsd.conf", "NATS Server Config File")
-	fs.StringVar(&nconfig.ConfigFile, "config", "/etc/nats/gnatsd.conf", "NATS Server Config File")
+	fs.Var(&fileSet, "c", "NATS Server Config File (may be repeated to specify more than one)")
+	fs.Var(&fileSet, "config", "NATS Server Config File (may be repeated to specify more than one)")
 	fs.IntVar(&nconfig.MaxRetries, "max-retries", 5, "Max attempts to trigger reload")
 	fs.IntVar(&nconfig.RetryWaitSecs, "retry-wait-secs", 2, "Time to back off when reloading fails before retrying")
 
 	fs.Parse(os.Args[1:])
 
+	nconfig.ConfigFiles = fileSet
+	if len(fileSet) == 0 {
+		nconfig.ConfigFiles = []string{"/etc/nats-config/gnatsd.conf"}
+	}
+
 	switch {
 	case showHelp:
 		flag.Usage()

example/example-nats-cluster-authfile.yaml

@@ -0,0 +1,35 @@
+# This is an example NatsCluster manifest which uses a 3rd party initContainer
+# to fetch the authorization credentials from outside kubernetes.
+#
+# An example of this could be consul-template getting user/passwords from vault.
+apiVersion: "nats.io/v1alpha2"
+kind: "NatsCluster"
+metadata:
+  name: my-nats-cluster
+  namespace: nats-stuff
+spec:
+  size: 5
+  version: "1.3.0"
+  service:
+    maxPayload: 20971520
+  pod:
+    volumeMounts:
+      - name: authconfig
+        mountPath: /etc/nats-config/authconfig
+  template:
+    serviceAccountName: my-secrets-getting-sa
+    initContainers:
+      - name: secret-getter
+        image: "busybox"
+        command: ["sh", "-c", "echo 'users = [ { user: 'foo', pass: 'bar' } ]' > /etc/nats-config/authconfig/auth.json"]
+        volumeMounts:
+          - name: authconfig
+            mountPath: /etc/nats-config/authconfig
+    volumes:
+      - name: authconfig
+        emptyDir: {}
+  auth:
+    # Needs to be under /etc/nats-config where nats looks
+    # for its config file, or it won't be able to be included
+    # by /etc/nats-config/gnatsd.conf
+    clientAuthFile: "authconfig/auth.json"

pkg/conf/natsconf.go

@@ -52,6 +52,7 @@ type AuthorizationConfig struct {
 	Timeout            int          `json:"timeout,omitempty"`
 	Users              []*User      `json:"users,omitempty"`
 	DefaultPermissions *Permissions `json:"default_permissions,omitempty"`
+	Include            string       `json:"include,omitempty"`
 }
 
 type User struct {

pkg/reloader/reloader.go

@@ -9,7 +9,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path"
+	"path/filepath"
 	"strconv"
 	"syscall"
 	"time"
@@ -20,7 +20,7 @@ import (
 // Config represents the configuration of the reloader.
 type Config struct {
 	PidFile       string
-	ConfigFile    string
+	ConfigFiles   []string
 	MaxRetries    int
 	RetryWaitSecs int
 }
@@ -39,10 +39,6 @@ type Reloader struct {
 
 	// quit shutsdown the reloader.
 	quit func()
-
-	// lastAppliedVersion is the last config update
-	// done by the proces..
-	lastAppliedVersion []byte
 }
 
 // Run starts the main loop.
@@ -94,41 +90,80 @@ func (r *Reloader) Run(ctx context.Context) error {
 	// Follow configuration updates in the directory where
 	// the config file is located and trigger reload when
 	// it is either recreated or written into.
-	if err := configWatcher.Add(path.Dir(r.ConfigFile)); err != nil {
-		return err
+	for i := range r.ConfigFiles {
+		// Ensure our paths are canonical
+		r.ConfigFiles[i], _ = filepath.Abs(r.ConfigFiles[i])
+		// Use directory here because k8s remounts the entire folder
+		// the config file lives in. So, watch the folder so we properly receive events.
+		if err := configWatcher.Add(filepath.Dir(r.ConfigFiles[i])); err != nil {
+			return err
+		}
 	}
 
 	attempts = 0
+	// lastConfigAppliedCache is the last config update
+	// applied by us
+	lastConfigAppliedCache := make(map[string][]byte)
+
+	// Preload config hashes, so we know their digests
+	// up front and avoid potentially reloading when unnecessary.
+	for _, configFile := range r.ConfigFiles {
+		h := sha256.New()
+		f, err := os.Open(configFile)
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(h, f); err != nil {
+			return err
+		}
+		digest := h.Sum(nil)
+		lastConfigAppliedCache[configFile] = digest
+	}
+
+WaitForEvent:
 	for {
 		select {
 		case <-ctx.Done():
 			return nil
 		case event := <-configWatcher.Events:
 			log.Printf("Event: %+v \n", event)
-			// FIXME: This captures all events in the same folder, should
-			// narrow down to updates to the config file involved only.
-			if event.Op != fsnotify.Write && event.Op != fsnotify.Create {
-				continue
-			}
-
-			h := sha256.New()
-			f, err := os.Open(r.ConfigFile)
+			touchedInfo, err := os.Stat(event.Name)
 			if err != nil {
-				log.Printf("Error: %s\n", err)
-				continue
-			}
-			if _, err := io.Copy(h, f); err != nil {
-				log.Printf("Error: %s\n", err)
 				continue
 			}
-			digest := h.Sum(nil)
-			if r.lastAppliedVersion != nil {
-				if bytes.Equal(r.lastAppliedVersion, digest) {
-					// Skip since no meaningful change
+
+			for _, configFile := range r.ConfigFiles {
+				configInfo, err := os.Stat(configFile)
+				if err != nil {
+					log.Printf("Error: %s\n", err)
+					continue WaitForEvent
+				}
+				if !os.SameFile(touchedInfo, configInfo) {
 					continue
 				}
+
+				h := sha256.New()
+				f, err := os.Open(configFile)
+				if err != nil {
+					log.Printf("Error: %s\n", err)
+					continue WaitForEvent
+				}
+				if _, err := io.Copy(h, f); err != nil {
+					log.Printf("Error: %s\n", err)
+					continue WaitForEvent
+				}
+				digest := h.Sum(nil)
+				lastConfigHash, ok := lastConfigAppliedCache[configFile]
+				if ok && bytes.Equal(lastConfigHash, digest) {
+					// No meaningful change or this is the first time we've checked
+					continue WaitForEvent
+				}
+				lastConfigAppliedCache[configFile] = digest
+
+				// We only get an event for one file at a time, we can stop checking
+				// config files here and continue with our business.
+				break
 			}
-			r.lastAppliedVersion = digest
 
 		case err := <-configWatcher.Errors:
 			log.Printf("Error: %s\n", err)

pkg/util/kubernetes/kubernetes.go

@@ -321,6 +321,11 @@ func addAuthConfig(
 			break
 		}
 		return nil
+	} else if cs.Auth.ClientsAuthFile != "" {
+		sconfig.Authorization = &natsconf.AuthorizationConfig{
+			Include: cs.Auth.ClientsAuthFile,
+		}
+		return nil
 	}
 	return nil
 }
@@ -427,9 +432,7 @@ func CreateConfigSecret(kubecli corev1client.CoreV1Interface, operatorcli natsal
 
 	// FIXME: Quoted "include" causes include to be ignored.
 	// Remove once using NATS v2.0 as the default container image.
-	if cluster.Pod != nil && cluster.Pod.AdvertiseExternalIP {
-		rawConfig = bytes.Replace(rawConfig, []byte(`"include":`), []byte("include "), 1)
-	}
+	rawConfig = bytes.Replace(rawConfig, []byte(`"include":`), []byte("include "), -1)
 
 	labels := LabelsForCluster(clusterName)
 	cm := &v1.Secret{
@@ -840,7 +843,12 @@ func NewNatsPodSpec(namespace, name, clusterName string, cs v1alpha2.ClusterSpec
 			imagePullPolicy = cs.Pod.ReloaderImagePullPolicy
 		}
 
-		reloaderContainer := natsPodReloaderContainer(image, imageTag, imagePullPolicy)
+		authFilePath := ""
+		if cs.Auth != nil {
+			authFilePath = cs.Auth.ClientsAuthFile
+		}
+
+		reloaderContainer := natsPodReloaderContainer(image, imageTag, imagePullPolicy, authFilePath)
 		reloaderContainer.VolumeMounts = volumeMounts
 		containers = append(containers, reloaderContainer)
 	}

pkg/util/kubernetes/pod.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	appsv1 "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -80,8 +80,8 @@ func natsPodContainer(clusterName, version string, serverImage string, enableCli
 }
 
 // natsPodReloaderContainer returns a NATS server pod container spec for configuration reloader.
-func natsPodReloaderContainer(image, tag, pullPolicy string) v1.Container {
-	return v1.Container{
+func natsPodReloaderContainer(image, tag, pullPolicy, authFilePath string) v1.Container {
+	container := v1.Container{
 		Name:            "reloader",
 		Image:           fmt.Sprintf("%s:%s", image, tag),
 		ImagePullPolicy: v1.PullPolicy(pullPolicy),
@@ -93,6 +93,10 @@ func natsPodReloaderContainer(image, tag, pullPolicy string) v1.Container {
 			constants.PidFilePath,
 		},
 	}
+	if authFilePath != "" {
+		container.Command = append(container.Command, "-config", authFilePath)
+	}
+	return container
 }
 
 // natsPodMetricsContainer returns a NATS server pod container spec for prometheus metrics exporter.