feat: add config package

.github/workflows/mod-update.yaml

@@ -0,0 +1,32 @@
+on:
+  - pull_request
+
+name: Update go.mod
+
+jobs:
+  update_go_mod:
+    runs-on: [ARM64, self-hosted, Linux]
+    outputs:
+      updated_go_mod: ${{ steps.commit_go_mod.outputs.committed }}
+    steps:
+      - name: Generate token
+        id: generate_token
+        uses: chanzuckerberg/[email protected]
+        with:
+          app_id: ${{ secrets.CZI_GITHUB_HELPER_APP_ID }}
+          private_key: ${{ secrets.CZI_GITHUB_HELPER_PK }}
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.generate_token.outputs.token }}
+          ref: ${{ github.event.pull_request.head.ref }}
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Update go.mod
+        run: go mod tidy
+      - uses: EndBug/add-and-commit@v9
+        id: commit_go_mod
+        with:
+          add: -A
+          message: ci - update go.mod

.github/workflows/release-please.yml

@@ -0,0 +1,28 @@
+on:
+    push:
+      branches:
+        - main
+
+name: release-please
+jobs:
+    release-please:
+      runs-on: [ARM64, self-hosted, Linux]
+      steps:
+        # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow
+        # For why we need to generate a token and not use the default
+        - name: Generate token
+          id: generate_token
+          uses: chanzuckerberg/[email protected]
+          with:
+            app_id: ${{ secrets.CZI_RELEASE_PLEASE_APP_ID }}
+            private_key: ${{ secrets.CZI_RELEASE_PLEASE_PK }}
+
+        - name: release please
+          uses: google-github-actions/release-please-action@v3
+          id: release
+          with:
+            release-type: simple
+            command: manifest
+            bump-minor-pre-major: true
+            token: ${{ steps.generate_token.outputs.token }}
+            monorepo-tags: true

.release-please-manifest.json

@@ -0,0 +1 @@
+{".":"1.11.1"}

README.md

@@ -12,6 +12,9 @@ This is a collection of Go libraries and projects used by other projects within
 ### aws
 An AWS client that aims to standardize the way we mock and write aws tests
 
+### config
+A utility for loading configuration from environment variables and files. See the [config](config/README.md) package for more information.
+
 ### lambda
 A collection of lambda functions
 

config/README.md

@@ -0,0 +1,104 @@
+# config
+
+A utility for loading configuration YAML files.
+
+## Design
+
+The `config` package provides a way to load configuration from YAML files. By default it will load a file named `app-config.yaml` in the root directory where your main package resides. Then it will load a file named `app-config.<env>.yaml` and use those values to override the previous configuration. The environment is determined by the value of the `APP_ENV` environment variable. If `APP_ENV` is not set, it will fall back to the value of the `DEPLOYEMENT_STAGE` environment variable. If neither are set, no override file will be loaded. The `config` package also supports loading configuration from a custom directory.
+
+### Custom Configuration Directory
+
+There are two ways to specify a custom directory to load configuration files from. The first is to set the `CONFIG_YAML_DIRECTORY` environment variable. The second is to use the `WithConfigYamlDir` option when calling `LoadConfiguration`.
+
+### Configuration Structs
+
+The `config` package uses the `mapstructure` package to load configuration into a struct. This allows for the use of tags to map configuration fields to struct fields. For example, the following YAML file:
+```yaml
+auth:
+  enable: true
+```
+can be loaded into the following struct:
+```go
+type AuthConfiguration struct {
+  Enable    *bool  `mapstructure:"enable"`
+}
+
+type Configuration struct {
+  Auth     AuthConfiguration     `mapstructure:"auth"`
+}
+```
+
+### Environment Variables
+
+The `config` package supports templated injection of environment variables in your YAML files to avoid storing sensitive values in the YAMLs. For example, the following YAML file:
+```yaml
+database:
+  driver: postgres
+  data_source_name: host={{.PLATFORM_DATABASE_HOST}} user={{.PLATFORM_DATABASE_USER}} password={{.PLATFORM_DATABASE_PASSWORD}} port={{.PLATFORM_DATABASE_PORT}} dbname={{.PLATFORM_DATABASE_NAME}}
+```
+will have the environment variables `PLATFORM_DATABASE_HOST`, `PLATFORM_DATABASE_USER`, `PLATFORM_DATABASE_PASSWORD`, `PLATFORM_DATABASE_PORT`, and `PLATFORM_DATABASE_NAME` injected into the `data_source_name` field prior to loading into your configuration struct.
+
+## Usage
+
+Example:
+```go
+package main
+
+import (
+  "fmt"
+
+  "github.com/chanzuckerberg/go-misc/config"
+)
+
+type AuthConfiguration struct {
+	Enable    *bool  `mapstructure:"enable"`
+}
+
+type ApiConfiguration struct {
+	Port                uint   `mapstructure:"port"`
+	LogLevel            string `mapstructure:"log_level"`
+}
+
+type DBDriver string
+
+func (d *DBDriver) String() string {
+	return string(*d)
+}
+
+const (
+	Sqlite   DBDriver = "sqlite"
+	Postgres DBDriver = "postgres"
+)
+
+type DatabaseConfiguration struct {
+	Driver         DBDriver `mapstructure:"driver"`
+	DataSourceName string   `mapstructure:"data_source_name"`
+}
+
+type Configuration struct {
+	Auth     AuthConfiguration     `mapstructure:"auth"`
+	Api      ApiConfiguration      `mapstructure:"api"`
+	Database DatabaseConfiguration `mapstructure:"database"`
+}
+
+func main() {
+  cfg := &Configuration{}
+
+  configOpts := []config.ConfigOption[Configuration]{
+    config.WithConfigYamlDir[Configuration]("./configs"),
+    config.WithConfigEditorFn(func(cfg *Configuration) error {
+      // default to having auth enabled
+      if cfg.Auth.Enable == nil {
+        enable := true
+        cfg.Auth.Enable = &enable
+      }
+      return nil
+    }),
+  }
+
+  err := config.LoadConfiguration(cfg, configOpts...)
+  if err != nil {
+    panic(fmt.Sprintf("Failed to load app configuration: %s", err.Error()))
+  }
+}
+```

config/config.go

@@ -0,0 +1,195 @@
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+)
+
+const defaultConfigYamlDir = "./"
+
+// ConfigEditorFn is a function that can be used to modify the configuration values after they have been loaded
+type ConfigEditorFn[T any] func(cfg *T) error
+
+type ConfigLoader[T any] struct {
+	ConfigEditors []ConfigEditorFn[T]
+	ConfigYamlDir string
+}
+
+// ConfigOption is a function that can be used to modify the configuration loader
+type ConfigOption[T any] func(*ConfigLoader[T]) error
+
+// WithConfigEditorFn allows setting up a callback function, which will be
+// called right after loading the configs. This can be used to mutate the config,
+// for example to set default values where none were set by the call to LoadConfiguration.
+func WithConfigEditorFn[T any](fn ConfigEditorFn[T]) ConfigOption[T] {
+	return func(c *ConfigLoader[T]) error {
+		c.ConfigEditors = append(c.ConfigEditors, fn)
+		return nil
+	}
+}
+
+func WithConfigYamlDir[T any](dir string) ConfigOption[T] {
+	return func(c *ConfigLoader[T]) error {
+		c.ConfigYamlDir = dir
+		return nil
+	}
+}
+
+// LoadConfiguration loads the configuration from the app-config.yaml and app-config.<env>.yaml files
+func LoadConfiguration[T any](cfg *T, opts ...ConfigOption[T]) error {
+	configYamlDir := defaultConfigYamlDir
+	if len(os.Getenv("CONFIG_YAML_DIRECTORY")) > 0 {
+		fmt.Println("CONFIG_YAML_DIRECTORY", os.Getenv("CONFIG_YAML_DIRECTORY"))
+		configYamlDir = os.Getenv("CONFIG_YAML_DIRECTORY")
+	}
+
+	loader := &ConfigLoader[T]{
+		ConfigEditors: []ConfigEditorFn[T]{},
+		ConfigYamlDir: configYamlDir,
+	}
+
+	for _, opt := range opts {
+		err := opt(loader)
+		if err != nil {
+			return fmt.Errorf("ConfigOption failed: %w", err)
+		}
+	}
+
+	loader.populateConfiguration(cfg)
+
+	for _, fn := range loader.ConfigEditors {
+		err := fn(cfg)
+		if err != nil {
+			return fmt.Errorf("ConfigEditorFn failed: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (c *ConfigLoader[T]) populateConfiguration(cfg *T) error {
+	configYamlDir := c.ConfigYamlDir
+	path, err := filepath.Abs(configYamlDir)
+	if err != nil {
+		return fmt.Errorf("failed to get absolute path of %s: %w", configYamlDir, err)
+	}
+
+	vpr := viper.New()
+	appConfigFile := filepath.Join(path, "app-config.yaml")
+	if _, err := os.Stat(appConfigFile); err == nil {
+		tmp, err := evaluateConfigWithEnvToTmp(appConfigFile)
+		if len(tmp) != 0 {
+			defer os.Remove(tmp)
+		}
+		if err != nil {
+			return err
+		}
+
+		vpr.SetConfigFile(tmp)
+		err = vpr.ReadInConfig()
+		if err != nil {
+			return fmt.Errorf("failed to read config file: %w", err)
+		}
+	}
+
+	envConfigFilename := fmt.Sprintf("app-config.%s.yaml", getAppEnv())
+	appEnvConfigFile := filepath.Join(path, envConfigFilename)
+	if _, err := os.Stat(appEnvConfigFile); err == nil {
+		tmp, err := evaluateConfigWithEnvToTmp(appEnvConfigFile)
+		if len(tmp) != 0 {
+			defer os.Remove(tmp)
+		}
+		if err != nil {
+			return err
+		}
+
+		vpr.SetConfigFile(tmp)
+		err = vpr.MergeInConfig()
+		if err != nil {
+			return fmt.Errorf("failed to merge env config: %w", err)
+		}
+	}
+
+	err = vpr.Unmarshal(cfg, viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc()))
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal configuration: %w", err)
+	}
+
+	return nil
+}
+
+func evaluateConfigWithEnvToTmp(configPath string) (string, error) {
+	tmp, err := os.CreateTemp("./", "*.yaml")
+	if err != nil {
+		return "", fmt.Errorf("unable to create a temp config file: %w", err)
+	}
+
+	cfile, err := os.Open(configPath)
+	if err != nil {
+		return "", fmt.Errorf("unable to open %s: %w", configPath, err)
+	}
+
+	_, err = evaluateConfigWithEnv(cfile, tmp)
+	if err != nil {
+		return "", fmt.Errorf("unable to populate the environment: %w", err)
+	}
+
+	return tmp.Name(), nil
+}
+
+func envToMap() map[string]string {
+	envMap := make(map[string]string)
+	for _, v := range os.Environ() {
+		s := strings.SplitN(v, "=", 2)
+		if len(s) != 2 {
+			continue
+		}
+		envMap[s[0]] = s[1]
+	}
+	return envMap
+}
+
+// evaluateConfigWithEnv reads a configuration reader and injects environment variables
+// that exist as part of the configuration in the form a go template. For example
+// {{.ENV_VAR1}} will be replace with the value of the environment variable ENV_VAR1.
+// Optional support for writting the contents to other places is supported by providing
+// other writers. By default, the evaluated configuartion is returned as a reader.
+func evaluateConfigWithEnv(configFile io.Reader, writers ...io.Writer) (io.Reader, error) {
+	envMap := envToMap()
+
+	b, err := io.ReadAll(configFile)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read the config file: %w", err)
+	}
+
+	t := template.New("appConfigTemplate")
+	tmpl, err := t.Parse(string(b))
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse template from: \n%s: %w", string(b), err)
+	}
+
+	populated := []byte{}
+	buff := bytes.NewBuffer(populated)
+	writers = append(writers, buff)
+	err = tmpl.Execute(io.MultiWriter(writers...), envMap)
+	if err != nil {
+		return nil, fmt.Errorf("unable to execute template: %w", err)
+	}
+	return buff, nil
+}
+
+func getAppEnv() string {
+	env := os.Getenv("APP_ENV")
+	if env == "" {
+		env = os.Getenv("DEPLOYMENT_STAGE")
+	}
+	return env
+}