Add support for Hashicorp Vault

Vault is supported for the following:
As a well-known filesystem for TLS cert, TLS key and SSH signing key.
For configuration secrets for cookie_secret, csrf_secret, oauth_client_id and oauth_client_secret options.

.travis.yml

@@ -4,10 +4,9 @@ services:
   - mongodb
 
 env:
-  - GO15VENDOREXPERIMENT=1 MYSQL_TEST_CONFIG="mysql:user:passwd:localhost" MONGO_TEST_CONFIG="mongo:user:passwd:localhost"
+  - MYSQL_TEST_CONFIG="mysql:user:passwd:localhost" MONGO_TEST_CONFIG="mongo:user:passwd:localhost"
 
 go:
-  - 1.6.3
   - 1.7
   - tip
 

README.md

@@ -15,6 +15,7 @@
 		- [Provider-specific options](#provider-specific-options)
 	- [ssh](#ssh)
 	- [aws](#aws)
+	- [vault](#vault)
 - [Usage](#usage)
 	- [Using cashier](#using-cashier)
 	- [Configuring SSH](#configuring-ssh)
@@ -79,25 +80,34 @@ docker run -it --rm -p 10000:10000 --name cashier -v $(pwd):/cashier nsheridan/c
 
 # Requirements
 ## Server
-Go 1.6 or later. May work with earlier versions but not tested.
+Go 1.7 or later. May work with earlier versions.
 
 ## Client
-OpenSSH 5.6 or newer.  
-A working SSH agent.  
-I have only tested this on Linux & OSX.  
+- OpenSSH 5.6 or newer.
+- A working SSH agent.
+
+Note: I have only tested this on Linux & OSX.
 
 # Configuration
 Configuration is divided into different sections: `server`, `auth`, `ssh`, and `aws`.
 
+## A note on files:
+For any option that takes a file path as a parameter (e.g. SSH signing key, TLS key, TLS cert), the path can be one of:
+
+- A relative or absolute filesystem path e.g. `/data/ssh_signing_key`, `tls/server.key`.
+- An AWS S3 bucket + object path starting with `/s3/` e.g. `/s3/my-bucket/ssh_signing_key`. You should add an [aws](#aws) config as needed.
+- A Google GCS bucket + object path starting with `/gcs/` e.g. `/gcs/my-bucket/ssh_signing_key`.
+- A [Vault](https://www.vaultproject.io) path + key starting with `/vault/` e.g. `/vault/secret/cashier/ssh_signing_key`. You should add a [vault](#vault) config as needed.
+
 ## server
 - `use_tls` : boolean. If this is set then `tls_key` and `tls_cert` are required.
-- `tls_key` : string. Path to the TLS key.
-- `tls_cert` : string. Path to the TLS cert.
+- `tls_key` : string. Path to the TLS key. See the [note](#a-note-on-files) on files above.
+- `tls_cert` : string. Path to the TLS cert. See the [note](#a-note-on-files) on files above.
 - `address` : string. IP address to listen on. If unset the server listens on all addresses.
 - `port` : int. Port to listen on.
 - `user` : string. User to which the server drops privileges to.
-- `cookie_secret`: string. Authentication key for the session cookie.
-- `csrf_secret`: string. Authentication key for CSRF protection.
+- `cookie_secret`: string. Authentication key for the session cookie. This can be a secret stored in a [vault](https://www.vaultproject.io/) using the form `/vault/path/key` e.g. `/vault/secret/cashier/cookie_secret`.
+- `csrf_secret`: string. Authentication key for CSRF protection. This can be a secret stored in a [vault](https://www.vaultproject.io/) using the form `/vault/path/key` e.g. `/vault/secret/cashier/csrf_secret`.
 - `http_logfile`: string. Path to the HTTP request log. Logs are written in the [Common Log Format](https://en.wikipedia.org/wiki/Common_Log_Format). If not set logs are written to stderr.
 - `datastore`: string. Datastore connection string. See [Datastore](#datastore).
 
@@ -131,8 +141,8 @@ Note that dbinit has no support for replica sets.
 
 ## auth
 - `provider` : string. Name of the oauth provider. Valid providers are currently "google" and "github".
-- `oauth_client_id` : string. Oauth Client ID.
-- `oauth_client_secret` : string. Oauth secret.
+- `oauth_client_id` : string. Oauth Client ID. This can be a secret stored in a [vault](https://www.vaultproject.io/) using the form `/vault/path/key` e.g. `/vault/secret/cashier/oauth_client_id`.
+- `oauth_client_secret` : string. Oauth secret. This can be a secret stored in a [vault](https://www.vaultproject.io/) using the form `/vault/path/key` e.g. `/vault/secret/cashier/oauth_client_secret`.
 - `oauth_callback_url` : string. URL that the Oauth provider will redirect to after user authorisation. The path is hardcoded to `"/auth/callback"` in the source.
 - `provider_opts` : object. Additional options for the provider.
 - `users_whitelist` : array of strings. Optional list of whitelisted usernames. If missing, all users of your current domain/organization are allowed to authenticate against cashierd. For Google auth a user is an email address. For GitHub auth a user is a GitHub username.
@@ -153,27 +163,34 @@ auth {
 }
 ```
 
+Supported options:
+
+
 | Provider |       Option | Notes                                                                                                                                  |
 |---------:|-------------:|----------------------------------------------------------------------------------------------------------------------------------------|
 | Google   |       domain | If this is unset then you must whitelist individual email addresses using `users_whitelist`.                                           |
 | Github   | organization | If this is unset then you must whitelist individual users using `users_whitelist`. The oauth client and secrets should be issued by the specified organization. |
 
-Supported options:
-
 ## ssh
-- `signing_key`: string. Path to the signing ssh private key you created earlier. This can be a S3 or GCS path using `/s3/<bucket>/<path/to/key>` or `/gcs/<bucket>/<path/to/key>` as appropriate. For S3 you should add an [aws](#aws) config as needed.
+- `signing_key`: string. Path to the signing ssh private key you created earlier. See the [note](#a-note-on-files) on files above.
 - `additional_principals`: array of string. By default certificates will have one principal set - the username portion of the requester's email address. If `additional_principals` is set, these will be added to the certificate e.g. if your production machines use shared user accounts.
 - `max_age`: string. If set the server will not issue certificates with an expiration value longer than this, regardless of what the client requests. Must be a valid Go [`time.Duration`](https://golang.org/pkg/time/#ParseDuration) string.
 - `permissions`: array of string. Actions the certificate can perform. See the [`-O` option to `ssh-keygen(1)`](http://man.openbsd.org/OpenBSD-current/man1/ssh-keygen.1) for a complete list.
 
 ## aws
-AWS configuration is only needed for accessing signing keys stored on S3, and isn't required even then.  
+AWS configuration is only needed for accessing signing keys stored on S3, and isn't totally necessary even then.  
 The S3 client can be configured using any of [the usual AWS-SDK means](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk) - environment variables, IAM roles etc.  
 It's strongly recommended that signing keys stored on S3 be locked down to specific IAM roles and encrypted using KMS.  
 
 - `region`: string. AWS region the bucket resides in, e.g. `us-east-1`.
-- `access_key`: string. AWS Access Key ID.
-- `secret_key`: string. AWS Secret Key.
+- `access_key`: string. AWS Access Key ID. This can be a secret stored in a [vault](https://www.vaultproject.io/) using the form `/vault/path/key` e.g. `/vault/secret/cashier/aws_access_key`.
+- `secret_key`: string. AWS Secret Key. This can be a secret stored in a [vault](https://www.vaultproject.io/) using the form `/vault/path/key` e.g. `/vault/secret/cashier/aws_secret_key`.
+
+## vault
+Vault support is currently a work-in-progress.
+
+- `address`: string. URL to the vault server.
+- `token`: string. Auth token for the vault.
 
 # Usage
 Cashier comes in two parts, a [cli](cmd/cashier) and a [server](cmd/cashierd).  

cmd/cashierd/main.go

@@ -28,13 +28,14 @@ import (
 	"github.com/nsheridan/cashier/server/auth"
 	"github.com/nsheridan/cashier/server/auth/github"
 	"github.com/nsheridan/cashier/server/auth/google"
-	"github.com/nsheridan/cashier/server/certutil"
 	"github.com/nsheridan/cashier/server/config"
-	"github.com/nsheridan/cashier/server/fs"
 	"github.com/nsheridan/cashier/server/signer"
 	"github.com/nsheridan/cashier/server/static"
 	"github.com/nsheridan/cashier/server/store"
 	"github.com/nsheridan/cashier/server/templates"
+	"github.com/nsheridan/cashier/server/util"
+	"github.com/nsheridan/cashier/server/wkfs/s3fs"
+	"github.com/nsheridan/cashier/server/wkfs/vaultfs"
 	"github.com/sid77/drop"
 )
 
@@ -167,7 +168,7 @@ func signHandler(a *appContext, w http.ResponseWriter, r *http.Request) (int, er
 	}
 	json.NewEncoder(w).Encode(&lib.SignResponse{
 		Status:   "ok",
-		Response: certutil.GetPublicKey(cert),
+		Response: util.GetPublicKey(cert),
 	})
 	return http.StatusOK, nil
 }
@@ -333,7 +334,10 @@ func main() {
 		log.Fatal(err)
 	}
 
-	fs.Register(config.AWS)
+	// Register well-known filesystems.
+	s3fs.Register(config.AWS)
+	vaultfs.Register(config.Vault)
+
 	signer, err := signer.New(config.SSH)
 	if err != nil {
 		log.Fatal(err)
@@ -378,7 +382,7 @@ func main() {
 	case "github":
 		authprovider, err = github.New(config.Auth)
 	default:
-		log.Fatalln("Unknown provider %s", config.Auth.Provider)
+		log.Fatalf("Unknown provider %s\n", config.Auth.Provider)
 	}
 	if err != nil {
 		log.Fatal(err)

example-server.conf

@@ -32,10 +32,16 @@ ssh {
   permissions = ["permit-pty", "permit-X11-forwarding", "permit-agent-forwarding", "permit-port-forwarding", "permit-user-rc"]  #  Permissions associated with a certificate
 }
 
-# Optional AWS config. if an aws config is present, the signing key can be read from S3 using the syntax `/s3/bucket/path/to/signing.key`.
+# Optional AWS config. if an aws config is present, then files (e.g. signing key or tls cert) can be read from S3 using the syntax `/s3/bucket/path/to/signing.key`.
 # These can also be set configured using the standard aws-sdk environment variables, IAM roles etc. https://github.com/aws/aws-sdk-go/wiki/configuring-sdk
 aws {
   region = "eu-west-1"
   access_key = "abcdef"
   secret_key = "xyz123"
 }
+
+# Optional Vault config. If a vault config is present then files (e.g. signing key or tls cert) can be read from a vault server using the syntax `/vault/secret/service/key_name`.
+vault {
+  address = "https://127.0.0.1:8200"
+  token = "83f01274-c6f0-4dae-aab9-13a6fc62772e"
+}

server/config/config.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 
 	"github.com/hashicorp/go-multierror"
+	"github.com/nsheridan/cashier/server/helpers/vault"
 	"github.com/spf13/viper"
 )
 
@@ -16,6 +17,7 @@ type Config struct {
 	Auth   *Auth   `mapstructure:"auth"`
 	SSH    *SSH    `mapstructure:"ssh"`
 	AWS    *AWS    `mapstructure:"aws"`
+	Vault  *Vault  `mapstructure:"vault"`
 }
 
 // unmarshalled holds the raw config.
@@ -24,6 +26,7 @@ type unmarshalled struct {
 	Auth   []Auth   `mapstructure:"auth"`
 	SSH    []SSH    `mapstructure:"ssh"`
 	AWS    []AWS    `mapstructure:"aws"`
+	Vault  []Vault  `mapstructure:"vault"`
 }
 
 // Server holds the configuration specific to the web server and sessions.
@@ -66,21 +69,31 @@ type AWS struct {
 	SecretKey string `mapstructure:"secret_key"`
 }
 
+// Vault holds Hashicorp Vault configuration.
+type Vault struct {
+	Address string `mapstructure:"address"`
+	Token   string `mapstructure:"token"`
+}
+
 func verifyConfig(u *unmarshalled) error {
 	var err error
 	if len(u.SSH) == 0 {
-		err = multierror.Append(errors.New("missing ssh config block"))
+		err = multierror.Append(errors.New("missing ssh config section"))
 	}
 	if len(u.Auth) == 0 {
-		err = multierror.Append(errors.New("missing auth config block"))
+		err = multierror.Append(errors.New("missing auth config section"))
 	}
 	if len(u.Server) == 0 {
-		err = multierror.Append(errors.New("missing server config block"))
+		err = multierror.Append(errors.New("missing server config section"))
 	}
 	if len(u.AWS) == 0 {
 		// AWS config is optional
 		u.AWS = append(u.AWS, AWS{})
 	}
+	if len(u.Vault) == 0 {
+		// Vault config is optional
+		u.Vault = append(u.Vault, Vault{})
+	}
 	return err
 }
 
@@ -106,6 +119,53 @@ func setFromEnv(u *unmarshalled) {
 	}
 }
 
+func setFromVault(u *unmarshalled) error {
+	if len(u.Vault) == 0 || u.Vault[0].Token == "" || u.Vault[0].Address == "" {
+		return nil
+	}
+	v, err := vault.NewClient(u.Vault[0].Address, u.Vault[0].Token)
+	if err != nil {
+		return err
+	}
+	get := func(value string) (string, error) {
+		if value[:7] == "/vault/" {
+			return v.Read(value)
+		}
+		return value, nil
+	}
+	if len(u.Auth) > 0 {
+		u.Auth[0].OauthClientID, err = get(u.Auth[0].OauthClientID)
+		if err != nil {
+			err = multierror.Append(err)
+		}
+		u.Auth[0].OauthClientSecret, err = get(u.Auth[0].OauthClientSecret)
+		if err != nil {
+			err = multierror.Append(err)
+		}
+	}
+	if len(u.Server) > 0 {
+		u.Server[0].CSRFSecret, err = get(u.Server[0].CSRFSecret)
+		if err != nil {
+			err = multierror.Append(err)
+		}
+		u.Server[0].CookieSecret, err = get(u.Server[0].CookieSecret)
+		if err != nil {
+			err = multierror.Append(err)
+		}
+	}
+	if len(u.AWS) > 0 {
+		u.AWS[0].AccessKey, err = get(u.AWS[0].AccessKey)
+		if err != nil {
+			err = multierror.Append(err)
+		}
+		u.AWS[0].SecretKey, err = get(u.AWS[0].SecretKey)
+		if err != nil {
+			err = multierror.Append(err)
+		}
+	}
+	return err
+}
+
 // ReadConfig parses a JSON configuration file into a Config struct.
 func ReadConfig(r io.Reader) (*Config, error) {
 	u := &unmarshalled{}
@@ -118,6 +178,9 @@ func ReadConfig(r io.Reader) (*Config, error) {
 		return nil, err
 	}
 	setFromEnv(u)
+	if err := setFromVault(u); err != nil {
+		return nil, err
+	}
 	if err := verifyConfig(u); err != nil {
 		return nil, err
 	}
@@ -126,5 +189,6 @@ func ReadConfig(r io.Reader) (*Config, error) {
 		Auth:   &u.Auth[0],
 		SSH:    &u.SSH[0],
 		AWS:    &u.AWS[0],
+		Vault:  &u.Vault[0],
 	}, nil
 }

server/helpers/vault/vault.go

@@ -0,0 +1,55 @@
+package vault
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/vault/api"
+)
+
+// NewClient returns a new vault client.
+func NewClient(address, token string) (*Client, error) {
+	config := &api.Config{
+		Address: address,
+	}
+	client, err := api.NewClient(config)
+	if err != nil {
+		return nil, err
+	}
+	client.SetToken(token)
+	return &Client{
+		vault: client,
+	}, nil
+}
+
+func parseName(name string) (path, key string) {
+	name = strings.TrimPrefix(name, "/vault/")
+	i := strings.LastIndex(name, "/")
+	if i < 0 {
+		return name, ""
+	}
+	return name[:i], name[i+1:]
+}
+
+// Client is a simple client for vault.
+type Client struct {
+	vault *api.Client
+}
+
+// Read returns a secret for a given path and key of the form `/vault/secret/path/key`.
+// If the requested key cannot be read the original string is returned along with an error.
+func (c *Client) Read(value string) (string, error) {
+	p, k := parseName(value)
+	data, err := c.vault.Logical().Read(p)
+	if err != nil {
+		return value, err
+	}
+	if data == nil {
+		return value, fmt.Errorf("no such key %s", k)
+	}
+	secret, ok := data.Data[k]
+	if !ok {
+		return value, fmt.Errorf("no such key %s", k)
+	}
+	return secret.(string), nil
+}

server/store/store.go

@@ -5,7 +5,7 @@ import (
 
 	"golang.org/x/crypto/ssh"
 
-	"github.com/nsheridan/cashier/server/certutil"
+	"github.com/nsheridan/cashier/server/util"
 )
 
 // CertStorer records issued certs in a persistent store for audit and
@@ -40,6 +40,6 @@ func parseCertificate(cert *ssh.Certificate) *CertRecord {
 		Principals: cert.ValidPrincipals,
 		CreatedAt:  parseTime(cert.ValidAfter),
 		Expires:    parseTime(cert.ValidBefore),
-		Raw:        certutil.GetPublicKey(cert),
+		Raw:        util.GetPublicKey(cert),
 	}
 }

server/certutil/util.go → server/util/util.go

@@ -1,4 +1,4 @@
-package certutil
+package util
 
 import "golang.org/x/crypto/ssh"
 

server/certutil/util_test.go → server/util/util_test.go

@@ -1,4 +1,4 @@
-package certutil
+package util
 
 import (
 	"testing"