Replace kube-lego + nginx ingress with traefik

chartpress.yaml

@@ -6,6 +6,8 @@ charts:
       published: https://jupyterhub.github.io/helm-chart
 
     images:
+      secret-sync:
+        valuesPath: proxy.secretSync.image
       hub:
         valuesPath: hub.image
         buildArgs:

images/secret-sync/Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.7-alpine
+
+RUN pip install --no-cache kubernetes
+
+COPY secret-sync.py /usr/local/bin/secret-sync.py

images/secret-sync/secret-sync.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""
+Helper script to perform two-way sync of files to k8s secret objects.
+
+traefik expects a JSON file (acme.json) to persist across time,
+to make sure Let's Encrypt certificates work. In kubernetes,
+pod restarts clear out the filesystem, making this hard. We could
+add a persistent volume to the proxy, but this is excessive for
+a single file.
+
+This script can do a 'two way' sync of a given file and a key
+in a kubernetes secret object. The file should be in an emptyDir
+volume in the traefik pod, which should also have this script
+running as a sidecar.
+
+## Kubernetes Secret -> File system
+
+This needs to happen only once when the pod starts - we do not
+support modifications to the secret by other actors. The
+'load' command is used to specify the secret name, key and
+path to load it into
+
+## File system -> Kubernetes secret
+
+traefik might write new contents to the acme.json file over
+time, and we need to sync it to the kubernetes secret object.
+Ideally, we would watch for changes to the file with inotify
+and update the secret object as needed. However, for now we
+just operate in a 30s loop. This is good enough, since
+traefik can always re-generate certs if needed.
+"""
+import sys
+import os
+import subprocess
+import argparse
+import time
+import tarfile
+import io
+import base64
+import logging
+from kubernetes import client, config
+
+def update_secret(namespace, secret_name, labels, key, value):
+    """
+    Update a secret object's key with the value
+    """
+    try:
+        config.load_kube_config()
+    except:
+        config.load_incluster_config()
+
+    v1 = client.CoreV1Api()
+    try:
+        secret = v1.read_namespaced_secret(namespace=namespace, name=secret_name)
+    except client.rest.ApiException as e:
+        if e.status == 404:
+            secret = client.V1Secret(
+                metadata=client.V1ObjectMeta(name=secret_name, labels=labels),
+                data={}
+            )
+            resp = v1.create_namespaced_secret(namespace=namespace, body=secret)
+            logging.info(f"Created secret {secret_name} since it does not exist")
+        else:
+            raise
+    # Value should be base64'd string
+    new_value = base64.standard_b64encode(value).decode()
+    if secret.data is None:
+        secret.data = {}
+    if new_value != secret.data.get(key):
+        secret.data[key] = base64.standard_b64encode(value).decode()
+        v1.patch_namespaced_secret(namespace=namespace, name=secret_name, body=secret)
+        logging.info(f"Updated secret {secret_name} with new value for key {key}")
+
+def get_secret_value(namespace, secret_name, key):
+    try:
+        config.load_kube_config()
+    except:
+        config.load_incluster_config()
+
+    v1 = client.CoreV1Api()
+    try:
+        secret = v1.read_namespaced_secret(namespace=namespace, name=secret_name)
+    except client.rest.ApiException as e:
+        if e.status == 404:
+            # Secret doesn't exist
+            return None
+        raise
+    if secret.data is None or key not in secret.data:
+        return None
+    return base64.standard_b64decode(secret.data[key])
+
+def setup_logging():
+    """
+    Set up root logger to log to stderr
+    """
+    logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO, stream=sys.stderr)
+
+def main():
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument(
+        '--namespace',
+        help='Namespace the secret exists in'
+    )
+
+    argparser.add_argument(
+        'action',
+        choices=['load', 'watch-save']
+    )
+
+    argparser.add_argument(
+        'secret_name',
+        help="Name of secret to sync with. Will be created if needed."
+    )
+
+    argparser.add_argument(
+        'key',
+        help="Key in secret object to sync file to"
+    )
+
+    argparser.add_argument(
+        'path',
+        help="Path in filesystem to sync to"
+    )
+
+    argparser.add_argument(
+        '--label',
+        help="Labels (of form key=value) to add to the k8s secret when it is created",
+        action="append"
+    )
+
+    args = argparser.parse_args()
+
+    setup_logging()
+
+    if not args.namespace:
+        try:
+            with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f:
+                args.namespace = f.read().strip()
+        except FileNotFoundError:
+            print("Can not determine a namespace, must be explicitly set with --namespace", file=sys.stderr)
+            sys.exit(1)
+
+    if args.action == 'load':
+        value = get_secret_value(args.namespace, args.secret_name, args.key)
+        if value:
+            with open(args.path, 'wb') as f:
+                f.write(value)
+                os.fchmod(f.fileno(), 0o600)
+    elif args.action == 'watch-save':
+        labels = {}
+        for label in args.label:
+            l_splits = label.split('=', 1)
+            labels[l_splits[0]] = l_splits[1]
+        # FIXME: use inotifiy
+        while True:
+            if os.path.exists(args.path):
+                with open(args.path, 'rb') as f:
+                    update_secret(args.namespace, args.secret_name, labels, args.key, f.read())
+            time.sleep(30)
+
+if __name__ == '__main__':
+    main()

jupyterhub/templates/proxy/autohttps/configmap.yaml

@@ -0,0 +1,106 @@
+{{- $HTTPS := (and .Values.proxy.https.hosts .Values.proxy.https.enabled) }}
+{{- $autoHTTPS := (and $HTTPS (eq .Values.proxy.https.type "letsencrypt")) }}
+{{- if $autoHTTPS -}}
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: traefik-proxy-config
+  labels:
+    {{- include "jupyterhub.labels" . | nindent 4 }}
+data:
+  traefik.toml: |
+    # traefik.toml file template
+    defaultEntryPoints = ["http", "https"]
+
+    logLevel = "INFO"
+    # log errors, which could be proxy errors
+    [accessLog]
+    format = "json"
+    [accessLog.filters]
+    statusCodes = ["500-999"]
+
+    [accessLog.fields.headers]
+    [accessLog.fields.headers.names]
+    Authorization = "redact"
+    Cookie = "redact"
+    Set-Cookie = "redact"
+    X-Xsrftoken = "redact"
+
+    [respondingTimeouts]
+    idleTimeout = "10m0s"
+
+    [entryPoints]
+      [entryPoints.http]
+      address = ":80"
+
+      [entryPoints.https]
+      address = ":443"
+
+    [wss]
+    protocol = "http"
+
+    [certificatesResolvers.le.acme]
+      email = {{ required "proxy.https.letsencrypt.contactEmail is a required field" .Values.proxy.https.letsencrypt.contactEmail | quote }}
+      storage = "/etc/acme/acme.json"
+      {{- if .Values.proxy.https.letsencrypt.acmeServer }}
+      caServer = {{ .Values.proxy.https.letsencrypt.acmeServer | quote }}
+      {{- end}}
+      [certificatesResolvers.le.acme.httpChallenge]
+        # used during the challenge
+        entryPoint = "http"
+    [providers]
+      [providers.file]
+        filename = '/etc/traefik/dynamic.toml'
+
+  dynamic.toml: |
+    [tls]
+      [tls.options]
+        [tls.options.default]
+          sniStrict = true
+          minVersion = "VersionTLS12"
+          # Adapted from https://ssl-config.mozilla.org/#server=traefik&server-version=1.7.12&config=intermediate
+          cipherSuites = [
+            "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+            "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+            "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+            "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
+            "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
+          ]
+
+    [http.middlewares]
+      [http.middlewares.redirect.redirectScheme]
+        scheme = "https"
+      [http.middlewares.hsts.headers]
+        stsSeconds = {{ int64 .Values.proxy.traefik.hsts.maxAge }}
+        {{ if .Values.proxy.traefik.hsts.includeSubdomains }}
+        stsIncludeSubdomains = true
+        {{- end }}
+
+
+    [http.routers]
+      [http.routers.httpredirect]
+        rule = "PathPrefix(`/`)"
+        service = "chp"
+        entrypoints = ["http"]
+        middlewares = ["redirect"]
+
+      [http.routers.chp]
+        rule = "PathPrefix(`/`)"
+        entrypoints = ["https"]
+        middlewares = ["hsts"]
+
+        service = "chp"
+          [http.routers.chp.tls]
+            options = "default"
+            certResolver = "le"
+            {{- range $host := .Values.proxy.https.hosts }}
+            [[http.routers.chp.tls.domains]]
+            main = "{{ $host }}"
+            {{- end}}
+
+    [http.services]
+      [http.services.chp.loadBalancer]
+        [[http.services.chp.loadBalancer.servers]]
+          url = "http://proxy-http:8000/"
+{{- end }}