diff --git a/dev-config.yaml b/dev-config.yaml index 9f9e98eaa5..8206cec09e 100644 --- a/dev-config.yaml +++ b/dev-config.yaml @@ -12,6 +12,7 @@ proxy: cpu: 0 networkPolicy: enabled: true + egress: [] # overrides allowance of 0.0.0.0/0 hub: cookieSecret: 1470700e01f77171c2c67b12130c25081dfbdf2697af8c2f2bd05621b31100bf @@ -27,7 +28,11 @@ hub: apiToken: 0cc05feaefeeb29179e924ffc6d3886ffacf0d1a28ab225f5c210436ffc5cfd5 networkPolicy: enabled: true - + egress: # overrides allowance of 0.0.0.0/0 + # In kind/k3s clusters the Kubernetes API server is exposing this port + - ports: + - protocol: TCP + port: 6443 singleuser: storage: @@ -36,21 +41,13 @@ singleuser: guarantee: null networkPolicy: enabled: true - # Block all egress apart from DNS and jupyter.org - # CIDR must match the allowed URL in test_singleuser_netpol + # For testing purposes in test_singleuser_netpol egress: - - ports: - ## port 53 is the default port for DNS queries - - port: 53 - protocol: UDP - to: - ## nslookup jupyter.org - ## - 104.28.9.110 - ## - 104.28.8.110 - ipBlock: - cidr: 104.28.9.110/32 + cidr: 104.28.9.110/32 # jupyter.org 1 - ipBlock: - cidr: 104.28.8.110/32 + cidr: 104.28.8.110/32 # jupyter.org 2 prePuller: hook: diff --git a/images/singleuser-sample/Dockerfile b/images/singleuser-sample/Dockerfile index 6e3aa6071e..29bcee28a6 100644 --- a/images/singleuser-sample/Dockerfile +++ b/images/singleuser-sample/Dockerfile @@ -14,7 +14,9 @@ ARG JUPYTERHUB_VERSION=1.1.* # NOTE: git is already available in the jupyter/minimal-notebook image. USER root RUN apt-get update && apt-get install --yes --no-install-recommends \ + dnsutils \ git \ + iputils-ping \ && rm -rf /var/lib/apt/lists/* USER $NB_USER diff --git a/jupyterhub/templates/hub/netpol.yaml b/jupyterhub/templates/hub/netpol.yaml index 361abd4d7d..bb4e419903 100644 --- a/jupyterhub/templates/hub/netpol.yaml +++ b/jupyterhub/templates/hub/netpol.yaml @@ -12,26 +12,48 @@ spec: policyTypes: - Ingress - Egress + ingress: - - from: + # allowed pods (hub.jupyter.org/network-access-hub) --> hub + - ports: + - port: http + from: - podSelector: matchLabels: hub.jupyter.org/network-access-hub: "true" - ports: - - port: http - {{- /* Useful if you want to give hub access to pods from other namespaces */}} - {{- if .Values.hub.networkPolicy.ingress}} - {{- .Values.hub.networkPolicy.ingress| toYaml | trimSuffix "\n" | nindent 4 }} + + {{- with .Values.hub.networkPolicy.ingress }} + # default: nothing --> hub + {{- . | toYaml | trimSuffix "\n" | nindent 4 }} {{- end }} + egress: - {{- /* - The default is to allow all egress for hub If you want to restrict it the - following egress is required - - proxy:8001 - - singleuser:8888 - - Kubernetes api-server - */}} - {{- if .Values.hub.networkPolicy.egress }} - {{- .Values.hub.networkPolicy.egress | toYaml | trimSuffix "\n" | nindent 4 }} + # hub --> proxy + - ports: + - port: 8001 + to: + - podSelector: + matchLabels: + {{- $_ := merge (dict "componentLabel" "proxy") . }} + {{- include "jupyterhub.matchLabels" $_ | nindent 14 }} + # hub --> singleuser-server + - ports: + - port: 8888 + to: + - podSelector: + matchLabels: + {{- $_ := merge (dict "componentLabel" "singleuser-server") . }} + {{- include "jupyterhub.matchLabels" $_ | nindent 14 }} + + # hub -> Kubernetes internal DNS + - ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + + {{- with .Values.hub.networkPolicy.egress }} + # hub --> default: everything + {{- . | toYaml | trimSuffix "\n" | nindent 4 }} {{- end }} {{- end }} diff --git a/jupyterhub/templates/proxy/netpol.yaml b/jupyterhub/templates/proxy/netpol.yaml index 64e57e5c64..855d0cc89e 100644 --- a/jupyterhub/templates/proxy/netpol.yaml +++ b/jupyterhub/templates/proxy/netpol.yaml @@ -16,37 +16,60 @@ spec: policyTypes: - Ingress - Egress + ingress: + # allowed pods (hub.jupyter.org/network-access-proxy-http) --> proxy (http/https port) - ports: - - port: http - {{- if or $manualHTTPS $manualHTTPSwithsecret}} - - port: https - {{- end }} - - from: + - port: http + {{- if or $manualHTTPS $manualHTTPSwithsecret }} + - port: https + {{- end }} + from: - podSelector: matchLabels: hub.jupyter.org/network-access-proxy-http: "true" - ports: - - port: http - - from: + + # allowed pods (hub.jupyter.org/network-access-proxy-api) --> proxy (api port) + - ports: + - port: api + from: - podSelector: matchLabels: hub.jupyter.org/network-access-proxy-api: "true" - ports: - - port: api - {{- /* Useful if you want to give proxy access to pods from other namespaces */}} - {{- if .Values.proxy.networkPolicy.ingress}} - {{- .Values.proxy.networkPolicy.ingress | toYaml | trimSuffix "\n" | nindent 4 }} + + {{- with .Values.proxy.networkPolicy.ingress}} + # default: nothing --> proxy + {{- . | toYaml | trimSuffix "\n" | nindent 4 }} {{- end }} + egress: - {{- /* - The default is to allow all egress for proxy If you want to restrict it the - following egress is required - - hub:8081 - - singleuser:8888 - - Kubernetes api-server - */}} - {{- if .Values.proxy.networkPolicy.egress }} - {{- .Values.proxy.networkPolicy.egress | toYaml | trimSuffix "\n" | nindent 4 }} + # proxy --> hub + - ports: + - port: 8081 + to: + - podSelector: + matchLabels: + {{- $_ := merge (dict "componentLabel" "hub") . }} + {{- include "jupyterhub.matchLabels" $_ | nindent 14 }} + + # proxy --> singleuser-server + - ports: + - port: 8888 + to: + - podSelector: + matchLabels: + {{- $_ := merge (dict "componentLabel" "singleuser-server") . }} + {{- include "jupyterhub.matchLabels" $_ | nindent 14 }} + + # proxy -> Kubernetes internal DNS + - ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + + {{- with .Values.proxy.networkPolicy.egress }} + # proxy --> default: everything + {{- . | toYaml | trimSuffix "\n" | nindent 4 }} {{- end }} {{- end }} diff --git a/jupyterhub/templates/singleuser/netpol.yaml b/jupyterhub/templates/singleuser/netpol.yaml index d6073dca54..f106aa5919 100644 --- a/jupyterhub/templates/singleuser/netpol.yaml +++ b/jupyterhub/templates/singleuser/netpol.yaml @@ -13,30 +13,40 @@ spec: policyTypes: - Ingress - Egress + ingress: - - from: + # allowed pods (hub.jupyter.org/network-access-singleuser) --> singleuser-server + - ports: + - port: notebook-port + from: - podSelector: matchLabels: hub.jupyter.org/network-access-singleuser: "true" - ports: - - port: 8888 - {{- /* Useful if you want to give user server access to pods from other namespaces */}} - {{- if .Values.singleuser.networkPolicy.ingress }} - {{- .Values.singleuser.networkPolicy.ingress | toYaml | trimSuffix "\n" | nindent 4 }} + + {{- with .Values.singleuser.networkPolicy.ingress }} + # default: nothing --> singleuser-server + {{- . | toYaml | trimSuffix "\n" | nindent 4 }} {{- end }} + egress: - - to: + # singleuser-server --> hub + - ports: + - port: 8081 + to: - podSelector: matchLabels: - {{- /* - Override componentLabel because we need the label of the - destination, not the source - */}} {{- $_ := merge (dict "componentLabel" "hub") . }} {{- include "jupyterhub.matchLabels" $_ | nindent 14 }} - ports: - - port: 8081 - {{- if .Values.singleuser.networkPolicy.egress }} - {{- .Values.singleuser.networkPolicy.egress | toYaml | trimSuffix "\n" | nindent 4 }} + + # singleuser-server -> Kubernetes internal DNS + - ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + + {{- with .Values.singleuser.networkPolicy.egress }} + # singleuser-server --> default: everything + {{- . | toYaml | trimSuffix "\n" | nindent 4 }} {{- end }} {{- end }} diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index 78edb1e730..144a36a9c1 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -68,6 +68,12 @@ hub: networkPolicy: enabled: false ingress: [] + ## egress for JupyterHub already includes Kubernetes internal DNS and + ## access to the proxy, but can be restricted further, but ensure to allow + ## access to the Kubernetes API server that couldn't be pinned ahead of + ## time. + ## + ## ref: https://stackoverflow.com/a/59016417/2220152 egress: - to: - ipBlock: diff --git a/tests/test_spawn.py b/tests/test_spawn.py index 1ad2ee5a23..f276931f9c 100644 --- a/tests/test_spawn.py +++ b/tests/test_spawn.py @@ -162,6 +162,24 @@ def test_singleuser_netpol(api_request, jupyter_user, request_data): print(server_model) pod_name = server_model["state"]["pod_name"] + c = subprocess.run([ + "kubectl", "exec", pod_name, + "--namespace", os.environ["Z2JH_KUBE_NAMESPACE"], + "--context", os.environ["Z2JH_KUBE_CONTEXT"], + "--", + "nslookup", "hub", + ]) + assert c.returncode == 0, "DNS issue: failed to resolve 'hub' from a singleuser-server" + + c = subprocess.run([ + "kubectl", "exec", pod_name, + "--namespace", os.environ["Z2JH_KUBE_NAMESPACE"], + "--context", os.environ["Z2JH_KUBE_CONTEXT"], + "--", + "nslookup", "jupyter.org", + ]) + assert c.returncode == 0, "DNS issue: failed to resolve 'jupyter.org' from a singleuser-server" + # Must match CIDR in singleuser.networkPolicy.egress. allowed_url = "http://jupyter.org" blocked_url = "http://mybinder.org" @@ -173,14 +191,14 @@ def test_singleuser_netpol(api_request, jupyter_user, request_data): "--", "wget", "--quiet", "--tries=1", "--timeout=3", allowed_url, ]) - assert c.returncode == 0, "Unable to get allowed domain (or failed to resolve the domain name)" + assert c.returncode == 0, "Unable to get allowed domain" c = subprocess.run([ "kubectl", "exec", pod_name, "--namespace", os.environ["Z2JH_KUBE_NAMESPACE"], "--context", os.environ["Z2JH_KUBE_CONTEXT"], "--", - "wget", "--quiet", "--tries=1", "--timeout=3", blocked_url, + "wget", "--quiet", "--server-response", "-O-", "--tries=1", "--timeout=3", blocked_url, ]) assert c.returncode > 0, "Blocked domain was allowed"