Merge pull request #216 from stlaz/proxy-endpoints

use a different port for proxy-specific endpoints

README.md

@@ -43,9 +43,8 @@ Kube-rbac-proxy flags:
       --auth-token-audiences strings                Comma-separated list of token audiences to accept. By default a token does not have to have any specific audience. It is recommended to set a specific audience.
       --client-ca-file string                       If set, any request presenting a client certificate signed by one of the authorities in the client-ca-file is authenticated with an identity corresponding to the CommonName of the client certificate.
       --config-file string                          Configuration file to configure kube-rbac-proxy.
-      --healthz-path string                         The path to serve the liveness check endpoint. If empty, the healthz endpoint will not be exposed.
       --ignore-paths strings                        Comma-separated list of paths against which kube-rbac-proxy pattern-matches the incoming request. If the requst matches, it will proxy the request without performing an authentication or authorization check. Cannot be used with --allow-paths.
-      --insecure-listen-address string              The address the kube-rbac-proxy HTTP server should listen on.
+      --insecure-listen-address string              [DEPRECATED] The address the kube-rbac-proxy HTTP server should listen on.
       --kubeconfig string                           Path to a kubeconfig file, specifying how to connect to the API server. If unset, in-cluster configuration will be used
       --oidc-ca-file string                         If set, the OpenID server's certificate will be verified by one of the authorities in the oidc-ca-file, otherwise the host's root CA set will be used.
       --oidc-clientID string                        The client ID for the OpenID Connect client, must be set if oidc-issuer-url is set.
@@ -54,6 +53,7 @@ Kube-rbac-proxy flags:
       --oidc-issuer string                          The URL of the OpenID issuer, only HTTPS scheme will be accepted. If set, it will be used to verify the OIDC JSON Web Token (JWT).
       --oidc-sign-alg stringArray                   Supported signing algorithms, default RS256 (default [RS256])
       --oidc-username-claim string                  Identifier of the user in JWT claim, by default set to 'email' (default "email")
+      --proxy-endpoints-port int                    The port to securely serve proxy-specific endpoints (such as '/healthz'). Uses the host from the '--secure-listen-address'.
       --secure-listen-address string                The address the kube-rbac-proxy HTTPs server should listen on.
       --tls-cert-file string                        File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert)
       --tls-cipher-suites strings                   Comma-separated list of cipher suites for the server. Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). If omitted, the default Go cipher suites will be used

cmd/kube-rbac-proxy/app/kube-rbac-proxy.go

@@ -29,6 +29,7 @@ import (
 	"os"
 	"os/signal"
 	"path"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -127,7 +128,7 @@ type configfile struct {
 type completedProxyRunOptions struct {
 	insecureListenAddress string // DEPRECATED
 	secureListenAddress   string
-	healthzPath           string
+	proxyEndpointsPort    int
 
 	upstreamURL      *url.URL
 	upstreamForceH2C bool
@@ -204,7 +205,7 @@ func Complete(o *options.ProxyRunOptions) (*completedProxyRunOptions, error) {
 	completed := &completedProxyRunOptions{
 		insecureListenAddress: o.InsecureListenAddress,
 		secureListenAddress:   o.SecureListenAddress,
-		healthzPath:           o.HealthzPath,
+		proxyEndpointsPort:    o.ProxyEndpointsPort,
 		upstreamForceH2C:      o.UpstreamForceH2C,
 
 		allowPaths:  o.AllowPaths,
@@ -353,10 +354,6 @@ func Run(cfg *completedProxyRunOptions) error {
 	mux := http.NewServeMux()
 	mux.Handle("/", handler)
 
-	if cfg.healthzPath != "" {
-		mux.HandleFunc(cfg.healthzPath, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) })
-	}
-
 	var gr run.Group
 	{
 		if cfg.secureListenAddress != "" {
@@ -413,24 +410,59 @@ func Run(cfg *completedProxyRunOptions) error {
 				return fmt.Errorf("failed to configure http2 server: %w", err)
 			}
 
-			klog.Infof("Starting TCP socket on %v", cfg.secureListenAddress)
-			l, err := net.Listen("tcp", cfg.secureListenAddress)
-			if err != nil {
-				return fmt.Errorf("failed to listen on secure address: %w", err)
-			}
-
 			gr.Add(func() error {
+				klog.Infof("Starting TCP socket on %v", cfg.secureListenAddress)
+				l, err := net.Listen("tcp", cfg.secureListenAddress)
+				if err != nil {
+					return fmt.Errorf("failed to listen on secure address: %w", err)
+				}
+				defer l.Close()
+
 				klog.Infof("Listening securely on %v", cfg.secureListenAddress)
 				tlsListener := tls.NewListener(l, srv.TLSConfig)
 				return srv.Serve(tlsListener)
 			}, func(err error) {
 				if err := srv.Shutdown(context.Background()); err != nil {
 					klog.Errorf("failed to gracefully shutdown server: %w", err)
 				}
-				if err := l.Close(); err != nil {
-					klog.Errorf("failed to gracefully close secure listener: %w", err)
-				}
 			})
+
+			if cfg.proxyEndpointsPort != 0 {
+				proxyEndpointsMux := http.NewServeMux()
+				proxyEndpointsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) })
+
+				proxyEndpointsSrv := &http.Server{
+					Handler:   proxyEndpointsMux,
+					TLSConfig: srv.TLSConfig.Clone(),
+				}
+
+				if err := http2.ConfigureServer(proxyEndpointsSrv, nil); err != nil {
+					return fmt.Errorf("failed to configure http2 server: %w", err)
+				}
+
+				gr.Add(func() error {
+					host, _, err := net.SplitHostPort(cfg.secureListenAddress)
+					if err != nil {
+						return fmt.Errorf("failed to split %q into host and port: %w", cfg.secureListenAddress, err)
+					}
+					endpointsAddr := net.JoinHostPort(host, strconv.Itoa(cfg.proxyEndpointsPort))
+
+					klog.Infof("Starting TCP socket on %v", endpointsAddr)
+					proxyListener, err := net.Listen("tcp", endpointsAddr)
+					if err != nil {
+						return fmt.Errorf("failed to listen on secure address: %w", err)
+					}
+					defer proxyListener.Close()
+
+					klog.Info("Listening securely on %v for proxy endpoints", endpointsAddr)
+					tlsListener := tls.NewListener(proxyListener, srv.TLSConfig)
+					return proxyEndpointsSrv.Serve(tlsListener)
+				}, func(err error) {
+					if err := proxyEndpointsSrv.Shutdown(context.Background()); err != nil {
+						klog.Errorf("failed to gracefully shutdown proxy endpoints server: %w", err)
+					}
+				})
+			}
 		}
 	}
 	{

cmd/kube-rbac-proxy/app/options/options.go

@@ -31,7 +31,7 @@ type ProxyRunOptions struct {
 
 	InsecureListenAddress string
 	SecureListenAddress   string
-	HealthzPath           string
+	ProxyEndpointsPort    int
 
 	Upstream           string
 	UpstreamForceH2C   bool
@@ -74,15 +74,15 @@ func (o *ProxyRunOptions) Flags() k8sapiflag.NamedFlagSets {
 	flagset := namedFlagSets.FlagSet("kube-rbac-proxy")
 
 	// kube-rbac-proxy flags
-	flagset.StringVar(&o.InsecureListenAddress, "insecure-listen-address", "", "The address the kube-rbac-proxy HTTP server should listen on.")
+	flagset.StringVar(&o.InsecureListenAddress, "insecure-listen-address", "", "[DEPRECATED] The address the kube-rbac-proxy HTTP server should listen on.")
 	flagset.StringVar(&o.SecureListenAddress, "secure-listen-address", "", "The address the kube-rbac-proxy HTTPs server should listen on.")
 	flagset.StringVar(&o.Upstream, "upstream", "", "The upstream URL to proxy to once requests have successfully been authenticated and authorized.")
 	flagset.BoolVar(&o.UpstreamForceH2C, "upstream-force-h2c", false, "Force h2c to communiate with the upstream. This is required when the upstream speaks h2c(http/2 cleartext - insecure variant of http/2) only. For example, go-grpc server in the insecure mode, such as helm's tiller w/o TLS, speaks h2c only")
 	flagset.StringVar(&o.UpstreamCAFile, "upstream-ca-file", "", "The CA the upstream uses for TLS connection. This is required when the upstream uses TLS and its own CA certificate")
 	flagset.StringVar(&o.ConfigFileName, "config-file", "", "Configuration file to configure kube-rbac-proxy.")
 	flagset.StringSliceVar(&o.AllowPaths, "allow-paths", nil, "Comma-separated list of paths against which kube-rbac-proxy pattern-matches the incoming request. If the request doesn't match, kube-rbac-proxy responds with a 404 status code. If omitted, the incoming request path isn't checked. Cannot be used with --ignore-paths.")
 	flagset.StringSliceVar(&o.IgnorePaths, "ignore-paths", nil, "Comma-separated list of paths against which kube-rbac-proxy pattern-matches the incoming request. If the requst matches, it will proxy the request without performing an authentication or authorization check. Cannot be used with --allow-paths.")
-	flagset.StringVar(&o.HealthzPath, "healthz-path", "", "The path to serve the liveness check endpoint. If empty, the healthz endpoint will not be exposed.")
+	flagset.IntVar(&o.ProxyEndpointsPort, "proxy-endpoints-port", 0, "The port to securely serve proxy-specific endpoints (such as '/healthz'). Uses the host from the '--secure-listen-address'.")
 
 	// TLS flags
 	flagset.StringVar(&o.TLS.CertFile, "tls-cert-file", "", "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert)")

test/e2e/allowpaths/deployment.yaml

@@ -19,13 +19,21 @@ spec:
           image: quay.io/brancz/kube-rbac-proxy:local
           args:
             - "--secure-listen-address=0.0.0.0:8443"
+            - "--proxy-endpoints-port=8643"
             - "--upstream=http://127.0.0.1:8081/"
             - "--allow-paths=/metrics,/api/v1/label/*/values"
             - "--logtostderr=true"
             - "--v=10"
           ports:
             - containerPort: 8443
               name: https
+            - containerPort: 8643
+              name: proxy
+          readinessProbe:
+            httpGet:
+              scheme: HTTPS
+              port: 8643
+              path: healthz
         - name: prometheus-example-app
           image: quay.io/brancz/prometheus-example-app:v0.1.0
           args: