From 067e14a2d1ecdfe8c18da6b0a0507cd4684e2c1c Mon Sep 17 00:00:00 2001 From: Stanislav Laznicka Date: Tue, 13 Dec 2022 11:10:26 +0100 Subject: [PATCH] use a different port for proxy-specific endpoints Using a port as a namespace for endpoints specific to the proxy is a bit more practical than using a path/subpath for each specific endpoint/endpoints as this introduced approach is less prone to upstream endpoint changes. --- README.md | 4 +- cmd/kube-rbac-proxy/app/kube-rbac-proxy.go | 62 ++++++++++++++++------ cmd/kube-rbac-proxy/app/options/options.go | 6 +-- test/e2e/allowpaths/deployment.yaml | 8 +++ 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 684ca83d7..e6c08e904 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,8 @@ Kube-rbac-proxy flags: --auth-token-audiences strings Comma-separated list of token audiences to accept. By default a token does not have to have any specific audience. It is recommended to set a specific audience. --client-ca-file string If set, any request presenting a client certificate signed by one of the authorities in the client-ca-file is authenticated with an identity corresponding to the CommonName of the client certificate. --config-file string Configuration file to configure kube-rbac-proxy. - --healthz-path string The path to serve the liveness check endpoint. If empty, the healthz endpoint will not be exposed. --ignore-paths strings Comma-separated list of paths against which kube-rbac-proxy pattern-matches the incoming request. If the requst matches, it will proxy the request without performing an authentication or authorization check. Cannot be used with --allow-paths. - --insecure-listen-address string The address the kube-rbac-proxy HTTP server should listen on. + --insecure-listen-address string [DEPRECATED] The address the kube-rbac-proxy HTTP server should listen on. --kubeconfig string Path to a kubeconfig file, specifying how to connect to the API server. If unset, in-cluster configuration will be used --oidc-ca-file string If set, the OpenID server's certificate will be verified by one of the authorities in the oidc-ca-file, otherwise the host's root CA set will be used. --oidc-clientID string The client ID for the OpenID Connect client, must be set if oidc-issuer-url is set. @@ -54,6 +53,7 @@ Kube-rbac-proxy flags: --oidc-issuer string The URL of the OpenID issuer, only HTTPS scheme will be accepted. If set, it will be used to verify the OIDC JSON Web Token (JWT). --oidc-sign-alg stringArray Supported signing algorithms, default RS256 (default [RS256]) --oidc-username-claim string Identifier of the user in JWT claim, by default set to 'email' (default "email") + --proxy-endpoints-port int The port to securely serve proxy-specific endpoints (such as '/healthz'). Uses the host from the '--secure-listen-address'. --secure-listen-address string The address the kube-rbac-proxy HTTPs server should listen on. --tls-cert-file string File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert) --tls-cipher-suites strings Comma-separated list of cipher suites for the server. Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). If omitted, the default Go cipher suites will be used diff --git a/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go b/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go index 8e6019e36..e7293a2ed 100644 --- a/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go +++ b/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go @@ -29,6 +29,7 @@ import ( "os" "os/signal" "path" + "strconv" "strings" "syscall" "time" @@ -127,7 +128,7 @@ type configfile struct { type completedProxyRunOptions struct { insecureListenAddress string // DEPRECATED secureListenAddress string - healthzPath string + proxyEndpointsPort int upstreamURL *url.URL upstreamForceH2C bool @@ -204,7 +205,7 @@ func Complete(o *options.ProxyRunOptions) (*completedProxyRunOptions, error) { completed := &completedProxyRunOptions{ insecureListenAddress: o.InsecureListenAddress, secureListenAddress: o.SecureListenAddress, - healthzPath: o.HealthzPath, + proxyEndpointsPort: o.ProxyEndpointsPort, upstreamForceH2C: o.UpstreamForceH2C, allowPaths: o.AllowPaths, @@ -353,10 +354,6 @@ func Run(cfg *completedProxyRunOptions) error { mux := http.NewServeMux() mux.Handle("/", handler) - if cfg.healthzPath != "" { - mux.HandleFunc(cfg.healthzPath, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) }) - } - var gr run.Group { if cfg.secureListenAddress != "" { @@ -413,13 +410,14 @@ func Run(cfg *completedProxyRunOptions) error { return fmt.Errorf("failed to configure http2 server: %w", err) } - klog.Infof("Starting TCP socket on %v", cfg.secureListenAddress) - l, err := net.Listen("tcp", cfg.secureListenAddress) - if err != nil { - return fmt.Errorf("failed to listen on secure address: %w", err) - } - gr.Add(func() error { + klog.Infof("Starting TCP socket on %v", cfg.secureListenAddress) + l, err := net.Listen("tcp", cfg.secureListenAddress) + if err != nil { + return fmt.Errorf("failed to listen on secure address: %w", err) + } + defer l.Close() + klog.Infof("Listening securely on %v", cfg.secureListenAddress) tlsListener := tls.NewListener(l, srv.TLSConfig) return srv.Serve(tlsListener) @@ -427,10 +425,44 @@ func Run(cfg *completedProxyRunOptions) error { if err := srv.Shutdown(context.Background()); err != nil { klog.Errorf("failed to gracefully shutdown server: %w", err) } - if err := l.Close(); err != nil { - klog.Errorf("failed to gracefully close secure listener: %w", err) - } }) + + if cfg.proxyEndpointsPort != 0 { + proxyEndpointsMux := http.NewServeMux() + proxyEndpointsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) }) + + proxyEndpointsSrv := &http.Server{ + Handler: proxyEndpointsMux, + TLSConfig: srv.TLSConfig.Clone(), + } + + if err := http2.ConfigureServer(proxyEndpointsSrv, nil); err != nil { + return fmt.Errorf("failed to configure http2 server: %w", err) + } + + gr.Add(func() error { + host, _, err := net.SplitHostPort(cfg.secureListenAddress) + if err != nil { + return fmt.Errorf("failed to split %q into host and port: %w", cfg.secureListenAddress, err) + } + endpointsAddr := net.JoinHostPort(host, strconv.Itoa(cfg.proxyEndpointsPort)) + + klog.Infof("Starting TCP socket on %v", endpointsAddr) + proxyListener, err := net.Listen("tcp", endpointsAddr) + if err != nil { + return fmt.Errorf("failed to listen on secure address: %w", err) + } + defer proxyListener.Close() + + klog.Info("Listening securely on %v for proxy endpoints", endpointsAddr) + tlsListener := tls.NewListener(proxyListener, srv.TLSConfig) + return proxyEndpointsSrv.Serve(tlsListener) + }, func(err error) { + if err := proxyEndpointsSrv.Shutdown(context.Background()); err != nil { + klog.Errorf("failed to gracefully shutdown proxy endpoints server: %w", err) + } + }) + } } } { diff --git a/cmd/kube-rbac-proxy/app/options/options.go b/cmd/kube-rbac-proxy/app/options/options.go index d435e5b84..9b64c8ce0 100644 --- a/cmd/kube-rbac-proxy/app/options/options.go +++ b/cmd/kube-rbac-proxy/app/options/options.go @@ -31,7 +31,7 @@ type ProxyRunOptions struct { InsecureListenAddress string SecureListenAddress string - HealthzPath string + ProxyEndpointsPort int Upstream string UpstreamForceH2C bool @@ -74,7 +74,7 @@ func (o *ProxyRunOptions) Flags() k8sapiflag.NamedFlagSets { flagset := namedFlagSets.FlagSet("kube-rbac-proxy") // kube-rbac-proxy flags - flagset.StringVar(&o.InsecureListenAddress, "insecure-listen-address", "", "The address the kube-rbac-proxy HTTP server should listen on.") + flagset.StringVar(&o.InsecureListenAddress, "insecure-listen-address", "", "[DEPRECATED] The address the kube-rbac-proxy HTTP server should listen on.") flagset.StringVar(&o.SecureListenAddress, "secure-listen-address", "", "The address the kube-rbac-proxy HTTPs server should listen on.") flagset.StringVar(&o.Upstream, "upstream", "", "The upstream URL to proxy to once requests have successfully been authenticated and authorized.") flagset.BoolVar(&o.UpstreamForceH2C, "upstream-force-h2c", false, "Force h2c to communiate with the upstream. This is required when the upstream speaks h2c(http/2 cleartext - insecure variant of http/2) only. For example, go-grpc server in the insecure mode, such as helm's tiller w/o TLS, speaks h2c only") @@ -82,7 +82,7 @@ func (o *ProxyRunOptions) Flags() k8sapiflag.NamedFlagSets { flagset.StringVar(&o.ConfigFileName, "config-file", "", "Configuration file to configure kube-rbac-proxy.") flagset.StringSliceVar(&o.AllowPaths, "allow-paths", nil, "Comma-separated list of paths against which kube-rbac-proxy pattern-matches the incoming request. If the request doesn't match, kube-rbac-proxy responds with a 404 status code. If omitted, the incoming request path isn't checked. Cannot be used with --ignore-paths.") flagset.StringSliceVar(&o.IgnorePaths, "ignore-paths", nil, "Comma-separated list of paths against which kube-rbac-proxy pattern-matches the incoming request. If the requst matches, it will proxy the request without performing an authentication or authorization check. Cannot be used with --allow-paths.") - flagset.StringVar(&o.HealthzPath, "healthz-path", "", "The path to serve the liveness check endpoint. If empty, the healthz endpoint will not be exposed.") + flagset.IntVar(&o.ProxyEndpointsPort, "proxy-endpoints-port", 0, "The port to securely serve proxy-specific endpoints (such as '/healthz'). Uses the host from the '--secure-listen-address'.") // TLS flags flagset.StringVar(&o.TLS.CertFile, "tls-cert-file", "", "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert)") diff --git a/test/e2e/allowpaths/deployment.yaml b/test/e2e/allowpaths/deployment.yaml index beae63daa..06dc38abf 100644 --- a/test/e2e/allowpaths/deployment.yaml +++ b/test/e2e/allowpaths/deployment.yaml @@ -19,6 +19,7 @@ spec: image: quay.io/brancz/kube-rbac-proxy:local args: - "--secure-listen-address=0.0.0.0:8443" + - "--proxy-endpoints-port=8643" - "--upstream=http://127.0.0.1:8081/" - "--allow-paths=/metrics,/api/v1/label/*/values" - "--logtostderr=true" @@ -26,6 +27,13 @@ spec: ports: - containerPort: 8443 name: https + - containerPort: 8643 + name: proxy + readinessProbe: + httpGet: + scheme: HTTPS + port: 8643 + path: healthz - name: prometheus-example-app image: quay.io/brancz/prometheus-example-app:v0.1.0 args: