http2: enable to disable, limit http2 resources

- close HTTP/2 connections on unauthenticatead users - set max streams per connection to 100 - set max frame size and stream buffer to 256 kb. - enable to disable HTTP/2 compeltely
ibihim · Oct 20, 2023 · ab438d5 · ab438d5
1 parent b0fa6e5
commit ab438d5
Show file tree

Hide file tree

Showing 13 changed files with 299 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -62,6 +62,9 @@ Kube-rbac-proxy flags:
       --auth-token-audiences strings                Comma-separated list of token audiences to accept. By default a token does not have to have any specific audience. It is recommended to set a specific audience.
       --client-ca-file string                       If set, any request presenting a client certificate signed by one of the authorities in the client-ca-file is authenticated with an identity corresponding to the CommonName of the client certificate.
       --config-file string                          Configuration file to configure kube-rbac-proxy.
+      --http2-disable                               Disable HTTP/2 support
+      --http2-max-concurrent-streams uint32         The maximum number of concurrent streams per HTTP/2 connection. (default 100)
+      --http2-max-size uint32                       The maximum number of bytes that the server will accept for frame size and buffer per stream in a HTTP/2 request. (default 262144)
       --ignore-paths strings                        Comma-separated list of paths against which kube-rbac-proxy pattern-matches the incoming request. If the requst matches, it will proxy the request without performing an authentication or authorization check. Cannot be used with --allow-paths.
       --insecure-listen-address string              [DEPRECATED] The address the kube-rbac-proxy HTTP server should listen on.
       --kubeconfig string                           Path to a kubeconfig file, specifying how to connect to the API server. If unset, in-cluster configuration will be used

diff --git a/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go b/cmd/kube-rbac-proxy/app/kube-rbac-proxy.go
@@ -134,6 +134,9 @@ type completedProxyRunOptions struct {
 	upstreamForceH2C bool
 	upstreamCABundle *x509.CertPool
 
+	http2Disable bool
+	http2Options *http2.Server
+
 	auth *proxy.Config
 	tls  *options.TLSConfig
 
@@ -250,6 +253,15 @@ func Complete(o *options.ProxyRunOptions) (*completedProxyRunOptions, error) {
 		return nil, fmt.Errorf("failed to instantiate Kubernetes client: %w", err)
 	}
 
+	completed.http2Disable = o.HTTP2Disable
+	completed.http2Options = &http2.Server{
+		IdleTimeout:                  90 * time.Second,
+		MaxConcurrentStreams:         o.HTTP2MaxConcurrentStreams,
+		MaxReadFrameSize:             o.HTTP2MaxSize,
+		MaxUploadBufferPerStream:     int32(o.HTTP2MaxSize),
+		MaxUploadBufferPerConnection: int32(o.HTTP2MaxSize) * int32(o.HTTP2MaxConcurrentStreams),
+	}
+
 	return completed, nil
 }
 
@@ -357,7 +369,10 @@ func Run(cfg *completedProxyRunOptions) error {
 	var gr run.Group
 	{
 		if cfg.secureListenAddress != "" {
-			srv := &http.Server{Handler: mux, TLSConfig: &tls.Config{}}
+			srv := &http.Server{
+				Handler:   mux,
+				TLSConfig: &tls.Config{},
+			}
 
 			if cfg.tls.CertFile == "" && cfg.tls.KeyFile == "" {
 				klog.Info("Generating self signed cert as no cert is provided")
@@ -406,8 +421,21 @@ func Run(cfg *completedProxyRunOptions) error {
 			srv.TLSConfig.MinVersion = version
 			srv.TLSConfig.ClientAuth = tls.RequestClientCert
 
-			if err := http2.ConfigureServer(srv, nil); err != nil {
-				return fmt.Errorf("failed to configure http2 server: %w", err)
+			if cfg.http2Disable {
+				// HTTP/2 is temporarily disabled due to CVE-2023-44487
+				// Programs that must disable HTTP/2 can do so by setting
+				// Transport.TLSNextProto (for clients) or Server.TLSNextProto
+				// (for servers) to a non-nil, empty map.
+				// https://pkg.go.dev/net/http
+				srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+
+				// For reference:
+				// https://github.com/kubernetes/kubernetes/blob/de054fbf9422d778568946de21a48c7330a6c1b7/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go#L55-L59
+				srv.TLSConfig.NextProtos = []string{"http/1.1"}
+			} else {
+				if err := http2.ConfigureServer(srv, cfg.http2Options); err != nil {
+					return fmt.Errorf("failed to configure http2 server: %w", err)
+				}
 			}
 
 			gr.Add(func() error {
@@ -436,8 +464,20 @@ func Run(cfg *completedProxyRunOptions) error {
 					TLSConfig: srv.TLSConfig.Clone(),
 				}
 
-				if err := http2.ConfigureServer(proxyEndpointsSrv, nil); err != nil {
-					return fmt.Errorf("failed to configure http2 server: %w", err)
+				if cfg.http2Disable {
+					// HTTP/2 is temporarily disabled due to CVE-2023-44487
+					// Programs that must disable HTTP/2 can do so by setting
+					// Transport.TLSNextProto (for clients) or Server.TLSNextProto
+					// (for servers) to a non-nil, empty map.
+					// https://pkg.go.dev/net/http
+					srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+					// For reference:
+					// https://github.com/kubernetes/kubernetes/blob/de054fbf9422d778568946de21a48c7330a6c1b7/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go#L55-L59
+					srv.TLSConfig.NextProtos = []string{"http/1.1"}
+				} else {
+					if err := http2.ConfigureServer(proxyEndpointsSrv, cfg.http2Options); err != nil {
+						return fmt.Errorf("failed to configure http2 server: %w", err)
+					}
 				}
 
 				gr.Add(func() error {
@@ -467,7 +507,12 @@ func Run(cfg *completedProxyRunOptions) error {
 	}
 	{
 		if cfg.insecureListenAddress != "" {
-			srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})}
+			srv := &http.Server{}
+			if cfg.http2Disable {
+				srv.Handler = mux
+			} else {
+				srv.Handler = h2c.NewHandler(mux, cfg.http2Options)
+			}
 
 			l, err := net.Listen("tcp", cfg.insecureListenAddress)
 			if err != nil {

diff --git a/cmd/kube-rbac-proxy/app/options/options.go b/cmd/kube-rbac-proxy/app/options/options.go
@@ -41,6 +41,10 @@ type ProxyRunOptions struct {
 	KubeconfigLocation string
 	AllowPaths         []string
 	IgnorePaths        []string
+
+	HTTP2Disable              bool
+	HTTP2MaxConcurrentStreams uint32
+	HTTP2MaxSize              uint32
 }
 
 type TLSConfig struct {
@@ -113,5 +117,10 @@ func (o *ProxyRunOptions) Flags() k8sapiflag.NamedFlagSets {
 	//Kubeconfig flag
 	flagset.StringVar(&o.KubeconfigLocation, "kubeconfig", "", "Path to a kubeconfig file, specifying how to connect to the API server. If unset, in-cluster configuration will be used")
 
+	// HTTP2 flags
+	flagset.BoolVar(&o.HTTP2Disable, "http2-disable", false, "Disable HTTP/2 support")
+	flagset.Uint32Var(&o.HTTP2MaxConcurrentStreams, "http2-max-concurrent-streams", 100, "The maximum number of concurrent streams per HTTP/2 connection.")
+	flagset.Uint32Var(&o.HTTP2MaxSize, "http2-max-size", 256*1024, "The maximum number of bytes that the server will accept for frame size and buffer per stream in a HTTP/2 request.")
+
 	return namedFlagSets
 }
diff --git a/test/e2e/http2.go b/test/e2e/http2.go
@@ -0,0 +1,105 @@
+/*
+Copyright 2023 the kube-rbac-proxy maintainers. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package e2e
+
+import (
+	"testing"
+
+	"github.com/brancz/kube-rbac-proxy/test/kubetest"
+	"k8s.io/client-go/kubernetes"
+)
+
+func testHTTP2(client kubernetes.Interface) kubetest.TestSuite {
+	return func(t *testing.T) {
+		command := `HTTP_VERSION=$(curl -sI --http2 --connect-timeout 5 -k --fail -w "%{http_version}\n" -o /dev/null https://kube-rbac-proxy.default.svc.cluster.local:8443/metrics); if [[ "$HTTP_VERSION" != "2" ]]; then echo "Did expect HTTP/2. Actual protocol: $HTTP_VERSION" > /proc/self/fd/2; exit 1; fi`
+
+		kubetest.Scenario{
+			Name: "With succeeding HTTP2-client",
+			Description: `
+				Expecting http/2 capable client to succeed to connect with http/2.
+			`,
+
+			Given: kubetest.Actions(
+				kubetest.CreatedManifests(
+					client,
+					"http2/clusterRole.yaml",
+					"http2/clusterRoleBinding.yaml",
+					"http2/deployment.yaml",
+					"http2/service.yaml",
+					"http2/serviceAccount.yaml",
+					"http2/clusterRole-client.yaml",
+					"http2/clusterRoleBinding-client.yaml",
+				),
+			),
+			When: kubetest.Actions(
+				kubetest.PodsAreReady(
+					client,
+					1,
+					"app=kube-rbac-proxy",
+				),
+				kubetest.ServiceIsReady(
+					client,
+					"kube-rbac-proxy",
+				),
+			),
+			Then: kubetest.Actions(
+				kubetest.ClientSucceeds(
+					client,
+					command,
+					nil,
+				),
+			),
+		}.Run(t)
+
+		kubetest.Scenario{
+			Name: "With failing HTTP2-client",
+			Description: `
+				Expecting http/2 capable client to fail to connect with http/2.
+			`,
+
+			Given: kubetest.Actions(
+				kubetest.CreatedManifests(
+					client,
+					"http2/clusterRole.yaml",
+					"http2/clusterRoleBinding.yaml",
+					"http2/deployment-no-http2.yaml",
+					"http2/service.yaml",
+					"http2/serviceAccount.yaml",
+					"http2/clusterRole-client.yaml",
+					"http2/clusterRoleBinding-client.yaml",
+				),
+			),
+			When: kubetest.Actions(
+				kubetest.PodsAreReady(
+					client,
+					1,
+					"app=kube-rbac-proxy",
+				),
+				kubetest.ServiceIsReady(
+					client,
+					"kube-rbac-proxy",
+				),
+			),
+			Then: kubetest.Actions(
+				kubetest.ClientFails(
+					client,
+					command,
+					nil,
+				),
+			),
+		}.Run(t)
+	}
+}
diff --git a/test/e2e/http2/clusterRole-client.yaml b/test/e2e/http2/clusterRole-client.yaml
@@ -0,0 +1,7 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics
+rules:
+  - nonResourceURLs: ["/metrics"]
+    verbs: ["get"]
diff --git a/test/e2e/http2/clusterRole.yaml b/test/e2e/http2/clusterRole.yaml
@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-rbac-proxy
+  namespace: default
+rules:
+  - apiGroups: ["authentication.k8s.io"]
+    resources:
+      - tokenreviews
+    verbs: ["create"]
+  - apiGroups: ["authorization.k8s.io"]
+    resources:
+      - subjectaccessreviews
+    verbs: ["create"]
diff --git a/test/e2e/http2/clusterRoleBinding-client.yaml b/test/e2e/http2/clusterRoleBinding-client.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: default
diff --git a/test/e2e/http2/clusterRoleBinding.yaml b/test/e2e/http2/clusterRoleBinding.yaml
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-rbac-proxy
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-rbac-proxy
+subjects:
+  - kind: ServiceAccount
+    name: kube-rbac-proxy
+    namespace: default
diff --git a/test/e2e/http2/deployment-no-http2.yaml b/test/e2e/http2/deployment-no-http2.yaml
@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-rbac-proxy
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kube-rbac-proxy
+  template:
+    metadata:
+      labels:
+        app: kube-rbac-proxy
+    spec:
+      serviceAccountName: kube-rbac-proxy
+      containers:
+        - name: kube-rbac-proxy
+          image: quay.io/brancz/kube-rbac-proxy:local
+          args:
+            - "--secure-listen-address=0.0.0.0:8443"
+            - "--upstream=http://127.0.0.1:8081/"
+            - "--ignore-paths=/metrics,/api/v1/*"
+            - "--logtostderr=true"
+            - "--http2-disable=true"
+            - "--v=10"
+          ports:
+            - containerPort: 8443
+              name: https
+        - name: prometheus-example-app
+          image: quay.io/brancz/prometheus-example-app:v0.1.0
+          args:
+            - "--bind=127.0.0.1:8081"
diff --git a/test/e2e/http2/deployment.yaml b/test/e2e/http2/deployment.yaml
@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-rbac-proxy
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kube-rbac-proxy
+  template:
+    metadata:
+      labels:
+        app: kube-rbac-proxy
+    spec:
+      serviceAccountName: kube-rbac-proxy
+      containers:
+        - name: kube-rbac-proxy
+          image: quay.io/brancz/kube-rbac-proxy:local
+          args:
+            - "--secure-listen-address=0.0.0.0:8443"
+            - "--upstream=http://127.0.0.1:8081/"
+            - "--ignore-paths=/metrics,/api/v1/*"
+            - "--logtostderr=true"
+            - "--v=10"
+          ports:
+            - containerPort: 8443
+              name: https
+        - name: prometheus-example-app
+          image: quay.io/brancz/prometheus-example-app:v0.1.0
+          args:
+            - "--bind=127.0.0.1:8081"
diff --git a/test/e2e/http2/service.yaml b/test/e2e/http2/service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: kube-rbac-proxy
+  name: kube-rbac-proxy
+  namespace: default
+spec:
+  ports:
+    - name: https
+      port: 8443
+      targetPort: https
+  selector:
+    app: kube-rbac-proxy
diff --git a/test/e2e/http2/serviceAccount.yaml b/test/e2e/http2/serviceAccount.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-rbac-proxy
+  namespace: default
diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go
@@ -59,6 +59,7 @@ func Test(t *testing.T) {
 		"IgnorePath":         testIgnorePaths(client),
 		"TLS":                testTLS(client),
 		"StaticAuthorizer":   testStaticAuthorizer(client),
+		"HTTP2":              testHTTP2(client),
 	}
 
 	for name, tc := range tests {