From 2db47315eef1bbe259c5c2dfe624547ea50afa71 Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Sun, 14 Apr 2024 11:07:11 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20grpc=20timeouts?= =?UTF-8?q?=20with=20proxy=20settings=20if=20backend=20is=20grpc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rootfs/etc/nginx/template/nginx.tmpl | 7 ++++ test/e2e/annotations/grpc.go | 51 ++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 4c0da2eb95..1f3f0ee378 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -1481,6 +1481,13 @@ stream { proxy_next_upstream_timeout {{ $location.Proxy.NextUpstreamTimeout }}; proxy_next_upstream_tries {{ $location.Proxy.NextUpstreamTries }}; + # Grpc settings + {{ if or (eq $location.BackendProtocol "GRPC") (eq $location.BackendProtocol "GRPCS")}} + grpc_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; + grpc_send_timeout {{ $location.Proxy.SendTimeout }}s; + grpc_read_timeout {{ $location.Proxy.ReadTimeout }}s; + {{ end }} + {{/* Add any additional configuration defined */}} {{ $location.ConfigurationSnippet }} diff --git a/test/e2e/annotations/grpc.go b/test/e2e/annotations/grpc.go index b3be82c2a7..e5aef7971b 100644 --- a/test/e2e/annotations/grpc.go +++ b/test/e2e/annotations/grpc.go @@ -35,7 +35,10 @@ import ( "k8s.io/ingress-nginx/test/e2e/framework" ) -const echoHost = "echo" +const ( + echoHost = "echo" + host = "grpc" +) var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { f := framework.NewDefaultFramework("grpc", framework.WithHTTPBunEnabled()) @@ -43,8 +46,6 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { ginkgo.It("should use grpc_pass in the configuration file", func() { f.NewGRPCFortuneTellerDeployment() - host := "grpc" - annotations := map[string]string{ "nginx.ingress.kubernetes.io/backend-protocol": "GRPC", } @@ -259,4 +260,48 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { metadata := res.GetMetadata() assert.Equal(ginkgo.GinkgoT(), metadata["content-type"].Values[0], "application/grpc") }) + + ginkgo.It("should set valid grpc timeouts for grpc", func() { + proxyConnectTimeout := "5" + proxySendTimeout := "30" + proxyReadtimeout := "30" + + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "grpc" + annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyConnectTimeout + annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxySendTimeout + annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyReadtimeout + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("grpc_connect_timeout %ss;", proxyConnectTimeout)) && + strings.Contains(server, fmt.Sprintf("grpc_send_timeout %ss;", proxySendTimeout)) && + strings.Contains(server, fmt.Sprintf("grpc_read_timeout %ss;", proxyReadtimeout)) + }) + }) + + ginkgo.It("should set valid grpc timeouts for grpcs", func() { + proxyConnectTimeout := "5" + proxySendTimeout := "30" + proxyReadtimeout := "30" + + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "grpcs" + annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyConnectTimeout + annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxySendTimeout + annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyReadtimeout + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("grpc_connect_timeout %ss;", proxyConnectTimeout)) && + strings.Contains(server, fmt.Sprintf("grpc_send_timeout %ss;", proxySendTimeout)) && + strings.Contains(server, fmt.Sprintf("grpc_read_timeout %ss;", proxyReadtimeout)) + }) + }) }) From b43137b41d0f5d5db56f10a7e1505c68a3d5e309 Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Sun, 14 Apr 2024 11:35:58 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=93=9D=20=20docs:=20Documentation=20o?= =?UTF-8?q?nly=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user-guide/nginx-configuration/annotations.md | 8 ++++++++ docs/user-guide/nginx-configuration/configmap.md | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md index 2230f80f73..2d8291d780 100755 --- a/docs/user-guide/nginx-configuration/annotations.md +++ b/docs/user-guide/nginx-configuration/annotations.md @@ -700,6 +700,14 @@ In some scenarios is required to have different values. To allow this we provide - `nginx.ingress.kubernetes.io/proxy-next-upstream-tries` - `nginx.ingress.kubernetes.io/proxy-request-buffering` +If you indicate [Backend Protocol](#backend-protocol) as `GRPC` or `GRPCS`, the following values will be applied to gRPC connection as well: + +``` +grpc_connect_timeout {{ proxy_connect_timeout }} +grpc_send_timeout {{ proxy_send_timeout }} +grpc_read_timeout {{ proxy_read_timeout }} +``` + Note: All timeout values are unitless and in seconds e.g. `nginx.ingress.kubernetes.io/proxy-read-timeout: "120"` sets a valid 120 seconds proxy read timeout. ### Proxy redirect diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md index 52c54c4fe4..162e3928ce 100644 --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -1142,14 +1142,20 @@ See NGINX [client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_mo Sets the timeout for [establishing a connection with a proxied server](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout). It should be noted that this timeout cannot usually exceed 75 seconds. +It will also set the [grpc_connect_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout) for gRPC connections. + ## proxy-read-timeout Sets the timeout in seconds for [reading a response from the proxied server](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout). The timeout is set only between two successive read operations, not for the transmission of the whole response. +It will also set the [grpc_read_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout) for gRPC connections. + ## proxy-send-timeout Sets the timeout in seconds for [transmitting a request to the proxied server](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout). The timeout is set only between two successive write operations, not for the transmission of the whole request. +It will also set the [grpc_send_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_send_timeout) for gRPC connections. + ## proxy-buffers-number Sets the number of the buffer used for [reading the first part of the response](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) received from the proxied server. This part usually contains a small response header. From 7ade80ebe356fd0b278f2c51dd59bfb1c90d9ee5 Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Sun, 14 Apr 2024 11:47:06 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=90=9B=20fix:=20uppercase=20for=20pro?= =?UTF-8?q?tocol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rootfs/etc/nginx/template/nginx.tmpl | 2 +- test/e2e/annotations/grpc.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 1f3f0ee378..93a04e3e62 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -1481,8 +1481,8 @@ stream { proxy_next_upstream_timeout {{ $location.Proxy.NextUpstreamTimeout }}; proxy_next_upstream_tries {{ $location.Proxy.NextUpstreamTries }}; + {{ if or (eq $location.BackendProtocol "GRPC") (eq $location.BackendProtocol "GRPCS") }} # Grpc settings - {{ if or (eq $location.BackendProtocol "GRPC") (eq $location.BackendProtocol "GRPCS")}} grpc_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; grpc_send_timeout {{ $location.Proxy.SendTimeout }}s; grpc_read_timeout {{ $location.Proxy.ReadTimeout }}s; diff --git a/test/e2e/annotations/grpc.go b/test/e2e/annotations/grpc.go index e5aef7971b..96cd227d99 100644 --- a/test/e2e/annotations/grpc.go +++ b/test/e2e/annotations/grpc.go @@ -267,7 +267,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { proxyReadtimeout := "30" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "grpc" + annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPC" annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyConnectTimeout annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxySendTimeout annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyReadtimeout @@ -289,7 +289,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { proxyReadtimeout := "30" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "grpcs" + annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPCS" annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyConnectTimeout annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxySendTimeout annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyReadtimeout From 044d125712047e9d1f350af8e58469a4323762d5 Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Sun, 14 Apr 2024 11:57:38 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=F0=9F=93=9D=20docs:=20grpc=20timeouts=20ex?= =?UTF-8?q?ample?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/examples/grpc/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/examples/grpc/README.md b/docs/examples/grpc/README.md index 508b23fb8f..23126c3453 100644 --- a/docs/examples/grpc/README.md +++ b/docs/examples/grpc/README.md @@ -166,11 +166,9 @@ This example demonstrates how to route traffic to a gRPC service through the Ing ### Notes on using response/request streams +> `grpc_read_timeout` and `grpc_send_timeout` will be set as `proxy_read_timeout` and `proxy_send_timeout` when you set backend protocol to `GRPC` or `GRPCS`. + 1. If your server only does response streaming and you expect a stream to be open longer than 60 seconds, you will have to change the `grpc_read_timeout` to accommodate this. 2. If your service only does request streaming and you expect a stream to be open longer than 60 seconds, you have to change the `grpc_send_timeout` and the `client_body_timeout`. 3. If you do both response and request streaming with an open stream longer than 60 seconds, you have to change all three timeouts: `grpc_read_timeout`, `grpc_send_timeout` and `client_body_timeout`. - -Values for the timeouts must be specified as e.g. `"1200s"`. - -> On the most recent versions of ingress-nginx, changing these timeouts requires using the `nginx.ingress.kubernetes.io/server-snippet` annotation. There are plans for future releases to allow using the Kubernetes annotations to define each timeout separately. From 41b01a569f98b2c3a139ef9380f79144300f6c35 Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Tue, 16 Apr 2024 11:44:22 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20links=20and=20?= =?UTF-8?q?default=20values=20for=20proxy=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user-guide/nginx-configuration/annotations.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md index 2d8291d780..d44d7981c8 100755 --- a/docs/user-guide/nginx-configuration/annotations.md +++ b/docs/user-guide/nginx-configuration/annotations.md @@ -700,13 +700,11 @@ In some scenarios is required to have different values. To allow this we provide - `nginx.ingress.kubernetes.io/proxy-next-upstream-tries` - `nginx.ingress.kubernetes.io/proxy-request-buffering` -If you indicate [Backend Protocol](#backend-protocol) as `GRPC` or `GRPCS`, the following values will be applied to gRPC connection as well: +If you indicate [Backend Protocol](#backend-protocol) as `GRPC` or `GRPCS`, the following grpc values will be set and inherited from proxy timeouts: -``` -grpc_connect_timeout {{ proxy_connect_timeout }} -grpc_send_timeout {{ proxy_send_timeout }} -grpc_read_timeout {{ proxy_read_timeout }} -``` +- [`grpc_connect_timeout=5s`](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout), from `nginx.ingress.kubernetes.io/proxy-connect-timeout` +- [`grpc_send_timeout=60s`](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_send_timeout), from `nginx.ingress.kubernetes.io/proxy-send-timeout` +- [`grpc_read_timeout=60s`](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout), from `nginx.ingress.kubernetes.io/proxy-read-timeout` Note: All timeout values are unitless and in seconds e.g. `nginx.ingress.kubernetes.io/proxy-read-timeout: "120"` sets a valid 120 seconds proxy read timeout. From 6eb6949c734205fa0bb6ce9a8617c3d421439ec7 Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Thu, 25 Apr 2024 11:12:25 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=A7=AA=20test:=20add=20e2e=20test=20f?= =?UTF-8?q?or=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + go.sum | 2 + test/e2e/annotations/grpc.go | 87 +++++++++++++++++------- test/e2e/framework/grpc_delay.go | 110 +++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 23 deletions(-) create mode 100644 test/e2e/framework/grpc_delay.go diff --git a/go.mod b/go.mod index cec60ef9c9..75782885ae 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( ) require ( + github.com/Anddd7/pb v0.0.0-20240425032658-369b0f6a404c github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 1ad706885a..d62944e1f5 100644 --- a/go.sum +++ b/go.sum @@ -597,6 +597,8 @@ cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcP dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Anddd7/pb v0.0.0-20240425032658-369b0f6a404c h1:uhBf0CHXi7nCFZXxHV7l1cBcYFEEVRK4FYxvm1l9lKg= +github.com/Anddd7/pb v0.0.0-20240425032658-369b0f6a404c/go.mod h1:vYWKbnXd2KAZHUECLPzSE0Er3FgiEmOdPtxwSIRihck= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= diff --git a/test/e2e/annotations/grpc.go b/test/e2e/annotations/grpc.go index 96cd227d99..95330e6e77 100644 --- a/test/e2e/annotations/grpc.go +++ b/test/e2e/annotations/grpc.go @@ -22,11 +22,13 @@ import ( "fmt" "strings" + delaypb "github.com/Anddd7/pb/grpcbin" pb "github.com/moul/pb/grpcbin/go-grpc" "github.com/onsi/ginkgo/v2" "github.com/stretchr/testify/assert" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -261,47 +263,86 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { assert.Equal(ginkgo.GinkgoT(), metadata["content-type"].Values[0], "application/grpc") }) - ginkgo.It("should set valid grpc timeouts for grpc", func() { - proxyConnectTimeout := "5" - proxySendTimeout := "30" - proxyReadtimeout := "30" + ginkgo.It("should return OK when request not exceed timeout", func() { + f.NewGRPCBinDelayDeployment() + + proxyTimeout := "10" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPC" - annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyConnectTimeout - annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxySendTimeout - annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyReadtimeout + annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyTimeout + annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxyTimeout + annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyTimeout + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, "grpcbin-delay", 50051, annotations) - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("grpc_connect_timeout %ss;", proxyConnectTimeout)) && - strings.Contains(server, fmt.Sprintf("grpc_send_timeout %ss;", proxySendTimeout)) && - strings.Contains(server, fmt.Sprintf("grpc_read_timeout %ss;", proxyReadtimeout)) + return strings.Contains(server, fmt.Sprintf("grpc_connect_timeout %ss;", proxyTimeout)) && + strings.Contains(server, fmt.Sprintf("grpc_send_timeout %ss;", proxyTimeout)) && + strings.Contains(server, fmt.Sprintf("grpc_read_timeout %ss;", proxyTimeout)) }) + + conn, err := grpc.Dial( + f.GetNginxIP()+":80", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithAuthority(host), + ) + assert.Nil(ginkgo.GinkgoT(), err, "error creating a connection") + defer conn.Close() + + client := delaypb.NewGrpcbinServiceClient(conn) + + res, err := client.Unary(context.Background(), &delaypb.UnaryRequest{ + Data: "hello", + }) + assert.Nil(ginkgo.GinkgoT(), err) + + metadata := res.GetResponseAttributes().RequestHeaders + assert.Equal(ginkgo.GinkgoT(), metadata["content-type"], "application/grpc") + assert.Equal(ginkgo.GinkgoT(), metadata[":authority"], host) }) - ginkgo.It("should set valid grpc timeouts for grpcs", func() { - proxyConnectTimeout := "5" - proxySendTimeout := "30" - proxyReadtimeout := "30" + ginkgo.It("should return Error when request exceed timeout", func() { + f.NewGRPCBinDelayDeployment() + + proxyTimeout := "10" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPCS" - annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyConnectTimeout - annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxySendTimeout - annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyReadtimeout + annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPC" + annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyTimeout + annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxyTimeout + annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyTimeout + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, "grpcbin-delay", 50051, annotations) - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("grpc_connect_timeout %ss;", proxyConnectTimeout)) && - strings.Contains(server, fmt.Sprintf("grpc_send_timeout %ss;", proxySendTimeout)) && - strings.Contains(server, fmt.Sprintf("grpc_read_timeout %ss;", proxyReadtimeout)) + return strings.Contains(server, fmt.Sprintf("grpc_connect_timeout %ss;", proxyTimeout)) && + strings.Contains(server, fmt.Sprintf("grpc_send_timeout %ss;", proxyTimeout)) && + strings.Contains(server, fmt.Sprintf("grpc_read_timeout %ss;", proxyTimeout)) }) + + conn, err := grpc.Dial( + f.GetNginxIP()+":80", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithAuthority(host), + ) + assert.Nil(ginkgo.GinkgoT(), err, "error creating a connection") + defer conn.Close() + + client := delaypb.NewGrpcbinServiceClient(conn) + + _, err = client.Unary(context.Background(), &delaypb.UnaryRequest{ + Data: "hello", + RequestAttributes: &delaypb.RequestAttributes{ + Delay: 15, + }, + }) + assert.Error(ginkgo.GinkgoT(), err) }) }) diff --git a/test/e2e/framework/grpc_delay.go b/test/e2e/framework/grpc_delay.go new file mode 100644 index 0000000000..406084c730 --- /dev/null +++ b/test/e2e/framework/grpc_delay.go @@ -0,0 +1,110 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//nolint:dupl // Ignore dupl errors for similar test case +package framework + +import ( + "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// NewGRPCBinDelayDeployment creates a new single replica +// deployment of the grpcbin image in a particular namespace +func (f *Framework) NewGRPCBinDelayDeployment() { + f.NewNewGRPCBinDelayDeploymentWithReplicas(1) +} + +// NewNewGRPCBinDelayDeploymentWithReplicas creates a new deployment of the +// grpcbin image in a particular namespace. Number of replicas is configurable +func (f *Framework) NewNewGRPCBinDelayDeploymentWithReplicas(replicas int32) { + name := "grpcbin-delay" + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: f.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: NewInt32(replicas), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": name, + }, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: NewInt64(0), + Containers: []corev1.Container{ + { + Name: name, + Image: "ghcr.io/anddd7/grpcbin:v1.0.5", + Env: []corev1.EnvVar{}, + Ports: []corev1.ContainerPort{ + { + Name: "grpc", + ContainerPort: 50051, + }, + }, + }, + }, + }, + }, + }, + } + + d := f.EnsureDeployment(deployment) + + err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, &metav1.ListOptions{ + LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), + }) + assert.Nil(ginkgo.GinkgoT(), err, "failed to wait for to become ready") + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: f.Namespace, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "grpc", + Port: 50051, + TargetPort: intstr.FromInt(50051), + Protocol: "TCP", + }, + }, + Selector: map[string]string{ + "app": name, + }, + }, + } + + f.EnsureService(service) + + err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, int(replicas)) + assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") +} From ae589edf8f8ce1d904e14d46d2fe4d6bf99d365d Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Thu, 25 Apr 2024 21:08:17 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=90=9B=20fix:=20upgrade=20to=201.0.6?= =?UTF-8?q?=20to=20fix=20nil=20pointer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/e2e/framework/grpc_delay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/framework/grpc_delay.go b/test/e2e/framework/grpc_delay.go index 406084c730..1efccc3678 100644 --- a/test/e2e/framework/grpc_delay.go +++ b/test/e2e/framework/grpc_delay.go @@ -61,7 +61,7 @@ func (f *Framework) NewNewGRPCBinDelayDeploymentWithReplicas(replicas int32) { Containers: []corev1.Container{ { Name: name, - Image: "ghcr.io/anddd7/grpcbin:v1.0.5", + Image: "ghcr.io/anddd7/grpcbin:v1.0.6", Env: []corev1.EnvVar{}, Ports: []corev1.ContainerPort{ { From fbf77e3ef1d11b76e12008bae29d0042c7d47c70 Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Thu, 25 Apr 2024 21:11:32 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=90=9B=20fix:=20lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/e2e/framework/grpc_delay.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/framework/grpc_delay.go b/test/e2e/framework/grpc_delay.go index 1efccc3678..58d10b2e97 100644 --- a/test/e2e/framework/grpc_delay.go +++ b/test/e2e/framework/grpc_delay.go @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -//nolint:dupl // Ignore dupl errors for similar test case package framework import ( From ee204a7b96a2af3f8737527daff07b46eaa4b599 Mon Sep 17 00:00:00 2001 From: Anddd7 Date: Thu, 2 May 2024 20:51:35 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=A7=AA=20test:=20trigger=20ci?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/e2e/annotations/grpc.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e/annotations/grpc.go b/test/e2e/annotations/grpc.go index 95330e6e77..60696e1395 100644 --- a/test/e2e/annotations/grpc.go +++ b/test/e2e/annotations/grpc.go @@ -267,6 +267,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { f.NewGRPCBinDelayDeployment() proxyTimeout := "10" + ingressName := "grpcbin-delay" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPC" @@ -274,7 +275,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxyTimeout annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyTimeout - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, "grpcbin-delay", 50051, annotations) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, ingressName, 50051, annotations) f.EnsureIngress(ing) @@ -309,6 +310,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { f.NewGRPCBinDelayDeployment() proxyTimeout := "10" + ingressName := "grpcbin-delay" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPC" @@ -316,7 +318,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxyTimeout annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyTimeout - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, "grpcbin-delay", 50051, annotations) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, ingressName, 50051, annotations) f.EnsureIngress(ing)