From d9e887bedc654282cddd4480239bbba0aa280c4b Mon Sep 17 00:00:00 2001 From: Chris Cotter Date: Thu, 9 Jun 2022 20:41:20 -0400 Subject: [PATCH 1/2] chore(gensupport): add retry invocation headers Adds invocation ID and attempt count to x-goog-api-client header for simple and resumable uploads. Note that this will also add this information to chunk upload requests for individual chunks for BQ and other APIs that do uploads using gensupport as well. Hopefully this isn't an issue. Similar to googleapis/google-cloud-go#6013 --- go.mod | 1 + go.sum | 1 + internal/gensupport/resumable.go | 18 +++++++++++++++++- internal/gensupport/send.go | 10 ++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 86ad5349d96..6e8967d2a17 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( cloud.google.com/go/compute v1.6.1 github.com/google/go-cmp v0.5.8 + github.com/google/uuid v1.1.2 github.com/googleapis/gax-go/v2 v2.4.0 go.opencensus.io v0.23.0 golang.org/x/net v0.0.0-20220607020251-c690dde0001d diff --git a/go.sum b/go.sum index 1af5ae00f82..5163a978af4 100644 --- a/go.sum +++ b/go.sum @@ -156,6 +156,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= diff --git a/internal/gensupport/resumable.go b/internal/gensupport/resumable.go index 0eae147fa92..54de000796d 100644 --- a/internal/gensupport/resumable.go +++ b/internal/gensupport/resumable.go @@ -10,8 +10,12 @@ import ( "fmt" "io" "net/http" + "strings" "sync" "time" + + "github.com/google/uuid" + "google.golang.org/api/internal" ) // ResumableUpload is used by the generated APIs to provide resumable uploads. @@ -38,6 +42,11 @@ type ResumableUpload struct { // ChunkRetryDeadline configures the per-chunk deadline after which no further // retries should happen. ChunkRetryDeadline time.Duration + + // Track current request invocation ID and attempt count for retry metric + // headers. + invocationID string + attempts int } // Progress returns the number of bytes uploaded at this point. @@ -72,6 +81,10 @@ func (rx *ResumableUpload) doUploadRequest(ctx context.Context, data io.Reader, req.Header.Set("Content-Type", rx.MediaType) req.Header.Set("User-Agent", rx.UserAgent) + baseXGoogHeader := "gl-go/" + GoVersion() + " gdcl/" + internal.Version + invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", rx.invocationID, rx.attempts) + req.Header.Set("X-Goog-Api-Client", strings.Join([]string{baseXGoogHeader, invocationHeader}, " ")) + // Google's upload endpoint uses status code 308 for a // different purpose than the "308 Permanent Redirect" // since-standardized in RFC 7238. Because of the conflict in @@ -178,9 +191,11 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err for { var pause time.Duration - // Each chunk gets its own initialized-at-zero backoff. + // Each chunk gets its own initialized-at-zero backoff and invocation ID. bo := rx.Retry.backoff() quitAfter := time.After(retryDeadline) + rx.attempts = 1 + rx.invocationID = uuid.New().String() // Retry loop for a single chunk. for { @@ -223,6 +238,7 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err break } + rx.attempts++ pause = bo.Pause() if resp != nil && resp.Body != nil { resp.Body.Close() diff --git a/internal/gensupport/send.go b/internal/gensupport/send.go index dd50cc20a58..d7f83cd8b07 100644 --- a/internal/gensupport/send.go +++ b/internal/gensupport/send.go @@ -8,9 +8,12 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" + "strings" "time" + "github.com/google/uuid" "github.com/googleapis/gax-go/v2" ) @@ -71,6 +74,9 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r var resp *http.Response var err error + attempts := 1 + invocationID := uuid.New().String() + baseXGoogHeader := req.Header.Get("X-Goog-Api-Client") // Loop to retry the request, up to the context deadline. var pause time.Duration @@ -109,6 +115,9 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r } return resp, err } + invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", invocationID, attempts) + xGoogHeader := strings.Join([]string{invocationHeader, baseXGoogHeader}, " ") + req.Header.Set("X-Goog-Api-Client", xGoogHeader) resp, err = client.Do(req.WithContext(ctx)) @@ -123,6 +132,7 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r if req.GetBody == nil || !errorFunc(status, err) { break } + attempts++ var errBody error req.Body, errBody = req.GetBody() if errBody != nil { From 7f5ebfdafd8fa1d4dd92053ac07bc0b3887c27d6 Mon Sep 17 00:00:00 2001 From: Chris Cotter Date: Tue, 21 Jun 2022 13:40:52 -0400 Subject: [PATCH 2/2] string formatter --- internal/gensupport/resumable.go | 2 +- internal/gensupport/send.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/gensupport/resumable.go b/internal/gensupport/resumable.go index 54de000796d..0c659188dda 100644 --- a/internal/gensupport/resumable.go +++ b/internal/gensupport/resumable.go @@ -82,7 +82,7 @@ func (rx *ResumableUpload) doUploadRequest(ctx context.Context, data io.Reader, req.Header.Set("User-Agent", rx.UserAgent) baseXGoogHeader := "gl-go/" + GoVersion() + " gdcl/" + internal.Version - invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", rx.invocationID, rx.attempts) + invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", rx.invocationID, rx.attempts) req.Header.Set("X-Goog-Api-Client", strings.Join([]string{baseXGoogHeader, invocationHeader}, " ")) // Google's upload endpoint uses status code 308 for a diff --git a/internal/gensupport/send.go b/internal/gensupport/send.go index d7f83cd8b07..70a8e01c1b3 100644 --- a/internal/gensupport/send.go +++ b/internal/gensupport/send.go @@ -115,7 +115,7 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r } return resp, err } - invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", invocationID, attempts) + invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", invocationID, attempts) xGoogHeader := strings.Join([]string{invocationHeader, baseXGoogHeader}, " ") req.Header.Set("X-Goog-Api-Client", xGoogHeader)