From 0b0b0836ff79fafc4048be2e2790f36b51a93ee7 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Mon, 12 Sep 2022 17:47:16 +0300 Subject: [PATCH 01/15] update support --- fakestorage/server.go | 1 + fakestorage/upload.go | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/fakestorage/server.go b/fakestorage/server.go index 73eed3c300..b09a19c79d 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -251,6 +251,7 @@ func (s *Server) buildMuxer() { s.mux.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject) s.mux.Path("/download/storage/v1/b/{bucketName}/o/{objectName:.+}").Methods(http.MethodGet).HandlerFunc(s.downloadObject) s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.insertObject)) + s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) s.mux.Path("/upload/resumable/{uploadId}").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) // Batch endpoint diff --git a/fakestorage/upload.go b/fakestorage/upload.go index 21664e1beb..a9def256fe 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -203,9 +203,19 @@ func (s *Server) checkUploadPreconditions(r *http.Request, bucketName string, ob return nil } +func normalizeFileName(name string) string { + for { + name = strings.Replace(name, "/", "", 1) + if !strings.HasPrefix(name, "/") { + break + } + } + return name +} + func (s *Server) simpleUpload(bucketName string, r *http.Request) jsonResponse { defer r.Body.Close() - name := r.URL.Query().Get("name") + name := normalizeFileName(r.URL.Query().Get("name")) predefinedACL := r.URL.Query().Get("predefinedAcl") contentEncoding := r.URL.Query().Get("contentEncoding") if name == "" { @@ -332,7 +342,7 @@ func (s *Server) multipartUpload(bucketName string, r *http.Request) jsonRespons return jsonResponse{errorMessage: err.Error()} } - objName := r.URL.Query().Get("name") + objName := normalizeFileName(r.URL.Query().Get("name")) predefinedACL := r.URL.Query().Get("predefinedAcl") if objName == "" { objName = metadata.Name @@ -372,7 +382,7 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons return jsonResponse{errorMessage: err.Error()} } } - objName := r.URL.Query().Get("name") + objName := normalizeFileName(r.URL.Query().Get("name")) if objName == "" { objName = metadata.Name } @@ -391,9 +401,10 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons } s.uploads.Store(uploadID, obj) header := make(http.Header) - header.Set("Location", s.URL()+"/upload/resumable/"+uploadID) + location := s.URL() + "/upload/storage/v1/b/" + bucketName + "/o?uploadType=resumable&name=" + objName + "&upload_id=" + uploadID + header.Set("Location", location) if r.Header.Get("X-Goog-Upload-Command") == "start" { - header.Set("X-Goog-Upload-URL", s.URL()+"/upload/resumable/"+uploadID) + header.Set("X-Goog-Upload-URL", location) header.Set("X-Goog-Upload-Status", "active") } return jsonResponse{ @@ -438,7 +449,7 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons // then has a status of "200 OK", with a header "X-Http-Status-Code-Override" // set to "308". func (s *Server) uploadFileContent(r *http.Request) jsonResponse { - uploadID := mux.Vars(r)["uploadId"] + uploadID := r.URL.Query().Get("upload_id") rawObj, ok := s.uploads.Load(uploadID) if !ok { return jsonResponse{status: http.StatusNotFound} From 85740a1ba78219aa752799a1c1a67d412e5450a4 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Mon, 12 Sep 2022 18:53:05 +0300 Subject: [PATCH 02/15] update support --- fakestorage/upload.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index a9def256fe..490d4feaff 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -204,13 +204,14 @@ func (s *Server) checkUploadPreconditions(r *http.Request, bucketName string, ob } func normalizeFileName(name string) string { - for { - name = strings.Replace(name, "/", "", 1) - if !strings.HasPrefix(name, "/") { - break - } - } return name + // for { + // name = strings.Replace(name, "/", "", 1) + // if !strings.HasPrefix(name, "/") { + // break + // } + // } + // return name } func (s *Server) simpleUpload(bucketName string, r *http.Request) jsonResponse { From be16d560405a91d270073f8c521e4de327f73147 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Mon, 12 Sep 2022 18:55:42 +0300 Subject: [PATCH 03/15] update support --- fakestorage/server.go | 2 +- fakestorage/upload.go | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/fakestorage/server.go b/fakestorage/server.go index b09a19c79d..d259413cf1 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -251,8 +251,8 @@ func (s *Server) buildMuxer() { s.mux.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject) s.mux.Path("/download/storage/v1/b/{bucketName}/o/{objectName:.+}").Methods(http.MethodGet).HandlerFunc(s.downloadObject) s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.insertObject)) - s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) s.mux.Path("/upload/resumable/{uploadId}").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) + s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) // Batch endpoint s.mux.MatcherFunc(s.publicHostMatcher).Path("/batch/storage/v1").Methods(http.MethodPost).HandlerFunc(s.handleBatchCall) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index 490d4feaff..a9def256fe 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -204,14 +204,13 @@ func (s *Server) checkUploadPreconditions(r *http.Request, bucketName string, ob } func normalizeFileName(name string) string { + for { + name = strings.Replace(name, "/", "", 1) + if !strings.HasPrefix(name, "/") { + break + } + } return name - // for { - // name = strings.Replace(name, "/", "", 1) - // if !strings.HasPrefix(name, "/") { - // break - // } - // } - // return name } func (s *Server) simpleUpload(bucketName string, r *http.Request) jsonResponse { From 434ddfe820150b4e8e95185664e086039c1ecfa7 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Mon, 12 Sep 2022 20:11:21 +0300 Subject: [PATCH 04/15] fix upload --- fakestorage/upload.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index a9def256fe..e92618a710 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -376,6 +376,10 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons contentEncoding := r.URL.Query().Get("contentEncoding") metadata := new(multipartMetadata) if r.Body != http.NoBody { + if r.URL.Query().Has("upload_id") { + return s.uploadFileContent(r) + } + var err error metadata, err = loadMetadata(r.Body) if err != nil { From 0e5e65fc060bbd67e1dd2e4fd7a4cd55de92738c Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Mon, 12 Sep 2022 20:23:42 +0300 Subject: [PATCH 05/15] fix routes order --- fakestorage/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fakestorage/server.go b/fakestorage/server.go index d259413cf1..b09a19c79d 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -251,8 +251,8 @@ func (s *Server) buildMuxer() { s.mux.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject) s.mux.Path("/download/storage/v1/b/{bucketName}/o/{objectName:.+}").Methods(http.MethodGet).HandlerFunc(s.downloadObject) s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.insertObject)) - s.mux.Path("/upload/resumable/{uploadId}").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) + s.mux.Path("/upload/resumable/{uploadId}").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) // Batch endpoint s.mux.MatcherFunc(s.publicHostMatcher).Path("/batch/storage/v1").Methods(http.MethodPost).HandlerFunc(s.handleBatchCall) From 41e8db7594af4433bf6b0818ef463df0764cd16d Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Tue, 13 Sep 2022 10:00:06 +0300 Subject: [PATCH 06/15] Fix normalizeFileName --- fakestorage/upload.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index e92618a710..5bcbf427ee 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -204,13 +204,7 @@ func (s *Server) checkUploadPreconditions(r *http.Request, bucketName string, ob } func normalizeFileName(name string) string { - for { - name = strings.Replace(name, "/", "", 1) - if !strings.HasPrefix(name, "/") { - break - } - } - return name + return strings.TrimLeft(name, "/") } func (s *Server) simpleUpload(bucketName string, r *http.Request) jsonResponse { From c62771be3c8a8c6414f159bbc485ea73abe8b0e0 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Tue, 13 Sep 2022 12:19:06 +0300 Subject: [PATCH 07/15] Fix escaping for mux vars --- fakestorage/bucket.go | 4 ++-- fakestorage/object.go | 34 ++++++++++++++++++++++++---------- fakestorage/server.go | 2 +- fakestorage/upload.go | 19 ++++++++----------- 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/fakestorage/bucket.go b/fakestorage/bucket.go index 49aedb1333..72bd7029be 100644 --- a/fakestorage/bucket.go +++ b/fakestorage/bucket.go @@ -105,7 +105,7 @@ func (s *Server) listBuckets(r *http.Request) jsonResponse { } func (s *Server) getBucket(r *http.Request) jsonResponse { - bucketName := mux.Vars(r)["bucketName"] + bucketName := unescapeMuxVars(mux.Vars(r))["bucketName"] bucket, err := s.backend.GetBucket(bucketName) if err != nil { return jsonResponse{status: http.StatusNotFound} @@ -114,7 +114,7 @@ func (s *Server) getBucket(r *http.Request) jsonResponse { } func (s *Server) deleteBucket(r *http.Request) jsonResponse { - bucketName := mux.Vars(r)["bucketName"] + bucketName := unescapeMuxVars(mux.Vars(r))["bucketName"] err := s.backend.DeleteBucket(bucketName) if err == backend.BucketNotFound { return jsonResponse{status: http.StatusNotFound} diff --git a/fakestorage/object.go b/fakestorage/object.go index 173be62074..c012315353 100644 --- a/fakestorage/object.go +++ b/fakestorage/object.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "net/http" + "net/url" "sort" "strconv" "strings" @@ -564,8 +565,21 @@ func (s *Server) objectWithGenerationOnValidGeneration(bucketName, objectName, g return s.GetObjectStreaming(bucketName, objectName) } +func unescapeMuxVars(vars map[string]string) map[string]string { + m := make(map[string]string) + for k, v := range vars { + r, err := url.PathUnescape(v) + if err == nil { + m[k] = r + } else { + m[k] = v + } + } + return m +} + func (s *Server) listObjects(r *http.Request) jsonResponse { - bucketName := mux.Vars(r)["bucketName"] + bucketName := unescapeMuxVars(mux.Vars(r))["bucketName"] objs, prefixes, err := s.ListObjectsWithOptions(bucketName, ListOptions{ Prefix: r.URL.Query().Get("prefix"), Delimiter: r.URL.Query().Get("delimiter"), @@ -587,7 +601,7 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { } handler := jsonToHTTPHandler(func(r *http.Request) jsonResponse { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) obj, err := s.objectWithGenerationOnValidGeneration(vars["bucketName"], vars["objectName"], r.FormValue("generation")) // Calling Close before checking err is okay on objects, and the object @@ -617,7 +631,7 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { } func (s *Server) deleteObject(r *http.Request) jsonResponse { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) obj, err := s.GetObjectStreaming(vars["bucketName"], vars["objectName"]) // Calling Close before checking err is okay on objects, and the object // may need to be closed whether or not there's an error. @@ -639,7 +653,7 @@ func (s *Server) deleteObject(r *http.Request) jsonResponse { } func (s *Server) listObjectACL(r *http.Request) jsonResponse { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) obj, err := s.GetObjectStreaming(vars["bucketName"], vars["objectName"]) if err != nil { @@ -651,7 +665,7 @@ func (s *Server) listObjectACL(r *http.Request) jsonResponse { } func (s *Server) setObjectACL(r *http.Request) jsonResponse { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) obj, err := s.GetObjectStreaming(vars["bucketName"], vars["objectName"]) if err != nil { @@ -689,7 +703,7 @@ func (s *Server) setObjectACL(r *http.Request) jsonResponse { } func (s *Server) rewriteObject(r *http.Request) jsonResponse { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) obj, err := s.objectWithGenerationOnValidGeneration(vars["sourceBucket"], vars["sourceObject"], r.FormValue("sourceGeneration")) // Calling Close before checking err is okay on objects, and the object // may need to be closed whether or not there's an error. @@ -744,7 +758,7 @@ func (s *Server) rewriteObject(r *http.Request) jsonResponse { } func (s *Server) downloadObject(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) obj, err := s.objectWithGenerationOnValidGeneration(vars["bucketName"], vars["objectName"], r.FormValue("generation")) // Calling Close before checking err is okay on objects, and the object // may need to be closed whether or not there's an error. @@ -901,7 +915,7 @@ func parseRange(rangeHeaderValue string, contentLength int64) (start int64, end } func (s *Server) patchObject(r *http.Request) jsonResponse { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) bucketName := vars["bucketName"] objectName := vars["objectName"] var metadata struct { @@ -928,7 +942,7 @@ func (s *Server) patchObject(r *http.Request) jsonResponse { } func (s *Server) updateObject(r *http.Request) jsonResponse { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) bucketName := vars["bucketName"] objectName := vars["objectName"] var metadata struct { @@ -955,7 +969,7 @@ func (s *Server) updateObject(r *http.Request) jsonResponse { } func (s *Server) composeObject(r *http.Request) jsonResponse { - vars := mux.Vars(r) + vars := unescapeMuxVars(mux.Vars(r)) bucketName := vars["bucketName"] destinationObject := vars["destinationObject"] diff --git a/fakestorage/server.go b/fakestorage/server.go index b09a19c79d..769d52100a 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -213,7 +213,7 @@ func newServer(options Options) (*Server, error) { func (s *Server) buildMuxer() { const apiPrefix = "/storage/v1" - s.mux = mux.NewRouter() + s.mux = mux.NewRouter().SkipClean(true).UseEncodedPath() // healthcheck s.mux.Path("/_internal/healthcheck").Methods(http.MethodGet).HandlerFunc(s.healthcheck) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index 5bcbf427ee..a5e5389e03 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -14,6 +14,7 @@ import ( "mime" "mime/multipart" "net/http" + "net/url" "strconv" "strings" @@ -46,7 +47,7 @@ type contentRange struct { } func (s *Server) insertObject(r *http.Request) jsonResponse { - bucketName := mux.Vars(r)["bucketName"] + bucketName := unescapeMuxVars(mux.Vars(r))["bucketName"] if _, err := s.backend.GetBucket(bucketName); err != nil { return jsonResponse{status: http.StatusNotFound} @@ -78,7 +79,7 @@ func (s *Server) insertObject(r *http.Request) jsonResponse { } func (s *Server) insertFormObject(r *http.Request) xmlResponse { - bucketName := mux.Vars(r)["bucketName"] + bucketName := unescapeMuxVars(mux.Vars(r))["bucketName"] if err := r.ParseMultipartForm(32 << 20); nil != err { return xmlResponse{errorMessage: "invalid form", status: http.StatusBadRequest} @@ -203,13 +204,9 @@ func (s *Server) checkUploadPreconditions(r *http.Request, bucketName string, ob return nil } -func normalizeFileName(name string) string { - return strings.TrimLeft(name, "/") -} - func (s *Server) simpleUpload(bucketName string, r *http.Request) jsonResponse { defer r.Body.Close() - name := normalizeFileName(r.URL.Query().Get("name")) + name := r.URL.Query().Get("name") predefinedACL := r.URL.Query().Get("predefinedAcl") contentEncoding := r.URL.Query().Get("contentEncoding") if name == "" { @@ -246,7 +243,7 @@ func (s notImplementedSeeker) Seek(offset int64, whence int) (int64, error) { func (s *Server) signedUpload(bucketName string, r *http.Request) jsonResponse { defer r.Body.Close() - name := mux.Vars(r)["objectName"] + name := r.URL.Query().Get("name") predefinedACL := r.URL.Query().Get("predefinedAcl") contentEncoding := r.URL.Query().Get("contentEncoding") @@ -336,7 +333,7 @@ func (s *Server) multipartUpload(bucketName string, r *http.Request) jsonRespons return jsonResponse{errorMessage: err.Error()} } - objName := normalizeFileName(r.URL.Query().Get("name")) + objName := r.URL.Query().Get("name") predefinedACL := r.URL.Query().Get("predefinedAcl") if objName == "" { objName = metadata.Name @@ -380,7 +377,7 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons return jsonResponse{errorMessage: err.Error()} } } - objName := normalizeFileName(r.URL.Query().Get("name")) + objName := r.URL.Query().Get("name") if objName == "" { objName = metadata.Name } @@ -399,7 +396,7 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons } s.uploads.Store(uploadID, obj) header := make(http.Header) - location := s.URL() + "/upload/storage/v1/b/" + bucketName + "/o?uploadType=resumable&name=" + objName + "&upload_id=" + uploadID + location := fmt.Sprintf("%s/upload/storage/v1/b/%s/o?uploadType=resumable&name=%s&upload_id=%s", s.URL(), bucketName, url.PathEscape(objName), uploadID) header.Set("Location", location) if r.Header.Get("X-Goog-Upload-Command") == "start" { header.Set("X-Goog-Upload-URL", location) From 279263659e0e9d1d70e11d1efe914a668db06902 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Tue, 13 Sep 2022 12:30:39 +0300 Subject: [PATCH 08/15] Fix escaping for mux vars --- fakestorage/upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index a5e5389e03..7fa4ae478d 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -243,7 +243,7 @@ func (s notImplementedSeeker) Seek(offset int64, whence int) (int64, error) { func (s *Server) signedUpload(bucketName string, r *http.Request) jsonResponse { defer r.Body.Close() - name := r.URL.Query().Get("name") + name := unescapeMuxVars(mux.Vars(r))["objectName"] predefinedACL := r.URL.Query().Get("predefinedAcl") contentEncoding := r.URL.Query().Get("contentEncoding") From 6412da78cc97ccceac3a34df775832b0543b6576 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Tue, 13 Sep 2022 14:11:34 +0300 Subject: [PATCH 09/15] Fix escaping for mux vars --- fakestorage/upload.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index 7fa4ae478d..abe5a2cb05 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -396,7 +396,13 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons } s.uploads.Store(uploadID, obj) header := make(http.Header) - location := fmt.Sprintf("%s/upload/storage/v1/b/%s/o?uploadType=resumable&name=%s&upload_id=%s", s.URL(), bucketName, url.PathEscape(objName), uploadID) + location := fmt.Sprintf( + "%s/upload/storage/v1/b/%s/o?uploadType=resumable&name=%s&upload_id=%s", + s.URL(), + url.PathEscape(bucketName), + url.PathEscape(objName), + uploadID, + ) header.Set("Location", location) if r.Header.Get("X-Goog-Upload-Command") == "start" { header.Set("X-Goog-Upload-URL", location) From a5991c7209a3b75955f0eb2e9d85e84bfb9fe165 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Tue, 13 Sep 2022 22:29:27 +0300 Subject: [PATCH 10/15] fix check and tests --- fakestorage/upload.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index abe5a2cb05..3f509063df 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -363,14 +363,13 @@ func (s *Server) multipartUpload(bucketName string, r *http.Request) jsonRespons } func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonResponse { + if r.URL.Query().Has("upload_id") { + return s.uploadFileContent(r) + } predefinedACL := r.URL.Query().Get("predefinedAcl") contentEncoding := r.URL.Query().Get("contentEncoding") metadata := new(multipartMetadata) if r.Body != http.NoBody { - if r.URL.Query().Has("upload_id") { - return s.uploadFileContent(r) - } - var err error metadata, err = loadMetadata(r.Body) if err != nil { From 07a0d5997f398d2eda382ca4ad270ea59c6fa7e4 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Wed, 14 Sep 2022 12:17:12 +0300 Subject: [PATCH 11/15] move unescapeMuxVars to server --- fakestorage/object.go | 14 -------------- fakestorage/server.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/fakestorage/object.go b/fakestorage/object.go index c012315353..3ae2cf0cd5 100644 --- a/fakestorage/object.go +++ b/fakestorage/object.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "net/http" - "net/url" "sort" "strconv" "strings" @@ -565,19 +564,6 @@ func (s *Server) objectWithGenerationOnValidGeneration(bucketName, objectName, g return s.GetObjectStreaming(bucketName, objectName) } -func unescapeMuxVars(vars map[string]string) map[string]string { - m := make(map[string]string) - for k, v := range vars { - r, err := url.PathUnescape(v) - if err == nil { - m[k] = r - } else { - m[k] = v - } - } - return m -} - func (s *Server) listObjects(r *http.Request) jsonResponse { bucketName := unescapeMuxVars(mux.Vars(r))["bucketName"] objs, prefixes, err := s.ListObjectsWithOptions(bucketName, ListOptions{ diff --git a/fakestorage/server.go b/fakestorage/server.go index 769d52100a..cc95fe1383 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -19,6 +19,7 @@ import ( "net/http/httptest" "net/http/httputil" "net/textproto" + "net/url" "strings" "sync" @@ -211,6 +212,19 @@ func newServer(options Options) (*Server, error) { return &s, nil } +func unescapeMuxVars(vars map[string]string) map[string]string { + m := make(map[string]string) + for k, v := range vars { + r, err := url.PathUnescape(v) + if err == nil { + m[k] = r + } else { + m[k] = v + } + } + return m +} + func (s *Server) buildMuxer() { const apiPrefix = "/storage/v1" s.mux = mux.NewRouter().SkipClean(true).UseEncodedPath() From 71bb72ea87df94d47e129992b8896554d0257546 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Mon, 19 Sep 2022 14:01:48 +0300 Subject: [PATCH 12/15] add new test cases --- fakestorage/upload_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go index 1687e516b3..836221f056 100644 --- a/fakestorage/upload_test.go +++ b/fakestorage/upload_test.go @@ -48,6 +48,18 @@ func TestServerClientObjectWriter(t *testing.T) { "other/interesting/object.txt", googleapi.MinUploadChunkSize, }, + { + "file with backslash at beginning", + "other-bucket", + "/some/other/object.txt", + googleapi.DefaultUploadChunkSize, + }, + { + "file with backslashes at name", + "other-bucket", + "//some//other//file.txt", + googleapi.MinUploadChunkSize, + }, } for _, test := range tests { From 8695101f1cd171729db453503037dcc384e38995 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Mon, 19 Sep 2022 14:04:17 +0300 Subject: [PATCH 13/15] add new test cases --- fakestorage/upload_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go index 836221f056..af476f3d5a 100644 --- a/fakestorage/upload_test.go +++ b/fakestorage/upload_test.go @@ -14,6 +14,7 @@ import ( "io" "mime/multipart" "net/http" + "net/url" "reflect" "strings" "testing" @@ -805,8 +806,8 @@ func TestServerGzippedUpload(t *testing.T) { t.Fatal(err) } - url := server.URL() - req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/upload/storage/v1/b/%s/o?name=testobj&uploadType=media", url, bucketName), &buf) + serverUrl := server.URL() + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/upload/storage/v1/b/%s/o?name=testobj&uploadType=media", serverUrl, url.PathEscape(bucketName)), &buf) if err != nil { t.Fatal(err) } From d18156ba71466a28969963155bf3b34e9159102c Mon Sep 17 00:00:00 2001 From: Oleksii Vasyliev Date: Mon, 19 Sep 2022 15:07:45 +0300 Subject: [PATCH 14/15] Update fakestorage/upload.go Co-authored-by: fsouza <108725+fsouza@users.noreply.github.com> --- fakestorage/upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fakestorage/upload.go b/fakestorage/upload.go index 3f509063df..127ecb2187 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -398,7 +398,7 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons location := fmt.Sprintf( "%s/upload/storage/v1/b/%s/o?uploadType=resumable&name=%s&upload_id=%s", s.URL(), - url.PathEscape(bucketName), + bucketName, url.PathEscape(objName), uploadID, ) From ed3cf5dd68df8094f5ffab49a0e074aaa92a3606 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Mon, 19 Sep 2022 15:09:22 +0300 Subject: [PATCH 15/15] remove bucket escaping --- fakestorage/upload_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go index af476f3d5a..da68465ea9 100644 --- a/fakestorage/upload_test.go +++ b/fakestorage/upload_test.go @@ -14,7 +14,6 @@ import ( "io" "mime/multipart" "net/http" - "net/url" "reflect" "strings" "testing" @@ -807,7 +806,7 @@ func TestServerGzippedUpload(t *testing.T) { } serverUrl := server.URL() - req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/upload/storage/v1/b/%s/o?name=testobj&uploadType=media", serverUrl, url.PathEscape(bucketName)), &buf) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/upload/storage/v1/b/%s/o?name=testobj&uploadType=media", serverUrl, bucketName), &buf) if err != nil { t.Fatal(err) }