From 28cb47cfaaebf10288b81f1fa056deb936c73486 Mon Sep 17 00:00:00 2001 From: Gergo Schmall Date: Fri, 4 Feb 2022 17:33:44 +0100 Subject: [PATCH 1/5] Handle batch calls Add a rudamental test for batch delete --- fakestorage/server.go | 75 ++++++++++++++++++++++++++++++++ fakestorage/server_test.go | 89 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) diff --git a/fakestorage/server.go b/fakestorage/server.go index 99f37ee20a..3e12c33e59 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -5,14 +5,20 @@ package fakestorage import ( + "bufio" + "bytes" "compress/gzip" "context" "fmt" "io" "mime" + "mime/multipart" "net" "net/http" "net/http/httptest" + "net/http/httputil" + "net/textproto" + "strings" "sync" "cloud.google.com/go/storage" @@ -231,6 +237,10 @@ func (s *Server) buildMuxer() { s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.insertObject)) s.mux.Path("/upload/resumable/{uploadId}").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent)) + // Batch endpoint + s.mux.Host(s.publicHost).Path("/batch/storage/v1").Methods(http.MethodPost).HandlerFunc(s.handleBatchCall) + s.mux.Path("/batch/storage/v1").Methods(http.MethodPost).HandlerFunc(s.handleBatchCall) + s.mux.Host(s.publicHost).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject) s.mux.Host("{bucketName:.+}").Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject) @@ -291,6 +301,71 @@ func (s *Server) Client() *storage.Client { return client } +func (s *Server) handleBatchCall(w http.ResponseWriter, r *http.Request) { + reader, err := r.MultipartReader() + if err != nil { + http.Error(w, "invalid Content-Type header", http.StatusBadRequest) + return + } + + mw := multipart.NewWriter(w) + w.Header().Set("Content-Type", "multipart/mixed; boundary=" + mw.Boundary()) + + w.WriteHeader(http.StatusOK) + part, err := reader.NextPart() + for ; err == nil; part, err = reader.NextPart() { + contentId := part.Header.Get("Content-ID") + if contentId == "" { + // missing content ID, skip + continue + } + + partHeaders := textproto.MIMEHeader{} + partHeaders.Set("Content-Type", "application/http") + partHeaders.Set("Content-ID", strings.Replace(contentId, "<", "\r\n" + + "\r\n" + + "DELETE /storage/v1/b/some-bucket/o/files/txt/text-01.txt HTTP/1.1\r\n" + + "accept: application/json\r\n" + + "content-length: 2\r\n" + + "\r\n" + + "{}\r\n" + + "--===============7330845974216740156==\r\n" + + "Content-Type: application/http\r\n" + + "Content-Transfer-Encoding: binary\r\n" + + "Content-ID: \r\n" + + "\r\n" + + "DELETE /storage/v1/b/some-bucket/o/files/txt/text-02.txt HTTP/1.1\r\n" + + "accept: application/json\r\n" + + "content-length: 2\r\n" + + "\r\n" + + "{}\r\n" + + "--===============7330845974216740156==\r\n" + + "Content-Type: application/http\r\n" + + "Content-Transfer-Encoding: binary\r\n" + + "Content-ID: \r\n" + + "\r\n" + + "DELETE /storage/v1/b/some-bucket/o/files/txt/text-03.txt HTTP/1.1\r\n" + + "accept: application/json\r\n" + + "content-length: 2\r\n" + + "\r\n" + + "{}\r\n" + + "--===============7330845974216740156==--\r\n" + + client := server.HTTPClient() + req, err := http.NewRequest(http.MethodPost, "https://127.0.0.1/batch/storage/v1", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Origin", "http://example.com") + req.Header.Set("Access-Control-Request-Method", http.MethodPost) + req.Header.Set("Content-Type", "multipart/mixed; boundary=\"===============7330845974216740156==\"") + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("wrong status returned\nwant %d\ngot %d", http.StatusOK, resp.StatusCode) + } + + it := server.Client().Bucket("some-bucket").Objects(context.Background(), nil) + _, err = it.Next() + if err != iterator.Done { + t.Errorf("Objects didn't get deleted by batch call") + } +} + type fakeEventFields struct { BucketName string Name string From b5bbcbcd00da26673c70043c6632388686e2f9ba Mon Sep 17 00:00:00 2001 From: Gergo Schmall Date: Tue, 15 Feb 2022 15:24:28 +0100 Subject: [PATCH 2/5] Fix lint errors --- fakestorage/server.go | 2 +- fakestorage/server_test.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fakestorage/server.go b/fakestorage/server.go index 3e12c33e59..f6600c99d4 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -309,7 +309,7 @@ func (s *Server) handleBatchCall(w http.ResponseWriter, r *http.Request) { } mw := multipart.NewWriter(w) - w.Header().Set("Content-Type", "multipart/mixed; boundary=" + mw.Boundary()) + w.Header().Set("Content-Type", "multipart/mixed; boundary="+mw.Boundary()) w.WriteHeader(http.StatusOK) part, err := reader.NextPart() diff --git a/fakestorage/server_test.go b/fakestorage/server_test.go index e40ca465c3..6eaa3d3107 100644 --- a/fakestorage/server_test.go +++ b/fakestorage/server_test.go @@ -7,13 +7,14 @@ package fakestorage import ( "bytes" "context" - "google.golang.org/api/iterator" "io" "io/ioutil" "net/http" "strings" "testing" + "google.golang.org/api/iterator" + "cloud.google.com/go/storage" "github.com/fsouza/fake-gcs-server/internal/backend" "github.com/fsouza/fake-gcs-server/internal/notification" @@ -671,7 +672,7 @@ func TestServerEventNotification(t *testing.T) { } func TestServerBatchRequest(t *testing.T) { - objects := []Object { + objects := []Object{ { ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "files/txt/text-01.txt"}, Content: []byte("something1"), From b1843ad597dfc60c550a7699a80bcf8de96be5bb Mon Sep 17 00:00:00 2001 From: Gergo Schmall Date: Mon, 21 Feb 2022 17:45:15 +0100 Subject: [PATCH 3/5] Quick fixes, renames --- fakestorage/server.go | 17 ++++++++--------- fakestorage/server_test.go | 2 -- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/fakestorage/server.go b/fakestorage/server.go index f6600c99d4..0d069f8c12 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -309,20 +309,21 @@ func (s *Server) handleBatchCall(w http.ResponseWriter, r *http.Request) { } mw := multipart.NewWriter(w) + defer mw.Close() w.Header().Set("Content-Type", "multipart/mixed; boundary="+mw.Boundary()) w.WriteHeader(http.StatusOK) part, err := reader.NextPart() for ; err == nil; part, err = reader.NextPart() { - contentId := part.Header.Get("Content-ID") - if contentId == "" { + contentID := part.Header.Get("Content-ID") + if contentID == "" { // missing content ID, skip continue } partHeaders := textproto.MIMEHeader{} partHeaders.Set("Content-Type", "application/http") - partHeaders.Set("Content-ID", strings.Replace(contentId, "<", " Date: Mon, 21 Feb 2022 19:02:06 +0100 Subject: [PATCH 4/5] Import fix --- fakestorage/server_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/fakestorage/server_test.go b/fakestorage/server_test.go index 435558e137..b58b6bafdb 100644 --- a/fakestorage/server_test.go +++ b/fakestorage/server_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "google.golang.org/api/iterator" "cloud.google.com/go/storage" "github.com/fsouza/fake-gcs-server/internal/backend" "github.com/fsouza/fake-gcs-server/internal/notification" From d33c6a136b4c26fc2ccb813183c837e1445aeb2e Mon Sep 17 00:00:00 2001 From: francisco souza Date: Mon, 21 Feb 2022 13:10:13 -0500 Subject: [PATCH 5/5] organize imports --- fakestorage/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fakestorage/server_test.go b/fakestorage/server_test.go index b58b6bafdb..3745d5fa00 100644 --- a/fakestorage/server_test.go +++ b/fakestorage/server_test.go @@ -13,11 +13,11 @@ import ( "strings" "testing" - "google.golang.org/api/iterator" "cloud.google.com/go/storage" "github.com/fsouza/fake-gcs-server/internal/backend" "github.com/fsouza/fake-gcs-server/internal/notification" "github.com/stretchr/testify/assert" + "google.golang.org/api/iterator" ) func TestNewServer(t *testing.T) {