Add support for multipart downloads for module-format Worker scripts (#…

…1040)

Co-authored-by: Jacob Bednarz <[email protected]>

.changelog/1040.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+workers: Support for multipart encoding for DownloadWorker on a module-format Worker script
+```

cloudflare.go

@@ -176,21 +176,41 @@ func (api *API) makeRequestContextWithHeaders(ctx context.Context, method, uri s
 	return api.makeRequestWithAuthTypeAndHeaders(ctx, method, uri, params, api.authType, headers)
 }
 
-// Deprecated: Use `makeRequestContextWithHeaders` instead.
-//nolint:unused
-func (api *API) makeRequestWithHeaders(method, uri string, params interface{}, headers http.Header) ([]byte, error) {
-	return api.makeRequestWithAuthTypeAndHeaders(context.Background(), method, uri, params, api.authType, headers)
-}
-
 func (api *API) makeRequestWithAuthType(ctx context.Context, method, uri string, params interface{}, authType int) ([]byte, error) {
 	return api.makeRequestWithAuthTypeAndHeaders(ctx, method, uri, params, authType, nil)
 }
 
+// APIResponse holds the structure for a response from the API. It looks alot
+// like `http.Response` however, uses a `[]byte` for the `Body` instead of a
+// `io.ReadCloser`.
+//
+// This may go away in the experimental client in favour of `http.Response`.
+type APIResponse struct {
+	Body       []byte
+	Status     string
+	StatusCode int
+	Headers    http.Header
+}
+
 func (api *API) makeRequestWithAuthTypeAndHeaders(ctx context.Context, method, uri string, params interface{}, authType int, headers http.Header) ([]byte, error) {
+	res, err := api.makeRequestWithAuthTypeAndHeadersComplete(ctx, method, uri, params, api.authType, headers)
+	if err != nil {
+		return nil, err
+	}
+	return res.Body, err
+}
+
+// Use this method if an API response can have different Content-Type headers and different body formats.
+func (api *API) makeRequestContextWithHeadersComplete(ctx context.Context, method, uri string, params interface{}, headers http.Header) (*APIResponse, error) {
+	return api.makeRequestWithAuthTypeAndHeadersComplete(ctx, method, uri, params, api.authType, headers)
+}
+
+func (api *API) makeRequestWithAuthTypeAndHeadersComplete(ctx context.Context, method, uri string, params interface{}, authType int, headers http.Header) (*APIResponse, error) {
 	var err error
 	var resp *http.Response
 	var respErr error
 	var respBody []byte
+
 	for i := 0; i <= api.retryPolicy.MaxRetries; i++ {
 		var reqBody io.Reader
 		if params != nil {
@@ -278,12 +298,14 @@ func (api *API) makeRequestWithAuthTypeAndHeaders(ctx context.Context, method, u
 			break
 		}
 	}
+
+	// still had an error after all retries
 	if respErr != nil {
 		return nil, respErr
 	}
 
 	if api.Debug {
-		fmt.Printf("cloudflare-go [DEBUG] RESPONSE StatusCode:%d Body:%#v RayID:%s\n", resp.StatusCode, string(respBody), resp.Header.Get("cf-ray"))
+		fmt.Printf("cloudflare-go [DEBUG] RESPONSE StatusCode:%d RayID:%s ContentType:%s Body:%#v\n", resp.StatusCode, resp.Header.Get("cf-ray"), resp.Header.Get("content-type"), string(respBody))
 	}
 
 	if resp.StatusCode >= http.StatusBadRequest {
@@ -341,7 +363,12 @@ func (api *API) makeRequestWithAuthTypeAndHeaders(ctx context.Context, method, u
 		}
 	}
 
-	return respBody, nil
+	return &APIResponse{
+		Body:       respBody,
+		StatusCode: resp.StatusCode,
+		Status:     resp.Status,
+		Headers:    resp.Header,
+	}, nil
 }
 
 // request makes a HTTP request to the given API endpoint, returning the raw

images.go

@@ -138,12 +138,11 @@ func (api *API) UploadImage(ctx context.Context, accountID string, upload ImageU
 	}
 	_ = w.Close()
 
-	res, err := api.makeRequestWithAuthTypeAndHeaders(
+	res, err := api.makeRequestContextWithHeaders(
 		ctx,
 		http.MethodPost,
 		uri,
 		body,
-		api.authType,
 		http.Header{
 			"Accept":       []string{"application/json"},
 			"Content-Type": []string{w.FormDataContentType()},

workers.go

@@ -8,9 +8,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"mime"
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
+	"strings"
 	"time"
 
 	"errors"
@@ -82,6 +85,7 @@ type WorkerListResponse struct {
 // WorkerScriptResponse wrapper struct for API response to worker script calls.
 type WorkerScriptResponse struct {
 	Response
+	Module       bool
 	WorkerScript `json:"result"`
 }
 
@@ -417,6 +421,7 @@ func (api *API) DownloadWorker(ctx context.Context, requestParams *WorkerRequest
 		return r, err
 	}
 	r.Script = string(res)
+	r.Module = false
 	r.Success = true
 	return r, nil
 }
@@ -429,12 +434,32 @@ func (api *API) downloadWorkerWithName(ctx context.Context, scriptName string) (
 		return WorkerScriptResponse{}, errors.New("account ID required")
 	}
 	uri := fmt.Sprintf("/accounts/%s/workers/scripts/%s", api.AccountID, scriptName)
-	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
+	res, err := api.makeRequestContextWithHeadersComplete(ctx, http.MethodGet, uri, nil, nil)
 	var r WorkerScriptResponse
 	if err != nil {
 		return r, err
 	}
-	r.Script = string(res)
+
+	// Check if the response type is multipart, in which case this was a module worker
+	mediaType, mediaParams, _ := mime.ParseMediaType(res.Headers.Get("content-type"))
+	if strings.HasPrefix(mediaType, "multipart/") {
+		bytesReader := bytes.NewReader(res.Body)
+		mimeReader := multipart.NewReader(bytesReader, mediaParams["boundary"])
+		mimePart, err := mimeReader.NextPart()
+		if err != nil {
+			return r, fmt.Errorf("could not get multipart response body: %w", err)
+		}
+		mimePartBody, err := ioutil.ReadAll(mimePart)
+		if err != nil {
+			return r, fmt.Errorf("could not read multipart response body: %w", err)
+		}
+		r.Script = string(mimePartBody)
+		r.Module = true
+	} else {
+		r.Script = string(res.Body)
+		r.Module = false
+	}
+
 	r.Success = true
 	return r, nil
 }
@@ -497,6 +522,7 @@ func (api *API) ListWorkerBindings(ctx context.Context, requestParams *WorkerReq
 		case WorkerWebAssemblyBindingType:
 			bindingListItem.Binding = WorkerWebAssemblyBinding{
 				Module: &bindingContentReader{
+					ctx:           ctx,
 					api:           api,
 					requestParams: requestParams,
 					bindingName:   name,
@@ -537,6 +563,7 @@ func (api *API) ListWorkerBindings(ctx context.Context, requestParams *WorkerReq
 type bindingContentReader struct {
 	api           *API
 	requestParams *WorkerRequestParams
+	ctx           context.Context
 	bindingName   string
 	content       []byte
 	position      int
@@ -546,7 +573,7 @@ func (b *bindingContentReader) Read(p []byte) (n int, err error) {
 	// Lazily load the content when Read() is first called
 	if b.content == nil {
 		uri := fmt.Sprintf("/accounts/%s/workers/scripts/%s/bindings/%s/content", b.api.AccountID, b.requestParams.ScriptName, b.bindingName)
-		res, err := b.api.makeRequest(http.MethodGet, uri, nil)
+		res, err := b.api.makeRequestContext(b.ctx, http.MethodGet, uri, nil)
 		if err != nil {
 			return 0, err
 		}

workers_test.go

@@ -23,7 +23,7 @@ const (
 }`
 	uploadWorkerResponseData = `{
     "result": {
-        "script": "addEventListener('fetch', event => {\n    event.passThroughOnException()\nevent.respondWith(handleRequest(event.request))\n})\n\nasync function handleRequest(request) {\n    return fetch(request)\n}",
+        "script": "addEventListener('fetch', event => {\n  event.passThroughOnException()\n  event.respondWith(handleRequest(event.request))\n})\n\nasync function handleRequest(request) {\n  return fetch(request)\n}",
         "etag": "279cf40d86d70b82f6cd3ba90a646b3ad995912da446836d7371c21c6a43977a",
         "size": 191,
         "modified_on": "2018-06-09T15:17:01.989141Z"
@@ -35,7 +35,7 @@ const (
 
 	uploadWorkerModuleResponseData = `{
     "result": {
-        "script": "export default {\n    async fetch(request, env, event) {\n     event.passThroughOnException()\n    return fetch(request)\n    }\n}",
+        "script": "export default {\n  async fetch(request, env, event) {\n    event.passThroughOnException()\n    return fetch(request)\n  }\n}",
         "etag": "279cf40d86d70b82f6cd3ba90a646b3ad995912da446836d7371c21c6a43977a",
         "size": 191,
         "modified_on": "2018-06-09T15:17:01.989141Z"
@@ -179,15 +179,38 @@ const (
   "errors": [],
   "messages": []
 }`
+	workerScript = `addEventListener('fetch', event => {
+  event.passThroughOnException()
+  event.respondWith(handleRequest(event.request))
+})
+
+async function handleRequest(request) {
+  return fetch(request)
+}`
+	workerModuleScript = `export default {
+  async fetch(request, env, event) {
+    event.passThroughOnException()
+    return fetch(request)
+  }
+}`
+	workerModuleScriptDownloadResponse = `
+--workermodulescriptdownload
+Content-Disposition: form-data; name="worker.js"
+
+export default {
+  async fetch(request, env, event) {
+    event.passThroughOnException()
+    return fetch(request)
+  }
+}
+--workermodulescriptdownload--
+`
 )
 
 var (
 	successResponse               = Response{Success: true, Errors: []ResponseInfo{}, Messages: []ResponseInfo{}}
-	workerScript                  = "addEventListener('fetch', event => {\n    event.passThroughOnException()\nevent.respondWith(handleRequest(event.request))\n})\n\nasync function handleRequest(request) {\n    return fetch(request)\n}"
-	workerModuleScript            = "export default {\n    async fetch(request, env, event) {\n     event.passThroughOnException()\n    return fetch(request)\n    }\n}"
 	deleteWorkerRouteResponseData = createWorkerRouteResponse
-
-	attachWorkerToDomainResponse = fmt.Sprintf(`{
+	attachWorkerToDomainResponse  = fmt.Sprintf(`{
     "result": {
         "id": "e7a57d8746e74ae49c25994dadb421b1",
 	"zone_id": "%s",
@@ -303,8 +326,9 @@ func TestWorkers_DeleteWorker(t *testing.T) {
 	})
 	res, err := client.DeleteWorker(context.Background(), &WorkerRequestParams{ZoneID: "foo"})
 	want := WorkerScriptResponse{
-		successResponse,
-		WorkerScript{}}
+		Response: successResponse,
+	}
+
 	if assert.NoError(t, err) {
 		assert.Equal(t, want.Response, res.Response)
 	}
@@ -321,8 +345,8 @@ func TestWorkers_DeleteWorkerWithName(t *testing.T) {
 	})
 	res, err := client.DeleteWorker(context.Background(), &WorkerRequestParams{ScriptName: "bar"})
 	want := WorkerScriptResponse{
-		successResponse,
-		WorkerScript{}}
+		Response: successResponse,
+	}
 	if assert.NoError(t, err) {
 		assert.Equal(t, want.Response, res.Response)
 	}
@@ -348,6 +372,7 @@ func TestWorkers_DownloadWorker(t *testing.T) {
 	res, err := client.DownloadWorker(context.Background(), &WorkerRequestParams{ZoneID: "foo"})
 	want := WorkerScriptResponse{
 		successResponse,
+		false,
 		WorkerScript{
 			Script: workerScript,
 		}}
@@ -368,6 +393,7 @@ func TestWorkers_DownloadWorkerWithName(t *testing.T) {
 	res, err := client.DownloadWorker(context.Background(), &WorkerRequestParams{ScriptName: "bar"})
 	want := WorkerScriptResponse{
 		successResponse,
+		false,
 		WorkerScript{
 			Script: workerScript,
 		}}
@@ -384,6 +410,27 @@ func TestWorkers_DownloadWorkerWithNameErrorsWithoutAccountId(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestWorkers_DownloadWorkerModule(t *testing.T) {
+	setup(UsingAccount("foo"))
+	defer teardown()
+
+	mux.HandleFunc("/accounts/foo/workers/scripts/bar", func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, http.MethodGet, r.Method, "Expected method 'GET', got %s", r.Method)
+		w.Header().Set("content-type", "multipart/form-data; boundary=workermodulescriptdownload")
+		fmt.Fprintf(w, workerModuleScriptDownloadResponse) //nolint
+	})
+	res, err := client.DownloadWorker(context.Background(), &WorkerRequestParams{ScriptName: "bar"})
+	want := WorkerScriptResponse{
+		successResponse,
+		true,
+		WorkerScript{
+			Script: workerModuleScript,
+		}}
+	if assert.NoError(t, err) {
+		assert.Equal(t, want.Script, res.Script)
+	}
+}
+
 func TestWorkers_ListWorkerScripts(t *testing.T) {
 	setup(UsingAccount("foo"))
 	defer teardown()
@@ -430,6 +477,7 @@ func TestWorkers_UploadWorker(t *testing.T) {
 	formattedTime, _ := time.Parse(time.RFC3339Nano, "2018-06-09T15:17:01.989141Z")
 	want := WorkerScriptResponse{
 		successResponse,
+		false,
 		WorkerScript{
 			Script: workerScript,
 			WorkerMetaData: WorkerMetaData{
@@ -468,6 +516,7 @@ func TestWorkers_UploadWorkerAsModule(t *testing.T) {
 	formattedTime, _ := time.Parse(time.RFC3339Nano, "2018-06-09T15:17:01.989141Z")
 	want := WorkerScriptResponse{
 		successResponse,
+		false,
 		WorkerScript{
 			Script: workerModuleScript,
 			WorkerMetaData: WorkerMetaData{
@@ -496,6 +545,7 @@ func TestWorkers_UploadWorkerWithName(t *testing.T) {
 	formattedTime, _ := time.Parse(time.RFC3339Nano, "2018-06-09T15:17:01.989141Z")
 	want := WorkerScriptResponse{
 		successResponse,
+		false,
 		WorkerScript{
 			Script: workerScript,
 			WorkerMetaData: WorkerMetaData{
@@ -524,6 +574,7 @@ func TestWorkers_UploadWorkerSingleScriptWithAccount(t *testing.T) {
 	formattedTime, _ := time.Parse(time.RFC3339Nano, "2018-06-09T15:17:01.989141Z")
 	want := WorkerScriptResponse{
 		successResponse,
+		false,
 		WorkerScript{
 			Script: workerScript,
 			WorkerMetaData: WorkerMetaData{
@@ -629,6 +680,7 @@ func TestWorkers_UploadWorkerWithInheritBinding(t *testing.T) {
 	formattedTime, _ := time.Parse(time.RFC3339Nano, "2018-06-09T15:17:01.989141Z")
 	want := WorkerScriptResponse{
 		successResponse,
+		false,
 		WorkerScript{
 			Script: workerScript,
 			WorkerMetaData: WorkerMetaData{