From 6b04b661409a1f2ad21555d5651371876ccd8503 Mon Sep 17 00:00:00 2001 From: Sunil Pai Date: Wed, 23 Feb 2022 12:05:55 +0000 Subject: [PATCH] feat: implement `[text_blobs]` This implements support for `[text_blobs]` as defined by https://github.com/cloudflare/wrangler/pull/1677. Text blobs can be defined in service-worker format with configuration in `wrangler.toml` as - ``` [text_blobs] MYTEXT = "./path/to/my-text.file" ``` The content of the file will then be available as the global `MYTEXT` inside your code. Note that this ONLY makes sense in service-worker format workers (for now). Workers Sites now uses `[text_blobs]` internally. Previously, we were inlining the asset manifest into the worker itself, but we now attach the asset manifest to the uploaded worker. I also added an additional example of Workers Sites with a modules format worker. --- .changeset/good-cats-remember.md | 18 +++ packages/example-sites-app/src/modules.js | 88 +++++++++++++ .../src/{index.js => service-worker.js} | 0 packages/wrangler/src/__tests__/dev.test.tsx | 1 + .../wrangler/src/__tests__/publish.test.ts | 122 +++++++++++++++++- packages/wrangler/src/api/form_data.ts | 37 +++++- packages/wrangler/src/api/worker.ts | 9 ++ packages/wrangler/src/config.ts | 26 +++- packages/wrangler/src/dev.tsx | 22 +++- packages/wrangler/src/index.tsx | 15 +++ packages/wrangler/src/publish.ts | 31 +++-- 11 files changed, 342 insertions(+), 27 deletions(-) create mode 100644 .changeset/good-cats-remember.md create mode 100644 packages/example-sites-app/src/modules.js rename packages/example-sites-app/src/{index.js => service-worker.js} (100%) diff --git a/.changeset/good-cats-remember.md b/.changeset/good-cats-remember.md new file mode 100644 index 000000000000..f002d14fea95 --- /dev/null +++ b/.changeset/good-cats-remember.md @@ -0,0 +1,18 @@ +--- +"wrangler": patch +--- + +feat: implement `[text_blobs]` + +This implements support for `[text_blobs]` as defined by https://github.com/cloudflare/wrangler/pull/1677. + +Text blobs can be defined in service-worker format with configuration in `wrangler.toml` as - + +``` +[text_blobs] +MYTEXT = "./path/to/my-text.file" +``` + +The content of the file will then be available as the global `MYTEXT` inside your code. Note that this ONLY makes sense in service-worker format workers (for now). + +Workers Sites now uses `[text_blobs]` internally. Previously, we were inlining the asset manifest into the worker itself, but we now attach the asset manifest to the uploaded worker. I also added an additional example of Workers Sites with a modules format worker. diff --git a/packages/example-sites-app/src/modules.js b/packages/example-sites-app/src/modules.js new file mode 100644 index 000000000000..45706600b718 --- /dev/null +++ b/packages/example-sites-app/src/modules.js @@ -0,0 +1,88 @@ +import { + getAssetFromKV, + mapRequestToAsset, +} from "@cloudflare/kv-asset-handler"; + +import manifestJSON from "__STATIC_CONTENT_MANIFEST"; +const assetManifest = JSON.parse(manifestJSON); + +/** + * The DEBUG flag will do two things that help during development: + * 1. we will skip caching on the edge, which makes it easier to + * debug. + * 2. we will return an error message on exception in your Response rather + * than the default 404.html page. + */ +const DEBUG = false; + +export default { + async fetch(request, env, ctx) { + let options = { + ASSET_NAMESPACE: env.__STATIC_CONTENT, + ASSET_MANIFEST: assetManifest, + }; + + /** + * You can add custom logic to how we fetch your assets + * by configuring the function `mapRequestToAsset` + */ + // options.mapRequestToAsset = handlePrefix(/^\/docs/) + + try { + if (DEBUG) { + // customize caching + options.cacheControl = { + bypassCache: true, + }; + } + + const page = await getAssetFromKV( + { + request, + waitUntil(promise) { + return ctx.waitUntil(promise); + }, + }, + options + ); + + // allow headers to be altered + const response = new Response(page.body, page); + + response.headers.set("X-XSS-Protection", "1; mode=block"); + response.headers.set("X-Content-Type-Options", "nosniff"); + response.headers.set("X-Frame-Options", "DENY"); + response.headers.set("Referrer-Policy", "unsafe-url"); + response.headers.set("Feature-Policy", "none"); + + return response; + } catch (e) { + // if an error is thrown try to serve the asset at 404.html + if (!DEBUG) { + try { + let notFoundResponse = await getAssetFromKV( + { + request, + waitUntil(promise) { + return ctx.waitUntil(promise); + }, + }, + { + ASSET_NAMESPACE: env.__STATIC_CONTENT, + ASSET_MANIFEST: assetManifest, + mapRequestToAsset: (req) => + new Request(`${new URL(req.url).origin}/404.html`, req), + } + ); + + return new Response(notFoundResponse.body, { + ...notFoundResponse, + status: 404, + }); + } catch (e) {} + } + + return new Response(e.message || e.toString(), { status: 500 }); + } + }, +}; diff --git a/packages/example-sites-app/src/index.js b/packages/example-sites-app/src/service-worker.js similarity index 100% rename from packages/example-sites-app/src/index.js rename to packages/example-sites-app/src/service-worker.js diff --git a/packages/wrangler/src/__tests__/dev.test.tsx b/packages/wrangler/src/__tests__/dev.test.tsx index 2b5db33bb2d2..d85be7d5b201 100644 --- a/packages/wrangler/src/__tests__/dev.test.tsx +++ b/packages/wrangler/src/__tests__/dev.test.tsx @@ -70,6 +70,7 @@ function renderDev({ durable_objects: { bindings: [] }, r2_buckets: [], wasm_modules: {}, + text_blobs: {}, unsafe: [], }, public: publicDir, diff --git a/packages/wrangler/src/__tests__/publish.test.ts b/packages/wrangler/src/__tests__/publish.test.ts index 61d271314c9e..805f653e9086 100644 --- a/packages/wrangler/src/__tests__/publish.test.ts +++ b/packages/wrangler/src/__tests__/publish.test.ts @@ -601,7 +601,7 @@ export default{ expect(std.err).toMatchInlineSnapshot(`""`); }); - it("when using a service worker type, it should inline an asset manifest, and bind to a namespace", async () => { + it("when using a service worker type, it should add an asset manifest as a text_blob, and bind to a namespace", async () => { const assets = [ { filePath: "assets/file-1.txt", content: "Content of file-1" }, { filePath: "assets/file-2.txt", content: "Content of file-2" }, @@ -620,13 +620,21 @@ export default{ writeAssets(assets); mockUploadWorkerRequest({ expectedType: "sw", - expectedEntry: `const __STATIC_CONTENT_MANIFEST = {"file-1.txt":"assets/file-1.2ca234f380.txt","file-2.txt":"assets/file-2.5938485188.txt"};`, + expectedModules: { + __STATIC_CONTENT_MANIFEST: + '{"file-1.txt":"assets/file-1.2ca234f380.txt","file-2.txt":"assets/file-2.5938485188.txt"}', + }, expectedBindings: [ { name: "__STATIC_CONTENT", namespace_id: "__test-name-workers_sites_assets-id", type: "kv_namespace", }, + { + name: "__STATIC_CONTENT_MANIFEST", + part: "__STATIC_CONTENT_MANIFEST", + type: "text_blob", + }, ], }); mockSubDomainRequest(); @@ -1617,6 +1625,116 @@ export default{ }); }); + describe("[text_blobs]", () => { + it("should be able to define text blobs for service-worker format workers", async () => { + writeWranglerToml({ + text_blobs: { + TESTTEXTBLOBNAME: "./path/to/text.file", + }, + }); + writeWorkerSource({ type: "sw" }); + fs.mkdirSync("./path/to", { recursive: true }); + fs.writeFileSync("./path/to/text.file", "SOME TEXT CONTENT"); + mockUploadWorkerRequest({ + expectedType: "sw", + expectedModules: { TESTTEXTBLOBNAME: "SOME TEXT CONTENT" }, + expectedBindings: [ + { + name: "TESTTEXTBLOBNAME", + part: "TESTTEXTBLOBNAME", + type: "text_blob", + }, + ], + }); + mockSubDomainRequest(); + await runWrangler("publish index.js"); + expect(std.out).toMatchInlineSnapshot(` + "Uploaded + test-name + (TIMINGS) + Published + test-name + (TIMINGS) + + test-name.test-sub-domain.workers.dev" + `); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.warn).toMatchInlineSnapshot(`""`); + }); + + it("should error when defining text blobs for modules format workers", async () => { + writeWranglerToml({ + text_blobs: { + TESTTEXTBLOBNAME: "./path/to/text.file", + }, + }); + writeWorkerSource({ type: "esm" }); + fs.mkdirSync("./path/to", { recursive: true }); + fs.writeFileSync("./path/to/text.file", "SOME TEXT CONTENT"); + + await expect( + runWrangler("publish index.js") + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"You cannot configure [text_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure \`[build.upload.rules]\` in your wrangler.toml"` + ); + expect(std.out).toMatchInlineSnapshot(`""`); + expect(std.err).toMatchInlineSnapshot(` + "You cannot configure [text_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure \`[build.upload.rules]\` in your wrangler.toml + + %s + If you think this is a bug then please create an issue at https://github.com/cloudflare/wrangler2/issues/new." + `); + expect(std.warn).toMatchInlineSnapshot(`""`); + }); + + it("should resolve text blobs relative to the wrangler.toml file", async () => { + fs.mkdirSync("./path/to/and/the/path/to/", { recursive: true }); + fs.writeFileSync( + "./path/to/wrangler.toml", + TOML.stringify({ + compatibility_date: "2022-01-12", + name: "test-name", + text_blobs: { + TESTTEXTBLOBNAME: "./and/the/path/to/text.file", + }, + }), + + "utf-8" + ); + + writeWorkerSource({ type: "sw" }); + fs.writeFileSync( + "./path/to/and/the/path/to/text.file", + "SOME TEXT CONTENT" + ); + mockUploadWorkerRequest({ + expectedType: "sw", + expectedModules: { TESTTEXTBLOBNAME: "SOME TEXT CONTENT" }, + expectedBindings: [ + { + name: "TESTTEXTBLOBNAME", + part: "TESTTEXTBLOBNAME", + type: "text_blob", + }, + ], + }); + mockSubDomainRequest(); + await runWrangler("publish index.js --config ./path/to/wrangler.toml"); + expect(std.out).toMatchInlineSnapshot(` + "Uploaded + test-name + (TIMINGS) + Published + test-name + (TIMINGS) + + test-name.test-sub-domain.workers.dev" + `); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.warn).toMatchInlineSnapshot(`""`); + }); + }); + describe("vars bindings", () => { it("should support json bindings", async () => { writeWranglerToml({ diff --git a/packages/wrangler/src/api/form_data.ts b/packages/wrangler/src/api/form_data.ts index e3c41333075e..1c5ce3efd109 100644 --- a/packages/wrangler/src/api/form_data.ts +++ b/packages/wrangler/src/api/form_data.ts @@ -37,6 +37,7 @@ export interface WorkerMetadata { | { type: "plain_text"; name: string; text: string } | { type: "json"; name: string; json: unknown } | { type: "wasm_module"; name: string; part: string } + | { type: "text_blob"; name: string; part: string } | { type: "durable_object_namespace"; name: string; @@ -54,7 +55,6 @@ export function toFormData(worker: CfWorkerInit): FormData { const formData = new FormData(); const { main, - modules, bindings, migrations, usage_model, @@ -62,6 +62,8 @@ export function toFormData(worker: CfWorkerInit): FormData { compatibility_flags, } = worker; + let { modules } = worker; + const metadataBindings: WorkerMetadata["bindings"] = []; bindings.kv_namespaces?.forEach(({ id, binding }) => { @@ -114,11 +116,28 @@ export function toFormData(worker: CfWorkerInit): FormData { ); } + for (const [name, filePath] of Object.entries(bindings.text_blobs || {})) { + metadataBindings.push({ + name, + type: "text_blob", + part: name, + }); + + if (name !== "__STATIC_CONTENT_MANIFEST") { + formData.set( + name, + new File([readFileSync(filePath)], filePath, { + type: "text/plain", + }) + ); + } + } + if (main.type === "commonjs") { // This is a service-worker format worker. - // So we convert all `.wasm` modules into `wasm_module` bindings. - for (const [index, module] of Object.entries(modules || [])) { + for (const module of Object.values([...(modules || [])])) { if (module.type === "compiled-wasm") { + // Convert all `.wasm` modules into `wasm_module` bindings. // The "name" of the module is a file path. We use it // to instead be a "part" of the body, and a reference // that we can use inside our source. This identifier has to be a valid @@ -139,7 +158,17 @@ export function toFormData(worker: CfWorkerInit): FormData { }) ); // And then remove it from the modules collection - modules?.splice(parseInt(index, 10), 1); + modules = modules?.filter((m) => m !== module); + } else if (module.name === "__STATIC_CONTENT_MANIFEST") { + // Add the manifest to the form data. + formData.set( + module.name, + new File([module.content], module.name, { + type: "text/plain", + }) + ); + // And then remove it from the modules collection + modules = modules?.filter((m) => m !== module); } } } diff --git a/packages/wrangler/src/api/worker.ts b/packages/wrangler/src/api/worker.ts index 52d3fe13cc18..e44dcc57d445 100644 --- a/packages/wrangler/src/api/worker.ts +++ b/packages/wrangler/src/api/worker.ts @@ -90,6 +90,14 @@ interface CfWasmModuleBindings { [key: string]: string; } +/** + * A binding to a text blob (in service worker format) + */ + +interface CfTextBlobBindings { + [key: string]: string; +} + /** * A Durable Object. */ @@ -145,6 +153,7 @@ export interface CfWorkerInit { vars: CfVars | undefined; kv_namespaces: CfKvNamespace[] | undefined; wasm_modules: CfWasmModuleBindings | undefined; + text_blobs: CfTextBlobBindings | undefined; durable_objects: { bindings: CfDurableObject[] } | undefined; r2_buckets: CfR2Bucket[] | undefined; unsafe: CfUnsafeBinding[] | undefined; diff --git a/packages/wrangler/src/config.ts b/packages/wrangler/src/config.ts index 6c5045db0c69..ee934472e119 100644 --- a/packages/wrangler/src/config.ts +++ b/packages/wrangler/src/config.ts @@ -240,6 +240,16 @@ export type Config = { [key: string]: string; }; + /** + * A list of text files that your worker should be bound to. This is + * the "legacy" way of binding to a text file. ES module workers should + * do proper module imports. + * NB: these ARE NOT inherited, and SHOULD NOT be duplicated across all environments. + */ + text_blobs?: { + [key: string]: string; + }; + /** * "Unsafe" tables for features that aren't directly supported by wrangler. * NB: these are not inherited, and HAVE to be duplicated across all environments. @@ -494,7 +504,13 @@ export type Config = { | undefined | Omit< Config, - "env" | "wasm_modules" | "migrations" | "site" | "dev" | "legacy_env" + | "env" + | "wasm_modules" + | "text_blobs" + | "migrations" + | "site" + | "dev" + | "legacy_env" >; }; }; @@ -554,7 +570,13 @@ export function normaliseAndValidateEnvironmentsConfig(config: Config) { type InheritedField = keyof Omit< Config, - "env" | "migrations" | "wasm_modules" | "site" | "dev" | "legacy_env" + | "env" + | "migrations" + | "wasm_modules" + | "text_blobs" + | "site" + | "dev" + | "legacy_env" >; const inheritedFields: InheritedField[] = [ diff --git a/packages/wrangler/src/dev.tsx b/packages/wrangler/src/dev.tsx index a0301482fb9e..80c9387fa487 100644 --- a/packages/wrangler/src/dev.tsx +++ b/packages/wrangler/src/dev.tsx @@ -77,6 +77,12 @@ function Dev(props: DevProps): JSX.Element { ); } + if (props.bindings.text_blobs && format === "modules") { + throw new Error( + "You cannot configure [text_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure `[build.upload.rules]` in your wrangler.toml" + ); + } + const bundle = useEsbuild({ entry, format, @@ -684,12 +690,7 @@ function useWorker(props: { true ); // TODO: cancellable? - let content = await readFile(bundle.path, "utf-8"); - if (format === "service-worker" && assets.manifest) { - content = `const __STATIC_CONTENT_MANIFEST = ${JSON.stringify( - assets.manifest - )};\n${content}`; - } + const content = await readFile(bundle.path, "utf-8"); const init: CfWorkerInit = { name, @@ -699,7 +700,7 @@ function useWorker(props: { content, }, modules: modules.concat( - assets.manifest && format === "modules" + assets.manifest ? { name: "__STATIC_CONTENT_MANIFEST", content: JSON.stringify(assets.manifest), @@ -714,6 +715,13 @@ function useWorker(props: { ? { binding: "__STATIC_CONTENT", id: assets.namespace } : [] ), + text_blobs: { + ...bindings.text_blobs, + ...(assets.manifest && + format === "service-worker" && { + __STATIC_CONTENT_MANIFEST: "__STATIC_CONTENT_MANIFEST", + }), + }, }, migrations: undefined, // no migrations in dev compatibility_date: compatibilityDate, diff --git a/packages/wrangler/src/index.tsx b/packages/wrangler/src/index.tsx index 8fb28fbc6003..195d9119d083 100644 --- a/packages/wrangler/src/index.tsx +++ b/packages/wrangler/src/index.tsx @@ -121,6 +121,18 @@ ${TOML.stringify({ config.wasm_modules = modules; } + if (configPath && "text_blobs" in config) { + // rewrite text_blobs paths to be absolute + const modules: Record = {}; + for (const [name, filePath] of Object.entries(config.text_blobs || {})) { + modules[name] = path.relative( + process.cwd(), + path.join(path.dirname(configPath), filePath) + ); + } + config.text_blobs = modules; + } + if ("unsafe" in config) { console.warn( "'unsafe' fields are experimental and may change or break at any time." @@ -856,6 +868,7 @@ export async function main(argv: string[]): Promise { ), vars: envRootObj.vars, wasm_modules: config.wasm_modules, + text_blobs: config.text_blobs, durable_objects: envRootObj.durable_objects, r2_buckets: envRootObj.r2_buckets, unsafe: envRootObj.unsafe?.bindings, @@ -1284,6 +1297,7 @@ export async function main(argv: string[]): Promise { ), vars: envRootObj.vars, wasm_modules: config.wasm_modules, + text_blobs: config.text_blobs, durable_objects: envRootObj.durable_objects, r2_buckets: envRootObj.r2_buckets, unsafe: envRootObj.unsafe?.bindings, @@ -1488,6 +1502,7 @@ export async function main(argv: string[]): Promise { durable_objects: { bindings: [] }, r2_buckets: [], wasm_modules: {}, + text_blobs: {}, unsafe: [], }, modules: [], diff --git a/packages/wrangler/src/publish.ts b/packages/wrangler/src/publish.ts index 4729b791d16f..dd8481205963 100644 --- a/packages/wrangler/src/publish.ts +++ b/packages/wrangler/src/publish.ts @@ -131,6 +131,12 @@ export default async function publish(props: Props): Promise { ); } + if ("text_blobs" in config && format === "modules") { + throw new Error( + "You cannot configure [text_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure `[build.upload.rules]` in your wrangler.toml" + ); + } + const { modules, resolvedEntryPointPath, bundleType } = await bundleWorker( props.entry, props.experimentalPublic, @@ -140,7 +146,7 @@ export default async function publish(props: Props): Promise { format ); - let content = readFileSync(resolvedEntryPointPath, { + const content = readFileSync(resolvedEntryPointPath, { encoding: "utf-8", }); @@ -202,23 +208,24 @@ export default async function publish(props: Props): Promise { ), vars: envRootObj.vars, wasm_modules: config.wasm_modules, + text_blobs: { + ...config.text_blobs, + ...(assets.manifest && + format === "service-worker" && { + __STATIC_CONTENT_MANIFEST: "__STATIC_CONTENT_MANIFEST", + }), + }, durable_objects: envRootObj.durable_objects, r2_buckets: envRootObj.r2_buckets, unsafe: envRootObj.unsafe?.bindings, }; if (assets.manifest) { - if (bundleType === "esm") { - modules.push({ - name: "__STATIC_CONTENT_MANIFEST", - content: JSON.stringify(assets.manifest), - type: "text", - }); - } else { - content = `const __STATIC_CONTENT_MANIFEST = ${JSON.stringify( - assets.manifest - )};\n${content}`; - } + modules.push({ + name: "__STATIC_CONTENT_MANIFEST", + content: JSON.stringify(assets.manifest), + type: "text", + }); } const worker: CfWorkerInit = {