diff --git a/docs/javascript/files.md b/docs/javascript/files.md index 5dfdd01dc..40eed68c5 100644 --- a/docs/javascript/files.md +++ b/docs/javascript/files.md @@ -2,7 +2,13 @@ TK Should this be called “working with data”? -You can load files the built-in `FileAttachment` function or the standard [`fetch`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch) API. We recommend `FileAttachment` because it supports many common data formats, including CSV, TSV, JSON, Apache Arrow, and SQLite. For example, here’s how to load a CSV file: +You can load files the built-in `FileAttachment` function. This is available by default in Markdown, but you can import it like so: + +```js echo +import {FileAttachment} from "npm:@observablehq/stdlib"; +``` + +`FileAttachment` supports many common data formats, including CSV, TSV, JSON, Apache Arrow, and SQLite. For example, here’s how to load a CSV file: ```js echo const gistemp = FileAttachment("gistemp.csv").csv({typed: true}); @@ -34,23 +40,6 @@ const gistemp = FileAttachment("gistemp.csv").csv().then((D) => D.map(coerceRow) TK An explanation of why coercing types as early as possible is important. -## Fetch - -Here’s `fetch` for comparison. - -```js run=false -import {autoType, csvParse} from "npm:d3-dsv"; - -const gistemp = fetch("./gistemp.csv").then(async (response) => { - if (!response.ok) throw new Error(`fetch error: ${response.status}`); - return csvParse(await response.text(), autoType); -}); -``` - -Use `fetch` if you prefer to stick to the web standards, you don’t mind writing a little extra code. 🥴 Also, you’ll need to use `fetch` to load files from imported ES modules; `FileAttachment` only works within Markdown. - -**Caution:** If you use `fetch` for a local file, the path *must* start with `./`, `../`, or `/`. This allows us to distinguish between local files and absolute URLs. But that’s a little silly, right? Because unlike `import`, you can’t `fetch` a bare module specifier, so we could be more generous and detect URLs using `/^\w+:/` instead. - ## Supported formats The following type-specific methods are supported: diff --git a/src/client/main.js b/src/client/main.js index b69e87b8c..c8e9c71b8 100644 --- a/src/client/main.js +++ b/src/client/main.js @@ -57,7 +57,7 @@ export function define(cell) { v.define(outputs.length ? `cell ${id}` : null, inputs, body); variables.push(v); for (const o of outputs) variables.push(main.variable(true).define(o, [`cell ${id}`], (exports) => exports[o])); - for (const f of files) registerFile(f.name, {url: f.path, mimeType: f.mimeType}); + for (const f of files) registerFile(f.name, f); for (const d of databases) registerDatabase(d.name, d); } diff --git a/src/client/stdlib/databaseClient.js b/src/client/stdlib/databaseClient.js index be86eac10..20688c78c 100644 --- a/src/client/stdlib/databaseClient.js +++ b/src/client/stdlib/databaseClient.js @@ -12,7 +12,7 @@ export function DatabaseClient(name) { return new DatabaseClientImpl(name, token); } -const DatabaseClientImpl = class DatabaseClient { +class DatabaseClientImpl { #token; constructor(name, token) { @@ -71,8 +71,9 @@ const DatabaseClientImpl = class DatabaseClient { async sql() { return this.query(...this.queryTag.apply(this, arguments)); } -}; +} +Object.defineProperty(DatabaseClientImpl, "name", {value: "DatabaseClient"}); // prevent mangling DatabaseClient.prototype = DatabaseClientImpl.prototype; // instanceof function coerceBuffer(d) { diff --git a/src/client/stdlib/fileAttachment.js b/src/client/stdlib/fileAttachment.js index 147c21bfc..12985888b 100644 --- a/src/client/stdlib/fileAttachment.js +++ b/src/client/stdlib/fileAttachment.js @@ -1,16 +1,18 @@ const files = new Map(); export function registerFile(name, file) { - if (file == null) files.delete(name); - else files.set(name, file); + const url = String(new URL(name, location.href)); + if (file == null) files.delete(url); + else files.set(url, file); } -export function FileAttachment(name) { +export function FileAttachment(name, base = location.href) { if (new.target !== undefined) throw new TypeError("FileAttachment is not a constructor"); - const file = files.get((name = `${name}`)); + const url = String(new URL(name, base)); + const file = files.get(url); if (!file) throw new Error(`File not found: ${name}`); - const {url, mimeType} = file; - return new FileAttachmentImpl(url, name, mimeType); + const {path, mimeType} = file; + return new FileAttachmentImpl(path, name.split("/").pop(), mimeType); } async function remote_fetch(file) { @@ -85,7 +87,7 @@ class AbstractFile { } } -const FileAttachmentImpl = class FileAttachment extends AbstractFile { +class FileAttachmentImpl extends AbstractFile { constructor(url, name, mimeType) { super(name, mimeType); Object.defineProperty(this, "_url", {value: url}); @@ -93,8 +95,9 @@ const FileAttachmentImpl = class FileAttachment extends AbstractFile { async url() { return (await this._url) + ""; } -}; +} +Object.defineProperty(FileAttachmentImpl, "name", {value: "FileAttachment"}); // prevent mangling FileAttachment.prototype = FileAttachmentImpl.prototype; // instanceof class ZipArchive { diff --git a/src/files.ts b/src/files.ts index 4ffa4f1a2..c580ad870 100644 --- a/src/files.ts +++ b/src/files.ts @@ -17,9 +17,9 @@ export function getLocalPath(sourcePath: string, name: string): string | null { export function fileReference(name: string, sourcePath: string): FileReference { return { - name, + name: relativeUrl(sourcePath, name), mimeType: mime.getType(name), - path: relativeUrl(sourcePath, resolvePath("_file", sourcePath, name)) + path: relativeUrl(sourcePath, join("_file", name)) }; } diff --git a/src/javascript.ts b/src/javascript.ts index 020035d9b..f9f7f2cc9 100644 --- a/src/javascript.ts +++ b/src/javascript.ts @@ -6,8 +6,6 @@ import {findAwaits} from "./javascript/awaits.js"; import {resolveDatabases} from "./javascript/databases.js"; import {findDeclarations} from "./javascript/declarations.js"; import {findFeatures} from "./javascript/features.js"; -import {rewriteFetches} from "./javascript/fetches.js"; -import {defaultGlobals} from "./javascript/globals.js"; import {findExports, findImportDeclarations, findImports} from "./javascript/imports.js"; import {createImportResolver, rewriteImports} from "./javascript/imports.js"; import {findReferences} from "./javascript/references.js"; @@ -20,9 +18,11 @@ export interface DatabaseReference { } export interface FileReference { + /** The relative path from the page to the original file (e.g., "./test.txt"). */ name: string; + /** The MIME type, if known; derived from the file extension. */ mimeType: string | null; - /** The relative path from the document to the file in _file */ + /** The relative path from the page to the file in _file (e.g., "../_file/sub/test.txt"). */ path: string; } @@ -93,7 +93,6 @@ export function transpileJavaScript(input: string, options: ParseOptions): Pendi output.insertRight(input.length, "\n))"); } await rewriteImports(output, node, sourcePath, createImportResolver(root, "_import")); - rewriteFetches(output, node, sourcePath); const result = `${node.async ? "async " : ""}(${inputs}) => { ${String(output)}${node.declarations?.length ? `\nreturn {${node.declarations.map(({name}) => name)}};` : ""} }`; @@ -143,7 +142,7 @@ export interface JavaScriptNode { } function parseJavaScript(input: string, options: ParseOptions): JavaScriptNode { - const {globals = defaultGlobals, inline = false, root, sourcePath} = options; + const {inline = false, root, sourcePath} = options; // First attempt to parse as an expression; if this fails, parse as a program. let expression = maybeParseExpression(input, parseOptions); if (expression?.type === "ClassExpression" && expression.id) expression = null; // treat named class as program @@ -152,16 +151,16 @@ function parseJavaScript(input: string, options: ParseOptions): JavaScriptNode { const body = expression ?? Parser.parse(input, parseOptions); const exports = findExports(body); if (exports.length) throw syntaxError("Unexpected token 'export'", exports[0], input); // disallow exports - const references = findReferences(body, globals); - findAssignments(body, references, globals, input); - const declarations = expression ? null : findDeclarations(body as Program, globals, input); - const {imports, fetches} = findImports(body, root, sourcePath); - const features = findFeatures(body, root, sourcePath, references, input); + const references = findReferences(body); + findAssignments(body, references, input); + const declarations = expression ? null : findDeclarations(body as Program, input); + const {imports, features: importedFeatures} = findImports(body, root, sourcePath); + const features = findFeatures(body, sourcePath, references, input); return { body, declarations, references, - features: [...features, ...fetches], + features: [...features, ...importedFeatures], imports, expression: !!expression, async: findAwaits(body).length > 0 diff --git a/src/javascript/assignments.ts b/src/javascript/assignments.ts index 21e25693b..c867c3b43 100644 --- a/src/javascript/assignments.ts +++ b/src/javascript/assignments.ts @@ -1,13 +1,14 @@ -import type {Expression, Node, Pattern, VariableDeclaration} from "acorn"; +import type {Expression, Identifier, Node, Pattern, VariableDeclaration} from "acorn"; import {simple} from "acorn-walk"; +import {defaultGlobals} from "./globals.js"; import {syntaxError} from "./syntaxError.js"; -export function findAssignments(node: Node, references: Node[], globals: Set, input: string): void { +export function findAssignments(node: Node, references: Identifier[], input: string): void { function checkConst(node: Expression | Pattern | VariableDeclaration) { switch (node.type) { case "Identifier": if (references.includes(node)) throw syntaxError(`Assignment to external variable '${node.name}'`, node, input); - if (globals.has(node.name)) throw syntaxError(`Assignment to global '${node.name}'`, node, input); + if (defaultGlobals.has(node.name)) throw syntaxError(`Assignment to global '${node.name}'`, node, input); break; case "ObjectPattern": node.properties.forEach((node) => checkConst(node.type === "Property" ? node.value : node)); diff --git a/src/javascript/declarations.ts b/src/javascript/declarations.ts index 13a2302e9..348bda133 100644 --- a/src/javascript/declarations.ts +++ b/src/javascript/declarations.ts @@ -1,11 +1,12 @@ import type {Identifier, Pattern, Program} from "acorn"; +import {defaultGlobals} from "./globals.js"; import {syntaxError} from "./syntaxError.js"; -export function findDeclarations(node: Program, globals: Set, input: string): Identifier[] { +export function findDeclarations(node: Program, input: string): Identifier[] { const declarations: Identifier[] = []; function declareLocal(node: Identifier) { - if (globals.has(node.name) || node.name === "arguments") { + if (defaultGlobals.has(node.name) || node.name === "arguments") { throw syntaxError(`Global '${node.name}' cannot be redefined`, node, input); } declarations.push(node); diff --git a/src/javascript/features.ts b/src/javascript/features.ts index bbd380ff8..bda090429 100644 --- a/src/javascript/features.ts +++ b/src/javascript/features.ts @@ -1,55 +1,50 @@ -import type {Identifier, Literal, Node, TemplateLiteral} from "acorn"; +import type {CallExpression, Identifier, Literal, Node, TemplateLiteral} from "acorn"; import {simple} from "acorn-walk"; import {getLocalPath} from "../files.js"; import type {Feature} from "../javascript.js"; import {syntaxError} from "./syntaxError.js"; -export function findFeatures( - node: Node, - root: string, - sourcePath: string, - references: Identifier[], - input: string -): Feature[] { +export function findFeatures(node: Node, path: string, references: Identifier[], input: string): Feature[] { const features: Feature[] = []; simple(node, { CallExpression(node) { - const { - callee, - arguments: args, - arguments: [arg] - } = node; - + const {callee} = node; // Ignore function calls that are not references to the feature. For // example, if there’s a local variable called Secret, that will mask the // built-in Secret and won’t be considered a feature. - if ( - callee.type !== "Identifier" || - (callee.name !== "Secret" && callee.name !== "FileAttachment" && callee.name !== "DatabaseClient") || - !references.includes(callee) - ) { - return; - } - - // Forbid dynamic calls. - if (args.length !== 1 || !isStringLiteral(arg)) { - throw syntaxError(`${callee.name} requires a single literal string argument`, node, input); - } - - // Forbid file attachments that are not local paths. - const value = getStringLiteralValue(arg); - if (callee.name === "FileAttachment" && !getLocalPath(sourcePath, value)) { - throw syntaxError(`non-local file path: "${value}"`, node, input); - } - - features.push({type: callee.name, name: value}); + if (callee.type !== "Identifier" || !references.includes(callee)) return; + const {name: type} = callee; + if (type !== "Secret" && type !== "FileAttachment" && type !== "DatabaseClient") return; + features.push(getFeature(type, node, path, input)); } }); return features; } +export function getFeature(type: Feature["type"], node: CallExpression, path: string, input: string): Feature { + const { + arguments: args, + arguments: [arg] + } = node; + + // Forbid dynamic calls. + if (args.length !== 1 || !isStringLiteral(arg)) { + throw syntaxError(`${type} requires a single literal string argument`, node, input); + } + + // Forbid file attachments that are not local paths; normalize the path. + let name: string | null = getStringLiteralValue(arg); + if (type === "FileAttachment") { + const localPath = getLocalPath(path, name); + if (!localPath) throw syntaxError(`non-local file path: ${name}`, node, input); + name = localPath; + } + + return {type, name}; +} + export function isStringLiteral(node: any): node is Literal | TemplateLiteral { return ( node && diff --git a/src/javascript/fetches.ts b/src/javascript/fetches.ts deleted file mode 100644 index c8c8047fb..000000000 --- a/src/javascript/fetches.ts +++ /dev/null @@ -1,74 +0,0 @@ -import {join} from "node:path"; -import type {CallExpression, Identifier, Node} from "acorn"; -import {simple} from "acorn-walk"; -import {type Feature, type JavaScriptNode} from "../javascript.js"; -import {type Sourcemap} from "../sourcemap.js"; -import {relativeUrl, resolvePath} from "../url.js"; -import {getStringLiteralValue, isStringLiteral} from "./features.js"; -import {defaultGlobals} from "./globals.js"; -import {isLocalImport} from "./imports.js"; -import {findReferences} from "./references.js"; - -export function rewriteFetches(output: Sourcemap, rootNode: JavaScriptNode, sourcePath: string): void { - simple(rootNode.body, { - CallExpression(node) { - rewriteIfLocalFetch(node, output, rootNode.references, sourcePath); - } - }); -} - -export function rewriteIfLocalFetch( - node: CallExpression, - output: Sourcemap, - references: Identifier[], - sourcePath: string, - {resolveMeta = false} = {} // if true, use import.meta to resolve at runtime; assumes _import -) { - if (isLocalFetch(node, references, sourcePath)) { - const arg = node.arguments[0]; - const value = getStringLiteralValue(arg); - const path = resolvePath("_file", sourcePath, value); - let result = JSON.stringify(relativeUrl(join(resolveMeta ? "_import" : ".", sourcePath), path)); - if (resolveMeta) result = `new URL(${result}, import.meta.url)`; // more support than import.meta.resolve - output.replaceLeft(arg.start, arg.end, result); - } -} - -// Promote fetches with static literals to file attachment references. -export function findFetches(body: Node, path: string) { - const references: Identifier[] = findReferences(body, defaultGlobals); - const fetches: Feature[] = []; - - simple(body, {CallExpression: findFetch}, undefined, path); - - function findFetch(node: CallExpression, sourcePath: string) { - maybeAddFetch(fetches, node, references, sourcePath); - } - - return fetches; -} - -export function maybeAddFetch( - features: Feature[], - node: CallExpression, - references: Identifier[], - sourcePath: string -): void { - if (isLocalFetch(node, references, sourcePath)) { - features.push({type: "FileAttachment", name: getStringLiteralValue(node.arguments[0])}); - } -} - -function isLocalFetch(node: CallExpression, references: Identifier[], sourcePath: string): boolean { - const { - callee, - arguments: [arg] - } = node; - return ( - callee.type === "Identifier" && - callee.name === "fetch" && - !references.includes(callee) && - isStringLiteral(arg) && - isLocalImport(getStringLiteralValue(arg), sourcePath) - ); -} diff --git a/src/javascript/imports.ts b/src/javascript/imports.ts index 2cef96083..40fdf61c2 100644 --- a/src/javascript/imports.ts +++ b/src/javascript/imports.ts @@ -2,7 +2,7 @@ import {createHash} from "node:crypto"; import {readFileSync} from "node:fs"; import {join} from "node:path"; import {Parser} from "acorn"; -import type {CallExpression, Identifier, Node, Program} from "acorn"; +import type {Identifier, Node, Program} from "acorn"; import type {ExportAllDeclaration, ExportNamedDeclaration, ImportDeclaration, ImportExpression} from "acorn"; import {simple} from "acorn-walk"; import {isEnoent} from "../error.js"; @@ -10,17 +10,16 @@ import {type Feature, type ImportReference, type JavaScriptNode} from "../javasc import {parseOptions} from "../javascript.js"; import {Sourcemap} from "../sourcemap.js"; import {relativeUrl, resolvePath} from "../url.js"; -import {getStringLiteralValue, isStringLiteral} from "./features.js"; -import {findFetches, maybeAddFetch, rewriteIfLocalFetch} from "./fetches.js"; +import {getFeature, getStringLiteralValue, isStringLiteral} from "./features.js"; import {defaultGlobals} from "./globals.js"; import {findReferences} from "./references.js"; type ImportNode = ImportDeclaration | ImportExpression; type ExportNode = ExportAllDeclaration | ExportNamedDeclaration; -export interface ImportsAndFetches { +export interface ImportsAndFeatures { imports: ImportReference[]; - fetches: Feature[]; + features: Feature[]; } /** @@ -47,17 +46,14 @@ export function findExports(body: Node): ExportNode[] { * Recursively processes any imported local ES modules. The returned transitive * import paths are relative to the given source path. */ - -export function findImports(body: Node, root: string, path: string): ImportsAndFetches { - const references: Identifier[] = findReferences(body, defaultGlobals); +export function findImports(body: Node, root: string, path: string): ImportsAndFeatures { const imports: ImportReference[] = []; - const fetches: Feature[] = []; + const features: Feature[] = []; const paths: string[] = []; simple(body, { ImportDeclaration: findImport, - ImportExpression: findImport, - CallExpression: findFetch + ImportExpression: findImport }); function findImport(node: ImportNode) { @@ -71,14 +67,10 @@ export function findImports(body: Node, root: string, path: string): ImportsAndF } } - function findFetch(node) { - maybeAddFetch(fetches, node, references, path); - } - // Recursively process any imported local ES modules. - const features = parseLocalImports(root, paths); - imports.push(...features.imports); - fetches.push(...features.fetches); + const transitive = parseLocalImports(root, paths); + imports.push(...transitive.imports); + features.push(...transitive.features); // Make all local paths relative to the source path. for (const i of imports) { @@ -87,7 +79,7 @@ export function findImports(body: Node, root: string, path: string): ImportsAndF } } - return {imports, fetches}; + return {imports, features}; } /** @@ -96,19 +88,19 @@ export function findImports(body: Node, root: string, path: string): ImportsAndF * appends to imports. The paths here are always relative to the root (unlike * findImports above!). */ -export function parseLocalImports(root: string, paths: string[]): ImportsAndFetches { +export function parseLocalImports(root: string, paths: string[]): ImportsAndFeatures { const imports: ImportReference[] = []; - const fetches: Feature[] = []; + const features: Feature[] = []; const set = new Set(paths); for (const path of set) { imports.push({type: "local", name: path}); try { const input = readFileSync(join(root, path), "utf-8"); - const program = Parser.parse(input, parseOptions); + const body = Parser.parse(input, parseOptions); simple( - program, + body, { ImportDeclaration: findImport, ImportExpression: findImport, @@ -118,7 +110,8 @@ export function parseLocalImports(root: string, paths: string[]): ImportsAndFetc undefined, path ); - fetches.push(...findFetches(program, path)); + + features.push(...findImportFeatures(body, path, input)); } catch (error) { if (!isEnoent(error) && !(error instanceof SyntaxError)) throw error; } @@ -136,13 +129,79 @@ export function parseLocalImports(root: string, paths: string[]): ImportsAndFetc } } - return {imports, fetches}; + return {imports, features}; +} + +/** + * Returns a map from Identifier to the feature type, such as FileAttachment. + * Note that this may be different than the identifier.name because of aliasing. + */ +export function getFeatureReferenceMap(node: Node): Map { + const declarations = new Set<{name: string}>(); + const alias = new Map(); + let globals: Set | undefined; + + // Find the declared local names of the imported symbol. Only named imports + // are supported. TODO Support namespace imports? + simple(node, { + ImportDeclaration(node) { + if (node.source.value === "npm:@observablehq/stdlib") { + for (const specifier of node.specifiers) { + if ( + specifier.type === "ImportSpecifier" && + specifier.imported.type === "Identifier" && + (specifier.imported.name === "FileAttachment" || + specifier.imported.name === "Secret" || + specifier.imported.name === "DatabaseClient") + ) { + declarations.add(specifier.local); + alias.set(specifier.local.name, specifier.imported.name); + } + } + } + } + }); + + // If the import is masking a global, don’t treat it as a global (since we’ll + // ignore the import declaration below). + for (const name of alias.keys()) { + if (defaultGlobals.has(name)) { + if (globals === undefined) globals = new Set(defaultGlobals); + globals.delete(name); + } + } + + function filterDeclaration(node: {name: string}): boolean { + return !declarations.has(node); // treat the imported declaration as unbound + } + + const references = findReferences(node, {globals, filterDeclaration}); + const map = new Map(); + for (const r of references) { + const type = alias.get(r.name); + if (type) map.set(r, type); + } + return map; +} + +export function findImportFeatures(node: Node, path: string, input: string): Feature[] { + const featureMap = getFeatureReferenceMap(node); + const features: Feature[] = []; + + simple(node, { + CallExpression(node) { + const type = featureMap.get(node.callee as Identifier); + if (type) features.push(getFeature(type, node, path, input)); + } + }); + + return features; } -/** Rewrites import specifiers in the specified ES module source. */ -export async function rewriteModule(input: string, sourcePath: string, resolver: ImportResolver): Promise { +/** Rewrites import specifiers and FileAttachment calls in the specified ES module source. */ +export async function rewriteModule(input: string, path: string, resolver: ImportResolver): Promise { const body = Parser.parse(input, parseOptions); - const references: Identifier[] = findReferences(body, defaultGlobals); + const featureMap = getFeatureReferenceMap(body); const output = new Sourcemap(input); const imports: (ImportNode | ExportNode)[] = []; @@ -151,8 +210,16 @@ export async function rewriteModule(input: string, sourcePath: string, resolver: ImportExpression: rewriteImport, ExportAllDeclaration: rewriteImport, ExportNamedDeclaration: rewriteImport, - CallExpression(node: CallExpression) { - rewriteIfLocalFetch(node, output, references, sourcePath, {resolveMeta: true}); + CallExpression(node) { + const type = featureMap.get(node.callee as Identifier); + if (type) { + const feature = getFeature(type, node, path, input); // validate syntax + if (feature.type === "FileAttachment") { + const arg = node.arguments[0]; + const result = JSON.stringify(relativeUrl(join("_import", path), feature.name)); + output.replaceLeft(arg.start, arg.end, `${result}, import.meta.url`); + } + } } }); @@ -165,7 +232,7 @@ export async function rewriteModule(input: string, sourcePath: string, resolver: output.replaceLeft( node.source.start, node.source.end, - JSON.stringify(await resolver(sourcePath, getStringLiteralValue(node.source))) + JSON.stringify(await resolver(path, getStringLiteralValue(node.source))) ); } } @@ -341,9 +408,7 @@ async function fetchModulePreloads(href: string): Promise | undefine function findImport(node: ImportNode | ExportNode) { if (isStringLiteral(node.source)) { const value = getStringLiteralValue(node.source); - if (["./", "../", "/"].some((prefix) => value.startsWith(prefix))) { - imports.add(String(new URL(value, href))); - } + if (isPathImport(value)) imports.add(String(new URL(value, href))); } } // TODO integrityCache.set(href, `sha384-${createHash("sha384").update(body).digest("base64")}`); @@ -426,9 +491,9 @@ function getModuleHash(root: string, path: string): string { if (!isEnoent(error)) throw error; } // TODO can’t simply concatenate here; we need a delimiter - const {imports, fetches} = parseLocalImports(root, [path]); - for (const i of [...imports, ...fetches]) { - if (i.type === "local") { + const {imports, features} = parseLocalImports(root, [path]); + for (const i of [...imports, ...features]) { + if (i.type === "local" || i.type === "FileAttachment") { try { hash.update(readFileSync(join(root, i.name), "utf-8")); } catch (error) { @@ -448,10 +513,12 @@ function rewriteImportSpecifier(node) { : `${node.imported.name}: ${node.local.name}`; } +export function isPathImport(specifier: string): boolean { + return ["./", "../", "/"].some((prefix) => specifier.startsWith(prefix)); +} + export function isLocalImport(specifier: string, path: string): boolean { - return ( - ["./", "../", "/"].some((prefix) => specifier.startsWith(prefix)) && !resolvePath(path, specifier).startsWith("../") - ); + return isPathImport(specifier) && !resolvePath(path, specifier).startsWith("../"); } function isNamespaceSpecifier(node) { diff --git a/src/javascript/references.ts b/src/javascript/references.ts index c7ddaf51d..d5d85da9c 100644 --- a/src/javascript/references.ts +++ b/src/javascript/references.ts @@ -15,14 +15,15 @@ import type { Program } from "acorn"; import {ancestor} from "acorn-walk"; +import {defaultGlobals} from "./globals.js"; // Based on https://github.com/ForbesLindesay/acorn-globals -// Copyright (c) 2014 Forbes Lindesay +// Portions copyright 2014 Forbes Lindesay. // https://github.com/ForbesLindesay/acorn-globals/blob/master/LICENSE -type Func = FunctionExpression | FunctionDeclaration | ArrowFunctionExpression | AnonymousFunctionDeclaration; +type FunctionNode = FunctionExpression | FunctionDeclaration | ArrowFunctionExpression | AnonymousFunctionDeclaration; -function isScope(node: Node): node is Func | Program { +function isScope(node: Node): node is FunctionNode | Program { return ( node.type === "FunctionExpression" || node.type === "FunctionDeclaration" || @@ -32,7 +33,7 @@ function isScope(node: Node): node is Func | Program { } // prettier-ignore -function isBlockScope(node: Node): node is Func | Program | BlockStatement | ForInStatement | ForOfStatement | ForStatement { +function isBlockScope(node: Node): node is FunctionNode | Program | BlockStatement | ForInStatement | ForOfStatement | ForStatement { return ( node.type === "BlockStatement" || node.type === "SwitchStatement" || @@ -43,9 +44,17 @@ function isBlockScope(node: Node): node is Func | Program | BlockStatement | For ); } -export function findReferences(node: Node, globals: Set): Identifier[] { +export function findReferences( + node: Node, + { + globals = defaultGlobals, + filterDeclaration = () => true + }: { + globals?: Set; + filterDeclaration?: (identifier: {name: string}) => any; + } = {} +): Identifier[] { const locals = new Map>(); - const globalSet = new Set(globals); const references: Identifier[] = []; function hasLocal(node: Node, name: string): boolean { @@ -54,6 +63,7 @@ export function findReferences(node: Node, globals: Set): Identifier[] { } function declareLocal(node: Node, id: {name: string}): void { + if (!filterDeclaration(id)) return; const l = locals.get(node); if (l) l.add(id.name); else locals.set(node, new Set([id.name])); @@ -63,7 +73,7 @@ export function findReferences(node: Node, globals: Set): Identifier[] { if (node.id) declareLocal(node, node.id); } - function declareFunction(node: Func) { + function declareFunction(node: FunctionNode) { node.params.forEach((param) => declarePattern(param, node)); if (node.id) declareLocal(node, node.id); if (node.type !== "ArrowFunctionExpression") declareLocal(node, {name: "arguments"}); @@ -139,7 +149,7 @@ export function findReferences(node: Node, globals: Set): Identifier[] { return; } } - if (!globalSet.has(name)) { + if (!globals.has(name)) { references.push(node); } } diff --git a/test/input/build/fetches/foo/foo.js b/test/input/build/fetches/foo/foo.js index 0a789ba14..78a1dd0ba 100644 --- a/test/input/build/fetches/foo/foo.js +++ b/test/input/build/fetches/foo/foo.js @@ -1,2 +1,3 @@ -export const fooJsonData = await fetch("./foo-data.json").then(d => d.json()); -export const fooCsvData = await fetch("./foo-data.csv").then(d => d.text()); +import {FileAttachment} from "npm:@observablehq/stdlib"; +export const fooJsonData = await FileAttachment("foo-data.json").json(); +export const fooCsvData = await FileAttachment("foo-data.csv").text(); diff --git a/test/input/build/fetches/top.js b/test/input/build/fetches/top.js index 9b45fda2d..4702eb3ae 100644 --- a/test/input/build/fetches/top.js +++ b/test/input/build/fetches/top.js @@ -1,3 +1,4 @@ +import {FileAttachment} from "npm:@observablehq/stdlib"; export {fooCsvData, fooJsonData} from "./foo/foo.js"; -export const topJsonData = await fetch("./top-data.json").then(d => d.json()); -export const topCsvData = await fetch("./top-data.csv").then(d => d.text()); +export const topJsonData = await FileAttachment("top-data.json").json(); +export const topCsvData = await FileAttachment("top-data.csv").text(); diff --git a/test/input/build/files/files.md b/test/input/build/files/files.md index e13d4e19d..4e1580146 100644 --- a/test/input/build/files/files.md +++ b/test/input/build/files/files.md @@ -2,14 +2,6 @@ -```js -fetch("./file-top.csv") -``` - -```js -fetch("./subsection/file-sub.csv") -``` - ```js FileAttachment("file-top.csv") ``` diff --git a/test/input/build/files/subsection/subfiles.md b/test/input/build/files/subsection/subfiles.md index 6449283ee..b30f5e306 100644 --- a/test/input/build/files/subsection/subfiles.md +++ b/test/input/build/files/subsection/subfiles.md @@ -1,14 +1,6 @@ -```js -fetch("../file-top.csv") -``` - -```js -fetch("./file-sub.csv") -``` - ```js FileAttachment("../file-top.csv") ``` diff --git a/test/input/build/imports/foo/foo.md b/test/input/build/imports/foo/foo.md index e6f758312..96435f545 100644 --- a/test/input/build/imports/foo/foo.md +++ b/test/input/build/imports/foo/foo.md @@ -7,5 +7,5 @@ import {top} from "/top.js"; display(bar); display(top); -fetch("/top.js"); +FileAttachment("/top.js"); ``` diff --git a/test/input/build/multi/index.md b/test/input/build/multi/index.md index 76767a922..783d6b813 100644 --- a/test/input/build/multi/index.md +++ b/test/input/build/multi/index.md @@ -9,6 +9,6 @@ Input.table(f1) ``` ```js -const f2 = fetch("./file2.csv"); +const f2 = FileAttachment("./file2.csv"); ``` diff --git a/test/input/fetch-parent-dir.md b/test/input/fetch-parent-dir.md index a64821832..3f92fb2c6 100644 --- a/test/input/fetch-parent-dir.md +++ b/test/input/fetch-parent-dir.md @@ -3,34 +3,17 @@ Trying to fetch from the parent directory should fail. ```js -const fail1 = fetch("../NOENT.md").then(d => d.text()) +const fail1 = FileAttachment("../NOENT.md").text() ``` ```js -const fail2 = FileAttachment("../NOENT.md").text() +const fail2 = FileAttachment("../README.md").text() ``` ```js -const fail3 = fetch("../README.md").then(d => d.text()) +const fail3 = FileAttachment("./NOENT.md").text() ``` ```js -const fail4 = FileAttachment("../README.md").text() +const ok1 = FileAttachment("./tex-expression.md").text() ``` - -```js -const fail5 = fetch("./NOENT.md").then(d => d.text()) -``` - -```js -const fail6 = FileAttachment("./NOENT.md").text() -``` - -```js -const ok1 = fetch("./tex-expression.md").then(d => d.text()) -``` - -```js -const ok2 = FileAttachment("./tex-expression.md").text() -``` - diff --git a/test/input/imports/baz.js b/test/input/imports/baz.js index d05c0a4eb..f9a426bff 100644 --- a/test/input/imports/baz.js +++ b/test/input/imports/baz.js @@ -1 +1,3 @@ -export const data = fetch("./fetch-local-data.json").then(d => d.json()); +import {FileAttachment} from "npm:@observablehq/stdlib"; + +export const data = FileAttachment("./fetch-local-data.json").json(); diff --git a/test/input/imports/fetch-local-data.json b/test/input/imports/fetch-local-data.json new file mode 100644 index 000000000..b5d8bb58d --- /dev/null +++ b/test/input/imports/fetch-local-data.json @@ -0,0 +1 @@ +[1, 2, 3] diff --git a/test/input/local-fetch.md b/test/input/local-fetch.md index 2ac27c95d..b8d63a5b7 100644 --- a/test/input/local-fetch.md +++ b/test/input/local-fetch.md @@ -1,5 +1,5 @@ # Local fetch ```js -fetch("./local-fetch.md") +FileAttachment("./local-fetch.md") ``` diff --git a/test/javascript/fetches-test.ts b/test/javascript/fetches-test.ts deleted file mode 100644 index 3669ba392..000000000 --- a/test/javascript/fetches-test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import assert from "node:assert"; -import type {ArrowFunctionExpression, CallExpression, Identifier} from "acorn"; -import {Parser} from "acorn"; -import {ascending} from "d3-array"; -import {rewriteIfLocalFetch} from "../../src/javascript/fetches.js"; -import {parseLocalImports} from "../../src/javascript/imports.js"; -import type {Feature} from "../../src/javascript.js"; -import {Sourcemap} from "../../src/sourcemap.js"; - -describe("parseLocalFetches(root, paths)", () => { - it("find all local fetches in one file", () => { - assert.deepStrictEqual(parseLocalImports("test/input/build/fetches", ["foo/foo.js"]).fetches.sort(compareImport), [ - {name: "./foo-data.csv", type: "FileAttachment"}, - {name: "./foo-data.json", type: "FileAttachment"} - ]); - }); - it("find all local fetches via transivite import", () => { - assert.deepStrictEqual(parseLocalImports("test/input/build/fetches", ["top.js"]).fetches.sort(compareImport), [ - {name: "./foo-data.csv", type: "FileAttachment"}, - {name: "./foo-data.json", type: "FileAttachment"}, - {name: "./top-data.csv", type: "FileAttachment"}, - {name: "./top-data.json", type: "FileAttachment"} - ]); - }); -}); - -function testFetch(target: string, sourcePath: string, resolveMeta = false): string { - const input = `fetch(${JSON.stringify(target)})`; - const node = Parser.parseExpressionAt(input, 0, {ecmaVersion: 13}) as CallExpression; - const output = new Sourcemap(input); - rewriteIfLocalFetch(node, output, [], sourcePath, {resolveMeta}); - return String(output); -} - -describe("rewriteIfLocalFetch(node, output, references, sourcePath, options)", () => { - it("rewrites relative fetches without meta", () => { - assert.strictEqual(testFetch("./test.txt", "test.js"), 'fetch("./_file/test.txt")'); - assert.strictEqual(testFetch("./sub/test.txt", "test.js"), 'fetch("./_file/sub/test.txt")'); - assert.strictEqual(testFetch("./test.txt", "sub/test.js"), 'fetch("../_file/sub/test.txt")'); - assert.strictEqual(testFetch("../test.txt", "sub/test.js"), 'fetch("../_file/test.txt")'); - }); - it("rewrites relative fetches with meta", () => { - assert.strictEqual(testFetch("./test.txt", "test.js", true), 'fetch(new URL("../_file/test.txt", import.meta.url))'); // prettier-ignore - assert.strictEqual(testFetch("./sub/test.txt", "test.js", true), 'fetch(new URL("../_file/sub/test.txt", import.meta.url))'); // prettier-ignore - assert.strictEqual(testFetch("./test.txt", "sub/test.js", true), 'fetch(new URL("../../_file/sub/test.txt", import.meta.url))'); // prettier-ignore - assert.strictEqual(testFetch("../test.txt", "sub/test.js", true), 'fetch(new URL("../../_file/test.txt", import.meta.url))'); // prettier-ignore - }); - it("ignores fetches that don’t start with ./, ../, or /", () => { - assert.strictEqual(testFetch("test.txt", "test.js"), 'fetch("test.txt")'); - assert.strictEqual(testFetch("sub/test.txt", "test.js"), 'fetch("sub/test.txt")'); - assert.strictEqual(testFetch("test.txt", "sub/test.js"), 'fetch("test.txt")'); - assert.strictEqual(testFetch("test.txt", "test.js", true), 'fetch("test.txt")'); - assert.strictEqual(testFetch("sub/test.txt", "test.js", true), 'fetch("sub/test.txt")'); - assert.strictEqual(testFetch("test.txt", "sub/test.js", true), 'fetch("test.txt")'); - }); - it("rewrites absolute fetches without meta", () => { - assert.strictEqual(testFetch("/test.txt", "test.js"), 'fetch("./_file/test.txt")'); - assert.strictEqual(testFetch("/sub/test.txt", "test.js"), 'fetch("./_file/sub/test.txt")'); - assert.strictEqual(testFetch("/test.txt", "sub/test.js"), 'fetch("../_file/test.txt")'); - }); - it("rewrites absolute fetches with meta", () => { - assert.strictEqual(testFetch("/test.txt", "test.js", true), 'fetch(new URL("../_file/test.txt", import.meta.url))'); // prettier-ignore - assert.strictEqual(testFetch("/sub/test.txt", "test.js", true), 'fetch(new URL("../_file/sub/test.txt", import.meta.url))'); // prettier-ignore - assert.strictEqual(testFetch("/test.txt", "sub/test.js", true), 'fetch(new URL("../../_file/test.txt", import.meta.url))'); // prettier-ignore - }); - it("does not ignore fetch if not masked by a reference", () => { - const input = '((fetch) => fetch("./test.txt"))(eval)'; - const node = Parser.parseExpressionAt(input, 0, {ecmaVersion: 13}) as CallExpression; - const call = (node.callee as ArrowFunctionExpression).body as CallExpression; - assert.strictEqual(input.slice(call.start, call.end), 'fetch("./test.txt")'); - const output = new Sourcemap(input); - rewriteIfLocalFetch(call, output, [], "test.js"); - assert.strictEqual(String(output), '((fetch) => fetch("./_file/test.txt"))(eval)'); - }); - it("ignores fetch if masked by a reference", () => { - const input = '((fetch) => fetch("./test.txt"))(eval)'; - const node = Parser.parseExpressionAt(input, 0, {ecmaVersion: 13}) as CallExpression; - const call = (node.callee as ArrowFunctionExpression).body as CallExpression; - assert.strictEqual(input.slice(call.start, call.end), 'fetch("./test.txt")'); - const output = new Sourcemap(input); - rewriteIfLocalFetch(call, output, [call.callee as Identifier], "test.js"); - assert.strictEqual(String(output), input); - }); - it("ignores non-identifier calls", () => { - const input = 'window.fetch("./test.txt")'; - const node = Parser.parseExpressionAt(input, 0, {ecmaVersion: 13}) as CallExpression; - const output = new Sourcemap(input); - rewriteIfLocalFetch(node, output, [], "test.js"); - assert.strictEqual(String(output), input); - }); - it("ignores non-fetch calls", () => { - const input = 'fletch("./test.txt")'; - const node = Parser.parseExpressionAt(input, 0, {ecmaVersion: 13}) as CallExpression; - const output = new Sourcemap(input); - rewriteIfLocalFetch(node, output, [], "test.js"); - assert.strictEqual(String(output), input); - }); - it("rewrites single-quoted literals", () => { - const input = "fetch('./test.txt')"; - const node = Parser.parseExpressionAt(input, 0, {ecmaVersion: 13}) as CallExpression; - const output = new Sourcemap(input); - rewriteIfLocalFetch(node, output, [], "test.js"); - assert.strictEqual(String(output), 'fetch("./_file/test.txt")'); - }); - it("rewrites template-quoted literals", () => { - const input = "fetch(`./test.txt`)"; - const node = Parser.parseExpressionAt(input, 0, {ecmaVersion: 13}) as CallExpression; - const output = new Sourcemap(input); - rewriteIfLocalFetch(node, output, [], "test.js"); - assert.strictEqual(String(output), 'fetch("./_file/test.txt")'); - }); - it("ignores non-literal calls", () => { - const input = "fetch(`./${'test'}.txt`)"; - const node = Parser.parseExpressionAt(input, 0, {ecmaVersion: 13}) as CallExpression; - const output = new Sourcemap(input); - rewriteIfLocalFetch(node, output, [], "test.js"); - assert.strictEqual(String(output), input); - }); - it("ignores URL fetches", () => { - assert.strictEqual(testFetch("https://example.com", "test.js"), 'fetch("https://example.com")'); - assert.strictEqual(testFetch("https://example.com", "sub/test.js"), 'fetch("https://example.com")'); - assert.strictEqual(testFetch("https://example.com", "test.js", true), 'fetch("https://example.com")'); - assert.strictEqual(testFetch("https://example.com", "sub/test.js", true), 'fetch("https://example.com")'); - }); - it("ignores non-local path fetches", () => { - assert.strictEqual(testFetch("../test.txt", "test.js"), 'fetch("../test.txt")'); - assert.strictEqual(testFetch("./../test.txt", "test.js"), 'fetch("./../test.txt")'); - assert.strictEqual(testFetch("../../test.txt", "sub/test.js"), 'fetch("../../test.txt")'); - assert.strictEqual(testFetch("./../../test.txt", "sub/test.js"), 'fetch("./../../test.txt")'); - }); -}); - -function compareImport(a: Feature, b: Feature): number { - return ascending(a.type, b.type) || ascending(a.name, b.name); -} diff --git a/test/javascript/imports-test.ts b/test/javascript/imports-test.ts index b90261c62..291b326ac 100644 --- a/test/javascript/imports-test.ts +++ b/test/javascript/imports-test.ts @@ -1,11 +1,13 @@ import assert from "node:assert"; +import type {Node, Program} from "acorn"; +import {Parser} from "acorn"; import {ascending} from "d3-array"; -import {parseLocalImports} from "../../src/javascript/imports.js"; -import {type ImportReference} from "../../src/javascript.js"; +import {getFeatureReferenceMap, parseLocalImports, rewriteModule} from "../../src/javascript/imports.js"; +import type {Feature, ImportReference} from "../../src/javascript.js"; describe("parseLocalImports(root, paths)", () => { it("finds all local imports in one file", () => { - assert.deepStrictEqual(parseLocalImports("test/input/build/imports", ["foo/foo.js"]).imports.sort(compareImport), [ + assert.deepStrictEqual(parseLocalImports("test/input/build/imports", ["foo/foo.js"]).imports.sort(order), [ {name: "npm:d3", type: "global"}, {name: "bar/bar.js", type: "local"}, {name: "bar/baz.js", type: "local"}, @@ -15,9 +17,7 @@ describe("parseLocalImports(root, paths)", () => { }); it("finds all local imports in multiple files", () => { assert.deepStrictEqual( - parseLocalImports("test/input/imports", ["transitive-static-import.js", "dynamic-import.js"]).imports.sort( - compareImport - ), + parseLocalImports("test/input/imports", ["transitive-static-import.js", "dynamic-import.js"]).imports.sort(order), [ {name: "bar.js", type: "local"}, {name: "dynamic-import.js", type: "local"}, @@ -28,7 +28,7 @@ describe("parseLocalImports(root, paths)", () => { }); it("ignores missing files", () => { assert.deepStrictEqual( - parseLocalImports("test/input/imports", ["static-import.js", "does-not-exist.js"]).imports.sort(compareImport), + parseLocalImports("test/input/imports", ["static-import.js", "does-not-exist.js"]).imports.sort(order), [ {name: "bar.js", type: "local"}, {name: "does-not-exist.js", type: "local"}, @@ -36,8 +36,165 @@ describe("parseLocalImports(root, paths)", () => { ] ); }); + it("find all local fetches in one file", () => { + assert.deepStrictEqual(parseLocalImports("test/input/build/fetches", ["foo/foo.js"]).features.sort(order), [ + {name: "foo/foo-data.csv", type: "FileAttachment"}, + {name: "foo/foo-data.json", type: "FileAttachment"} + ]); + }); + it("find all local fetches via transitive import", () => { + assert.deepStrictEqual(parseLocalImports("test/input/build/fetches", ["top.js"]).features.sort(order), [ + {name: "foo/foo-data.csv", type: "FileAttachment"}, + {name: "foo/foo-data.json", type: "FileAttachment"}, + {name: "top-data.csv", type: "FileAttachment"}, + {name: "top-data.json", type: "FileAttachment"} + ]); + }); +}); + +describe("findImportFeatureReferences(node)", () => { + it("finds the import declaration", () => { + const node = parse('import {FileAttachment} from "npm:@observablehq/stdlib";\nFileAttachment("file.txt");'); + assert.deepStrictEqual(Array.from(getFeatureReferenceMap(node).keys(), object), [ + {type: "Identifier", name: "FileAttachment", start: 57, end: 71} + ]); + }); + it("finds the import declaration if aliased", () => { + const node = parse('import {FileAttachment as F} from "npm:@observablehq/stdlib";\nF("file.txt");'); + assert.deepStrictEqual(Array.from(getFeatureReferenceMap(node).keys(), object), [ + {type: "Identifier", name: "F", start: 62, end: 63} + ]); + }); + it("finds the import declaration if aliased and masking a global", () => { + const node = parse('import {FileAttachment as File} from "npm:@observablehq/stdlib";\nFile("file.txt");'); + assert.deepStrictEqual(Array.from(getFeatureReferenceMap(node).keys(), object), [ + {type: "Identifier", name: "File", start: 65, end: 69} + ]); + }); + it("finds the import declaration if multiple aliases", () => { + const node = parse('import {FileAttachment as F, FileAttachment as G} from "npm:@observablehq/stdlib";\nF("file.txt");\nG("file.txt");'); // prettier-ignore + assert.deepStrictEqual(Array.from(getFeatureReferenceMap(node).keys(), object), [ + {type: "Identifier", name: "F", start: 83, end: 84}, + {type: "Identifier", name: "G", start: 98, end: 99} + ]); + }); + it("ignores import declarations from another module", () => { + const node = parse('import {FileAttachment as F} from "npm:@observablehq/not-stdlib";\nF("file.txt");'); + assert.deepStrictEqual(Array.from(getFeatureReferenceMap(node).keys(), object), []); + }); + it.skip("supports namespace imports", () => { + const node = parse('import * as O from "npm:@observablehq/stdlib";\nO.FileAttachment("file.txt");'); + assert.deepStrictEqual(Array.from(getFeatureReferenceMap(node).keys(), object), [ + {type: "Identifier", name: "FileAttachment", start: 49, end: 63} + ]); + }); + it("ignores masked references", () => { + const node = parse('import {FileAttachment} from "npm:@observablehq/stdlib";\n((FileAttachment) => FileAttachment("file.txt"))(String);'); // prettier-ignore + assert.deepStrictEqual(Array.from(getFeatureReferenceMap(node).keys(), object), []); + }); +}); + +async function testFile(target: string, path: string): Promise { + const input = `import {FileAttachment} from "npm:@observablehq/stdlib";\nFileAttachment(${JSON.stringify(target)})`; + const output = await rewriteModule(input, path, async (path, specifier) => specifier); + return output.split("\n").pop()!; +} + +describe("rewriteModule(input, path, resolver)", () => { + it("rewrites relative files with import.meta.resolve", async () => { + assert.strictEqual(await testFile("./test.txt", "test.js"), 'FileAttachment("../test.txt", import.meta.url)'); // prettier-ignore + assert.strictEqual(await testFile("./sub/test.txt", "test.js"), 'FileAttachment("../sub/test.txt", import.meta.url)'); // prettier-ignore + assert.strictEqual(await testFile("./test.txt", "sub/test.js"), 'FileAttachment("../../sub/test.txt", import.meta.url)'); // prettier-ignore + assert.strictEqual(await testFile("../test.txt", "sub/test.js"), 'FileAttachment("../../test.txt", import.meta.url)'); // prettier-ignore + }); + it("does not require paths to start with ./, ../, or /", async () => { + assert.strictEqual(await testFile("test.txt", "test.js"), 'FileAttachment("../test.txt", import.meta.url)'); // prettier-ignore + assert.strictEqual(await testFile("sub/test.txt", "test.js"), 'FileAttachment("../sub/test.txt", import.meta.url)'); // prettier-ignore + assert.strictEqual(await testFile("test.txt", "sub/test.js"), 'FileAttachment("../../sub/test.txt", import.meta.url)'); // prettier-ignore + }); + it("rewrites absolute files with meta", async () => { + assert.strictEqual(await testFile("/test.txt", "test.js"), 'FileAttachment("../test.txt", import.meta.url)'); // prettier-ignore + assert.strictEqual(await testFile("/sub/test.txt", "test.js"), 'FileAttachment("../sub/test.txt", import.meta.url)'); // prettier-ignore + assert.strictEqual(await testFile("/test.txt", "sub/test.js"), 'FileAttachment("../../test.txt", import.meta.url)'); // prettier-ignore + }); + it("ignores FileAttachment if masked by a reference", async () => { + const input = 'import {FileAttachment} from "npm:@observablehq/stdlib";\n((FileAttachment) => FileAttachment("./test.txt"))(eval)'; // prettier-ignore + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, '((FileAttachment) => FileAttachment("./test.txt"))(eval)'); + }); + it("ignores FileAttachment if not imported", async () => { + const input = 'import {Generators} from "npm:@observablehq/stdlib";\nFileAttachment("./test.txt")'; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, 'FileAttachment("./test.txt")'); + }); + it("ignores FileAttachment if a comma expression", async () => { + const input = 'import {FileAttachment} from "npm:@observablehq/stdlib";\n(1, FileAttachment)("./test.txt")'; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, '(1, FileAttachment)("./test.txt")'); + }); + it("ignores FileAttachment if not imported from @observablehq/stdlib", async () => { + const input = 'import {FileAttachment} from "npm:@observablehq/not-stdlib";\nFileAttachment("./test.txt")'; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, 'FileAttachment("./test.txt")'); + }); + it("rewrites FileAttachment when aliased", async () => { + const input = 'import {FileAttachment as F} from "npm:@observablehq/stdlib";\nF("./test.txt")'; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, 'F("../test.txt", import.meta.url)'); + }); + it("rewrites FileAttachment when aliased to a global", async () => { + const input = 'import {FileAttachment as File} from "npm:@observablehq/stdlib";\nFile("./test.txt")'; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, 'File("../test.txt", import.meta.url)'); + }); + it.skip("rewrites FileAttachment when imported as a namespace", async () => { + const input = 'import * as O from "npm:@observablehq/stdlib";\nO.FileAttachment("./test.txt")'; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, 'O.FileAttachment("../test.txt", import.meta.url)'); + }); + it("ignores non-FileAttachment calls", async () => { + const input = 'import {FileAttachment} from "npm:@observablehq/stdlib";\nFile("./test.txt")'; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, 'File("./test.txt")'); + }); + it("rewrites single-quoted literals", async () => { + const input = "import {FileAttachment} from \"npm:@observablehq/stdlib\";\nFileAttachment('./test.txt')"; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, 'FileAttachment("../test.txt", import.meta.url)'); + }); + it("rewrites template-quoted literals", async () => { + const input = 'import {FileAttachment} from "npm:@observablehq/stdlib";\nFileAttachment(`./test.txt`)'; + const output = (await rewriteModule(input, "test.js", async (path, specifier) => specifier)).split("\n").pop()!; + assert.strictEqual(output, 'FileAttachment("../test.txt", import.meta.url)'); + }); + it("throws a syntax error with non-literal calls", async () => { + const input = "import {FileAttachment} from \"npm:@observablehq/stdlib\";\nFileAttachment(`./${'test'}.txt`)"; + await assert.rejects(() => rewriteModule(input, "test.js", async (path, specifier) => specifier), /FileAttachment requires a single literal string/); // prettier-ignore + }); + it("throws a syntax error with URL fetches", async () => { + const input = 'import {FileAttachment} from "npm:@observablehq/stdlib";\nFileAttachment("https://example.com")'; + await assert.rejects(() => rewriteModule(input, "test.js", async (path, specifier) => specifier), /non-local file path/); // prettier-ignore + }); + it("ignores non-local path fetches", async () => { + const input1 = 'import {FileAttachment} from "npm:@observablehq/stdlib";\nFileAttachment("../test.txt")'; + const input2 = 'import {FileAttachment} from "npm:@observablehq/stdlib";\nFileAttachment("./../test.txt")'; + const input3 = 'import {FileAttachment} from "npm:@observablehq/stdlib";\nFileAttachment("../../test.txt")'; + const input4 = 'import {FileAttachment} from "npm:@observablehq/stdlib";\nFileAttachment("./../../test.txt")'; + await assert.rejects(() => rewriteModule(input1, "test.js", async (path, specifier) => specifier), /non-local file path/); // prettier-ignore + await assert.rejects(() => rewriteModule(input2, "test.js", async (path, specifier) => specifier), /non-local file path/); // prettier-ignore + await assert.rejects(() => rewriteModule(input3, "sub/test.js", async (path, specifier) => specifier), /non-local file path/); // prettier-ignore + await assert.rejects(() => rewriteModule(input4, "sub/test.js", async (path, specifier) => specifier), /non-local file path/); // prettier-ignore + }); }); -function compareImport(a: ImportReference, b: ImportReference): number { +function parse(input: string): Program { + return Parser.parse(input, {ecmaVersion: 13, sourceType: "module"}); +} + +function object(node: Node) { + return {...node}; +} + +function order(a: ImportReference | Feature, b: ImportReference | Feature): number { return ascending(a.type, b.type) || ascending(a.name, b.name); } diff --git a/test/markdown-test.ts b/test/markdown-test.ts index 268c5273c..c17cf8162 100644 --- a/test/markdown-test.ts +++ b/test/markdown-test.ts @@ -102,12 +102,12 @@ describe("normalizePieceHtml adds local file attachments", () => { assert.deepEqual(context.files, [ { mimeType: "image/jpeg", - name: "large.jpg", + name: "./large.jpg", path: "./_file/large.jpg" }, { mimeType: "image/jpeg", - name: "small.jpg", + name: "./small.jpg", path: "./_file/small.jpg" } ]); @@ -127,7 +127,7 @@ describe("normalizePieceHtml adds local file attachments", () => { assert.deepEqual(context.files, [ { mimeType: "video/quicktime", - name: "observable.mov", + name: "./observable.mov", path: "./_file/observable.mov" } ]); @@ -153,12 +153,12 @@ describe("normalizePieceHtml adds local file attachments", () => { assert.deepEqual(context.files, [ { mimeType: "video/mp4", - name: "observable.mp4", + name: "./observable.mp4", path: "./_file/observable.mp4" }, { mimeType: "video/quicktime", - name: "observable.mov", + name: "./observable.mov", path: "./_file/observable.mov" } ]); @@ -182,12 +182,12 @@ describe("normalizePieceHtml adds local file attachments", () => { assert.deepEqual(context.files, [ { mimeType: "image/png", - name: "observable-logo-narrow.png", + name: "./observable-logo-narrow.png", path: "./_file/observable-logo-narrow.png" }, { mimeType: "image/png", - name: "observable-logo-wide.png", + name: "./observable-logo-wide.png", path: "./_file/observable-logo-wide.png" } ]); @@ -226,7 +226,7 @@ describe("normalizePieceHtml only adds local files", () => { assert.deepEqual(context.files, [ { mimeType: "image/jpeg", - name: "small.jpg", + name: "./small.jpg", path: "./_file/small.jpg" } ]); @@ -252,7 +252,7 @@ describe("normalizePieceHtml only adds local files", () => { assert.deepEqual(context.files, [ { mimeType: "video/quicktime", - name: "observable.mov", + name: "./observable.mov", path: "./_file/observable.mov" } ]); diff --git a/test/output/build/archives/tar.html b/test/output/build/archives/tar.html index c18db6fad..a7f7e5fbd 100644 --- a/test/output/build/archives/tar.html +++ b/test/output/build/archives/tar.html @@ -14,43 +14,43 @@ import {define} from "./_observablehq/client.js"; -define({id: "d5134368", inputs: ["FileAttachment","display"], files: [{"name":"static-tar/file.txt","mimeType":"text/plain","path":"./_file/static-tar/file.txt"}], body: async (FileAttachment,display) => { +define({id: "d5134368", inputs: ["FileAttachment","display"], files: [{"name":"./static-tar/file.txt","mimeType":"text/plain","path":"./_file/static-tar/file.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("static-tar/file.txt").text() )) }}); -define({id: "a0c06958", inputs: ["FileAttachment","display"], files: [{"name":"static-tgz/file.txt","mimeType":"text/plain","path":"./_file/static-tgz/file.txt"}], body: async (FileAttachment,display) => { +define({id: "a0c06958", inputs: ["FileAttachment","display"], files: [{"name":"./static-tgz/file.txt","mimeType":"text/plain","path":"./_file/static-tgz/file.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("static-tgz/file.txt").text() )) }}); -define({id: "d84cd7fb", inputs: ["FileAttachment","display"], files: [{"name":"static-tar/does-not-exist.txt","mimeType":"text/plain","path":"./_file/static-tar/does-not-exist.txt"}], body: async (FileAttachment,display) => { +define({id: "d84cd7fb", inputs: ["FileAttachment","display"], files: [{"name":"./static-tar/does-not-exist.txt","mimeType":"text/plain","path":"./_file/static-tar/does-not-exist.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("static-tar/does-not-exist.txt").text() )) }}); -define({id: "86bd51aa", inputs: ["FileAttachment","display"], files: [{"name":"dynamic-tar/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar/file.txt"}], body: async (FileAttachment,display) => { +define({id: "86bd51aa", inputs: ["FileAttachment","display"], files: [{"name":"./dynamic-tar/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar/file.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("dynamic-tar/file.txt").text() )) }}); -define({id: "95938c22", inputs: ["FileAttachment","display"], files: [{"name":"dynamic-tar/does-not-exist.txt","mimeType":"text/plain","path":"./_file/dynamic-tar/does-not-exist.txt"}], body: async (FileAttachment,display) => { +define({id: "95938c22", inputs: ["FileAttachment","display"], files: [{"name":"./dynamic-tar/does-not-exist.txt","mimeType":"text/plain","path":"./_file/dynamic-tar/does-not-exist.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("dynamic-tar/does-not-exist.txt").text() )) }}); -define({id: "7e5740fd", inputs: ["FileAttachment","display"], files: [{"name":"dynamic-tar-gz/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar-gz/file.txt"}], body: async (FileAttachment,display) => { +define({id: "7e5740fd", inputs: ["FileAttachment","display"], files: [{"name":"./dynamic-tar-gz/file.txt","mimeType":"text/plain","path":"./_file/dynamic-tar-gz/file.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("dynamic-tar-gz/file.txt").text() )) }}); -define({id: "d0a58efd", inputs: ["FileAttachment","display"], files: [{"name":"dynamic-tar-gz/does-not-exist.txt","mimeType":"text/plain","path":"./_file/dynamic-tar-gz/does-not-exist.txt"}], body: async (FileAttachment,display) => { +define({id: "d0a58efd", inputs: ["FileAttachment","display"], files: [{"name":"./dynamic-tar-gz/does-not-exist.txt","mimeType":"text/plain","path":"./_file/dynamic-tar-gz/does-not-exist.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("dynamic-tar-gz/does-not-exist.txt").text() )) diff --git a/test/output/build/archives/zip.html b/test/output/build/archives/zip.html index 2938d197b..e0f0a8883 100644 --- a/test/output/build/archives/zip.html +++ b/test/output/build/archives/zip.html @@ -14,25 +14,25 @@ import {define} from "./_observablehq/client.js"; -define({id: "d3b9d0ee", inputs: ["FileAttachment","display"], files: [{"name":"static/file.txt","mimeType":"text/plain","path":"./_file/static/file.txt"}], body: async (FileAttachment,display) => { +define({id: "d3b9d0ee", inputs: ["FileAttachment","display"], files: [{"name":"./static/file.txt","mimeType":"text/plain","path":"./_file/static/file.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("static/file.txt").text() )) }}); -define({id: "bab54217", inputs: ["FileAttachment","display"], files: [{"name":"static/not-found.txt","mimeType":"text/plain","path":"./_file/static/not-found.txt"}], body: async (FileAttachment,display) => { +define({id: "bab54217", inputs: ["FileAttachment","display"], files: [{"name":"./static/not-found.txt","mimeType":"text/plain","path":"./_file/static/not-found.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("static/not-found.txt").text() )) }}); -define({id: "11eec300", inputs: ["FileAttachment","display"], files: [{"name":"dynamic/file.txt","mimeType":"text/plain","path":"./_file/dynamic/file.txt"}], body: async (FileAttachment,display) => { +define({id: "11eec300", inputs: ["FileAttachment","display"], files: [{"name":"./dynamic/file.txt","mimeType":"text/plain","path":"./_file/dynamic/file.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("dynamic/file.txt").text() )) }}); -define({id: "ee2310f3", inputs: ["FileAttachment","display"], files: [{"name":"dynamic/not-found.txt","mimeType":"text/plain","path":"./_file/dynamic/not-found.txt"}], body: async (FileAttachment,display) => { +define({id: "ee2310f3", inputs: ["FileAttachment","display"], files: [{"name":"./dynamic/not-found.txt","mimeType":"text/plain","path":"./_file/dynamic/not-found.txt"}], body: async (FileAttachment,display) => { display(( await FileAttachment("dynamic/not-found.txt").text() )) diff --git a/test/output/build/fetches/_file/foo/foo-data.csv b/test/output/build/fetches/_file/foo/foo-data.csv new file mode 100644 index 000000000..d91c6bdab --- /dev/null +++ b/test/output/build/fetches/_file/foo/foo-data.csv @@ -0,0 +1,3 @@ +eruid,description +batman,uses technology +superman,flies through the air diff --git a/test/output/build/fetches/_file/foo/foo-data.json b/test/output/build/fetches/_file/foo/foo-data.json new file mode 100644 index 000000000..b5d8bb58d --- /dev/null +++ b/test/output/build/fetches/_file/foo/foo-data.json @@ -0,0 +1 @@ +[1, 2, 3] diff --git a/test/output/build/fetches/_import/foo/foo.js b/test/output/build/fetches/_import/foo/foo.js index 0e20474d3..130f9d719 100644 --- a/test/output/build/fetches/_import/foo/foo.js +++ b/test/output/build/fetches/_import/foo/foo.js @@ -1,2 +1,3 @@ -export const fooJsonData = await fetch(new URL("../../_file/foo/foo-data.json", import.meta.url)).then(d => d.json()); -export const fooCsvData = await fetch(new URL("../../_file/foo/foo-data.csv", import.meta.url)).then(d => d.text()); +import {FileAttachment} from "../../_observablehq/stdlib.js"; +export const fooJsonData = await FileAttachment("../../foo/foo-data.json", import.meta.url).json(); +export const fooCsvData = await FileAttachment("../../foo/foo-data.csv", import.meta.url).text(); diff --git a/test/output/build/fetches/_import/top.js b/test/output/build/fetches/_import/top.js index d8dc9599b..94d5dba46 100644 --- a/test/output/build/fetches/_import/top.js +++ b/test/output/build/fetches/_import/top.js @@ -1,3 +1,4 @@ -export {fooCsvData, fooJsonData} from "./foo/foo.js?sha=ed706415f035efdd17afdfeeb09a5ff51231b2a43553a47ae89a6bf03039b1ee"; -export const topJsonData = await fetch(new URL("../_file/top-data.json", import.meta.url)).then(d => d.json()); -export const topCsvData = await fetch(new URL("../_file/top-data.csv", import.meta.url)).then(d => d.text()); +import {FileAttachment} from "../_observablehq/stdlib.js"; +export {fooCsvData, fooJsonData} from "./foo/foo.js?sha=ddc538dfc10d83a59458d5893c89191ef3b2c9b1c02ef6da055423f37388ecf4"; +export const topJsonData = await FileAttachment("../top-data.json", import.meta.url).json(); +export const topCsvData = await FileAttachment("../top-data.csv", import.meta.url).text(); diff --git a/test/output/build/fetches/foo.html b/test/output/build/fetches/foo.html index ce86c3365..113c413ec 100644 --- a/test/output/build/fetches/foo.html +++ b/test/output/build/fetches/foo.html @@ -7,7 +7,7 @@ - + @@ -15,8 +15,8 @@ import {define} from "./_observablehq/client.js"; -define({id: "47a695da", inputs: ["display"], outputs: ["fooJsonData","fooCsvData"], files: [{"name":"./foo-data.json","mimeType":"application/json","path":"./_file/foo-data.json"},{"name":"./foo-data.csv","mimeType":"text/csv","path":"./_file/foo-data.csv"}], body: async (display) => { -const {fooJsonData, fooCsvData} = await import("./_import/foo/foo.js?sha=ed706415f035efdd17afdfeeb09a5ff51231b2a43553a47ae89a6bf03039b1ee"); +define({id: "47a695da", inputs: ["display"], outputs: ["fooJsonData","fooCsvData"], files: [{"name":"./foo/foo-data.json","mimeType":"application/json","path":"./_file/foo/foo-data.json"},{"name":"./foo/foo-data.csv","mimeType":"text/csv","path":"./_file/foo/foo-data.csv"}], body: async (display) => { +const {fooJsonData, fooCsvData} = await import("./_import/foo/foo.js?sha=ddc538dfc10d83a59458d5893c89191ef3b2c9b1c02ef6da055423f37388ecf4"); display(fooJsonData); display(fooCsvData); diff --git a/test/output/build/fetches/top.html b/test/output/build/fetches/top.html index 843ccd644..626e39c0b 100644 --- a/test/output/build/fetches/top.html +++ b/test/output/build/fetches/top.html @@ -7,8 +7,8 @@ - - + + @@ -16,8 +16,8 @@ import {define} from "./_observablehq/client.js"; -define({id: "cb908c08", inputs: ["display"], outputs: ["fooCsvData","fooJsonData","topCsvData","topJsonData"], files: [{"name":"./top-data.json","mimeType":"application/json","path":"./_file/top-data.json"},{"name":"./top-data.csv","mimeType":"text/csv","path":"./_file/top-data.csv"},{"name":"./foo-data.json","mimeType":"application/json","path":"./_file/foo-data.json"},{"name":"./foo-data.csv","mimeType":"text/csv","path":"./_file/foo-data.csv"}], body: async (display) => { -const {fooCsvData, fooJsonData, topCsvData, topJsonData} = await import("./_import/top.js?sha=b8624239dee696ef885eb778f2e949ea1072ef47180f602502b2d4ad5aa10abf"); +define({id: "cb908c08", inputs: ["display"], outputs: ["fooCsvData","fooJsonData","topCsvData","topJsonData"], files: [{"name":"./top-data.json","mimeType":"application/json","path":"./_file/top-data.json"},{"name":"./top-data.csv","mimeType":"text/csv","path":"./_file/top-data.csv"},{"name":"./foo/foo-data.json","mimeType":"application/json","path":"./_file/foo/foo-data.json"},{"name":"./foo/foo-data.csv","mimeType":"text/csv","path":"./_file/foo/foo-data.csv"}], body: async (display) => { +const {fooCsvData, fooJsonData, topCsvData, topJsonData} = await import("./_import/top.js?sha=a713c9f9e5b3c1e456a16cd100388f7eacfe48ee5fcaf7772f405abe53541cd0"); display(fooJsonData); display(fooCsvData); diff --git a/test/output/build/files/files.html b/test/output/build/files/files.html index 6803340fc..e80d5700c 100644 --- a/test/output/build/files/files.html +++ b/test/output/build/files/files.html @@ -13,25 +13,13 @@ import {define} from "./_observablehq/client.js"; -define({id: "a7808707", inputs: ["display"], files: [{"name":"./file-top.csv","mimeType":"text/csv","path":"./_file/file-top.csv"}], body: (display) => { -display(( -fetch("./_file/file-top.csv") -)) -}}); - -define({id: "03b99abc", inputs: ["display"], files: [{"name":"./subsection/file-sub.csv","mimeType":"text/csv","path":"./_file/subsection/file-sub.csv"}], body: (display) => { -display(( -fetch("./_file/subsection/file-sub.csv") -)) -}}); - -define({id: "10037545", inputs: ["FileAttachment","display"], files: [{"name":"file-top.csv","mimeType":"text/csv","path":"./_file/file-top.csv"}], body: (FileAttachment,display) => { +define({id: "10037545", inputs: ["FileAttachment","display"], files: [{"name":"./file-top.csv","mimeType":"text/csv","path":"./_file/file-top.csv"}], body: (FileAttachment,display) => { display(( FileAttachment("file-top.csv") )) }}); -define({id: "453a8147", inputs: ["FileAttachment","display"], files: [{"name":"subsection/file-sub.csv","mimeType":"text/csv","path":"./_file/subsection/file-sub.csv"}], body: (FileAttachment,display) => { +define({id: "453a8147", inputs: ["FileAttachment","display"], files: [{"name":"./subsection/file-sub.csv","mimeType":"text/csv","path":"./_file/subsection/file-sub.csv"}], body: (FileAttachment,display) => { display(( FileAttachment("subsection/file-sub.csv") )) @@ -59,9 +47,7 @@ -
-
-
+