From 7b8296fc93a8444897bfba6e30535d00cdb1f7c7 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Fri, 16 Dec 2022 20:00:34 -0500 Subject: [PATCH 01/50] web standard APIs adoption --- greenwood.config.js | 3 +- packages/cli/src/index.js | 10 ++--- packages/cli/src/lifecycles/compile.js | 2 +- packages/cli/src/lifecycles/config.js | 55 +++++++++++--------------- packages/cli/src/lifecycles/context.js | 24 +++++------ 5 files changed, 41 insertions(+), 53 deletions(-) diff --git a/greenwood.config.js b/greenwood.config.js index 332980001..782957d54 100644 --- a/greenwood.config.js +++ b/greenwood.config.js @@ -6,10 +6,9 @@ import { greenwoodPluginPolyfills } from '@greenwood/plugin-polyfills'; import { greenwoodPluginPostCss } from '@greenwood/plugin-postcss'; import { greenwoodPluginRendererPuppeteer } from '@greenwood/plugin-renderer-puppeteer'; import rollupPluginAnalyzer from 'rollup-plugin-analyzer'; -import { fileURLToPath, URL } from 'url'; export default { - workspace: fileURLToPath(new URL('./www', import.meta.url)), + workspace: new URL('./www/', import.meta.url), prerender: true, optimization: 'inline', staticRouter: true, diff --git a/packages/cli/src/index.js b/packages/cli/src/index.js index f398ed2e7..f968f9729 100755 --- a/packages/cli/src/index.js +++ b/packages/cli/src/index.js @@ -69,22 +69,22 @@ const run = async() => { switch (command) { case 'build': - await (await import('./commands/build.js')).runProductionBuild(compilation); + // await (await import('./commands/build.js')).runProductionBuild(compilation); break; case 'develop': - await (await import('./commands/develop.js')).runDevServer(compilation); + // await (await import('./commands/develop.js')).runDevServer(compilation); break; case 'serve': process.env.__GWD_COMMAND__ = 'build'; - await (await import('./commands/build.js')).runProductionBuild(compilation); - await (await import('./commands/serve.js')).runProdServer(compilation); + // await (await import('./commands/build.js')).runProductionBuild(compilation); + // await (await import('./commands/serve.js')).runProdServer(compilation); break; case 'eject': - await (await import('./commands/eject.js')).ejectConfiguration(compilation); + // await (await import('./commands/eject.js')).ejectConfiguration(compilation); break; default: diff --git a/packages/cli/src/lifecycles/compile.js b/packages/cli/src/lifecycles/compile.js index a8d02f83a..749dce8af 100644 --- a/packages/cli/src/lifecycles/compile.js +++ b/packages/cli/src/lifecycles/compile.js @@ -22,7 +22,7 @@ const generateCompilation = () => { // generate a graph of all pages / components to build console.info('Generating graph of workspace files...'); - compilation = await generateGraph(compilation); + // compilation = await generateGraph(compilation); resolve(compilation); } catch (err) { diff --git a/packages/cli/src/lifecycles/config.js b/packages/cli/src/lifecycles/config.js index 1e4fbdaf8..a931ceef8 100644 --- a/packages/cli/src/lifecycles/config.js +++ b/packages/cli/src/lifecycles/config.js @@ -1,23 +1,23 @@ import fs from 'fs'; -import path from 'path'; -import { fileURLToPath, pathToFileURL, URL } from 'url'; -// get and "tag" all plugins provided / maintained by the @greenwood/cli -// and include as the default set, with all user plugins getting appended -const greenwoodPluginsBasePath = fileURLToPath(new URL('../plugins', import.meta.url)); +const cwd = new URL(`file://${process.cwd()}/`); +const greenwoodPluginsDirectoryUrl = new URL('../plugins/', import.meta.url); const PLUGINS_FLATTENED_DEPTH = 2; +// get and "tag" all plugins provided / maintained by the @greenwood/cli +// and include as the default set, with all user plugins getting appended +// TODO could probably refactored to use a for loop const greenwoodPlugins = (await Promise.all([ - path.join(greenwoodPluginsBasePath, 'copy'), - path.join(greenwoodPluginsBasePath, 'renderer'), - path.join(greenwoodPluginsBasePath, 'resource'), - path.join(greenwoodPluginsBasePath, 'server') -].map(async (pluginDirectory) => { - const files = await fs.promises.readdir(pluginDirectory); + new URL('./copy/', greenwoodPluginsDirectoryUrl), + new URL('./renderer/', greenwoodPluginsDirectoryUrl), + new URL('./resource/', greenwoodPluginsDirectoryUrl), + new URL('./server/', greenwoodPluginsDirectoryUrl) +].map(async (pluginDirectoryUrl) => { + const files = await fs.promises.readdir(pluginDirectoryUrl); return await Promise.all(files.map(async(file) => { - const importPaTh = pathToFileURL(`${pluginDirectory}${path.sep}${file}`); - const pluginImport = await import(importPaTh); + const importUrl = new URL(`./${file}`, pluginDirectoryUrl); + const pluginImport = await import(importUrl); const plugin = pluginImport[Object.keys(pluginImport)[0]]; return Array.isArray(plugin) @@ -34,7 +34,7 @@ const greenwoodPlugins = (await Promise.all([ const optimizations = ['default', 'none', 'static', 'inline']; const pluginTypes = ['copy', 'context', 'resource', 'rollup', 'server', 'source', 'renderer']; const defaultConfig = { - workspace: path.join(process.cwd(), 'src'), + workspace: new URL('./src/', cwd), devServer: { hud: true, port: 1984, @@ -55,35 +55,24 @@ const readAndMergeConfig = async() => { return new Promise(async (resolve, reject) => { try { // deep clone of default config + const configUrl = new URL('./greenwood.config.js', cwd); let customConfig = Object.assign({}, defaultConfig); - if (fs.existsSync(path.join(process.cwd(), 'greenwood.config.js'))) { - const userCfgFile = (await import(pathToFileURL(path.join(process.cwd(), 'greenwood.config.js')))).default; + if (fs.existsSync(configUrl.pathname)) { + const userCfgFile = (await import(configUrl)).default; const { workspace, devServer, markdown, optimization, plugins, port, prerender, staticRouter, pagesDirectory, templatesDirectory, interpolateFrontmatter } = userCfgFile; // workspace validation if (workspace) { - if (typeof workspace !== 'string') { - reject('Error: greenwood.config.js workspace path must be a string'); + if (!workspace instanceof URL) { + reject('Error: greenwood.config.js workspace must be an instance of URL'); } - if (!path.isAbsolute(workspace)) { - // prepend relative path with current directory - customConfig.workspace = path.join(process.cwd(), workspace); + if (!fs.existsSync(workspace.pathname)) { + reject('Error: greenwood.config.js workspace doesn\'t exist! Please double check your configuration.'); } - if (path.isAbsolute(workspace)) { - // use the users provided path - customConfig.workspace = workspace; - } - - if (!fs.existsSync(customConfig.workspace)) { - reject('Error: greenwood.config.js workspace doesn\'t exist! \n' + - 'common issues to check might be: \n' + - '- typo in your workspace directory name, or in greenwood.config.js \n' + - '- if using relative paths, make sure your workspace is in the same cwd as _greenwood.config.js_ \n' + - '- consider using an absolute path, e.g. new URL(\'/your/relative/path/\', import.meta.url)'); - } + customConfig.workspace = workspace; } if (typeof optimization === 'string' && optimizations.indexOf(optimization.toLowerCase()) >= 0) { diff --git a/packages/cli/src/lifecycles/context.js b/packages/cli/src/lifecycles/context.js index d52a3ccee..ce434a2f5 100644 --- a/packages/cli/src/lifecycles/context.js +++ b/packages/cli/src/lifecycles/context.js @@ -1,19 +1,19 @@ import fs from 'fs'; -import path from 'path'; -import { fileURLToPath, URL } from 'url'; const initContext = async({ config }) => { - const scratchDir = path.join(process.cwd(), './.greenwood'); - const outputDir = path.join(process.cwd(), './public'); - const dataDir = fileURLToPath(new URL('../data', import.meta.url)); return new Promise(async (resolve, reject) => { try { - const projectDirectory = process.cwd(); - const userWorkspace = path.join(config.workspace); - const apisDir = path.join(userWorkspace, 'api/'); - const pagesDir = path.join(userWorkspace, `${config.pagesDirectory}/`); - const userTemplatesDir = path.join(userWorkspace, `${config.templatesDirectory}/`); + const { workspace, pagesDirectory, templatesDirectory } = config; + + const projectDirectory = new URL(`file://${process.cwd()}/`); + const scratchDir = new URL('./.greenwood/', projectDirectory); + const outputDir = new URL('./public/', projectDirectory); + const dataDir = new URL('../data/', import.meta.url); + const userWorkspace = workspace; + const apisDir = new URL('./apis/', userWorkspace); + const pagesDir = new URL(`./${pagesDirectory}/`, userWorkspace); + const userTemplatesDir = new URL(`./${templatesDirectory}/`, userWorkspace); const context = { dataDir, @@ -26,8 +26,8 @@ const initContext = async({ config }) => { projectDirectory }; - if (!fs.existsSync(scratchDir)) { - fs.mkdirSync(scratchDir, { + if (!fs.existsSync(scratchDir.pathname)) { + fs.mkdirSync(scratchDir.pathname, { recursive: true }); } From 52deb4b582f3c98cdba606ba331b9905d9bf9a5b Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Sat, 17 Dec 2022 20:06:11 -0500 Subject: [PATCH 02/50] update graph lifecyle to work with URLs --- packages/cli/src/lifecycles/compile.js | 2 +- packages/cli/src/lifecycles/graph.js | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/lifecycles/compile.js b/packages/cli/src/lifecycles/compile.js index 749dce8af..a8d02f83a 100644 --- a/packages/cli/src/lifecycles/compile.js +++ b/packages/cli/src/lifecycles/compile.js @@ -22,7 +22,7 @@ const generateCompilation = () => { // generate a graph of all pages / components to build console.info('Generating graph of workspace files...'); - // compilation = await generateGraph(compilation); + compilation = await generateGraph(compilation); resolve(compilation); } catch (err) { diff --git a/packages/cli/src/lifecycles/graph.js b/packages/cli/src/lifecycles/graph.js index 2f6381668..ca7cae380 100644 --- a/packages/cli/src/lifecycles/graph.js +++ b/packages/cli/src/lifecycles/graph.js @@ -214,16 +214,16 @@ const generateGraph = async (compilation) => { }; console.debug('building from local sources...'); - if (fs.existsSync(path.join(userWorkspace, 'index.html'))) { // SPA + if (fs.existsSync(path.join(userWorkspace.pathname, 'index.html'))) { // SPA graph = [{ ...graph[0], - path: `${userWorkspace}${path.sep}index.html`, + path: `${userWorkspace.pathname}index.html`, isSPA: true }]; } else { const oldGraph = graph[0]; - graph = fs.existsSync(pagesDir) ? await walkDirectoryForPages(pagesDir) : graph; + graph = fs.existsSync(pagesDir.pathname) ? await walkDirectoryForPages(pagesDir.pathname) : graph; const has404Page = graph.filter(page => page.route === '/404/').length === 1; @@ -279,11 +279,11 @@ const generateGraph = async (compilation) => { compilation.graph = graph; - if (!fs.existsSync(context.scratchDir)) { - await fs.promises.mkdir(context.scratchDir); + if (!fs.existsSync(context.scratchDir.pathname)) { + await fs.promises.mkdir(context.scratchDir.pathname); } - await fs.promises.writeFile(`${context.scratchDir}/graph.json`, JSON.stringify(compilation.graph)); + await fs.promises.writeFile(`${context.scratchDir.pathname}graph.json`, JSON.stringify(compilation.graph)); resolve(compilation); } catch (err) { From 12a21891c723fcd04882ae866b737bc6705c1508 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Sat, 24 Dec 2022 20:56:41 -0500 Subject: [PATCH 03/50] web standards refactoring for Resource plugins resolve lifecycle --- packages/cli/src/lib/resource-interface.js | 106 ++++---- packages/cli/src/lifecycles/serve.js | 228 ++++++++++-------- .../plugins/resource/plugin-node-modules.js | 26 +- .../plugins/resource/plugin-static-router.js | 19 +- .../plugins/resource/plugin-user-workspace.js | 33 +-- .../src/plugins/server/plugin-livereload.js | 19 +- 6 files changed, 220 insertions(+), 211 deletions(-) diff --git a/packages/cli/src/lib/resource-interface.js b/packages/cli/src/lib/resource-interface.js index adb07900d..cb2e2e7b3 100644 --- a/packages/cli/src/lib/resource-interface.js +++ b/packages/cli/src/lib/resource-interface.js @@ -1,5 +1,4 @@ -import fs from 'fs'; -import path from 'path'; +// import fs from 'fs'; class ResourceInterface { constructor(compilation, options = {}) { @@ -11,87 +10,94 @@ class ResourceInterface { // get rid of things like query string parameters // that will break when trying to use with fs - getBareUrlPath(url) { - return url.replace(/\?(.*)/, ''); - } + // TODO URLs will not contain query strings by default, right? + // getBareUrlPath(url) { + // console.debug('getBareUrlPath', { url }); + // return url.replace(/\?(.*)/, ''); + // } // turn relative paths into relatively absolute based on a known root directory // e.g. "../styles/theme.css" -> `${userWorkspace}/styles/theme.css` - resolveRelativeUrl(root, url) { - if (fs.existsSync(path.join(root, url))) { - return url; - } - - let reducedUrl; - - url.split('/') - .filter((segment) => segment !== '') - .reduce((acc, segment) => { - const reducedPath = url.replace(`${acc}/${segment}`, ''); - - if (path.extname(reducedPath) !== '' && fs.existsSync(path.join(root, reducedPath))) { - reducedUrl = reducedPath; - } - return `${acc}/${segment}`; - }, ''); - - return reducedUrl; - } + // resolveRelativeUrl(root, pathname) { + // // console.debug('getBareUrlPath', { root, pathname }); + // if (fs.existsSync(new URL(pathname, root).pathname)) { + // return url; + // } + + // let reducedPathname; + + // pathname.split('/') + // .filter((segment) => segment !== '') + // .reduce((acc, segment) => { + // // console.debug({ acc, segment }); + // const reducedPath = pathname.replace(`${acc}/${segment}`, ''); + + // // console.debug({ reducedPath }); + // // console.debug(new URL(`.${reducedPath}`, root).pathname); + // if (reducedPath.split('.').pop() !== '' && fs.existsSync(new URL(`.${reducedPath}`, root).pathname)) { + // reducedPathname = reducedPath; + // } + // return `${acc}/${segment}`; + // }, ''); + + // // console.debug({ reducedPathname }); + // return reducedPathname; + // } // test if this plugin should change a relative URL from the browser to an absolute path on disk // like for node_modules/ resolution. not commonly needed by most resource plugins // return true | false // eslint-disable-next-line no-unused-vars - async shouldResolve(url) { - return Promise.resolve(false); - } + // async shouldResolve(url) { + // return Promise.resolve(false); + // } - // return an absolute path - async resolve(url) { - return Promise.resolve(url); - } + // // return an absolute path + // async resolve(url) { + // return Promise.resolve(url); + // } // test if this plugin should be used to process a given url / header combo the browser and retu // ex: ` - - `); - - resolve({ - body: newContents - }); - } catch (e) { - reject(e); - } + // handle for monorepos too + const userPackageJson = fs.existsSync(new URL('./package.json', userWorkspace).pathname) + ? JSON.parse(await fs.promises.readFile(new URL('./package.json', userWorkspace), 'utf-8')) // its a monorepo? + : fs.existsSync(new URL('./package.json', projectDirectory).pathname) + ? JSON.parse(await fs.promises.readFile(new URL('./package.json', projectDirectory))) + : {}; + + // if there are dependencies and we haven't generated the importMap already + // walk the project's package.json for all its direct dependencies + // for each entry found in dependencies, find its entry point + // then walk its entry point (e.g. index.js) for imports / exports to add to the importMap + // and then walk its package.json for transitive dependencies and all those import / exports + importMap = !importMap && Object.keys(userPackageJson.dependencies || []).length > 0 + ? await walkPackageJson(userPackageJson) + : importMap || {}; + + // apply import map and shim for users + body = body.replace('', ` + + + + `); + + // TODO avoid having to rebuild response each time? + return new Response(body, { + headers: response.headers }); } } diff --git a/packages/cli/src/plugins/resource/plugin-standard-font.js b/packages/cli/src/plugins/resource/plugin-standard-font.js index ee5f1c400..8358b46c8 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-font.js +++ b/packages/cli/src/plugins/resource/plugin-standard-font.js @@ -29,20 +29,6 @@ class StandardFontResource extends ResourceInterface { 'Content-Type': contentType } }); - // return new Promise(async (resolve, reject) => { - // try { - // const ext = path.extname(url).replace('.', ''); - // const contentType = ext === 'eot' ? 'application/vnd.ms-fontobject' : ext; - // const body = await fs.promises.readFile(url); - - // resolve({ - // body, - // contentType - // }); - // } catch (e) { - // reject(e); - // } - // }); } } diff --git a/packages/cli/src/plugins/resource/plugin-standard-html.js b/packages/cli/src/plugins/resource/plugin-standard-html.js index 9c2ed525a..67b542158 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-html.js +++ b/packages/cli/src/plugins/resource/plugin-standard-html.js @@ -207,8 +207,6 @@ class StandardHtmlResource extends ResourceInterface { const pathname = url.pathname; // const relativeUrl = this.getRelativeUserworkspaceUrl(url).replace(/\\/g, '/'); // and handle for windows const isClientSideRoute = this.compilation.graph[0].isSPA && pathname.split('.').pop() === '' && (request.headers?.accept || '').indexOf(this.contentType) >= 0; - console.debug('StandardHtmlResource.shouldServe', url.pathname); - console.debug('HEADERS', request.headers); const hasMatchingRoute = this.compilation.graph.filter((node) => { return node.route === pathname; }).length === 1; @@ -216,9 +214,7 @@ class StandardHtmlResource extends ResourceInterface { return hasMatchingRoute || isClientSideRoute; } - async serve(url, request) { - console.debug('StandardHtmlResource.serve', url.pathname); - console.debug('HEADERS', request.headers); + async serve(url) { const { config } = this.compilation; const { pagesDir, userTemplatesDir } = this.compilation.context; const { interpolateFrontmatter } = config; @@ -392,7 +388,7 @@ class StandardHtmlResource extends ResourceInterface { `); } - console.debug({ body }); + // TODO avoid having to rebuild response each time? return new Response(body, { headers: { 'Content-Type': this.contentType diff --git a/packages/cli/src/plugins/resource/plugin-standard-image.js b/packages/cli/src/plugins/resource/plugin-standard-image.js index b1f04725c..941172ad9 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-image.js +++ b/packages/cli/src/plugins/resource/plugin-standard-image.js @@ -21,7 +21,7 @@ class StandardFontResource extends ResourceInterface { async serve(url) { const extension = url.pathname.split('.').pop(); - const type = extension === 'svg' ? `${ext}+xml` : extension; + const type = extension === 'svg' ? `${extension}+xml` : extension; const exts = [...this.extensions]; const isIco = extension === 'ico'; let body = ''; @@ -33,6 +33,7 @@ class StandardFontResource extends ResourceInterface { if (extension === 'svg') { body = await fs.promises.readFile(url, 'utf-8'); } else { + // TODO this doesn't seem to work body = await fs.promises.readFile(url); } } else if (isIco) { @@ -40,9 +41,10 @@ class StandardFontResource extends ResourceInterface { body = await fs.promises.readFile(url); } + // TODO avoid having to rebuild response each time? return new Response(body, { headers: { - 'Content-Type': contentType + 'content-type': contentType } }); } diff --git a/packages/cli/src/plugins/resource/plugin-static-router.js b/packages/cli/src/plugins/resource/plugin-static-router.js index e68cb56af..2ed02f648 100644 --- a/packages/cli/src/plugins/resource/plugin-static-router.js +++ b/packages/cli/src/plugins/resource/plugin-static-router.js @@ -27,27 +27,26 @@ class StaticRouterResource extends ResourceInterface { return new Request(`file://${routerUrl.pathname}`); } - async shouldIntercept(url, body, headers = { request: {} }) { - const contentType = headers.request['content-type']; + async shouldIntercept(url, request, response) { + const { pathname } = url; + const contentType = response.headers.get['content-type']; - return Promise.resolve(process.env.__GWD_COMMAND__ === 'build' // eslint-disable-line no-underscore-dangle + // TODO should this also happen during development too? + return process.env.__GWD_COMMAND__ === 'build' // eslint-disable-line no-underscore-dangle && this.compilation.config.staticRouter - && url !== '404.html' - && (path.extname(url) === '.html' || (contentType && contentType.indexOf('text/html') >= 0))); + && pathname.startsWith('/404') + && pathname.split('.').pop() === 'html' || (contentType && contentType.indexOf('text/html') >= 0); } - async intercept(url, body) { - return new Promise(async (resolve, reject) => { - try { - body = body.replace('', ` - \n - - `); + async intercept(url, request, response) { + const body = response.body.replace('', ` + \n + + `); - resolve({ body }); - } catch (e) { - reject(e); - } + // TODO avoid having to rebuild response each time? + return new Response(body, { + headers: response.headers }); } diff --git a/packages/cli/src/plugins/server/plugin-livereload.js b/packages/cli/src/plugins/server/plugin-livereload.js index bb90ba480..751cda66e 100644 --- a/packages/cli/src/plugins/server/plugin-livereload.js +++ b/packages/cli/src/plugins/server/plugin-livereload.js @@ -51,24 +51,23 @@ class LiveReloadServer extends ServerInterface { class LiveReloadResource extends ResourceInterface { - async shouldIntercept(url, body, headers) { - const { accept } = headers.request; + async shouldIntercept(url, request, response) { + const contentType = response.headers.get('content-type'); - return Promise.resolve(accept && accept.indexOf('text/html') >= 0 && process.env.__GWD_COMMAND__ === 'develop'); // eslint-disable-line no-underscore-dangle + return contentType.indexOf('text/html') >= 0 && process.env.__GWD_COMMAND__ === 'develop'; // eslint-disable-line no-underscore-dangle } - async intercept(url, body) { - return new Promise((resolve, reject) => { - try { - const contents = body.replace('', ` - - - `); + async intercept(url, request, response) { + let body = await response.text(); + + body = body.replace('', ` + + + `); - resolve({ body: contents }); - } catch (e) { - reject(e); - } + // TODO avoid having to rebuild response each time? + return new Response(body, { + headers: response.headers }); } } From 482506bdba80993e40dd80a96e757008c30fc89e Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Mon, 26 Dec 2022 17:50:27 -0500 Subject: [PATCH 06/50] clean up --- packages/cli/src/lifecycles/serve.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cli/src/lifecycles/serve.js b/packages/cli/src/lifecycles/serve.js index c113c6519..a3657f3c4 100644 --- a/packages/cli/src/lifecycles/serve.js +++ b/packages/cli/src/lifecycles/serve.js @@ -102,7 +102,6 @@ async function getDevServer(compilation) { ctx.set('Content-Type', response.headers.get('content-type')); ctx.body = await response.text(); - ctx.body = t; } catch (e) { console.error(e); } From bec54a4d711972b18f0ff86fb4bb4d80c6791ef7 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Mon, 26 Dec 2022 21:17:44 -0500 Subject: [PATCH 07/50] restore app templating to standard HTML resource plugin --- .../plugins/resource/plugin-standard-html.js | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/packages/cli/src/plugins/resource/plugin-standard-html.js b/packages/cli/src/plugins/resource/plugin-standard-html.js index 67b542158..eca1772ae 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-html.js +++ b/packages/cli/src/plugins/resource/plugin-standard-html.js @@ -16,7 +16,6 @@ import remarkParse from 'remark-parse'; import remarkRehype from 'remark-rehype'; import { ResourceInterface } from '../../lib/resource-interface.js'; import unified from 'unified'; -import { fileURLToPath } from 'url'; import { Worker } from 'worker_threads'; function getCustomPageTemplates(contextPlugins, templateName) { @@ -24,49 +23,51 @@ function getCustomPageTemplates(contextPlugins, templateName) { .map(plugin => plugin.templates) .flat() .filter((templateDir) => { - return templateName && fs.existsSync(path.join(templateDir, `${templateName}.html`)); + return templateName && fs.existsSync(new URL(`./${templateName}.html`, templateDir).pathname); }); } -const getPageTemplate = (fullPath, templatesDir, template, contextPlugins = [], pagesDir) => { +// TODO use URL more here +const getPageTemplate = async (filePath, templatesDir, template, contextPlugins = [], pagesDir) => { const customPluginDefaultPageTemplates = getCustomPageTemplates(contextPlugins, 'page'); const customPluginPageTemplates = getCustomPageTemplates(contextPlugins, template); - const is404Page = path.basename(fullPath).indexOf('404') === 0 && path.extname(fullPath) === '.html'; + const extension = filePath.split('.').pop(); + const is404Page = path.basename(filePath).indexOf('404') === 0 && extension === 'html'; let contents; - if (template && customPluginPageTemplates.length > 0 || fs.existsSync(`${templatesDir}/${template}.html`)) { + if (template && customPluginPageTemplates.length > 0 || fs.existsSync(new URL(`./${template}.html`, templatesDir).pathname)) { // use a custom template, usually from markdown frontmatter contents = customPluginPageTemplates.length > 0 - ? fs.readFileSync(`${customPluginPageTemplates[0]}/${template}.html`, 'utf-8') - : fs.readFileSync(`${templatesDir}/${template}.html`, 'utf-8'); - } else if (path.extname(fullPath) === '.html' && fs.existsSync(fullPath)) { + ? await fs.promises.readFile(`${customPluginPageTemplates[0]}/${template}.html`, 'utf-8') + : await fs.promises.readFile(new URL(`./${template}.html`, templatesDir), 'utf-8'); + } else if (extension === 'html' && fs.existsSync(filePath)) { // if the page is already HTML, use that as the template, NOT accounting for 404 pages - contents = fs.readFileSync(fullPath, 'utf-8'); + contents = await fs.promises.readFile(filePath, 'utf-8'); } else if (customPluginDefaultPageTemplates.length > 0 || (!is404Page && fs.existsSync(`${templatesDir}/page.html`))) { // else look for default page template from the user // and 404 pages should be their own "top level" template contents = customPluginDefaultPageTemplates.length > 0 - ? fs.readFileSync(`${customPluginDefaultPageTemplates[0]}/page.html`, 'utf-8') - : fs.readFileSync(`${templatesDir}/page.html`, 'utf-8'); - } else if (is404Page && !fs.existsSync(path.join(pagesDir, '404.html'))) { - contents = fs.readFileSync(fileURLToPath(new URL('../../templates/404.html', import.meta.url)), 'utf-8'); + ? await fs.promises.readFile(`${customPluginDefaultPageTemplates[0]}/page.html`, 'utf-8') + : await fs.promises.readFile(new URL('./page.html', templatesDir).pathname); + } else if (is404Page && !fs.existsSync(new URL('./404.html', pagesDir).pathname)) { + contents = await fs.promises.readFile(new URL('../../templates/404.html', import.meta.url).pathname, 'utf-8'); } else { // fallback to using Greenwood's stock page template - contents = fs.readFileSync(fileURLToPath(new URL('../../templates/page.html', import.meta.url)), 'utf-8'); + contents = await fs.promises.readFile(new URL('../../templates/page.html', import.meta.url).pathname, 'utf-8'); } return contents; }; -const getAppTemplate = (pageTemplateContents, templatesDir, customImports = [], contextPlugins, enableHud, frontmatterTitle) => { - const userAppTemplatePath = `${templatesDir}app.html`; +const getAppTemplate = async (pageTemplateContents, templatesDir, customImports = [], contextPlugins, enableHud, frontmatterTitle) => { + const userAppTemplatePath = new URL('./app.html', templatesDir); const customAppTemplates = getCustomPageTemplates(contextPlugins, 'app'); let mergedTemplateContents = ''; let appTemplateContents = customAppTemplates.length > 0 - ? fs.readFileSync(`${customAppTemplates[0]}/app.html`, 'utf-8') - : fs.existsSync(userAppTemplatePath) - ? fs.readFileSync(userAppTemplatePath, 'utf-8') - : fs.readFileSync(fileURLToPath(new URL('../../templates/app.html', import.meta.url)), 'utf-8'); + ? await fs.promises.readFile(`${customAppTemplates[0]}/app.html`) + : fs.existsSync(userAppTemplatePath.pathname) + ? await fs.promises.readFile(userAppTemplatePath, 'utf-8') + : await fs.promises.readFile(new URL('../../templates/app.html', import.meta.url), 'utf-8'); const pageRoot = htmlparser.parse(pageTemplateContents, { script: true, @@ -169,14 +170,14 @@ const getAppTemplate = (pageTemplateContents, templatesDir, customImports = [], return mergedTemplateContents; }; -const getUserScripts = (contents, context) => { +const getUserScripts = async (contents, context) => { // https://lit.dev/docs/tools/requirements/#polyfills if (process.env.__GWD_COMMAND__ === 'build') { // eslint-disable-line no-underscore-dangle const { projectDirectory, userWorkspace } = context; - const dependencies = fs.existsSync(path.join(userWorkspace, 'package.json')) // handle monorepos first - ? JSON.parse(fs.readFileSync(path.join(userWorkspace, 'package.json'), 'utf-8')).dependencies - : fs.existsSync(path.join(projectDirectory, 'package.json')) - ? JSON.parse(fs.readFileSync(path.join(projectDirectory, 'package.json'), 'utf-8')).dependencies + const dependencies = fs.existsSync(new URL('./package.json', userWorkspace).pathname) // handle monorepos first + ? JSON.parse(await fs.promises.readFile(new URL('./package.json', userWorkspace), 'utf-8')).dependencies + : fs.existsSync(new URL('./package.json', projectDirectory).pathname) + ? JSON.parse(await fs.promises.readFile(new URL('./package.json', projectDirectory), 'utf-8')).dependencies : {}; const litPolyfill = dependencies && dependencies.lit @@ -224,8 +225,8 @@ class StandardHtmlResource extends ResourceInterface { const matchingRoute = isClientSideRoute ? this.compilation.graph[0] : this.compilation.graph.filter((node) => node.route === pathname)[0]; - const fullPath = !matchingRoute.external ? matchingRoute.path : ''; - const isMarkdownContent = pathname.split('.').pop() === 'md'; + const filePath = !matchingRoute.external ? matchingRoute.path : ''; + const isMarkdownContent = matchingRoute.filename.split('.').pop() === 'md'; let customImports = []; let body = ''; @@ -242,7 +243,7 @@ class StandardHtmlResource extends ResourceInterface { } if (isMarkdownContent) { - const markdownContents = await fs.promises.readFile(fullPath, 'utf-8'); + const markdownContents = await fs.promises.readFile(filePath, 'utf-8'); const rehypePlugins = []; const remarkPlugins = []; @@ -342,13 +343,13 @@ class StandardHtmlResource extends ResourceInterface { }); if (isClientSideRoute) { - body = fs.readFileSync(fullPath, 'utf-8'); + body = fs.readFileSync(filePath, 'utf-8'); } else { - body = ssrTemplate ? ssrTemplate : getPageTemplate(fullPath, userTemplatesDir, template, contextPlugins, pagesDir); + body = ssrTemplate ? ssrTemplate : await getPageTemplate(filePath, userTemplatesDir, template, contextPlugins, pagesDir); } - body = getAppTemplate(body, userTemplatesDir, customImports, contextPlugins, config.devServer.hud, title); - body = getUserScripts(body, this.compilation.context); + body = await getAppTemplate(body, userTemplatesDir, customImports, contextPlugins, config.devServer.hud, title); + body = await getUserScripts(body, this.compilation.context); if (processedMarkdown) { const wrappedCustomElementRegex = /

<[a-zA-Z]*-[a-zA-Z](.*)>(.*)<\/[a-zA-Z]*-[a-zA-Z](.*)><\/p>/g; From 2eca31c951e8f389b5d7e933ba283653e6d9d534 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 27 Dec 2022 09:14:19 -0500 Subject: [PATCH 08/50] WIP refactoring plugins for web standardization --- .../plugins/resource/plugin-standard-html.js | 5 +- .../plugins/resource/plugin-static-router.js | 2 +- packages/plugin-graphql/src/core/cache.js | 10 +-- packages/plugin-graphql/src/index.js | 44 ++++++------ packages/plugin-import-css/src/index.js | 51 +++++--------- packages/plugin-import-json/src/index.js | 61 +++++++++------- packages/plugin-include-html/src/index.js | 69 +++++++++---------- packages/plugin-postcss/src/index.js | 46 +++++-------- 8 files changed, 133 insertions(+), 155 deletions(-) diff --git a/packages/cli/src/plugins/resource/plugin-standard-html.js b/packages/cli/src/plugins/resource/plugin-standard-html.js index eca1772ae..772f7f478 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-html.js +++ b/packages/cli/src/plugins/resource/plugin-standard-html.js @@ -32,6 +32,7 @@ const getPageTemplate = async (filePath, templatesDir, template, contextPlugins const customPluginDefaultPageTemplates = getCustomPageTemplates(contextPlugins, 'page'); const customPluginPageTemplates = getCustomPageTemplates(contextPlugins, template); const extension = filePath.split('.').pop(); + console.debug({ filePath, extension, template }); const is404Page = path.basename(filePath).indexOf('404') === 0 && extension === 'html'; let contents; @@ -43,12 +44,12 @@ const getPageTemplate = async (filePath, templatesDir, template, contextPlugins } else if (extension === 'html' && fs.existsSync(filePath)) { // if the page is already HTML, use that as the template, NOT accounting for 404 pages contents = await fs.promises.readFile(filePath, 'utf-8'); - } else if (customPluginDefaultPageTemplates.length > 0 || (!is404Page && fs.existsSync(`${templatesDir}/page.html`))) { + } else if (customPluginDefaultPageTemplates.length > 0 || (!is404Page && fs.existsSync(new URL('./page.html', templatesDir).pathname))) { // else look for default page template from the user // and 404 pages should be their own "top level" template contents = customPluginDefaultPageTemplates.length > 0 ? await fs.promises.readFile(`${customPluginDefaultPageTemplates[0]}/page.html`, 'utf-8') - : await fs.promises.readFile(new URL('./page.html', templatesDir).pathname); + : await fs.promises.readFile(new URL('./page.html', templatesDir), 'utf-8'); } else if (is404Page && !fs.existsSync(new URL('./404.html', pagesDir).pathname)) { contents = await fs.promises.readFile(new URL('../../templates/404.html', import.meta.url).pathname, 'utf-8'); } else { diff --git a/packages/cli/src/plugins/resource/plugin-static-router.js b/packages/cli/src/plugins/resource/plugin-static-router.js index 2ed02f648..b3fe7c930 100644 --- a/packages/cli/src/plugins/resource/plugin-static-router.js +++ b/packages/cli/src/plugins/resource/plugin-static-router.js @@ -34,7 +34,7 @@ class StaticRouterResource extends ResourceInterface { // TODO should this also happen during development too? return process.env.__GWD_COMMAND__ === 'build' // eslint-disable-line no-underscore-dangle && this.compilation.config.staticRouter - && pathname.startsWith('/404') + && !pathname.startsWith('/404') && pathname.split('.').pop() === 'html' || (contentType && contentType.indexOf('text/html') >= 0); } diff --git a/packages/plugin-graphql/src/core/cache.js b/packages/plugin-graphql/src/core/cache.js index b3f070149..eda38878b 100644 --- a/packages/plugin-graphql/src/core/cache.js +++ b/packages/plugin-graphql/src/core/cache.js @@ -28,14 +28,14 @@ const createCache = async (req, context) => { const cache = JSON.stringify(data); const queryHash = getQueryHash(query, variables); const hashFilename = `${queryHash}-cache.json`; - const cachePath = `${outputDir}/${hashFilename}`; + const cachePath = new URL(`./${hashFilename}`, outputDir); - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir); + if (!fs.existsSync(outputDir.pathname)) { + fs.mkdirSync(outputDir.pathname); } - if (!fs.existsSync(cachePath)) { - fs.writeFileSync(cachePath, cache, 'utf8'); + if (!fs.existsSync(cachePath.pathname)) { + fs.writeFileSync(cachePath.pathname, cache, 'utf8'); } } diff --git a/packages/plugin-graphql/src/index.js b/packages/plugin-graphql/src/index.js index bcf3c0f5a..3933b278b 100644 --- a/packages/plugin-graphql/src/index.js +++ b/packages/plugin-graphql/src/index.js @@ -19,41 +19,39 @@ const importMap = { class GraphQLResource extends ResourceInterface { constructor(compilation, options = {}) { super(compilation, options); - this.extensions = ['.gql']; + this.extensions = ['gql']; this.contentType = ['text/javascript', 'text/html']; } + async shouldServe(url) { + return url.protocol === 'file:' && this.extensions.indexOf(url.pathname.split('.').pop()) >= 0; + } + async serve(url) { - return new Promise(async (resolve, reject) => { - try { - const js = await fs.promises.readFile(url, 'utf-8'); - const body = ` - export default \`${js}\`; - `; + const js = await fs.promises.readFile(url, 'utf-8'); + const body = ` + export default \`${js}\`; + `; - resolve({ - body, - contentType: this.contentType[0] - }); - } catch (e) { - reject(e); + // TODO avoid having to rebuild response each time? + return new Response(body, { + headers: { + 'Content-Type': this.contentType[0] } }); } - async shouldIntercept(url, body, headers) { - return Promise.resolve(headers.request.accept && headers.request.accept.indexOf(this.contentType[1]) >= 0); + async shouldIntercept(url, request, response) { + return response.headers.get('content-type').indexOf('text/html') >= 0; } - async intercept(url, body) { - return new Promise(async (resolve, reject) => { - try { - const newBody = mergeImportMap(body, importMap); + async intercept(url, request, response) { + const body = await response.text(); + const newBody = mergeImportMap(body, importMap); - resolve({ body: newBody }); - } catch (e) { - reject(e); - } + // TODO avoid having to rebuild response each time? + return new Response(newBody, { + headers: response.headers }); } diff --git a/packages/plugin-import-css/src/index.js b/packages/plugin-import-css/src/index.js index 8c186cb58..d4811ab61 100644 --- a/packages/plugin-import-css/src/index.js +++ b/packages/plugin-import-css/src/index.js @@ -4,59 +4,46 @@ * */ import fs from 'fs'; -import path from 'path'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; -import { pathToFileURL } from 'url'; class ImportCssResource extends ResourceInterface { constructor(compilation, options) { super(compilation, options); - this.extensions = ['.css']; + this.extensions = ['css']; this.contentType = 'text/javascript'; } - // TODO resolve as part of https://github.com/ProjectEvergreen/greenwood/issues/952 - async shouldServe() { - return false; - } - // https://github.com/ProjectEvergreen/greenwood/issues/700 async shouldResolve(url) { - const isCssInDisguise = url.endsWith(this.extensions[0]) && fs.existsSync(`${url}.js`); - - return Promise.resolve(isCssInDisguise); + return url.pathname.endsWith(`.${this.extensions[0]}`) && fs.existsSync(`${url.pathname}.js`); } async resolve(url) { - return Promise.resolve(`${url}.js`); + return new Request(`file://${url.pathname}.js`); } - async shouldIntercept(url, body, headers = { request: {} }) { - const { originalUrl = '' } = headers.request; - const accept = headers.request.accept || ''; - const isCssFile = path.extname(url) === this.extensions[0]; - const notFromBrowser = accept.indexOf('text/css') < 0 && accept.indexOf('application/signed-exchange') < 0; + async shouldIntercept(url, request) { + const { pathname } = url; + const accepts = request.headers.get('accept') || ''; + const isCssFile = pathname.split('.').pop() === this.extensions[0]; + const notFromBrowser = accepts.indexOf('text/css') < 0 && accepts.indexOf('application/signed-exchange') < 0; // https://github.com/ProjectEvergreen/greenwood/issues/492 - const isCssInJs = originalUrl.indexOf('?type=css') >= 0 + const isCssInJs = url.searchParams.has('type') && url.searchParams.get('type') === this.extensions[0] || isCssFile && notFromBrowser - || isCssFile && notFromBrowser && url.indexOf('/node_modules/') >= 0; + || isCssFile && notFromBrowser && pathname.startsWith('/node_modules/'); - return Promise.resolve(isCssInJs); + return isCssInJs; } - async intercept(url, body) { - return new Promise(async (resolve, reject) => { - try { - const finalBody = body || await fs.promises.readFile(pathToFileURL(url), 'utf-8'); - const cssInJsBody = `const css = \`${finalBody.replace(/\r?\n|\r/g, ' ').replace(/\\/g, '\\\\')}\`;\nexport default css;`; - - resolve({ - body: cssInJsBody, - contentType: this.contentType - }); - } catch (e) { - reject(e); + async intercept(url, request, response) { + // TODO why do we need to check for body first? + const body = await response.text(); // => body || await fs.promises.readFile(pathToFileURL(url), 'utf-8'); + const cssInJsBody = `const css = \`${body.replace(/\r?\n|\r/g, ' ').replace(/\\/g, '\\\\')}\`;\nexport default css;`; + + return new Response(cssInJsBody, { + headers: { + 'content-type': this.contentType } }); } diff --git a/packages/plugin-import-json/src/index.js b/packages/plugin-import-json/src/index.js index 99f0d0642..b72773b99 100644 --- a/packages/plugin-import-json/src/index.js +++ b/packages/plugin-import-json/src/index.js @@ -10,40 +10,51 @@ import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js' class ImportJsonResource extends ResourceInterface { constructor(compilation, options) { super(compilation, options); - this.extensions = ['.json']; + this.extensions = ['json']; this.contentType = 'text/javascript'; } - // TODO resolve as part of https://github.com/ProjectEvergreen/greenwood/issues/952 - async shouldServe() { - return false; - } - // TODO handle it from node_modules too, when without `?type=json` - async shouldIntercept(url, body, headers) { - const { originalUrl } = headers.request; - const type = this.extensions[0].replace('.', ''); + async shouldIntercept(url) { + // const { originalUrl } = headers.request; + // const type = this.extensions[0].replace('.', ''); - return Promise.resolve(originalUrl && originalUrl.indexOf(`?type=${type}`) >= 0); + // return Promise.resolve(originalUrl && originalUrl.indexOf(`?type=${type}`) >= 0); + return url.searchParams.has('type') && url.searchParams.get('type') === this.extensions[0]; } - async intercept(url, body = '') { - return new Promise(async (resolve, reject) => { - try { - // TODO better way to handle this? - // https://github.com/ProjectEvergreen/greenwood/issues/948 - const raw = body === '' - ? await fs.promises.readFile(url, 'utf-8') - : body; - - resolve({ - body: `export default ${JSON.stringify(raw)}`, - contentType: this.contentType - }); - } catch (e) { - reject(e); + async intercept(url, request, response) { + // TODO better way to handle this? + // https://github.com/ProjectEvergreen/greenwood/issues/948 + const body = await response.text() === '' + ? await fs.promises.readFile(url, 'utf-8') + : await response.text(); + + return new Response(`export default ${JSON.stringify(body)}`, { + headers: { + 'content-type': this.contentType } }); + // resolve({ + // body: `export default ${JSON.stringify(raw)}`, + // contentType: this.contentType + // }); + // return new Promise(async (resolve, reject) => { + // try { + // // TODO better way to handle this? + // // https://github.com/ProjectEvergreen/greenwood/issues/948 + // const raw = body === '' + // ? await fs.promises.readFile(url, 'utf-8') + // : body; + + // resolve({ + // body: `export default ${JSON.stringify(raw)}`, + // contentType: this.contentType + // }); + // } catch (e) { + // reject(e); + // } + // }); } } diff --git a/packages/plugin-include-html/src/index.js b/packages/plugin-include-html/src/index.js index 031b9bec8..d9c508cf3 100644 --- a/packages/plugin-include-html/src/index.js +++ b/packages/plugin-include-html/src/index.js @@ -1,7 +1,5 @@ import fs from 'fs'; -import path from 'path'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; -import { pathToFileURL } from 'url'; class IncludeHtmlResource extends ResourceInterface { constructor(compilation, options) { @@ -10,44 +8,41 @@ class IncludeHtmlResource extends ResourceInterface { this.contentType = 'text/html'; } - async shouldIntercept(url, body, headers) { - return Promise.resolve(headers.response['content-type'] === this.contentType); + async shouldIntercept(url, request, response) { + return response.headers.get('content-type').indexOf(this.contentType) >= 0; } - async intercept(url, body) { - return new Promise(async (resolve, reject) => { - try { - const includeLinksRegexMatches = body.match(//g); - const includeCustomElementssRegexMatches = body.match(/<[a-zA-Z]*-[a-zA-Z](.*)>(.*)<\/[a-zA-Z]*-[a-zA-Z](.*)>/g); - - if (includeLinksRegexMatches) { - includeLinksRegexMatches - .filter(link => link.indexOf('rel="html"') > 0) - .forEach((link) => { - const href = link.match(/href="(.*)"/)[1]; - const includeContents = fs.readFileSync(path.join(this.compilation.context.userWorkspace, href), 'utf-8'); - - body = body.replace(link, includeContents); - }); - } - - if (includeCustomElementssRegexMatches) { - const customElementTags = includeCustomElementssRegexMatches.filter(customElementTag => customElementTag.indexOf('src=') > 0); - - for (const tag of customElementTags) { - const src = tag.match(/src="(.*)"/)[1]; - const filepath = path.join(this.compilation.context.userWorkspace, this.getBareUrlPath(src.replace(/\.\.\//g, ''))); - const { getData, getTemplate } = await import(pathToFileURL(filepath)); - const includeContents = await getTemplate(await getData()); - - body = body.replace(tag, includeContents); - } - } - - resolve({ body }); - } catch (e) { - reject(e); + async intercept(url, request, response) { + let body = await response.text(); + const includeLinksRegexMatches = body.match(//g); + const includeCustomElementsRegexMatches = body.match(/<[a-zA-Z]*-[a-zA-Z](.*)>(.*)<\/[a-zA-Z]*-[a-zA-Z](.*)>/g); + + if (includeLinksRegexMatches) { + const htmlIncludeLinks = includeLinksRegexMatches.filter(link => link.indexOf('rel="html"') > 0); + + for (const link of htmlIncludeLinks) { + const href = link.match(/href="(.*)"/)[1]; + const includeContents = await fs.readFile(new URL(href, this.compilation.context.userWorkspace), 'utf-8'); + + body = body.replace(link, includeContents); + } + } + + if (includeCustomElementsRegexMatches) { + const customElementTags = includeCustomElementsRegexMatches.filter(customElementTag => customElementTag.indexOf('src=') > 0); + + for (const tag of customElementTags) { + const src = tag.match(/src="(.*)"/)[1]; + const srcUrl = new URL(`./${src.replace(/\.\.\//g, '')}`, this.compilation.context.userWorkspace); + const { getData, getTemplate } = await import(srcUrl); + const includeContents = await getTemplate(await getData()); + + body = body.replace(tag, includeContents); } + } + + return new Response(body, { + headers: response.headers }); } } diff --git a/packages/plugin-postcss/src/index.js b/packages/plugin-postcss/src/index.js index 41bffa49e..921e317dd 100644 --- a/packages/plugin-postcss/src/index.js +++ b/packages/plugin-postcss/src/index.js @@ -4,17 +4,15 @@ * */ import fs from 'fs'; -import path from 'path'; import postcss from 'postcss'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; -import { pathToFileURL } from 'url'; async function getConfig (compilation, extendConfig = false) { const { projectDirectory } = compilation.context; const configFile = 'postcss.config'; - const defaultConfig = (await import(new URL(`${configFile}.js`, import.meta.url).pathname)).default; - const userConfig = fs.existsSync(path.join(projectDirectory, `${configFile}.mjs`)) - ? (await import(pathToFileURL(path.join(projectDirectory, `${configFile}.mjs`)))).default + const defaultConfig = (await import(new URL(`${configFile}.js`, import.meta.url))).default; + const userConfig = fs.existsSync(new URL(`./${configFile}.mjs`, projectDirectory).pathname) + ? (await import(new URL(`./${configFile}.mjs`, projectDirectory))).default : {}; let finalConfig = Object.assign({}, userConfig); @@ -30,37 +28,25 @@ async function getConfig (compilation, extendConfig = false) { class PostCssResource extends ResourceInterface { constructor(compilation, options) { super(compilation, options); - this.extensions = ['.css']; + this.extensions = ['css']; this.contentType = ['text/css']; } - async shouldServe() { - return false; - } - - isCssFile(url) { - return path.extname(url) === '.css'; - } - async shouldIntercept(url) { - return Promise.resolve(this.isCssFile(url)); + return url.protocol === 'file:' && url.pathname.split('.').pop() === this.extensions[0]; } - async intercept(url, body) { - return new Promise(async(resolve, reject) => { - try { - const config = await getConfig(this.compilation, this.options.extendConfig); - const plugins = config.plugins || []; - const css = plugins.length > 0 - ? (await postcss(plugins).process(body, { from: url })).css - : body; - - resolve({ - body: css - }); - } catch (e) { - reject(e); - } + async intercept(url, request, response) { + const config = await getConfig(this.compilation, this.options.extendConfig); + const plugins = config.plugins || []; + const body = await response.text(); + const css = plugins.length > 0 + ? (await postcss(plugins).process(body, { from: url.pathname })).css + : body; + + // TODO avoid having to rebuild response each time? + return new Response(css, { + headers: response.headers }); } } From 769b8a48df7ad989e0a570fc86f3a9eee54ffdcf Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Sat, 31 Dec 2022 13:13:11 -0500 Subject: [PATCH 09/50] handle develop command middleware support for handling binary or text response types --- packages/cli/src/lifecycles/serve.js | 18 +++++++++----- .../plugins/resource/plugin-standard-html.js | 1 - .../plugins/resource/plugin-standard-image.js | 24 ++++--------------- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/packages/cli/src/lifecycles/serve.js b/packages/cli/src/lifecycles/serve.js index a3657f3c4..a7f5fbb70 100644 --- a/packages/cli/src/lifecycles/serve.js +++ b/packages/cli/src/lifecycles/serve.js @@ -2,6 +2,7 @@ import fs from 'fs'; // import { hashString } from '../lib/hashing-utils.js'; import path from 'path'; import Koa from 'koa'; +import { Readable } from 'stream'; import { ResourceInterface } from '../lib/resource-interface.js'; async function getDevServer(compilation) { @@ -46,6 +47,7 @@ async function getDevServer(compilation) { ctx.url = request.url; } catch (e) { + ctx.status = 500; console.error(e); } @@ -56,13 +58,13 @@ async function getDevServer(compilation) { app.use(async (ctx, next) => { try { const url = new URL(ctx.url); - const request = new Request(url, { + const request = new Request(url.href, { method: ctx.request.method, headers: ctx.request.header }); - let response = new Response(ctx.message, { + let response = new Response(null, { status: ctx.response.status, - headers: ctx.response.header + headers: new Headers() }); for (const plugin of resourcePlugins) { @@ -72,9 +74,12 @@ async function getDevServer(compilation) { } // TODO would be nice if Koa (or other framework) could just a Response object directly - ctx.set('Content-Type', response.headers.get('content-type')); - ctx.body = await response.text(); + // not sure why we have to use `Readable.from`, does this couple us to NodeJS? + ctx.body = response.body ? Readable.from(response.body) : ''; + ctx.type = response.headers.get('content-type'); + ctx.status = response.status; } catch (e) { + ctx.status = 500; console.error(e); } @@ -100,9 +105,10 @@ async function getDevServer(compilation) { } } + ctx.body = response.body ? Readable.from(response.body) : ''; ctx.set('Content-Type', response.headers.get('content-type')); - ctx.body = await response.text(); } catch (e) { + ctx.status = 500; console.error(e); } diff --git a/packages/cli/src/plugins/resource/plugin-standard-html.js b/packages/cli/src/plugins/resource/plugin-standard-html.js index 772f7f478..72c0f41d7 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-html.js +++ b/packages/cli/src/plugins/resource/plugin-standard-html.js @@ -32,7 +32,6 @@ const getPageTemplate = async (filePath, templatesDir, template, contextPlugins const customPluginDefaultPageTemplates = getCustomPageTemplates(contextPlugins, 'page'); const customPluginPageTemplates = getCustomPageTemplates(contextPlugins, template); const extension = filePath.split('.').pop(); - console.debug({ filePath, extension, template }); const is404Page = path.basename(filePath).indexOf('404') === 0 && extension === 'html'; let contents; diff --git a/packages/cli/src/plugins/resource/plugin-standard-image.js b/packages/cli/src/plugins/resource/plugin-standard-image.js index 941172ad9..b031df9f1 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-image.js +++ b/packages/cli/src/plugins/resource/plugin-standard-image.js @@ -22,29 +22,15 @@ class StandardFontResource extends ResourceInterface { async serve(url) { const extension = url.pathname.split('.').pop(); const type = extension === 'svg' ? `${extension}+xml` : extension; - const exts = [...this.extensions]; - const isIco = extension === 'ico'; - let body = ''; - let contentType = ''; - - if (exts.includes(extension)) { - contentType = `image/${type}`; - - if (extension === 'svg') { - body = await fs.promises.readFile(url, 'utf-8'); - } else { - // TODO this doesn't seem to work - body = await fs.promises.readFile(url); - } - } else if (isIco) { - contentType = 'image/x-icon'; - body = await fs.promises.readFile(url); - } + const body = await fs.promises.readFile(url); + const contentType = extension === 'ico' + ? 'x-icon' + : type; // TODO avoid having to rebuild response each time? return new Response(body, { headers: { - 'content-type': contentType + 'content-type': `image/${contentType}` } }); } From ca4ec6372309b54728355d6ef0d4eeebfd1b7352 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Sat, 31 Dec 2022 17:08:12 -0500 Subject: [PATCH 10/50] restore nested relative routes resolution and general standard plugin refactoring --- packages/cli/src/lib/resource-interface.js | 63 +++++++++---------- packages/cli/src/lifecycles/serve.js | 3 + .../plugins/resource/plugin-node-modules.js | 24 +++---- .../plugins/resource/plugin-standard-html.js | 27 +++----- .../plugins/resource/plugin-user-workspace.js | 11 +--- 5 files changed, 50 insertions(+), 78 deletions(-) diff --git a/packages/cli/src/lib/resource-interface.js b/packages/cli/src/lib/resource-interface.js index cb2e2e7b3..8aa600b7a 100644 --- a/packages/cli/src/lib/resource-interface.js +++ b/packages/cli/src/lib/resource-interface.js @@ -1,4 +1,4 @@ -// import fs from 'fs'; +import fs from 'fs'; class ResourceInterface { constructor(compilation, options = {}) { @@ -8,41 +8,36 @@ class ResourceInterface { this.contentType = ''; // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types } - // get rid of things like query string parameters - // that will break when trying to use with fs - // TODO URLs will not contain query strings by default, right? - // getBareUrlPath(url) { - // console.debug('getBareUrlPath', { url }); - // return url.replace(/\?(.*)/, ''); - // } + hasExtension(url) { + const extension = url.pathname.split('.').pop(); + + return extension !== '' && !extension.startsWith('/'); + } // turn relative paths into relatively absolute based on a known root directory - // e.g. "../styles/theme.css" -> `${userWorkspace}/styles/theme.css` - // resolveRelativeUrl(root, pathname) { - // // console.debug('getBareUrlPath', { root, pathname }); - // if (fs.existsSync(new URL(pathname, root).pathname)) { - // return url; - // } - - // let reducedPathname; - - // pathname.split('/') - // .filter((segment) => segment !== '') - // .reduce((acc, segment) => { - // // console.debug({ acc, segment }); - // const reducedPath = pathname.replace(`${acc}/${segment}`, ''); - - // // console.debug({ reducedPath }); - // // console.debug(new URL(`.${reducedPath}`, root).pathname); - // if (reducedPath.split('.').pop() !== '' && fs.existsSync(new URL(`.${reducedPath}`, root).pathname)) { - // reducedPathname = reducedPath; - // } - // return `${acc}/${segment}`; - // }, ''); - - // // console.debug({ reducedPathname }); - // return reducedPathname; - // } + // * deep link route - /blog/releases/some-post + // * and a nested path in the template - ../../styles/theme.css + // so will get resolved as `${rootUrl}/styles/theme.css` + resolveForRelativeUrl(url, rootUrl) { + let reducedUrl; + + if (fs.existsSync(new URL(`.${url.pathname}`, rootUrl).pathname)) { + return new URL(`.${url.pathname}`, rootUrl); + } + + url.pathname.split('/') + .filter((segment) => segment !== '') + .reduce((acc, segment) => { + const reducedPath = url.pathname.replace(`${acc}/${segment}`, ''); + + if (reducedPath !== '' && fs.existsSync(new URL(`.${reducedPath}`, rootUrl).pathname)) { + reducedUrl = new URL(`.${reducedPath}`, rootUrl); + } + return `${acc}/${segment}`; + }, ''); + + return reducedUrl; + } // test if this plugin should change a relative URL from the browser to an absolute path on disk // like for node_modules/ resolution. not commonly needed by most resource plugins diff --git a/packages/cli/src/lifecycles/serve.js b/packages/cli/src/lifecycles/serve.js index a7f5fbb70..68fd45531 100644 --- a/packages/cli/src/lifecycles/serve.js +++ b/packages/cli/src/lifecycles/serve.js @@ -34,6 +34,7 @@ async function getDevServer(compilation) { app.use(async (ctx, next) => { try { const url = new URL(`http://localhost:${compilation.config.port}${ctx.url}`); + let request = new Request(url, { method: ctx.request.method, headers: ctx.request.header @@ -105,6 +106,8 @@ async function getDevServer(compilation) { } } + // TODO would be nice if Koa (or other framework) could just a Response object directly + // not sure why we have to use `Readable.from`, does this couple us to NodeJS? ctx.body = response.body ? Readable.from(response.body) : ''; ctx.set('Content-Type', response.headers.get('content-type')); } catch (e) { diff --git a/packages/cli/src/plugins/resource/plugin-node-modules.js b/packages/cli/src/plugins/resource/plugin-node-modules.js index 4f7dd7e26..0d16f9ae3 100644 --- a/packages/cli/src/plugins/resource/plugin-node-modules.js +++ b/packages/cli/src/plugins/resource/plugin-node-modules.js @@ -4,7 +4,6 @@ * */ import fs from 'fs'; -import path from 'path'; import { nodeResolve } from '@rollup/plugin-node-resolve'; import replace from '@rollup/plugin-replace'; import { getNodeModulesLocationForPackage, getPackageNameFromUrl } from '../../lib/node-modules-utils.js'; @@ -27,29 +26,20 @@ class NodeModulesResource extends ResourceInterface { // TODO convert node modules util to URL async resolve(url) { const { projectDirectory } = this.compilation.context; - const pathname = url.pathname; + const { pathname } = url; const packageName = getPackageNameFromUrl(pathname); const absoluteNodeModulesLocation = await getNodeModulesLocationForPackage(packageName); const packagePathPieces = pathname.split('node_modules/')[1].split('/'); // double split to handle node_modules within nested paths - let absoluteNodeModulesUrl; - - if (absoluteNodeModulesLocation) { - absoluteNodeModulesUrl = `${absoluteNodeModulesLocation}${packagePathPieces.join('/').replace(packageName, '')}`; - } else { - const isAbsoluteNodeModulesFile = fs.existsSync(path.join(projectDirectory, pathname)); - - absoluteNodeModulesUrl = isAbsoluteNodeModulesFile - ? new URL(`.${pathname}`, userWorkspace) - : this.resolveRelativeUrl(projectDirectory, barePath) - ? new URL(this.resolveRelativeUrl(projectDirectory, pathname), userWorkspace) - : pathname; - } + // use node modules resolution logic first, else hope for the best from the root of the project + const absoluteNodeModulesPathname = absoluteNodeModulesLocation + ? `${absoluteNodeModulesLocation}${packagePathPieces.join('/').replace(packageName, '')}` + : this.resolveForRelativeUrl(url, projectDirectory).pathname; - return new Request(`file://${absoluteNodeModulesUrl}`); + return new Request(`file://${absoluteNodeModulesPathname}`); } async shouldServe(url) { - return url.protocol === 'file:' && url.pathname.startsWith('/node_modules/'); + return this.hasExtension(url) && url.pathname.startsWith('/node_modules/'); } async serve(url) { diff --git a/packages/cli/src/plugins/resource/plugin-standard-html.js b/packages/cli/src/plugins/resource/plugin-standard-html.js index 72c0f41d7..7a020def9 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-html.js +++ b/packages/cli/src/plugins/resource/plugin-standard-html.js @@ -200,31 +200,20 @@ class StandardHtmlResource extends ResourceInterface { this.contentType = 'text/html'; } - getRelativeUserworkspaceUrl(url) { - return path.normalize(url.replace(this.compilation.context.userWorkspace, '')); - } - - async shouldServe(url, request) { - const pathname = url.pathname; - // const relativeUrl = this.getRelativeUserworkspaceUrl(url).replace(/\\/g, '/'); // and handle for windows - const isClientSideRoute = this.compilation.graph[0].isSPA && pathname.split('.').pop() === '' && (request.headers?.accept || '').indexOf(this.contentType) >= 0; - const hasMatchingRoute = this.compilation.graph.filter((node) => { - return node.route === pathname; - }).length === 1; + async shouldServe(url) { + const { protocol, pathname } = url; + const hasMatchingPageRoute = this.compilation.graph.find(node => node.route === pathname); - return hasMatchingRoute || isClientSideRoute; + return protocol.startsWith('http') && hasMatchingPageRoute; } async serve(url) { const { config } = this.compilation; const { pagesDir, userTemplatesDir } = this.compilation.context; const { interpolateFrontmatter } = config; - const pathname = url.pathname; - // const relativeUrl = this.getRelativeUserworkspaceUrl(url).replace(/\\/g, '/'); // and handle for windows; - const isClientSideRoute = this.compilation.graph[0].isSPA; - const matchingRoute = isClientSideRoute - ? this.compilation.graph[0] - : this.compilation.graph.filter((node) => node.route === pathname)[0]; + const { pathname } = url; + const isSpaRoute = this.compilation.graph[0].isSPA; + const matchingRoute = this.compilation.graph.find((node) => node.route === pathname); const filePath = !matchingRoute.external ? matchingRoute.path : ''; const isMarkdownContent = matchingRoute.filename.split('.').pop() === 'md'; @@ -342,7 +331,7 @@ class StandardHtmlResource extends ResourceInterface { return plugin.provider(this.compilation); }); - if (isClientSideRoute) { + if (isSpaRoute) { body = fs.readFileSync(filePath, 'utf-8'); } else { body = ssrTemplate ? ssrTemplate : await getPageTemplate(filePath, userTemplatesDir, template, contextPlugins, pagesDir); diff --git a/packages/cli/src/plugins/resource/plugin-user-workspace.js b/packages/cli/src/plugins/resource/plugin-user-workspace.js index 649cc7200..8b7fe21ff 100644 --- a/packages/cli/src/plugins/resource/plugin-user-workspace.js +++ b/packages/cli/src/plugins/resource/plugin-user-workspace.js @@ -4,7 +4,6 @@ * This sets the default value for requests in Greenwood. * */ -import fs from 'fs'; import { ResourceInterface } from '../../lib/resource-interface.js'; class UserWorkspaceResource extends ResourceInterface { @@ -15,19 +14,15 @@ class UserWorkspaceResource extends ResourceInterface { async shouldResolve(url) { const { userWorkspace } = this.compilation.context; - const pathname = url.pathname; - const isWorkspaceFile = pathname !== '/' - && pathname.split('.').pop() !== '' - && fs.existsSync(new URL(`.${pathname}`, userWorkspace).pathname); - return !pathname.startsWith('/node_modules/') && isWorkspaceFile; + return this.hasExtension(url) && this.resolveForRelativeUrl(url, userWorkspace); } async resolve(url) { const { userWorkspace } = this.compilation.context; - const workspaceUrl = new URL(`.${url.pathname}`, userWorkspace); + const workspaceUrl = this.resolveForRelativeUrl(url, userWorkspace); - return new Request(`file://${workspaceUrl.pathname}`); + return new Request(workspaceUrl); } } From 27a8fc488460f0011938d743b93b4a1c6952e553 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Mon, 2 Jan 2023 13:39:43 -0500 Subject: [PATCH 11/50] group greenwood plugins --- greenwood.config.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/greenwood.config.js b/greenwood.config.js index 782957d54..525eeee88 100644 --- a/greenwood.config.js +++ b/greenwood.config.js @@ -19,6 +19,8 @@ export default { greenwoodPluginPostCss(), greenwoodPluginImportJson(), greenwoodPluginImportCss(), + greenwoodPluginIncludeHTML(), + greenwoodPluginRendererPuppeteer(), { type: 'rollup', name: 'rollup-plugin-analyzer', @@ -32,9 +34,7 @@ export default { }) ]; } - }, - greenwoodPluginIncludeHTML(), - greenwoodPluginRendererPuppeteer() + } ], markdown: { plugins: [ From f8c571f60484878b1d3cced605f4dab9b26ca3b7 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Mon, 2 Jan 2023 13:40:15 -0500 Subject: [PATCH 12/50] adapt intercept lifecycles for web standards signature --- packages/plugin-polyfills/src/index.js | 88 ++++++++++--------- .../src/plugins/resource.js | 18 ++-- 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/packages/plugin-polyfills/src/index.js b/packages/plugin-polyfills/src/index.js index f45346093..75d645d56 100644 --- a/packages/plugin-polyfills/src/index.js +++ b/packages/plugin-polyfills/src/index.js @@ -14,55 +14,57 @@ class PolyfillsResource extends ResourceInterface { }; } - async shouldIntercept(url, body, headers = { request: {} }) { - return Promise.resolve(headers.request['content-type'] && headers.request['content-type'].indexOf('text/html') >= 0); + async shouldIntercept(url) { + const { protocol, pathname } = url; + const { wc, lit, dsd } = this.options; + const hasMatchingPageRoute = this.compilation.graph.find(node => node.route === pathname); + const isEnabled = wc || lit || dsd; + + return isEnabled && protocol.startsWith('http') && hasMatchingPageRoute; } - async intercept(url, body) { - return new Promise(async (resolve, reject) => { - try { - let newHtml = body; + async intercept(url, request, response) { + const { wc, lit, dsd } = this.options; + let body = await response.text(); - // standard WC polyfill - if (this.options.wc) { - newHtml = newHtml.replace('', ` - - - `); - } + // standard WC polyfill + if (wc) { + body = body.replace('', ` + + + `); + } - // append Lit polyfill next to make sure it comes before WC polyfill - if (this.options.lit) { - newHtml = newHtml.replace('', ` - - - `); - } + // append Lit polyfill next to make sure it comes before WC polyfill + if (lit) { + body = body.replace('', ` + + + `); + } - // lastly, Declarative Shadow DOM polyfill - if (this.options.dsd) { - newHtml = newHtml.replace('', ` - - - `); - } + // lastly, Declarative Shadow DOM polyfill + if (dsd) { + body = body.replace('', ` + + + `); + } - resolve({ body: newHtml }); - } catch (e) { - reject(e); - } + return new Response(body, { + headers: response.headers }); } } diff --git a/packages/plugin-renderer-puppeteer/src/plugins/resource.js b/packages/plugin-renderer-puppeteer/src/plugins/resource.js index d83b9020a..b224eab8b 100644 --- a/packages/plugin-renderer-puppeteer/src/plugins/resource.js +++ b/packages/plugin-renderer-puppeteer/src/plugins/resource.js @@ -1,4 +1,3 @@ -import path from 'path'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; class PuppeteerResource extends ResourceInterface { @@ -9,19 +8,24 @@ class PuppeteerResource extends ResourceInterface { this.contentType = 'text/html'; } - async shouldIntercept(url, body, headers = {}) { - const shouldIntercept = url.endsWith(path.sep) && headers.request && headers.request.accept.indexOf(this.contentType) >= 0; - - return process.env.__GWD_COMMAND__ === 'build' && shouldIntercept;// eslint-disable-line no-underscore-dangle + async shouldIntercept(url) { + const { protocol, pathname } = url; + const hasMatchingPageRoute = this.compilation.graph.find(node => node.route === pathname); + + return process.env.__GWD_COMMAND__ === 'build' && protocol.startsWith('http') && hasMatchingPageRoute; // eslint-disable-line no-underscore-dangle } - async intercept(url, body) { + async intercept(url, request, response) { + let body = await response.text(); + body = body.replace('', ` `); - return Promise.resolve({ body }); + return new Response(body, { + headers: response.headers + }); } } From 7020c1bffc6141b27739c8ef185b24be75a82176 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Mon, 2 Jan 2023 18:00:33 -0500 Subject: [PATCH 13/50] adapt pre-render build lifecycle and refactor page serving and intercepting --- packages/cli/src/lib/resource-utils.js | 11 +- packages/cli/src/lifecycles/config.js | 2 +- packages/cli/src/lifecycles/graph.js | 4 +- packages/cli/src/lifecycles/prerender.js | 134 ++++++++++-------- .../src/plugins/copy/plugin-copy-assets.js | 11 +- .../src/plugins/copy/plugin-copy-favicon.js | 11 +- .../plugins/copy/plugin-copy-graph-json.js | 8 +- .../src/plugins/copy/plugin-copy-robots.js | 11 +- .../plugins/resource/plugin-static-router.js | 2 +- packages/plugin-graphql/src/index.js | 2 +- packages/plugin-import-json/src/index.js | 24 ---- packages/plugin-polyfills/src/index.js | 17 ++- .../src/plugins/resource.js | 11 +- 13 files changed, 117 insertions(+), 131 deletions(-) diff --git a/packages/cli/src/lib/resource-utils.js b/packages/cli/src/lib/resource-utils.js index 506f3b0ba..409c5d0b7 100644 --- a/packages/cli/src/lib/resource-utils.js +++ b/packages/cli/src/lib/resource-utils.js @@ -1,8 +1,7 @@ import fs from 'fs'; import { hashString } from '../lib/hashing-utils.js'; -import path from 'path'; -import { pathToFileURL } from 'url'; +// TODO could make async? function modelResource(context, type, src = undefined, contents = undefined, optimizationAttr = undefined, rawAttributes = undefined) { const { projectDirectory, scratchDir, userWorkspace } = context; const extension = type === 'script' ? 'js' : 'css'; @@ -10,15 +9,15 @@ function modelResource(context, type, src = undefined, contents = undefined, opt let sourcePathURL; if (src) { - sourcePathURL = src.indexOf('/node_modules') === 0 - ? pathToFileURL(path.join(projectDirectory, src)) // TODO (good first issue) get "real" location of node modules - : pathToFileURL(path.join(userWorkspace, src.replace(/\.\.\//g, '').replace('./', ''))); + sourcePathURL = src.startsWith('/node_modules') + ? new URL(`./${src}`, projectDirectory) // pathToFileURL(path.join(projectDirectory, src)) // TODO (good first issue) get "real" location of node modules + : new URL(`./${src.replace(/\.\.\//g, '').replace('./', '')}`, userWorkspace); // pathToFileURL(path.join(userWorkspace, src.replace(/\.\.\//g, '').replace('./', ''))); contents = fs.readFileSync(sourcePathURL, 'utf-8'); } else { const scratchFileName = hashString(contents); - sourcePathURL = pathToFileURL(path.join(scratchDir, `${scratchFileName}.${extension}`)); + sourcePathURL = new URL(`./${scratchFileName}.${extension}`, scratchDir); fs.writeFileSync(sourcePathURL, contents); } diff --git a/packages/cli/src/lifecycles/config.js b/packages/cli/src/lifecycles/config.js index a931ceef8..1f71338f3 100644 --- a/packages/cli/src/lifecycles/config.js +++ b/packages/cli/src/lifecycles/config.js @@ -194,7 +194,7 @@ const readAndMergeConfig = async() => { } // SPA should _not_ prerender unless if user has specified prerender should be true - if (prerender === undefined && fs.existsSync(path.join(customConfig.workspace, 'index.html'))) { + if (prerender === undefined && fs.existsSync(new URL('./index.html', customConfig.workspace))) { customConfig.prerender = false; } diff --git a/packages/cli/src/lifecycles/graph.js b/packages/cli/src/lifecycles/graph.js index ca7cae380..2593d1468 100644 --- a/packages/cli/src/lifecycles/graph.js +++ b/packages/cli/src/lifecycles/graph.js @@ -6,6 +6,8 @@ import path from 'path'; import toc from 'markdown-toc'; import { Worker } from 'worker_threads'; +// TODO convert graph to use URLs +// https://github.com/ProjectEvergreen/greenwood/issues/952 const generateGraph = async (compilation) => { return new Promise(async (resolve, reject) => { @@ -35,7 +37,7 @@ const generateGraph = async (compilation) => { const extension = path.extname(filename); const isStatic = extension === '.md' || extension === '.html'; const isDynamic = extension === '.js'; - const relativePagePath = fullPath.substring(pagesDir.length - 1, fullPath.length); + const relativePagePath = fullPath.substring(pagesDir.pathname.length - 1, fullPath.length); const relativeWorkspacePath = directory.replace(process.cwd(), '').replace(path.sep, ''); let route = relativePagePath .replace(extension, '') diff --git a/packages/cli/src/lifecycles/prerender.js b/packages/cli/src/lifecycles/prerender.js index 65437e773..b5a6bc040 100644 --- a/packages/cli/src/lifecycles/prerender.js +++ b/packages/cli/src/lifecycles/prerender.js @@ -2,16 +2,15 @@ import fs from 'fs'; import htmlparser from 'node-html-parser'; import { modelResource } from '../lib/resource-utils.js'; import os from 'os'; -import path from 'path'; import { WorkerPool } from '../lib/threadpool.js'; function isLocalLink(url = '') { return url !== '' && (url.indexOf('http') !== 0 && url.indexOf('//') !== 0); } -function createOutputDirectory(route, outputPathDir) { - if (route !== '/404/' && !fs.existsSync(outputPathDir)) { - fs.mkdirSync(outputPathDir, { +function createOutputDirectory(route, { pathname }) { + if (route !== '/404/' && !fs.existsSync(pathname)) { + fs.mkdirSync(pathname, { recursive: true }); } @@ -72,66 +71,73 @@ function trackResourcesForRoute(html, compilation, route) { return resources; } -async function interceptPage(compilation, contents, route) { - const headers = { - request: { 'accept': 'text/html', 'content-type': 'text/html' }, - response: { 'content-type': 'text/html' } - }; - const interceptResources = compilation.config.plugins.filter((plugin) => { - return plugin.type === 'resource' && plugin.name !== 'plugin-node-modules:resource'; - }).map((plugin) => { - return plugin.provider(compilation); - }).filter((provider) => { - return provider.shouldIntercept && provider.intercept; - }); +async function servePage(url, request, plugins) { + let response = new Response(''); + + for (const plugin of plugins) { + if (plugin.shouldServe && await plugin.shouldServe(url, request)) { + response = await plugin.serve(url, request); + } + } + + return response; +} + +async function interceptPage(url, request, plugins, body) { + const headers = new Headers(); + headers.append('content-type', 'text/html'); - const htmlIntercepted = await interceptResources.reduce(async (htmlPromise, resource) => { - const html = (await htmlPromise).body; - const shouldIntercept = await resource.shouldIntercept(route, html, headers); + let response = new Response(body, { headers }); - return shouldIntercept - ? resource.intercept(route, html, headers) - : htmlPromise; - }, Promise.resolve({ body: contents })); + for (const plugin of plugins) { + if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request, response)) { + response = await plugin.intercept(url, request, response); + } + } - return htmlIntercepted; + return response; +} + +function getPluginInstances (compilation) { + return [...compilation.config.plugins] + .map((plugin) => { + return plugin.provider(compilation); + }); } async function preRenderCompilationWorker(compilation, workerPrerender) { const pages = compilation.graph.filter(page => !page.isSSR || (page.isSSR && page.data.static) || (page.isSSR && compilation.config.prerender)); const { scratchDir } = compilation.context; + const plugins = getPluginInstances(compilation); console.info('pages to generate', `\n ${pages.map(page => page.route).join('\n ')}`); const pool = new WorkerPool(os.cpus().length, workerPrerender.workerUrl); for (const page of pages) { - const { outputPath, route } = page; - const outputPathDir = path.join(scratchDir, route); - const htmlResource = compilation.config.plugins.filter((plugin) => { - return plugin.name === 'plugin-standard-html'; - }).map((plugin) => { - return plugin.provider(compilation); - })[0]; - let html; + const { route, outputPath } = page; + const outputDirUrl = new URL(`./${route}`, scratchDir); + const outputPathUrl = new URL(`./${outputPath}`, scratchDir); + const url = new URL(`http://localhost:${compilation.config.port}${route}`); + const request = new Request(url); - html = (await htmlResource.serve(route)).body; - html = (await interceptPage(compilation, html, route)).body; + let body = await (await servePage(url, request, plugins)).text(); + body = await (await interceptPage(url, request, plugins, html)).text(); - createOutputDirectory(route, outputPathDir); + createOutputDirectory(route, outputDirUrl); - const resources = trackResourcesForRoute(html, compilation, route); + const resources = trackResourcesForRoute(body, compilation, route); const scripts = resources .filter(resource => resource.type === 'script') .map(resource => resource.sourcePathURL.href); - html = await new Promise((resolve, reject) => { + body = await new Promise((resolve, reject) => { pool.runTask({ modulePath: null, compilation: JSON.stringify(compilation), route, prerender: true, - htmlContents: html, + htmlContents: body, scripts: JSON.stringify(scripts) }, (err, result) => { if (err) { @@ -142,7 +148,7 @@ async function preRenderCompilationWorker(compilation, workerPrerender) { }); }); - await fs.promises.writeFile(path.join(scratchDir, outputPath), html); + await fs.promises.writeFile(outputPathUrl, body); console.info('generated page...', route); } @@ -151,28 +157,32 @@ async function preRenderCompilationWorker(compilation, workerPrerender) { async function preRenderCompilationCustom(compilation, customPrerender) { const { scratchDir } = compilation.context; const renderer = (await import(customPrerender.customUrl)).default; + const plugins = getPluginInstances(compilation); console.info('pages to generate', `\n ${compilation.graph.map(page => page.route).join('\n ')}`); - await renderer(compilation, async (page, contents) => { - const { outputPath, route } = page; - const outputPathDir = path.join(scratchDir, route); + await renderer(compilation, async (page, body) => { + const { route, outputPath } = page; + const outputDirUrl = new URL(`./${route}`, scratchDir); + const outputPathUrl = new URL(`./${outputPath}`, scratchDir); + const url = new URL(`http://localhost:${compilation.config.port}${route}`); + const request = new Request(url); // clean up special Greenwood dev only assets that would come through if prerendering with a headless browser - contents = contents.replace(/`, ` - - `); - } else if (optimizationAttr === 'static' || optimization === 'static') { - body = body.replace(``, ''); - } - } else if (type === 'link') { - if (!optimizationAttr && (optimization !== 'none' && optimization !== 'inline')) { - const optimizedFilePath = `/${optimizedFileName}`; - - body = body.replace(src, optimizedFilePath); - body = body.replace('', ` - - - `); - } else if (optimizationAttr === 'inline' || optimization === 'inline') { - // https://github.com/ProjectEvergreen/greenwood/issues/810 - // when pre-rendering, puppeteer normalizes everything to - // but if not using pre-rendering, then it could come out as - // not great, but best we can do for now until #742 - body = body.replace(``, ` - - `).replace(``, ` - - `); - } - } + const { pathname } = url; + const pageResources = this.compilation.graph.find(page => page.outputPath === pathname || page.route === pathname).imports; + let body = await response.text(); + + for (const pageResource of pageResources) { + const keyedResource = this.compilation.resources.get(pageResource.sourcePathURL.pathname); + const { contents, src, type, optimizationAttr, optimizedFileContents, optimizedFileName, rawAttributes } = keyedResource; + + if (src) { + if (type === 'script') { + if (!optimizationAttr && optimization === 'default') { + const optimizedFilePath = `/${optimizedFileName}`; + + body = body.replace(src, optimizedFilePath); + body = body.replace('', ` + + + `); + } else if (optimizationAttr === 'inline' || optimization === 'inline') { + const isModule = rawAttributes.indexOf('type="module') >= 0 ? ' type="module"' : ''; + + body = body.replace(``, ` + + `); + } else if (optimizationAttr === 'static' || optimization === 'static') { + body = body.replace(``, ''); + } + } else if (type === 'link') { + if (!optimizationAttr && (optimization !== 'none' && optimization !== 'inline')) { + const optimizedFilePath = `/${optimizedFileName}`; + + body = body.replace(src, optimizedFilePath); + body = body.replace('', ` + + + `); + } else if (optimizationAttr === 'inline' || optimization === 'inline') { + // https://github.com/ProjectEvergreen/greenwood/issues/810 + // when pre-rendering, puppeteer normalizes everything to + // but if not using pre-rendering, then it could come out as + // not great, but best we can do for now until #742 + body = body.replace(``, ` + + `).replace(``, ` + + `); + } + } + } else { + if (type === 'script') { + if (optimizationAttr === 'static' || optimization === 'static') { + body = body.replace(``, ''); + } else if (optimizationAttr === 'none') { + body = body.replace(contents, contents.replace(/\.\//g, '/').replace(/\$/g, '$$$')); } else { - if (type === 'script') { - if (optimizationAttr === 'static' || optimization === 'static') { - body = body.replace(``, ''); - } else if (optimizationAttr === 'none') { - body = body.replace(contents, contents.replace(/\.\//g, '/').replace(/\$/g, '$$$')); - } else { - body = body.replace(contents, optimizedFileContents.replace(/\.\//g, '/').replace(/\$/g, '$$$')); - } - } else if (type === 'style') { - body = body.replace(contents, optimizedFileContents); - } + body = body.replace(contents, optimizedFileContents.replace(/\.\//g, '/').replace(/\$/g, '$$$')); } + } else if (type === 'style') { + body = body.replace(contents, optimizedFileContents); } + } + } - // TODO clean up lit-polyfill as part of https://github.com/ProjectEvergreen/greenwood/issues/728 - body = body.replace(/ - - `.replace(/\n/g, '').replace(/ /g, '')) - .replace(/(.*)<\/body>/s, ` - \n - - - ${bodyContents.replace(/\$/g, '$$$')}\n - - - ${routeTags.join('\n')} - - `); - - resolve(body); - } catch (e) { - reject(e); + return ` + + `; + }); + + if (isStaticRoute) { + if (!fs.existsSync(outputPartialDirPath)) { + fs.mkdirSync(outputPartialDirPath, { + recursive: true + }); } + + await fs.promises.writeFile(new URL('./index.html', outputPartialDirUrl), partial); + } + + body = body.replace('', ` + + + `.replace(/\n/g, '').replace(/ /g, '')) + .replace(/(.*)<\/body>/s, ` + \n + + + ${partial.replace(/\$/g, '$$$')}\n + + + ${routeTags.join('\n')} + + `); + + // TODO avoid having to rebuild response each time? + return new Response(body, { + headers: response.headers }); } } diff --git a/packages/plugin-graphql/src/index.js b/packages/plugin-graphql/src/index.js index 45e5383ed..48cd70f60 100644 --- a/packages/plugin-graphql/src/index.js +++ b/packages/plugin-graphql/src/index.js @@ -1,7 +1,6 @@ import fs from 'fs'; import { graphqlServer } from './core/server.js'; import { mergeImportMap } from '@greenwood/cli/src/lib/walker-package-ranger.js'; -import path from 'path'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; import { ServerInterface } from '@greenwood/cli/src/lib/server-interface.js'; import rollupPluginAlias from '@rollup/plugin-alias'; @@ -36,7 +35,7 @@ class GraphQLResource extends ResourceInterface { // TODO avoid having to rebuild response each time? return new Response(body, { headers: { - 'Content-Type': this.contentType[0] + 'content-type': this.contentType[0] } }); } @@ -55,24 +54,23 @@ class GraphQLResource extends ResourceInterface { }); } - async shouldOptimize(url = '', body, headers = {}) { - return Promise.resolve(path.extname(url) === '.html' || (headers.request && headers.request['content-type'].indexOf('text/html') >= 0)); + async shouldOptimize(url, response) { + return response.headers.get('content-type').indexOf(this.contentType[1]) >= 0; } - async optimize(url, body) { - return new Promise((resolve, reject) => { - try { - body = body.replace('', ` - - - `); - - resolve(body); - } catch (e) { - reject(e); - } + async optimize(url, response) { + let body = await response.text(); + + body = body.replace('', ` + + + `); + + // TODO avoid having to rebuild response each time? + return new Response(body, { + headers: response.headers }); } } From 5deb291d4df4e941ab9bcd4b11e69156d00a3862 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 3 Jan 2023 08:50:43 -0500 Subject: [PATCH 15/50] refactor bundle and optimize lifecycles --- packages/cli/src/lifecycles/bundle.js | 69 ++++++++++++++++----------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/packages/cli/src/lifecycles/bundle.js b/packages/cli/src/lifecycles/bundle.js index 39d4de559..5d24b9381 100644 --- a/packages/cli/src/lifecycles/bundle.js +++ b/packages/cli/src/lifecycles/bundle.js @@ -1,3 +1,4 @@ +/* eslint-disable max-depth */ import fs from 'fs'; import { getRollupConfig } from '../config/rollup.config.js'; import { hashString } from '../lib/hashing-utils.js'; @@ -18,39 +19,42 @@ async function cleanUpResources(compilation) { } } -async function optimizeStaticPages(compilation, optimizeResources) { +async function optimizeStaticPages(compilation, plugins) { const { scratchDir, outputDir } = compilation.context; return Promise.all(compilation.graph .filter(page => !page.isSSR || (page.isSSR && page.data.static) || (page.isSSR && compilation.config.prerender)) .map(async (page) => { const { route, outputPath } = page; - const html = await fs.promises.readFile(path.join(scratchDir, outputPath), 'utf-8'); + const url = new URL(`http://localhost:${compilation.config.port}${route}`); + const contents = await fs.promises.readFile(new URL(`./${outputPath}`, scratchDir), 'utf-8'); + const headers = new Headers(); - if (route !== '/404/' && !fs.existsSync(path.join(outputDir, route))) { - fs.mkdirSync(path.join(outputDir, route), { + headers.append('content-type', 'text/html'); + + if (route !== '/404/' && !fs.existsSync(new URL(`.${route}`, outputDir).pathname)) { + fs.mkdirSync(new URL(`.${route}`, outputDir).pathname, { recursive: true }); } - let htmlOptimized = await optimizeResources.reduce(async (htmlPromise, resource) => { - const contents = await htmlPromise; - const shouldOptimize = await resource.shouldOptimize(outputPath, contents); + let response = new Response(contents, { headers }); - return shouldOptimize - ? resource.optimize(outputPath, contents) - : Promise.resolve(contents); - }, Promise.resolve(html)); + for (const plugin of plugins) { + if (plugin.shouldOptimize && await plugin.shouldOptimize(url, response)) { + response = await plugin.optimize(url, response); + } + } // clean up optimization markers - htmlOptimized = htmlOptimized.replace(/data-gwd-opt=".*[a-z]"/g, ''); + const body = (await response.text()).replace(/data-gwd-opt=".*[a-z]"/g, ''); - await fs.promises.writeFile(path.join(outputDir, outputPath), htmlOptimized); + await fs.promises.writeFile(new URL(`./${outputPath}`, outputDir), body); }) ); } -async function bundleStyleResources(compilation, optimizationPlugins) { +async function bundleStyleResources(compilation, plugins) { const { outputDir } = compilation.context; for (const resource of compilation.resources.values()) { @@ -74,7 +78,10 @@ async function bundleStyleResources(compilation, optimizationPlugins) { optimizedFileName = `${hashString(contents)}.css`; } - const outputPathRoot = path.join(outputDir, path.dirname(optimizedFileName)); + const outputPathRoot = new URL(`./${optimizedFileName}`, outputDir).pathname + .split('/') + .slice(0, -1) + .join('/'); if (!fs.existsSync(outputPathRoot)) { fs.mkdirSync(outputPathRoot, { @@ -85,22 +92,28 @@ async function bundleStyleResources(compilation, optimizationPlugins) { if (compilation.config.optimization === 'none' || optimizationAttr === 'none') { optimizedFileContents = contents; } else { - const url = resource.sourcePathURL.pathname; - let optimizedStyles = await fs.promises.readFile(url, 'utf-8'); + const url = resource.sourcePathURL; + const body = await fs.promises.readFile(url, 'utf-8'); + const headers = new Headers(); + const request = new Request(url); + + headers.append('content-type', 'text/css'); + + let response = new Response(body, { headers }); - for (const plugin of optimizationPlugins) { - optimizedStyles = await plugin.shouldIntercept(url, optimizedStyles) - ? (await plugin.intercept(url, optimizedStyles)).body - : optimizedStyles; + for (const plugin of plugins) { + if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request, response)) { + response = await plugin.intercept(url, request, response); + } } - for (const plugin of optimizationPlugins) { - optimizedStyles = await plugin.shouldOptimize(url, optimizedStyles) - ? await plugin.optimize(url, optimizedStyles) - : optimizedStyles; + for (const plugin of plugins) { + if (plugin.shouldOptimize && await plugin.shouldOptimize(url, response)) { + response = await plugin.optimize(url, response); + } } - optimizedFileContents = optimizedStyles; + optimizedFileContents = await response.text(); } compilation.resources.set(resourceKey, { @@ -109,7 +122,7 @@ async function bundleStyleResources(compilation, optimizationPlugins) { optimizedFileContents }); - await fs.promises.writeFile(path.join(outputDir, optimizedFileName), optimizedFileContents); + await fs.promises.writeFile(new URL(`./${optimizedFileName}`, outputDir), optimizedFileContents); } } } @@ -152,7 +165,7 @@ const bundleCompilation = async (compilation) => { console.info('optimizing static pages....'); - await optimizeStaticPages(compilation, optimizeResourcePlugins.filter(plugin => plugin.contentType.includes('text/html'))); + await optimizeStaticPages(compilation, optimizeResourcePlugins); await cleanUpResources(compilation); resolve(); From 812c94385ab332ba6e28fa8da34739e9a3514e76 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 3 Jan 2023 08:56:32 -0500 Subject: [PATCH 16/50] restore cleanupResources --- packages/cli/src/lifecycles/bundle.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/lifecycles/bundle.js b/packages/cli/src/lifecycles/bundle.js index 5d24b9381..a3c90b5dd 100644 --- a/packages/cli/src/lifecycles/bundle.js +++ b/packages/cli/src/lifecycles/bundle.js @@ -14,7 +14,7 @@ async function cleanUpResources(compilation) { const optAttr = ['inline', 'static'].indexOf(optimizationAttr) >= 0; if (optimizedFileName && (!src || (optAttr || optConfig))) { - fs.unlinkSync(path.join(outputDir, optimizedFileName)); + fs.unlinkSync(new URL(`./${optimizedFileName}`, outputDir).pathname); } } } @@ -67,6 +67,7 @@ async function bundleStyleResources(compilation, plugins) { let optimizedFileContents; if (src) { + // TODO remove path. usage const basename = path.basename(srcPath); const basenamePieces = path.basename(srcPath).split('.'); const fileNamePieces = srcPath.split('/').filter(piece => piece !== ''); // normalize by removing any leading /'s From 39fd81ed23dfbbc1237ec3ff5506918e8fd83840 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 3 Jan 2023 11:13:33 -0500 Subject: [PATCH 17/50] restore copy lifecyle and all of build command --- packages/cli/src/commands/build.js | 4 +- packages/cli/src/lib/node-modules-utils.js | 1 + packages/cli/src/lifecycles/copy.js | 98 ++++++++++------------ packages/plugin-polyfills/src/index.js | 15 ++-- 4 files changed, 53 insertions(+), 65 deletions(-) diff --git a/packages/cli/src/commands/build.js b/packages/cli/src/commands/build.js index e5ce43348..275e66071 100644 --- a/packages/cli/src/commands/build.js +++ b/packages/cli/src/commands/build.js @@ -15,8 +15,8 @@ const runProductionBuild = async (compilation) => { ? compilation.config.plugins.filter(plugin => plugin.type === 'renderer')[0].provider(compilation) : {}; - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir); + if (!fs.existsSync(outputDir.pathname)) { + fs.mkdirSync(outputDir.pathname); } if (prerender || prerenderPlugin.prerender) { diff --git a/packages/cli/src/lib/node-modules-utils.js b/packages/cli/src/lib/node-modules-utils.js index 393d03c9f..07db68c4c 100644 --- a/packages/cli/src/lib/node-modules-utils.js +++ b/packages/cli/src/lib/node-modules-utils.js @@ -1,3 +1,4 @@ +// TODO convert this to use / return URLs import { createRequire } from 'module'; // https://stackoverflow.com/a/62499498/417806 import fs from 'fs'; import path from 'path'; diff --git a/packages/cli/src/lifecycles/copy.js b/packages/cli/src/lifecycles/copy.js index 4000bf52a..09c3a35c6 100644 --- a/packages/cli/src/lifecycles/copy.js +++ b/packages/cli/src/lifecycles/copy.js @@ -14,11 +14,11 @@ async function rreaddir (dir, allFiles = []) { } // https://stackoverflow.com/a/30405105/417806 -async function copyFile(source, target) { +async function copyFile(source, target, projectDirectory) { try { - console.info(`copying file... ${source.replace(`${process.cwd()}/`, '')}`); - const rd = fs.createReadStream(source); - const wr = fs.createWriteStream(target); + console.info(`copying file... ${source.pathname.replace(projectDirectory.pathname, '')}`); + const rd = fs.createReadStream(source.pathname); + const wr = fs.createWriteStream(target.pathname); return await new Promise((resolve, reject) => { rd.on('error', reject); @@ -33,67 +33,55 @@ async function copyFile(source, target) { } } -async function copyDirectory(from, to) { - return new Promise(async(resolve, reject) => { - try { - console.info(`copying directory... ${from.replace(`${process.cwd()}/`, '')}`); - const files = await rreaddir(from); - - if (files.length > 0) { - if (!fs.existsSync(to)) { - fs.mkdirSync(to, { - recursive: true - }); +async function copyDirectory(fromUrl, toUrl, projectDirectory) { + try { + console.info(`copying directory... ${fromUrl.pathname.replace(projectDirectory.pathname, '')}`); + const files = await rreaddir(fromUrl.pathname); + + if (files.length > 0) { + if (!fs.existsSync(toUrl.pathname)) { + fs.mkdirSync(toUrl.pathname, { + recursive: true + }); + } + await Promise.all(files.filter((filePath) => { + const target = filePath.replace(fromUrl.pathname, toUrl.pathname); + const isDirectory = fs.lstatSync(filePath).isDirectory(); + + if (isDirectory && !fs.existsSync(target)) { + fs.mkdirSync(target); + } else if (!isDirectory) { + return filePath; } - await Promise.all(files.filter((asset) => { - const target = asset.replace(from, to); - const isDirectory = path.extname(target) === ''; + }).map((filePath) => { + const sourceUrl = new URL(`file://${filePath}`); + const targetUrl = new URL(`file://${filePath.replace(fromUrl.pathname, toUrl.pathname)}`); - if (isDirectory && !fs.existsSync(target)) { - fs.mkdirSync(target); - } else if (!isDirectory) { - return asset; - } - }).map((asset) => { - const target = asset.replace(from, to); - - return copyFile(asset, target); - })); - } - resolve(); - } catch (e) { - reject(e); + return copyFile(sourceUrl, targetUrl, projectDirectory); + })); } - }); + } catch (e) { + console.error('ERROR', e); + } } -const copyAssets = (compilation) => { - - return new Promise(async (resolve, reject) => { - try { - const copyPlugins = compilation.config.plugins.filter(plugin => plugin.type === 'copy'); +const copyAssets = async (compilation) => { + const copyPlugins = compilation.config.plugins.filter(plugin => plugin.type === 'copy'); + const { projectDirectory } = compilation.context; - for (const plugin of copyPlugins) { - const locations = await plugin.provider(compilation); + for (const plugin of copyPlugins) { + const locations = await plugin.provider(compilation); - for (const location of locations) { - const { from, to } = location; + for (const location of locations) { + const { from, to } = location; - if (path.extname(from) === '') { - // copy directory - await copyDirectory(from, to); - } else { - // copy file - await copyFile(from, to); - } - } + if (from.pathname.endsWith('/')) { + await copyDirectory(from, to, projectDirectory); + } else { + await copyFile(from, to, projectDirectory); } - - resolve(); - } catch (err) { - reject(err); } - }); + } }; export { copyAssets }; \ No newline at end of file diff --git a/packages/plugin-polyfills/src/index.js b/packages/plugin-polyfills/src/index.js index a4c0d4e94..d160d35b6 100644 --- a/packages/plugin-polyfills/src/index.js +++ b/packages/plugin-polyfills/src/index.js @@ -1,5 +1,4 @@ import { getNodeModulesLocationForPackage } from '@greenwood/cli/src/lib/node-modules-utils.js'; -import path from 'path'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; class PolyfillsResource extends ResourceInterface { @@ -80,21 +79,21 @@ const greenwoodPluginPolyfills = (options = {}) => { type: 'copy', name: 'plugin-copy-polyfills', provider: async (compilation) => { - // TODO convert this and node utils to use URL const { outputDir } = compilation.context; const polyfillPackageName = '@webcomponents/webcomponentsjs'; const polyfillNodeModulesLocation = await getNodeModulesLocationForPackage(polyfillPackageName); const litNodeModulesLocation = await getNodeModulesLocationForPackage('lit'); + const standardPolyfills = [{ - from: path.join(polyfillNodeModulesLocation, 'webcomponents-loader.js'), - to: path.join(outputDir.pathname, 'webcomponents-loader.js') + from: new URL('./webcomponents-loader.js', new URL(`file://${polyfillNodeModulesLocation}/`)), + to: new URL('./webcomponents-loader.js', outputDir) }, { - from: path.join(polyfillNodeModulesLocation, 'bundles'), - to: path.join(outputDir.pathname, 'bundles') + from: new URL('./bundles/', new URL(`file://${polyfillNodeModulesLocation}/`)), + to: new URL('./bundles/', outputDir) }]; const litPolyfills = [{ - from: path.join(litNodeModulesLocation, 'polyfill-support.js'), - to: path.join(outputDir.pathname, 'polyfill-support.js') + from: new URL('./polyfill-support.js', new URL(`file://${litNodeModulesLocation}/`)), + to: new URL('./polyfill-support.js', outputDir) }]; return [ From 957a986cd62b4e59dc3c16f0723cec37bb8f5751 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 3 Jan 2023 13:48:05 -0500 Subject: [PATCH 18/50] restore serve command and full website prerendering --- packages/cli/src/index.js | 10 +- packages/cli/src/lifecycles/serve.js | 192 ++++++------------ .../resource/plugin-standard-javascript.js | 2 +- 3 files changed, 71 insertions(+), 133 deletions(-) diff --git a/packages/cli/src/index.js b/packages/cli/src/index.js index f968f9729..f398ed2e7 100755 --- a/packages/cli/src/index.js +++ b/packages/cli/src/index.js @@ -69,22 +69,22 @@ const run = async() => { switch (command) { case 'build': - // await (await import('./commands/build.js')).runProductionBuild(compilation); + await (await import('./commands/build.js')).runProductionBuild(compilation); break; case 'develop': - // await (await import('./commands/develop.js')).runDevServer(compilation); + await (await import('./commands/develop.js')).runDevServer(compilation); break; case 'serve': process.env.__GWD_COMMAND__ = 'build'; - // await (await import('./commands/build.js')).runProductionBuild(compilation); - // await (await import('./commands/serve.js')).runProdServer(compilation); + await (await import('./commands/build.js')).runProductionBuild(compilation); + await (await import('./commands/serve.js')).runProdServer(compilation); break; case 'eject': - // await (await import('./commands/eject.js')).ejectConfiguration(compilation); + await (await import('./commands/eject.js')).ejectConfiguration(compilation); break; default: diff --git a/packages/cli/src/lifecycles/serve.js b/packages/cli/src/lifecycles/serve.js index 68fd45531..57ee37b8d 100644 --- a/packages/cli/src/lifecycles/serve.js +++ b/packages/cli/src/lifecycles/serve.js @@ -1,6 +1,5 @@ import fs from 'fs'; // import { hashString } from '../lib/hashing-utils.js'; -import path from 'path'; import Koa from 'koa'; import { Readable } from 'stream'; import { ResourceInterface } from '../lib/resource-interface.js'; @@ -153,52 +152,38 @@ async function getDevServer(compilation) { async function getStaticServer(compilation, composable) { const app = new Koa(); - const standardResources = compilation.config.plugins.filter((plugin) => { - // html is intentionally omitted - return plugin.isGreenwoodDefaultPlugin - && plugin.type === 'resource' - && ((plugin.name.indexOf('plugin-standard') >= 0 // allow standard web resources - && plugin.name.indexOf('plugin-standard-html') < 0) // but _not_ our markdown / HTML plugin - || plugin.name.indexOf('plugin-source-maps') >= 0); // and source maps - }).map((plugin) => { - return plugin.provider(compilation); + const { outputDir } = compilation.context; + const standardResourcePlugins = compilation.config.plugins.filter((plugin) => { + return plugin.type === 'resource' + && plugin.isGreenwoodDefaultPlugin + && plugin.name !== 'plugin-standard-html'; }); app.use(async (ctx, next) => { - const { outputDir, userWorkspace } = compilation.context; - const url = ctx.request.url.replace(/\?(.*)/, ''); // get rid of things like query string parameters - - // only handle static output routes, eg. public/about.html - if (url.endsWith('/') || url.endsWith('.html')) { - const barePath = fs.existsSync(path.join(userWorkspace, 'index.html')) // SPA - ? 'index.html' - : url.endsWith('/') - ? path.join(url, 'index.html') - : url; - - if (fs.existsSync(path.join(outputDir, barePath))) { - const contents = await fs.promises.readFile(path.join(outputDir, barePath), 'utf-8'); - - ctx.set('content-type', 'text/html'); - ctx.body = contents; - } + const url = new URL(`http://localhost:8080${ctx.url}`); + const matchingRoute = compilation.graph.find(page => page.route === url.pathname); + + if (matchingRoute || url.pathname.split('.').pop() === 'html') { + const pathname = matchingRoute ? matchingRoute.outputPath : url.pathname; + const body = await fs.promises.readFile(new URL(`./${pathname}`, outputDir), 'utf-8'); + + ctx.set('content-type', 'text/html'); + ctx.body = body; } await next(); }); app.use(async (ctx, next) => { - const url = ctx.request.url; + const url = new URL(`http://localhost:8080${ctx.url}`); if (compilation.config.devServer.proxy) { - const proxyPlugin = compilation.config.plugins.filter((plugin) => { - return plugin.name === 'plugin-dev-proxy'; - }).map((plugin) => { - return plugin.provider(compilation); - })[0]; - - if (url !== '/' && await proxyPlugin.shouldServe(url)) { - ctx.body = (await proxyPlugin.serve(url)).body; + const proxyPlugin = standardResourcePlugins + .find((plugin) => plugin.name === 'plugin-dev-proxy') + .provider(compilation); + + if (await proxyPlugin.shouldServe(url)) { + ctx.body = (await proxyPlugin.serve(url)).text(); } } @@ -206,38 +191,24 @@ async function getStaticServer(compilation, composable) { }); app.use(async (ctx, next) => { - const responseAccumulator = { - body: ctx.body, - contentType: ctx.response.header['content-type'] - }; - - const reducedResponse = await standardResources.reduce(async (responsePromise, resource) => { - const response = await responsePromise; - const url = ctx.url.replace(/\?(.*)/, ''); - const { headers } = ctx.response; - const outputPathUrl = path.join(compilation.context.outputDir, url); - const shouldServe = await resource.shouldServe(outputPathUrl, { - request: ctx.headers, - response: headers - }); - - if (shouldServe) { - const resolvedResource = await resource.serve(outputPathUrl, { - request: ctx.headers, - response: headers - }); - - return Promise.resolve({ - ...response, - ...resolvedResource - }); - } else { - return Promise.resolve(response); + const url = new URL(`.${ctx.url}`, outputDir.href); + const resourcePlugins = standardResourcePlugins.map((plugin) => { + return plugin.provider(compilation); + }); + const request = new Request(url.href); + let response = null; + + for (const plugin of resourcePlugins) { + if (plugin.shouldServe && await plugin.shouldServe(url, request)) { + response = await plugin.serve(url, request); } - }, Promise.resolve(responseAccumulator)); + } - ctx.set('content-type', reducedResponse.contentType); - ctx.body = reducedResponse.body; + if (response) { + ctx.body = Readable.from(response.body); + ctx.type = response.headers.get('content-type'); + ctx.status = response.status; + } if (composable) { await next(); @@ -249,81 +220,48 @@ async function getStaticServer(compilation, composable) { async function getHybridServer(compilation) { const app = await getStaticServer(compilation, true); - const apiResource = compilation.config.plugins.filter((plugin) => { - return plugin.isGreenwoodDefaultPlugin - && plugin.type === 'resource' - && plugin.name.indexOf('plugin-api-routes') === 0; - }).map((plugin) => { - return plugin.provider(compilation); - })[0]; + const resourcePlugins = compilation.config.plugins.filter((plugin) => { + return plugin.type === 'resource'; + }); app.use(async (ctx) => { - const url = ctx.request.url.replace(/\?(.*)/, ''); // get rid of things like query string parameters - const isApiRoute = await apiResource.shouldServe(url); - const matchingRoute = compilation.graph.filter((node) => { - return node.route === url; - })[0] || { data: {} }; + const url = new URL(`http://localhost:8080${ctx.url}`); + const isApiRoute = url.pathname.startsWith('/api'); + const matchingRoute = compilation.graph.find((node) => node.route === url) || { data: {} }; if (matchingRoute.isSSR && !matchingRoute.data.static) { - // TODO would be nice to pull these plugins once instead of one every request - const headers = { - request: { 'accept': 'text/html', 'content-type': 'text/html' }, - response: { 'content-type': 'text/html' } - }; - const standardHtmlResource = compilation.config.plugins.filter((plugin) => { + const standardHtmlResource = [resourcePlugins.find((plugin) => { return plugin.isGreenwoodDefaultPlugin - && plugin.type === 'resource' && plugin.name.indexOf('plugin-standard-html') === 0; - }).map((plugin) => { - return plugin.provider(compilation); - })[0]; - let body; - - const interceptResources = compilation.config.plugins.filter((plugin) => { - return plugin.type === 'resource' && !plugin.isGreenwoodDefaultPlugin; - }).map((plugin) => { - return plugin.provider(compilation); - }).filter((provider) => { - return provider.shouldIntercept && provider.intercept; - }); + })].provider(compilation); + let response = await standardHtmlResource.serve(url, request); - body = (await standardHtmlResource.serve(url)).body; - body = (await interceptResources.reduce(async (htmlPromise, resource) => { - const html = (await htmlPromise).body; - const shouldIntercept = await resource.shouldIntercept(url, html, headers); - - return shouldIntercept - ? resource.intercept(url, html, headers) - : htmlPromise; - }, Promise.resolve({ url, body }))).body; - - const optimizeResources = compilation.config.plugins.filter((plugin) => { - return plugin.type === 'resource'; - }).map((plugin) => { - return plugin.provider(compilation); - }).filter((provider) => { - return provider.shouldOptimize && provider.optimize; - }); - - body = await optimizeResources.reduce(async (htmlPromise, resource) => { - const html = await htmlPromise; - const shouldOptimize = await resource.shouldOptimize(url, html, headers); + for (const plugin of resourcePlugins) { + if (plugin.shouldIntercept && await plugin.shouldIntercept(url, response)) { + response = await plugin.intercept(url, response); + } + } - return shouldOptimize - ? resource.optimize(url, html, headers) - : Promise.resolve(html); - }, Promise.resolve(body)); + for (const plugin of resourcePlugins) { + if (plugin.shouldOptimize && await plugin.shouldOptimize(url, response)) { + response = await plugin.optimize(url, response); + } + } - ctx.status = 200; + ctx.body = response.body; ctx.set('content-type', 'text/html'); - ctx.body = body; + ctx.status = 200; } else if (isApiRoute) { // TODO just use response - const { body, resp } = await apiResource.serve(ctx.request.url); + const apiResource = standardResourcePlugins.find((plugin) => { + return plugin.isGreenwoodDefaultPlugin + && plugin.name === 'plugin-api-routes'; + }).provider(compilation); + const response = await apiResource.serve(ctx.request.url); ctx.status = 200; - ctx.set('content-type', resp.headers.get('content-type')); - ctx.body = body; + ctx.set('content-type', response.headers.get('content-type')); + ctx.body = response.body; } }); diff --git a/packages/cli/src/plugins/resource/plugin-standard-javascript.js b/packages/cli/src/plugins/resource/plugin-standard-javascript.js index fce2a6d70..e85641280 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-javascript.js +++ b/packages/cli/src/plugins/resource/plugin-standard-javascript.js @@ -16,7 +16,7 @@ class StandardJavaScriptResource extends ResourceInterface { } async shouldServe(url) { - return url.protocol === 'file:' && this.extensions.indexOf(url.pathname.split('.').pop()) >= 0; + return url.protocol === 'file:' && this.extensions.includes(url.pathname.split('.').pop()); } async serve(url) { From 87e8d6390325461f022756d61812df6673070aa1 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Sat, 7 Jan 2023 16:48:46 -0500 Subject: [PATCH 19/50] restore specs and fixing missed upgrades --- packages/cli/src/lib/resource-utils.js | 5 ++++- packages/cli/src/lifecycles/config.js | 4 ++-- packages/cli/src/lifecycles/context.js | 2 +- .../cli/src/plugins/resource/plugin-api-routes.js | 9 +++++---- .../cli/src/plugins/resource/plugin-dev-proxy.js | 14 ++++++++++++-- .../cli/src/plugins/resource/plugin-source-maps.js | 7 ++++--- .../src/plugins/resource/plugin-standard-html.js | 2 +- .../src/plugins/resource/plugin-static-router.js | 14 ++++++++------ .../src/plugins/resource/plugin-user-workspace.js | 4 +++- test/smoke-test.js | 2 +- 10 files changed, 41 insertions(+), 22 deletions(-) diff --git a/packages/cli/src/lib/resource-utils.js b/packages/cli/src/lib/resource-utils.js index 31c6c91cb..5752a2258 100644 --- a/packages/cli/src/lib/resource-utils.js +++ b/packages/cli/src/lib/resource-utils.js @@ -9,9 +9,12 @@ function modelResource(context, type, src = undefined, contents = undefined, opt let sourcePathURL; if (src) { + // TODO more elegant way to normalize paths with ../, ./, /, etc? sourcePathURL = src.startsWith('/node_modules') ? new URL(`.${src}`, projectDirectory) - : new URL(`${src.replace(/\.\.\//g, '').replace('./', '')}`, userWorkspace); + : src.startsWith('/') + ? new URL(`.${src}`, userWorkspace) + : new URL(`./${src.replace(/\.\.\//g, '').replace('./', '')}`, userWorkspace); contents = fs.readFileSync(sourcePathURL, 'utf-8'); } else { diff --git a/packages/cli/src/lifecycles/config.js b/packages/cli/src/lifecycles/config.js index 1f71338f3..0d5a6c468 100644 --- a/packages/cli/src/lifecycles/config.js +++ b/packages/cli/src/lifecycles/config.js @@ -64,7 +64,7 @@ const readAndMergeConfig = async() => { // workspace validation if (workspace) { - if (!workspace instanceof URL) { + if (!(workspace instanceof URL)) { reject('Error: greenwood.config.js workspace must be an instance of URL'); } @@ -207,7 +207,7 @@ const readAndMergeConfig = async() => { } } else { // SPA should _not_ prerender unless if user has specified prerender should be true - if (fs.existsSync(path.join(customConfig.workspace, 'index.html'))) { + if (fs.existsSync(new URL('./index.html', customConfig.workspace).pathname)) { customConfig.prerender = false; } } diff --git a/packages/cli/src/lifecycles/context.js b/packages/cli/src/lifecycles/context.js index ce434a2f5..e2bdaf332 100644 --- a/packages/cli/src/lifecycles/context.js +++ b/packages/cli/src/lifecycles/context.js @@ -11,7 +11,7 @@ const initContext = async({ config }) => { const outputDir = new URL('./public/', projectDirectory); const dataDir = new URL('../data/', import.meta.url); const userWorkspace = workspace; - const apisDir = new URL('./apis/', userWorkspace); + const apisDir = new URL('./api/', userWorkspace); const pagesDir = new URL(`./${pagesDirectory}/`, userWorkspace); const userTemplatesDir = new URL(`./${templatesDirectory}/`, userWorkspace); diff --git a/packages/cli/src/plugins/resource/plugin-api-routes.js b/packages/cli/src/plugins/resource/plugin-api-routes.js index 8a1de6c33..3dcd7462a 100644 --- a/packages/cli/src/plugins/resource/plugin-api-routes.js +++ b/packages/cli/src/plugins/resource/plugin-api-routes.js @@ -12,15 +12,17 @@ class ApiRoutesResource extends ResourceInterface { } async shouldServe(url) { + const apiPathUrl = new URL(`.${url.pathname.replace('/api', '')}.js`, this.compilation.context.apisDir); + // TODO Could this existence check be derived from the graph instead, like pages are? // https://github.com/ProjectEvergreen/greenwood/issues/946 return url.protocol.indexOf('http') === 0 && url.pathname.startsWith('/api') - && fs.existsSync(this.compilation.context.apisDir, url.pathname.replace('/api/', '')); + && fs.existsSync(apiPathUrl.pathname); } async serve(url, request) { - let href = new URL(`./${url.pathname.replace('/api/', '')}.js`, `file://${this.compilation.context.apisDir}`).href; + let href = new URL(`./${url.pathname.replace('/api/', '')}.js`, `file://${this.compilation.context.apisDir.pathname}`).href; // https://github.com/nodejs/modules/issues/307#issuecomment-1165387383 if (process.env.__GWD_COMMAND__ === 'develop') { // eslint-disable-line no-underscore-dangle @@ -31,9 +33,8 @@ class ApiRoutesResource extends ResourceInterface { const req = new Request(new URL(`${request.url.origin}${url}`), { ...request }); - const resp = await handler(req); - return resp; + return await handler(req); } } diff --git a/packages/cli/src/plugins/resource/plugin-dev-proxy.js b/packages/cli/src/plugins/resource/plugin-dev-proxy.js index 3141eca53..b45a40873 100644 --- a/packages/cli/src/plugins/resource/plugin-dev-proxy.js +++ b/packages/cli/src/plugins/resource/plugin-dev-proxy.js @@ -20,9 +20,19 @@ class DevProxyResource extends ResourceInterface { } async serve(url, request) { - return fetch(request, { - ...request + const { pathname } = url; + const proxies = this.compilation.config.devServer.proxy; + const proxyBaseUrl = Object.entries(proxies).reduce((acc, entry) => { + return pathname.indexOf(entry[0]) >= 0 + ? `${entry[1]}${pathname}` + : acc; + }, pathname); + const requestProxied = new Request(`${proxyBaseUrl}${url.search}`, { + method: request.method, + headers: request.header }); + + return await fetch(requestProxied); } } diff --git a/packages/cli/src/plugins/resource/plugin-source-maps.js b/packages/cli/src/plugins/resource/plugin-source-maps.js index 7613408f5..92fb1d4d4 100644 --- a/packages/cli/src/plugins/resource/plugin-source-maps.js +++ b/packages/cli/src/plugins/resource/plugin-source-maps.js @@ -9,11 +9,12 @@ import { ResourceInterface } from '../../lib/resource-interface.js'; class SourceMapsResource extends ResourceInterface { constructor(compilation, options) { super(compilation, options); - this.extensions = ['.map']; + this.extensions = ['map']; + this.contentType = 'application/json'; } async shouldServe(url) { - return `.${url.pathname.split('.').pop()}` === this.extensions[0] && fs.existsSync(url.pathname); + return url.pathname.split('.').pop() === this.extensions[0] && fs.existsSync(url.pathname); } async serve(url) { @@ -21,7 +22,7 @@ class SourceMapsResource extends ResourceInterface { return new Response(body, { headers: { - 'Content-Type': 'text/javascript' + 'content-type': this.contentType } }); } diff --git a/packages/cli/src/plugins/resource/plugin-standard-html.js b/packages/cli/src/plugins/resource/plugin-standard-html.js index eec1ec413..8a19b1173 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-html.js +++ b/packages/cli/src/plugins/resource/plugin-standard-html.js @@ -215,7 +215,7 @@ class StandardHtmlResource extends ResourceInterface { const isSpaRoute = this.compilation.graph[0].isSPA; const matchingRoute = this.compilation.graph.find((node) => node.route === pathname); const filePath = !matchingRoute.external ? matchingRoute.path : ''; - const isMarkdownContent = matchingRoute.filename.split('.').pop() === 'md'; + const isMarkdownContent = (matchingRoute?.filename || '').split('.').pop() === 'md'; let customImports = []; let body = ''; diff --git a/packages/cli/src/plugins/resource/plugin-static-router.js b/packages/cli/src/plugins/resource/plugin-static-router.js index b574efaad..f61634ad2 100644 --- a/packages/cli/src/plugins/resource/plugin-static-router.js +++ b/packages/cli/src/plugins/resource/plugin-static-router.js @@ -27,18 +27,20 @@ class StaticRouterResource extends ResourceInterface { } async shouldIntercept(url, request, response) { - const { pathname } = url; - const contentType = response.headers.get['content-type']; + const { pathname, protocol } = url; + const contentType = response.headers.get['content-type'] || ''; // TODO should this also happen during development too? return process.env.__GWD_COMMAND__ === 'build' // eslint-disable-line no-underscore-dangle && this.compilation.config.staticRouter && !pathname.startsWith('/404') - && pathname.split('.').pop() === 'html' || (contentType && contentType.indexOf(this.contentType) >= 0); + && protocol === 'http:' || contentType.indexOf(this.contentType) >= 0; } async intercept(url, request, response) { - const body = response.body.replace('', ` + let body = await response.text(); + + body = body.replace('', ` \n `); @@ -56,8 +58,8 @@ class StaticRouterResource extends ResourceInterface { } async optimize(url, response) { - const { pathname } = url; let body = await response.text(); + const { pathname } = url; const isStaticRoute = this.compilation.graph.find(page => page.route === pathname && !page.isSSR); const { outputDir } = this.compilation.context; const partial = body.match(/(.*)<\/body>/s)[0].replace('', '').replace('', ''); @@ -76,7 +78,7 @@ class StaticRouterResource extends ResourceInterface { ? '' : page.route.slice(0, page.route.lastIndexOf('/')); - if (url === page.outputPath) { + if (pathname === page.route) { currentTemplate = template; } return ` diff --git a/packages/cli/src/plugins/resource/plugin-user-workspace.js b/packages/cli/src/plugins/resource/plugin-user-workspace.js index 8b7fe21ff..2bd002997 100644 --- a/packages/cli/src/plugins/resource/plugin-user-workspace.js +++ b/packages/cli/src/plugins/resource/plugin-user-workspace.js @@ -15,7 +15,9 @@ class UserWorkspaceResource extends ResourceInterface { async shouldResolve(url) { const { userWorkspace } = this.compilation.context; - return this.hasExtension(url) && this.resolveForRelativeUrl(url, userWorkspace); + return this.hasExtension(url) + && !url.pathname.startsWith('/node_modules') + && this.resolveForRelativeUrl(url, userWorkspace); } async resolve(url) { diff --git a/test/smoke-test.js b/test/smoke-test.js index 27a22fc04..ac7f8c83a 100644 --- a/test/smoke-test.js +++ b/test/smoke-test.js @@ -231,7 +231,7 @@ function serve(label) { }); it('should return the correct content type', function(done) { - expect(response.headers['content-type']).to.equal('text/html'); + expect(response.headers['content-type']).to.contain('text/html'); done(); }); From a283a496af09c8009d40870bcec51040e2db603c Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 10 Jan 2023 09:39:55 -0500 Subject: [PATCH 20/50] all CLI specs passing --- packages/cli/src/commands/build.js | 4 +- packages/cli/src/commands/eject.js | 12 +++--- packages/cli/src/config/rollup.config.js | 4 +- packages/cli/src/lifecycles/bundle.js | 2 +- packages/cli/src/lifecycles/graph.js | 2 +- packages/cli/src/lifecycles/prerender.js | 12 ++---- packages/cli/src/lifecycles/serve.js | 43 ++++++++++++------- .../plugins/resource/plugin-node-modules.js | 7 ++- .../plugins/resource/plugin-standard-html.js | 26 +++++------ .../greenwood.config.js | 4 +- .../build.config.error-workspace.spec.js | 4 +- .../greenwood.config.js | 4 +- .../build.default.ssr.spec.js | 2 +- .../theme-pack-context-plugin.js | 7 +-- .../build.plugins.copy/greenwood.config.js | 13 ++---- .../greenwood.config.js | 25 ++++++----- .../develop.default/develop.default.spec.js | 36 +++++++++------- .../develop.plugins.context.spec.js | 4 +- .../greenwood.config.js | 13 +++--- .../serve.default.api.spec.js | 2 +- .../cases/serve.default/serve.default.spec.js | 2 +- .../test/cases/theme-pack/greenwood.config.js | 7 +-- .../test/cases/theme-pack/my-theme-pack.js | 6 +-- .../theme-pack/theme-pack.develop.spec.js | 4 +- 24 files changed, 125 insertions(+), 120 deletions(-) diff --git a/packages/cli/src/commands/build.js b/packages/cli/src/commands/build.js index 275e66071..76e9d9857 100644 --- a/packages/cli/src/commands/build.js +++ b/packages/cli/src/commands/build.js @@ -11,8 +11,8 @@ const runProductionBuild = async (compilation) => { try { const { prerender } = compilation.config; const outputDir = compilation.context.outputDir; - const prerenderPlugin = (compilation.config.plugins.filter(plugin => plugin.type === 'renderer') || []).length === 1 - ? compilation.config.plugins.filter(plugin => plugin.type === 'renderer')[0].provider(compilation) + const prerenderPlugin = compilation.config.plugins.find(plugin => plugin.type === 'renderer') + ? compilation.config.plugins.find(plugin => plugin.type === 'renderer').provider(compilation) : {}; if (!fs.existsSync(outputDir.pathname)) { diff --git a/packages/cli/src/commands/eject.js b/packages/cli/src/commands/eject.js index e93792ff7..c0b41d6e7 100644 --- a/packages/cli/src/commands/eject.js +++ b/packages/cli/src/commands/eject.js @@ -1,18 +1,16 @@ import fs from 'fs'; -import path from 'path'; -import { fileURLToPath, URL } from 'url'; const ejectConfiguration = async (compilation) => { return new Promise(async (resolve, reject) => { try { - const configFilePath = fileURLToPath(new URL('../config', import.meta.url)); - const configFiles = fs.readdirSync(configFilePath); + const configFileDirUrl = new URL('../config/', import.meta.url); + const configFiles = await fs.promises.readdir(configFileDirUrl); configFiles.forEach((configFile) => { - const from = path.join(configFilePath, configFile); - const to = `${compilation.context.projectDirectory}/${configFile}`; + const from = new URL(`./${configFile}`, configFileDirUrl); + const to = new URL(`./${configFile}`, compilation.context.projectDirectory); - fs.copyFileSync(from, to); + fs.copyFileSync(from.pathname, to.pathname); console.log(`Ejected ${configFile} successfully.`); }); diff --git a/packages/cli/src/config/rollup.config.js b/packages/cli/src/config/rollup.config.js index 4de4b1ceb..50eb91330 100644 --- a/packages/cli/src/config/rollup.config.js +++ b/packages/cli/src/config/rollup.config.js @@ -12,8 +12,8 @@ function greenwoodResourceLoader (compilation) { resolveId(id) { const { userWorkspace } = compilation.context; - if ((id.indexOf('./') === 0 || id.indexOf('/') === 0) && fs.existsSync(new URL(`./${id}`, userWorkspace).pathname)) { - return new URL(`./${id.replace(/\?type=(.*)/, '')}`, userWorkspace); + if ((id.indexOf('./') === 0 || id.indexOf('/') === 0) && fs.existsSync(new URL(`./${id.replace(/\?type=(.*)/, '')}`, userWorkspace).pathname)) { + return new URL(`./${id.replace(/\?type=(.*)/, '')}`, userWorkspace).pathname; } return null; diff --git a/packages/cli/src/lifecycles/bundle.js b/packages/cli/src/lifecycles/bundle.js index a3c90b5dd..0e13b9dc4 100644 --- a/packages/cli/src/lifecycles/bundle.js +++ b/packages/cli/src/lifecycles/bundle.js @@ -161,7 +161,7 @@ const bundleCompilation = async (compilation) => { await Promise.all([ await bundleScriptResources(compilation), - await bundleStyleResources(compilation, optimizeResourcePlugins.filter(plugin => plugin.contentType.includes('text/css'))) + await bundleStyleResources(compilation, optimizeResourcePlugins) ]); console.info('optimizing static pages....'); diff --git a/packages/cli/src/lifecycles/graph.js b/packages/cli/src/lifecycles/graph.js index 2593d1468..ed6d26337 100644 --- a/packages/cli/src/lifecycles/graph.js +++ b/packages/cli/src/lifecycles/graph.js @@ -216,7 +216,7 @@ const generateGraph = async (compilation) => { }; console.debug('building from local sources...'); - if (fs.existsSync(path.join(userWorkspace.pathname, 'index.html'))) { // SPA + if (fs.existsSync(new URL('./index.html', userWorkspace).pathname)) { // SPA graph = [{ ...graph[0], path: `${userWorkspace.pathname}index.html`, diff --git a/packages/cli/src/lifecycles/prerender.js b/packages/cli/src/lifecycles/prerender.js index 6df72a19d..5b9f42742 100644 --- a/packages/cli/src/lifecycles/prerender.js +++ b/packages/cli/src/lifecycles/prerender.js @@ -100,6 +100,7 @@ async function interceptPage(url, request, plugins, body) { function getPluginInstances (compilation) { return [...compilation.config.plugins] + .filter(plugin => plugin.type === 'resource' && plugin.name !== 'plugin-node-modules:resource') .map((plugin) => { return plugin.provider(compilation); }); @@ -116,13 +117,13 @@ async function preRenderCompilationWorker(compilation, workerPrerender) { for (const page of pages) { const { route, outputPath } = page; - const outputDirUrl = new URL(`./${route}`, scratchDir); + const outputDirUrl = new URL(`./${route}/`, scratchDir); const outputPathUrl = new URL(`./${outputPath}`, scratchDir); const url = new URL(`http://localhost:${compilation.config.port}${route}`); const request = new Request(url); let body = await (await servePage(url, request, plugins)).text(); - body = await (await interceptPage(url, request, plugins, html)).text(); + body = await (await interceptPage(url, request, plugins, body)).text(); createOutputDirectory(route, outputDirUrl); @@ -157,7 +158,6 @@ async function preRenderCompilationWorker(compilation, workerPrerender) { async function preRenderCompilationCustom(compilation, customPrerender) { const { scratchDir } = compilation.context; const renderer = (await import(customPrerender.customUrl)).default; - const plugins = getPluginInstances(compilation); console.info('pages to generate', `\n ${compilation.graph.map(page => page.route).join('\n ')}`); @@ -165,8 +165,6 @@ async function preRenderCompilationCustom(compilation, customPrerender) { const { route, outputPath } = page; const outputDirUrl = new URL(`./${route}`, scratchDir); const outputPathUrl = new URL(`./${outputPath}`, scratchDir); - const url = new URL(`http://localhost:${compilation.config.port}${route}`); - const request = new Request(url); // clean up special Greenwood dev only assets that would come through if prerendering with a headless browser body = body.replace(/ - - - `); - - resolve({ body: newHtml }); - } catch (e) { - reject(e); - } + let body = await response.text(); + + body = body.replace('', ` + + + + + `); + + return new Response(body, { + headers: response.headers }); } } diff --git a/packages/plugin-google-analytics/test/cases/default/default.spec.js b/packages/plugin-google-analytics/test/cases/default/default.spec.js index 6d191c76b..4db2a6af1 100644 --- a/packages/plugin-google-analytics/test/cases/default/default.spec.js +++ b/packages/plugin-google-analytics/test/cases/default/default.spec.js @@ -79,20 +79,20 @@ describe('Build Greenwood With: ', function() { it('should have the expected code with users analyticsId', function() { const expectedContent = ` - var getOutboundLink = function(url) { - gtag('event', 'click', { - 'event_category': 'outbound', - 'event_label': url, - 'transport_type': 'beacon' - }); - } - window.dataLayer = window.dataLayer || []; - function gtag(){dataLayer.push(arguments);} - gtag('js', new Date()); - gtag('config', '${mockAnalyticsId}', { 'anonymize_ip': true }); + var getOutboundLink = function(url) { + gtag('event', 'click', { + 'event_category': 'outbound', + 'event_label': url, + 'transport_type': 'beacon' + }); + } + window.dataLayer = window.dataLayer || []; + function gtag(){dataLayer.push(arguments);} + gtag('js', new Date()); + gtag('config', '${mockAnalyticsId}', { 'anonymize_ip': true }); `; - expect(inlineScript[0].textContent).to.contain(expectedContent); + expect(inlineScript[0].textContent.trim().replace(/\n/g, '').replace(/ /g, '')).to.contain(expectedContent.trim().replace(/\n/g, '').replace(/ /g, '')); }); it('should only have one external Google script tag', function() { diff --git a/packages/plugin-google-analytics/test/cases/option-anonymous/option-anonymous.spec.js b/packages/plugin-google-analytics/test/cases/option-anonymous/option-anonymous.spec.js index 057d6c390..b3035f1c3 100644 --- a/packages/plugin-google-analytics/test/cases/option-anonymous/option-anonymous.spec.js +++ b/packages/plugin-google-analytics/test/cases/option-anonymous/option-anonymous.spec.js @@ -15,7 +15,7 @@ * plugins: [{ * greenwoodPluginGoogleAnalytics({ * analyticsId: 'UA-123456-1', - * anonymouse: false + * anonymous: false * }) * }] * @@ -81,7 +81,7 @@ describe('Build Greenwood With: ', function() { gtag('config', '${mockAnalyticsId}', { 'anonymize_ip': false }); `; - expect(inlineScript[0].textContent).to.contain(expectedContent); + expect(inlineScript[0].textContent.trim().replace(/\n/g, '').replace(/ /g, '')).to.contain(expectedContent.trim().replace(/\n/g, '').replace(/ /g, '')); }); }); diff --git a/packages/plugin-graphql/src/schema/schema.js b/packages/plugin-graphql/src/schema/schema.js index 15b95bc21..cb1a34634 100644 --- a/packages/plugin-graphql/src/schema/schema.js +++ b/packages/plugin-graphql/src/schema/schema.js @@ -3,13 +3,11 @@ import { configTypeDefs, configResolvers } from './config.js'; import { graphTypeDefs, graphResolvers } from './graph.js'; import fs from 'fs'; import gql from 'graphql-tag'; -import path from 'path'; -import { pathToFileURL } from 'url'; const createSchema = async (compilation) => { const { graph } = compilation; const uniqueCustomDataDefKeys = {}; - const customSchemasPath = `${compilation.context.userWorkspace}/data/schema`; + const customSchemasUrl = new URL('./data/schema/', compilation.context.userWorkspace); const customUserResolvers = []; const customUserDefs = []; let customDataDefsString = ''; @@ -36,13 +34,13 @@ const createSchema = async (compilation) => { } `; - if (fs.existsSync(customSchemasPath)) { + if (fs.existsSync(customSchemasUrl.pathname)) { console.log('custom schemas directory detected, scanning...'); - const schemaPaths = (await fs.promises.readdir(customSchemasPath)) - .filter(file => path.extname(file) === '.js'); + const schemaPaths = (await fs.promises.readdir(customSchemasUrl)) + .filter(file => file.split('.').pop() === 'js'); for (const schemaPath of schemaPaths) { - const { customTypeDefs, customResolvers } = await import(pathToFileURL(`${customSchemasPath}/${schemaPath}`)); + const { customTypeDefs, customResolvers } = await import(new URL(`./${schemaPath}`, customSchemasUrl)); customUserDefs.push(customTypeDefs); customUserResolvers.push(customResolvers); diff --git a/packages/plugin-graphql/test/cases/develop.default/develop.default.spec.js b/packages/plugin-graphql/test/cases/develop.default/develop.default.spec.js index 0b3e9b908..84b016e8b 100644 --- a/packages/plugin-graphql/test/cases/develop.default/develop.default.spec.js +++ b/packages/plugin-graphql/test/cases/develop.default/develop.default.spec.js @@ -89,7 +89,7 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.contain('text/html'); + expect(response.headers['content-type']).to.contain('text/html'); done(); }); @@ -134,7 +134,7 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.equal('text/javascript'); + expect(response.headers['content-type']).to.contain('text/javascript'); done(); }); @@ -170,7 +170,7 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.equal('text/javascript'); + expect(response.headers['content-type']).to.contain('text/javascript'); done(); }); diff --git a/packages/plugin-graphql/test/cases/qraphql-server/graphql-server.spec.js b/packages/plugin-graphql/test/cases/qraphql-server/graphql-server.spec.js index c8c2f70ac..b44c12069 100644 --- a/packages/plugin-graphql/test/cases/qraphql-server/graphql-server.spec.js +++ b/packages/plugin-graphql/test/cases/qraphql-server/graphql-server.spec.js @@ -81,7 +81,7 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.contain('text/html'); + expect(response.headers['content-type']).to.contain('text/html'); done(); }); }); @@ -125,12 +125,12 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.contain('application/json'); + expect(response.headers['content-type']).to.contain('application/json'); done(); }); it('should return the expected query response', function(done) { - expect(response.body.data.config.workspace).to.equal(path.join(outputPath, 'src')); + expect(response.body.data.config.workspace).to.equal(new URL('./src/', import.meta.url).href); done(); }); }); diff --git a/packages/plugin-renderer-lit/test/cases/build.default/build.default.spec.js b/packages/plugin-renderer-lit/test/cases/build.default/build.default.spec.js index 6009de6cc..a31503280 100644 --- a/packages/plugin-renderer-lit/test/cases/build.default/build.default.spec.js +++ b/packages/plugin-renderer-lit/test/cases/build.default/build.default.spec.js @@ -182,7 +182,7 @@ describe('Build Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.contain('text/html'); + expect(response.headers['content-type']).to.contain('text/html'); done(); }); diff --git a/packages/plugin-typescript/src/index.js b/packages/plugin-typescript/src/index.js index 55d4d717d..2ac09bbf7 100644 --- a/packages/plugin-typescript/src/index.js +++ b/packages/plugin-typescript/src/index.js @@ -4,7 +4,6 @@ * */ import fs from 'fs'; -import path from 'path'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; import tsc from 'typescript'; @@ -17,7 +16,7 @@ const defaultCompilerOptions = { function getCompilerOptions (projectDirectory, extendConfig) { const customOptions = extendConfig - ? JSON.parse(fs.readFileSync(path.join(projectDirectory, 'tsconfig.json'), 'utf-8')) + ? JSON.parse(fs.readFileSync(new URL('./tsconfig.json', projectDirectory).pathname, 'utf-8')) : { compilerOptions: {} }; return { @@ -29,27 +28,28 @@ function getCompilerOptions (projectDirectory, extendConfig) { class TypeScriptResource extends ResourceInterface { constructor(compilation, options) { super(compilation, options); - this.extensions = ['.ts']; + this.extensions = ['ts']; this.contentType = 'text/javascript'; } - async serve(url) { - return new Promise(async (resolve, reject) => { - try { - const { projectDirectory } = this.compilation.context; - const source = await fs.promises.readFile(url, 'utf-8'); - const compilerOptions = getCompilerOptions(projectDirectory, this.options.extendConfig); + async shouldServe(url) { + const { pathname, protocol } = url; + const isTsFile = protocol === 'file:' && pathname.split('.').pop() === this.extensions[0]; + + return isTsFile || isTsFile && url.searchParams.has('type') && url.searchParams.get('type') === this.extensions[0]; + } - // https://github.com/microsoft/TypeScript/wiki/Using-the-Compiler-API - const result = tsc.transpileModule(source, { compilerOptions }); + async serve(url) { + const { projectDirectory } = this.compilation.context; + const source = await fs.promises.readFile(url, 'utf-8'); + const compilerOptions = getCompilerOptions(projectDirectory, this.options.extendConfig); + // https://github.com/microsoft/TypeScript/wiki/Using-the-Compiler-API + const body = tsc.transpileModule(source, { compilerOptions }).outputText; - resolve({ - body: result.outputText, - contentType: this.contentType - }); - } catch (e) { - reject(e); - } + return new Response(body, { + headers: new Headers({ + 'Content-Type': this.contentType + }) }); } } diff --git a/packages/plugin-typescript/test/cases/develop.default/develop.default.spec.js b/packages/plugin-typescript/test/cases/develop.default/develop.default.spec.js index 46589369a..15f741133 100644 --- a/packages/plugin-typescript/test/cases/develop.default/develop.default.spec.js +++ b/packages/plugin-typescript/test/cases/develop.default/develop.default.spec.js @@ -82,7 +82,7 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.equal('text/javascript'); + expect(response.headers['content-type']).to.contain('text/javascript'); done(); }); From 655f6bcdf1ebb947c76489bf05ea5a44fb12abb9 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Sat, 14 Jan 2023 16:05:03 -0500 Subject: [PATCH 23/50] getting package plugin specs passing --- .../test/cases/develop.default/develop.default.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/plugin-import-css/test/cases/develop.default/develop.default.spec.js b/packages/plugin-import-css/test/cases/develop.default/develop.default.spec.js index 142f5d600..035711775 100644 --- a/packages/plugin-import-css/test/cases/develop.default/develop.default.spec.js +++ b/packages/plugin-import-css/test/cases/develop.default/develop.default.spec.js @@ -89,7 +89,7 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.equal('text/javascript'); + expect(response.headers['content-type']).to.contain('text/javascript'); done(); }); @@ -132,7 +132,7 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.equal('text/javascript'); + expect(response.headers['content-type']).to.contain('text/javascript'); done(); }); From 82f2ae5aced8a19806a6c56a5f144d76e2605b22 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Sat, 14 Jan 2023 20:20:27 -0500 Subject: [PATCH 24/50] getting package plugin specs passing --- packages/plugin-import-json/src/index.js | 16 ++++++++++------ .../develop.default/develop.default.spec.js | 6 +++--- .../test/cases/develop.default/src/main.json | 5 +---- packages/plugin-include-html/src/index.js | 3 ++- .../build.default.link-tag.spec.js | 2 +- packages/plugin-postcss/src/index.js | 6 +++--- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/packages/plugin-import-json/src/index.js b/packages/plugin-import-json/src/index.js index 76467497e..35ea7afee 100644 --- a/packages/plugin-import-json/src/index.js +++ b/packages/plugin-import-json/src/index.js @@ -4,7 +4,7 @@ * This is a Greenwood default plugin. * */ -import fs from 'fs'; +// import fs from 'fs'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; class ImportJsonResource extends ResourceInterface { @@ -16,17 +16,21 @@ class ImportJsonResource extends ResourceInterface { // TODO handle it from node_modules too, when without `?type=json` async shouldIntercept(url) { - return url.searchParams.has('type') && url.searchParams.get('type') === this.extensions[0]; + const { pathname } = url; + + return pathname.split('.').pop() === this.extensions[0] || (url.searchParams.has('type') && url.searchParams.get('type') === this.extensions[0]); } async intercept(url, request, response) { // TODO better way to handle this? // https://github.com/ProjectEvergreen/greenwood/issues/948 - const body = await response.text() === '' - ? await fs.promises.readFile(url, 'utf-8') - : await response.text(); + const body = await response.text(); + // TODO need to support an empty body to read from disc? + // const json = body === '' + // ? await fs.promises.readFile(url, 'utf-8') + // : body; - return new Response(`export default ${JSON.stringify(body)}`, { + return new Response(`export default ${body}`, { headers: new Headers({ 'Content-Type': this.contentType }) diff --git a/packages/plugin-import-json/test/cases/develop.default/develop.default.spec.js b/packages/plugin-import-json/test/cases/develop.default/develop.default.spec.js index a91f85454..26c24e787 100644 --- a/packages/plugin-import-json/test/cases/develop.default/develop.default.spec.js +++ b/packages/plugin-import-json/test/cases/develop.default/develop.default.spec.js @@ -13,8 +13,8 @@ * * { * plugins: [{ - * ...greenwoodPluginImportJson() - * }] + * greenwoodPluginImportJson() + * }] * } * * @@ -89,7 +89,7 @@ describe('Develop Greenwood With: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.equal('text/javascript'); + expect(response.headers['content-type']).to.equal('text/javascript'); done(); }); diff --git a/packages/plugin-import-json/test/cases/develop.default/src/main.json b/packages/plugin-import-json/test/cases/develop.default/src/main.json index ba21e8844..ea0ea4f9a 100644 --- a/packages/plugin-import-json/test/cases/develop.default/src/main.json +++ b/packages/plugin-import-json/test/cases/develop.default/src/main.json @@ -1,4 +1 @@ -{ - "status": 200, - "message": "got json" -} \ No newline at end of file +{"status":200,"message":"got json"} \ No newline at end of file diff --git a/packages/plugin-include-html/src/index.js b/packages/plugin-include-html/src/index.js index c85cc5e32..f9c81a218 100644 --- a/packages/plugin-include-html/src/index.js +++ b/packages/plugin-include-html/src/index.js @@ -22,7 +22,8 @@ class IncludeHtmlResource extends ResourceInterface { for (const link of htmlIncludeLinks) { const href = link.match(/href="(.*)"/)[1]; - const includeContents = await fs.readFile(new URL(href, this.compilation.context.userWorkspace), 'utf-8'); + const prefix = href.startsWith('/') ? '.' : ''; + const includeContents = await fs.promises.readFile(new URL(`${prefix}${href}`, this.compilation.context.userWorkspace), 'utf-8'); body = body.replace(link, includeContents); } diff --git a/packages/plugin-include-html/test/cases/build.default-link-tag/build.default.link-tag.spec.js b/packages/plugin-include-html/test/cases/build.default-link-tag/build.default.link-tag.spec.js index b78b66b6f..68392a49b 100644 --- a/packages/plugin-include-html/test/cases/build.default-link-tag/build.default.link-tag.spec.js +++ b/packages/plugin-include-html/test/cases/build.default-link-tag/build.default.link-tag.spec.js @@ -36,7 +36,7 @@ import { fileURLToPath, URL } from 'url'; const expect = chai.expect; describe('Build Greenwood With HTML Include Plugin: ', function() { - const LABEL = 'Using Custom Element feature'; + const LABEL = 'Using Link Tag feature'; const cliPath = path.join(process.cwd(), 'packages/cli/src/index.js'); const outputPath = fileURLToPath(new URL('.', import.meta.url)); let runner; diff --git a/packages/plugin-postcss/src/index.js b/packages/plugin-postcss/src/index.js index 921e317dd..4cee5635b 100644 --- a/packages/plugin-postcss/src/index.js +++ b/packages/plugin-postcss/src/index.js @@ -10,13 +10,13 @@ import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js' async function getConfig (compilation, extendConfig = false) { const { projectDirectory } = compilation.context; const configFile = 'postcss.config'; - const defaultConfig = (await import(new URL(`${configFile}.js`, import.meta.url))).default; + const defaultConfig = (await import(new URL(`./${configFile}.js`, import.meta.url))).default; const userConfig = fs.existsSync(new URL(`./${configFile}.mjs`, projectDirectory).pathname) ? (await import(new URL(`./${configFile}.mjs`, projectDirectory))).default : {}; let finalConfig = Object.assign({}, userConfig); - if (userConfig && extendConfig) { + if (userConfig && extendConfig) { finalConfig.plugins = Array.isArray(userConfig.plugins) ? [...defaultConfig.plugins, ...userConfig.plugins] : [...defaultConfig.plugins]; @@ -43,7 +43,7 @@ class PostCssResource extends ResourceInterface { const css = plugins.length > 0 ? (await postcss(plugins).process(body, { from: url.pathname })).css : body; - + // TODO avoid having to rebuild response each time? return new Response(css, { headers: response.headers From 79969f13191f86227add68fb553db4523d02fd51 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Sat, 14 Jan 2023 20:25:41 -0500 Subject: [PATCH 25/50] getting package plugin specs passing --- packages/cli/src/lifecycles/bundle.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/lifecycles/bundle.js b/packages/cli/src/lifecycles/bundle.js index 9994b17db..904419bfb 100644 --- a/packages/cli/src/lifecycles/bundle.js +++ b/packages/cli/src/lifecycles/bundle.js @@ -107,7 +107,7 @@ async function bundleStyleResources(compilation, resourcePlugins) { const thisResponse = await plugin.intercept(url, request.clone(), intermediateResponse.clone()); if (thisResponse.headers.get('Content-Type').indexOf(contentType) >= 0) { - return Promise.resolve(intermediateResponse.clone()); + return Promise.resolve(thisResponse.clone()); } } From fc56315b1ee36919da4adda01676977e0b62dca7 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Mon, 16 Jan 2023 10:25:03 -0500 Subject: [PATCH 26/50] all plugin specs passing --- packages/cli/src/config/rollup.config.js | 13 +++++--- packages/plugin-babel/src/index.js | 34 ++++++++------------ packages/plugin-import-commonjs/src/index.js | 31 +++++++----------- 3 files changed, 34 insertions(+), 44 deletions(-) diff --git a/packages/cli/src/config/rollup.config.js b/packages/cli/src/config/rollup.config.js index 50eb91330..c56dc3476 100644 --- a/packages/cli/src/config/rollup.config.js +++ b/packages/cli/src/config/rollup.config.js @@ -10,22 +10,25 @@ function greenwoodResourceLoader (compilation) { return { name: 'greenwood-resource-loader', resolveId(id) { + const normalizedId = id.replace(/\?type=(.*)/, ''); const { userWorkspace } = compilation.context; - if ((id.indexOf('./') === 0 || id.indexOf('/') === 0) && fs.existsSync(new URL(`./${id.replace(/\?type=(.*)/, '')}`, userWorkspace).pathname)) { - return new URL(`./${id.replace(/\?type=(.*)/, '')}`, userWorkspace).pathname; + if ((id.indexOf('./') === 0 || id.indexOf('/') === 0) && fs.existsSync(new URL(`./${normalizedId}`, userWorkspace).pathname)) { + return new URL(`./${normalizedId}`, userWorkspace).pathname; } return null; }, async load(id) { - const url = new URL(`file://${id}`); - const extension = url.pathname.split('.').pop(); + const pathname = id.indexOf('?') >= 0 ? id.slice(0, id.indexOf('?')) : id; + const extension = pathname.split('.').pop(); - if (extension !== 'js') { + if (extension !== '' && extension !== 'js') { + const url = new URL(`file://${pathname}`); const request = new Request(url.href); let response = new Response(''); + // TODO should this use the reduce pattern too? for (const plugin of resourcePlugins) { if (plugin.shouldServe && await plugin.shouldServe(url, request)) { response = await plugin.serve(url, request); diff --git a/packages/plugin-babel/src/index.js b/packages/plugin-babel/src/index.js index 364f9c051..4a6c0b90a 100644 --- a/packages/plugin-babel/src/index.js +++ b/packages/plugin-babel/src/index.js @@ -5,24 +5,23 @@ */ import babel from '@babel/core'; import fs from 'fs'; -import path from 'path'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; import rollupBabelPlugin from '@rollup/plugin-babel'; -async function getConfig (compilation, extendConfig = false) { +async function getConfig(compilation, extendConfig = false) { const { projectDirectory } = compilation.context; const configFile = 'babel.config.mjs'; - const defaultConfig = (await import(new URL(configFile, import.meta.url).pathname)).default; - const userConfig = fs.existsSync(path.join(projectDirectory, configFile)) + const defaultConfig = (await import(new URL(`./${configFile}`, import.meta.url))).default; + const userConfig = fs.existsSync(new URL(`./${configFile}`, projectDirectory).pathname) ? (await import(`${projectDirectory}/${configFile}`)).default : {}; let finalConfig = Object.assign({}, userConfig); - + if (extendConfig) { finalConfig.presets = Array.isArray(userConfig.presets) ? [...defaultConfig.presets, ...userConfig.presets] : [...defaultConfig.presets]; - + finalConfig.plugins = Array.isArray(userConfig.plugins) ? [...defaultConfig.plugins, ...userConfig.plugins] : [...defaultConfig.plugins]; @@ -34,26 +33,21 @@ async function getConfig (compilation, extendConfig = false) { class BabelResource extends ResourceInterface { constructor(compilation, options) { super(compilation, options); - this.extensions = ['.js']; + this.extensions = ['js']; this.contentType = ['text/javascript']; } async shouldIntercept(url) { - return Promise.resolve(path.extname(url) === this.extensions[0] && url.indexOf('node_modules/') < 0); + return url.pathname.split('.').pop() === this.extensions[0] && !url.pathname.startsWith('/node_modules/'); } - async intercept(url, body) { - return new Promise(async(resolve, reject) => { - try { - const config = await getConfig(this.compilation, this.options.extendConfig); - const result = await babel.transform(body, config); - - resolve({ - body: result.code - }); - } catch (e) { - reject(e); - } + async intercept(url, request, response) { + const config = await getConfig(this.compilation, this.options.extendConfig); + const body = await response.text(); + const result = await babel.transform(body, config); + + return new Response(result.code, { + headers: response.headers }); } } diff --git a/packages/plugin-import-commonjs/src/index.js b/packages/plugin-import-commonjs/src/index.js index d1506296f..d7610e5a3 100644 --- a/packages/plugin-import-commonjs/src/index.js +++ b/packages/plugin-import-commonjs/src/index.js @@ -5,7 +5,6 @@ */ import commonjs from '@rollup/plugin-commonjs'; import fs from 'fs'; -import path from 'path'; import { parse, init } from 'cjs-module-lexer'; import { ResourceInterface } from '@greenwood/cli/src/lib/resource-interface.js'; import rollupStream from '@rollup/stream'; @@ -13,9 +12,10 @@ import rollupStream from '@rollup/stream'; // bit of a workaround for now, but maybe this could be supported by cjs-module-lexar natively? // https://github.com/guybedford/cjs-module-lexer/issues/35 const testForCjsModule = async(url) => { + const { pathname } = url; let isCommonJs = false; - if (path.extname(url) === '.js' && (/node_modules/).test(url) && url.indexOf('es-module-shims.js') < 0) { + if (pathname.split('.').pop() === '.js' && pathname.startsWith('/node_modules/') && pathname.indexOf('es-module-shims.js') < 0) { try { await init(); const body = await fs.promises.readFile(url, 'utf-8'); @@ -26,7 +26,7 @@ const testForCjsModule = async(url) => { const { message } = e; const isProbablyLexarErrorSoIgnore = message.indexOf('Unexpected import statement in CJS module.') >= 0 || message.indexOf('Unexpected export statement in CJS module.') >= 0; - + if (!isProbablyLexarErrorSoIgnore) { // we probably _shouldn't_ ignore this, so let's log it since we don't want to swallow all errors console.error(e); @@ -34,7 +34,7 @@ const testForCjsModule = async(url) => { } } - return Promise.resolve(isCommonJs); + return isCommonJs; }; class ImportCommonJsResource extends ResourceInterface { @@ -43,23 +43,16 @@ class ImportCommonJsResource extends ResourceInterface { } async shouldIntercept(url) { - return new Promise(async (resolve, reject) => { - try { - const isCommonJs = await testForCjsModule(url); - - return resolve(isCommonJs); - } catch (e) { - console.error(e); - reject(e); - } - }); + return await testForCjsModule(url); } async intercept(url) { + const { pathname } = url; + return new Promise(async(resolve, reject) => { try { const options = { - input: url, + input: pathname, output: { format: 'esm' }, plugins: [ commonjs() @@ -70,10 +63,10 @@ class ImportCommonJsResource extends ResourceInterface { stream.on('data', (data) => (bundle += data)); stream.on('end', () => { - console.debug(`proccessed module "${url}" as a CommonJS module type.`); - resolve({ - body: bundle - }); + console.debug(`processed module "${pathname}" as a CommonJS module type.`); + resolve(new Response(bundle, { + headers: response.headers + })); }); } catch (e) { reject(e); From 0c76736af2fa354c3e833133ae0d360b856358a9 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Mon, 16 Jan 2023 10:25:51 -0500 Subject: [PATCH 27/50] update spec for content type --- .../init/test/cases/develop.default/develop.default.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/init/test/cases/develop.default/develop.default.spec.js b/packages/init/test/cases/develop.default/develop.default.spec.js index 866f2281d..9cf7ecc51 100644 --- a/packages/init/test/cases/develop.default/develop.default.spec.js +++ b/packages/init/test/cases/develop.default/develop.default.spec.js @@ -113,7 +113,7 @@ xdescribe('Scaffold Greenwood and Run Develop command: ', function() { }); it('should return the correct content type', function(done) { - expect(response.headers['Content-Type']).to.contain('text/html'); + expect(response.headers['content-type']).to.contain('text/html'); done(); }); From 0e265382c62f6964c019a70b3094c5a668f5fc3b Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 17 Jan 2023 20:40:46 -0500 Subject: [PATCH 28/50] exp CSS import specs passing --- packages/cli/src/loader.js | 122 ++++++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 30 deletions(-) diff --git a/packages/cli/src/loader.js b/packages/cli/src/loader.js index dc1f1c1e1..69d9870ea 100644 --- a/packages/cli/src/loader.js +++ b/packages/cli/src/loader.js @@ -1,23 +1,72 @@ -import path from 'path'; +import fs from 'fs/promises'; import { readAndMergeConfig as initConfig } from './lifecycles/config.js'; -import { URL, fileURLToPath } from 'url'; const config = await initConfig(); -const plugins = config.plugins.filter(plugin => plugin.type === 'resource' && !plugin.isGreenwoodDefaultPlugin).map(plugin => plugin.provider({ +const resourcePlugins = config.plugins.filter(plugin => plugin.type === 'resource' && !plugin.isGreenwoodDefaultPlugin).map(plugin => plugin.provider({ context: { - projectDirectory: process.cwd() + projectDirectory: new URL(`file://${process.cwd()}`) } })); -function getCustomLoaderPlugins(url, body, headers) { - return plugins.filter(plugin => plugin.extensions.includes(path.extname(url)) && (plugin.shouldServe(url, body, headers) || plugin.shouldIntercept(url, body, headers))); +async function getCustomLoaderResponse(url, body = '', checkOnly = false) { + console.debug('getCustomLoaderResponse', { url, body, checkOnly }); + const headers = new Headers({ + 'Content-Type': 'text/javascript' + }); + const request = new Request(url.href, { headers }); + const initResponse = new Response(body, { headers }); + let response = initResponse; // new Response(body); + let shouldHandle = false; + + // TODO should this use the reduce pattern too? + for (const plugin of resourcePlugins) { + if (plugin.shouldServe && await plugin.shouldServe(url, request.clone())) { + shouldHandle = true; + + if (!checkOnly) { + response = await plugin.serve(url, request.clone()); + } + } + } + + for (const plugin of resourcePlugins) { + if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request.clone(), response.clone())) { + shouldHandle = true; + + if (!checkOnly) { + response = await plugin.intercept(url, request.clone(), response.clone()); + } + } + } + // let response = await resourcePlugins.reduce(async (responsePromise, plugin) => { + // return plugin.shouldServe && await plugin.shouldServe(url, request.clone()) + // ? Promise.resolve(await plugin.serve(url, request.clone())) + // : Promise.resolve(await responsePromise); + // }, Promise.resolve(initResponse.clone())); + + // response = await resourcePlugins.reduce(async (responsePromise, plugin) => { + // const intermediateResponse = await responsePromise; + // return plugin.shouldIntercept && await plugin.shouldIntercept(url, request.clone(), intermediateResponse.clone()) + // ? Promise.resolve(await plugin.intercept(url, request.clone(), await intermediateResponse.clone())) + // : Promise.resolve(responsePromise); + // }, Promise.resolve(initResponse.clone())); + + return { + shouldHandle, + response + }; } // https://nodejs.org/docs/latest-v18.x/api/esm.html#resolvespecifier-context-nextresolve -export function resolve(specifier, context, defaultResolve) { +export async function resolve(specifier, context, defaultResolve) { + console.log('my resolve', { specifier }); const { baseURL } = context; - if (getCustomLoaderPlugins(specifier).length > 0) { + const { shouldHandle } = await getCustomLoaderResponse(new URL(specifier), null, true); + + console.debug('resolve shouldHandle????', { specifier, shouldHandle }); + if (shouldHandle) { + console.log('handlign!!!!!!!@@@@@', { specifier }); return { url: new URL(specifier, baseURL).href, shortCircuit: true @@ -29,35 +78,48 @@ export function resolve(specifier, context, defaultResolve) { // https://nodejs.org/docs/latest-v18.x/api/esm.html#loadurl-context-nextload export async function load(source, context, defaultLoad) { - const resourcePlugins = getCustomLoaderPlugins(source); - const extension = path.extname(source).replace('.', ''); - - if (resourcePlugins.length) { - const headers = { - request: { - originalUrl: `${source}?type=${extension}`, - accept: '' - } - }; - let contents = ''; + console.debug('my load', { source, context }); + const extension = source.split('.').pop(); + const { shouldHandle } = await getCustomLoaderResponse(new URL('', `${source}?type=${extension}`), null, true); - for (const plugin of resourcePlugins) { - if (await plugin.shouldServe(source, headers)) { - contents = (await plugin.serve(source, headers)).body || contents; - } - } + console.debug({ shouldHandle }); - for (const plugin of resourcePlugins) { - if (await plugin.shouldIntercept(fileURLToPath(source), contents, headers)) { - contents = (await plugin.intercept(fileURLToPath(source), contents, headers)).body || contents; - } - } + if (shouldHandle) { + console.log('we have a hit for !!!!!', { source }); + const contents = await fs.readFile(new URL(source), 'utf-8'); + console.debug('what goes in???????', { contents }); + const { response } = await getCustomLoaderResponse(new URL('', `${source}?type=${extension}`), contents); + console.debug({ response }); + const body = await response.text(); + + console.debug('must come out!!!?????????', { body }); + + // const headers = { + // request: { + // originalUrl: `${source}?type=${extension}`, + // accept: '' + // } + // }; + // let contents = ''; + + // // TODO should this use the reduce pattern too? + // for (const plugin of resourcePlugins) { + // if (await plugin.shouldServe(source, headers)) { + // contents = (await plugin.serve(source, headers)).body || contents; + // } + // } + + // for (const plugin of resourcePlugins) { + // if (await plugin.shouldIntercept(fileURLToPath(source), contents, headers)) { + // contents = (await plugin.intercept(fileURLToPath(source), contents, headers)).body || contents; + // } + // } // TODO better way to handle remove export default? // https://github.com/ProjectEvergreen/greenwood/issues/948 return { format: extension === 'json' ? 'json' : 'module', - source: extension === 'json' ? JSON.parse(contents.replace('export default ', '')) : contents, + source: extension === 'json' ? JSON.parse(body.replace('export default ', '')) : body, shortCircuit: true }; } From c9f4ad02a16f44f6885dec84ab8fb2dc714d829f Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 17 Jan 2023 21:30:12 -0500 Subject: [PATCH 29/50] exp JSON import specs passing --- packages/cli/src/loader.js | 46 ++++-------------------- packages/plugin-import-json/src/index.js | 7 ++-- 2 files changed, 11 insertions(+), 42 deletions(-) diff --git a/packages/cli/src/loader.js b/packages/cli/src/loader.js index 69d9870ea..0c2c26183 100644 --- a/packages/cli/src/loader.js +++ b/packages/cli/src/loader.js @@ -38,18 +38,6 @@ async function getCustomLoaderResponse(url, body = '', checkOnly = false) { } } } - // let response = await resourcePlugins.reduce(async (responsePromise, plugin) => { - // return plugin.shouldServe && await plugin.shouldServe(url, request.clone()) - // ? Promise.resolve(await plugin.serve(url, request.clone())) - // : Promise.resolve(await responsePromise); - // }, Promise.resolve(initResponse.clone())); - - // response = await resourcePlugins.reduce(async (responsePromise, plugin) => { - // const intermediateResponse = await responsePromise; - // return plugin.shouldIntercept && await plugin.shouldIntercept(url, request.clone(), intermediateResponse.clone()) - // ? Promise.resolve(await plugin.intercept(url, request.clone(), await intermediateResponse.clone())) - // : Promise.resolve(responsePromise); - // }, Promise.resolve(initResponse.clone())); return { shouldHandle, @@ -80,46 +68,24 @@ export async function resolve(specifier, context, defaultResolve) { export async function load(source, context, defaultLoad) { console.debug('my load', { source, context }); const extension = source.split('.').pop(); - const { shouldHandle } = await getCustomLoaderResponse(new URL('', `${source}?type=${extension}`), null, true); + const url = new URL('', `${source}?type=${extension}`); + const { shouldHandle } = await getCustomLoaderResponse(url, null, true); - console.debug({ shouldHandle }); + console.debug({ url, shouldHandle, extension }); if (shouldHandle) { console.log('we have a hit for !!!!!', { source }); const contents = await fs.readFile(new URL(source), 'utf-8'); console.debug('what goes in???????', { contents }); - const { response } = await getCustomLoaderResponse(new URL('', `${source}?type=${extension}`), contents); - console.debug({ response }); + const { response } = await getCustomLoaderResponse(url, contents); + console.debug('$$$$$', { response }); const body = await response.text(); - console.debug('must come out!!!?????????', { body }); - - // const headers = { - // request: { - // originalUrl: `${source}?type=${extension}`, - // accept: '' - // } - // }; - // let contents = ''; - - // // TODO should this use the reduce pattern too? - // for (const plugin of resourcePlugins) { - // if (await plugin.shouldServe(source, headers)) { - // contents = (await plugin.serve(source, headers)).body || contents; - // } - // } - - // for (const plugin of resourcePlugins) { - // if (await plugin.shouldIntercept(fileURLToPath(source), contents, headers)) { - // contents = (await plugin.intercept(fileURLToPath(source), contents, headers)).body || contents; - // } - // } - // TODO better way to handle remove export default? // https://github.com/ProjectEvergreen/greenwood/issues/948 return { format: extension === 'json' ? 'json' : 'module', - source: extension === 'json' ? JSON.parse(body.replace('export default ', '')) : body, + source: extension === 'json' ? JSON.stringify(JSON.parse(contents.replace('export default ', ''))) : body, shortCircuit: true }; } diff --git a/packages/plugin-import-json/src/index.js b/packages/plugin-import-json/src/index.js index 35ea7afee..efffdbeba 100644 --- a/packages/plugin-import-json/src/index.js +++ b/packages/plugin-import-json/src/index.js @@ -22,15 +22,18 @@ class ImportJsonResource extends ResourceInterface { } async intercept(url, request, response) { + console.log('JSON intercept!?!?!?!', { url }); // TODO better way to handle this? // https://github.com/ProjectEvergreen/greenwood/issues/948 - const body = await response.text(); + const json = await response.json(); + const body = `export default ${JSON.stringify(json)}`; // TODO need to support an empty body to read from disc? // const json = body === '' // ? await fs.promises.readFile(url, 'utf-8') // : body; + console.log('JSON return', { body }); - return new Response(`export default ${body}`, { + return new Response(body, { headers: new Headers({ 'Content-Type': this.contentType }) From af739ff2c95779458038b7bf0bb800a93bc6b66c Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 17 Jan 2023 21:38:38 -0500 Subject: [PATCH 30/50] clean up console logs --- packages/cli/src/loader.js | 7 ------- packages/plugin-import-json/src/index.js | 8 -------- 2 files changed, 15 deletions(-) diff --git a/packages/cli/src/loader.js b/packages/cli/src/loader.js index 0c2c26183..0e1c129b2 100644 --- a/packages/cli/src/loader.js +++ b/packages/cli/src/loader.js @@ -9,7 +9,6 @@ const resourcePlugins = config.plugins.filter(plugin => plugin.type === 'resourc })); async function getCustomLoaderResponse(url, body = '', checkOnly = false) { - console.debug('getCustomLoaderResponse', { url, body, checkOnly }); const headers = new Headers({ 'Content-Type': 'text/javascript' }); @@ -66,19 +65,13 @@ export async function resolve(specifier, context, defaultResolve) { // https://nodejs.org/docs/latest-v18.x/api/esm.html#loadurl-context-nextload export async function load(source, context, defaultLoad) { - console.debug('my load', { source, context }); const extension = source.split('.').pop(); const url = new URL('', `${source}?type=${extension}`); const { shouldHandle } = await getCustomLoaderResponse(url, null, true); - console.debug({ url, shouldHandle, extension }); - if (shouldHandle) { - console.log('we have a hit for !!!!!', { source }); const contents = await fs.readFile(new URL(source), 'utf-8'); - console.debug('what goes in???????', { contents }); const { response } = await getCustomLoaderResponse(url, contents); - console.debug('$$$$$', { response }); const body = await response.text(); // TODO better way to handle remove export default? diff --git a/packages/plugin-import-json/src/index.js b/packages/plugin-import-json/src/index.js index efffdbeba..5f2780f65 100644 --- a/packages/plugin-import-json/src/index.js +++ b/packages/plugin-import-json/src/index.js @@ -22,16 +22,8 @@ class ImportJsonResource extends ResourceInterface { } async intercept(url, request, response) { - console.log('JSON intercept!?!?!?!', { url }); - // TODO better way to handle this? - // https://github.com/ProjectEvergreen/greenwood/issues/948 const json = await response.json(); const body = `export default ${JSON.stringify(json)}`; - // TODO need to support an empty body to read from disc? - // const json = body === '' - // ? await fs.promises.readFile(url, 'utf-8') - // : body; - console.log('JSON return', { body }); return new Response(body, { headers: new Headers({ From e4f3f496e9e1a4130b94288fa670e801e8227373 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Wed, 18 Jan 2023 17:04:45 -0500 Subject: [PATCH 31/50] resolve max listeners warning --- packages/cli/src/lifecycles/bundle.js | 2 +- packages/cli/src/lifecycles/serve.js | 16 ++++++++-------- packages/cli/src/loader.js | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/lifecycles/bundle.js b/packages/cli/src/lifecycles/bundle.js index 904419bfb..ea4f8dc56 100644 --- a/packages/cli/src/lifecycles/bundle.js +++ b/packages/cli/src/lifecycles/bundle.js @@ -104,7 +104,7 @@ async function bundleStyleResources(compilation, resourcePlugins) { const shouldIntercept = plugin.shouldIntercept && await plugin.shouldIntercept(url, request, intermediateResponse.clone()); if (shouldIntercept) { - const thisResponse = await plugin.intercept(url, request.clone(), intermediateResponse.clone()); + const thisResponse = await plugin.intercept(url, request, intermediateResponse.clone()); if (thisResponse.headers.get('Content-Type').indexOf(contentType) >= 0) { return Promise.resolve(thisResponse.clone()); diff --git a/packages/cli/src/lifecycles/serve.js b/packages/cli/src/lifecycles/serve.js index 8f56fd3cd..10fa0f28f 100644 --- a/packages/cli/src/lifecycles/serve.js +++ b/packages/cli/src/lifecycles/serve.js @@ -62,8 +62,8 @@ async function getDevServer(compilation) { const initResponse = new Response(null, { status }); const request = new Request(url.href, { method, headers: header }); const response = await resourcePlugins.reduce(async (responsePromise, plugin) => { - return plugin.shouldServe && await plugin.shouldServe(url, request.clone()) - ? Promise.resolve(await plugin.serve(url, request.clone())) + return plugin.shouldServe && await plugin.shouldServe(url, request) + ? Promise.resolve(await plugin.serve(url, request)) : Promise.resolve(await responsePromise); }, Promise.resolve(initResponse.clone())); @@ -94,8 +94,8 @@ async function getDevServer(compilation) { }); const response = await resourcePlugins.reduce(async (responsePromise, plugin) => { const intermediateResponse = await responsePromise; - return plugin.shouldIntercept && await plugin.shouldIntercept(url, request.clone(), intermediateResponse.clone()) - ? Promise.resolve(await plugin.intercept(url, request.clone(), await intermediateResponse.clone())) + return plugin.shouldIntercept && await plugin.shouldIntercept(url, request, intermediateResponse.clone()) + ? Promise.resolve(await plugin.intercept(url, request, await intermediateResponse.clone())) : Promise.resolve(responsePromise); }, Promise.resolve(initResponse.clone())); @@ -180,8 +180,8 @@ async function getStaticServer(compilation, composable) { .find((plugin) => plugin.name === 'plugin-dev-proxy') .provider(compilation); - if (await proxyPlugin.shouldServe(url, request.clone())) { - const response = await proxyPlugin.serve(url, request.clone()); + if (await proxyPlugin.shouldServe(url, request)) { + const response = await proxyPlugin.serve(url, request); ctx.body = Readable.from(response.body); ctx.set('Content-Type', response.headers.get('Content-Type')); @@ -202,8 +202,8 @@ async function getStaticServer(compilation, composable) { headers: new Headers(ctx.response.header) }); const response = await resourcePlugins.reduce(async (responsePromise, plugin) => { - return plugin.shouldServe && await plugin.shouldServe(url, request.clone()) - ? Promise.resolve(await plugin.serve(url, request.clone())) + return plugin.shouldServe && await plugin.shouldServe(url, request) + ? Promise.resolve(await plugin.serve(url, request)) : responsePromise; }, Promise.resolve(initResponse)); diff --git a/packages/cli/src/loader.js b/packages/cli/src/loader.js index 0e1c129b2..5c34129d0 100644 --- a/packages/cli/src/loader.js +++ b/packages/cli/src/loader.js @@ -19,21 +19,21 @@ async function getCustomLoaderResponse(url, body = '', checkOnly = false) { // TODO should this use the reduce pattern too? for (const plugin of resourcePlugins) { - if (plugin.shouldServe && await plugin.shouldServe(url, request.clone())) { + if (plugin.shouldServe && await plugin.shouldServe(url, request)) { shouldHandle = true; if (!checkOnly) { - response = await plugin.serve(url, request.clone()); + response = await plugin.serve(url, request); } } } for (const plugin of resourcePlugins) { - if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request.clone(), response.clone())) { + if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request, response.clone())) { shouldHandle = true; if (!checkOnly) { - response = await plugin.intercept(url, request.clone(), response.clone()); + response = await plugin.intercept(url, request, response.clone()); } } } From ceaf66dd67d7e3bafe03ead5ab367aa053ace141 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Thu, 19 Jan 2023 08:56:57 -0500 Subject: [PATCH 32/50] restore E-Tag middleware for development --- packages/cli/src/lifecycles/serve.js | 59 ++++++++++++++-------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/packages/cli/src/lifecycles/serve.js b/packages/cli/src/lifecycles/serve.js index 10fa0f28f..0bfc3fa6e 100644 --- a/packages/cli/src/lifecycles/serve.js +++ b/packages/cli/src/lifecycles/serve.js @@ -1,5 +1,5 @@ import fs from 'fs'; -// import { hashString } from '../lib/hashing-utils.js'; +import { hashString } from '../lib/hashing-utils.js'; import Koa from 'koa'; import { Readable } from 'stream'; import { ResourceInterface } from '../lib/resource-interface.js'; @@ -113,35 +113,34 @@ async function getDevServer(compilation) { // ETag Support - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag // https://stackoverflow.com/questions/43659756/chrome-ignores-the-etag-header-and-just-uses-the-in-memory-cache-disk-cache - // app.use(async (ctx) => { - // const body = ctx.response.body; - // const { url } = ctx; - - // // don't interfere with external requests or API calls, binary files, or JSON - // // and only run in development - // if (process.env.__GWD_COMMAND__ === 'develop' && path.extname(url) !== '' && url.indexOf('http') !== 0) { // eslint-disable-line no-underscore-dangle - // if (!body || Buffer.isBuffer(body)) { - // // console.warn(`no body for => ${ctx.url}`); - // } else { - // const inm = ctx.headers['if-none-match']; - // const etagHash = path.extname(ctx.request.headers.originalUrl) === '.json' - // ? hashString(JSON.stringify(body)) - // : hashString(body); - - // if (inm && inm === etagHash) { - // ctx.status = 304; - // ctx.body = null; - // ctx.set('Etag', etagHash); - // ctx.set('Cache-Control', 'no-cache'); - // } else if (!inm || inm !== etagHash) { - // ctx.set('Etag', etagHash); - // } - // } - // } - - // }); - - return Promise.resolve(app); + app.use(async (ctx) => { + const body = ctx.response.body; + const url = new URL(ctx.url); + + // don't interfere with external requests or API calls, binary files, or JSON + // and only run in development + if (process.env.__GWD_COMMAND__ === 'develop' && url.protocol === 'file:') { // eslint-disable-line no-underscore-dangle + if (!body || Buffer.isBuffer(body)) { + // console.warn(`no body for => ${ctx.url}`); + } else { + const inm = ctx.headers['if-none-match']; + const etagHash = url.pathname.split('.').pop() === 'json' + ? hashString(JSON.stringify(body)) + : hashString(body); + + if (inm && inm === etagHash) { + ctx.status = 304; + ctx.body = null; + ctx.set('Etag', etagHash); + ctx.set('Cache-Control', 'no-cache'); + } else if (!inm || inm !== etagHash) { + ctx.set('Etag', etagHash); + } + } + } + }); + + return app; } async function getStaticServer(compilation, composable) { From 6fba001401c4e03b69730fba992cbfc1dea8eea0 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Tue, 24 Jan 2023 09:10:48 -0500 Subject: [PATCH 33/50] merged response bundling working --- packages/cli/src/config/rollup.config.js | 4 +-- packages/cli/src/lib/resource-utils.js | 24 ++++++++++++-- packages/cli/src/lifecycles/bundle.js | 10 +++--- packages/cli/src/lifecycles/prerender.js | 8 ++--- packages/cli/src/lifecycles/serve.js | 32 ++++++++++++------- .../plugins/resource/plugin-node-modules.js | 5 +-- .../plugins/resource/plugin-standard-html.js | 1 - .../plugins/resource/plugin-standard-image.js | 1 - .../plugins/resource/plugin-static-router.js | 14 +++----- .../src/plugins/server/plugin-livereload.js | 5 +-- packages/plugin-graphql/src/index.js | 17 +++------- packages/plugin-import-css/src/index.js | 3 +- packages/plugin-polyfills/src/index.js | 4 +-- packages/plugin-postcss/src/index.js | 5 +-- .../src/plugins/resource.js | 4 +-- 15 files changed, 69 insertions(+), 68 deletions(-) diff --git a/packages/cli/src/config/rollup.config.js b/packages/cli/src/config/rollup.config.js index c56dc3476..78c657132 100644 --- a/packages/cli/src/config/rollup.config.js +++ b/packages/cli/src/config/rollup.config.js @@ -36,8 +36,8 @@ function greenwoodResourceLoader (compilation) { } for (const plugin of resourcePlugins) { - if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request, response)) { - response = await plugin.intercept(url, request, response); + if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request, response.clone())) { + response = await plugin.intercept(url, request, response.clone()); } } diff --git a/packages/cli/src/lib/resource-utils.js b/packages/cli/src/lib/resource-utils.js index 5752a2258..daddb0c9e 100644 --- a/packages/cli/src/lib/resource-utils.js +++ b/packages/cli/src/lib/resource-utils.js @@ -1,7 +1,6 @@ import fs from 'fs'; import { hashString } from '../lib/hashing-utils.js'; -// TODO could make async? function modelResource(context, type, src = undefined, contents = undefined, optimizationAttr = undefined, rawAttributes = undefined) { const { projectDirectory, scratchDir, userWorkspace } = context; const extension = type === 'script' ? 'js' : 'css'; @@ -49,4 +48,25 @@ function modelResource(context, type, src = undefined, contents = undefined, opt }; } -export { modelResource }; \ No newline at end of file +function mergeResponse(destination, source) { + const headers = destination.headers || new Headers(); + + source.headers.forEach((value, key) => { + // TODO better way to handle Response automatically setting content-type + const isDefaultHeader = key.toLowerCase() === 'content-type' && value === 'text/plain;charset=UTF-8'; + + if (!isDefaultHeader) { + headers.set(key, value); + } + }); + + // TODO handle merging in state (aborted, type, status, etc) + return new Response(source.body, { + headers + }); +} + +export { + mergeResponse, + modelResource +}; \ No newline at end of file diff --git a/packages/cli/src/lifecycles/bundle.js b/packages/cli/src/lifecycles/bundle.js index ea4f8dc56..1ddfe001a 100644 --- a/packages/cli/src/lifecycles/bundle.js +++ b/packages/cli/src/lifecycles/bundle.js @@ -2,6 +2,7 @@ import fs from 'fs'; import { getRollupConfig } from '../config/rollup.config.js'; import { hashString } from '../lib/hashing-utils.js'; +import { mergeResponse } from '../lib/resource-utils.js'; import path from 'path'; import { rollup } from 'rollup'; @@ -67,7 +68,6 @@ async function bundleStyleResources(compilation, resourcePlugins) { let optimizedFileContents; if (src) { - // TODO remove path. usage const basename = path.basename(srcPath); const basenamePieces = path.basename(srcPath).split('.'); const fileNamePieces = srcPath.split('/').filter(piece => piece !== ''); // normalize by removing any leading /'s @@ -104,10 +104,12 @@ async function bundleStyleResources(compilation, resourcePlugins) { const shouldIntercept = plugin.shouldIntercept && await plugin.shouldIntercept(url, request, intermediateResponse.clone()); if (shouldIntercept) { - const thisResponse = await plugin.intercept(url, request, intermediateResponse.clone()); + const currentResponse = await plugin.intercept(url, request, intermediateResponse.clone()); + const mergedResponse = mergeResponse(intermediateResponse.clone(), currentResponse.clone()); - if (thisResponse.headers.get('Content-Type').indexOf(contentType) >= 0) { - return Promise.resolve(thisResponse.clone()); + // TODO better way to handle Response automatically setting content-type + if ((mergedResponse.headers.get('Content-Type') || mergedResponse.headers.get('content-type')).indexOf(contentType) >= 0) { + return Promise.resolve(mergedResponse.clone()); } } diff --git a/packages/cli/src/lifecycles/prerender.js b/packages/cli/src/lifecycles/prerender.js index 6cd26d44d..ba0729c8e 100644 --- a/packages/cli/src/lifecycles/prerender.js +++ b/packages/cli/src/lifecycles/prerender.js @@ -77,6 +77,7 @@ async function servePage(url, request, plugins) { for (const plugin of plugins) { if (plugin.shouldServe && await plugin.shouldServe(url, request)) { response = await plugin.serve(url, request); + break; } } @@ -84,10 +85,9 @@ async function servePage(url, request, plugins) { } async function interceptPage(url, request, plugins, body) { - const headers = new Headers(); - headers.append('Content-Type', 'text/html'); - - let response = new Response(body, { headers }); + let response = new Response(body, { + headers: new Headers({ 'Content-Type': 'text/html' }) + }); for (const plugin of plugins) { if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request, response)) { diff --git a/packages/cli/src/lifecycles/serve.js b/packages/cli/src/lifecycles/serve.js index 0bfc3fa6e..4275937e3 100644 --- a/packages/cli/src/lifecycles/serve.js +++ b/packages/cli/src/lifecycles/serve.js @@ -1,6 +1,7 @@ import fs from 'fs'; import { hashString } from '../lib/hashing-utils.js'; import Koa from 'koa'; +import { mergeResponse } from '../lib/resource-utils.js'; import { Readable } from 'stream'; import { ResourceInterface } from '../lib/resource-interface.js'; @@ -35,7 +36,7 @@ async function getDevServer(compilation) { const url = new URL(`http://localhost:${compilation.config.port}${ctx.url}`); const initRequest = new Request(url, { method: ctx.request.method, - headers: ctx.request.header + headers: new Headers(ctx.request.header) }); const request = await resourcePlugins.reduce(async (requestPromise, plugin) => { const intermediateRequest = await requestPromise; @@ -59,13 +60,15 @@ async function getDevServer(compilation) { const url = new URL(ctx.url); const { method, header } = ctx.request; const { status } = ctx.response; - const initResponse = new Response(null, { status }); - const request = new Request(url.href, { method, headers: header }); - const response = await resourcePlugins.reduce(async (responsePromise, plugin) => { - return plugin.shouldServe && await plugin.shouldServe(url, request) - ? Promise.resolve(await plugin.serve(url, request)) - : Promise.resolve(await responsePromise); - }, Promise.resolve(initResponse.clone())); + const request = new Request(url.href, { method, headers: new Headers(header) }); + let response = new Response(null, { status }); + + for (const plugin of resourcePlugins) { + if (plugin.shouldServe && await plugin.shouldServe(url, request)) { + response = await plugin.serve(url, request); + break; + } + } // TODO would be nice if Koa (or other framework) could just a Response object directly // not sure why we have to use `Readable.from`, does this couple us to NodeJS? @@ -86,7 +89,7 @@ async function getDevServer(compilation) { const url = new URL(ctx.url); const request = new Request(url, { method: ctx.request.method, - headers: ctx.request.header + headers: new Headers(ctx.request.header) }); const initResponse = new Response(ctx.body, { status: ctx.response.status, @@ -94,9 +97,14 @@ async function getDevServer(compilation) { }); const response = await resourcePlugins.reduce(async (responsePromise, plugin) => { const intermediateResponse = await responsePromise; - return plugin.shouldIntercept && await plugin.shouldIntercept(url, request, intermediateResponse.clone()) - ? Promise.resolve(await plugin.intercept(url, request, await intermediateResponse.clone())) - : Promise.resolve(responsePromise); + if (plugin.shouldIntercept && await plugin.shouldIntercept(url, request, intermediateResponse.clone())) { + const current = await plugin.intercept(url, request, await intermediateResponse.clone()); + const merged = mergeResponse(intermediateResponse.clone(), current); + + return Promise.resolve(merged); + } else { + return Promise.resolve(await responsePromise); + } }, Promise.resolve(initResponse.clone())); // TODO would be nice if Koa (or other framework) could just a Response object directly diff --git a/packages/cli/src/plugins/resource/plugin-node-modules.js b/packages/cli/src/plugins/resource/plugin-node-modules.js index 1ceb9b87f..b82cc3159 100644 --- a/packages/cli/src/plugins/resource/plugin-node-modules.js +++ b/packages/cli/src/plugins/resource/plugin-node-modules.js @@ -103,10 +103,7 @@ class NodeModulesResource extends ResourceInterface { `); - // TODO avoid having to rebuild response each time? - return new Response(body, { - headers: response.headers - }); + return new Response(body); } } diff --git a/packages/cli/src/plugins/resource/plugin-standard-html.js b/packages/cli/src/plugins/resource/plugin-standard-html.js index e64d2ad43..3657d3ab6 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-html.js +++ b/packages/cli/src/plugins/resource/plugin-standard-html.js @@ -378,7 +378,6 @@ class StandardHtmlResource extends ResourceInterface { `); } - // TODO avoid having to rebuild response each time? return new Response(body, { headers: new Headers({ 'Content-Type': this.contentType diff --git a/packages/cli/src/plugins/resource/plugin-standard-image.js b/packages/cli/src/plugins/resource/plugin-standard-image.js index a198dc51c..f371a2a9f 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-image.js +++ b/packages/cli/src/plugins/resource/plugin-standard-image.js @@ -27,7 +27,6 @@ class StandardFontResource extends ResourceInterface { ? 'x-icon' : type; - // TODO avoid having to rebuild response each time? return new Response(body, { headers: new Headers({ 'Content-Type': `image/${contentType}` diff --git a/packages/cli/src/plugins/resource/plugin-static-router.js b/packages/cli/src/plugins/resource/plugin-static-router.js index 6a9258b76..bc3acf9e9 100644 --- a/packages/cli/src/plugins/resource/plugin-static-router.js +++ b/packages/cli/src/plugins/resource/plugin-static-router.js @@ -28,7 +28,7 @@ class StaticRouterResource extends ResourceInterface { async shouldIntercept(url, request, response) { const { pathname, protocol } = url; - const contentType = response.headers.get['Content-Type'] || ''; + const contentType = response.headers.get('Content-Type') || ''; // TODO should this also happen during development too? return process.env.__GWD_COMMAND__ === 'build' // eslint-disable-line no-underscore-dangle @@ -45,16 +45,13 @@ class StaticRouterResource extends ResourceInterface { `); - // TODO avoid having to rebuild response each time? - return new Response(body, { - headers: response.headers - }); + return new Response(body); } async shouldOptimize(url, response) { return this.compilation.config.staticRouter && !url.pathname.startsWith('/404') - && response.headers.get('Content-Type').indexOf(this.contentType) >= 0; + && (response.headers.get('Content-Type') || response.headers.get('content-type')).indexOf(this.contentType) >= 0; } async optimize(url, response) { @@ -114,10 +111,7 @@ class StaticRouterResource extends ResourceInterface { `); - // TODO avoid having to rebuild response each time? - return new Response(body, { - headers: response.headers - }); + return new Response(body); } } diff --git a/packages/cli/src/plugins/server/plugin-livereload.js b/packages/cli/src/plugins/server/plugin-livereload.js index cee2521bf..8042260f4 100644 --- a/packages/cli/src/plugins/server/plugin-livereload.js +++ b/packages/cli/src/plugins/server/plugin-livereload.js @@ -65,10 +65,7 @@ class LiveReloadResource extends ResourceInterface { `); - // TODO avoid having to rebuild response each time? - return new Response(body, { - headers: response.headers - }); + return new Response(body); } } diff --git a/packages/plugin-graphql/src/index.js b/packages/plugin-graphql/src/index.js index 876964dfd..8c0f228e6 100644 --- a/packages/plugin-graphql/src/index.js +++ b/packages/plugin-graphql/src/index.js @@ -32,11 +32,10 @@ class GraphQLResource extends ResourceInterface { export default \`${js}\`; `; - // TODO avoid having to rebuild response each time? return new Response(body, { - headers: { + headers: new Headers({ 'Content-Type': this.contentType[0] - } + }) }); } @@ -48,14 +47,11 @@ class GraphQLResource extends ResourceInterface { const body = await response.text(); const newBody = mergeImportMap(body, importMap); - // TODO avoid having to rebuild response each time? - return new Response(newBody, { - headers: response.headers - }); + return new Response(newBody); } async shouldOptimize(url, response) { - return response.headers.get('Content-Type').indexOf(this.contentType[1]) >= 0; + return (response.headers.get('Content-Type') || response.headers.get('content-type')).indexOf(this.contentType[1]) >= 0; } async optimize(url, response) { @@ -68,10 +64,7 @@ class GraphQLResource extends ResourceInterface { `); - // TODO avoid having to rebuild response each time? - return new Response(body, { - headers: response.headers - }); + return new Response(body); } } diff --git a/packages/plugin-import-css/src/index.js b/packages/plugin-import-css/src/index.js index 0f72e878a..6619fd648 100644 --- a/packages/plugin-import-css/src/index.js +++ b/packages/plugin-import-css/src/index.js @@ -37,8 +37,7 @@ class ImportCssResource extends ResourceInterface { } async intercept(url, request, response) { - // TODO why do we need to check for body first? - const body = await response.text(); // => body || await fs.promises.readFile(pathToFileURL(url), 'utf-8'); + const body = await response.text(); const cssInJsBody = `const css = \`${body.replace(/\r?\n|\r/g, ' ').replace(/\\/g, '\\\\')}\`;\nexport default css;`; return new Response(cssInJsBody, { diff --git a/packages/plugin-polyfills/src/index.js b/packages/plugin-polyfills/src/index.js index afb08a201..e56252bd5 100644 --- a/packages/plugin-polyfills/src/index.js +++ b/packages/plugin-polyfills/src/index.js @@ -64,9 +64,7 @@ class PolyfillsResource extends ResourceInterface { `); } - return new Response(body, { - headers: response.headers - }); + return new Response(body); } } diff --git a/packages/plugin-postcss/src/index.js b/packages/plugin-postcss/src/index.js index 4cee5635b..7f952f48b 100644 --- a/packages/plugin-postcss/src/index.js +++ b/packages/plugin-postcss/src/index.js @@ -44,10 +44,7 @@ class PostCssResource extends ResourceInterface { ? (await postcss(plugins).process(body, { from: url.pathname })).css : body; - // TODO avoid having to rebuild response each time? - return new Response(css, { - headers: response.headers - }); + return new Response(css); } } diff --git a/packages/plugin-renderer-puppeteer/src/plugins/resource.js b/packages/plugin-renderer-puppeteer/src/plugins/resource.js index ade391022..509f2faaf 100644 --- a/packages/plugin-renderer-puppeteer/src/plugins/resource.js +++ b/packages/plugin-renderer-puppeteer/src/plugins/resource.js @@ -24,9 +24,7 @@ class PuppeteerResource extends ResourceInterface { `); - return new Response(body, { - headers: response.headers - }); + return new Response(body); } } From eada5e3fc17418dd99101f96bbac2f4537f5740b Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Thu, 26 Jan 2023 08:32:53 -0500 Subject: [PATCH 34/50] restore optimized graphql behavior --- packages/cli/src/lifecycles/bundle.js | 13 ++++++------- packages/plugin-graphql/src/index.js | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/lifecycles/bundle.js b/packages/cli/src/lifecycles/bundle.js index 1ddfe001a..5e50fae0d 100644 --- a/packages/cli/src/lifecycles/bundle.js +++ b/packages/cli/src/lifecycles/bundle.js @@ -29,9 +29,8 @@ async function optimizeStaticPages(compilation, plugins) { const { route, outputPath } = page; const url = new URL(`http://localhost:${compilation.config.port}${route}`); const contents = await fs.promises.readFile(new URL(`./${outputPath}`, scratchDir), 'utf-8'); - const headers = new Headers(); - - headers.append('Content-Type', 'text/html'); + const headers = new Headers({ 'Content-Type': 'text/html' }); + let response = new Response(contents, { headers }); if (route !== '/404/' && !fs.existsSync(new URL(`.${route}`, outputDir).pathname)) { fs.mkdirSync(new URL(`.${route}`, outputDir).pathname, { @@ -39,11 +38,11 @@ async function optimizeStaticPages(compilation, plugins) { }); } - let response = new Response(contents, { headers }); - for (const plugin of plugins) { - if (plugin.shouldOptimize && await plugin.shouldOptimize(url, response)) { - response = await plugin.optimize(url, response); + if (plugin.shouldOptimize && await plugin.shouldOptimize(url, response.clone())) { + const currentResponse = await plugin.optimize(url, response.clone()); + + response = mergeResponse(response.clone(), currentResponse.clone()); } } diff --git a/packages/plugin-graphql/src/index.js b/packages/plugin-graphql/src/index.js index 8c0f228e6..9bda02ef8 100644 --- a/packages/plugin-graphql/src/index.js +++ b/packages/plugin-graphql/src/index.js @@ -51,7 +51,7 @@ class GraphQLResource extends ResourceInterface { } async shouldOptimize(url, response) { - return (response.headers.get('Content-Type') || response.headers.get('content-type')).indexOf(this.contentType[1]) >= 0; + return response.headers.get('Content-Type').indexOf(this.contentType[1]) >= 0; } async optimize(url, response) { From 4441a2bbbddda399d0c2537e30320c74baeb9c05 Mon Sep 17 00:00:00 2001 From: Owen Buckley Date: Thu, 26 Jan 2023 18:12:52 -0500 Subject: [PATCH 35/50] update docs --- packages/cli/src/lib/resource-interface.js | 56 ----- .../plugins/resource/plugin-standard-css.js | 4 +- .../plugins/resource/plugin-standard-html.js | 4 +- www/pages/docs/configuration.md | 4 +- www/pages/guides/theme-packs.md | 17 +- www/pages/plugins/context.md | 6 +- www/pages/plugins/copy.md | 17 +- www/pages/plugins/index.md | 25 +- www/pages/plugins/resource.md | 233 ++++++++++-------- www/pages/plugins/rollup.md | 2 +- 10 files changed, 147 insertions(+), 221 deletions(-) diff --git a/packages/cli/src/lib/resource-interface.js b/packages/cli/src/lib/resource-interface.js index 8aa600b7a..57660c992 100644 --- a/packages/cli/src/lib/resource-interface.js +++ b/packages/cli/src/lib/resource-interface.js @@ -5,7 +5,6 @@ class ResourceInterface { this.compilation = compilation; this.options = options; this.extensions = []; - this.contentType = ''; // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types } hasExtension(url) { @@ -38,61 +37,6 @@ class ResourceInterface { return reducedUrl; } - - // test if this plugin should change a relative URL from the browser to an absolute path on disk - // like for node_modules/ resolution. not commonly needed by most resource plugins - // return true | false - // eslint-disable-next-line no-unused-vars - // async shouldResolve(url) { - // return Promise.resolve(false); - // } - - // // return an absolute path - // async resolve(url) { - // return Promise.resolve(url); - // } - - // test if this plugin should be used to process a given url / header combo the browser and retu - // ex: ` or diff --git a/packages/cli/src/lifecycles/bundle.js b/packages/cli/src/lifecycles/bundle.js index 6f685b143..c672944f7 100644 --- a/packages/cli/src/lifecycles/bundle.js +++ b/packages/cli/src/lifecycles/bundle.js @@ -1,5 +1,5 @@ /* eslint-disable max-depth */ -import fs from 'fs'; +import fs from 'fs/promises'; import { getRollupConfig } from '../config/rollup.config.js'; import { hashString } from '../lib/hashing-utils.js'; import { mergeResponse } from '../lib/resource-utils.js'; @@ -15,7 +15,7 @@ async function cleanUpResources(compilation) { const optAttr = ['inline', 'static'].indexOf(optimizationAttr) >= 0; if (optimizedFileName && (!src || (optAttr || optConfig))) { - fs.unlinkSync(new URL(`./${optimizedFileName}`, outputDir).pathname); + await fs.unlink(new URL(`./${optimizedFileName}`, outputDir).pathname); } } } @@ -28,12 +28,14 @@ async function optimizeStaticPages(compilation, plugins) { .map(async (page) => { const { route, outputPath } = page; const url = new URL(`http://localhost:${compilation.config.port}${route}`); - const contents = await fs.promises.readFile(new URL(`./${outputPath}`, scratchDir), 'utf-8'); + const contents = await fs.readFile(new URL(`./${outputPath}`, scratchDir), 'utf-8'); const headers = new Headers({ 'Content-Type': 'text/html' }); let response = new Response(contents, { headers }); - if (route !== '/404/' && !fs.existsSync(new URL(`.${route}`, outputDir).pathname)) { - fs.mkdirSync(new URL(`.${route}`, outputDir).pathname, { + try { + await fs.access(new URL(`.${route}`, outputDir)) + } catch (error) { + await fs.mkdir(new URL(`.${route}`, outputDir), { recursive: true }); } @@ -49,7 +51,7 @@ async function optimizeStaticPages(compilation, plugins) { // clean up optimization markers const body = (await response.text()).replace(/data-gwd-opt=".*[a-z]"/g, ''); - await fs.promises.writeFile(new URL(`./${outputPath}`, outputDir), body); + await fs.writeFile(new URL(`./${outputPath}`, outputDir), body); }) ); } @@ -83,8 +85,11 @@ async function bundleStyleResources(compilation, resourcePlugins) { .slice(0, -1) .join('/'); - if (!fs.existsSync(outputPathRoot)) { - fs.mkdirSync(outputPathRoot, { + console.debug('???', new URL(`file://${outputPathRoot}`)) + try { + fs.access(new URL(`file://${outputPathRoot}`)); + } catch (error) { + fs.mkdir(new URL(`file://${outputPathRoot}`), { recursive: true }); } @@ -132,7 +137,7 @@ async function bundleStyleResources(compilation, resourcePlugins) { optimizedFileContents }); - await fs.promises.writeFile(new URL(`./${optimizedFileName}`, outputDir), optimizedFileContents); + await fs.writeFile(new URL(`./${optimizedFileName}`, outputDir), optimizedFileContents); } } } diff --git a/packages/cli/src/lifecycles/config.js b/packages/cli/src/lifecycles/config.js index 76be78bab..4b09fb835 100644 --- a/packages/cli/src/lifecycles/config.js +++ b/packages/cli/src/lifecycles/config.js @@ -64,7 +64,7 @@ const readAndMergeConfig = async() => { try { await fs.access(configUrl); hasConfigFile = true; - } catch(e) { + } catch (e) { } @@ -72,7 +72,7 @@ const readAndMergeConfig = async() => { try { await fs.access(new URL('./index.html', customConfig.workspace)); isSPA = true; - } catch(e) { + } catch (e) { } diff --git a/packages/cli/src/lifecycles/copy.js b/packages/cli/src/lifecycles/copy.js index 09c3a35c6..257190b22 100644 --- a/packages/cli/src/lifecycles/copy.js +++ b/packages/cli/src/lifecycles/copy.js @@ -36,22 +36,29 @@ async function copyFile(source, target, projectDirectory) { async function copyDirectory(fromUrl, toUrl, projectDirectory) { try { console.info(`copying directory... ${fromUrl.pathname.replace(projectDirectory.pathname, '')}`); - const files = await rreaddir(fromUrl.pathname); + const files = await rreaddir(fromUrl); if (files.length > 0) { - if (!fs.existsSync(toUrl.pathname)) { - fs.mkdirSync(toUrl.pathname, { + try { + await fs.promises.access(toUrl); + } catch (e) { + await fs.promises.mkdir(toUrl, { recursive: true }); } + await Promise.all(files.filter((filePath) => { - const target = filePath.replace(fromUrl.pathname, toUrl.pathname); - const isDirectory = fs.lstatSync(filePath).isDirectory(); + const targetUrl = `file://${filePath.replace(fromUrl.pathname, toUrl.pathname)}`; + const isDirectory = (await fs.promises.lstat(targetUrl)).isDirectory(); - if (isDirectory && !fs.existsSync(target)) { - fs.mkdirSync(target); - } else if (!isDirectory) { - return filePath; + try { + if (isDirectory) { + await fs.promises.access(targetUrl); + } else if (!isDirectory) { + return filePath; + } + } catch (e) { + await fs.promises.mkdir(targetUrl); } }).map((filePath) => { const sourceUrl = new URL(`file://${filePath}`); diff --git a/packages/cli/src/lifecycles/graph.js b/packages/cli/src/lifecycles/graph.js index 9b8f9388a..405baae33 100644 --- a/packages/cli/src/lifecycles/graph.js +++ b/packages/cli/src/lifecycles/graph.js @@ -132,10 +132,10 @@ const generateGraph = async (compilation) => { worker.on('message', (result) => { if (result.frontmatter) { - const resources = (result.frontmatter.imports || []).map((resource) => { + const resources = (result.frontmatter.imports || []).map(async (resource) => { const type = resource.split('.').pop() === 'js' ? 'script' : 'link'; - return modelResource(compilation.context, type, resource); + return await modelResource(compilation.context, type, resource); }); result.frontmatter.imports = resources; diff --git a/packages/cli/src/lifecycles/prerender.js b/packages/cli/src/lifecycles/prerender.js index 6b8a30838..adf86fef7 100644 --- a/packages/cli/src/lifecycles/prerender.js +++ b/packages/cli/src/lifecycles/prerender.js @@ -1,4 +1,4 @@ -import fs from 'fs'; +import fs from 'fs/promises'; import htmlparser from 'node-html-parser'; import { modelResource } from '../lib/resource-utils.js'; import os from 'os'; @@ -8,9 +8,13 @@ function isLocalLink(url = '') { return url !== '' && (url.indexOf('http') !== 0 && url.indexOf('//') !== 0); } -function createOutputDirectory(route, { pathname }) { - if (route !== '/404/' && !fs.existsSync(pathname)) { - fs.mkdirSync(pathname, { +async function createOutputDirectory(route, outputDir) { + try { + if (route !== '/404/') { + await fs.access(outputDir); + } + } catch (e) { + await fs.mkdir(outputDir, { recursive: true }); } @@ -40,16 +44,16 @@ function trackResourcesForRoute(html, compilation, route) { if (src) { // - return modelResource(context, 'script', src, null, optimizationAttr, rawAttrs); + return await modelResource(context, 'script', src, null, optimizationAttr, rawAttrs); } else if (script.rawText) { // - return modelResource(context, 'script', null, script.rawText, optimizationAttr, rawAttrs); + return await modelResource(context, 'script', null, script.rawText, optimizationAttr, rawAttrs); } }); const styles = root.querySelectorAll('style') .filter(style => !(/\$/).test(style.rawText) && !(//).test(style.rawText)) // filter out Shady DOM