diff --git a/.npmrc b/.npmrc index 5789e699f..9daa01a03 100644 --- a/.npmrc +++ b/.npmrc @@ -1,5 +1,5 @@ registry = https://registry.npmjs.org -ignore-scripts = false +ignore-scripts = true public-hoist-pattern[]=*@types* diff --git a/frontend/app/index.html b/frontend/app/index.html index 4f176f5c0..7276ec22a 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -7,8 +7,6 @@ - - {{content-for "head"}} diff --git a/frontend/app/services/-compile/babel/index.ts b/frontend/app/services/-compile/babel/index.ts index bec2776e7..65a9dda5c 100644 --- a/frontend/app/services/-compile/babel/index.ts +++ b/frontend/app/services/-compile/babel/index.ts @@ -1,19 +1,19 @@ +import { compileJS as cjs } from 'ember-repl'; // import config from 'limber/config/environment'; -// import { compile as worker } from './worker'; +import { compile as worker } from './worker'; -// import type { ExtractedCode } from '../markdown-to-ember'; -// import type { AsyncReturnType } from 'type-fest'; +import type { AsyncReturnType } from 'type-fest'; +import { ExtractedCode } from '../markdown-to-ember'; -// type CompiledViaWorker = AsyncReturnType; -// type CompiledViaCJS = AsyncReturnType; -// export type CompileOutput = CompiledViaCJS[0] | CompiledViaWorker[0]; +type CompiledViaWorker = AsyncReturnType; +type CompiledViaCJS = AsyncReturnType; +export type CompileOutput = CompiledViaCJS | CompiledViaWorker; -// export async function compileJS(_id: string, js: ExtractedCode[]): Promise { -// if (config.SERVICE_WORKER) { -// return worker(js); -// } +export async function compileJS(info: ExtractedCode): Promise { + // if (config.SERVICE_WORKER) { + return worker(info); + // } -// // use compileAll from -compile/index -// // return cjs(js); -// } + // return cjs(js); +} diff --git a/frontend/app/services/-compile/babel/worker.ts b/frontend/app/services/-compile/babel/worker.ts index b80381a65..d19c09de0 100644 --- a/frontend/app/services/-compile/babel/worker.ts +++ b/frontend/app/services/-compile/babel/worker.ts @@ -2,74 +2,82 @@ import type { ExtractedCode } from '../markdown-to-ember'; let isRegistered = false; -export async function compile(js: ExtractedCode[]) { - let moduleInfos = await compileModules(js); - - let missing = moduleInfos.filter(({ importPath }) => !importPath); - - if (missing.length > 0) { - let first = missing[0]; - - throw new Error(`Component, ${first.name}, failed to compile`); +class CompileError extends Error { + constructor( + public message: string, + public response: Response, + public code: string, + public url: string + ) { + super(message); } - - return moduleInfos; } -export async function compileModules(js: ExtractedCode[]) { +export async function compile(js: ExtractedCode) { if (!isRegistered) { - await installImportMap(); + // await installImportMap(); await setupServiceWorker(); + await establishRequireJSEntries(); isRegistered = true; } - let responses = await Promise.all( - js.map(async ({ name, code }) => { - let qps = new URLSearchParams(); + let { name, code } = js; + + let qps = new URLSearchParams(); + + qps.set('n', name); + qps.set('q', code); - qps.set('n', name); - qps.set('q', code); + let url = `/compile-sw?${qps}`; - let response = await fetch(`/compile-sw?${qps}`); - let { importPath } = await response.json(); + let response = await fetch(url, { + headers: { 'Content-Type': 'application/json' }, + }); - return { name, importPath }; - }) - ); + if (response.status !== 200) { + throw new CompileError( + `${response.status} | Could not compile code. See console for details.`, + response, + code, + url + ); + } + + let { importPath } = await response.json(); - return responses; + return { name, importPath }; } -async function installImportMap() { - let script = document.createElement('script'); +// async function installImportMap() { +// let script = document.createElement('script'); - script.setAttribute('type', 'importmap'); +// script.setAttribute('type', 'importmap'); - // let response = await import( - // /* webpackIgnore: true */ - // 'https://raw.githubusercontent.com/ef4/mho/a4391e53891f3f6321f0a8f36de88ec23511dbee/ember-app/importmap.json' - // ); - // External Import maps are not supported yet - // let response = await fetch( - // 'https://raw.githubusercontent.com/ef4/mho/a4391e53891f3f6321f0a8f36de88ec23511dbee/ember-app/importmap.json' - // ); - // let importmap = await response.text(); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - let importmap = (await import('/mho-importmap.json')).default; +// // let response = await import( +// // /* webpackIgnore: true */ +// // 'https://raw.githubusercontent.com/ef4/mho/a4391e53891f3f6321f0a8f36de88ec23511dbee/ember-app/importmap.json' +// // ); +// // External Import maps are not supported yet +// // let response = await fetch( +// // 'https://raw.githubusercontent.com/ef4/mho/a4391e53891f3f6321f0a8f36de88ec23511dbee/ember-app/importmap.json' +// // ); +// // let importmap = await response.text(); +// // eslint-disable-next-line @typescript-eslint/ban-ts-comment +// // @ts-ignore +// let importmap = (await import('/mho-importmap.json')).default; - console.debug({ importmap }); +// console.debug({ importmap }); - script.innerHTML = JSON.stringify(importmap); - document.body.appendChild(script); -} +// script.innerHTML = JSON.stringify(importmap); +// document.body.appendChild(script); +// } async function setupServiceWorker() { if ('serviceWorker' in navigator) { - let registration = await navigator.serviceWorker.register('/transpilation-worker.js'); + let registration = await navigator.serviceWorker.register('/transpile.js'); - // registration.update(); + registration.update(); console.info('ServiceWorker registration successful with scope: ', registration.scope); @@ -82,8 +90,34 @@ async function setupServiceWorker() { }, 50); }); + console.info('ServiceWorker activated.'); + return registration; } throw new Error(`ServiceWorker is required`); } + +async function establishRequireJSEntries() { + await fetch('/populate-sw', { + method: 'POST', + // Require JS is private API, but we need to swap out imports in the code + // snippets for accessing the same requirejs modules that are in this app. + // + // This is to + // - reduce overall shipped JS + // + // However, if we were to make the rendering area an entirely different app, + // we could then isolate compile errors and not have fatal problems that require + // a browser page refresh -- this may be the best thing to do in the long term. + // + // But for now, we need to at least try the requirejs stuff, because it's a requirement + // for design-system REPLs + // + // eslint-disable-next-line @typescript-eslint/no-explicit-any + body: JSON.stringify(Object.keys((window.requirejs as any).entries)), + headers: { + 'Content-Type': 'application/json', + }, + }); +} diff --git a/frontend/app/services/-compile/index.ts b/frontend/app/services/-compile/index.ts index 70084d412..330e985b8 100644 --- a/frontend/app/services/-compile/index.ts +++ b/frontend/app/services/-compile/index.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/ban-types */ -import { compileHBS, compileJS, invocationName } from 'ember-repl'; +import { compileHBS, invocationName } from 'ember-repl'; +import { compileJS } from './babel'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error import CopyMenu from 'limber/components/limber/copy-menu'; @@ -19,14 +20,14 @@ interface CompilationResult { errorLine?: number; } -export async function compileAll(js: { code: string }[]) { +export async function compileAll(js: ExtractedCode[]) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore let { COMPONENT_MAP } = await import('/ember-repl/component-map.js'); let modules = await Promise.all( - js.map(async ({ code }) => { - return await compileJS(code, COMPONENT_MAP); + js.map(async (info) => { + return await compileJS(info); }) ); @@ -71,13 +72,13 @@ export async function compile(glimdownInput: string): Promise compiled.map(async (info) => { // using web worker + import maps is not available yet (need firefox support) // (and to somehow be able to point at npm) - // - // if ('importPath' in info) { - // return scope.push({ - // moduleName: name, - // component: await import(/* webpackIgnore: true */ info.importPath), - // }); - // } + + if ('importPath' in info) { + return scope.push({ + moduleName: name, + component: await import(/* webpackIgnore: true */ info.importPath), + }); + } return scope.push(info); }) @@ -88,9 +89,6 @@ export async function compile(glimdownInput: string): Promise scope.push(compileHBS(code)); } } catch (error) { - console.info({ scope }); - console.error(error); - return { error, rootTemplate }; } } diff --git a/frontend/ember-cli-build.js b/frontend/ember-cli-build.js index 898a4de5b..6fdff5e14 100644 --- a/frontend/ember-cli-build.js +++ b/frontend/ember-cli-build.js @@ -18,6 +18,18 @@ module.exports = function (defaults) { `); let config = { + contentFor(_, __, type) { + if (type === 'head' && isProduction) { + return ` + + ` + } + + this._super.call(this, ...arguments); + }, 'ember-cli-terser': { enabled: MINIFY, }, @@ -44,6 +56,8 @@ module.exports = function (defaults) { return require('@embroider/compat').compatBuild(app, Webpack, { extraPublicTrees: [ + // Service Worker / transpilation + require('@nullvoxpopuli/limber-transpilation/broccoli-funnel')(), // Mobile Editor require('@nullvoxpopuli/limber-codemirror/broccoli-funnel')(), // Desktop Editor diff --git a/packages/transpilation/broccoli-funnel.js b/packages/transpilation/broccoli-funnel.js new file mode 100644 index 000000000..8fce569b9 --- /dev/null +++ b/packages/transpilation/broccoli-funnel.js @@ -0,0 +1,18 @@ +'use strict'; + +const path = require('path'); +const Funnel = require('broccoli-funnel'); + +const SRC_FILES = path.join(__dirname, 'dist'); + +/** + * This broccoli funnel is for copying the built assets to a target + * app's public folder. No building occurs + * + */ +module.exports = function distWatcher() { + return new Funnel(SRC_FILES, { + // dist or whatever the root of the output directory is + destDir: '/', + }); +}; diff --git a/packages/transpilation/build.js b/packages/transpilation/build.js index 9e75196cb..83922e255 100644 --- a/packages/transpilation/build.js +++ b/packages/transpilation/build.js @@ -1,62 +1,30 @@ -'use strict'; - const path = require('path'); -const os = require('os'); -const fs = require('fs'); const esbuild = require('esbuild'); -const appFolder = path.join(__dirname, '..', 'app'); -const workerRoot = path.join(appFolder, 'workers'); - -function detectWorkers() { - let workers = {}; - let dir = fs.readdirSync(workerRoot); - - for (let i = 0; i < dir.length; i++) { - let name = dir[i]; - - workers[name] = path.join(workerRoot, name, 'index.js'); - } - - return workers; -} - -function configureWorkerTree({ isProduction, buildDir }) { - return ([name, entryPath]) => { - esbuild.buildSync({ - loader: { '.ts': 'ts', '.js': 'js' }, - entryPoints: [entryPath], - bundle: true, - outfile: path.join(buildDir, `${name}.js`), - format: 'esm', - minify: isProduction, - sourcemap: !isProduction, - // incremental: true, - // tsconfig: path.join(appFolder, 'tsconfig.json'), - }); - }; -} - -function buildWorkers(env) { - let inputs = detectWorkers(); - let workerBuilder = configureWorkerTree(env); - - // separate build from ember, will be detached, won't watch - Object.entries(inputs).map(workerBuilder); -} - -function workersFunnel({ isProduction }) { - let buildDir = fs.mkdtempSync(path.join(os.tmpdir(), 'limber--workers')); - - let options = { - isProduction, - buildDir, - }; - - // outputs {buildDir}/highlighting.js - buildWorkers(options); -} - -module.exports = { - workersFunnel, -}; +const entry = path.join(__dirname, 'src', 'index.ts'); +const buildDir = path.join(__dirname, 'dist'); + +const isWatching = process.argv.includes('--watch') +const isProduction = !isWatching; + +esbuild.build({ + loader: { '.ts': 'ts', '.js': 'js' }, + entryPoints: [entry], + bundle: true, + outfile: path.join(buildDir, `transpile.js`), + format: 'esm', + minify: isProduction, + sourcemap: isProduction, + ...( + isWatching ? { + incremental: true, + watch: { + onRebuild(error, result) { + if (error) console.error('watch build failed:', error) + else console.log('watch build succeeded:', result) + }, + }, + } + : {} + ) +}); diff --git a/packages/transpilation/package.json b/packages/transpilation/package.json index 5611c14e5..463daee3a 100644 --- a/packages/transpilation/package.json +++ b/packages/transpilation/package.json @@ -6,13 +6,11 @@ "author": "NullVoxPopuli", "main": "dist/limber-worker.js", "scripts": { - "prepare": "webpack", + "start": "node build.js --watch", + "build": "node build.js", "lint:fix": "npm-run-all --aggregate-output --parallel 'lint:*:fix'", "lint:js": "eslint . --cache", - "lint:js:fix": "eslint . --fix", - "start": "webpack --watch", - "cp": "cp ./dist/limber-worker.js ../limber/public/transpilation-worker.js", - "prod": "PRODUCTION=true webpack" + "lint:js:fix": "eslint . --fix" }, "browser": { "path": "path-browserify", @@ -45,10 +43,7 @@ "@types/htmlbars-inline-precompile": "^3.0.0", "babel-loader": "^8.2.3", "esbuild": "0.14.27", - "typescript": "^4.6.2", - "webpack": "^5.70.0", - "webpack-cli": "^4.9.2", - "webpack-node-externals": "^3.0.0" + "typescript": "^4.6.2" }, "engines": { "node": ">= v16.14.1" diff --git a/packages/transpilation/src/babel.ts b/packages/transpilation/src/babel.ts new file mode 100644 index 000000000..7da5e621e --- /dev/null +++ b/packages/transpilation/src/babel.ts @@ -0,0 +1,25 @@ +interface Info { + name: string; + code: string; +} + +/** + * Multi-phase "compiling"! + * 1. Swap out imports from requireJS as const-definitions in the via + * `import { tracked } from '@glimmer/tracking';` + * -> const { tracked } = requirejs('@glimmer/tracking'); + * `import Component from '@ember/component';` + * -> const { default: Component } = requirejs('@ember/component'); + * + * Outstanding question: where does '@glimmer/component' live? + * where would we get it from? + * + * 2. Run template transform + * 3. Compile out the decorators + * + */ +export function compileGJS({ name, code }: Info, requireJSList: Set) { + console.log(name, code, requireJSList); + + return ' '; +} diff --git a/packages/transpilation/src/fetch-handler.ts b/packages/transpilation/src/fetch-handler.ts index 4001858e6..9d2baea55 100644 --- a/packages/transpilation/src/fetch-handler.ts +++ b/packages/transpilation/src/fetch-handler.ts @@ -1,97 +1,114 @@ -// import { compileGJS } from './babel'; - -// // const CACHE_NAME = 'babel-compilation-and-module-service'; -// const URLS = ['/compile-sw', /^\/module-sw\//]; - -// const COMPILE_CACHE = new Map(); - -// export async function handleFetch(event: FetchEvent): Promise { -// const url = new URL(event.request.url); - -// console.info('handleFetch', url.pathname); - -// if (!URLS.some((matcher) => url.pathname.match(matcher))) { -// return fetch(event.request); -// } - -// if (COMPILE_CACHE.has(url.pathname)) { -// return moduleResponse(url.pathname); -// } - -// if (url.pathname === '/compile-sw') { -// return maybe(() => compile(url)); -// } - -// return error(`Unhandled URL: ${url.pathname}`); -// } - -// async function maybe(op: () => Return | Promise) { -// try { -// return await op(); -// } catch (e) { -// return error(e); -// } -// } - -// function error(msg: Error | string, status = 500) { -// let payload: string | Error | Record; - -// if (typeof msg === 'string') { -// payload = msg; -// } else if (msg instanceof TypeError) { -// payload = { -// ...msg, -// name: msg.name, -// message: msg.message, -// stack: msg.stack, -// }; -// } else { -// payload = msg; -// } - -// return new Response(JSON.stringify({ error: payload }), { -// status, -// headers: { -// 'Content-Type': 'application/json', -// }, -// }); -// } - -// function moduleResponse(pathName: string) { -// let code = COMPILE_CACHE.get(pathName); - -// if (!code) { -// throw new Error(`Code has not been compiled. call /compile-sw with the code`); -// } - -// return new Response(code, { -// headers: { -// 'Content-Type': 'application/javascript', -// }, -// }); -// } - -// async function compile(url: URL) { -// let qps = new URLSearchParams(url.search); -// let name = qps.get('n'); -// let code = qps.get('q'); -// let modulePath = `/module-sw/${name}.js`; - -// if (!name || !code) { -// throw new Error( -// `Both name and code are required. Make sure than the n and q query params are specified` -// ); -// } - -// let compiled = await compileGJS({ name, code }); - -// COMPILE_CACHE.set(modulePath, compiled); - -// let response = new Response(JSON.stringify({ importPath: modulePath }), { -// headers: { -// 'Content-Type': 'application/json', -// }, -// }); - -// return response; -// } +import { compileGJS } from './babel'; + +// const CACHE_NAME = 'babel-compilation-and-module-service'; +const URLS = ['/compile-sw', /^\/module-sw\//, '/populate-sw']; + +const COMPILE_CACHE = new Map(); +let REQUIRE_JS_LIST: Set; + +export async function handleFetch(event: FetchEvent): Promise { + const url = new URL(event.request.url); + + if (!URLS.some((matcher) => url.pathname.match(matcher))) { + return fetch(event.request); + } + + console.info('handleFetch', url.pathname); + + if (COMPILE_CACHE.has(url.pathname)) { + return moduleResponse(url.pathname); + } + + if (url.pathname === '/populate-sw') { + return maybe(() => populateRequireJSModules(event)); + } + + if (url.pathname === '/compile-sw') { + return maybe(() => compile(url)); + } + + return error(`Unhandled URL: ${url.pathname}`); +} + +async function maybe(op: () => Return | Promise) { + try { + return await op(); + } catch (e) { + return error(e); + } +} + +function error(msg: Error | string, status = 500) { + let payload: string | Error | Record; + + if (typeof msg === 'string') { + payload = msg; + } else if (msg instanceof TypeError) { + payload = { + ...msg, + name: msg.name, + message: msg.message, + stack: msg.stack, + }; + } else { + payload = msg; + } + + return new Response(JSON.stringify({ error: payload }), { + status, + headers: { + 'Content-Type': 'application/json', + }, + }); +} + +async function populateRequireJSModules(event: FetchEvent) { + let list = await event.request.json(); + + REQUIRE_JS_LIST = new Set(list); + + return new Response('{ "status": "done" }', { + headers: { + 'Content-Type': 'application/javascript', + }, + }); +} + +function moduleResponse(pathName: string) { + let code = COMPILE_CACHE.get(pathName); + + if (!code) { + throw new Error(`Code has not been compiled. call /compile-sw with the code`); + } + + return new Response(code, { + headers: { + 'Content-Type': 'application/javascript', + }, + }); +} + +async function compile(url: URL) { + let qps = new URLSearchParams(url.search); + let name = qps.get('n'); + let code = qps.get('q'); + let modulePath = `/module-sw/${name}.js`; + + if (!name || !code) { + throw new Error( + `Both name and code are required. Make sure than the n and q query params are specified` + ); + } + + let compiled = await compileGJS({ name, code }, REQUIRE_JS_LIST); + + COMPILE_CACHE.set(modulePath, compiled); + + let response = new Response(JSON.stringify({ importPath: modulePath }), { + headers: { + 'Content-Type': 'application/json', + }, + }); + + return response; +} diff --git a/packages/transpilation/src/index.ts b/packages/transpilation/src/index.ts index ffd516c33..3271fe980 100644 --- a/packages/transpilation/src/index.ts +++ b/packages/transpilation/src/index.ts @@ -1,57 +1,61 @@ -// import { handleFetch } from './fetch-handler'; +import { handleFetch } from './fetch-handler'; -// const worker = self as unknown as ServiceWorkerGlobalScope; +const worker = self as unknown as ServiceWorkerGlobalScope; -// /** -// * For a given glimdown document id, we will compile -// * N components within that glimdown, and return an object -// * map of an arbitrary name of the default export to the URL -// * for which the module may be imported from. -// * -// * Since the set of modules is uniqueish to the glimdown -// * document id, we'll try to keep a history of 10 most recent -// * compiles, so that quick edits don't need to do extra work -// * -// * example: -// * -// * POST /compile.sw -// * id: gmd.id, -// * components: [{ name: string, code: string }] -// * -// * => -// * -// * { -// * [name] => "url/to/import" -// * } -// * -// * which will then turn in to (roughly): -// * -// * for (let [name, importPath] of response) { -// * let module = await import(importPath); -// * -// * owner.register(`component:${name}`, module); -// * } -// * -// * and the <${name} /> will be swapped in to the ember -// * variant of the glimdown for invocation -// * -// */ -// worker.addEventListener('install', () => { -// // force moving on to activation even if another service worker had control -// worker.skipWaiting(); -// }); +/** + * For a given glimdown document id, we will compile + * N components within that glimdown, and return an object + * map of an arbitrary name of the default export to the URL + * for which the module may be imported from. + * + * Since the set of modules is uniqueish to the glimdown + * document id, we'll try to keep a history of 10 most recent + * compiles, so that quick edits don't need to do extra work + * + * example: + * + * POST /compile.sw + * id: gmd.id, + * components: [{ name: string, code: string }] + * + * => + * + * { + * [name] => "url/to/import" + * } + * + * which will then turn in to (roughly): + * + * for (let [name, importPath] of response) { + * let module = await import(importPath); + * + * owner.register(`component:${name}`, module); + * } + * + * and the <${name} /> will be swapped in to the ember + * variant of the glimdown for invocation + * + */ +worker.addEventListener('install', () => { + // force moving on to activation even if another service worker had control + worker.skipWaiting(); +}); -// worker.addEventListener('activate', (event) => { -// // Claim any clients immediately, so that the page will be under SW control without reloading. -// event.waitUntil(worker.clients.claim()); -// console.info(`\ -// Service Worker installed successfully! +worker.addEventListener('activate', (event) => { + // Claim any clients immediately, so that the page will be under SW control without reloading. + // + // https://developer.mozilla.org/en-US/docs/Web/API/Clients/claim + event.waitUntil(worker.clients.claim()); + console.info(`\ + Service Worker installed successfully! -// This service worker is used for compiling JavaScript -// and providing modules to the main thread. -// `); -// }); + This service worker is used for compiling JavaScript + and providing modules to the main thread. + `); +}); -// worker.addEventListener('fetch', (event) => { -// event.respondWith(handleFetch(event)); -// }); +worker.addEventListener('fetch', (event) => { + let response = handleFetch(event); + + event.respondWith(response); +}); diff --git a/packages/transpilation/webpack.config.js b/packages/transpilation/webpack.config.js deleted file mode 100644 index 6f3c12153..000000000 --- a/packages/transpilation/webpack.config.js +++ /dev/null @@ -1,44 +0,0 @@ -'use strict'; - -const path = require('path'); -const webpack = require('webpack'); - -module.exports = { - mode: process.env.PRODUCTION != null ? 'production' : 'development', - entry: { - worker: './src/index.ts', - }, - target: 'webworker', - devtool: process.env.PRODUCTION != null ? false : 'inline-source-map', - plugins: [ - new webpack.ProvidePlugin({ - process: 'process', - Buffer: 'buffer', - }), - ], - module: { - rules: [ - { - test: /\.ts$/i, - use: ['babel-loader'], - }, - ], - }, - output: { - filename: 'limber-[name].js', - }, - resolve: { - extensions: ['.ts', '.js', '.json'], - fallback: { - fs: false, - path: require.resolve('path-browserify'), - '@ember/template-compilation': require.resolve( - 'ember-source/dist/ember-template-compiler.js' - ), - }, - alias: { - // this prevents complaining about require.extensions - // handlebars: 'handlebars/dist/cjs/handlebars.js', - }, - }, -};