diff --git a/.jestSetup.js b/.jestSetup.js index 60ffd7f15782d..7c86cf8eb3440 100644 --- a/.jestSetup.js +++ b/.jestSetup.js @@ -5,7 +5,30 @@ if ( typeof globalThis.TextEncoder === "undefined" || typeof globalThis.TextDecoder === "undefined" ) { - const utils = require("util"); - globalThis.TextEncoder = utils.TextEncoder; - globalThis.TextDecoder = utils.TextDecoder; + const utils = require("util") + globalThis.TextEncoder = utils.TextEncoder + globalThis.TextDecoder = utils.TextDecoder } + +jest.mock(`gatsby-worker`, () => { + const gatsbyWorker = jest.requireActual(`gatsby-worker`) + + const { WorkerPool: OriginalWorkerPool } = gatsbyWorker + + class WorkerPoolThatCanUseTS extends OriginalWorkerPool { + constructor(workerPath, options) { + options.env = { + ...(options.env ?? {}), + NODE_OPTIONS: `--require ${require.resolve( + `./packages/gatsby/src/utils/worker/__tests__/test-helpers/ts-register.js` + )}`, + } + super(workerPath, options) + } + } + + return { + ...gatsbyWorker, + WorkerPool: WorkerPoolThatCanUseTS, + } +}) diff --git a/packages/gatsby/src/utils/parcel/__tests__/compile-gatsby-files.ts b/packages/gatsby/src/utils/parcel/__tests__/compile-gatsby-files.ts index 76ec8a9fabaf2..def5de6064dce 100644 --- a/packages/gatsby/src/utils/parcel/__tests__/compile-gatsby-files.ts +++ b/packages/gatsby/src/utils/parcel/__tests__/compile-gatsby-files.ts @@ -18,9 +18,10 @@ const dir = { misnamedJS: `${__dirname}/fixtures/misnamed-js`, misnamedTS: `${__dirname}/fixtures/misnamed-ts`, gatsbyNodeAsDirectory: `${__dirname}/fixtures/gatsby-node-as-directory`, + errorInCode: `${__dirname}/fixtures/error-in-code-ts`, } -jest.setTimeout(15000) +jest.setTimeout(60_000) jest.mock(`@parcel/core`, () => { const parcelCore = jest.requireActual(`@parcel/core`) @@ -175,6 +176,37 @@ describe(`gatsby file compilation`, () => { }) }) }) + + it(`handles errors in TS code`, async () => { + process.chdir(dir.errorInCode) + await remove(`${dir.errorInCode}/.cache`) + await compileGatsbyFiles(dir.errorInCode) + + expect(reporterPanicMock).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + Object { + "context": Object { + "filePath": "/gatsby-node.ts", + "generalMessage": "Expected ';', '}' or ", + "hints": null, + "origin": "@parcel/transformer-js", + "specificMessage": "This is the expression part of an expression statement", + }, + "id": "11901", + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": undefined, + }, + ], + } + `) + }) }) describe(`gatsby-node directory is allowed`, () => { diff --git a/packages/gatsby/src/utils/parcel/__tests__/fixtures/error-in-code-ts/gatsby-node.ts b/packages/gatsby/src/utils/parcel/__tests__/fixtures/error-in-code-ts/gatsby-node.ts new file mode 100644 index 0000000000000..a72e769a773d8 --- /dev/null +++ b/packages/gatsby/src/utils/parcel/__tests__/fixtures/error-in-code-ts/gatsby-node.ts @@ -0,0 +1,47 @@ +import { GatsbyNode } from "gatsby" +import { working } from "../utils/say-what-ts" +import { createPages } from "../utils/create-pages-ts" + +this is wrong syntax that should't compile + +export const onPreInit: GatsbyNode["onPreInit"] = ({ reporter }) => { + reporter.info(working) +} + +type Character = { + id: string + name: string +} + +export const sourceNodes: GatsbyNode["sourceNodes"] = async ({ actions, createNodeId, createContentDigest }) => { + const { createNode } = actions + + let characters: Array = [ + { + id: `0`, + name: `A` + }, + { + id: `1`, + name: `B` + } + ] + + characters.forEach((character: Character) => { + const node = { + ...character, + id: createNodeId(`characters-${character.id}`), + parent: null, + children: [], + internal: { + type: 'Character', + content: JSON.stringify(character), + contentDigest: createContentDigest(character), + }, + } + + createNode(node) + }) +} + +export { createPages } diff --git a/packages/gatsby/src/utils/parcel/compile-gatsby-files.ts b/packages/gatsby/src/utils/parcel/compile-gatsby-files.ts index 5b71db2cee73b..6edde27b95f86 100644 --- a/packages/gatsby/src/utils/parcel/compile-gatsby-files.ts +++ b/packages/gatsby/src/utils/parcel/compile-gatsby-files.ts @@ -3,6 +3,7 @@ import { LMDBCache, Cache } from "@parcel/cache" import path from "path" import type { Diagnostic } from "@parcel/diagnostic" import reporter from "gatsby-cli/lib/reporter" +import { WorkerPool } from "gatsby-worker" import { ensureDir, emptyDir, existsSync, remove, readdir } from "fs-extra" import telemetry from "gatsby-telemetry" import { isNearMatch } from "../is-near-match" @@ -52,6 +53,28 @@ export function constructParcel(siteRoot: string, cache?: Cache): Parcel { }) } +interface IProcessBundle { + filePath: string + mainEntryPath?: string +} + +type RunParcelReturn = Array + +export async function runParcel(siteRoot: string): Promise { + const cache = new LMDBCache(getCacheDir(siteRoot)) as unknown as Cache + const parcel = constructParcel(siteRoot, cache) + const { bundleGraph } = await parcel.run() + const bundles = bundleGraph.getBundles() + // bundles is not serializable, so we need to extract the data we need + // so it crosses IPC boundaries + return bundles.map(bundle => { + return { + filePath: bundle.filePath, + mainEntryPath: bundle.getMainEntry()?.filePath, + } + }) +} + /** * Compile known gatsby-* files (e.g. `gatsby-config`, `gatsby-node`) * and output in `/.cache/compiled`. @@ -107,33 +130,59 @@ export async function compileGatsbyFiles( }) } + const worker = new WorkerPool( + require.resolve(`./compile-gatsby-files`), + { + numWorkers: 1, + } + ) + const distDir = `${siteRoot}/${COMPILED_CACHE_DIR}` await ensureDir(distDir) await emptyDir(distDir) await exponentialBackoff(retry) - // for whatever reason TS thinks LMDBCache is some browser Cache and not actually Parcel's Cache - // so we force type it to Parcel's Cache - const cache = new LMDBCache(getCacheDir(siteRoot)) as unknown as Cache - const parcel = constructParcel(siteRoot, cache) - const { bundleGraph } = await parcel.run() - let cacheClosePromise = Promise.resolve() + let bundles: RunParcelReturn = [] try { - // @ts-ignore store is public field on LMDBCache class, but public interface for Cache - // doesn't have it. There doesn't seem to be proper public API for this, so we have to - // resort to reaching into internals. Just in case this is wrapped in try/catch if - // parcel changes internals in future (closing cache is only needed when retrying - // so the if the change happens we shouldn't fail on happy builds) - cacheClosePromise = cache.store.close() - } catch (e) { - reporter.verbose(`Failed to close parcel cache\n${e.toString()}`) + // sometimes parcel segfaults which is not something we can recover from, so we run parcel + // in child process and IF it fails we try to delete parcel's cache (this seems to "fix" the problem + // causing segfaults?) and retry few times + // not ideal, but having gatsby segfaulting is really frustrating and common remedy is to clean + // entire .cache for users, which is not ideal either especially when we can just delete parcel's cache + // and to recover automatically + bundles = await worker.single.runParcel(siteRoot) + } catch (error) { + if (error.diagnostics) { + handleErrors(error.diagnostics) + return + } else if (retry >= RETRY_COUNT) { + reporter.panic({ + id: `11904`, + error, + context: { + siteRoot, + retries: RETRY_COUNT, + sourceMessage: error.message, + }, + }) + } else { + await exponentialBackoff(retry) + try { + await remove(getCacheDir(siteRoot)) + } catch { + // in windows we might get "EBUSY" errors if LMDB failed to close, so this try/catch is + // to prevent EBUSY errors from potentially hiding real import errors + } + await compileGatsbyFiles(siteRoot, retry + 1) + return + } + } finally { + worker.end() } await exponentialBackoff(retry) - const bundles = bundleGraph.getBundles() - if (bundles.length === 0) return let compiledTSFilesCount = 0 @@ -150,7 +199,7 @@ export async function compileGatsbyFiles( siteRoot, retries: RETRY_COUNT, compiledFileLocation: bundle.filePath, - sourceFileLocation: bundle.getMainEntry()?.filePath, + sourceFileLocation: bundle.mainEntryPath, }, }) } else if (retry > 0) { @@ -165,9 +214,6 @@ export async function compileGatsbyFiles( ) } - // sometimes parcel cache gets in weird state and we need to clear the cache - await cacheClosePromise - try { await remove(getCacheDir(siteRoot)) } catch { @@ -179,7 +225,7 @@ export async function compileGatsbyFiles( return } - const mainEntry = bundle.getMainEntry()?.filePath + const mainEntry = bundle.mainEntryPath // mainEntry won't exist for shared chunks if (mainEntry) { if (mainEntry.endsWith(`.ts`)) {