diff --git a/.circleci/config.yml b/.circleci/config.yml index 2af8a75829f42..e17a153324ebc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -313,6 +313,13 @@ jobs: test_path: integration-tests/esm-in-gatsby-files test_command: yarn test + integration_tests_lmdb_regeneration: + executor: node + steps: + - e2e-test: + test_path: integration-tests/lmdb-regeneration + test_command: yarn test + e2e_tests_path-prefix: <<: *e2e-executor steps: @@ -590,6 +597,8 @@ workflows: <<: *e2e-test-workflow - integration_tests_esm_in_gatsby_files: <<: *e2e-test-workflow + - integration_tests_lmdb_regeneration: + <<: *e2e-test-workflow - integration_tests_gatsby_cli: requires: - bootstrap diff --git a/integration-tests/lmdb-regeneration/.gitignore b/integration-tests/lmdb-regeneration/.gitignore new file mode 100644 index 0000000000000..f9fd0a59de1f5 --- /dev/null +++ b/integration-tests/lmdb-regeneration/.gitignore @@ -0,0 +1,3 @@ +__tests__/__debug__ +node_modules +yarn.lock diff --git a/integration-tests/lmdb-regeneration/.npmrc b/integration-tests/lmdb-regeneration/.npmrc new file mode 100644 index 0000000000000..1a456a8d1c769 --- /dev/null +++ b/integration-tests/lmdb-regeneration/.npmrc @@ -0,0 +1 @@ +build_from_source=true diff --git a/integration-tests/lmdb-regeneration/LICENSE b/integration-tests/lmdb-regeneration/LICENSE new file mode 100644 index 0000000000000..20f91f2b3c52b --- /dev/null +++ b/integration-tests/lmdb-regeneration/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 gatsbyjs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/integration-tests/lmdb-regeneration/README.md b/integration-tests/lmdb-regeneration/README.md new file mode 100644 index 0000000000000..8df148ba4dc0d --- /dev/null +++ b/integration-tests/lmdb-regeneration/README.md @@ -0,0 +1,3 @@ +## Artifacts test suite + +This integration test suite helps us assert our mechanism to heal back from a broken lmdb installation is working by validating we install and use the correct prebundled binary for our platform diff --git a/integration-tests/lmdb-regeneration/__tests__/index.js b/integration-tests/lmdb-regeneration/__tests__/index.js new file mode 100644 index 0000000000000..3da14d6ce0d54 --- /dev/null +++ b/integration-tests/lmdb-regeneration/__tests__/index.js @@ -0,0 +1,59 @@ +const path = require(`path`) +const execa = require(`execa`) +const mod = require("module") +const fs = require(`fs-extra`) + +jest.setTimeout(100000) + +const rootPath = path.resolve(__dirname, "../") + +describe(`Lmdb regeneration`, () => { + test(`gatbsy build detects lmdb setup built from source and installs pre-buit package`, async () => { + const lmdbNodeModulesPath = path.resolve(rootPath, "node_modules", "lmdb") + // Make sure we clear out the current `@lmdb` optional dependencies + const pathsToRemove = [ + path.resolve(rootPath, "node_modules", "@lmdb"), + path.resolve(rootPath, "node_modules", "gatsby", "node_modules", "@lmdb"), + ] + for (let path of pathsToRemove) { + fs.rmSync(path, { force: true, recursive: true }) + } + // Check the lmdb instance we have installed does have a binary built from source since we need it to reproduce the fix we're trying to test + // If this check fails then it means our fixture is wrong and we're relying on an lmdb instance with prebuilt binaries + const builtFromSource = fs.existsSync( + path.resolve(lmdbNodeModulesPath, "build", "Release", "lmdb.node") + ) + expect(builtFromSource).toEqual(true) + + const options = { + stderr: `inherit`, + stdout: `inherit`, + cwd: rootPath, + } + const gatsbyBin = path.resolve(rootPath, `node_modules`, `gatsby`, `cli.js`) + await execa(gatsbyBin, [`build`], options) + + // lmdb module with prebuilt binaries for our platform + const lmdbPackage = `@lmdb/lmdb-${process.platform}-${process.arch}` + + // If the fix worked correctly we should have installed the prebuilt binary for our platform under our `.cache` directory + const lmdbRequire = mod.createRequire( + path.resolve(rootPath, ".cache", "internal-packages", "package.json") + ) + expect(() => { + lmdbRequire.resolve(lmdbPackage) + }).not.toThrow() + + // The resulting query-engine bundle should not contain the binary built from source + const binaryBuiltFromSource = path.resolve( + rootPath, + ".cache", + "query-engine", + "assets", + "build", + "Release", + "lmdb.node" + ) + expect(fs.existsSync(binaryBuiltFromSource)).toEqual(false) + }) +}) diff --git a/integration-tests/lmdb-regeneration/gatsby-config.js b/integration-tests/lmdb-regeneration/gatsby-config.js new file mode 100644 index 0000000000000..9e31e5d618ad2 --- /dev/null +++ b/integration-tests/lmdb-regeneration/gatsby-config.js @@ -0,0 +1,9 @@ +module.exports = { + siteMetadata: { + siteUrl: `https://www.yourdomain.tld`, + }, + plugins: [], + flags: { + DEV_SSR: true, + }, +} diff --git a/integration-tests/lmdb-regeneration/jest.config.js b/integration-tests/lmdb-regeneration/jest.config.js new file mode 100644 index 0000000000000..4e5a78b25d7bf --- /dev/null +++ b/integration-tests/lmdb-regeneration/jest.config.js @@ -0,0 +1,3 @@ +module.exports = { + testPathIgnorePatterns: [`/node_modules/`, `__tests__/fixtures`, `.cache`], +} diff --git a/integration-tests/lmdb-regeneration/package.json b/integration-tests/lmdb-regeneration/package.json new file mode 100644 index 0000000000000..7cd2ca82f4e72 --- /dev/null +++ b/integration-tests/lmdb-regeneration/package.json @@ -0,0 +1,22 @@ +{ + "name": "lmdb-regeneration", + "private": true, + "author": "Sid Chatterjee", + "description": "A simplified bare-bones starter for Gatsby with DSG", + "version": "0.1.0", + "license": "MIT", + "scripts": { + "build": "gatsby build", + "clean": "gatsby clean", + "test": "jest --runInBand" + }, + "dependencies": { + "gatsby": "next", + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "fs-extra": "^11.1.0", + "jest": "^29.3.1" + } +} diff --git a/integration-tests/lmdb-regeneration/src/images/icon.png b/integration-tests/lmdb-regeneration/src/images/icon.png new file mode 100644 index 0000000000000..38b2fb0e467e0 Binary files /dev/null and b/integration-tests/lmdb-regeneration/src/images/icon.png differ diff --git a/integration-tests/lmdb-regeneration/src/pages/404.jsx b/integration-tests/lmdb-regeneration/src/pages/404.jsx new file mode 100644 index 0000000000000..d7b40627e81ae --- /dev/null +++ b/integration-tests/lmdb-regeneration/src/pages/404.jsx @@ -0,0 +1,49 @@ +import * as React from "react" +import { Link } from "gatsby" + +const pageStyles = { + color: "#232129", + padding: "96px", + fontFamily: "-apple-system, Roboto, sans-serif, serif", +} +const headingStyles = { + marginTop: 0, + marginBottom: 64, + maxWidth: 320, +} + +const paragraphStyles = { + marginBottom: 48, +} +const codeStyles = { + color: "#8A6534", + padding: 4, + backgroundColor: "#FFF4DB", + fontSize: "1.25rem", + borderRadius: 4, +} + +const NotFoundPage = () => { + return ( +
+

Page not found

+

+ Sorry 😔, we couldn’t find what you were looking for. +
+ {process.env.NODE_ENV === "development" ? ( + <> +
+ Try creating a page in src/pages/. +
+ + ) : null} +
+ Go home. +

+
+ ) +} + +export default NotFoundPage + +export const Head = () => Not found diff --git a/integration-tests/lmdb-regeneration/src/pages/test-dsg.jsx b/integration-tests/lmdb-regeneration/src/pages/test-dsg.jsx new file mode 100644 index 0000000000000..db06e1909a02a --- /dev/null +++ b/integration-tests/lmdb-regeneration/src/pages/test-dsg.jsx @@ -0,0 +1,17 @@ +import * as React from "react"; + +const DSG = () => { + return

DSG

; +}; + +export default DSG; + +export const Head = () => DSG; + +export async function config() { + return () => { + return { + defer: true, + }; + }; +} diff --git a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts index 8b5d99f4e41c2..4b87d2e5115a9 100644 --- a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts +++ b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts @@ -2,8 +2,10 @@ import * as path from "path" import * as fs from "fs-extra" +import execa, { Options as ExecaOptions } from "execa" import webpack, { Module, NormalModule, Compilation } from "webpack" import ConcatenatedModule from "webpack/lib/optimize/ConcatenatedModule" +import { dependencies } from "gatsby/package.json" import { printQueryEnginePlugins } from "./print-plugins" import mod from "module" import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging" @@ -12,6 +14,7 @@ import { schemaCustomizationAPIs } from "./print-plugins" import type { GatsbyNodeAPI } from "../../redux/types" import * as nodeApis from "../../utils/api-node-docs" import { store } from "../../redux" +import { PackageJson } from "../../.." type Reporter = typeof reporter @@ -35,6 +38,92 @@ function getApisToRemoveForQueryEngine(): Array { return apisToRemove } +const getInternalPackagesCacheDir = (): string => + path.join(process.cwd(), `.cache/internal-packages`) + +// Create a directory and JS module where we install internally used packages +const createInternalPackagesCacheDir = async (): Promise => { + const cacheDir = getInternalPackagesCacheDir() + await fs.ensureDir(cacheDir) + await fs.emptyDir(cacheDir) + + const packageJsonPath = path.join(cacheDir, `package.json`) + + await fs.outputJson(packageJsonPath, { + name: `gatsby-internal-packages`, + description: `This directory contains internal packages installed by Gatsby used to comply with the current platform requirements`, + version: `1.0.0`, + private: true, + author: `Gatsby`, + license: `MIT`, + }) +} + +// lmdb module with prebuilt binaries for our platform +const lmdbPackage = `@lmdb/lmdb-${process.platform}-${process.arch}` + +// Detect if the prebuilt binaries for lmdb have been installed. These are installed under @lmdb and are tied to each platform/arch. We've seen instances where regular installations lack these modules because of a broken lockfile or skipping optional dependencies installs +function installPrebuiltLmdb(): boolean { + // Read lmdb's package.json, go through its optional depedencies and validate if there's a prebuilt lmdb module with a compatible binary to our platform and arch + let packageJson: PackageJson + try { + const modulePath = path + .dirname(require.resolve(`lmdb`)) + .replace(`/dist`, ``) + const packageJsonPath = path.join(modulePath, `package.json`) + packageJson = JSON.parse(fs.readFileSync(packageJsonPath, `utf-8`)) + } catch (e) { + // If we fail to read lmdb's package.json there's bigger problems here so just skip installation + return false + } + // If there's no lmdb prebuilt package for our arch/platform listed as optional dep no point in trying to install it + const { optionalDependencies } = packageJson + if (!optionalDependencies) return false + if (!Object.keys(optionalDependencies).find(p => p === lmdbPackage)) + return false + try { + const lmdbRequire = mod.createRequire(require.resolve(`lmdb`)) + lmdbRequire.resolve(lmdbPackage) + return false + } catch (e) { + return true + } +} + +// Install lmdb's native system module under our internal cache if we detect the current installation +// isn't using the pre-build binaries +async function installIfMissingLmdb(): Promise { + if (!installPrebuiltLmdb()) return undefined + + await createInternalPackagesCacheDir() + + const cacheDir = getInternalPackagesCacheDir() + const options: ExecaOptions = { + stderr: `inherit`, + cwd: cacheDir, + } + + const npmAdditionalCliArgs = [ + `--no-progress`, + `--no-audit`, + `--no-fund`, + `--loglevel`, + `error`, + `--color`, + `always`, + `--legacy-peer-deps`, + `--save-exact`, + ] + + await execa( + `npm`, + [`install`, ...npmAdditionalCliArgs, `${lmdbPackage}@${dependencies.lmdb}`], + options + ) + + return path.join(cacheDir, `node_modules`, lmdbPackage) +} + export async function createGraphqlEngineBundle( rootDir: string, reporter: Reporter, @@ -57,6 +146,19 @@ export async function createGraphqlEngineBundle( require.resolve(`gatsby-plugin-typescript`) ) + // Alternative lmdb path we've created to self heal from a "broken" lmdb installation + const alternativeLmdbPath = await installIfMissingLmdb() + + // We force a specific lmdb binary module if we detected a broken lmdb installation or if we detect the presence of an adapter + let forcedLmdbBinaryModule: string | undefined = undefined + if (store.getState().adapter.instance) { + forcedLmdbBinaryModule = `${lmdbPackage}/node.abi83.glibc.node` + } + // We always force the binary if we've installed an alternative path + if (alternativeLmdbPath) { + forcedLmdbBinaryModule = `${alternativeLmdbPath}/node.abi83.glibc.node` + } + const compiler = webpack({ name: `Query Engine`, // mode: `production`, @@ -121,9 +223,7 @@ export async function createGraphqlEngineBundle( { loader: require.resolve(`./lmdb-bundling-patch`), options: { - forcedBinaryModule: store.getState().adapter.instance - ? `@lmdb/lmdb-${process.platform}-${process.arch}/node.abi83.glibc.node` - : undefined, + forcedBinaryModule: forcedLmdbBinaryModule, }, }, ],