diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2af8a75829f42..e17a153324ebc 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -313,6 +313,13 @@ jobs:
test_path: integration-tests/esm-in-gatsby-files
test_command: yarn test
+ integration_tests_lmdb_regeneration:
+ executor: node
+ steps:
+ - e2e-test:
+ test_path: integration-tests/lmdb-regeneration
+ test_command: yarn test
+
e2e_tests_path-prefix:
<<: *e2e-executor
steps:
@@ -590,6 +597,8 @@ workflows:
<<: *e2e-test-workflow
- integration_tests_esm_in_gatsby_files:
<<: *e2e-test-workflow
+ - integration_tests_lmdb_regeneration:
+ <<: *e2e-test-workflow
- integration_tests_gatsby_cli:
requires:
- bootstrap
diff --git a/integration-tests/lmdb-regeneration/.gitignore b/integration-tests/lmdb-regeneration/.gitignore
new file mode 100644
index 0000000000000..f9fd0a59de1f5
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/.gitignore
@@ -0,0 +1,3 @@
+__tests__/__debug__
+node_modules
+yarn.lock
diff --git a/integration-tests/lmdb-regeneration/.npmrc b/integration-tests/lmdb-regeneration/.npmrc
new file mode 100644
index 0000000000000..1a456a8d1c769
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/.npmrc
@@ -0,0 +1 @@
+build_from_source=true
diff --git a/integration-tests/lmdb-regeneration/LICENSE b/integration-tests/lmdb-regeneration/LICENSE
new file mode 100644
index 0000000000000..20f91f2b3c52b
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/integration-tests/lmdb-regeneration/README.md b/integration-tests/lmdb-regeneration/README.md
new file mode 100644
index 0000000000000..8df148ba4dc0d
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/README.md
@@ -0,0 +1,3 @@
+## Artifacts test suite
+
+This integration test suite helps us assert our mechanism to heal back from a broken lmdb installation is working by validating we install and use the correct prebundled binary for our platform
diff --git a/integration-tests/lmdb-regeneration/__tests__/index.js b/integration-tests/lmdb-regeneration/__tests__/index.js
new file mode 100644
index 0000000000000..3da14d6ce0d54
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/__tests__/index.js
@@ -0,0 +1,59 @@
+const path = require(`path`)
+const execa = require(`execa`)
+const mod = require("module")
+const fs = require(`fs-extra`)
+
+jest.setTimeout(100000)
+
+const rootPath = path.resolve(__dirname, "../")
+
+describe(`Lmdb regeneration`, () => {
+ test(`gatbsy build detects lmdb setup built from source and installs pre-buit package`, async () => {
+ const lmdbNodeModulesPath = path.resolve(rootPath, "node_modules", "lmdb")
+ // Make sure we clear out the current `@lmdb` optional dependencies
+ const pathsToRemove = [
+ path.resolve(rootPath, "node_modules", "@lmdb"),
+ path.resolve(rootPath, "node_modules", "gatsby", "node_modules", "@lmdb"),
+ ]
+ for (let path of pathsToRemove) {
+ fs.rmSync(path, { force: true, recursive: true })
+ }
+ // Check the lmdb instance we have installed does have a binary built from source since we need it to reproduce the fix we're trying to test
+ // If this check fails then it means our fixture is wrong and we're relying on an lmdb instance with prebuilt binaries
+ const builtFromSource = fs.existsSync(
+ path.resolve(lmdbNodeModulesPath, "build", "Release", "lmdb.node")
+ )
+ expect(builtFromSource).toEqual(true)
+
+ const options = {
+ stderr: `inherit`,
+ stdout: `inherit`,
+ cwd: rootPath,
+ }
+ const gatsbyBin = path.resolve(rootPath, `node_modules`, `gatsby`, `cli.js`)
+ await execa(gatsbyBin, [`build`], options)
+
+ // lmdb module with prebuilt binaries for our platform
+ const lmdbPackage = `@lmdb/lmdb-${process.platform}-${process.arch}`
+
+ // If the fix worked correctly we should have installed the prebuilt binary for our platform under our `.cache` directory
+ const lmdbRequire = mod.createRequire(
+ path.resolve(rootPath, ".cache", "internal-packages", "package.json")
+ )
+ expect(() => {
+ lmdbRequire.resolve(lmdbPackage)
+ }).not.toThrow()
+
+ // The resulting query-engine bundle should not contain the binary built from source
+ const binaryBuiltFromSource = path.resolve(
+ rootPath,
+ ".cache",
+ "query-engine",
+ "assets",
+ "build",
+ "Release",
+ "lmdb.node"
+ )
+ expect(fs.existsSync(binaryBuiltFromSource)).toEqual(false)
+ })
+})
diff --git a/integration-tests/lmdb-regeneration/gatsby-config.js b/integration-tests/lmdb-regeneration/gatsby-config.js
new file mode 100644
index 0000000000000..9e31e5d618ad2
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/gatsby-config.js
@@ -0,0 +1,9 @@
+module.exports = {
+ siteMetadata: {
+ siteUrl: `https://www.yourdomain.tld`,
+ },
+ plugins: [],
+ flags: {
+ DEV_SSR: true,
+ },
+}
diff --git a/integration-tests/lmdb-regeneration/jest.config.js b/integration-tests/lmdb-regeneration/jest.config.js
new file mode 100644
index 0000000000000..4e5a78b25d7bf
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/jest.config.js
@@ -0,0 +1,3 @@
+module.exports = {
+ testPathIgnorePatterns: [`/node_modules/`, `__tests__/fixtures`, `.cache`],
+}
diff --git a/integration-tests/lmdb-regeneration/package.json b/integration-tests/lmdb-regeneration/package.json
new file mode 100644
index 0000000000000..7cd2ca82f4e72
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/package.json
@@ -0,0 +1,22 @@
+{
+ "name": "lmdb-regeneration",
+ "private": true,
+ "author": "Sid Chatterjee",
+ "description": "A simplified bare-bones starter for Gatsby with DSG",
+ "version": "0.1.0",
+ "license": "MIT",
+ "scripts": {
+ "build": "gatsby build",
+ "clean": "gatsby clean",
+ "test": "jest --runInBand"
+ },
+ "dependencies": {
+ "gatsby": "next",
+ "react": "^18.2.0",
+ "react-dom": "^18.2.0"
+ },
+ "devDependencies": {
+ "fs-extra": "^11.1.0",
+ "jest": "^29.3.1"
+ }
+}
diff --git a/integration-tests/lmdb-regeneration/src/images/icon.png b/integration-tests/lmdb-regeneration/src/images/icon.png
new file mode 100644
index 0000000000000..38b2fb0e467e0
Binary files /dev/null and b/integration-tests/lmdb-regeneration/src/images/icon.png differ
diff --git a/integration-tests/lmdb-regeneration/src/pages/404.jsx b/integration-tests/lmdb-regeneration/src/pages/404.jsx
new file mode 100644
index 0000000000000..d7b40627e81ae
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/src/pages/404.jsx
@@ -0,0 +1,49 @@
+import * as React from "react"
+import { Link } from "gatsby"
+
+const pageStyles = {
+ color: "#232129",
+ padding: "96px",
+ fontFamily: "-apple-system, Roboto, sans-serif, serif",
+}
+const headingStyles = {
+ marginTop: 0,
+ marginBottom: 64,
+ maxWidth: 320,
+}
+
+const paragraphStyles = {
+ marginBottom: 48,
+}
+const codeStyles = {
+ color: "#8A6534",
+ padding: 4,
+ backgroundColor: "#FFF4DB",
+ fontSize: "1.25rem",
+ borderRadius: 4,
+}
+
+const NotFoundPage = () => {
+ return (
+
+ Page not found
+
+ Sorry 😔, we couldn’t find what you were looking for.
+
+ {process.env.NODE_ENV === "development" ? (
+ <>
+
+ Try creating a page in src/pages/
.
+
+ >
+ ) : null}
+
+ Go home.
+
+
+ )
+}
+
+export default NotFoundPage
+
+export const Head = () =>
Not found
diff --git a/integration-tests/lmdb-regeneration/src/pages/test-dsg.jsx b/integration-tests/lmdb-regeneration/src/pages/test-dsg.jsx
new file mode 100644
index 0000000000000..db06e1909a02a
--- /dev/null
+++ b/integration-tests/lmdb-regeneration/src/pages/test-dsg.jsx
@@ -0,0 +1,17 @@
+import * as React from "react";
+
+const DSG = () => {
+ return DSG
;
+};
+
+export default DSG;
+
+export const Head = () => DSG;
+
+export async function config() {
+ return () => {
+ return {
+ defer: true,
+ };
+ };
+}
diff --git a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts
index 8b5d99f4e41c2..4b87d2e5115a9 100644
--- a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts
+++ b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts
@@ -2,8 +2,10 @@
import * as path from "path"
import * as fs from "fs-extra"
+import execa, { Options as ExecaOptions } from "execa"
import webpack, { Module, NormalModule, Compilation } from "webpack"
import ConcatenatedModule from "webpack/lib/optimize/ConcatenatedModule"
+import { dependencies } from "gatsby/package.json"
import { printQueryEnginePlugins } from "./print-plugins"
import mod from "module"
import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging"
@@ -12,6 +14,7 @@ import { schemaCustomizationAPIs } from "./print-plugins"
import type { GatsbyNodeAPI } from "../../redux/types"
import * as nodeApis from "../../utils/api-node-docs"
import { store } from "../../redux"
+import { PackageJson } from "../../.."
type Reporter = typeof reporter
@@ -35,6 +38,92 @@ function getApisToRemoveForQueryEngine(): Array {
return apisToRemove
}
+const getInternalPackagesCacheDir = (): string =>
+ path.join(process.cwd(), `.cache/internal-packages`)
+
+// Create a directory and JS module where we install internally used packages
+const createInternalPackagesCacheDir = async (): Promise => {
+ const cacheDir = getInternalPackagesCacheDir()
+ await fs.ensureDir(cacheDir)
+ await fs.emptyDir(cacheDir)
+
+ const packageJsonPath = path.join(cacheDir, `package.json`)
+
+ await fs.outputJson(packageJsonPath, {
+ name: `gatsby-internal-packages`,
+ description: `This directory contains internal packages installed by Gatsby used to comply with the current platform requirements`,
+ version: `1.0.0`,
+ private: true,
+ author: `Gatsby`,
+ license: `MIT`,
+ })
+}
+
+// lmdb module with prebuilt binaries for our platform
+const lmdbPackage = `@lmdb/lmdb-${process.platform}-${process.arch}`
+
+// Detect if the prebuilt binaries for lmdb have been installed. These are installed under @lmdb and are tied to each platform/arch. We've seen instances where regular installations lack these modules because of a broken lockfile or skipping optional dependencies installs
+function installPrebuiltLmdb(): boolean {
+ // Read lmdb's package.json, go through its optional depedencies and validate if there's a prebuilt lmdb module with a compatible binary to our platform and arch
+ let packageJson: PackageJson
+ try {
+ const modulePath = path
+ .dirname(require.resolve(`lmdb`))
+ .replace(`/dist`, ``)
+ const packageJsonPath = path.join(modulePath, `package.json`)
+ packageJson = JSON.parse(fs.readFileSync(packageJsonPath, `utf-8`))
+ } catch (e) {
+ // If we fail to read lmdb's package.json there's bigger problems here so just skip installation
+ return false
+ }
+ // If there's no lmdb prebuilt package for our arch/platform listed as optional dep no point in trying to install it
+ const { optionalDependencies } = packageJson
+ if (!optionalDependencies) return false
+ if (!Object.keys(optionalDependencies).find(p => p === lmdbPackage))
+ return false
+ try {
+ const lmdbRequire = mod.createRequire(require.resolve(`lmdb`))
+ lmdbRequire.resolve(lmdbPackage)
+ return false
+ } catch (e) {
+ return true
+ }
+}
+
+// Install lmdb's native system module under our internal cache if we detect the current installation
+// isn't using the pre-build binaries
+async function installIfMissingLmdb(): Promise {
+ if (!installPrebuiltLmdb()) return undefined
+
+ await createInternalPackagesCacheDir()
+
+ const cacheDir = getInternalPackagesCacheDir()
+ const options: ExecaOptions = {
+ stderr: `inherit`,
+ cwd: cacheDir,
+ }
+
+ const npmAdditionalCliArgs = [
+ `--no-progress`,
+ `--no-audit`,
+ `--no-fund`,
+ `--loglevel`,
+ `error`,
+ `--color`,
+ `always`,
+ `--legacy-peer-deps`,
+ `--save-exact`,
+ ]
+
+ await execa(
+ `npm`,
+ [`install`, ...npmAdditionalCliArgs, `${lmdbPackage}@${dependencies.lmdb}`],
+ options
+ )
+
+ return path.join(cacheDir, `node_modules`, lmdbPackage)
+}
+
export async function createGraphqlEngineBundle(
rootDir: string,
reporter: Reporter,
@@ -57,6 +146,19 @@ export async function createGraphqlEngineBundle(
require.resolve(`gatsby-plugin-typescript`)
)
+ // Alternative lmdb path we've created to self heal from a "broken" lmdb installation
+ const alternativeLmdbPath = await installIfMissingLmdb()
+
+ // We force a specific lmdb binary module if we detected a broken lmdb installation or if we detect the presence of an adapter
+ let forcedLmdbBinaryModule: string | undefined = undefined
+ if (store.getState().adapter.instance) {
+ forcedLmdbBinaryModule = `${lmdbPackage}/node.abi83.glibc.node`
+ }
+ // We always force the binary if we've installed an alternative path
+ if (alternativeLmdbPath) {
+ forcedLmdbBinaryModule = `${alternativeLmdbPath}/node.abi83.glibc.node`
+ }
+
const compiler = webpack({
name: `Query Engine`,
// mode: `production`,
@@ -121,9 +223,7 @@ export async function createGraphqlEngineBundle(
{
loader: require.resolve(`./lmdb-bundling-patch`),
options: {
- forcedBinaryModule: store.getState().adapter.instance
- ? `@lmdb/lmdb-${process.platform}-${process.arch}/node.abi83.glibc.node`
- : undefined,
+ forcedBinaryModule: forcedLmdbBinaryModule,
},
},
],