fix: re-install lmdb when detecting the absence of @LMDB prebuilt pac…

…kages (#38691)

* fix: re-install lmdb when detecting the absence of @LMDB prebuilt packages

* chore: set the correct root lmdb and add further comments

* chore: linter

* chore: linting

* chore: make the install check for reliable and install @LMDB instead of lmdb

* fix: no need to touch the patch, we can leverage the force binary flag

* chore: add integration test for lmdb fix

* chore: add CCI config for new integration test

* add integration_tests_esm_in_gatsby_files to jobs

* fix: integration test for lmdb-regeneration

* fix: cli path in test

* chore: actually add lmdb's dist directory

* fix: back to the npmrc and just rm -rf @LMDB modules

---------

Co-authored-by: Michal Piechowiak <[email protected]>

.circleci/config.yml

@@ -313,6 +313,13 @@ jobs:
           test_path: integration-tests/esm-in-gatsby-files
           test_command: yarn test
 
+  integration_tests_lmdb_regeneration:
+    executor: node
+    steps:
+      - e2e-test:
+          test_path: integration-tests/lmdb-regeneration
+          test_command: yarn test
+
   e2e_tests_path-prefix:
     <<: *e2e-executor
     steps:
@@ -590,6 +597,8 @@ workflows:
           <<: *e2e-test-workflow
       - integration_tests_esm_in_gatsby_files:
           <<: *e2e-test-workflow
+      - integration_tests_lmdb_regeneration:
+          <<: *e2e-test-workflow
       - integration_tests_gatsby_cli:
           requires:
             - bootstrap

integration-tests/lmdb-regeneration/.gitignore

@@ -0,0 +1,3 @@
+__tests__/__debug__
+node_modules
+yarn.lock

integration-tests/lmdb-regeneration/.npmrc

@@ -0,0 +1 @@
+build_from_source=true

integration-tests/lmdb-regeneration/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

integration-tests/lmdb-regeneration/README.md

@@ -0,0 +1,3 @@
+## Artifacts test suite
+
+This integration test suite helps us assert our mechanism to heal back from a broken lmdb installation is working by validating we install and use the correct prebundled binary for our platform

integration-tests/lmdb-regeneration/__tests__/index.js

@@ -0,0 +1,59 @@
+const path = require(`path`)
+const execa = require(`execa`)
+const mod = require("module")
+const fs = require(`fs-extra`)
+
+jest.setTimeout(100000)
+
+const rootPath = path.resolve(__dirname, "../")
+
+describe(`Lmdb regeneration`, () => {
+  test(`gatbsy build detects lmdb setup built from source and installs pre-buit package`, async () => {
+    const lmdbNodeModulesPath = path.resolve(rootPath, "node_modules", "lmdb")
+    // Make sure we clear out the current `@lmdb` optional dependencies
+    const pathsToRemove = [
+      path.resolve(rootPath, "node_modules", "@lmdb"),
+      path.resolve(rootPath, "node_modules", "gatsby", "node_modules", "@lmdb"),
+    ]
+    for (let path of pathsToRemove) {
+      fs.rmSync(path, { force: true, recursive: true })
+    }
+    // Check the lmdb instance we have installed does have a binary built from source since we need it to reproduce the fix we're trying to test
+    // If this check fails then it means our fixture is wrong and we're relying on an lmdb instance with prebuilt binaries
+    const builtFromSource = fs.existsSync(
+      path.resolve(lmdbNodeModulesPath, "build", "Release", "lmdb.node")
+    )
+    expect(builtFromSource).toEqual(true)
+
+    const options = {
+      stderr: `inherit`,
+      stdout: `inherit`,
+      cwd: rootPath,
+    }
+    const gatsbyBin = path.resolve(rootPath, `node_modules`, `gatsby`, `cli.js`)
+    await execa(gatsbyBin, [`build`], options)
+
+    // lmdb module with prebuilt binaries for our platform
+    const lmdbPackage = `@lmdb/lmdb-${process.platform}-${process.arch}`
+
+    // If the fix worked correctly we should have installed the prebuilt binary for our platform under our `.cache` directory
+    const lmdbRequire = mod.createRequire(
+      path.resolve(rootPath, ".cache", "internal-packages", "package.json")
+    )
+    expect(() => {
+      lmdbRequire.resolve(lmdbPackage)
+    }).not.toThrow()
+
+    // The resulting query-engine bundle should not contain the binary built from source
+    const binaryBuiltFromSource = path.resolve(
+      rootPath,
+      ".cache",
+      "query-engine",
+      "assets",
+      "build",
+      "Release",
+      "lmdb.node"
+    )
+    expect(fs.existsSync(binaryBuiltFromSource)).toEqual(false)
+  })
+})

integration-tests/lmdb-regeneration/gatsby-config.js

@@ -0,0 +1,9 @@
+module.exports = {
+  siteMetadata: {
+    siteUrl: `https://www.yourdomain.tld`,
+  },
+  plugins: [],
+  flags: {
+    DEV_SSR: true,
+  },
+}

integration-tests/lmdb-regeneration/jest.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  testPathIgnorePatterns: [`/node_modules/`, `__tests__/fixtures`, `.cache`],
+}

integration-tests/lmdb-regeneration/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "lmdb-regeneration",
+  "private": true,
+  "author": "Sid Chatterjee",
+  "description": "A simplified bare-bones starter for Gatsby with DSG",
+  "version": "0.1.0",
+  "license": "MIT",
+  "scripts": {
+    "build": "gatsby build",
+    "clean": "gatsby clean",
+    "test": "jest --runInBand"
+  },
+  "dependencies": {
+    "gatsby": "next",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  },
+  "devDependencies": {
+    "fs-extra": "^11.1.0",
+    "jest": "^29.3.1"
+  }
+}

integration-tests/lmdb-regeneration/src/pages/404.jsx

@@ -0,0 +1,49 @@
+import * as React from "react"
+import { Link } from "gatsby"
+
+const pageStyles = {
+  color: "#232129",
+  padding: "96px",
+  fontFamily: "-apple-system, Roboto, sans-serif, serif",
+}
+const headingStyles = {
+  marginTop: 0,
+  marginBottom: 64,
+  maxWidth: 320,
+}
+
+const paragraphStyles = {
+  marginBottom: 48,
+}
+const codeStyles = {
+  color: "#8A6534",
+  padding: 4,
+  backgroundColor: "#FFF4DB",
+  fontSize: "1.25rem",
+  borderRadius: 4,
+}
+
+const NotFoundPage = () => {
+  return (
+    <main style={pageStyles}>
+      <h1 style={headingStyles}>Page not found</h1>
+      <p style={paragraphStyles}>
+        Sorry 😔, we couldn’t find what you were looking for.
+        <br />
+        {process.env.NODE_ENV === "development" ? (
+          <>
+            <br />
+            Try creating a page in <code style={codeStyles}>src/pages/</code>.
+            <br />
+          </>
+        ) : null}
+        <br />
+        <Link to="/">Go home</Link>.
+      </p>
+    </main>
+  )
+}
+
+export default NotFoundPage
+
+export const Head = () => <title>Not found</title>

integration-tests/lmdb-regeneration/src/pages/test-dsg.jsx

@@ -0,0 +1,17 @@
+import * as React from "react";
+
+const DSG = () => {
+  return <h1>DSG</h1>;
+};
+
+export default DSG;
+
+export const Head = () => <title>DSG</title>;
+
+export async function config() {
+  return () => {
+    return {
+      defer: true,
+    };
+  };
+}

packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts

@@ -2,8 +2,10 @@
 
 import * as path from "path"
 import * as fs from "fs-extra"
+import execa, { Options as ExecaOptions } from "execa"
 import webpack, { Module, NormalModule, Compilation } from "webpack"
 import ConcatenatedModule from "webpack/lib/optimize/ConcatenatedModule"
+import { dependencies } from "gatsby/package.json"
 import { printQueryEnginePlugins } from "./print-plugins"
 import mod from "module"
 import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging"
@@ -12,6 +14,7 @@ import { schemaCustomizationAPIs } from "./print-plugins"
 import type { GatsbyNodeAPI } from "../../redux/types"
 import * as nodeApis from "../../utils/api-node-docs"
 import { store } from "../../redux"
+import { PackageJson } from "../../.."
 
 type Reporter = typeof reporter
 
@@ -35,6 +38,92 @@ function getApisToRemoveForQueryEngine(): Array<GatsbyNodeAPI> {
   return apisToRemove
 }
 
+const getInternalPackagesCacheDir = (): string =>
+  path.join(process.cwd(), `.cache/internal-packages`)
+
+// Create a directory and JS module where we install internally used packages
+const createInternalPackagesCacheDir = async (): Promise<void> => {
+  const cacheDir = getInternalPackagesCacheDir()
+  await fs.ensureDir(cacheDir)
+  await fs.emptyDir(cacheDir)
+
+  const packageJsonPath = path.join(cacheDir, `package.json`)
+
+  await fs.outputJson(packageJsonPath, {
+    name: `gatsby-internal-packages`,
+    description: `This directory contains internal packages installed by Gatsby used to comply with the current platform requirements`,
+    version: `1.0.0`,
+    private: true,
+    author: `Gatsby`,
+    license: `MIT`,
+  })
+}
+
+// lmdb module with prebuilt binaries for our platform
+const lmdbPackage = `@lmdb/lmdb-${process.platform}-${process.arch}`
+
+// Detect if the prebuilt binaries for lmdb have been installed. These are installed under @lmdb and are tied to each platform/arch. We've seen instances where regular installations lack these modules because of a broken lockfile or skipping optional dependencies installs
+function installPrebuiltLmdb(): boolean {
+  // Read lmdb's package.json, go through its optional depedencies and validate if there's a prebuilt lmdb module with a compatible binary to our platform and arch
+  let packageJson: PackageJson
+  try {
+    const modulePath = path
+      .dirname(require.resolve(`lmdb`))
+      .replace(`/dist`, ``)
+    const packageJsonPath = path.join(modulePath, `package.json`)
+    packageJson = JSON.parse(fs.readFileSync(packageJsonPath, `utf-8`))
+  } catch (e) {
+    // If we fail to read lmdb's package.json there's bigger problems here so just skip installation
+    return false
+  }
+  // If there's no lmdb prebuilt package for our arch/platform listed as optional dep no point in trying to install it
+  const { optionalDependencies } = packageJson
+  if (!optionalDependencies) return false
+  if (!Object.keys(optionalDependencies).find(p => p === lmdbPackage))
+    return false
+  try {
+    const lmdbRequire = mod.createRequire(require.resolve(`lmdb`))
+    lmdbRequire.resolve(lmdbPackage)
+    return false
+  } catch (e) {
+    return true
+  }
+}
+
+// Install lmdb's native system module under our internal cache if we detect the current installation
+// isn't using the pre-build binaries
+async function installIfMissingLmdb(): Promise<string | undefined> {
+  if (!installPrebuiltLmdb()) return undefined
+
+  await createInternalPackagesCacheDir()
+
+  const cacheDir = getInternalPackagesCacheDir()
+  const options: ExecaOptions = {
+    stderr: `inherit`,
+    cwd: cacheDir,
+  }
+
+  const npmAdditionalCliArgs = [
+    `--no-progress`,
+    `--no-audit`,
+    `--no-fund`,
+    `--loglevel`,
+    `error`,
+    `--color`,
+    `always`,
+    `--legacy-peer-deps`,
+    `--save-exact`,
+  ]
+
+  await execa(
+    `npm`,
+    [`install`, ...npmAdditionalCliArgs, `${lmdbPackage}@${dependencies.lmdb}`],
+    options
+  )
+
+  return path.join(cacheDir, `node_modules`, lmdbPackage)
+}
+
 export async function createGraphqlEngineBundle(
   rootDir: string,
   reporter: Reporter,
@@ -57,6 +146,19 @@ export async function createGraphqlEngineBundle(
     require.resolve(`gatsby-plugin-typescript`)
   )
 
+  // Alternative lmdb path we've created to self heal from a "broken" lmdb installation
+  const alternativeLmdbPath = await installIfMissingLmdb()
+
+  // We force a specific lmdb binary module if we detected a broken lmdb installation or if we detect the presence of an adapter
+  let forcedLmdbBinaryModule: string | undefined = undefined
+  if (store.getState().adapter.instance) {
+    forcedLmdbBinaryModule = `${lmdbPackage}/node.abi83.glibc.node`
+  }
+  // We always force the binary if we've installed an alternative path
+  if (alternativeLmdbPath) {
+    forcedLmdbBinaryModule = `${alternativeLmdbPath}/node.abi83.glibc.node`
+  }
+
   const compiler = webpack({
     name: `Query Engine`,
     // mode: `production`,
@@ -121,9 +223,7 @@ export async function createGraphqlEngineBundle(
                 {
                   loader: require.resolve(`./lmdb-bundling-patch`),
                   options: {
-                    forcedBinaryModule: store.getState().adapter.instance
-                      ? `@lmdb/lmdb-${process.platform}-${process.arch}/node.abi83.glibc.node`
-                      : undefined,
+                    forcedBinaryModule: forcedLmdbBinaryModule,
                   },
                 },
               ],