add integration test

integration-tests/gatsby-pipeline/__tests__/fetch-remote-file/index.js

@@ -0,0 +1,63 @@
+/**
+ * We want to make sure that fetch-remote-file is working with multi workers.
+ */
+
+const execa = require(`execa`)
+const path = require(`path`)
+const glob = require(`glob`)
+const fs = require(`fs-extra`)
+const md5File = require(`md5-file`)
+const basePath = path.resolve(__dirname, `../../`)
+
+const cleanDirs = () =>
+  Promise.all([
+    fs.emptyDir(`${basePath}/public`),
+    fs.emptyDir(`${basePath}/.cache`),
+  ])
+
+describe(`fetch-remote-file`, () => {
+  beforeAll(async () => {
+    await cleanDirs()
+    await execa(`yarn`, [`build`], {
+      cwd: basePath,
+      // we want to force 1 query per worker
+      env: { NODE_ENV: `production`, GATSBY_PARALLEL_QUERY_CHUNK_SIZE: `1` },
+    })
+  }, 60 * 1000)
+
+  it("should have the correct md5", async () => {
+    expect(
+      await md5File(
+        path.join(
+          __dirname,
+          "../..",
+          "public/images/50c58a791de3c2303e62084d731799eb/photoA.jpg"
+        )
+      )
+    ).toEqual("a9e57a66a10b2d26a1999a4685d7c9ef")
+    expect(
+      await md5File(
+        path.join(
+          __dirname,
+          "../..",
+          "public/images/4910e745c3c453b8795d6ba65c79d99b/photoB.jpg"
+        )
+      )
+    ).toEqual("c305dc5c5db45cc773231a507af5116d")
+    expect(
+      await md5File(
+        path.join(
+          __dirname,
+          "../..",
+          "public/images/fb673e75e9534b3cc2d2e24085386d48/photoC.jpg"
+        )
+      )
+    ).toEqual("4ba953ba27236727d7abe7d5b8916432")
+  })
+
+  it("should have conflict between workers", async () => {
+    const files = await fs.readdir(path.join(__dirname, "../../.cache/workers"))
+
+    expect(files.length).toBe(4)
+  })
+})

integration-tests/gatsby-pipeline/gatsby-node.js

@@ -0,0 +1,69 @@
+const { fetchRemoteFile } = require("gatsby-core-utils/fetch-remote-file")
+const { slash } = require("gatsby-core-utils")
+const path = require("path")
+const fs = require("fs-extra")
+
+/** @type{import('gatsby').createSchemaCustomization} */
+exports.createSchemaCustomization = ({ actions, schema, cache, reporter }) => {
+  actions.createTypes(
+    schema.buildObjectType({
+      name: "MyRemoteFile",
+      fields: {
+        url: "String!",
+        publicUrl: {
+          type: "String!",
+          async resolve(source) {
+            const filePath = await fetchRemoteFile({
+              name: path.basename(source.name, path.extname(source.name)),
+              ext: path.extname(source.name),
+              url: source.url,
+              directory: "./public/images",
+            })
+
+            const dir = path.join(global.__GATSBY.root, ".cache", "workers")
+            await fs.ensureDir(dir)
+            await fs.createFile(
+              `${path.join(dir, `worker-${process.env.GATSBY_WORKER_ID}`)}`
+            )
+
+            const workers = (await cache.get("workers")) ?? []
+            workers.push(process.env.GATSBY_WORKER_ID)
+
+            return `${slash(filePath.replace(/^public/, ""))}`
+          },
+        },
+      },
+      interfaces: ["Node"],
+    })
+  )
+}
+
+/** @type {imporg('gatsby').sourceNodes} */
+exports.sourceNodes = ({ actions, createNodeId, createContentDigest }) => {
+  const items = [
+    {
+      name: "photoA.jpg",
+      url: "https://images.unsplash.com/photo-1517849845537-4d257902454a?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
+    },
+    {
+      name: "photoB.jpg",
+      url: "https://images.unsplash.com/photo-1552053831-71594a27632d?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
+    },
+    {
+      name: "photoC.jpg",
+      url: "https://images.unsplash.com/photo-1561037404-61cd46aa615b?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80",
+    },
+  ]
+
+  items.forEach((item, index) => {
+    actions.createNode({
+      id: createNodeId(`remote-file-${index}`),
+      name: item.name,
+      url: item.url,
+      internal: {
+        type: "MyRemoteFile",
+        contentDigest: createContentDigest(item.url),
+      },
+    })
+  })
+}

integration-tests/gatsby-pipeline/package.json

@@ -4,13 +4,13 @@
   "version": "1.0.0",
   "author": "Kyle Mathews <[email protected]>",
   "dependencies": {
-    "gatsby": "latest",
+    "gatsby": "4.8.0-next.1-dev-1645011114818",
     "gatsby-image": "latest",
-    "gatsby-plugin-gatsby-cloud": "latest",
-    "gatsby-plugin-react-helmet": "latest",
-    "gatsby-plugin-sharp": "latest",
-    "gatsby-source-filesystem": "latest",
-    "gatsby-transformer-sharp": "latest",
+    "gatsby-plugin-gatsby-cloud": "4.8.0-next.1-dev-1645011114818",
+    "gatsby-plugin-react-helmet": "5.8.0-next.0-dev-1645011114818",
+    "gatsby-plugin-sharp": "4.8.0-next.0-dev-1645011114818",
+    "gatsby-source-filesystem": "4.8.0-next.0-dev-1645011114818",
+    "gatsby-transformer-sharp": "4.8.0-next.0-dev-1645011114818",
     "prop-types": "^15.7.2",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
@@ -26,11 +26,12 @@
     "test": "jest"
   },
   "devDependencies": {
+    "cross-env": "^7.0.3",
     "execa": "^4.0.3",
     "fs-extra": "^9.0.1",
+    "jest": "^27.2.1",
     "md5-file": "^5.0.0",
     "node-fetch": "^2.6.0",
-    "jest": "^27.2.1",
     "tree-kill": "^1.2.2"
   },
   "repository": {
@@ -40,4 +41,4 @@
   "engines": {
     "node": ">=12.13.0"
   }
-}
+}

integration-tests/gatsby-pipeline/src/pages/fetch-remote-a.js

@@ -0,0 +1,32 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteA = ({ data }) => {
+  console.log({ data })
+  return (
+    <Layout>
+      <SEO title="Fetch Remote A" />
+
+      <pre
+        dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
+      />
+      <Link to="/">Go back to the homepage</Link>
+    </Layout>
+  )
+}
+
+export default FetchRemoteA
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`

integration-tests/gatsby-pipeline/src/pages/fetch-remote-b.js

@@ -0,0 +1,32 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    <Layout>
+      <SEO title="Fetch Remote B" />
+
+      <pre
+        dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
+      />
+
+      <Link to="/">Go back to the homepage</Link>
+    </Layout>
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`

integration-tests/gatsby-pipeline/src/pages/fetch-remote-c.js

@@ -0,0 +1,31 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    <Layout>
+      <SEO title="Fetch Remote C" />
+
+      <pre
+        dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
+      />
+      <Link to="/">Go back to the homepage</Link>
+    </Layout>
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`

integration-tests/gatsby-pipeline/src/pages/fetch-remote-d.js

@@ -0,0 +1,31 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    <Layout>
+      <SEO title="Fetch Remote B" />
+
+      <pre
+        dangerouslySetInnerHTML={{ __html: JSON.stringify(data, null, 2) }}
+      />
+      <Link to="/">Go back to the homepage</Link>
+    </Layout>
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`

packages/gatsby-core-utils/src/fetch-remote-file.ts

@@ -8,6 +8,7 @@ import {
   getRemoteFileExtension,
   createFilePath,
 } from "./filename-utils"
+import { slash } from "./path"
 import { requestRemoteNode } from "./remote-file-utils/fetch-file"
 import { getStorage, getDatabaseDir } from "./utils/get-storage"
 import { createMutex } from "./mutex"
@@ -182,7 +183,9 @@ async function fetchFile({
       httpOptions
     )
 
-    const filename = createFilePath(path.join(fileDirectory, digest), name, ext)
+    const filename = slash(
+      createFilePath(path.join(fileDirectory, digest), name, ext)
+    )
     if (response.statusCode === 200) {
       // Save the response headers for future requests.
       // If the user did not provide an extension and we couldn't get one from remote file, try and guess one
@@ -200,7 +203,7 @@ async function fetchFile({
         cacheKey,
         extension: ext,
         headers: response.headers.etag ? { etag: response.headers.etag } : {},
-        directory: fileDirectory,
+        directory: slash(fileDirectory),
         path: filename.replace(fileDirectory, ``),
       })
     } else if (response.statusCode === 304) {