[v2] improve how we load GraphQL query results in development/production (aka Ludicrous Mode)

packages/gatsby-source-filesystem/package.json

@@ -16,7 +16,8 @@
     "mime": "^2.2.0",
     "pretty-bytes": "^4.0.2",
     "slash": "^1.0.0",
-    "valid-url": "^1.0.9"
+    "valid-url": "^1.0.9",
+    "xstate": "^3.1.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0-beta.42",

packages/gatsby-source-filesystem/src/create-file-node.js

@@ -7,13 +7,6 @@ const prettyBytes = require(`pretty-bytes`)
 const md5File = require(`bluebird`).promisify(require(`md5-file`))
 const crypto = require(`crypto`)
 
-const createId = path => {
-  const slashed = slash(path)
-  return `${slashed} absPath of file`
-}
-
-exports.createId = createId
-
 exports.createFileNode = async (
   pathToFile,
   createNodeId,

packages/gatsby-source-filesystem/src/gatsby-node.js

@@ -1,10 +1,77 @@
 const chokidar = require(`chokidar`)
 const fs = require(`fs`)
+const { Machine } = require(`xstate`)
 
 const { createId, createFileNode } = require(`./create-file-node`)
 
+/**
+ * Create a state machine to manage Chokidar's not-ready/ready states and for
+ * emitting file system events into Gatsby.
+ *
+ * On the latter, this solves the problem where if you call createNode for the
+ * same File node in quick succession, this can leave Gatsby's internal state
+ * in disarray causing queries to fail. The latter state machine tracks when
+ * Gatsby is "processing" a node update or when it's "idle". If updates come in
+ * while Gatsby is processing, we queue them until the system returns to an
+ * "idle" state.
+ */
+const fsMachine = Machine({
+  key: "emitFSEvents",
+  parallel: true,
+  strict: true,
+  states: {
+    CHOKIDAR: {
+      initial: `CHOKIDAR_NOT_READY`,
+      states: {
+        CHOKIDAR_NOT_READY: {
+          on: {
+            CHOKIDAR_READY: "CHOKIDAR_WATCHING",
+            BOOTSTRAP_FINISHED: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED",
+          },
+        },
+        CHOKIDAR_WATCHING: {
+          on: {
+            BOOTSTRAP_FINISHED: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED",
+            CHOKIDAR_READY: "CHOKIDAR_WATCHING",
+          },
+        },
+        CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED: {
+          on: {
+            CHOKIDAR_READY: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED",
+          },
+        },
+      },
+    },
+    PROCESSING: {
+      initial: `BOOTSTRAPPING`,
+      states: {
+        BOOTSTRAPPING: {
+          on: {
+            BOOTSTRAP_FINISHED: "IDLE",
+          },
+        },
+        IDLE: {
+          on: {
+            EMIT_FS_EVENT: `PROCESSING`,
+          },
+        },
+        PROCESSING: {
+          on: {
+            QUERY_QUEUE_DRAINED: `IDLE`,
+            TOUCH_NODE: `IDLE`,
+          },
+        },
+      },
+    },
+  },
+})
+
+let currentState = fsMachine.initialState
+
+const fileQueue = new Map()
+
 exports.sourceNodes = (
-  { actions, getNode, createNodeId, hasNodeChanged, reporter },
+  { actions, getNode, createNodeId, hasNodeChanged, reporter, emitter },
   pluginOptions
 ) => {
   const { createNode, deleteNode } = actions
@@ -21,8 +88,36 @@ Please pick a path to an existing directory.
 See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/
       `)
   }
+  let fileNodeQueue = new Map()
+
+  // Once bootstrap is finished, we only let one File node update go through
+  // the system at a time.
+  emitter.on(`BOOTSTRAP_FINISHED`, () => {
+    currentState = fsMachine.transition(
+      currentState.value,
+      `BOOTSTRAP_FINISHED`
+    )
+  })
+  emitter.on(`TOUCH_NODE`, () => {
+    // If we create a node which is the same as the previous version, createNode
+    // returns TOUCH_NODE and then nothing else happens so we listen to that
+    // to return the state back to IDLE.
+    currentState = fsMachine.transition(currentState.value, `TOUCH_NODE`)
+  })
 
-  let ready = false
+  emitter.on(`QUERY_QUEUE_DRAINED`, () => {
+    currentState = fsMachine.transition(
+      currentState.value,
+      `QUERY_QUEUE_DRAINED`
+    )
+    // If we have any updates queued, run one of them now.
+    if (fileNodeQueue.size > 0) {
+      const toProcess = fileNodeQueue.get(Array.from(fileNodeQueue.keys())[0])
+      fileNodeQueue.delete(toProcess.id)
+      currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+      createNode(toProcess)
+    }
+  })
 
   const watcher = chokidar.watch(pluginOptions.path, {
     ignored: [
@@ -36,8 +131,21 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/
     ],
   })
 
-  const createAndProcessNode = path =>
-    createFileNode(path, createNodeId, pluginOptions).then(createNode)
+  const createAndProcessNode = path => {
+    const fileNodePromise = createFileNode(
+      path,
+      createNodeId,
+      pluginOptions
+    ).then(fileNode => {
+      if (currentState.value.PROCESSING === `PROCESSING`) {
+        fileNodeQueue.set(fileNode.id, fileNode)
+      } else {
+        currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+        createNode(fileNode)
+      }
+    })
+    return fileNodePromise
+  }
 
   // For every path that is reported before the 'ready' event, we throw them
   // into a queue and then flush the queue when 'ready' event arrives.
@@ -50,49 +158,68 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/
   }
 
   watcher.on(`add`, path => {
-    if (ready) {
-      reporter.info(`added file at ${path}`)
+    if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+      if (
+        currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+      ) {
+        reporter.info(`added file at ${path}`)
+      }
       createAndProcessNode(path).catch(err => reporter.error(err))
     } else {
       pathQueue.push(path)
     }
   })
 
   watcher.on(`change`, path => {
-    reporter.info(`changed file at ${path}`)
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`changed file at ${path}`)
+    }
     createAndProcessNode(path).catch(err => reporter.error(err))
   })
 
   watcher.on(`unlink`, path => {
-    reporter.info(`file deleted at ${path}`)
-    const node = getNode(createId(path))
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`file deleted at ${path}`)
+    }
+    const node = getNode(createNodeId(path))
     // It's possible the file node was never created as sometimes tools will
     // write and then immediately delete temporary files to the file system.
     if (node) {
+      currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
       deleteNode(node.id, node)
     }
   })
 
   watcher.on(`addDir`, path => {
-    if (ready) {
-      reporter.info(`added directory at ${path}`)
+    if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+      if (
+        currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+      ) {
+        reporter.info(`added directory at ${path}`)
+      }
       createAndProcessNode(path).catch(err => reporter.error(err))
     } else {
       pathQueue.push(path)
     }
   })
 
   watcher.on(`unlinkDir`, path => {
-    reporter.info(`directory deleted at ${path}`)
-    const node = getNode(createId(path))
+    if (
+      currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+    ) {
+      reporter.info(`directory deleted at ${path}`)
+    }
+    const node = getNode(createNodeId(path))
     deleteNode(node.id, node)
   })
 
   return new Promise((resolve, reject) => {
     watcher.on(`ready`, () => {
-      if (ready) return
-
-      ready = true
+      currentState = fsMachine.transition(currentState.value, `CHOKIDAR_READY`)
       flushPathQueue().then(resolve, reject)
     })
   })

packages/gatsby/cache-dir/app.js

@@ -11,7 +11,12 @@ window.___emitter = emitter
 // Let the site/plugins run code very early.
 apiRunnerAsync(`onClientEntry`).then(() => {
   // Hook up the client to socket.io on server
-  socketIo()
+  const socket = socketIo()
+  if (socket) {
+    socket.on(`reload`, () => {
+      window.location.reload()
+    })
+  }
 
   /**
    * Service Workers are persistent by nature. They stick around,

packages/gatsby/cache-dir/component-renderer.js

@@ -126,7 +126,7 @@ class ComponentRenderer extends React.Component {
     })
     const replacementComponent = pluginResponses[0]
     // If page.
-    if (this.props.page) {
+    if (this.props.isPage) {
       if (this.state.pageResources) {
         return (
           replacementComponent ||
@@ -163,7 +163,7 @@ class ComponentRenderer extends React.Component {
 }
 
 ComponentRenderer.propTypes = {
-  page: PropTypes.bool,
+  isPage: PropTypes.bool,
   layout: PropTypes.bool,
   location: PropTypes.object,
 }

packages/gatsby/cache-dir/json-store.js

@@ -0,0 +1,98 @@
+import React, { createElement } from "react"
+import { Route } from "react-router-dom"
+import ComponentRenderer from "./component-renderer"
+import syncRequires from "./sync-requires"
+import socketIo from "./socketIo"
+import omit from "lodash/omit"
+import get from "lodash/get"
+
+const getPathFromProps = props => {
+  if (props.isPage) {
+    return get(props.pageResources, `page.path`)
+  } else {
+    return `/dev-404-page/`
+  }
+}
+
+class JSONStore extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      data: {},
+      path: null,
+    }
+
+    this.setPageData = this.setPageData.bind(this)
+
+    this.socket = socketIo()
+    this.socket.on(`queryResult`, this.setPageData)
+  }
+
+  componentWillMount() {
+    this.registerPath(getPathFromProps(this.props))
+  }
+
+  componentWillReceiveProps(nextProps) {
+    const { path } = this.state
+    const newPath = getPathFromProps(nextProps)
+
+    if (path !== newPath) {
+      this.unregisterPath(path)
+      this.registerPath(newPath)
+    }
+  }
+
+  registerPath(path) {
+    this.setState({ path })
+    this.socket.emit(`registerPath`, path)
+  }
+
+  unregisterPath(path) {
+    this.setState({ path: null })
+    this.socket.emit(`unregisterPath`, path)
+  }
+
+  componentWillUnmount() {
+    this.unregisterPath(this.state.path)
+  }
+
+  setPageData({ path, result }) {
+    this.setState({
+      data: {
+        ...this.state.data,
+        [path]: result,
+      },
+    })
+  }
+
+  render() {
+    const { isPage, pages, pageResources } = this.props
+    const data = this.state.data[this.state.path]
+    const propsWithoutPages = omit(this.props, `pages`)
+
+    if (!data) {
+      return null
+    } else if (isPage) {
+      return createElement(ComponentRenderer, {
+        key: `normal-page`,
+        ...propsWithoutPages,
+        ...data,
+      })
+    } else {
+      const dev404Page = pages.find(p => /^\/dev-404-page/.test(p.path))
+      return createElement(Route, {
+        key: `404-page`,
+        component: props =>
+          createElement(
+            syncRequires.components[dev404Page.componentChunkName],
+            {
+              ...propsWithoutPages,
+              ...data,
+            }
+          ),
+      })
+    }
+  }
+}
+
+export default JSONStore