feat(gatsby): Optimize creating many child nodes from one parent

packages/gatsby-transformer-json/src/gatsby-node.js

@@ -6,6 +6,8 @@ function unstable_shouldOnCreateNode({ node }) {
   return node.internal.mediaType === `application/json`
 }
 
+const typeCache = new Map()
+
 async function onCreateNode(
   { node, actions, loadNodeContent, createNodeId, createContentDigest },
   pluginOptions
@@ -20,15 +22,35 @@ async function onCreateNode(
     } else if (pluginOptions && _.isString(pluginOptions.typeName)) {
       return pluginOptions.typeName
     } else if (node.internal.type !== `File`) {
-      return _.upperFirst(_.camelCase(`${node.internal.type} Json`))
+      if (typeCache.has(node.internal.type)) {
+        return typeCache.get(node.internal.type)
+      } else {
+        const type = _.upperFirst(_.camelCase(`${node.internal.type} Json`))
+        typeCache.set(node.internal.type, type)
+        return type
+      }
     } else if (isArray) {
-      return _.upperFirst(_.camelCase(`${node.name} Json`))
+      if (typeCache.has(node.name)) {
+        return typeCache.get(node.name)
+      } else {
+        const type = _.upperFirst(_.camelCase(`${node.name} Json`))
+        typeCache.set(node.name, type)
+        return type
+      }
     } else {
-      return _.upperFirst(_.camelCase(`${path.basename(node.dir)} Json`))
+      if (typeCache.has(node.dir)) {
+        return typeCache.get(node.dir)
+      } else {
+        const type = _.upperFirst(
+          _.camelCase(`${path.basename(node.dir)} Json`)
+        )
+        typeCache.set(node.dir, type)
+        return type
+      }
     }
   }
 
-  async function transformObject(obj, id, type) {
+  function transformObject(obj, id, type) {
     const jsonNode = {
       ...obj,
       id,
@@ -42,7 +64,7 @@ async function onCreateNode(
     if (obj.id) {
       jsonNode[`jsonId`] = obj.id
     }
-    await createNode(jsonNode)
+    createNode(jsonNode)
     createParentChildLink({ parent: node, child: jsonNode })
   }
 
@@ -59,18 +81,35 @@ async function onCreateNode(
     throw new Error(`Unable to parse JSON: ${hint}`)
   }
 
-  if (_.isArray(parsedContent)) {
-    for (let i = 0, l = parsedContent.length; i < l; i++) {
-      const obj = parsedContent[i]
-
-      await transformObject(
+  async function transformArrayChunk({ chunk, startCount }) {
+    for (let i = 0, l = chunk.length; i < l; i++) {
+      const obj = chunk[i]
+      transformObject(
         obj,
-        createNodeId(`${node.id} [${i}] >>> JSON`),
-        getType({ node, object: obj, isArray: true })
+        createNodeId(`${node.id} [${i + startCount}] >>> JSON`),
+        getType({
+          node,
+          object: obj,
+          isArray: true,
+        })
+      )
+      await new Promise(resolve =>
+        setImmediate(() => {
+          resolve()
+        })
       )
     }
+  }
+
+  if (_.isArray(parsedContent)) {
+    const chunks = _.chunk(parsedContent, 100)
+    let count = 0
+    for await (const chunk of chunks) {
+      await transformArrayChunk({ chunk, startCount: count })
+      count += chunk.length
+    }
   } else if (_.isPlainObject(parsedContent)) {
-    await transformObject(
+    transformObject(
       parsedContent,
       createNodeId(`${node.id} >>> JSON`),
       getType({ node, object: parsedContent, isArray: false })

packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts

@@ -7,6 +7,7 @@ import { IDataStore, ILmdbDatabases, IQueryResult } from "../types"
 import { emitter, replaceReducer } from "../../redux"
 import { GatsbyIterable } from "../common/iterable"
 import { doRunQuery } from "./query/run-query"
+import _ from "lodash"
 import {
   IRunFilterArg,
   runFastFiltersAndSort,
@@ -219,6 +220,8 @@ async function runQuery(args: IRunFilterArg): Promise<IQueryResult> {
 
 let lastOperationPromise: Promise<any> = Promise.resolve()
 
+const debounceFunctionsPerNodeType = new Map()
+
 function updateDataStore(action: ActionsUnion): void {
   switch (action.type) {
     case `DELETE_CACHE`: {
@@ -237,10 +240,31 @@ function updateDataStore(action: ActionsUnion): void {
       clearIndexes()
       break
     }
+    case `ADD_CHILD_NODE_TO_PARENT_NODE`: {
+      let fn
+      const dbs = getDatabases()
+      const key = action.payload.id + action.payload.internal.type
+      if (!debounceFunctionsPerNodeType.has(key)) {
+        fn = _.debounce(
+          _action => {
+            updateNodes(dbs.nodes, _action)
+            updateNodesByType(dbs.nodesByType, _action)
+          },
+          100,
+          { leading: true, trailing: true }
+        )
+        debounceFunctionsPerNodeType.set(key, fn)
+      } else {
+        fn = debounceFunctionsPerNodeType.get(key)
+      }
+
+      // Call the debounce function.
+      fn(action)
+      break
+    }
     case `CREATE_NODE`:
     case `DELETE_NODE`:
     case `ADD_FIELD_TO_NODE`:
-    case `ADD_CHILD_NODE_TO_PARENT_NODE`:
     case `MATERIALIZE_PAGE_MODE`: {
       const dbs = getDatabases()
       const operationPromise = Promise.all([