fix(gatsby-source-drupal): Ensure all new nodes are created before cr…

…eating relationships (#33864)

* fix(gatsby-source-drupal): Ensure all new nodes are created before creating relationships

* fixes

* Add support for webhook bodies as well

packages/gatsby-source-drupal/src/__tests__/fixtures/1593545806.json

@@ -98,6 +98,69 @@
           }
         }
       }
+    },
+    {
+      "jsonapi": {
+        "version": "1.0",
+        "meta": {
+          "links": {
+            "self": {
+              "href": "http://jsonapi.org/format/1.0/"
+            }
+          }
+        }
+      },
+      "data": {
+        "type": "node--article",
+        "id": "article-10",
+        "attributes": {
+          "id": 100,
+          "uuid": "article-10",
+          "title": "Article #10",
+          "body": "Aliquam non varius libero, sit amet consequat ex. Aenean porta turpis quis vulputate blandit. Suspendisse in porta erat. Sed sit amet scelerisque turpis, at rutrum mauris. Sed tempor eleifend lobortis. Proin maximus, massa sed dignissim sollicitudin, quam risus mattis justo, sit amet aliquam odio ligula quis urna. Interdum et malesuada fames ac ante ipsum primis in faucibus. Ut mollis leo nisi, at interdum urna fermentum ut. Fusce id suscipit neque, eu fermentum lacus. Donec egestas laoreet felis ac luctus. Vestibulum molestie mattis ante, a vulputate nunc ullamcorper at. Ut hendrerit ipsum eget gravida ultricies."
+        },
+        "relationships": {
+          "field_tags": {
+            "data": [
+              {
+                "type": "taxonomy_term--tags",
+                "id": "tag-10"
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "jsonapi": {
+        "version": "1.0",
+        "meta": {
+          "links": {
+            "self": {
+              "href": "http://jsonapi.org/format/1.0/"
+            }
+          }
+        }
+      },
+      "data": {
+      "type": "taxonomy_term--tags",
+      "id": "tag-10",
+      "attributes": {
+        "id": 110,
+        "uuid": "tag-10",
+        "langcode": "en",
+        "name": "Tag #10",
+        "description": null,
+        "weight": 0,
+        "changed": 1523031646,
+        "default_langcode": true,
+        "path": {
+          "alias": null,
+          "pid": null,
+          "langcode": "en"
+        }
+        }
+      }
     }
   ]
 }

packages/gatsby-source-drupal/src/__tests__/fixtures/webhook-body-multiple-nodes.json

@@ -0,0 +1,45 @@
+{
+  "action": "update",
+  "data": [
+    {
+        "type": "node--article",
+        "id": "article-10",
+        "attributes": {
+          "id": 100,
+          "uuid": "article-10",
+          "title": "Article #10",
+          "body": "Aliquam non varius libero, sit amet consequat ex. Aenean porta turpis quis vulputate blandit. Suspendisse in porta erat. Sed sit amet scelerisque turpis, at rutrum mauris. Sed tempor eleifend lobortis. Proin maximus, massa sed dignissim sollicitudin, quam risus mattis justo, sit amet aliquam odio ligula quis urna. Interdum et malesuada fames ac ante ipsum primis in faucibus. Ut mollis leo nisi, at interdum urna fermentum ut. Fusce id suscipit neque, eu fermentum lacus. Donec egestas laoreet felis ac luctus. Vestibulum molestie mattis ante, a vulputate nunc ullamcorper at. Ut hendrerit ipsum eget gravida ultricies."
+        },
+        "relationships": {
+          "field_tags": {
+            "data": [
+              {
+                "type": "taxonomy_term--tags",
+                "id": "tag-10"
+              }
+            ]
+          }
+        }
+    },
+    {
+    "type": "taxonomy_term--tags",
+      "id": "tag-10",
+      "attributes": {
+        "id": 110,
+        "uuid": "tag-10",
+        "langcode": "en",
+        "name": "Tag #10",
+        "description": null,
+        "weight": 0,
+        "changed": 1523031646,
+        "default_langcode": true,
+        "path": {
+          "alias": null,
+          "pid": null,
+          "langcode": "en"
+        }
+      }
+    }
+  ]
+}
+

packages/gatsby-source-drupal/src/__tests__/index.js

@@ -355,6 +355,26 @@ describe(`gatsby-source-drupal`, () => {
         })
       })
     })
+    describe(`multiple entities in webhook body`, () => {
+      let resp
+      beforeAll(async () => {
+        const webhookBody = require(`./fixtures/webhook-body-multiple-nodes.json`)
+        await sourceNodes(
+          {
+            ...args,
+            webhookBody,
+          },
+          { baseUrl }
+        )
+      })
+
+      it(`Relationships`, async () => {
+        expect(
+          nodes[createNodeId(`und.article-10`)].relationships.field_tags___NODE
+            .length
+        ).toBe(1)
+      })
+    })
 
     describe(`Insert content`, () => {
       it(`Node doesn't exist before webhook`, () => {
@@ -567,6 +587,10 @@ describe(`gatsby-source-drupal`, () => {
           nodes[createNodeId(`und.article-2`)].relationships
             .field_tertiary_image___NODE_image___NODE
         ).toBe(undefined)
+        expect(
+          nodes[createNodeId(`und.article-10`)].relationships.field_tags___NODE
+            .length
+        ).toBe(1)
       })
 
       it(`Back references`, () => {

packages/gatsby-source-drupal/src/gatsby-node.js

@@ -16,6 +16,7 @@ const {
   storeRefsLookups,
   handleReferences,
   handleWebhookUpdate,
+  createNodeIfItDoesNotExist,
   handleDeletedNode,
 } = require(`./utils`)
 
@@ -234,6 +235,17 @@ ${JSON.stringify(webhookBody, null, 4)}`
         nodesToUpdate = [data]
       }
 
+      for (const nodeToUpdate of nodesToUpdate) {
+        await createNodeIfItDoesNotExist({
+          nodeToUpdate,
+          actions,
+          createNodeId,
+          createContentDigest,
+          getNode,
+          reporter,
+        })
+      }
+
       for (const nodeToUpdate of nodesToUpdate) {
         await handleWebhookUpdate(
           {
@@ -345,6 +357,31 @@ ${JSON.stringify(webhookBody, null, 4)}`
 
           // Process sync data from Drupal.
           const nodesToSync = res.body.entities
+
+          // First create all nodes that we haven't seen before. That
+          // way we can create relationships correctly next as the nodes
+          // will exist in Gatsby.
+          for (const nodeSyncData of nodesToSync) {
+            if (nodeSyncData.action === `delete`) {
+              continue
+            }
+
+            let nodesToUpdate = nodeSyncData.data
+            if (!Array.isArray(nodeSyncData.data)) {
+              nodesToUpdate = [nodeSyncData.data]
+            }
+            for (const nodeToUpdate of nodesToUpdate) {
+              createNodeIfItDoesNotExist({
+                nodeToUpdate,
+                actions,
+                createNodeId,
+                createContentDigest,
+                getNode,
+                reporter,
+              })
+            }
+          }
+
           for (const nodeSyncData of nodesToSync) {
             if (nodeSyncData.action === `delete`) {
               handleDeletedNode({

packages/gatsby-source-drupal/src/utils.js

@@ -240,6 +240,50 @@ const handleDeletedNode = async ({
   return deletedNode
 }
 
+function createNodeIfItDoesNotExist({
+  nodeToUpdate,
+  actions,
+  createNodeId,
+  createContentDigest,
+  getNode,
+  reporter,
+}) {
+  if (!nodeToUpdate) {
+    reporter.warn(
+      `The updated node was empty. The fact you're seeing this warning means there's probably a bug in how we're creating and processing updates from Drupal.
+
+${JSON.stringify(nodeToUpdate, null, 4)}
+      `
+    )
+
+    return
+  }
+
+  const { createNode } = actions
+  const newNodeId = createNodeId(
+    createNodeIdWithVersion(
+      nodeToUpdate.id,
+      nodeToUpdate.type,
+      getOptions().languageConfig ? nodeToUpdate.langcode : `und`,
+      nodeToUpdate.meta?.target_version,
+      getOptions().entityReferenceRevisions
+    )
+  )
+
+  const oldNode = getNode(newNodeId)
+  // Node doesn't yet exist so we'll create it now.
+  if (!oldNode) {
+    const newNode = nodeFromData(
+      nodeToUpdate,
+      createNodeId,
+      getOptions().entityReferenceRevisions
+    )
+
+    newNode.internal.contentDigest = createContentDigest(newNode)
+    createNode(newNode)
+  }
+}
+
 const handleWebhookUpdate = async (
   {
     nodeToUpdate,
@@ -409,3 +453,4 @@ function drupalCreateNodeManifest({
 
 exports.handleWebhookUpdate = handleWebhookUpdate
 exports.handleDeletedNode = handleDeletedNode
+exports.createNodeIfItDoesNotExist = createNodeIfItDoesNotExist