From a610d8d71d4b391fecc3b29bd8aa7b520e0a55b5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 25 Oct 2021 10:14:59 -0700 Subject: [PATCH 1/2] feat(gatsby-source-drupal): More ports from customer fixes --- packages/gatsby-source-drupal/README.md | 2 +- packages/gatsby-source-drupal/package.json | 12 ++++++------ .../gatsby-source-drupal/src/gatsby-node.js | 18 ++++++++++++++++-- packages/gatsby-source-drupal/src/utils.js | 4 +++- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md index 16eee5609108e..732653c6932fc 100644 --- a/packages/gatsby-source-drupal/README.md +++ b/packages/gatsby-source-drupal/README.md @@ -2,7 +2,7 @@ Source plugin for pulling data (including images) into Gatsby from Drupal sites. -Pulls data from Drupal 8 sites with the +It pulls data from Drupal 8 sites with the [Drupal JSONAPI module](https://www.drupal.org/project/jsonapi) installed. An example site built with the headless Drupal distro diff --git a/packages/gatsby-source-drupal/package.json b/packages/gatsby-source-drupal/package.json index bc692b4d75bda..daaabb49a0d9e 100644 --- a/packages/gatsby-source-drupal/package.json +++ b/packages/gatsby-source-drupal/package.json @@ -1,7 +1,7 @@ { "name": "gatsby-source-drupal", "description": "Gatsby source plugin for building websites using the Drupal CMS as a data source", - "version": "5.1.0-next.0", + "version": "4.14.0-next.6", "author": "Kyle Mathews ", "bugs": { "url": "https://github.com/gatsbyjs/gatsby/issues" @@ -13,22 +13,22 @@ "bluebird": "^3.7.2", "body-parser": "^1.19.0", "fastq": "^1.11.1", - "gatsby-source-filesystem": "^4.1.0-next.0", + "opentracing": "^0.14.4", + "gatsby-source-filesystem": "^3.14.0-next.2", "got": "^11.8.2", "http2-wrapper": "^2.0.5", "lodash": "^4.17.21", - "opentracing": "^0.14.5", "tiny-async-pool": "^1.2.0", "url-join": "^4.0.1" }, "devDependencies": { "@babel/cli": "^7.15.4", "@babel/core": "^7.15.5", - "babel-preset-gatsby-package": "^2.1.0-next.0", + "babel-preset-gatsby-package": "^1.14.0-next.2", "cross-env": "^7.0.3" }, "engines": { - "node": ">=14.15.0" + "node": ">=12.13.0" }, "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme", "keywords": [ @@ -38,7 +38,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": "^4.0.0-next" + "gatsby": "^3.0.0-next.0" }, "repository": { "type": "git", diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index 7ca4bdb25a0ac..3bb2f218118b7 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -62,6 +62,9 @@ async function worker([url, options]) { const response = await got(url, { agent, cache: false, + timeout: { + request: 15000, + }, // request: http2wrapper.auto, // http2: true, ...options, @@ -253,7 +256,7 @@ ${JSON.stringify(webhookBody, null, 4)}` // lastFetched isn't set so do a full rebuild. if (!lastFetched) { - setPluginStatus({ lastFetched: new Date().getTime() }) + setPluginStatus({ lastFetched: Math.floor(new Date().getTime() / 1000) }) requireFullRebuild = true } else { const drupalFetchIncrementalActivity = reporter.activityTimer( @@ -264,9 +267,14 @@ ${JSON.stringify(webhookBody, null, 4)}` drupalFetchIncrementalActivity.start() try { + console.time(`drupal: gatsby-fastbuilds/sync`) // Hit fastbuilds endpoint with the lastFetched date. const res = await requestQueue.push([ - urlJoin(baseUrl, `gatsby-fastbuilds/sync/`, lastFetched.toString()), + urlJoin( + baseUrl, + `gatsby-fastbuilds/sync/`, + Math.floor(lastFetched).toString() + ), { username: basicAuth.username, password: basicAuth.password, @@ -277,6 +285,8 @@ ${JSON.stringify(webhookBody, null, 4)}` }, ]) + console.timeEnd(`drupal: gatsby-fastbuilds/sync`) + // Fastbuilds returns a -1 if: // - the timestamp has expired // - if old fastbuild logs were purged @@ -289,6 +299,7 @@ ${JSON.stringify(webhookBody, null, 4)}` } else { // Touch nodes so they are not garbage collected by Gatsby. if (initialSourcing) { + console.time(`drupal: touchNode`) const touchNodesSpan = tracer.startSpan(`sourceNodes.touchNodes`, { childOf: fastBuildsSpan, }) @@ -300,6 +311,7 @@ ${JSON.stringify(webhookBody, null, 4)}` touchNode(node) } }) + console.timeEnd(`drupal: touchNode`) touchNodesSpan.setTag(`sourceNodes.touchNodes.count`, touchCount) touchNodesSpan.finish() } @@ -315,6 +327,7 @@ ${JSON.stringify(webhookBody, null, 4)}` ) // Process sync data from Drupal. + console.time(`drupal: process synced data`) const nodesToSync = res.body.entities for (const nodeSyncData of nodesToSync) { if (nodeSyncData.action === `delete`) { @@ -353,6 +366,7 @@ ${JSON.stringify(webhookBody, null, 4)}` } } } + console.timeEnd(`drupal: process synced data`) createNodesSpan.finish() setPluginStatus({ lastFetched: res.body.timestamp }) diff --git a/packages/gatsby-source-drupal/src/utils.js b/packages/gatsby-source-drupal/src/utils.js index eda40f3437987..558833008eb2f 100644 --- a/packages/gatsby-source-drupal/src/utils.js +++ b/packages/gatsby-source-drupal/src/utils.js @@ -333,7 +333,9 @@ ${JSON.stringify(nodeToUpdate, null, 4)} } node.internal.contentDigest = createContentDigest(node) createNode(node) - reporter.log(`Updated Gatsby node: ${node.id}`) + reporter.log( + `Updated Gatsby node — id: ${node.id} type: ${node.internal.type}` + ) } } From a6ce4ebde89d77cc9b58ab813d8612e9f76604e6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 25 Oct 2021 10:19:53 -0700 Subject: [PATCH 2/2] cleanups --- packages/gatsby-source-drupal/package.json | 12 ++++++------ packages/gatsby-source-drupal/src/gatsby-node.js | 8 +------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/gatsby-source-drupal/package.json b/packages/gatsby-source-drupal/package.json index daaabb49a0d9e..bc692b4d75bda 100644 --- a/packages/gatsby-source-drupal/package.json +++ b/packages/gatsby-source-drupal/package.json @@ -1,7 +1,7 @@ { "name": "gatsby-source-drupal", "description": "Gatsby source plugin for building websites using the Drupal CMS as a data source", - "version": "4.14.0-next.6", + "version": "5.1.0-next.0", "author": "Kyle Mathews ", "bugs": { "url": "https://github.com/gatsbyjs/gatsby/issues" @@ -13,22 +13,22 @@ "bluebird": "^3.7.2", "body-parser": "^1.19.0", "fastq": "^1.11.1", - "opentracing": "^0.14.4", - "gatsby-source-filesystem": "^3.14.0-next.2", + "gatsby-source-filesystem": "^4.1.0-next.0", "got": "^11.8.2", "http2-wrapper": "^2.0.5", "lodash": "^4.17.21", + "opentracing": "^0.14.5", "tiny-async-pool": "^1.2.0", "url-join": "^4.0.1" }, "devDependencies": { "@babel/cli": "^7.15.4", "@babel/core": "^7.15.5", - "babel-preset-gatsby-package": "^1.14.0-next.2", + "babel-preset-gatsby-package": "^2.1.0-next.0", "cross-env": "^7.0.3" }, "engines": { - "node": ">=12.13.0" + "node": ">=14.15.0" }, "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme", "keywords": [ @@ -38,7 +38,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": "^3.0.0-next.0" + "gatsby": "^4.0.0-next" }, "repository": { "type": "git", diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index 3bb2f218118b7..9e40259605a13 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -63,6 +63,7 @@ async function worker([url, options]) { agent, cache: false, timeout: { + // Occasionally requests to Drupal stall. Set a 15s timeout to retry in this case. request: 15000, }, // request: http2wrapper.auto, @@ -267,7 +268,6 @@ ${JSON.stringify(webhookBody, null, 4)}` drupalFetchIncrementalActivity.start() try { - console.time(`drupal: gatsby-fastbuilds/sync`) // Hit fastbuilds endpoint with the lastFetched date. const res = await requestQueue.push([ urlJoin( @@ -285,8 +285,6 @@ ${JSON.stringify(webhookBody, null, 4)}` }, ]) - console.timeEnd(`drupal: gatsby-fastbuilds/sync`) - // Fastbuilds returns a -1 if: // - the timestamp has expired // - if old fastbuild logs were purged @@ -299,7 +297,6 @@ ${JSON.stringify(webhookBody, null, 4)}` } else { // Touch nodes so they are not garbage collected by Gatsby. if (initialSourcing) { - console.time(`drupal: touchNode`) const touchNodesSpan = tracer.startSpan(`sourceNodes.touchNodes`, { childOf: fastBuildsSpan, }) @@ -311,7 +308,6 @@ ${JSON.stringify(webhookBody, null, 4)}` touchNode(node) } }) - console.timeEnd(`drupal: touchNode`) touchNodesSpan.setTag(`sourceNodes.touchNodes.count`, touchCount) touchNodesSpan.finish() } @@ -327,7 +323,6 @@ ${JSON.stringify(webhookBody, null, 4)}` ) // Process sync data from Drupal. - console.time(`drupal: process synced data`) const nodesToSync = res.body.entities for (const nodeSyncData of nodesToSync) { if (nodeSyncData.action === `delete`) { @@ -366,7 +361,6 @@ ${JSON.stringify(webhookBody, null, 4)}` } } } - console.timeEnd(`drupal: process synced data`) createNodesSpan.finish() setPluginStatus({ lastFetched: res.body.timestamp })