diff --git a/packages/gatsby-source-drupal/package.json b/packages/gatsby-source-drupal/package.json index 447e0b237e4b2..a275494341193 100644 --- a/packages/gatsby-source-drupal/package.json +++ b/packages/gatsby-source-drupal/package.json @@ -8,10 +8,12 @@ }, "dependencies": { "@babel/runtime": "^7.15.4", + "@opentelemetry/semantic-conventions": "0.24.0", "agentkeepalive": "^4.1.4", "bluebird": "^3.7.2", "body-parser": "^1.19.0", "fastq": "^1.11.1", + "opentracing": "^0.14.4", "gatsby-source-filesystem": "^3.14.0-next.2", "got": "^11.8.2", "http2-wrapper": "^2.0.5", diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index 953429d4a1851..cb7237def7fb0 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -553,7 +553,8 @@ describe(`gatsby-source-drupal`, () => { expect(reporter.activityTimer).toHaveBeenCalledTimes(1) expect(reporter.activityTimer).toHaveBeenNthCalledWith( 1, - `Fetch all data from Drupal` + `Fetch all data from Drupal`, + { parentSpan: undefined } ) expect(activity.start).toHaveBeenCalledTimes(1) @@ -576,11 +577,13 @@ describe(`gatsby-source-drupal`, () => { expect(reporter.activityTimer).toHaveBeenCalledTimes(2) expect(reporter.activityTimer).toHaveBeenNthCalledWith( 1, - `Fetch all data from Drupal` + `Fetch all data from Drupal`, + { parentSpan: undefined } ) expect(reporter.activityTimer).toHaveBeenNthCalledWith( 2, - `Remote file download` + `Remote file download`, + { parentSpan: undefined } ) expect(activity.start).toHaveBeenCalledTimes(2) @@ -639,7 +642,8 @@ describe(`gatsby-source-drupal`, () => { expect(reporter.activityTimer).toHaveBeenCalledTimes(1) expect(reporter.activityTimer).toHaveBeenNthCalledWith( 1, - `Fetch incremental changes from Drupal` + `Fetch incremental changes from Drupal`, + { parentSpan: {} } ) expect(activity.start).toHaveBeenCalledTimes(1) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index 53a77d41e5da9..9db0caefe4da5 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -3,17 +3,14 @@ const _ = require(`lodash`) const urlJoin = require(`url-join`) import HttpAgent from "agentkeepalive" // const http2wrapper = require(`http2-wrapper`) +const opentracing = require(`opentracing`) +const { SemanticAttributes } = require(`@opentelemetry/semantic-conventions`) const { HttpsAgent } = HttpAgent const { setOptions, getOptions } = require(`./plugin-options`) -const { - nodeFromData, - downloadFile, - isFileNode, - createNodeIdWithVersion, -} = require(`./normalize`) +const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`) const { handleReferences, handleWebhookUpdate, @@ -31,6 +28,20 @@ let apiRequestCount = 0 let initialSourcing = true let globalReporter async function worker([url, options]) { + const tracer = opentracing.globalTracer() + const httpSpan = tracer.startSpan(`http.get`, { + childOf: options.parentSpan, + }) + const parsedUrl = new URL(url) + httpSpan.setTag(SemanticAttributes.HTTP_URL, url) + httpSpan.setTag(SemanticAttributes.HTTP_HOST, parsedUrl.host) + httpSpan.setTag( + SemanticAttributes.HTTP_SCHEME, + parsedUrl.protocol.replace(/:$/, ``) + ) + httpSpan.setTag(SemanticAttributes.HTTP_TARGET, parsedUrl.pathname) + httpSpan.setTag(`plugin`, `gatsby-source-drupal`) + // Log out progress during the initial sourcing. if (initialSourcing) { apiRequestCount += 1 @@ -48,13 +59,25 @@ async function worker([url, options]) { } } - return got(url, { + const response = await got(url, { agent, cache: false, // request: http2wrapper.auto, // http2: true, ...options, }) + + httpSpan.setTag(SemanticAttributes.HTTP_STATUS_CODE, response.statusCode) + httpSpan.setTag(SemanticAttributes.HTTP_METHOD, `GET`) + httpSpan.setTag(SemanticAttributes.NET_PEER_IP, response.ip) + httpSpan.setTag( + SemanticAttributes.HTTP_RESPONSE_CONTENT_LENGTH, + response.rawBody?.length + ) + + httpSpan.finish() + + return response } const requestQueue = require(`fastq`).promise(worker, 20) @@ -97,6 +120,8 @@ exports.sourceNodes = async ( }, pluginOptions ) => { + const tracer = opentracing.globalTracer() + globalReporter = reporter const { baseUrl, @@ -213,6 +238,12 @@ ${JSON.stringify(webhookBody, null, 4)} } if (fastBuilds) { + const fastBuildsSpan = tracer.startSpan(`sourceNodes.fetch`, { + childOf: parentSpan, + }) + fastBuildsSpan.setTag(`plugin`, `gatsby-source-drupal`) + fastBuildsSpan.setTag(`sourceNodes.fetch.type`, `delta`) + const lastFetched = store.getState().status.plugins?.[`gatsby-source-drupal`]?.lastFetched ?? 0 @@ -222,7 +253,8 @@ ${JSON.stringify(webhookBody, null, 4)} ) const drupalFetchIncrementalActivity = reporter.activityTimer( - `Fetch incremental changes from Drupal` + `Fetch incremental changes from Drupal`, + { parentSpan: fastBuildsSpan } ) let requireFullRebuild = false @@ -238,6 +270,7 @@ ${JSON.stringify(webhookBody, null, 4)} headers, searchParams: params, responseType: `json`, + parentSpan: fastBuildsSpan, }, ]) @@ -251,12 +284,31 @@ ${JSON.stringify(webhookBody, null, 4)} setPluginStatus({ lastFetched: res.body.timestamp }) requireFullRebuild = true } else { + const touchNodesSpan = tracer.startSpan(`sourceNodes.touchNodes`, { + childOf: fastBuildsSpan, + }) + touchNodesSpan.setTag(`plugin`, `gatsby-source-drupal`) + // Touch nodes so they are not garbage collected by Gatsby. + let touchCount = 0 getNodes().forEach(node => { if (node.internal.owner === `gatsby-source-drupal`) { + touchCount += 1 touchNode(node) } }) + touchNodesSpan.setTag(`sourceNodes.touchNodes.count`, touchCount) + touchNodesSpan.finish() + + const createNodesSpan = tracer.startSpan(`sourceNodes.createNodes`, { + childOf: parentSpan, + }) + createNodesSpan.setTag(`plugin`, `gatsby-source-drupal`) + createNodesSpan.setTag(`sourceNodes.fetch.type`, `delta`) + createNodesSpan.setTag( + `sourceNodes.createNodes.count`, + res.body.entities?.length + ) // Process sync data from Drupal. const nodesToSync = res.body.entities @@ -298,14 +350,19 @@ ${JSON.stringify(webhookBody, null, 4)} } } + createNodesSpan.finish() setPluginStatus({ lastFetched: res.body.timestamp }) } } catch (e) { gracefullyRethrow(drupalFetchIncrementalActivity, e) + + drupalFetchIncrementalActivity.end() + fastBuildsSpan.finish() return } drupalFetchIncrementalActivity.end() + fastBuildsSpan.finish() if (!requireFullRebuild) { return @@ -313,8 +370,14 @@ ${JSON.stringify(webhookBody, null, 4)} } const drupalFetchActivity = reporter.activityTimer( - `Fetch all data from Drupal` + `Fetch all data from Drupal`, + { parentSpan } ) + const fullFetchSpan = tracer.startSpan(`sourceNodes.fetch`, { + childOf: parentSpan, + }) + fullFetchSpan.setTag(`plugin`, `gatsby-source-drupal`) + fullFetchSpan.setTag(`sourceNodes.fetch.type`, `full`) // Fetch articles. reporter.info(`Starting to fetch all data from Drupal`) @@ -332,6 +395,7 @@ ${JSON.stringify(webhookBody, null, 4)} headers, searchParams: params, responseType: `json`, + parentSpan: fullFetchSpan, }, ]) allData = await Promise.all( @@ -380,6 +444,7 @@ ${JSON.stringify(webhookBody, null, 4)} password: basicAuth.password, headers, responseType: `json`, + parentSpan: fullFetchSpan, }, ]) } catch (error) { @@ -479,6 +544,13 @@ ${JSON.stringify(webhookBody, null, 4)} } drupalFetchActivity.end() + fullFetchSpan.finish() + + const createNodesSpan = tracer.startSpan(`sourceNodes.createNodes`, { + childOf: parentSpan, + }) + createNodesSpan.setTag(`plugin`, `gatsby-source-drupal`) + createNodesSpan.setTag(`sourceNodes.fetch.type`, `full`) const nodes = new Map() @@ -492,6 +564,8 @@ ${JSON.stringify(webhookBody, null, 4)} }) }) + createNodesSpan.setTag(`sourceNodes.createNodes.count`, nodes.size) + // second pass - handle relationships and back references nodes.forEach(node => { handleReferences(node, { @@ -510,8 +584,10 @@ ${JSON.stringify(webhookBody, null, 4)} const fileNodes = [...nodes.values()].filter(isFileNode) if (fileNodes.length) { - const downloadingFilesActivity = - reporter.activityTimer(`Remote file download`) + const downloadingFilesActivity = reporter.activityTimer( + `Remote file download`, + { parentSpan } + ) downloadingFilesActivity.start() try { await asyncPool(concurrentFileRequests, fileNodes, async node => { @@ -545,6 +621,7 @@ ${JSON.stringify(webhookBody, null, 4)} // We're now done with the initial sourcing. initialSourcing = false + createNodesSpan.finish() return } diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index d573f83d3b0c1..e4f3e1779844b 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -375,6 +375,7 @@ const runAPI = async (plugin, api, args, activity) => { const apiCallArgs = [ { ...args, + parentSpan: pluginSpan, basePath: pathPrefix, pathPrefix: publicPath, actions, diff --git a/yarn.lock b/yarn.lock index 92dceb9fbb622..c5a4fe7bca187 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3723,6 +3723,11 @@ resolved "https://registry.yarnpkg.com/@open-draft/until/-/until-1.0.3.tgz#db9cc719191a62e7d9200f6e7bab21c5b848adca" integrity sha512-Aq58f5HiWdyDlFffbbSjAlv596h/cOnt2DO1w3DOC7OJ5EHs0hd/nycJfiu9RJbT6Yk6F1knnRRXNSpxoIVZ9Q== +"@opentelemetry/semantic-conventions@0.24.0": + version "0.24.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-0.24.0.tgz#1028ef0e0923b24916158d80d2ddfd67ea8b6740" + integrity sha512-a/szuMQV0Quy0/M7kKdglcbRSoorleyyOwbTNNJ32O+RBN766wbQlMTvdimImTmwYWGr+NJOni1EcC242WlRcA== + "@pmmmwh/react-refresh-webpack-plugin@^0.4.3": version "0.4.3" resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.4.3.tgz#1eec460596d200c0236bf195b078a5d1df89b766"