Skip to content

Commit

Permalink
feat(gatsby-source-drupal): Add tracing for full/delta fetches and ht…
Browse files Browse the repository at this point in the history 
…tp requests (#33142)

* feat(gatsby-source-drupal): Add tracing for full/delta fetches and http requests

* Track creating nodes

* don't create a parent sourceNodes span (redundant) and standardize other names

* Fix tests

* Use @opentelemetry/semantic-conventions for names

Co-authored-by: gatsbybot <[email protected]>
  • Loading branch information
@KyleAMathews
KyleAMathews and gatsbybot authored Sep 13, 2021
1 parent d31645c commit 91187da
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 15 deletions.
2 changes: 2 additions & 0 deletions packages/gatsby-source-drupal/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@
},
"dependencies": {
"@babel/runtime": "^7.15.4",
"@opentelemetry/semantic-conventions": "0.24.0",
"agentkeepalive": "^4.1.4",
"bluebird": "^3.7.2",
"body-parser": "^1.19.0",
"fastq": "^1.11.1",
"opentracing": "^0.14.4",
"gatsby-source-filesystem": "^3.14.0-next.2",
"got": "^11.8.2",
"http2-wrapper": "^2.0.5",
Expand Down
12 changes: 8 additions & 4 deletions packages/gatsby-source-drupal/src/__tests__/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,8 @@ describe(`gatsby-source-drupal`, () => {
expect(reporter.activityTimer).toHaveBeenCalledTimes(1)
expect(reporter.activityTimer).toHaveBeenNthCalledWith(
1,
`Fetch all data from Drupal`
`Fetch all data from Drupal`,
{ parentSpan: undefined }
)

expect(activity.start).toHaveBeenCalledTimes(1)
Expand All @@ -576,11 +577,13 @@ describe(`gatsby-source-drupal`, () => {
expect(reporter.activityTimer).toHaveBeenCalledTimes(2)
expect(reporter.activityTimer).toHaveBeenNthCalledWith(
1,
`Fetch all data from Drupal`
`Fetch all data from Drupal`,
{ parentSpan: undefined }
)
expect(reporter.activityTimer).toHaveBeenNthCalledWith(
2,
`Remote file download`
`Remote file download`,
{ parentSpan: undefined }
)

expect(activity.start).toHaveBeenCalledTimes(2)
Expand Down Expand Up @@ -639,7 +642,8 @@ describe(`gatsby-source-drupal`, () => {
expect(reporter.activityTimer).toHaveBeenCalledTimes(1)
expect(reporter.activityTimer).toHaveBeenNthCalledWith(
1,
`Fetch incremental changes from Drupal`
`Fetch incremental changes from Drupal`,
{ parentSpan: {} }
)

expect(activity.start).toHaveBeenCalledTimes(1)
Expand Down
99 changes: 88 additions & 11 deletions packages/gatsby-source-drupal/src/gatsby-node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,14 @@ const _ = require(`lodash`)
const urlJoin = require(`url-join`)
import HttpAgent from "agentkeepalive"
// const http2wrapper = require(`http2-wrapper`)
const opentracing = require(`opentracing`)
const { SemanticAttributes } = require(`@opentelemetry/semantic-conventions`)

const { HttpsAgent } = HttpAgent

const { setOptions, getOptions } = require(`./plugin-options`)

const {
nodeFromData,
downloadFile,
isFileNode,
createNodeIdWithVersion,
} = require(`./normalize`)
const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`)
const {
handleReferences,
handleWebhookUpdate,
Expand All @@ -31,6 +28,20 @@ let apiRequestCount = 0
let initialSourcing = true
let globalReporter
async function worker([url, options]) {
const tracer = opentracing.globalTracer()
const httpSpan = tracer.startSpan(`http.get`, {
childOf: options.parentSpan,
})
const parsedUrl = new URL(url)
httpSpan.setTag(SemanticAttributes.HTTP_URL, url)
httpSpan.setTag(SemanticAttributes.HTTP_HOST, parsedUrl.host)
httpSpan.setTag(
SemanticAttributes.HTTP_SCHEME,
parsedUrl.protocol.replace(/:$/, ``)
)
httpSpan.setTag(SemanticAttributes.HTTP_TARGET, parsedUrl.pathname)
httpSpan.setTag(`plugin`, `gatsby-source-drupal`)

// Log out progress during the initial sourcing.
if (initialSourcing) {
apiRequestCount += 1
Expand All @@ -48,13 +59,25 @@ async function worker([url, options]) {
}
}

return got(url, {
const response = await got(url, {
agent,
cache: false,
// request: http2wrapper.auto,
// http2: true,
...options,
})

httpSpan.setTag(SemanticAttributes.HTTP_STATUS_CODE, response.statusCode)
httpSpan.setTag(SemanticAttributes.HTTP_METHOD, `GET`)
httpSpan.setTag(SemanticAttributes.NET_PEER_IP, response.ip)
httpSpan.setTag(
SemanticAttributes.HTTP_RESPONSE_CONTENT_LENGTH,
response.rawBody?.length
)

httpSpan.finish()

return response
}

const requestQueue = require(`fastq`).promise(worker, 20)
Expand Down Expand Up @@ -97,6 +120,8 @@ exports.sourceNodes = async (
},
pluginOptions
) => {
const tracer = opentracing.globalTracer()

globalReporter = reporter
const {
baseUrl,
Expand Down Expand Up @@ -213,6 +238,12 @@ ${JSON.stringify(webhookBody, null, 4)}
}

if (fastBuilds) {
const fastBuildsSpan = tracer.startSpan(`sourceNodes.fetch`, {
childOf: parentSpan,
})
fastBuildsSpan.setTag(`plugin`, `gatsby-source-drupal`)
fastBuildsSpan.setTag(`sourceNodes.fetch.type`, `delta`)

const lastFetched =
store.getState().status.plugins?.[`gatsby-source-drupal`]?.lastFetched ??
0
Expand All @@ -222,7 +253,8 @@ ${JSON.stringify(webhookBody, null, 4)}
)

const drupalFetchIncrementalActivity = reporter.activityTimer(
`Fetch incremental changes from Drupal`
`Fetch incremental changes from Drupal`,
{ parentSpan: fastBuildsSpan }
)
let requireFullRebuild = false

Expand All @@ -238,6 +270,7 @@ ${JSON.stringify(webhookBody, null, 4)}
headers,
searchParams: params,
responseType: `json`,
parentSpan: fastBuildsSpan,
},
])

Expand All @@ -251,12 +284,31 @@ ${JSON.stringify(webhookBody, null, 4)}
setPluginStatus({ lastFetched: res.body.timestamp })
requireFullRebuild = true
} else {
const touchNodesSpan = tracer.startSpan(`sourceNodes.touchNodes`, {
childOf: fastBuildsSpan,
})
touchNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)

// Touch nodes so they are not garbage collected by Gatsby.
let touchCount = 0
getNodes().forEach(node => {
if (node.internal.owner === `gatsby-source-drupal`) {
touchCount += 1
touchNode(node)
}
})
touchNodesSpan.setTag(`sourceNodes.touchNodes.count`, touchCount)
touchNodesSpan.finish()

const createNodesSpan = tracer.startSpan(`sourceNodes.createNodes`, {
childOf: parentSpan,
})
createNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
createNodesSpan.setTag(`sourceNodes.fetch.type`, `delta`)
createNodesSpan.setTag(
`sourceNodes.createNodes.count`,
res.body.entities?.length
)

// Process sync data from Drupal.
const nodesToSync = res.body.entities
Expand Down Expand Up @@ -298,23 +350,34 @@ ${JSON.stringify(webhookBody, null, 4)}
}
}

createNodesSpan.finish()
setPluginStatus({ lastFetched: res.body.timestamp })
}
} catch (e) {
gracefullyRethrow(drupalFetchIncrementalActivity, e)

drupalFetchIncrementalActivity.end()
fastBuildsSpan.finish()
return
}

drupalFetchIncrementalActivity.end()
fastBuildsSpan.finish()

if (!requireFullRebuild) {
return
}
}

const drupalFetchActivity = reporter.activityTimer(
`Fetch all data from Drupal`
`Fetch all data from Drupal`,
{ parentSpan }
)
const fullFetchSpan = tracer.startSpan(`sourceNodes.fetch`, {
childOf: parentSpan,
})
fullFetchSpan.setTag(`plugin`, `gatsby-source-drupal`)
fullFetchSpan.setTag(`sourceNodes.fetch.type`, `full`)

// Fetch articles.
reporter.info(`Starting to fetch all data from Drupal`)
Expand All @@ -332,6 +395,7 @@ ${JSON.stringify(webhookBody, null, 4)}
headers,
searchParams: params,
responseType: `json`,
parentSpan: fullFetchSpan,
},
])
allData = await Promise.all(
Expand Down Expand Up @@ -380,6 +444,7 @@ ${JSON.stringify(webhookBody, null, 4)}
password: basicAuth.password,
headers,
responseType: `json`,
parentSpan: fullFetchSpan,
},
])
} catch (error) {
Expand Down Expand Up @@ -479,6 +544,13 @@ ${JSON.stringify(webhookBody, null, 4)}
}

drupalFetchActivity.end()
fullFetchSpan.finish()

const createNodesSpan = tracer.startSpan(`sourceNodes.createNodes`, {
childOf: parentSpan,
})
createNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
createNodesSpan.setTag(`sourceNodes.fetch.type`, `full`)

const nodes = new Map()

Expand All @@ -492,6 +564,8 @@ ${JSON.stringify(webhookBody, null, 4)}
})
})

createNodesSpan.setTag(`sourceNodes.createNodes.count`, nodes.size)

// second pass - handle relationships and back references
nodes.forEach(node => {
handleReferences(node, {
Expand All @@ -510,8 +584,10 @@ ${JSON.stringify(webhookBody, null, 4)}
const fileNodes = [...nodes.values()].filter(isFileNode)

if (fileNodes.length) {
const downloadingFilesActivity =
reporter.activityTimer(`Remote file download`)
const downloadingFilesActivity = reporter.activityTimer(
`Remote file download`,
{ parentSpan }
)
downloadingFilesActivity.start()
try {
await asyncPool(concurrentFileRequests, fileNodes, async node => {
Expand Down Expand Up @@ -545,6 +621,7 @@ ${JSON.stringify(webhookBody, null, 4)}
// We're now done with the initial sourcing.
initialSourcing = false

createNodesSpan.finish()
return
}

Expand Down
1 change: 1 addition & 0 deletions packages/gatsby/src/utils/api-runner-node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,7 @@ const runAPI = async (plugin, api, args, activity) => {
const apiCallArgs = [
{
...args,
parentSpan: pluginSpan,
basePath: pathPrefix,
pathPrefix: publicPath,
actions,
Expand Down
5 changes: 5 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3723,6 +3723,11 @@
resolved "https://registry.yarnpkg.com/@open-draft/until/-/until-1.0.3.tgz#db9cc719191a62e7d9200f6e7bab21c5b848adca"
integrity sha512-Aq58f5HiWdyDlFffbbSjAlv596h/cOnt2DO1w3DOC7OJ5EHs0hd/nycJfiu9RJbT6Yk6F1knnRRXNSpxoIVZ9Q==

"@opentelemetry/[email protected]":
version "0.24.0"
resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-0.24.0.tgz#1028ef0e0923b24916158d80d2ddfd67ea8b6740"
integrity sha512-a/szuMQV0Quy0/M7kKdglcbRSoorleyyOwbTNNJ32O+RBN766wbQlMTvdimImTmwYWGr+NJOni1EcC242WlRcA==

"@pmmmwh/react-refresh-webpack-plugin@^0.4.3":
version "0.4.3"
resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.4.3.tgz#1eec460596d200c0236bf195b078a5d1df89b766"
Expand Down

0 comments on commit 91187da

Please sign in to comment.