From 200455f48c6d99f126bde58f8c545b34ae09bd2e Mon Sep 17 00:00:00 2001 From: Nell Shamrell Date: Wed, 10 Nov 2021 14:52:54 -0800 Subject: [PATCH 1/8] exp Signed-off-by: Nell Shamrell --- app.js | 1 + routes/definitions.js | 1 + 2 files changed, 2 insertions(+) diff --git a/app.js b/app.js index 16fb2cce..6af0f673 100644 --- a/app.js +++ b/app.js @@ -156,6 +156,7 @@ function createApp(config) { // catch 404 and forward to error handler const requestHandler = (req, res, next) => { logger.info('Error when handling a request', { rawUrl: req._parsedUrl._raw, baseUrl: req.baseUrl, originalUrl: req.originalUrl, params: req.params, route: req.route, url: req.url }) + logger.info('Route not found', { route: req.url }) const err = new Error('Not Found') err.status = 404 next(err) diff --git a/routes/definitions.js b/routes/definitions.js index fcbc017b..256de329 100644 --- a/routes/definitions.js +++ b/routes/definitions.js @@ -13,6 +13,7 @@ const logger = require('../providers/logging/logger') // API for serving consumers and API router.get('/:type/:provider/:namespace/:name/:revision/pr/:pr', asyncMiddleware(getDefinition)) router.get('/:type/:provider/:namespace/:name/:revision', asyncMiddleware(getDefinition)) +router.get('/:type/:provider/:namespace/:name/:revision/:extra', asyncMiddleware(getDefinition)) async function getDefinition(request, response) { const log = logger() From 7417684ab99cc636b328990d207242d18af0eb8e Mon Sep 17 00:00:00 2001 From: Nell Shamrell Date: Wed, 10 Nov 2021 17:34:30 -0800 Subject: [PATCH 2/8] brute force solution for dealing with additional slashes in namespaces Signed-off-by: Nell Shamrell --- routes/definitions.js | 52 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/routes/definitions.js b/routes/definitions.js index 256de329..70ba2a3b 100644 --- a/routes/definitions.js +++ b/routes/definitions.js @@ -13,12 +13,60 @@ const logger = require('../providers/logging/logger') // API for serving consumers and API router.get('/:type/:provider/:namespace/:name/:revision/pr/:pr', asyncMiddleware(getDefinition)) router.get('/:type/:provider/:namespace/:name/:revision', asyncMiddleware(getDefinition)) -router.get('/:type/:provider/:namespace/:name/:revision/:extra', asyncMiddleware(getDefinition)) + +// When a request for a component with slashes in the namespace comes in +// They are initially encoded with url encoding like this go/golang/github.com%2fgorilla/mux/v1.7.3 +// However, when it comes through a load balancer, the load balancer sometimes decodes the encoding to +// go/golang/github.com/gorilla/mux/v1.7.3 +// which causes routing errors unless we allow for additional fields +// We currently allow up to one extra field +router.get('/:type/:provider/:namespace/:name/:revision/:extra1?/:extra2?/:extra3?', asyncMiddleware(getDefinition)) async function getDefinition(request, response) { const log = logger() log.info('getDefinition route hit', { ts: new Date().toISOString(), requestParams: request.params }) - const coordinates = utils.toEntityCoordinatesFromRequest(request) + + // Painful way of handling go namespaces with multiple slashes + // Unfortunately, it seems the best option without doing a massive + // rearchitecture of the entire coordinate system + if (request.params.type == "go" && request.params.provider == "golang") { + let namespaceNameRevision = `${request.params.namespace}/${request.params.name}/${request.params.revision}` + + if (request.params.extra1) { + namespaceNameRevision += `/${request.params.extra1}` + } + + if (request.params.extra2) { + namespaceNameRevision += `/${request.params.extra2}` + } + + if (request.params.extra3) { + namespaceNameRevision += `/${request.params.extra3}` + } + + console.log(namespaceNameRevision) + + let splitString = namespaceNameRevision.split("/") + + // Pull off the last part of the string as the revision + const revision = splitString.pop() + const name = splitString.pop() + + const nameSpace = splitString.join('/') + + console.log(revision) + console.log(name) + console.log(nameSpace) + + request.params.namespace = nameSpace + request.params.name = name + request.params.revision = revision + + coordinates = utils.toEntityCoordinatesFromRequest(request) + } else { + coordinates = utils.toEntityCoordinatesFromRequest(request) + } + const pr = request.params.pr const force = request.query.force const expand = request.query.expand === '-files' ? '-files' : null // only support '-files' for now From 334cf308f39a0f3c4cd2a17b46e742cf00e6c61b Mon Sep 17 00:00:00 2001 From: Nell Shamrell Date: Thu, 11 Nov 2021 10:39:52 -0800 Subject: [PATCH 3/8] adds new function to create entity coordinates from arguments Signed-off-by: Nell Shamrell --- lib/utils.js | 11 +++++++++++ test/lib/util.js | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/utils.js b/lib/utils.js index 021940ff..580b10bd 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -32,6 +32,16 @@ function toEntityCoordinatesFromRequest(request) { ) } +function toEntityCoordinatesFromArgs(args) { + return new EntityCoordinates( + args['type'], + args['provider'], + args['namespace'] === '-' ? null : reEncodeSlashes(args['namespace']), + args['name'], + args['revision'] + ) +} + // When someone requests a component with a slash in the namespace // They encode that slash as %2f // For example: https://clearlydefined.io/definitions/go/golang/rsc.io%2fquote/v3/v3.1.0 @@ -442,6 +452,7 @@ const _licenseUrlOverrides = [ module.exports = { toEntityCoordinatesFromRequest, toResultCoordinatesFromRequest, + toEntityCoordinatesFromArgs, getLatestVersion, extractDate, setIfValue, diff --git a/test/lib/util.js b/test/lib/util.js index 1d99eb10..b76ce7be 100644 --- a/test/lib/util.js +++ b/test/lib/util.js @@ -488,6 +488,25 @@ describe('Utils toEntityCoordinatesFromRequest', () => { }) }) +describe('Utils toEntityCoordinatesFromArgs', () => { + const args = { + type: 'go', + provider: 'golang', + namespace: 'rsc.io/quote', + name: 'v3', + revision: 'v3.1.0' + } + + it('should turn the args into entity coordinates', () => { + const result = utils.toEntityCoordinatesFromArgs(args) + expect(result.type).to.eq('go') + expect(result.provider).to.eq('golang') + expect(result.namespace).to.eq('rsc.io%2fquote') + expect(result.name).to.eq('v3') + expect(result.revision).to.eq('v3.1.0') + }) +}) + describe('Utils getLicenseLocations', () => { const npmRequest = { params: { From 9148e6b327595605120cf2657708ccb3d2b85b4d Mon Sep 17 00:00:00 2001 From: Nell Shamrell Date: Thu, 11 Nov 2021 10:46:45 -0800 Subject: [PATCH 4/8] refactoring Signed-off-by: Nell Shamrell --- routes/definitions.js | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/routes/definitions.js b/routes/definitions.js index 70ba2a3b..4a585a5a 100644 --- a/routes/definitions.js +++ b/routes/definitions.js @@ -44,8 +44,6 @@ async function getDefinition(request, response) { namespaceNameRevision += `/${request.params.extra3}` } - console.log(namespaceNameRevision) - let splitString = namespaceNameRevision.split("/") // Pull off the last part of the string as the revision @@ -54,15 +52,15 @@ async function getDefinition(request, response) { const nameSpace = splitString.join('/') - console.log(revision) - console.log(name) - console.log(nameSpace) - - request.params.namespace = nameSpace - request.params.name = name - request.params.revision = revision - - coordinates = utils.toEntityCoordinatesFromRequest(request) + coordinates = utils.toEntityCoordinatesFromArgs( + { + 'type': request.params.type, + 'provider': request.params.provider, + 'namespace': nameSpace, + 'name': name, + 'revision': revision + } + ) } else { coordinates = utils.toEntityCoordinatesFromRequest(request) } From 2f2378c0a2a7c0558a1f3c29c97561112d6e99d8 Mon Sep 17 00:00:00 2001 From: Nell Shamrell Date: Thu, 11 Nov 2021 11:03:38 -0800 Subject: [PATCH 5/8] more refactoring and cleanup Signed-off-by: Nell Shamrell --- lib/utils.js | 21 ++++++++++++++++++++- routes/definitions.js | 18 ++++-------------- test/lib/util.js | 20 ++++++++++++++++++++ 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 580b10bd..178e6001 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -53,6 +53,24 @@ function reEncodeSlashes(namespace) { return `${namespace.replace(/\//g, '%2f')}` } +function parseNamespaceNameRevision(request) { + let namespaceNameRevision = `${request.params.namespace}/${request.params.name}/${request.params.revision}` + + if (request.params.extra1) { + namespaceNameRevision += `/${request.params.extra1}` + } + + if (request.params.extra2) { + namespaceNameRevision += `/${request.params.extra2}` + } + + if (request.params.extra3) { + namespaceNameRevision += `/${request.params.extra3}` + } + + return namespaceNameRevision +} + function getLatestVersion(versions) { if (!Array.isArray(versions)) return versions if (versions.length === 0) return null @@ -467,5 +485,6 @@ module.exports = { isLicenseFile, simplifyAttributions, isDeclaredLicense, - parseUrn + parseUrn, + parseNamespaceNameRevision } diff --git a/routes/definitions.js b/routes/definitions.js index 4a585a5a..2a37df4c 100644 --- a/routes/definitions.js +++ b/routes/definitions.js @@ -30,26 +30,16 @@ async function getDefinition(request, response) { // Unfortunately, it seems the best option without doing a massive // rearchitecture of the entire coordinate system if (request.params.type == "go" && request.params.provider == "golang") { - let namespaceNameRevision = `${request.params.namespace}/${request.params.name}/${request.params.revision}` - - if (request.params.extra1) { - namespaceNameRevision += `/${request.params.extra1}` - } - - if (request.params.extra2) { - namespaceNameRevision += `/${request.params.extra2}` - } - - if (request.params.extra3) { - namespaceNameRevision += `/${request.params.extra3}` - } - + let namespaceNameRevision = utils.parseNamespaceNameRevision(request) let splitString = namespaceNameRevision.split("/") // Pull off the last part of the string as the revision const revision = splitString.pop() + + // Pull of next part of the string as the name const name = splitString.pop() + // Join the rest of the string as the namespace const nameSpace = splitString.join('/') coordinates = utils.toEntityCoordinatesFromArgs( diff --git a/test/lib/util.js b/test/lib/util.js index b76ce7be..a84cde63 100644 --- a/test/lib/util.js +++ b/test/lib/util.js @@ -507,6 +507,26 @@ describe('Utils toEntityCoordinatesFromArgs', () => { }) }) +describe('Utils parseNamespaceNameRevision', () => { + const fakeSlashNamespaceRequest = { + params: { + type: 'go', + provider: 'golang', + namespace: 'rsc.io/quote', + name: 'v3', + revision: 'foo', + extra1: 'bar', + extra2: 'bah', + extra3: 'v3.1.0', + } + } + + it('parses the args into one string', () => { + const result = utils.parseNamespaceNameRevision(fakeSlashNamespaceRequest) + expect(result).to.eq('rsc.io/quote/v3/foo/bar/bah/v3.1.0') + }) +}) + describe('Utils getLicenseLocations', () => { const npmRequest = { params: { From cdddaba56107503cd87c073feb5992e10584d40e Mon Sep 17 00:00:00 2001 From: Nell Shamrell Date: Thu, 11 Nov 2021 11:16:43 -0800 Subject: [PATCH 6/8] corrects ci errors Signed-off-by: Nell Shamrell --- routes/definitions.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/routes/definitions.js b/routes/definitions.js index 2a37df4c..ac8f4a0e 100644 --- a/routes/definitions.js +++ b/routes/definitions.js @@ -26,12 +26,14 @@ async function getDefinition(request, response) { const log = logger() log.info('getDefinition route hit', { ts: new Date().toISOString(), requestParams: request.params }) + let coordinates + // Painful way of handling go namespaces with multiple slashes // Unfortunately, it seems the best option without doing a massive // rearchitecture of the entire coordinate system - if (request.params.type == "go" && request.params.provider == "golang") { + if (request.params.type == 'go' && request.params.provider == 'golang') { let namespaceNameRevision = utils.parseNamespaceNameRevision(request) - let splitString = namespaceNameRevision.split("/") + let splitString = namespaceNameRevision.split('/') // Pull off the last part of the string as the revision const revision = splitString.pop() From 86d54ac81507d25b64dee17ad4ed87378da1bfb1 Mon Sep 17 00:00:00 2001 From: Nell Shamrell Date: Thu, 11 Nov 2021 14:36:44 -0800 Subject: [PATCH 7/8] more cleanup Signed-off-by: Nell Shamrell --- app.js | 1 - routes/definitions.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app.js b/app.js index 6af0f673..16fb2cce 100644 --- a/app.js +++ b/app.js @@ -156,7 +156,6 @@ function createApp(config) { // catch 404 and forward to error handler const requestHandler = (req, res, next) => { logger.info('Error when handling a request', { rawUrl: req._parsedUrl._raw, baseUrl: req.baseUrl, originalUrl: req.originalUrl, params: req.params, route: req.route, url: req.url }) - logger.info('Route not found', { route: req.url }) const err = new Error('Not Found') err.status = 404 next(err) diff --git a/routes/definitions.js b/routes/definitions.js index ac8f4a0e..9ae148e8 100644 --- a/routes/definitions.js +++ b/routes/definitions.js @@ -19,7 +19,7 @@ router.get('/:type/:provider/:namespace/:name/:revision', asyncMiddleware(getDef // However, when it comes through a load balancer, the load balancer sometimes decodes the encoding to // go/golang/github.com/gorilla/mux/v1.7.3 // which causes routing errors unless we allow for additional fields -// We currently allow up to one extra field +// We currently allow up to three extra fields (that means up to three slashes in the namespace) router.get('/:type/:provider/:namespace/:name/:revision/:extra1?/:extra2?/:extra3?', asyncMiddleware(getDefinition)) async function getDefinition(request, response) { From a498e544fca038f38d7d7ad8ca0cd5914fe6755b Mon Sep 17 00:00:00 2001 From: Nell Shamrell Date: Fri, 12 Nov 2021 11:32:54 -0800 Subject: [PATCH 8/8] corrects == to === Signed-off-by: Nell Shamrell --- routes/definitions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/definitions.js b/routes/definitions.js index 9ae148e8..a1fee5b1 100644 --- a/routes/definitions.js +++ b/routes/definitions.js @@ -31,7 +31,7 @@ async function getDefinition(request, response) { // Painful way of handling go namespaces with multiple slashes // Unfortunately, it seems the best option without doing a massive // rearchitecture of the entire coordinate system - if (request.params.type == 'go' && request.params.provider == 'golang') { + if (request.params.type === 'go' && request.params.provider === 'golang') { let namespaceNameRevision = utils.parseNamespaceNameRevision(request) let splitString = namespaceNameRevision.split('/')