Merge pull request #444 from clearlydefined/add-go-support-revamp

adds support for fetching and extracting go packages to the crawler

config/cdConfig.js

@@ -32,6 +32,7 @@ module.exports = {
     cratesio: {},
     debian: { cdFileLocation: cd_file.location },
     git: {},
+    go: {},
     mavenCentral: {},
     mavenGoogle: {},
     npmjs: {},
@@ -51,6 +52,7 @@ module.exports = {
       installDir: config.get('FOSSOLOGY_HOME') || '/mnt/c/git/fo/fossology/src/'
     },
     gem: { githubToken },
+    go: { githubToken },
     licensee: {},
     maven: { githubToken },
     npm: { githubToken },

config/map.js

@@ -50,6 +50,15 @@ const deb = {
   fossology
 }
 
+const go = {
+  _type: 'go',
+  source,
+  clearlydefined,
+  licensee,
+  scancode,
+  fossology
+}
+
 const maven = {
   _type: 'maven',
   source,
@@ -108,6 +117,7 @@ const _package = {
   npm,
   crate,
   deb,
+  go,
   maven,
   nuget,
   pod,
@@ -135,6 +145,7 @@ const entities = {
   npm,
   crate,
   deb,
+  go,
   maven,
   nuget,
   composer,

lib/entitySpec.js

@@ -39,9 +39,8 @@ class EntitySpec {
     const revisionPart = this.revision ? `:revision:${this.revision}` : ''
     const toolVersionPart = this.toolVersion ? `:${this.toolVersion}` : ''
     const toolPart = this.tool ? `:tool:${this.tool}` : ''
-    return `urn:${this.type}:${this.provider}:${this.namespace || '-'}:${
-      this.name
-    }${revisionPart}${toolPart}${toolVersionPart}`
+    return `urn:${this.type}:${this.provider}:${this.namespace || '-'}:${this.name
+      }${revisionPart}${toolPart}${toolVersionPart}`
   }
 
   toUrl() {
@@ -52,9 +51,8 @@ class EntitySpec {
     const revisionPart = this.revision ? `/${this.revision}` : ''
     const toolVersionPart = this.toolVersion ? `/${this.toolVersion}` : ''
     const toolPart = this.tool ? `/tool/${this.tool}` : ''
-    return `${this.type}/${this.provider}/${this.namespace || '-'}/${
-      this.name
-    }${revisionPart}${toolPart}${toolVersionPart}`
+    return `${this.type}/${this.provider}/${this.namespace || '-'}/${this.name
+      }${revisionPart}${toolPart}${toolVersionPart}`
   }
 }
 

providers/fetch/goFetch.js

@@ -0,0 +1,111 @@
+const { clone } = require('lodash')
+const requestPromise = require('request-promise-native')
+const AbstractFetch = require('./abstractFetch')
+const nodeRequest = require('request')
+const fs = require('fs')
+
+class GoFetch extends AbstractFetch {
+
+  canHandle(request) {
+    const spec = this.toSpec(request)
+    return spec && spec.provider === 'golang'
+  }
+
+  async handle(request) {
+    const spec = this.toSpec(request)
+    if (!spec.revision) spec.revision = await this._getLatestVersion(spec)
+    if (!spec.revision) return this.markSkip(request)
+
+    request.url = spec.toUrl()
+
+    super.handle(request)
+
+    const info = await this._getInfo(spec)
+    if (!info) return this.markSkip(request)
+
+    const artifact = this.createTempFile(request)
+    const artifactResult = await this._getArtifact(spec, artifact.name)
+    if (!artifactResult) return this.markSkip(request)
+
+    const dir = this.createTempDir(request)
+
+    await this.decompress(artifact.name, dir.name)
+
+    const hashes = await this.computeHashes(artifact.name)
+    const releaseDate = info.Time
+
+    request.document = this._createDocument(dir, releaseDate, hashes)
+    request.contentOrigin = 'origin'
+    request.casedSpec = clone(spec)
+
+    return request
+  }
+
+  async _getLatestVersion(spec) {
+    const initial_url = `https://${spec.provider}/${spec.namespace}/${spec.name}/@v/list`
+    const replace_encoded_url = this._replace_encodings(initial_url)
+    const url = replace_encoded_url.replace(/null\//g, '')
+
+    const response = await requestPromise({ url })
+    const versions = response.toString().split('\n').sort()
+
+    // return last version in sorted versions array
+    return versions[versions.length - 1]
+  }
+
+  _convert_to_versions_array(versions_string) {
+    versions_string.split('\n').sort()
+  }
+
+  _createDocument(dir, releaseDate, hashes) {
+    return { location: dir.name, releaseDate, hashes }
+  }
+
+  _buildUrl(spec, extension = '.zip') {
+    let initial_url = `https://proxy.golang.org/${spec.namespace}/${spec.name}/@v/${spec.revision}${extension}`
+    return this._replace_encodings(this._remove_blank_fields(initial_url))
+  }
+
+  _remove_blank_fields(url) {
+    return `${url.replace(/-\//g, '')}`
+  }
+
+  _replace_encodings(url) {
+    return `${url.replace(/%2f/g, '/')}`
+  }
+
+  async _getArtifact(spec, destination) {
+    const url = this._buildUrl(spec)
+
+    const status = await new Promise(resolve => {
+      nodeRequest
+        .get(url, (error, response) => {
+          if (error) this.logger.error(this._google_proxy_error_string(error))
+          if (response.statusCode !== 200) return resolve(false)
+        })
+        .pipe(fs.createWriteStream(destination).on('finish', () => resolve(true)))
+    })
+
+    if (status) return true
+  }
+
+  async _getInfo(spec) {
+    const url = this._buildUrl(spec, '.info')
+    let content
+
+    try {
+      content = await requestPromise({ url })
+    } catch (error) {
+      if (error.statusCode === 404) return null
+      else throw this._google_proxy_error_string(error)
+    }
+
+    return JSON.parse(content.toString())
+  }
+
+  _google_proxy_error_string(error) {
+    return `Error encountered when querying proxy.golang.org. Please check whether the component has a valid go.mod file. ${error}`
+  }
+}
+
+module.exports = options => new GoFetch(options)

providers/index.js

@@ -16,6 +16,7 @@ module.exports = {
     cratesio: require('./fetch/cratesioFetch'),
     debian: require('./fetch/debianFetch'),
     git: require('./fetch/gitCloner'),
+    go: require('./fetch/goFetch'),
     mavenCentral: require('./fetch/mavencentralFetch'),
     mavenGoogle: require('./fetch/mavenGoogleFetch'),
     npmjs: require('./fetch/npmjsFetch'),
@@ -30,6 +31,7 @@ module.exports = {
     deb: require('./process/debExtract'),
     debsrc: require('./process/debsrcExtract'),
     gem: require('./process/gemExtract'),
+    go: require('./process/goExtract'),
     licensee: require('./process/licensee'),
     maven: require('./process/mavenExtract'),
     npm: require('./process/npmExtract'),

providers/process/goExtract.js

@@ -0,0 +1,35 @@
+const AbstractClearlyDefinedProcessor = require('./abstractClearlyDefinedProcessor')
+const sourceDiscovery = require('../../lib/sourceDiscovery')
+const { merge } = require('lodash')
+
+class GoExtract extends AbstractClearlyDefinedProcessor {
+  constructor(options, sourceFinder) {
+    super(options)
+    this.sourceFinder = sourceFinder
+  }
+
+  canHandle(request) {
+    const spec = this.toSpec(request)
+    return request.type === 'go' && spec && spec.type === 'go'
+  }
+
+  get toolVersion() {
+    return '1.0.0'
+  }
+
+  async handle(request) {
+    if (this.isProcessing(request)) {
+      await super.handle(request)
+      const spec = this.toSpec(request)
+      this._createDocument(request, spec)
+    }
+    this.linkAndQueueTool(request, 'licensee')
+    this.linkAndQueueTool(request, 'scancode')
+  }
+
+  _createDocument(request) {
+    request.document = merge(this.clone(request.document))
+  }
+}
+
+module.exports = (options, sourceFinder) => new GoExtract(options, sourceFinder || sourceDiscovery)

providers/process/package.js

@@ -3,7 +3,7 @@
 
 const AbstractProcessor = require('./abstractProcessor')
 
-const supportedTypes = ['npm', 'crate', 'maven', 'nuget', 'gem', 'pod', 'pypi', 'composer', 'deb']
+const supportedTypes = ['npm', 'crate', 'maven', 'nuget', 'gem', 'go', 'pod', 'pypi', 'composer', 'deb']
 
 class PackageProcessor extends AbstractProcessor {
   shouldFetch() {

test/fixtures/go/list

@@ -0,0 +1,10 @@
+v1.3.0
+v1.5.1
+v1.0.0
+v1.5.3-pre1
+v1.5.0
+v1.5.2
+v1.2.1
+v1.4.0
+v1.2.0
+v1.1.0

test/fixtures/go/v1.3.0.info

@@ -0,0 +1 @@
+{"Version":"v1.3.0","Time":"2018-02-14T00:54:53Z"}

test/fixtures/go/v1.3.0.mod

@@ -0,0 +1 @@
+module "rsc.io/quote"

test/unit/lib/entitySpecTests.js

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
+// SPDX-License-Identifier: MIT
+
+const expect = require('chai').expect
+const EntitySpec = require('../../../lib/entitySpec')
+
+describe('entitySpec', () => {
+  it('creates an EntitySpec from a url', () => {
+    const entityFromUrl = EntitySpec.fromUrl('cd:/go/golang/rsc.io/quote/v3.1.0')
+
+    expect(entityFromUrl.type).to.eq('go')
+    expect(entityFromUrl.provider).to.eq('golang')
+    expect(entityFromUrl.namespace).to.eq('rsc.io')
+    expect(entityFromUrl.name).to.eq('quote')
+    expect(entityFromUrl.revision).to.eq('v3.1.0')
+  })
+
+  it('creates an EntitySpec from a Maven url', () => {
+    const entityFromUrl = EntitySpec.fromUrl('cd:/maven/mavencentral/org.eclipse.xtext/org.eclipse.xtext.common.types/2.25.0')
+
+    expect(entityFromUrl.namespace).to.eq('org.eclipse.xtext')
+    expect(entityFromUrl.name).to.eq('org.eclipse.xtext.common.types')
+    expect(entityFromUrl.revision).to.eq('2.25.0')
+  })
+})