From 668e99c11a12699d9b1aca20cf48e1969055acf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20I=2E=20Ni=C8=9Bu?= <10091929+novalex@users.noreply.github.com> Date: Wed, 19 Jul 2023 12:15:57 +0300 Subject: [PATCH] feat: add SCM report test support [ZEN-668] (#4745) * chore: update code-client version * feat: implement report functionality for SCM projects * test: split report tests from regular snyk code tests * test: refactor report tests, remove redundant mocked Sarif test * test: add SCM report sample response and test * fix: add project data to analysisContext * fix: pass analysisContext to analyzeScmProject * test: make exit code test name clearer * chore: refactor analysis methods and data flow --- .gitleaksignore | 2 + package-lock.json | 95 ++++- package.json | 2 +- src/lib/plugins/sast/analysis.ts | 202 +++++++---- src/lib/plugins/sast/types.ts | 7 +- src/lib/types.ts | 3 + .../sample-analyze-scm-project-response.json | 269 +++++++++++++++ .../snyk-code/snyk-code-test-report.spec.ts | 266 ++++++++++++++ .../unit/snyk-code/snyk-code-test.spec.ts | 324 ++++-------------- 9 files changed, 825 insertions(+), 345 deletions(-) create mode 100644 test/fixtures/sast/sample-analyze-scm-project-response.json create mode 100644 test/jest/unit/snyk-code/snyk-code-test-report.spec.ts diff --git a/.gitleaksignore b/.gitleaksignore index 2f5ee19d68..eb9c21e375 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -77,3 +77,5 @@ ac6208df74be9a21f2d14caaaee8aec98195b336:test/fixtures/iac/terraform-plan/tf-pla cba65a3a91c64db2ee92c87e5972602b6c959586:test/fixtures/sast/sample-analyze-folders-response.json:generic-api-key:3 6380d9d4147491cadee99113701516ebb8242836:src/cli/commands/test/iac-local-execution/parsers/hcl2json.js:generic-api-key:9827 c2de35484dcad696a6ee32f2fa317d5cfaffc133:test/fixtures/code/sample-analyze-folders-response.json:generic-api-key:3 +dc5190fcfbbc315bab3ed99a14262242c55bf9b2:test/fixtures/sast/sample-analyze-scm-project-response.json:generic-api-key:4 +0fb9746f823f48b2802269569e8575f4dfd3d95d:help/cli-commands/iac-test.md:snyk-api-token:219 diff --git a/package-lock.json b/package-lock.json index 92f83911eb..ff39018cb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@sentry/node": "^7.34.0", "@snyk/cli-interface": "2.12.0", "@snyk/cloud-config-parser": "^1.14.5", - "@snyk/code-client": "^4.18.5", + "@snyk/code-client": "^4.19.0", "@snyk/dep-graph": "^2.6.1", "@snyk/docker-registry-v2-client": "^2.10.0", "@snyk/fix": "file:packages/snyk-fix", @@ -2133,9 +2133,9 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/@snyk/code-client": { - "version": "4.18.5", - "resolved": "https://registry.npmjs.org/@snyk/code-client/-/code-client-4.18.5.tgz", - "integrity": "sha512-b5gTJLS7qBGniV4j3r8LtRGO7L9Bi3Nwgl76ROBQKcUANziyk2b375wSbt9dqlc+MKzzrC5KdYYt5PeV4M0Yfg==", + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@snyk/code-client/-/code-client-4.19.0.tgz", + "integrity": "sha512-IB/oYSCROGY04h74UYSs+nhCpNVPqWDZ7DO5nuzLun8YfNiUMh98V3OdUl+DMrYBxKUUE2HoLEr+GfPkPUpVzA==", "dependencies": { "@deepcode/dcignore": "^1.0.4", "@types/flat-cache": "^2.0.0", @@ -2144,18 +2144,53 @@ "@types/lodash.union": "^4.6.6", "@types/sarif": "^2.1.4", "@types/uuid": "^8.3.1", - "fast-glob": "^3.2.11", + "fast-glob": "3.3.0", "ignore": "^5.1.8", "lodash.omit": "^4.5.0", "lodash.pick": "^4.4.0", "lodash.union": "^4.6.0", "multimatch": "^5.0.0", - "needle": "~3.0.0", + "needle": "~3.2.0", "p-map": "^3.0.0", "uuid": "^8.3.2", "yaml": "^1.10.2" } }, + "node_modules/@snyk/code-client/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/@snyk/code-client/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@snyk/code-client/node_modules/needle": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-3.2.0.tgz", + "integrity": "sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==", + "dependencies": { + "debug": "^3.2.6", + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + }, + "bin": { + "needle": "bin/needle" + }, + "engines": { + "node": ">= 4.4.x" + } + }, "node_modules/@snyk/code-client/node_modules/p-map": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", @@ -8088,9 +8123,9 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-glob": { - "version": "3.2.11", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.11.tgz", - "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.0.tgz", + "integrity": "sha512-ChDuvbOypPuNjO8yIDf36x7BlZX1smcUMTTcyoIjycexOxd6DFsKsg21qVBzEmr3G7fUKIRy2/psii+CIUt7FA==", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -22522,9 +22557,9 @@ } }, "@snyk/code-client": { - "version": "4.18.5", - "resolved": "https://registry.npmjs.org/@snyk/code-client/-/code-client-4.18.5.tgz", - "integrity": "sha512-b5gTJLS7qBGniV4j3r8LtRGO7L9Bi3Nwgl76ROBQKcUANziyk2b375wSbt9dqlc+MKzzrC5KdYYt5PeV4M0Yfg==", + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@snyk/code-client/-/code-client-4.19.0.tgz", + "integrity": "sha512-IB/oYSCROGY04h74UYSs+nhCpNVPqWDZ7DO5nuzLun8YfNiUMh98V3OdUl+DMrYBxKUUE2HoLEr+GfPkPUpVzA==", "requires": { "@deepcode/dcignore": "^1.0.4", "@types/flat-cache": "^2.0.0", @@ -22533,18 +22568,44 @@ "@types/lodash.union": "^4.6.6", "@types/sarif": "^2.1.4", "@types/uuid": "^8.3.1", - "fast-glob": "^3.2.11", + "fast-glob": "3.3.0", "ignore": "^5.1.8", "lodash.omit": "^4.5.0", "lodash.pick": "^4.4.0", "lodash.union": "^4.6.0", "multimatch": "^5.0.0", - "needle": "~3.0.0", + "needle": "~3.2.0", "p-map": "^3.0.0", "uuid": "^8.3.2", "yaml": "^1.10.2" }, "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + }, + "needle": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-3.2.0.tgz", + "integrity": "sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==", + "requires": { + "debug": "^3.2.6", + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + } + }, "p-map": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", @@ -27426,9 +27487,9 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "fast-glob": { - "version": "3.2.11", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.11.tgz", - "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.0.tgz", + "integrity": "sha512-ChDuvbOypPuNjO8yIDf36x7BlZX1smcUMTTcyoIjycexOxd6DFsKsg21qVBzEmr3G7fUKIRy2/psii+CIUt7FA==", "requires": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", diff --git a/package.json b/package.json index 2564a57805..2887130e63 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "@sentry/node": "^7.34.0", "@snyk/cli-interface": "2.12.0", "@snyk/cloud-config-parser": "^1.14.5", - "@snyk/code-client": "^4.18.5", + "@snyk/code-client": "^4.19.0", "@snyk/dep-graph": "^2.6.1", "@snyk/docker-registry-v2-client": "^2.10.0", "@snyk/fix": "file:packages/snyk-fix", diff --git a/src/lib/plugins/sast/analysis.ts b/src/lib/plugins/sast/analysis.ts index aa6f685ef1..5853bd5c47 100644 --- a/src/lib/plugins/sast/analysis.ts +++ b/src/lib/plugins/sast/analysis.ts @@ -1,7 +1,10 @@ import { analyzeFolders, + analyzeScmProject, AnalysisSeverity, MAX_FILE_SIZE, + FileAnalysis, + ScmAnalysis, } from '@snyk/code-client'; import { ReportingDescriptor, Result } from 'sarif'; import { SEVERITY } from '../../snyk-test/legacy'; @@ -28,6 +31,27 @@ import { getCodeClientProxyUrl } from '../../code-config'; const debug = debugLib('snyk-code'); +type GetCodeAnalysisArgs = { + options: Options; + fileOptions: { + paths: string[]; + }; + connectionOptions: { + org?: string; + source: string; + baseURL: string; + requestId: string; + sessionToken: string; + }; + analysisOptions: { + severity: AnalysisSeverity; + }; + supportedLanguages?: string[]; +}; + +/** + * Bootstrap and trigger a Code test, then return the results. + */ export async function getCodeTestResults( root: string, options: Options, @@ -36,42 +60,16 @@ export async function getCodeTestResults( ): Promise { await spinner.clearAll(); analysisProgressUpdate(); - const codeAnalysis = await getCodeAnalysis( - root, - options, - sastSettings, - requestId, - ); - spinner.clearAll(); - if (!codeAnalysis) { - return null; - } - - return { - reportResults: codeAnalysis.reportResults, - analysisResults: codeAnalysis.analysisResults, - }; -} - -async function getCodeAnalysis( - root: string, - options: Options, - sastSettings: SastSettings, - requestId: string, -) { const isLocalCodeEngineEnabled = isLocalCodeEngine(sastSettings); if (isLocalCodeEngineEnabled) { validateLocalCodeEngineUrl(sastSettings.localCodeEngine.url); } - const source = 'snyk-cli'; const baseURL = isLocalCodeEngineEnabled ? sastSettings.localCodeEngine.url : getCodeClientProxyUrl(); - const org = sastSettings.org; - // TODO(james) This mirrors the implementation in request.ts and we need to use this for deeproxy calls // This ensures we support lowercase http(s)_proxy values as well // The weird IF around it ensures we don't create an envvar with @@ -91,54 +89,120 @@ async function getCodeAnalysis( }); } - const sessionToken = getAuthHeader(); - - const severity = options.severityThreshold - ? severityToAnalysisSeverity(options.severityThreshold) - : AnalysisSeverity.info; - - const result = await analyzeFolders({ - connection: { + const analysisArgs = { + options, + fileOptions: { + paths: [root], + }, + connectionOptions: { baseURL, - sessionToken, - source, + sessionToken: getAuthHeader(), + source: 'snyk-cli', requestId, - org, + org: sastSettings.org, + }, + analysisOptions: { + severity: options.severityThreshold + ? severityToAnalysisSeverity(options.severityThreshold) + : AnalysisSeverity.info, + }, + supportedLanguages: sastSettings.supportedLanguages, + }; + + const codeAnalysis = await getCodeAnalysis(analysisArgs); + + spinner.clearAll(); + + if (!codeAnalysis) { + return null; + } + + return { + reportResults: codeAnalysis.reportResults, + analysisResults: codeAnalysis.analysisResults, + }; +} + +/** + * Performs Code analysis and returns normalised results. + * Analysis method (i.e. file-based or SCM) is chosen based on flow options. + */ +async function getCodeAnalysis( + args: GetCodeAnalysisArgs, +): Promise { + const { + options, + fileOptions, + analysisOptions, + connectionOptions, + supportedLanguages, + } = args; + + const analysisContext = { + initiator: 'CLI', + flow: connectionOptions.source, + projectName: config.PROJECT_NAME, // back-compat + project: { + name: options['project-name'] || config.PROJECT_NAME || 'unknown', + publicId: options['project-id'] || 'unknown', + type: 'sast', + }, + org: { + name: connectionOptions.org || 'unknown', + displayName: 'unknown', + publicId: 'unknown', + flags: {}, }, - analysisOptions: { severity }, - fileOptions: { paths: [root] }, - ...(options.report && { + } as const; + + let result: FileAnalysis | ScmAnalysis | null = null; + + // When the "report" arg is provided the test results are published on the platform. + const isReportFlow = options.report ?? false; + // We differentiate between file-based reporting flows + // and SCM-based ones by looking at the "project-id" arg. + const isScmReportFlow = isReportFlow && options['project-id']; + + if (isScmReportFlow) { + // Run an SCM analysis test with reporting. + result = await analyzeScmProject({ + connection: connectionOptions, + analysisOptions, reportOptions: { - enabled: options.report ?? false, - projectName: options['project-name'], - targetName: options['target-name'], - targetRef: options['target-reference'], - remoteRepoUrl: options['remote-repo-url'], + projectId: options['project-id'], + commitId: options['commit-id'], }, - }), - analysisContext: { - initiator: 'CLI', - flow: source, - projectName: config.PROJECT_NAME, - org: { - name: sastSettings.org || 'unknown', - displayName: 'unknown', - publicId: 'unknown', - flags: {}, - }, - }, - languages: sastSettings.supportedLanguages, - }); - - if (result?.fileBundle.skippedOversizedFiles?.length) { - debug( - '\n', - chalk.yellow( - `Warning!\nFiles were skipped in the analysis due to their size being greater than ${MAX_FILE_SIZE}B. Skipped files: ${[ - ...result.fileBundle.skippedOversizedFiles, - ].join(', ')}`, - ), - ); + analysisContext, + }); + } else { + // Run a file-based test, optionally with reporting. + result = await analyzeFolders({ + connection: connectionOptions, + analysisOptions, + fileOptions, + ...(isReportFlow && { + reportOptions: { + enabled: true, + projectName: options['project-name'], + targetName: options['target-name'], + targetRef: options['target-reference'], + remoteRepoUrl: options['remote-repo-url'], + }, + }), + analysisContext, + languages: supportedLanguages, + }); + + if (result?.fileBundle.skippedOversizedFiles?.length) { + debug( + '\n', + chalk.yellow( + `Warning!\nFiles were skipped in the analysis due to their size being greater than ${MAX_FILE_SIZE}B. Skipped files: ${[ + ...result.fileBundle.skippedOversizedFiles, + ].join(', ')}`, + ), + ); + } } if (!result || result.analysisResults.type !== 'sarif') { diff --git a/src/lib/plugins/sast/types.ts b/src/lib/plugins/sast/types.ts index bf79a8b4d9..b4c7de352f 100644 --- a/src/lib/plugins/sast/types.ts +++ b/src/lib/plugins/sast/types.ts @@ -3,6 +3,7 @@ import { AnalysisResultSarif, FileAnalysis, ReportResult, + ScmAnalysis, } from '@snyk/code-client'; interface LocalCodeEngine { @@ -31,6 +32,6 @@ export interface CodeTestResults { analysisResults: AnalysisResultSarif; } -export interface CodeAnalysisResults extends FileAnalysis { - analysisResults: AnalysisResultSarif; -} +export type CodeAnalysisResults = + | (FileAnalysis & { analysisResults: AnalysisResultSarif }) + | ScmAnalysis; diff --git a/src/lib/types.ts b/src/lib/types.ts index e206643d40..74d6f9844d 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -87,6 +87,9 @@ export interface Options { report?: boolean; 'var-file'?: string; 'target-name'?: string; + // Used only with the Code (SAST) plugin. Allows running tests with reporting for existing projects. + 'project-id'?: string; + 'commit-id'?: string; } // TODO(kyegupov): catch accessing ['undefined-properties'] via noImplicitAny diff --git a/test/fixtures/sast/sample-analyze-scm-project-response.json b/test/fixtures/sast/sample-analyze-scm-project-response.json new file mode 100644 index 0000000000..1358ec1009 --- /dev/null +++ b/test/fixtures/sast/sample-analyze-scm-project-response.json @@ -0,0 +1,269 @@ +{ + "connection": { + "baseURL": "http://proxy.acme--snyk-url.io", + "sessionToken": "912D0461-5CCD-417B-8073-1305D1D896C2", + "source": "snyk-cli" + }, + "analysisOptions": { "severity": 1 }, + "analysisResults": { + "type": "sarif", + "sarif": { + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "SnykCode", + "semanticVersion": "1.0.0", + "version": "1.0.0", + "rules": [ + { + "id": "javascript/HttpToHttps", + "name": "HttpToHttps", + "shortDescription": { + "text": "Cleartext Transmission of Sensitive Information" + }, + "defaultConfiguration": { + "level": "warning" + }, + "help": { + "markdown": "", + "text": "" + }, + "properties": { + "tags": [ + "javascript", + "maintenance", + "http", + "server" + ], + "categories":["Security"], + "precision": "very-high", + "cwe": [ + "CWE-319" + ] + } + }, + { + "id": "javascript/DisablePoweredBy", + "name": "DisablePoweredBy", + "shortDescription": { + "text": "Information Exposure" + }, + "defaultConfiguration": { + "level": "warning" + }, + "help": { + "markdown": "", + "text": "" + }, + "properties": { + "tags": [ + "javascript", + "maintenance", + "express", + "server", + "helmet" + ], + "categories":["Security"], + "precision": "very-high", + "cwe": [ + "CWE-200" + ] + } + } + ] + } + }, + "results": [ + { + "ruleId": "javascript/HttpToHttps", + "ruleIndex": 0, + "level": "warning", + "message": { + "text": "http (used in require) is an insecure protocol and should not be used in new code.", + "markdown": "{0} (used in {1}) is an insecure protocol and should not be used in new code.", + "arguments": [ + "[http](0)", + "[require](1)" + ] + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "sample-project/goof/app.js", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 12, + "endLine": 12, + "startColumn": 12, + "endColumn": 26 + } + } + } + ], + "fingerprints": { + "0": "8ecbfa60577a4d25a3c18f790761ea95" + }, + "codeFlows": [ + { + "threadFlows": [ + { + "locations": [ + { + "location": { + "id": 0, + "physicalLocation": { + "artifactLocation": { + "uri": "app.js", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 12, + "endLine": 12, + "startColumn": 20, + "endColumn": 25 + } + } + } + }, + { + "location": { + "id": 1, + "physicalLocation": { + "artifactLocation": { + "uri": "app.js", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 12, + "endLine": 12, + "startColumn": 12, + "endColumn": 26 + } + } + } + } + ] + } + ] + } + ], + "suppressions": [ + { + "justification": "test-ignore-1", + "kind": "external" + } + ] + }, + { + "ruleId": "javascript/DisablePoweredBy", + "ruleIndex": 1, + "level": "warning", + "message": { + "text": "Disable X-Powered-By header for your Express app (consider using Helmet middleware), because it exposes information about the used framework to potential attackers.", + "markdown": "Disable X-Powered-By header for your {0} (consider using Helmet middleware), because it exposes information about the used framework to potential attackers.", + "arguments": [ + "[Express app](0)" + ] + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "sample-project/goof/app.js", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 27, + "endLine": 27, + "startColumn": 11, + "endColumn": 19 + } + } + } + ], + "fingerprints": { + "0": "8ecbfa60577a4d25a3c18f790761ea95" + }, + "codeFlows": [ + { + "threadFlows": [ + { + "locations": [ + { + "location": { + "id": 0, + "physicalLocation": { + "artifactLocation": { + "uri": "app.js", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 27, + "endLine": 27, + "startColumn": 11, + "endColumn": 19 + } + } + } + } + ] + } + ] + } + ], + "suppressions": [ + { + "justification": "test-ignore-2", + "kind": "external" + } + ] + } + ], + "properties": { + "coverage": [ + { + "files": 8, + "isSupported": true, + "lang": "JavaScript" + }, + { + "files": 1, + "isSupported": true, + "lang": "HTML" + } + ] + } + } + ] + }, + "status": "COMPLETE", + "timing": { + "fetchingCode": 3, + "analysis": 5043, + "queue": 5 + }, + "coverage": [ + { + "files": 8, + "isSupported": true, + "lang": "JavaScript" + }, + { + "files": 1, + "isSupported": true, + "lang": "HTML" + } + ] + }, + "reportResults": { + "projectId": "test-scm-project-id", + "snapshotId": "test-scm-snapshot-id", + "reportUrl": "test/scm/report/url" + } + } + \ No newline at end of file diff --git a/test/jest/unit/snyk-code/snyk-code-test-report.spec.ts b/test/jest/unit/snyk-code/snyk-code-test-report.spec.ts new file mode 100644 index 0000000000..cbef07275d --- /dev/null +++ b/test/jest/unit/snyk-code/snyk-code-test-report.spec.ts @@ -0,0 +1,266 @@ +import * as path from 'path'; +import { analyzeFolders, analyzeScmProject } from '@snyk/code-client'; + +import { loadJson } from '../../../utils'; +import { ArgsOptions } from '../../../../src/cli/args'; +import snykTest from '../../../../src/cli/commands/test'; +import * as ecosystems from '../../../../src/lib/ecosystems'; +import * as checks from '../../../../src/lib/plugins/sast/checks'; +import * as analysis from '../../../../src/lib/plugins/sast/analysis'; + +const { getCodeTestResults } = analysis; + +jest.mock('@snyk/code-client'); +const analyzeFoldersMock = analyzeFolders as jest.Mock; +const analyzeScmProjectMock = analyzeScmProject as jest.Mock; + +describe('Test snyk code with --report', () => { + let isSastEnabledForOrgSpy; + let trackUsageSpy; + let getCodeTestResultsSpy; + + const fixturePath = path.join(__dirname, '../../../fixtures/sast'); + + const sampleAnalyzeFoldersWithReportAndIgnoresResponse = loadJson( + path.join( + fixturePath, + 'sample-analyze-folders-with-report-and-ignores-response.json', + ), + ); + const sampleAnalyzeFoldersWithReportAndIgnoresOnlyResponse = loadJson( + path.join( + fixturePath, + 'sample-analyze-folders-with-report-and-ignores-only-response.json', + ), + ); + const sampleAnalyzeScmProjectResponse = loadJson( + path.join(fixturePath, 'sample-analyze-scm-project-response.json'), + ); + + const sastSettings = { + sastEnabled: true, + localCodeEngine: { + url: '', + allowCloudUpload: true, + enabled: false, + }, + }; + + beforeAll(() => { + isSastEnabledForOrgSpy = jest.spyOn(checks, 'getSastSettingsForOrg'); + trackUsageSpy = jest.spyOn(checks, 'trackUsage'); + getCodeTestResultsSpy = jest.spyOn(analysis, 'getCodeTestResults'); + }); + + beforeEach(() => { + isSastEnabledForOrgSpy.mockResolvedValueOnce({ + sastEnabled: true, + localCodeEngine: { + enabled: false, + }, + }); + trackUsageSpy.mockResolvedValue({}); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('file-based report flow - analyzeFolders', () => { + it('should return the right report results response', async () => { + const reportOptions = { + enabled: true, + projectName: 'test-project-name', + targetName: 'test-target-name', + targetRef: 'test-target-ref', + remoteRepoUrl: 'https://github.com/owner/repo', + }; + + analyzeFoldersMock.mockResolvedValue( + sampleAnalyzeFoldersWithReportAndIgnoresResponse, + ); + + const actual = await getCodeTestResults( + '.', + { + path: '', + code: true, + report: true, + 'project-name': reportOptions.projectName, + 'target-name': reportOptions.targetName, + 'target-reference': reportOptions.targetRef, + 'remote-repo-url': reportOptions.remoteRepoUrl, + }, + sastSettings, + 'test-id', + ); + + expect(analyzeFoldersMock).toHaveBeenCalledWith( + expect.objectContaining({ + reportOptions, + analysisContext: expect.objectContaining({ + flow: 'snyk-cli', + initiator: 'CLI', + project: { + name: reportOptions.projectName, + publicId: 'unknown', + type: 'sast', + }, + }), + }), + ); + + const expectedReportResults = { + projectId: 'test-project-id', + snapshotId: 'test-snapshot-id', + reportUrl: 'test/report/url', + }; + + expect(actual?.reportResults).toEqual(expectedReportResults); + }); + }); + + describe('SCM-based report flow - analyzeScmProject', () => { + it('should return the right report results response', async () => { + const reportOptions = { + projectId: 'test-scm-project-id', + commitId: 'test-commit-id', + }; + + analyzeScmProjectMock.mockResolvedValue(sampleAnalyzeScmProjectResponse); + + const actual = await getCodeTestResults( + '.', + { + path: '', + code: true, + report: true, + 'project-id': reportOptions.projectId, + 'commit-id': reportOptions.commitId, + }, + sastSettings, + 'test-id', + ); + + expect(analyzeScmProjectMock).toHaveBeenCalledWith( + expect.objectContaining({ + reportOptions, + analysisContext: expect.objectContaining({ + flow: 'snyk-cli', + initiator: 'CLI', + project: { + name: 'unknown', + publicId: reportOptions.projectId, + type: 'sast', + }, + }), + }), + ); + + const expectedReportResults = { + projectId: 'test-scm-project-id', + snapshotId: 'test-scm-snapshot-id', + reportUrl: 'test/scm/report/url', + }; + + expect(actual?.reportResults).toEqual(expectedReportResults); + }); + }); + + describe('exit codes', () => { + it('should exit with correct code (1) when issues are found (including ignored issues)', async () => { + const options: ArgsOptions = { + path: '', + traverseNodeModules: false, + showVulnPaths: 'none', + code: true, + report: true, + projectName: 'test-project', + _: [], + _doubleDashArgs: [], + }; + + getCodeTestResultsSpy.mockResolvedValue( + sampleAnalyzeFoldersWithReportAndIgnoresResponse, + ); + + await expect(snykTest('some/path', options)).rejects.toThrowError(); + }); + + it('should exit with correct code (0) when only ignored issues are found', async () => { + const options: ArgsOptions = { + path: '', + traverseNodeModules: false, + showVulnPaths: 'none', + code: true, + report: true, + projectName: 'test-project', + _: [], + _doubleDashArgs: [], + }; + + getCodeTestResultsSpy.mockResolvedValue( + sampleAnalyzeFoldersWithReportAndIgnoresOnlyResponse, + ); + + await expect(snykTest('some/path', options)).resolves.not.toThrowError(); + }); + }); + + describe('error handling', () => { + it.each([ + [ + 'disabled FF', + { + apiName: 'initReport', + statusCode: 400, + statusText: 'Bad request', + }, + 'Make sure this feature is enabled by contacting support.', + ], + [ + 'SARIF too large', + { + apiName: 'getReport', + statusCode: 400, + statusText: 'Analysis result set too large', + }, + 'The findings for this project may exceed the allowed size limit.', + ], + [ + 'analysis failed', + { + apiName: 'getReport', + statusCode: 500, + statusText: 'Analysis failed', + }, + "One or more of Snyk's services may be temporarily unavailable.", + ], + [ + 'bad gateway', + { + apiName: 'initReport', + statusCode: 502, + statusText: 'Bad Gateway', + }, + "One or more of Snyk's services may be temporarily unavailable.", + ], + ])( + 'when code-client fails, throw customized message for %s', + async (_, codeClientError, expectedErrorUserMessage) => { + getCodeTestResultsSpy.mockRejectedValue(codeClientError); + + await expect( + ecosystems.testEcosystem('code', ['.'], { + path: '', + code: true, + report: true, + }), + ).rejects.toHaveProperty( + 'userMessage', + expect.stringContaining(expectedErrorUserMessage), + ); + }, + ); + }); +}); diff --git a/test/jest/unit/snyk-code/snyk-code-test.spec.ts b/test/jest/unit/snyk-code/snyk-code-test.spec.ts index 84f4800beb..813c5c5e2b 100644 --- a/test/jest/unit/snyk-code/snyk-code-test.spec.ts +++ b/test/jest/unit/snyk-code/snyk-code-test.spec.ts @@ -1,7 +1,9 @@ +import * as os from 'os'; import * as fs from 'fs'; import * as path from 'path'; import stripAnsi from 'strip-ansi'; import { analyzeFolders, AnalysisSeverity } from '@snyk/code-client'; + jest.mock('@snyk/code-client'); const analyzeFoldersMock = analyzeFolders as jest.Mock; @@ -12,46 +14,33 @@ import * as analysis from '../../../../src/lib/plugins/sast/analysis'; import { Options, TestOptions } from '../../../../src/lib/types'; import * as ecosystems from '../../../../src/lib/ecosystems'; import * as analytics from '../../../../src/lib/analytics'; -import snykTest from '../../../../src/cli/commands/test/'; +import snykTest from '../../../../src/cli/commands/test'; import { jsonStringifyLargeObject } from '../../../../src/lib/json'; import { ArgsOptions } from '../../../../src/cli/args'; import * as codeConfig from '../../../../src/lib/code-config'; const { getCodeTestResults } = analysis; -import * as os from 'os'; describe('Test snyk code', () => { let apiUserConfig; let isSastEnabledForOrgSpy; let trackUsageSpy; + const failedCodeTestMessage = "Failed to run 'code test'"; const fakeApiKey = '123456789'; const baseURL = codeConfig.getCodeClientProxyUrl(); const LCEbaseURL = 'https://my-proxy-server'; + + const fixturePath = path.join(__dirname, '../../../fixtures/sast'); + const sampleSarifResponse = loadJson( - path.join(__dirname, '/../../../fixtures/sast/sample-sarif.json'), + path.join(fixturePath, 'sample-sarif.json'), ); const sampleAnalyzeFoldersResponse = loadJson( - path.join( - __dirname, - '/../../../fixtures/sast/sample-analyze-folders-response.json', - ), - ); - const sampleAnalyzeFoldersWithReportAndIgnoresResponse = loadJson( - path.join( - __dirname, - '/../../../fixtures/sast/sample-analyze-folders-with-report-and-ignores-response.json', - ), - ); - const sampleAnalyzeFoldersWithReportAndIgnoresOnlyResponse = loadJson( - path.join( - __dirname, - '/../../../fixtures/sast/sample-analyze-folders-with-report-and-ignores-only-response.json', - ), + path.join(fixturePath, 'sample-analyze-folders-response.json'), ); const isWindows = os.platform().indexOf('win') === 0; - const fixturePath = path.join(__dirname, '../../../fixtures', 'sast'); const cwd = process.cwd(); function readFixture(filename: string) { @@ -106,7 +95,11 @@ describe('Test snyk code', () => { const sastSettings = { sastEnabled: true, - localCodeEngine: { url: '', allowCloudUpload: true, enabled: false }, + localCodeEngine: { + url: '', + allowCloudUpload: true, + enabled: false, + }, }; const analyzeFoldersSpy = analyzeFoldersMock.mockResolvedValue( @@ -218,7 +211,10 @@ describe('Test snyk code', () => { }); it('should throw error when response code is not 200', async () => { - const error = { code: 401, message: 'Invalid auth token' }; + const error = { + code: 401, + message: 'Invalid auth token', + }; isSastEnabledForOrgSpy.mockRejectedValue(error); const expected = new Error(error.message); @@ -234,7 +230,10 @@ describe('Test snyk code', () => { }); it('should throw error correctly from outside of ecosystem flow when response code is not 200', async () => { - const error = { code: 401, message: 'Invalid auth token' }; + const error = { + code: 401, + message: 'Invalid auth token', + }; isSastEnabledForOrgSpy.mockRejectedValue(error); const expected = new Error(error.message); @@ -260,7 +259,11 @@ describe('Test snyk code', () => { }); await expect( - snykTest('some/path', { code: true, _: [], _doubleDashArgs: [] }), + snykTest('some/path', { + code: true, + _: [], + _doubleDashArgs: [], + }), ).rejects.toHaveProperty( 'userMessage', 'Snyk Code is not supported for org: enable in Settings > Snyk Code', @@ -274,7 +277,11 @@ describe('Test snyk code', () => { }); await expect( - snykTest('some/path', { code: true, _: [], _doubleDashArgs: [] }), + snykTest('some/path', { + code: true, + _: [], + _doubleDashArgs: [], + }), ).rejects.toHaveProperty('userMessage', 'error from api: org not found'); }); @@ -291,7 +298,11 @@ describe('Test snyk code', () => { }); await expect( - snykTest('some/path', { code: true, _: [], _doubleDashArgs: [] }), + snykTest('some/path', { + code: true, + _: [], + _doubleDashArgs: [], + }), ).rejects.toHaveProperty('userMessage', 'Test limit reached!'); }); @@ -299,17 +310,26 @@ describe('Test snyk code', () => { { name: 'should write only sarif result to file when only `--sarif-file-output` is used', - options: { 'sarif-file-output': true, 'json-file-output': false }, + options: { + 'sarif-file-output': true, + 'json-file-output': false, + }, }, { name: 'should write only json result to file when only `--json-file-output` is used', - options: { 'sarif-file-output': false, 'json-file-output': true }, + options: { + 'sarif-file-output': false, + 'json-file-output': true, + }, }, { name: 'should write sarif and json results to file when `--sarif-file-output` and `--json-file-output` are used', - options: { 'sarif-file-output': true, 'json-file-output': true }, + options: { + 'sarif-file-output': true, + 'json-file-output': true, + }, }, ])('$name', async (args) => { const options: ArgsOptions = { @@ -392,113 +412,6 @@ describe('Test snyk code', () => { } }); - it('should create sarif result including suppressions (ignored issues) - for report flow', async () => { - const sastSettings = { - sastEnabled: true, - localCodeEngine: { url: '', allowCloudUpload: true, enabled: false }, - }; - - // First get results without ignores - it should not ignore when report is disabled - analyzeFoldersMock.mockResolvedValue( - sampleAnalyzeFoldersWithReportAndIgnoresResponse, - ); - const resultWithoutIgnores = await getCodeTestResults( - '.', - { - path: '', - code: true, - report: false, - }, - sastSettings, - 'test-id', - ); - - const sarifWithoutIgnores = - resultWithoutIgnores?.analysisResults.sarif.runs[0].results; - if (!sarifWithoutIgnores) throw new Error('A value was expected'); - - // Then get the results with ignores - ignore when report is enabled - analyzeFoldersMock.mockResolvedValue( - sampleAnalyzeFoldersWithReportAndIgnoresResponse, - ); - const resultWithIgnores = await getCodeTestResults( - '.', - { - path: '', - code: true, - report: true, - }, - sastSettings, - 'test-id', - ); - - const sarifWithIgnores = - resultWithIgnores?.analysisResults.sarif.runs[0].results; - if (!sarifWithIgnores) throw new Error('A value was expected'); - - expect(sarifWithoutIgnores.length).toBeGreaterThan(0); - expect(sarifWithIgnores.length).toBeGreaterThan(0); - expect(sarifWithIgnores.length).toBe(sarifWithoutIgnores.length); - - let numSuppressions = 0; - sarifWithIgnores.forEach((result) => { - numSuppressions += result.suppressions?.length ?? 0; - }); - expect(numSuppressions).toBeGreaterThan(0); - }); - - it('should exit with correct code (1) when ignored issues are found - for report flow', async () => { - const options: ArgsOptions = { - path: '', - traverseNodeModules: false, - showVulnPaths: 'none', - code: true, - report: true, - projectName: 'test-project', - _: [], - _doubleDashArgs: [], - }; - - analyzeFoldersMock.mockResolvedValue( - sampleAnalyzeFoldersWithReportAndIgnoresResponse, - ); - isSastEnabledForOrgSpy.mockResolvedValueOnce({ - sastEnabled: true, - localCodeEngine: { - enabled: false, - }, - }); - trackUsageSpy.mockResolvedValue({}); - - await expect(snykTest('some/path', options)).rejects.toThrowError(); - }); - - it('should exit with correct code (0) when only ignored issues are found - for report flow', async () => { - const options: ArgsOptions = { - path: '', - traverseNodeModules: false, - showVulnPaths: 'none', - code: true, - report: true, - projectName: 'test-project', - _: [], - _doubleDashArgs: [], - }; - - analyzeFoldersMock.mockResolvedValue( - sampleAnalyzeFoldersWithReportAndIgnoresOnlyResponse, - ); - isSastEnabledForOrgSpy.mockResolvedValueOnce({ - sastEnabled: true, - localCodeEngine: { - enabled: false, - }, - }); - trackUsageSpy.mockResolvedValue({}); - - await expect(snykTest('some/path', options)).resolves.not.toThrowError(); - }); - describe('Default org test in CLI output', () => { beforeAll(() => { userConfig.set('org', 'defaultOrg'); @@ -605,6 +518,7 @@ describe('Test snyk code', () => { name: 'defaultOrg', publicId: 'unknown', }, + project: expect.any(Object), }, analysisOptions: expect.any(Object), connection: expect.any(Object), @@ -815,71 +729,6 @@ describe('Test snyk code', () => { ).rejects.toHaveProperty('message', "Failed to run 'code test'"); }); - it.each([ - [ - 'disabled FF', - { - apiName: 'initReport', - statusCode: 400, - statusText: 'Bad request', - }, - 'Make sure this feature is enabled by contacting support.', - ], - [ - 'SARIF too large', - { - apiName: 'getReport', - statusCode: 400, - statusText: 'Analysis result set too large', - }, - 'The findings for this project may exceed the allowed size limit.', - ], - [ - 'analysis failed', - { - apiName: 'getReport', - statusCode: 500, - statusText: 'Analysis failed', - }, - "One or more of Snyk's services may be temporarily unavailable.", - ], - [ - 'bad gateway', - { - apiName: 'initReport', - statusCode: 502, - statusText: 'Bad Gateway', - }, - "One or more of Snyk's services may be temporarily unavailable.", - ], - ])( - 'When code-client fails in the report flow, throw customized message for %s', - async (testName, codeClientError, expectedErrorUserMessage) => { - jest - .spyOn(analysis, 'getCodeTestResults') - .mockRejectedValue(codeClientError); - - isSastEnabledForOrgSpy.mockResolvedValueOnce({ - sastEnabled: true, - localCodeEngine: { - enabled: false, - }, - }); - trackUsageSpy.mockResolvedValue({}); - - await expect( - ecosystems.testEcosystem('code', ['.'], { - path: '', - code: true, - report: true, - }), - ).rejects.toHaveProperty( - 'userMessage', - expect.stringContaining(expectedErrorUserMessage), - ); - }, - ); - it('analyzeFolders should be called with the right arguments', async () => { const baseURL = expect.any(String); const sessionToken = `token ${fakeApiKey}`; @@ -901,15 +750,20 @@ describe('Test snyk code', () => { analysisContext: { flow: 'snyk-cli', initiator: 'CLI', - org: expect.anything(), + org: expect.any(Object), projectName: undefined, + project: expect.any(Object), }, languages: undefined, }; const sastSettings = { sastEnabled: true, - localCodeEngine: { url: '', allowCloudUpload: true, enabled: false }, + localCodeEngine: { + url: '', + allowCloudUpload: true, + enabled: false, + }, }; const analyzeFoldersSpy = analyzeFoldersMock.mockResolvedValue( @@ -931,7 +785,11 @@ describe('Test snyk code', () => { it('analyzeFolders should return the right sarif response', async () => { const sastSettings = { sastEnabled: true, - localCodeEngine: { url: '', allowCloudUpload: true, enabled: false }, + localCodeEngine: { + url: '', + allowCloudUpload: true, + enabled: false, + }, }; analyzeFoldersMock.mockResolvedValue(sampleAnalyzeFoldersResponse); @@ -948,55 +806,6 @@ describe('Test snyk code', () => { expect(actual?.analysisResults.sarif).toEqual(sampleSarifResponse); }); - it('analyzeFolders with report enabled should return the right report results response', async () => { - const sastSettings = { - sastEnabled: true, - localCodeEngine: { url: '', allowCloudUpload: true, enabled: false }, - }; - - const reportOptions = { - projectName: 'test-project-name', - targetName: 'test-target-name', - targetRef: 'test-target-ref', - remoteRepoUrl: 'https://github.com/owner/repo', - }; - - analyzeFoldersMock.mockResolvedValue( - sampleAnalyzeFoldersWithReportAndIgnoresResponse, - ); - const actual = await getCodeTestResults( - '.', - { - path: '', - code: true, - report: true, - 'project-name': reportOptions.projectName, - 'target-name': reportOptions.targetName, - 'target-reference': reportOptions.targetRef, - 'remote-repo-url': reportOptions.remoteRepoUrl, - }, - sastSettings, - 'test-id', - ); - - expect(analyzeFoldersMock).toHaveBeenCalledWith( - expect.objectContaining({ - reportOptions: { - enabled: true, - ...reportOptions, - }, - }), - ); - - const expectedReportResults = { - projectId: 'test-project-id', - snapshotId: 'test-snapshot-id', - reportUrl: 'test/report/url', - }; - - expect(actual?.reportResults).toEqual(expectedReportResults); - }); - it.each([ [ "use LCE's url as base when LCE is enabled", @@ -1044,8 +853,9 @@ describe('Test snyk code', () => { analysisContext: { flow: 'snyk-cli', initiator: 'CLI', - org: expect.anything(), + org: expect.any(Object), projectName: undefined, + project: expect.any(Object), }, languages: undefined, }; @@ -1098,7 +908,11 @@ describe('Test snyk code', () => { it('Local code engine - should throw error, when enabled and url is missing', async () => { const sastSettings = { sastEnabled: true, - localCodeEngine: { url: '', allowCloudUpload: true, enabled: true }, + localCodeEngine: { + url: '', + allowCloudUpload: true, + enabled: true, + }, }; await expect(