From 60fcf3fd35f0d8e0938edd23919e5c79c5086853 Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Wed, 28 Aug 2024 23:36:39 +0200 Subject: [PATCH 1/9] Fixed multithreaded gradle Signed-off-by: Roland Asmann --- docs/ENV.md | 1 + index.js | 66 ++++++++++++++++++++++++++++++++++------------------- utils.js | 29 +++++++++++------------ 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/docs/ENV.md b/docs/ENV.md index 94913f8cd..7b21d1b46 100644 --- a/docs/ENV.md +++ b/docs/ENV.md @@ -24,6 +24,7 @@ The following environment variables are available to configure the bom generatio | GRADLE_HOME | Specify gradle home | | GRADLE_CMD | Set to override gradle command | | GRADLE_DEPENDENCY_TASK | By default cdxgen use the task "dependencies" to collect packages. Set to override the task name. | +| GRADLE_SKIP_MODULES | Comma-separated list of modules to skip during the "dependencies" task. This can be useful if you have modules that would fail the gradle build, eg when they do not have dependencies in the given configuration. Use "root" if the top most module should be skipped, use the name (without leading ":") for all others. | | SBT_CACHE_DIR | Specify sbt cache directory. Useful for class name resolving | | FETCH_LICENSE | Set this variable to `true` or `1` to fetch license information from the registry. npm and golang | | SEARCH_MAVEN_ORG | If maven metadata is missing in jar file, a search is performed on search.maven.org. Set to `false` or `0` to disable search. (defaults to `true`) | diff --git a/index.js b/index.js index 24de7362f..ee22834fa 100644 --- a/index.js +++ b/index.js @@ -1753,7 +1753,7 @@ export async function createJavaBom(path, options) { isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options) ) { const gradleCmd = getGradleCommand(gradleRootPath, null); - const defaultDepTaskArgs = ["--console", "plain", "--build-cache"]; + const defaultTaskArgs = ["--console", "plain", "--build-cache"]; allProjects.push(parentComponent); let depTaskWithArgs = ["dependencies"]; let relevantTasks = ["dependencies"]; @@ -1761,42 +1761,58 @@ export async function createJavaBom(path, options) { depTaskWithArgs = process.env.GRADLE_DEPENDENCY_TASK.split(" "); relevantTasks = process.env.GRADLE_DEPENDENCY_TASK.split(" "); } - let gradleDepArgs = []; - gradleDepArgs = gradleDepArgs - .concat(depTaskWithArgs.slice(1)) - .concat(defaultDepTaskArgs); + let gradleArgs = []; + gradleArgs = gradleArgs.concat(defaultTaskArgs); // Support custom GRADLE_ARGS such as --configuration runtimeClassPath (used for all tasks) if (process.env.GRADLE_ARGS) { const addArgs = process.env.GRADLE_ARGS.split(" "); - gradleDepArgs = gradleDepArgs.concat(addArgs); + gradleArgs = gradleArgs.concat(addArgs); } // gradle args only for the dependencies task + let gradleDepArgs = []; + gradleDepArgs = gradleDepArgs.concat(depTaskWithArgs.slice(1)); if (process.env.GRADLE_ARGS_DEPENDENCIES) { const addArgs = process.env.GRADLE_ARGS_DEPENDENCIES.split(" "); gradleDepArgs = gradleDepArgs.concat(addArgs); } if (process.env.GRADLE_MULTI_THREADED) { - gradleDepArgs.push(depTaskWithArgs[0]); + const modulesToSkip = process.env.GRADLE_SKIP_MODULES + ? process.env.GRADLE_SKIP_MODULES.split(",") + : []; + let gradleSubProjectDepArgs = []; + if (!modulesToSkip.includes("root")) { + gradleSubProjectDepArgs.push(depTaskWithArgs[0]); + gradleSubProjectDepArgs = gradleSubProjectDepArgs.concat(gradleDepArgs); + } for (const sp of allProjects) { //create single command for dependencies tasks on all subprojects - if (sp.purl !== parentComponent.purl) { - gradleDepArgs.push(`:${sp.name}:${depTaskWithArgs[0]}`); + if ( + sp.purl !== parentComponent.purl && + !modulesToSkip.includes(sp.name) + ) { + gradleSubProjectDepArgs.push(`:${sp.name}:${depTaskWithArgs[0]}`); + gradleSubProjectDepArgs = + gradleSubProjectDepArgs.concat(gradleDepArgs); } } console.log( "Executing", gradleCmd, - gradleDepArgs.join(" "), + gradleArgs.concat(gradleSubProjectDepArgs).join(" "), "in", gradleRootPath, ); - const sresult = spawnSync(gradleCmd, gradleDepArgs, { - cwd: gradleRootPath, - encoding: "utf-8", - timeout: TIMEOUT_MS, - maxBuffer: MAX_BUFFER, - }); + const sresult = spawnSync( + gradleCmd, + gradleArgs.concat(gradleSubProjectDepArgs), + { + cwd: gradleRootPath, + encoding: "utf-8", + timeout: TIMEOUT_MS, + maxBuffer: MAX_BUFFER, + }, + ); if (sresult.status !== 0 || sresult.error) { if (options.failOnError || DEBUG_MODE) { @@ -1862,16 +1878,20 @@ export async function createJavaBom(path, options) { console.log( "Executing", gradleCmd, - gradleSubProjectDepArgs.join(" "), + gradleArgs.concat(gradleSubProjectDepArgs).join(" "), "in", gradleRootPath, ); - const sresult = spawnSync(gradleCmd, gradleSubProjectDepArgs, { - cwd: gradleRootPath, - encoding: "utf-8", - timeout: TIMEOUT_MS, - maxBuffer: MAX_BUFFER, - }); + const sresult = spawnSync( + gradleCmd, + gradleArgs.concat(gradleSubProjectDepArgs), + { + cwd: gradleRootPath, + encoding: "utf-8", + timeout: TIMEOUT_MS, + maxBuffer: MAX_BUFFER, + }, + ); if (sresult.status !== 0 || sresult.error) { if (options.failOnError || DEBUG_MODE) { console.error(sresult.stdout, sresult.stderr); diff --git a/utils.js b/utils.js index 9bc0f20d1..cba5f8550 100644 --- a/utils.js +++ b/utils.js @@ -2894,34 +2894,31 @@ export function executeParallelGradleProperties(dir, rootPath, allProjectsStr) { version: "latest", }, }; - let parallelPropTaskArgs = ["properties"]; - for (const spstr of allProjectsStr) { - parallelPropTaskArgs.push(`${spstr}:properties`); - } - - let gradlePropertiesArgs = ["--console", "plain", "--build-cache"]; const gradleCmd = getGradleCommand(dir, rootPath); // common gradle args, used for all tasks + let gradleArgs = ["--console", "plain", "--build-cache"]; if (process.env.GRADLE_ARGS) { const addArgs = process.env.GRADLE_ARGS.split(" "); - gradlePropertiesArgs = gradlePropertiesArgs.concat(addArgs); + gradleArgs = gradleArgs.concat(addArgs); } + // gradle args only for the properties task + let gradlePropertiesArgs = []; if (process.env.GRADLE_ARGS_PROPERTIES) { const addArgs = process.env.GRADLE_ARGS_PROPERTIES.split(" "); gradlePropertiesArgs = gradlePropertiesArgs.concat(addArgs); } - parallelPropTaskArgs = parallelPropTaskArgs.concat(gradlePropertiesArgs); - console.log( - "Executing", - gradleCmd, - parallelPropTaskArgs.join(" "), - "in", - dir, - ); - const result = spawnSync(gradleCmd, parallelPropTaskArgs, { + gradleArgs.push("properties"); + gradleArgs = gradleArgs.concat(gradlePropertiesArgs); + for (const spstr of allProjectsStr) { + gradleArgs.push(`${spstr}:properties`); + gradleArgs = gradleArgs.concat(gradlePropertiesArgs); + } + + console.log("Executing", gradleCmd, gradleArgs.join(" "), "in", dir); + const result = spawnSync(gradleCmd, gradleArgs, { cwd: dir, encoding: "utf-8", shell: isWin, From 2962cdf82d9dd239a6dd958d3f900d951119b7ae Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Sun, 1 Sep 2024 14:43:20 +0200 Subject: [PATCH 2/9] Renamed some variables for better readability of the code Signed-off-by: Roland Asmann --- utils.js | 62 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/utils.js b/utils.js index cba5f8550..a0b29bf91 100644 --- a/utils.js +++ b/utils.js @@ -2524,7 +2524,7 @@ export function parseGradleDep( const keys_cache = {}; const deps_keys_cache = {}; let last_level = 0; - let last_purl = decodeURIComponent( + let last_bomref = decodeURIComponent( new PackageURL( "maven", rootProject.group, @@ -2534,10 +2534,10 @@ export function parseGradleDep( null, ).toString(), ); - const first_purl = last_purl; - let last_project_purl = first_purl; + const first_bomref = last_bomref; + let last_project_bomref = first_bomref; const level_trees = {}; - level_trees[last_purl] = []; + level_trees[last_bomref] = []; let scope = undefined; let profileName = undefined; if (retMap?.projects) { @@ -2556,9 +2556,9 @@ export function parseGradleDep( ), ); } - level_trees[last_purl] = subDependsOn; + level_trees[last_bomref] = subDependsOn; } - let stack = [last_purl]; + let stack = [last_bomref]; const depRegex = /^.*?--- +(?[^\s:]+) ?:(?[^\s:]+)(?::(?:{strictly [[]?)?(?[^,\s:}]+))?(?:})?(?:[^->]* +-> +(?:(?[^\s:]+):(?[^\s:]+):)?(?[^\s:]+))?/gm; for (let rline of rawOutput.split("\n")) { @@ -2579,9 +2579,9 @@ export function parseGradleDep( rline.startsWith("\\--- ") ) { last_level = 1; - last_project_purl = first_purl; - last_purl = last_project_purl; - stack = [first_purl]; + last_project_bomref = first_bomref; + last_bomref = last_project_bomref; + stack = [first_bomref]; } if (rline.includes(" - ")) { profileName = rline.split(" - ")[0]; @@ -2626,11 +2626,11 @@ export function parseGradleDep( { type: "jar" }, null, ).toString(); - const purlString = decodeURIComponent(purl); - keys_cache[`${purlString}_${last_purl}`] = true; + const bomRef = decodeURIComponent(purl); + keys_cache[`${bomRef}_${last_bomref}`] = true; // Filter duplicates - if (!deps_keys_cache[purlString]) { - deps_keys_cache[purlString] = true; + if (!deps_keys_cache[bomRef]) { + deps_keys_cache[bomRef] = true; const adep = { group: group !== "project" ? group : rootProjectGroup, name: name, @@ -2638,7 +2638,7 @@ export function parseGradleDep( qualifiers: { type: "jar" }, }; adep["purl"] = purl; - adep["bom-ref"] = purlString; + adep["bom-ref"] = bomRef; if (scope) { adep["scope"] = scope; } @@ -2652,38 +2652,38 @@ export function parseGradleDep( } deps.push(adep); } - if (!level_trees[purlString]) { - level_trees[purlString] = []; + if (!level_trees[bomRef]) { + level_trees[bomRef] = []; } if (level === 0) { - stack = [first_purl]; - stack.push(purlString); - } else if (last_purl === "") { - stack.push(purlString); + stack = [first_bomref]; + stack.push(bomRef); + } else if (last_bomref === "") { + stack.push(bomRef); } else if (level > last_level) { - const cnodes = level_trees[last_purl] || []; - if (!cnodes.includes(purlString)) { - cnodes.push(purlString); + const cnodes = level_trees[last_bomref] || []; + if (!cnodes.includes(bomRef)) { + cnodes.push(bomRef); } - level_trees[last_purl] = cnodes; - if (stack[stack.length - 1] !== purlString) { - stack.push(purlString); + level_trees[last_bomref] = cnodes; + if (stack[stack.length - 1] !== bomRef) { + stack.push(bomRef); } } else { for (let i = level; i <= last_level; i++) { stack.pop(); } const last_stack = - stack.length > 0 ? stack[stack.length - 1] : last_project_purl; + stack.length > 0 ? stack[stack.length - 1] : last_project_bomref; const cnodes = level_trees[last_stack] || []; - if (!cnodes.includes(purlString)) { - cnodes.push(purlString); + if (!cnodes.includes(bomRef)) { + cnodes.push(bomRef); } level_trees[last_stack] = cnodes; - stack.push(purlString); + stack.push(bomRef); } last_level = level; - last_purl = purlString; + last_bomref = bomRef; } } } From 7372787476f17d94afdaeafbbd7d08875ac93184 Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Sun, 1 Sep 2024 17:24:56 +0200 Subject: [PATCH 3/9] This method is returning a String in non-error case, so error should not return an object! Moved returning the empty String to after logging of information on how to debug errors. Signed-off-by: Roland Asmann --- utils.js | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/utils.js b/utils.js index a0b29bf91..1568ba2a6 100644 --- a/utils.js +++ b/utils.js @@ -2887,13 +2887,6 @@ export function parseGradleProperties(rawOutput) { * @returns {string} The combined output for all subprojects of the Gradle properties task */ export function executeParallelGradleProperties(dir, rootPath, allProjectsStr) { - const defaultProps = { - rootProject: "root", - projects: [], - metadata: { - version: "latest", - }, - }; const gradleCmd = getGradleCommand(dir, rootPath); // common gradle args, used for all tasks @@ -2925,9 +2918,6 @@ export function executeParallelGradleProperties(dir, rootPath, allProjectsStr) { }); if (result.status !== 0 || result.error) { if (result.stderr) { - if (result.stderr.includes("does not exist")) { - return defaultProps; - } console.error(result.stdout, result.stderr); console.log( "1. Check if the correct version of java and gradle are installed and available in PATH. For example, some project might require Java 11 with gradle 7.\n cdxgen container image bundles Java 21 with gradle 8 which might be incompatible.", @@ -2940,6 +2930,9 @@ export function executeParallelGradleProperties(dir, rootPath, allProjectsStr) { "3. Check if the SBOM is generated for the correct root project for your application.", ); } + if (result.stderr.includes("does not exist")) { + return ""; + } } } const stdout = result.stdout; From 336fb6a2fceed7e3917b62ec78f40a8c5932d016 Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Sun, 1 Sep 2024 17:27:42 +0200 Subject: [PATCH 4/9] Removed a parameter that was ALWAYS null Signed-off-by: Roland Asmann --- index.js | 5 ++--- utils.js | 10 ++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index ee22834fa..3cf6ec7f6 100644 --- a/index.js +++ b/index.js @@ -1621,7 +1621,7 @@ export async function createJavaBom(path, options) { gradleFiles?.length && isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options) ) { - let retMap = executeGradleProperties(gradleRootPath, null, null); + let retMap = executeGradleProperties(gradleRootPath, null); const allProjectsStr = retMap.projects || []; const rootProject = retMap.rootProject; if (rootProject) { @@ -1646,7 +1646,6 @@ export async function createJavaBom(path, options) { if (process.env.GRADLE_MULTI_THREADED) { const parallelPropTaskOut = executeParallelGradleProperties( gradleRootPath, - null, allProjectsStr, ); const splitPropTaskOut = splitOutputByGradleProjects( @@ -1702,7 +1701,7 @@ export async function createJavaBom(path, options) { } } else { for (const spstr of allProjectsStr) { - retMap = executeGradleProperties(gradleRootPath, null, spstr); + retMap = executeGradleProperties(gradleRootPath, spstr); const rootSubProject = retMap.rootProject; if (rootSubProject) { const rspName = rootSubProject.replace(/^:/, ""); diff --git a/utils.js b/utils.js index 1568ba2a6..42bebc706 100644 --- a/utils.js +++ b/utils.js @@ -2881,13 +2881,12 @@ export function parseGradleProperties(rawOutput) { * Execute gradle properties command using multi-threading and return parsed output * * @param {string} dir Directory to execute the command - * @param {string} rootPath Root directory * @param {array} allProjectsStr List of all sub-projects (including the preceding `:`) * * @returns {string} The combined output for all subprojects of the Gradle properties task */ -export function executeParallelGradleProperties(dir, rootPath, allProjectsStr) { - const gradleCmd = getGradleCommand(dir, rootPath); +export function executeParallelGradleProperties(dir, allProjectsStr) { + const gradleCmd = getGradleCommand(dir, null); // common gradle args, used for all tasks let gradleArgs = ["--console", "plain", "--build-cache"]; @@ -2946,10 +2945,9 @@ export function executeParallelGradleProperties(dir, rootPath, allProjectsStr) { * Execute gradle properties command and return parsed output * * @param {string} dir Directory to execute the command - * @param {string} rootPath Root directory * @param {string} subProject Sub project name */ -export function executeGradleProperties(dir, rootPath, subProject) { +export function executeGradleProperties(dir, subProject) { const defaultProps = { rootProject: subProject, projects: [], @@ -2969,7 +2967,7 @@ export function executeGradleProperties(dir, rootPath, subProject) { "plain", "--build-cache", ]; - const gradleCmd = getGradleCommand(dir, rootPath); + const gradleCmd = getGradleCommand(dir, null); // common gradle args, used for all tasks if (process.env.GRADLE_ARGS) { const addArgs = process.env.GRADLE_ARGS.split(" "); From f9534ad0d03c3060139c433e8ed4896229bc0824 Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Sun, 1 Sep 2024 19:22:45 +0200 Subject: [PATCH 5/9] No need to get properties for the root again Signed-off-by: Roland Asmann --- utils.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils.js b/utils.js index 42bebc706..806bcdf34 100644 --- a/utils.js +++ b/utils.js @@ -2902,8 +2902,6 @@ export function executeParallelGradleProperties(dir, allProjectsStr) { gradlePropertiesArgs = gradlePropertiesArgs.concat(addArgs); } - gradleArgs.push("properties"); - gradleArgs = gradleArgs.concat(gradlePropertiesArgs); for (const spstr of allProjectsStr) { gradleArgs.push(`${spstr}:properties`); gradleArgs = gradleArgs.concat(gradlePropertiesArgs); From 76e6de54692ec0f6eb3c12b4682545acc1260d8a Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Sun, 1 Sep 2024 23:50:22 +0200 Subject: [PATCH 6/9] Also parse ignored modules, but use the empty String as input Signed-off-by: Roland Asmann --- index.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 3cf6ec7f6..c84c462ef 100644 --- a/index.js +++ b/index.js @@ -1826,12 +1826,9 @@ export async function createJavaBom(path, options) { cmdOutput, relevantTasks, ); - for (const [key, singleProjectDepOut] of perProjectOutput.entries()) { - const sp = allProjects - .filter((project) => project.name === key) - .pop(); + for (const sp of allProjects) { const parsedList = parseGradleDep( - singleProjectDepOut, + perProjectOutput.has(sp.name) ? perProjectOutput.get(sp.name) : "", sp.group || parentComponent.group, sp.name, sp.version?.length && sp.version !== "latest" From 9e0c1267022fba0b3938fdbc8f41dcc92918d10b Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Thu, 5 Sep 2024 04:47:38 +0200 Subject: [PATCH 7/9] Added a helper-method to build the list of arguments for all gradle calls Signed-off-by: Roland Asmann --- index.js | 104 +++++++++++++++++++++++-------------------------------- utils.js | 83 +++++++++++++++++++++----------------------- 2 files changed, 83 insertions(+), 104 deletions(-) diff --git a/index.js b/index.js index c84c462ef..35ae9415f 100644 --- a/index.js +++ b/index.js @@ -42,6 +42,7 @@ import { addEvidenceForDotnet, addEvidenceForImports, addPlugin, + buildGradleCommandArguments, checksumFile, cleanupPlugin, collectGradleDependencies, @@ -1751,38 +1752,19 @@ export async function createJavaBom(path, options) { options.installDeps && isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options) ) { - const gradleCmd = getGradleCommand(gradleRootPath, null); - const defaultTaskArgs = ["--console", "plain", "--build-cache"]; allProjects.push(parentComponent); - let depTaskWithArgs = ["dependencies"]; - let relevantTasks = ["dependencies"]; - if (process.env.GRADLE_DEPENDENCY_TASK) { - depTaskWithArgs = process.env.GRADLE_DEPENDENCY_TASK.split(" "); - relevantTasks = process.env.GRADLE_DEPENDENCY_TASK.split(" "); - } - let gradleArgs = []; - gradleArgs = gradleArgs.concat(defaultTaskArgs); - // Support custom GRADLE_ARGS such as --configuration runtimeClassPath (used for all tasks) - if (process.env.GRADLE_ARGS) { - const addArgs = process.env.GRADLE_ARGS.split(" "); - gradleArgs = gradleArgs.concat(addArgs); - } - // gradle args only for the dependencies task - let gradleDepArgs = []; - gradleDepArgs = gradleDepArgs.concat(depTaskWithArgs.slice(1)); - if (process.env.GRADLE_ARGS_DEPENDENCIES) { - const addArgs = process.env.GRADLE_ARGS_DEPENDENCIES.split(" "); - gradleDepArgs = gradleDepArgs.concat(addArgs); - } + const gradleCmd = getGradleCommand(gradleRootPath, null); + const gradleDepTask = process.env.GRADLE_DEPENDENCY_TASK + ? process.env.GRADLE_DEPENDENCY_TASK + : "dependencies"; if (process.env.GRADLE_MULTI_THREADED) { + const gradleSubCommands = []; const modulesToSkip = process.env.GRADLE_SKIP_MODULES ? process.env.GRADLE_SKIP_MODULES.split(",") : []; - let gradleSubProjectDepArgs = []; if (!modulesToSkip.includes("root")) { - gradleSubProjectDepArgs.push(depTaskWithArgs[0]); - gradleSubProjectDepArgs = gradleSubProjectDepArgs.concat(gradleDepArgs); + gradleSubCommands.push(gradleDepTask); } for (const sp of allProjects) { //create single command for dependencies tasks on all subprojects @@ -1790,28 +1772,29 @@ export async function createJavaBom(path, options) { sp.purl !== parentComponent.purl && !modulesToSkip.includes(sp.name) ) { - gradleSubProjectDepArgs.push(`:${sp.name}:${depTaskWithArgs[0]}`); - gradleSubProjectDepArgs = - gradleSubProjectDepArgs.concat(gradleDepArgs); + gradleSubCommands.push(`:${sp.name}:${gradleDepTask}`); } } + const gradleArguments = buildGradleCommandArguments( + process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [], + gradleSubCommands, + process.env.GRADLE_ARGS_DEPENDENCIES + ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ") + : [], + ); console.log( "Executing", gradleCmd, - gradleArgs.concat(gradleSubProjectDepArgs).join(" "), + gradleArguments.join(" "), "in", gradleRootPath, ); - const sresult = spawnSync( - gradleCmd, - gradleArgs.concat(gradleSubProjectDepArgs), - { - cwd: gradleRootPath, - encoding: "utf-8", - timeout: TIMEOUT_MS, - maxBuffer: MAX_BUFFER, - }, - ); + const sresult = spawnSync(gradleCmd, gradleArguments, { + cwd: gradleRootPath, + encoding: "utf-8", + timeout: TIMEOUT_MS, + maxBuffer: MAX_BUFFER, + }); if (sresult.status !== 0 || sresult.error) { if (options.failOnError || DEBUG_MODE) { @@ -1822,10 +1805,9 @@ export async function createJavaBom(path, options) { const sstdout = sresult.stdout; if (sstdout) { const cmdOutput = Buffer.from(sstdout).toString(); - const perProjectOutput = splitOutputByGradleProjects( - cmdOutput, - relevantTasks, - ); + const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [ + gradleDepTask, + ]); for (const sp of allProjects) { const parsedList = parseGradleDep( perProjectOutput.has(sp.name) ? perProjectOutput.get(sp.name) : "", @@ -1863,31 +1845,31 @@ export async function createJavaBom(path, options) { ); } for (const sp of allProjects) { - let gradleSubProjectDepArgs = [ - sp.purl === parentComponent.purl - ? depTaskWithArgs[0] - : `:${sp.name}:${depTaskWithArgs[0]}`, - ]; - gradleSubProjectDepArgs = gradleSubProjectDepArgs.concat(gradleDepArgs); + const gradleArguments = buildGradleCommandArguments( + process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [], + [ + sp.purl === parentComponent.purl + ? gradleDepTask + : `:${sp.name}:${gradleDepTask}`, + ], + process.env.GRADLE_ARGS_DEPENDENCIES + ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ") + : [], + ); - gradleSubProjectDepArgs.push("-q"); console.log( "Executing", gradleCmd, - gradleArgs.concat(gradleSubProjectDepArgs).join(" "), + gradleArguments.join(" "), "in", gradleRootPath, ); - const sresult = spawnSync( - gradleCmd, - gradleArgs.concat(gradleSubProjectDepArgs), - { - cwd: gradleRootPath, - encoding: "utf-8", - timeout: TIMEOUT_MS, - maxBuffer: MAX_BUFFER, - }, - ); + const sresult = spawnSync(gradleCmd, gradleArguments, { + cwd: gradleRootPath, + encoding: "utf-8", + timeout: TIMEOUT_MS, + maxBuffer: MAX_BUFFER, + }); if (sresult.status !== 0 || sresult.error) { if (options.failOnError || DEBUG_MODE) { console.error(sresult.stdout, sresult.stderr); diff --git a/utils.js b/utils.js index 806bcdf34..9390166e4 100644 --- a/utils.js +++ b/utils.js @@ -2887,25 +2887,13 @@ export function parseGradleProperties(rawOutput) { */ export function executeParallelGradleProperties(dir, allProjectsStr) { const gradleCmd = getGradleCommand(dir, null); - - // common gradle args, used for all tasks - let gradleArgs = ["--console", "plain", "--build-cache"]; - if (process.env.GRADLE_ARGS) { - const addArgs = process.env.GRADLE_ARGS.split(" "); - gradleArgs = gradleArgs.concat(addArgs); - } - - // gradle args only for the properties task - let gradlePropertiesArgs = []; - if (process.env.GRADLE_ARGS_PROPERTIES) { - const addArgs = process.env.GRADLE_ARGS_PROPERTIES.split(" "); - gradlePropertiesArgs = gradlePropertiesArgs.concat(addArgs); - } - - for (const spstr of allProjectsStr) { - gradleArgs.push(`${spstr}:properties`); - gradleArgs = gradleArgs.concat(gradlePropertiesArgs); - } + const gradleArgs = buildGradleCommandArguments( + process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [], + allProjectsStr.map((project) => `${project}:properties`), + process.env.GRADLE_ARGS_PROPERTIES + ? process.env.GRADLE_ARGS_PROPERTIES.split(" ") + : [], + ); console.log("Executing", gradleCmd, gradleArgs.join(" "), "in", dir); const result = spawnSync(gradleCmd, gradleArgs, { @@ -2958,32 +2946,16 @@ export function executeGradleProperties(dir, subProject) { if (subProject && subProject.match(/:/g).length >= 2) { return defaultProps; } - let gradlePropertiesArgs = [ - subProject ? `${subProject}:properties` : "properties", - "-q", - "--console", - "plain", - "--build-cache", - ]; const gradleCmd = getGradleCommand(dir, null); - // common gradle args, used for all tasks - if (process.env.GRADLE_ARGS) { - const addArgs = process.env.GRADLE_ARGS.split(" "); - gradlePropertiesArgs = gradlePropertiesArgs.concat(addArgs); - } - // gradle args only for the properties task - if (process.env.GRADLE_ARGS_PROPERTIES) { - const addArgs = process.env.GRADLE_ARGS_PROPERTIES.split(" "); - gradlePropertiesArgs = gradlePropertiesArgs.concat(addArgs); - } - console.log( - "Executing", - gradleCmd, - gradlePropertiesArgs.join(" "), - "in", - dir, + const gradleArguments = buildGradleCommandArguments( + process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [], + [subProject ? `${subProject}:properties` : "properties"], + process.env.GRADLE_ARGS_PROPERTIES + ? process.env.GRADLE_ARGS_PROPERTIES.split(" ") + : [], ); - const result = spawnSync(gradleCmd, gradlePropertiesArgs, { + console.log("Executing", gradleCmd, gradleArguments.join(" "), "in", dir); + const result = spawnSync(gradleCmd, gradleArguments, { cwd: dir, encoding: "utf-8", shell: isWin, @@ -9933,6 +9905,31 @@ export function getGradleCommand(srcPath, rootPath) { } return gradleCmd; } + +/** + * Method to combine the general gradle arguments, the sub-commands and the sub-commands' arguments in the correct way + * + * @param {string[]} gradleArguments The general gradle arguments, which must only be added once + * @param {string[]} gradleSubCommands The sub-commands that are to be executed by gradle + * @param {string[]} gradleSubCommandArguments The arguments specific to the sub-command(s), which much be added PER sub-command + * + * @returns {string[]} Array of arguments to be added to the gradle command + */ +export function buildGradleCommandArguments( + gradleArguments, + gradleSubCommands, + gradleSubCommandArguments, +) { + let allGradleArguments = ["--console", "plain", "--build-cache"].concat( + gradleArguments, + ); + for (const gradleSubCommand of gradleSubCommands) { + allGradleArguments.push(gradleSubCommand); + allGradleArguments = allGradleArguments.concat(gradleSubCommandArguments); + } + return allGradleArguments; +} + /** * Method to split the output produced by Gradle using parallel processing by project * From 59112cff6aca7b5627dde0792498a17462925d2f Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Thu, 5 Sep 2024 04:56:55 +0200 Subject: [PATCH 8/9] Added repotest for multithreaded gradle Signed-off-by: Roland Asmann --- .github/workflows/repotests.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml index a60114902..f031f6457 100644 --- a/.github/workflows/repotests.yml +++ b/.github/workflows/repotests.yml @@ -63,6 +63,9 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12' + - name: pip install custom-json-diff + run: | + pip install custom-json-diff - uses: actions/checkout@v4 with: repository: 'ShiftLeftSecurity/shiftleft-java-example' @@ -243,6 +246,10 @@ jobs: repository: 'owasp-dep-scan/blint' path: 'repotests/blint' ref: 'v2.2.2' + - uses: actions/checkout@v4 + with: + repository: 'malice00/cdxgen-expo-test' + path: 'repotests/expo-test' - uses: dtolnay/rust-toolchain@stable - name: setup sdkman run: | @@ -440,6 +447,13 @@ jobs: bin/cdxgen.js -p -t python repotests/blint -o bomresults/bom-blint-deep.json --deep bin/cdxgen.js -p -t java repotests/broken-mvn-wrapper -o bomresults/bom-broken-mvn-wrapper.json shell: bash + - name: repotests expo + run: | + cd repotests/expo-test && npm ci && cd ../.. + GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" bin/cdxgen.js -p -t gradle repostests/expo-test -o bomresults/bom-expo.json + GRADLE_MULTI_THREADED=true GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root,expo-modules-core\$android-annotation,expo-modules-core\$android-annotation-processor bin/cdxgen.js -p -t gradle repostests/expo-test -o bomresults/bom-expo-multi.json + custom-json-diff -i bomresults/bom-expo.json bomresults/bom-expo-multi.json -o bomresults/diff-expo + shell: bash - name: jenkins plugins run: | mkdir -p jenkins From a0c818e4a880789ee42769adc7fe5819bfc01064 Mon Sep 17 00:00:00 2001 From: Roland Asmann Date: Thu, 5 Sep 2024 06:06:10 +0200 Subject: [PATCH 9/9] Added DEBUG to figure out what is going on Signed-off-by: Roland Asmann --- .github/workflows/repotests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml index f031f6457..5e54f9d61 100644 --- a/.github/workflows/repotests.yml +++ b/.github/workflows/repotests.yml @@ -450,8 +450,8 @@ jobs: - name: repotests expo run: | cd repotests/expo-test && npm ci && cd ../.. - GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" bin/cdxgen.js -p -t gradle repostests/expo-test -o bomresults/bom-expo.json - GRADLE_MULTI_THREADED=true GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root,expo-modules-core\$android-annotation,expo-modules-core\$android-annotation-processor bin/cdxgen.js -p -t gradle repostests/expo-test -o bomresults/bom-expo-multi.json + CDXGEN_DEBUG_MODE=debug GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" bin/cdxgen.js -p -t gradle repostests/expo-test -o bomresults/bom-expo.json + CDXGEN_DEBUG_MODE=debug GRADLE_MULTI_THREADED=true GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root,expo-modules-core\$android-annotation,expo-modules-core\$android-annotation-processor bin/cdxgen.js -p -t gradle repostests/expo-test -o bomresults/bom-expo-multi.json custom-json-diff -i bomresults/bom-expo.json bomresults/bom-expo-multi.json -o bomresults/diff-expo shell: bash - name: jenkins plugins