Fixed multithreaded gradle (#1337)

CycloneDX · Sep 9, 2024 · e7be44f · e7be44f
1 parent 806d212
commit e7be44f
Show file tree

Hide file tree

Showing 4 changed files with 141 additions and 145 deletions.
diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -63,6 +63,9 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
+      - name: pip install custom-json-diff
+        run: |
+          pip install custom-json-diff
       - uses: actions/checkout@v4
         with:
           repository: 'ShiftLeftSecurity/shiftleft-java-example'
@@ -243,6 +246,10 @@ jobs:
           repository: 'owasp-dep-scan/blint'
           path: 'repotests/blint'
           ref: 'v2.2.2'
+      - uses: actions/checkout@v4
+        with:
+          repository: 'malice00/cdxgen-expo-test'
+          path: 'repotests/expo-test'
       - uses: dtolnay/rust-toolchain@stable
       - name: setup sdkman
         run: |
@@ -440,6 +447,13 @@ jobs:
           bin/cdxgen.js -p -t python repotests/blint -o bomresults/bom-blint-deep.json --deep
           bin/cdxgen.js -p -t java repotests/broken-mvn-wrapper -o bomresults/bom-broken-mvn-wrapper.json
         shell: bash
+      - name: repotests expo
+        run: |
+          cd repotests/expo-test && npm ci && cd ../..
+          CDXGEN_DEBUG_MODE=debug GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" bin/cdxgen.js -p -t gradle repostests/expo-test -o bomresults/bom-expo.json
+          CDXGEN_DEBUG_MODE=debug GRADLE_MULTI_THREADED=true GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root,expo-modules-core\$android-annotation,expo-modules-core\$android-annotation-processor bin/cdxgen.js -p -t gradle repostests/expo-test -o bomresults/bom-expo-multi.json
+          custom-json-diff -i bomresults/bom-expo.json bomresults/bom-expo-multi.json -o bomresults/diff-expo
+        shell: bash
       - name: jenkins plugins
         run: |
           mkdir -p jenkins

diff --git a/docs/ENV.md b/docs/ENV.md
@@ -24,6 +24,7 @@ The following environment variables are available to configure the bom generatio
 | GRADLE_HOME                  | Specify gradle home                                                                                                                                                                                                                                                                                              |
 | GRADLE_CMD                   | Set to override gradle command                                                                                                                                                                                                                                                                                   |
 | GRADLE_DEPENDENCY_TASK       | By default cdxgen use the task "dependencies" to collect packages. Set to override the task name.                                                                                                                                                                                                                |
+| GRADLE_SKIP_MODULES          | Comma-separated list of modules to skip during the "dependencies" task. This can be useful if you have modules that would fail the gradle build, eg when they do not have dependencies in the given configuration. Use "root" if the top most module should be skipped, use the name (without leading ":") for all others. |
 | SBT_CACHE_DIR                | Specify sbt cache directory. Useful for class name resolving                                                                                                                                                                                                                                                     |
 | FETCH_LICENSE                | Set this variable to `true` or `1` to fetch license information from the registry. npm and golang                                                                                                                                                                                                                |
 | SEARCH_MAVEN_ORG             | If maven metadata is missing in jar file, a search is performed on search.maven.org. Set to `false` or `0` to disable search. (defaults to `true`)                                                                                                                                                               |

diff --git a/index.js b/index.js
@@ -42,6 +42,7 @@ import {
   addEvidenceForDotnet,
   addEvidenceForImports,
   addPlugin,
+  buildGradleCommandArguments,
   checksumFile,
   cleanupPlugin,
   collectGradleDependencies,
@@ -1621,7 +1622,7 @@ export async function createJavaBom(path, options) {
     gradleFiles?.length &&
     isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options)
   ) {
-    let retMap = executeGradleProperties(gradleRootPath, null, null);
+    let retMap = executeGradleProperties(gradleRootPath, null);
     const allProjectsStr = retMap.projects || [];
     const rootProject = retMap.rootProject;
     if (rootProject) {
@@ -1646,7 +1647,6 @@ export async function createJavaBom(path, options) {
       if (process.env.GRADLE_MULTI_THREADED) {
         const parallelPropTaskOut = executeParallelGradleProperties(
           gradleRootPath,
-          null,
           allProjectsStr,
         );
         const splitPropTaskOut = splitOutputByGradleProjects(
@@ -1702,7 +1702,7 @@ export async function createJavaBom(path, options) {
         }
       } else {
         for (const spstr of allProjectsStr) {
-          retMap = executeGradleProperties(gradleRootPath, null, spstr);
+          retMap = executeGradleProperties(gradleRootPath, spstr);
           const rootSubProject = retMap.rootProject;
           if (rootSubProject) {
             const rspName = rootSubProject.replace(/^:/, "");
@@ -1752,46 +1752,44 @@ export async function createJavaBom(path, options) {
     options.installDeps &&
     isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options)
   ) {
-    const gradleCmd = getGradleCommand(gradleRootPath, null);
-    const defaultDepTaskArgs = ["--console", "plain", "--build-cache"];
     allProjects.push(parentComponent);
-    let depTaskWithArgs = ["dependencies"];
-    let relevantTasks = ["dependencies"];
-    if (process.env.GRADLE_DEPENDENCY_TASK) {
-      depTaskWithArgs = process.env.GRADLE_DEPENDENCY_TASK.split(" ");
-      relevantTasks = process.env.GRADLE_DEPENDENCY_TASK.split(" ");
-    }
-    let gradleDepArgs = [];
-    gradleDepArgs = gradleDepArgs
-      .concat(depTaskWithArgs.slice(1))
-      .concat(defaultDepTaskArgs);
-    // Support custom GRADLE_ARGS such as --configuration runtimeClassPath (used for all tasks)
-    if (process.env.GRADLE_ARGS) {
-      const addArgs = process.env.GRADLE_ARGS.split(" ");
-      gradleDepArgs = gradleDepArgs.concat(addArgs);
-    }
-    // gradle args only for the dependencies task
-    if (process.env.GRADLE_ARGS_DEPENDENCIES) {
-      const addArgs = process.env.GRADLE_ARGS_DEPENDENCIES.split(" ");
-      gradleDepArgs = gradleDepArgs.concat(addArgs);
-    }
+    const gradleCmd = getGradleCommand(gradleRootPath, null);
+    const gradleDepTask = process.env.GRADLE_DEPENDENCY_TASK
+      ? process.env.GRADLE_DEPENDENCY_TASK
+      : "dependencies";
 
     if (process.env.GRADLE_MULTI_THREADED) {
-      gradleDepArgs.push(depTaskWithArgs[0]);
+      const gradleSubCommands = [];
+      const modulesToSkip = process.env.GRADLE_SKIP_MODULES
+        ? process.env.GRADLE_SKIP_MODULES.split(",")
+        : [];
+      if (!modulesToSkip.includes("root")) {
+        gradleSubCommands.push(gradleDepTask);
+      }
       for (const sp of allProjects) {
         //create single command for dependencies tasks on all subprojects
-        if (sp.purl !== parentComponent.purl) {
-          gradleDepArgs.push(`:${sp.name}:${depTaskWithArgs[0]}`);
+        if (
+          sp.purl !== parentComponent.purl &&
+          !modulesToSkip.includes(sp.name)
+        ) {
+          gradleSubCommands.push(`:${sp.name}:${gradleDepTask}`);
         }
       }
+      const gradleArguments = buildGradleCommandArguments(
+        process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
+        gradleSubCommands,
+        process.env.GRADLE_ARGS_DEPENDENCIES
+          ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ")
+          : [],
+      );
       console.log(
         "Executing",
         gradleCmd,
-        gradleDepArgs.join(" "),
+        gradleArguments.join(" "),
         "in",
         gradleRootPath,
       );
-      const sresult = spawnSync(gradleCmd, gradleDepArgs, {
+      const sresult = spawnSync(gradleCmd, gradleArguments, {
         cwd: gradleRootPath,
         encoding: "utf-8",
         timeout: TIMEOUT_MS,
@@ -1807,16 +1805,12 @@ export async function createJavaBom(path, options) {
       const sstdout = sresult.stdout;
       if (sstdout) {
         const cmdOutput = Buffer.from(sstdout).toString();
-        const perProjectOutput = splitOutputByGradleProjects(
-          cmdOutput,
-          relevantTasks,
-        );
-        for (const [key, singleProjectDepOut] of perProjectOutput.entries()) {
-          const sp = allProjects
-            .filter((project) => project.name === key)
-            .pop();
+        const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [
+          gradleDepTask,
+        ]);
+        for (const sp of allProjects) {
           const parsedList = parseGradleDep(
-            singleProjectDepOut,
+            perProjectOutput.has(sp.name) ? perProjectOutput.get(sp.name) : "",
             sp.group || parentComponent.group,
             sp.name,
             sp.version?.length && sp.version !== "latest"
@@ -1851,22 +1845,26 @@ export async function createJavaBom(path, options) {
         );
       }
       for (const sp of allProjects) {
-        let gradleSubProjectDepArgs = [
-          sp.purl === parentComponent.purl
-            ? depTaskWithArgs[0]
-            : `:${sp.name}:${depTaskWithArgs[0]}`,
-        ];
-        gradleSubProjectDepArgs = gradleSubProjectDepArgs.concat(gradleDepArgs);
+        const gradleArguments = buildGradleCommandArguments(
+          process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
+          [
+            sp.purl === parentComponent.purl
+              ? gradleDepTask
+              : `:${sp.name}:${gradleDepTask}`,
+          ],
+          process.env.GRADLE_ARGS_DEPENDENCIES
+            ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ")
+            : [],
+        );
 
-        gradleSubProjectDepArgs.push("-q");
         console.log(
           "Executing",
           gradleCmd,
-          gradleSubProjectDepArgs.join(" "),
+          gradleArguments.join(" "),
           "in",
           gradleRootPath,
         );
-        const sresult = spawnSync(gradleCmd, gradleSubProjectDepArgs, {
+        const sresult = spawnSync(gradleCmd, gradleArguments, {
           cwd: gradleRootPath,
           encoding: "utf-8",
           timeout: TIMEOUT_MS,