Merge pull request #17955 from tamasvajk/binlog/multiple

C#: Change extractor to accept multiple `binlog` files

csharp/codeql-extractor.yml

@@ -70,4 +70,4 @@ options:
     description: >
       [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
       This option only works when `--build-mode none` is also specified.
-    type: string
+    type: array

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -106,10 +106,10 @@ public static ExitCode Run(string[] args)
                 var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
                 var pathTransformer = new PathTransformer(canonicalPathCache);
 
-                if (options.BinaryLogPath is string binlogPath)
+                if (options.BinaryLogPaths is string[] binlogPaths)
                 {
                     logger.LogInfo(" Running binary log analysis.");
-                    return RunBinaryLogAnalysis(analyzerStopwatch, options, binlogPath, logger, canonicalPathCache, pathTransformer);
+                    return RunBinaryLogAnalysis(analyzerStopwatch, options, binlogPaths, logger, canonicalPathCache, pathTransformer);
                 }
                 else
                 {
@@ -124,6 +124,25 @@ public static ExitCode Run(string[] args)
             }
         }
 
+        private static ExitCode RunBinaryLogAnalysis(Stopwatch stopwatch, Options options, string[] binlogPaths, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
+        {
+            var allFailed = true;
+            foreach (var binlogPath in binlogPaths)
+            {
+                var exit = RunBinaryLogAnalysis(stopwatch, options, binlogPath, logger, canonicalPathCache, pathTransformer);
+                switch (exit)
+                {
+                    case ExitCode.Ok:
+                    case ExitCode.Errors:
+                        allFailed = false;
+                        break;
+                    case ExitCode.Failed:
+                        break;
+                }
+            }
+            return allFailed ? ExitCode.Failed : ExitCode.Ok;
+        }
+
         private static ExitCode RunBinaryLogAnalysis(Stopwatch stopwatch, Options options, string binlogPath, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
         {
             logger.LogInfo($"Reading compiler calls from binary log {binlogPath}");
@@ -190,11 +209,11 @@ static bool filter(CompilerCall compilerCall)
                     switch (exit)
                     {
                         case ExitCode.Ok:
-                            allFailed &= false;
+                            allFailed = false;
                             logger.LogInfo($"  Compilation {diagnosticName} succeeded");
                             break;
                         case ExitCode.Errors:
-                            allFailed &= false;
+                            allFailed = false;
                             logger.LogWarning($"  Compilation {diagnosticName} had errors");
                             break;
                         case ExitCode.Failed:

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

@@ -33,9 +33,9 @@ public sealed class Options : CommonOptions
         public bool AssemblySensitiveTrap { get; private set; } = false;
 
         /// <summary>
-        /// The path to the binary log file, or null if unspecified.
+        /// The paths to the binary log files, or null if unspecified.
         /// </summary>
-        public string? BinaryLogPath { get; set; }
+        public string[]? BinaryLogPaths { get; set; }
 
         public static Options CreateWithEnvironment(string[] arguments)
         {
@@ -71,7 +71,7 @@ public override bool HandleOption(string key, string value)
                     ProjectsToLoad.Add(value);
                     return true;
                 case "binlog":
-                    BinaryLogPath = value;
+                    BinaryLogPaths = value.Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries);
                     return true;
                 default:
                     return base.HandleOption(key, value);

csharp/ql/integration-tests/all-platforms/binlog_multiple/Files.expected

@@ -0,0 +1,10 @@
+| a/A.cs:0:0:0:0 | a/A.cs |
+| a/obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs:0:0:0:0 | a/obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs |
+| a/obj/Debug/net8.0/test.AssemblyInfo.cs:0:0:0:0 | a/obj/Debug/net8.0/test.AssemblyInfo.cs |
+| a/obj/Debug/net8.0/test.GlobalUsings.g.cs:0:0:0:0 | a/obj/Debug/net8.0/test.GlobalUsings.g.cs |
+| b/B.cs:0:0:0:0 | b/B.cs |
+| b/obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs:0:0:0:0 | b/obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs |
+| b/obj/Debug/net8.0/test.AssemblyInfo.cs:0:0:0:0 | b/obj/Debug/net8.0/test.AssemblyInfo.cs |
+| b/obj/Debug/net8.0/test.GlobalUsings.g.cs:0:0:0:0 | b/obj/Debug/net8.0/test.GlobalUsings.g.cs |
+| generated/a/test.csproj (net8.0)/System.Text.RegularExpressions.Generator/System.Text.RegularExpressions.Generator.RegexGenerator/RegexGenerator.g.cs:0:0:0:0 | generated/a/test.csproj (net8.0)/System.Text.RegularExpressions.Generator/System.Text.RegularExpressions.Generator.RegexGenerator/RegexGenerator.g.cs |
+| generated/b/test.csproj (net8.0)/System.Text.RegularExpressions.Generator/System.Text.RegularExpressions.Generator.RegexGenerator/RegexGenerator.g.cs:0:0:0:0 | generated/b/test.csproj (net8.0)/System.Text.RegularExpressions.Generator/System.Text.RegularExpressions.Generator.RegexGenerator/RegexGenerator.g.cs |

csharp/ql/integration-tests/all-platforms/binlog_multiple/Files.ql

@@ -0,0 +1,5 @@
+import csharp
+
+from File f
+where f.fromSource()
+select f

csharp/ql/integration-tests/all-platforms/binlog_multiple/a/A.cs

@@ -0,0 +1,9 @@
+using System.Text.RegularExpressions;
+
+var dummy = "dummy";
+
+partial class Test
+{
+    [GeneratedRegex("abc|def", RegexOptions.IgnoreCase, "en-US")]
+    private static partial Regex AbcOrDefGeneratedRegex();
+}

csharp/ql/integration-tests/all-platforms/binlog_multiple/a/test.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

csharp/ql/integration-tests/all-platforms/binlog_multiple/b/B.cs

@@ -0,0 +1,9 @@
+using System.Text.RegularExpressions;
+
+var dummy = "dummy";
+
+partial class Test
+{
+    [GeneratedRegex("abc|def", RegexOptions.IgnoreCase, "en-US")]
+    private static partial Regex AbcOrDefGeneratedRegex();
+}

csharp/ql/integration-tests/all-platforms/binlog_multiple/b/test.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

csharp/ql/integration-tests/all-platforms/binlog_multiple/diagnostics.expected

@@ -0,0 +1,42 @@
+{
+  "markdownMessage": "C# analysis with build-mode 'none' completed.",
+  "severity": "unknown",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/complete",
+    "name": "C# analysis with build-mode 'none' completed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# was extracted with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/mode-active",
+    "name": "C# was extracted with build-mode set to 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# was extracted with the experimental 'binlog' option.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/binlog",
+    "name": "C# was extracted with the experimental 'binlog' option"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/all-platforms/binlog_multiple/global.json

@@ -0,0 +1,5 @@
+{
+    "sdk": {
+        "version": "8.0.101"
+    }
+}

csharp/ql/integration-tests/all-platforms/binlog_multiple/test.py

@@ -0,0 +1,7 @@
+import commands
+
+
+def test(codeql, csharp):
+    commands.run(["dotnet", "build", "a/test.csproj", "/bl:a.binlog"])
+    commands.run(["dotnet", "build", "b/test.csproj", "/bl:b.binlog"])
+    codeql.database.create(build_mode="none", extractor_option=["binlog=a.binlog", "binlog=b.binlog"])