Merge pull request #64277 from mavasani/SarifEnhancements

Enhance rule metadata and suppression info in SARIF V2 errorlog
dotnet · Nov 18, 2022 · 33c8ae8 · 33c8ae8
2 parents dc04e49 + 202720e
commit 33c8ae8
Show file tree

Hide file tree

Showing 18 changed files with 632 additions and 71 deletions.
diff --git a/docs/compilers/Error Log Format.md b/docs/compilers/Error Log Format.md
@@ -3,4 +3,139 @@ the command line to log all diagnostics in a structured, JSON format.
 
 The log format is SARIF (Static Analysis Results Interchange Format):
 See https://sarifweb.azurewebsites.net/ for the format specification,
-JSON schema, and other related resources.
+JSON schema, and other related resources.
+
+## SARIF v2 format
+
+This section provides a high level overview of the contents of the SARIF v2 error log file generated by the C# and Visual Basic compilers. A more formal and detailed schema specification for the SARIF v2 format is specified at http://json.schemastore.org/sarif-2.1.0.
+
+1. `schema` and `version` information: The first two lines of the error log file specifies the SARIF schema and version information. For example:
+
+```json
+"$schema": "http://json.schemastore.org/sarif-2.1.0",
+"version": "2.1.0",
+```
+
+2. `runs` information: The core entry in the error log file is the `runs` section with a single run entry within it for the build. The run entry has 3 main parts:
+  1. `results` section: This section contains an array of result entries, where each result corresponds to information about a reported compiler or analyzer `Diagnostic`. More details in [`Result` format for each compiler or analyzer `Diagnostic` instance](#result-format-for-each-compiler-or-analyzer-diagnostic-instance).
+  2. `tools` section: This section contains information about the compiler build and version. Additionally, it contains a `rules` array, where each rule entry corresponds to metadata or `DiagnosticDescriptor` information about each reported analyzer diagnostic. More details in [`Rule` format for each analyzer supported `DiagnosticDescriptor` instance](#rule-format-for-each-analyzer-supported-diagnosticdescriptor-instance)
+  3. `columnKind` section: This section contains information about the unit in which the tool measures columns. C# and Visual Basic compilers uses utf16 code units.
+
+Example `runs` section, with stripped off `results` and `rules` sections:
+```json
+"runs": [
+{
+  "results": [
+  ],
+  "tool": {
+    "driver": {
+      "name": "Microsoft (R) Visual C# Compiler",
+      "version": "4.4.0-dev (<developer build>)",
+      "dottedQuadFileVersion": "42.42.42.42",
+      "semanticVersion": "42.42.42",
+      "language": "en-US",
+      "rules": [
+      ]
+    }
+  },
+  "columnKind": "utf16CodeUnits"
+}
+]
+```
+
+### `Result` format for each compiler or analyzer `Diagnostic` instance
+
+The results section contains an array of result entries, where each result corresponds to information about a reported compiler or analyzer `Diagnostic`. It contains the below data:
+1. `ruleId`: Rule ID or diagnostic ID for the diagnostic.
+2. `ruleIndex`: Index into the `rules` section for the underlying `DiagnosticDescriptor` for the diagnostic. See [`Rule` format for each analyzer supported `DiagnosticDescriptor` instance](#rule-format-for-each-analyzer-supported-diagnosticdescriptor-instance) for more details.
+3. `level`: Severity level for the diagnostic, such as `error`, `warning`, `note`, etc.
+4. `message`: User facing message for the diagnostic.
+5. `suppressions`: For each diagnostic instance that is suppressed with a pragma directive, SuppressMessageAttribute or via a DiagnosticSuppressor, the result entry contains `suppressions` section with the following data:
+  1. `kind` information: C# and Visual Basic compilers support a single `inSource` suppression kind.
+  2. `justification` information: Justification pertaining to the suppression. Currently, this field is populated only for `SuppressMessageAttribute` based suppression, with a non-null justification argument.
+  3. `suppressionType` property with one of these three values: `Pramga Directive`, `SuppressMessageAttribute`, or `DiagnosticSuppressor`. This data helps analyze the preferred in-source suppression mechanisms in a code base.
+6. `locations`: One or more locations associated with the diagnostic.
+7. `properties`: One or more custom key-value string pairs associated with the diagnostic.
+
+Example `result` entry:
+```json
+{
+  "ruleId": "CA1822",
+  "ruleIndex": 97,
+  "level": "warning",
+  "message": {
+    "text": "Member 'M2' does not access instance data and can be marked as static"
+  },
+  "suppressions": [
+  {
+    "kind": "inSource",
+    "justification": "<Pending>",
+    "properties": {
+      "suppressionType": "SuppressMessageAttribute"
+    }
+  }
+  ],
+  "locations": [
+  {
+    "physicalLocation": {
+      "artifactLocation": {
+        "uri": "file:///C:/source/repos/ClassLibrary1/Class1.cs"
+      },
+      "region": {
+        "startLine": 13,
+        "startColumn": 10,
+        "endLine": 13,
+        "endColumn": 12
+      }
+    }
+  }
+  ],
+  "properties": {
+    "warningLevel": 1
+  }
+}
+```
+
+### `Rule` format for each analyzer supported `DiagnosticDescriptor` instance
+
+The `rules` section contains a rule entry that corresponds to metadata or `DiagnosticDescriptor` information about supported descriptors for each analyzer. We report the `DiagnosticDescriptor` info for all analyzers that were provided to the compilation, regardless of whether or not they executed on the entire compilation, part of the compilation or were disabled for the entire compilation. A `rule` entry contains the below data:
+1. `id`: Rule ID or Diagnostic ID associated with the descriptor.
+2. `shortDescription`: User facing short description or `Title` associated with the descriptor.
+3. `fullDescription`: User facing full description or `Description` associated with the descriptor.
+4. `defaultConfiguration`: Default severity level for the diagnostics reported for the descriptor, such as `error`, `warning`, `note`, etc.
+5. `helpUri`: Help uri for help information associated with the descriptor.
+6. `properties`: One or more custom properties associated with the descriptor. It includes the following:
+  1. `category`: `Category` associated with the descriptor, such as, `Design`, `Performance`, `Security`, etc.
+  2. `isEverSuppressed` and `suppressionKinds`: If a rule had either a source suppression or was disabled for part or whole of the compilation via options, the rule metadata contains a special flag `isEverSuppressed = true` and an array `suppressionKinds` with either or both of the below suppression kinds:
+    1. `inSource` suppression kind for one or more reported diagnostic(s) that were suppressed through pragma directive, SuppressMessageAttribute or a DiagnosticSuppressor.
+    2. `external` suppression kind for diagnostic ID that is disabled either for the entire compilation (via global options such as /nowarn, ruleset, globalconfig, etc.) or for certain files or folders in the compilation (via editorconfig options).
+  3. `tags`: An array of one of more `CustomTags` associated with the descriptor.
+
+Example `rule` entry:
+```json
+{
+  "id": "CA1001",
+  "shortDescription": {
+    "text": "Types that own disposable fields should be disposable"
+  },
+  "fullDescription": {
+    "text": "A class declares and implements an instance field that is a System.IDisposable type, and the class does not implement IDisposable. A class that declares an IDisposable field indirectly owns an unmanaged resource and should implement the IDisposable interface."
+  },
+  "defaultConfiguration": {
+    "level": "note"
+  },
+  "helpUri": "https://docs.microsoft.com/dotnet/fundamentals/code-analysis/quality-rules/ca1001",
+  "properties": {
+    "category": "Design",
+    "isEverSuppressed": "true",
+    "suppressionKinds": [
+      "external"
+    ],
+    "tags": [
+      "PortedFromFxCop",
+      "Telemetry",
+      "EnabledRuleInAggressiveMode"
+    ]
+  }
+}
+```
diff --git a/src/Compilers/CSharp/Test/CommandLine/SarifErrorLoggerTests.cs b/src/Compilers/CSharp/Test/CommandLine/SarifErrorLoggerTests.cs
@@ -4,9 +4,11 @@
 
 #nullable disable
 
+using System;
 using System.Collections.Immutable;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using Microsoft.CodeAnalysis.CSharp.Test.Utilities;
 using Microsoft.CodeAnalysis.Diagnostics;
 using Xunit;
@@ -20,9 +22,9 @@ public abstract class SarifErrorLoggerTests : CommandLineTestBase
         protected abstract string ErrorLogQualifier { get; }
         internal abstract string GetExpectedOutputForNoDiagnostics(CommonCompiler cmd);
         internal abstract string GetExpectedOutputForSimpleCompilerDiagnostics(CommonCompiler cmd, string sourceFile);
-        internal abstract string GetExpectedOutputForSimpleCompilerDiagnosticsSuppressed(CommonCompiler cmd, string sourceFile);
+        internal abstract string GetExpectedOutputForSimpleCompilerDiagnosticsSuppressed(CommonCompiler cmd, string sourceFile, params string[] suppressionKinds);
         internal abstract string GetExpectedOutputForAnalyzerDiagnosticsWithAndWithoutLocation(MockCSharpCompiler cmd);
-        internal abstract string GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(MockCSharpCompiler cmd, string justification);
+        internal abstract string GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(MockCSharpCompiler cmd, string justification, string suppressionType, params string[] suppressionKinds);
 
         protected void NoDiagnosticsImpl()
         {
@@ -118,7 +120,7 @@ public class C
             Assert.NotEqual(0, exitCode);
 
             var actualOutput = File.ReadAllText(errorLogFile).Trim();
-            string expectedOutput = GetExpectedOutputForSimpleCompilerDiagnosticsSuppressed(cmd, sourceFile);
+            string expectedOutput = GetExpectedOutputForSimpleCompilerDiagnosticsSuppressed(cmd, sourceFile, suppressionKinds: "inSource");
 
             Assert.Equal(expectedOutput, actualOutput);
 
@@ -188,7 +190,7 @@ class C
             Assert.NotEqual(0, exitCode);
 
             var actualOutput = File.ReadAllText(errorLogFile).Trim();
-            string expectedOutput = GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(cmd, "Justification1");
+            string expectedOutput = GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(cmd, "Justification1", suppressionType: "SuppressMessageAttribute", suppressionKinds: "inSource");
 
             Assert.Equal(expectedOutput, actualOutput);
 
@@ -223,7 +225,7 @@ class C
             Assert.NotEqual(0, exitCode);
 
             var actualOutput = File.ReadAllText(errorLogFile).Trim();
-            string expectedOutput = GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(cmd, null);
+            string expectedOutput = GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(cmd, null, suppressionType: "SuppressMessageAttribute", suppressionKinds: "inSource");
 
             Assert.Equal(expectedOutput, actualOutput);
 
@@ -258,7 +260,7 @@ class C
             Assert.NotEqual(0, exitCode);
 
             var actualOutput = File.ReadAllText(errorLogFile).Trim();
-            string expectedOutput = GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(cmd, "");
+            string expectedOutput = GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(cmd, "", suppressionType: "SuppressMessageAttribute", suppressionKinds: "inSource");
 
             Assert.Equal(expectedOutput, actualOutput);
 
@@ -293,7 +295,7 @@ class C
             Assert.NotEqual(0, exitCode);
 
             var actualOutput = File.ReadAllText(errorLogFile).Trim();
-            string expectedOutput = GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(cmd, null);
+            string expectedOutput = GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(cmd, null, suppressionType: "SuppressMessageAttribute", suppressionKinds: "inSource");
 
             Assert.Equal(expectedOutput, actualOutput);
 

diff --git a/src/Compilers/CSharp/Test/CommandLine/SarifV1ErrorLoggerTests.cs b/src/Compilers/CSharp/Test/CommandLine/SarifV1ErrorLoggerTests.cs
@@ -111,7 +111,7 @@ public void SimpleCompilerDiagnostics()
             SimpleCompilerDiagnosticsImpl();
         }
 
-        internal override string GetExpectedOutputForSimpleCompilerDiagnosticsSuppressed(CommonCompiler cmd, string sourceFile)
+        internal override string GetExpectedOutputForSimpleCompilerDiagnosticsSuppressed(CommonCompiler cmd, string sourceFile, params string[] suppressionKinds)
         {
             var expectedHeader = GetExpectedErrorLogHeader(cmd);
             var expectedIssues = string.Format(@"
@@ -196,7 +196,7 @@ internal override string GetExpectedOutputForAnalyzerDiagnosticsWithAndWithoutLo
             return expectedHeader + expectedIssues;
         }
 
-        internal override string GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(MockCSharpCompiler cmd, string justification)
+        internal override string GetExpectedOutputForAnalyzerDiagnosticsWithSuppression(MockCSharpCompiler cmd, string justification, string suppressionType, params string[] suppressionKinds)
         {
             var expectedHeader = GetExpectedErrorLogHeader(cmd);
             var expectedIssues = AnalyzerForErrorLogTest.GetExpectedV1ErrorLogWithSuppressionResultsAndRulesText(cmd.Compilation);