License scanning for VMR (#17442)

eng/install-scancode.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html#installation-as-a-library-via-pip
+
+pyEnvPath="/tmp/scancode-env"
+python3 -m venv $pyEnvPath
+source $pyEnvPath/bin/activate
+pip install scancode-toolkit
+deactivate
+
+# Setup a script which executes scancode in the virtual environment
+cat > /usr/local/bin/scancode << EOF
+#!/bin/bash
+set -euo pipefail
+source $pyEnvPath/bin/activate
+scancode "\$@"
+deactivate
+EOF
+
+chmod +x /usr/local/bin/scancode

eng/pipelines/source-build-license-scan.yml

@@ -0,0 +1,137 @@
+# Pipeline documentation at https://github.com/dotnet/dotnet/blob/main/docs/license-scanning.md
+
+schedules:
+- cron: "0 7 * * 1-5"
+  displayName: Run on weekdays at 7am UTC
+  branches:
+    include:
+    - main
+    - release/*
+
+pr: none
+trigger: none
+
+parameters:
+# Provides a way to scan a specific repo. If not provided, all repos of the VMR will be scanned.
+- name: specificRepoName
+  type: string
+  displayName: "Specific repo name to scan (e.g. runtime, sdk). If empty, scans all repos of the VMR."
+  default: " " # Set it to an empty string to allow it be an optional parameter
+
+variables:
+  installerRoot: '$(Build.SourcesDirectory)/src/installer'
+
+jobs:
+- job: Setup
+  pool:
+    name: NetCore1ESPool-Svc-Internal
+    demands: ImageOverride -equals 1es-ubuntu-2004
+  steps:
+  - script: |
+      vmrSrcDir="$(Build.SourcesDirectory)/src"
+      
+      # Builds an Azure DevOps matrix definition. Each entry in the matrix is a path,
+      # allowing a job to be run for each src repo.
+      matrix=""
+
+      # Trim leading/trailing spaces from the repo name
+      specificRepoName=$(echo "${{ parameters.specificRepoName }}" | awk '{$1=$1};1')
+
+      # If the repo name is provided, only scan that repo.
+      if [ ! -z "$specificRepoName" ]; then
+        matrix="\"$specificRepoName\": { \"repoPath\": \"$vmrSrcDir/$specificRepoName\" }"
+      else
+        for dir in $vmrSrcDir/*/
+        do
+          if [ ! -z "$matrix" ]; then
+            matrix="$matrix,"
+          fi
+          repoName=$(basename $dir)
+          matrix="$matrix \"$repoName\": { \"repoPath\": \"$dir\" }"
+        done
+      fi
+
+      matrix="{ $matrix }"
+
+      echo "##vso[task.setvariable variable=matrix;isOutput=true]$matrix"
+    name: GetMatrix
+    displayName: Get Matrix
+
+- job: LicenseScan
+  dependsOn: Setup
+  pool:
+    name: NetCore1ESPool-Svc-Internal
+    demands: ImageOverride -equals 1es-ubuntu-2004
+  timeoutInMinutes: 420
+  strategy:
+    matrix: $[ dependencies.Setup.outputs['GetMatrix.matrix'] ]
+  steps:
+
+  - script: $(Build.SourcesDirectory)/prep.sh --no-artifacts --no-bootstrap --no-prebuilts
+    displayName: 'Install .NET SDK'
+
+  - task: PipAuthenticate@1
+    displayName: 'Pip Authenticate'
+    inputs: 
+      artifactFeeds: public/dotnet-public-pypi
+      onlyAddExtraIndex: false
+
+  - script: $(installerRoot)/eng/install-scancode.sh
+    displayName: Install Scancode
+
+  - script: >
+      $(Build.SourcesDirectory)/.dotnet/dotnet test
+      $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj
+      --filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.LicenseScanTests.ScanForLicenses"
+      --logger:'trx;LogFileName=$(Agent.JobName)_LicenseScan.trx'
+      --logger:'console;verbosity=detailed'
+      -c Release
+      -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog 
+      -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log
+      -clp:v=m
+      -e SMOKE_TESTS_LICENSE_SCAN_PATH=$(repoPath)
+      -e SMOKE_TESTS_RUNNING_IN_CI=true
+      -e SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS=false
+      -e SMOKE_TESTS_TARGET_RID=linux-x64
+      -e SMOKE_TESTS_PORTABLE_RID=linux-x64
+    displayName: Run Tests
+    workingDirectory: $(Build.SourcesDirectory)
+
+  - script: |
+      set -x
+      targetFolder=$(Build.StagingDirectory)/BuildLogs/
+      mkdir -p ${targetFolder}
+      cd "$(Build.SourcesDirectory)"
+      find artifacts/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \;
+      echo "Updated:"
+      find test/ -type f -name "Updated*.json"
+      find test/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \;
+      echo "Results:"
+      find test/ -type f -name "scancode-results*.json"
+      find test/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \;
+      echo "All:"
+      ls -R test/
+      echo "BuildLogs:"
+      ls -R ${targetFolder}
+    displayName: Prepare BuildLogs staging directory
+    continueOnError: true
+    condition: succeededOrFailed()
+
+  - publish: '$(Build.StagingDirectory)/BuildLogs'
+    artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
+    displayName: Publish BuildLogs
+    continueOnError: true
+    condition: succeededOrFailed()
+
+  - task: PublishTestResults@2
+    displayName: Publish Test Results
+    condition: succeededOrFailed()
+    continueOnError: true
+    inputs:
+      testRunner: vSTest
+      testResultsFiles: '*.trx'
+      searchFolder: $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults
+      mergeTestResults: true
+      publishRunAttachments: true
+      testRunTitle: $(Agent.JobName)

src/SourceBuild/content/docs/license-scanning.md

@@ -0,0 +1,24 @@
+# License Scanning
+
+The VMR is regularly scanned for license references to ensure that only open-source license are used where relevant.
+
+License scanning pipline: https://dev.azure.com/dnceng/internal/_build?definitionId=1301 (internal only)
+
+License scanning test: https://github.com/dotnet/dotnet/blob/main/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs
+
+By default, running the pipeline will scan all repos within the VMR which takes several hours to run.
+The pipeline can be triggered manually to target a specific repo within the VMR by setting the `specificRepoName` parameter.
+This value should be the name of the repo within the VMR (i.e. a name of a directory within https://github.com/dotnet/dotnet/tree/main/src).
+To test source modifications intended to resolve a license issue, apply the change in an internal branch of the VMR.
+Run this pipeline, targeting your branch, and set the `specificRepoName` parameter to the name of the repo containing the change.
+
+The output of the pipeline is a set of test results and logs.
+The logs are published as an artifact and can be found at test/Microsoft.DotNet/SourceBuild.SmokeTests/bin/Release/netX.0/logs.
+It consists of the following:
+  * `UpdatedLicenses.<repo-name>.json`: This is the output of that gets compared to the stored baseline.
+    If they're the same, the test passes; if not, it fails. By comparing this file to the baseline, one can determine which new license
+    references have been introduced.
+    If everything is deemed to be acceptable, the developer can either update the allowed licenses, update the exclusions file, update the
+    baseline, or any combination.
+  * `scancode-results.json`: This is the raw output that comes from scancode. This file is useful for diagnostic purposes because it tells you
+    the exact line number of where a license has been detected in a file.

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs

@@ -41,12 +41,12 @@ public static void CompareEntries(string baselineFileName, IOrderedEnumerable<st
             Assert.Null(message);
         }
 
-        public static void CompareBaselineContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper, bool warnOnDiffs = false)
+        public static void CompareBaselineContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper, bool warnOnDiffs = false, string baselineSubDir = "")
         {
-            string actualFilePath = Path.Combine(DotNetHelper.LogsDirectory, $"Updated{baselineFileName}");
+            string actualFilePath = Path.Combine(TestBase.LogsDirectory, $"Updated{baselineFileName}");
             File.WriteAllText(actualFilePath, actualContents);
 
-            CompareFiles(GetBaselineFilePath(baselineFileName), actualFilePath, outputHelper, warnOnDiffs);
+            CompareFiles(GetBaselineFilePath(baselineFileName, baselineSubDir), actualFilePath, outputHelper, warnOnDiffs);
         }
 
         public static void CompareFiles(string expectedFilePath, string actualFilePath, ITestOutputHelper outputHelper, bool warnOnDiffs = false)
@@ -87,7 +87,8 @@ public static string DiffFiles(string file1Path, string file2Path, ITestOutputHe
 
         public static string GetAssetsDirectory() => Path.Combine(Directory.GetCurrentDirectory(), "assets");
 
-        public static string GetBaselineFilePath(string baselineFileName) => Path.Combine(GetAssetsDirectory(), "baselines", baselineFileName);
+        public static string GetBaselineFilePath(string baselineFileName, string baselineSubDir = "") =>
+            Path.Combine(GetAssetsDirectory(), "baselines", baselineSubDir, baselineFileName);
 
         public static string RemoveNetTfmPaths(string source)
         {

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BasicScenarioTests.cs

@@ -15,7 +15,7 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 /// <see cref="WebScenarioTests"/> for related web scenarios.
 /// They are encapsulated in a separate testclass so that they can be run in parallel.
 /// </summary>
-public class BasicScenarioTests : SmokeTests
+public class BasicScenarioTests : SdkTests
 {
     public BasicScenarioTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
 

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

@@ -20,7 +20,9 @@ internal static class Config
     public const string SourceBuiltArtifactsPathEnv = "SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH";
     public const string TargetRidEnv = "SMOKE_TESTS_TARGET_RID";
     public const string WarnSdkContentDiffsEnv = "SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS";
+    public const string WarnLicenseScanDiffsEnv = "SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS";
     public const string RunningInCIEnv = "SMOKE_TESTS_RUNNING_IN_CI";
+    public const string LicenseScanPathEnv = "SMOKE_TESTS_LICENSE_SCAN_PATH";
 
     public static string DotNetDirectory { get; } =
         Environment.GetEnvironmentVariable(DotNetDirectoryEnv) ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet");
@@ -31,15 +33,18 @@ internal static class Config
     public static string? PrereqsPath { get; } = Environment.GetEnvironmentVariable(PrereqsPathEnv);
     public static string? CustomPackagesPath { get; } = Environment.GetEnvironmentVariable(CustomPackagesPathEnv);
     public static string? SdkTarballPath { get; } = Environment.GetEnvironmentVariable(SdkTarballPathEnv);
-    public static string SourceBuiltArtifactsPath { get; } = Environment.GetEnvironmentVariable(SourceBuiltArtifactsPathEnv) ??
-        throw new InvalidOperationException($"'{Config.SourceBuiltArtifactsPathEnv}' must be specified");
+    public static string? SourceBuiltArtifactsPath { get; } = Environment.GetEnvironmentVariable(SourceBuiltArtifactsPathEnv);
     public static string TargetRid { get; } = Environment.GetEnvironmentVariable(TargetRidEnv) ??
         throw new InvalidOperationException($"'{Config.TargetRidEnv}' must be specified");
     public static string TargetArchitecture { get; } = TargetRid.Split('-')[1];
     public static bool WarnOnSdkContentDiffs { get; } =
         bool.TryParse(Environment.GetEnvironmentVariable(WarnSdkContentDiffsEnv), out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs;
+    public static bool WarnOnLicenseScanDiffs { get; } =
+        bool.TryParse(Environment.GetEnvironmentVariable(WarnLicenseScanDiffsEnv), out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs;
 
     // Indicates whether the tests are being run in the context of a CI pipeline
     public static bool RunningInCI { get; } =
         bool.TryParse(Environment.GetEnvironmentVariable(RunningInCIEnv), out bool runningInCI) && runningInCI;
+
+    public static string? LicenseScanPath { get; } = Environment.GetEnvironmentVariable(LicenseScanPathEnv);
 }

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DebugTests.cs

@@ -12,7 +12,7 @@
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
-public class DebugTests : SmokeTests
+public class DebugTests : SdkTests
 {
     private record ScanResult(string FileName, bool HasDebugInfo, bool HasDebugAbbrevs, bool HasFileSymbols, bool HasGnuDebugLink);
 

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
-public class DotNetFormatTests : SmokeTests
+public class DotNetFormatTests : SdkTests
 {
     private const string TestFileName = "FormatTest.cs";
     private const string UnformattedFileName = "FormatTestUnformatted.cs";

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs

@@ -3,14 +3,12 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Net.Sockets;
-using System.Threading.Tasks;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -21,7 +19,6 @@ internal class DotNetHelper
     private static readonly object s_lockObj = new();
 
     public static string DotNetPath { get; } = Path.Combine(Config.DotNetDirectory, "dotnet");
-    public static string LogsDirectory { get; } = Path.Combine(Directory.GetCurrentDirectory(), "logs");
     public static string PackagesDirectory { get; } = Path.Combine(Directory.GetCurrentDirectory(), "packages");
     public static string ProjectsDirectory { get; } = Path.Combine(Directory.GetCurrentDirectory(), $"projects-{DateTime.Now:yyyyMMddHHmmssffff}");
 
@@ -56,11 +53,6 @@ public DotNetHelper(ITestOutputHelper outputHelper)
             {
                 Directory.CreateDirectory(PackagesDirectory);
             }
-
-            if (!Directory.Exists(LogsDirectory))
-            {
-                Directory.CreateDirectory(LogsDirectory);
-            }
         }
     }
 
@@ -261,7 +253,7 @@ private static string GetBinLogOption(string projectName, string command, string
             fileName += $"-{differentiator}";
         }
 
-        return $"/bl:{Path.Combine(LogsDirectory, $"{fileName}.binlog")}";
+        return $"/bl:{Path.Combine(TestBase.LogsDirectory, $"{fileName}.binlog")}";
     }
 
     private static bool DetermineIsMonoRuntime(string dotnetRoot)

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetWatchTests.cs

@@ -9,7 +9,7 @@
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
-public class DotNetWatchTests : SmokeTests
+public class DotNetWatchTests : SdkTests
 {
     public DotNetWatchTests(ITestOutputHelper outputHelper) : base(outputHelper) { }