Generate test certificates in pipeline (microsoft#4747)

The tests use test certificates for codesigning and for HTTPS with the
localhost server. These test certificates were stored as secure files in
the pipelines.

This PR changes the test pipeline to generate the certificates on each
run so that we don't need to create new certificates when they expire,
or if we want to recreate the pipelines in a different ADO account.

* Added pipeline steps to create/install the test certificates for
codesigning and HTTPS. The scripts export the path and password (a GUID)
as pipeline variables for use in subsequent tasks
* Removed unnecessary BinSkim task (already included in official
pipelines)
* Removed unused certificates from the code (unrelated to the ones being
replaced)
* Updated hash tests that were using the existing certificates to use a
different file, as the cert is no longer constant
* Added collection of the logs from the localhost server and index
creation; this requires stopping the server after it is used

.github/actions/spelling/expect.txt

@@ -30,7 +30,6 @@ appname
 appshutdown
 APPTERMINATION
 archs
-argumentlist
 ARMNT
 arp
 arphelper
@@ -390,7 +389,6 @@ Peet
 peetdev
 PEGI
 pfn
-pfxpath
 pgp
 Pherson
 pid
@@ -469,7 +467,6 @@ savepoint
 schematab
 Scm
 sddl
-SECUREFILEPATH
 secureobject
 securestring
 seof

azure-pipelines.yml

@@ -259,7 +259,8 @@ jobs:
   - template: templates/e2e-setup.yml
     parameters:
       sourceDir: $(Build.SourcesDirectory)
-      localhostWebServerArgs: '-BuildRoot $(artifactsDir)\E2ETests\LocalhostWebServer -StaticFileRoot $(Agent.TempDirectory)\TestLocalIndex -LocalSourceJson $(Build.SourcesDirectory)\src\AppInstallerCLIE2ETests\TestData\localsource.json -TestDataPath $(Build.SourcesDirectory)\src\AppInstallerCLIE2ETests\TestData -SourceCert $(Build.SourcesDirectory)\src\AppInstallerCLIE2ETests\TestData\AppInstallerTest.cer -ExitBeforeRun'
+      localhostWebServerArgs: '-BuildRoot $(artifactsDir)\E2ETests\LocalhostWebServer -StaticFileRoot $(Agent.TempDirectory)\TestLocalIndex -LocalSourceJson $(Build.SourcesDirectory)\src\AppInstallerCLIE2ETests\TestData\localsource.json -TestDataPath $(Build.SourcesDirectory)\src\AppInstallerCLIE2ETests\TestData -ExitBeforeRun'
+      signingCertOutDir: $(artifactsDir)\E2ETests
 
   - task: CopyFiles@2
     displayName: 'Copy TestLocalIndex'
@@ -302,28 +303,6 @@ jobs:
       verbosity: 'Verbose'
       alertWarningLevel: 'High'
 
-  # Run BimSkim for all the binaries
-  - task: BinSkim@4
-    displayName: 'Run BinSkim '
-    inputs:
-      arguments: 'analyze
-        "$(buildOutDir)\AppInstallerCLI\winget.exe"
-        "$(buildOutDir)\WinGetUtil\WinGetUtil.dll"
-        "$(buildOutDir)\WindowsPackageManager\WindowsPackageManager.dll"
-        "$(buildOutDir)\Microsoft.Management.Deployment.InProc\Microsoft.Management.Deployment.InProc.dll"
-        "$(Build.SourcesDirectory)\src\WinGetUtilInterop\bin\WinGetUtil*Interop.dll"
-        "$(buildOutDir)\UndockedRegFreeWinRT\winrtact.dll"
-        "$(buildOutDir)\Microsoft.WinGet.Client.Cmdlets\Microsoft.WinGet.Client*.dll"
-        "$(buildOutDir)\ConfigurationRemotingServer\ConfigurationRemoting*Server.dll"
-        "$(buildOutDir)\ConfigurationRemotingServer\ConfigurationRemoting*Server.exe"
-        "$(buildOutDir)\ConfigurationRemotingServer\Microsoft.Management.Configuration*.dll"
-        "$(buildOutDir)\Microsoft.Management.Configuration\Microsoft.Management.Configuration*.dll"
-        "$(buildOutDir)\Microsoft.Management.Configuration.OutOfProc\Microsoft.Management.Configuration*.dll"
-        --config default --recurse'
-
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
-    displayName: 'Publish Security Analysis Logs'
-
 # Test job runs tests using build artifacts
 
 - job: 'Test'
@@ -431,7 +410,7 @@ jobs:
   - template: templates/e2e-setup.yml
     parameters:
       sourceDir: $(Build.SourcesDirectory)
-      localhostWebServerArgs: '-BuildRoot $(buildOutDir)\E2ETests\LocalhostWebServer -StaticFileRoot $(buildOutDir)\E2ETests\TestLocalIndex -SourceCert $(Build.SourcesDirectory)\src\AppInstallerCLIE2ETests\TestData\AppInstallerTest.cer'
+      localhostWebServerArgs: '-BuildRoot $(buildOutDir)\E2ETests\LocalhostWebServer -StaticFileRoot $(buildOutDir)\E2ETests\TestLocalIndex -SourceCert $(buildOutDir)\E2ETests\TestSigningCert.cer'
 
   - template: templates/e2e-test.template.yml
     parameters:
@@ -511,6 +490,10 @@ jobs:
       arguments: '-TargetLocation $(artifactsDir)\ConfigOOPTestsLog'
     condition: succeededOrFailed()
 
+  - powershell: Get-Process LocalhostWebServer | Stop-Process
+    displayName: Stop LocalhostWebServer
+    condition: succeededOrFailed()
+
   - task: PublishPipelineArtifact@1
     displayName: Publish Pipeline Artifacts
     inputs:
@@ -524,6 +507,8 @@ jobs:
   timeoutInMinutes: 120
   dependsOn: 'Build'
   condition: succeeded('Build')
+  variables:
+    buildOutDir: $(Pipeline.Workspace)\Build.x64Release
 
   steps:
   - task: DownloadPipelineArtifact@2
@@ -532,7 +517,7 @@ jobs:
   - task: CopyFiles@2
     displayName: 'Copy x64 PowerShell Binaries to Output'
     inputs:
-      SourceFolder: '$(Pipeline.Workspace)\Build.x64release\PowerShell'
+      SourceFolder: '$(buildOutDir)\PowerShell'
       Contents: '**\*'
       TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
@@ -566,14 +551,14 @@ jobs:
       targetType: 'inline'
       script: |
         Get-ChildItem AppxPackages\AppInstallerCLIPackage_0.0.2.0_Test\Dependencies\x64 -Filter *.appx | %{ Add-AppxPackage $_.FullName }
-      workingDirectory: $(Pipeline.Workspace)\Build.x64release\
+      workingDirectory: $(buildOutDir)
 
   - template: templates/e2e-setup.yml
     parameters:
       sourceDir: $(Build.SourcesDirectory)
-      localhostWebServerArgs: '-BuildRoot $(Pipeline.Workspace)\Build.x64release\E2ETests\LocalhostWebServer -StaticFileRoot $(Pipeline.Workspace)\Build.x64release\E2ETests\TestLocalIndex -SourceCert $(Build.SourcesDirectory)\src\AppInstallerCLIE2ETests\TestData\AppInstallerTest.cer'
+      localhostWebServerArgs: '-BuildRoot $(buildOutDir)\E2ETests\LocalhostWebServer -StaticFileRoot $(buildOutDir)\E2ETests\TestLocalIndex -SourceCert $(buildOutDir)\E2ETests\TestSigningCert.cer'
 
-  - pwsh: .\RunTests.ps1 -testModulesPath $(Build.ArtifactStagingDirectory) -outputPath $(Pipeline.Workspace)\PesterTest -packageLayoutPath $(Pipeline.Workspace)\Build.x64release\DevPackage
+  - pwsh: .\RunTests.ps1 -testModulesPath $(Build.ArtifactStagingDirectory) -outputPath $(Pipeline.Workspace)\PesterTest -packageLayoutPath $(buildOutDir)\DevPackage
     workingDirectory: $(Build.SourcesDirectory)\src\PowerShell\tests\
     displayName: Run PowerShell 7 Tests
 
@@ -582,6 +567,10 @@ jobs:
     displayName: Run Windows PowerShell Tests
     condition: succeededOrFailed()
 
+  - powershell: Get-Process LocalhostWebServer | Stop-Process
+    displayName: Stop LocalhostWebServer
+    condition: succeededOrFailed()
+
   - task: PublishTestResults@2
     displayName: Publish Pester Test Results PowerShell 7
     inputs:

src/AppInstallerCLIE2ETests/Constants.cs

@@ -45,9 +45,6 @@ public class Constants
         public const string TestSourceType = "Microsoft.PreIndexed.Package";
         public const string TestSourceIdentifier = @"WingetE2E.Tests_8wekyb3d8bbwe";
 
-        public const string AppInstallerTestCert = "AppInstallerTest.cer";
-        public const string AppInstallerTestCertThumbprint = "d03e7a688b388b1edde8476a627531c49db88017";
-
         public const string AICLIPackageFamilyName = "WinGetDevCLI_8wekyb3d8bbwe";
         public const string AICLIPackageName = "WinGetDevCLI";
         public const string AICLIPackagePublisherHash = "8wekyb3d8bbwe";

src/AppInstallerCLIE2ETests/HashCommand.cs

@@ -21,9 +21,9 @@ public class HashCommand : BaseCommand
         [Test]
         public void HashFile()
         {
-            var result = TestCommon.RunAICLICommand("hash", TestCommon.GetTestDataFile("AppInstallerTest.cer"));
+            var result = TestCommon.RunAICLICommand("hash", TestCommon.GetTestDataFile("AppInstallerTestMsiInstaller.msi"));
             Assert.AreEqual(Constants.ErrorCode.S_OK, result.ExitCode);
-            Assert.True(result.StdOut.Contains("9b4c49ad7e47afd97d2e666e93347745e1647c55f1a7ebba6d31b7dd5f69ee68"));
+            Assert.True(result.StdOut.Contains("21d90ee9b3569590c624836ef50bf39791c7184869c227eedc00585e1f39b4de"));
         }
 
         /// <summary>
@@ -44,9 +44,9 @@ public void HashMSIX()
         [Test]
         public void HashInvalidMSIX()
         {
-            var result = TestCommon.RunAICLICommand("hash", TestCommon.GetTestDataFile("AppInstallerTest.cer") + " -m");
+            var result = TestCommon.RunAICLICommand("hash", TestCommon.GetTestDataFile("AppInstallerTestMsiInstaller.msi") + " -m");
             Assert.AreEqual(Constants.ErrorCode.OPC_E_ZIP_MISSING_END_OF_CENTRAL_DIRECTORY, result.ExitCode);
-            Assert.True(result.StdOut.Contains("9b4c49ad7e47afd97d2e666e93347745e1647c55f1a7ebba6d31b7dd5f69ee68"));
+            Assert.True(result.StdOut.Contains("21d90ee9b3569590c624836ef50bf39791c7184869c227eedc00585e1f39b4de"));
             Assert.True(result.StdOut.Contains("Please verify that the input file is a valid, signed MSIX."));
         }
 

src/AppInstallerCLIE2ETests/SetUpFixture.cs

@@ -41,8 +41,6 @@ public void Setup()
 
                 shouldRevertDefaultFileTypeRiskOnExit = this.DecreaseFileTypeRisk(".exe;.msi", false);
 
-                Assert.True(TestCommon.RunCommand("certutil.exe", "-addstore -f \"TRUSTEDPEOPLE\" " + TestCommon.GetTestDataFile(Constants.AppInstallerTestCert)), "Add AppInstallerTestCert");
-
                 if (testParams.PackagedContext)
                 {
                     if (testParams.LooseFileRegistration)
@@ -83,8 +81,6 @@ public void TearDown()
                     this.DecreaseFileTypeRisk(defaultFileTypes, true);
                 }
 
-                TestCommon.RunCommand("certutil.exe", $"-delstore \"TRUSTEDPEOPLE\" {Constants.AppInstallerTestCertThumbprint}");
-
                 TestCommon.PublishE2ETestLogs();
 
                 if (TestSetup.Parameters.PackagedContext)

src/AppInstallerCLIE2ETests/TestData/localsource.json

@@ -44,6 +44,7 @@
     }
   ],
   "Signature": {
-    "CertFile": "%APPINSTALLERTEST_SECUREFILEPATH%"
+    "CertFile": "%TestSigningCert_PfxPath%",
+    "Password": "%TestSigningCert_Password%"
   }
 }

src/LocalhostWebServer/Program.cs

@@ -164,7 +164,7 @@ private static void CopyDirectoryRecursive(string sourceDir, string destDir)
             foreach (string file in files)
             {
                 string dest = Path.Combine(destDir, Path.GetFileName(file));
-                File.Copy(file, dest);
+                File.Copy(file, dest, overwrite: true);
             }
 
             string[] directories = Directory.GetDirectories(sourceDir);

src/LocalhostWebServer/Run-LocalhostWebServer.ps1

@@ -54,8 +54,19 @@ if (-not [System.String]::IsNullOrEmpty($sourceCert))
 
 Push-Location $BuildRoot
 
-$Local:process = Start-Process -FilePath "LocalhostWebServer.exe" -ArgumentList "StaticFileRoot=$StaticFileRoot CertPath=$CertPath CertPassword=$CertPassword OutCertFile=$OutCertFile LocalSourceJson=$LocalSourceJson TestDataPath=$TestDataPath ExitBeforeRun=$ExitBeforeRun" -PassThru
+$startProcessArguments = @{
+    FilePath = Join-Path $BuildRoot "LocalhostWebServer.exe"
+    ArgumentList = "StaticFileRoot=$StaticFileRoot CertPath=$CertPath CertPassword=$CertPassword OutCertFile=$OutCertFile LocalSourceJson=$LocalSourceJson TestDataPath=$TestDataPath ExitBeforeRun=$ExitBeforeRun"
+    PassThru = $true
+}
+
+if (-not [System.string]::IsNullOrEmpty($env:artifactsDir))
+{
+    $startProcessArguments.RedirectStandardOutput = Join-Path $env:artifactsDir "LocalhostWebServer.out"
+    $startProcessArguments.RedirectStandardError = Join-Path $env:artifactsDir "LocalhostWebServer.err"
+}
 
+$Local:process = Start-Process @startProcessArguments
 
 if ($ExitBeforeRun)
 {

templates/e2e-setup.yml

@@ -4,31 +4,48 @@ parameters:
   type: string
 - name: localhostWebServerArgs
   type: string
+- name: signingCertOutDir
+  type: string
+  default: $(Agent.TempDirectory)
 
 steps:
-  - task: DownloadSecureFile@1
-    name: AppInstallerTest
-    displayName: 'Download Source Package Certificate'
-    inputs:
-      secureFile: 'AppInstallerTest.pfx'
+  - pwsh: |
+      $newCertArguments = @{
+        Type = "Custom"
+        Subject = "CN=Microsoft Corporation, O=Microsoft Corporation, L=Redmond, S=Washington, C=US"
+        KeyUsage = "DigitalSignature"
+        TextExtension = @("2.5.29.37={text}1.3.6.1.5.5.7.3.3", "2.5.29.19={text}")
+        CertStoreLocation = "Cert:\CurrentUser\My"
+      }
+      $cert = New-SelfSignedCertificate @newCertArguments
 
-  - task: DownloadSecureFile@1
-    name: HTTPSDevCert
-    displayName: 'Download Kestrel Certificate'
-    inputs:
-      secureFile: 'HTTPSDevCertV2.pfx'
+      $certPfxPath = Join-Path $(Agent.TempDirectory) TestSigningCert.pfx
+      $certCerPath = Join-Path ${{ parameters.signingCertOutDir }} TestSigningCert.cer
+      $certPassword = (New-Guid).ToString()
+      $certSecurePassword = ConvertTo-SecureString $certPassword -AsPlainText
 
-  - task: PowerShell@2
-    displayName: Install Root Certificate
-    inputs:
-      filePath: '${{ parameters.sourceDir }}\src\LocalhostWebServer\InstallDevCert.ps1'
-      arguments: '-pfxpath $(HTTPSDevCert.secureFilePath) -password microsoft'
+      Export-PfxCertificate -Cert $cert -FilePath $certPfxPath -Password $certSecurePassword
+      Export-Certificate -Cert $cert -FilePath $certCerPath
 
-  - task: PowerShell@2
+      Write-Host "##vso[task.setvariable variable=TestSigningCert.PfxPath;]$certPfxPath"
+      Write-Host "##vso[task.setvariable variable=TestSigningCert.CerPath;]$certCerPath"
+      Write-Host "##vso[task.setvariable variable=TestSigningCert.Password;]$certPassword"
+    displayName: Create test codesigning cert
+
+  - pwsh: |
+      $httpsCertPath = Join-Path $(Agent.TempDirectory) HttpsCert.pfx
+      $httpsCertPassword = (New-Guid).ToString()
+      dotnet dev-certs https --export-path $httpsCertPath --password $httpsCertPassword
+
+      $securePassword = ConvertTo-SecureString $httpsCertPassword -AsPlainText
+      Import-PfxCertificate -FilePath $httpsCertPath -Password $securePassword -CertStoreLocation Cert:\LocalMachine\Root
+
+      Write-Host "##vso[task.setvariable variable=HttpsCert.Path;]$httpsCertPath"
+      Write-Host "##vso[task.setvariable variable=HttpsCert.Password;]$httpsCertPassword"
+    displayName: Create and install localhost HTTPS cert
+
+  - pwsh: ${{ parameters.sourceDir }}\src\LocalhostWebServer\Run-LocalhostWebServer.ps1 -CertPath $(HttpsCert.Path) -CertPassword $(HttpsCert.Password) -OutCertFile $(Agent.TempDirectory)\servercert.cer ${{ parameters.localhostWebServerArgs }}
     displayName: Launch LocalhostWebServer
-    inputs:
-      filePath: '${{ parameters.sourceDir }}\src\LocalhostWebServer\Run-LocalhostWebServer.ps1'
-      arguments: '-CertPath $(HTTPSDevCert.secureFilePath) -CertPassword microsoft -OutCertFile $(Agent.TempDirectory)\servercert.cer ${{ parameters.localhostWebServerArgs }}'
 
   - task: PowerShell@2
     displayName: Setup Local PS Repository