diff --git a/Directory.Build.props b/Directory.Build.props index 90c9e5082e..8e038fae3a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -39,6 +39,7 @@ amd64 $(NuGetPackageRoot)microsoft.developercontrolplane.$(BuildOs)-$(BuildArch)/$(MicrosoftDeveloperControlPlanedarwinamd64PackageVersion)/tools/ + $([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix', 'tests')) $(MSBuildThisFileDirectory)/artifacts/bin/Aspire.Dashboard/$(Configuration)/net8.0/ diff --git a/Directory.Build.targets b/Directory.Build.targets index e2c9aa3192..cbd14c71d4 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -22,5 +22,5 @@ - + diff --git a/eng/pipelines/azure-pipelines.yml b/eng/pipelines/azure-pipelines.yml index e2609376bd..3cfa1373d6 100644 --- a/eng/pipelines/azure-pipelines.yml +++ b/eng/pipelines/azure-pipelines.yml @@ -98,7 +98,7 @@ stages: jobs: - job: windows - timeoutInMinutes: 30 + timeoutInMinutes: 45 pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: @@ -131,7 +131,7 @@ stages: - ${{ if eq(variables._RunAsPublic, True) }}: - job: linux - timeoutInMinutes: 30 + timeoutInMinutes: 45 pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: @@ -167,7 +167,7 @@ stages: # ---------------------------------------------------------------- - stage: codecoverage displayName: CodeCoverage - dependsOn: + dependsOn: - build condition: succeeded('build') variables: diff --git a/eng/pipelines/templates/BuildAndTest.yml b/eng/pipelines/templates/BuildAndTest.yml index f51392a33f..3f5ecb325b 100644 --- a/eng/pipelines/templates/BuildAndTest.yml +++ b/eng/pipelines/templates/BuildAndTest.yml @@ -22,6 +22,7 @@ steps: - script: ${{ parameters.buildScript }} -restore -build -configuration ${{ parameters.buildConfig }} + /p:ArchiveTests=true /bl:${{ parameters.repoLogPath }}/build.binlog $(_OfficialBuildIdArgs) displayName: Build @@ -29,9 +30,37 @@ steps: - ${{ if ne(parameters.skipTests, 'true') }}: - script: ${{ parameters.dotnetScript }} dotnet-coverage collect --settings $(Build.SourcesDirectory)/eng/CodeCoverage.config + --output ${{ parameters.repoTestResultsPath }}/NonHelixTests.cobertura.xml + "${{ parameters.buildScript }} -testnobuild -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)" + displayName: Run non-helix tests + + - template: /eng/pipelines/templates/send-to-helix.yml + parameters: + HelixProjectPath: '$(Build.SourcesDirectory)/tests/send-to-helix.proj' + HelixProjectArguments: /p:RunWithCodeCoverage=true /p:RepoTestResultsPath=${{ parameters.repoTestResultsPath }} + + ${{ if eq(parameters.isWindows, 'true') }}: + HelixTargetQueues: Windows.11.Amd64.Client.Open + ${{ if ne(parameters.isWindows, 'true') }}: + HelixTargetQueues: Ubuntu.2204.Amd64.Open + + IsWindows: ${{ parameters.isWindows }} + Creator: $(Build.DefinitionName) + HelixBuild: $(Build.BuildNumber) + HelixAccessToken: $(HelixApiAccessToken) + + - task: CopyFiles@2 + inputs: + Contents: '${{ parameters.repoTestResultsPath }}/**/*.cobertura.xml' + TargetFolder: '${{ parameters.repoTestResultsPath }}/collected' + flattenFolders: true + displayName: Gather coverage results (cobertura.xml) + + - script: $(Build.SourcesDirectory)/.dotnet/dotnet dotnet-coverage merge + ${{ parameters.repoTestResultsPath }}/collected/*.cobertura.xml + --output-format cobertura --output ${{ parameters.repoTestResultsPath }}/$(Agent.Os)_$(Agent.JobName).cobertura.xml - "${{ parameters.buildScript }} -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)" - displayName: Run tests + displayName: Merge code coverage reports - task: PublishBuildArtifacts@1 inputs: diff --git a/eng/pipelines/templates/send-to-helix.yml b/eng/pipelines/templates/send-to-helix.yml new file mode 100644 index 0000000000..5dcd7ea792 --- /dev/null +++ b/eng/pipelines/templates/send-to-helix.yml @@ -0,0 +1,95 @@ +# This is a modified copy of eng/common/templates/steps/send-to-helix.yml +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixProjectPath: '' # required -- path to the project file to build + HelixProjectArguments: '' # optional -- arguments passed to the build command + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run helix tests' # optional -- rename the beginning of the displayName of the steps in AzDO + IsWindows: false # optional + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - ${{ if and(parameters.condition, eq(parameters.IsWindows, 'true')) }}: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 ${{ parameters.HelixProjectPath }} ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} + - ${{ if and(parameters.condition, ne(parameters.IsWindows, 'true')) }}: + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh ${{ parameters.HelixProjectPath }} ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/testing/.runsettings b/eng/testing/.runsettings new file mode 100644 index 0000000000..578840a73a --- /dev/null +++ b/eng/testing/.runsettings @@ -0,0 +1,23 @@ + + + + + 300000 + + category!=failing + + + + + + TestResults.trx + + + + + normal + + + + + diff --git a/tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj b/tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj index ba590719f6..3302f81107 100644 --- a/tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj +++ b/tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj @@ -2,6 +2,7 @@ $(NetCurrent) + true diff --git a/tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj b/tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj index 2be42ac724..37c80171b8 100644 --- a/tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj +++ b/tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj @@ -2,6 +2,9 @@ $(NetCurrent) + + + true diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets new file mode 100644 index 0000000000..32f2946920 --- /dev/null +++ b/tests/Directory.Build.targets @@ -0,0 +1,22 @@ + + + + true + + false + true + + + + + + + + + + + + + diff --git a/tests/send-to-helix.proj b/tests/send-to-helix.proj new file mode 100644 index 0000000000..b7b8c5afc0 --- /dev/null +++ b/tests/send-to-helix.proj @@ -0,0 +1,101 @@ + + + + msbuild + + <_workItemTimeout>00:20:00 + $(TestArchiveTestsDir)**/*.zip + + true + sdk + + $([System.IO.File]::ReadAllText('$(RepoRoot)global.json')) + $([System.Text.RegularExpressions.Regex]::Match($(GlobalJsonContent), '(%3F<="dotnet": ").*(%3F=")')) + + + + <_SupportDataStagingDir>$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix', 'support-data')) + <_HelixLogsPath Condition="'$(OS)' != 'Windows_NT'">$HELIX_WORKITEM_UPLOAD_ROOT/logs + <_HelixLogsPath Condition="'$(OS)' == 'Windows_NT'">%HELIX_WORKITEM_UPLOAD_ROOT%/logs + + <_HelixCorrelationPayloadEnvVar Condition="'$(OS)' != 'Windows_NT'">$HELIX_CORRELATION_PAYLOAD + <_HelixCorrelationPayloadEnvVar Condition="'$(OS)' == 'Windows_NT'">%HELIX_CORRELATION_PAYLOAD% + + <_TestNameEnvVar Condition="'$(OS)' != 'Windows_NT'">$TEST_NAME + <_TestNameEnvVar Condition="'$(OS)' == 'Windows_NT'">%TEST_NAME% + + + + <_TestCoverageCommand Include="$(_HelixCorrelationPayloadEnvVar)/dotnet-coverage/dotnet-coverage collect" /> + <_TestCoverageCommand Include="--settings $(_HelixCorrelationPayloadEnvVar)/support-data/CodeCoverage.config" /> + <_TestCoverageCommand Include="--output $(_HelixLogsPath)/$(_TestNameEnvVar).cobertura.xml" /> + + <_TestRunCommandArguments Include="dotnet test" /> + <_TestRunCommandArguments Include="-s .runsettings" /> + <_TestRunCommandArguments Include="$(_TestNameEnvVar).dll" /> + <_TestRunCommandArguments Include="--ResultsDirectory:$(_HelixLogsPath)" /> + + + + + + + $(RepoTestResultsPath) + + <_TestRunCommand Condition="'$(RunWithCodeCoverage)' == 'true'">@(_TestCoverageCommand, ' ') "@(_TestRunCommandArguments, ' ')" + <_TestRunCommand Condition="'$(RunWithCodeCoverage)' != 'true'">@(_TestRunCommandArguments, ' ') + + + + + + <_DefaultWorkItems Include="$(WorkItemArchiveWildCard)" /> + + + %(Identity) + set "TEST_NAME=%(FileName)" + export "TEST_NAME=%(FileName)" + $(_TestRunCommand) + $(_workItemTimeout) + + + logs/%(FileName).cobertura.xml + + + + + + + + + + + + <_DotNetToolJsonPath>$(RepoRoot).config/dotnet-tools.json + <_DotNetToolJsonContent>$([System.IO.File]::ReadAllText($(_DotNetToolJsonPath))) + <_DotNetCoverageVersionRegex>"dotnet-coverage":\s*{\s*"version":\s*"([^"]*)" + <_DotNetCoverageToolVersion>$([System.Text.RegularExpressions.Regex]::Match($(_DotNetToolJsonContent), '$(_DotNetCoverageVersionRegex)').Groups[1].Value) + + + + + + + + + + + + + + + + + + diff --git a/tests/testproject/Directory.Build.props b/tests/testproject/Directory.Build.props new file mode 100644 index 0000000000..4922346540 --- /dev/null +++ b/tests/testproject/Directory.Build.props @@ -0,0 +1,7 @@ + + + true + + + +