diff --git a/build/Helix/AzurePipelinesHelperScripts.ps1 b/build/Helix/AzurePipelinesHelperScripts.ps1 index 5574a9b532b..8934a8548d1 100644 --- a/build/Helix/AzurePipelinesHelperScripts.ps1 +++ b/build/Helix/AzurePipelinesHelperScripts.ps1 @@ -29,4 +29,147 @@ function GetQueryTestRunsUri $baseUri = GetAzureDevOpsBaseUri -CollectionUri $CollectionUri -TeamProject $TeamProject $queryUri = "$baseUri/_apis/test/runs?buildUri=$BuildUri$includeRunDetailsParameter&api-version=5.0" return $queryUri +} + +function Get-HelixJobTypeFromTestRun +{ + Param ($testRun) + + $testRunSingleResultUri = "$($testRun.url)/results?`$top=1&`$skip=0&api-version=5.1" + $singleTestResult = Invoke-RestMethod -Uri $testRunSingleResultUri -Method Get -Headers $azureDevOpsRestApiHeaders + $count = $singleTestResult.value.Length + if($count -eq 0) + { + # If the count is 0, then results have not yet been reported for this run. + # We only care about completed runs with results, so it is ok to just return 'UNKNOWN' for this run. + return "UNKNOWN" + } + else + { + $info = ConvertFrom-Json $singleTestResult.value.comment + $helixJobId = $info.HelixJobId + $job = Invoke-RestMethodWithRetries "https://helix.dot.net/api/2019-06-17/jobs/${helixJobId}?access_token=${HelixAccessToken}" + return $job.Type + } +} + +function Append-HelixAccessTokenToUrl +{ + Param ([string]$url, [string]$token) + if($url.Contains("?")) + { + $url = "$($url)&access_token=$($token)" + } + else + { + $url = "$($url)?access_token=$($token)" + } + return $url +} + + +# The Helix Rest api is sometimes unreliable. So we call these apis with retry logic. +# Note: The Azure DevOps apis are stable and do not need to be called with this retry logic. +$helixApiRetries = 0 +$helixApiRetriesMax = 10 + +function Download-StringWithRetries +{ + Param ([string]$fileName, [string]$url) + + $result = "" + $done = $false + + while(!($done)) + { + try + { + Write-Host "Downloading $fileName" + $result = (New-Object System.Net.WebClient).DownloadString($url) + $done = $true + } + catch + { + Write-Host "Failed to download $fileName $($PSItem.Exception)" + + $helixApiRetries = $helixApiRetries + 1 + if($helixApiRetries -lt $helixApiRetriesMax) + { + Write-Host "Sleep and retry download of $fileName" + Start-Sleep 60 + } + else + { + throw "Failed to download $fileName" + } + } + } + + return $result +} + +function Invoke-RestMethodWithRetries +{ + Param ([string]$url,$Headers) + + $result = @() + $done = $false + + while(!($done)) + { + try + { + $result = Invoke-RestMethod -Uri $url -Method Get -Headers $Headers + $done = $true + } + catch + { + Write-Host "Failed to invoke Rest method $($PSItem.Exception)" + + $helixApiRetries = $helixApiRetries + 1 + if($helixApiRetries -lt $helixApiRetriesMax) + { + Write-Host "Sleep and retry invoke" + Start-Sleep 60 + } + else + { + throw "Failed to invoke Rest method" + } + } + } + + return $result +} + +function Download-FileWithRetries +{ + Param ([string]$fileurl, [string]$destination) + + $done = $false + + while(!($done)) + { + try + { + Write-Host "Downloading $destination" + $webClient.DownloadFile($fileurl, $destination) + $done = $true + } + catch + { + Write-Host "Failed to download $destination $($PSItem.Exception)" + + $helixApiRetries = $helixApiRetries + 1 + if($helixApiRetries -lt $helixApiRetriesMax) + { + Write-Host "Sleep and retry download of $destination" + Start-Sleep 60 + } + else + { + throw "Failed to download $destination" + } + } + } } \ No newline at end of file diff --git a/build/Helix/ProcessHelixFiles.ps1 b/build/Helix/ProcessHelixFiles.ps1 index 81f47deba34..f74187219c1 100644 --- a/build/Helix/ProcessHelixFiles.ps1 +++ b/build/Helix/ProcessHelixFiles.ps1 @@ -40,72 +40,71 @@ $azureDevOpsRestApiHeaders = @{ $queryUri = GetQueryTestRunsUri -CollectionUri $CollectionUri -TeamProject $TeamProject -BuildUri $BuildUri -IncludeRunDetails Write-Host "queryUri = $queryUri" -$testRuns = Invoke-RestMethod -Uri $queryUri -Method Get -Headers $azureDevOpsRestApiHeaders +$testRuns = Invoke-RestMethodWithRetries $queryUri -Headers $azureDevOpsRestApiHeaders $webClient = New-Object System.Net.WebClient [System.Collections.Generic.List[string]]$workItems = @() foreach ($testRun in $testRuns.value) { Write-Host "testRunUri = $testRun.url" - $testResults = Invoke-RestMethod -Uri "$($testRun.url)/results?api-version=5.0" -Method Get -Headers $azureDevOpsRestApiHeaders + $testResults = Invoke-RestMethodWithRetries "$($testRun.url)/results?api-version=5.0" -Headers $azureDevOpsRestApiHeaders $isTestRunNameShown = $false foreach ($testResult in $testResults.value) { - $info = ConvertFrom-Json $testResult.comment - $helixJobId = $info.HelixJobId - $helixWorkItemName = $info.HelixWorkItemName + $info = ConvertFrom-Json $testResult.comment + $helixJobId = $info.HelixJobId + $helixWorkItemName = $info.HelixWorkItemName - $workItem = "$helixJobId-$helixWorkItemName" + $workItem = "$helixJobId-$helixWorkItemName" - Write-Host "Helix Work Item = $workItem" + Write-Host "Helix Work Item = $workItem" - if (-not $workItems.Contains($workItem)) + if (-not $workItems.Contains($workItem)) + { + $workItems.Add($workItem) + $filesQueryUri = "https://helix.dot.net/api/2019-06-17/jobs/$helixJobId/workitems/$helixWorkItemName/files$accessTokenParam" + $files = Invoke-RestMethodWithRetries $filesQueryUri + + $screenShots = $files | where { $_.Name.EndsWith(".jpg") } + $dumps = $files | where { $_.Name.EndsWith(".dmp") } + $pgcFiles = $files | where { $_.Name.EndsWith(".pgc") } + if ($screenShots.Count + $dumps.Count + $pgcFiles.Count -gt 0) { - $workItems.Add($workItem) - $filesQueryUri = "https://helix.dot.net/api/2019-06-17/jobs/$helixJobId/workitems/$helixWorkItemName/files$accessTokenParam" - $files = Invoke-RestMethod -Uri $filesQueryUri -Method Get - - $screenShots = $files | where { $_.Name.EndsWith(".jpg") } - $dumps = $files | where { $_.Name.EndsWith(".dmp") } - $pgcFiles = $files | where { $_.Name.EndsWith(".pgc") } - if ($screenShots.Count + $dumps.Count + $pgcFiles.Count -gt 0) + if(-Not $isTestRunNameShown) { - if(-Not $isTestRunNameShown) - { - Out-File -FilePath $helixLinkFile -Append -InputObject "

$($testRun.name)

" - $isTestRunNameShown = $true - } - Out-File -FilePath $helixLinkFile -Append -InputObject "

$helixWorkItemName

" - Generate-File-Links $screenShots "Screenshots" - Generate-File-Links $dumps "CrashDumps" - Generate-File-Links $pgcFiles "PGC files" - $misc = $files | where { ($screenShots -NotContains $_) -And ($dumps -NotContains $_) -And ($visualTreeVerificationFiles -NotContains $_) -And ($pgcFiles -NotContains $_) } - Generate-File-Links $misc "Misc" - - foreach($pgcFile in $pgcFiles) - { - $flavorPath = $testResult.automatedTestName.Split('.')[0] - $archPath = $testResult.automatedTestName.Split('.')[1] - $fileName = $pgcFile.Name - $fullPath = "$OutputFolder\PGO\$flavorPath\$archPath" - $destination = "$fullPath\$fileName" + Out-File -FilePath $helixLinkFile -Append -InputObject "

$($testRun.name)

" + $isTestRunNameShown = $true + } + Out-File -FilePath $helixLinkFile -Append -InputObject "

$helixWorkItemName

" + Generate-File-Links $screenShots "Screenshots" + Generate-File-Links $dumps "CrashDumps" + Generate-File-Links $pgcFiles "PGC files" + $misc = $files | where { ($screenShots -NotContains $_) -And ($dumps -NotContains $_) -And ($visualTreeVerificationFiles -NotContains $_) -And ($pgcFiles -NotContains $_) } + Generate-File-Links $misc "Misc" + + foreach($pgcFile in $pgcFiles) + { + $flavorPath = $testResult.automatedTestName.Split('.')[0] + $archPath = $testResult.automatedTestName.Split('.')[1] + $fileName = $pgcFile.Name + $fullPath = "$OutputFolder\PGO\$flavorPath\$archPath" + $destination = "$fullPath\$fileName" - Write-Host "Copying $($pgcFile.Name) to $destination" + Write-Host "Copying $($pgcFile.Name) to $destination" - if (-Not (Test-Path $fullPath)) - { - New-Item $fullPath -ItemType Directory - } + if (-Not (Test-Path $fullPath)) + { + New-Item $fullPath -ItemType Directory + } - $link = $pgcFile.Link + $link = $pgcFile.Link - Write-Host "Downloading $link to $destination" + Write-Host "Downloading $link to $destination" - $webClient.DownloadFile($link, $destination) - } + Download-FileWithRetries $link $destination } } - + } } } diff --git a/build/pipelines/templates/build-console-pgo.yml b/build/pipelines/templates/build-console-pgo.yml index 49dcd3a6c9d..201bb75012a 100644 --- a/build/pipelines/templates/build-console-pgo.yml +++ b/build/pipelines/templates/build-console-pgo.yml @@ -35,6 +35,7 @@ jobs: - template: helix-processtestresults-job.yml parameters: name: 'ProcessTestResults' + pgoArtifact: 'PGO' dependsOn: - RunTestsInHelix condition: succeededOrFailed()