Try to fix PGO artifact name. Update helpers from MUX. Use retries fu…

…nctions for url query and downloads as obviously MUX had a problem with it being sporatic too.
microsoft · May 9, 2021 · d868f9e · d868f9e
1 parent c95e0f8
commit d868f9e
Show file tree

Hide file tree

Showing 3 changed files with 188 additions and 45 deletions.
diff --git a/build/Helix/AzurePipelinesHelperScripts.ps1 b/build/Helix/AzurePipelinesHelperScripts.ps1
@@ -29,4 +29,147 @@ function GetQueryTestRunsUri
     $baseUri = GetAzureDevOpsBaseUri -CollectionUri $CollectionUri -TeamProject $TeamProject
     $queryUri = "$baseUri/_apis/test/runs?buildUri=$BuildUri$includeRunDetailsParameter&api-version=5.0"
     return $queryUri
+}
+
+function Get-HelixJobTypeFromTestRun
+{
+    Param ($testRun)
+
+    $testRunSingleResultUri = "$($testRun.url)/results?`$top=1&`$skip=0&api-version=5.1"
+    $singleTestResult = Invoke-RestMethod -Uri $testRunSingleResultUri -Method Get -Headers $azureDevOpsRestApiHeaders
+    $count = $singleTestResult.value.Length
+    if($count -eq 0)
+    {
+        # If the count is 0, then results have not yet been reported for this run.
+        # We only care about completed runs with results, so it is ok to just return 'UNKNOWN' for this run.
+        return "UNKNOWN"
+    }
+    else
+    {
+        $info = ConvertFrom-Json $singleTestResult.value.comment
+        $helixJobId = $info.HelixJobId
+        $job = Invoke-RestMethodWithRetries "https://helix.dot.net/api/2019-06-17/jobs/${helixJobId}?access_token=${HelixAccessToken}"
+        return $job.Type
+    }
+}
+
+function Append-HelixAccessTokenToUrl
+{
+    Param ([string]$url, [string]$token)
+    if($url.Contains("?"))
+    {
+        $url = "$($url)&access_token=$($token)"
+    }
+    else
+    {
+        $url = "$($url)?access_token=$($token)"
+    }
+    return $url
+}
+
+
+# The Helix Rest api is sometimes unreliable. So we call these apis with retry logic. 
+# Note: The Azure DevOps apis are stable and do not need to be called with this retry logic.
+$helixApiRetries = 0
+$helixApiRetriesMax = 10 
+
+function Download-StringWithRetries
+{
+    Param ([string]$fileName, [string]$url)
+
+    $result = ""
+    $done = $false
+
+    while(!($done))
+    {
+        try
+        {
+            Write-Host "Downloading $fileName"
+            $result = (New-Object System.Net.WebClient).DownloadString($url)
+            $done = $true
+        }
+        catch
+        {
+            Write-Host "Failed to download $fileName $($PSItem.Exception)"
+
+            $helixApiRetries = $helixApiRetries + 1
+            if($helixApiRetries -lt $helixApiRetriesMax)
+            {
+                Write-Host "Sleep and retry download of $fileName"
+                Start-Sleep 60
+            }
+            else
+            {
+                throw "Failed to download $fileName"
+            }
+        }
+    }
+
+    return $result
+}
+
+function Invoke-RestMethodWithRetries
+{
+    Param ([string]$url,$Headers)
+
+    $result = @()
+    $done = $false
+
+    while(!($done))
+    {
+        try
+        {
+            $result = Invoke-RestMethod -Uri $url -Method Get -Headers $Headers
+            $done = $true
+        }
+        catch
+        {
+            Write-Host "Failed to invoke Rest method $($PSItem.Exception)"
+
+            $helixApiRetries = $helixApiRetries + 1
+            if($helixApiRetries -lt $helixApiRetriesMax)
+            {
+                Write-Host "Sleep and retry invoke"
+                Start-Sleep 60
+            }
+            else
+            {
+                throw "Failed to invoke Rest method"
+            }
+        }
+    }
+
+    return $result
+}
+
+function Download-FileWithRetries
+{
+    Param ([string]$fileurl, [string]$destination)
+
+    $done = $false
+
+    while(!($done))
+    {
+        try
+        {
+            Write-Host "Downloading $destination"
+            $webClient.DownloadFile($fileurl, $destination)
+            $done = $true
+        }
+        catch
+        {
+            Write-Host "Failed to download $destination $($PSItem.Exception)"
+
+            $helixApiRetries = $helixApiRetries + 1
+            if($helixApiRetries -lt $helixApiRetriesMax)
+            {
+                Write-Host "Sleep and retry download of $destination"
+                Start-Sleep 60
+            }
+            else
+            {
+                throw "Failed to download $destination"
+            }
+        }
+    }
 }
diff --git a/build/Helix/ProcessHelixFiles.ps1 b/build/Helix/ProcessHelixFiles.ps1
@@ -40,72 +40,71 @@ $azureDevOpsRestApiHeaders = @{
 $queryUri = GetQueryTestRunsUri -CollectionUri $CollectionUri -TeamProject $TeamProject -BuildUri $BuildUri -IncludeRunDetails
 Write-Host "queryUri = $queryUri"
 
-$testRuns = Invoke-RestMethod -Uri $queryUri -Method Get -Headers $azureDevOpsRestApiHeaders
+$testRuns = Invoke-RestMethodWithRetries $queryUri -Headers $azureDevOpsRestApiHeaders
 $webClient = New-Object System.Net.WebClient
 [System.Collections.Generic.List[string]]$workItems = @()
 
 foreach ($testRun in $testRuns.value)
 {
     Write-Host "testRunUri = $testRun.url"
-    $testResults = Invoke-RestMethod -Uri "$($testRun.url)/results?api-version=5.0" -Method Get -Headers $azureDevOpsRestApiHeaders
+    $testResults = Invoke-RestMethodWithRetries "$($testRun.url)/results?api-version=5.0" -Headers $azureDevOpsRestApiHeaders
     $isTestRunNameShown = $false
 
     foreach ($testResult in $testResults.value)
     {
-            $info = ConvertFrom-Json $testResult.comment
-            $helixJobId = $info.HelixJobId
-            $helixWorkItemName = $info.HelixWorkItemName
+        $info = ConvertFrom-Json $testResult.comment
+        $helixJobId = $info.HelixJobId
+        $helixWorkItemName = $info.HelixWorkItemName
 
-            $workItem = "$helixJobId-$helixWorkItemName"
+        $workItem = "$helixJobId-$helixWorkItemName"
 
-            Write-Host "Helix Work Item = $workItem"
+        Write-Host "Helix Work Item = $workItem"
 
-            if (-not $workItems.Contains($workItem))
+        if (-not $workItems.Contains($workItem))
+        {
+            $workItems.Add($workItem)
+            $filesQueryUri = "https://helix.dot.net/api/2019-06-17/jobs/$helixJobId/workitems/$helixWorkItemName/files$accessTokenParam"
+            $files = Invoke-RestMethodWithRetries $filesQueryUri
+
+            $screenShots = $files | where { $_.Name.EndsWith(".jpg") }
+            $dumps = $files | where { $_.Name.EndsWith(".dmp") }
+            $pgcFiles = $files | where { $_.Name.EndsWith(".pgc") }
+            if ($screenShots.Count + $dumps.Count + $pgcFiles.Count -gt 0)
             {
-                $workItems.Add($workItem)
-                $filesQueryUri = "https://helix.dot.net/api/2019-06-17/jobs/$helixJobId/workitems/$helixWorkItemName/files$accessTokenParam"
-                $files = Invoke-RestMethod -Uri $filesQueryUri -Method Get
-
-                $screenShots = $files | where { $_.Name.EndsWith(".jpg") }
-                $dumps = $files | where { $_.Name.EndsWith(".dmp") }
-                $pgcFiles = $files | where { $_.Name.EndsWith(".pgc") }
-                if ($screenShots.Count + $dumps.Count + $pgcFiles.Count -gt 0)
+                if(-Not $isTestRunNameShown)
                 {
-                    if(-Not $isTestRunNameShown)
-                    {
-                        Out-File -FilePath $helixLinkFile -Append -InputObject "<h2>$($testRun.name)</h2>"
-                        $isTestRunNameShown = $true
-                    }
-                    Out-File -FilePath $helixLinkFile -Append -InputObject "<h3>$helixWorkItemName</h3>"
-                    Generate-File-Links $screenShots "Screenshots"
-                    Generate-File-Links $dumps "CrashDumps"
-                    Generate-File-Links $pgcFiles "PGC files"
-                    $misc = $files | where { ($screenShots -NotContains $_) -And ($dumps -NotContains $_) -And ($visualTreeVerificationFiles -NotContains $_) -And ($pgcFiles -NotContains $_) }
-                    Generate-File-Links $misc "Misc"
-
-                    foreach($pgcFile in $pgcFiles)
-                    {
-                        $flavorPath = $testResult.automatedTestName.Split('.')[0]
-                        $archPath = $testResult.automatedTestName.Split('.')[1]
-                        $fileName = $pgcFile.Name
-                        $fullPath = "$OutputFolder\PGO\$flavorPath\$archPath"
-                        $destination = "$fullPath\$fileName"
+                    Out-File -FilePath $helixLinkFile -Append -InputObject "<h2>$($testRun.name)</h2>"
+                    $isTestRunNameShown = $true
+                }
+                Out-File -FilePath $helixLinkFile -Append -InputObject "<h3>$helixWorkItemName</h3>"
+                Generate-File-Links $screenShots "Screenshots"
+                Generate-File-Links $dumps "CrashDumps"
+                Generate-File-Links $pgcFiles "PGC files"
+                $misc = $files | where { ($screenShots -NotContains $_) -And ($dumps -NotContains $_) -And ($visualTreeVerificationFiles -NotContains $_) -And ($pgcFiles -NotContains $_) }
+                Generate-File-Links $misc "Misc"
+
+                foreach($pgcFile in $pgcFiles)
+                {
+                    $flavorPath = $testResult.automatedTestName.Split('.')[0]
+                    $archPath = $testResult.automatedTestName.Split('.')[1]
+                    $fileName = $pgcFile.Name
+                    $fullPath = "$OutputFolder\PGO\$flavorPath\$archPath"
+                    $destination = "$fullPath\$fileName"
 
-                        Write-Host "Copying $($pgcFile.Name) to $destination"
+                    Write-Host "Copying $($pgcFile.Name) to $destination"
 
-                        if (-Not (Test-Path $fullPath))
-                        {
-                            New-Item $fullPath -ItemType Directory
-                        }
+                    if (-Not (Test-Path $fullPath))
+                    {
+                        New-Item $fullPath -ItemType Directory
+                    }
 
-                        $link = $pgcFile.Link
+                    $link = $pgcFile.Link
 
-                        Write-Host "Downloading $link to $destination"
+                    Write-Host "Downloading $link to $destination"
 
-                        $webClient.DownloadFile($link, $destination)
-                    }
+                    Download-FileWithRetries $link $destination
                 }
             }
-
+        }
     }
 }
diff --git a/build/pipelines/templates/build-console-pgo.yml b/build/pipelines/templates/build-console-pgo.yml
@@ -35,6 +35,7 @@ jobs:
 - template: helix-processtestresults-job.yml
   parameters:
     name: 'ProcessTestResults'
+    pgoArtifact: 'PGO'
     dependsOn:
     - RunTestsInHelix
     condition: succeededOrFailed()