From 9de4867ee5c02cf4035b34f09d39969bbe605c5a Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 19 Nov 2021 18:21:35 -0500 Subject: [PATCH] Update subscription configuration merge jobs to use secret handler --- .../TestResources/New-TestResources.ps1 | 40 +------- .../build-test-resource-config.yml | 45 ++------- eng/common/TestResources/config-functions.ps1 | 93 +++++++++++++++++++ 3 files changed, 102 insertions(+), 76 deletions(-) create mode 100644 eng/common/TestResources/config-functions.ps1 diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 8a0658072e6e..0fd2969b6997 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -79,6 +79,8 @@ param ( [switch] $OutFile ) +. $PSScriptRoot/config-functions.ps1 + # By default stop for any error. if (!$PSBoundParameters.ContainsKey('ErrorAction')) { $ErrorActionPreference = 'Stop' @@ -186,42 +188,6 @@ function BuildDeploymentOutputs([string]$serviceDirectoryPrefix, [object]$azCont return $deploymentOutputs } -function ShouldMarkValueAsSecret([string]$serviceDirectoryPrefix, [string]$key, [string]$value, [array]$allowedValues) { - $logOutputNonSecret = @( - # Environment Variables - "RESOURCEGROUP_NAME", - # Deployment Outputs - "CLIENT_ID", - "TENANT_ID", - "SUBSCRIPTION_ID", - "RESOURCE_GROUP", - "LOCATION", - "ENVIRONMENT", - "AZURE_AUTHORITY_HOST", - "RESOURCE_MANAGER_URL", - "SERVICE_MANAGEMENT_URL" - ) - - $suffix1 = $key -replace $serviceDirectoryPrefix, "" - $suffix2 = $key -replace "AZURE_", "" - $variants = @($key, $suffix1, $suffix2) - Write-Host "non secret" - Write-Host "$serviceDirectoryPrefix" - Write-Host "$key $suffix1 $suffix2" - Write-Host "$($allowedValues | ConvertTo-Json)" - if ($variants | Where-Object { $logOutputNonSecret -contains $_ }) { - Write-Host "false for variant" - return $false - } - - if ($allowedValues -contains $value) { - Write-Host "false for allowed" - return $false - } - - return $true -} - function SetDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [object]$templateFile) { $serviceDirectoryPrefix = $serviceName.ToUpperInvariant() + "_" $deploymentOutputs = BuildDeploymentOutputs $serviceDirectoryPrefix $azContext $deployment @@ -258,8 +224,8 @@ function SetDeploymentOutputs([string]$serviceName, [object]$azContext, [object] if (ShouldMarkValueAsSecret $serviceDirectoryPrefix $key $value $notSecretValues) { # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. - Write-Host "Setting variable as secret '$key': $value" Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$value" + Write-Host "Setting variable as secret '$key': $value" } else { Write-Host "Setting variable '$key': $value" $notSecretValues += $value diff --git a/eng/common/TestResources/build-test-resource-config.yml b/eng/common/TestResources/build-test-resource-config.yml index 9aeaca918df6..ca1b7ce7c21c 100644 --- a/eng/common/TestResources/build-test-resource-config.yml +++ b/eng/common/TestResources/build-test-resource-config.yml @@ -13,22 +13,8 @@ steps: ${{ parameters.SubscriptionConfiguration }} '@ | ConvertFrom-Json -AsHashtable - foreach($pair in $config.GetEnumerator()) { - if ($pair.Value -is [Hashtable]) { - foreach($nestedPair in $pair.Value.GetEnumerator()) { - # Mark values as secret so we don't print json blobs containing secrets in the logs. - # Prepend underscore to the variable name, so we can still access the variable names via environment - # variables if they get set subsequently. - Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)" - } - } else { - Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)" - } - } - - Write-Host ($config | ConvertTo-Json) - $serialized = $config | ConvertTo-Json -Compress - Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" + . ./eng/common/TestResources/config-functions.ps1 + SetSubscriptionConfiguration $config displayName: Initialize SubscriptionConfiguration variable - ${{ if parameters.SubscriptionConfigurations }}: @@ -39,33 +25,14 @@ steps: - ${{ each config in parameters.SubscriptionConfigurations }}: - pwsh: | - $config = @' + $configBase = @' $(SubscriptionConfiguration) '@ | ConvertFrom-Json -AsHashtable - $addToConfig = @' + $config = @' ${{ config }} '@ | ConvertFrom-Json -AsHashtable - foreach ($pair in $addToConfig.GetEnumerator()) { - if ($pair.Value -is [Hashtable]) { - if (!$config.ContainsKey($pair.Name)) { - $config[$pair.Name] = @{} - } - foreach($nestedPair in $pair.Value.GetEnumerator()) { - # Mark values as secret so we don't print json blobs containing secrets in the logs. - # Prepend underscore to the variable name, so we can still access the variable names via environment - # variables if they get set subsequently. - Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)" - $config[$pair.Name][$nestedPair.Name] = $nestedPair.Value - } - } else { - Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)" - $config[$pair.Name] = $pair.Value - } - } - - $serialized = $config | ConvertTo-Json -Compress - Write-Host ($config | ConvertTo-Json) - Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" + . ./eng/common/TestResources/config-functions.ps1 + UpdateSubscriptionConfiguration $configBase $config displayName: Merge Test Resource Configurations diff --git a/eng/common/TestResources/config-functions.ps1 b/eng/common/TestResources/config-functions.ps1 new file mode 100644 index 000000000000..dfaa17de5158 --- /dev/null +++ b/eng/common/TestResources/config-functions.ps1 @@ -0,0 +1,93 @@ +function ShouldMarkValueAsSecret([string]$serviceDirectoryPrefix, [string]$key, [string]$value, [array]$allowedValues = @()) +{ + $logOutputNonSecret = @( + # Environment Variables + "RESOURCEGROUP_NAME", + # Deployment Outputs + "CLIENT_ID", + "TENANT_ID", + "SUBSCRIPTION_ID", + "RESOURCE_GROUP", + "LOCATION", + "ENVIRONMENT", + "AZURE_AUTHORITY_HOST", + "RESOURCE_MANAGER_URL", + "SERVICE_MANAGEMENT_URL", + "ENDPOINT_SUFFIX", + # This is used in many places and is harder to extract from the base subscription config, so hardcode it for now. + "STORAGE_ENDPOINT_SUFFIX", + # Parameters + "Environment", + "SubscriptionId", + "TenantId", + "TestApplicationId", + "TestApplicationOid", + "ProvisionerApplicationId" + ) + + $suffix1 = $key -replace $serviceDirectoryPrefix, "" + $suffix2 = $key -replace "AZURE_", "" + $variants = @($key, $suffix1, $suffix2) + if ($variants | Where-Object { $logOutputNonSecret -contains $_ }) { + return $false + } + + if ($allowedValues -contains $value) { + return $false + } + + return $true +} + +function SetSubscriptionConfiguration([object]$subscriptionConfiguration) +{ + foreach($pair in $subscriptionConfiguration.GetEnumerator()) { + if ($pair.Value -is [Hashtable]) { + foreach($nestedPair in $pair.Value.GetEnumerator()) { + # Mark values as secret so we don't print json blobs containing secrets in the logs. + # Prepend underscore to the variable name, so we can still access the variable names via environment + # variables if they get set subsequently. + if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) { + Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)" + } + } + } else { + if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) { + Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)" + } + } + } + + Write-Host ($subscriptionConfiguration | ConvertTo-Json) + $serialized = $subscriptionConfiguration | ConvertTo-Json -Compress + Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" +} + +function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration) +{ + foreach ($pair in $subscriptionConfiguration.GetEnumerator()) { + if ($pair.Value -is [Hashtable]) { + if (!$subscriptionConfigurationBase.ContainsKey($pair.Name)) { + $subscriptionConfigurationBase[$pair.Name] = @{} + } + foreach($nestedPair in $pair.Value.GetEnumerator()) { + # Mark values as secret so we don't print json blobs containing secrets in the logs. + # Prepend underscore to the variable name, so we can still access the variable names via environment + # variables if they get set subsequently. + if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) { + Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)" + } + $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value + } + } else { + if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) { + Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)" + } + $subscriptionConfigurationBase[$pair.Name] = $pair.Value + } + } + + $serialized = $subscriptionConfigurationBase | ConvertTo-Json -Compress + Write-Host ($subscriptionConfigurationBase | ConvertTo-Json) + Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" +} \ No newline at end of file