diff --git a/common/perf-test-core/README.md b/common/perf-test-core/README.md index 66ac6bcfbd9f0..76cad2ccc4a4d 100644 --- a/common/perf-test-core/README.md +++ b/common/perf-test-core/README.md @@ -32,8 +32,7 @@ helps developers create their performance tests for their APIs. ## Contributing -If you would like to become an active contributor to this project please follow the instructions provided in [Microsoft -Azure Projects Contribution Guidelines](https://azure.github.io/guidelines.html). +For details on contributing to this repository, see the [contributing guide](https://github.com/Azure/azure-sdk-for-java/blob/master/CONTRIBUTING.md). 1. Fork it 1. Create your feature branch (`git checkout -b my-new-feature`) diff --git a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml index 42252c13d1b07..e4484116f26ec 100755 --- a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml +++ b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml @@ -348,6 +348,14 @@ + + + + + + + + diff --git a/eng/code-quality-reports/src/main/resources/revapi/revapi.json b/eng/code-quality-reports/src/main/resources/revapi/revapi.json index 26e55d5d88a7c..fa2c1d3d77397 100644 --- a/eng/code-quality-reports/src/main/resources/revapi/revapi.json +++ b/eng/code-quality-reports/src/main/resources/revapi/revapi.json @@ -269,6 +269,92 @@ "code": "java.annotation.added", "new": "class com.azure.storage.blob.models.PageList", "justification": "Annotation required to resolve deserialization bug." + },{ + "code": "java.method.returnTypeChanged", + "old": "method com.azure.core.util.IterableStream com.azure.core.amqp.models.AmqpDataBody::getData()", + "new": "method java.util.List com.azure.core.amqp.models.AmqpDataBody::getData()", + "justification": "Updated " + }, + { + "code": "java.class.removed", + "old": "class com.azure.core.amqp.models.AmqpDataBody", + "justification": "Renamed as AmqpMessageBody." + }, + { + "code": "java.class.kindChanged", + "old": "interface com.azure.core.amqp.models.AmqpMessageBody", + "new": "class com.azure.core.amqp.models.AmqpMessageBody", + "justification": "AmqpMessageBody is class representing all the new AMQP data types." + }, + { + "code": "java.class.nowFinal", + "old": "interface com.azure.core.amqp.models.AmqpMessageBody", + "new": "class com.azure.core.amqp.models.AmqpMessageBody", + "justification": "Made it final." + }, + { + "code": "java.class.removed", + "old": "enum com.azure.core.amqp.models.AmqpBodyType", + "justification": "Because It is renamed to AmqpMessageBodyType" + }, + { + "code": "java.method.returnTypeChanged", + "old": "method com.azure.core.amqp.models.AmqpBodyType com.azure.core.amqp.models.AmqpMessageBody::getBodyType()", + "new": "method com.azure.core.amqp.models.AmqpMessageBodyType com.azure.core.amqp.models.AmqpMessageBody::getBodyType()", + "justification": "Renamed to match AmqpMessage prefix." + }, + { + "code": "java.method.returnTypeChanged", + "old": "method java.lang.String com.azure.core.amqp.models.AmqpMessageProperties::getCorrelationId()", + "new": "method com.azure.core.amqp.models.AmqpMessageId com.azure.core.amqp.models.AmqpMessageProperties::getCorrelationId()", + "justification": "New return type AmqpMessageId." + }, + { + "code": "java.method.returnTypeChanged", + "old": "method java.lang.String com.azure.core.amqp.models.AmqpMessageProperties::getMessageId()", + "new": "method com.azure.core.amqp.models.AmqpMessageId com.azure.core.amqp.models.AmqpMessageProperties::getMessageId()", + "justification": "New return type." + }, + { + "code": "java.method.returnTypeChanged", + "old": "method java.lang.String com.azure.core.amqp.models.AmqpMessageProperties::getReplyTo()", + "new": "method com.azure.core.amqp.models.AmqpAddress com.azure.core.amqp.models.AmqpMessageProperties::getReplyTo()", + "justification": "New return type." + }, + { + "code": "java.method.returnTypeChanged", + "old": "method java.lang.String com.azure.core.amqp.models.AmqpMessageProperties::getTo()", + "new": "method com.azure.core.amqp.models.AmqpAddress com.azure.core.amqp.models.AmqpMessageProperties::getTo()", + "justification": "New return type." + }, + { + "code": "java.method.parameterTypeChanged", + "old": "parameter com.azure.core.amqp.models.AmqpMessageProperties com.azure.core.amqp.models.AmqpMessageProperties::setCorrelationId(===java.lang.String===)", + "new": "parameter com.azure.core.amqp.models.AmqpMessageProperties com.azure.core.amqp.models.AmqpMessageProperties::setCorrelationId(===com.azure.core.amqp.models.AmqpMessageId===)", + "justification": "Introduced new type AmqpMessageId." + }, + { + "code": "java.method.parameterTypeChanged", + "old": "parameter com.azure.core.amqp.models.AmqpMessageProperties com.azure.core.amqp.models.AmqpMessageProperties::setMessageId(===java.lang.String===)", + "new": "parameter com.azure.core.amqp.models.AmqpMessageProperties com.azure.core.amqp.models.AmqpMessageProperties::setMessageId(===com.azure.core.amqp.models.AmqpMessageId===)", + "justification":"Introduced new type AmqpMessageId." + }, + { + "code": "java.method.parameterTypeChanged", + "old": "parameter com.azure.core.amqp.models.AmqpMessageProperties com.azure.core.amqp.models.AmqpMessageProperties::setReplyTo(===java.lang.String===)", + "new": "parameter com.azure.core.amqp.models.AmqpMessageProperties com.azure.core.amqp.models.AmqpMessageProperties::setReplyTo(===com.azure.core.amqp.models.AmqpAddress===)", + "justification": "Introduced new type AmqpAddress." + }, + { + "code": "java.method.parameterTypeChanged", + "old": "parameter com.azure.core.amqp.models.AmqpMessageProperties com.azure.core.amqp.models.AmqpMessageProperties::setTo(===java.lang.String===)", + "new": "parameter com.azure.core.amqp.models.AmqpMessageProperties com.azure.core.amqp.models.AmqpMessageProperties::setTo(===com.azure.core.amqp.models.AmqpAddress===)", + "justification": "Introduced new type AmqpAddress." + }, + { + "code": "java.method.removed", + "old": "method void com.azure.core.amqp.models.AmqpAnnotatedMessage::(com.azure.core.amqp.models.AmqpAnnotatedMessage)", + "justification": "Removed copy constructor, It is not required for Service bus message." } ] } diff --git a/eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml b/eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml index d8292539a9b37..27e67e802b68f 100755 --- a/eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml +++ b/eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml @@ -1351,6 +1351,13 @@ + + + + + + + @@ -2297,7 +2304,7 @@ - + @@ -2390,7 +2397,7 @@ - + @@ -2421,4 +2428,21 @@ + + + + + + + + + + + + + + + diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 4c03816c12843..25f0b18c31cf6 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -125,356 +125,387 @@ $exitActions = @({ } }) -trap { - # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content. - $exitActions.Invoke() +New-Variable -Name 'initialContext' -Value (Get-AzContext) -Option Constant +if ($initialContext) { + $exitActions += { + Write-Verbose "Restoring initial context: $($initialContext.Account)" + $null = $initialContext | Select-AzContext + } } -# Enumerate test resources to deploy. Fail if none found. -$repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path -$root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path -$templateFileName = 'test-resources.json' -$templateFiles = @() -# Azure SDK Developer Playground -$defaultSubscription = "faa080af-c1d8-40ad-9cce-e1a450ca5b57" +# try..finally will also trap Ctrl+C. +try { -Write-Verbose "Checking for '$templateFileName' files under '$root'" -Get-ChildItem -Path $root -Filter $templateFileName -Recurse | ForEach-Object { - $templateFile = $_.FullName + # Enumerate test resources to deploy. Fail if none found. + $repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path + $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path + $templateFileName = 'test-resources.json' + $templateFiles = @() + # Azure SDK Developer Playground + $defaultSubscription = "faa080af-c1d8-40ad-9cce-e1a450ca5b57" - Write-Verbose "Found template '$templateFile'" - $templateFiles += $templateFile -} + Write-Verbose "Checking for '$templateFileName' files under '$root'" + Get-ChildItem -Path $root -Filter $templateFileName -Recurse | ForEach-Object { + $templateFile = $_.FullName -if (!$templateFiles) { - Write-Warning -Message "No template files found under '$root'" - exit -} + Write-Verbose "Found template '$templateFile'" + $templateFiles += $templateFile + } -$UserName = if ($env:USER) { $env:USER } else { "${env:USERNAME}" } + if (!$templateFiles) { + Write-Warning -Message "No template files found under '$root'" + exit + } -# If no base name is specified use current user name -if (!$BaseName) { - $BaseName = "$UserName$ServiceDirectory" + $UserName = if ($env:USER) { $env:USER } else { "${env:USERNAME}" } + # Remove spaces, etc. that may be in $UserName + $UserName = $UserName -replace '\W' + + # Make sure $BaseName is set. + if ($CI) { + $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16) + Log "Generated base name '$BaseName' for CI build" + } elseif (!$BaseName) { + $BaseName = "$UserName$ServiceDirectory" + Log "BaseName was not set. Using default base name: '$BaseName'" + } # Make sure pre- and post-scripts are passed formerly required arguments. $PSBoundParameters['BaseName'] = $BaseName - Log "BaseName was not set. Using default base name: '$BaseName'" -} + # Try detecting repos that support OutFile and defaulting to it + if (!$CI -and !$PSBoundParameters.ContainsKey('OutFile') -and $IsWindows) { + # TODO: find a better way to detect the language + if (Test-Path "$repositoryRoot/eng/service.proj") { + $OutFile = $true + Log "Detected .NET repository. Defaulting OutFile to true. Test environment settings would be stored into the file so you don't need to set environment variables manually." + } + } -# Try detecting repos that support OutFile and defaulting to it -if (!$CI -and !$PSBoundParameters.ContainsKey('OutFile') -and $IsWindows) { - # TODO: find a better way to detect the language - if (Test-Path "$repositoryRoot/eng/service.proj") { - $OutFile = $true - Log "Detected .NET repository. Defaulting OutFile to true. Test environment settings would be stored into the file so you don't need to set environment variables manually." + # If no location is specified use safe default locations for the given + # environment. If no matching environment is found $Location remains an empty + # string. + if (!$Location) { + $Location = @{ + 'AzureCloud' = 'westus2'; + 'AzureUSGovernment' = 'usgovvirginia'; + 'AzureChinaCloud' = 'chinaeast2'; + 'Dogfood' = 'westus' + }[$Environment] + + Write-Verbose "Location was not set. Using default location for environment: '$Location'" } -} -# If no location is specified use safe default locations for the given -# environment. If no matching environment is found $Location remains an empty -# string. -if (!$Location) { - $Location = @{ - 'AzureCloud' = 'westus2'; - 'AzureUSGovernment' = 'usgovvirginia'; - 'AzureChinaCloud' = 'chinaeast2'; - 'Dogfood' = 'westus' - }[$Environment] - - Write-Verbose "Location was not set. Using default location for environment: '$Location'" -} + if (!$CI) { -if (!$CI) { + # Make sure the user is logged in to create a service principal. + $context = Get-AzContext; + if (!$context) { + Log "You are not logged in; connecting to 'Azure SDK Developer Playground'" + $context = (Connect-AzAccount -Subscription $defaultSubscription).Context + } - # Make sure the user is logged in to create a service principal. - $context = Get-AzContext; - if (!$context) { - Log "You are not logged in; connecting to 'Azure SDK Developer Playground'" - $context = (Connect-AzAccount -Subscription $defaultSubscription).Context - } + # If no test application ID is specified during an interactive session, create a new service principal. + if (!$TestApplicationId) { - # If no test application ID is specified during an interactive session, create a new service principal. - if (!$TestApplicationId) { - Log "TestApplicationId was not specified; creating a new service principal" - $servicePrincipal = New-AzADServicePrincipal -Role Owner + # Cache the created service principal in this session for frequent reuse. + $servicePrincipal = if ($AzureTestPrincipal) { + Log "TestApplicationId was not specified; loading the cached service principal" + $AzureTestPrincipal + } else { + Log "TestApplicationId was not specified; creating a new service principal" + $global:AzureTestPrincipal = New-AzADServicePrincipal -Role Owner - $TestApplicationId = $servicePrincipal.ApplicationId - $TestApplicationSecret = (ConvertFrom-SecureString $servicePrincipal.Secret -AsPlainText); + Log "Created service principal '$AzureTestPrincipal'" + $AzureTestPrincipal + } - # Make sure pre- and post-scripts are passed formerly required arguments. - $PSBoundParameters['TestApplicationId'] = $TestApplicationId - $PSBoundParameters['TestApplicationSecret'] = $TestApplicationSecret + $TestApplicationId = $servicePrincipal.ApplicationId + $TestApplicationSecret = (ConvertFrom-SecureString $servicePrincipal.Secret -AsPlainText); - Log "Created service principal '$TestApplicationId'" - } + # Make sure pre- and post-scripts are passed formerly required arguments. + $PSBoundParameters['TestApplicationId'] = $TestApplicationId + $PSBoundParameters['TestApplicationSecret'] = $TestApplicationSecret + } - if (!$ProvisionerApplicationId) { - $ProvisionerApplicationId = $TestApplicationId - $ProvisionerApplicationSecret = $TestApplicationSecret - $TenantId = $context.Tenant.Id + if (!$ProvisionerApplicationId) { + $ProvisionerApplicationId = $TestApplicationId + $ProvisionerApplicationSecret = $TestApplicationSecret + $TenantId = $context.Tenant.Id + } } -} -# Log in as and run pre- and post-scripts as the provisioner service principal. -if ($ProvisionerApplicationId) { - $null = Disable-AzContextAutosave -Scope Process + # Log in as and run pre- and post-scripts as the provisioner service principal. + if ($ProvisionerApplicationId) { + $null = Disable-AzContextAutosave -Scope Process - Log "Logging into service principal '$ProvisionerApplicationId'" - $provisionerSecret = ConvertTo-SecureString -String $ProvisionerApplicationSecret -AsPlainText -Force - $provisionerCredential = [System.Management.Automation.PSCredential]::new($ProvisionerApplicationId, $provisionerSecret) + Log "Logging into service principal '$ProvisionerApplicationId'" + $provisionerSecret = ConvertTo-SecureString -String $ProvisionerApplicationSecret -AsPlainText -Force + $provisionerCredential = [System.Management.Automation.PSCredential]::new($ProvisionerApplicationId, $provisionerSecret) - # Use the given subscription ID if provided. - $subscriptionArgs = if ($SubscriptionId) { - @{SubscriptionId = $SubscriptionId} - } else { - @{} - } + # Use the given subscription ID if provided. + $subscriptionArgs = if ($SubscriptionId) { + @{SubscriptionId = $SubscriptionId} + } else { + @{} + } - $provisionerAccount = Retry { - Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs - } + $provisionerAccount = Retry { + Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs + } - $exitActions += { - Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'" + $exitActions += { + Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'" - # Only attempt to disconnect if the -WhatIf flag was not set. Otherwise, this call is not necessary and will fail. - if ($PSCmdlet.ShouldProcess($ProvisionerApplicationId)) { - $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + # Only attempt to disconnect if the -WhatIf flag was not set. Otherwise, this call is not necessary and will fail. + if ($PSCmdlet.ShouldProcess($ProvisionerApplicationId)) { + $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + } } } -} -# Get test application OID from ID if not already provided. -if ($TestApplicationId -and !$TestApplicationOid) { - $testServicePrincipal = Retry { - Get-AzADServicePrincipal -ApplicationId $TestApplicationId - } + # Get test application OID from ID if not already provided. + if ($TestApplicationId -and !$TestApplicationOid) { + $testServicePrincipal = Retry { + Get-AzADServicePrincipal -ApplicationId $TestApplicationId + } - if ($testServicePrincipal -and $testServicePrincipal.Id) { - $script:TestApplicationOid = $testServicePrincipal.Id + if ($testServicePrincipal -and $testServicePrincipal.Id) { + $script:TestApplicationOid = $testServicePrincipal.Id + } } -} - -# Determine the Azure context that the script is running in. -$context = Get-AzContext; -# If the ServiceDirectory is an absolute path use the last directory name -# (e.g. D:\foo\bar\ -> bar) -$serviceName = if (Split-Path -IsAbsolute $ServiceDirectory) { - Split-Path -Leaf $ServiceDirectory -} else { - $ServiceDirectory -} - -if ($CI) { - $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16) - Write-Verbose "Generated base name '$BaseName' for CI build" -} - -$ResourceGroupName = if ($ResourceGroupName) { - $ResourceGroupName -} elseif ($CI) { - # Format the resource group name based on resource group naming recommendations and limitations. - "rg-{0}-$BaseName" -f ($serviceName -replace '[\\\/:]', '-').Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-') -} else { - "rg-$BaseName" -} - -# Tag the resource group to be deleted after a certain number of hours if specified. -$tags = @{ - Creator = $UserName - ServiceDirectory = $ServiceDirectory -} - -if ($PSBoundParameters.ContainsKey('DeleteAfterHours')) { - $deleteAfter = [DateTime]::UtcNow.AddHours($DeleteAfterHours) - $tags.Add('DeleteAfter', $deleteAfter.ToString('o')) -} + # Determine the Azure context that the script is running in. + $context = Get-AzContext; -if ($CI) { - # Add tags for the current CI job. - $tags += @{ - BuildId = "${env:BUILD_BUILDID}" - BuildJob = "${env:AGENT_JOBNAME}" - BuildNumber = "${env:BUILD_BUILDNUMBER}" - BuildReason = "${env:BUILD_REASON}" + # If the ServiceDirectory is an absolute path use the last directory name + # (e.g. D:\foo\bar\ -> bar) + $serviceName = if (Split-Path -IsAbsolute $ServiceDirectory) { + Split-Path -Leaf $ServiceDirectory + } else { + $ServiceDirectory } - # Set the resource group name variable. - Write-Host "Setting variable 'AZURE_RESOURCEGROUP_NAME': $ResourceGroupName" - Write-Host "##vso[task.setvariable variable=AZURE_RESOURCEGROUP_NAME;]$ResourceGroupName" - if ($EnvironmentVariables.ContainsKey('AZURE_RESOURCEGROUP_NAME') -and ` - $EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'] -ne $ResourceGroupName) - { - Write-Warning ("Overwriting 'EnvironmentVariables.AZURE_RESOURCEGROUP_NAME' with value " + - "'$($EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'])' " + "to new value '$($ResourceGroupName)'") + $ResourceGroupName = if ($ResourceGroupName) { + $ResourceGroupName + } elseif ($CI) { + # Format the resource group name based on resource group naming recommendations and limitations. + "rg-{0}-$BaseName" -f ($serviceName -replace '[\\\/:]', '-').Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-') + } else { + "rg-$BaseName" } - $EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'] = $ResourceGroupName -} - -Log "Creating resource group '$ResourceGroupName' in location '$Location'" -$resourceGroup = Retry { - New-AzResourceGroup -Name "$ResourceGroupName" -Location $Location -Tag $tags -Force:$Force -} -if ($resourceGroup.ProvisioningState -eq 'Succeeded') { - # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway. - Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'" -} -elseif (($resourceGroup -eq $null) -and (-not $PSCmdlet.ShouldProcess($resourceGroupName))) { - # If the -WhatIf flag was passed, there will be no resource group created. Fake it. - $resourceGroup = [PSCustomObject]@{ - ResourceGroupName = $resourceGroupName - Location = $Location + # Tag the resource group to be deleted after a certain number of hours if specified. + $tags = @{ + Creator = $UserName + ServiceDirectory = $ServiceDirectory } -} - -# Populate the template parameters and merge any additional specified. -$templateParameters = @{ - baseName = $BaseName - testApplicationId = $TestApplicationId - testApplicationOid = "$TestApplicationOid" -} -if ($TenantId) { - $templateParameters.Add('tenantId', $TenantId) -} -if ($TestApplicationSecret) { - $templateParameters.Add('testApplicationSecret', $TestApplicationSecret) -} + if ($PSBoundParameters.ContainsKey('DeleteAfterHours')) { + $deleteAfter = [DateTime]::UtcNow.AddHours($DeleteAfterHours) + $tags.Add('DeleteAfter', $deleteAfter.ToString('o')) + } -MergeHashes $ArmTemplateParameters $(Get-Variable templateParameters) -MergeHashes $AdditionalParameters $(Get-Variable templateParameters) + if ($CI) { + # Add tags for the current CI job. + $tags += @{ + BuildId = "${env:BUILD_BUILDID}" + BuildJob = "${env:AGENT_JOBNAME}" + BuildNumber = "${env:BUILD_BUILDNUMBER}" + BuildReason = "${env:BUILD_REASON}" + } -# Include environment-specific parameters only if not already provided as part of the "ArmTemplateParameters" -if (($context.Environment.StorageEndpointSuffix) -and (-not ($templateParameters.ContainsKey('storageEndpointSuffix')))) { - $templateParameters.Add('storageEndpointSuffix', $context.Environment.StorageEndpointSuffix) -} + # Set the resource group name variable. + Write-Host "Setting variable 'AZURE_RESOURCEGROUP_NAME': $ResourceGroupName" + Write-Host "##vso[task.setvariable variable=AZURE_RESOURCEGROUP_NAME;]$ResourceGroupName" + if ($EnvironmentVariables.ContainsKey('AZURE_RESOURCEGROUP_NAME') -and ` + $EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'] -ne $ResourceGroupName) + { + Write-Warning ("Overwriting 'EnvironmentVariables.AZURE_RESOURCEGROUP_NAME' with value " + + "'$($EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'])' " + "to new value '$($ResourceGroupName)'") + } + $EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'] = $ResourceGroupName + } -# Try to detect the shell based on the parent process name (e.g. launch via shebang). -$shell, $shellExportFormat = if (($parentProcessName = (Get-Process -Id $PID).Parent.ProcessName) -and $parentProcessName -eq 'cmd') { - 'cmd', 'set {0}={1}' -} elseif (@('bash', 'csh', 'tcsh', 'zsh') -contains $parentProcessName) { - 'shell', 'export {0}={1}' -} else { - 'PowerShell', '${{env:{0}}} = ''{1}''' -} + Log "Creating resource group '$ResourceGroupName' in location '$Location'" + $resourceGroup = Retry { + New-AzResourceGroup -Name "$ResourceGroupName" -Location $Location -Tag $tags -Force:$Force + } -# Deploy the templates -foreach ($templateFile in $templateFiles) { - # Deployment fails if we pass in more parameters than are defined. - Write-Verbose "Removing unnecessary parameters from template '$templateFile'" - $templateJson = Get-Content -LiteralPath $templateFile | ConvertFrom-Json - $templateParameterNames = $templateJson.parameters.PSObject.Properties.Name - - $templateFileParameters = $templateParameters.Clone() - foreach ($key in $templateParameters.Keys) { - if ($templateParameterNames -notcontains $key) { - Write-Verbose "Removing unnecessary parameter '$key'" - $templateFileParameters.Remove($key) + if ($resourceGroup.ProvisioningState -eq 'Succeeded') { + # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway. + Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'" + } + elseif (!$resourceGroup -and !$PSCmdlet.ShouldProcess($resourceGroupName)) { + # If the -WhatIf flag was passed, there will be no resource group created. Fake it. + $resourceGroup = [PSCustomObject]@{ + ResourceGroupName = $resourceGroupName + Location = $Location } } - $preDeploymentScript = $templateFile | Split-Path | Join-Path -ChildPath 'test-resources-pre.ps1' - if (Test-Path $preDeploymentScript) { - Log "Invoking pre-deployment script '$preDeploymentScript'" - &$preDeploymentScript -ResourceGroupName $ResourceGroupName @PSBoundParameters + # Populate the template parameters and merge any additional specified. + $templateParameters = @{ + baseName = $BaseName + testApplicationId = $TestApplicationId + testApplicationOid = "$TestApplicationOid" } - Log "Deploying template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'" - $deployment = Retry { - New-AzResourceGroupDeployment -Name $BaseName -ResourceGroupName $resourceGroup.ResourceGroupName -TemplateFile $templateFile -TemplateParameterObject $templateFileParameters + if ($TenantId) { + $templateParameters.Add('tenantId', $TenantId) } - - if ($deployment.ProvisioningState -eq 'Succeeded') { - # New-AzResourceGroupDeployment would've written an error and stopped the pipeline by default anyway. - Write-Verbose "Successfully deployed template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'" + if ($TestApplicationSecret) { + $templateParameters.Add('testApplicationSecret', $TestApplicationSecret) } - $serviceDirectoryPrefix = $serviceName.ToUpperInvariant() + "_" - - # Add default values - $deploymentOutputs = @{ - "$($serviceDirectoryPrefix)CLIENT_ID" = $TestApplicationId; - "$($serviceDirectoryPrefix)CLIENT_SECRET" = $TestApplicationSecret; - "$($serviceDirectoryPrefix)TENANT_ID" = $context.Tenant.Id; - "$($serviceDirectoryPrefix)SUBSCRIPTION_ID" = $context.Subscription.Id; - "$($serviceDirectoryPrefix)RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; - "$($serviceDirectoryPrefix)LOCATION" = $resourceGroup.Location; - "$($serviceDirectoryPrefix)ENVIRONMENT" = $context.Environment.Name; - "$($serviceDirectoryPrefix)AZURE_AUTHORITY_HOST" = $context.Environment.ActiveDirectoryAuthority; - "$($serviceDirectoryPrefix)RESOURCE_MANAGER_URL" = $context.Environment.ResourceManagerUrl; - "$($serviceDirectoryPrefix)SERVICE_MANAGEMENT_URL" = $context.Environment.ServiceManagementUrl; - "$($serviceDirectoryPrefix)STORAGE_ENDPOINT_SUFFIX" = $context.Environment.StorageEndpointSuffix; + MergeHashes $ArmTemplateParameters $(Get-Variable templateParameters) + MergeHashes $AdditionalParameters $(Get-Variable templateParameters) + + # Include environment-specific parameters only if not already provided as part of the "ArmTemplateParameters" + if (($context.Environment.StorageEndpointSuffix) -and (-not ($templateParameters.ContainsKey('storageEndpointSuffix')))) { + $templateParameters.Add('storageEndpointSuffix', $context.Environment.StorageEndpointSuffix) } - MergeHashes $EnvironmentVariables $(Get-Variable deploymentOutputs) + # Try to detect the shell based on the parent process name (e.g. launch via shebang). + $shell, $shellExportFormat = if (($parentProcessName = (Get-Process -Id $PID).Parent.ProcessName) -and $parentProcessName -eq 'cmd') { + 'cmd', 'set {0}={1}' + } elseif (@('bash', 'csh', 'tcsh', 'zsh') -contains $parentProcessName) { + 'shell', 'export {0}={1}' + } else { + 'PowerShell', '${{env:{0}}} = ''{1}''' + } - foreach ($key in $deployment.Outputs.Keys) { - $variable = $deployment.Outputs[$key] + # Deploy the templates + foreach ($templateFile in $templateFiles) { + # Deployment fails if we pass in more parameters than are defined. + Write-Verbose "Removing unnecessary parameters from template '$templateFile'" + $templateJson = Get-Content -LiteralPath $templateFile | ConvertFrom-Json + $templateParameterNames = $templateJson.parameters.PSObject.Properties.Name + + $templateFileParameters = $templateParameters.Clone() + foreach ($key in $templateParameters.Keys) { + if ($templateParameterNames -notcontains $key) { + Write-Verbose "Removing unnecessary parameter '$key'" + $templateFileParameters.Remove($key) + } + } - # Work around bug that makes the first few characters of environment variables be lowercase. - $key = $key.ToUpperInvariant() + $preDeploymentScript = $templateFile | Split-Path | Join-Path -ChildPath 'test-resources-pre.ps1' + if (Test-Path $preDeploymentScript) { + Log "Invoking pre-deployment script '$preDeploymentScript'" + &$preDeploymentScript -ResourceGroupName $ResourceGroupName @PSBoundParameters + } - if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { - $deploymentOutputs[$key] = $variable.Value + Log "Deploying template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'" + $deployment = Retry { + $lastDebugPreference = $DebugPreference + try { + if ($CI) { + $DebugPreference = "Continue" + } + New-AzResourceGroupDeployment -Name $BaseName -ResourceGroupName $resourceGroup.ResourceGroupName -TemplateFile $templateFile -TemplateParameterObject $templateFileParameters + } catch { + Write-Output @" +##################################################### +# For help debugging live test provisioning issues, # +# see http://aka.ms/azsdk/engsys/live-test-help, # +##################################################### +"@ + throw + } finally { + $DebugPreference = $lastDebugPreference + } } - } - if ($OutFile) { - if (!$IsWindows) { - Write-Host "File option is supported only on Windows" + if ($deployment.ProvisioningState -eq 'Succeeded') { + # New-AzResourceGroupDeployment would've written an error and stopped the pipeline by default anyway. + Write-Verbose "Successfully deployed template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'" } - $outputFile = "$templateFile.env" + $serviceDirectoryPrefix = $serviceName.ToUpperInvariant() + "_" + + # Add default values + $deploymentOutputs = @{ + "$($serviceDirectoryPrefix)CLIENT_ID" = $TestApplicationId; + "$($serviceDirectoryPrefix)CLIENT_SECRET" = $TestApplicationSecret; + "$($serviceDirectoryPrefix)TENANT_ID" = $context.Tenant.Id; + "$($serviceDirectoryPrefix)SUBSCRIPTION_ID" = $context.Subscription.Id; + "$($serviceDirectoryPrefix)RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; + "$($serviceDirectoryPrefix)LOCATION" = $resourceGroup.Location; + "$($serviceDirectoryPrefix)ENVIRONMENT" = $context.Environment.Name; + "$($serviceDirectoryPrefix)AZURE_AUTHORITY_HOST" = $context.Environment.ActiveDirectoryAuthority; + "$($serviceDirectoryPrefix)RESOURCE_MANAGER_URL" = $context.Environment.ResourceManagerUrl; + "$($serviceDirectoryPrefix)SERVICE_MANAGEMENT_URL" = $context.Environment.ServiceManagementUrl; + "$($serviceDirectoryPrefix)STORAGE_ENDPOINT_SUFFIX" = $context.Environment.StorageEndpointSuffix; + } - $environmentText = $deploymentOutputs | ConvertTo-Json; - $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText) - $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + MergeHashes $EnvironmentVariables $(Get-Variable deploymentOutputs) - Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + foreach ($key in $deployment.Outputs.Keys) { + $variable = $deployment.Outputs[$key] - Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" - } else { + # Work around bug that makes the first few characters of environment variables be lowercase. + $key = $key.ToUpperInvariant() - if (!$CI) { - # Write an extra new line to isolate the environment variables for easy reading. - Log "Persist the following environment variables based on your detected shell ($shell):`n" + if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { + $deploymentOutputs[$key] = $variable.Value + } } - foreach ($key in $deploymentOutputs.Keys) { - $value = $deploymentOutputs[$key] - $EnvironmentVariables[$key] = $value + if ($OutFile) { + if (!$IsWindows) { + Write-Host "File option is supported only on Windows" + } - if ($CI) { - # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. - # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. - Write-Host "Setting variable '$key': ***" - Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($value)" - Write-Host "##vso[task.setvariable variable=$key;]$($value)" - } else { - Write-Host ($shellExportFormat -f $key, $value) + $outputFile = "$templateFile.env" + + $environmentText = $deploymentOutputs | ConvertTo-Json; + $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText) + $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + + Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + + Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" + } else { + + if (!$CI) { + # Write an extra new line to isolate the environment variables for easy reading. + Log "Persist the following environment variables based on your detected shell ($shell):`n" + } + + foreach ($key in $deploymentOutputs.Keys) { + $value = $deploymentOutputs[$key] + $EnvironmentVariables[$key] = $value + + if ($CI) { + # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. + # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. + Write-Host "Setting variable '$key': ***" + Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($value)" + Write-Host "##vso[task.setvariable variable=$key;]$($value)" + } else { + Write-Host ($shellExportFormat -f $key, $value) + } + } + + if ($key) { + # Isolate the environment variables for easy reading. + Write-Host "`n" + $key = $null } } - if ($key) { - # Isolate the environment variables for easy reading. - Write-Host "`n" - $key = $null + $postDeploymentScript = $templateFile | Split-Path | Join-Path -ChildPath 'test-resources-post.ps1' + if (Test-Path $postDeploymentScript) { + Log "Invoking post-deployment script '$postDeploymentScript'" + &$postDeploymentScript -ResourceGroupName $ResourceGroupName -DeploymentOutputs $deploymentOutputs @PSBoundParameters } } - $postDeploymentScript = $templateFile | Split-Path | Join-Path -ChildPath 'test-resources-post.ps1' - if (Test-Path $postDeploymentScript) { - Log "Invoking post-deployment script '$postDeploymentScript'" - &$postDeploymentScript -ResourceGroupName $ResourceGroupName -DeploymentOutputs $deploymentOutputs @PSBoundParameters - } +} finally { + $exitActions.Invoke() } -$exitActions.Invoke() - # Suppress output locally if ($CI) { return $EnvironmentVariables @@ -616,12 +647,7 @@ The environment file would be scoped to the current repository directory. .EXAMPLE Connect-AzAccount -Subscription "REPLACE_WITH_SUBSCRIPTION_ID" -$testAadApp = New-AzADServicePrincipal -Role Owner -DisplayName 'azure-sdk-live-test-app' -New-TestResources.ps1 ` - -BaseName 'uuid123' ` - -ServiceDirectory 'keyvault' ` - -TestApplicationId $testAadApp.ApplicationId.ToString() ` - -TestApplicationSecret (ConvertFrom-SecureString $testAadApp.Secret -AsPlainText) +New-TestResources.ps1 -ServiceDirectory 'keyvault' Run this in a desktop environment to create new AAD apps and Service Principals that can be used to provision resources and run live tests. diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md index 515a51bd373cd..096c7e70aa37e 100644 --- a/eng/common/TestResources/New-TestResources.ps1.md +++ b/eng/common/TestResources/New-TestResources.ps1.md @@ -56,12 +56,7 @@ specified in $ProvisionerApplicationId and $ProvisionerApplicationSecret. ### EXAMPLE 1 ``` Connect-AzAccount -Subscription "REPLACE_WITH_SUBSCRIPTION_ID" -$testAadApp = New-AzADServicePrincipal -Role Owner -DisplayName 'azure-sdk-live-test-app' -New-TestResources.ps1 ` - -BaseName 'uuid123' ` - -ServiceDirectory 'keyvault' ` - -TestApplicationId $testAadApp.ApplicationId.ToString() ` - -TestApplicationSecret (ConvertFrom-SecureString $testAadApp.Secret -AsPlainText) +New-TestResources.ps1 -ServiceDirectory 'keyvault' ``` Run this in a desktop environment to create new AAD apps and Service Principals diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml index 031c4b37cccd1..a9d693b99420e 100644 --- a/eng/common/TestResources/deploy-test-resources.yml +++ b/eng/common/TestResources/deploy-test-resources.yml @@ -1,7 +1,7 @@ parameters: ServiceDirectory: not-set ArmTemplateParameters: '@{}' - DeleteAfterHours: 24 + DeleteAfterHours: 8 Location: '' SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) diff --git a/eng/common/docgeneration/Generate-DocIndex.ps1 b/eng/common/docgeneration/Generate-DocIndex.ps1 new file mode 100644 index 0000000000000..84fd562715508 --- /dev/null +++ b/eng/common/docgeneration/Generate-DocIndex.ps1 @@ -0,0 +1,196 @@ +# Generates an index page for cataloging different versions of the Docs +[CmdletBinding()] +Param ( + $DocFx, + $RepoRoot, + $DocGenDir, + $DocOutDir = "${RepoRoot}/docfx_project", + $DocfxJsonPath = "${PSScriptRoot}\docfx.json", + $MainJsPath = "${PSScriptRoot}\templates\matthews\styles\main.js" +) +. "${PSScriptRoot}\..\scripts\common.ps1" +$GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" + +# Given the metadata url under https://github.com/Azure/azure-sdk/tree/master/_data/releases/latest, +# the function will return the csv metadata back as part of response. +function Get-CSVMetadata ([string]$MetadataUri) { + $metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv + return $metadataResponse +} + +# Given the github io blob storage url and language regex, +# the helper function will return a list of artifact names. +function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobArtifactsReplacement) { + LogDebug "Reading artifact from storage blob ..." + $returnedArtifacts = @() + $pageToken = "" + Do { + $resp = "" + if (!$pageToken) { + # First page call. + $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrl + } + else { + # Next page call + $blobStorageUrlPageToken = $blobStorageUrl + "&marker=$pageToken" + $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrlPageToken + } + # Convert to xml documents. + $xmlDoc = [xml](removeBomFromString $resp) + foreach ($elem in $xmlDoc.EnumerationResults.Blobs.BlobPrefix) { + # What service return like "dotnet/Azure.AI.Anomalydetector/", needs to fetch out "Azure.AI.Anomalydetector" + $artifact = $elem.Name -replace $blobDirectoryRegex, $blobArtifactsReplacement + $returnedArtifacts += $artifact + } + # Fetch page token + $pageToken = $xmlDoc.EnumerationResults.NextMarker + } while ($pageToken) + return $returnedArtifacts + } + +# The sequence of Bom bytes differs by different encoding. +# The helper function here is only to strip the utf-8 encoding system as it is used by blob storage list api. +# Return the original string if not in BOM utf-8 sequence. +function RemoveBomFromString([string]$bomAwareString) { + if ($bomAwareString.length -le 3) { + return $bomAwareString + } + $bomPatternByteArray = [byte[]] (0xef, 0xbb, 0xbf) + # The default encoding for powershell is ISO-8859-1, so converting bytes with the encoding. + $bomAwareBytes = [Text.Encoding]::GetEncoding(28591).GetBytes($bomAwareString.Substring(0, 3)) + if (@(Compare-Object $bomPatternByteArray $bomAwareBytes -SyncWindow 0).Length -eq 0) { + return $bomAwareString.Substring(3) + } + return $bomAwareString +} + +function Get-TocMapping { + Param ( + [Parameter(Mandatory = $true)] [Object[]] $metadata, + [Parameter(Mandatory = $true)] [String[]] $artifacts + ) + # Used for sorting the toc display order + $orderServiceMapping = @{} + + foreach ($artifact in $artifacts) { + $packageInfo = $metadata | ? {$_.Package -eq $artifact} + + if ($packageInfo -and $packageInfo[0].Hide -eq 'true') { + LogDebug "The artifact $artifact set 'Hide' to 'true'." + continue + } + $serviceName = "" + $displayName = "" + if (!$packageInfo) { + LogWarning "There is no artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " + # If no service name retrieved, print out warning message, and put it into Other page. + $serviceName = "Other" + } + elseif (!$packageInfo[0].ServiceName) { + LogWarning "There is no service name for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " + # If no service name retrieved, print out warning message, and put it into Other page. + $serviceName = "Other" + $displayName = $packageInfo[0].DisplayName.Trim() + } + else { + if ($packageInfo.Length -gt 1) { + LogWarning "There are more than 1 packages fetched out for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " + } + $serviceName = $packageInfo[0].ServiceName.Trim() + $displayName = $packageInfo[0].DisplayName.Trim() + } + $orderServiceMapping[$artifact] = @($serviceName, $displayName) + } + return $orderServiceMapping +} + +function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) { + LogDebug "Start generating the docfx toc and build docfx site..." + + LogDebug "Initializing Default DocFx Site..." + & $($DocFx) init -q -o "${DocOutDir}" + # The line below is used for testing in local + #docfx init -q -o "${DocOutDir}" + LogDebug "Copying template and configuration..." + New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force + Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse + Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force + $YmlPath = "${DocOutDir}/api" + New-Item -Path $YmlPath -Name "toc.yml" -Force + $visitedService = @{} + # Sort and display toc service name by alphabetical order, and then sort artifact by order. + foreach ($serviceMapping in ($tocContent.GetEnumerator() | Sort-Object Value[0], Key)) { + $artifact = $serviceMapping.Key + $serviceName = $serviceMapping.Value[0] + $displayName = $serviceMapping.Value[1] + + $fileName = ($serviceName -replace '\s', '').ToLower().Trim() + if ($visitedService.ContainsKey($serviceName)) { + if ($displayName) { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact`n##### ($displayName)" + } + else { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact" + } + } + else { + Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: ${serviceName}`r`n href: ${fileName}.md" + New-Item -Path $YmlPath -Name "${fileName}.md" -Force + if ($displayName) { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact`n##### ($displayName)" + } + else { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact" + } + $visitedService[$serviceName] = $true + } + } + + # Generate toc homepage. + LogDebug "Creating Site Title and Navigation..." + New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force + Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for $lang APIs`r`n href: api/`r`n homepage: api/index.md" + + LogDebug "Copying root markdowns" + Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force + Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force + + LogDebug "Building site..." + & $($DocFx) build "${DocOutDir}/docfx.json" + # The line below is used for testing in local + #docfx build "${DocOutDir}/docfx.json" + Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force +} + +function Mutate-Files { + Param ( + [Parameter(Mandatory=$true)] [String]$appTitle, + [Parameter(Mandatory=$true)] [String]$lang, + [Parameter(Mandatory=$true)] [String]$indexhtmlloc, + [Parameter(Mandatory=$false)] [String]$packageRegex = "`"`"", + [Parameter(Mandatory=$false)] [String]$regexReplacement = "" + ) + # Update docfx.json + $docfxContent = Get-Content -Path $DocfxJsonPath -Raw + $docfxContent = $docfxContent -replace "`"_appTitle`": `"`"", "`"_appTitle`": `"$appTitle`"" + $docfxContent = $docfxContent -replace "`"_appFooter`": `"`"", "`"_appFooter`": `"$appTitle`"" + Set-Content -Path $DocfxJsonPath -Value $docfxContent + # Update main.js var lang + $mainJsContent = Get-Content -Path $MainJsPath -Raw + $mainJsContent = $mainJsContent -replace "var SELECTED_LANGUAGE = ''", "var SELECTED_LANGUAGE = '$lang'" + # Update main.js var index html + $mainJsContent = $mainJsContent -replace "var INDEX_HTML = ''", "var INDEX_HTML = '$indexhtmlloc'" + # Update main.js package regex and replacement + $mainJsContent = $mainJsContent -replace "var PACKAGE_REGEX = ''", "var PACKAGE_REGEX = $packageRegex" + $mainJsContent = $mainJsContent -replace "var PACKAGE_REPLACEMENT = ''", "var PACKAGE_REPLACEMENT = `"$regexReplacement`"" + Set-Content -Path $MainJsPath -Value $mainJsContent -NoNewline +} + +if ($GetGithubIoDocIndexFn -and (Test-Path "function:$GetGithubIoDocIndexFn")) +{ + &$GetGithubIoDocIndexFn +} +else +{ + LogWarning "The function '$GetGithubIoDocIndexFn' was not found." +} diff --git a/eng/common/docgeneration/assets/logo.svg b/eng/common/docgeneration/assets/logo.svg new file mode 100644 index 0000000000000..5da99f4048866 --- /dev/null +++ b/eng/common/docgeneration/assets/logo.svg @@ -0,0 +1,76 @@ + + + + diff --git a/eng/common/docgeneration/docfx.json b/eng/common/docgeneration/docfx.json new file mode 100644 index 0000000000000..6b113dafd3c0d --- /dev/null +++ b/eng/common/docgeneration/docfx.json @@ -0,0 +1,78 @@ +{ + "metadata": [ + { + "src": [ + { + "files": [ + "src/**.csproj" + ] + } + ], + "dest": "api", + "disableGitFeatures": false, + "disableDefaultFilter": false + } + ], + "build": { + "content": [ + { + "files": [ + "api/**.yml", + "api/**.md", + "api/index.md" + ] + }, + { + "files": [ + "toc.yml", + "*.md" + ] + } + ], + "resource": [ + { + "files": [ + "images/**" + ] + } + ], + "overwrite": [ + { + "files": [ + "apidoc/**.md" + ], + "exclude": [ + "obj/**", + "_site/**" + ] + } + ], + "dest": "_site", + "globalMetadataFiles": [], + "fileMetadataFiles": [], + "template": [ + "default", + "templates/matthews" + ], + "postProcessors": [], + "markdownEngineName": "markdig", + "noLangKeyword": false, + "keepFileLink": false, + "cleanupCacheHistory": false, + "disableGitFeatures": false, + "globalMetadata": { + "_appTitle": "", + "_appFooter": "", + "_enableSearch": false, + "_enableNewTab": true, + "_appFaviconPath": "https://c.s-microsoft.com/favicon.ico?v2", + "_disableContribution": true + } + } +} + + + + + + diff --git a/eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial new file mode 100644 index 0000000000000..43a33d0120a38 --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial @@ -0,0 +1,17 @@ +{{^_disableContribution}} +
+ {{#docurl}} + + {{/docurl}} + {{#sourceurl}} + + {{/sourceurl}} +
+{{/_disableContribution}} + + diff --git a/eng/common/docgeneration/templates/matthews/partials/class.header.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/class.header.tmpl.partial new file mode 100644 index 0000000000000..49a27d827322b --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/class.header.tmpl.partial @@ -0,0 +1,100 @@ +

{{>partials/title}}

+
{{{summary}}}
+
{{{conceptual}}}
+ +{{#inClass}} +
+
{{__global.inheritance}}
+ {{#inheritance}} +
{{{specName.0.value}}}
+ {{/inheritance}} +
{{name.0.value}}
+
+{{/inClass}} + +{{#derivedClasses}} +
{{{specName.0.value}}}
+{{/derivedClasses}} + +{{#inheritedMembers.0}} +
+
{{__global.inheritedMembers}}
+{{/inheritedMembers.0}} +{{#inheritedMembers}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/inheritedMembers}} +{{#inheritedMembers.0}} +
+{{/inheritedMembers.0}} + +
{{__global.namespace}}: {{namespace}}
+
{{__global.assembly}}: {{assemblies.0}}.dll
+ +
{{__global.syntax}}
+
+ 
{{syntax.content.0.value}}
+
+ +{{#syntax.parameters.0}} +
{{__global.parameters}}
+ +{{/syntax.parameters.0}} +{{#syntax.parameters}} + + + +{{/syntax.parameters}} +{{#syntax.parameters.0}} +
+ {{{type.specName.0.value}}} + {{{id}}} +

{{{description}}}

+
+{{/syntax.parameters.0}} + +{{#syntax.return}} +
{{__global.returns}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/syntax.return}} + +{{#syntax.typeParameters.0}} +
{{__global.typeParameters}}
+ +{{/syntax.typeParameters.0}} +{{#syntax.typeParameters}} + + + +{{/syntax.typeParameters}} +{{#syntax.typeParameters.0}} +
+ {{{id}}} +

{{{description}}}

+
+{{/syntax.typeParameters.0}} + +{{#remarks}} +
{{__global.remarks}}
+
{{{remarks}}}
+{{/remarks}} + +{{#example.0}} +
{{__global.examples}}
+{{/example.0}} +{{#example}} +{{{.}}} +{{/example}} diff --git a/eng/common/docgeneration/templates/matthews/partials/class.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/class.tmpl.partial new file mode 100644 index 0000000000000..5f00b822cc76d --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/class.tmpl.partial @@ -0,0 +1,210 @@ +{{>partials/class.header}} +{{#children}} +

{{>partials/classSubtitle}}

+{{#children}} + +{{^_disableContribution}} +{{#docurl}} + + + + +{{/docurl}} +{{#sourceurl}} + + + +{{/sourceurl}} +{{/_disableContribution}} + +{{#overload}} + +{{/overload}} + +

{{name.0.value}}

+ +
+
{{{summary}}}
+
{{{conceptual}}}
+
{{__global.declaration}}
+ +{{#syntax}} +
+ 
{{syntax.content.0.value}}
+
+ +{{#parameters.0}} +
{{__global.parameters}}
+ +{{/parameters.0}} +{{#parameters}} + + + +{{/parameters}} +{{#parameters.0}} +
+ {{{type.specName.0.value}}} + {{{id}}} +

{{{description}}}

+
+{{/parameters.0}} + +{{#return}} +
{{__global.returns}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/return}} + +{{#typeParameters.0}} +
{{__global.typeParameters}}
+ +{{/typeParameters.0}} +{{#typeParameters}} + + + +{{/typeParameters}} +{{#typeParameters.0}} +
+ {{{id}}} +

{{{description}}}

+
+{{/typeParameters.0}} + +{{#fieldValue}} +
{{__global.fieldValue}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/fieldValue}} + +{{#propertyValue}} +
{{__global.propertyValue}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/propertyValue}} + +{{#eventType}} +
{{__global.eventType}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/eventType}} +{{/syntax}} + +{{#overridden}} +
{{__global.overrides}}
+
+{{/overridden}} + +{{#implements.0}} +
{{__global.implements}}
+{{/implements.0}} +{{#implements}} + {{#definition}} +
+ {{/definition}} + {{^definition}} +
+ {{/definition}} +{{/implements}} + +{{#remarks}} +
{{__global.remarks}}
+
{{{remarks}}}
+{{/remarks}} + +{{#example.0}} +
{{__global.examples}}
+{{/example.0}} +{{#example}} +{{{.}}} +{{/example}} + +{{#exceptions.0}} +
{{__global.exceptions}}
+ +{{/exceptions.0}} +{{#exceptions}} + + + +{{/exceptions}} +{{#exceptions.0}} +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/exceptions.0}} + +{{#seealso.0}} +
{{__global.seealso}}
+
+{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
{{{type.specName.0.value}}}
+ {{/isCref}} + {{^isCref}} +
{{{url}}}
+ {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
+{{/seealso.0}} +
+{{/children}} +{{/children}} + +{{#extensionMethods.0}} +

{{__global.extensionMethods}}

+{{/extensionMethods.0}} +{{#extensionMethods}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/extensionMethods}} + +{{#seealso.0}} +

{{__global.seealso}}

+
+{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
{{{type.specName.0.value}}}
+ {{/isCref}} + {{^isCref}} +
{{{url}}}
+ {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
+{{/seealso.0}} diff --git a/eng/common/docgeneration/templates/matthews/partials/enum.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/enum.tmpl.partial new file mode 100644 index 0000000000000..91e7ede1ac83d --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/enum.tmpl.partial @@ -0,0 +1,24 @@ +{{>partials/class.header}} +{{#children}} + {{#children}} +

{{name.0.value}}

+ +
+

{{{summary}}}

+
+ {{/children}} +{{/children}} + +{{#extensionMethods.0}} +

{{__global.extensionMethods}}

+{{/extensionMethods.0}} +{{#extensionMethods}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/extensionMethods}} diff --git a/eng/common/docgeneration/templates/matthews/partials/namespace.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/namespace.tmpl.partial new file mode 100644 index 0000000000000..f607a3dc61bd7 --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/namespace.tmpl.partial @@ -0,0 +1,17 @@ +

{{>partials/title}}

+
{{{summary}}}
+
{{{conceptual}}}
+
{{{remarks}}}
+{{#children}} +

{{>partials/namespaceSubtitle}}

+ + {{#children}} + + + + {{/children}} +
+

+

{{{summary}}}

+
+{{/children}} diff --git a/eng/common/docgeneration/templates/matthews/styles/main.css b/eng/common/docgeneration/templates/matthews/styles/main.css new file mode 100644 index 0000000000000..21eb0c3567f36 --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/styles/main.css @@ -0,0 +1,302 @@ +@import url("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css"); + +/* Clickability fix for selector on sm devices */ +@media (min-width: 768px) and (max-width: 991px) { + article h1:first-of-type:before { + height: 0; + margin-top: 0; + } +} + +#search { + border: none; +} + +.fa-code { + font-size: 19px; +} + +.sidetoc, +body .toc, +.sidefilter, +.sidetoggle { + background-color: #f9fbe7; +} + +.sidenav, +.toc-toggle { + padding: 0; +} + +.sidetoggle { + padding-bottom: 15px; +} + +/* Remove center align from Navbar and Collapsible section */ +.collapse.in, +.collapsing { + text-align: unset; +} + +article h4 { + border-bottom: none; + line-height: normal; +} + +@media (min-width: 768px) { + .sidetoc, .sidefilter { + margin-left: -15px; + } +} + +@media (max-width: 767px) { + .navbar-collapse { + text-align: center !important; + } + + .navbar-collapse li .active { + border-radius: 20px; + } +} + +/* Collapsible Sections + ------------------------------------------------------- */ +.expander:after { + font-family: 'Glyphicons Halflings'; + content: "\e260"; + margin-left: 5px; + color: grey; + font-size: small; +} + +.expander.collapsed:after { + content: "\e259"; +} + +/* Floating buttons + ------------------------------------------------------- */ +.fab { + width: 40px; + height: 40px; + text-align: center; + padding: 11px 0 0 0; + border: none; + outline: none; + color: #FFF; + border-radius: 100%; + box-shadow: 0 3px 6px rgba(0,0,0,0.16), 0 3px 6px rgba(0,0,0,0.23); + transition:.3s; +} + +.fab:hover { + transform: scale(1.1); +} + +.fab + .fab { + margin-right: 15px; +} + +.contribution-panel { + z-index: 1000; + position: fixed; + right: 30px; + top: 70px; +} + +/* Bootstrap docs like sidebar + ------------------------------------------------------- */ +.affix h5 { + display: none; +} + +/* active & hover links */ +.affix ul > li > a:hover, +.affix ul > li.active > a, +.affix ul > li > a:focus { + color: #563d7c; + text-decoration: none; + background-color: transparent; + border-left-color: #563d7c; +} + +/* all active links */ +.affix ul > li.active > a, +.affix ul > li.active:hover > a, +.affix ul > li.active:focus >a { + font-weight: 700; +} + +/* nested active links */ +.affix ul ul > li.active > a, +.affix ul ul > li.active:hover > a, +.affix ul ul > li.active:focus > a { + font-weight: 500; +} + +/* all links */ +.affix ul > li > a { + color: #999; + border-left: 2px solid transparent; + padding: 4px 20px; + font-size: 13px; + font-weight: 400; +} + +/* nested links */ +.affix ul ul > li > a { + padding-top: 1px; + padding-bottom: 1px; + padding-left: 30px; + font-size: 12px; +} + +/* hide inactive nested list */ +.affix ul ul { + display: none; +} + +/* show active nested list */ +.affix ul > li.active > ul { + display: block; +} + +.affix > ul > li > a:before { + content: ''; +} + +.affix ul ul > li > a:before { + content: ''; +} + +/* Style Buttons + ------------------------------------------------------- */ +.btn-warning { + background-color: #0071c5; +} + +.btn-info { + background-color: #0071c5; +} + +/* Navbar Hamburger + ------------------------------------------------------- */ +.icon-bar { + transition: 0.4s; +} + +/* Rotate first bar */ +.change .icon-bar:nth-of-type(2) { + transform: rotate(-45deg) translate(-4px, 5px) ; +} + +/* Fade out the second bar */ +.change .icon-bar:nth-of-type(3) { + opacity: 0; +} + +/* Rotate last bar */ +.change .icon-bar:nth-of-type(4) { + transform: rotate(45deg) translate(-4px, -5px) ; +} + +/* Custom Navbar + ------------------------------------------------------- */ +.navbar-inverse { + background-color: #0071c5; + opacity: 0.95; + border-color: #0071c5; +} +.navbar-inverse .navbar-brand { + color: #ffffff; +} +.navbar-inverse .navbar-brand:hover, +.navbar-inverse .navbar-brand:focus { + color: #ecdbff; +} +.navbar-inverse .navbar-text { + color: #ffffff; +} +.navbar-inverse .navbar-nav > li > a { + color: #ffffff; +} +.navbar-inverse .navbar-nav > li > a:hover, +.navbar-inverse .navbar-nav > li > a:focus { + color: #ecdbff; +} +.navbar-inverse .navbar-nav > .active > a, +.navbar-inverse .navbar-nav > .active > a:hover, +.navbar-inverse .navbar-nav > .active > a:focus { + color: #ecdbff; + background-color: #0071c5; +} +.navbar-inverse .navbar-nav > .open > a, +.navbar-inverse .navbar-nav > .open > a:hover, +.navbar-inverse .navbar-nav > .open > a:focus { + color: #ecdbff; + background-color: #0071c5; +} +.navbar-inverse .navbar-toggle { + border-color: #0071c5; +} +.navbar-inverse .navbar-toggle:hover, +.navbar-inverse .navbar-toggle:focus { + background-color: #0071c5; +} +.navbar-inverse .navbar-toggle .icon-bar { + background-color: #ffffff; +} +.navbar-inverse .navbar-collapse, +.navbar-inverse .navbar-form { + border: none; +} +.navbar-inverse .navbar-link { + color: #ffffff; +} +.navbar-inverse .navbar-link:hover { + color: #ecdbff; +} +.versionarrow { + margin-left: 0.8em; + margin-top: -1.5em; + margin-bottom: -1em; + padding: 1em; +} + +.versionarrow::before { + position: absolute; + content: ''; + width: 0; + height: 0; + border: .5em solid transparent; + border-left-color: gray; + transform-origin: 0 50%; + transition: transform .1s; + margin-top: 0.2em; +} + + +.versionarrow.disable { + text-decoration: line-through; +} + +.versionarrow.down::before { + transform: rotate(90deg); + margin-top: 0em; + transition: transform .1s; +} + +@media (max-width: 767px) { + .navbar-inverse .navbar-nav .open .dropdown-menu > li > a { + color: #ffffff; + } + .navbar-inverse .navbar-nav .open .dropdown-menu > li > a:hover, + .navbar-inverse .navbar-nav .open .dropdown-menu > li > a:focus { + color: #ecdbff; + } + .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a, + .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a:hover, + .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a:focus { + color: #ecdbff; + background-color: #0071c5; + } +} \ No newline at end of file diff --git a/eng/common/docgeneration/templates/matthews/styles/main.js b/eng/common/docgeneration/templates/matthews/styles/main.js new file mode 100644 index 0000000000000..73ccd77747d0a --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/styles/main.js @@ -0,0 +1,224 @@ +// Use container fluid +var containers = $(".container"); +containers.removeClass("container"); +containers.addClass("container-fluid"); + +WINDOW_CONTENTS = window.location.href.split('/') +var SELECTED_LANGUAGE = '' +var INDEX_HTML = '' +var PACKAGE_REGEX = '' +var PACKAGE_REPLACEMENT = '' + +ATTR1 = '[System.ComponentModel.EditorBrowsable]\n<' + +// Navbar Hamburger +$(function () { + $(".navbar-toggle").click(function () { + $(this).toggleClass("change"); + }) +}) + +// Select list to replace affix on small screens +$(function () { + var navItems = $(".sideaffix .level1 > li"); + + if (navItems.length == 0) { + return; + } + + var selector = $("