Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

{Packaging/MSI} Use embeddable python for MSI #14300

Merged
merged 6 commits into from
Jul 30, 2020
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@
*.bat eol=crlf
# The az script for Git Bash/Cygwin should be LF
build_scripts/windows/scripts/az eol=lf
# sh scripts should be LF
*.sh eol=lf
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why need this? Will git applicaiton get conflict with it?

Copy link
Member Author

@fengzhou-msft fengzhou-msft Jul 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would like to reuse the scripts/ci/build.sh to run on Git Bash. Without this setting, all files checked out on Windows will be converted to crlf and will have error when run with bash.

12 changes: 6 additions & 6 deletions build_scripts/windows/scripts/build.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ if "%CLI_VERSION%"=="" (
echo Please set the CLI_VERSION environment variable, e.g. 2.0.13
goto ERROR
)
set PYTHON_VERSION=3.6.6
set PYTHON_VERSION=3.6.8

set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip"
set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python366-32.zip"
set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-32.zip"
set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip"

:: Set up the output directory and temp. directories
Expand All @@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR%
set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch
set BUILDING_DIR=%ARTIFACTS_DIR%\cli
set WIX_DIR=%ARTIFACTS_DIR%\wix
set PYTHON_DIR=%ARTIFACTS_DIR%\Python366-32
set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-32
set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change

set REPO_ROOT=%~dp0..\..\..
Expand Down Expand Up @@ -75,10 +75,10 @@ if not exist %PYTHON_DIR% (
mkdir %PYTHON_DIR%
pushd %PYTHON_DIR%
echo Downloading Python.
curl -o Python366-32.zip %PYTHON_DOWNLOAD_URL% -k
unzip -q Python366-32.zip
curl -o Python368-32.zip %PYTHON_DOWNLOAD_URL% -k
unzip -q Python368-32.zip
if %errorlevel% neq 0 goto ERROR
del Python366-32.zip
del Python368-32.zip
echo Python downloaded and extracted successfully.
popd
)
Expand Down
21 changes: 21 additions & 0 deletions build_scripts/windows/scripts/setup_msi_test.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Prerequisites:
# 1. Install the MSI built with current branch
# 2. Run bash azure-cli\scripts\ci\build.sh with Git Bash first to generate artifacts under azure-cli\artifacts\build so we can use the testsdk and fulltest wheels.

# Elevate to Admin
If (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator"))
{
$arguments = "& '" + $myinvocation.mycommand.definition + "'"
Start-Process powershell -Verb runAs -ArgumentList $arguments
}

& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest
& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest-xdist

$testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_testsdk*.whl | Select-Object Name
& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($testsdk.Name)

$fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name
& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name)

& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py
25 changes: 25 additions & 0 deletions build_scripts/windows/scripts/test_msi_package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

# Invoke this script in Powershell with:
# & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' test_msi_package.py

import os
import sys
import subprocess

root_dir = 'C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\CLI2\\Lib\\site-packages\\azure\\cli\\command_modules'

mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__']

for mod_name in mod_list:
exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN',
'--junit-xml', '{}\\azure_cli_test_result\\{}.xml'.format(os.path.expanduser('~'), mod_name), '-n', 'auto', '--pyargs', 'azure.cli.command_modules.{}'.format(mod_name)])
if exit_code != 0 and exit_code != 5: # exit code is 5 when there is no tests collected in the module
sys.exit(exit_code)

exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN',
'--junit-xml', '{}\\azure_cli_test_result\\azure-cli-core.xml'.format(os.path.expanduser('~')), '-n', 'auto', '--pyargs', 'azure.cli.core', '--import-mode=append'])
sys.exit(exit_code)
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ class AmsSpTests(ScenarioTest):
@StorageAccountPreparer(parameter_name='storage_account_for_create')
@AllowLargeResponse()
def test_ams_sp_create_reset(self, resource_group, storage_account_for_create):
with mock.patch('azure.cli.command_modules.ams._utils._gen_guid', side_effect=self.create_guid):
with mock.patch('azure.cli.command_modules.ams.operations.sp._gen_guid', side_effect=self.create_guid):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

amsname = self.create_random_name(prefix='ams', length=12)

self.kwargs.update({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,16 +69,16 @@ def test_ams_streaming_endpoint_create_with_akamai(self, storage_account_for_cre
'scaleUnits': 4,
'tags': 'foo=bar',
'ips': '1.1.1.1 2.2.2.2',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'identifier': 'id1',
'expiration': '2030-12-31T16:00:00-08:00',
'base64Key': 'dGVzdGlkMQ=='
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --ips {ips} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --ips {ips} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('name', '{streamingEndpointName}'),
self.check('resourceGroup', '{rg}'),
self.check('location', 'North Europe'),
Expand Down Expand Up @@ -124,14 +124,14 @@ def test_ams_streaming_endpoint_update(self, storage_account_for_create):
'maxCacheAge': 11,
'scaleUnits': 5,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'ip': '4.4.4.4'
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('name', '{streamingEndpointName}'),
self.check('resourceGroup', '{rg}'),
self.check('location', 'Australia East'),
Expand All @@ -155,12 +155,12 @@ def test_ams_streaming_endpoint_update(self, storage_account_for_create):
'description': 'test streaming description2',
'maxCacheAge': 9,
'tags': 'foo2=bar2 foo3=bar3',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'ips': '1.1.1.1 2.2.2.2 192.168.0.0/28'
})

self.cmd('az ams streaming-endpoint update -g {rg} -a {amsname} -n {streamingEndpointName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --tags {tags} --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint update -g {rg} -a {amsname} -n {streamingEndpointName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --tags {tags} --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('name', '{streamingEndpointName}'),
self.check('cdnProvider', '{cdnProvider}'),
self.check('cdnProfile', '{cdnProfile}'),
Expand Down Expand Up @@ -201,13 +201,13 @@ def test_ams_streaming_endpoint_create(self, storage_account_for_create):
'maxCacheAge': 11,
'scaleUnits': 6,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('name', '{streamingEndpointName}'),
self.check('resourceGroup', '{rg}'),
self.check('location', 'Canada Central'),
Expand Down Expand Up @@ -240,13 +240,13 @@ def test_ams_streaming_endpoint_show(self, storage_account_for_show):
'maxCacheAge': 11,
'scaleUnits': 7,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"')
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"')

self.cmd('az ams streaming-endpoint show -g {rg} -a {amsname} -n {streamingEndpointName}', checks=[
self.check('name', '{streamingEndpointName}'),
Expand Down Expand Up @@ -288,13 +288,13 @@ def test_ams_streaming_endpoint_delete(self, storage_account_for_delete):
'maxCacheAge': 11,
'scaleUnits': 8,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName1} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"')
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName1} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"')

self.cmd('az ams streaming-endpoint list -g {rg} -a {amsname}', checks=[
self.check('length(@)', 2)
Expand Down Expand Up @@ -325,13 +325,13 @@ def test_ams_streaming_endpoint_scale(self, storage_account_for_scale):
'scaleUnits': 9,
'scaleUnits2': 10,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('scaleUnits', '{scaleUnits}')
])

Expand Down Expand Up @@ -468,4 +468,5 @@ def test_ams_streaming_endpoint_list(self, storage_account_for_create):
# Helper functions

def _normalize_filename(cmd, string):
return '"' + string.replace('\\', '/') + '"'
import platform
return '"' + string.replace('\\', '/') + '"' if platform.system() != 'Windows' else string