From ad8603b33c45dcd64960190e060e23ede4a0d9eb Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Tue, 18 May 2021 14:00:15 -0700 Subject: [PATCH 01/13] first sketch of the change fixes removed extra blank lines changes regarding param renaming added ssl tests added more detail to the unit test additional import moved pem files out of public folder fixed import chenged import changed import unit tests fix unit test fix fixed unit tests fixed unit test unit test fix changes int test cert and key --- .../partner_extensions/AzureMLKubernetes.py | 55 ++++++++++++++----- .../latest/data/cert_and_key_encoded.txt | 2 + .../tests/latest/data/test_cert.pem | 1 + .../tests/latest/data/test_key.pem | 1 + .../tests/latest/test_azureml_extension.py | 32 +++++++++++ testing/test/extensions/data/test_cert.pem | 1 + testing/test/extensions/data/test_key.pem | 1 + .../public/AzureMLKubernetes.Tests.ps1 | 45 +++++++++++++++ 8 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 src/k8s-extension/azext_k8s_extension/tests/latest/data/cert_and_key_encoded.txt create mode 100644 src/k8s-extension/azext_k8s_extension/tests/latest/data/test_cert.pem create mode 100644 src/k8s-extension/azext_k8s_extension/tests/latest/data/test_key.pem create mode 100644 src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py create mode 100644 testing/test/extensions/data/test_cert.pem create mode 100644 testing/test/extensions/data/test_key.pem diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index d907e7d3c6..51479c86e6 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -66,6 +66,11 @@ def __init__(self): self.SERVICE_BUS_JOB_STATE_TOPIC = 'jobstate-updatedby-computeprovider' self.SERVICE_BUS_JOB_STATE_SUB = 'compute-scheduler-jobstate' + # constants for enableing SSL in inference + self.sslKeyPemFile = 'sslKeyPemFile' + self.sslCertPemFile = 'sslCertPemFile' + self.allowInsecureConnections = 'allowInsecureConnections' + # reference mapping self.reference_mapping = { self.RELAY_SERVER_CONNECTION_STRING: [self.RELAY_CONNECTION_STRING_KEY, self.RELAY_CONNECTION_STRING_DEPRECATED_KEY], @@ -168,6 +173,7 @@ def __validate_config(self, configuration_settings, configuration_protected_sett if enable_inference: logger.warning("The installed AzureML extension for AML inference is experimental and not covered by customer support. Please use with discretion.") self.__validate_scoring_fe_settings(configuration_settings, configuration_protected_settings) + self.__set_up_inference_ssl(configuration_settings, configuration_protected_settings) elif not (enable_training or enable_inference): raise InvalidArgumentValueError( "Please create Microsoft.AzureML.Kubernetes extension instance either " @@ -181,32 +187,53 @@ def __validate_config(self, configuration_settings, configuration_protected_sett configuration_protected_settings.pop(self.ENABLE_INFERENCE, None) def __validate_scoring_fe_settings(self, configuration_settings, configuration_protected_settings): - clusterPurpose = _get_value_from_config_protected_config( - 'clusterPurpose', configuration_settings, configuration_protected_settings) - if clusterPurpose and clusterPurpose not in ["DevTest", "FastProd"]: - raise InvalidArgumentValueError( - "Accepted values for '--configuration-settings clusterPurpose' " - "are 'DevTest' and 'FastProd'") - - feSslCert = _get_value_from_config_protected_config( - 'scoringFe.sslCert', configuration_settings, configuration_protected_settings) - sslKey = _get_value_from_config_protected_config( - 'scoringFe.sslKey', configuration_settings, configuration_protected_settings) + experimentalCluster = _get_value_from_config_protected_config( + 'experimental', configuration_settings, configuration_protected_settings) + experimentalCluster = str(experimentalCluster).lower() == 'true' + if experimentalCluster: + configuration_settings['clusterPurpose'] = 'DevTest' + else: + configuration_settings['clusterPurpose'] = 'FastProd' + feSslCertFile = configuration_protected_settings.get(self.sslCertPemFile) + feSslKeyFile = configuration_protected_settings.get(self.sslKeyPemFile) allowInsecureConnections = _get_value_from_config_protected_config( - 'allowInsecureConnections', configuration_settings, configuration_protected_settings) + self.allowInsecureConnections, configuration_settings, configuration_protected_settings) allowInsecureConnections = str(allowInsecureConnections).lower() == 'true' - if (not feSslCert or not sslKey) and not allowInsecureConnections: + if (not feSslCertFile or not feSslKeyFile) and not allowInsecureConnections: raise InvalidArgumentValueError( "Provide ssl certificate and key. " "Otherwise explicitly allow insecure connection by specifying " "'--configuration-settings allowInsecureConnections=true'") feIsInternalLoadBalancer = _get_value_from_config_protected_config( - 'scoringFe.serviceType.internalLoadBalancer', configuration_settings, configuration_protected_settings) + 'privateEndpointILB', configuration_settings, configuration_protected_settings) feIsInternalLoadBalancer = str(feIsInternalLoadBalancer).lower() == 'true' if feIsInternalLoadBalancer: logger.warning( 'Internal load balancer only supported on AKS and AKS Engine Clusters.') + configuration_protected_settings['scoringFe.privateEndpointILB'] = feIsInternalLoadBalancer + + def __set_up_inference_ssl(self, configuration_settings, configuration_protected_settings): + allowInsecureConnections = _get_value_from_config_protected_config( + self.allowInsecureConnections, configuration_settings, configuration_protected_settings) + allowInsecureConnections = str(allowInsecureConnections).lower() == 'true' + if not allowInsecureConnections: + import base64 + feSslCertFile = configuration_protected_settings.get(self.sslCertPemFile) + feSslKeyFile = configuration_protected_settings.get(self.sslKeyPemFile) + with open(feSslCertFile) as f: + cert_data = f.read() + cert_data_bytes = cert_data.encode("ascii") + ssl_cert = base64.b64encode(cert_data_bytes) + configuration_protected_settings['scoringFe.sslCert'] = ssl_cert + with open(feSslKeyFile) as f: + key_data = f.read() + key_data_bytes = key_data.encode("ascii") + ssl_key = base64.b64encode(key_data_bytes) + configuration_protected_settings['scoringFe.sslKey'] = ssl_key + else: + logger.warning( + 'SSL is not set up. Allowing insecure connection to the deployed services') def __create_required_resource( self, cmd, configuration_settings, configuration_protected_settings, subscription_id, resource_group_name, diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/data/cert_and_key_encoded.txt b/src/k8s-extension/azext_k8s_extension/tests/latest/data/cert_and_key_encoded.txt new file mode 100644 index 0000000000..4c2cb46c83 --- /dev/null +++ b/src/k8s-extension/azext_k8s_extension/tests/latest/data/cert_and_key_encoded.txt @@ -0,0 +1,2 @@ +dGVzdGNlcnQ= +dGVzdGtleQ== \ No newline at end of file diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/data/test_cert.pem b/src/k8s-extension/azext_k8s_extension/tests/latest/data/test_cert.pem new file mode 100644 index 0000000000..e7529e3fde --- /dev/null +++ b/src/k8s-extension/azext_k8s_extension/tests/latest/data/test_cert.pem @@ -0,0 +1 @@ +testcert \ No newline at end of file diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/data/test_key.pem b/src/k8s-extension/azext_k8s_extension/tests/latest/data/test_key.pem new file mode 100644 index 0000000000..7ef00201c7 --- /dev/null +++ b/src/k8s-extension/azext_k8s_extension/tests/latest/data/test_key.pem @@ -0,0 +1 @@ +testkey \ No newline at end of file diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py b/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py new file mode 100644 index 0000000000..47c1f52cf2 --- /dev/null +++ b/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py @@ -0,0 +1,32 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +from azext_k8s_extension.partner_extensions.AzureMLKubernetes import AzureMLKubernetes + + +TEST_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), '..')) + + +class TestAzureMlExtension(unittest.TestCase): + + def test_set_up_inference_ssl(self): + azremlk8sInstance = AzureMLKubernetes() + config = {'allowInsecureConnections': 'false'} + # read and encode dummy cert and key + sslKeyPemFile = os.path.join(TEST_DIR, 'data', 'test_key.pem') + sslCertPemFile = os.path.join(TEST_DIR, 'data', 'test_cert.pem') + protected_config = {'sslKeyPemFile': sslKeyPemFile, 'sslCertPemFile': sslCertPemFile} + azremlk8sInstance._AzureMLKubernetes__set_up_inference_ssl(config, protected_config) + self.assertTrue('scoringFe.sslCert' in protected_config) + self.assertTrue('scoringFe.sslKey' in protected_config) + encoded_cert_and_key_file = os.path.join(TEST_DIR, 'data', 'cert_and_key_encoded.txt') + with open(encoded_cert_and_key_file, "rb") as text_file: + cert = text_file.readline().rstrip() + self.assertEquals(cert, protected_config['scoringFe.sslCert']) + key = text_file.readline() + self.assertEquals(key, protected_config['scoringFe.sslKey']) \ No newline at end of file diff --git a/testing/test/extensions/data/test_cert.pem b/testing/test/extensions/data/test_cert.pem new file mode 100644 index 0000000000..e7529e3fde --- /dev/null +++ b/testing/test/extensions/data/test_cert.pem @@ -0,0 +1 @@ +testcert \ No newline at end of file diff --git a/testing/test/extensions/data/test_key.pem b/testing/test/extensions/data/test_key.pem new file mode 100644 index 0000000000..7ef00201c7 --- /dev/null +++ b/testing/test/extensions/data/test_key.pem @@ -0,0 +1 @@ +testkey \ No newline at end of file diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index 35790d5f89..c2c58e376b 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -150,4 +150,49 @@ Describe 'AzureML Kubernetes Testing' { $badOut | Should -Not -BeNullOrEmpty $output | Should -BeNullOrEmpty } + + It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { + Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True clusterPurpose=DevTest --config-protected sslKeyPemFile=../data/test_key.pem sslCertPemFile=../data/test_cert.pem" -ErrorVariable badOut + $badOut | Should -BeNullOrEmpty + + $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + $badOut | Should -BeNullOrEmpty + + $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # Loop and retry until the extension installs + $n = 0 + do + { + if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) { + break + } + Start-Sleep -Seconds 20 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + + # check if relay is populated + $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + $relayResourceID | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster with inference enabled" { + # cleanup the relay and servicebus + $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey + $relayNamespaceName = $relayResourceID.split("/")[8] + $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8] + az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName + az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName + + $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + $badOut | Should -BeNullOrEmpty + + # Extension should not be found on the cluster + $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + $badOut | Should -Not -BeNullOrEmpty + $output | Should -BeNullOrEmpty + } } From 5c07778f6fd70c1399965d0b77012c417f16f19d Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Fri, 21 May 2021 18:22:00 -0700 Subject: [PATCH 02/13] test protected config --- .../azext_k8s_extension/partner_extensions/AzureMLKubernetes.py | 2 ++ testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index 51479c86e6..edd022ebf0 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -194,6 +194,8 @@ def __validate_scoring_fe_settings(self, configuration_settings, configuration_p configuration_settings['clusterPurpose'] = 'DevTest' else: configuration_settings['clusterPurpose'] = 'FastProd' + print(configuration_settings) + print(configuration_protected_settings) feSslCertFile = configuration_protected_settings.get(self.sslCertPemFile) feSslKeyFile = configuration_protected_settings.get(self.sslKeyPemFile) allowInsecureConnections = _get_value_from_config_protected_config( diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index c2c58e376b..f8f79d569a 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -152,7 +152,7 @@ Describe 'AzureML Kubernetes Testing' { } It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True clusterPurpose=DevTest --config-protected sslKeyPemFile=../data/test_key.pem sslCertPemFile=../data/test_cert.pem" -ErrorVariable badOut + Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableIidentity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net clusterPurpose=DevTest --config-protected sslKeyPemFile=../data/test_key.pem sslCertPemFile=../data/test_cert.pem" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut From 23c0338f8923b86f122cf266691756b2610eff0d Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Sun, 23 May 2021 22:49:32 -0700 Subject: [PATCH 03/13] fix test typo --- testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index f8f79d569a..68509edeaa 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -152,7 +152,7 @@ Describe 'AzureML Kubernetes Testing' { } It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableIidentity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net clusterPurpose=DevTest --config-protected sslKeyPemFile=../data/test_key.pem sslCertPemFile=../data/test_cert.pem" -ErrorVariable badOut + Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net experimental=True --config-protected sslKeyPemFile=../data/test_key.pem sslCertPemFile=../data/test_cert.pem" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut From 916691f371041be89857e3fb07717d58a2ad98f2 Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Mon, 24 May 2021 00:59:54 -0700 Subject: [PATCH 04/13] temporary changes reverted --- .../azext_k8s_extension/partner_extensions/AzureMLKubernetes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index edd022ebf0..51479c86e6 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -194,8 +194,6 @@ def __validate_scoring_fe_settings(self, configuration_settings, configuration_p configuration_settings['clusterPurpose'] = 'DevTest' else: configuration_settings['clusterPurpose'] = 'FastProd' - print(configuration_settings) - print(configuration_protected_settings) feSslCertFile = configuration_protected_settings.get(self.sslCertPemFile) feSslKeyFile = configuration_protected_settings.get(self.sslKeyPemFile) allowInsecureConnections = _get_value_from_config_protected_config( From 0bc89c7006f6db9ac78be981dd927a44c6f32ba0 Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Tue, 25 May 2021 00:44:11 -0700 Subject: [PATCH 05/13] fixing tests --- testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index 68509edeaa..d2a5de8440 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -152,7 +152,9 @@ Describe 'AzureML Kubernetes Testing' { } It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net experimental=True --config-protected sslKeyPemFile=../data/test_key.pem sslCertPemFile=../data/test_cert.pem" -ErrorVariable badOut + $sslKeyPemFile = (Split-Path $PSScriptRoot -Parent) + "\data\test_key.pem" + $sslCertPemFile = (Split-Path $PSScriptRoot -Parent) + "\data\test_cert.pem" + Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net experimental=True --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut From 8caf752a2c754765d2309bc05c870ba786186d86 Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Tue, 25 May 2021 10:13:42 -0700 Subject: [PATCH 06/13] fixed file paths --- testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 | 4 ++-- testing/test/extensions/public/test.ps1 | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 testing/test/extensions/public/test.ps1 diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index d2a5de8440..315f34f0f0 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -152,8 +152,8 @@ Describe 'AzureML Kubernetes Testing' { } It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { - $sslKeyPemFile = (Split-Path $PSScriptRoot -Parent) + "\data\test_key.pem" - $sslCertPemFile = (Split-Path $PSScriptRoot -Parent) + "\data\test_cert.pem" + $sslKeyPemFile = Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "test_key.pem" + $sslCertPemFile = Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "test_cert.pem" Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net experimental=True --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty diff --git a/testing/test/extensions/public/test.ps1 b/testing/test/extensions/public/test.ps1 new file mode 100644 index 0000000000..8b5b24d077 --- /dev/null +++ b/testing/test/extensions/public/test.ps1 @@ -0,0 +1,4 @@ +$sslKeyPemFile = Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "test_key.pem" +$sslCertPemFile = Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "test_cert.pem" + +Write-Output $sslKeyPemFile \ No newline at end of file From ae49b99cacad90276fcdfb9e11b4367e6bb56db9 Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Tue, 25 May 2021 11:31:21 -0700 Subject: [PATCH 07/13] removed accidentally added file --- testing/test/extensions/public/test.ps1 | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 testing/test/extensions/public/test.ps1 diff --git a/testing/test/extensions/public/test.ps1 b/testing/test/extensions/public/test.ps1 deleted file mode 100644 index 8b5b24d077..0000000000 --- a/testing/test/extensions/public/test.ps1 +++ /dev/null @@ -1,4 +0,0 @@ -$sslKeyPemFile = Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "test_key.pem" -$sslCertPemFile = Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "test_cert.pem" - -Write-Output $sslKeyPemFile \ No newline at end of file From 26e2a8ec45037c478fb10da3d3f53c15da0d978d Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Thu, 27 May 2021 22:54:35 -0700 Subject: [PATCH 08/13] changes according to review comments --- .../partner_extensions/AzureMLKubernetes.py | 2 +- .../tests/latest/data/{ => azure_ml}/cert_and_key_encoded.txt | 0 .../tests/latest/data/{ => azure_ml}/test_cert.pem | 0 .../tests/latest/data/{ => azure_ml}/test_key.pem | 0 .../tests/latest/test_azureml_extension.py | 4 ++-- testing/test/extensions/data/{ => azure_ml}/test_cert.pem | 0 testing/test/extensions/data/{ => azure_ml}/test_key.pem | 0 testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 | 4 ++-- 8 files changed, 5 insertions(+), 5 deletions(-) rename src/k8s-extension/azext_k8s_extension/tests/latest/data/{ => azure_ml}/cert_and_key_encoded.txt (100%) rename src/k8s-extension/azext_k8s_extension/tests/latest/data/{ => azure_ml}/test_cert.pem (100%) rename src/k8s-extension/azext_k8s_extension/tests/latest/data/{ => azure_ml}/test_key.pem (100%) rename testing/test/extensions/data/{ => azure_ml}/test_cert.pem (100%) rename testing/test/extensions/data/{ => azure_ml}/test_key.pem (100%) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index 51479c86e6..f0fdfbcafd 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -66,7 +66,7 @@ def __init__(self): self.SERVICE_BUS_JOB_STATE_TOPIC = 'jobstate-updatedby-computeprovider' self.SERVICE_BUS_JOB_STATE_SUB = 'compute-scheduler-jobstate' - # constants for enableing SSL in inference + # constants for enabling SSL in inference self.sslKeyPemFile = 'sslKeyPemFile' self.sslCertPemFile = 'sslCertPemFile' self.allowInsecureConnections = 'allowInsecureConnections' diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/data/cert_and_key_encoded.txt b/src/k8s-extension/azext_k8s_extension/tests/latest/data/azure_ml/cert_and_key_encoded.txt similarity index 100% rename from src/k8s-extension/azext_k8s_extension/tests/latest/data/cert_and_key_encoded.txt rename to src/k8s-extension/azext_k8s_extension/tests/latest/data/azure_ml/cert_and_key_encoded.txt diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/data/test_cert.pem b/src/k8s-extension/azext_k8s_extension/tests/latest/data/azure_ml/test_cert.pem similarity index 100% rename from src/k8s-extension/azext_k8s_extension/tests/latest/data/test_cert.pem rename to src/k8s-extension/azext_k8s_extension/tests/latest/data/azure_ml/test_cert.pem diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/data/test_key.pem b/src/k8s-extension/azext_k8s_extension/tests/latest/data/azure_ml/test_key.pem similarity index 100% rename from src/k8s-extension/azext_k8s_extension/tests/latest/data/test_key.pem rename to src/k8s-extension/azext_k8s_extension/tests/latest/data/azure_ml/test_key.pem diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py b/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py index 47c1f52cf2..6f2731cf1c 100644 --- a/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py +++ b/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py @@ -18,8 +18,8 @@ def test_set_up_inference_ssl(self): azremlk8sInstance = AzureMLKubernetes() config = {'allowInsecureConnections': 'false'} # read and encode dummy cert and key - sslKeyPemFile = os.path.join(TEST_DIR, 'data', 'test_key.pem') - sslCertPemFile = os.path.join(TEST_DIR, 'data', 'test_cert.pem') + sslKeyPemFile = os.path.join(TEST_DIR, 'data', 'azure_ml', 'test_key.pem') + sslCertPemFile = os.path.join(TEST_DIR, 'data', 'azure_ml', 'test_cert.pem') protected_config = {'sslKeyPemFile': sslKeyPemFile, 'sslCertPemFile': sslCertPemFile} azremlk8sInstance._AzureMLKubernetes__set_up_inference_ssl(config, protected_config) self.assertTrue('scoringFe.sslCert' in protected_config) diff --git a/testing/test/extensions/data/test_cert.pem b/testing/test/extensions/data/azure_ml/test_cert.pem similarity index 100% rename from testing/test/extensions/data/test_cert.pem rename to testing/test/extensions/data/azure_ml/test_cert.pem diff --git a/testing/test/extensions/data/test_key.pem b/testing/test/extensions/data/azure_ml/test_key.pem similarity index 100% rename from testing/test/extensions/data/test_key.pem rename to testing/test/extensions/data/azure_ml/test_key.pem diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index 315f34f0f0..77b1cbdb34 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -152,8 +152,8 @@ Describe 'AzureML Kubernetes Testing' { } It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { - $sslKeyPemFile = Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "test_key.pem" - $sslCertPemFile = Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "test_cert.pem" + $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem" + $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem" Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net experimental=True --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty From 2b21bfde1cda031cb42cbd7acfc1622bc772ce5e Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Thu, 27 May 2021 23:20:36 -0700 Subject: [PATCH 09/13] more changes according to review comments --- .../azext_k8s_extension/partner_extensions/AzureMLKubernetes.py | 2 +- .../azext_k8s_extension/tests/latest/test_azureml_extension.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index f0fdfbcafd..5f753daf2e 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -233,7 +233,7 @@ def __set_up_inference_ssl(self, configuration_settings, configuration_protected configuration_protected_settings['scoringFe.sslKey'] = ssl_key else: logger.warning( - 'SSL is not set up. Allowing insecure connection to the deployed services') + 'SSL is not enabled. Allowing insecure connections to the deployed services.') def __create_required_resource( self, cmd, configuration_settings, configuration_protected_settings, subscription_id, resource_group_name, diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py b/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py index 6f2731cf1c..26d0b85abf 100644 --- a/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py +++ b/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py @@ -24,7 +24,7 @@ def test_set_up_inference_ssl(self): azremlk8sInstance._AzureMLKubernetes__set_up_inference_ssl(config, protected_config) self.assertTrue('scoringFe.sslCert' in protected_config) self.assertTrue('scoringFe.sslKey' in protected_config) - encoded_cert_and_key_file = os.path.join(TEST_DIR, 'data', 'cert_and_key_encoded.txt') + encoded_cert_and_key_file = os.path.join(TEST_DIR, 'data', 'azure_ml', 'cert_and_key_encoded.txt') with open(encoded_cert_and_key_file, "rb") as text_file: cert = text_file.readline().rstrip() self.assertEquals(cert, protected_config['scoringFe.sslCert']) From 47f601e8b60351f9f111848c8638daa156c6c8cf Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Thu, 27 May 2021 23:30:39 -0700 Subject: [PATCH 10/13] changes according to review comments --- .../partner_extensions/AzureMLKubernetes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index 5f753daf2e..298f08177f 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -70,6 +70,7 @@ def __init__(self): self.sslKeyPemFile = 'sslKeyPemFile' self.sslCertPemFile = 'sslCertPemFile' self.allowInsecureConnections = 'allowInsecureConnections' + self.privateEndpointILB = 'privateEndpointILB' # reference mapping self.reference_mapping = { @@ -206,12 +207,12 @@ def __validate_scoring_fe_settings(self, configuration_settings, configuration_p "'--configuration-settings allowInsecureConnections=true'") feIsInternalLoadBalancer = _get_value_from_config_protected_config( - 'privateEndpointILB', configuration_settings, configuration_protected_settings) + self.privateEndpointILB, configuration_settings, configuration_protected_settings) feIsInternalLoadBalancer = str(feIsInternalLoadBalancer).lower() == 'true' if feIsInternalLoadBalancer: logger.warning( 'Internal load balancer only supported on AKS and AKS Engine Clusters.') - configuration_protected_settings['scoringFe.privateEndpointILB'] = feIsInternalLoadBalancer + configuration_protected_settings['scoringFe.%s' % self.privateEndpointILB] = feIsInternalLoadBalancer def __set_up_inference_ssl(self, configuration_settings, configuration_protected_settings): allowInsecureConnections = _get_value_from_config_protected_config( From 3593c8008303be1a7323c928584c20f3a6a682bd Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Wed, 2 Jun 2021 17:00:31 -0700 Subject: [PATCH 11/13] fixed decode error --- .../partner_extensions/AzureMLKubernetes.py | 4 ++-- .../tests/latest/test_azureml_extension.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index 229abcb749..2438427ac5 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -225,12 +225,12 @@ def __set_up_inference_ssl(self, configuration_settings, configuration_protected with open(feSslCertFile) as f: cert_data = f.read() cert_data_bytes = cert_data.encode("ascii") - ssl_cert = base64.b64encode(cert_data_bytes) + ssl_cert = base64.b64encode(cert_data_bytes).decode() configuration_protected_settings['scoringFe.sslCert'] = ssl_cert with open(feSslKeyFile) as f: key_data = f.read() key_data_bytes = key_data.encode("ascii") - ssl_key = base64.b64encode(key_data_bytes) + ssl_key = base64.b64encode(key_data_bytes).decode() configuration_protected_settings['scoringFe.sslKey'] = ssl_key else: logger.warning( diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py b/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py index 26d0b85abf..6814a57839 100644 --- a/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py +++ b/src/k8s-extension/azext_k8s_extension/tests/latest/test_azureml_extension.py @@ -25,7 +25,7 @@ def test_set_up_inference_ssl(self): self.assertTrue('scoringFe.sslCert' in protected_config) self.assertTrue('scoringFe.sslKey' in protected_config) encoded_cert_and_key_file = os.path.join(TEST_DIR, 'data', 'azure_ml', 'cert_and_key_encoded.txt') - with open(encoded_cert_and_key_file, "rb") as text_file: + with open(encoded_cert_and_key_file, "r") as text_file: cert = text_file.readline().rstrip() self.assertEquals(cert, protected_config['scoringFe.sslCert']) key = text_file.readline() From 4e72f002df6d4acc98f5bbc90970967a9398a9e5 Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Wed, 2 Jun 2021 18:21:04 -0700 Subject: [PATCH 12/13] renamed the experimental param --- .../partner_extensions/AzureMLKubernetes.py | 2 +- testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index 2438427ac5..68e06a8469 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -189,7 +189,7 @@ def __validate_config(self, configuration_settings, configuration_protected_sett def __validate_scoring_fe_settings(self, configuration_settings, configuration_protected_settings): experimentalCluster = _get_value_from_config_protected_config( - 'experimental', configuration_settings, configuration_protected_settings) + 'inferenceLoadBalancerHA', configuration_settings, configuration_protected_settings) experimentalCluster = str(experimentalCluster).lower() == 'true' if experimentalCluster: configuration_settings['clusterPurpose'] = 'DevTest' diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index 77b1cbdb34..ac5573ad95 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -107,7 +107,7 @@ Describe 'AzureML Kubernetes Testing' { } It 'Creates the extension and checks that it onboards correctly with inference enabled' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True clusterPurpose=DevTest" -ErrorVariable badOut + Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True inferenceLoadBalancerHA=true" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut @@ -154,7 +154,7 @@ Describe 'AzureML Kubernetes Testing' { It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem" $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem" - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net experimental=True --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut + Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net inferenceLoadBalancerHA=True --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut From cd28455db7d0d1f9d222876b07d6c57970336072 Mon Sep 17 00:00:00 2001 From: Lia Kazakova Date: Thu, 3 Jun 2021 10:48:15 -0700 Subject: [PATCH 13/13] renamed fe related extension params --- .../partner_extensions/AzureMLKubernetes.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index 68e06a8469..d724822e57 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -71,6 +71,7 @@ def __init__(self): self.sslCertPemFile = 'sslCertPemFile' self.allowInsecureConnections = 'allowInsecureConnections' self.privateEndpointILB = 'privateEndpointILB' + self.privateEndpointNodeport = 'privateEndpointNodeport' # reference mapping self.reference_mapping = { @@ -206,13 +207,23 @@ def __validate_scoring_fe_settings(self, configuration_settings, configuration_p "Otherwise explicitly allow insecure connection by specifying " "'--configuration-settings allowInsecureConnections=true'") + feIsNodePort = _get_value_from_config_protected_config( + self.privateEndpointNodeport, configuration_settings, configuration_protected_settings) + feIsNodePort = str(feIsNodePort).lower() == 'true' feIsInternalLoadBalancer = _get_value_from_config_protected_config( self.privateEndpointILB, configuration_settings, configuration_protected_settings) feIsInternalLoadBalancer = str(feIsInternalLoadBalancer).lower() == 'true' - if feIsInternalLoadBalancer: + + if feIsNodePort and feIsInternalLoadBalancer: + raise InvalidArgumentValueError( + "Specify either privateEndpointNodeport=true or privateEndpointILB=true, but not both.") + elif feIsNodePort: + configuration_settings['scoringFe.serviceType.nodePort'] = feIsNodePort + elif feIsInternalLoadBalancer: + configuration_settings['scoringFe.serviceType.internalLoadBalancer'] = feIsInternalLoadBalancer logger.warning( 'Internal load balancer only supported on AKS and AKS Engine Clusters.') - configuration_protected_settings['scoringFe.%s' % self.privateEndpointILB] = feIsInternalLoadBalancer + def __set_up_inference_ssl(self, configuration_settings, configuration_protected_settings): allowInsecureConnections = _get_value_from_config_protected_config(