From 4529c18d65457d1754f3fc39ea50783e4994182a Mon Sep 17 00:00:00 2001 From: Tom Care Date: Thu, 27 Feb 2020 11:19:26 -0800 Subject: [PATCH] Rename pipeline YAMLs for clarity (#212) * Rename pipeline and pipeline template YAMLs to new convention * Rename references to pipelines and pipeline templates * Sort pipeline entries in code_description.md --- .../{azdo-abtest-pipeline.yml => abtest.yml} | 14 +++++++------- ...e-pipeline.yml => code-quality-template.yml} | 0 .pipelines/diabetes_regression-ci-image.yml | 2 +- ...ild-train.yml => diabetes_regression-ci.yml} | 12 ++++++------ ...s_regression-get-model-version-template.yml} | 0 ... diabetes_regression-variables-template.yml} | 0 ...lm-install.yml => helm-install-template.yml} | 0 ...lm-upgrade.yml => helm-upgrade-template.yml} | 2 +- .pipelines/{azdo-pr-build-train.yml => pr.yml} | 4 ++-- bootstrap/bootstrap.py | 10 +++++----- docs/canary_ab_deployment.md | 4 ++-- docs/code_description.md | 17 +++++++---------- docs/custom_container.md | 4 ++-- docs/getting_started.md | 14 +++++++------- ....yml => iac-create-environment-pipeline.yml} | 0 ....yml => iac-remove-environment-pipeline.yml} | 0 16 files changed, 40 insertions(+), 43 deletions(-) rename .pipelines/{azdo-abtest-pipeline.yml => abtest.yml} (93%) rename .pipelines/{azdo-base-pipeline.yml => code-quality-template.yml} (100%) rename .pipelines/{diabetes_regression-ci-build-train.yml => diabetes_regression-ci.yml} (95%) rename .pipelines/{diabetes_regression-template-get-model-version.yml => diabetes_regression-get-model-version-template.yml} (100%) rename .pipelines/{diabetes_regression-variables.yml => diabetes_regression-variables-template.yml} (100%) rename .pipelines/{azdo-helm-install.yml => helm-install-template.yml} (100%) rename .pipelines/{azdo-helm-upgrade.yml => helm-upgrade-template.yml} (93%) rename .pipelines/{azdo-pr-build-train.yml => pr.yml} (78%) rename environment_setup/{iac-create-environment.yml => iac-create-environment-pipeline.yml} (100%) rename environment_setup/{iac-remove-environment.yml => iac-remove-environment-pipeline.yml} (100%) diff --git a/.pipelines/azdo-abtest-pipeline.yml b/.pipelines/abtest.yml similarity index 93% rename from .pipelines/azdo-abtest-pipeline.yml rename to .pipelines/abtest.yml index d6728d81..0d64e8c8 100644 --- a/.pipelines/azdo-abtest-pipeline.yml +++ b/.pipelines/abtest.yml @@ -69,7 +69,7 @@ stages: IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)" echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION" displayName: 'Get Image Location' - - template: azdo-helm-upgrade.yml + - template: helm-upgrade-template.yml parameters: chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model' releaseName: $(blueReleaseName) @@ -81,7 +81,7 @@ stages: displayName: 50 50 rollout to blue environment timeoutInMinutes: 0 steps: - - template: azdo-helm-upgrade.yml + - template: helm-upgrade-template.yml parameters: chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio' releaseName: 'abtest-istio' @@ -96,7 +96,7 @@ stages: runOnce: deploy: steps: - - template: azdo-helm-upgrade.yml + - template: helm-upgrade-template.yml parameters: chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio' releaseName: 'abtest-istio' @@ -113,7 +113,7 @@ stages: runOnce: deploy: steps: - - template: azdo-helm-upgrade.yml + - template: helm-upgrade-template.yml parameters: chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio' releaseName: 'abtest-istio' @@ -134,7 +134,7 @@ stages: IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)" echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION" displayName: 'Get Image Location' - - template: azdo-helm-upgrade.yml + - template: helm-upgrade-template.yml parameters: chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model' releaseName: $(greenReleaseName) @@ -145,7 +145,7 @@ stages: - job: 'Prod_Rollout_100' timeoutInMinutes: 0 steps: - - template: azdo-helm-upgrade.yml + - template: helm-upgrade-template.yml parameters: chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio' releaseName: 'abtest-istio' @@ -157,7 +157,7 @@ stages: - job: 'blue_disable' timeoutInMinutes: 0 steps: - - template: azdo-helm-install.yml + - template: helm-install-template.yml - task: HelmDeploy@0 displayName: 'helm uninstall blue' inputs: diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/code-quality-template.yml similarity index 100% rename from .pipelines/azdo-base-pipeline.yml rename to .pipelines/code-quality-template.yml diff --git a/.pipelines/diabetes_regression-ci-image.yml b/.pipelines/diabetes_regression-ci-image.yml index 63041db5..6282fd31 100644 --- a/.pipelines/diabetes_regression-ci-image.yml +++ b/.pipelines/diabetes_regression-ci-image.yml @@ -1,4 +1,4 @@ -# Builds the container image that is used by other pipelines for scoring. +# Pipeline for building the container image that is used by other pipelines for scoring. resources: containers: diff --git a/.pipelines/diabetes_regression-ci-build-train.yml b/.pipelines/diabetes_regression-ci.yml similarity index 95% rename from .pipelines/diabetes_regression-ci-build-train.yml rename to .pipelines/diabetes_regression-ci.yml index 96cf1d26..b3504dfe 100644 --- a/.pipelines/diabetes_regression-ci-build-train.yml +++ b/.pipelines/diabetes_regression-ci.yml @@ -18,7 +18,7 @@ trigger: - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py variables: -- template: diabetes_regression-variables.yml +- template: diabetes_regression-variables-template.yml - group: devopsforai-aml-vg pool: @@ -34,7 +34,7 @@ stages: container: mlops timeoutInMinutes: 0 steps: - - template: azdo-base-pipeline.yml + - template: code-quality-template.yml - task: AzureCLI@1 inputs: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' @@ -93,7 +93,7 @@ stages: container: mlops timeoutInMinutes: 0 steps: - - template: diabetes_regression-template-get-model-version.yml + - template: diabetes_regression-get-model-version-template.yml - stage: 'Deploy_ACI' displayName: 'Deploy to ACI' @@ -105,7 +105,7 @@ stages: container: mlops timeoutInMinutes: 0 steps: - - template: diabetes_regression-template-get-model-version.yml + - template: diabetes_regression-get-model-version-template.yml - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0 displayName: 'Azure ML Model Deploy' inputs: @@ -138,7 +138,7 @@ stages: container: mlops timeoutInMinutes: 0 steps: - - template: diabetes_regression-template-get-model-version.yml + - template: diabetes_regression-get-model-version-template.yml - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0 displayName: 'Azure ML Model Deploy' inputs: @@ -172,7 +172,7 @@ stages: container: mlops timeoutInMinutes: 0 steps: - - template: diabetes_regression-template-get-model-version.yml + - template: diabetes_regression-get-model-version-template.yml - task: AzureCLI@1 displayName: 'Create scoring image and set IMAGE_LOCATION variable' inputs: diff --git a/.pipelines/diabetes_regression-template-get-model-version.yml b/.pipelines/diabetes_regression-get-model-version-template.yml similarity index 100% rename from .pipelines/diabetes_regression-template-get-model-version.yml rename to .pipelines/diabetes_regression-get-model-version-template.yml diff --git a/.pipelines/diabetes_regression-variables.yml b/.pipelines/diabetes_regression-variables-template.yml similarity index 100% rename from .pipelines/diabetes_regression-variables.yml rename to .pipelines/diabetes_regression-variables-template.yml diff --git a/.pipelines/azdo-helm-install.yml b/.pipelines/helm-install-template.yml similarity index 100% rename from .pipelines/azdo-helm-install.yml rename to .pipelines/helm-install-template.yml diff --git a/.pipelines/azdo-helm-upgrade.yml b/.pipelines/helm-upgrade-template.yml similarity index 93% rename from .pipelines/azdo-helm-upgrade.yml rename to .pipelines/helm-upgrade-template.yml index 11b9616b..4f75c8ed 100644 --- a/.pipelines/azdo-helm-upgrade.yml +++ b/.pipelines/helm-upgrade-template.yml @@ -5,7 +5,7 @@ parameters: overrideValues: '' steps: -- template: azdo-helm-install.yml +- template: helm-install-template.yml - task: HelmDeploy@0 displayName: 'helm upgrade' inputs: diff --git a/.pipelines/azdo-pr-build-train.yml b/.pipelines/pr.yml similarity index 78% rename from .pipelines/azdo-pr-build-train.yml rename to .pipelines/pr.yml index 9c48202a..765a5fef 100644 --- a/.pipelines/azdo-pr-build-train.yml +++ b/.pipelines/pr.yml @@ -17,8 +17,8 @@ pool: container: mlops variables: -- template: diabetes_regression-variables.yml +- template: diabetes_regression-variables-template.yml - group: devopsforai-aml-vg steps: -- template: azdo-base-pipeline.yml +- template: code-quality-template.yml diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 571d3443..92af061b 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -90,12 +90,12 @@ def validateargs(self): def replaceprojectname(project_dir, project_name, rename_name): # Replace instances of rename_name within files with project_name dirs = [r".env.example", - r".pipelines/azdo-base-pipeline.yml", - r".pipelines/azdo-pr-build-train.yml", - r".pipelines/diabetes_regression-ci-build-train.yml", + r".pipelines/code-quality-template.yml", + r".pipelines/pr.yml", + r".pipelines/diabetes_regression-ci.yml", r".pipelines/diabetes_regression-ci-image.yml", - r".pipelines/diabetes_regression-template-get-model-version.yml", # NOQA: E501 - r".pipelines/diabetes_regression-variables.yml", + r".pipelines/diabetes_regression-get-model-version-template.yml", # NOQA: E501 + r".pipelines/diabetes_regression-variables-template.yml", r"environment_setup/Dockerfile", r"environment_setup/install_requirements.sh", r"ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py", # NOQA: E501 diff --git a/docs/canary_ab_deployment.md b/docs/canary_ab_deployment.md index 7e5703b4..4a85ea15 100644 --- a/docs/canary_ab_deployment.md +++ b/docs/canary_ab_deployment.md @@ -30,7 +30,7 @@ There are some extra variables that you need to setup in ***devopsforai-aml-vg** #### 3. Configure a pipeline to build and deploy a scoring Image -Import and run the [azdo-abtest-pipeline.yml](./.pipelines/azdo-abtest-pipeline.yml) multistage deployment pipeline. +Import and run the [abtest.yml](./.pipelines/abtest.yml) multistage deployment pipeline. The result of the pipeline will be a registered Docker image in the ACR repository attached to the AML Service: @@ -46,7 +46,7 @@ model-green 1/1 1 1 19h #### 4. Build a new Scoring Image -Change value of the ***SCORE_SCRIPT*** variable in the [azdo-abtest-pipeline.yml](./.pipelines/azdo-abtest-pipeline.yml) to point to ***scoring/scoreA.py*** and merge it to the master branch. +Change value of the ***SCORE_SCRIPT*** variable in the [abtest.yml](./.pipelines/abtest.yml) to point to ***scoring/scoreA.py*** and merge it to the master branch. **Note:** ***scoreA.py*** and ***scoreB.py*** files used in this tutorial are just mockups returning either "New Model A" or "New Model B" respectively. They are used to demonstrate the concept of testing two scoring images with different models or scoring code. In real life you would implement a scoring file similar to [score.py](./../code/scoring/score.py) (see [getting started](./getting_started.md)). diff --git a/docs/code_description.md b/docs/code_description.md index 3ef54603..b09cf0a1 100644 --- a/docs/code_description.md +++ b/docs/code_description.md @@ -43,15 +43,13 @@ The repository provides a template with folders structure suitable for maintaini ### Pipelines -- `.pipelines/azdo-base-pipeline.yml` : a pipeline template used by ci-build-train pipeline and pr-build-train pipelines. It contains steps performing linting, data and unit testing. -- `.pipelines/diabetes_regression-ci-build-train.yml` : a pipeline triggered when the code is merged into **master**. It performs linting, data integrity testing, unit testing, building and publishing an ML pipeline. -- `.pipelines/azdo-pr-build-train.yml` : a pipeline triggered when a **pull request** to the **master** branch is created. It performs linting, data integrity testing and unit testing only. -- `.pipelines/diabetes_regression-ci-image.yml` : a pipeline building a scoring image for the diabetes regression model. -- `.pipelines/diabetes_regression-template-get-model-version.yml` : a pipeline template used by the `.pipelines/diabetes_regression-ci-build-train.yml` pipeline. It finds out if a new model was registered and retrieves a version of the new model. -- `.pipelines/azdo-abtest-pipeline.yml` : a pipeline demonstrating [Canary deployment strategy](./docs/canary_ab_deployment.md). -- `.pipelines/azdo-helm-*.yml` : pipeline templates used by the `.pipelines/azdo-abtest-pipeline.yml` pipeline. - - +- `.pipelines/abtest.yml` : a pipeline demonstrating [Canary deployment strategy](./docs/canary_ab_deployment.md). +- `.pipelines/code-quality-template.yml` : a pipeline template used by the CI and PR pipelines. It contains steps performing linting, data and unit testing. +- `.pipelines/diabetes_regression-ci-image.yml` : a pipeline building a scoring image for the diabetes regression model. +- `.pipelines/diabetes_regression-ci.yml` : a pipeline triggered when the code is merged into **master**. It performs linting, data integrity testing, unit testing, building and publishing an ML pipeline. +- `.pipelines/diabetes_regression-get-model-version-template.yml` : a pipeline template used by the `.pipelines/diabetes_regression-ci.yml` pipeline. It finds out if a new model was registered and retrieves a version of the new model. +- `.pipelines/helm-*.yml` : pipeline templates used by the `.pipelines/abtest.yml` pipeline. +- `.pipelines/pr.yml` : a pipeline triggered when a **pull request** to the **master** branch is created. It performs linting, data integrity testing and unit testing only. ### ML Services @@ -90,4 +88,3 @@ The repository provides a template with folders structure suitable for maintaini - `diabetes_regression/scoring/score.py` : a scoring script which is about to be packed into a Docker Image along with a model while being deployed to QA/Prod environment. - `diabetes_regression/scoring/inference_config.yml`, `deployment_config_aci.yml`, `deployment_config_aks.yml` : configuration files for the [AML Model Deploy](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.private-vss-services-azureml&ssr=false#overview) pipeline task for ACI and AKS deployment targets. - `diabetes_regression/scoring/scoreA.py`, `diabetes_regression/scoring/scoreB.py` : simplified scoring files for the [Canary deployment sample](./docs/canary_ab_deployment.md). - diff --git a/docs/custom_container.md b/docs/custom_container.md index fcc0e449..94851d84 100644 --- a/docs/custom_container.md +++ b/docs/custom_container.md @@ -57,7 +57,7 @@ run from that image. ## Modify the model pipeline -Modify the model pipeline file [diabetes_regression-ci-build-train.yml](../.pipelines/diabetes_regression-ci-build-train.yml) by replacing this section: +Modify the model pipeline file [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml) by replacing this section: ``` resources: @@ -94,6 +94,6 @@ A better approach would be to use a distinct name for your modified environment, By changing the name of the image in your branch in both the container build pipeline [environment_setup/docker-image-pipeline.yml](../environment_setup/docker-image-pipeline.yml) and the model pipeline file -[diabetes_regression-ci-build-train.yml](../.pipelines/diabetes_regression-ci-build-train.yml), +[diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml), and running both pipelines in sequence on your branch, you avoid any branch conflicts, and the name does not have to be changed after merging to master. diff --git a/docs/getting_started.md b/docs/getting_started.md index 472a7a3f..5bbfe0bb 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -58,7 +58,7 @@ the BASE_NAME value should not exceed 10 characters and it should contain number The **RESOURCE_GROUP** parameter is used as the name for the resource group that will hold the Azure resources for the solution. If providing an existing AML Workspace, set this value to the corresponding resource group name. -The **AZURE_RM_SVC_CONNECTION** parameter is used by the [Azure DevOps pipeline]((../environment_setup/iac-create-environment.yml)) that creates the Azure ML workspace and associated resources through Azure Resource Manager. The pipeline requires an **Azure Resource Manager** +The **AZURE_RM_SVC_CONNECTION** parameter is used by the [Azure DevOps pipeline]((../environment_setup/iac-create-environment-pipeline.yml)) that creates the Azure ML workspace and associated resources through Azure Resource Manager. The pipeline requires an **Azure Resource Manager** [service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection). ![create service connection](./images/create-rm-service-connection.png) @@ -88,7 +88,7 @@ For instructions on how to set up a local development environment, refer to the ### Azure DevOps configuration -For using Azure DevOps Pipelines all other variables are stored in the file `.pipelines/diabetes_regression-variables.yml`. Using the default values as a starting point, adjust the variables to suit your requirements. +For using Azure DevOps Pipelines all other variables are stored in the file `.pipelines/diabetes_regression-variables-template.yml`. Using the default values as a starting point, adjust the variables to suit your requirements. **Note:** In `diabetes_regression` folder you can find `config.json` file that we would recommend to use in order to provide parameters for training, evaluation and scoring scripts. An example of a such parameter is a hyperparameter of a training algorithm: in our case it's the ridge regression [*alpha* hyperparameter](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html). We don't provide any special serializers for this config file. So, it's up to you which template to support there. @@ -102,7 +102,7 @@ Up until now you should have: The easiest way to create all required resources (Resource Group, ML Workspace, Container Registry, Storage Account, etc.) is to leverage an -"Infrastructure as Code" [pipeline in this repository](../environment_setup/iac-create-environment.yml). This **IaC** pipeline takes care of setting up +"Infrastructure as Code" [pipeline in this repository](../environment_setup/iac-create-environment-pipeline.yml). This **IaC** pipeline takes care of setting up all required resources based on these [ARM templates](../environment_setup/arm-templates/cloud-environment.json). ### Create a Build IaC Pipeline @@ -111,7 +111,7 @@ In your Azure DevOps project, create a build pipeline from your forked repositor ![build connnect step](./images/build-connect.png) -Select the **Existing Azure Pipelines YAML file** option and set the path to [/environment_setup/iac-create-environment.yml](../environment_setup/iac-create-environment.yml): +Select the **Existing Azure Pipelines YAML file** option and set the path to [/environment_setup/iac-create-environment-pipeline.yml](../environment_setup/iac-create-environment-pipeline.yml): ![configure step](./images/select-iac-pipeline.png) @@ -123,7 +123,7 @@ Check out the newly created resources in the [Azure Portal](https://portal.azure ![created resources](./images/created-resources.png) -(Optional) To remove the resources created for this project you can use the [/environment_setup/iac-remove-environment.yml](../environment_setup/iac-remove-environment.yml) definition or you can just delete the resource group in the [Azure Portal](https://portal.azure.com). +(Optional) To remove the resources created for this project you can use the [/environment_setup/iac-remove-environment-pipeline.yml](../environment_setup/iac-remove-environment-pipeline.yml) definition or you can just delete the resource group in the [Azure Portal](https://portal.azure.com). **Note:** The training ML pipeline uses a [sample diabetes dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) as training data. To use your own data, you need to [create a Dataset](https://docs.microsoft.com/azure/machine-learning/how-to-create-register-datasets) in your workspace and specify its name in a DATASET_NAME variable in the ***devopsforai-aml-vg*** variable group. You will also need to modify the test cases in the **ml_service/util/smoke_test_scoring_service.py** script to match the schema of the training features in your dataset. @@ -157,7 +157,7 @@ performs linting, unit testing and publishes a training pipeline. ### Set up the Pipeline In your [Azure DevOps](https://dev.azure.com) project create and run a new build -pipeline referring to the [diabetes_regression-ci-build-train.yml](../.pipelines/diabetes_regression-ci-build-train.yml) +pipeline referring to the [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml) pipeline definition in your forked repository: ![configure ci build pipeline](./images/ci-build-pipeline-configure.png) @@ -199,7 +199,7 @@ Wait until the pipeline finishes and verify that there is a new model in the **M ![trained model](./images/trained-model.png) -To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci-build-train.yml` pipeline to `false`. This can also be overridden at runtime execution of the pipeline. +To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci.yml` pipeline to `false`. This can also be overridden at runtime execution of the pipeline. To skip model training and registration, and deploy a model successfully registered by a previous build (for testing changes to the score file or inference configuration), add the variable `MODEL_BUILD_ID` when the pipeline is queued, and set the value to the id of the previous build. diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment-pipeline.yml similarity index 100% rename from environment_setup/iac-create-environment.yml rename to environment_setup/iac-create-environment-pipeline.yml diff --git a/environment_setup/iac-remove-environment.yml b/environment_setup/iac-remove-environment-pipeline.yml similarity index 100% rename from environment_setup/iac-remove-environment.yml rename to environment_setup/iac-remove-environment-pipeline.yml