Rename pipeline YAMLs for clarity (#212)

* Rename pipeline and pipeline template YAMLs to new convention

* Rename references to pipelines and pipeline templates

* Sort pipeline entries in code_description.md

.pipelines/azdo-abtest-pipeline.yml → .pipelines/abtest.yml

@@ -69,7 +69,7 @@ stages:
               IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)"
               echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
             displayName: 'Get Image Location'
-          - template: azdo-helm-upgrade.yml
+          - template: helm-upgrade-template.yml
             parameters:
               chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model'
               releaseName: $(blueReleaseName) 
@@ -81,7 +81,7 @@ stages:
     displayName: 50 50 rollout to blue environment
     timeoutInMinutes: 0
     steps:
-    - template: azdo-helm-upgrade.yml
+    - template: helm-upgrade-template.yml
       parameters:
         chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio'
         releaseName: 'abtest-istio' 
@@ -96,7 +96,7 @@ stages:
       runOnce:
         deploy:
           steps:
-          - template: azdo-helm-upgrade.yml
+          - template: helm-upgrade-template.yml
             parameters:
               chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio'
               releaseName: 'abtest-istio' 
@@ -113,7 +113,7 @@ stages:
       runOnce:
         deploy:
           steps:
-          - template: azdo-helm-upgrade.yml
+          - template: helm-upgrade-template.yml
             parameters:
               chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio'
               releaseName: 'abtest-istio' 
@@ -134,7 +134,7 @@ stages:
               IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)"
               echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
             displayName: 'Get Image Location'
-          - template: azdo-helm-upgrade.yml
+          - template: helm-upgrade-template.yml
             parameters:
               chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model'
               releaseName: $(greenReleaseName) 
@@ -145,7 +145,7 @@ stages:
   - job: 'Prod_Rollout_100'
     timeoutInMinutes: 0
     steps:
-    - template: azdo-helm-upgrade.yml
+    - template: helm-upgrade-template.yml
       parameters:
         chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio'
         releaseName: 'abtest-istio' 
@@ -157,7 +157,7 @@ stages:
   - job: 'blue_disable'
     timeoutInMinutes: 0
     steps:
-    - template: azdo-helm-install.yml
+    - template: helm-install-template.yml
     - task: HelmDeploy@0
       displayName: 'helm uninstall blue'
       inputs:

.pipelines/diabetes_regression-ci-image.yml

@@ -1,4 +1,4 @@
-# Builds the container image that is used by other pipelines for scoring.
+# Pipeline for building the container image that is used by other pipelines for scoring.
 
 resources:
   containers:

.pipelines/diabetes_regression-ci-build-train.yml → .pipelines/diabetes_regression-ci.yml

@@ -18,7 +18,7 @@ trigger:
     - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py
 
 variables:
-- template: diabetes_regression-variables.yml
+- template: diabetes_regression-variables-template.yml
 - group: devopsforai-aml-vg
 
 pool:
@@ -34,7 +34,7 @@ stages:
     container: mlops
     timeoutInMinutes: 0
     steps:
-    - template: azdo-base-pipeline.yml
+    - template: code-quality-template.yml
     - task: AzureCLI@1
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
@@ -93,7 +93,7 @@ stages:
     container: mlops
     timeoutInMinutes: 0
     steps:
-    - template: diabetes_regression-template-get-model-version.yml
+    - template: diabetes_regression-get-model-version-template.yml
 
 - stage: 'Deploy_ACI'
   displayName: 'Deploy to ACI'
@@ -105,7 +105,7 @@ stages:
     container: mlops
     timeoutInMinutes: 0
     steps:
-    - template: diabetes_regression-template-get-model-version.yml
+    - template: diabetes_regression-get-model-version-template.yml
     - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
       displayName: 'Azure ML Model Deploy'
       inputs:
@@ -138,7 +138,7 @@ stages:
     container: mlops
     timeoutInMinutes: 0
     steps:
-    - template: diabetes_regression-template-get-model-version.yml
+    - template: diabetes_regression-get-model-version-template.yml
     - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
       displayName: 'Azure ML Model Deploy'
       inputs:
@@ -172,7 +172,7 @@ stages:
     container: mlops
     timeoutInMinutes: 0
     steps:
-    - template: diabetes_regression-template-get-model-version.yml
+    - template: diabetes_regression-get-model-version-template.yml
     - task: AzureCLI@1
       displayName: 'Create scoring image and set IMAGE_LOCATION variable'
       inputs:

.pipelines/azdo-helm-upgrade.yml → .pipelines/helm-upgrade-template.yml

@@ -5,7 +5,7 @@ parameters:
   overrideValues: ''
 
 steps:
-- template: azdo-helm-install.yml
+- template: helm-install-template.yml
 - task: HelmDeploy@0
   displayName: 'helm upgrade'
   inputs:

.pipelines/azdo-pr-build-train.yml → .pipelines/pr.yml

@@ -17,8 +17,8 @@ pool:
 container: mlops
 
 variables:
-- template: diabetes_regression-variables.yml
+- template: diabetes_regression-variables-template.yml
 - group: devopsforai-aml-vg
 
 steps:
-- template: azdo-base-pipeline.yml
+- template: code-quality-template.yml

bootstrap/bootstrap.py

@@ -90,12 +90,12 @@ def validateargs(self):
 def replaceprojectname(project_dir, project_name, rename_name):
     # Replace instances of rename_name within files with project_name
     dirs = [r".env.example",
-            r".pipelines/azdo-base-pipeline.yml",
-            r".pipelines/azdo-pr-build-train.yml",
-            r".pipelines/diabetes_regression-ci-build-train.yml",
+            r".pipelines/code-quality-template.yml",
+            r".pipelines/pr.yml",
+            r".pipelines/diabetes_regression-ci.yml",
             r".pipelines/diabetes_regression-ci-image.yml",
-            r".pipelines/diabetes_regression-template-get-model-version.yml",  # NOQA: E501
-            r".pipelines/diabetes_regression-variables.yml",
+            r".pipelines/diabetes_regression-get-model-version-template.yml",  # NOQA: E501
+            r".pipelines/diabetes_regression-variables-template.yml",
             r"environment_setup/Dockerfile",
             r"environment_setup/install_requirements.sh",
             r"ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py",  # NOQA: E501

docs/canary_ab_deployment.md

@@ -30,7 +30,7 @@ There are some extra variables that you need to setup in ***devopsforai-aml-vg**
 
 #### 3. Configure a pipeline to build and deploy a scoring Image
 
-Import and run the [azdo-abtest-pipeline.yml](./.pipelines/azdo-abtest-pipeline.yml) multistage deployment pipeline.
+Import and run the [abtest.yml](./.pipelines/abtest.yml) multistage deployment pipeline.
 
 The result of the pipeline will be a registered Docker image in the ACR repository attached to the AML Service:
 
@@ -46,7 +46,7 @@ model-green   1/1     1            1           19h
 
 #### 4. Build a new Scoring Image
 
-Change value of the ***SCORE_SCRIPT*** variable in the [azdo-abtest-pipeline.yml](./.pipelines/azdo-abtest-pipeline.yml) to point to ***scoring/scoreA.py*** and merge it to the master branch.
+Change value of the ***SCORE_SCRIPT*** variable in the [abtest.yml](./.pipelines/abtest.yml) to point to ***scoring/scoreA.py*** and merge it to the master branch.
 
 **Note:** ***scoreA.py*** and ***scoreB.py*** files used in this tutorial are just mockups returning either "New Model A" or "New Model B" respectively. They are used to demonstrate the concept of testing two scoring images with different models or scoring code. In real life you would implement a scoring file similar to [score.py](./../code/scoring/score.py) (see [getting started](./getting_started.md)).
 

docs/code_description.md

@@ -43,15 +43,13 @@ The repository provides a template with folders structure suitable for maintaini
 
 ### Pipelines
 
-- `.pipelines/azdo-base-pipeline.yml` : a pipeline template used by ci-build-train pipeline and pr-build-train pipelines. It contains steps performing linting, data and unit testing.  
-- `.pipelines/diabetes_regression-ci-build-train.yml` : a pipeline triggered when the code is merged into **master**. It performs linting, data integrity testing, unit testing, building and publishing an ML pipeline.
-- `.pipelines/azdo-pr-build-train.yml` : a pipeline triggered when a **pull request** to the **master** branch is created. It performs linting, data integrity testing and unit testing only.
-- `.pipelines/diabetes_regression-ci-image.yml` : a pipeline building a scoring image for the diabetes regression model. 
-- `.pipelines/diabetes_regression-template-get-model-version.yml` : a pipeline template used by the `.pipelines/diabetes_regression-ci-build-train.yml` pipeline. It finds out if a new model was registered and retrieves a version of the new model.
-- `.pipelines/azdo-abtest-pipeline.yml` : a pipeline demonstrating [Canary deployment strategy](./docs/canary_ab_deployment.md).
-- `.pipelines/azdo-helm-*.yml` : pipeline templates used by the `.pipelines/azdo-abtest-pipeline.yml` pipeline.
-
-
+- `.pipelines/abtest.yml` : a pipeline demonstrating [Canary deployment strategy](./docs/canary_ab_deployment.md).
+- `.pipelines/code-quality-template.yml` : a pipeline template used by the CI and PR pipelines. It contains steps performing linting, data and unit testing.
+- `.pipelines/diabetes_regression-ci-image.yml` : a pipeline building a scoring image for the diabetes regression model.
+- `.pipelines/diabetes_regression-ci.yml` : a pipeline triggered when the code is merged into **master**. It performs linting, data integrity testing, unit testing, building and publishing an ML pipeline.
+- `.pipelines/diabetes_regression-get-model-version-template.yml` : a pipeline template used by the `.pipelines/diabetes_regression-ci.yml` pipeline. It finds out if a new model was registered and retrieves a version of the new model.
+- `.pipelines/helm-*.yml` : pipeline templates used by the `.pipelines/abtest.yml` pipeline.
+- `.pipelines/pr.yml` : a pipeline triggered when a **pull request** to the **master** branch is created. It performs linting, data integrity testing and unit testing only.
 
 ### ML Services
 
@@ -90,4 +88,3 @@ The repository provides a template with folders structure suitable for maintaini
 - `diabetes_regression/scoring/score.py` : a scoring script which is about to be packed into a Docker Image along with a model while being deployed to QA/Prod environment.
 - `diabetes_regression/scoring/inference_config.yml`, `deployment_config_aci.yml`, `deployment_config_aks.yml` : configuration files for the [AML Model Deploy](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.private-vss-services-azureml&ssr=false#overview) pipeline task for ACI and AKS deployment targets.
 - `diabetes_regression/scoring/scoreA.py`, `diabetes_regression/scoring/scoreB.py` : simplified scoring files for the [Canary deployment sample](./docs/canary_ab_deployment.md).
-

docs/custom_container.md

@@ -57,7 +57,7 @@ run from that image.
 
 ## Modify the model pipeline
 
-Modify the model pipeline file [diabetes_regression-ci-build-train.yml](../.pipelines/diabetes_regression-ci-build-train.yml) by replacing this section:
+Modify the model pipeline file [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml) by replacing this section:
 
 ```
 resources:
@@ -94,6 +94,6 @@ A better approach would be to use a distinct name for your modified environment,
 By changing the name of the image in your branch in both the container build pipeline
 [environment_setup/docker-image-pipeline.yml](../environment_setup/docker-image-pipeline.yml)
 and the model pipeline file
-[diabetes_regression-ci-build-train.yml](../.pipelines/diabetes_regression-ci-build-train.yml),
+[diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml),
 and running both pipelines in sequence on your branch,
 you avoid any branch conflicts, and the name does not have to be changed after merging to master.

docs/getting_started.md

@@ -58,7 +58,7 @@ the BASE_NAME value should not exceed 10 characters and it should contain number
 
 The **RESOURCE_GROUP** parameter is used as the name for the resource group that will hold the Azure resources for the solution. If providing an existing AML Workspace, set this value to the corresponding resource group name.
 
-The **AZURE_RM_SVC_CONNECTION** parameter is used by the [Azure DevOps pipeline]((../environment_setup/iac-create-environment.yml)) that creates the Azure ML workspace and associated resources through Azure Resource Manager. The pipeline requires an **Azure Resource Manager**
+The **AZURE_RM_SVC_CONNECTION** parameter is used by the [Azure DevOps pipeline]((../environment_setup/iac-create-environment-pipeline.yml)) that creates the Azure ML workspace and associated resources through Azure Resource Manager. The pipeline requires an **Azure Resource Manager**
 [service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection).
 
 ![create service connection](./images/create-rm-service-connection.png)
@@ -88,7 +88,7 @@ For instructions on how to set up a local development environment, refer to the
 
 ### Azure DevOps configuration
 
-For using Azure DevOps Pipelines all other variables are stored in the file `.pipelines/diabetes_regression-variables.yml`. Using the default values as a starting point, adjust the variables to suit your requirements.
+For using Azure DevOps Pipelines all other variables are stored in the file `.pipelines/diabetes_regression-variables-template.yml`. Using the default values as a starting point, adjust the variables to suit your requirements.
 
 **Note:** In `diabetes_regression` folder you can find `config.json` file that we would recommend to use in order to provide parameters for training, evaluation and scoring scripts. An example of a such parameter is a hyperparameter of a training algorithm: in our case it's the ridge regression [*alpha* hyperparameter](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html). We don't provide any special serializers for this config file. So, it's up to you which template to support there.
 
@@ -102,7 +102,7 @@ Up until now you should have:
 
 The easiest way to create all required resources (Resource Group, ML Workspace,
 Container Registry, Storage Account, etc.) is to leverage an
-"Infrastructure as Code" [pipeline in this repository](../environment_setup/iac-create-environment.yml). This **IaC** pipeline takes care of setting up
+"Infrastructure as Code" [pipeline in this repository](../environment_setup/iac-create-environment-pipeline.yml). This **IaC** pipeline takes care of setting up
 all required resources based on these [ARM templates](../environment_setup/arm-templates/cloud-environment.json).
 
 ### Create a Build IaC Pipeline
@@ -111,7 +111,7 @@ In your Azure DevOps project, create a build pipeline from your forked repositor
 
 ![build connnect step](./images/build-connect.png)
 
-Select the **Existing Azure Pipelines YAML file** option and set the path to [/environment_setup/iac-create-environment.yml](../environment_setup/iac-create-environment.yml):
+Select the **Existing Azure Pipelines YAML file** option and set the path to [/environment_setup/iac-create-environment-pipeline.yml](../environment_setup/iac-create-environment-pipeline.yml):
 
 ![configure step](./images/select-iac-pipeline.png)
 
@@ -123,7 +123,7 @@ Check out the newly created resources in the [Azure Portal](https://portal.azure
 
 ![created resources](./images/created-resources.png)
 
-(Optional) To remove the resources created for this project you can use the [/environment_setup/iac-remove-environment.yml](../environment_setup/iac-remove-environment.yml) definition or you can just delete the resource group in the [Azure Portal](https://portal.azure.com).
+(Optional) To remove the resources created for this project you can use the [/environment_setup/iac-remove-environment-pipeline.yml](../environment_setup/iac-remove-environment-pipeline.yml) definition or you can just delete the resource group in the [Azure Portal](https://portal.azure.com).
 
 **Note:** The training ML pipeline uses a [sample diabetes dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) as training data. To use your own data, you need to [create a Dataset](https://docs.microsoft.com/azure/machine-learning/how-to-create-register-datasets) in your workspace and specify its name in a DATASET_NAME variable in the ***devopsforai-aml-vg*** variable group. You will also need to modify the test cases in the **ml_service/util/smoke_test_scoring_service.py** script to match the schema of the training features in your dataset.
 
@@ -157,7 +157,7 @@ performs linting, unit testing and publishes a training pipeline.
 ### Set up the Pipeline
 
 In your [Azure DevOps](https://dev.azure.com) project create and run a new build
-pipeline referring to the [diabetes_regression-ci-build-train.yml](../.pipelines/diabetes_regression-ci-build-train.yml)
+pipeline referring to the [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml)
 pipeline definition in your forked repository:
 
 ![configure ci build pipeline](./images/ci-build-pipeline-configure.png)
@@ -199,7 +199,7 @@ Wait until the pipeline finishes and verify that there is a new model in the **M
 
 ![trained model](./images/trained-model.png)
 
-To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci-build-train.yml` pipeline to `false`.  This can also be overridden at runtime execution of the pipeline.
+To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci.yml` pipeline to `false`.  This can also be overridden at runtime execution of the pipeline.
 
 To skip model training and registration, and deploy a model successfully registered by a previous build (for testing changes to the score file or inference configuration), add the variable `MODEL_BUILD_ID` when the pipeline is queued, and set the value to the id of the previous build.