[ci] Paralize azure pipeline (sonic-net#1040)

1. Setup pipeline without manual effort when checkout new release branch. 2. Use correct branch when downloading artifacts or checkout relative repos. 3. Clear downloaded artifacts to avoid using outdated dependencies. 4. Use commonlib pipeline to download libnl3 and libyang instead of vs image build, to increase success rate. 5. Add weekly build to keep artifacts remaining.
Pterosaur · May 18, 2022 · 564bea7 · 564bea7
1 parent 57ed180
commit 564bea7
Show file tree

Hide file tree

Showing 5 changed files with 137 additions and 69 deletions.
diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml
@@ -32,40 +32,48 @@ jobs:
     vmImage: 'ubuntu-20.04'
 
   steps:
+  - checkout: self
+    clean: true
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
+      path: $(Build.ArtifactStagingDirectory)/download
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.sairedis_artifact_name }}
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic sairedis deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.swss_artifact_name }}
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss artifact"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 142
+      pipeline: Azure.sonic-buildimage.official.vs
       artifact: sonic-buildimage.vs
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-    displayName: "Download sonic buildimage"
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: '**/target/docker-sonic-vs.gz'
+    displayName: "Download sonic-buildimage docker-sonic-vs"
   - script: |
+      set -ex
       echo $(Build.DefinitionName).$(Build.BuildNumber)
 
-      docker load < ../target/docker-sonic-vs.gz
+      docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
 
       mkdir -p .azure-pipelines/docker-sonic-vs/debs
 
-      cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs
+      cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
 
       pushd .azure-pipelines
 
@@ -74,7 +82,8 @@ jobs:
       popd
 
       docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
-    displayName: "Build sonic-docker-vs"
+      rm -rf $(Build.ArtifactStagingDirectory)/download
+    displayName: "Build docker-sonic-vs"
 
   - publish: $(Build.ArtifactStagingDirectory)/
     artifact: ${{ parameters.artifact_name }}

diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml
@@ -21,6 +21,9 @@ parameters:
 - name: sonic_slave
   type: string
 
+- name: debian_version
+  type: string
+
 - name: sairedis_artifact_name
   type: string
 
@@ -38,13 +41,22 @@ jobs:
   pool:
     ${{ if ne(parameters.pool, 'default') }}:
       name: ${{ parameters.pool }}
-    ${{ if eq(parameters.pool, 'default') }}:
+    ${{ else }}:
       vmImage: 'ubuntu-20.04'
 
   container:
     image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest
 
   steps:
+  - checkout: sonic-swss
+    submodules: true
+    clean: true
+  - script: |
+      set -ex
+      git checkout $(BUILD_BRANCH)
+      git submodule update
+      git status
+    displayName: Set up sonic-swss branch
   - script: |
       sudo apt-get install -y libhiredis0.14 libhiredis-dev
       sudo apt-get install -y libzmq5 libzmq3-dev
@@ -60,51 +72,57 @@ jobs:
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        libswsscommon_1.0.0_*.deb
+        libswsscommon-dev_1.0.0*.deb
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.sairedis_artifact_name }}
-    displayName: "Download sonic sairedis deb packages"
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        libsaivs_*.deb
+        libsaivs-dev_*.deb
+        libsairedis_*.deb
+        libsairedis-dev_*.deb
+        libsaimetadata_*.deb
+        libsaimetadata-dev_*.deb
+        syncd-vs_*.deb
+    displayName: "Download pre-stage built ${{ parameters.sairedis_artifact_name }}"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 1
-      artifact: sonic-buildimage.vs
+      pipeline: Azure.sonic-buildimage.common_libs
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-    displayName: "Download sonic buildimage"
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      artifact: common-lib
+      patterns: |
+        target/debs/${{ parameters.debian_version }}/libnl-3*.deb
+        target/debs/${{ parameters.debian_version }}/libnl-genl*.deb
+        target/debs/${{ parameters.debian_version }}/libnl-route*.deb
+        target/debs/${{ parameters.debian_version }}/libnl-nf*.deb
+    displayName: "Download common libs"
+
   - script: |
-      sudo dpkg -i target/debs/buster/libnl-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb
-      sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
-      sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
-      sudo dpkg -i libsaivs_*.deb
-      sudo dpkg -i libsaivs-dev_*.deb
-      sudo dpkg -i libsairedis_*.deb
-      sudo dpkg -i libsairedis-dev_*.deb
-      sudo dpkg -i libsaimetadata_*.deb
-      sudo dpkg -i libsaimetadata-dev_*.deb
-      sudo dpkg -i syncd-vs_*.deb
-    workingDirectory: $(Pipeline.Workspace)
+      set -ex
+      sudo dpkg -i $(find ./download -name *.deb)
+      rm -rf download || true
+    workingDirectory: $(Build.ArtifactStagingDirectory)
     displayName: "Install libnl3, sonic swss common, and sairedis"
-  - checkout: sonic-swss
-    path: s
-    submodules: true
   - script: |
+      set -ex
+      rm ../*.deb || true
       ./autogen.sh
-      dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
+      dpkg-buildpackage -us -uc -b -j$(nproc)
+      mv ../*.deb $(Build.ArtifactStagingDirectory)
     displayName: "Compile sonic swss"
-  - publish: $(System.DefaultWorkingDirectory)/
+  - publish: $(Build.ArtifactStagingDirectory)
     artifact: ${{ parameters.artifact_name }}
     displayName: "Archive swss debian packages"
diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml
@@ -51,7 +51,7 @@ jobs:
   pool:
     ${{ if ne(parameters.pool, 'default') }}:
       name: ${{ parameters.pool }}
-    ${{ if eq(parameters.pool, 'default') }}:
+    ${{ else }}:
       vmImage: 'ubuntu-20.04'
 
   container:
@@ -97,22 +97,25 @@ jobs:
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-      path: '$(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss common deb packages"
   - script: |
-      cd $(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}
-      sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
-      sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
-    workingDirectory: $(Pipeline.Workspace)
+      set -ex
+      sudo dpkg -i download/libswsscommon_1.0.0_${{ parameters.arch }}.deb
+      sudo dpkg -i download/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
+      rm -rf download
+    workingDirectory: $(Build.ArtifactStagingDirectory)
     displayName: "Install sonic swss Common"
   - script: |
       set -ex
+      rm ../*.deb || true
       ./autogen.sh
-      fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs && cp ../*.deb .
+      fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs
+      mv ../*.deb .
     displayName: "Compile sonic sairedis with coverage enabled"
   - script: |
       sudo cp azsyslog.conf /etc/rsyslog.conf

diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml
@@ -18,33 +18,39 @@ jobs:
       ls -A1 | xargs -I{} sudo rm -rf {}
     displayName: "Clean workspace"
   - checkout: self
+    clean: true
+    displayName: "Checkout sonic-sairedis"
+  - checkout: sonic-swss
+    clean: true
+    displayName: "Checkout sonic-swss"
+  - script: |
+      set -ex
+      cd sonic-swss
+      git checkout $(BUILD_BRANCH)
+    displayName: Set up sonic-swss branch
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: docker-sonic-vs
-    displayName: "Download docker sonic vs image"
-
+      path: $(Build.ArtifactStagingDirectory)/download
+    displayName: "Download pre-stage built docker-sonic-vs"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: sonic-swss-common.amd64.ubuntu20_04
+      path: $(Build.ArtifactStagingDirectory)/download
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
     displayName: "Download sonic swss common deb packages"
 
-  - checkout: self
-    displayName: "Checkout sonic-sairedis"
-  - checkout: sonic-swss
-    displayName: "Checkout sonic-swss"
-
   - script: |
-      set -x
+      set -ex
       sudo sonic-sairedis/.azure-pipelines/build_and_install_module.sh
 
       sudo apt-get install -y libhiredis0.14
-      sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
-      sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
+      sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
 
       # install packages for vs test
       sudo apt-get install -y net-tools bridge-utils vlan
@@ -53,12 +59,13 @@ jobs:
     displayName: "Install dependencies"
 
   - script: |
-      set -x
-      sudo docker load -i ../docker-sonic-vs.gz
+      set -ex
+      sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz
       docker ps
       ip netns list
       pushd sonic-swss/tests
       sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+      rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"
 
   - task: PublishTestResults@2

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -3,10 +3,29 @@
 # Add steps that build, run tests, deploy, and more:
 # https://aka.ms/yaml
 
+pr:
+- master
+- 202???
+- 201???
+
 trigger:
+  batch: true
   branches:
     include:
-      - "*"
+    - master
+    - 202???
+    - 201???
+
+# this part need to be set in UI
+schedules:
+- cron: "0 0 * * 6"
+  displayName: Weekly build
+  branches:
+    include:
+    - master
+    - 202???
+    - 201???
+  always: true
 
 resources:
   repositories:
@@ -15,14 +34,25 @@ resources:
     name: Azure/sonic-swss
     endpoint: build
 
+parameters:
+  - name: debian_version
+    type: string
+    default: buster
+variables:
+  - name: BUILD_BRANCH
+    ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+      value: $(System.PullRequest.TargetBranch)
+    ${{ else }}:
+      value: $(Build.SourceBranchName)
+
 stages:
 - stage: Build
 
   jobs:
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: amd64
-      sonic_slave: sonic-slave-buster
+      sonic_slave: sonic-slave-${{ parameters.debian_version }}
       swss_common_artifact_name: sonic-swss-common
       artifact_name: sonic-sairedis
       syslog_artifact_name: sonic-sairedis.syslog
@@ -36,19 +66,19 @@ stages:
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: armhf
-      timeout: 240
+      timeout: 180
       pool: sonicbld-armhf
-      sonic_slave: sonic-slave-buster-armhf
+      sonic_slave: sonic-slave-${{ parameters.debian_version }}-armhf
       swss_common_artifact_name: sonic-swss-common.armhf
       artifact_name: sonic-sairedis.armhf
       syslog_artifact_name: sonic-sairedis.syslog.armhf
 
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: arm64
-      timeout: 240
+      timeout: 180
       pool: sonicbld-arm64
-      sonic_slave: sonic-slave-buster-arm64
+      sonic_slave: sonic-slave-${{ parameters.debian_version }}-arm64
       swss_common_artifact_name: sonic-swss-common.arm64
       artifact_name: sonic-sairedis.arm64
       syslog_artifact_name: sonic-sairedis.syslog.arm64
@@ -60,10 +90,11 @@ stages:
   - template: .azure-pipelines/build-swss-template.yml
     parameters:
       arch: amd64
-      sonic_slave: sonic-slave-buster
+      sonic_slave: sonic-slave-${{ parameters.debian_version }}
       swss_common_artifact_name: sonic-swss-common
       sairedis_artifact_name: sonic-sairedis
       artifact_name: sonic-swss
+      debian_version: ${{ parameters.debian_version }}
 
 - stage: BuildDocker
   dependsOn: BuildSwss