[CI] Update sonic image official build template. (#7198)

* draft * fix * Update new-build-template.yml * Update azure-pipelines-build.yml * Update azure-pipelines-job-groups.yml * fix * fix * draft * fix * fix * fix * fix * fix * fix Signed-off-by: Shilong Liu <[email protected]> Co-authored-by: Sonic Automation <[email protected]>
sonic-net · May 12, 2021 · 9cc3721 · 9cc3721
1 parent 413103c
commit 9cc3721
Show file tree

Hide file tree

Showing 4 changed files with 239 additions and 0 deletions.
diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml
@@ -0,0 +1,99 @@
+# The azure pipeline template for Official build, and upgrade version build
+
+parameters:
+- name: 'jobFilters'
+  type: object
+  default: ''
+- name: 'buildOptions'
+  type: string
+  default: 'SONIC_CONFIG_BUILD_JOBS=1'
+- name: 'preSteps'
+  type: stepList
+  default: []
+- name: 'postSteps'
+  type: stepList
+  default: []
+
+jobs:
+- template: azure-pipelines-image-template.yml
+  parameters:
+    jobFilters: ${{ parameters.jobFilters }}
+    preSteps: ${{ parameters.preSteps }}
+    postSteps: ${{ parameters.postSteps }}
+    jobVariables:
+      PLATFORM: $(GROUP_NAME)
+      PLATFORM_ARCH: amd64
+      BUILD_OPTIONS: ${{ parameters.buildOptions }}
+      dbg_image: false
+      swi_image: false
+      raw_image: false
+      sync_rpc_image: false
+      platform_rpc: false
+    jobGroups:
+      - name: vs
+        variables:
+          dbg_image: true
+      - name: barefoot
+        variables:
+          swi_image: true
+      - name: broadcom
+        variables:
+          dbg_image: true
+          swi_image: true
+          raw_image: true
+          sync_rpc_image: true
+          platform_rpc: brcm
+      - name: centec
+        variables:
+          dbg_image: true
+          sync_rpc_image: true
+          platform_rpc: centec
+      - name: centec-arm64
+        pool: sonicbld_8c
+        variables:
+          arch: arm64
+          timeoutInMinutes: 1800
+      - name: generic
+        variables:
+          dbg_image: true
+      - name: innovium
+        variables:
+          swi_image: true
+      - name: marvell-armhf
+        pool: sonicbld_8c
+        variables:
+          arch: armhf
+          timeoutInMinutes: 1800
+      - name: mellanox
+        variables:
+          dbg_image: true
+          sync_rpc_image: true
+          platform_rpc: mlnx
+      - name: nephos
+        variables:
+          dbg_image: true
+          sync_rpc_image: true
+          platform_rpc: nephos
+    buildSteps:
+      - bash: |
+          if [ $(GROUP_NAME) == vs ]; then
+            if [ $(dbg_image) == true ]; then
+              make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
+            fi
+            make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+          else
+            if [ $(dbg_image) == true ]; then
+              make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin && \
+                mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
+            fi
+            if [ $(swi_image) == true ]; then
+              make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
+            fi
+            if [ $(raw_image) == true ]; then
+              make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
+            fi
+            if [ $(sync_rpc_image) == true ]; then
+              make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+            fi
+            make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
+          fi
diff --git a/.azure-pipelines/azure-pipelines-image-template.yml b/.azure-pipelines/azure-pipelines-image-template.yml
@@ -0,0 +1,54 @@
+# The azure pipeline template for PR build, Official build, and upgrade version build
+
+parameters:
+- name: 'jobFilters'
+  type: object
+  default: ''
+- name: 'preSteps'
+  type: stepList
+  default: []
+- name: 'buildSteps'
+  type: stepList
+  default: []
+- name: 'postSteps'
+  type: stepList
+  default: []
+- name: jobGroups
+  type: object
+  default: []
+- name: jobVariables
+  type: object
+  default: []
+jobs:
+- template: azure-pipelines-job-groups.yml
+  parameters:
+    jobFilters: ${{ parameters.jobFilters }}
+    jobVariables: ${{ parameters.jobVariables }}
+    preSteps:
+      - template: cleanup.yml
+      - ${{ parameters. preSteps }}
+      - script: |
+          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM) | grep -E -q "^(vs|broadcom|mellanox)$"; then
+            CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM)"
+            BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
+            echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
+          fi
+        displayName: "Make build options"
+      - checkout: self
+        submodules: recursive
+        displayName: 'Checkout code'
+      - script: |
+          sudo modprobe overlay
+          sudo apt-get install -y acl
+          export DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker
+          sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
+          ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM) PLATFORM_ARCH=$(PLATFORM_ARCH) configure
+        displayName: 'Make configure'
+    postSteps:
+      - publish: $(System.DefaultWorkingDirectory)/target
+        artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
+        displayName: "Archive sonic image"
+      - ${{ parameters.postSteps }}
+      - template: cleanup.yml
+    jobGroups: ${{ parameters.jobGroups }}
+    buildSteps: ${{ parameters.buildSteps }}
diff --git a/.azure-pipelines/azure-pipelines-job-groups.yml b/.azure-pipelines/azure-pipelines-job-groups.yml
@@ -0,0 +1,57 @@
+parameters:
+- name: 'preSteps'
+  type: stepList
+  default: []
+- name: 'buildSteps'
+  type: stepList
+  default: []
+- name: "postSteps"
+  type: stepList
+  default: []
+- name: 'jobGroups'
+  type: object
+  default: []
+- name: 'jobVariables'
+  type: object
+  default: {}
+- name: 'scriptEnv'
+  type: object
+  default: ''
+- name: 'timeoutInMinutes'
+  type: 'number'
+  default: 600
+- name: 'jobFilters'
+  type: object
+  default: ''
+
+jobs:
+- ${{ each jobGroup in parameters.jobGroups }}:
+  - ${{ if or(eq(parameters.jobFilters, ''), containsValue(parameters.jobFilters, jobGroup.name), endswith(variables['Build.DefinitionName'], format('.{0}{1}', jobGroup.name, jobGroup.extName))) }}:
+    - job: ${{ replace(format('{0}{1}', jobGroup.name, jobGroup.extName), '-', '_') }}
+      ${{ each pair in jobGroup }}:
+        ${{ if not(in(pair.key, 'job', 'name', 'extName', 'variables', 'steps', 'script', 'scriptEnv')) }}:
+          ${{ pair.key }}: ${{ pair.value }}
+      ${{ if eq(jobGroup.timeoutInMinutes, '') }}:
+        timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+      variables:
+        GROUP_NAME: ${{ jobGroup.name }}
+        GROUP_EXTNAME: '${{ jobGroup.extName }}'
+        GROUP_NAMES: ${{ join(',', parameters.jobGroups.*.name ) }}
+        ${{ if ne(jobGroup.variables, '') }}:
+          ${{ jobGroup.variables }}
+        ${{ each para in parameters.jobvariables }}:
+          ${{ if eq(jobGroup.variables[para.key], '') }}:
+            ${{ para.key }}: ${{ para.value }}
+      steps:
+      - ${{ parameters.preSteps }}
+      - ${{ if ne(jobGroup.script, '') }}:
+        - script: |
+            ${{ jobGroup.script }}
+          env:
+            ${{ if ne(parameters.scriptEnv, '') }}:
+              ${{ parameters.scriptEnv }}
+          displayName: 'JobScript'
+      - ${{ if ne(jobGroup.steps, '') }}:
+        - ${{ jobGroup.steps }}
+      - ${{ parameters.buildSteps }}
+      - ${{ parameters.postSteps }}
diff --git a/.azure-pipelines/official-build.yml b/.azure-pipelines/official-build.yml
@@ -0,0 +1,29 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+schedules:
+- cron: "0 8 * * *"
+  displayName: Daily midnight build
+  branches:
+    include:
+    - master
+    - 202012
+  always: true
+
+trigger: none
+pr: none
+
+stages:
+- stage: Build
+  pool: sonicbld
+  variables:
+    CACHE_MODE: wcache
+    ${{ if eq(variables['Build.SourceBranchName'], '202012') }}:
+      VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web'
+  jobs:
+  - template: azure-pipelines-build.yml
+    parameters:
+      buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
+      jobFilters: none