Merge branch 'master' into as5835-54x_20210604_pddf

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -20,6 +20,20 @@ schedules:
 
 pool: sonicbld
 
+parameters:
+- name: 'jobFilters'
+  type: object
+  default:
+  - vs
+  - barefoot
+  - broadcom
+  - centec
+  - centec-arm64
+  - generic
+  - innovium
+  - marvell-armhf
+  - mellanox
+
 stages:
 - stage: Build
   variables:
@@ -28,6 +42,7 @@ stages:
   jobs:
   - template: azure-pipelines-build.yml
     parameters:
+      jobFilters: ${{ parameters.jobFilters }}
       buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
       - script: |

.azure-pipelines/official-build-cisco-8000.yml

@@ -22,13 +22,19 @@ resources:
     name: Cisco-8000-sonic/platform-cisco-8000
     endpoint: cisco-connection
 
+variables:
+- group: SONIC-AKV-STROAGE-1
+- name: StorageSASKey
+  value: $(sonicstorage-SasToken)
+
 stages:
 - stage: Build
   pool: sonic
   variables:
     CACHE_MODE: wcache
     SKIP_CHECKOUT: true
     TERM: ''
+    PACKAGE_URL: "https://sonicstorage.blob.core.windows.net/packages"
 
   jobs:
   - template: azure-pipelines-build.yml
@@ -60,5 +66,29 @@ stages:
           make PLATFORM=cisco-8000 platform/cisco-8000
           tar xfz $(System.ArtifactsDirectory)/artifactory-*.tar.gz -C platform/cisco-8000
         displayName: 'Setup cisco artifacts'
+      - script: |
+          set -ex
+          filename=$(find platform/cisco-8000/artifactory/sonic -name cisco-* -type f | head -n 1)
+          if [ -z "$filename" ]; then
+            echo "Cisco sai package not found" 1>&2
+            exit 1
+          fi
+          cd $(dirname $filename)
+          echo "PWD=$(pwd)"
+          ls -l *.deb
+          while read -r package; do
+            # Cisco version format: <VERSION>-sai-<sai-ver>-<distribution>-<COMMIT HASH>
+            # The <sai-ver> may contain several values in one build, the part is skipped when publishing to storage
+            # See https://github.com/Cisco-8000-sonic/sdk/blob/master/azure-pipelines.yml
+            # The $PACKAGE_URL is only accessible for AZP
+            version=$(echo $package | awk -F_ '{print $(NF-1)}' | cut -d- -f1,2,4,5)
+            package_url="$PACKAGE_URL/sai/ciscosai/master/$version/$package"
+            echo "Override package $package from $package_url"
+            wget "$package_url$StorageSASKey" -O "$package"
+          done < <(ls *.deb)
+        env:
+          StorageSASKey: $(StorageSASKey)
+        condition: ne(variables['Build.Reason'], 'PullRequest')
+        displayName: "Override cisco sai packages"
       jobGroups:
       - name: cisco-8000

Makefile.work

@@ -115,6 +115,10 @@ rules/config.user:
 include rules/config
 -include rules/config.user
 
+ifneq ($(DEFAULT_CONTAINER_REGISTRY),)
+override DEFAULT_CONTAINER_REGISTRY := $(DEFAULT_CONTAINER_REGISTRY)/
+endif
+
 ifeq ($(ENABLE_DOCKER_BASE_PULL),)
 	override ENABLE_DOCKER_BASE_PULL = n
 endif
@@ -139,9 +143,9 @@ $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
     scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
-$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
+$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
-$(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
+$(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
 
 # Add the versions in the tag, if the version change, need to rebuild the slave
 SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* src/sonic-build-hooks/hooks/* | sha1sum | awk '{print substr($$1,0,11);}')
@@ -291,6 +295,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
+                           SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
                            ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \
                            ENABLE_AUTO_TECH_SUPPORT=$(ENABLE_AUTO_TECH_SUPPORT) \

build_debian.sh

@@ -67,6 +67,9 @@ mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR
 mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/x86_64-grub
 touch $FILESYSTEM_ROOT/$PLATFORM_DIR/firsttime
 
+## ensure proc is mounted
+sudo mount proc /proc -t proc || true
+
 ## make / as a mountpoint in chroot env, needed by dockerd
 pushd $FILESYSTEM_ROOT
 sudo mount --bind . .
@@ -190,7 +193,7 @@ if [ -f platform/$CONFIGURED_PLATFORM/modules ]; then
 fi
 
 ## Add mtd and uboot firmware tools package
-sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install u-boot-tools mtd-utils device-tree-compiler
+sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install u-boot-tools libubootenv-tool mtd-utils device-tree-compiler
 
 ## Install docker
 echo '[INFO] Install docker'
@@ -559,6 +562,9 @@ if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
     fi
 fi
 
+# Collect host image version files before cleanup
+scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
+
 # Remove GCC
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc
 
@@ -571,6 +577,9 @@ sudo LANG=C chroot $FILESYSTEM_ROOT bash -c 'rm -rf /usr/share/doc/* /usr/share/
 ## Clean up proxy
 [ -n "$http_proxy" ] && sudo rm -f $FILESYSTEM_ROOT/etc/apt/apt.conf.d/01proxy
 
+## Clean up pip cache
+sudo LANG=C chroot $FILESYSTEM_ROOT pip3 cache purge
+
 ## Umount all
 echo '[INFO] Umount all'
 ## Display all process details access /proc
@@ -590,7 +599,6 @@ sudo rm -f $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS
 ## Note: -x to skip directories on different file systems, such as /proc
 sudo du -hsx $FILESYSTEM_ROOT
 sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker
-scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
 sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -comp zstd -b 1M -e boot -e var/lib/docker -e $PLATFORM_DIR
 
 # Ensure admin gid is 1000

build_image.sh

@@ -68,6 +68,8 @@ generate_kvm_image()
 
 generate_onie_installer_image()
 {
+    output_file=$OUTPUT_ONIE_IMAGE
+    [ -n "$1" ] && output_file=$1
     # Copy platform-specific ONIE installer config files where onie-mk-demo.sh expects them
     rm -rf ./installer/x86_64/platforms/
     mkdir -p ./installer/x86_64/platforms/
@@ -83,7 +85,7 @@ generate_onie_installer_image()
     ## Generate an ONIE installer image
     ## Note: Don't leave blank between lines. It is single line command.
     ./onie-mk-demo.sh $TARGET_PLATFORM $TARGET_MACHINE $TARGET_PLATFORM-$TARGET_MACHINE-$ONIEIMAGE_VERSION \
-          installer platform/$TARGET_MACHINE/platform.conf $OUTPUT_ONIE_IMAGE OS $IMAGE_VERSION $ONIE_IMAGE_PART_SIZE \
+          installer platform/$TARGET_MACHINE/platform.conf $output_file OS $IMAGE_VERSION $ONIE_IMAGE_PART_SIZE \
           $ONIE_INSTALLER_PAYLOAD
 }
 
@@ -119,12 +121,13 @@ if [ "$IMAGE_TYPE" = "onie" ]; then
 elif [ "$IMAGE_TYPE" = "raw" ]; then
 
     echo "Build RAW image"
+    tmp_output_onie_image=${OUTPUT_ONIE_IMAGE}.tmp
     mkdir -p `dirname $OUTPUT_RAW_IMAGE`
     sudo rm -f $OUTPUT_RAW_IMAGE
 
     generate_device_list "./installer/$TARGET_PLATFORM/platforms_asic"
 
-    generate_onie_installer_image
+    generate_onie_installer_image "$tmp_output_onie_image"
 
     echo "Creating SONiC raw partition : $OUTPUT_RAW_IMAGE of size $RAW_IMAGE_DISK_SIZE MB"
     fallocate -l "$RAW_IMAGE_DISK_SIZE"M $OUTPUT_RAW_IMAGE
@@ -135,8 +138,9 @@ elif [ "$IMAGE_TYPE" = "raw" ]; then
     ## Generate a partition dump that can be used to 'dd' in-lieu of using the onie-nos-installer
     ## Run the installer
     ## The 'build' install mode of the installer is used to generate this dump.
-    sudo chmod a+x $OUTPUT_ONIE_IMAGE
-    sudo ./$OUTPUT_ONIE_IMAGE
+    sudo chmod a+x $tmp_output_onie_image
+    sudo ./$tmp_output_onie_image
+    rm $tmp_output_onie_image
 
     [ -r $OUTPUT_RAW_IMAGE ] || {
         echo "Error : $OUTPUT_RAW_IMAGE not generated!"

device/accton/x86_64-accton_as4630_54pe-r0/Accton-AS4630-54PE/hx5-as4630-48x1G+4x25G+2x100G.bcm

@@ -1,4 +1,4 @@
-stable_size=71303168
+stable_size=76303168
 
 #polarity/lanemap is using TH2 style.
 core_clock_frequency=893

device/accton/x86_64-accton_as4630_54pe-r0/sonic_platform/__init__.py

@@ -0,0 +1,2 @@
+__all__ = [ "platform", "chassis", "sfp", "eeprom", "component", "psu", "thermal", "fan", "fan_drawer" ]
+from . import platform