diff --git a/.gitignore b/.gitignore index eda891212d53..2a2d70646c78 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,10 @@ platform/broadcom/sonic-platform-modules-dell/s6100/modules/dell_s6100_lpc.c platform/broadcom/sonic-platform-modules-dell/z9100/modules/dell_ich.c platform/broadcom/sonic-platform-modules-dell/z9100/modules/dell_mailbox.c platform/broadcom/sonic-platform-modules-dell/z9264f/sonic_platform/ipmihelper.py + +# buildinfo +files/build/buildinfo +files/build/tmp +dockers/**/buildinfo +platform/**/buildinfo +sonic-slave*/**/buildinfo diff --git a/Makefile b/Makefile index c717da3d40a3..140ebe8b3c5a 100644 --- a/Makefile +++ b/Makefile @@ -38,3 +38,7 @@ ifeq ($(NOSTRETCH), 0) BLDENV=stretch make -f Makefile.work $@ endif BLDENV=buster make -f Makefile.work $@ + +# Freeze the versions, see more detail options: scripts/versions_manager.py freeze -h +freeze: + @scripts/versions_manager.py freeze $(FREEZE_VERSION_OPTIONS) diff --git a/Makefile.work b/Makefile.work index 79f521488fe6..8d28377ae194 100644 --- a/Makefile.work +++ b/Makefile.work @@ -89,11 +89,24 @@ else SLAVE_DIR = sonic-slave-jessie endif -SLAVE_BASE_TAG = $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile && sha1sum $(SLAVE_DIR)/Dockerfile | awk '{print substr($$1,0,11);}') -SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}') +include rules/config + SLAVE_BASE_IMAGE = $(SLAVE_DIR) SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER) +# Generate the version control build info +$(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + scripts/generate_buildinfo_config.sh) + +# Generate the slave Dockerfile, and prepare build info for it +$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile) +$(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV)) + +# Add the versions in the tag, if the version change, need to rebuild the slave +SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}') +SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}') + OVERLAY_MODULE_CHECK := \ lsmod | grep -q "^overlay " &>/dev/null || \ zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \ @@ -118,8 +131,6 @@ DOCKER_RUN := docker run --rm=true --privileged --init \ -i$(if $(TERM),t,) \ $(SONIC_BUILDER_EXTRA_CMDLINE) -include rules/config - ifneq ($(DOCKER_BUILDER_USER_MOUNT),) DOCKER_RUN += $(foreach mount,$(subst $(comma), ,$(DOCKER_BUILDER_USER_MOUNT)), $(addprefix -v , $(mount))) endif @@ -218,6 +229,7 @@ SONIC_BUILD_INSTRUCTION := make \ EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \ BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \ SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \ + SLAVE_DIR=$(SLAVE_DIR) \ $(SONIC_OVERRIDE_BUILD_VARS) .PHONY: sonic-slave-build sonic-slave-bash init reset @@ -234,27 +246,35 @@ endif endif @$(OVERLAY_MODULE_CHECK) + @pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd + @cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_BASE_IMAGE)/buildinfo @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \ - $(DOCKER_BASE_BUILD) ; } + $(DOCKER_BASE_BUILD) ; \ + scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target ; } @docker inspect --type image $(SLAVE_IMAGE):$(SLAVE_TAG) &> /dev/null || \ { echo Image $(SLAVE_IMAGE):$(SLAVE_TAG) not found. Building... ; \ $(DOCKER_BUILD) ; } ifeq "$(KEEP_SLAVE_ON)" "yes" ifdef SOURCE_FOLDER - @$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; /bin/bash" + @$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash" else - @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; /bin/bash" + @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash" endif else - @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) $(SONIC_BUILD_INSTRUCTION) $@ + @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?" endif -sonic-slave-base-build : +sonic-build-hooks: + @pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd + @cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_BASE_IMAGE)/buildinfo + +sonic-slave-base-build : sonic-build-hooks @$(OVERLAY_MODULE_CHECK) @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \ - $(DOCKER_BASE_BUILD) ; } + $(DOCKER_BASE_BUILD) ; \ + scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target ; } sonic-slave-build : sonic-slave-base-build @docker inspect --type image $(SLAVE_IMAGE):$(SLAVE_TAG) &> /dev/null || \ diff --git a/build_debian.sh b/build_debian.sh index 8b4b9f57c245..4838d3c87796 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -40,6 +40,8 @@ PLATFORM_DIR=platform ## Hostname for the linux image HOSTNAME=sonic DEFAULT_USERINFO="Default admin user,,," +BUILD_TOOL_PATH=src/sonic-build-hooks/buildinfo +TRUSTED_GPG_DIR=$BUILD_TOOL_PATH/trusted.gpg.d ## Read ONIE image related config file . ./onie-image.conf @@ -70,16 +72,12 @@ pushd $FILESYSTEM_ROOT sudo mount --bind . . popd -## Build a basic Debian system by debootstrap -echo '[INFO] Debootstrap...' -if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then - # qemu arm bin executable for cross-building - sudo mkdir -p $FILESYSTEM_ROOT/usr/bin - sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true - sudo http_proxy=$http_proxy debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian -else - sudo http_proxy=$http_proxy debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian -fi +## Build the host debian base system +echo '[INFO] Build host debian base system...' +TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT + +# Prepare buildinfo +sudo scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy ## Config hostname and hosts, otherwise 'sudo ...' will complain 'sudo: unable to resolve host ...' sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo '$HOSTNAME' > /etc/hostname" @@ -100,6 +98,9 @@ echo '[INFO] Mount all' ## Output all the mounted device for troubleshooting sudo LANG=C chroot $FILESYSTEM_ROOT mount +## Install the trusted gpg public keys +[ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/ + ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/ @@ -584,6 +585,8 @@ sudo du -hsx $FILESYSTEM_ROOT sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR +scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT + ## Compress docker files pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd diff --git a/dockers/docker-base-buster/sources.list b/dockers/docker-base-buster/sources.list index 4a94e3161ab1..0eef72d9fa2d 100644 --- a/dockers/docker-base-buster/sources.list +++ b/dockers/docker-base-buster/sources.list @@ -6,3 +6,8 @@ deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster mai deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free + +# Debian mirror supports multiple versions for a package +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.arm64 b/dockers/docker-base-buster/sources.list.arm64 index d4450c25952e..6375734e99e6 100644 --- a/dockers/docker-base-buster/sources.list.arm64 +++ b/dockers/docker-base-buster/sources.list.arm64 @@ -6,3 +6,6 @@ deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free deb [arch=arm64] http://deb.debian.org/debian/ buster-backports main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.armhf b/dockers/docker-base-buster/sources.list.armhf index a053a509e148..a03af1a33ac0 100644 --- a/dockers/docker-base-buster/sources.list.armhf +++ b/dockers/docker-base-buster/sources.list.armhf @@ -6,3 +6,6 @@ deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free deb [arch=armhf] http://deb.debian.org/debian/ buster-backports main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list b/dockers/docker-base-stretch/sources.list index 752b1bb4fc60..0c29b339bb87 100644 --- a/dockers/docker-base-stretch/sources.list +++ b/dockers/docker-base-stretch/sources.list @@ -6,3 +6,6 @@ deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch ma deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch-backports main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list.arm64 b/dockers/docker-base-stretch/sources.list.arm64 index f5fe1d391427..520c46519919 100644 --- a/dockers/docker-base-stretch/sources.list.arm64 +++ b/dockers/docker-base-stretch/sources.list.arm64 @@ -6,3 +6,6 @@ deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free deb [arch=arm64] http://deb.debian.org/debian/ stretch-backports main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list.armhf b/dockers/docker-base-stretch/sources.list.armhf index 90c4d351f09c..58077f310424 100644 --- a/dockers/docker-base-stretch/sources.list.armhf +++ b/dockers/docker-base-stretch/sources.list.armhf @@ -6,3 +6,6 @@ deb-src [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free deb [arch=armhf] http://deb.debian.org/debian/ stretch-backports main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/rules/config b/rules/config index 8f486d8170e0..fdad3dc44bc8 100644 --- a/rules/config +++ b/rules/config @@ -162,3 +162,20 @@ K8s_GCR_IO_PAUSE_VERSION = 3.2 # CA_CERT = # The relative path is build root folder. SONIC_ENABLE_IMAGE_SIGNATURE ?= n + +# PACKAGE_URL_PREFIX - the package url prefix +PACKAGE_URL_PREFIX ?= https://packages.trafficmanager.net/public/packages + +# TRUSTED_GPG_URLS - the trusted gpgs, separated by comma +TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,https://packages.microsoft.com/keys/microsoft.asc + +# SONIC_VERSION_CONTROL_COMPONENTS - Valid values: none|all|components..., the components consist of one or multiple: deb,py2,py3,web,git,docker, seperated by comma +# none : disable the version control +# all : enable the version control for all components +# deb : debian packages +# py2 : python2 packages +# py3 : python3 pakcages +# web : web packages, downloaded by wget, curl +# git : git repositories, donloaded by git clone +# docker: docker base images +SONIC_VERSION_CONTROL_COMPONENTS ?= none diff --git a/scripts/build_debian_base_system.sh b/scripts/build_debian_base_system.sh new file mode 100755 index 000000000000..a85b3b5bf0ee --- /dev/null +++ b/scripts/build_debian_base_system.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +CONFIGURED_ARCH=$1 +IMAGE_DISTRO=$2 +FILESYSTEM_ROOT=$3 +http_proxy=$4 + +TARGET=$TARGET_PATH +[ -z "$TARGET" ] && TARGET=target + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +TARGET_BASEIMAGE_PATH=$TARGET/versions/host-base-image +mkdir -p $TARGET_BASEIMAGE_PATH + +generate_version_file() +{ + sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg-query -W -f '\${Package}==\${Version}\n'" > $TARGET_BASEIMAGE_PATH/versions-deb-${IMAGE_DISTRO}-${CONFIGURED_ARCH} +} + +if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ]; then + if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then + # qemu arm bin executable for cross-building + sudo mkdir -p $FILESYSTEM_ROOT/usr/bin + sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true + sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian + else + sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian + fi + RET=$? + if [ $RET -ne 0 ]; then + exit $RET + fi + + generate_version_file + exit $RET +fi + +ARCH=$(dpkg --print-architecture) +DISTRO=$(grep CODENAME /etc/os-release | cut -d= -f2) +if [ "$ARCH" != "$CONFIGURED_ARCH" ] || [ "$DISTRO" != "$IMAGE_DISTRO" ]; then + "Not support to build different ARCH/DISTRO ${CONFIGURED_ARCH}:${$IMAGE_DISTRO} in ${ARCH}:${DISTRO}." + exit 1 +fi + +BASE_VERSIONS=files/build/versions/host-base-image/versions-deb-${IMAGE_DISTRO} +BASEIMAGE_TARBALLPATH=$TARGET/baseimage +BASEIMAGE_TARBALL=$(realpath -e $TARGET)/baseimage.tgz + +rm -rf $BASEIMAGE_TARBALLPATH $BASEIMAGE_TARBALL + +ARCHIEVES=$BASEIMAGE_TARBALLPATH/var/cache/apt/archives +APTLIST=$BASEIMAGE_TARBALLPATH/var/lib/apt/lists +TARGET_DEBOOTSTRAP=$BASEIMAGE_TARBALLPATH/debootstrap +APTDEBIAN="$APTLIST/deb.debian.org_debian_dists_buster_main_binary-${CONFIGURED_ARCH}_Packages" +DEBPATHS=$TARGET_DEBOOTSTRAP/debpaths +DEBOOTSTRAP_BASE=$TARGET_DEBOOTSTRAP/base +DEBOOTSTRAP_REQUIRED=$TARGET_DEBOOTSTRAP/required +[ -d $BASEIMAGE_TARBALLPATH ] && rm -rf $BASEIMAGE_TARBALLPATH +mkdir -p $ARCHIEVES +mkdir -p $APTLIST +mkdir -p $TARGET_DEBOOTSTRAP +PACKAGES=$(sed -E 's/=(=[^=]*)$/\1/' $BASE_VERSIONS) +URL_ARR=($(apt-get download --print-uris $PACKAGES | cut -d" " -f1 | tr -d "'")) +PACKAGE_ARR=($PACKAGES) +LENGTH=${#PACKAGE_ARR[@]} +for ((i=0;i> $DEBOOTSTRAP_REQUIRED + echo "$packagename /var/cache/apt/archives/$filename" >> $DEBPATHS +done +touch $APTDEBIAN +touch $DEBOOTSTRAP_BASE +(cd $BASEIMAGE_TARBALLPATH && tar -zcf $BASEIMAGE_TARBALL .) + +sudo debootstrap --verbose --variant=minbase --arch $CONFIGURED_ARCH --unpack-tarball=$BASEIMAGE_TARBALL $IMAGE_DISTRO $FILESYSTEM_ROOT +RET=$? +if [ $RET -ne 0 ]; then + exit $RET +fi + +generate_version_file diff --git a/scripts/collect_build_version_files.sh b/scripts/collect_build_version_files.sh new file mode 100755 index 000000000000..b650e421996b --- /dev/null +++ b/scripts/collect_build_version_files.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +RET=$1 +BLDENV=$2 +TARGET_PATH=$3 + +TIMESTAMP=$(date +"%Y%m%d%H%M%S") +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +[ -z "$BLDENV" ] && BLDENV=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) +[ -z "$BLDENV" ] && exit $RET + +[ -z "$TARGET_PATH" ] && TARGET_PATH=./target + +VERSION_BUILD_PATH=$TARGET_PATH/versions/build +VERSION_SLAVE_PATH=$VERSION_BUILD_PATH/build-sonic-slave-${BLDENV} +LOG_VERSION_PATH=$VERSION_BUILD_PATH/log-${TIMESTAMP} + +sudo chmod -R a+rw $BUILDINFO_PATH +collect_version_files $LOG_VERSION_PATH +([ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls $BUILD_VERSION_PATH/)" ]) && cp -rf $BUILD_VERSION_PATH/* $LOG_VERSION_PATH/ +mkdir -p $VERSION_SLAVE_PATH + +scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH + +rm -rf $BUILD_VERSION_PATH/* + +exit $RET diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh new file mode 100755 index 000000000000..73f0a9b53198 --- /dev/null +++ b/scripts/collect_docker_version_files.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +DOCKER_IMAGE=$1 +TARGET_PATH=$2 + +[ -z "$TARGET_PATH" ] && TARGET_PATH=./target + +DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1) +DOCKER_CONTAINER=$DOCKER_IMAGE_NAME +TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME + +[ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH +mkdir -p $TARGET_VERSIONS_PATH + +export DOCKER_CLI_EXPERIMENTAL=enabled + +# Remove the old docker container if existing +if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then + docker container rm $DOCKER_IMAGE > /dev/null +fi +docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE +docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ +docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/ +docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/ +docker container rm $DOCKER_CONTAINER diff --git a/scripts/collect_host_image_version_files.sh b/scripts/collect_host_image_version_files.sh new file mode 100755 index 000000000000..3e06bcfb2345 --- /dev/null +++ b/scripts/collect_host_image_version_files.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +TARGET=$1 +FILESYSTEM_ROOT=$2 +VERSIONS_PATH=$TARGET/versions/host-image + +mkdir -p $VERSIONS_PATH + +sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo + +cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/ +cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/ diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh new file mode 100755 index 000000000000..fe7657a6b6c9 --- /dev/null +++ b/scripts/generate_buildinfo_config.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +BUILDINFO_PATH=src/sonic-build-hooks + +BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config + +mkdir -p $BUILDINFO_PATH/buildinfo/config + +echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG +echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG diff --git a/scripts/prepare_debian_image_buildinfo.sh b/scripts/prepare_debian_image_buildinfo.sh new file mode 100755 index 000000000000..912e0de0b25f --- /dev/null +++ b/scripts/prepare_debian_image_buildinfo.sh @@ -0,0 +1,29 @@ +#!/bin/bash + + +ARCH=$1 +DISTRO=$2 +FILESYSTEM_ROOT=$3 + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh +VERSION_DEB_PREFERENCE="01-versions-deb" +BUILDINFO_PATH=${FILESYSTEM_ROOT}/usr/local/share/buildinfo +BUILDINFO_VERSION_PATH=${FILESYSTEM_ROOT}/usr/local/share/buildinfo/versions +BUILDINFO_VERSION_DEB=${BUILDINFO_VERSION_PATH}/${VERSION_DEB_PREFERENCE} +OVERRIDE_VERSION_PATH=files/build/versions/host-image +DIFF_VERSIONS_PATH=$BUILDINFO_PATH/diff-versions + +mkdir -p $BUILDINFO_PATH + +# Copy the build info config +cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH/ + +# Generate version lock files +scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -m "$OVERRIDE_VERSION_PATH" -d "$DISTRO" -a "$ARCH" + +if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then + cp -f $BUILDINFO_VERSION_DEB ${FILESYSTEM_ROOT}/etc/apt/preferences.d/ +fi + +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb" +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo" diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh new file mode 100755 index 000000000000..aa3aaaa4bed5 --- /dev/null +++ b/scripts/prepare_docker_buildinfo.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +IMAGENAME=$1 +DOCKERFILE=$2 +ARCH=$3 +DOCKERFILE_TARGE=$4 +DISTRO=$5 + +[ -z "$BUILD_SLAVE" ] && BUILD_SLAVE=n +[ -z "$DOCKERFILE_TARGE" ] && DOCKERFILE_TARGE=$DOCKERFILE +DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE") +BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo" +BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions" + +[ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH +mkdir -p $BUILDINFO_VERSION_PATH + +# Get the debian distribution from the docker base image +if [ -z "$DISTRO" ]; then + DOCKER_BASE_IMAGE=$(grep "^FROM" $DOCKERFILE | head -n 1 | awk '{print $2}') + DISTRO=$(docker run --rm --entrypoint "" $DOCKER_BASE_IMAGE cat /etc/os-release | grep VERSION_CODENAME | cut -d= -f2) + [ -z "$DISTRO" ] && DISTRO=jessie +fi + +DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo +COPY ["buildinfo", "/usr/local/share/buildinfo"] +RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb +RUN pre_run_buildinfo' + +# Add the auto-generate code if it is not added in the target Dockerfile +if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then + # Insert the docker build script before the RUN command + LINE_NUMBER=$(grep -Fn -m 1 'RUN' $DOCKERFILE | cut -d: -f1) + TEMP_FILE=$(mktemp) + awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE + + # Append the docker build script at the end of the docker file + echo -e "\nRUN post_run_buildinfo" >> $TEMP_FILE + + cat $TEMP_FILE > $DOCKERFILE_TARGE + rm -f $TEMP_FILE +fi + +# Copy the build info config +cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH + +# Generate the version lock files +scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH" + +touch $BUILDINFO_VERSION_PATH/versions-deb diff --git a/scripts/prepare_slave_container_buildinfo.sh b/scripts/prepare_slave_container_buildinfo.sh new file mode 100755 index 000000000000..d66bbe816104 --- /dev/null +++ b/scripts/prepare_slave_container_buildinfo.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +SLAVE_DIR=$1 +ARCH=$2 +DISTRO=$3 + +# Enable the build hooks +symlink_build_hooks + +# Build the slave running config +cp -rf $SLAVE_DIR/buildinfo/* /usr/local/share/buildinfo/ +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +# Build the slave version config +[ -d /usr/local/share/buildinfo/versions ] && rm -rf /usr/local/share/buildinfo/versions +scripts/versions_manager.py generate -t "/usr/local/share/buildinfo/versions" -n "build-${SLAVE_DIR}" -d "$DISTRO" -a "$ARCH" +touch ${BUILDINFO_PATH}/versions/versions-deb + +rm -f /etc/apt/preferences.d/01-versions-deb +([ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]) && cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ +exit 0 diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py new file mode 100755 index 000000000000..59621cdfbbc7 --- /dev/null +++ b/scripts/versions_manager.py @@ -0,0 +1,666 @@ +#!/usr/bin/python3 + +import argparse +import glob +import os +import sys + +ALL_DIST = 'all' +ALL_ARCH = 'all' +DEFAULT_MODULE = 'default' +DEFAULT_VERSION_PATH = 'files/build/versions' +VERSION_PREFIX="versions-" +VERSION_DEB_PREFERENCE = '01-versions-deb' +DEFAULT_OVERWRITE_COMPONENTS=['deb', 'py2', 'py3'] +SLAVE_INDIVIDULE_VERSION = False + + +class Component: + ''' + The component consists of mutiple packages + + ctype -- Component Type, such as deb, py2, etc + dist -- Distribution, such as stretch, buster, etc + arch -- Architectrue, such as amd64, arm64, etc + + ''' + def __init__(self, versions, ctype, dist=ALL_DIST, arch=ALL_ARCH): + self.versions = versions + self.ctype = ctype + if not dist: + dist = ALL_DIST + if not arch: + arch = ALL_ARCH + self.dist = dist + self.arch = arch + + @classmethod + def get_versions(cls, version_file): + result = {} + if not os.path.exists(version_file): + return result + with open(version_file) as fp: + for line in fp.readlines(): + offset = line.rfind('==') + if offset > 0: + package = line[:offset].strip() + version = line[offset+2:].strip() + result[package] = version + return result + + def clone(self): + return Component(self.versions.copy(), self.ctype, self.dist, self.arch) + + def merge(self, versions, overwritten=True): + for package in versions: + if overwritten or package not in self.versions: + self.versions[package] = versions[package] + + def subtract(self, versions): + for package in versions: + if package in self.versions and self.versions[package] == versions[package]: + del self.versions[package] + + def dump(self, config=False, priority=999): + result = [] + for package in sorted(self.versions.keys(), key=str.casefold): + if config and self.ctype == 'deb': + lines = 'Package: {0}\nPin: version {1}\nPin-Priority: {2}\n\n'.format(package, self.versions[package], priority) + result.append(lines) + else: + result.append('{0}=={1}'.format(package, self.versions[package])) + return "\n".join(result) + + def dump_to_file(self, version_file, config=False, priority=999): + if len(self.versions) <= 0: + return + with open(version_file, 'w') as f: + f.write(self.dump(config, priority)) + + def dump_to_path(self, file_path, config=False, priority=999): + if len(self.versions) <= 0: + return + if not os.path.exists(file_path): + os.makedirs(file_path) + filename = self.get_filename() + if config and self.ctype == 'deb': + none_config_file_path = os.path.join(file_path, filename) + self.dump_to_file(none_config_file_path, False, priority) + filename = VERSION_DEB_PREFERENCE + file_path = os.path.join(file_path, filename) + self.dump_to_file(file_path, config, priority) + + # Check if the self component can be overwritten by the input component + def check_overwritable(self, component, for_all_dist=False, for_all_arch=False): + if self.ctype != component.ctype: + return False + if self.dist != component.dist and not (for_all_dist and self.dist == ALL_DIST): + return False + if self.arch != component.arch and not (for_all_arch and self.arch == ALL_ARCH): + return False + return True + + # Check if the self component can inherit the package versions from the input component + def check_inheritable(self, component): + if self.ctype != component.ctype: + return False + if self.dist != component.dist and component.dist != ALL_DIST: + return False + if self.arch != component.arch and component.arch != ALL_ARCH: + return False + return True + + ''' + Get the file name + + The file name format: versions-{ctype}-{dist}-{arch} + If {arch} is all, then the file name format: versions-{ctype}-{dist} + if {arch} is all and {dist} is all, then the file name format: versions-{ctype} + ''' + def get_filename(self): + filename = VERSION_PREFIX + self.ctype + dist = self.dist + if self.arch and self.arch != ALL_ARCH: + if not dist: + dist = ALL_DIST + return filename + '-' + dist + '-' + self.arch + if dist and self.dist != ALL_DIST: + filename = filename + '-' + dist + return filename + + def get_order_keys(self): + dist = self.dist + if not dist or dist == ALL_DIST: + dist = '' + arch = self.arch + if not arch or arch == ALL_ARCH: + arch = '' + return (self.ctype, dist, arch) + + def clean_info(self, clean_dist=True, clean_arch=True, force=False): + if clean_dist: + if force or self.ctype != 'deb': + self.dist = ALL_DIST + if clean_arch: + self.arch = ALL_ARCH + + +class VersionModule: + ''' + The version module represents a build target, such as docker image, host image, consists of multiple components. + + name -- The name of the image, such as sonic-slave-buster, docker-lldp, etc + ''' + def __init__(self, name=None, components=None): + self.name = name + self.components = components + + # Overwrite the docker/host image/base image versions + def overwrite(self, module, for_all_dist=False, for_all_arch=False): + # Overwrite from generic one to detail one + # For examples: versions-deb overwrtten by versions-deb-buster, and versions-deb-buster overwritten by versions-deb-buster-amd64 + components = sorted(module.components, key = lambda x : x.get_order_keys()) + for merge_component in components: + merged = False + for component in self.components: + if component.check_overwritable(merge_component, for_all_dist=for_all_dist, for_all_arch=for_all_arch): + component.merge(merge_component.versions, True) + merged = True + if not merged: + tmp_component = merge_component.clone() + tmp_component.clean_info(clean_dist=for_all_dist, clean_arch=for_all_arch) + self.components.append(tmp_component) + self.adjust() + + def get_config_module(self, default_module, dist, arch): + if self.is_individule_version(): + return self + module = default_module + if not self.is_aggregatable_module(self.name): + module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) + return self._get_config_module(module, dist, arch) + + def _get_config_module(self, default_module, dist, arch): + module = default_module.clone() + default_ctype_components = module._get_components_per_ctypes() + module.overwrite(self) + config_components = [] + ctype_components = module._get_components_per_ctypes() + for ctype in default_ctype_components: + if ctype not in ctype_components: + ctype_components[ctype] = [] + for components in ctype_components.values(): + config_component = self._get_config_for_ctype(components, dist, arch) + config_components.append(config_component) + config_module = VersionModule(self.name, config_components) + return config_module + + def _get_config_for_ctype(self, components, dist, arch): + result = Component({}, components[0].ctype, dist, arch) + for component in sorted(components, key = lambda x : x.get_order_keys()): + if result.check_inheritable(component): + result.merge(component.versions, True) + return result + + def subtract(self, default_module): + module = self.clone() + result = [] + ctype_components = module._get_components_per_ctypes() + for ctype in ctype_components: + components = ctype_components[ctype] + components = sorted(components, key = lambda x : x.get_order_keys()) + for i in range(0, len(components)): + component = components[i] + base_module = VersionModule(self.name, components[0:i]) + config_module = base_module._get_config_module(default_module, component.dist, component.arch) + config_components = config_module._get_components_by_ctype(ctype) + if len(config_components) > 0: + config_component = config_components[0] + component.subtract(config_component.versions) + if len(component.versions): + result.append(component) + self.components = result + + def adjust(self): + result_components = [] + ctype_components = self._get_components_per_ctypes() + for components in ctype_components.values(): + result_components += self._adjust_components_for_ctype(components) + self.components = result_components + + def _get_components_by_ctype(self, ctype): + components = [] + for component in self.components: + if component.ctype == ctype: + components.append(component) + return components + + def _adjust_components_for_ctype(self, components): + components = sorted(components, key = lambda x : x.get_order_keys()) + result = [] + for i in range(0, len(components)): + component = components[i] + inheritable_component = Component({}, component.ctype) + for j in range(0, i): + base_component = components[j] + if component.check_inheritable(base_component): + inheritable_component.merge(base_component.versions, True) + component.subtract(inheritable_component.versions) + if len(component.versions) > 0: + result.append(component) + return result + + def _get_components_per_ctypes(self): + result = {} + for component in self.components: + components = result.get(component.ctype, []) + components.append(component) + result[component.ctype] = components + return result + + def load(self, image_path, filter_ctype=None, filter_dist=None, filter_arch=None): + version_file_pattern = os.path.join(image_path, VERSION_PREFIX) + '*' + file_paths = glob.glob(version_file_pattern) + components = [] + self.name = os.path.basename(image_path) + self.components = components + for file_path in file_paths: + filename = os.path.basename(file_path) + items = filename.split('-') + if len(items) < 2: + continue + ctype = items[1] + if filter_ctype and filter_ctype != ctype: + continue + dist = '' + arch = '' + if len(items) > 2: + dist = items[2] + if filter_dist and dist and filter_dist != dist: + continue + if len(items) > 3: + arch = items[3] + if filter_arch and arch and filter_arch != arch: + continue + versions = Component.get_versions(file_path) + component = Component(versions, ctype, dist, arch) + components.append(component) + + def load_from_target(self, image_path): + post_versions = os.path.join(image_path, 'post-versions') + if os.path.exists(post_versions): + self.load(post_versions) + self.name = os.path.basename(image_path) + pre_versions = os.path.join(image_path, 'pre-versions') + if os.path.exists(pre_versions): + pre_module = VersionModule() + pre_module.load(pre_versions) + self.subtract(pre_module) + else: + self.load(image_path) + + def dump(self, module_path, config=False, priority=999): + version_file_pattern = os.path.join(module_path, VERSION_PREFIX + '*') + for filename in glob.glob(version_file_pattern): + os.remove(filename) + for component in self.components: + component.dump_to_path(module_path, config, priority) + + def filter(self, ctypes=[]): + if 'all' in ctypes: + return self + components = [] + for component in self.components: + if component.ctype in ctypes: + components.append(component) + self.components = components + + def clean_info(self, clean_dist=True, clean_arch=True, force=False): + for component in self.components: + component.clean_info(clean_dist=clean_dist, clean_arch=clean_arch, force=force) + + def clone(self, ctypes=None, exclude_ctypes=None): + components = [] + for component in self.components: + if exclude_ctypes and component.ctype in exclude_ctypes: + continue + if ctypes and component.ctype not in ctypes: + continue + components.append(component.clone()) + return VersionModule(self.name, components) + + def is_slave_module(self): + return self.name.startswith('sonic-slave-') + + # Do not inherit the version from the default module + def is_individule_version(self): + return self.is_slave_module() and SLAVE_INDIVIDULE_VERSION + + @classmethod + def is_aggregatable_module(cls, module_name): + if module_name.startswith('sonic-slave-'): + return False + if module_name.startswith('build-sonic-slave-'): + return False + if module_name == DEFAULT_MODULE: + return False + if module_name == 'host-image' or module_name == 'host-base-image': + return False + return True + + @classmethod + def get_module_path_by_name(cls, source_path, module_name): + common_modules = ['default', 'host-image', 'host-base-image'] + if module_name in common_modules: + return os.path.join(source_path, 'files/build/versions', module_name) + if module_name.startswith('build-sonic-slave-'): + return os.path.join(source_path, 'files/build/versions/build', module_name) + return os.path.join(source_path, 'files/build/versions/dockers', module_name) + +class VersionBuild: + ''' + The VersionBuild consists of multiple version modules. + + ''' + def __init__(self, target_path="./target", source_path='.'): + self.target_path = target_path + self.source_path = source_path + self.modules = {} + + def load_from_target(self): + dockers_path = os.path.join(self.target_path, 'versions/dockers') + build_path = os.path.join(self.target_path, 'versions/build') + modules = {} + self.modules = modules + file_paths = glob.glob(dockers_path + '/*') + file_paths += glob.glob(build_path + '/build-*') + file_paths.append(os.path.join(self.target_path, 'versions/host-image')) + file_paths.append(os.path.join(self.target_path, 'versions/host-base-image')) + for file_path in file_paths: + if not os.path.isdir(file_path): + continue + module = VersionModule() + module.load_from_target(file_path) + modules[module.name] = module + self._merge_dgb_modules() + + def load_from_source(self): + # Load default versions and host image versions + versions_path = os.path.join(self.source_path, 'files/build/versions') + dockers_path = os.path.join(versions_path, "dockers") + build_path = os.path.join(versions_path, "build") + paths = [os.path.join(versions_path, 'default')] + paths += glob.glob(versions_path + '/host-*') + paths += glob.glob(dockers_path + '/*') + paths += glob.glob(build_path + '/*') + modules = {} + self.modules = modules + for image_path in paths: + module = VersionModule() + module.load(image_path) + modules[module.name] = module + + def overwrite(self, build, for_all_dist=False, for_all_arch=False): + for target_module in build.modules.values(): + module = self.modules.get(target_module.name, None) + tmp_module = target_module.clone() + tmp_module.clean_info(for_all_dist, for_all_arch) + if module: + module.overwrite(tmp_module, for_all_dist=for_all_dist, for_all_arch=for_all_arch) + else: + self.modules[target_module.name] = tmp_module + + def dump(self): + for module in self.modules.values(): + module_path = self.get_module_path(module) + module.dump(module_path) + + def subtract(self, default_module): + none_aggregatable_module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) + for module in self.modules.values(): + if module.name == DEFAULT_MODULE: + continue + if module.name == 'host-base-image': + continue + if module.is_individule_version(): + continue + tmp_module = default_module + if not module.is_aggregatable_module(module.name): + tmp_module = none_aggregatable_module + module.subtract(tmp_module) + + def freeze(self, rebuild=False, for_all_dist=False, for_all_arch=False, ctypes=['all']): + if rebuild: + self.load_from_target() + self.filter(ctypes=ctypes) + default_module = self.get_default_module() + self._clean_component_info() + self.subtract(default_module) + self.modules[DEFAULT_MODULE] = default_module + self.dump() + return + self.load_from_source() + default_module = self.modules.get(DEFAULT_MODULE, None) + target_build = VersionBuild(self.target_path, self.source_path) + target_build.load_from_target() + target_build.filter(ctypes=ctypes) + if not default_module: + raise Exception("The default versions does not exist") + for module in target_build.modules.values(): + if module.is_individule_version(): + continue + tmp_module = module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) + default_module.overwrite(tmp_module, for_all_dist=True, for_all_arch=True) + target_build.subtract(default_module) + self.overwrite(target_build, for_all_dist=for_all_dist, for_all_arch=for_all_arch) + self.dump() + + def filter(self, ctypes=[]): + for module in self.modules.values(): + module.filter(ctypes=ctypes) + + def get_default_module(self): + if DEFAULT_MODULE in self.modules: + return self.modules[DEFAULT_MODULE] + ctypes = self.get_component_types() + dists = self.get_dists() + components = [] + for ctype in ctypes: + if ctype == 'deb': + for dist in dists: + versions = self._get_versions(ctype, dist) + common_versions = self._get_common_versions(versions) + component = Component(common_versions, ctype, dist) + components.append(component) + else: + versions = self._get_versions(ctype) + common_versions = self._get_common_versions(versions) + component = Component(common_versions, ctype) + components.append(component) + return VersionModule(DEFAULT_MODULE, components) + + def get_aggregatable_modules(self): + modules = {} + for module_name in self.modules: + if not VersionModule.is_aggregatable_module(module_name): + continue + module = self.modules[module_name] + modules[module_name] = module + return modules + + def get_components(self): + components = [] + for module_name in self.modules: + module = self.modules[module_name] + for component in module.components: + components.append(component) + return components + + def get_component_types(self): + ctypes = [] + for module_name in self.modules: + module = self.modules[module_name] + for component in module.components: + if component.ctype not in ctypes: + ctypes.append(component.ctype) + return ctypes + + def get_dists(self): + dists = [] + components = self.get_components() + for component in components: + if component.dist not in dists: + dists.append(component.dist) + return dists + + def get_archs(self): + archs = [] + components = self.get_components() + for component in components: + if component.arch not in archs: + archs.append(component.arch) + return archs + + def get_module_path(self, module): + return self.get_module_path_by_name(module.name) + + def get_module_path_by_name(self, module_name): + return VersionModule.get_module_path_by_name(self.source_path, module_name) + + def _merge_dgb_modules(self): + dbg_modules = [] + for module_name in self.modules: + if not module_name.endswith('-dbg'): + continue + dbg_modules.append(module_name) + base_module_name = module_name[:-4] + if base_module_name not in self.modules: + raise Exception('The Module {0} not found'.format(base_module_name)) + base_module = self.modules[base_module_name] + dbg_module = self.modules[module_name] + base_module.overwrite(dbg_module) + for module_name in dbg_modules: + del self.modules[module_name] + + def _clean_component_info(self, clean_dist=True, clean_arch=True): + for module in self.modules.values(): + module.clean_info(clean_dist, clean_arch) + + def _get_versions(self, ctype, dist=None, arch=None): + versions = {} + modules = self.get_aggregatable_modules() + for module_name in self.modules: + if module_name not in modules: + temp_module = self.modules[module_name].clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) + modules[module_name] = temp_module + for module in modules.values(): + for component in module.components: + if ctype != component.ctype: + continue + if dist and dist != component.dist: + continue + if arch and arch != component.arch: + continue + for package in component.versions: + version = component.versions[package] + package_versions = versions.get(package, []) + if version not in package_versions: + package_versions.append(version) + versions[package] = package_versions + return versions + + def _get_common_versions(self, versions): + common_versions = {} + for package in versions: + package_versions = versions[package] + if len(package_versions) == 1: + common_versions[package] = package_versions[0] + return common_versions + + +class VersionManagerCommands: + def __init__(self): + usage = 'version_manager.py []\n\n' + usage = usage + 'The most commonly used commands are:\n' + usage = usage + ' freeze Freeze the version files\n' + usage = usage + ' generate Generate the version files\n' + usage = usage + ' merge Merge the version files' + parser = argparse.ArgumentParser(description='Version manager', usage=usage) + parser.add_argument('command', help='Subcommand to run') + args = parser.parse_args(sys.argv[1:2]) + if not hasattr(self, args.command): + print('Unrecognized command: {0}'.format(args.command)) + parser.print_help() + exit(1) + getattr(self, args.command)() + + def freeze(self): + parser = argparse.ArgumentParser(description = 'Freeze the version files') + parser.add_argument('-t', '--target_path', default='./target', help='target path') + parser.add_argument('-s', '--source_path', default='.', help='source path') + + # store_true which implies default=False + parser.add_argument('-r', '--rebuild', action='store_true', help='rebuild all versions') + parser.add_argument('-d', '--for_all_dist', action='store_true', help='apply the versions for all distributions') + parser.add_argument('-a', '--for_all_arch', action='store_true', help='apply the versions for all architectures') + parser.add_argument('-c', '--ctypes', default='all', help='component types to freeze') + args = parser.parse_args(sys.argv[2:]) + ctypes = args.ctypes.split(',') + if len(ctypes) == 0: + ctypes = ['all'] + build = VersionBuild(target_path=args.target_path, source_path=args.source_path) + build.freeze(rebuild=args.rebuild, for_all_dist=args.for_all_dist, for_all_arch=args.for_all_arch, ctypes=ctypes) + + def merge(self): + parser = argparse.ArgumentParser(description = 'Merge the version files') + parser.add_argument('-t', '--target_path', required=True, help='target path to save the merged version files') + parser.add_argument('-m', '--module_path', default=None, help='merge path, use the target path if not specified') + parser.add_argument('-b', '--base_path', required=True, help='base path, merge to the module path') + parser.add_argument('-e', '--exclude_module_path', default=None, help='exclude module path') + args = parser.parse_args(sys.argv[2:]) + module_path = args.module_path + if not module_path: + module_path = args.target_path + if not os.path.exists(module_path): + print('The module path {0} does not exist'.format(module_path)) + if not os.path.exists(args.target_path): + os.makedirs(args.target_path) + module = VersionModule() + module.load(module_path) + base_module = VersionModule() + base_module.load(args.base_path) + module.overwrite(base_module) + if args.exclude_module_path: + exclude_module = VersionModule() + exclude_module.load(args.exclude_module_path) + module.subtract(exclude_module) + module.dump(args.target_path) + + def generate(self): + parser = argparse.ArgumentParser(description = 'Generate the version files') + parser.add_argument('-t', '--target_path', required=True, help='target path to generate the version lock files') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-n', '--module_name', help="module name, such as docker-lldp, sonic-slave-buster, etc") + group.add_argument('-m', '--module_path', help="module apth, such as files/docker/versions/dockers/docker-lldp, files/docker/versions/dockers/sonic-slave-buster, etc") + parser.add_argument('-s', '--source_path', default='.', help='source path') + parser.add_argument('-d', '--distribution', required=True, help="distribution") + parser.add_argument('-a', '--architecture', required=True, help="architecture") + parser.add_argument('-p', '--priority', default=999, help="priority of the debian apt preference") + + args = parser.parse_args(sys.argv[2:]) + module_path = args.module_path + if not module_path: + module_path = VersionModule.get_module_path_by_name(args.source_path, args.module_name) + if not os.path.exists(args.target_path): + os.makedirs(args.target_path) + module = VersionModule() + module.load(module_path, filter_dist=args.distribution, filter_arch=args.architecture) + default_module_path = VersionModule.get_module_path_by_name(args.source_path, DEFAULT_MODULE) + default_module = VersionModule() + default_module.load(default_module_path, filter_dist=args.distribution, filter_arch=args.architecture) + config = module.get_config_module(default_module, args.distribution, args.architecture) + config.clean_info(force=True) + config.dump(args.target_path, config=True, priority=args.priority) + +if __name__ == "__main__": + VersionManagerCommands() diff --git a/slave.mk b/slave.mk index e818eeed16d9..6bd66ff589cd 100644 --- a/slave.mk +++ b/slave.mk @@ -100,6 +100,14 @@ list : include $(RULES_PATH)/config -include $(RULES_PATH)/config.user + +############################################################################### +## Version control related exports +############################################################################### +export PACKAGE_URL_PREFIX +export TRUSTED_GPG_URLS +export SONIC_VERSION_CONTROL_COMPONENTS + ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y) ENABLE_PFCWD_ON_START = y endif @@ -257,6 +265,9 @@ else $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH)) endif +# Overwrite the buildinfo in slave container +$(shell sudo scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)) + include Makefile.cache ifeq ($(SONIC_USE_DOCKER_BUILDKIT),y) @@ -637,6 +648,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g $(HEADER) # Apply series of patches if exist if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && QUILT_PATCHES=../$(notdir $($*.gz_PATH)).patch quilt push -a; popd; fi + # Prepare docker build info + scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo docker info $(LOG) docker build --squash --no-cache \ --build-arg http_proxy=$(HTTP_PROXY) \ @@ -646,7 +659,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g --build-arg guid=$(GUID) \ --build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \ --label Tag=$(SONIC_IMAGE_VERSION) \ + -f $(TARGET_DOCKERFILE)/Dockerfile.buildinfo \ -t $* $($*.gz_PATH) $(LOG) + scripts/collect_docker_version_files.sh $* $(TARGET_PATH) docker save $* | gzip -c > $@ # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -720,6 +735,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform $(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_whls=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_PYTHON_WHEELS)))\n" | awk '!a[$$0]++')) $(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_PACKAGES)))\n" | awk '!a[$$0]++')) j2 $($*.gz_PATH)/Dockerfile.j2 > $($*.gz_PATH)/Dockerfile + # Prepare docker build info + PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ + scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) docker info $(LOG) docker build --squash --no-cache \ --build-arg http_proxy=$(HTTP_PROXY) \ @@ -732,6 +752,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform --build-arg frr_user_gid=$(FRR_USER_GID) \ --label Tag=$(SONIC_IMAGE_VERSION) \ -t $* $($*.gz_PATH) $(LOG) + scripts/collect_docker_version_files.sh $* $(TARGET_PATH) docker save $* | gzip -c > $@ # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -764,6 +785,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG $(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_IMAGE_PACKAGES)))\n" | awk '!a[$$0]++')) ./build_debug_docker_j2.sh $* $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs > $($*.gz_PATH)/Dockerfile-dbg.j2 j2 $($*.gz_PATH)/Dockerfile-dbg.j2 > $($*.gz_PATH)/Dockerfile-dbg + # Prepare docker build info + PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ + scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH) docker info $(LOG) docker build \ $(if $($*.gz_DBG_DEPENDS), --squash --no-cache, --no-cache) \ @@ -773,6 +799,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG --label Tag=$(SONIC_IMAGE_VERSION) \ --file $($*.gz_PATH)/Dockerfile-dbg \ -t $*-dbg $($*.gz_PATH) $(LOG) + scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) docker save $*-dbg | gzip -c > $@ # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -970,6 +997,10 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ USERNAME="$(USERNAME)" \ PASSWORD="$(PASSWORD)" \ IMAGE_TYPE=$($*_IMAGE_TYPE) \ + TARGET_PATH=$(TARGET_PATH) \ + SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ + PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ ./build_debian.sh $(LOG) USERNAME="$(USERNAME)" \ diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index 4a6500a48f2d..e62ca9462610 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -16,7 +16,9 @@ RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bust echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list + echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list {%- if CONFIGURED_ARCH == "armhf" %} RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \ @@ -25,7 +27,9 @@ RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non- echo "deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list + echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \ + echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list {%- elif CONFIGURED_ARCH == "arm64" %} RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \ echo "deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \ @@ -33,7 +37,9 @@ RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non- echo "deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list + echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list {%- endif %} ## Make apt-get non-interactive diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index 6d4f8b4acdfd..c1e9d10951e0 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -14,7 +14,10 @@ RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stre echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian stretch-backports main" >> /etc/apt/sources.list + echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian stretch-backports main" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.microsoft.com/debian/9/prod stretch main" >> /etc/apt/sources.list {%- if CONFIGURED_ARCH == "armhf" %} RUN echo "deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \ @@ -23,7 +26,9 @@ RUN echo "deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non echo "deb-src [arch=armhf] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list + echo 'deb [arch=armhf] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \ + echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list {%- elif CONFIGURED_ARCH == "arm64" %} RUN echo "deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \ echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ @@ -31,7 +36,9 @@ RUN echo "deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list + echo 'deb [arch=arm64] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list {%- endif %} ## Make apt-get non-interactive diff --git a/src/sonic-build-hooks/.gitignore b/src/sonic-build-hooks/.gitignore new file mode 100644 index 000000000000..4490afae4a5e --- /dev/null +++ b/src/sonic-build-hooks/.gitignore @@ -0,0 +1,2 @@ +buildinfo +tmp diff --git a/src/sonic-build-hooks/Makefile b/src/sonic-build-hooks/Makefile new file mode 100644 index 000000000000..786af9056332 --- /dev/null +++ b/src/sonic-build-hooks/Makefile @@ -0,0 +1,39 @@ +SONIC_BUILD_HOOKS = sonic-build-hooks +SONIC_BUILD_HOOKS_VERSION = 1.0 +SONIC_BUILD_HOOKS_PACKAGE = $(SONIC_BUILD_HOOKS)_$(SONIC_BUILD_HOOKS_VERSION)_all.deb + +BUILDINFO_DIR = buildinfo +TMP_DIR = tmp +SYMBOL_LINKS_SRC_DIR = ../../usr/local/share/buildinfo/scripts +SYMBOL_LINKS = symlink_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files +SONIC_BUILD_HOOKS_TARGET = $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS_PACKAGE) +BUILD_ROOT_DIR = $(TMP_DIR)/$(SONIC_BUILD_HOOKS) +DEBIAN_DIR = $(BUILD_ROOT_DIR)/DEBIAN +TRUSTED_GPG_PATH = $(BUILD_ROOT_DIR)/etc/apt/trusted.gpg.d +INSTALL_PATH = $(BUILD_ROOT_DIR)/usr/local/share/buildinfo +SYMBOL_LINK_PATH = $(BUILD_ROOT_DIR)/usr/sbin +SCRIPTS_PATH = $(INSTALL_PATH)/scripts +HOOKS_PATH = $(INSTALL_PATH)/hooks +DPKGTOOL = $(shell which dpkg-deb) + + +# If the depk-deb not installed, use the docker container to make the debian package +ifeq ($(shell which dpkg-deb),) +BUILD_COMMAND=docker run --user $(shell id -u):$(shell id -g) --rm -v $(shell pwd):/build debian:buster bash -c 'cd /build; dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_HOOKS) $(SONIC_BUILD_HOOKS_TARGET)' +else +BUILD_COMMAND=dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_HOOKS) $(SONIC_BUILD_HOOKS_TARGET) +endif + +DEPENDS := $(shell find scripts hooks debian -type f) +$(SONIC_BUILD_HOOKS_TARGET): $(DEPENDS) + @rm -rf $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS) $(TMP_DIR) + @mkdir -p $(DEBIAN_DIR) $(SCRIPTS_PATH) $(HOOKS_PATH) $(SYMBOL_LINK_PATH) $(TRUSTED_GPG_PATH) $(BUILDINFO_DIR) + @chmod 0775 $(DEBIAN_DIR) + @cp debian/* $(DEBIAN_DIR)/ + @cp scripts/* $(SCRIPTS_PATH)/ + @cp hooks/* $(HOOKS_PATH)/ + @for url in $$(echo $(TRUSTED_GPG_URLS) | sed 's/[,;]/ /g'); do wget -q "$$url" -P "$(TRUSTED_GPG_PATH)/"; done + @for f in $(SYMBOL_LINKS); do ln -s $(SYMBOL_LINKS_SRC_DIR)/$$f $(SYMBOL_LINK_PATH)/$$f; done + @$(BUILD_COMMAND) + +all: $(SONIC_BUILD_HOOKS_TARGET) diff --git a/src/sonic-build-hooks/debian/control b/src/sonic-build-hooks/debian/control new file mode 100644 index 000000000000..9e2454be9d95 --- /dev/null +++ b/src/sonic-build-hooks/debian/control @@ -0,0 +1,10 @@ +Package: sonic-build-hooks +Version: 1.0 +Section: devel +Priority: optional +Architecture: all +Depends: +Maintainer: SONiC +Description: sonic build hooks + Hooks the build tools, such as apt-get, wget, pip, etc. + It is used to monitor and control the packages installed during the build. diff --git a/src/sonic-build-hooks/hooks/apt-get b/src/sonic-build-hooks/hooks/apt-get new file mode 100755 index 000000000000..9ae50a118650 --- /dev/null +++ b/src/sonic-build-hooks/hooks/apt-get @@ -0,0 +1,38 @@ +#!/bin/bash + +INSTALL= +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +REAL_COMMAND=$(get_command apt-get) +if [ -z "$REAL_COMMAND" ]; then + echo "The command apt-get does not exist." 1>&2 + exit 1 +fi + +VERSION_FILE="/usr/local/share/buildinfo/versions/versions-deb" +if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then + for para in $@ + do + if [[ "$para" != -* ]]; then + continue + fi + if [ ! -z "$INSTALL" ]; then + if [[ "$para" == *=* ]]; then + continue + elif [[ "$para" == *=* ]]; then + continue + else + package=$para + if ! grep -q "^${package}=" $VERSION_FILE; then + echo "The version of the package ${package} is not specified." + exit 1 + fi + fi + elif [[ "$para" == "install" ]]; then + INSTALL=y + fi + done +fi + + +$REAL_COMMAND "$@" diff --git a/src/sonic-build-hooks/hooks/curl b/src/sonic-build-hooks/hooks/curl new file mode 100755 index 000000000000..58bfe0395e55 --- /dev/null +++ b/src/sonic-build-hooks/hooks/curl @@ -0,0 +1,6 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh +[ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/curl + +REAL_COMMAND=$REAL_COMMAND download_packages "$@" diff --git a/src/sonic-build-hooks/hooks/pip b/src/sonic-build-hooks/hooks/pip new file mode 120000 index 000000000000..b31c5d8ca9d0 --- /dev/null +++ b/src/sonic-build-hooks/hooks/pip @@ -0,0 +1 @@ +pip2 \ No newline at end of file diff --git a/src/sonic-build-hooks/hooks/pip2 b/src/sonic-build-hooks/hooks/pip2 new file mode 100755 index 000000000000..e2ccfff2aac0 --- /dev/null +++ b/src/sonic-build-hooks/hooks/pip2 @@ -0,0 +1,13 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +VERSION_FILE="$BUILDINFO_PATH/versions/versions-py2" +REAL_COMMAND=$(get_command pip2) + +if [ ! -x "$REAL_COMMAND" ]; then + echo "The command pip2 not found" 1>&2 + exit 1 +fi + +PIP_VERSION_FILE=$VERSION_FILE ENABLE_VERSION_CONTROL_PY=$ENABLE_VERSION_CONTROL_PY2 REAL_COMMAND=$REAL_COMMAND run_pip_command "$@" diff --git a/src/sonic-build-hooks/hooks/pip3 b/src/sonic-build-hooks/hooks/pip3 new file mode 100755 index 000000000000..36a8fd93a13d --- /dev/null +++ b/src/sonic-build-hooks/hooks/pip3 @@ -0,0 +1,12 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +VERSION_FILE="$BUILDINFO_PATH/versions/versions-py3" +REAL_COMMAND=$(get_command pip3) +if [ ! -x "$REAL_COMMAND" ]; then + echo "The command pip3 not found" 1>&2 + exit 1 +fi + +PIP_VERSION_FILE=$VERSION_FILE ENABLE_VERSION_CONTROL_PY=$ENABLE_VERSION_CONTROL_PY2 REAL_COMMAND=$REAL_COMMAND run_pip_command "$@" diff --git a/src/sonic-build-hooks/hooks/wget b/src/sonic-build-hooks/hooks/wget new file mode 100755 index 000000000000..c4cb1a3d1be9 --- /dev/null +++ b/src/sonic-build-hooks/hooks/wget @@ -0,0 +1,15 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh +[ -z $REAL_COMMAND ] && REAL_COMMAND=$(get_command wget) +if [ -z "$REAL_COMMAND" ]; then + echo "The command wget does not exist." 1>&2 + exit 1 +fi + +if [ "$SKIP_BUILD_HOOK" == y ]; then + $REAL_COMMAND "$@" + exit $? +fi + +REAL_COMMAND=$REAL_COMMAND download_packages "$@" diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh new file mode 100755 index 000000000000..d19558c544e3 --- /dev/null +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +BUILDINFO_PATH=/usr/local/share/buildinfo + +LOG_PATH=$BUILDINFO_PATH/log +VERSION_PATH=$BUILDINFO_PATH/versions +PRE_VERSION_PATH=$BUILDINFO_PATH/pre-versions +DIFF_VERSION_PATH=$BUILDINFO_PATH/diff-versions +BUILD_VERSION_PATH=$BUILDINFO_PATH/build-versions +POST_VERSION_PATH=$BUILDINFO_PATH/post-versions +VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb +WEB_VERSION_FILE=$VERSION_PATH/versions-web +BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web + +. $BUILDINFO_PATH/config/buildinfo.config + +URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#") + +log_err() +{ + echo "$1" >> $LOG_PATH/error.log + echo "$1" 1>&2 +} + +get_command() +{ + local path=$(echo $PATH | sed 's#[^:]*buildinfo/scripts:##' | sed "s#/usr/sbin:##") + local command=$(PATH=$path which $1) + echo $command +} + +check_version_control() +{ + if [[ ",$SONIC_VERSION_CONTROL_COMPONENTS," == *,all,* ]] || [[ ",$SONIC_VERSION_CONTROL_COMPONENTS," == *,$1,* ]]; then + echo "y" + else + echo "n" + fi +} + +get_url_version() +{ + local package_url=$1 + /usr/bin/curl -ks $package_url | md5sum | cut -d' ' -f1 +} + +check_if_url_exist() +{ + local url=$1 + if /usr/bin/curl --output /dev/null --silent --head --fail "$1" > /dev/null 2>&1; then + echo y + else + echo n + fi +} + +download_packages() +{ + local parameters=("$@") + local filenames= + declare -A filenames + for (( i=0; i<${#parameters[@]}; i++ )) + do + local para=${parameters[$i]} + local nexti=$((i+1)) + if [[ "$para" == *://* ]]; then + local url=$para + local real_version= + if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then + local version= + local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) + [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}') + if [ -z "$version" ]; then + echo "Failed to verify the package: $url, the version is not specified" 2>&1 + exit 1 + fi + + local version_filename="${filename}-${version}" + local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}" + local url_exist=$(check_if_url_exist $proxy_url) + if [ "$url_exist" == y ]; then + parameters[$i]=$proxy_url + filenames[$version_filename]=$filename + real_version=$version + else + real_version=$(get_url_version $url) + if [ "$real_version" != "$version" ]; then + echo "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2 + exit 1 + fi + fi + else + real_version=$(get_url_version $url) + fi + + echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE} + fi + done + + $REAL_COMMAND "${parameters[@]}" + local result=$? + + for filename in "${!filenames[@]}" + do + [ -f "$filename" ] && mv "$filename" "${filenames[$filename]}" + done + + return $result +} + +run_pip_command() +{ + parameters=("$@") + + if [ ! -x "$REAL_COMMAND" ] && [ " $1" == "freeze" ]; then + return 1 + fi + + if [ "$ENABLE_VERSION_CONTROL_PY" != "y" ]; then + $REAL_COMMAND "$@" + return $? + fi + + local found=n + local install=n + local pip_version_file=$PIP_VERSION_FILE + local tmp_version_file=$(mktemp) + [ -f "$pip_version_file" ] && cp -f $pip_version_file $tmp_version_file + for para in "${parameters[@]}" + do + ([ "$para" == "-c" ] || [ "$para" == "--constraint" ]) && found=y + if [ "$para" == "install" ]; then + install=y + elif [[ "$para" == *.whl ]]; then + package_name=$(echo $para | cut -d- -f1 | tr _ .) + sed "/^${package_name}==/d" -i $tmp_version_file + fi + done + + if [ "$found" == "n" ] && [ "$install" == "y" ]; then + parameters+=("-c") + parameters+=("${tmp_version_file}") + fi + + $REAL_COMMAND "${parameters[@]}" + local result=$? + rm $tmp_version_file + return $result +} + +ENABLE_VERSION_CONTROL_DEB=$(check_version_control "deb") +ENABLE_VERSION_CONTROL_PY2=$(check_version_control "py2") +ENABLE_VERSION_CONTROL_PY3=$(check_version_control "py3") +ENABLE_VERSION_CONTROL_WEB=$(check_version_control "web") +ENABLE_VERSION_CONTROL_GIT=$(check_version_control "git") +ENABLE_VERSION_CONTROL_DOCKER=$(check_version_control "docker") diff --git a/src/sonic-build-hooks/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files new file mode 100755 index 000000000000..b62beb4a2115 --- /dev/null +++ b/src/sonic-build-hooks/scripts/collect_version_files @@ -0,0 +1,14 @@ +#!/bin/bash + +TARGET_PATH=$1 +ARCH=$(dpkg --print-architecture) +DIST=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) +([ -z "$DIST" ] && grep -q jessie /etc/os-release) && DIST=jessie + +mkdir -p $TARGET_PATH +chmod a+rw $TARGET_PATH +dpkg-query -W -f '${Package}==${Version}\n' > "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" +([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze > "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" +([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze > "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" + +exit 0 diff --git a/src/sonic-build-hooks/scripts/post_run_buildinfo b/src/sonic-build-hooks/scripts/post_run_buildinfo new file mode 100755 index 000000000000..a8dab41021b1 --- /dev/null +++ b/src/sonic-build-hooks/scripts/post_run_buildinfo @@ -0,0 +1,14 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +[ -d $POST_VERSION_PATH ] && rm -rf $POST_VERSION_PATH + +# Collect the version files +collect_version_files $POST_VERSION_PATH + +[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH +rm -rf $BUILD_VERSION_PATH/* + +# Disable the build hooks +symlink_build_hooks -d diff --git a/src/sonic-build-hooks/scripts/pre_run_buildinfo b/src/sonic-build-hooks/scripts/pre_run_buildinfo new file mode 100755 index 000000000000..a8450690b6ff --- /dev/null +++ b/src/sonic-build-hooks/scripts/pre_run_buildinfo @@ -0,0 +1,20 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +[ -d $DIFF_VERSION_PATH ] && rm -rf $DIFF_VERSION_PATH +mkdir -p $DIFF_VERSION_PATH +mkdir -p $BUILD_VERSION_PATH +mkdir -p $LOG_PATH + +[ -d $PRE_VERSION_PATH ] && rm -rf $PRE_VERSION_PATH +collect_version_files $PRE_VERSION_PATH +symlink_build_hooks + +chmod -R a+rw $BUILDINFO_PATH + +if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]; then + cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ +fi + +exit 0 diff --git a/src/sonic-build-hooks/scripts/symlink_build_hooks b/src/sonic-build-hooks/scripts/symlink_build_hooks new file mode 100755 index 000000000000..7218575ee9ab --- /dev/null +++ b/src/sonic-build-hooks/scripts/symlink_build_hooks @@ -0,0 +1,34 @@ +#!/bin/bash + +HOOK_PATH=/usr/local/share/buildinfo/hooks +TARGET_PATH=/usr/sbin +FILES=$(ls $HOOK_PATH) + +usage() +{ + echo "Usage: $0 [-d]" + exit 1 +} + +DISABLE=n +while getopts "d" opt; do + case $opt in + d) + DISABLE=y + ;; + *) + usage + ;; + esac +done + +for f in $FILES +do + if [ $DISABLE == "n" ]; then + [ ! -e $TARGET_PATH/$f ] && ln -s $HOOK_PATH/$f $TARGET_PATH/$f + else + ([ -e $TARGET_PATH/$f ] && ls -l $TARGET_PATH/$f | grep -q $HOOK_PATH) && rm -f $TARGET_PATH/$f + fi +done + +exit 0