From 6453b3ac1b961bbff5b608bc29f23708cabfcf70 Mon Sep 17 00:00:00 2001 From: Jason Tsai Date: Sun, 10 Jun 2018 02:22:28 +0800 Subject: [PATCH] Fix build errors in branch 201712 (#1775) * Fix links of debian git server (#1755) * [dockers] Prevent apt-get from installing suggested and recommended packages by default (#1666) * [cfggen]: Fix build by fixing pyangbind version (#1633) * [Ingrasys] Add platform support for S9280-64X with Barefoot ASIC * Revert "[Ingrasys] Add platform support for S9280-64X with Barefoot ASIC" This reverts commit 29b9d904f67170c0faa8b2b8a4ead84f905325f1. Separate S9280-64X platform driver files into a new PR --- dockers/docker-base/Dockerfile.j2 | 3 +- .../docker-base/no_install_recommend_suggest | 5 ++ dockers/docker-database/Dockerfile.j2 | 46 +++++----- dockers/docker-fpm-quagga/Dockerfile.j2 | 31 ++++--- dockers/docker-snmp-sv2/Dockerfile.j2 | 84 +++++++++++-------- platform/p4/docker-sonic-p4.mk | 2 +- platform/vs/docker-sonic-vs.mk | 2 +- platform/vs/docker-sonic-vs/Dockerfile.j2 | 25 +++--- rules/docker-database.mk | 2 +- rules/docker-snmp-sv2.mk | 2 +- sonic-slave/Dockerfile | 16 ++-- src/initramfs-tools/Makefile | 2 +- src/isc-dhcp/Makefile | 2 +- src/libnl3/Makefile | 2 +- src/libteam/Makefile | 2 +- src/sonic-config-engine/setup.py | 2 +- 16 files changed, 129 insertions(+), 99 deletions(-) create mode 100644 dockers/docker-base/no_install_recommend_suggest diff --git a/dockers/docker-base/Dockerfile.j2 b/dockers/docker-base/Dockerfile.j2 index ff52cf88e7d3..db693540fb32 100644 --- a/dockers/docker-base/Dockerfile.j2 +++ b/dockers/docker-base/Dockerfile.j2 @@ -18,8 +18,9 @@ RUN rm -rf \ ENV DEBIAN_FRONTEND=noninteractive # Configure data sources for apt/dpkg -COPY ["sources.list", "/etc/apt/sources.list"] COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] +COPY ["sources.list", "/etc/apt/sources.list"] +COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] RUN apt-get update # Pre-install fundamental packages diff --git a/dockers/docker-base/no_install_recommend_suggest b/dockers/docker-base/no_install_recommend_suggest new file mode 100644 index 000000000000..b5bca577de1e --- /dev/null +++ b/dockers/docker-base/no_install_recommend_suggest @@ -0,0 +1,5 @@ +# Instruct apt-get to NOT install "recommended" or "suggested" packages by +# default when installing a package. + +APT::Install-Recommends "false"; +APT::Install-Suggests "false"; diff --git a/dockers/docker-database/Dockerfile.j2 b/dockers/docker-database/Dockerfile.j2 index 7fb3401184f2..4690e1587c14 100644 --- a/dockers/docker-database/Dockerfile.j2 +++ b/dockers/docker-database/Dockerfile.j2 @@ -1,34 +1,34 @@ FROM docker-config-engine -## Make apt-get non-interactive +# Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Update apt's cache of available packages RUN apt-get update -COPY \ -{% for deb in docker_database_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor -%} -debs/ - -## Install redis-tools dependencies -## TODO: implicitly install dependencies -RUN apt-get -y install libjemalloc1 - -RUN dpkg -i \ -{% for deb in docker_database_debs.split(' ') -%} -debs/{{ deb }}{{' '}} +{% if docker_database_debs.strip() -%} +# Copy locally-built Debian package dependencies +{%- for deb in docker_database_debs.split(' ') %} +COPY debs/{{ deb }} /debs/ {%- endfor %} -## Clean up -RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y -RUN rm -rf /debs - -RUN sed -ri 's/^(save .*$)/# \1/g; \ - s/^daemonize yes$/daemonize no/; \ - s/^logfile .*$/logfile ""/; \ - s/^# syslog-enabled no$/syslog-enabled no/; \ - s/^# unixsocket/unixsocket/; \ +# Install locally-built Debian packages and implicitly install their dependencies +{%- for deb in docker_database_debs.split(' ') %} +RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /debs/{{ deb }} +{%- endfor %} +{%- endif %} + +# Clean up +RUN apt-get clean -y +RUN apt-get autoclean -y +RUN apt-get autoremove -y +RUN rm -rf /debs ~/.cache + +RUN sed -ri 's/^(save .*$)/# \1/g; \ + s/^daemonize yes$/daemonize no/; \ + s/^logfile .*$/logfile ""/; \ + s/^# syslog-enabled no$/syslog-enabled no/; \ + s/^# unixsocket/unixsocket/; \ s/^client-output-buffer-limit pubsub [0-9]+mb [0-9]+mb [0-9]+/client-output-buffer-limit pubsub 0 0 0/ \ ' /etc/redis/redis.conf diff --git a/dockers/docker-fpm-quagga/Dockerfile.j2 b/dockers/docker-fpm-quagga/Dockerfile.j2 index c10cf0fd71d9..8836459e4e93 100644 --- a/dockers/docker-fpm-quagga/Dockerfile.j2 +++ b/dockers/docker-fpm-quagga/Dockerfile.j2 @@ -1,26 +1,31 @@ FROM docker-config-engine -## Make apt-get non-interactive +# Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Update apt's cache of available packages RUN apt-get update -RUN apt-get install -y libdbus-1-3 libdaemon0 libjansson4 +# Install required packages +RUN apt-get install -y libdbus-1-3 libdaemon0 libjansson4 -COPY \ -{% for deb in docker_fpm_quagga_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor -%} -debs/ +{% if docker_fpm_quagga_debs.strip() -%} +# Copy locally-built Debian package dependencies +{%- for deb in docker_fpm_quagga_debs.split(' ') %} +COPY debs/{{ deb }} /debs/ +{%- endfor %} -RUN dpkg -i \ -{% for deb in docker_fpm_quagga_debs.split(' ') -%} -debs/{{ deb }}{{' '}} +# Install locally-built Debian packages and implicitly install their dependencies +{%- for deb in docker_fpm_quagga_debs.split(' ') %} +RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /debs/{{ deb }} {%- endfor %} +{%- endif %} -## Clean up -RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y -RUN rm -rf /debs +# Clean up +RUN apt-get clean -y +RUN apt-get autoclean -y +RUN apt-get autoremove -y +RUN rm -rf /debs ~/.cache COPY ["bgpcfgd", "start.sh", "/usr/bin/"] COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] diff --git a/dockers/docker-snmp-sv2/Dockerfile.j2 b/dockers/docker-snmp-sv2/Dockerfile.j2 index 3c5473951116..a6d5a98a82a3 100644 --- a/dockers/docker-snmp-sv2/Dockerfile.j2 +++ b/dockers/docker-snmp-sv2/Dockerfile.j2 @@ -1,50 +1,62 @@ FROM docker-config-engine -COPY [ \ -{% for deb in docker_snmp_sv2_debs.split(' ') -%} - "debs/{{ deb }}", -{%- endfor %} \ - "/debs/"] - -# Install Python SwSSSDK (SNMP subagent dependency) -COPY python-wheels/sonic_utilities-*-py3-*.whl /python-wheels/ -COPY python-wheels/swsssdk-*-py3-*.whl /python-wheels/ -COPY python-wheels/asyncsnmp-*-py3-*.whl /python-wheels/ - -# enable -O for all Python calls +# Enable -O for all Python calls ENV PYTHONOPTIMIZE 1 -## Make apt-get non-interactive +# Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive -# install libsnmp30 dependencies -# install libpython3.6-dev dependencies -# install pip dependencies -# TODO: remove libpython3.6-dev, its and pip's dependencies if we can get pip3 directly -# install subagent -# clean up -RUN apt-get update && apt-get install -y libperl5.20 libpci3 libwrap0 \ - libexpat1-dev \ - curl gcc && \ - dpkg -i \ -{% for deb in docker_snmp_sv2_debs.split(' ') -%} - debs/{{ deb }}{{' '}} -{%- endfor %} && \ - rm -rf /debs && \ - curl https://bootstrap.pypa.io/get-pip.py | python3.6 && \ - python3.6 -m pip install --no-cache-dir /python-wheels/*py3*.whl hiredis && \ - rm -rf /python-wheels && \ - python3.6 -m sonic_ax_impl install && \ - apt-get -y purge libpython3.6-dev libexpat1-dev curl gcc && \ - apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y --purge && \ - find / | grep -E "__pycache__" | xargs rm -rf && \ - rm -rf ~/.cache +# Update apt's cache of available packages +RUN apt-get update + +# Install curl so we can download and install pip later +# Also install major root CA certificates for curl to reference +RUN apt-get install -y curl ca-certificates + +# Install gcc which is required for installing hiredis +RUN apt-get install -y gcc + +{% if docker_snmp_sv2_debs.strip() -%} +# Copy locally-built Debian package dependencies +{%- for deb in docker_snmp_sv2_debs.split(' ') %} +COPY debs/{{ deb }} /debs/ +{%- endfor %} + +# Install locally-built Debian packages and implicitly install their dependencies +{%- for deb in docker_snmp_sv2_debs.split(' ') %} +RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /debs/{{ deb }} +{%- endfor %} +{%- endif %} + +# Install up-to-date version of pip +RUN curl https://bootstrap.pypa.io/get-pip.py | python3.6 +RUN python3.6 -m pip install --no-cache-dir hiredis + +{% if docker_snmp_sv2_whls.strip() -%} +# Copy locally-built Python wheel dependencies +{%- for whl in docker_snmp_sv2_whls.split(' ') %} +COPY python-wheels/{{ whl }} /python-wheels/ +{%- endfor %} + +# Install locally-built Python wheel dependencies +{%- for whl in docker_snmp_sv2_whls.split(' ') %} +RUN pip install /python-wheels/{{ whl }} +{%- endfor %} +{% endif %} + +RUN python3.6 -m sonic_ax_impl install + +# Clean up +RUN apt-get -y purge libpython3.6-dev curl gcc +RUN apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y --purge +RUN find / | grep -E "__pycache__" | xargs rm -rf +RUN rm -rf /debs /python-wheels ~/.cache COPY ["start.sh", "/usr/bin/"] COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] COPY ["*.j2", "/usr/share/sonic/templates/"] -## Although exposing ports is not needed for host net mode, keep it for possible bridge mode +# Although exposing ports is not needed for host net mode, keep it for possible bridge mode EXPOSE 161/udp 162/udp ENTRYPOINT ["/usr/bin/supervisord"] diff --git a/platform/p4/docker-sonic-p4.mk b/platform/p4/docker-sonic-p4.mk index ebe7ceb080a2..696bac8bfaef 100644 --- a/platform/p4/docker-sonic-p4.mk +++ b/platform/p4/docker-sonic-p4.mk @@ -5,8 +5,8 @@ $(DOCKER_SONIC_P4)_PATH = $(PLATFORM_PATH)/docker-sonic-p4 $(DOCKER_SONIC_P4)_DEPENDS += $(SWSS) \ $(SYNCD) \ $(P4_SWITCH) \ - $(REDIS_SERVER) \ $(REDIS_TOOLS) \ + $(REDIS_SERVER) \ $(PYTHON_SWSSCOMMON) \ $(LIBTEAMDCT) \ $(LIBTEAM_UTILS) \ diff --git a/platform/vs/docker-sonic-vs.mk b/platform/vs/docker-sonic-vs.mk index 656c53644637..ce0f4f3dea09 100644 --- a/platform/vs/docker-sonic-vs.mk +++ b/platform/vs/docker-sonic-vs.mk @@ -4,8 +4,8 @@ DOCKER_SONIC_VS = docker-sonic-vs.gz $(DOCKER_SONIC_VS)_PATH = $(PLATFORM_PATH)/docker-sonic-vs $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS) \ $(SYNCD_VS) \ - $(REDIS_SERVER) \ $(REDIS_TOOLS) \ + $(REDIS_SERVER) \ $(PYTHON_SWSSCOMMON) \ $(LIBTEAMDCT) \ $(LIBTEAM_UTILS) \ diff --git a/platform/vs/docker-sonic-vs/Dockerfile.j2 b/platform/vs/docker-sonic-vs/Dockerfile.j2 index a26c3f475fa6..b66a3507e91a 100644 --- a/platform/vs/docker-sonic-vs/Dockerfile.j2 +++ b/platform/vs/docker-sonic-vs/Dockerfile.j2 @@ -43,20 +43,23 @@ RUN apt-get install -y net-tools \ RUN pip install setuptools RUN pip install py2_ipaddress -COPY \ -{% for deb in docker_sonic_vs_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor -%} -debs/ +{% if docker_sonic_vs_debs.strip() -%} +# Copy locally-built Debian package dependencies +{%- for deb in docker_sonic_vs_debs.split(' ') %} +COPY debs/{{ deb }} /debs/ +{%- endfor %} -RUN dpkg -i \ -{% for deb in docker_sonic_vs_debs.split(' ') -%} -debs/{{ deb }}{{' '}} +# Install locally-built Debian packages and implicitly install their dependencies +{%- for deb in docker_sonic_vs_debs.split(' ') %} +RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /debs/{{ deb }} {%- endfor %} +{%- endif %} -## Clean up -RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y -RUN rm -rf /debs +# Clean up +RUN apt-get clean -y +RUN apt-get autoclean -y +RUN apt-get autoremove -y +RUN rm -rf /debs ~/.cache RUN sed -ri 's/^(save .*$)/# \1/g; \ s/^daemonize yes$/daemonize no/; \ diff --git a/rules/docker-database.mk b/rules/docker-database.mk index ef77047e8431..7625f8b37ab6 100644 --- a/rules/docker-database.mk +++ b/rules/docker-database.mk @@ -2,7 +2,7 @@ DOCKER_DATABASE = docker-database.gz $(DOCKER_DATABASE)_PATH = $(DOCKERS_PATH)/docker-database -$(DOCKER_DATABASE)_DEPENDS += $(REDIS_SERVER) $(REDIS_TOOLS) +$(DOCKER_DATABASE)_DEPENDS += $(REDIS_TOOLS) $(REDIS_SERVER) $(DOCKER_DATABASE)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE) SONIC_DOCKER_IMAGES += $(DOCKER_DATABASE) SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_DATABASE) diff --git a/rules/docker-snmp-sv2.mk b/rules/docker-snmp-sv2.mk index 95fdcc25775c..e8eabd1e08eb 100644 --- a/rules/docker-snmp-sv2.mk +++ b/rules/docker-snmp-sv2.mk @@ -4,7 +4,7 @@ DOCKER_SNMP_SV2 = docker-snmp-sv2.gz $(DOCKER_SNMP_SV2)_PATH = $(DOCKERS_PATH)/docker-snmp-sv2 ## TODO: remove LIBPY3_DEV if we can get pip3 directly $(DOCKER_SNMP_SV2)_DEPENDS += $(SNMP) $(SNMPD) $(PY3) $(LIBPY3_DEV) -$(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(ASYNCSNMP_PY3) +$(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(SONIC_PLATFORM_COMMON_PY3) $(SWSSSDK_PY3) $(ASYNCSNMP_PY3) $(DOCKER_SNMP_SV2)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE) SONIC_DOCKER_IMAGES += $(DOCKER_SNMP_SV2) SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_SNMP_SV2) diff --git a/sonic-slave/Dockerfile b/sonic-slave/Dockerfile index a43bdd5c1550..5b3c7156f5b3 100644 --- a/sonic-slave/Dockerfile +++ b/sonic-slave/Dockerfile @@ -189,7 +189,6 @@ RUN apt-get update && apt-get install -y \ libjs-sphinxdoc \ libjs-underscore \ python-docutils \ - python-jinja2 \ python-markupsafe \ python-pygments \ python-roman \ @@ -198,7 +197,6 @@ RUN apt-get update && apt-get install -y \ python3-sphinx \ # For sonic config engine testing python-lxml \ - python-jinja2 \ python-netaddr \ python-ipaddr \ python-yaml \ @@ -234,16 +232,22 @@ RUN export VERSION=1.8.3 \ && echo 'export GOROOT=/usr/local/go' >> /etc/bash.bashrc \ && echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc +# Upgrade pip2 +# Note: use pip2 specific version so jinja2 2.10 will install +RUN python2 -m pip install -U pip==9.0.3 + # For p4 build RUN pip install \ ctypesgen \ crc16 -# For templating -RUN pip install j2cli - # For sonic config engine testing -RUN pip install pyangbind==0.5.10 +RUN pip install pyangbind==0.6.0 +# Note: force upgrade debian packaged jinja2, if installed +RUN pip install --force-reinstall --upgrade jinja2>=2.10 + +# For templating (requiring jinja2) +RUN pip install j2cli # For sonic utilities testing RUN pip install click-default-group click natsort tabulate diff --git a/src/initramfs-tools/Makefile b/src/initramfs-tools/Makefile index 95d5c24d1185..c5010e876d8b 100644 --- a/src/initramfs-tools/Makefile +++ b/src/initramfs-tools/Makefile @@ -7,7 +7,7 @@ MAIN_TARGET = initramfs-tools_$(INITRAMFS_TOOLS_VERSION)_all.deb $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # Obtaining the initramfs-tools rm -rf ./initramfs-tools - git clone --branch v0.120 https://anonscm.debian.org/git/kernel/initramfs-tools.git ./initramfs-tools + git clone --branch v0.120 https://salsa.debian.org/kernel-team/initramfs-tools.git ./initramfs-tools # Patch pushd ./initramfs-tools diff --git a/src/isc-dhcp/Makefile b/src/isc-dhcp/Makefile index 5b1cb8407005..ce4fbc62608f 100644 --- a/src/isc-dhcp/Makefile +++ b/src/isc-dhcp/Makefile @@ -10,7 +10,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : rm -rf ./isc-dhcp # Clone isc-dhcp repo - git clone https://anonscm.debian.org/cgit/pkg-dhcp/isc-dhcp.git + git clone https://salsa.debian.org/berni/isc-dhcp.git pushd ./isc-dhcp # Reset HEAD to the commit of the proper tag diff --git a/src/libnl3/Makefile b/src/libnl3/Makefile index efd234424dff..6f50e4e389fa 100644 --- a/src/libnl3/Makefile +++ b/src/libnl3/Makefile @@ -16,7 +16,7 @@ DERIVED_TARGETS = libnl-3-dev_$(LIBNL3_VERSION)_amd64.deb \ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # Obtaining the libnl3 rm -rf ./libnl3 - git clone https://anonscm.debian.org/git/collab-maint/libnl3.git + git clone https://github.com/qiluo-msft/libnl3.git pushd ./libnl3 git checkout -f b77c0e49cb diff --git a/src/libteam/Makefile b/src/libteam/Makefile index c96266a5c7fd..a20f8dbd35b0 100644 --- a/src/libteam/Makefile +++ b/src/libteam/Makefile @@ -22,7 +22,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : popd # Obtain debian packaging - git clone https://anonscm.debian.org/git/collab-maint/libteam.git tmp + git clone https://salsa.debian.org/debian/libteam.git tmp pushd ./tmp git checkout -f da006f2 # v1.26 popd diff --git a/src/sonic-config-engine/setup.py b/src/sonic-config-engine/setup.py index 296b7a03a6b9..215e3163a29d 100755 --- a/src/sonic-config-engine/setup.py +++ b/src/sonic-config-engine/setup.py @@ -17,6 +17,6 @@ def get_test_suite(): url='https://github.com/Azure/sonic-buildimage', py_modules=['portconfig', 'minigraph', 'openconfig_acl', 'sonic_platform'], scripts=['sonic-cfggen'], - install_requires=['lxml', 'jinja2', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind'], + install_requires=['lxml', 'jinja2>=2.10', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind==0.6.0'], test_suite='setup.get_test_suite', )