From 1fe12b8e8c511256a328b58c897b39f25f86e62c Mon Sep 17 00:00:00 2001 From: winlin Date: Sat, 13 Aug 2022 10:27:14 +0800 Subject: [PATCH] Speedup test by using cache image. --- .github/workflows/codeql-analysis.yml | 5 +- .github/workflows/test.yml | 71 ++++++++++++++++----------- trunk/Dockerfile | 10 ---- trunk/Dockerfile.builds | 55 ++++++++------------- trunk/Dockerfile.cov | 2 +- trunk/Dockerfile.test | 2 +- trunk/auto/depends.sh | 16 ++++++ 7 files changed, 84 insertions(+), 77 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d61dadbd3d..38c3146a30 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,10 @@ jobs: - name: Build SRS run: | - cd trunk && ./configure && make + echo "pwd: $(pwd), who: $(whoami)" + docker run --rm -v $(pwd):$(pwd) -w $(pwd)/trunk ossrs/srs:ubuntu20-cache \ + bash -c "./configure --jobs=2 && chmod 777 -R objs" + cd trunk && ./configure --jobs=2 && make -j2 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35276d944b..1fb57ae72e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,8 +5,10 @@ on: [push, pull_request] jobs: build-centos7: - name: actions-test-build-centos7 + name: build-centos7 runs-on: ubuntu-20.04 + needs: + - multile-arch-amd64 steps: - name: Checkout repository uses: actions/checkout@v2 @@ -23,8 +25,10 @@ jobs: run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target centos7-ansi-no-ffmpeg . build-centos6: - name: actions-test-build-centos6 + name: build-centos6 runs-on: ubuntu-20.04 + needs: + - multile-arch-amd64 steps: - name: Checkout repository uses: actions/checkout@v2 @@ -34,21 +38,11 @@ jobs: - name: Build on CentOS6, with SRT run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target centos6-srt . - build-centos8: - name: actions-test-build-centos8 - runs-on: ubuntu-20.04 - steps: - - name: Checkout repository - uses: actions/checkout@v2 - # Build for CentOS 8 - - name: Build on CentOS8, baseline - run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target centos8-baseline . - - name: Build on CentOS8, with SRT - run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target centos8-srt . - build-ubuntu16: - name: actions-test-build-ubuntu16 + name: build-ubuntu16 runs-on: ubuntu-20.04 + needs: + - multile-arch-amd64 steps: - name: Checkout repository uses: actions/checkout@v2 @@ -59,8 +53,10 @@ jobs: run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target ubuntu16-srt . build-ubuntu18: - name: actions-test-build-ubuntu18 + name: build-ubuntu18 runs-on: ubuntu-20.04 + needs: + - multile-arch-amd64 steps: - name: Checkout repository uses: actions/checkout@v2 @@ -71,8 +67,10 @@ jobs: run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target ubuntu18-srt . build-ubuntu20: - name: actions-test-build-ubuntu20 + name: build-ubuntu20 runs-on: ubuntu-20.04 + needs: + - multile-arch-amd64 steps: - name: Checkout repository uses: actions/checkout@v2 @@ -82,40 +80,51 @@ jobs: - name: Build on Ubuntu20, with SRT run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target ubuntu20-srt . - build-cross: - name: actions-test-build-cross + build-cross-arm: + name: build-cross-arm runs-on: ubuntu-20.04 + needs: + - multile-arch-amd64 steps: - name: Checkout repository uses: actions/checkout@v2 - # Cross Build for ARMv7 - name: Cross Build for ARMv7 on Ubuntu16 run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target ubuntu16-cross-armv7 . - name: Cross Build for ARMv7 on Ubuntu20 run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target ubuntu20-cross-armv7 . - # Cross Build for AARCH64 + + build-cross-aarch64: + name: build-cross-aarch64 + runs-on: ubuntu-20.04 + needs: + - multile-arch-amd64 + steps: + - name: Checkout repository + uses: actions/checkout@v2 - name: Cross Build for AARCH64 on Ubuntu16 run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target ubuntu16-cross-aarch64 . - name: Cross Build for AARCH64 on Ubuntu20 run: DOCKER_BUILDKIT=1 docker build -f trunk/Dockerfile.builds --target ubuntu20-cross-aarch64 . build: - name: actions-test-build + name: build needs: - build-centos7 - build-centos6 - - build-centos8 - build-ubuntu16 - build-ubuntu18 - build-ubuntu20 - - build-cross + - build-cross-arm + - build-cross-aarch64 runs-on: ubuntu-20.04 steps: - run: echo 'Build done' utest: - name: actions-test-utest + name: utest runs-on: ubuntu-20.04 + needs: + - multile-arch-amd64 steps: - name: Checkout repository uses: actions/checkout@v2 @@ -130,8 +139,10 @@ jobs: run: docker run --rm srs:test bash -c 'make && ./objs/srs -c conf/regression-test.conf && cd 3rdparty/srs-bench && make && ./objs/srs_test -test.v' coverage: - name: actions-test-coverage + name: coverage runs-on: ubuntu-20.04 + needs: + - utest steps: - name: Checkout repository uses: actions/checkout@v2 @@ -161,7 +172,7 @@ jobs: # multile-arch-armv7: - name: actions-test-multile-arch-armv7 + name: multile-arch-armv7 runs-on: ubuntu-20.04 steps: - name: Checkout repository @@ -181,7 +192,7 @@ jobs: -f trunk/Dockerfile . multile-arch-aarch64: - name: actions-test-multile-arch-aarch64 + name: multile-arch-aarch64 runs-on: ubuntu-20.04 steps: - name: Checkout repository @@ -201,7 +212,7 @@ jobs: -f trunk/Dockerfile . multile-arch-amd64: - name: actions-test-multile-arch-amd64 + name: multile-arch-amd64 runs-on: ubuntu-20.04 steps: - name: Checkout repository @@ -231,7 +242,7 @@ jobs: - run: echo 'Artifacts done' done: - name: actions-test-done + name: done needs: - build - utest diff --git a/trunk/Dockerfile b/trunk/Dockerfile index ea55ecdb9e..2f00b255fa 100644 --- a/trunk/Dockerfile +++ b/trunk/Dockerfile @@ -22,16 +22,6 @@ RUN apt-get update && apt-get install -y gcc make g++ patch unzip perl git COPY . /srs WORKDIR /srs/trunk -# Use srs-cache from base image. -RUN if [[ -d /usr/local/srs-cache/srs/trunk/objs ]]; then \ - ./configure --generate-objs=on && \ - SOURCE_DIR=$(ls -d /usr/local/srs-cache/srs/trunk/objs/Platform-*) && \ - TARGET_DIR=$(ls -d objs/Platform-*) && \ - echo "Build from cache, source=$SOURCE_DIR, target=$TARGET_DIR" && \ - rm -rf $TARGET_DIR && mv $SOURCE_DIR $TARGET_DIR && \ - du -sh $TARGET_DIR && ls -lh objs; \ - fi - # Build and install SRS. # Note that SRT is enabled by default, so we configure without --srt=on. RUN ./configure --jobs=${JOBS} && make -j${JOBS} && make install diff --git a/trunk/Dockerfile.builds b/trunk/Dockerfile.builds index 7111eee3ed..97d74b070f 100644 --- a/trunk/Dockerfile.builds +++ b/trunk/Dockerfile.builds @@ -1,87 +1,74 @@ ######################################################## -FROM ossrs/srs:dev AS centos7-baseline +FROM ossrs/srs:dev-cache AS centos7-baseline COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=off && make -j2 -FROM ossrs/srs:dev AS centos7-no-webrtc +FROM ossrs/srs:dev-cache AS centos7-no-webrtc COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=off --rtc=off && make -j2 -FROM ossrs/srs:dev AS centos7-no-asm +FROM ossrs/srs:dev-cache AS centos7-no-asm COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=off --nasm=off --srtp-nasm=off && make -j2 -FROM ossrs/srs:dev AS centos7-srt +FROM ossrs/srs:dev-cache AS centos7-srt COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=on && make -j2 -FROM ossrs/srs:dev AS centos7-ansi-no-ffmpeg +FROM ossrs/srs:dev-cache AS centos7-ansi-no-ffmpeg COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=off --cxx11=off --cxx14=off --ffmpeg-fit=off && make -j2 ######################################################## -FROM ossrs/srs:dev6 AS centos6-baseline +FROM ossrs/srs:dev6-cache AS centos6-baseline COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=off --cxx11=off --cxx14=off && make -j2 -FROM ossrs/srs:dev6 AS centos6-srt +FROM ossrs/srs:dev6-cache AS centos6-srt COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=on --cxx11=off --cxx14=off && make -j2 ######################################################## -FROM ossrs/srs:dev8 AS centos8-baseline +FROM ossrs/srs:ubuntu16-cache AS ubuntu16-baseline COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=off && make -j2 -FROM ossrs/srs:dev8 AS centos8-srt +FROM ossrs/srs:ubuntu16-cache AS ubuntu16-srt COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=on && make -j2 ######################################################## -FROM ossrs/srs:ubuntu16 AS ubuntu16-baseline +FROM ossrs/srs:ubuntu18-cache AS ubuntu18-baseline COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=off && make -j2 -FROM ossrs/srs:ubuntu16 AS ubuntu16-srt +FROM ossrs/srs:ubuntu18-cache AS ubuntu18-srt COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=on && make -j2 ######################################################## -FROM ossrs/srs:ubuntu18 AS ubuntu18-baseline +FROM ossrs/srs:ubuntu20-cache AS ubuntu20-baseline COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=off && make -j2 -FROM ossrs/srs:ubuntu18 AS ubuntu18-srt +FROM ossrs/srs:ubuntu20-cache AS ubuntu20-srt COPY . /srs RUN cd /srs/trunk && ./configure --jobs=2 --srt=on && make -j2 ######################################################## -FROM ossrs/srs:ubuntu20 AS ubuntu20-baseline +FROM ossrs/srs:ubuntu16-cross-arm AS ubuntu16-cross-armv7 COPY . /srs -RUN cd /srs/trunk && ./configure --jobs=2 --srt=off && make -j2 - -FROM ossrs/srs:ubuntu20 AS ubuntu20-srt -COPY . /srs -RUN cd /srs/trunk && ./configure --jobs=2 --srt=on && make -j2 - -######################################################## -FROM ossrs/srs:ubuntu16 AS ubuntu16-cross-armv7 -COPY . /srs -RUN cd /srs/trunk && ./configure --jobs=2 --srt=off --cross-build --cc=arm-linux-gnueabihf-gcc --cxx=arm-linux-gnueabihf-g++ \ - --ar=arm-linux-gnueabihf-ar --ld=arm-linux-gnueabihf-ld --randlib=arm-linux-gnueabihf-randlib && make -j2 +RUN cd /srs/trunk && ./configure --jobs=2 --cross-build --cross-prefix=arm-linux-gnueabihf- && make -j2 -FROM ossrs/srs:ubuntu16 AS ubuntu16-cross-aarch64 +FROM ossrs/srs:ubuntu16-cross-aarch64 AS ubuntu16-cross-aarch64 COPY . /srs -RUN cd /srs/trunk && ./configure --jobs=2 --srt=off --cross-build --cc=aarch64-linux-gnu-gcc --cxx=aarch64-linux-gnu-g++ \ - --ar=aarch64-linux-gnu-ar --ld=aarch64-linux-gnu-ld --randlib=aarch64-linux-gnu-randlib && make -j2 +RUN cd /srs/trunk && ./configure --jobs=2 --cross-build --cross-prefix=aarch64-linux-gnu- && make -j2 ######################################################## -FROM ossrs/srs:ubuntu20 AS ubuntu20-cross-armv7 +FROM ossrs/srs:ubuntu20-cross-arm AS ubuntu20-cross-armv7 COPY . /srs -RUN cd /srs/trunk && ./configure --jobs=2 --srt=off --cross-build --cc=arm-linux-gnueabihf-gcc --cxx=arm-linux-gnueabihf-g++ \ - --ar=arm-linux-gnueabihf-ar --ld=arm-linux-gnueabihf-ld --randlib=arm-linux-gnueabihf-randlib && make -j2 +RUN cd /srs/trunk && ./configure --jobs=2 --cross-build --cross-prefix=arm-linux-gnueabihf- && make -j2 -FROM ossrs/srs:ubuntu20 AS ubuntu20-cross-aarch64 +FROM ossrs/srs:ubuntu20-cross-aarch64 AS ubuntu20-cross-aarch64 COPY . /srs -RUN cd /srs/trunk && ./configure --jobs=2 --srt=off --cross-build --cc=aarch64-linux-gnu-gcc --cxx=aarch64-linux-gnu-g++ \ - --ar=aarch64-linux-gnu-ar --ld=aarch64-linux-gnu-ld --randlib=aarch64-linux-gnu-randlib && make -j2 +RUN cd /srs/trunk && ./configure --jobs=2 --cross-build --cross-prefix=aarch64-linux-gnu- && make -j2 diff --git a/trunk/Dockerfile.cov b/trunk/Dockerfile.cov index d86c7cfab4..6a745d29f9 100644 --- a/trunk/Dockerfile.cov +++ b/trunk/Dockerfile.cov @@ -1,4 +1,4 @@ -FROM ossrs/srs:dev-gcc7 +FROM ossrs/srs:dev-gcc7-cache # Install depends tools. RUN yum install -y gcc make gcc-c++ patch unzip perl git diff --git a/trunk/Dockerfile.test b/trunk/Dockerfile.test index 4559b7b619..cd12950f78 100644 --- a/trunk/Dockerfile.test +++ b/trunk/Dockerfile.test @@ -1,4 +1,4 @@ -FROM ossrs/srs:dev-gcc7 +FROM ossrs/srs:dev-gcc7-cache # Install depends tools. RUN yum install -y gcc make gcc-c++ patch unzip perl git diff --git a/trunk/auto/depends.sh b/trunk/auto/depends.sh index 65fab11127..fd18e7d3a8 100755 --- a/trunk/auto/depends.sh +++ b/trunk/auto/depends.sh @@ -369,6 +369,22 @@ if [[ $OS_IS_UBUNTU = NO && $OS_IS_CENTOS = NO && $OS_IS_OSX = NO && $SRS_CROSS_ exit 1 fi +##################################################################################### +# Try to load cache if exists /usr/local/srs-cache +##################################################################################### +# Use srs-cache from base image. +if [[ -d /usr/local/srs-cache/srs/trunk/objs && $(pwd) != "/usr/local/srs-cache/srs/trunk" ]]; then + SOURCE_DIR=$(ls -d /usr/local/srs-cache/srs/trunk/objs/Platform-* 2>/dev/null|head -n 1) + if [[ -d $SOURCE_DIR ]]; then + TARGET_DIR=${SRS_OBJS}/${SRS_PLATFORM} && + echo "Build from cache, source=$SOURCE_DIR, target=$TARGET_DIR" && + rm -rf $TARGET_DIR && mkdir -p ${SRS_OBJS} && cp -R $SOURCE_DIR $TARGET_DIR && + du -sh /usr/local/srs-cache/srs/trunk/objs/Platform-* && + du -sh objs/Platform-* && + ls -lh objs + fi +fi + ##################################################################################### # state-threads #####################################################################################