From 2539cf5ed432efb6025d10719261f29dce9358fb Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Wed, 14 Aug 2019 13:11:44 +0300 Subject: [PATCH 1/4] Switching cache from local to Radis --- .gitlab-ci.yml | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f03d79625cd..2b641e4ca83 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,15 +4,15 @@ stages: - publish - optional -image: parity/parity-ci-linux:latest +image: registry.parity.io/pub/ci/parity-ci-linux:latest variables: GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: recursive CI_SERVER_NAME: "GitLab CI" CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_JOB_NAME}" - SCCACHE_DIR: "/ci-cache/${CI_PROJECT_NAME}/sccache" CARGO_TARGET: x86_64-unknown-linux-gnu + .no_git: &no_git # disable git strategy variables: GIT_STRATEGY: none @@ -41,22 +41,7 @@ variables: before_script: - rustup show - cargo --version - - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_debug.log - RUST_LOG=sccache=debug - sccache --start-server - - sccache -s - after_script: - # sccache debug info - - if test -e sccache_debug.log; - then - echo "_____All crate-types:_____"; - grep 'parse_arguments.*--crate-type' sccache_debug.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c; - echo "_____Non-cacheable reasons:_____"; - grep CannotCache sccache_debug.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c; - else - echo "_____No logs from sccache_____"; - exit 0; - fi + - SCCACHE_REDIS=redis://:${REDIS}@172.17.0.1/0 sccache --start-server tags: - linux-docker @@ -66,7 +51,6 @@ variables: <<: *collect_artifacts script: - scripts/gitlab/build-linux.sh - - sccache -s after_script: - mkdir -p tools - cp -r scripts/docker/hub/* ./tools @@ -84,21 +68,18 @@ cargo-check 0 3: <<: *docker-cache-status script: - time cargo check --target $CARGO_TARGET --locked --no-default-features --verbose --color=always - - sccache -s cargo-check 1 3: stage: test <<: *docker-cache-status script: - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features --verbose --color=always - - sccache -s cargo-check 2 3: stage: test <<: *docker-cache-status script: - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose --color=always - - sccache -s cargo-audit: stage: test @@ -112,21 +93,18 @@ validate-chainspecs: <<: *docker-cache-status script: - ./scripts/gitlab/validate-chainspecs.sh - - sccache -s test-cpp: stage: build <<: *docker-cache-status script: - ./scripts/gitlab/test-cpp.sh - - sccache -s test-linux: stage: build <<: *docker-cache-status script: - ./scripts/gitlab/test-linux.sh stable - - sccache -s test-linux-beta: stage: build @@ -134,7 +112,6 @@ test-linux-beta: <<: *docker-cache-status script: - ./scripts/gitlab/test-linux.sh beta - - sccache -s test-linux-nightly: stage: build @@ -142,7 +119,6 @@ test-linux-nightly: <<: *docker-cache-status script: - ./scripts/gitlab/test-linux.sh nightly - - sccache -s allow_failure: true build-android: From 984681fef99fd7e1e75d155b0f8783f38962223f Mon Sep 17 00:00:00 2001 From: Denis P Date: Wed, 14 Aug 2019 15:27:07 +0200 Subject: [PATCH 2/4] sccache -s won't work. All the images are taken from own registry --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b641e4ca83..1ac00ddd133 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,11 +37,11 @@ variables: .docker-cache-status: &docker-cache-status variables: CARGO_HOME: "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}" + SCCACHE_REDIS: "redis://:${REDIS}@172.17.0.1/0" dependencies: [] before_script: - rustup show - cargo --version - - SCCACHE_REDIS=redis://:${REDIS}@172.17.0.1/0 sccache --start-server tags: - linux-docker @@ -123,7 +123,7 @@ test-linux-nightly: build-android: <<: *build-on-linux - image: parity/parity-ci-android:stretch + image: registry.parity.io/pub/ci/parity-ci-android:stretch variables: CARGO_TARGET: armv7-linux-androideabi @@ -134,21 +134,21 @@ build-linux: build-linux-i386: <<: *build-on-linux only: *releaseable_branches - image: parity/parity-ci-i386:latest + image: registry.parity.io/pub/ci/parity-ci-i386:latest variables: CARGO_TARGET: i686-unknown-linux-gnu build-linux-arm64: <<: *build-on-linux only: *releaseable_branches - image: parity/parity-ci-arm64:latest + image: registry.parity.io/pub/ci/parity-ci-arm64:latest variables: CARGO_TARGET: aarch64-unknown-linux-gnu build-linux-armhf: <<: *build-on-linux only: *releaseable_branches - image: parity/parity-ci-armhf:latest + image: registry.parity.io/pub/ci/parity-ci-armhf:latest variables: CARGO_TARGET: armv7-unknown-linux-gnueabihf @@ -260,7 +260,7 @@ publish-onchain: - linux-docker publish-awss3-release: - image: parity/awscli:latest + image: registry.parity.io/pub/ci/awscli:latest stage: publish only: *releaseable_branches <<: *no_git @@ -287,7 +287,7 @@ publish-awss3-release: publish-docs: stage: publish - image: parity/parity-ci-docs:latest + image: registry.parity.io/pub/ci/parity-ci-docs:latest only: - tags except: From 408765b119e60872fb121c2e0b637fb32cd10ecb Mon Sep 17 00:00:00 2001 From: Denis P Date: Thu, 15 Aug 2019 11:27:18 +0200 Subject: [PATCH 3/4] new images addresses --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ac00ddd133..8b328088ab0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - publish - optional -image: registry.parity.io/pub/ci/parity-ci-linux:latest +image: $CI_REGISTRY/parity/infrastructure/scripts/parity-ci-linux:latest variables: GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: recursive @@ -123,7 +123,7 @@ test-linux-nightly: build-android: <<: *build-on-linux - image: registry.parity.io/pub/ci/parity-ci-android:stretch + image: $CI_REGISTRY/parity/infrastructure/scripts/parity-ci-android:stretch variables: CARGO_TARGET: armv7-linux-androideabi @@ -134,21 +134,21 @@ build-linux: build-linux-i386: <<: *build-on-linux only: *releaseable_branches - image: registry.parity.io/pub/ci/parity-ci-i386:latest + image: $CI_REGISTRY/parity/infrastructure/scripts/parity-ci-i386:latest variables: CARGO_TARGET: i686-unknown-linux-gnu build-linux-arm64: <<: *build-on-linux only: *releaseable_branches - image: registry.parity.io/pub/ci/parity-ci-arm64:latest + image: $CI_REGISTRY/parity/infrastructure/scripts/parity-ci-arm64:latest variables: CARGO_TARGET: aarch64-unknown-linux-gnu build-linux-armhf: <<: *build-on-linux only: *releaseable_branches - image: registry.parity.io/pub/ci/parity-ci-armhf:latest + image: $CI_REGISTRY/parity/infrastructure/scripts/parity-ci-armhf:latest variables: CARGO_TARGET: armv7-unknown-linux-gnueabihf @@ -260,7 +260,7 @@ publish-onchain: - linux-docker publish-awss3-release: - image: registry.parity.io/pub/ci/awscli:latest + image: $CI_REGISTRY/parity/infrastructure/scripts/awscli:latest stage: publish only: *releaseable_branches <<: *no_git @@ -287,7 +287,7 @@ publish-awss3-release: publish-docs: stage: publish - image: registry.parity.io/pub/ci/parity-ci-docs:latest + image: $CI_REGISTRY/parity/infrastructure/scripts/parity-ci-docs:latest only: - tags except: From ca20ad9d8ac48e6ab418721fe8feac7cfed62e6e Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Wed, 2 Oct 2019 22:16:12 +0300 Subject: [PATCH 4/4] statisticss after stop sccache server sccache CC&CXX --- .gitlab-ci.yml | 6 ++++-- scripts/gitlab/build-linux.sh | 4 ++++ scripts/gitlab/test-cpp.sh | 4 ++++ scripts/gitlab/test-linux.sh | 6 +++++- scripts/gitlab/validate-chainspecs.sh | 3 ++- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b328088ab0..91390d86d63 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,6 @@ variables: .docker-cache-status: &docker-cache-status variables: CARGO_HOME: "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}" - SCCACHE_REDIS: "redis://:${REDIS}@172.17.0.1/0" dependencies: [] before_script: - rustup show @@ -68,18 +67,21 @@ cargo-check 0 3: <<: *docker-cache-status script: - time cargo check --target $CARGO_TARGET --locked --no-default-features --verbose --color=always + - sccache --stop-server cargo-check 1 3: stage: test <<: *docker-cache-status script: - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features --verbose --color=always + - sccache --stop-server cargo-check 2 3: stage: test <<: *docker-cache-status script: - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose --color=always + - sccache --stop-server cargo-audit: stage: test @@ -199,7 +201,7 @@ publish-docker: DOCKER_DRIVER: overlay2 GIT_STRATEGY: none # DOCKERFILE: tools/Dockerfile - # CONTAINER_IMAGE: parity/parity + # CONTAINER_IMAGE: parity/parity script: # we stopped pushing nightlies to dockerhub, will push to own registry prb. - ./tools/publish-docker.sh diff --git a/scripts/gitlab/build-linux.sh b/scripts/gitlab/build-linux.sh index 75a8682bb54..e0f2876c000 100755 --- a/scripts/gitlab/build-linux.sh +++ b/scripts/gitlab/build-linux.sh @@ -3,6 +3,8 @@ set -e # fail on any error set -u # treat unset variables as error +export CC="sccache "$CC +export CXX="sccache "$CXX echo "__________Show ENVIROMENT__________" echo "CI_SERVER_NAME: " $CI_SERVER_NAME echo "CARGO_HOME: " $CARGO_HOME @@ -55,3 +57,5 @@ do echo "> ${binary} cannot be hashed with cross-compiled binary (keccak256)" fi done +#show sccache statistics +sccache --stop-server diff --git a/scripts/gitlab/test-cpp.sh b/scripts/gitlab/test-cpp.sh index 1cbd58a30cf..b61db1bb2f4 100755 --- a/scripts/gitlab/test-cpp.sh +++ b/scripts/gitlab/test-cpp.sh @@ -4,6 +4,8 @@ set -e # fail on any error set -u # treat unset variables as error #use nproc `linux only THREADS=$(nproc) +export CC="sccache gcc" +export CXX="sccache g++" echo "________Running the C++ example________" DIR=parity-clib/examples/cpp/build @@ -15,3 +17,5 @@ make VERBOSE=1 -j $THREADS # that to happen on CI cd - rm -rf $DIR +#show sccache statistics +sccache --stop-server diff --git a/scripts/gitlab/test-linux.sh b/scripts/gitlab/test-linux.sh index 2ad2ab9be28..396f445979a 100755 --- a/scripts/gitlab/test-linux.sh +++ b/scripts/gitlab/test-linux.sh @@ -4,7 +4,8 @@ echo "________Running test-linux.sh________" set -e # fail on any error set -u # treat unset variables as error - +export CC="sccache gcc" +export CXX="sccache g++" FEATURES="json-tests,ci-skip-tests" OPTIONS="--release" #use nproc `linux only @@ -16,3 +17,6 @@ rustup show echo "________Running Parity Full Test Suite________" # Why are we using RUSTFLAGS? See https://github.com/paritytech/parity-ethereum/pull/10719 CARGO_INCREMENTAL=0 RUSTFLAGS="-C opt-level=3 -C overflow-checks=on -C debuginfo=2 -Ctarget-feature=+aes,+sse2,+ssse3" time cargo test $OPTIONS --features "$FEATURES" --locked --all --target $CARGO_TARGET --verbose --color=never -- --test-threads $THREADS + +#show sccache statistics +sccache --stop-server diff --git a/scripts/gitlab/validate-chainspecs.sh b/scripts/gitlab/validate-chainspecs.sh index 58391e13126..cf5c126f7b3 100755 --- a/scripts/gitlab/validate-chainspecs.sh +++ b/scripts/gitlab/validate-chainspecs.sh @@ -15,5 +15,6 @@ done for spec in ethcore/res/ethereum/*.json; do if ! ./target/release/chainspec "$spec"; then ERR=1; fi done - +#show sccache statistics +sccache --stop-server exit $ERR