ci: chocolatey caching

Optimised Windows CI/CD setup by internalising and caching chocolatey packages Fixes #397
MatrixAI · Jul 12, 2022 · e476835 · e476835
1 parent d95e516
commit e476835
Show file tree

Hide file tree

Showing 6 changed files with 178 additions and 135 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -34,6 +34,8 @@ cache:
     - ./tmp/ts-node-cache/
     # Homebrew cache is only used by the macos runner
     - ./tmp/Homebrew
+    # Chocolatey cache is only used by the windows runner
+    - ./tmp/chocolatey/
     # `jest` cache is configured in jest.config.js
     - ./tmp/jest/
 
@@ -63,56 +65,56 @@ check:lint:
     - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
       when: manual
 
-# check:nix-dry:
-#   stage: check
-#   needs: []
-#   script:
-#     - nix-build -v -v --dry-run ./release.nix
-#   rules:
-#     # Runs on feature and staging commits and ignores version commits
-#     - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-#     # Runs on tag pipeline where the tag is a prerelease or release version
-#     - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-#     # Manually run on commits other than master and ignore version commits
-#     - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-#       when: manual
+check:nix-dry:
+  stage: check
+  needs: []
+  script:
+    - nix-build -v -v --dry-run ./release.nix
+  rules:
+    # Runs on feature and staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Manually run on commits other than master and ignore version commits
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+      when: manual
 
 check:test-generate:
   stage: check
   needs: []
   script:
     - >
       nix-shell --run '
-      ./scripts/check:test-generate.sh > ./tmp/check:test.yml
+      ./scripts/check-test-generate.sh > ./tmp/check-test.yml
       '
   artifacts:
     when: always
     paths:
-      - ./tmp/check:test.yml
+      - ./tmp/check-test.yml
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
     # Manually run on commits other than master and staging and ignore version commits
     - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
       when: manual
 
-# check:test:
-#   stage: check
-#   needs:
-#     - check:test-generate
-#   inherit:
-#     variables: false
-#   trigger:
-#     include:
-#       - artifact: tmp/check:test.yml
-#         job: check:test-generate
-#     strategy: depend
-#   rules:
-#     # Runs on staging commits and ignores version commits
-#     - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-#     # Manually run on commits other than master and staging and ignore version commits
-#     - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-#       when: manual
+check:test:
+  stage: check
+  needs:
+    - check:test-generate
+  inherit:
+    variables: false
+  trigger:
+    include:
+      - artifact: tmp/check-test.yml
+        job: check:test-generate
+    strategy: depend
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Manually run on commits other than master and staging and ignore version commits
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+      when: manual
 
 build:merge:
   stage: build
@@ -148,18 +150,17 @@ build:platforms-generate:
   script:
     - >
         nix-shell --run '
-        ./scripts/build:platforms-generate.sh > ./tmp/build:platforms.yml
+        ./scripts/build-platforms-generate.sh > ./tmp/build-platforms.yml
         '
   artifacts:
     when: always
     paths:
-      - ./tmp/build:platforms.yml
+      - ./tmp/build-platforms.yml
   rules:
     # Runs on staging commits and ignores version commits
-    - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-    # Manually run on commits other than master and staging and ignore version commits
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-      when: manual
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 
 build:platforms:
   stage: build
@@ -169,15 +170,14 @@ build:platforms:
     variables: false
   trigger:
     include:
-      - artifact: tmp/build:platforms.yml
+      - artifact: tmp/build-platforms.yml
         job: build:platforms-generate
     strategy: depend
   rules:
     # Runs on staging commits and ignores version commits
-    - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-    # Manually run on commits other than master and staging and ignore version commits
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-      when: manual
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 
 build:dist:
   stage: build

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -98,7 +98,6 @@
     "google-protobuf": "^3.14.0",
     "ip-num": "^1.3.3-0",
     "isomorphic-git": "^1.8.1",
-    "jest-junit": "^13.2.0",
     "jose": "^4.3.6",
     "lexicographic-integer": "^1.1.0",
     "multiformats": "^9.4.8",

diff --git a/scripts/build:platforms-generate.sh → scripts/build-platforms-generate.sh b/scripts/build:platforms-generate.sh → scripts/build-platforms-generate.sh
@@ -36,6 +36,8 @@ cache:
     - ./tmp/ts-node-cache/
     # Homebrew cache is only used by the macos runner
     - ./tmp/Homebrew
+    # Chocolatey cache is only used by the windows runner
+    - ./tmp/chocolatey/
     # `jest` cache is configured in jest.config.js
     - ./tmp/jest/
 
@@ -47,61 +49,61 @@ EOF
 
 printf "\n"
 
-# # Each test directory has its own job
-# for test_dir in tests/**/*/; do
-#   test_files=("$test_dir"*.test.ts)
-#   if [ ${#test_files[@]} -eq 0 ]; then
-#     continue
-#   fi
-#   # Remove trailing slash
-#   test_dir="${test_dir%\/}"
-#   # Remove `tests/` prefix
-#   test_dir="${test_dir#*/}"
-#   cat << EOF
-# build:linux $test_dir:
-#   stage: build
-#   needs: []
-#   script:
-#     - >
-#         nix-shell --run '
-#         npm test -- --ci --coverage ${test_files[@]};
-#         '
-#   artifacts:
-#     when: always
-#     reports:
-#       junit:
-#         - ./tmp/junit/junit.xml
-#       coverage_report:
-#         coverage_format: cobertura
-#         path: ./tmp/coverage/cobertura-coverage.xml
-#   coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
-# EOF
-#   printf "\n"
-# done
-
-# # All top-level test files are accumulated into 1 job
-# test_files=(tests/*.test.ts)
-# cat << EOF
-# build:linux index:
-#   stage: build
-#   needs: []
-#   script:
-#     - >
-#         nix-shell --run '
-#         npm test -- --ci --coverage ${test_files[@]};
-#         '
-#   artifacts:
-#     when: always
-#     reports:
-#       junit:
-#         - ./tmp/junit/junit.xml
-#       coverage_report:
-#         coverage_format: cobertura
-#         path: ./tmp/coverage/cobertura-coverage.xml
-#   coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
-# EOF
-
-# printf "\n"
+# Each test directory has its own job
+for test_dir in tests/**/*/; do
+  test_files=("$test_dir"*.test.ts)
+  if [ ${#test_files[@]} -eq 0 ]; then
+    continue
+  fi
+  # Remove trailing slash
+  test_dir="${test_dir%\/}"
+  # Remove `tests/` prefix
+  test_dir="${test_dir#*/}"
+  cat << EOF
+build:linux $test_dir:
+  stage: build
+  needs: []
+  script:
+    - >
+        nix-shell --run '
+        npm test -- --ci --coverage --runInBand ${test_files[@]};
+        '
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
+EOF
+  printf "\n"
+done
+
+# All top-level test files are accumulated into 1 job
+test_files=(tests/*.test.ts)
+cat << EOF
+build:linux index:
+  stage: build
+  needs: []
+  script:
+    - >
+        nix-shell --run '
+        npm test -- --ci --coverage --runInBand ${test_files[@]};
+        '
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
+EOF
+
+printf "\n"
 
 # Using shards to optimise tests
 # In the future we can incorporate test durations rather than using
@@ -110,38 +112,38 @@ printf "\n"
 # Number of parallel shards to split the test suite into
 CI_PARALLEL=2
 
-# cat << "EOF"
-# build:windows:
-#   stage: build
-#   needs: []
-# EOF
-# cat << EOF
-#   parallel: $CI_PARALLEL
-# EOF
-# cat << "EOF"
-#   tags:
-#     - windows
-#   before_script:
-#     - mkdir -Force "$CI_PROJECT_DIR/tmp"
-#     - choco install nodejs --version=16.14.2 -y
-#     - refreshenv
-#   script:
-#     - npm config set msvs_version 2019
-#     - npm install --ignore-scripts
-#     - $env:Path = "$(npm bin);" + $env:Path
-#     - npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
-#   artifacts:
-#     when: always
-#     reports:
-#       junit:
-#         - ./tmp/junit/junit.xml
-#       coverage_report:
-#         coverage_format: cobertura
-#         path: ./tmp/coverage/cobertura-coverage.xml
-#   coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
-# EOF
-
-# printf "\n"
+cat << "EOF"
+build:windows:
+  stage: build
+  needs: []
+EOF
+cat << EOF
+  parallel: $CI_PARALLEL
+EOF
+cat << "EOF"
+  tags:
+    - windows
+  before_script:
+    - mkdir -Force "$CI_PROJECT_DIR/tmp"
+    - .\scripts\choco-install.ps1
+    - refreshenv
+  script:
+    - npm config set msvs_version 2019
+    - npm install --ignore-scripts
+    - $env:Path = "$(npm bin);" + $env:Path
+    - npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL --maxWorkers=50%
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
+EOF
+
+printf "\n"
 
 cat << "EOF"
 build:macos:
@@ -169,12 +171,16 @@ cat << "EOF"
   script:
     - npm install --ignore-scripts
     - export PATH="$(npm bin):$PATH"
-    - npm test -- --ci --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL --maxWorkers=50%
+    - npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL --maxWorkers=50%
   artifacts:
     when: always
     reports:
       junit:
         - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
 EOF
 
 printf "\n"
diff --git a/scripts/check:test-generate.sh → scripts/check-test-generate.sh b/scripts/check:test-generate.sh → scripts/check-test-generate.sh
@@ -28,6 +28,7 @@ variables:
 # Cached directories shared between jobs & pipelines per-branch per-runner
 cache:
   key: $CI_COMMIT_REF_SLUG
+  when: 'always'
   paths:
     - ./tmp/npm/
     - ./tmp/ts-node-cache/
@@ -59,7 +60,7 @@ check:test $test_dir:
   script:
     - >
         nix-shell --run '
-        npm test -- --ci --coverage ${test_files[@]};
+        npm test -- --ci --coverage --runInBand ${test_files[@]};
         '
   artifacts:
     when: always
@@ -83,7 +84,7 @@ check:test index:
   script:
     - >
         nix-shell --run '
-        npm test -- --ci --coverage ${test_files[@]};
+        npm test -- --ci --coverage --runInBand ${test_files[@]};
         '
   artifacts:
     when: always