Skip to content

Commit

Permalink
CI: try to make caching more reliable (#25259)
Browse files Browse the repository at this point in the history 
- `~/.yarn/cache` is now restored from an hierarchical cache key, if no precise match is found, we fallback to less precise ones. 
- The yarn install in `fixtures/dom` is also cached. Notably, is utilizes the cache from root, but stores into its more precise key.
- Steps running in root no longer have a `yarn install` and rely on the cache from the setup step.
- Retry `yarn install` once on failure.
  • Loading branch information
@kassens
kassens authored Sep 14, 2022
1 parent afe664d commit c327b91
Showing 1 changed file with 90 additions and 64 deletions.
154 changes: 90 additions & 64 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,93 @@ aliases:
- &restore_yarn_cache
restore_cache:
name: Restore yarn cache
key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn
keys:
- v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
- v1-yarn_cache-{{ arch }}-
- v1-yarn_cache-

- &yarn_install
run:
name: Install dependencies
command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn

- &yarn_install_retry
run:
name: Install dependencies (retry)
when: on_fail
command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn

- &save_yarn_cache
save_cache:
name: Save yarn cache
key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
paths:
- ~/.cache/yarn

- &restore_yarn_cache_fixtures_dom
restore_cache:
name: Restore yarn cache for fixtures/dom
keys:
- v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom
- v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
- v1-yarn_cache-{{ arch }}-
- v1-yarn_cache-

- &yarn_install_fixtures_dom
run:
name: Install dependencies in fixtures/dom
working_directory: fixtures/dom
command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn

- &prepare_node_modules_cache_key
- &yarn_install_fixtures_dom_retry
run:
name: Preparing node_modules cache key
command: yarn workspaces info | head -n -1 > workspace_info.txt
name: Install dependencies in fixtures/dom (retry)
when: on_fail
working_directory: fixtures/dom
command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn

- &save_yarn_cache_fixtures_dom
save_cache:
name: Save yarn cache for fixtures/dom
key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom
paths:
- ~/.cache/yarn

- &save_node_modules
save_cache:
name: Save node_modules cache
# Cache only for the current revision to prevent cache injections from
# malicious PRs.
key: v1-node_modules-{{ arch }}-{{ .Revision }}
paths:
- node_modules
- packages/eslint-plugin-react-hooks/node_modules
- packages/react-art/node_modules
- packages/react-client/node_modules
- packages/react-devtools-core/node_modules
- packages/react-devtools-extensions/node_modules
- packages/react-devtools-inline/node_modules
- packages/react-devtools-shared/node_modules
- packages/react-devtools-shell/node_modules
- packages/react-devtools-timeline/node_modules
- packages/react-devtools/node_modules
- packages/react-dom/node_modules
- packages/react-interactions/node_modules
- packages/react-native-renderer/node_modules
- packages/react-reconciler/node_modules
- packages/react-server-dom-relay/node_modules
- packages/react-server-dom-webpack/node_modules
- packages/react-server-native-relay/node_modules
- packages/react-server/node_modules
- packages/react-test-renderer/node_modules
- packages/react/node_modules
- packages/scheduler/node_modules

- &restore_node_modules
restore_cache:
name: Restore node_modules cache
keys:
- v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules
- v1-node_modules-{{ arch }}-{{ .Revision }}

- &TEST_PARALLELISM 20

Expand Down Expand Up @@ -49,35 +124,18 @@ jobs:
name: Nodejs Version
command: node --version
- *restore_yarn_cache
- run:
name: Install Packages
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
- run: yarn workspaces info | head -n -1 > workspace_info.txt
- save_cache:
# Store the yarn cache globally for all lock files with this same
# checksum. This will speed up the setup job for all PRs where the
# lockfile is the same.
name: Save yarn cache for future installs
key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn
paths:
- ~/.cache/yarn
- save_cache:
# Store node_modules for all jobs in this workflow so that they don't
# need to each run a yarn install for each job. This will speed up
# all jobs run on this branch with the same lockfile.
name: Save node_modules cache
# This cache key is per branch, a yarn install in setup is required.
key: v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules
paths:
- node_modules
- *restore_node_modules
- *yarn_install
- *yarn_install_retry
- *save_yarn_cache
- *save_node_modules

yarn_lint:
docker: *docker
environment: *environment

steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run: node ./scripts/prettier/index
- run: node ./scripts/tasks/eslint
Expand All @@ -92,7 +150,6 @@ jobs:

steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run: node ./scripts/tasks/flow-ci

Expand All @@ -102,7 +159,6 @@ jobs:

steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
command: |
Expand All @@ -119,7 +175,6 @@ jobs:
parallelism: 40
steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run: yarn build-combined
- persist_to_workspace:
Expand All @@ -135,7 +190,6 @@ jobs:
type: string
steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Download artifacts for revision
Expand All @@ -153,7 +207,6 @@ jobs:
environment: *environment
steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Download artifacts for base revision
Expand Down Expand Up @@ -182,7 +235,6 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA
# Compress build directory into a single tarball for easy download
Expand All @@ -202,7 +254,6 @@ jobs:
- attach_workspace:
at: .
- run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
command: node ./scripts/tasks/danger
Expand All @@ -214,11 +265,7 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Install Packages
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
- run:
environment:
RELEASE_CHANNEL: experimental
Expand All @@ -233,11 +280,7 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Install Packages
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
- run:
name: Playwright install deps
command: |
Expand All @@ -259,11 +302,7 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Install nested packages from Yarn cache
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
- run: ./scripts/circleci/download_devtools_regression_build.js << parameters.version >> --replaceBuild
- run: node ./scripts/jest/jest-cli.js --build --project devtools --release-channel=experimental --reactVersion << parameters.version >> --ci

Expand All @@ -278,11 +317,7 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Install nested packages from Yarn cache
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
- run:
name: Playwright install deps
command: |
Expand All @@ -306,7 +341,6 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- run: yarn lint-build

Expand All @@ -317,7 +351,6 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- run: yarn check-release-dependencies

Expand All @@ -328,7 +361,6 @@ jobs:
steps:
- checkout
- attach_workspace: *attach_workspace
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Search build artifacts for unminified errors
Expand All @@ -345,7 +377,6 @@ jobs:
type: string
steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run: yarn test <<parameters.args>> --ci

Expand All @@ -360,11 +391,7 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Install nested packages from Yarn cache
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
- run: yarn test --build <<parameters.args>> --ci

RELEASE_CHANNEL_stable_yarn_test_dom_fixtures:
Expand All @@ -374,15 +401,17 @@ jobs:
- checkout
- attach_workspace:
at: .
- *prepare_node_modules_cache_key
- *restore_node_modules
- *restore_yarn_cache_fixtures_dom
- *yarn_install_fixtures_dom
- *yarn_install_fixtures_dom_retry
- *save_yarn_cache_fixtures_dom
- run:
name: Run DOM fixture tests
environment:
RELEASE_CHANNEL: stable
working_directory: fixtures/dom
command: |
cd fixtures/dom
yarn --frozen-lockfile
yarn prestart
yarn test --maxWorkers=2
Expand All @@ -391,7 +420,6 @@ jobs:
environment: *environment
steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Run fuzz tests
Expand All @@ -411,7 +439,6 @@ jobs:
environment: *environment
steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Run publish script
Expand All @@ -430,7 +457,6 @@ jobs:
environment: *environment
steps:
- checkout
- *prepare_node_modules_cache_key
- *restore_node_modules
- run:
name: Fetch revisions that contain an intentional fork
Expand Down

0 comments on commit c327b91

Please sign in to comment.