From 64bc070bc40bfc20b70920c84965fc3b2f537fcf Mon Sep 17 00:00:00 2001 From: Jose Luis Pereira Date: Wed, 15 Nov 2023 15:21:07 +0000 Subject: [PATCH] Implement copy and copy-immutable strategies --- .github/workflows/push.yaml | 127 +++++++++++++++++++++++++++++++++--- README.md | 16 ++++- action.yml | 4 ++ dist/main.js | 30 +++++++-- dist/post.js | 46 ++++++++++--- src/lib/getVars.ts | 12 +++- src/main.ts | 22 +++++-- src/post.ts | 27 ++++++-- 8 files changed, 247 insertions(+), 37 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 194ef27..426ea38 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -1,9 +1,6 @@ name: Push on: push -env: - SHARED_CACHE_RUN_KEY: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }} - jobs: tests: runs-on: ubuntu-latest @@ -12,7 +9,41 @@ jobs: - run: npm install - run: npm run all - first-run: + test-move-strategy-run1: + runs-on: + group: local-action-cache + steps: + - uses: actions/checkout@v3 + - name: 'Run first time without cache' + id: 'first-run' + uses: ./ + with: + path: './demo-output.txt' + key: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }}-move + - name: Assert output + if: steps.first-run.outputs.cache-hit == 'true' + run: echo "Should not have done cache hit" && exit 1 + - name: Write cache file + run: echo "demo-results" > ./demo-output.txt + + test-move-strategy-run2: + runs-on: + group: local-action-cache + needs: [test-move-strategy-run1] + steps: + - uses: actions/checkout@v3 + - uses: ./ + id: 'second-run' + with: + path: './demo-output.txt' + key: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }}-move + - name: Assert output + if: steps.second-run.outputs.cache-hit != 'true' + run: echo "Should have hit cache" && exit 1 + - name: Check contents + run: test $(cat ./demo-output.txt) != "demo-results" && echo "Wrong cached contents $(cat ./demo-output.txt)" && exit 1 + + test-copy-strategy-run1: runs-on: group: local-action-cache steps: @@ -22,26 +53,104 @@ jobs: uses: ./ with: path: './demo-output.txt' - key: ${{ env.SHARED_CACHE_RUN_KEY }} + key: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }}-copy + strategy: copy - name: Assert output if: steps.first-run.outputs.cache-hit == 'true' run: echo "Should not have done cache hit" && exit 1 - name: Write cache file run: echo "demo-results" > ./demo-output.txt - second-run: + test-copy-strategy-run2: runs-on: group: local-action-cache - needs: [first-run] + needs: [test-copy-strategy-run1] steps: - uses: actions/checkout@v3 - uses: ./ id: 'second-run' with: path: './demo-output.txt' - key: ${{ env.SHARED_CACHE_RUN_KEY }} + key: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }}-copy + strategy: copy-immutable - name: Assert output if: steps.second-run.outputs.cache-hit != 'true' run: echo "Should have hit cache" && exit 1 - name: Check contents - run: cat ./demo-output.txt + run: test $(cat ./demo-output.txt) != "demo-results" && echo "Wrong cached contents $(cat ./demo-output.txt)" && exit 1 + - name: Change contents + run: echo "changed-results" > ./demo-output.txt + + test-copy-strategy-run3: + runs-on: + group: local-action-cache + needs: [test-copy-strategy-run2] + steps: + - uses: actions/checkout@v3 + - uses: ./ + id: 'third-run' + with: + path: './demo-output.txt' + key: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }}-copy + strategy: copy + - name: Assert output + if: steps.third-run.outputs.cache-hit != 'true' + run: echo "Should have hit cache" && exit 1 + - name: Check contents + run: test $(cat ./demo-output.txt) != "changed-results" && echo "Wrong cached contents $(cat ./demo-output.txt)" && exit 1 + + test-copy-strategy-immutable-run1: + runs-on: + group: local-action-cache + steps: + - uses: actions/checkout@v3 + - name: 'Run first time without cache' + id: 'first-run' + uses: ./ + with: + path: './demo-output.txt' + key: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }}-copy-immutable + strategy: copy-immutable + - name: Assert output + if: steps.first-run.outputs.cache-hit == 'true' + run: echo "Should not have done cache hit" && exit 1 + - name: Write cache file + run: echo "demo-results" > ./demo-output.txt + + test-copy-strategy-immutable-run2: + runs-on: + group: local-action-cache + needs: [test-copy-strategy-immutable-run1] + steps: + - uses: actions/checkout@v3 + - uses: ./ + id: 'second-run' + with: + path: './demo-output.txt' + key: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }}-copy-immutable + strategy: copy-immutable + - name: Assert output + if: steps.second-run.outputs.cache-hit != 'true' + run: echo "Should have hit cache" && exit 1 + - name: Check contents + run: test $(cat ./demo-output.txt) != "demo-results" && echo "Wrong cached contents $(cat ./demo-output.txt)" && exit 1 + - name: Change contents + run: echo "changed-results" > ./demo-output.txt + + test-copy-strategy-immutable-run3: + runs-on: + group: local-action-cache + needs: [test-copy-strategy-immutable-run2] + steps: + - uses: actions/checkout@v3 + - uses: ./ + id: 'third-run' + with: + path: './demo-output.txt' + key: CACHE_V1-${{ github.run_id }}-${{ github.run_attempt }}-copy-immutable + strategy: copy-immutable + - name: Assert output + if: steps.third-run.outputs.cache-hit != 'true' + run: echo "Should have hit cache" && exit 1 + - name: Check contents + run: test $(cat ./demo-output.txt) != "demo-results" && echo "Wrong cached contents $(cat ./demo-output.txt)" && exit 1 diff --git a/README.md b/README.md index 86c4fd5..b2add6e 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,16 @@ jobs: The `key` param is optional and you can use it to invalidate cache. +### Caching strategy + +Define how you want your cache to be used. `move` is the default strategy: + +| Name | Description | Usage | Observations | +| -------------- | ------------------------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------ | +| Move | Moves cache on job start, moves back on job end | `strategy: move` | Uses the filesystem move syscalls, cache saving and restoration is instant. Does not work across disks/remote shares. | +| Copy | Recursiverly copies cache on job start, refreshes cache on job end | `strategy: copy` | Works across disks/remote shares. | +| Copy Immutable | Recursiverly copies cache on job start, does not refresh cache on job end | `strategy: copy-immutable` | Works across disks/remote shares. | + # How it works The first time `action-local-cache` is used in a runner it will take the given path and create a folder structure inside the `$RUNNER_TOOL_CACHE` dir (usually `_work/_tool` inside the runner's workspace), prefixing the user/org name, repo name, and key. @@ -44,11 +54,11 @@ For example, when running the usage example above inside a workflow for the `Mas Since there's no `api/node_modules/` dir inside, the restore step is skipped and the `cache-hit` output variable will be set to `false`; because of that, the next step which install the dependencies will run. -After the job is successfully completed, `action-local-cache` will take the `api/node_modules` dir generated by the install dependencies step and move it to the previously created dir structure before the runner workspace is wiped. +After the job is successfully completed, `action-local-cache` will take the `api/node_modules` dir generated by the install dependencies step and move/copy it to the previously created dir structure before the runner workspace is wiped. -On a next run under the same runner instance, the cache will be moved back to the working directory and the `cache-output` variable will be set to `true`. +On a next run under the same runner instance, the cache will be moved/copied back to the working directory and the `cache-output` variable will be set to `true`. -After all steps complete the folder will be moved back to the cache dir and so on. Since it is using the filesystem move syscalls, cache saving and restoration is instant. +After all steps complete the folder will be moved/copied back to the cache dir and so on. # LICENSE diff --git a/action.yml b/action.yml index 77b1a6c..05c15ca 100644 --- a/action.yml +++ b/action.yml @@ -7,6 +7,10 @@ inputs: path: description: 'The file or folder to be cached' required: true + strategy: + description: 'Caching mechanism to be used' + required: false + default: 'move' outputs: cache-hit: description: 'A boolean value to indicate if cache was found and restored' diff --git a/dist/main.js b/dist/main.js index 17428e7..8c44dc4 100644 --- a/dist/main.js +++ b/dist/main.js @@ -2443,7 +2443,7 @@ var require_io = __commonJS({ var assert_1 = __require("assert"); var path2 = __importStar(__require("path")); var ioUtil = __importStar(require_io_util()); - function cp(source, dest, options = {}) { + function cp2(source, dest, options = {}) { return __awaiter(this, void 0, void 0, function* () { const { force, recursive, copySourceDirectory } = readCopyOptions(options); const destStat = (yield ioUtil.exists(dest)) ? yield ioUtil.stat(dest) : null; @@ -2469,7 +2469,7 @@ var require_io = __commonJS({ } }); } - exports.cp = cp; + exports.cp = cp2; function mv2(source, dest, options = {}) { return __awaiter(this, void 0, void 0, function* () { if (yield ioUtil.exists(dest)) { @@ -2869,6 +2869,12 @@ var import_io_util = __toESM(require_io_util()); var core = __toESM(require_core()); var { GITHUB_REPOSITORY, RUNNER_TOOL_CACHE } = process.env; var CWD = process.cwd(); +var Strategy = /* @__PURE__ */ ((Strategy2) => { + Strategy2["CopyImmutable"] = "copy-immutable"; + Strategy2["Copy"] = "copy"; + Strategy2["Move"] = "move"; + return Strategy2; +})(Strategy || {}); var getVars = () => { if (!RUNNER_TOOL_CACHE) { throw new TypeError("Expected RUNNER_TOOL_CACHE environment variable to be defined."); @@ -2878,10 +2884,13 @@ var getVars = () => { } const options = { key: core.getInput("key") || "no-key", - path: core.getInput("path") + path: core.getInput("path"), + strategy: core.getInput("strategy") }; if (!options.path) { throw new TypeError("path is required but was not provided."); + } else if (!Object.values(Strategy).includes(options.strategy)) { + throw new TypeError(`Unknown strategy ${options.strategy}`); } const cacheDir = path__default.default.join(RUNNER_TOOL_CACHE, GITHUB_REPOSITORY, options.key); const cachePath = path__default.default.join(cacheDir, options.path); @@ -2922,8 +2931,19 @@ async function main() { const { cachePath, targetDir, targetPath, options } = getVars(); if (await (0, import_io_util.exists)(cachePath)) { await (0, import_io.mkdirP)(targetDir); - await (0, import_io.mv)(cachePath, targetPath, { force: true }); - log_default.info(`Cache found and restored to ${options.path}`); + switch (options.strategy) { + case "copy-immutable" /* CopyImmutable */: + case "copy" /* Copy */: + await (0, import_io.cp)(cachePath, targetPath, { + copySourceDirectory: false, + recursive: true + }); + break; + case "move" /* Move */: + await (0, import_io.mv)(cachePath, targetPath, { force: true }); + break; + } + log_default.info(`Cache found and restored to ${options.path} with ${options.strategy} strategy`); (0, import_core.setOutput)("cache-hit", true); } else { log_default.info(`Skipping: cache not found for ${options.path}.`); diff --git a/dist/post.js b/dist/post.js index 4cea074..0c4f8e3 100644 --- a/dist/post.js +++ b/dist/post.js @@ -2270,7 +2270,7 @@ var require_io_util = __commonJS({ exports.IS_WINDOWS = process.platform === "win32"; exports.UV_FS_O_EXLOCK = 268435456; exports.READONLY = fs.constants.O_RDONLY; - function exists(fsPath) { + function exists2(fsPath) { return __awaiter(this, void 0, void 0, function* () { try { yield exports.stat(fsPath); @@ -2283,7 +2283,7 @@ var require_io_util = __commonJS({ return true; }); } - exports.exists = exists; + exports.exists = exists2; function isDirectory(fsPath, useStat = false) { return __awaiter(this, void 0, void 0, function* () { const stats = useStat ? yield exports.stat(fsPath) : yield exports.lstat(fsPath); @@ -2443,7 +2443,7 @@ var require_io = __commonJS({ var assert_1 = __require("assert"); var path2 = __importStar(__require("path")); var ioUtil = __importStar(require_io_util()); - function cp(source, dest, options = {}) { + function cp2(source, dest, options = {}) { return __awaiter(this, void 0, void 0, function* () { const { force, recursive, copySourceDirectory } = readCopyOptions(options); const destStat = (yield ioUtil.exists(dest)) ? yield ioUtil.stat(dest) : null; @@ -2469,7 +2469,7 @@ var require_io = __commonJS({ } }); } - exports.cp = cp; + exports.cp = cp2; function mv2(source, dest, options = {}) { return __awaiter(this, void 0, void 0, function* () { if (yield ioUtil.exists(dest)) { @@ -2480,7 +2480,7 @@ var require_io = __commonJS({ } if (destExists) { if (options.force == null || options.force) { - yield rmRF(dest); + yield rmRF2(dest); } else { throw new Error("Destination already exists"); } @@ -2491,7 +2491,7 @@ var require_io = __commonJS({ }); } exports.mv = mv2; - function rmRF(inputPath) { + function rmRF2(inputPath) { return __awaiter(this, void 0, void 0, function* () { if (ioUtil.IS_WINDOWS) { if (/[*"<>|]/.test(inputPath)) { @@ -2510,7 +2510,7 @@ var require_io = __commonJS({ } }); } - exports.rmRF = rmRF; + exports.rmRF = rmRF2; function mkdirP2(fsPath) { return __awaiter(this, void 0, void 0, function* () { assert_1.ok(fsPath, "a path argument must be provided"); @@ -2868,6 +2868,12 @@ var import_io = __toESM(require_io()); var core = __toESM(require_core()); var { GITHUB_REPOSITORY, RUNNER_TOOL_CACHE } = process.env; var CWD = process.cwd(); +var Strategy = /* @__PURE__ */ ((Strategy2) => { + Strategy2["CopyImmutable"] = "copy-immutable"; + Strategy2["Copy"] = "copy"; + Strategy2["Move"] = "move"; + return Strategy2; +})(Strategy || {}); var getVars = () => { if (!RUNNER_TOOL_CACHE) { throw new TypeError("Expected RUNNER_TOOL_CACHE environment variable to be defined."); @@ -2877,10 +2883,13 @@ var getVars = () => { } const options = { key: core.getInput("key") || "no-key", - path: core.getInput("path") + path: core.getInput("path"), + strategy: core.getInput("strategy") }; if (!options.path) { throw new TypeError("path is required but was not provided."); + } else if (!Object.values(Strategy).includes(options.strategy)) { + throw new TypeError(`Unknown strategy ${options.strategy}`); } const cacheDir = path__default.default.join(RUNNER_TOOL_CACHE, GITHUB_REPOSITORY, options.key); const cachePath = path__default.default.join(cacheDir, options.path); @@ -2916,11 +2925,28 @@ if (process.env.LOG_LEVEL) { var log_default = import_loglevel.default; // src/post.ts +var import_io_util = __toESM(require_io_util()); async function post() { try { - const { cacheDir, targetPath, cachePath } = getVars(); + const { cacheDir, targetPath, cachePath, options } = getVars(); await (0, import_io.mkdirP)(cacheDir); - await (0, import_io.mv)(targetPath, cachePath, { force: true }); + switch (options.strategy) { + case "copy-immutable" /* CopyImmutable */: + if (await (0, import_io_util.exists)(cachePath)) { + log_default.info(`Cache already exists, skipping`); + return; + } + await (0, import_io.cp)(targetPath, cachePath, { copySourceDirectory: true, recursive: true }); + break; + case "copy" /* Copy */: + await (0, import_io.rmRF)(cachePath); + await (0, import_io.cp)(targetPath, cachePath, { copySourceDirectory: true, recursive: true }); + break; + case "move" /* Move */: + await (0, import_io.mv)(targetPath, cachePath, { force: true }); + break; + } + log_default.info(`Cache saved to ${cachePath} with ${options.strategy} strategy`); } catch (error) { log_default.trace(error); (0, import_core.setFailed)(isErrorLike(error) ? error.message : `unknown error: ${error}`); diff --git a/src/lib/getVars.ts b/src/lib/getVars.ts index bda77c7..784ef80 100644 --- a/src/lib/getVars.ts +++ b/src/lib/getVars.ts @@ -5,12 +5,19 @@ import * as core from '@actions/core' const { GITHUB_REPOSITORY, RUNNER_TOOL_CACHE } = process.env const CWD = process.cwd() +export enum Strategy { + CopyImmutable = 'copy-immutable', + Copy = 'copy', + Move = 'move', +} + type Vars = { cacheDir: string cachePath: string options: { key: string - path: string + path: string, + strategy: Strategy } targetDir: string targetPath: string @@ -28,10 +35,13 @@ export const getVars = (): Vars => { const options = { key: core.getInput('key') || 'no-key', path: core.getInput('path'), + strategy: core.getInput('strategy') as Strategy, } if (!options.path) { throw new TypeError('path is required but was not provided.') + } else if (!Object.values(Strategy).includes(options.strategy)) { + throw new TypeError(`Unknown strategy ${options.strategy}`) } const cacheDir = path.join(RUNNER_TOOL_CACHE, GITHUB_REPOSITORY, options.key) diff --git a/src/main.ts b/src/main.ts index d35f7c0..9521086 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,19 +1,31 @@ import { setFailed, setOutput } from '@actions/core' -import { mkdirP, mv } from '@actions/io/' +import { mkdirP, mv, cp } from '@actions/io/' import { exists } from '@actions/io/lib/io-util' -import { getVars } from './lib/getVars' +import { Strategy, getVars } from './lib/getVars' import { isErrorLike } from './lib/isErrorLike' import log from './lib/log' async function main(): Promise { try { const { cachePath, targetDir, targetPath, options } = getVars() - if (await exists(cachePath)) { await mkdirP(targetDir) - await mv(cachePath, targetPath, { force: true }) - log.info(`Cache found and restored to ${options.path}`) + + switch (options.strategy) { + case Strategy.CopyImmutable: + case Strategy.Copy: + await cp(cachePath, targetPath, { + copySourceDirectory: false, + recursive: true, + }) + break + case Strategy.Move: + await mv(cachePath, targetPath, { force: true }) + break + } + + log.info(`Cache found and restored to ${options.path} with ${options.strategy} strategy`) setOutput('cache-hit', true) } else { log.info(`Skipping: cache not found for ${options.path}.`) diff --git a/src/post.ts b/src/post.ts index c999b51..4da5b68 100644 --- a/src/post.ts +++ b/src/post.ts @@ -1,16 +1,35 @@ import { setFailed } from '@actions/core' -import { mkdirP, mv } from '@actions/io' +import { mkdirP, mv, cp, rmRF } from '@actions/io' -import { getVars } from './lib/getVars' +import { Strategy, getVars } from './lib/getVars' import { isErrorLike } from './lib/isErrorLike' import log from './lib/log' +import { exists } from '@actions/io/lib/io-util' async function post(): Promise { try { - const { cacheDir, targetPath, cachePath } = getVars() + const { cacheDir, targetPath, cachePath, options } = getVars() await mkdirP(cacheDir) - await mv(targetPath, cachePath, { force: true }) + + switch (options.strategy) { + case Strategy.CopyImmutable: + if (await exists(cachePath)) { + log.info(`Cache already exists, skipping`) + return + } + await cp(targetPath, cachePath, { copySourceDirectory: true, recursive: true }) + break + case Strategy.Copy: + await rmRF(cachePath) + await cp(targetPath, cachePath, { copySourceDirectory: true, recursive: true }) + break + case Strategy.Move: + await mv(targetPath, cachePath, { force: true }) + break + } + + log.info(`Cache saved to ${cachePath} with ${options.strategy} strategy`) } catch (error: unknown) { log.trace(error) setFailed(isErrorLike(error) ? error.message : `unknown error: ${error}`)