From 54e89c96cb0abd33d2a333f7c1929c08c0b02a78 Mon Sep 17 00:00:00 2001 From: Kevin Peterson Date: Mon, 6 Nov 2023 21:11:05 -0800 Subject: [PATCH] Remove PIP keys from the cache key and hash function --- .../aws-cdk-lib/core/lib/asset-staging.ts | 18 ++- .../aws-cdk-lib/core/test/staging.test.ts | 109 ++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/packages/aws-cdk-lib/core/lib/asset-staging.ts b/packages/aws-cdk-lib/core/lib/asset-staging.ts index 438c9e9749051..faf7f10ca4320 100644 --- a/packages/aws-cdk-lib/core/lib/asset-staging.ts +++ b/packages/aws-cdk-lib/core/lib/asset-staging.ts @@ -487,7 +487,7 @@ export class AssetStaging extends Construct { // If we're bundling an asset, include the bundling configuration in the hash if (bundling) { - hash.update(JSON.stringify(bundling)); + hash.update(JSON.stringify(bundling, sanitizeHashValue)); } return hash.digest('hex'); @@ -538,7 +538,7 @@ function determineHashType(assetHashType?: AssetHashType, customSourceFingerprin */ function calculateCacheKey(props: A): string { return crypto.createHash('sha256') - .update(JSON.stringify(sortObject(props))) + .update(JSON.stringify(sortObject(props), sanitizeHashValue)) .digest('hex'); } @@ -556,6 +556,20 @@ function sortObject(object: { [key: string]: any }): { [key: string]: any } { return ret; } +/** + * Remove the auth token from the URLs if present. + */ +function sanitizeHashValue(key: string, value: any): any { + if (key === 'PIP_INDEX_URL' || key === 'PIP_EXTRA_INDEX_URL') { + let url = new URL(value); + if (url.password) { + url.password = ''; + return url.toString(); + } + } + return value; +} + /** * Returns the single archive file of a directory or undefined */ diff --git a/packages/aws-cdk-lib/core/test/staging.test.ts b/packages/aws-cdk-lib/core/test/staging.test.ts index 333e300dfd0fe..3f2a1cd6f86a6 100644 --- a/packages/aws-cdk-lib/core/test/staging.test.ts +++ b/packages/aws-cdk-lib/core/test/staging.test.ts @@ -496,6 +496,53 @@ describe('staging', () => { ]); }); + test('bundler ignores secret tokens in code artifact URLs', () => { + // GIVEN + const app = new App({ context: { [cxapi.NEW_STYLE_STACK_SYNTHESIS_CONTEXT]: false } }); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: DockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.SUCCESS], + environment: { + PIP_INDEX_URL: 'https://aws:MY_SECRET_TOKEN@your-code-repo.d.codeartifact.us-west-2.amazonaws.com/pypi/python/simple/', + }, + }, + }); + + new AssetStaging(stack, 'AssetWithDifferentBundlingOptions', { + sourcePath: directory, + bundling: { + image: DockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.SUCCESS], + environment: { + PIP_INDEX_URL: 'https://aws:MY_OTHER_SECRET_TOKEN@your-code-repo.d.codeartifact.us-west-2.amazonaws.com/pypi/python/simple/', + }, + }, + }); + + // THEN + const assembly = app.synth(); + + // We're testing that docker was run once, only for the first Asset, since the only difference is the token. + expect( + readDockerStubInputConcat()).toEqual( + `run --rm ${USER_ARG} -v /input:/asset-input:delegated -v /output:/asset-output:delegated --env PIP_INDEX_URL=https://aws:MY_SECRET_TOKEN@your-code-repo.d.codeartifact.us-west-2.amazonaws.com/pypi/python/simple/ -w /asset-input alpine DOCKER_STUB_SUCCESS`, + ); + + expect(fs.readdirSync(assembly.directory)).toEqual([ + 'asset.2de2347dd01e3f43a463652635acaae09539cdf32769d9a60ac0ad4622b1e943', // 'Asset' + 'cdk.out', + 'manifest.json', + 'stack.template.json', + 'tree.json', + ]); + }); + test('bundler outputs to intermediate dir and renames to asset', () => { // GIVEN const app = new App({ context: { [cxapi.NEW_STYLE_STACK_SYNTHESIS_CONTEXT]: false } }); @@ -609,6 +656,68 @@ describe('staging', () => { ]); }); + test('bundler re-uses assets from previous synths, ignoring tokens', () => { + // GIVEN + const TEST_OUTDIR = path.join(__dirname, 'cdk.out'); + if (fs.existsSync(TEST_OUTDIR)) { + fs.removeSync(TEST_OUTDIR); + } + + const app = new App({ outdir: TEST_OUTDIR, context: { [cxapi.NEW_STYLE_STACK_SYNTHESIS_CONTEXT]: false } }); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: DockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.SUCCESS], + environment: { + PIP_EXTRA_INDEX_URL: 'https://aws:MY_SECRET_TOKEN@your-code-repo.d.codeartifact.us-west-2.amazonaws.com/pypi/python/simple/', + }, + }, + }); + + // Clear asset hash cache to show that during the second synth bundling + // will consider the existing bundling dir (file system cache). + AssetStaging.clearAssetHashCache(); + + // GIVEN + const app2 = new App({ outdir: TEST_OUTDIR, context: { [cxapi.NEW_STYLE_STACK_SYNTHESIS_CONTEXT]: false } }); + const stack2 = new Stack(app2, 'stack'); + + // WHEN + new AssetStaging(stack2, 'Asset', { + sourcePath: directory, + bundling: { + image: DockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.SUCCESS], + environment: { + PIP_EXTRA_INDEX_URL: 'https://aws:MY_OTHER_SECRET_TOKEN@your-code-repo.d.codeartifact.us-west-2.amazonaws.com/pypi/python/simple/', + }, + }, + }); + + // THEN + const appAssembly = app.synth(); + const app2Assembly = app2.synth(); + + expect( + readDockerStubInputConcat()).toEqual( + `run --rm ${USER_ARG} -v /input:/asset-input:delegated -v /output:/asset-output:delegated --env PIP_EXTRA_INDEX_URL=https://aws:MY_SECRET_TOKEN@your-code-repo.d.codeartifact.us-west-2.amazonaws.com/pypi/python/simple/ -w /asset-input alpine DOCKER_STUB_SUCCESS`, + ); + + expect(appAssembly.directory).toEqual(app2Assembly.directory); + expect(fs.readdirSync(appAssembly.directory)).toEqual([ + 'asset.ec1d4062c578dacd630d64166a7d1efcd472e570e085a63f8857f6c674491bac', + 'cdk.out', + 'manifest.json', + 'stack.template.json', + 'tree.json', + ]); + }); + test('bundling throws when /asset-output is empty', () => { // GIVEN const app = new App();