fix(cli): asset existence check is slow for many assets

packages/aws-cdk/lib/util/parallel.ts

@@ -0,0 +1,44 @@
+/**
+ * Run a number of promise generators with max parallelism
+ *
+ * Order is not maintained between the input and output.
+ */
+export async function parallelPromises<A>(n: number, promises: Array<() => Promise<A>>): Promise<Array<A>> {
+  const ret = new Array<A>();
+  let count = 0;
+  let error: Error | undefined;
+  const queue = [...promises];
+
+  return new Promise((ok, ko) => {
+    tick();
+
+    function tick() {
+      if (count === 0 && error) {
+        ko(error);
+        return;
+      }
+      if (count === 0 && queue.length === 0) {
+        ok(ret);
+        return;
+      }
+
+      while (count < n && queue.length > 0 && !error) {
+        const next = queue.shift();
+        if (next !== undefined) {
+          start(next);
+        }
+      }
+    }
+
+    function start(fn: () => Promise<A>) {
+      count += 1;
+      fn()
+        .then((result) => { ret.push(result); })
+        .catch((e) => { error = e; })
+        .finally(() => {
+          count -= 1;
+          tick();
+        });
+    }
+  });
+}

packages/aws-cdk/lib/util/work-graph-builder.ts

@@ -27,7 +27,7 @@ export class WorkGraphBuilder {
     this.graph.addNodes({
       type: 'stack',
       id: `${this.idPrefix}${artifact.id}`,
-      dependencies: new Set(this.getDepIds(artifact.dependencies)),
+      dependencies: new Set(this.getDepIds(onlyStacks(artifact.dependencies))),
       stack: artifact,
       deploymentState: DeploymentState.PENDING,
       priority: WorkGraphBuilder.PRIORITIES.stack,

packages/aws-cdk/lib/util/work-graph.ts

@@ -1,4 +1,6 @@
+import { parallelPromises } from './parallel';
 import { WorkNode, DeploymentState, StackNode, AssetBuildNode, AssetPublishNode } from './work-graph-types';
+import { debug, trace } from '../logging';
 
 export type Concurrency = number | Record<WorkNode['type'], number>;
 
@@ -215,15 +217,19 @@ export class WorkGraph {
     function renderNode(id: string, node: WorkNode): string[] {
       const ret = [];
       if (node.deploymentState === DeploymentState.COMPLETED) {
-        ret.push(`  "${id}" [style=filled,fillcolor=yellow];`);
+        ret.push(`  "${simplifyId(id)}" [style=filled,fillcolor=yellow];`);
       } else {
-        ret.push(`  "${id}";`);
+        ret.push(`  "${simplifyId(id)}";`);
       }
       for (const dep of node.dependencies) {
-        ret.push(`  "${id}" -> "${dep}";`);
+        ret.push(`  "${simplifyId(id)}" -> "${simplifyId(dep)}";`);
       }
       return ret;
     }
+
+    function simplifyId(id: string) {
+      return id.replace(/([0-9a-f]{6})[0-9a-f]{6,}/g, '$1');
+    }
   }
 
   /**
@@ -234,12 +240,8 @@ export class WorkGraph {
    */
   public removeUnavailableDependencies() {
     for (const node of Object.values(this.nodes)) {
-      const removeDeps = [];
-      for (const dep of node.dependencies) {
-        if (this.nodes[dep] === undefined) {
-          removeDeps.push(dep);
-        }
-      }
+      const removeDeps = Array.from(node.dependencies).filter((dep) => this.nodes[dep] === undefined);
+
       removeDeps.forEach((d) => {
         node.dependencies.delete(d);
       });
@@ -249,16 +251,25 @@ export class WorkGraph {
   /**
    * Remove all asset publishing steps for assets that are already published, and then build
    * that aren't used anymore.
+   *
+   * Do this in parallel, because there may be a lot of assets in an application (seen in practice: >100 assets)
    */
   public async removeUnnecessaryAssets(isUnnecessary: (x: AssetPublishNode) => Promise<boolean>) {
+    debug('Checking for previously published assets');
+
     const publishes = this.nodesOfType('asset-publish');
-    for (const assetNode of publishes) {
-      const unnecessary = await isUnnecessary(assetNode);
-      if (unnecessary) {
-        this.removeNode(assetNode);
-      }
+
+    const classifiedNodes = await parallelPromises(
+      8,
+      publishes.map((assetNode) => async() => [assetNode, await isUnnecessary(assetNode)] as const));
+
+    const alreadyPublished = classifiedNodes.filter(([_, unnecessary]) => unnecessary).map(([assetNode, _]) => assetNode);
+    for (const assetNode of alreadyPublished) {
+      this.removeNode(assetNode);
     }
 
+    debug(`${publishes.length} total assets, ${publishes.length - alreadyPublished.length} still need to be published`);
+
     // Now also remove any asset build steps that don't have any dependencies on them anymore
     const unusedBuilds = this.nodesOfType('asset-build').filter(build => this.dependees(build).length === 0);
     for (const unusedBuild of unusedBuilds) {
@@ -288,7 +299,8 @@ export class WorkGraph {
 
     if (this.readyPool.length === 0 && activeCount === 0 && pendingCount > 0) {
       const cycle = this.findCycle() ?? ['No cycle found!'];
-      throw new Error(`Unable to make progress anymore, dependency cycle between remaining artifacts: ${cycle.join(' -> ')}`);
+      trace(`Cycle ${cycle.join(' -> ')} in graph ${this}`);
+      throw new Error(`Unable to make progress anymore, dependency cycle between remaining artifacts: ${cycle.join(' -> ')} (run with -vv for full graph)`);
     }
   }
 

packages/aws-cdk/test/util/parallel.test.ts

@@ -0,0 +1,33 @@
+import { parallelPromises } from '../../lib/util/parallel';
+import { sleep } from '../util';
+
+test('parallelPromises', async () => {
+  const N = 4;
+  const J = 100;
+
+  let jobsDone = 0;
+  let concurrent = 0;
+  let maxConcurrent = 0;
+
+  const jobs = range(J).map(() => async () => {
+    concurrent += 1;
+    maxConcurrent = Math.max(concurrent, maxConcurrent);
+    await sleep(Math.round(Math.random() * 100));
+    concurrent -= 1;
+    jobsDone += 1;
+  });
+
+  await parallelPromises(N, jobs);
+
+  expect(maxConcurrent).toBeLessThanOrEqual(N);
+  expect(maxConcurrent).toBeGreaterThan(1);
+  expect(jobsDone).toEqual(J);
+});
+
+function range(n: number) {
+  const ret = new Array<number>();
+  for (let i = 0; i < n; i++) {
+    ret.push(i);
+  }
+  return ret;
+}