From 53b215a91e822fdd0bf3b94af627b1ae2cf7a0a6 Mon Sep 17 00:00:00 2001 From: mattcompiles Date: Fri, 3 Mar 2023 02:27:09 +1100 Subject: [PATCH] Use BitSet for bundler intersections (#8862) --- .../bundlers/default/src/DefaultBundler.js | 101 +++++++------- packages/core/utils/package.json | 3 +- packages/core/utils/src/BitSet.js | 126 ++++++++++++++++++ packages/core/utils/src/index.js | 1 + packages/core/utils/test/BitSet.test.js | 119 +++++++++++++++++ packages/dev/eslint-config/index.js | 2 +- .../runtimes/hmr/src/loaders/hmr-runtime.js | 2 +- 7 files changed, 301 insertions(+), 53 deletions(-) create mode 100644 packages/core/utils/src/BitSet.js create mode 100644 packages/core/utils/test/BitSet.test.js diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 3eefdc4ed62..76827fa3a91 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -1,5 +1,5 @@ +/* eslint-disable */ // @flow strict-local - import type { Asset, Bundle as LegacyBundle, @@ -19,13 +19,7 @@ import {ContentGraph, Graph} from '@parcel/graph'; import invariant from 'assert'; import {ALL_EDGE_TYPES} from '@parcel/graph'; import {Bundler} from '@parcel/plugin'; -import { - setIntersect, - setUnion, - setEqual, - validateSchema, - DefaultMap, -} from '@parcel/utils'; +import {setEqual, validateSchema, DefaultMap, BitSet} from '@parcel/utils'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -56,14 +50,12 @@ const HTTP_OPTIONS = { }, }; -type AssetId = string; - /* BundleRoot - An asset that is the main entry of a Bundle. */ type BundleRoot = Asset; export type Bundle = {| uniqueKey: ?string, assets: Set, - internalizedAssetIds: Array, + internalizedAssets?: BitSet, bundleBehavior?: ?BundleBehavior, needsStableName: boolean, mainEntryAsset: ?Asset, @@ -230,17 +222,17 @@ function decorateLegacyGraph( for (let [, idealBundle] of idealBundleGraph.nodes) { if (idealBundle === 'root') continue; let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); - for (let internalized of idealBundle.internalizedAssetIds) { - let incomingDeps = bundleGraph.getIncomingDependencies( - bundleGraph.getAssetById(internalized), - ); - for (let incomingDep of incomingDeps) { - if ( - incomingDep.priority === 'lazy' && - incomingDep.specifierType !== 'url' && - bundle.hasDependency(incomingDep) - ) { - bundleGraph.internalizeAsyncDependency(bundle, incomingDep); + if (idealBundle.internalizedAssets) { + for (let internalized of idealBundle.internalizedAssets.values()) { + let incomingDeps = bundleGraph.getIncomingDependencies(internalized); + for (let incomingDep of incomingDeps) { + if ( + incomingDep.priority === 'lazy' && + incomingDep.specifierType !== 'url' && + bundle.hasDependency(incomingDep) + ) { + bundleGraph.internalizeAsyncDependency(bundle, incomingDep); + } } } } @@ -594,6 +586,9 @@ function createIdealGraph( } }, }); + + let assetSet = BitSet.from(assets); + // Step Merge Type Change Bundles: Clean up type change bundles within the exact same bundlegroups for (let [nodeIdA, a] of bundleGraph.nodes) { //if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes @@ -657,6 +652,10 @@ function createIdealGraph( if (node.type === 'dependency') { let dependency = node.value; + if (assetGraph.isDependencySkipped(dependency)) { + actions.skipChildren(); + } + if (dependencyBundleGraph.hasContentKey(dependency.id)) { if (dependency.priority !== 'sync') { let assets = assetGraph.getDependencyAssets(dependency); @@ -710,11 +709,11 @@ function createIdealGraph( } // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets - let ancestorAssets: Map> = new Map(); + let ancestorAssets: Map> = new Map(); for (let entry of entries.keys()) { // Initialize an empty set of ancestors available to entries - ancestorAssets.set(entry, new Set()); + ancestorAssets.set(entry, assetSet.cloneEmpty()); } // Step Determine Availability @@ -741,9 +740,9 @@ function createIdealGraph( // it belongs to. It's the intersection of those sets. let available; if (bundleRoot.bundleBehavior === 'isolated') { - available = new Set(); + available = assetSet.cloneEmpty(); } else { - available = new Set(ancestorAssets.get(bundleRoot)); + available = nullthrows(ancestorAssets.get(bundleRoot)).clone(); for (let bundleIdInGroup of [ bundleGroupId, ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), @@ -777,7 +776,7 @@ function createIdealGraph( nodeId, ALL_EDGE_TYPES, ); - let parallelAvailability: Set = new Set(); + let parallelAvailability = assetSet.cloneEmpty(); for (let childId of children) { let child = bundleRootGraph.getNode(childId); @@ -799,23 +798,21 @@ function createIdealGraph( // it will only assume availability of assets it has under any circumstance const childAvailableAssets = ancestorAssets.get(child); let currentChildAvailable = isParallel - ? setUnion(parallelAvailability, available) + ? BitSet.union(parallelAvailability, available) : available; if (childAvailableAssets != null) { - setIntersect(childAvailableAssets, currentChildAvailable); + childAvailableAssets.intersect(currentChildAvailable); } else { - ancestorAssets.set(child, new Set(currentChildAvailable)); + ancestorAssets.set(child, currentChildAvailable.clone()); } if (isParallel) { - let assetsFromBundleRoot = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(child.id), - ) - .map(id => nullthrows(reachableRoots.getNode(id))); - parallelAvailability = setUnion( - parallelAvailability, - assetsFromBundleRoot, - ); + for (let reachableNodeId of reachableRoots.getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(child.id), + )) { + let asset = nullthrows(reachableRoots.getNode(reachableNodeId)); + + parallelAvailability.add(asset); + } parallelAvailability.add(child); //The next sibling should have older sibling available via parallel } } @@ -848,7 +845,11 @@ function createIdealGraph( nullthrows(bundles.get(parent.id)), ); invariant(parentBundle != null && parentBundle !== 'root'); - parentBundle.internalizedAssetIds.push(bundleRoot.id); + if (!parentBundle.internalizedAssets) { + parentBundle.internalizedAssets = assetSet.cloneEmpty(); + } + + parentBundle.internalizedAssets.add(bundleRoot); } else { canDelete = false; } @@ -970,20 +971,22 @@ function createIdealGraph( env: firstSourceBundle.env, }); bundle.sourceBundles = new Set(sourceBundles); - let sharedInternalizedAssets = new Set( - firstSourceBundle.internalizedAssetIds, - ); + let sharedInternalizedAssets = firstSourceBundle.internalizedAssets + ? firstSourceBundle.internalizedAssets.clone() + : assetSet.cloneEmpty(); for (let p of sourceBundles) { let parentBundle = nullthrows(bundleGraph.getNode(p)); invariant(parentBundle !== 'root'); if (parentBundle === firstSourceBundle) continue; - setIntersect( - sharedInternalizedAssets, - new Set(parentBundle.internalizedAssetIds), - ); + + if (parentBundle.internalizedAssets) { + sharedInternalizedAssets.intersect(parentBundle.internalizedAssets); + } else { + sharedInternalizedAssets.clear(); + } } - bundle.internalizedAssetIds = [...sharedInternalizedAssets]; + bundle.internalizedAssets = sharedInternalizedAssets; bundleId = bundleGraph.addNode(bundle); bundles.set(key, bundleId); } else { @@ -1257,7 +1260,6 @@ function createBundle(opts: {| return { uniqueKey: opts.uniqueKey, assets: new Set(), - internalizedAssetIds: [], mainEntryAsset: null, size: 0, sourceBundles: new Set(), @@ -1273,7 +1275,6 @@ function createBundle(opts: {| return { uniqueKey: opts.uniqueKey, assets: new Set([asset]), - internalizedAssetIds: [], mainEntryAsset: asset, size: asset.stats.size, sourceBundles: new Set(), diff --git a/packages/core/utils/package.json b/packages/core/utils/package.json index 5fe5f48a4e0..a95fc01ca14 100644 --- a/packages/core/utils/package.json +++ b/packages/core/utils/package.json @@ -39,7 +39,8 @@ "@parcel/logger": "2.8.3", "@parcel/markdown-ansi": "2.8.3", "@parcel/source-map": "^2.1.1", - "chalk": "^4.1.0" + "chalk": "^4.1.0", + "nullthrows": "^1.1.1" }, "devDependencies": { "@iarna/toml": "^2.2.0", diff --git a/packages/core/utils/src/BitSet.js b/packages/core/utils/src/BitSet.js new file mode 100644 index 00000000000..39db524338c --- /dev/null +++ b/packages/core/utils/src/BitSet.js @@ -0,0 +1,126 @@ +// @flow strict-local +import nullthrows from 'nullthrows'; + +// As our current version of flow doesn't support BigInt's, these values/types +// have been hoisted to keep the flow errors to a minimum. This can be removed +// if we upgrade to a flow version that supports BigInt's +// $FlowFixMe +type TmpBigInt = bigint; +// $FlowFixMe +const BIGINT_ZERO = 0n; +// $FlowFixMe +const BIGINT_ONE = 1n; +// $FlowFixMe +let numberToBigInt = (v: number): TmpBigInt => BigInt(v); + +let bitUnion = (a: TmpBigInt, b: TmpBigInt): TmpBigInt => a | b; + +export class BitSet { + _value: TmpBigInt; + _lookup: Map; + _items: Array; + + constructor({ + initial, + items, + lookup, + }: {| + items: Array, + lookup: Map, + initial?: BitSet | TmpBigInt, + |}) { + if (initial instanceof BitSet) { + this._value = initial?._value; + } else if (initial) { + this._value = initial; + } else { + this._value = BIGINT_ZERO; + } + + this._items = items; + this._lookup = lookup; + } + + static from(items: Array): BitSet { + let lookup: Map = new Map(); + for (let i = 0; i < items.length; i++) { + lookup.set(items[i], numberToBigInt(i)); + } + + return new BitSet({items, lookup}); + } + + static union(a: BitSet, b: BitSet): BitSet { + return new BitSet({ + initial: bitUnion(a._value, b._value), + lookup: a._lookup, + items: a._items, + }); + } + + #getIndex(item: Item) { + return nullthrows(this._lookup.get(item), 'Item is missing from BitSet'); + } + + add(item: Item) { + this._value |= BIGINT_ONE << this.#getIndex(item); + } + + delete(item: Item) { + this._value &= ~(BIGINT_ONE << this.#getIndex(item)); + } + + has(item: Item): boolean { + return Boolean(this._value & (BIGINT_ONE << this.#getIndex(item))); + } + + intersect(v: BitSet) { + this._value = this._value & v._value; + } + + union(v: BitSet) { + this._value = bitUnion(this._value, v._value); + } + + clear() { + this._value = BIGINT_ZERO; + } + + cloneEmpty(): BitSet { + return new BitSet({ + lookup: this._lookup, + items: this._items, + }); + } + + clone(): BitSet { + return new BitSet({ + lookup: this._lookup, + items: this._items, + initial: this._value, + }); + } + + values(): Array { + let values = []; + let tmpValue = this._value; + let i; + + // This implementation is optimized for BitSets that contain a very small percentage + // of items compared to the total number of potential items. This makes sense for + // our bundler use-cases where Sets often contain <1% coverage of the total item count. + // In cases where Sets contain a larger percentage of the total items, a regular looping + // strategy would be more performant. + while (tmpValue > BIGINT_ZERO) { + // Get last set bit + i = tmpValue.toString(2).length - 1; + + values.push(this._items[i]); + + // Unset last set bit + tmpValue &= ~(BIGINT_ONE << numberToBigInt(i)); + } + + return values; + } +} diff --git a/packages/core/utils/src/index.js b/packages/core/utils/src/index.js index 63bf339e0ab..6d6a75c5534 100644 --- a/packages/core/utils/src/index.js +++ b/packages/core/utils/src/index.js @@ -73,3 +73,4 @@ export { loadSourceMap, remapSourceLocation, } from './sourcemap'; +export {BitSet} from './BitSet'; diff --git a/packages/core/utils/test/BitSet.test.js b/packages/core/utils/test/BitSet.test.js new file mode 100644 index 00000000000..c21656c37a1 --- /dev/null +++ b/packages/core/utils/test/BitSet.test.js @@ -0,0 +1,119 @@ +// @flow strict-local + +import assert from 'assert'; +import {BitSet} from '../src/BitSet'; + +function assertValues(set: BitSet, values: Array) { + let setValues = set.values(); + + for (let value of values) { + assert(set.has(value), 'Set.has returned false'); + assert( + setValues.some(v => v === value), + 'Set values is missing value', + ); + } + + assert( + setValues.length === values.length, + `Expected ${values.length} values but got ${setValues.length}`, + ); +} + +describe('BitSet', () => { + it('cloneEmpty should return an empty set', () => { + let set1 = BitSet.from([1, 2, 3, 4, 5]); + set1.add(1); + set1.add(3); + + let set2 = set1.cloneEmpty(); + + assertValues(set2, []); + }); + + it('clone should return a set with the same values', () => { + let set1 = BitSet.from([1, 2, 3, 4, 5]); + set1.add(1); + set1.add(3); + + let set2 = set1.clone(); + + assertValues(set2, [1, 3]); + }); + + it('clear should remove all values from the set', () => { + let set1 = BitSet.from([1, 2, 3, 4, 5]); + set1.add(1); + set1.add(3); + + set1.clear(); + + assertValues(set1, []); + }); + + it('delete should remove values from the set', () => { + let set1 = BitSet.from([1, 2, 3, 4, 5]); + set1.add(1); + set1.add(3); + set1.add(5); + + set1.delete(3); + + assertValues(set1, [1, 5]); + }); + + it('should intersect with another BitSet', () => { + let set1 = BitSet.from([1, 2, 3, 4, 5]); + set1.add(1); + set1.add(3); + + let set2 = set1.cloneEmpty(); + set2.add(3); + set2.add(5); + + set1.intersect(set2); + assertValues(set1, [3]); + }); + + it('should union with another BitSet', () => { + let set1 = BitSet.from([1, 2, 3, 4, 5]); + set1.add(1); + set1.add(3); + + let set2 = set1.cloneEmpty(); + set2.add(3); + set2.add(5); + + set1.union(set2); + assertValues(set1, [1, 3, 5]); + }); + + it('BitSet.union should create a new BitSet with the union', () => { + let set1 = BitSet.from([1, 2, 3, 4, 5]); + set1.add(1); + set1.add(3); + + let set2 = set1.cloneEmpty(); + set2.add(3); + set2.add(5); + + let set3 = BitSet.union(set1, set2); + assertValues(set1, [1, 3]); + assertValues(set2, [3, 5]); + assertValues(set3, [1, 3, 5]); + }); + + it('returns an array of all values', () => { + let set = BitSet.from([1, 2, 3, 4]); + set.add(1); + set.add(3); + + assertValues(set, [3, 1]); + }); + + it('should return an error if a new item is added', () => { + let set = BitSet.from([1, 2, 3, 4]); + + assert.throws(() => set.add(5), /Item is missing from BitSet/); + }); +}); diff --git a/packages/dev/eslint-config/index.js b/packages/dev/eslint-config/index.js index 0abfa81961a..0cf9f4451e7 100644 --- a/packages/dev/eslint-config/index.js +++ b/packages/dev/eslint-config/index.js @@ -18,8 +18,8 @@ module.exports = { sourceType: 'module', }, env: { + es2020: true, node: true, - es6: true, }, globals: { parcelRequire: true, diff --git a/packages/runtimes/hmr/src/loaders/hmr-runtime.js b/packages/runtimes/hmr/src/loaders/hmr-runtime.js index 399219202d9..7514eedb3f3 100644 --- a/packages/runtimes/hmr/src/loaders/hmr-runtime.js +++ b/packages/runtimes/hmr/src/loaders/hmr-runtime.js @@ -1,5 +1,5 @@ // @flow -/* global HMR_HOST, HMR_PORT, HMR_ENV_HASH, HMR_SECURE, chrome, browser, globalThis, __parcel__import__, __parcel__importScripts__, ServiceWorkerGlobalScope */ +/* global HMR_HOST, HMR_PORT, HMR_ENV_HASH, HMR_SECURE, chrome, browser, __parcel__import__, __parcel__importScripts__, ServiceWorkerGlobalScope */ /*:: import type {