Skip to content

Commit

Permalink
Use BitSet for bundler intersections (#8862)
Browse files Browse the repository at this point in the history
  • Loading branch information
mattcompiles authored Mar 2, 2023
1 parent 28a2744 commit 53b215a
Show file tree
Hide file tree
Showing 7 changed files with 301 additions and 53 deletions.
101 changes: 51 additions & 50 deletions packages/bundlers/default/src/DefaultBundler.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* eslint-disable */
// @flow strict-local

import type {
Asset,
Bundle as LegacyBundle,
Expand All @@ -19,13 +19,7 @@ import {ContentGraph, Graph} from '@parcel/graph';
import invariant from 'assert';
import {ALL_EDGE_TYPES} from '@parcel/graph';
import {Bundler} from '@parcel/plugin';
import {
setIntersect,
setUnion,
setEqual,
validateSchema,
DefaultMap,
} from '@parcel/utils';
import {setEqual, validateSchema, DefaultMap, BitSet} from '@parcel/utils';
import nullthrows from 'nullthrows';
import {encodeJSONKeyComponent} from '@parcel/diagnostic';

Expand Down Expand Up @@ -56,14 +50,12 @@ const HTTP_OPTIONS = {
},
};

type AssetId = string;

/* BundleRoot - An asset that is the main entry of a Bundle. */
type BundleRoot = Asset;
export type Bundle = {|
uniqueKey: ?string,
assets: Set<Asset>,
internalizedAssetIds: Array<AssetId>,
internalizedAssets?: BitSet<Asset>,
bundleBehavior?: ?BundleBehavior,
needsStableName: boolean,
mainEntryAsset: ?Asset,
Expand Down Expand Up @@ -230,17 +222,17 @@ function decorateLegacyGraph(
for (let [, idealBundle] of idealBundleGraph.nodes) {
if (idealBundle === 'root') continue;
let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle));
for (let internalized of idealBundle.internalizedAssetIds) {
let incomingDeps = bundleGraph.getIncomingDependencies(
bundleGraph.getAssetById(internalized),
);
for (let incomingDep of incomingDeps) {
if (
incomingDep.priority === 'lazy' &&
incomingDep.specifierType !== 'url' &&
bundle.hasDependency(incomingDep)
) {
bundleGraph.internalizeAsyncDependency(bundle, incomingDep);
if (idealBundle.internalizedAssets) {
for (let internalized of idealBundle.internalizedAssets.values()) {
let incomingDeps = bundleGraph.getIncomingDependencies(internalized);
for (let incomingDep of incomingDeps) {
if (
incomingDep.priority === 'lazy' &&
incomingDep.specifierType !== 'url' &&
bundle.hasDependency(incomingDep)
) {
bundleGraph.internalizeAsyncDependency(bundle, incomingDep);
}
}
}
}
Expand Down Expand Up @@ -594,6 +586,9 @@ function createIdealGraph(
}
},
});

let assetSet = BitSet.from(assets);

// Step Merge Type Change Bundles: Clean up type change bundles within the exact same bundlegroups
for (let [nodeIdA, a] of bundleGraph.nodes) {
//if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes
Expand Down Expand Up @@ -657,6 +652,10 @@ function createIdealGraph(
if (node.type === 'dependency') {
let dependency = node.value;

if (assetGraph.isDependencySkipped(dependency)) {
actions.skipChildren();
}

if (dependencyBundleGraph.hasContentKey(dependency.id)) {
if (dependency.priority !== 'sync') {
let assets = assetGraph.getDependencyAssets(dependency);
Expand Down Expand Up @@ -710,11 +709,11 @@ function createIdealGraph(
}
// Maps a given bundleRoot to the assets reachable from it,
// and the bundleRoots reachable from each of these assets
let ancestorAssets: Map<BundleRoot, Set<Asset>> = new Map();
let ancestorAssets: Map<BundleRoot, BitSet<Asset>> = new Map();

for (let entry of entries.keys()) {
// Initialize an empty set of ancestors available to entries
ancestorAssets.set(entry, new Set());
ancestorAssets.set(entry, assetSet.cloneEmpty());
}

// Step Determine Availability
Expand All @@ -741,9 +740,9 @@ function createIdealGraph(
// it belongs to. It's the intersection of those sets.
let available;
if (bundleRoot.bundleBehavior === 'isolated') {
available = new Set();
available = assetSet.cloneEmpty();
} else {
available = new Set(ancestorAssets.get(bundleRoot));
available = nullthrows(ancestorAssets.get(bundleRoot)).clone();
for (let bundleIdInGroup of [
bundleGroupId,
...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId),
Expand Down Expand Up @@ -777,7 +776,7 @@ function createIdealGraph(
nodeId,
ALL_EDGE_TYPES,
);
let parallelAvailability: Set<BundleRoot> = new Set();
let parallelAvailability = assetSet.cloneEmpty();

for (let childId of children) {
let child = bundleRootGraph.getNode(childId);
Expand All @@ -799,23 +798,21 @@ function createIdealGraph(
// it will only assume availability of assets it has under any circumstance
const childAvailableAssets = ancestorAssets.get(child);
let currentChildAvailable = isParallel
? setUnion(parallelAvailability, available)
? BitSet.union(parallelAvailability, available)
: available;
if (childAvailableAssets != null) {
setIntersect(childAvailableAssets, currentChildAvailable);
childAvailableAssets.intersect(currentChildAvailable);
} else {
ancestorAssets.set(child, new Set(currentChildAvailable));
ancestorAssets.set(child, currentChildAvailable.clone());
}
if (isParallel) {
let assetsFromBundleRoot = reachableRoots
.getNodeIdsConnectedFrom(
reachableRoots.getNodeIdByContentKey(child.id),
)
.map(id => nullthrows(reachableRoots.getNode(id)));
parallelAvailability = setUnion(
parallelAvailability,
assetsFromBundleRoot,
);
for (let reachableNodeId of reachableRoots.getNodeIdsConnectedFrom(
reachableRoots.getNodeIdByContentKey(child.id),
)) {
let asset = nullthrows(reachableRoots.getNode(reachableNodeId));

parallelAvailability.add(asset);
}
parallelAvailability.add(child); //The next sibling should have older sibling available via parallel
}
}
Expand Down Expand Up @@ -848,7 +845,11 @@ function createIdealGraph(
nullthrows(bundles.get(parent.id)),
);
invariant(parentBundle != null && parentBundle !== 'root');
parentBundle.internalizedAssetIds.push(bundleRoot.id);
if (!parentBundle.internalizedAssets) {
parentBundle.internalizedAssets = assetSet.cloneEmpty();
}

parentBundle.internalizedAssets.add(bundleRoot);
} else {
canDelete = false;
}
Expand Down Expand Up @@ -970,20 +971,22 @@ function createIdealGraph(
env: firstSourceBundle.env,
});
bundle.sourceBundles = new Set(sourceBundles);
let sharedInternalizedAssets = new Set(
firstSourceBundle.internalizedAssetIds,
);
let sharedInternalizedAssets = firstSourceBundle.internalizedAssets
? firstSourceBundle.internalizedAssets.clone()
: assetSet.cloneEmpty();

for (let p of sourceBundles) {
let parentBundle = nullthrows(bundleGraph.getNode(p));
invariant(parentBundle !== 'root');
if (parentBundle === firstSourceBundle) continue;
setIntersect(
sharedInternalizedAssets,
new Set(parentBundle.internalizedAssetIds),
);

if (parentBundle.internalizedAssets) {
sharedInternalizedAssets.intersect(parentBundle.internalizedAssets);
} else {
sharedInternalizedAssets.clear();
}
}
bundle.internalizedAssetIds = [...sharedInternalizedAssets];
bundle.internalizedAssets = sharedInternalizedAssets;
bundleId = bundleGraph.addNode(bundle);
bundles.set(key, bundleId);
} else {
Expand Down Expand Up @@ -1257,7 +1260,6 @@ function createBundle(opts: {|
return {
uniqueKey: opts.uniqueKey,
assets: new Set(),
internalizedAssetIds: [],
mainEntryAsset: null,
size: 0,
sourceBundles: new Set(),
Expand All @@ -1273,7 +1275,6 @@ function createBundle(opts: {|
return {
uniqueKey: opts.uniqueKey,
assets: new Set([asset]),
internalizedAssetIds: [],
mainEntryAsset: asset,
size: asset.stats.size,
sourceBundles: new Set(),
Expand Down
3 changes: 2 additions & 1 deletion packages/core/utils/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@
"@parcel/logger": "2.8.3",
"@parcel/markdown-ansi": "2.8.3",
"@parcel/source-map": "^2.1.1",
"chalk": "^4.1.0"
"chalk": "^4.1.0",
"nullthrows": "^1.1.1"
},
"devDependencies": {
"@iarna/toml": "^2.2.0",
Expand Down
126 changes: 126 additions & 0 deletions packages/core/utils/src/BitSet.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
// @flow strict-local
import nullthrows from 'nullthrows';

// As our current version of flow doesn't support BigInt's, these values/types
// have been hoisted to keep the flow errors to a minimum. This can be removed
// if we upgrade to a flow version that supports BigInt's
// $FlowFixMe
type TmpBigInt = bigint;
// $FlowFixMe
const BIGINT_ZERO = 0n;
// $FlowFixMe
const BIGINT_ONE = 1n;
// $FlowFixMe
let numberToBigInt = (v: number): TmpBigInt => BigInt(v);

let bitUnion = (a: TmpBigInt, b: TmpBigInt): TmpBigInt => a | b;

export class BitSet<Item> {
_value: TmpBigInt;
_lookup: Map<Item, TmpBigInt>;
_items: Array<Item>;

constructor({
initial,
items,
lookup,
}: {|
items: Array<Item>,
lookup: Map<Item, number>,
initial?: BitSet<Item> | TmpBigInt,
|}) {
if (initial instanceof BitSet) {
this._value = initial?._value;
} else if (initial) {
this._value = initial;
} else {
this._value = BIGINT_ZERO;
}

this._items = items;
this._lookup = lookup;
}

static from(items: Array<Item>): BitSet<Item> {
let lookup: Map<Item, TmpBigInt> = new Map();
for (let i = 0; i < items.length; i++) {
lookup.set(items[i], numberToBigInt(i));
}

return new BitSet({items, lookup});
}

static union(a: BitSet<Item>, b: BitSet<Item>): BitSet<Item> {
return new BitSet({
initial: bitUnion(a._value, b._value),
lookup: a._lookup,
items: a._items,
});
}

#getIndex(item: Item) {
return nullthrows(this._lookup.get(item), 'Item is missing from BitSet');
}

add(item: Item) {
this._value |= BIGINT_ONE << this.#getIndex(item);
}

delete(item: Item) {
this._value &= ~(BIGINT_ONE << this.#getIndex(item));
}

has(item: Item): boolean {
return Boolean(this._value & (BIGINT_ONE << this.#getIndex(item)));
}

intersect(v: BitSet<Item>) {
this._value = this._value & v._value;
}

union(v: BitSet<Item>) {
this._value = bitUnion(this._value, v._value);
}

clear() {
this._value = BIGINT_ZERO;
}

cloneEmpty(): BitSet<Item> {
return new BitSet({
lookup: this._lookup,
items: this._items,
});
}

clone(): BitSet<Item> {
return new BitSet({
lookup: this._lookup,
items: this._items,
initial: this._value,
});
}

values(): Array<Item> {
let values = [];
let tmpValue = this._value;
let i;

// This implementation is optimized for BitSets that contain a very small percentage
// of items compared to the total number of potential items. This makes sense for
// our bundler use-cases where Sets often contain <1% coverage of the total item count.
// In cases where Sets contain a larger percentage of the total items, a regular looping
// strategy would be more performant.
while (tmpValue > BIGINT_ZERO) {
// Get last set bit
i = tmpValue.toString(2).length - 1;

values.push(this._items[i]);

// Unset last set bit
tmpValue &= ~(BIGINT_ONE << numberToBigInt(i));
}

return values;
}
}
1 change: 1 addition & 0 deletions packages/core/utils/src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,4 @@ export {
loadSourceMap,
remapSourceLocation,
} from './sourcemap';
export {BitSet} from './BitSet';
Loading

0 comments on commit 53b215a

Please sign in to comment.