Skip to content

Commit

Permalink
fix: pattern based compression
Browse files Browse the repository at this point in the history
  • Loading branch information
@erights
erights committed Oct 8, 2022
1 parent 421d477 commit 4192893
Show file tree
Hide file tree
Showing 9 changed files with 687 additions and 54 deletions.
1 change: 1 addition & 0 deletions packages/ERTP/src/paymentLedger.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ export const vivifyPaymentLedger = (
const paymentLedger = provideDurableWeakMapStore(
issuerBaggage,
'paymentLedger',
{ valueShape: amountShape },
);

/**
Expand Down
52 changes: 30 additions & 22 deletions packages/SwingSet/src/liveslots/collectionManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
assertKeyPattern,
assertPattern,
matches,
fit,
compareRank,
M,
zeroPad,
Expand All @@ -13,6 +14,8 @@ import {
isEncodedRemotable,
makeCopySet,
makeCopyMap,
mustCompress,
decompress,
} from '@agoric/store';
import { Far, passStyleOf } from '@endo/marshal';
import { decodeToJustin } from '@endo/marshal/src/marshal-justin.js';
Expand Down Expand Up @@ -213,7 +216,7 @@ export function makeCollectionManager(
return storeKindInfo[kindName].kindID;
}

// Not that it's only used for this purpose, what should it be called?
// Now that it's only used for this purpose, what should it be called?
// TODO Should we be using the new encodeBigInt scheme instead, anyway?
const BIGINT_TAG_LEN = 10;

Expand Down Expand Up @@ -257,6 +260,23 @@ export function makeCollectionManager(
const dbKeyPrefix = `vc.${collectionID}.`;
let currentGenerationNumber = 0;

const keyLabel = `invalid key type for collection ${q(label)}`;
const valueLabel = `invalid value type for collection ${q(label)}`;

const serializeValue = value => {
if (valueShape === undefined) {
return serialize(value);
}
return serialize(mustCompress(value, valueShape, valueLabel));
};

const unserializeValue = data => {
if (valueShape === undefined) {
return unserialize(data);
}
return decompress(unserialize(data), valueShape);
};

function prefix(dbEntryKey) {
return `${dbKeyPrefix}${dbEntryKey}`;
}
Expand Down Expand Up @@ -331,11 +351,10 @@ export function makeCollectionManager(
}

function get(key) {
matches(key, keyShape) ||
assert.fail(X`invalid key type for collection ${q(label)}`);
fit(key, keyShape, keyLabel);
const result = syscall.vatstoreGet(keyToDBKey(key));
if (result) {
return unserialize(JSON.parse(result));
return unserializeValue(JSON.parse(result));
}
assert.fail(X`key ${key} not found in collection ${q(label)}`);
}
Expand All @@ -351,16 +370,11 @@ export function makeCollectionManager(
}

function init(key, value) {
matches(key, keyShape) ||
assert.fail(X`invalid key type for collection ${q(label)}`);
fit(key, keyShape, keyLabel);
!has(key) ||
assert.fail(X`key ${key} already registered in collection ${q(label)}`);
if (valueShape) {
matches(value, valueShape) ||
assert.fail(X`invalid value type for collection ${q(label)}`);
}
currentGenerationNumber += 1;
const serializedValue = serialize(value);
const serializedValue = serializeValue(value);
assertAcceptableSyscallCapdataSize([serializedValue]);
if (durable) {
serializedValue.slots.forEach((vref, slotIndex) => {
Expand Down Expand Up @@ -388,13 +402,8 @@ export function makeCollectionManager(
}

function set(key, value) {
matches(key, keyShape) ||
assert.fail(X`invalid key type for collection ${q(label)}`);
if (valueShape) {
matches(value, valueShape) ||
assert.fail(X`invalid value type for collection ${q(label)}`);
}
const after = serialize(harden(value));
fit(key, keyShape, keyLabel);
const after = serializeValue(harden(value));
assertAcceptableSyscallCapdataSize([after]);
if (durable) {
after.slots.forEach((vref, i) => {
Expand All @@ -412,8 +421,7 @@ export function makeCollectionManager(
}

function deleteInternal(key) {
matches(key, keyShape) ||
assert.fail(X`invalid key type for collection ${q(label)}`);
fit(key, keyShape, keyLabel);
const dbKey = keyToDBKey(key);
const rawValue = syscall.vatstoreGet(dbKey);
assert(rawValue, X`key ${key} not found in collection ${q(label)}`);
Expand Down Expand Up @@ -472,7 +480,7 @@ export function makeCollectionManager(
if (dbKey < end) {
priorDBKey = dbKey;
if (ignoreKeys) {
const value = unserialize(JSON.parse(dbValue));
const value = unserializeValue(JSON.parse(dbValue));
if (matches(value, valuePatt)) {
yield [undefined, value];
}
Expand All @@ -484,7 +492,7 @@ export function makeCollectionManager(
} else {
const key = dbKeyToKey(dbKey);
if (matches(key, keyPatt)) {
const value = unserialize(JSON.parse(dbValue));
const value = unserializeValue(JSON.parse(dbValue));
if (matches(value, valuePatt)) {
yield [key, value];
}
Expand Down
52 changes: 37 additions & 15 deletions packages/SwingSet/test/stores/test-collections.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,9 @@ test('constrain map key shape', t => {
t.is(stringsOnly.get('skey'), 'this should work');
t.throws(
() => stringsOnly.init(29, 'this should not work'),
m('invalid key type for collection "map key strings only"'),
m(
'invalid key type for collection "map key strings only": number 29 - Must be a string',
),
);

const noStrings = makeScalarBigMapStore('map key no strings', {
Expand All @@ -184,27 +186,31 @@ test('constrain map key shape', t => {
noStrings.init(true, 'boolean ok');
t.throws(
() => noStrings.init('foo', 'string not ok?'),
m('invalid key type for collection "map key no strings"'),
m(
'invalid key type for collection "map key no strings": "foo" - Must fail negated pattern: "[match:string]"',
),
);
t.is(noStrings.get(47), 'number ok');
t.is(noStrings.get(true), 'boolean ok');
t.falsy(noStrings.has('foo'));
t.throws(
() => noStrings.get('foo'),
m('invalid key type for collection "map key no strings"'),
m(
'invalid key type for collection "map key no strings": "foo" - Must fail negated pattern: "[match:string]"',
),
);

const only47 = makeScalarBigMapStore('map key only 47', { keyShape: 47 });
only47.init(47, 'this number ok');
t.throws(
() => only47.init(29, 'this number not ok?'),
m('invalid key type for collection "map key only 47"'),
m('invalid key type for collection "map key only 47": 29 - Must be: 47'),
);
t.is(only47.get(47), 'this number ok');
t.falsy(only47.has(29));
t.throws(
() => only47.get(29),
m('invalid key type for collection "map key only 47"'),
m('invalid key type for collection "map key only 47": 29 - Must be: 47'),
);

const lt47 = makeScalarBigMapStore('map key less than 47', {
Expand All @@ -213,13 +219,17 @@ test('constrain map key shape', t => {
lt47.init(29, 'this number ok');
t.throws(
() => lt47.init(53, 'this number not ok?'),
m('invalid key type for collection "map key less than 47"'),
m(
'invalid key type for collection "map key less than 47": 53 - Must be < 47',
),
);
t.is(lt47.get(29), 'this number ok');
t.falsy(lt47.has(53));
t.throws(
() => lt47.get(53),
m('invalid key type for collection "map key less than 47"'),
m(
'invalid key type for collection "map key less than 47": 53 - Must be < 47',
),
);
lt47.init(11, 'lower value');
lt47.init(46, 'higher value');
Expand All @@ -235,7 +245,9 @@ test('constrain map value shape', t => {
t.is(stringsOnly.get('sval'), 'string value');
t.throws(
() => stringsOnly.init('nval', 29),
m('invalid value type for collection "map value strings only"'),
m(
'invalid value type for collection "map value strings only": number 29 - Must be a string',
),
);

const noStrings = makeScalarBigMapStore('map value no strings', {
Expand All @@ -245,7 +257,9 @@ test('constrain map value shape', t => {
noStrings.init('bkey', true);
t.throws(
() => noStrings.init('skey', 'string not ok?'),
m('invalid value type for collection "map value no strings"'),
m(
'invalid value type for collection "map value no strings": "string not ok?" - Must fail negated pattern: "[match:string]"',
),
);
t.is(noStrings.get('nkey'), 47);
t.is(noStrings.get('bkey'), true);
Expand All @@ -257,7 +271,9 @@ test('constrain map value shape', t => {
only47.init('47key', 47);
t.throws(
() => only47.init('29key', 29),
m('invalid value type for collection "map value only 47"'),
m(
'invalid value type for collection "map value only 47": 29 - Must be: 47',
),
);
t.is(only47.get('47key'), 47);
t.falsy(only47.has('29key'));
Expand All @@ -268,7 +284,9 @@ test('constrain map value shape', t => {
lt47.init('29key', 29);
t.throws(
() => lt47.init('53key', 53),
m('invalid value type for collection "map value less than 47"'),
m(
'invalid value type for collection "map value less than 47": 53 - Must be < 47',
),
);
t.is(lt47.get('29key'), 29);
t.falsy(lt47.has('53key'));
Expand All @@ -288,7 +306,9 @@ test('constrain set key shape', t => {
t.truthy(stringsOnly.has('skey'));
t.throws(
() => stringsOnly.add(29),
m('invalid key type for collection "strings only set"'),
m(
'invalid key type for collection "strings only set": number 29 - Must be a string',
),
);

const noStrings = makeScalarBigSetStore('no strings set', {
Expand All @@ -298,7 +318,9 @@ test('constrain set key shape', t => {
noStrings.add(true);
t.throws(
() => noStrings.add('foo?'),
m('invalid key type for collection "no strings set"'),
m(
'invalid key type for collection "no strings set": "foo?" - Must fail negated pattern: "[match:string]"',
),
);
t.truthy(noStrings.has(47));
t.truthy(noStrings.has(true));
Expand All @@ -311,7 +333,7 @@ test('constrain set key shape', t => {
t.falsy(only47.has(29));
t.throws(
() => only47.add(29),
m('invalid key type for collection "only 47 set"'),
m('invalid key type for collection "only 47 set": 29 - Must be: 47'),
);

const lt47 = makeScalarBigSetStore('less than 47 set', {
Expand All @@ -320,7 +342,7 @@ test('constrain set key shape', t => {
lt47.add(29);
t.throws(
() => lt47.add(53),
m('invalid key type for collection "less than 47 set"'),
m('invalid key type for collection "less than 47 set": 53 - Must be < 47'),
);
t.truthy(lt47.has(29));
t.falsy(lt47.has(53));
Expand Down
2 changes: 2 additions & 0 deletions packages/store/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ export {
fit,
} from './patterns/patternMatchers.js';

export { compress, mustCompress, decompress } from './patterns/compress.js';

export {
defendPrototype,
initEmpty,
Expand Down
Loading

0 comments on commit 4192893

Please sign in to comment.