fix: pattern based compression

packages/ERTP/src/paymentLedger.js

@@ -139,6 +139,7 @@ export const vivifyPaymentLedger = (
   const paymentLedger = provideDurableWeakMapStore(
     issuerBaggage,
     'paymentLedger',
+    { valueShape: amountShape },
   );
 
   /**

packages/ERTP/src/purse.js

@@ -1,3 +1,4 @@
+import { M } from '@agoric/store';
 import { vivifyFarClassKit, makeScalarBigSetStore } from '@agoric/vat-data';
 import { AmountMath } from './amountMath.js';
 import { makeTransientNotifierKit } from './transientNotifier.js';
@@ -12,6 +13,8 @@ export const vivifyPurseKind = (
   PurseIKit,
   purseMethods,
 ) => {
+  const amountShape = brand.getAmountShape();
+
   // Note: Virtual for high cardinality, but *not* durable, and so
   // broken across an upgrade.
   const { provideNotifier, update: updateBalance } = makeTransientNotifierKit();
@@ -111,6 +114,12 @@ export const vivifyPurseKind = (
         },
       },
     },
+    {
+      stateShape: {
+        currentBalance: amountShape,
+        recoverySet: M.remotable('recoverySet'),
+      },
+    },
   );
   return () => makePurseKit().purse;
 };

packages/SwingSet/src/liveslots/collectionManager.js

@@ -5,6 +5,7 @@ import {
   assertKeyPattern,
   assertPattern,
   matches,
+  fit,
   compareRank,
   M,
   zeroPad,
@@ -13,6 +14,8 @@ import {
   isEncodedRemotable,
   makeCopySet,
   makeCopyMap,
+  mustCompress,
+  decompress,
 } from '@agoric/store';
 import { Far, passStyleOf } from '@endo/marshal';
 import { decodeToJustin } from '@endo/marshal/src/marshal-justin.js';
@@ -213,7 +216,7 @@ export function makeCollectionManager(
     return storeKindInfo[kindName].kindID;
   }
 
-  // Not that it's only used for this purpose, what should it be called?
+  // Now that it's only used for this purpose, what should it be called?
   // TODO Should we be using the new encodeBigInt scheme instead, anyway?
   const BIGINT_TAG_LEN = 10;
 
@@ -257,6 +260,23 @@ export function makeCollectionManager(
     const dbKeyPrefix = `vc.${collectionID}.`;
     let currentGenerationNumber = 0;
 
+    const keyLabel = `invalid key type for collection ${q(label)}`;
+    const valueLabel = `invalid value type for collection ${q(label)}`;
+
+    const serializeValue = value => {
+      if (valueShape === undefined) {
+        return serialize(value);
+      }
+      return serialize(mustCompress(value, valueShape, valueLabel));
+    };
+
+    const unserializeValue = data => {
+      if (valueShape === undefined) {
+        return unserialize(data);
+      }
+      return decompress(unserialize(data), valueShape);
+    };
+
     function prefix(dbEntryKey) {
       return `${dbKeyPrefix}${dbEntryKey}`;
     }
@@ -331,11 +351,10 @@ export function makeCollectionManager(
     }
 
     function get(key) {
-      matches(key, keyShape) ||
-        assert.fail(X`invalid key type for collection ${q(label)}`);
+      fit(key, keyShape, keyLabel);
       const result = syscall.vatstoreGet(keyToDBKey(key));
       if (result) {
-        return unserialize(JSON.parse(result));
+        return unserializeValue(JSON.parse(result));
       }
       assert.fail(X`key ${key} not found in collection ${q(label)}`);
     }
@@ -351,16 +370,11 @@ export function makeCollectionManager(
     }
 
     function init(key, value) {
-      matches(key, keyShape) ||
-        assert.fail(X`invalid key type for collection ${q(label)}`);
+      fit(key, keyShape, keyLabel);
       !has(key) ||
         assert.fail(X`key ${key} already registered in collection ${q(label)}`);
-      if (valueShape) {
-        matches(value, valueShape) ||
-          assert.fail(X`invalid value type for collection ${q(label)}`);
-      }
+      const serializedValue = serializeValue(value);
       currentGenerationNumber += 1;
-      const serializedValue = serialize(value);
       assertAcceptableSyscallCapdataSize([serializedValue]);
       if (durable) {
         serializedValue.slots.forEach((vref, slotIndex) => {
@@ -388,13 +402,8 @@ export function makeCollectionManager(
     }
 
     function set(key, value) {
-      matches(key, keyShape) ||
-        assert.fail(X`invalid key type for collection ${q(label)}`);
-      if (valueShape) {
-        matches(value, valueShape) ||
-          assert.fail(X`invalid value type for collection ${q(label)}`);
-      }
-      const after = serialize(harden(value));
+      fit(key, keyShape, keyLabel);
+      const after = serializeValue(harden(value));
       assertAcceptableSyscallCapdataSize([after]);
       if (durable) {
         after.slots.forEach((vref, i) => {
@@ -412,8 +421,7 @@ export function makeCollectionManager(
     }
 
     function deleteInternal(key) {
-      matches(key, keyShape) ||
-        assert.fail(X`invalid key type for collection ${q(label)}`);
+      fit(key, keyShape, keyLabel);
       const dbKey = keyToDBKey(key);
       const rawValue = syscall.vatstoreGet(dbKey);
       assert(rawValue, X`key ${key} not found in collection ${q(label)}`);
@@ -472,7 +480,7 @@ export function makeCollectionManager(
           if (dbKey < end) {
             priorDBKey = dbKey;
             if (ignoreKeys) {
-              const value = unserialize(JSON.parse(dbValue));
+              const value = unserializeValue(JSON.parse(dbValue));
               if (matches(value, valuePatt)) {
                 yield [undefined, value];
               }
@@ -484,7 +492,7 @@ export function makeCollectionManager(
             } else {
               const key = dbKeyToKey(dbKey);
               if (matches(key, keyPatt)) {
-                const value = unserialize(JSON.parse(dbValue));
+                const value = unserializeValue(JSON.parse(dbValue));
                 if (matches(value, valuePatt)) {
                   yield [key, value];
                 }

packages/SwingSet/src/liveslots/virtualObjectManager.js

@@ -1,13 +1,20 @@
 // @ts-check
-/* eslint-disable no-use-before-define, jsdoc/require-returns-type */
+/* eslint-disable no-use-before-define */
 
-import { assert, details as X, q } from '@agoric/assert';
-import { defendPrototype } from '@agoric/store';
-import { Far } from '@endo/marshal';
+import {
+  assertPattern,
+  decompress,
+  defendPrototype,
+  mustCompress,
+} from '@agoric/store';
+import { Far, hasOwnPropertyOf, passStyleOf } from '@endo/marshal';
 import { parseVatSlot } from '../lib/parseVatSlots.js';
 
 /** @template T @typedef {import('@agoric/vat-data').DefineKindOptions<T>} DefineKindOptions */
 
+const { ownKeys } = Reflect;
+const { details: X, quote: q } = assert;
+
 // import { kdebug } from './kdebug.js';
 
 // Marker associated to flag objects that should be held onto strongly if
@@ -28,7 +35,7 @@ const unweakable = new WeakSet();
  * @param {(baseRef: string, rawState: object) => void} store  Function to
  *   store raw object state by its baseRef
  *
- * @returns An LRU cache of (up to) the given size
+ * @returns {object} An LRU cache of (up to) the given size
  *
  * This cache is part of the virtual object manager and is not intended to be
  * used independently; it is exported only for the benefit of test code.
@@ -146,24 +153,28 @@ export function makeCache(size, fetch, store) {
 /**
  * Create a new virtual object manager.  There is one of these for each vat.
  *
- * @param {*} syscall  Vat's syscall object, used to access the vatstore operations.
- * @param {*} vrm  Virtual reference manager, to handle reference counting and GC
- *   of virtual references.
- * @param {() => number} allocateExportID  Function to allocate the next object
- *   export ID for the enclosing vat.
- * @param {(val: object) => string} _getSlotForVal  A function that returns the
- *   object ID (vref) for a given object, if any.  their corresponding export
- *   IDs
- * @param {*} registerValue  Function to register a new slot+value in liveSlot's
- *   various tables
- * @param {import('@endo/marshal').Serialize<unknown>} serialize  Serializer for this vat
- * @param {import('@endo/marshal').Unserialize<unknown>} unserialize  Unserializer for this vat
- * @param {number} cacheSize  How many virtual objects this manager should cache
- *   in memory.
- * @param {*} assertAcceptableSyscallCapdataSize  Function to check for oversized
- *   syscall params
+ * @param {*} syscall
+ * Vat's syscall object, used to access the vatstore operations.
+ * @param {*} vrm
+ * Virtual reference manager, to handle reference counting and GC
+ * of virtual references.
+ * @param {() => number} allocateExportID
+ * Function to allocate the next object export ID for the enclosing vat.
+ * @param {(val: object) => string} _getSlotForVal
+ * A function that returns the object ID (vref) for a given object, if any.
+ * their corresponding export IDs
+ * @param {*} registerValue
+ * Function to register a new slot+value in liveSlot's various tables
+ * @param {import('@endo/marshal').Serialize<unknown>} serialize
+ * Serializer for this vat
+ * @param {import('@endo/marshal').Unserialize<unknown>} unserialize
+ * Unserializer for this vat
+ * @param {number} cacheSize
+ * How many virtual objects this manager should cache in memory.
+ * @param {*} assertAcceptableSyscallCapdataSize
+ * Function to check for oversized syscall params
  *
- * @returns a new virtual object manager.
+ * @returns {object} a new virtual object manager.
  *
  * The virtual object manager allows the creation of persistent objects that do
  * not need to occupy memory when they are not in use.  It provides five
@@ -585,13 +596,46 @@ export function makeVirtualObjectManager(
   ) {
     const {
       finish,
+      stateShape = undefined,
       thisfulMethods = false,
       interfaceGuard = undefined,
     } = options;
     let facetNames;
     let contextMapTemplate;
     let prototypeTemplate;
 
+    harden(stateShape);
+    stateShape === undefined ||
+      passStyleOf(stateShape) === 'copyRecord' ||
+      assert.fail(X`A stateShape must be a copyRecord: ${q(stateShape)}`);
+    assertPattern(stateShape);
+
+    const serializeSlot = (slotState, prop) => {
+      if (stateShape === undefined) {
+        return serialize(slotState);
+      }
+      hasOwnPropertyOf(stateShape, prop) ||
+        assert.fail(
+          X`State must only have fields described by stateShape: ${q(
+            ownKeys(stateShape),
+          )}`,
+        );
+      return serialize(mustCompress(slotState, stateShape[prop], prop));
+    };
+
+    const unserializeSlot = (slotData, prop) => {
+      if (stateShape === undefined) {
+        return unserialize(slotData);
+      }
+      hasOwnPropertyOf(stateShape, prop) ||
+        assert.fail(
+          X`State only has fields described by stateShape: ${q(
+            ownKeys(stateShape),
+          )}`,
+        );
+      return decompress(unserialize(slotData), stateShape[prop]);
+    };
+
     const facetiousness = assessFacetiousness(behavior);
     switch (facetiousness) {
       case 'one': {
@@ -695,12 +739,12 @@ export function makeVirtualObjectManager(
         Object.defineProperty(state, prop, {
           get: () => {
             ensureState();
-            return unserialize(innerSelf.rawState[prop]);
+            return unserializeSlot(innerSelf.rawState[prop], prop);
           },
           set: value => {
             ensureState();
             const before = innerSelf.rawState[prop];
-            const after = serialize(value);
+            const after = serializeSlot(value, prop);
             assertAcceptableSyscallCapdataSize([after]);
             if (isDurable) {
               after.slots.forEach((vref, index) => {
@@ -793,11 +837,12 @@ export function makeVirtualObjectManager(
       const initialData = init ? init(...args) : {};
       const rawState = {};
       for (const prop of Object.getOwnPropertyNames(initialData)) {
-        const data = serialize(initialData[prop]);
+        const data = serializeSlot(initialData[prop], prop);
         assertAcceptableSyscallCapdataSize([data]);
         if (isDurable) {
           data.slots.forEach(vref => {
-            assert(vrm.isDurable(vref), X`value for ${q(prop)} is not durable`);
+            vrm.isDurable(vref) ||
+              assert.fail(X`value for ${q(prop)} is not durable`);
           });
         }
         data.slots.forEach(vrm.addReachableVref);