feat(swingset): hash kernel state changes into 'crankhash'

The multiple members of a consensus machines are supposed to perform
identical exection of every crank. To detect any possible divergence as
quickly as possible, the kernel maintains the "crankhash": a
constantly-updated string which incorporates (by SHA256 hash) a copy of every
DB write and delete.

`controller.getCrankHash()` can be run after one or more cranks have
finished, and the resulting hex string can be e.g. stored in a host
application consensus state vector. If two members diverge in a way that
causes their swingset state to differ, they will have different crankhashes,
and the consensus state vectors will diverge. This should cause at least one
of them to fall out of consensus.

Some keys are excluded from consensus: currently just those involving vat
snapshots and the truncation (non-initial starting point) of the transcript.

refs #3442

packages/SwingSet/src/controller.js

@@ -12,6 +12,7 @@ import { assert, details as X } from '@agoric/assert';
 import { importBundle } from '@agoric/import-bundle';
 import { xsnap, recordXSnap } from '@agoric/xsnap';
 
+import { createSHA256 } from './hasher.js';
 import engineGC from './engine-gc.js';
 import { WeakRef, FinalizationRegistry } from './weakref.js';
 import { startSubprocessWorker } from './spawnSubprocessWorker.js';
@@ -271,6 +272,7 @@ export async function makeSwingsetController(
     WeakRef,
     FinalizationRegistry,
     gcAndFinalize: makeGcAndFinalize(engineGC),
+    createSHA256,
   };
 
   const kernelOptions = { verbose, warehousePolicy, overrideVatManagerOptions };
@@ -327,6 +329,10 @@ export async function makeSwingsetController(
       return defensiveCopy(kernel.getStatus());
     },
 
+    getCrankHash() {
+      return kernel.getCrankHash();
+    },
+
     pinVatRoot(vatName) {
       const vatID = kernel.vatNameToID(vatName);
       const kref = kernel.getRootObject(vatID);

packages/SwingSet/src/hasher.js

@@ -0,0 +1,32 @@
+import { assert } from '@agoric/assert';
+
+import { createHash } from 'crypto';
+
+/**
+ * @typedef { (initial: string?) => {
+ *             add: (more: string) => void,
+ *             finish: () => string,
+ *            }
+ *          } CreateSHA256
+ */
+
+/** @type { CreateSHA256 } */
+function createSHA256(initial = undefined) {
+  const hash = createHash('sha256');
+  let done = false;
+  function add(more) {
+    assert(!done);
+    hash.update(more);
+  }
+  function finish() {
+    assert(!done);
+    done = true;
+    return hash.digest('hex');
+  }
+  if (initial) {
+    add(initial);
+  }
+  return harden({ add, finish });
+}
+harden(createSHA256);
+export { createSHA256 };

packages/SwingSet/src/kernel/initializeKernel.js

@@ -2,6 +2,7 @@
 
 import { makeMarshal, Far } from '@agoric/marshal';
 import { assert, details as X } from '@agoric/assert';
+import { createSHA256 } from '../hasher.js';
 import { assertKnownOptions } from '../assertOptions.js';
 import { insistVatID } from './id.js';
 import { makeVatSlot } from '../parseVatSlots.js';
@@ -17,7 +18,8 @@ export function initializeKernel(config, hostStorage, verbose = false) {
   const logStartup = verbose ? console.debug : () => 0;
   insistStorageAPI(hostStorage.kvStore);
 
-  const kernelKeeper = makeKernelKeeper(hostStorage);
+  const kernelSlog = null;
+  const kernelKeeper = makeKernelKeeper(hostStorage, kernelSlog, createSHA256);
 
   const wasInitialized = kernelKeeper.getInitialized();
   assert(!wasInitialized);

packages/SwingSet/src/kernel/kernel.js

@@ -126,6 +126,7 @@ export default function buildKernel(
     WeakRef,
     FinalizationRegistry,
     gcAndFinalize,
+    createSHA256,
   } = kernelEndowments;
   deviceEndowments = { ...deviceEndowments }; // copy so we can modify
   const {
@@ -142,7 +143,7 @@ export default function buildKernel(
     ? makeSlogger(slogCallbacks, writeSlogObject)
     : makeDummySlogger(slogCallbacks, makeConsole);
 
-  const kernelKeeper = makeKernelKeeper(hostStorage, kernelSlog);
+  const kernelKeeper = makeKernelKeeper(hostStorage, kernelSlog, createSHA256);
 
   let started = false;
 
@@ -1206,6 +1207,10 @@ export default function buildKernel(
       });
     },
 
+    getCrankHash() {
+      return kernelKeeper.getCrankHash();
+    },
+
     dump() {
       // note: dump().log is not deterministic, since log() does not go
       // through the syscall interface (and we replay transcripts one vat at

packages/SwingSet/src/kernel/state/kernelKeeper.js

@@ -131,18 +131,43 @@ const FIRST_METER_ID = 1n;
 /**
  * @param {HostStore} hostStorage
  * @param {KernelSlog} kernelSlog
+ * @param {CreateSHA256} createSHA256
  */
-export default function makeKernelKeeper(hostStorage, kernelSlog) {
+export default function makeKernelKeeper(
+  hostStorage,
+  kernelSlog,
+  createSHA256,
+) {
   // the kernelKeeper wraps the host's raw key-value store in a crank buffer
   const rawKVStore = hostStorage.kvStore;
   insistStorageAPI(rawKVStore);
 
+  /**
+   * @param { string } key
+   * @returns { boolean }
+   */
+  function isConsensusKey(key) {
+    if (/^v\d+\.lastSnapshot$/.test(key)) {
+      return false;
+    }
+    if (key.startsWith('snapshot.')) {
+      return false;
+    }
+    return true;
+  }
+
   const { abortCrank, commitCrank, enhancedCrankBuffer: kvStore } = wrapStorage(
     rawKVStore,
+    createSHA256,
+    isConsensusKey,
   );
   insistEnhancedStorageAPI(kvStore);
   const { streamStore, snapStore } = hostStorage;
 
+  function getCrankHash() {
+    return rawKVStore.get('crankhash');
+  }
+
   /**
    * @param {string} key
    * @returns {string}
@@ -1337,6 +1362,7 @@ export default function makeKernelKeeper(hostStorage, kernelSlog) {
     kvStore,
     abortCrank,
     commitCrank,
+    getCrankHash,
 
     dump,
   });

packages/SwingSet/src/kernel/state/storageWrapper.js

@@ -18,19 +18,29 @@ import { insistStorageAPI } from '../../storageAPI.js';
  * that buffers any mutations until told to commit them.
  *
  * @param {*} kvStore  The storage object that this crank buffer will be based on.
- *
+ * @param {CreateSHA256}  createSHA256
+ * @param { (key: string) => bool } isConsensusKey
  * @returns {*} an object {
- *   crankBuffer,  // crank buffer as described, wrapping `kvStore`
- *   commitCrank,  // function to save buffered mutations to `kvStore`
- *   abortCrank,   // function to discard buffered mutations
+ * crankBuffer,  // crank buffer as described, wrapping `kvStore`
+ * commitCrank,  // function to save buffered mutations to `kvStore`
+ * abortCrank,   // function to discard buffered mutations
  * }
  */
-export function buildCrankBuffer(kvStore) {
+export function buildCrankBuffer(
+  kvStore,
+  createSHA256,
+  isConsensusKey = () => true,
+) {
   insistStorageAPI(kvStore);
+  let crankhasher;
+  function resetCrankHash() {
+    crankhasher = createSHA256();
+  }
 
   // to avoid confusion, additions and deletions should never share a key
   const additions = new Map();
   const deletions = new Set();
+  resetCrankHash();
 
   const crankBuffer = {
     has(key) {
@@ -76,17 +86,32 @@ export function buildCrankBuffer(kvStore) {
       assert.typeof(value, 'string');
       additions.set(key, value);
       deletions.delete(key);
+      if (isConsensusKey(key)) {
+        crankhasher.add('add');
+        crankhasher.add('\n');
+        crankhasher.add(key);
+        crankhasher.add('\n');
+        crankhasher.add(value);
+        crankhasher.add('\n');
+      }
     },
 
     delete(key) {
       assert.typeof(key, 'string');
       additions.delete(key);
       deletions.add(key);
+      if (isConsensusKey(key)) {
+        crankhasher.add('delete');
+        crankhasher.add('\n');
+        crankhasher.add(key);
+        crankhasher.add('\n');
+      }
     },
   };
 
   /**
-   * Flush any buffered mutations to the underlying storage.
+   * Flush any buffered mutations to the underlying storage, and update the
+   * crankhash.
    */
   function commitCrank() {
     for (const [key, value] of additions) {
@@ -97,6 +122,22 @@ export function buildCrankBuffer(kvStore) {
     }
     additions.clear();
     deletions.clear();
+    const crankhash = crankhasher.finish();
+    resetCrankHash();
+
+    let oldCrankhash = kvStore.get('crankhash');
+    if (oldCrankhash === undefined) {
+      oldCrankhash = '';
+    }
+    const hasher = createSHA256('crankhash\n');
+    hasher.add(oldCrankhash);
+    hasher.add('\n');
+    hasher.add(crankhash);
+    hasher.add('\n');
+    const newCrankhash = hasher.finish();
+    kvStore.set('crankhash', newCrankhash);
+
+    return { crankhash, newCrankhash };
   }
 
   /**
@@ -105,6 +146,7 @@ export function buildCrankBuffer(kvStore) {
   function abortCrank() {
     additions.clear();
     deletions.clear();
+    resetCrankHash();
   }
 
   return harden({ crankBuffer, commitCrank, abortCrank });
@@ -166,9 +208,13 @@ export function addHelpers(kvStore) {
 // write-back buffer wrapper (the CrankBuffer), but the keeper is unaware of
 // that.
 
-export function wrapStorage(kvStore) {
+export function wrapStorage(kvStore, createSHA256, isConsensusKey) {
   insistStorageAPI(kvStore);
-  const { crankBuffer, commitCrank, abortCrank } = buildCrankBuffer(kvStore);
+  const { crankBuffer, commitCrank, abortCrank } = buildCrankBuffer(
+    kvStore,
+    createSHA256,
+    isConsensusKey,
+  );
   const enhancedCrankBuffer = addHelpers(crankBuffer);
   return { enhancedCrankBuffer, commitCrank, abortCrank };
 }

packages/SwingSet/test/test-clist.js

@@ -2,13 +2,14 @@ import { test } from '../tools/prepare-test-env-ava.js';
 
 // eslint-disable-next-line import/order
 import { initSimpleSwingStore } from '@agoric/swing-store-simple';
+import { createSHA256 } from '../src/hasher.js';
 import { makeDummySlogger } from '../src/kernel/slogger.js';
 import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js';
 
 test(`clist reachability`, async t => {
   const slog = makeDummySlogger({});
   const hostStorage = initSimpleSwingStore();
-  const kk = makeKernelKeeper(hostStorage, slog);
+  const kk = makeKernelKeeper(hostStorage, slog, createSHA256);
   const s = kk.kvStore;
   kk.createStartingKernelState('local');
   const vatID = kk.allocateUnusedVatID();
@@ -93,7 +94,7 @@ test(`clist reachability`, async t => {
 test('getImporters', async t => {
   const slog = makeDummySlogger({});
   const hostStorage = initSimpleSwingStore();
-  const kk = makeKernelKeeper(hostStorage, slog);
+  const kk = makeKernelKeeper(hostStorage, slog, createSHA256);
 
   kk.createStartingKernelState('local');
   const vatID1 = kk.allocateUnusedVatID();

packages/SwingSet/test/test-controller.js

@@ -86,6 +86,11 @@ async function simpleCall(t) {
 
   controller.log('2');
   t.is(controller.dump().log[1], '2');
+
+  // hash determined experimentally: will change if the initial kernel state
+  // ever changes
+  const h = '08b771e7bbf966c2b4dd98737c5e7e827fc4e688160557579f89649f87e4ec4f';
+  t.is(controller.getCrankHash(), h);
 }
 
 test('simple call', async t => {

packages/SwingSet/test/test-gc-kernel.js

@@ -4,6 +4,7 @@ import { test } from '../tools/prepare-test-env-ava.js';
 
 import { WeakRef, FinalizationRegistry } from '../src/weakref.js';
 import { waitUntilQuiescent } from '../src/waitUntilQuiescent.js';
+import { createSHA256 } from '../src/hasher.js';
 
 import buildKernel from '../src/kernel/index.js';
 import { initializeKernel } from '../src/kernel/initializeKernel.js';
@@ -51,6 +52,7 @@ function makeEndowments() {
     writeSlogObject,
     WeakRef,
     FinalizationRegistry,
+    createSHA256,
   };
 }
 

packages/SwingSet/test/test-hasher.js

@@ -0,0 +1,36 @@
+import { test } from '../tools/prepare-test-env-ava.js';
+
+// eslint-disable-next-line import/order
+import { createSHA256 } from '../src/hasher.js';
+
+test('createSHA256', t => {
+  t.is(
+    createSHA256().finish(),
+    'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+  );
+
+  const h1 = createSHA256('a');
+  t.is(
+    h1.finish(),
+    'ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb',
+  );
+
+  const h2 = createSHA256();
+  h2.add('a');
+  t.is(
+    h2.finish(),
+    'ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb',
+  );
+
+  const h3 = createSHA256('a');
+  h3.add('b');
+  t.is(
+    h3.finish(),
+    'fb8e20fc2e4c3f248c60c39bd652f3c1347298bb977b8b4d5903b85055620603',
+  );
+
+  const h4 = createSHA256();
+  h4.finish();
+  t.throws(h4.add);
+  t.throws(h4.finish);
+});

packages/SwingSet/test/test-kernel.js

@@ -5,6 +5,7 @@ import anylogger from 'anylogger';
 import { assert, details as X } from '@agoric/assert';
 import { WeakRef, FinalizationRegistry } from '../src/weakref.js';
 import { waitUntilQuiescent } from '../src/waitUntilQuiescent.js';
+import { createSHA256 } from '../src/hasher.js';
 
 import buildKernel from '../src/kernel/index.js';
 import { initializeKernel } from '../src/kernel/initializeKernel.js';
@@ -62,6 +63,7 @@ function makeEndowments() {
     makeConsole,
     WeakRef,
     FinalizationRegistry,
+    createSHA256,
   };
 }