Gracefully shut down node on critical errors like full disk

bin/node

@@ -50,6 +50,26 @@ process.on('SIGINT', async () => {
   await node.close();
 });
 
+node.on('abort', async (err) => {
+  const timeout = setTimeout(() => {
+    console.error('Shutdown is taking a long time. Exitting.');
+    process.exit(3);
+  }, 5000);
+
+  timeout.unref();
+
+  try {
+    console.error('Shutting down...');
+    await node.close();
+    clearTimeout(timeout);
+    console.error(err.stack);
+    process.exit(2);
+  } catch (e) {
+    console.error(`Error occurred during shutdown: ${e.message}`);
+    process.exit(3);
+  }
+});
+
 (async () => {
   await node.ensure();
   await node.open();

bin/spvnode

@@ -67,6 +67,26 @@ process.on('SIGINT', async () => {
   await node.close();
 });
 
+node.on('abort', async (err) => {
+  const timeout = setTimeout(() => {
+    console.error('Shutdown is taking a long time. Exitting.');
+    process.exit(3);
+  }, 5000);
+
+  timeout.unref();
+
+  try {
+    console.error('Shutting down...');
+    await node.close();
+    clearTimeout(timeout);
+    console.error(err.stack);
+    process.exit(2);
+  } catch (e) {
+    console.error(`Error occurred during shutdown: ${e.message}`);
+    process.exit(3);
+  }
+});
+
 (async () => {
   await node.ensure();
   await node.open();

lib/blockchain/chain.js

@@ -26,6 +26,7 @@ const Script = require('../script/script');
 const {VerifyError} = require('../protocol/errors');
 const {OwnershipProof} = require('../covenants/ownership');
 const AirdropProof = require('../primitives/airdropproof');
+const {CriticalError} = require('../errors');
 const thresholdStates = common.thresholdStates;
 const {states} = NameState;
 
@@ -996,8 +997,11 @@ class Chain extends AsyncEmitter {
       const ns = await view.getNameState(this.db, nameHash);
 
       if (ns.isNull()) {
-        if (!covenant.isClaim() && !covenant.isOpen())
-          throw new Error('Database inconsistency.');
+        if (!covenant.isClaim() && !covenant.isOpen()) {
+          const error = new CriticalError('Database inconsistency.');
+          this.emit('abort', error);
+          throw error;
+        }
 
         const name = covenant.get(2);
         ns.set(name, height);
@@ -1893,7 +1897,13 @@ class Chain extends AsyncEmitter {
     }
 
     // Save block and connect inputs.
-    await this.db.save(entry, block, view);
+    try {
+      await this.db.save(entry, block, view);
+    } catch (e) {
+      const error = new CriticalError(e.message);
+      this.emit('abort', error);
+      throw error;
+    }
 
     // Expose the new state.
     this.tip = entry;
@@ -1952,7 +1962,13 @@ class Chain extends AsyncEmitter {
         entry.height, util.hex32(entry.version));
     }
 
-    await this.db.save(entry, block);
+    try {
+      await this.db.save(entry, block);
+    } catch (e) {
+      const error = new CriticalError(e.message);
+      this.emit('abort', error);
+      throw error;
+    }
 
     this.logger.warning('Heads up: Competing chain at height %d:'
       + ' tip-height=%d competitor-height=%d'
@@ -2156,6 +2172,10 @@ class Chain extends AsyncEmitter {
       await this.db.compactTree(entry);
       await this.syncTree();
       this.emit('tree compact end', entry.treeRoot, entry);
+    } catch(e) {
+      const error = new CriticalError(e.message);
+      this.emit('abort', error);
+      throw error;
     } finally {
       unlock();
     }

lib/errors.js

@@ -0,0 +1,41 @@
+/*!
+ * errors.js - internal error objects for hsd
+ * Copyright (c) 2022 The Handshake Developers (MIT License).
+ * https://github.com/handshake-org/hsd
+ */
+
+'use strict';
+
+/**
+ * @module errors
+ */
+
+/**
+ * Critical Error
+ * An error severe enough to warrant shutting down the node.
+ * @extends Error
+ */
+
+class CriticalError extends Error {
+  /**
+   * Create a verify error.
+   * @constructor
+   * @param {String} msg
+   */
+
+  constructor(msg) {
+    super();
+
+    this.type = 'CriticalError';
+    this.message = `Critical Error: ${msg}`;
+
+    if (Error.captureStackTrace)
+      Error.captureStackTrace(this, CriticalError);
+  }
+}
+
+/*
+ * Expose
+ */
+
+exports.CriticalError = CriticalError;

lib/hsd.js

@@ -72,6 +72,9 @@ hsd.define('Rules', './covenants/rules');
 hsd.define('dns', './dns/server');
 hsd.define('resource', './dns/resource');
 
+// Errors
+hsd.define('errors', './errors');
+
 // HD
 hsd.define('hd', './hd');
 hsd.define('HDPrivateKey', './hd/private');

lib/node/fullnode.js

@@ -197,6 +197,8 @@ class FullNode extends Node {
   init() {
     // Bind to errors
     this.chain.on('error', err => this.error(err));
+    this.chain.on('abort', err => this.abort(err));
+
     this.mempool.on('error', err => this.error(err));
     this.pool.on('error', err => this.error(err));
     this.miner.on('error', err => this.error(err));
@@ -345,6 +347,7 @@ class FullNode extends Node {
     await this.handleClose();
 
     this.logger.info('Node is closed.');
+    this.emit('closed');
   }
 
   /**

lib/node/node.js

@@ -305,6 +305,17 @@ class Node extends EventEmitter {
     this.emit('error', err);
   }
 
+  /**
+   * Emit and log an abort error.
+   * @private
+   * @param {Error} err
+   */
+
+  abort(err) {
+    this.logger.error(err);
+    this.emit('abort', err);
+  }
+
   /**
    * Get node uptime in seconds.
    * @returns {Number}
@@ -361,8 +372,10 @@ class Node extends EventEmitter {
 
     this.stack.push(instance);
 
-    if (typeof instance.on === 'function')
+    if (typeof instance.on === 'function') {
       instance.on('error', err => this.error(err));
+      instance.on('abort', msg => this.abort(msg));
+    }
 
     return instance;
   }

lib/node/spvnode.js

@@ -130,6 +130,8 @@ class SPVNode extends Node {
   init() {
     // Bind to errors
     this.chain.on('error', err => this.error(err));
+    this.chain.on('abort', err => this.abort(err));
+
     this.pool.on('error', err => this.error(err));
 
     if (this.http)
@@ -214,6 +216,9 @@ class SPVNode extends Node {
     await this.pool.close();
     await this.chain.close();
     await this.handleClose();
+
+    this.logger.info('Node is closed.');
+    this.emit('closed');
   }
 
   /**

test/node-critical-error-test.js

@@ -0,0 +1,142 @@
+/* eslint-env mocha */
+
+'use strict';
+
+const assert = require('bsert');
+const FullNode = require('../lib/node/fullnode');
+const {rimraf, testdir} = require('./util/common');
+
+describe('Node Critical Error', function() {
+  this.timeout(30000);
+
+  let prefix, node;
+
+  beforeEach(async () => {
+    prefix = testdir('hsd-critical-error-test');
+    node = new FullNode({
+      memory: false,
+      network: 'regtest',
+      prefix
+    });
+    await node.ensure();
+    await node.open();
+  });
+
+  afterEach(async () => {
+    if (node && node.opened)
+      await node.close();
+    await rimraf(prefix);
+  });
+
+  async function mineBlocks(node, count) {
+    for (let i = 0; i < count; i++) {
+      if (!node || !node.opened)
+        break;
+
+      const block = await node.miner.mineBlock(
+        null,
+        'rs1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqn6kda'
+      );
+
+      try {
+        // We are catching this error in the test but normally
+        // it would bubble up all the way from blockstore to Peer,
+        // where it is caught and logged, then we disconnect the peer
+        // that sent us whatever data caused the error (even if it's our fault!)
+        //
+        // [error] (net) Could not write block.
+        //   at FileBlockStore._write (hsd/lib/blockstore/file.js:424:13)
+        //   at async FileBatch.write (hsd/lib/blockstore/file.js:755:11)
+        //   at async ChainDB.commit (hsd/lib/blockchain/chaindb.js:332:7)
+        //   at async ChainDB.save (hsd/lib/blockchain/chaindb.js:1531:5)
+        //   at async Chain.setBestChain (hsd/lib/blockchain/chain.js:1835:5)
+        //   at async Chain.connect (hsd/lib/blockchain/chain.js:2236:7)
+        //   at async Chain._add (hsd/lib/blockchain/chain.js:2145:19)
+        //   at async Chain.add (hsd/lib/blockchain/chain.js:2073:14)
+        //   at async Pool._addBlock (hsd/lib/net/pool.js:2459:15)
+        //   at async Pool.addBlock (hsd/lib/net/pool.js:2426:14)
+        //   at async Pool.handleBlock (hsd/lib/net/pool.js:2410:5)
+        //   at async Pool.handlePacket (hsd/lib/net/pool.js:1331:9)
+        //   at async Peer.handlePacket (hsd/lib/net/peer.js:1549:7)
+        //   at async Peer.readPacket (hsd/lib/net/peer.js:1486:11)
+        //   at async Parser.<anonymous> (hsd/lib/net/peer.js:185:9)
+        await node.chain.add(block);
+      } catch (e) {
+        assert.strictEqual(e.message, 'Critical Error: Disk full!');
+        assert.strictEqual(e.type, 'CriticalError');
+        break;
+      }
+    }
+  }
+
+  it('should not run out of disk space', async () => {
+    await mineBlocks(node, 100);
+    assert.strictEqual(node.chain.height, 100);
+    assert.strictEqual(node.opened, true);
+    assert.strictEqual(node.chain.opened, true);
+    assert.strictEqual(node.chain.db.db.loaded, true);
+    assert.strictEqual(node.chain.db.blocks.db.loaded, true);
+    await node.close();
+    assert.strictEqual(node.opened, false);
+    assert.strictEqual(node.chain.opened, false);
+    assert.strictEqual(node.chain.db.db.loaded, false);
+    assert.strictEqual(node.chain.db.blocks.db.loaded, false);
+  });
+
+  it('should run out of disk space on block write and abort', async () => {
+    const waiter = new Promise((resolve) => {
+      node.once('closed', () => resolve());
+    });
+
+    node.on('abort', async () => {
+      try {
+        await node.close();
+      } catch (e) {
+        ;
+      }
+    });
+
+    await mineBlocks(node, 99);
+    node.chain.db.db.batch = () => {
+      return {
+        clear: () => {},
+        put: () => {},
+        del: () => {},
+        write: () => {
+          throw new Error('Disk full!');
+        }
+      };
+    };
+    await mineBlocks(node, 1);
+    await waiter;
+    assert.strictEqual(node.opened, false);
+    assert.strictEqual(node.chain.opened, false);
+    assert.strictEqual(node.chain.db.db.loaded, false);
+    assert.strictEqual(node.chain.db.blocks.db.loaded, false);
+  });
+
+  it('should run out of disk space on tree commit and abort', async () => {
+    const waiter = new Promise((resolve) => {
+      node.once('closed', () => resolve());
+    });
+
+    node.on('abort', async () => {
+      try {
+        await node.close();
+      } catch (e) {
+        ;
+      }
+    });
+
+    await mineBlocks(node, 50);
+    node.chain.db.tree.store.commit = () => {
+      throw new Error('Disk full!');
+    };
+    await mineBlocks(node, 50);
+    await waiter;
+    assert.strictEqual(node.opened, false);
+    assert.strictEqual(node.chain.opened, false);
+    assert.strictEqual(node.chain.db.db.loaded, false);
+    assert.strictEqual(node.chain.db.blocks.db.loaded, false);
+  });
+});