Skip to content
This repository has been archived by the owner on Jan 19, 2021. It is now read-only.

Commit

Permalink
Extract db-related methods from baseTrie (#74)
Browse files Browse the repository at this point in the history
* Mv raw methods to DB class
* Add ScratchDB which CheckpointTrie uses
* Rename checkpoint-trie to checkpointTrie
* Add comments to scratch
* Fix linting errors
* Regenerate docs
* Rm raw methods and their tests
* Rename DB._db to DB._leveldb
* Make createScratchReadStream private
* Fix linting error and jsdoc comments
* Regenerate docs
* Add tests for db, scratch and more cases for checkpoint
* Fix checkpoint copy, add tests for checkpoint and secure copy

Signed-off-by: Sina Mahmoodi <[email protected]>
  • Loading branch information
s1na authored Jan 22, 2019
1 parent 029b5fe commit c315566
Show file tree
Hide file tree
Showing 14 changed files with 654 additions and 417 deletions.
338 changes: 208 additions & 130 deletions docs/index.md

Large diffs are not rendered by default.

104 changes: 13 additions & 91 deletions src/baseTrie.js
Original file line number Diff line number Diff line change
@@ -1,23 +1,25 @@
const assert = require('assert')
const level = require('level-mem')
const async = require('async')
const rlp = require('rlp')
const ethUtil = require('ethereumjs-util')
const semaphore = require('semaphore')
const DB = require('./db')
const TrieNode = require('./trieNode')
const ReadStream = require('./readStream')
const PrioritizedTaskExecutor = require('./prioritizedTaskExecutor')
const { callTogether, asyncFirstSeries } = require('./util/async')
const { callTogether } = require('./util/async')
const { stringToNibbles, matchingNibbleLength, doKeysMatch } = require('./util/nibbles')

/**
* Use `require('merkel-patricia-tree')` for the base interface. In Ethereum applications stick with the Secure Trie Overlay `require('merkel-patricia-tree/secure')`. The API for the raw and the secure interface are about the same
* Use `require('merkel-patricia-tree')` for the base interface. In Ethereum applications
* stick with the Secure Trie Overlay `require('merkel-patricia-tree/secure')`.
* The API for the raw and the secure interface are about the same
* @class Trie
* @public
* @param {Object} [db] An instance of [levelup](https://github.com/rvagg/node-levelup/) or a compatible API. If the db is `null` or left undefined, then the trie will be stored in memory via [memdown](https://github.com/rvagg/memdown)
* @param {Object} [db] An instance of `DB`.
* If the db is `null` or left undefined, then the trie will be stored in memory via [memdown](https://github.com/Level/memdown)
* @param {Buffer|String} [root] A hex `String` or `Buffer` for the root of a previously stored trie
* @prop {Buffer} root The current root of the `trie`
* @prop {Boolean} isCheckpoint determines if you are saving to a checkpoint or directly to the db
* @prop {Buffer} EMPTY_TRIE_ROOT the Root for an empty trie
*/
module.exports = class Trie {
Expand All @@ -26,11 +28,7 @@ module.exports = class Trie {
this.EMPTY_TRIE_ROOT = ethUtil.SHA3_RLP
this.sem = semaphore(1)

// setup dbs
this.db = db || level()

this._getDBs = [this.db]
this._putDBs = [this.db]
this.db = db || new DB()

Object.defineProperty(this, 'root', {
set(value) {
Expand All @@ -49,8 +47,6 @@ module.exports = class Trie {
})

this.root = root

this.putRaw = this._putRaw
}

/**
Expand Down Expand Up @@ -133,38 +129,12 @@ module.exports = class Trie {
})
}

/**
* Retrieves a raw value in the underlying db
* @method getRaw
* @memberof Trie
* @param {Buffer} key
* @param {Function} callback A callback `Function`, which is given the arguments `err` - for errors that may have occured and `value` - the found value in a `Buffer` or if no value was found `null`.
*/
getRaw (key, cb) {
key = ethUtil.toBuffer(key)

function dbGet (db, cb2) {
db.get(key, {
keyEncoding: 'binary',
valueEncoding: 'binary'
}, (err, foundNode) => {
if (err || !foundNode) {
cb2(null, null)
} else {
cb2(null, foundNode)
}
})
}

asyncFirstSeries(this._getDBs, dbGet, cb)
}

// retrieves a node from dbs by hash
_lookupNode (node, cb) {
if (TrieNode.isRawNode(node)) {
cb(new TrieNode(node))
} else {
this.getRaw(node, (err, value) => {
this.db.get(node, (err, value) => {
if (err) {
throw err
}
Expand All @@ -178,60 +148,11 @@ module.exports = class Trie {
}
}

/**
* Writes a value directly to the underlining db
* @method putRaw
* @memberof Trie
* @param {Buffer|String} key The key as a `Buffer` or `String`
* @param {Buffer} value The value to be stored
* @param {Function} callback A callback `Function`, which is given the argument `err` - for errors that may have occured
*/
// TODO: remove the proxy method when changing the caching
_putRaw (key, val, cb) {
function dbPut (db, cb2) {
db.put(key, val, {
keyEncoding: 'binary',
valueEncoding: 'binary'
}, cb2)
}

async.each(this._putDBs, dbPut, cb)
}

/**
* Removes a raw value in the underlying db
* @method delRaw
* @memberof Trie
* @param {Buffer|String} key
* @param {Function} callback A callback `Function`, which is given the argument `err` - for errors that may have occured
*/
delRaw (key, cb) {
function del (db, cb2) {
db.del(key, {
keyEncoding: 'binary'
}, cb2)
}

async.each(this._putDBs, del, cb)
}

// writes a single node to dbs
_putNode (node, cb) {
const hash = node.hash()
const serialized = node.serialize()
this._putRaw(hash, serialized, cb)
}

// writes many nodes to db
_batchNodes (opStack, cb) {
function dbBatch (db, cb) {
db.batch(opStack, {
keyEncoding: 'binary',
valueEncoding: 'binary'
}, cb)
}

async.each(this._putDBs, dbBatch, cb)
this.db.put(hash, serialized, cb)
}

/**
Expand Down Expand Up @@ -560,7 +481,7 @@ module.exports = class Trie {
this.root = lastRoot
}

this._batchNodes(opStack, cb)
this.db.batch(opStack, cb)
}

_deleteNode (key, stack, cb) {
Expand Down Expand Up @@ -732,7 +653,8 @@ module.exports = class Trie {
// creates a new trie backed by the same db
// and starting at the same root
copy () {
return new Trie(this.db, this.root)
const db = this.db.copy()
return new Trie(db, this.root)
}

/**
Expand Down
88 changes: 36 additions & 52 deletions src/checkpoint-trie.js → src/checkpointTrie.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
const async = require('async')
const level = require('level-mem')
const WriteStream = require('level-ws')
const BaseTrie = require('./baseTrie')
const proof = require('./proof.js')
const ScratchReadStream = require('./scratchReadStream')
const ScratchDB = require('./scratch')
const { callTogether } = require('./util/async')

module.exports = class CheckpointTrie extends BaseTrie {
constructor (...args) {
super(...args)
// Reference to main DB instance
this._mainDB = this.db
// DB instance used for checkpoints
this._scratch = null
// Roots of trie at the moment of checkpoint
this._checkpoints = []
}

Expand Down Expand Up @@ -71,7 +75,8 @@ module.exports = class CheckpointTrie extends BaseTrie {

/**
* Reverts the trie to the state it was at when `checkpoint` was first called.
* If during a nested checkpoint, only sets parent as current checkpoint.
* If during a nested checkpoint, sets root to most recent checkpoint, and sets
* parent checkpoint as current.
* @method revert
* @param {Function} cb the callback
*/
Expand All @@ -93,78 +98,57 @@ module.exports = class CheckpointTrie extends BaseTrie {

/**
* Returns a copy of the underlying trie with the interface
* of CheckpointTrie.
* of CheckpointTrie. If during a checkpoint, the copy will
* contain the checkpointing metadata (incl. reference to the same scratch).
* @method copy
*/
copy () {
const trie = new CheckpointTrie(this.db, this.root)
trie._scratch = this._scratch
// trie._checkpoints = this._checkpoints.slice()
return trie
}

/**
* Returns a `ScratchReadStream` based on the state updates
* since checkpoint.
* @method createScratchReadStream
*/
createScratchReadStream (scratch) {
const trie = this.copy()
scratch = scratch || this._scratch
// Only read from the scratch
trie._getDBs = [scratch]
trie._scratch = scratch
return new ScratchReadStream(trie)
}

/**
* Puts kv-pair directly to db, ignoring checkpoints.
* @private
*/
_overridePutRaw (key, val, cb) {
const dbPut = (db, cb2) => {
db.put(key, val, {
keyEncoding: 'binary',
valueEncoding: 'binary'
}, cb2)
const db = this._mainDB.copy()
const trie = new CheckpointTrie(db, this.root)
if (this.isCheckpoint) {
trie._checkpoints = this._checkpoints.slice()
trie._scratch = this._scratch.copy()
trie.db = trie._scratch
}
async.each(this.__putDBs, dbPut, cb)
return trie
}

/**
* Enter into checkpoint mode.
* @private
*/
_enterCpMode () {
this._scratch = level()
this._getDBs = [this._scratch].concat(this._getDBs)
this.__putDBs = this._putDBs
this._putDBs = [this._scratch]
this._putRaw = this.putRaw
this.putRaw = this._overridePutRaw
this._scratch = new ScratchDB(this._mainDB)
this.db = this._scratch
}

/**
* Exit from checkpoint mode.
* @private
*/
_exitCpMode (commitState, cb) {
var scratch = this._scratch
const scratch = this._scratch
this._scratch = null
this._getDBs = this._getDBs.slice(1)
this._putDBs = this.__putDBs
this.putRaw = this._putRaw

const flushScratch = (db, cb) => {
this.createScratchReadStream(scratch)
.pipe(WriteStream(db))
.on('close', cb)
}
this.db = this._mainDB

if (commitState) {
async.map(this._putDBs, flushScratch, cb)
this._createScratchReadStream(scratch)
.pipe(WriteStream(this.db))
.on('close', cb)
} else {
cb()
async.nextTick(cb)
}
}

/**
* Returns a `ScratchReadStream` based on the state updates
* since checkpoint.
* @method createScratchReadStream
* @private
*/
_createScratchReadStream (scratch) {
scratch = scratch || this._scratch
const trie = new BaseTrie(scratch, this.root)
return new ScratchReadStream(trie)
}
}
83 changes: 83 additions & 0 deletions src/db.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
const level = require('level-mem')

const ENCODING_OPTS = { keyEncoding: 'binary', valueEncoding: 'binary' }

/**
* DB is a thin wrapper around the underlying levelup db,
* which validates inputs and sets encoding type.
*/
module.exports = class DB {
/**
* Initialize a DB instance. If `leveldb` is not provided, DB
* defaults to an [in-memory store](https://github.com/Level/memdown).
* @param {Object} [leveldb] - An abstract-leveldown compliant store
*/
constructor (leveldb) {
this._leveldb = leveldb || level()
}

/**
* Retrieves a raw value from leveldb.
* @param {Buffer} key
* @param {Function} cb A callback `Function`, which is given the arguments
* `err` - for errors that may have occured
* and `value` - the found value in a `Buffer` or if no value was found `null`.
*/
get (key, cb) {
if (!Buffer.isBuffer(key)) throw new Error('Invalid input: expected buffer')

this._leveldb.get(key, ENCODING_OPTS, (err, v) => {
if (err || !v) {
cb(null, null)
} else {
cb(null, v)
}
})
}

/**
* Writes a value directly to leveldb.
* @param {Buffer} key The key as a `Buffer` or `String`
* @param {Buffer} value The value to be stored
* @param {Function} cb A callback `Function`, which is given the argument
* `err` - for errors that may have occured
*/
put (key, val, cb) {
if (!Buffer.isBuffer(key)) throw new Error('Invalid input: expected buffer')
if (!Buffer.isBuffer(val)) throw new Error('Invalid input: expected buffer')

this._leveldb.put(key, val, ENCODING_OPTS, cb)
}

/**
* Removes a raw value in the underlying leveldb.
* @param {Buffer} key
* @param {Function} cb A callback `Function`, which is given the argument
* `err` - for errors that may have occured
*/
del (key, cb) {
if (!Buffer.isBuffer(key)) throw new Error('Invalid input: expected buffer')

this._leveldb.del(key, ENCODING_OPTS, cb)
}

/**
* Performs a batch operation on db.
* @param {Array} opStack A stack of levelup operations
* @param {Function} cb A callback `Function`, which is given the argument
* `err` - for errors that may have occured
*/
batch (opStack, cb) {
if (!Array.isArray(opStack)) throw new Error('Invalid input: expected buffer')

this._leveldb.batch(opStack, ENCODING_OPTS, cb)
}

/**
* Returns a copy of the DB instance, with a reference
* to the **same** underlying leveldb instance.
*/
copy () {
return new DB(this._leveldb)
}
}
2 changes: 1 addition & 1 deletion src/index.js
Original file line number Diff line number Diff line change
@@ -1 +1 @@
module.exports = require('./checkpoint-trie')
module.exports = require('./checkpointTrie')
Loading

0 comments on commit c315566

Please sign in to comment.