rename methods for clarity

update docs
update changelog

rebuild docs

CHANGELOG.md

@@ -6,11 +6,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 (modification: no type change headlines) and this project adheres to
 [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [4.1.0] - 2020-12-03
+## UNRELEASED
 
 ## New features
 
-There is a new exported `WalkController` object. Also, `trie.walkTrie` is now a public method. Using the `WalkController`, one can create their own custom ways to traverse the trie.
+There is a new exported `WalkController` class. Also, `trie.walkTrie()` is now a public method. Using the `WalkController`, one can create own custom ways to traverse the trie. [#135](https://github.com/ethereumjs/merkle-patricia-tree/pull/135)
 
 ## [4.0.0] - 2020-04-17
 

docs/classes/_util_walkcontroller_.walkcontroller.md

@@ -20,7 +20,7 @@ WalkController is an interface to control how the trie is being traversed.
 
 * [allChildren](_util_walkcontroller_.walkcontroller.md#allchildren)
 * [onlyBranchIndex](_util_walkcontroller_.walkcontroller.md#onlybranchindex)
-* [pushNode](_util_walkcontroller_.walkcontroller.md#pushnode)
+* [pushNodeToQueue](_util_walkcontroller_.walkcontroller.md#pushnodetoqueue)
 * [newWalk](_util_walkcontroller_.walkcontroller.md#static-newwalk)
 
 ## Properties
@@ -53,15 +53,15 @@ ___
 
 ▸ **allChildren**(`node`: TrieNode, `key`: Nibbles): *void*
 
-*Defined in [util/walkController.ts:63](https://github.com/ethereumjs/merkle-patricia-tree/blob/master/src/util/walkController.ts#L63)*
+*Defined in [util/walkController.ts:59](https://github.com/ethereumjs/merkle-patricia-tree/blob/master/src/util/walkController.ts#L59)*
 
-Run all children of a node. Priority of these nodes are the key length of the children
+Run all children of a node. Priority of these nodes are the key length of the children.
 
 **Parameters:**
 
 Name | Type | Default | Description |
 ------ | ------ | ------ | ------ |
-`node` | TrieNode | - | Node to get all children of and call onNode on |
+`node` | TrieNode | - | Node to get all children of and call onNode on. |
 `key` | Nibbles | [] | The current `key` which would yield the `node` when trying to get this node with a `get` operation.  |
 
 **Returns:** *void*
@@ -70,30 +70,32 @@ ___
 
 ###  onlyBranchIndex
 
-▸ **onlyBranchIndex**(`node`: BranchNode, `key`: Nibbles, `childIndex`: number, `priority?`: undefined | number): *Promise‹void›*
+▸ **onlyBranchIndex**(`node`: BranchNode, `key`: Nibbles, `childIndex`: number, `priority?`: undefined | number): *void*
 
-*Defined in [util/walkController.ts:106](https://github.com/ethereumjs/merkle-patricia-tree/blob/master/src/util/walkController.ts#L106)*
+*Defined in [util/walkController.ts:105](https://github.com/ethereumjs/merkle-patricia-tree/blob/master/src/util/walkController.ts#L105)*
 
-Push a branch of a certain BranchNode to the event queue
+Push a branch of a certain BranchNode to the event queue.
 
 **Parameters:**
 
 Name | Type | Default | Description |
 ------ | ------ | ------ | ------ |
-`node` | BranchNode | - | The node to select a branch on. Should be a BranchNode |
-`key` | Nibbles | [] | The current key which leads to the corresponding node |
-`childIndex` | number | - | The child index to add to the event queue |
-`priority?` | undefined | number | - | Optional priority of the event, defaults to the total key length  |
+`node` | BranchNode | - | The node to select a branch on. Should be a BranchNode. |
+`key` | Nibbles | [] | The current key which leads to the corresponding node. |
+`childIndex` | number | - | The child index to add to the event queue. |
+`priority?` | undefined | number | - | Optional priority of the event, defaults to the total key length.  |
 
-**Returns:** *Promise‹void›*
+**Returns:** *void*
 
 ___
 
-###  pushNode
+###  pushNodeToQueue
+
+▸ **pushNodeToQueue**(`nodeRef`: Buffer, `key`: Nibbles, `priority?`: undefined | number): *void*
 
-▸ **pushNode**(`nodeRef`: Buffer, `key`: Nibbles, `priority?`: undefined | number): *void*
+*Defined in [util/walkController.ts:87](https://github.com/ethereumjs/merkle-patricia-tree/blob/master/src/util/walkController.ts#L87)*
 
-*Defined in [util/walkController.ts:91](https://github.com/ethereumjs/merkle-patricia-tree/blob/master/src/util/walkController.ts#L91)*
+Push a node to the queue. If the queue has places left for tasks, the node is executed immediately, otherwise it is queued.
 
 **Parameters:**
 
@@ -111,17 +113,17 @@ ___
 
 ▸ **newWalk**(`onNode`: [FoundNodeFunction](../modules/_basetrie_.md#foundnodefunction), `trie`: BaseTrie, `root`: Buffer, `poolSize?`: undefined | number): *Promise‹void›*
 
-*Defined in [util/walkController.ts:34](https://github.com/ethereumjs/merkle-patricia-tree/blob/master/src/util/walkController.ts#L34)*
+*Defined in [util/walkController.ts:35](https://github.com/ethereumjs/merkle-patricia-tree/blob/master/src/util/walkController.ts#L35)*
 
 Async function to create and start a new walk over a trie.
 
 **Parameters:**
 
 Name | Type | Description |
 ------ | ------ | ------ |
-`onNode` | [FoundNodeFunction](../modules/_basetrie_.md#foundnodefunction) | The `FoundNodeFunction to call if a node is found |
-`trie` | BaseTrie | The trie to walk on |
-`root` | Buffer | The root key to walk on |
+`onNode` | [FoundNodeFunction](../modules/_basetrie_.md#foundnodefunction) | The `FoundNodeFunction to call if a node is found. |
+`trie` | BaseTrie | The trie to walk on. |
+`root` | Buffer | The root key to walk on. |
 `poolSize?` | undefined | number | Task execution pool size to prevent OOM errors. Defaults to 500.  |
 
 **Returns:** *Promise‹void›*

src/baseTrie.ts

@@ -4,7 +4,7 @@ import { keccak, KECCAK256_RLP } from 'ethereumjs-util'
 import { DB, BatchDBOp, PutBatch } from './db'
 import { TrieReadStream as ReadStream } from './readStream'
 import { bufferToNibbles, matchingNibbleLength, doKeysMatch } from './util/nibbles'
-import WalkController from './util/walkController'
+import { WalkController } from './util/walkController'
 import {
   TrieNode,
   decodeNode,
@@ -180,7 +180,7 @@ export class Trie {
             } else {
               // node found, continuing search
               // this can be optimized as this calls getBranch again.
-              await walkController.onlyBranchIndex(node, keyProgress, branchIndex)
+              walkController.onlyBranchIndex(node, keyProgress, branchIndex)
             }
           }
         } else if (node instanceof LeafNode) {

src/index.ts

@@ -1,5 +1,4 @@
 export { CheckpointTrie } from './checkpointTrie'
 export { SecureTrie } from './secure'
 export { Trie as BaseTrie } from './baseTrie'
-import WalkController from './util/walkController'
-export { WalkController }
+export { WalkController } from './util/walkController'

src/prioritizedTaskExecutor.ts

@@ -24,20 +24,23 @@ export class PrioritizedTaskExecutor {
   }
 
   /**
-   * Executes the task.
+   * Executes the task or queues it if no spots are available.
+   * When a task is added, check if there are spots left in the pool.
+   * If a spot is available, claim that spot and give back the spot once the asynchronous task has been resolved.
+   * When no spots are available, add the task to the task queue. The task will be executed at some point when another task has been resolved.
    * @private
    * @param priority The priority of the task
    * @param fn The function that accepts the callback, which must be called upon the task completion.
    */
-  execute(priority: number, fn: Function) {
+  executeOrQueue(priority: number, fn: Function) {
     if (this.currentPoolSize < this.maxPoolSize) {
       this.currentPoolSize++
       fn(() => {
         this.currentPoolSize--
         if (this.queue.length > 0) {
           this.queue.sort((a, b) => b.priority - a.priority)
           const item = this.queue.shift()
-          this.execute(item!.priority, item!.fn)
+          this.executeOrQueue(item!.priority, item!.fn)
         }
       })
     } else {

src/util/walkController.ts

@@ -6,16 +6,17 @@ import { BranchNode, ExtensionNode, LeafNode, Nibbles, TrieNode } from '../trieN
 /**
  * WalkController is an interface to control how the trie is being traversed.
  */
-export default class WalkController {
+export class WalkController {
   readonly onNode: FoundNodeFunction
   readonly taskExecutor: PrioritizedTaskExecutor
   readonly trie: BaseTrie
   private resolve: Function
 
   /**
    * Creates a new WalkController
-   * @param onNode - The `FoundNodeFunction` to call if a node is found
-   * @param trie - The `Trie` to walk on
+   * @param onNode - The `FoundNodeFunction` to call if a node is found.
+   * @param trie - The `Trie` to walk on.
+   * @param poolSize - The size of the task queue.
    */
   private constructor(onNode: FoundNodeFunction, trie: BaseTrie, poolSize: number) {
     this.onNode = onNode
@@ -26,9 +27,9 @@ export default class WalkController {
 
   /**
    * Async function to create and start a new walk over a trie.
-   * @param onNode - The `FoundNodeFunction to call if a node is found
-   * @param trie - The trie to walk on
-   * @param root - The root key to walk on
+   * @param onNode - The `FoundNodeFunction to call if a node is found.
+   * @param trie - The trie to walk on.
+   * @param root - The root key to walk on.
    * @param poolSize - Task execution pool size to prevent OOM errors. Defaults to 500.
    */
   static async newWalk(
@@ -42,22 +43,17 @@ export default class WalkController {
   }
 
   private async startWalk(root: Buffer): Promise<void> {
-    return await new Promise((resolve, reject) => {
+    // eslint-disable-next-line no-async-promise-executor
+    return await new Promise(async (resolve) => {
       this.resolve = resolve
-      this.trie
-        ._lookupNode(root)
-        .then((node) => {
-          this.processNode(root, node, [])
-        })
-        .catch((e) => {
-          reject(e)
-        })
+      const node = await this.trie._lookupNode(root)
+      this.processNode(root, node, [])
     })
   }
 
   /**
-   * Run all children of a node. Priority of these nodes are the key length of the children
-   * @param node - Node to get all children of and call onNode on
+   * Run all children of a node. Priority of these nodes are the key length of the children.
+   * @param node - Node to get all children of and call onNode on.
    * @param key - The current `key` which would yield the `node` when trying to get this node with a `get` operation.
    */
   allChildren(node: TrieNode, key: Nibbles = []) {
@@ -78,37 +74,35 @@ export default class WalkController {
       const childRef = child[1] as Buffer
       const childKey = key.concat(keyExtension)
       const priority = childKey.length
-      this.pushNode(childRef, childKey, priority)
+      this.pushNodeToQueue(childRef, childKey, priority)
     }
   }
 
   /**
-   *
+   * Push a node to the queue. If the queue has places left for tasks, the node is executed immediately, otherwise it is queued.
    * @param nodeRef - Push a node reference to the event queue. This reference is a 32-byte keccak hash of the value corresponding to the `key`.
    * @param key - The current key.
    * @param priority - Optional priority, defaults to key length
    */
-  pushNode(nodeRef: Buffer, key: Nibbles = [], priority?: number) {
-    this.taskExecutor.execute(priority ?? key.length, async (taskCallback: Function) => {
-      const childNode = await this.trie._lookupNode(nodeRef)
-      taskCallback()
-      this.processNode(nodeRef as Buffer, childNode as TrieNode, key)
-    })
+  pushNodeToQueue(nodeRef: Buffer, key: Nibbles = [], priority?: number) {
+    this.taskExecutor.executeOrQueue(
+      priority ?? key.length,
+      async (taskFinishedCallback: Function) => {
+        const childNode = await this.trie._lookupNode(nodeRef)
+        taskFinishedCallback() // this marks the current task as finished. If there are any tasks left in the queue, this will immediately execute the first task.
+        this.processNode(nodeRef as Buffer, childNode as TrieNode, key)
+      }
+    )
   }
 
   /**
-   * Push a branch of a certain BranchNode to the event queue
-   * @param node - The node to select a branch on. Should be a BranchNode
-   * @param key - The current key which leads to the corresponding node
-   * @param childIndex - The child index to add to the event queue
-   * @param priority - Optional priority of the event, defaults to the total key length
+   * Push a branch of a certain BranchNode to the event queue.
+   * @param node - The node to select a branch on. Should be a BranchNode.
+   * @param key - The current key which leads to the corresponding node.
+   * @param childIndex - The child index to add to the event queue.
+   * @param priority - Optional priority of the event, defaults to the total key length.
    */
-  async onlyBranchIndex(
-    node: BranchNode,
-    key: Nibbles = [],
-    childIndex: number,
-    priority?: number
-  ) {
+  onlyBranchIndex(node: BranchNode, key: Nibbles = [], childIndex: number, priority?: number) {
     if (!(node instanceof BranchNode)) {
       throw new Error('Expected branch node')
     }
@@ -119,7 +113,7 @@ export default class WalkController {
     const childKey = key.slice() // This copies the key to a new array.
     childKey.push(childIndex)
     const prio = priority ?? childKey.length
-    this.pushNode(childRef as Buffer, childKey, prio)
+    this.pushNodeToQueue(childRef as Buffer, childKey, prio)
   }
 
   private processNode(nodeRef: Buffer, node: TrieNode | null, key: Nibbles = []) {

test/prioritizedTaskExecutor.spec.ts

@@ -8,7 +8,7 @@ tape('prioritized task executor test', function (t) {
   const callbacks = [] as any
   const executionOrder = [] as any
   tasks.forEach(function (task) {
-    taskExecutor.execute(task, function (cb: Function) {
+    taskExecutor.executeOrQueue(task, function (cb: Function) {
       executionOrder.push(task)
       callbacks.push(cb)
     })