updating patricia trie code from ethereum and general refactoring (#310)

ledger/test/test_ledger.py

@@ -50,7 +50,9 @@ def ledger(tempdir):
     ledger = Ledger(CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
                     dataDir=tempdir, serializer=ledgerSerializer)
     ledger.reset()
-    return ledger
+    yield ledger
+    # close the file
+    ledger.stop()
 
 
 def testAddTxn(ledger):

plenum/test/common/test_parse_ledger.py

@@ -3,36 +3,38 @@
 
 from ledger.compact_merkle_tree import CompactMerkleTree
 from ledger.ledger import Ledger
-from plenum.common.constants import TXN_TYPE, TARGET_NYM, DATA, NAME, ALIAS, SERVICES, VALIDATOR
+from plenum.common.constants import TXN_TYPE, TARGET_NYM, DATA, NAME, ALIAS, SERVICES, VALIDATOR, VERKEY
 from plenum.common.stack_manager import TxnStackManager
 
 whitelist = ['substring not found']
 
+
 @pytest.fixture(scope="module")
 def tdirWithLedger(tdir):
-    tree = CompactMerkleTree()
     ledger = Ledger(CompactMerkleTree(), dataDir=tdir)
     for d in range(3):
-        txn = { TXN_TYPE: '0',
-                TARGET_NYM: base58.b58encode(b'whatever'),
-                DATA: {
-                NAME: str(d),
-                ALIAS: 'test' + str(d),
-                SERVICES: [VALIDATOR],
-                }
-              }
+        txn = {TXN_TYPE: '0',
+               TARGET_NYM: base58.b58encode(b'whatever'),
+               DATA: {
+                   NAME: str(d),
+                   ALIAS: 'test' + str(d),
+                   SERVICES: [VALIDATOR],
+               }
+               }
         if d == 1:
             txn[TARGET_NYM] = "invalid===="
         ledger.add(txn)
     return ledger
-
+
+
 """
 Test that invalid base58 TARGET_NYM in pool_transaction raises the proper exception (INDY-150)
 """
 
-def test_parse_non_base58_txn_type_field_raises_descriptive_error(tdirWithLedger,tdir):
+
+def test_parse_non_base58_txn_type_field_raises_descriptive_error(tdirWithLedger, tdir):
     with pytest.raises(ValueError) as excinfo:
         ledger = Ledger(CompactMerkleTree(), dataDir=tdir)
         _, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger)
-    assert("verkey" in str(excinfo.value))
+    assert (VERKEY in str(excinfo.value))
     ledger.stop()

plenum/test/common/test_random_string.py

@@ -1,6 +1,7 @@
 import time
 from plenum.common.util import randomString
 
+
 # Checks if the function randomString() is returning correct
 # length random string for various lengths
 def test_random_string1():

plenum/test/node_catchup/test_ledger_status_quorum.py

@@ -1,5 +1,3 @@
-from plenum.common.messages.node_messages import LedgerStatus, DOMAIN_LEDGER_ID
-
 from plenum.common.ledger_manager import LedgerManager
 from plenum.common.util import getMaxFailures
 

state/pruning_state.py

@@ -1,4 +1,5 @@
 from binascii import unhexlify
+from typing import Optional
 
 from state.db.persistent_db import PersistentDB
 from state.kv.kv_store import KeyValueStorage
@@ -10,15 +11,17 @@
 
 
 class PruningState(State):
-    # This class is used to store the
-    # committed root hash of the trie in the db.
-    # The committed root hash is only updated once a batch gets written to the
-    # ledger. It might happen that a few batches are in 3 phase commit and the
-    # node crashes. Now when the node restarts, it restores the db from the
-    # committed root hash and all entries for uncommitted batches will be
-    # ignored
-
-    # some key that does not collide with any state variable's name
+    """
+    This class is used to store the
+    committed root hash of the trie in the db.
+    The committed root hash is only updated once a batch gets written to the
+    ledger. It might happen that a few batches are in 3 phase commit and the
+    node crashes. Now when the node restarts, it restores the db from the
+    committed root hash and all entries for uncommitted batches will be
+    ignored
+    """
+
+    # SOME KEY THAT DOES NOT COLLIDE WITH ANY STATE VARIABLE'S NAME
     rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b'
 
     def __init__(self, keyValueStorage: KeyValueStorage):
@@ -50,7 +53,7 @@ def committedHead(self):
     def set(self, key: bytes, value: bytes):
         self._trie.update(key, rlp_encode([value]))
 
-    def get(self, key: bytes, isCommitted: bool = True):
+    def get(self, key: bytes, isCommitted: bool = True) -> Optional[bytes]:
         if not isCommitted:
             val = self._trie.get(key)
         else:

state/state.py

@@ -30,19 +30,22 @@ def revertToHead(self, headHash=None):
     def close(self):
         raise NotImplementedError
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def head(self):
         # The current head of the state, if the state is a merkle tree then
         # head is the root
         raise NotImplementedError
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def committedHead(self):
         # The committed head of the state, if the state is a merkle tree then
         # head is the root
         raise NotImplementedError
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def headHash(self):
         """
         The hash of the current head of the state, if the state is a merkle
@@ -51,10 +54,12 @@ def headHash(self):
         """
         raise NotImplementedError
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def committedHeadHash(self):
         raise NotImplementedError
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def isEmpty(self):
         raise NotImplementedError

state/trie/pruning_trie.py

@@ -871,6 +871,49 @@ def to_dict(self):
             res[key] = value
         return res
 
+    def iter_branch(self):
+        for key_str, value in self._iter_branch(self.root_node):
+            if key_str:
+                nibbles = [int(x) for x in key_str.split(b'+')]
+            else:
+                nibbles = []
+            key = nibbles_to_bin(without_terminator(nibbles))
+            yield key, value
+
+    def _iter_branch(self, node):
+        """yield (key, value) stored in this and the descendant nodes
+        :param node: node in form of list, or BLANK_NODE
+
+        .. note::
+            Here key is in full form, rather than key of the individual node
+        """
+        if node == BLANK_NODE:
+            raise StopIteration
+
+        node_type = self._get_node_type(node)
+
+        if is_key_value_type(node_type):
+            nibbles = without_terminator(unpack_to_nibbles(node[0]))
+            key = b'+'.join([to_string(x) for x in nibbles])
+            if node_type == NODE_TYPE_EXTENSION:
+                sub_tree = self._iter_branch(self._decode_to_node(node[1]))
+            else:
+                sub_tree = [(to_string(NIBBLE_TERMINATOR), node[1])]
+
+            # prepend key of this node to the keys of children
+            for sub_key, sub_value in sub_tree:
+                full_key = (key + b'+' + sub_key).strip(b'+')
+                yield (full_key, sub_value)
+
+        elif node_type == NODE_TYPE_BRANCH:
+            for i in range(16):
+                sub_tree = self._iter_branch(self._decode_to_node(node[i]))
+                for sub_key, sub_value in sub_tree:
+                    full_key = (str_to_bytes(str(i)) + b'+' + sub_key).strip(b'+')
+                    yield (full_key, sub_value)
+            if node[16]:
+                yield (to_string(NIBBLE_TERMINATOR), node[-1])
+
     def get(self, key):
         return self._get(self.root_node, bin_to_nibbles(to_string(key)))
 
@@ -936,6 +979,25 @@ def get_at(self, root_node, key):
         """
         return self._get(root_node, bin_to_nibbles(to_string(key)))
 
+    @staticmethod
+    def verify_spv_proof(root, key, proof_nodes):
+        proof.push(VERIFYING, proof_nodes)
+        t = Trie(KeyValueStorageInMemory())
+
+        for i, node in enumerate(proof_nodes):
+            R = rlp_encode(node)
+            H = sha3(R)
+            t._db.put(H, R)
+        try:
+            t.root_hash = root
+            t.get(key)
+            proof.pop()
+            return True
+        except Exception as e:
+            print(e)
+            proof.pop()
+            return False
+
 
 if __name__ == "__main__":