HashStore is used to recover when the ledger starts, fixed the calcu…

ledger/compact_merkle_tree.py

@@ -261,10 +261,24 @@ def leafCount(self) -> int:
     def nodeCount(self) -> int:
         return self.hashStore.nodeCount
 
-    def verifyConsistency(self, expectedLeafCount = -1) -> bool:
-        if expectedLeafCount > 0 and expectedLeafCount != self.leafCount:
+    @staticmethod
+    def get_expected_node_count(leaf_count):
+        """
+        The number of nodes is the number of full subtrees present
+        """
+        count = 0
+        while leaf_count > 1:
+            leaf_count //= 2
+            count += leaf_count
+        return count
+
+    def verify_consistency(self, expected_leaf_count) -> bool:
+        """
+        Check that the tree has same leaf count as expected and the
+        number of nodes are also as expected
+        """
+        if expected_leaf_count != self.leafCount:
             raise ConsistencyVerificationFailed()
-        expectedNodeCount = count_bits_set(self.leafCount)
-        if not expectedNodeCount == self.nodeCount:
+        if self.get_expected_node_count(self.leafCount) != self.nodeCount:
             raise ConsistencyVerificationFailed()
-        return True
+        return True

ledger/ledger.py

@@ -73,34 +73,28 @@ def recoverTree(self):
                             .format(type(self.tree)))
 
 
-        # ATTENTION!
-        # This functionality is disabled until better consistency verification
-        # implemented - always using recovery from transaction log
-        # from ledger.stores.memory_hash_store import MemoryHashStore
-        # from ledger.util import ConsistencyVerificationFailed
-        # if not self.tree.hashStore \
-        #         or isinstance(self.tree.hashStore, MemoryHashStore) \
-        #         or self.tree.leafCount == 0:
-        #     logging.info("Recovering tree from transaction log")
-        #     self.recoverTreeFromTxnLog()
-        # else:
-        #     try:
-        #         logging.info("Recovering tree from hash store of size {}".
-        #                       format(self.tree.leafCount))
-        #         self.recoverTreeFromHashStore()
-        #     except ConsistencyVerificationFailed:
-        #         logging.error("Consistency verification of merkle tree "
-        #                       "from hash store failed, "
-        #                       "falling back to transaction log")
-        #         self.recoverTreeFromTxnLog()
-
-        logging.debug("Recovering tree from transaction log")
+        from ledger.stores.memory_hash_store import MemoryHashStore
+        from ledger.util import ConsistencyVerificationFailed
         start = time.perf_counter()
-        self.recoverTreeFromTxnLog()
+        if not self.tree.hashStore \
+                or isinstance(self.tree.hashStore, MemoryHashStore) \
+                or self.tree.leafCount == 0:
+            logging.debug("Recovering tree from transaction log")
+            self.recoverTreeFromTxnLog()
+        else:
+            try:
+                logging.debug("Recovering tree from hash store of size {}".
+                              format(self.tree.leafCount))
+                self.recoverTreeFromHashStore()
+            except ConsistencyVerificationFailed:
+                logging.error("Consistency verification of merkle tree "
+                              "from hash store failed, "
+                              "falling back to transaction log")
+                self.recoverTreeFromTxnLog()
+
         end = time.perf_counter()
         t = end - start
-        logging.debug("Recovered tree from transaction log in {} seconds".
-                      format(t))
+        logging.debug("Recovered tree in {} seconds".format(t))
 
     def recoverTreeFromTxnLog(self):
         # TODO: in this and some other lines specific fields of
@@ -118,7 +112,7 @@ def recoverTreeFromHashStore(self):
         hashes = list(reversed(self.tree.inclusion_proof(treeSize,
                                                          treeSize + 1)))
         self.tree._update(self.tree.leafCount, hashes)
-        self.tree.verifyConsistency(self._transactionLog.numKeys)
+        self.tree.verify_consistency(self._transactionLog.numKeys)
 
     def add(self, leaf):
         self._addToStore(leaf)

ledger/merkle_tree.py

@@ -69,6 +69,6 @@ def nodeCount(self) -> int:
         """
 
     @abstractmethod
-    def verifyConsistency(self, expectedLeafCount) -> bool:
+    def verify_consistency(self, expectedLeafCount) -> bool:
         """
         """

ledger/stores/hash_store.py

@@ -137,6 +137,14 @@ def readNodeByTree(self, start, height=None):
         pos = self.getNodePosition(start, height)
         return self.readNode(pos)
 
+    @property
+    def is_consistent(self) -> bool:
+        """
+        Returns True if number of nodes are consistent with number of leaves
+        """
+        from ledger.compact_merkle_tree import CompactMerkleTree
+        return self.nodeCount == CompactMerkleTree.get_expected_node_count(self.leafCount)
+
     @staticmethod
     def _validatePos(start, end=None):
         if end:

ledger/test/test_file_hash_store.py

@@ -51,6 +51,14 @@ def testSimpleReadWrite(nodesLeaves, tempdir):
     for i, n in enumerate(nds):
         assert nodes[i][2] == n
 
+    # Check that hash store can be closed and re-opened and the contents remain same
+    leaf_count = fhs.leafCount
+    node_count = fhs.nodeCount
+    fhs.close()
+    reopened_hash_store = FileHashStore(tempdir)
+    assert reopened_hash_store.leafCount == leaf_count
+    assert reopened_hash_store.nodeCount == node_count
+
 
 def testIncorrectWrites(tempdir):
     fhs = FileHashStore(tempdir, leafSize=50, nodeSize=50)

ledger/test/test_ledger.py

@@ -107,7 +107,7 @@ def testRecoverLedgerFromHashStore(tempdir):
     fhs = FileHashStore(tempdir)
     tree = CompactMerkleTree(hashStore=fhs)
     ledger = Ledger(tree=tree, dataDir=tempdir)
-    for d in range(10):
+    for d in range(100):
         ledger.add(str(d).encode())
     updatedTree = ledger.tree
     ledger.stop()

ledger/test/test_merkle_proof.py

@@ -110,6 +110,9 @@
 """
 
 
+TXN_COUNT = 1000
+
+
 @pytest.yield_fixture(scope="module", params=['File', 'Memory'])
 def hashStore(request, tdir):
     if request.param == 'File':
@@ -141,15 +144,13 @@ def hasherAndTree(hasher):
 def addTxns(hasherAndTree):
     h, m = hasherAndTree
 
-    txn_count = 1000
-
     auditPaths = []
-    for d in range(txn_count):
+    for d in range(TXN_COUNT):
         serNo = d+1
         data = str(serNo).encode()
         auditPaths.append([hexlify(h) for h in m.append(data)])
-
-    return txn_count, auditPaths
+        print(m.hashStore.leafCount, m.hashStore.nodeCount)
+    return TXN_COUNT, auditPaths
 
 
 @pytest.fixture()
@@ -200,14 +201,16 @@ def testCompactMerkleTree2(hasherAndTree, verifier):
 def testCompactMerkleTree(hasherAndTree, verifier):
     h, m = hasherAndTree
     printEvery = 1000
-    count = 1000
+    count = TXN_COUNT
     for d in range(count):
         data = str(d + 1).encode()
         data_hex = hexlify(data)
         audit_path = m.append(data)
         audit_path_hex = [hexlify(h) for h in audit_path]
         incl_proof = m.inclusion_proof(d, d+1)
         assert audit_path == incl_proof
+        assert m.nodeCount == m.get_expected_node_count(m.leafCount)
+        assert m.hashStore.is_consistent
         if d % printEvery == 0:
             show(h, m, data_hex)
             print("audit path is {}".format(audit_path_hex))

plenum/cli/cli.py

@@ -252,10 +252,10 @@ def __init__(self, looper, basedirpath, nodeReg=None, cliNodeReg=None,
             eventloop=eventloop,
             output=out)
 
-        RAETVerbosity = getRAETLogLevelFromConfig("RAETLogLevelCli",
-                                                  Console.Wordage.mute,
-                                                  self.config)
-        RAETLogFile = getRAETLogFilePath("RAETLogFilePathCli", self.config)
+        # RAETVerbosity = getRAETLogLevelFromConfig("RAETLogLevelCli",
+        #                                           Console.Wordage.mute,
+        #                                           self.config)
+        # RAETLogFile = getRAETLogFilePath("RAETLogFilePathCli", self.config)
         # Patch stdout in something that will always print *above* the prompt
         # when something is written to stdout.
         sys.stdout = self.cli.stdout_proxy()

plenum/test/input_validation/message_validation/test_prepare_message.py

@@ -1,9 +1,6 @@
-import pytest
-
 from collections import OrderedDict
 from plenum.common.messages.fields import NonNegativeNumberField, \
-    NonEmptyStringField, \
-    HexField, MerkleRootField, AnyValueField
+    NonEmptyStringField, MerkleRootField
 from plenum.common.messages.node_messages import Prepare
 
 EXPECTED_ORDERED_FIELDS = OrderedDict([

plenum/test/node_catchup/test_node_request_consistency_proof.py

@@ -1,7 +1,4 @@
 import types
-from random import randint
-
-import pytest
 
 from plenum.common.constants import DOMAIN_LEDGER_ID, CONSISTENCY_PROOF
 from plenum.common.ledger import Ledger