Merge pull request hyperledger#68 from hyperledger/master

Master

crypto/bls/indy_crypto/bls_crypto_indy_crypto.py

@@ -1,10 +1,12 @@
+import logging
 from logging import getLogger
 from typing import Sequence, Optional
 
 import base58
 from crypto.bls.bls_crypto import GroupParams, BlsGroupParamsLoader, BlsCryptoVerifier, BlsCryptoSigner
 from indy_crypto.bls import BlsEntity, Generator, VerKey, SignKey, Bls, Signature, MultiSignature
 
+logging.getLogger("indy_crypto").setLevel(logging.WARNING)
 logger = getLogger()
 
 

ledger/compact_merkle_tree.py

@@ -6,8 +6,8 @@
 from ledger.hash_stores.hash_store import HashStore
 from ledger.hash_stores.memory_hash_store import MemoryHashStore
 from ledger.tree_hasher import TreeHasher
-from ledger.util import count_bits_set, lowest_bit_set
 from ledger.util import ConsistencyVerificationFailed
+from ledger.util import count_bits_set, lowest_bit_set
 
 
 class CompactMerkleTree(merkle_tree.MerkleTree):
@@ -249,7 +249,7 @@ def _path(self, m, start_n: int, end_n: int):
                 return self._path(m - k, start_n + k, end_n) + [
                     (start_n, start_n + k)]
 
-    def get_tree_head(self, seq: int=None):
+    def get_tree_head(self, seq: int = None):
         if seq is None:
             seq = self.tree_size
         if seq > self.tree_size:
@@ -288,3 +288,8 @@ def verify_consistency(self, expected_leaf_count) -> bool:
         if self.get_expected_node_count(self.leafCount) != self.nodeCount:
             raise ConsistencyVerificationFailed()
         return True
+
+    def reset(self):
+        self.hashStore.reset()
+        self._update(tree_size=0,
+                     hashes=())

ledger/ledger.py

@@ -83,7 +83,8 @@ def recoverTree(self):
 
     def recoverTreeFromTxnLog(self):
         # TODO: in this and some other lines specific fields of
-        self.tree.hashStore.reset()
+        self.tree.reset()
+        self.seqNo = 0
         for key, entry in self._transactionLog.iterator():
             if self.txn_serializer != self.hash_serializer:
                 entry = self.serialize_for_tree(

ledger/merkle_tree.py

@@ -1,5 +1,5 @@
 from abc import abstractmethod, ABCMeta
-from typing import List, Tuple
+from typing import Tuple
 
 from ledger.hash_stores.hash_store import HashStore
 
@@ -81,3 +81,9 @@ def verify_consistency(self, expectedLeafCount) -> bool:
     def hashStore(self) -> HashStore:
         """
         """
+
+    @property
+    @abstractmethod
+    def reset(self):
+        """
+        """

ledger/test/conftest.py

@@ -6,8 +6,8 @@
 from common.serializers.msgpack_serializer import MsgPackSerializer
 from common.serializers.signing_serializer import SigningSerializer
 from ledger.genesis_txn.genesis_txn_file_util import create_genesis_txn_init_ledger
-from ledger.genesis_txn.genesis_txn_initiator_from_file import GenesisTxnInitiatorFromFile
-from ledger.test.helper import create_ledger
+from ledger.test.helper import create_ledger, create_ledger_text_file_storage, create_ledger_chunked_file_storage, \
+    create_ledger_leveldb_file_storage
 
 
 @pytest.fixture(scope='module')
@@ -83,6 +83,16 @@ def ledger(request, genesis_txn_file, tempdir, txn_serializer, hash_serializer):
     ledger.stop()
 
 
+@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage', 'LeveldbStorage'])
+def create_ledger_callable(request):
+    if request.param == 'TextFileStorage':
+        return create_ledger_text_file_storage
+    elif request.param == 'ChunkedFileStorage':
+        return create_ledger_chunked_file_storage
+    elif request.param == 'LeveldbStorage':
+        return create_ledger_leveldb_file_storage
+
+
 @pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage', 'LeveldbStorage'])
 def ledger_no_genesis(request, tempdir, txn_serializer, hash_serializer):
     ledger = create_ledger(request, txn_serializer, hash_serializer, tempdir)

ledger/test/test_ledger.py

@@ -9,11 +9,9 @@
 from ledger.compact_merkle_tree import CompactMerkleTree
 from ledger.test.conftest import orderedFields
 from ledger.test.helper import NoTransactionRecoveryLedger, \
-    check_ledger_generator, create_ledger_text_file_storage, create_ledger_chunked_file_storage, \
-    create_ledger_leveldb_file_storage, create_default_ledger, random_txn
+    check_ledger_generator, create_ledger_text_file_storage, create_default_ledger, random_txn
 from ledger.test.test_file_hash_store import generateHashes
 from ledger.util import ConsistencyVerificationFailed, F
-from storage.text_file_store import TextFileStore
 
 
 def b64e(s):
@@ -89,25 +87,11 @@ def test_query_merkle_info(ledger, genesis_txns, genesis_txn_file):
 """
 
 
-def test_recover_merkle_tree_from_txn_log_text_file(tempdir, txn_serializer, hash_serializer, genesis_txn_file):
-    check_recover_merkle_tree_from_txn_log(create_ledger_text_file_storage,
-                                           tempdir, txn_serializer, hash_serializer, genesis_txn_file)
-
-
-def test_recover_merkle_tree_from_txn_log_chunked_file(tempdir, txn_serializer, hash_serializer, genesis_txn_file):
-    check_recover_merkle_tree_from_txn_log(create_ledger_chunked_file_storage,
-                                           tempdir, txn_serializer, hash_serializer, genesis_txn_file)
-
-
-def test_recover_merkle_tree_from_txn_log_leveldb_file(tempdir, txn_serializer, hash_serializer, genesis_txn_file):
-    check_recover_merkle_tree_from_txn_log(create_ledger_leveldb_file_storage,
-                                           tempdir, txn_serializer, hash_serializer, genesis_txn_file)
-
-
-def check_recover_merkle_tree_from_txn_log(create_ledger_func, tempdir, txn_serializer, hash_serializer, genesis_txn_file):
-    ledger = create_ledger_func(
+def test_recover_merkle_tree_from_txn_log(create_ledger_callable, tempdir,
+                                          txn_serializer, hash_serializer, genesis_txn_file):
+    ledger = create_ledger_callable(
         txn_serializer, hash_serializer, tempdir, genesis_txn_file)
-    for d in range(100):
+    for d in range(5):
         ledger.add(random_txn(d))
     # delete hash store, so that the only option for recovering is txn log
     ledger.tree.hashStore.reset()
@@ -119,8 +103,8 @@ def check_recover_merkle_tree_from_txn_log(create_ledger_func, tempdir, txn_seri
     root_hash_before = ledger.root_hash
     hashes_before = ledger.tree.hashes
 
-    restartedLedger = create_ledger_func(
-        txn_serializer, hash_serializer, tempdir, genesis_txn_file)
+    restartedLedger = create_ledger_callable(txn_serializer,
+                                             hash_serializer, tempdir, genesis_txn_file)
 
     assert size_before == restartedLedger.size
     assert root_hash_before == restartedLedger.root_hash
@@ -129,18 +113,23 @@ def check_recover_merkle_tree_from_txn_log(create_ledger_func, tempdir, txn_seri
     assert tree_size_before == restartedLedger.tree.tree_size
 
 
-def test_recover_merkle_tree_from_hash_store(tempdir):
-    ledger = create_default_ledger(tempdir)
-    for d in range(100):
+def test_recover_merkle_tree_from_hash_store(create_ledger_callable, tempdir,
+                                             txn_serializer, hash_serializer, genesis_txn_file):
+    ledger = create_ledger_callable(
+        txn_serializer, hash_serializer, tempdir, genesis_txn_file)
+    for d in range(5):
         ledger.add(random_txn(d))
     ledger.stop()
+
     size_before = ledger.size
     tree_root_hash_before = ledger.tree.root_hash
     tree_size_before = ledger.tree.tree_size
     root_hash_before = ledger.root_hash
     hashes_before = ledger.tree.hashes
 
-    restartedLedger = create_default_ledger(tempdir)
+    restartedLedger = create_ledger_callable(txn_serializer,
+                                             hash_serializer, tempdir, genesis_txn_file)
+
     assert size_before == restartedLedger.size
     assert root_hash_before == restartedLedger.root_hash
     assert hashes_before == restartedLedger.tree.hashes
@@ -151,7 +140,7 @@ def test_recover_merkle_tree_from_hash_store(tempdir):
 def test_recover_ledger_new_fields_to_txns_added(tempdir):
     ledger = create_ledger_text_file_storage(
         CompactSerializer(orderedFields), None, tempdir)
-    for d in range(100):
+    for d in range(5):
         ledger.add(random_txn(d))
     updatedTree = ledger.tree
     ledger.stop()
@@ -215,6 +204,68 @@ def test_consistency_verification_on_startup_case_2(tempdir):
     ledger.stop()
 
 
+def test_recover_merkle_tree_from_txn_log_if_hash_store_runs_ahead(create_ledger_callable, tempdir,
+                                                                   txn_serializer, hash_serializer,
+                                                                   genesis_txn_file):
+    '''
+    Check that tree can be recovered from txn log if recovering from hash store failed
+    (we have one more txn in hash store than in txn log, so consistency verification fails).
+    '''
+    ledger = create_ledger_callable(
+        txn_serializer, hash_serializer, tempdir, genesis_txn_file)
+    for d in range(5):
+        ledger.add(random_txn(d))
+
+    size_before = ledger.size
+    tree_root_hash_before = ledger.tree.root_hash
+    tree_size_before = ledger.tree.tree_size
+    root_hash_before = ledger.root_hash
+    hashes_before = ledger.tree.hashes
+
+    # add to hash store only
+    ledger._addToTree(ledger.serialize_for_tree(random_txn(50)),
+                      serialized=True)
+    ledger.stop()
+
+    restartedLedger = create_ledger_callable(txn_serializer,
+                                             hash_serializer, tempdir, genesis_txn_file)
+
+    assert size_before == restartedLedger.size
+    assert root_hash_before == restartedLedger.root_hash
+    assert hashes_before == restartedLedger.tree.hashes
+    assert tree_root_hash_before == restartedLedger.tree.root_hash
+    assert tree_size_before == restartedLedger.tree.tree_size
+
+
+def test_recover_merkle_tree_from_txn_log_if_hash_store_lags_behind(create_ledger_callable, tempdir,
+                                                                    txn_serializer, hash_serializer,
+                                                                    genesis_txn_file):
+    '''
+    Check that tree can be recovered from txn log if recovering from hash store failed
+    (we have one more txn in txn log than in hash store, so consistency verification fails).
+    '''
+    ledger = create_ledger_callable(
+        txn_serializer, hash_serializer, tempdir, genesis_txn_file)
+    for d in range(100):
+        ledger.add(random_txn(d))
+
+    # add to txn log only
+    ledger._addToStore(ledger.serialize_for_txn_log(random_txn(50)),
+                       serialized=True)
+    ledger.stop()
+
+    size_before = ledger.size
+    tree_size_before = ledger.tree.tree_size
+
+    restartedLedger = create_ledger_callable(txn_serializer,
+                                             hash_serializer, tempdir, genesis_txn_file)
+
+    # root hashes will be not the same as before (since we recoverd based on txn log)
+    # the new size is 1 greater than before since we recovered from txn log which contained one more txn
+    assert size_before + 1 == restartedLedger.size
+    assert tree_size_before + 1 == restartedLedger.tree.tree_size
+
+
 def test_start_ledger_without_new_line_appended_to_last_record(tempdir, txn_serializer):
     if isinstance(txn_serializer, MsgPackSerializer):
         # MsgPack is a binary one, not compatible with TextFileStorage

plenum/client/client.py

@@ -80,8 +80,8 @@ def __init__(self,
         :param ha: tuple of host and port
         """
         self.config = config or getConfig()
-        self.basedirpath = basedirpath or os.path.join(self.config.baseDir,
-                                                       self.config.NETWORK_NAME)
+        self.basedirpath = basedirpath or os.path.join(self.config.CLI_NETWORK_DIR, self.config.NETWORK_NAME)
+        self.basedirpath = os.path.expanduser(self.basedirpath)
 
         signer = Signer(sighex)
         sighex = signer.keyraw
@@ -139,7 +139,7 @@ def __init__(self,
                          ha=cha,
                          main=False,  # stops incoming vacuous joins
                          auth_mode=AuthMode.ALLOW_ANY.value)
-        stackargs['basedirpath'] = basedirpath
+        stackargs['basedirpath'] = self.basedirpath
         self.created = time.perf_counter()
 
         # noinspection PyCallingNonCallable

plenum/server/node.py

@@ -2394,8 +2394,9 @@ def executeBatch(self, view_no, pp_seq_no: int, pp_time: float,
                              format(self, old))
 
     def updateSeqNoMap(self, committedTxns):
-        self.seqNoDB.addBatch((txn[f.IDENTIFIER.nm], txn[f.REQ_ID.nm],
-                               txn[F.seqNo.name]) for txn in committedTxns)
+        if all([txn.get(f.REQ_ID.nm, None) for txn in committedTxns]):
+            self.seqNoDB.addBatch((txn[f.IDENTIFIER.nm], txn[f.REQ_ID.nm],
+                                   txn[F.seqNo.name]) for txn in committedTxns)
 
     def commitAndSendReplies(self, reqHandler, ppTime, reqs: List[Request],
                              stateRoot, txnRoot) -> List:

plenum/test/node_catchup/test_req_id_key_error.py

@@ -0,0 +1,21 @@
+from plenum.common.types import f
+from plenum.common.txn_util import reqToTxn
+from plenum.test.helper import signed_random_requests
+from stp_core.common.log import getlogger
+
+logger = getlogger()
+
+def test_req_id_key_error(testNode, wallet1):
+    #create random transactions
+    count_of_txn = 3
+    reqs = signed_random_requests(wallet1, count_of_txn)
+    txns = []
+    #prepare transactions and remove reqId from
+    for i, req in enumerate(reqs):
+        txnreq = reqToTxn(req)
+        txnreq[f.SEQ_NO.nm] = i
+        txnreq.pop(f.REQ_ID.nm)
+        txns.append(txnreq)
+    node = testNode
+    logger.debug(txns)
+    node.updateSeqNoMap(txns)