From 0bdd8f9663b7ca79e4a78543b8a5298211700f51 Mon Sep 17 00:00:00 2001 From: ashcherbakov Date: Tue, 21 Nov 2017 15:53:20 +0300 Subject: [PATCH 1/4] INDY-893: fix recovering tree from txn log (#451) * fix recovering tree from txn log - fixed recovering merkle tree after restart from txn log, if recovering from hash store failed because of consistency verification error (it can be since add to ledger is not atomic operation). Signed-off-by: ashcherbakov * add missing line Signed-off-by: ashcherbakov * rename tests Signed-off-by: ashcherbakov --- ledger/compact_merkle_tree.py | 9 ++- ledger/ledger.py | 3 +- ledger/merkle_tree.py | 8 ++- ledger/test/conftest.py | 14 ++++- ledger/test/test_ledger.py | 107 +++++++++++++++++++++++++--------- 5 files changed, 107 insertions(+), 34 deletions(-) diff --git a/ledger/compact_merkle_tree.py b/ledger/compact_merkle_tree.py index 0209fd83d1..bcbfaf74ed 100644 --- a/ledger/compact_merkle_tree.py +++ b/ledger/compact_merkle_tree.py @@ -6,8 +6,8 @@ from ledger.hash_stores.hash_store import HashStore from ledger.hash_stores.memory_hash_store import MemoryHashStore from ledger.tree_hasher import TreeHasher -from ledger.util import count_bits_set, lowest_bit_set from ledger.util import ConsistencyVerificationFailed +from ledger.util import count_bits_set, lowest_bit_set class CompactMerkleTree(merkle_tree.MerkleTree): @@ -249,7 +249,7 @@ def _path(self, m, start_n: int, end_n: int): return self._path(m - k, start_n + k, end_n) + [ (start_n, start_n + k)] - def get_tree_head(self, seq: int=None): + def get_tree_head(self, seq: int = None): if seq is None: seq = self.tree_size if seq > self.tree_size: @@ -288,3 +288,8 @@ def verify_consistency(self, expected_leaf_count) -> bool: if self.get_expected_node_count(self.leafCount) != self.nodeCount: raise ConsistencyVerificationFailed() return True + + def reset(self): + self.hashStore.reset() + self._update(tree_size=0, + hashes=()) diff --git a/ledger/ledger.py b/ledger/ledger.py index 25a2e919b0..e38d7256ea 100644 --- a/ledger/ledger.py +++ b/ledger/ledger.py @@ -83,7 +83,8 @@ def recoverTree(self): def recoverTreeFromTxnLog(self): # TODO: in this and some other lines specific fields of - self.tree.hashStore.reset() + self.tree.reset() + self.seqNo = 0 for key, entry in self._transactionLog.iterator(): if self.txn_serializer != self.hash_serializer: entry = self.serialize_for_tree( diff --git a/ledger/merkle_tree.py b/ledger/merkle_tree.py index 7bab70445b..356842f672 100644 --- a/ledger/merkle_tree.py +++ b/ledger/merkle_tree.py @@ -1,5 +1,5 @@ from abc import abstractmethod, ABCMeta -from typing import List, Tuple +from typing import Tuple from ledger.hash_stores.hash_store import HashStore @@ -81,3 +81,9 @@ def verify_consistency(self, expectedLeafCount) -> bool: def hashStore(self) -> HashStore: """ """ + + @property + @abstractmethod + def reset(self): + """ + """ diff --git a/ledger/test/conftest.py b/ledger/test/conftest.py index 1a2ae94f4d..834ba96ae2 100644 --- a/ledger/test/conftest.py +++ b/ledger/test/conftest.py @@ -6,8 +6,8 @@ from common.serializers.msgpack_serializer import MsgPackSerializer from common.serializers.signing_serializer import SigningSerializer from ledger.genesis_txn.genesis_txn_file_util import create_genesis_txn_init_ledger -from ledger.genesis_txn.genesis_txn_initiator_from_file import GenesisTxnInitiatorFromFile -from ledger.test.helper import create_ledger +from ledger.test.helper import create_ledger, create_ledger_text_file_storage, create_ledger_chunked_file_storage, \ + create_ledger_leveldb_file_storage @pytest.fixture(scope='module') @@ -83,6 +83,16 @@ def ledger(request, genesis_txn_file, tempdir, txn_serializer, hash_serializer): ledger.stop() +@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage', 'LeveldbStorage']) +def create_ledger_callable(request): + if request.param == 'TextFileStorage': + return create_ledger_text_file_storage + elif request.param == 'ChunkedFileStorage': + return create_ledger_chunked_file_storage + elif request.param == 'LeveldbStorage': + return create_ledger_leveldb_file_storage + + @pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage', 'LeveldbStorage']) def ledger_no_genesis(request, tempdir, txn_serializer, hash_serializer): ledger = create_ledger(request, txn_serializer, hash_serializer, tempdir) diff --git a/ledger/test/test_ledger.py b/ledger/test/test_ledger.py index 926c817032..5f63daf005 100644 --- a/ledger/test/test_ledger.py +++ b/ledger/test/test_ledger.py @@ -9,11 +9,9 @@ from ledger.compact_merkle_tree import CompactMerkleTree from ledger.test.conftest import orderedFields from ledger.test.helper import NoTransactionRecoveryLedger, \ - check_ledger_generator, create_ledger_text_file_storage, create_ledger_chunked_file_storage, \ - create_ledger_leveldb_file_storage, create_default_ledger, random_txn + check_ledger_generator, create_ledger_text_file_storage, create_default_ledger, random_txn from ledger.test.test_file_hash_store import generateHashes from ledger.util import ConsistencyVerificationFailed, F -from storage.text_file_store import TextFileStore def b64e(s): @@ -89,25 +87,11 @@ def test_query_merkle_info(ledger, genesis_txns, genesis_txn_file): """ -def test_recover_merkle_tree_from_txn_log_text_file(tempdir, txn_serializer, hash_serializer, genesis_txn_file): - check_recover_merkle_tree_from_txn_log(create_ledger_text_file_storage, - tempdir, txn_serializer, hash_serializer, genesis_txn_file) - - -def test_recover_merkle_tree_from_txn_log_chunked_file(tempdir, txn_serializer, hash_serializer, genesis_txn_file): - check_recover_merkle_tree_from_txn_log(create_ledger_chunked_file_storage, - tempdir, txn_serializer, hash_serializer, genesis_txn_file) - - -def test_recover_merkle_tree_from_txn_log_leveldb_file(tempdir, txn_serializer, hash_serializer, genesis_txn_file): - check_recover_merkle_tree_from_txn_log(create_ledger_leveldb_file_storage, - tempdir, txn_serializer, hash_serializer, genesis_txn_file) - - -def check_recover_merkle_tree_from_txn_log(create_ledger_func, tempdir, txn_serializer, hash_serializer, genesis_txn_file): - ledger = create_ledger_func( +def test_recover_merkle_tree_from_txn_log(create_ledger_callable, tempdir, + txn_serializer, hash_serializer, genesis_txn_file): + ledger = create_ledger_callable( txn_serializer, hash_serializer, tempdir, genesis_txn_file) - for d in range(100): + for d in range(5): ledger.add(random_txn(d)) # delete hash store, so that the only option for recovering is txn log ledger.tree.hashStore.reset() @@ -119,8 +103,8 @@ def check_recover_merkle_tree_from_txn_log(create_ledger_func, tempdir, txn_seri root_hash_before = ledger.root_hash hashes_before = ledger.tree.hashes - restartedLedger = create_ledger_func( - txn_serializer, hash_serializer, tempdir, genesis_txn_file) + restartedLedger = create_ledger_callable(txn_serializer, + hash_serializer, tempdir, genesis_txn_file) assert size_before == restartedLedger.size assert root_hash_before == restartedLedger.root_hash @@ -129,18 +113,23 @@ def check_recover_merkle_tree_from_txn_log(create_ledger_func, tempdir, txn_seri assert tree_size_before == restartedLedger.tree.tree_size -def test_recover_merkle_tree_from_hash_store(tempdir): - ledger = create_default_ledger(tempdir) - for d in range(100): +def test_recover_merkle_tree_from_hash_store(create_ledger_callable, tempdir, + txn_serializer, hash_serializer, genesis_txn_file): + ledger = create_ledger_callable( + txn_serializer, hash_serializer, tempdir, genesis_txn_file) + for d in range(5): ledger.add(random_txn(d)) ledger.stop() + size_before = ledger.size tree_root_hash_before = ledger.tree.root_hash tree_size_before = ledger.tree.tree_size root_hash_before = ledger.root_hash hashes_before = ledger.tree.hashes - restartedLedger = create_default_ledger(tempdir) + restartedLedger = create_ledger_callable(txn_serializer, + hash_serializer, tempdir, genesis_txn_file) + assert size_before == restartedLedger.size assert root_hash_before == restartedLedger.root_hash assert hashes_before == restartedLedger.tree.hashes @@ -151,7 +140,7 @@ def test_recover_merkle_tree_from_hash_store(tempdir): def test_recover_ledger_new_fields_to_txns_added(tempdir): ledger = create_ledger_text_file_storage( CompactSerializer(orderedFields), None, tempdir) - for d in range(100): + for d in range(5): ledger.add(random_txn(d)) updatedTree = ledger.tree ledger.stop() @@ -215,6 +204,68 @@ def test_consistency_verification_on_startup_case_2(tempdir): ledger.stop() +def test_recover_merkle_tree_from_txn_log_if_hash_store_runs_ahead(create_ledger_callable, tempdir, + txn_serializer, hash_serializer, + genesis_txn_file): + ''' + Check that tree can be recovered from txn log if recovering from hash store failed + (we have one more txn in hash store than in txn log, so consistency verification fails). + ''' + ledger = create_ledger_callable( + txn_serializer, hash_serializer, tempdir, genesis_txn_file) + for d in range(5): + ledger.add(random_txn(d)) + + size_before = ledger.size + tree_root_hash_before = ledger.tree.root_hash + tree_size_before = ledger.tree.tree_size + root_hash_before = ledger.root_hash + hashes_before = ledger.tree.hashes + + # add to hash store only + ledger._addToTree(ledger.serialize_for_tree(random_txn(50)), + serialized=True) + ledger.stop() + + restartedLedger = create_ledger_callable(txn_serializer, + hash_serializer, tempdir, genesis_txn_file) + + assert size_before == restartedLedger.size + assert root_hash_before == restartedLedger.root_hash + assert hashes_before == restartedLedger.tree.hashes + assert tree_root_hash_before == restartedLedger.tree.root_hash + assert tree_size_before == restartedLedger.tree.tree_size + + +def test_recover_merkle_tree_from_txn_log_if_hash_store_lags_behind(create_ledger_callable, tempdir, + txn_serializer, hash_serializer, + genesis_txn_file): + ''' + Check that tree can be recovered from txn log if recovering from hash store failed + (we have one more txn in txn log than in hash store, so consistency verification fails). + ''' + ledger = create_ledger_callable( + txn_serializer, hash_serializer, tempdir, genesis_txn_file) + for d in range(100): + ledger.add(random_txn(d)) + + # add to txn log only + ledger._addToStore(ledger.serialize_for_txn_log(random_txn(50)), + serialized=True) + ledger.stop() + + size_before = ledger.size + tree_size_before = ledger.tree.tree_size + + restartedLedger = create_ledger_callable(txn_serializer, + hash_serializer, tempdir, genesis_txn_file) + + # root hashes will be not the same as before (since we recoverd based on txn log) + # the new size is 1 greater than before since we recovered from txn log which contained one more txn + assert size_before + 1 == restartedLedger.size + assert tree_size_before + 1 == restartedLedger.tree.tree_size + + def test_start_ledger_without_new_line_appended_to_last_record(tempdir, txn_serializer): if isinstance(txn_serializer, MsgPackSerializer): # MsgPack is a binary one, not compatible with TextFileStorage From 912beadeb4b427e11c214602af8e0bf599fe2f6e Mon Sep 17 00:00:00 2001 From: ashcherbakov Date: Wed, 22 Nov 2017 11:50:09 +0300 Subject: [PATCH 2/4] do not show DEBUG logs from indy-crypto Signed-off-by: ashcherbakov --- crypto/bls/indy_crypto/bls_crypto_indy_crypto.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crypto/bls/indy_crypto/bls_crypto_indy_crypto.py b/crypto/bls/indy_crypto/bls_crypto_indy_crypto.py index a51bd6168c..e63b00cbea 100644 --- a/crypto/bls/indy_crypto/bls_crypto_indy_crypto.py +++ b/crypto/bls/indy_crypto/bls_crypto_indy_crypto.py @@ -1,3 +1,4 @@ +import logging from logging import getLogger from typing import Sequence, Optional @@ -5,6 +6,7 @@ from crypto.bls.bls_crypto import GroupParams, BlsGroupParamsLoader, BlsCryptoVerifier, BlsCryptoSigner from indy_crypto.bls import BlsEntity, Generator, VerKey, SignKey, Bls, Signature, MultiSignature +logging.getLogger("indy_crypto").setLevel(logging.WARNING) logger = getLogger() From e943a546a258402b0d607f4a074a745c8757af8f Mon Sep 17 00:00:00 2001 From: Dmitry Surnin Date: Tue, 21 Nov 2017 19:12:59 +0300 Subject: [PATCH 3/4] Fix client basedir initialization Signed-off-by: Dmitry Surnin --- plenum/client/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plenum/client/client.py b/plenum/client/client.py index 4d04b3b386..f736b9caa1 100644 --- a/plenum/client/client.py +++ b/plenum/client/client.py @@ -80,8 +80,8 @@ def __init__(self, :param ha: tuple of host and port """ self.config = config or getConfig() - self.basedirpath = basedirpath or os.path.join(self.config.baseDir, - self.config.NETWORK_NAME) + self.basedirpath = basedirpath or os.path.join(self.config.CLI_NETWORK_DIR, self.config.NETWORK_NAME) + self.basedirpath = os.path.expanduser(self.basedirpath) signer = Signer(sighex) sighex = signer.keyraw @@ -139,7 +139,7 @@ def __init__(self, ha=cha, main=False, # stops incoming vacuous joins auth_mode=AuthMode.ALLOW_ANY.value) - stackargs['basedirpath'] = basedirpath + stackargs['basedirpath'] = self.basedirpath self.created = time.perf_counter() # noinspection PyCallingNonCallable From 4a32a74f91d54de41741f2281b4a966139be476c Mon Sep 17 00:00:00 2001 From: anikitinDSR Date: Wed, 22 Nov 2017 18:15:09 +0300 Subject: [PATCH 4/4] [INDY-948] Hotfix for KeyError exception (#459) Signed-off-by: Andrew Nikitin --- plenum/server/node.py | 5 +++-- .../node_catchup/test_req_id_key_error.py | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 plenum/test/node_catchup/test_req_id_key_error.py diff --git a/plenum/server/node.py b/plenum/server/node.py index 8b3b1528ef..032e200035 100644 --- a/plenum/server/node.py +++ b/plenum/server/node.py @@ -2394,8 +2394,9 @@ def executeBatch(self, view_no, pp_seq_no: int, pp_time: float, format(self, old)) def updateSeqNoMap(self, committedTxns): - self.seqNoDB.addBatch((txn[f.IDENTIFIER.nm], txn[f.REQ_ID.nm], - txn[F.seqNo.name]) for txn in committedTxns) + if all([txn.get(f.REQ_ID.nm, None) for txn in committedTxns]): + self.seqNoDB.addBatch((txn[f.IDENTIFIER.nm], txn[f.REQ_ID.nm], + txn[F.seqNo.name]) for txn in committedTxns) def commitAndSendReplies(self, reqHandler, ppTime, reqs: List[Request], stateRoot, txnRoot) -> List: diff --git a/plenum/test/node_catchup/test_req_id_key_error.py b/plenum/test/node_catchup/test_req_id_key_error.py new file mode 100644 index 0000000000..6d55e0e416 --- /dev/null +++ b/plenum/test/node_catchup/test_req_id_key_error.py @@ -0,0 +1,21 @@ +from plenum.common.types import f +from plenum.common.txn_util import reqToTxn +from plenum.test.helper import signed_random_requests +from stp_core.common.log import getlogger + +logger = getlogger() + +def test_req_id_key_error(testNode, wallet1): + #create random transactions + count_of_txn = 3 + reqs = signed_random_requests(wallet1, count_of_txn) + txns = [] + #prepare transactions and remove reqId from + for i, req in enumerate(reqs): + txnreq = reqToTxn(req) + txnreq[f.SEQ_NO.nm] = i + txnreq.pop(f.REQ_ID.nm) + txns.append(txnreq) + node = testNode + logger.debug(txns) + node.updateSeqNoMap(txns)