ZStack takes config

plenum/common/config_util.py

@@ -1,6 +1,7 @@
 import os
 from importlib import import_module
 from importlib.util import module_from_spec, spec_from_file_location
+from stp_core.common.config.util import getConfig as STPConfig
 
 
 CONFIG = None
@@ -35,7 +36,10 @@ def getConfig(homeDir=None):
     """
     global CONFIG
     if not CONFIG:
-        refConfig = import_module("plenum.config")
+        stp_config = STPConfig(homeDir)
+        plenum_config = import_module("plenum.config")
+        refConfig = stp_config
+        refConfig.__dict__.update(plenum_config.__dict__)
         try:
             homeDir = os.path.expanduser(homeDir or "~")
 

plenum/common/stacks.py

@@ -2,18 +2,19 @@
 
 from plenum import config
 from plenum.common.batched import Batched, logger
+from plenum.common.config_util import getConfig
 from plenum.common.message_processor import MessageProcessor
 from stp_raet.rstack import SimpleRStack, KITRStack
 from stp_core.types import HA
 from stp_zmq.zstack import SimpleZStack, KITZStack
 
 
 class ClientZStack(SimpleZStack, MessageProcessor):
-    def __init__(self, stackParams: dict, msgHandler: Callable, seed=None):
+    def __init__(self, stackParams: dict, msgHandler: Callable, seed=None,
+                 config=None):
+        config = config or getConfig()
         SimpleZStack.__init__(self, stackParams, msgHandler, seed=seed,
-                              onlyListener=True,
-                              listenerQuota=config.LISTENER_MESSAGE_QUOTA,
-                              remoteQuota=config.REMOTES_MESSAGE_QUOTA)
+                              onlyListener=True, config=config)
         MessageProcessor.__init__(self, allowDictOnly=False)
         self.connectedClients = set()
 
@@ -55,12 +56,12 @@ def transmitToClients(self, msg: Any, remoteNames: List[str]):
 
 class NodeZStack(Batched, KITZStack):
     def __init__(self, stackParams: dict, msgHandler: Callable,
-                 registry: Dict[str, HA], seed=None, sighex: str=None):
+                 registry: Dict[str, HA], seed=None, sighex: str=None,
+                 config=None):
+        config = config or getConfig()
         Batched.__init__(self)
         KITZStack.__init__(self, stackParams, msgHandler, registry=registry,
-                           seed=seed, sighex=sighex,
-                           listenerQuota=config.LISTENER_MESSAGE_QUOTA,
-                           remoteQuota=config.REMOTES_MESSAGE_QUOTA)
+                           seed=seed, sighex=sighex, config=config)
         MessageProcessor.__init__(self, allowDictOnly=False)
 
     # TODO: Reconsider defaulting `reSetupAuth` to True.

plenum/common/test_network_setup.py

@@ -14,7 +14,8 @@
 
 from plenum.common.keygen_utils import initLocalKeys
 from plenum.common.constants import STEWARD, CLIENT_STACK_SUFFIX, TRUSTEE
-from plenum.common.util import hexToFriendly, adict
+from plenum.common.util import hexToFriendly
+from stp_core.common.util import adict
 
 
 class TestNetworkSetup:

plenum/common/util.py

@@ -1,15 +1,16 @@
 import asyncio
 import collections
+import functools
 import glob
 import inspect
 import ipaddress
 import itertools
 import json
 import logging
+import math
 import os
 import random
 import time
-import math
 from binascii import unhexlify, hexlify
 from collections import Counter, defaultdict
 from collections import OrderedDict
@@ -20,19 +21,20 @@
 
 import base58
 import libnacl.secret
-from libnacl import randombytes, randombytes_uniform
 import psutil
 from jsonpickle import encode, decode
+from libnacl import randombytes, randombytes_uniform
 from six import iteritems, string_types
 
 from ledger.util import F
 from plenum.cli.constants import WALLET_FILE_EXTENSION
 from plenum.common.error import error
 from stp_core.crypto.util import isHexKey, isHex
 from stp_core.network.exceptions import \
-    MissingEndpoint, \
     InvalidEndpointIpAddress, InvalidEndpointPort
-import functools
+
+# Do not remove the next import until imports in sovrin are fixed
+from stp_core.common.util import adict
 
 
 T = TypeVar('T')
@@ -218,37 +220,6 @@ def prime_gen() -> int:
             D[x] = p
 
 
-class adict(dict):
-    """Dict with attr access to keys."""
-    marker = object()
-
-    def __init__(self, **kwargs):
-        super().__init__()
-        for key in kwargs:
-            self.__setitem__(key, kwargs[key])
-
-    def __setitem__(self, key, value):
-        if isinstance(value, dict) and not isinstance(value, adict):
-            value = adict(**value)
-        super(adict, self).__setitem__(key, value)
-
-    def __getitem__(self, key):
-        found = self.get(key, adict.marker)
-        if found is adict.marker:
-            found = adict()
-            super(adict, self).__setitem__(key, found)
-        return found
-
-    def copy(self):
-        return self.__copy__()
-
-    def __copy__(self):
-        return adict(**self)
-
-    __setattr__ = __setitem__
-    __getattr__ = __getitem__
-
-
 async def untilTrue(condition, *args, timeout=5) -> bool:
     """
     Keep checking the condition till it is true or a timeout is reached

plenum/test/batching_3pc/catch-up/test_3pc_paused_during_catch_up.py

@@ -1,5 +1,6 @@
 from plenum.common.types import Prepare
-from plenum.test.batching_3pc.helper import make_node_syncing, fail_on_execute_batch_on_master
+from plenum.test.batching_3pc.helper import make_node_syncing, \
+    fail_on_execute_batch_on_master
 from plenum.test.helper import sendRandomRequests
 from plenum.test.test_node import getNonPrimaryReplicas
 

plenum/test/cli/conftest.py

@@ -7,7 +7,7 @@
 
 from stp_core.loop.eventually import eventually
 from stp_core.loop.looper import Looper
-from plenum.common.util import adict
+from stp_core.common.util import adict
 from plenum.test.cli.helper import newCLI, waitAllNodesUp, loadPlugin, \
     doByCtx
 from stp_core.network.port_dispenser import genHa

plenum/test/instances/test_multiple_commit.py

@@ -5,7 +5,7 @@
 
 from stp_core.loop.eventually import eventually
 from plenum.common.types import Commit
-from plenum.common.util import adict
+from stp_core.common.util import adict
 from plenum.server.suspicion_codes import Suspicions
 from plenum.test.helper import getNodeSuspicions, whitelistNode
 from plenum.test.malicious_behaviors_node import makeNodeFaulty, \

plenum/test/instances/test_multiple_pre_prepare.py

@@ -4,7 +4,7 @@
 
 from stp_core.loop.eventually import eventually
 from plenum.common.types import PrePrepare
-from plenum.common.util import adict
+from stp_core.common.util import adict
 from plenum.server.suspicion_codes import Suspicions
 from plenum.test.helper import getNodeSuspicions
 from plenum.test.instances.helper import sentPrepare

plenum/test/instances/test_multiple_prepare.py

@@ -4,7 +4,7 @@
 
 from stp_core.loop.eventually import eventually
 from plenum.common.types import Prepare
-from plenum.common.util import adict
+from stp_core.common.util import adict
 from plenum.server.suspicion_codes import Suspicions
 from plenum.test.helper import getNodeSuspicions, whitelistNode
 from plenum.test.malicious_behaviors_node import makeNodeFaulty, \

plenum/test/instances/test_pre_prepare_digest.py

@@ -4,7 +4,7 @@
 
 from stp_core.loop.eventually import eventually
 from plenum.common.types import PrePrepare
-from plenum.common.util import adict
+from stp_core.common.util import adict
 from plenum.server.suspicion_codes import Suspicions
 from plenum.test.helper import getNodeSuspicions
 from plenum.test.instances.helper import sentPrepare

plenum/test/monitoring/test_avg_latency.py

@@ -1,4 +1,3 @@
-from stp_core.loop.eventually import eventually
 from stp_core.common.log import getlogger
 from stp_core.loop.looper import Looper
 from plenum.server.node import Node

plenum/test/monitoring/test_instance_change_with_Delta.py

@@ -2,15 +2,13 @@
 
 import pytest
 
-from plenum.common.types import PrePrepare
-from plenum.common.util import adict
+from stp_core.common.util import adict
 from plenum.server.node import Node
 from plenum.test import waits
 from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies
 from plenum.test.malicious_behaviors_node import slow_primary
 from plenum.test.test_node import getPrimaryReplica
 from plenum.test.view_change.helper import provoke_and_wait_for_view_change
-from plenum.test.helper import waitForViewChange
 from stp_core.common.log import getlogger
 from stp_core.loop.eventually import eventually
 

plenum/test/monitoring/test_instance_change_with_req_Lambda.py

@@ -1,10 +1,7 @@
-from functools import partial
-
 import pytest
 
-from stp_core.loop.eventually import eventually
 from plenum.common.types import PrePrepare, f
-from plenum.common.util import adict
+from stp_core.common.util import adict
 from plenum.test.helper import waitForViewChange, \
     sendReqsToNodesAndVerifySuffReplies
 from plenum.test.test_node import getPrimaryReplica

plenum/test/monitoring/test_throughput.py

@@ -1,6 +1,5 @@
 from typing import Iterable
 
-from stp_core.loop.eventually import eventually
 from stp_core.common.log import getlogger
 from plenum.server.node import Node
 from plenum.test.helper import sendRandomRequest, \

plenum/test/node_catchup/helper.py

@@ -2,7 +2,6 @@
 
 from plenum.common.util import check_if_all_equal_in_list
 from stp_core.common.log import getlogger
-from stp_zmq.zstack import KITZStack
 from typing import Iterable
 
 from plenum.common.constants import POOL_LEDGER_ID, DOMAIN_LEDGER_ID
@@ -11,9 +10,8 @@
 from plenum.test.helper import checkLedgerEquality, checkStateEquality, \
     check_seqno_db_equality, assertEquality, check_last_ordered_3pc
 from plenum.test.test_client import TestClient
-from plenum.test.test_node import TestNode, TestNodeSet
+from plenum.test.test_node import TestNode
 from plenum.test import waits
-from plenum.common import util
 import pytest
 
 

plenum/test/node_catchup/test_catchup_delayed_nodes.py

@@ -1,6 +1,5 @@
 import pytest
 
-from stp_core.loop.eventually import eventually
 from stp_core.common.log import getlogger
 
 from plenum.test import waits

plenum/test/node_catchup/test_catchup_demoted.py

@@ -1,12 +1,13 @@
 from plenum.common.constants import ALIAS, SERVICES, VALIDATOR
 from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies
-from plenum.test.node_catchup.conftest import whitelist
 from plenum.test.node_catchup.helper import waitNodeDataEquality, \
-    checkNodeDataForInequality, checkNodeDataForEquality
+    checkNodeDataForInequality
 from plenum.test.pool_transactions.helper import \
     updateNodeData
 from stp_core.common.log import getlogger
 
+from plenum.test.node_catchup.conftest import whitelist
+
 logger = getlogger()
 
 

plenum/test/node_catchup/test_catchup_while_new_request_incoming.py

@@ -8,7 +8,7 @@
     sendReqsToNodesAndVerifySuffReplies
 from plenum.test.node_catchup.helper import checkNodeDataForEquality
 from plenum.test.pool_transactions.helper import addNewStewardAndNode
-from plenum.test.test_node import checkNodesConnected, TestNode
+from plenum.test.test_node import TestNode
 from stp_core.loop.eventually import eventually
 
 

plenum/test/node_catchup/test_discard_view_no.py

@@ -1,11 +1,7 @@
 import time
-from functools import partial
-
-import pytest
 
 from stp_core.loop.eventually import eventually
-from plenum.common.types import Nomination, PrePrepare, ViewChangeDone
-from plenum.common.util import randomString
+from plenum.common.types import PrePrepare
 from plenum.common.constants import DOMAIN_LEDGER_ID
 from plenum.test.helper import checkDiscardMsg
 from plenum.test.view_change.helper import ensure_view_change

plenum/test/node_catchup/test_new_node_catchup.py

@@ -3,10 +3,8 @@
 import pytest
 
 from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState
-from plenum.common.util import updateNamedTuple
-from plenum.test.delayers import cqDelay, cr_delay
+from plenum.test.delayers import cr_delay
 from plenum.test.spy_helpers import get_count
-from stp_zmq.zstack import KITZStack
 
 from stp_core.loop.eventually import eventually
 from plenum.common.types import HA
@@ -15,7 +13,8 @@
     check_last_ordered_3pc
 from plenum.test.node_catchup.helper import waitNodeDataEquality, \
     check_ledger_state
-from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected
+from plenum.test.pool_transactions.helper import \
+    disconnect_node_and_ensure_disconnected
 from plenum.test.test_ledger_manager import TestLedgerManager
 from plenum.test.test_node import checkNodesConnected, TestNode
 from plenum.test import waits

plenum/test/node_catchup/test_node_catchup_after_disconnect.py

@@ -1,7 +1,9 @@
 from stp_core.common.log import getlogger
 from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies
-from plenum.test.node_catchup.helper import waitNodeDataEquality, waitNodeDataInequality, checkNodeDataForEquality
-from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected, reconnect_node_and_ensure_connected
+from plenum.test.node_catchup.helper import waitNodeDataEquality, \
+    waitNodeDataInequality, checkNodeDataForEquality
+from plenum.test.pool_transactions.helper import \
+    disconnect_node_and_ensure_disconnected, reconnect_node_and_ensure_connected
 
 # Do not remove the next import
 from plenum.test.node_catchup.conftest import whitelist

plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py

@@ -5,7 +5,6 @@
 from stp_core.common.log import getlogger
 from plenum.common.constants import TXN_TYPE, DOMAIN_LEDGER_ID
 from plenum.common.types import CatchupReq, f, CatchupRep
-from plenum.test.helper import sendRandomRequests
 from plenum.test.node_catchup.helper import waitNodeDataEquality
 from plenum.test.test_node import checkNodesConnected, getNonPrimaryReplicas
 from plenum.test import waits

plenum/test/node_catchup/test_node_request_consistency_proof.py

@@ -5,7 +5,6 @@
 
 from plenum.common.constants import DOMAIN_LEDGER_ID
 from plenum.common.ledger import Ledger
-from stp_core.loop.eventually import eventually
 from stp_core.common.log import getlogger
 from plenum.common.types import LedgerStatus
 from plenum.test.helper import sendRandomRequests

plenum/test/node_catchup/test_node_request_missing_transactions.py

@@ -1,4 +1,3 @@
-import time
 import types
 
 import pytest

plenum/test/node_catchup/test_revert_during_catchup.py

@@ -2,8 +2,7 @@
 
 from plenum.common.constants import DOMAIN_LEDGER_ID
 from plenum.test import waits
-from plenum.test.batching_3pc.conftest import tconf
-from plenum.test.delayers import cDelay, cpDelay, cqDelay
+from plenum.test.delayers import cDelay, cqDelay
 from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \
     check_last_ordered_3pc
 from plenum.test.node_catchup.helper import waitNodeDataInequality, \
@@ -14,6 +13,9 @@
 from plenum.test.view_change.helper import ensure_view_change
 from stp_core.loop.eventually import eventually
 
+# Do not remove the next imports
+from plenum.test.batching_3pc.conftest import tconf
+
 Max3PCBatchSize = 2
 
 

plenum/test/node_request/node_request_helper.py

@@ -1,10 +1,8 @@
 import time
 from functools import partial
 
-from stp_core.loop.eventually import eventuallyAll
 from plenum.common.types import PrePrepare, OPERATION, f
 from plenum.common.constants import DOMAIN_LEDGER_ID
-from plenum.common.types import OPERATION, f
 from plenum.common.util import getMaxFailures
 from plenum.server.node import Node
 from plenum.server.replica import Replica