Clean up tests and functions

tests/core/middleware/test_filter_middleware.py

@@ -3,11 +3,11 @@
 from web3 import Web3
 from web3.middleware import (
     construct_result_generator_middleware,
-    filter_middleware,
+    local_filter_middleware,
 )
 from web3.middleware.filter import (
     iter_latest_block_ranges,
-    range_counter,
+    block_range_chunker,
 )
 from web3.providers.base import (
     BaseProvider,
@@ -22,7 +22,7 @@ def make_request(self, method, params):
         ))
 
 
-@pytest.fixture(scope='module')
+@pytest.fixture(scope='function')
 def iter_block_number(start=0):
     def iterator():
         block_number = start
@@ -35,25 +35,25 @@ def iterator():
     return block_number
 
 
-@pytest.fixture(scope='module')
+@pytest.fixture(scope='function')
 def result_generator_middleware(iter_block_number):
     return construct_result_generator_middleware({
-        'eth_getLogs': lambda *args: [],
+        'eth_getLogs': lambda *_: ["middleware"],
+        'eth_getBlockByNumber': lambda *_: type('block', (object,), {'hash': 'middleware'}),
         'net_version': lambda *_: 1,
         'eth_blockNumber': lambda *_: next(iter_block_number),
     })
 
 
-@pytest.fixture(scope='module')
+@pytest.fixture(scope='function')
 def w3_base():
     return Web3(providers=[DummyProvider()], middlewares=[])
 
 
-@pytest.fixture(scope='module')
+@pytest.fixture(scope='function')
 def w3(w3_base, result_generator_middleware):
     w3_base.middleware_stack.add(result_generator_middleware)
-    w3_base.middleware_stack.add(filter_middleware)
-    w3_base.eth.filter(filter_params={'fromBlock': 'latest'})
+    w3_base.middleware_stack.add(local_filter_middleware)
     return w3_base
 
 
@@ -77,12 +77,12 @@ def w3(w3_base, result_generator_middleware):
     ]),
     (5, 0, TypeError),
 ])
-def test_range_counter(start, stop, expected):
+def test_request_block_chunker(start, stop, expected):
     if isinstance(expected, type) and issubclass(expected, Exception):
         with pytest.raises(expected):
-            range_counter(start, stop)
+            block_range_chunker(start, stop)
     else:
-        for actual, expected in zip(range_counter(start, stop), expected):
+        for actual, expected in zip(block_range_chunker(start, stop), expected):
             assert actual == expected
 
 
@@ -116,16 +116,21 @@ def test_iter_latest_block_ranges(
         assert actual_tuple == expected_tuple
 
 
-@pytest.mark.parametrize("method,args,expected", [
-    ('eth_newFilter', [{}], '0x1'),
-    ('eth_newPendingBlockFilter', [{}], NotImplementedError),
-    ('eth_newBlockFilter', [{}], '0x2'),
-    ('eth_getFilterChanges', ['0x0'], []),
-    ('eth_getFilterLogs', ['0x0'], []),
-])
-def test_filter_middleware(w3, method, args, expected):
-    if isinstance(expected, type) and issubclass(expected, Exception):
-        with pytest.raises(expected):
-            w3.manager.request_blocking(method, args)
-    else:
-        assert w3.manager.request_blocking(method, args) == expected
+def test_local_filter_middleware(w3, iter_block_number):
+    block_filter = w3.eth.filter('latest')
+    assert block_filter.filter_id == '0x0'
+    iter_block_number.send(1)
+
+    with pytest.raises(NotImplementedError):
+        w3.eth.filter('pending')
+
+    log_filter = w3.eth.filter(filter_params={'fromBlock': 'latest'})
+    assert log_filter.filter_id == "0x1"
+
+    assert w3.eth.getFilterChanges("0x0") == ["middleware"]
+
+    iter_block_number.send(2)
+    results = w3.eth.getFilterChanges("0x1")
+    assert results == ["middleware"]
+
+    assert w3.eth.getFilterLogs("0x1") == ["middleware"]

web3/middleware/__init__.py

@@ -18,7 +18,7 @@
     construct_exception_handler_middleware,
 )
 from .filter import (  # noqa: F401
-    filter_middleware,
+    local_filter_middleware,
 )
 from .fixture import (  # noqa: F401
     construct_fixture_middleware,

web3/middleware/filter.py

@@ -1,5 +1,3 @@
-import itertools
-
 from eth_utils import (
     apply_key_map,
     to_hex,
@@ -10,29 +8,128 @@
 )
 
 
-def range_counter(start, stop=None, step=5):
-    if stop is not None and start > stop:
+def segment_count(start, stop=None, step=5):
+    """Creates a segment counting generator
+
+    The generator returns tuple pairs of integers
+    that correspond to segments in the provided range.
+
+    :param start: The initial value of the counting range
+    :param stop: Optional, if provided the last value in the
+    counting range
+    :param step: Optional, the segment length. Default is 5.
+    :type start: int
+    :type stop: int
+    :return: returns a generator object
+
+
+    Example:
+
+    >>> segment_counter = segment_count(start=0, stop=10, step=3)
+    >>> next(segment_counter)
+    (0, 3)
+    >>> next(segment_counter)
+    (3, 6)
+    >>> next(segment_counter)
+    (6, 9)
+    >>> next(segment_counter) #  Remainder is also returned
+    (9, 10)
+    """
+    def finite(start, stop, step):
+        #  If the initial range is less than the step
+        #  just return (start, stop)
+        if stop < step:
+            yield (start, stop)
+            return
+        for i in zip(
+                range(start, stop - step, step),
+                range(start + step, stop, step)):
+            yield i
+        else:
+            remainder = (stop - start) % step
+            #  Handle the remainder
+            if remainder:
+                yield (stop - remainder, stop)
+
+    def infinite(start, step):
+        for i in zip(
+                count(start, step),
+                count(start + step, step)):
+            yield i
+
+    if stop is None:
+        return infinite(start, step)
+    else:
+        return finite(start, stop, step)
+
+
+def block_range_chunker(start_block, last_block=None, step=5):
+    """Returns 2-tuple ranges describing chunks of block from start_block to last_block
+
+       Segments do not overlap to facilitate use as ``toBlock``, ``fromBlock``
+       json-rpc arguments, which are both inclusive.
+    """
+
+    if last_block is not None and start_block > last_block:
         raise TypeError(
             "Incompatible start and stop arguments.",
             "Start must be less than or equal to stop.")
 
-    def range_counter(start, stop, step):
-        def iter_ranges(start, step):
-            yield start, start + step - 1
-            yield from iter_ranges(start + step, step)
+    def block_range_chunker(start_block, last_block, step):
+        for from_block, to_block in segment_count(start_block, last_block + 1, step):
+            yield (from_block, to_block - 1)
 
-        ranges = iter_ranges(start, step)
-        while True:
-            _from, _to = next(ranges)
+    return block_range_chunker(start_block, last_block, step)
 
-            # Handle final case where stop is less than a step away
-            if stop is not None and stop < _to:
-                yield _from, stop
-                return
 
-            yield _from, _to
+def iter_latest_block(web3, to_block=None):
+    """Returns a generator that dispenses the latest block, if
+    any new blocks have been mined since last iteration.
 
-    return range_counter(start, stop, step)
+    If there are no new blocks None is returned.
+
+    If ``to_block`` is defined, ``StopIteration`` is raised
+    after to_block is reached.
+
+    >>> mined_blocks = dispense_mined_blocks(web3, 0, 10)
+    >>> next(new_blocks)  # Latest block = 0
+    0
+    >>> next(new_blocks)  # No new blocks
+    >>> next(new_blocks)  # Latest block = 1
+    1
+    >>> next(new_blocks)  # Latest block = 10
+    10
+    >>> next(new_blocks)
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+      StopIteration
+    >>>
+    """
+    _last = None
+    while True:
+        latest_block = web3.eth.blockNumber
+        if (latest_block is not None and
+                to_block is not None and
+                latest_block > to_block):
+            return
+        #  No new blocks since last iteration.
+        if _last is not None and _last == latest_block:
+            yield None
+        else:
+            yield latest_block
+        _last = latest_block
+
+
+def count(start=0, step=1):
+    """A counter you can update with send()
+    """
+    i = start
+    while True:
+        val = (yield i)
+        if val is not None:
+            i = val
+        else:
+            i += step
 
 
 def iter_latest_block_ranges(web3, from_block, to_block=None):
@@ -50,45 +147,32 @@ def iter_latest_block_ranges(web3, from_block, to_block=None):
     >>> next(blocks_to_filter)  # latest block number = 50
     (46, 50)
     """
-    # Return full range if all blocks exist
-    if to_block is not None and to_block <= web3.eth.blockNumber:
-        return from_block, to_block
-
-    def get_block_range(from_block, to_block):
-
-        # If to_block has been mined,
-        # and last yield didn't reach to_block.
-        if to_block is not None and web3.eth.blockNumber >= to_block and from_block < to_block:
-            yield from_block, to_block
-            return
-
-        current_block = web3.eth.blockNumber
-        # If to_block hasn't been mined yet
-        if to_block is None or to_block > current_block:
-            # If from_block doesnt exist yet:
-            if from_block > current_block:
-                yield (None, None)
-            # Return mined block range
-            else:
-                yield from_block, current_block
-
-            yield from get_block_range(current_block + 1, to_block)
-
-    yield from get_block_range(from_block, to_block)
+    _from_block = from_block
+    for _latest_block in iter_latest_block(web3, to_block):
+        if _latest_block is None:
+            yield (None, None)
+        else:
+            yield (_from_block, _latest_block)
+            _from_block = _latest_block + 1
 
 
-def request_getLogs(
+def get_logs_multipart(
         web3,
         startBlock,
         stopBlock,
         address,
         topics,
-        chunk_size):
-    block_chunker = range_counter(startBlock, stopBlock)
-    for chunk in block_chunker:
+        max_blocks):
+    """Use to make large requests to ``eth_getLogs``
+
+    The getLog request is broken into multiple calls of the max number of blocks
+    ``max_blocks``.
+    """
+    chunks = block_range_chunker(startBlock, stopBlock, max_blocks)
+    for from_block, to_block in chunks:
         params = {
-            'fromBlock': chunk[0],
-            'toBlock': chunk[1],
+            'fromBlock': from_block,
+            'toBlock': to_block,
             'address': address,
             'topics': topics
         }
@@ -147,24 +231,24 @@ def _get_filter_changes(self):
 
             yield list(
                 concat(
-                    request_getLogs(
+                    get_logs_multipart(
                         self.web3,
                         start,
                         stop,
                         self.address,
                         self.topics,
-                        chunk_size=5)))
+                        max_blocks=5)))
 
     def get_logs(self):
         return list(
             concat(
-                request_getLogs(
+                get_logs_multipart(
                     self.web3,
                     self.from_block,
                     self.to_block,
                     self.address,
                     self.topics,
-                    chunk_size=5)))
+                    max_blocks=5)))
 
 
 FILTER_PARAMS_KEY_MAP = {
@@ -207,9 +291,9 @@ def block_hashes_in_range(web3, block_range):
         yield web3.eth.getBlock(block_number).hash
 
 
-def filter_middleware(make_request, web3):
+def local_filter_middleware(make_request, web3):
     filters = {}
-    filter_id_counter = map(to_hex, itertools.count())
+    filter_id_counter = map(to_hex, count())
 
     def middleware(method, params):
         if method in NEW_FILTER_METHODS: