Merge pull request #732 from dylanjw/filtering-middleware

node independent filter middleware

docs/middleware.rst

@@ -368,3 +368,25 @@ which uses a prototype PoA for it's development mode and the Rinkeby test networ
 Unfortunately, it does deviate from the yellow paper specification, which constrains the
 ``extraData`` field in each block to a maximum of 32-bytes. Geth's PoA uses more than
 32 bytes, so this middleware modifies the block data a bit before returning it.
+
+.. _local-filter:
+
+Locally Managed Log and Block Filters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This middleware provides an alternative to ethereum node managed filters. When used, Log and
+Block filter logic are handled locally while using the same web3 filter api. Filter results are
+retrieved using JSON-RPC endpoints that don't rely on server state. 
+
+.. code-block:: python
+
+    >>> from web3 import Web3, EthereumTesterProvider
+    >>> w3 = Web3(EthereumTesterProvider)
+    >>> from web3.middleware import local_filter_middleware
+    >>> w3.middleware_stack.add(local_filter_middleware())
+
+    #  Normal block and log filter apis behave as before.
+    >>> block_filter = w3.eth.filter("latest")
+
+    >>> log_filter = myContract.events.myEvent.build_filter().deploy()
+

tests/core/filtering/conftest.py

@@ -9,16 +9,29 @@
 )
 
 from web3 import Web3
+from web3.middleware import (
+    local_filter_middleware,
+)
 from web3.providers.eth_tester import (
     EthereumTesterProvider,
 )
 
 
 @pytest.fixture()
+def tester_snapshot(web3):
+    return web3.providers[0].ethereum_tester.take_snapshot()
+
+
+@pytest.fixture(
+    scope='function',
+    params=[True, False],
+    ids=["local_filter_middleware", "node_based_filter"])
 def web3(request):
+    use_filter_middleware = request.param
     provider = EthereumTesterProvider()
     w3 = Web3(provider)
-
+    if use_filter_middleware:
+        w3.middleware_stack.add(local_filter_middleware)
     return w3
 
 

tests/core/filtering/test_basic_filter_tests.py

@@ -0,0 +1,40 @@
+
+
+def test_filtering_sequential_blocks_with_bounded_range(
+        web3,
+        emitter,
+        Emitter,
+        wait_for_transaction):
+    builder = emitter.events.LogNoArguments.build_filter()
+    builder.fromBlock = "latest"
+
+    initial_block_number = web3.eth.blockNumber
+
+    builder.toBlock = initial_block_number + 100
+    filter_ = builder.deploy(web3)
+    for i in range(100):
+        emitter.functions.logNoArgs(which=1).transact()
+    assert web3.eth.blockNumber == initial_block_number + 100
+    assert len(filter_.get_new_entries()) == 100
+
+
+def test_filtering_starting_block_range(
+        web3,
+        emitter,
+        Emitter,
+        wait_for_transaction):
+    for i in range(10):
+        emitter.functions.logNoArgs(which=1).transact()
+    builder = emitter.events.LogNoArguments.build_filter()
+    filter_ = builder.deploy(web3)
+    initial_block_number = web3.eth.blockNumber
+    for i in range(10):
+        emitter.functions.logNoArgs(which=1).transact()
+    assert web3.eth.blockNumber == initial_block_number + 10
+    assert len(filter_.get_new_entries()) == 10
+
+
+def test_requesting_results_with_no_new_blocks(web3, emitter):
+    builder = emitter.events.LogNoArguments.build_filter()
+    filter_ = builder.deploy(web3)
+    assert len(filter_.get_new_entries()) == 0

tests/core/filtering/test_contract_data_filters.py

@@ -56,6 +56,14 @@ def array_values(draw):
     return (matching, non_matching)
 
 
+def clear_chain_state(web3, snapshot):
+    """Clear chain state
+    Hypothesis doesn't allow function scoped fixtures to re-run between test runs
+    so chain state needs to be explicitly cleared
+    """
+    web3.providers[0].ethereum_tester.revert_to_snapshot(snapshot)
+
+
 @pytest.mark.parametrize('api_style', ('v4', 'build_filter'))
 @given(vals=dynamic_values())
 @settings(max_examples=5, deadline=None)
@@ -65,7 +73,9 @@ def test_data_filters_with_dynamic_arguments(
         create_filter,
         emitter,
         api_style,
+        tester_snapshot,
         vals):
+    clear_chain_state(web3, tester_snapshot)
 
     if api_style == 'build_filter':
         filter_builder = emitter.events.LogDynamicArgs.build_filter()
@@ -104,7 +114,9 @@ def test_data_filters_with_fixed_arguments(
         create_filter,
         api_style,
         vals,
+        tester_snapshot,
         request):
+    clear_chain_state(web3, tester_snapshot)
 
     if api_style == 'build_filter':
         filter_builder = emitter.events.LogQuadrupleArg.build_filter()
@@ -156,7 +168,9 @@ def test_data_filters_with_list_arguments(
         create_filter,
         api_style,
         vals,
+        tester_snapshot,
         request):
+    clear_chain_state(web3, tester_snapshot)
 
     matching, non_matching = vals
 

tests/core/filtering/test_contract_topic_filters.py

@@ -58,6 +58,14 @@ def array_values(draw):
     return (matching, non_matching)
 
 
+def clear_chain_state(web3, snapshot):
+    """Clear chain state
+    Hypothesis doesn't allow function scoped fixtures to re-run between test runs
+    so chain state needs to be explicitly cleared
+    """
+    web3.providers[0].ethereum_tester.revert_to_snapshot(snapshot)
+
+
 @pytest.mark.parametrize('api_style', ('v4', 'build_filter'))
 @given(vals=dynamic_values())
 @settings(max_examples=5, deadline=None)
@@ -68,7 +76,9 @@ def test_topic_filters_with_dynamic_arguments(
         emitter_event_ids,
         create_filter,
         api_style,
+        tester_snapshot,
         vals):
+    clear_chain_state(web3, tester_snapshot)
 
     if api_style == 'build_filter':
         filter_builder = emitter.events.LogDynamicArgs.build_filter()
@@ -110,7 +120,9 @@ def test_topic_filters_with_fixed_arguments(
         call_as_instance,
         create_filter,
         api_style,
+        tester_snapshot,
         vals):
+    clear_chain_state(web3, tester_snapshot)
 
     if api_style == 'build_filter':
         filter_builder = emitter.events.LogQuadrupleWithIndex.build_filter()
@@ -162,7 +174,9 @@ def test_topic_filters_with_list_arguments(
         call_as_instance,
         create_filter,
         api_style,
+        tester_snapshot,
         vals):
+    clear_chain_state(web3, tester_snapshot)
 
     matching, non_matching = vals
 

tests/core/filtering/test_filters_against_many_blocks.py

@@ -60,7 +60,7 @@ def test_event_filter_new_events(
         is_match = bool(random.randint(0, 1))
         if is_match:
             expected_match_counter += 1
-            matching_transact()
+            wait_for_transaction(web3, matching_transact())
             pad_with_transactions(web3)
             continue
         non_matching_transact()

tests/core/middleware/test_filter_middleware.py

@@ -0,0 +1,148 @@
+import pytest
+
+from web3 import Web3
+from web3.middleware import (
+    construct_result_generator_middleware,
+    local_filter_middleware,
+)
+from web3.middleware.filter import (
+    block_ranges,
+    iter_latest_block_ranges,
+)
+from web3.providers.base import (
+    BaseProvider,
+)
+
+
+class DummyProvider(BaseProvider):
+    def make_request(self, method, params):
+        raise NotImplementedError("Cannot make request for {0}:{1}".format(
+            method,
+            params,
+        ))
+
+
+@pytest.fixture(scope='function')
+def iter_block_number(start=0):
+    def iterator():
+        block_number = start
+        while True:
+            sent_value = (yield block_number)
+            if sent_value is not None:
+                block_number = sent_value
+    block_number = iterator()
+    next(block_number)
+    return block_number
+
+
+@pytest.fixture(scope='function')
+def result_generator_middleware(iter_block_number):
+    return construct_result_generator_middleware({
+        'eth_getLogs': lambda *_: ["middleware"],
+        'eth_getBlockByNumber': lambda *_: type('block', (object,), {'hash': 'middleware'}),
+        'net_version': lambda *_: 1,
+        'eth_blockNumber': lambda *_: next(iter_block_number),
+    })
+
+
+@pytest.fixture(scope='function')
+def w3_base():
+    return Web3(providers=[DummyProvider()], middlewares=[])
+
+
+@pytest.fixture(scope='function')
+def w3(w3_base, result_generator_middleware):
+    w3_base.middleware_stack.add(result_generator_middleware)
+    w3_base.middleware_stack.add(local_filter_middleware)
+    return w3_base
+
+
+@pytest.mark.parametrize("start, stop, expected", [
+    (2, 7, [
+        (2, 6),
+        (7, 7)
+    ]),
+    (0, 12, [
+        (0, 4),
+        (5, 9),
+        (10, 12)
+    ]),
+    (0, 15, [
+        (0, 4),
+        (5, 9),
+        (10, 14),
+        (15, 15)
+    ]),
+    (0, 0, [
+        (0, 0),
+    ]),
+    (1, 1, [
+        (1, 1),
+    ]),
+    (5, 0, TypeError),
+])
+def test_block_ranges(start, stop, expected):
+    if isinstance(expected, type) and issubclass(expected, Exception):
+        with pytest.raises(expected):
+            block_ranges(start, stop)
+    else:
+        actual = tuple(block_ranges(start, stop))
+        assert len(actual) == len(expected)
+        for actual, expected in zip(actual, expected):
+            assert actual == expected
+
+
+@pytest.mark.parametrize("from_block,to_block,current_block,expected", [
+    (0, 10, [10], [
+        (0, 10),
+    ]),
+    (0, 55, [0, 19, 55], [
+        (0, 0),
+        (1, 19),
+        (20, 55),
+    ]),
+])
+def test_iter_latest_block_ranges(
+        w3,
+        iter_block_number,
+        from_block,
+        to_block,
+        current_block,
+        expected):
+    latest_block_ranges = iter_latest_block_ranges(w3, from_block, to_block)
+    for index, block in enumerate(current_block):
+        iter_block_number.send(block)
+        expected_tuple = expected[index]
+        actual_tuple = next(latest_block_ranges)
+        assert actual_tuple == expected_tuple
+
+
+def test_pending_block_filter_middleware(w3):
+    with pytest.raises(NotImplementedError):
+        w3.eth.filter('pending')
+
+
+def test_local_filter_middleware(w3, iter_block_number):
+    block_filter = w3.eth.filter('latest')
+    iter_block_number.send(1)
+
+    log_filter = w3.eth.filter(filter_params={'fromBlock': 'latest'})
+
+    assert w3.eth.getFilterChanges(block_filter.filter_id) == ["middleware"]
+
+    iter_block_number.send(2)
+    results = w3.eth.getFilterChanges(log_filter.filter_id)
+    assert results == ["middleware"]
+
+    assert w3.eth.getFilterLogs(log_filter.filter_id) == ["middleware"]
+
+    filter_ids = (
+        block_filter.filter_id,
+        log_filter.filter_id
+    )
+
+    # Test that all ids are str types
+    assert all(isinstance(_filter_id, (str,)) for _filter_id in filter_ids)
+
+    # Test that all ids are unique
+    assert len(filter_ids) == len(set(filter_ids))

web3/middleware/__init__.py

@@ -17,6 +17,9 @@
 from .exception_handling import (  # noqa: F401
     construct_exception_handler_middleware,
 )
+from .filter import (  # noqa: F401
+    local_filter_middleware,
+)
 from .fixture import (  # noqa: F401
     construct_fixture_middleware,
     construct_result_generator_middleware,