diff --git a/.circleci/config.yml b/.circleci/config.yml index 854edd5b47..bd6cb28de4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -248,60 +248,60 @@ jobs: # Please don't use this key for any shenanigans WEB3_INFURA_PROJECT_ID: 7707850c2fb7465ebe6f150d67182e22 - py37-integration-goethereum-ipc_eth: + py37-integration-goethereum-ipc: <<: *geth_steps docker: - image: cimg/python:3.7 environment: - TOXENV: py37-integration-goethereum-ipc_eth + TOXENV: py37-integration-goethereum-ipc GETH_VERSION: v1.10.23 - py37-integration-goethereum-ipc_non_eth: + py37-integration-goethereum-ipc_flaky: <<: *geth_steps docker: - image: cimg/python:3.7 environment: - TOXENV: py37-integration-goethereum-ipc_non_eth + TOXENV: py37-integration-goethereum-ipc_flaky GETH_VERSION: v1.10.23 - py37-integration-goethereum-http_eth: + py37-integration-goethereum-http: <<: *geth_steps docker: - image: cimg/python:3.7 environment: - TOXENV: py37-integration-goethereum-http_eth + TOXENV: py37-integration-goethereum-http GETH_VERSION: v1.10.23 - py37-integration-goethereum-http_non_eth: + py37-integration-goethereum-http_async: <<: *geth_steps docker: - image: cimg/python:3.7 environment: - TOXENV: py37-integration-goethereum-http_non_eth + TOXENV: py37-integration-goethereum-http_async GETH_VERSION: v1.10.23 - py37-integration-goethereum-http_async: + py37-integration-goethereum-http_flaky: <<: *geth_steps docker: - image: cimg/python:3.7 environment: - TOXENV: py37-integration-goethereum-http_async + TOXENV: py37-integration-goethereum-http_flaky GETH_VERSION: v1.10.23 - py37-integration-goethereum-ws_eth: + py37-integration-goethereum-ws: <<: *geth_steps docker: - image: cimg/python:3.7 environment: - TOXENV: py37-integration-goethereum-ws_eth + TOXENV: py37-integration-goethereum-ws GETH_VERSION: v1.10.23 - py37-integration-goethereum-ws_non_eth: + py37-integration-goethereum-ws_flaky: <<: *geth_steps docker: - image: cimg/python:3.7 environment: - TOXENV: py37-integration-goethereum-ws_non_eth + TOXENV: py37-integration-goethereum-ws_flaky GETH_VERSION: v1.10.23 py37-integration-ethtester-pyevm: @@ -350,60 +350,60 @@ jobs: # Please don't use this key for any shenanigans WEB3_INFURA_PROJECT_ID: 7707850c2fb7465ebe6f150d67182e22 - py38-integration-goethereum-ipc_eth: + py38-integration-goethereum-ipc: <<: *geth_steps docker: - image: cimg/python:3.8 environment: - TOXENV: py38-integration-goethereum-ipc_eth + TOXENV: py38-integration-goethereum-ipc GETH_VERSION: v1.10.23 - py38-integration-goethereum-ipc_non_eth: + py38-integration-goethereum-ipc_flaky: <<: *geth_steps docker: - image: cimg/python:3.8 environment: - TOXENV: py38-integration-goethereum-ipc_non_eth + TOXENV: py38-integration-goethereum-ipc_flaky GETH_VERSION: v1.10.23 - py38-integration-goethereum-http_eth: + py38-integration-goethereum-http: <<: *geth_steps docker: - image: cimg/python:3.8 environment: - TOXENV: py38-integration-goethereum-http_eth + TOXENV: py38-integration-goethereum-http GETH_VERSION: v1.10.23 - py38-integration-goethereum-http_non_eth: + py38-integration-goethereum-http_async: <<: *geth_steps docker: - image: cimg/python:3.8 environment: - TOXENV: py38-integration-goethereum-http_non_eth + TOXENV: py38-integration-goethereum-http_async GETH_VERSION: v1.10.23 - py38-integration-goethereum-http_async: + py38-integration-goethereum-http_flaky: <<: *geth_steps docker: - image: cimg/python:3.8 environment: - TOXENV: py38-integration-goethereum-http_async + TOXENV: py38-integration-goethereum-http_flaky GETH_VERSION: v1.10.23 - py38-integration-goethereum-ws_eth: + py38-integration-goethereum-ws: <<: *geth_steps docker: - image: cimg/python:3.8 environment: - TOXENV: py38-integration-goethereum-ws_eth + TOXENV: py38-integration-goethereum-ws GETH_VERSION: v1.10.23 - py38-integration-goethereum-ws_non_eth: + py38-integration-goethereum-ws_flaky: <<: *geth_steps docker: - image: cimg/python:3.8 environment: - TOXENV: py38-integration-goethereum-ws_non_eth + TOXENV: py38-integration-goethereum-ws_flaky GETH_VERSION: v1.10.23 py38-integration-ethtester-pyevm: @@ -447,60 +447,60 @@ jobs: # Please don't use this key for any shenanigans WEB3_INFURA_PROJECT_ID: 7707850c2fb7465ebe6f150d67182e22 - py39-integration-goethereum-ipc_eth: + py39-integration-goethereum-ipc: <<: *geth_steps docker: - image: cimg/python:3.9 environment: - TOXENV: py39-integration-goethereum-ipc_eth + TOXENV: py39-integration-goethereum-ipc GETH_VERSION: v1.10.23 - py39-integration-goethereum-ipc_non_eth: + py39-integration-goethereum-ipc_flaky: <<: *geth_steps docker: - image: cimg/python:3.9 environment: - TOXENV: py39-integration-goethereum-ipc_non_eth + TOXENV: py39-integration-goethereum-ipc_flaky GETH_VERSION: v1.10.23 - py39-integration-goethereum-http_eth: + py39-integration-goethereum-http: <<: *geth_steps docker: - image: cimg/python:3.9 environment: - TOXENV: py39-integration-goethereum-http_eth + TOXENV: py39-integration-goethereum-http GETH_VERSION: v1.10.23 - py39-integration-goethereum-http_non_eth: + py39-integration-goethereum-http_async: <<: *geth_steps docker: - image: cimg/python:3.9 environment: - TOXENV: py39-integration-goethereum-http_non_eth + TOXENV: py39-integration-goethereum-http_async GETH_VERSION: v1.10.23 - py39-integration-goethereum-http_async: + py39-integration-goethereum-http_flaky: <<: *geth_steps docker: - image: cimg/python:3.9 environment: - TOXENV: py39-integration-goethereum-http_async + TOXENV: py39-integration-goethereum-http_flaky GETH_VERSION: v1.10.23 - py39-integration-goethereum-ws_eth: + py39-integration-goethereum-ws: <<: *geth_steps docker: - image: cimg/python:3.9 environment: - TOXENV: py39-integration-goethereum-ws_eth + TOXENV: py39-integration-goethereum-ws GETH_VERSION: v1.10.23 - py39-integration-goethereum-ws_non_eth: + py39-integration-goethereum-ws_flaky: <<: *geth_steps docker: - image: cimg/python:3.9 environment: - TOXENV: py39-integration-goethereum-ws_non_eth + TOXENV: py39-integration-goethereum-ws_flaky GETH_VERSION: v1.10.23 py39-integration-ethtester-pyevm: @@ -544,60 +544,60 @@ jobs: # Please don't use this key for any shenanigans WEB3_INFURA_PROJECT_ID: 7707850c2fb7465ebe6f150d67182e22 - py310-integration-goethereum-ipc_eth: + py310-integration-goethereum-ipc: <<: *geth_steps docker: - image: cimg/python:3.10 environment: - TOXENV: py310-integration-goethereum-ipc_eth + TOXENV: py310-integration-goethereum-ipc GETH_VERSION: v1.10.23 - py310-integration-goethereum-ipc_non_eth: + py310-integration-goethereum-ipc_flaky: <<: *geth_steps docker: - image: cimg/python:3.10 environment: - TOXENV: py310-integration-goethereum-ipc_non_eth + TOXENV: py310-integration-goethereum-ipc_flaky GETH_VERSION: v1.10.23 - py310-integration-goethereum-http_eth: + py310-integration-goethereum-http: <<: *geth_steps docker: - image: cimg/python:3.10 environment: - TOXENV: py310-integration-goethereum-http_eth + TOXENV: py310-integration-goethereum-http GETH_VERSION: v1.10.23 - py310-integration-goethereum-http_non_eth: + py310-integration-goethereum-http_async: <<: *geth_steps docker: - image: cimg/python:3.10 environment: - TOXENV: py310-integration-goethereum-http_non_eth + TOXENV: py310-integration-goethereum-http_async GETH_VERSION: v1.10.23 - py310-integration-goethereum-http_async: + py310-integration-goethereum-http_flaky: <<: *geth_steps docker: - image: cimg/python:3.10 environment: - TOXENV: py310-integration-goethereum-http_async + TOXENV: py310-integration-goethereum-http_flaky GETH_VERSION: v1.10.23 - py310-integration-goethereum-ws_eth: + py310-integration-goethereum-ws: <<: *geth_steps docker: - image: cimg/python:3.10 environment: - TOXENV: py310-integration-goethereum-ws_eth + TOXENV: py310-integration-goethereum-ws GETH_VERSION: v1.10.23 - py310-integration-goethereum-ws_non_eth: + py310-integration-goethereum-ws_flaky: <<: *geth_steps docker: - image: cimg/python:3.10 environment: - TOXENV: py310-integration-goethereum-ws_non_eth + TOXENV: py310-integration-goethereum-ws_flaky GETH_VERSION: v1.10.23 py310-integration-ethtester-pyevm: @@ -637,46 +637,46 @@ workflows: - benchmark - py37-ens - py37-ethpm - - py37-integration-goethereum-ipc_eth - - py37-integration-goethereum-ipc_non_eth - - py37-integration-goethereum-http_eth - - py37-integration-goethereum-http_non_eth + - py37-integration-goethereum-ipc + - py37-integration-goethereum-ipc_flaky + - py37-integration-goethereum-http - py37-integration-goethereum-http_async - - py37-integration-goethereum-ws_eth - - py37-integration-goethereum-ws_non_eth + - py37-integration-goethereum-http_flaky + - py37-integration-goethereum-ws + - py37-integration-goethereum-ws_flaky - py37-integration-ethtester-pyevm - py37-wheel-cli - py37-wheel-cli-windows - py38-ens - py38-ethpm - - py38-integration-goethereum-ipc_eth - - py38-integration-goethereum-ipc_non_eth - - py38-integration-goethereum-http_eth - - py38-integration-goethereum-http_non_eth + - py38-integration-goethereum-ipc + - py38-integration-goethereum-ipc_flaky + - py38-integration-goethereum-http - py38-integration-goethereum-http_async - - py38-integration-goethereum-ws_eth - - py38-integration-goethereum-ws_non_eth + - py38-integration-goethereum-http_flaky + - py38-integration-goethereum-ws + - py38-integration-goethereum-ws_flaky - py38-integration-ethtester-pyevm - py38-wheel-cli - py39-ens - py39-ethpm - - py39-integration-goethereum-ipc_eth - - py39-integration-goethereum-ipc_non_eth - - py39-integration-goethereum-http_eth - - py39-integration-goethereum-http_non_eth + - py39-integration-goethereum-ipc + - py39-integration-goethereum-ipc_flaky + - py39-integration-goethereum-http - py39-integration-goethereum-http_async - - py39-integration-goethereum-ws_eth - - py39-integration-goethereum-ws_non_eth + - py39-integration-goethereum-http_flaky + - py39-integration-goethereum-ws + - py39-integration-goethereum-ws_flaky - py39-integration-ethtester-pyevm - py39-wheel-cli - py310-ens - py310-ethpm - - py310-integration-goethereum-ipc_eth - - py310-integration-goethereum-ipc_non_eth - - py310-integration-goethereum-http_eth - - py310-integration-goethereum-http_non_eth + - py310-integration-goethereum-ipc + - py310-integration-goethereum-ipc_flaky + - py310-integration-goethereum-http - py310-integration-goethereum-http_async - - py310-integration-goethereum-ws_eth - - py310-integration-goethereum-ws_non_eth + - py310-integration-goethereum-http_flaky + - py310-integration-goethereum-ws + - py310-integration-goethereum-ws_flaky - py310-integration-ethtester-pyevm - py310-wheel-cli diff --git a/newsfragments/2638.misc.rst b/newsfragments/2638.misc.rst new file mode 100644 index 0000000000..767871a3eb --- /dev/null +++ b/newsfragments/2638.misc.rst @@ -0,0 +1 @@ +Run some flaky tests related to unlocked account fixtures as a separate test run in the CI builds. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 62e6f0657f..a2339dbc81 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -18,6 +18,47 @@ REVERT_CONTRACT_BYTECODE, ) +# --- integration test configurations --- # + + +def pytest_addoption(parser): + parser.addoption("--flaky", action="store_true") + + +def pytest_collection_modifyitems(items, config): + """ + It is ideal to keep this configuration as simple as possible so that we don't + risk missing some tests. + """ + # TODO: See if there is a better way to address the timeout issues present + # in unlocked account tests. + + flaky_tests = [] + non_flaky_tests = [] + + for item in items: + if ( + # Unlocked account tests are problematic - separate them into their own + # test run. + any( + _ in item.fixturenames + for _ in ("unlocked_account", "unlocked_account_dual_type") + ) + # Leave offchain_lookup tests split between eth sync and async tests as + # those can conflict with each other as well. + and "offchain_lookup" not in item.name + ): + flaky_tests.append(item) + else: + non_flaky_tests.append(item) + + if config.option.flaky: + items[:] = flaky_tests + config.hook.pytest_deselected(items=non_flaky_tests) + else: + items[:] = non_flaky_tests + config.hook.pytest_deselected(items=flaky_tests) + @pytest.fixture(scope="module") def math_contract_factory(w3): diff --git a/tox.ini b/tox.ini index cecf799da5..63428676e0 100644 --- a/tox.ini +++ b/tox.ini @@ -34,13 +34,13 @@ commands= core: pytest {posargs:tests/core} ens: pytest {posargs:tests/ens} ethpm: pytest {posargs:tests/ethpm} - integration-goethereum-ipc_eth: pytest {posargs:tests/integration/go_ethereum/test_goethereum_ipc.py -k EthModule} - integration-goethereum-ipc_non_eth: pytest {posargs:tests/integration/go_ethereum/test_goethereum_ipc.py -k "not EthModule"} - integration-goethereum-http_eth: pytest {posargs:tests/integration/go_ethereum/test_goethereum_http.py -k "EthModule and not Async"} - integration-goethereum-http_non_eth: pytest {posargs:tests/integration/go_ethereum/test_goethereum_http.py -k "not EthModule and not Async"} + integration-goethereum-ipc: pytest {posargs:tests/integration/go_ethereum/test_goethereum_ipc.py} + integration-goethereum-ipc_flaky: pytest {posargs:tests/integration/go_ethereum/test_goethereum_ipc.py --flaky} + integration-goethereum-http: pytest {posargs:tests/integration/go_ethereum/test_goethereum_http.py -k "not Async"} integration-goethereum-http_async: pytest {posargs:tests/integration/go_ethereum/test_goethereum_http.py -k Async} - integration-goethereum-ws_eth: pytest {posargs:tests/integration/go_ethereum/test_goethereum_ws.py -k EthModule} - integration-goethereum-ws_non_eth: pytest {posargs:tests/integration/go_ethereum/test_goethereum_ws.py -k "not EthModule"} + integration-goethereum-http_flaky: pytest {posargs:tests/integration/go_ethereum/test_goethereum_http.py --flaky} + integration-goethereum-ws: pytest {posargs:tests/integration/go_ethereum/test_goethereum_ws.py} + integration-goethereum-ws_flaky: pytest {posargs:tests/integration/go_ethereum/test_goethereum_ws.py --flaky} integration-ethtester: pytest {posargs:tests/integration/test_ethereum_tester.py} docs: make -C {toxinidir} validate-docs deps =