🧪 Rewire pytest fixtures avoiding import loops

This patch also refactors and reduces the duplication of the previously
existing fixtures for retrieving different multidict module
implementations and makes the c-extension testing controllable by a CLI
option on the pytest level.

Fixes aio-libs#837

tests/conftest.py

@@ -1,25 +1,142 @@
+from __future__ import annotations
+
 import pickle
+from argparse import BooleanOptionalAction
+from dataclasses import dataclass
+from functools import cached_property
+from importlib import import_module
+from types import ModuleType
+from typing import Callable, Type
 
 import pytest
 
-from multidict._compat import USE_EXTENSIONS
+from multidict import MultiMapping, MutableMultiMapping
+
+
+@dataclass(frozen=True)
+class MultidictImplementation:
+    is_pure_python: bool
 
-OPTIONAL_CYTHON = (
-    ()
-    if USE_EXTENSIONS
-    else pytest.mark.skip(reason="No extensions available")
+    @cached_property
+    def tag(self) -> str:
+        return "pure-python" if self.is_pure_python else "c-extension"
+
+    @cached_property
+    def imported_module(self) -> ModuleType:
+        importable_module = (
+            "_multidict_py" if self.is_pure_python
+            else "_multidict"
+        )
+        return import_module(f"multidict.{importable_module}")
+
+    def __str__(self):
+        return f"{self.tag}-module"
+
+
+@pytest.fixture(
+    scope="session",
+    params=(
+        MultidictImplementation(is_pure_python=False),
+        MultidictImplementation(is_pure_python=True),
+    ),
+    ids=str,
 )
+def multidict_implementation(request) -> MultidictImplementation:
+    multidict_implementation = request.param
+    test_c_extensions = request.config.getoption("--c-extensions") is True
 
+    if not test_c_extensions and not multidict_implementation.is_pure_python:
+        pytest.skip("C-extension testing not requested")
 
-@pytest.fixture(  # type: ignore[call-overload]
+    return multidict_implementation
+
+
+@pytest.fixture(scope="session")
+def multidict_module(
+    multidict_implementation: MultidictImplementation,
+) -> ModuleType:
+    return multidict_implementation.imported_module
+
+
+@pytest.fixture(
     scope="session",
-    params=[
-        pytest.param("multidict._multidict", marks=OPTIONAL_CYTHON),  # type: ignore
-        "multidict._multidict_py",
-    ],
+    params=("MultiDict", "CIMultiDict"),
+    ids=("case-sensitive", "case-insensitive"),
 )
-def _multidict(request):
-    return pytest.importorskip(request.param)
+def any_multidict_class_name(request: pytest.FixtureRequest) -> str:
+    return request.param
+
+
+@pytest.fixture(scope="session")
+def multidict_class(
+    any_multidict_class_name: str,
+    multidict_module: ModuleType,
+) -> Type[MutableMultiMapping[str]]:
+    return getattr(multidict_module, any_multidict_class_name)
+
+
+@pytest.fixture(scope="session")
+def case_sensitive_multidict_class(
+    multidict_module: ModuleType,
+) -> Type[MutableMultiMapping[str]]:
+    return multidict_module.MultiDict
+
+
+@pytest.fixture(scope="session")
+def case_insensitive_multidict_class(
+    multidict_module: ModuleType,
+) -> Type[MutableMultiMapping[str]]:
+    return multidict_module.CIMultiDict
+
+
+@pytest.fixture(scope="session")
+def case_insensitive_str_class(multidict_module: ModuleType) -> Type[str]:
+    return multidict_module.istr
+
+
+@pytest.fixture(scope="session")
+def any_multidict_proxy_class_name(any_multidict_class_name: str) -> str:
+    return f"{any_multidict_class_name}Proxy"
+
+
+@pytest.fixture(scope="session")
+def any_multidict_proxy_class(
+    any_multidict_proxy_class_name: str,
+    multidict_module: ModuleType,
+) -> Type[MultiMapping[str]]:
+    return getattr(multidict_module, any_multidict_proxy_class_name)
+
+
+@pytest.fixture(scope="session")
+def case_sensitive_multidict_proxy_class(
+    multidict_module: ModuleType,
+) -> Type[MutableMultiMapping[str]]:
+    return multidict_module.MultiDictProxy
+
+
+@pytest.fixture(scope="session")
+def case_insensitive_multidict_proxy_class(
+    multidict_module: ModuleType,
+) -> Type[MutableMultiMapping[str]]:
+    return multidict_module.CIMultiDictProxy
+
+
+@pytest.fixture(scope="session")
+def multidict_getversion_callable(multidict_module: ModuleType) -> Callable:
+    return multidict_module.getversion
+
+
+def pytest_addoption(
+    parser: pytest.Parser,
+    pluginmanager: pytest.PytestPluginManager,
+) -> None:
+    del pluginmanager
+    parser.addoption(
+        "--c-extensions",  # disabled with `--no-c-extensions`
+        action=BooleanOptionalAction,
+        default=True,
+        help="Test C-extensions (on by default)",
+    )
 
 
 def pytest_generate_tests(metafunc):

tests/gen_pickles.py

@@ -1,31 +1,24 @@
 import pickle
 
-from multidict._compat import USE_EXTENSIONS
-from multidict._multidict_py import CIMultiDict as PyCIMultiDict  # noqa
-from multidict._multidict_py import MultiDict as PyMultiDict  # noqa
+import multidict
 
-try:
-    from multidict._multidict import (  # type: ignore # noqa
-        CIMultiDict,
-        MultiDict,
-    )
-except ImportError:
-    pass
 
-
-def write(name, proto):
-    cls = globals()[name]
+def write(tag, cls, proto):
     d = cls([("a", 1), ("a", 2)])
-    with open("{}.pickle.{}".format(name.lower(), proto), "wb") as f:
+    file_basename = f"{cls.__name__.lower()}-{tag}"
+    with open(f"{file_basename}.pickle.{proto}", "wb") as f:
         pickle.dump(d, f, proto)
 
 
 def generate():
-    if not USE_EXTENSIONS:
-        raise RuntimeError("C Extension is required")
+    _impl_map = {
+        "c-extension": multidict._multidict,
+        "pure-python": multidict._multidict_py,
+    }
     for proto in range(pickle.HIGHEST_PROTOCOL + 1):
-        for name in ("MultiDict", "CIMultiDict", "PyMultiDict", "PyCIMultiDict"):
-            write(name, proto)
+        for tag, impl in _impl_map.items():
+            for cls in impl.CIMultiDict, impl.MultiDict:
+                write(tag, cls, proto)
 
 
 if __name__ == "__main__":

tests/test_abc.py

@@ -3,43 +3,6 @@
 import pytest
 
 from multidict import MultiMapping, MutableMultiMapping
-from multidict._compat import USE_EXTENSIONS
-from multidict._multidict_py import CIMultiDict as PyCIMultiDict
-from multidict._multidict_py import CIMultiDictProxy as PyCIMultiDictProxy
-from multidict._multidict_py import MultiDict as PyMultiDict  # noqa: E402
-from multidict._multidict_py import MultiDictProxy as PyMultiDictProxy
-
-if USE_EXTENSIONS:
-    from multidict._multidict import (  # type: ignore
-        CIMultiDict,
-        CIMultiDictProxy,
-        MultiDict,
-        MultiDictProxy,
-    )
-
-
-@pytest.fixture(
-    params=([MultiDict, CIMultiDict] if USE_EXTENSIONS else [])
-    + [PyMultiDict, PyCIMultiDict],
-    ids=(["MultiDict", "CIMultiDict"] if USE_EXTENSIONS else [])
-    + ["PyMultiDict", "PyCIMultiDict"],
-)
-def cls(request):
-    return request.param
-
-
-@pytest.fixture(
-    params=(
-        [(MultiDictProxy, MultiDict), (CIMultiDictProxy, CIMultiDict)]
-        if USE_EXTENSIONS
-        else []
-    )
-    + [(PyMultiDictProxy, PyMultiDict), (PyCIMultiDictProxy, PyCIMultiDict)],
-    ids=(["MultiDictProxy", "CIMultiDictProxy"] if USE_EXTENSIONS else [])
-    + ["PyMultiDictProxy", "PyCIMultiDictProxy"],
-)
-def proxy_classes(request):
-    return request.param
 
 
 def test_abc_inheritance():
@@ -116,15 +79,14 @@ def test_abc_popall():
         B().popall("key")
 
 
-def test_multidict_inheritance(cls):
-    assert issubclass(cls, MultiMapping)
-    assert issubclass(cls, MutableMultiMapping)
+def test_multidict_inheritance(multidict_class):
+    assert issubclass(multidict_class, MultiMapping)
+    assert issubclass(multidict_class, MutableMultiMapping)
 
 
-def test_proxy_inheritance(proxy_classes):
-    proxy, _ = proxy_classes
-    assert issubclass(proxy, MultiMapping)
-    assert not issubclass(proxy, MutableMultiMapping)
+def test_proxy_inheritance(any_multidict_proxy_class):
+    assert issubclass(any_multidict_proxy_class, MultiMapping)
+    assert not issubclass(any_multidict_proxy_class, MutableMultiMapping)
 
 
 def test_generic_type_in_runtime():

tests/test_copy.py

@@ -1,79 +1,38 @@
 import copy
 
-import pytest
 
-from multidict._compat import USE_EXTENSIONS
-from multidict._multidict_py import CIMultiDict as PyCIMultiDict
-from multidict._multidict_py import CIMultiDictProxy as PyCIMultiDictProxy
-from multidict._multidict_py import MultiDict as PyMultiDict  # noqa: E402
-from multidict._multidict_py import MultiDictProxy as PyMultiDictProxy
-
-if USE_EXTENSIONS:
-    from multidict._multidict import (  # type: ignore
-        CIMultiDict,
-        CIMultiDictProxy,
-        MultiDict,
-        MultiDictProxy,
-    )
-
-
-@pytest.fixture(
-    params=([MultiDict, CIMultiDict] if USE_EXTENSIONS else [])
-    + [PyMultiDict, PyCIMultiDict],
-    ids=(["MultiDict", "CIMultiDict"] if USE_EXTENSIONS else [])
-    + ["PyMultiDict", "PyCIMultiDict"],
-)
-def cls(request):
-    return request.param
-
-
-@pytest.fixture(
-    params=(
-        [(MultiDictProxy, MultiDict), (CIMultiDictProxy, CIMultiDict)]
-        if USE_EXTENSIONS
-        else []
-    )
-    + [(PyMultiDictProxy, PyMultiDict), (PyCIMultiDictProxy, PyCIMultiDict)],
-    ids=(["MultiDictProxy", "CIMultiDictProxy"] if USE_EXTENSIONS else [])
-    + ["PyMultiDictProxy", "PyCIMultiDictProxy"],
-)
-def proxy_classes(request):
-    return request.param
-
-
-def test_copy(cls):
-    d = cls()
+def test_copy(multidict_class):
+    d = multidict_class()
     d["foo"] = 6
     d2 = d.copy()
     d2["foo"] = 7
     assert d["foo"] == 6
     assert d2["foo"] == 7
 
 
-def test_copy_proxy(proxy_classes):
-    proxy_cls, dict_cls = proxy_classes
-    d = dict_cls()
+def test_copy_proxy(multidict_class, any_multidict_proxy_class):
+    d = multidict_class()
     d["foo"] = 6
-    p = proxy_cls(d)
+    p = any_multidict_proxy_class(d)
     d2 = p.copy()
     d2["foo"] = 7
     assert d["foo"] == 6
     assert p["foo"] == 6
     assert d2["foo"] == 7
 
 
-def test_copy_std_copy(cls):
-    d = cls()
+def test_copy_std_copy(multidict_class):
+    d = multidict_class()
     d["foo"] = 6
     d2 = copy.copy(d)
     d2["foo"] = 7
     assert d["foo"] == 6
     assert d2["foo"] == 7
 
 
-def test_ci_multidict_clone(cls):
-    d = cls(foo=6)
-    d2 = cls(d)
+def test_ci_multidict_clone(multidict_class):
+    d = multidict_class(foo=6)
+    d2 = multidict_class(d)
     d2["foo"] = 7
     assert d["foo"] == 6
     assert d2["foo"] == 7