Add a --use-zipapp option to the test suite

.github/workflows/ci.yml

@@ -221,6 +221,35 @@ jobs:
         env:
           TEMP: "R:\\Temp"
 
+  tests-zipapp:
+    name: tests / zipapp
+    runs-on: ubuntu-latest
+
+    needs: [pre-commit, packaging, determine-changes]
+    if: >-
+      needs.determine-changes.outputs.tests == 'true' ||
+      github.event_name != 'pull_request'
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+
+      - name: Install Ubuntu dependencies
+        run: sudo apt-get install bzr
+
+      - run: pip install nox 'virtualenv<20' 'setuptools != 60.6.0'
+
+      # Main check
+      - name: Run integration tests
+        run: >-
+          nox -s test-3.10 --
+          -m integration
+          --verbose --numprocesses auto --showlocals
+          --durations=5
+          --use-zipapp
+
   # TODO: Remove this when we add Python 3.11 to CI.
   tests-importlib-metadata:
     name: tests for importlib.metadata backend

news/11250.feature.rst

@@ -0,0 +1 @@
+Add an option to run the test suite with pip built as a zipapp.

tests/conftest.py

@@ -19,6 +19,7 @@
     Union,
 )
 from unittest.mock import patch
+from zipfile import ZipFile
 
 import pytest
 
@@ -33,6 +34,7 @@
 from installer.destinations import SchemeDictionaryDestination
 from installer.sources import WheelFile
 
+from pip import __file__ as pip_location
 from pip._internal.cli.main import main as pip_entry_point
 from pip._internal.locations import _USE_SYSCONFIG
 from pip._internal.utils.temp_dir import global_tempdir_manager
@@ -85,6 +87,12 @@ def pytest_addoption(parser: Parser) -> None:
         default=None,
         help="use given proxy in session network tests",
     )
+    parser.addoption(
+        "--use-zipapp",
+        action="store_true",
+        default=False,
+        help="use a zipapp when running pip in tests",
+    )
 
 
 def pytest_collection_modifyitems(config: Config, items: List[pytest.Function]) -> None:
@@ -513,10 +521,13 @@ def __call__(
 
 @pytest.fixture(scope="session")
 def script_factory(
-    virtualenv_factory: Callable[[Path], VirtualEnvironment], deprecated_python: bool
+    virtualenv_factory: Callable[[Path], VirtualEnvironment],
+    deprecated_python: bool,
+    zipapp: Optional[str],
 ) -> ScriptFactory:
     def factory(
-        tmpdir: Path, virtualenv: Optional[VirtualEnvironment] = None
+        tmpdir: Path,
+        virtualenv: Optional[VirtualEnvironment] = None,
     ) -> PipTestEnvironment:
         if virtualenv is None:
             virtualenv = virtualenv_factory(tmpdir.joinpath("venv"))
@@ -535,13 +546,64 @@ def factory(
             assert_no_temp=True,
             # Deprecated python versions produce an extra deprecation warning
             pip_expect_warning=deprecated_python,
+            # Tell the Test Environment if we want to run pip via a zipapp
+            zipapp=zipapp,
         )
 
     return factory
 
 
+ZIPAPP_MAIN = """\
+#!/usr/bin/env python
+
+import os
+import runpy
+import sys
+
+lib = os.path.join(os.path.dirname(__file__), "lib")
+sys.path.insert(0, lib)
+
+runpy.run_module("pip", run_name="__main__")
+"""
+
+
+def make_zipapp_from_pip(zipapp_name: Path) -> None:
+    pip_dir = Path(pip_location).parent
+    with zipapp_name.open("wb") as zipapp_file:
+        zipapp_file.write(b"#!/usr/bin/env python\n")
+        with ZipFile(zipapp_file, "w") as zipapp:
+            for pip_file in pip_dir.rglob("*"):
+                if pip_file.suffix == ".pyc":
+                    continue
+                if pip_file.name == "__pycache__":
+                    continue
+                rel_name = pip_file.relative_to(pip_dir.parent)
+                zipapp.write(pip_file, arcname=f"lib/{rel_name}")
+            zipapp.writestr("__main__.py", ZIPAPP_MAIN)
+
+
+@pytest.fixture(scope="session")
+def zipapp(
+    request: pytest.FixtureRequest, tmpdir_factory: pytest.TempPathFactory
+) -> Optional[str]:
+    """
+    If the user requested for pip to be run from a zipapp, build that zipapp
+    and return its location. If the user didn't request a zipapp, return None.
+
+    This fixture is session scoped, so the zipapp will only be created once.
+    """
+    if not request.config.getoption("--use-zipapp"):
+        return None
+
+    temp_location = tmpdir_factory.mktemp("zipapp")
+    pyz_file = temp_location / "pip.pyz"
+    make_zipapp_from_pip(pyz_file)
+    return str(pyz_file)
+
+
 @pytest.fixture
 def script(
+    request: pytest.FixtureRequest,
     tmpdir: Path,
     virtualenv: VirtualEnvironment,
     script_factory: ScriptFactory,

tests/functional/test_cli.py

@@ -16,6 +16,9 @@
     ],
 )
 def test_entrypoints_work(entrypoint: str, script: PipTestEnvironment) -> None:
+    if script.zipapp:
+        pytest.skip("Zipapp does not include entrypoints")
+
     fake_pkg = script.temp_path / "fake_pkg"
     fake_pkg.mkdir()
     fake_pkg.joinpath("setup.py").write_text(

tests/functional/test_completion.py

@@ -107,7 +107,12 @@ def test_completion_for_supported_shells(
     Test getting completion for bash shell
     """
     result = script_with_launchers.pip("completion", "--" + shell, use_module=False)
-    assert completion in result.stdout, str(result.stdout)
+    actual = str(result.stdout)
+    if script_with_launchers.zipapp:
+        # The zipapp reports its name as "pip.pyz", but the expected
+        # output assumes "pip"
+        actual = actual.replace("pip.pyz", "pip")
+    assert completion in actual, actual
 
 
 @pytest.fixture(scope="session")

tests/functional/test_pip_runner_script.py

@@ -12,7 +12,9 @@ def test_runner_work_in_environments_with_no_pip(
 
     # Ensure there's no pip installed in the environment
     script.pip("uninstall", "pip", "--yes", use_module=True)
-    script.pip("--version", expect_error=True)
+    # We don't use script.pip to check here, as when testing a
+    # zipapp, script.pip will run pip from the zipapp.
+    script.run("python", "-c", "import pip", expect_error=True)
 
     # The runner script should still invoke a usable pip
     result = script.run("python", os.fspath(runner), "--version")

tests/lib/__init__.py

@@ -505,6 +505,7 @@ def __init__(
         *args: Any,
         virtualenv: VirtualEnvironment,
         pip_expect_warning: bool = False,
+        zipapp: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
         # Store paths related to the virtual environment
@@ -551,6 +552,9 @@ def __init__(
         # (useful for Python version deprecation)
         self.pip_expect_warning = pip_expect_warning
 
+        # The name of an (optional) zipapp to use when running pip
+        self.zipapp = zipapp
+
         # Call the TestFileEnvironment __init__
         super().__init__(base_path, *args, **kwargs)
 
@@ -585,6 +589,10 @@ def __init__(
     def _ignore_file(self, fn: str) -> bool:
         if fn.endswith("__pycache__") or fn.endswith(".pyc"):
             result = True
+        elif self.zipapp and fn.endswith("cacert.pem"):
+            # Temporary copies of cacert.pem are extracted
+            # when running from a zipapp
+            result = True
         else:
             result = super()._ignore_file(fn)
         return result
@@ -696,7 +704,10 @@ def pip(
         __tracebackhide__ = True
         if self.pip_expect_warning:
             kwargs["allow_stderr_warning"] = True
-        if use_module:
+        if self.zipapp:
+            exe = "python"
+            args = (self.zipapp,) + args
+        elif use_module:
             exe = "python"
             args = ("-m", "pip") + args
         else:

tests/lib/test_lib.py

@@ -41,6 +41,10 @@ def test_correct_pip_version(script: PipTestEnvironment) -> None:
     """
     Check we are running proper version of pip in run_pip.
     """
+
+    if script.zipapp:
+        pytest.skip("Test relies on the pip under test being in the filesystem")
+
     # output is like:
     # pip PIPVERSION from PIPDIRECTORY (python PYVERSION)
     result = script.pip("--version")