_dependency_source: Return a SkippedDependency when the resolver can't find a project on PyPI

CHANGELOG.md

@@ -18,6 +18,12 @@ All versions prior to 0.0.9 are untracked.
   as a bare option, `--desc` is equivalent to `--desc on`
   ([#153](https://github.com/trailofbits/pip-audit/pull/153))
 
+* Dependency resolution: The PyPI-based dependency resolver no longer throws
+  an uncaught exception on package resolution errors; instead, the package
+  is marked as skipped and an appropriate warning or fatal error (in
+  `--strict` mode) is produced
+  ([#162](https://github.com/trailofbits/pip-audit/pull/162))
+
 ### Removed
 
 ## [1.0.0] - 2021-12-1

pip_audit/_dependency_source/interface.py

@@ -8,7 +8,7 @@
 
 from packaging.requirements import Requirement
 
-from pip_audit._service import Dependency, ResolvedDependency
+from pip_audit._service import Dependency
 
 
 class DependencySource(ABC):
@@ -48,15 +48,15 @@ class DependencyResolver(ABC):
     """
 
     @abstractmethod
-    def resolve(self, req: Requirement) -> List[ResolvedDependency]:  # pragma: no cover
+    def resolve(self, req: Requirement) -> List[Dependency]:  # pragma: no cover
         """
-        Resolve a single `Requirement` into a list of concrete `Dependency` instances.
+        Resolve a single `Requirement` into a list of `Dependency` instances.
         """
         raise NotImplementedError
 
     def resolve_all(
         self, reqs: Iterator[Requirement]
-    ) -> Iterator[Tuple[Requirement, List[ResolvedDependency]]]:
+    ) -> Iterator[Tuple[Requirement, List[Dependency]]]:
         """
         Resolve a collection of `Requirement`s into their respective `Dependency` sets.
 

pip_audit/_dependency_source/requirement.py

@@ -3,7 +3,7 @@
 """
 
 from pathlib import Path
-from typing import Iterator, List, Optional, Set
+from typing import Iterator, List, Optional, Set, cast
 
 from packaging.requirements import Requirement
 from pip_api import parse_requirements
@@ -16,6 +16,7 @@
     DependencySourceError,
 )
 from pip_audit._service import Dependency
+from pip_audit._service.interface import ResolvedDependency, SkippedDependency
 from pip_audit._state import AuditState
 
 
@@ -64,10 +65,13 @@ def collect(self) -> Iterator[Dependency]:
                         # Don't allow duplicate dependencies to be returned
                         if dep in collected:
                             continue
-                        if self.state is not None:
-                            self.state.update_state(
-                                f"Collecting {dep.name} ({dep.version})"
-                            )  # pragma: no cover
+                        if self.state is not None:  # pragma: no cover
+                            if dep.is_skipped():
+                                dep = cast(SkippedDependency, dep)
+                                self.state.update_state(f"Skipping {dep.name}: {dep.skip_reason}")
+                            else:
+                                dep = cast(ResolvedDependency, dep)
+                                self.state.update_state(f"Collecting {dep.name} ({dep.version})")
                         collected.add(dep)
                         yield dep
             except DependencyResolverError as dre:

pip_audit/_dependency_source/resolvelib/pypi_provider.py

@@ -185,6 +185,8 @@ def get_project_from_pypi(project, extras, timeout: Optional[int], state: Option
     """Return candidates created from the project name and extras."""
     url = "https://pypi.org/simple/{}".format(project)
     response: requests.Response = requests.get(url, timeout=timeout)
+    if response.status_code == 404:
+        raise PyPINotFoundError(f'Could not find project "{project}" on PyPI')
     response.raise_for_status()
     data = response.content
     doc = html5lib.parse(data, namespaceHTMLElements=False)
@@ -291,3 +293,11 @@ def get_dependencies(self, candidate):
         See `resolvelib.providers.AbstractProvider.get_dependencies`.
         """
         return candidate.dependencies
+
+
+class PyPINotFoundError(Exception):
+    """
+    An error to signify that the provider could not find the requested project on PyPI.
+    """
+
+    pass

pip_audit/_dependency_source/resolvelib/resolvelib.py

@@ -3,17 +3,20 @@
 `Resolver` that uses PyPI as an information source.
 """
 
+import logging
 from typing import List, Optional
 
 from packaging.requirements import Requirement
 from requests.exceptions import HTTPError
 from resolvelib import BaseReporter, Resolver
 
 from pip_audit._dependency_source import DependencyResolver, DependencyResolverError
-from pip_audit._service.interface import ResolvedDependency
+from pip_audit._service.interface import Dependency, ResolvedDependency, SkippedDependency
 from pip_audit._state import AuditState
 
-from .pypi_provider import PyPIProvider
+from .pypi_provider import PyPINotFoundError, PyPIProvider
+
+logger = logging.getLogger(__name__)
 
 
 class ResolveLibResolver(DependencyResolver):
@@ -32,13 +35,17 @@ def __init__(self, timeout: Optional[int] = None, state: Optional[AuditState] =
         self.reporter = BaseReporter()
         self.resolver: Resolver = Resolver(self.provider, self.reporter)
 
-    def resolve(self, req: Requirement) -> List[ResolvedDependency]:
+    def resolve(self, req: Requirement) -> List[Dependency]:
         """
         Resolve the given `Requirement` into a `Dependency` list.
         """
-        deps: List[ResolvedDependency] = []
+        deps: List[Dependency] = []
         try:
             result = self.resolver.resolve([req])
+        except PyPINotFoundError as e:
+            skip_reason = str(e)
+            logger.debug(skip_reason)
+            return [SkippedDependency(name=req.name, skip_reason=skip_reason)]
         except HTTPError as e:
             raise ResolveLibResolverError("failed to resolve dependencies") from e
         for name, candidate in result.mapping.items():

test/dependency_source/test_resolvelib.py

@@ -10,13 +10,14 @@
 
 from pip_audit._dependency_source import resolvelib
 from pip_audit._dependency_source.resolvelib import pypi_provider
-from pip_audit._service.interface import Dependency, ResolvedDependency
+from pip_audit._service.interface import Dependency, ResolvedDependency, SkippedDependency
 
 
 def get_package_mock(data):
     class Doc:
         def __init__(self, content):
             self.content = content
+            self.status_code = 200
 
         def raise_for_status(self):
             pass
@@ -239,6 +240,9 @@ def test_resolvelib_sdist_invalid_suffix(monkeypatch):
 def test_resolvelib_http_error(monkeypatch):
     def get_http_error_mock():
         class Doc:
+            def __init__(self):
+                self.status_code = 400
+
             def raise_for_status(self):
                 raise HTTPError
 
@@ -250,3 +254,24 @@ def raise_for_status(self):
     req = Requirement("flask==2.0.1")
     with pytest.raises(resolvelib.ResolveLibResolverError):
         dict(resolver.resolve_all([req]))
+
+
+def test_resolvelib_http_notfound(monkeypatch):
+    def get_http_not_found_mock():
+        class Doc:
+            def __init__(self):
+                self.status_code = 404
+
+        return Doc()
+
+    monkeypatch.setattr(requests, "get", lambda _url, **kwargs: get_http_not_found_mock())
+
+    resolver = resolvelib.ResolveLibResolver()
+    req = Requirement("flask==2.0.1")
+    resolved_deps = dict(resolver.resolve_all([req]))
+    assert len(resolved_deps) == 1
+    expected_deps = [
+        SkippedDependency(name="flask", skip_reason='Could not find project "flask" on PyPI')
+    ]
+    assert req in resolved_deps
+    assert resolved_deps[req] == expected_deps