662 duplicates are not supported in requirements.txt when run with disable pip

pip_audit/_dependency_source/requirement.py

@@ -224,18 +224,23 @@ def _fix_file(self, filename: Path, fix_version: ResolvedFixVersion) -> None:
         # Check ahead of time for anything invalid in the requirements file since we don't want to
         # encounter this while writing out the file. Check for duplicate requirements and lines that
         # failed to parse.
-        req_names: set[str] = set()
+        req_specifiers: dict[str, SpecifierSet] = dict()
+
         for req in reqs:
             if (
                 isinstance(req, InstallRequirement)
                 and (req.marker is None or req.marker.evaluate())
                 and req.req is not None
             ):
-                if req.name in req_names:
+                duplicate_req_specifier = req_specifiers.get(req.name)
+
+                if not duplicate_req_specifier:
+                    req_specifiers[req.name] = req.specifier
+
+                elif duplicate_req_specifier != req.specifier:
                     raise RequirementFixError(
                         f"package {req.name} has duplicate requirements: {str(req)}"
                     )
-                req_names.add(req.name)
             elif isinstance(req, InvalidRequirementLine):
                 raise RequirementFixError(
                     f"requirement file {filename} has invalid requirement: {str(req)}"
@@ -292,7 +297,7 @@ def _collect_preresolved_deps(
         """
         Collect pre-resolved (pinned) dependencies.
         """
-        req_names: set[str] = set()
+        req_specifiers: dict[str, SpecifierSet] = dict()
         for req in reqs:
             if not req.hash_options and require_hashes:
                 raise RequirementSourceError(f"requirement {req.dumps()} does not contain a hash")
@@ -313,12 +318,21 @@ def _collect_preresolved_deps(
             if req.marker is not None and not req.marker.evaluate():
                 continue  # pragma: no cover
 
-            # This means we have a duplicate requirement for the same package
-            if req.name in req_names:
+            duplicate_req_specifier = req_specifiers.get(req.name)
+
+            if not duplicate_req_specifier:
+                req_specifiers[req.name] = req.specifier
+
+            # We have a duplicate requirement for the same package
+            # but different specifiers, meaning a badly resolved requirements.txt
+            elif duplicate_req_specifier != req.specifier:
                 raise RequirementSourceError(
                     f"package {req.name} has duplicate requirements: {str(req)}"
                 )
-            req_names.add(req.name)
+            else:
+                # We have a duplicate requirement for the same package and the specifier matches
+                # As they would return the same result from the audit, there no need to yield it a second time.
+                continue
 
             # NOTE: URL dependencies cannot be pinned, so skipping them
             # makes sense (under the same principle of skipping dependencies

test/dependency_source/test_requirement.py

@@ -526,6 +526,30 @@ def test_requirement_source_disable_pip_duplicate_dependencies(req_file):
         [(req_file(), "flask==1.0\nflask==1.0")], disable_pip=True, no_deps=True
     )
 
+    specs = list(source.collect())
+
+    # If the dependency list has duplicates, then converting to a set will reduce the length of the
+    # collection
+    assert len(specs) == len(set(specs))
+
+
+def test_requirement_source_disable_pip_duplicate_dependencies_with_extras(req_file):
+    source = _init_requirement(
+        [(req_file(), "aiohttp==3.9\naiohttp[speedups]==3.9")], disable_pip=True, no_deps=True
+    )
+
+    specs = list(source.collect())
+
+    # If the dependency list has duplicates, then converting to a set will reduce the length of the
+    # collection
+    assert len(specs) == len(set(specs))
+
+
+def test_requirement_source_disable_pip_duplicate_dependencies_diff_specifier(req_file):
+    source = _init_requirement(
+        [(req_file(), "flask==1.0\nflask==2.0")], disable_pip=True, no_deps=True
+    )
+
     with pytest.raises(DependencySourceError):
         list(source.collect())