662 duplicates are not supported in requirements.txt when run with disable pip

pip_audit/_dependency_source/pyproject.py

@@ -142,8 +142,8 @@ def fix(self, fix_version: ResolvedFixVersion) -> None:
             # Now dump the new edited TOML to the temporary file.
             toml.dump(pyproject_data, tmp)
 
-            # And replace the original `pyproject.toml` file.
-            os.replace(tmp.name, self.filename)
+        # And replace the original `pyproject.toml` file.
+        os.replace(tmp.name, self.filename)
 
 
 class PyProjectSourceError(DependencySourceError):

pip_audit/_dependency_source/requirement.py

@@ -225,18 +225,23 @@ def _fix_file(self, filename: Path, fix_version: ResolvedFixVersion) -> None:
         # Check ahead of time for anything invalid in the requirements file since we don't want to
         # encounter this while writing out the file. Check for duplicate requirements and lines that
         # failed to parse.
-        req_names: set[str] = set()
+        req_specifiers: dict[str, SpecifierSet] = dict()
+
         for req in reqs:
             if (
                 isinstance(req, InstallRequirement)
                 and (req.marker is None or req.marker.evaluate())
                 and req.req is not None
             ):
-                if req.name in req_names:
+                duplicate_req_specifier = req_specifiers.get(req.name)
+
+                if not duplicate_req_specifier:
+                    req_specifiers[req.name] = req.specifier
+
+                elif duplicate_req_specifier != req.specifier:
                     raise RequirementFixError(
                         f"package {req.name} has duplicate requirements: {str(req)}"
                     )
-                req_names.add(req.name)
             elif isinstance(req, InvalidRequirementLine):
                 raise RequirementFixError(
                     f"requirement file {filename} has invalid requirement: {str(req)}"
@@ -294,7 +299,7 @@ def _collect_preresolved_deps(
         """
         Collect pre-resolved (pinned) dependencies.
         """
-        req_names: set[str] = set()
+        req_specifiers: dict[str, SpecifierSet] = dict()
         for req in reqs:
             if not req.hash_options and require_hashes:
                 raise RequirementSourceError(f"requirement {req.dumps()} does not contain a hash")
@@ -315,12 +320,21 @@ def _collect_preresolved_deps(
             if req.marker is not None and not req.marker.evaluate():
                 continue  # pragma: no cover
 
-            # This means we have a duplicate requirement for the same package
-            if req.name in req_names:
+            duplicate_req_specifier = req_specifiers.get(req.name)
+
+            if not duplicate_req_specifier:
+                req_specifiers[req.name] = req.specifier
+
+            # We have a duplicate requirement for the same package
+            # but different specifiers, meaning a badly resolved requirements.txt
+            elif duplicate_req_specifier != req.specifier:
                 raise RequirementSourceError(
                     f"package {req.name} has duplicate requirements: {str(req)}"
                 )
-            req_names.add(req.name)
+            else:
+                # We have a duplicate requirement for the same package and the specifier matches
+                # As they would return the same result from the audit, there no need to yield it a second time.
+                continue
 
             # NOTE: URL dependencies cannot be pinned, so skipping them
             # makes sense (under the same principle of skipping dependencies

pip_audit/_subprocess.py

@@ -52,9 +52,8 @@ def run(args: Sequence[str], *, log_stdout: bool = False, state: AuditState = Au
     # once `stdout` hits EOF, so we don't have to worry about that blocking.
     while not terminated:
         terminated = process.poll() is not None
-        # NOTE(ww): Buffer size chosen arbitrarily here and below.
-        stdout += process.stdout.read(4096)  # type: ignore
-        stderr += process.stderr.read(4096)  # type: ignore
+        stdout += process.stdout.read()  # type: ignore
+        stderr += process.stderr.read()  # type: ignore
         state.update_state(
             f"Running {pretty_args}",
             stdout.decode(errors="replace") if log_stdout else None,

test/dependency_source/test_requirement.py

@@ -523,6 +523,30 @@ def test_requirement_source_disable_pip_duplicate_dependencies(req_file):
         [(req_file(), "flask==1.0\nflask==1.0")], disable_pip=True, no_deps=True
     )
 
+    specs = list(source.collect())
+
+    # If the dependency list has duplicates, then converting to a set will reduce the length of the
+    # collection
+    assert len(specs) == len(set(specs))
+
+
+def test_requirement_source_disable_pip_duplicate_dependencies_with_extras(req_file):
+    source = _init_requirement(
+        [(req_file(), "aiohttp==3.9\naiohttp[speedups]==3.9")], disable_pip=True, no_deps=True
+    )
+
+    specs = list(source.collect())
+
+    # If the dependency list has duplicates, then converting to a set will reduce the length of the
+    # collection
+    assert len(specs) == len(set(specs))
+
+
+def test_requirement_source_disable_pip_duplicate_dependencies_diff_specifier(req_file):
+    source = _init_requirement(
+        [(req_file(), "flask==1.0\nflask==2.0")], disable_pip=True, no_deps=True
+    )
+
     with pytest.raises(DependencySourceError):
         list(source.collect())