From 0d6d0a79dfef62c27f7d614f02cbb08da2a07896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Houlbr=C3=A8que?= Date: Wed, 21 Feb 2024 14:22:08 +0100 Subject: [PATCH] Remove duplicated packages --- pre_commit_hooks/requirements_txt_fixer.py | 11 ++++++++++- tests/requirements_txt_fixer_test.py | 6 ++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pre_commit_hooks/requirements_txt_fixer.py b/pre_commit_hooks/requirements_txt_fixer.py index 58843940..261acc97 100644 --- a/pre_commit_hooks/requirements_txt_fixer.py +++ b/pre_commit_hooks/requirements_txt_fixer.py @@ -45,6 +45,11 @@ def __lt__(self, requirement: Requirement) -> bool: elif requirement.value == b'\n': return False else: + # if 2 requirements have the same name, the one with comments + # needs to go first (so that when removing duplicates, the one + # with comments is kept) + if self.name == requirement.name: + return bool(self.comments) > bool(requirement.comments) return self.name < requirement.name def is_complete(self) -> bool: @@ -113,10 +118,14 @@ def fix_requirements(f: IO[bytes]) -> int: if req.value != b'pkg-resources==0.0.0\n' ] + # sort the requirements and remove duplicates + prev = None for requirement in sorted(requirements): after.extend(requirement.comments) assert requirement.value, requirement.value - after.append(requirement.value) + if prev is None or requirement.value != prev.value: + after.append(requirement.value) + prev = requirement after.extend(rest) after_string = b''.join(after) diff --git a/tests/requirements_txt_fixer_test.py b/tests/requirements_txt_fixer_test.py index b725afa2..c400be1f 100644 --- a/tests/requirements_txt_fixer_test.py +++ b/tests/requirements_txt_fixer_test.py @@ -68,6 +68,12 @@ b'f<=2\n' b'g<2\n', ), + (b'a==1\nb==1\na==1\n', FAIL, b'a==1\nb==1\n'), + ( + b'a==1\nb==1\n#comment about a\na==1\n', + FAIL, + b'#comment about a\na==1\nb==1\n', + ), (b'ocflib\nDjango\nPyMySQL\n', FAIL, b'Django\nocflib\nPyMySQL\n'), ( b'-e git+ssh://git_url@tag#egg=ocflib\nDjango\nPyMySQL\n',