Add support for reading dependencies form urls in requirements.txt (#100

)

* added support for reading dependencies form urls in requirements.txt
* added unit tests

deptry/dependency_getter/requirements_txt.py

@@ -77,17 +77,17 @@ def _extract_dependency_from_line(self, line: str) -> Dependency:
         else:
             return None
 
-    @staticmethod
-    def _find_dependency_name_in(line):
+    def _find_dependency_name_in(self, line):
         """
         Find the dependency name of a dependency specified according to the pip-standards for requirement.txt
         """
-        match = re.search("^[a-zA-Z0-9-_]+", line)
-        if match and not match.group(0)[0] == "-":
-            name = match.group(0)
-            return name
+        if self._line_is_url(line):
+            return self._extract_name_from_url(line)
         else:
-            return None
+            match = re.search("^[^-][a-zA-Z0-9-_]+", line)
+            if match:
+                return match.group(0)
+        return None
 
     @staticmethod
     def _remove_comments_from(line):
@@ -110,3 +110,23 @@ def _log_dependencies(self, dependencies: List[Dependency]) -> None:
         for dependency in dependencies:
             logging.debug(str(dependency))
         logging.debug("")
+
+    @staticmethod
+    def _line_is_url(line):
+        return re.search("^(http|https|git\+https)", line)
+
+    @staticmethod
+    def _extract_name_from_url(line):
+
+        # for url like git+https://github.com/name/python-module.git@0d6dc38d58
+        match = re.search("\/((?:(?!\/).)*?)\.git", line)
+        if match:
+            return match.group(1)
+
+        # for url like https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
+        match = re.search("\/((?:(?!\/).)*?)\/archive\/", line)
+        if match:
+            return match.group(1)
+
+        logging.warning(f"Could not parse dependency name from url {line}")
+        return None

tests/test_requirements_txt_dependency_getter.py

@@ -18,24 +18,11 @@ def test_parse_requirements_txt(tmp_path):
 SomeProject3 ; sys_platform == 'win32'
 requests [security] >= 2.8.1, == 2.8.* ; python_version < "2.7"
 # This is a comment, to show how #-prefixed lines are ignored.
-# It is possible to specify requirements as plain names.
 pytest
 pytest-cov
 beautifulsoup4
-
-# The syntax supported here is the same as that of requirement specifiers.
 docopt == 0.6.1
 requests [security] >= 2.8.1, == 2.8.* ; python_version < "2.7"
-urllib3 @ https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
-
-# It is possible to refer to other requirement files or constraints files.
--r other-requirements.txt
--c constraints.txt
-
-# It is possible to refer to specific local distribution paths.
-./downloads/numpy-1.9.2-cp34-none-win32.whl
-
-# It is possible to refer to URLs.
 """
 
     with run_within_dir(tmp_path):
@@ -44,7 +31,7 @@ def test_parse_requirements_txt(tmp_path):
 
         dependencies = RequirementsTxtDependencyGetter().get()
 
-        assert len(dependencies) == 18
+        assert len(dependencies) == 17
 
         assert dependencies[1].name == "colorama"
         assert not dependencies[1].is_conditional
@@ -62,6 +49,28 @@ def test_parse_requirements_txt(tmp_path):
         assert "requests" in dependencies[11].top_levels
 
 
+def test_parse_requirements_txt_urls(tmp_path):
+
+    fake_requirements_txt = """
+urllib3 @ https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
+https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
+git+https://github.com/baz/foo-bar.git@asd#egg=foo-bar
+git+https://github.com/baz/foo-bar.git@asd
+"""
+
+    with run_within_dir(tmp_path):
+        with open("requirements.txt", "w") as f:
+            f.write(fake_requirements_txt)
+
+        dependencies = RequirementsTxtDependencyGetter().get()
+
+        assert len(dependencies) == 4
+        assert dependencies[0].name == "urllib3"
+        assert dependencies[1].name == "urllib3"
+        assert dependencies[2].name == "foo-bar"
+        assert dependencies[3].name == "foo-bar"
+
+
 def test_with_name(tmp_path):
 
     with run_within_dir(tmp_path):