Merge pull request #96 from DataDog/88-guarddog-fails-to-parse-the-wh…

…ole-requirementstxt-after-encountering-package-name-starting-with-git+https

Handle requirement file parsing errors

guarddog/scanners/project_scanner.py

@@ -2,7 +2,7 @@
 import os
 import re
 import sys
-from pprint import pprint
+from typing import List
 
 import pathos
 import pkg_resources
@@ -68,13 +68,13 @@ def sanitize_requirements(self, requirements) -> list[str]:
 
         return sanitized_lines
 
-    def parse_requirements(self, requirements) -> dict:
+    def parse_requirements(self, requirements: List[str]) -> dict:
         """
         Parses requirements.txt specification and finds all valid
         versions of each dependency
 
         Args:
-            requirements (str): contents of requirements.txt file
+            requirements (List[str]): contents of requirements.txt file
 
         Returns:
             dict: mapping of dependencies to valid versions
@@ -97,8 +97,22 @@ def versions(package_name):
 
         dependencies = {}
 
+        def safe_parse_requirements(req):
+            parsed = pkg_resources.parse_requirements(req)
+            while True:
+                try:
+                    yield next(parsed)
+                except StopIteration:
+                    break
+                except Exception as e:
+                    sys.stderr.write(f"Error when parsing requirements, received error {str(e)}. This entry will be "
+                                     "ignored.\n")
+                    yield None
+
         try:
-            for requirement in pkg_resources.parse_requirements(sanitized_requirements):
+            for requirement in safe_parse_requirements(sanitized_requirements):
+                if requirement is None:
+                    continue
                 valid_versions = None
                 project_exists_on_pypi = True
                 for spec in requirement.specs:

tests/core/test_requirements_scanner.py

@@ -1,8 +1,26 @@
+import pkg_resources
+
 from guarddog.scanners.project_scanner import RequirementsScanner
 
 
 # Regression test for https://github.com/DataDog/guarddog/issues/78
-def test_requirements_scanner_on_non_existing_package():
+def test_requirements_scanner():
     scanner = RequirementsScanner()
-    result = scanner.parse_requirements(["not-a-real-package==1.0.0"])
-    assert "not-a-real-package" not in result
+    result = scanner.parse_requirements(["not-a-real-package==1.0.0", "flask==2.2.2"])
+    assert "not-a-real-package" not in result  # ignoring non existing packages
+    assert "flask" in result
+
+
+# Regression test for https://github.com/DataDog/guarddog/issues/88
+def test_requirements_scanner_on_git_url_packages():
+    scanner = RequirementsScanner()
+    result = scanner.parse_requirements([
+        "flask==2.2.2",
+        "http://wxpython.org/Phoenix/snapshot-builds/wxPython_Phoenix-3.0.3.dev1820+49a8884-cp34-none-win_amd64.whl",
+        "guarddog @ git+https://github.com/DataDog/guarddog.git",
+        "git+https://github.com/DataDog/guarddog.git"
+    ])
+    assert "guarddog" in result
+    assert "flask" in result
+    assert len(result) == 2
+