From 42a88927e591e622cea9f82e238e14670b2ae8c5 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 6 Sep 2023 16:26:02 +0530 Subject: [PATCH 1/8] Handle invalid constraints sequence exception - issue https://github.com/nexB/univers/issues/118 Signed-off-by: Keshav Priyadarshi --- packagedb/api.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index 77bc9d94..6a5aec96 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -749,18 +749,23 @@ def resolve_versions(parsed_purl, vers): all_versions = get_all_versions(parsed_purl) or [] - return [ - str( - PackageURL( - type=parsed_purl.type, - namespace=parsed_purl.namespace, - name=parsed_purl.name, - version=version.string, - ) - ) - for version in all_versions - if version in version_range - ] + result = [] + + for version in all_versions: + try: + if version in version_range: + package_url = PackageURL( + type=parsed_purl.type, + namespace=parsed_purl.namespace, + name=parsed_purl.name, + version=version.string, + ) + result.append(str(package_url)) + except Exception: + # Skip the ``Invalid constraints sequence`` Exception + pass + + return result def get_all_versions(purl: PackageURL): """ From 228137f87eec2a796fa27bdddb5144b2c388dcd7 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 6 Sep 2023 16:30:46 +0530 Subject: [PATCH 2/8] Handle potential invalid version string Signed-off-by: Keshav Priyadarshi --- packagedb/api.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index 6a5aec96..a076b7b0 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -45,7 +45,7 @@ from univers import versions from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import InvalidVersionRange +from univers.versions import InvalidVersion from univers.version_range import VersionRange @@ -750,7 +750,6 @@ def resolve_versions(parsed_purl, vers): all_versions = get_all_versions(parsed_purl) or [] result = [] - for version in all_versions: try: if version in version_range: @@ -783,10 +782,17 @@ def get_all_versions(purl: PackageURL): if not package_name or not versionAPI: return - all_versions = versionAPI().fetch(package_name) + all_versions = versionAPI().fetch(package_name) or [] versionClass = VERSION_CLASS_BY_PACKAGE_TYPE.get(purl.type) - return [versionClass(package_version.value) for package_version in all_versions] + result = [] + for package_version in all_versions: + try: + result.append(versionClass(package_version.value)) + except InvalidVersion: + pass + + return result VERSION_CLASS_BY_PACKAGE_TYPE = {pkg_type: range_class.version_class for pkg_type, range_class in RANGE_CLASS_BY_SCHEMES.items()} From e72f492bb4c0effe066022569cdbf8d09f360767 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 6 Sep 2023 16:33:21 +0530 Subject: [PATCH 3/8] Skip vers resolution for unsupported ecosystems Signed-off-by: Keshav Priyadarshi --- packagedb/api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index a076b7b0..7e917bfa 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -405,8 +405,9 @@ def index_packages(self, request, *args, **kwargs): packages = request.data.get('packages') or [] queued_packages = [] unqueued_packages = [] + supported_ecosystem = ["maven","npm"] - unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages) + unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages, supported_ecosystem) for purl in unique_purls: is_routable_purl = priority_router.is_routable(purl) @@ -691,7 +692,7 @@ class PackageSetViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = PackageSetAPISerializer -def get_resolved_purls(packages): +def get_resolved_purls(packages, supported_ecosystem): """ Take a list of dict containing purl or version-less purl along with vers and return a list of resolved purls, a list of unsupported purls, and a @@ -718,7 +719,7 @@ def get_resolved_purls(packages): unique_resolved_purls.add(purl) continue - if not vers: + if not vers or parsed_purl.type not in supported_ecosystem: unsupported_purls.add(purl) continue From 6d0521dda80df71947449c4af79b92a05fb920cf Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 7 Sep 2023 21:25:23 +0530 Subject: [PATCH 4/8] Bump univers to v30.10.1 Signed-off-by: Keshav Priyadarshi --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b7e95f8f..6babc4d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,7 +56,7 @@ install_requires = scancode-toolkit[full] == 32.0.6 urlpy == 0.5 matchcode-toolkit >= 1.1.1 - univers == 30.10.0 + univers == 30.10.1 setup_requires = setuptools_scm[toml] >= 4 python_requires = >=3.8 From a126277bb63e680a495f02df9dd2bf3a5be91ee3 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 8 Sep 2023 17:40:33 +0530 Subject: [PATCH 5/8] Add logging for exceptions Signed-off-by: Keshav Priyadarshi --- packagedb/api.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index 7e917bfa..3c18de4d 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import logging from django.core.exceptions import ValidationError from django.db.models import Q from django_filters.rest_framework import FilterSet @@ -48,6 +49,7 @@ from univers.versions import InvalidVersion from univers.version_range import VersionRange +logger = logging.getLogger(__name__) class PackageResourcePurlFilter(Filter): def filter(self, qs, value): @@ -405,9 +407,9 @@ def index_packages(self, request, *args, **kwargs): packages = request.data.get('packages') or [] queued_packages = [] unqueued_packages = [] - supported_ecosystem = ["maven","npm"] + supported_ecosystems = ["maven", "npm"] - unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages, supported_ecosystem) + unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages, supported_ecosystems) for purl in unique_purls: is_routable_purl = priority_router.is_routable(purl) @@ -692,7 +694,7 @@ class PackageSetViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = PackageSetAPISerializer -def get_resolved_purls(packages, supported_ecosystem): +def get_resolved_purls(packages, supported_ecosystems): """ Take a list of dict containing purl or version-less purl along with vers and return a list of resolved purls, a list of unsupported purls, and a @@ -719,7 +721,7 @@ def get_resolved_purls(packages, supported_ecosystem): unique_resolved_purls.add(purl) continue - if not vers or parsed_purl.type not in supported_ecosystem: + if not vers or parsed_purl.type not in supported_ecosystems: unsupported_purls.add(purl) continue @@ -763,7 +765,8 @@ def resolve_versions(parsed_purl, vers): result.append(str(package_url)) except Exception: # Skip the ``Invalid constraints sequence`` Exception - pass + logger.warning(f"Invalid constraints sequence in '{vers}' for '{parsed_purl}'") + return return result @@ -791,6 +794,7 @@ def get_all_versions(purl: PackageURL): try: result.append(versionClass(package_version.value)) except InvalidVersion: + logger.warning(f"Invalid version '{package_version.value}' for '{purl}'") pass return result From f03ad38d1e275593194ffa33decff38bab814aef Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 12 Sep 2023 14:49:06 +0530 Subject: [PATCH 6/8] Bump univers to v30.11.0 Signed-off-by: Keshav Priyadarshi --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6babc4d8..895b5b84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,7 +56,7 @@ install_requires = scancode-toolkit[full] == 32.0.6 urlpy == 0.5 matchcode-toolkit >= 1.1.1 - univers == 30.10.1 + univers == 30.11.0 setup_requires = setuptools_scm[toml] >= 4 python_requires = >=3.8 From ea27db46cd576f9eefc12a359e8ac8961b573b67 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 12 Sep 2023 14:58:10 +0530 Subject: [PATCH 7/8] Handle InvalidConstraintsError exception Signed-off-by: Keshav Priyadarshi --- packagedb/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index 3c18de4d..b32bf73e 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -48,6 +48,7 @@ from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.versions import InvalidVersion from univers.version_range import VersionRange +from univers.version_constraint import InvalidConstraintsError logger = logging.getLogger(__name__) @@ -763,8 +764,7 @@ def resolve_versions(parsed_purl, vers): version=version.string, ) result.append(str(package_url)) - except Exception: - # Skip the ``Invalid constraints sequence`` Exception + except InvalidConstraintsError: logger.warning(f"Invalid constraints sequence in '{vers}' for '{parsed_purl}'") return From b7d323b814190689c674afca1476f704d1f73e38 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 12 Sep 2023 09:55:21 -0700 Subject: [PATCH 8/8] Update test expectation Signed-off-by: Jono Yang --- minecode/tests/testfiles/directories/ls-lr-expected.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/minecode/tests/testfiles/directories/ls-lr-expected.json b/minecode/tests/testfiles/directories/ls-lr-expected.json index f6da0f88..9b088b89 100644 --- a/minecode/tests/testfiles/directories/ls-lr-expected.json +++ b/minecode/tests/testfiles/directories/ls-lr-expected.json @@ -3,7 +3,7 @@ "path":"README", "type":"f", "size":1499, - "date":"2022-09", + "date":"2023-09", "target":null }, { @@ -17,7 +17,7 @@ "path":"README.html", "type":"f", "size":3185, - "date":"2022-09", + "date":"2023-09", "target":null }, { @@ -45,7 +45,7 @@ "path":"dists/README", "type":"f", "size":932, - "date":"2022-09", + "date":"2023-09", "target":null }, {