From 406069cfe5dee585099afa56198580a5a120fe8f Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 30 Jul 2024 18:56:15 +0400 Subject: [PATCH 1/9] Add all the functions for a SPDX dependency resolution #1145 Signed-off-by: tdruez --- scanpipe/pipes/resolve.py | 53 +++- scanpipe/pipes/spdx.py | 12 + .../data/spdx/SPDXJSONExample-v2.3.spdx.json | 289 ++++++++++++++++++ scanpipe/tests/pipes/test_resolve.py | 42 ++- scanpipe/tests/pipes/test_spdx.py | 6 + 5 files changed, 396 insertions(+), 6 deletions(-) create mode 100644 scanpipe/tests/data/spdx/SPDXJSONExample-v2.3.spdx.json diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 61d014450..86ef49d52 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -265,8 +265,12 @@ def convert_spdx_expression(license_expression_spdx): return get_license_detections_and_expression(license_expression_spdx)[1] -def spdx_package_to_discovered_package_data(spdx_package): +def spdx_package_to_package_data(spdx_package): + """Convert the provided spdx_package into package_data.""" package_url_dict = {} + # Store the original "SPDXID" as package_uid for dependencies resolution. + package_uid = spdx_package.spdx_id + for ref in spdx_package.external_refs: if ref.type == "purl": purl = ref.locator @@ -283,6 +287,7 @@ def spdx_package_to_discovered_package_data(spdx_package): declared_expression = convert_spdx_expression(declared_license_expression_spdx) package_data = { + "package_uid": package_uid, "name": spdx_package.name, "download_url": spdx_package.download_location, "declared_license_expression": declared_expression, @@ -305,8 +310,28 @@ def spdx_package_to_discovered_package_data(spdx_package): } -def resolve_spdx_packages(input_location): - """Resolve the packages from the `input_location` SPDX document file.""" +def spdx_relationship_to_dependency_data(spdx_relationship): + """Convert the provided spdx_relationship into dependency_data.""" + # spdx_id is a dependency of related_spdx_id + if spdx_relationship.is_dependency_relationship: + for_package_uid = spdx_relationship.related_spdx_id + resolve_to_package_uid = spdx_relationship.spdx_id + else: # spdx_id depends on related_spdx_id + for_package_uid = spdx_relationship.spdx_id + resolve_to_package_uid = spdx_relationship.related_spdx_id + + dependency_data = { + "for_package_uid": for_package_uid, + "resolve_to_package_uid": resolve_to_package_uid, + "is_runtime": True, + "is_resolved": True, + "is_direct": True, + } + return dependency_data + + +def get_spdx_document_from_file(input_location): + """Return the loaded SPDX document from the `input_location` file.""" input_path = Path(input_location) spdx_document = json.loads(input_path.read_text()) @@ -315,12 +340,32 @@ def resolve_spdx_packages(input_location): except Exception as e: raise Exception(f'SPDX document "{input_path.name}" is not valid: {e}') + return spdx_document + + +def resolve_spdx_packages(input_location): + """Resolve the packages from the `input_location` SPDX document file.""" + spdx_document = get_spdx_document_from_file(input_location) return [ - spdx_package_to_discovered_package_data(spdx.Package.from_data(spdx_package)) + spdx_package_to_package_data(spdx.Package.from_data(spdx_package)) for spdx_package in spdx_document.get("packages", []) ] +def resolve_spdx_dependencies(input_location): + """Resolve the dependencies from the `input_location` SPDX document file.""" + spdx_document = get_spdx_document_from_file(input_location) + spdx_relationships = [ + spdx.Relationship.from_data(spdx_relationship) + for spdx_relationship in spdx_document.get("relationships", []) + ] + + return [ + spdx_relationship_to_dependency_data(spdx_relationship) + for spdx_relationship in spdx_relationships + ] + + def get_default_package_type(input_location): """ Return the package type associated with the provided `input_location`. diff --git a/scanpipe/pipes/spdx.py b/scanpipe/pipes/spdx.py index 9192d7ed7..815cfa7f2 100644 --- a/scanpipe/pipes/spdx.py +++ b/scanpipe/pipes/spdx.py @@ -522,6 +522,18 @@ def from_data(cls, data): comment=data.get("comment"), ) + @property + def is_dependency_relationship(self): + """ + Return True if this relationship type implies that the spdx_id element + is a dependency of related_spdx_id. + """ + reverse_dependency_types = ["ANCESTOR_OF", "CONTAINS", "DEPENDS_ON"] + # Every others types implies that the spdx_id element is a dependency of + # related_spdx_id. Such as: + # "DEPENDENCY_OF", "DESCENDANT_OF", "PACKAGE_OF", "CONTAINED_BY", ... + return self.relationship.upper() not in reverse_dependency_types + @dataclass class Document: diff --git a/scanpipe/tests/data/spdx/SPDXJSONExample-v2.3.spdx.json b/scanpipe/tests/data/spdx/SPDXJSONExample-v2.3.spdx.json new file mode 100644 index 000000000..4b2057dfe --- /dev/null +++ b/scanpipe/tests/data/spdx/SPDXJSONExample-v2.3.spdx.json @@ -0,0 +1,289 @@ +{ + "SPDXID" : "SPDXRef-DOCUMENT", + "spdxVersion" : "SPDX-2.3", + "creationInfo" : { + "comment" : "This package has been shipped in source and binary form.\nThe binaries were created with gcc 4.5.1 and expect to link to\ncompatible system run time libraries.", + "created" : "2010-01-29T18:30:22Z", + "creators" : [ "Tool: LicenseFind-1.0", "Organization: ExampleCodeInspect ()", "Person: Jane Doe ()" ], + "licenseListVersion" : "3.17" + }, + "name" : "SPDX-Tools-v2.0", + "dataLicense" : "CC0-1.0", + "comment" : "This document was created using SPDX 2.0 using licenses from the web site.", + "externalDocumentRefs" : [ { + "externalDocumentId" : "DocumentRef-spdx-tool-1.2", + "checksum" : { + "algorithm" : "SHA1", + "checksumValue" : "d6a770ba38583ed4bb4525bd96e50461655d2759" + }, + "spdxDocument" : "http://spdx.org/spdxdocs/spdx-tools-v1.2-3F2504E0-4F89-41D3-9A0C-0305E82C3301" + } ], + "hasExtractedLicensingInfos" : [ { + "licenseId" : "LicenseRef-1", + "extractedText" : "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/" + }, { + "licenseId" : "LicenseRef-2", + "extractedText" : "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n© Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: \n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. \nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. \nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. \nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + }, { + "licenseId" : "LicenseRef-4", + "extractedText" : "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/" + }, { + "licenseId" : "LicenseRef-Beerware-4.2", + "comment" : "The beerware license has a couple of other standard variants.", + "extractedText" : "\"THE BEER-WARE LICENSE\" (Revision 42):\nphk@FreeBSD.ORG wrote this file. As long as you retain this notice you\ncan do whatever you want with this stuff. If we meet some day, and you think this stuff is worth it, you can buy me a beer in return Poul-Henning Kamp", + "name" : "Beer-Ware License (Version 42)", + "seeAlsos" : [ "http://people.freebsd.org/~phk/" ] + }, { + "licenseId" : "LicenseRef-3", + "comment" : "This is tye CyperNeko License", + "extractedText" : "The CyberNeko Software License, Version 1.0\n\n \n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n \nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer. \n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment: \n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior \n written permission. For written permission, please contact \n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, \nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT \nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR \nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, \nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE \nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, \nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "name" : "CyberNeko License", + "seeAlsos" : [ "http://people.apache.org/~andyc/neko/LICENSE", "http://justasample.url.com" ] + } ], + "annotations" : [ { + "annotationDate" : "2010-01-29T18:30:22Z", + "annotationType" : "OTHER", + "annotator" : "Person: Jane Doe ()", + "comment" : "Document level annotation" + }, { + "annotationDate" : "2010-02-10T00:00:00Z", + "annotationType" : "REVIEW", + "annotator" : "Person: Joe Reviewer", + "comment" : "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses" + }, { + "annotationDate" : "2011-03-13T00:00:00Z", + "annotationType" : "REVIEW", + "annotator" : "Person: Suzanne Reviewer", + "comment" : "Another example reviewer." + } ], + "documentDescribes" : [ "SPDXRef-File", "SPDXRef-Package" ], + "documentNamespace" : "http://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "packages" : [ { + "SPDXID" : "SPDXRef-Package", + "annotations" : [ { + "annotationDate" : "2011-01-29T18:30:22Z", + "annotationType" : "OTHER", + "annotator" : "Person: Package Commenter", + "comment" : "Package level annotation" + } ], + "attributionTexts" : [ "The GNU C Library is free software. See the file COPYING.LIB for copying conditions, and LICENSES for notices about a few contributions that require these additional notices to be distributed. License copyright years may be listed using range notation, e.g., 1996-2015, indicating that every year in the range, inclusive, is a copyrightable year that would otherwise be listed individually." ], + "builtDate" : "2011-01-29T18:30:22Z", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "624c1abb3664f4b35547e7c73864ad24" + }, { + "algorithm" : "SHA1", + "checksumValue" : "85ed0817af83a24ad8da68c2b5094de69833983c" + }, { + "algorithm" : "SHA256", + "checksumValue" : "11b6d3ee554eedf79299905a98f9b9a04e498210b59f15094c916c91d150efcd" + }, { + "algorithm" : "BLAKE2b-384", + "checksumValue" : "aaabd89c926ab525c242e6621f2f5fa73aa4afe3d9e24aed727faaadd6af38b620bdb623dd2b4788b1c8086984af8706" + } ], + "copyrightText" : "Copyright 2008-2010 John Smith", + "description" : "The GNU C Library defines functions that are specified by the ISO C standard, as well as additional features specific to POSIX and other derivatives of the Unix operating system, and extensions specific to GNU systems.", + "downloadLocation" : "http://ftp.gnu.org/gnu/glibc/glibc-ports-2.15.tar.gz", + "externalRefs" : [ { + "referenceCategory" : "SECURITY", + "referenceLocator" : "cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:*", + "referenceType" : "cpe23Type" + }, { + "comment" : "This is the external ref for Acme", + "referenceCategory" : "OTHER", + "referenceLocator" : "acmecorp/acmenator/4.1.3-alpha", + "referenceType" : "http://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#LocationRef-acmeforge" + } ], + "filesAnalyzed" : true, + "homepage" : "http://ftp.gnu.org/gnu/glibc", + "licenseComments" : "The license for this project changed with the release of version x.y. The version of the project included here post-dates the license change.", + "licenseConcluded" : "(LGPL-2.0-only OR LicenseRef-3)", + "licenseDeclared" : "(LGPL-2.0-only AND LicenseRef-3)", + "licenseInfoFromFiles" : [ "GPL-2.0-only", "LicenseRef-2", "LicenseRef-1" ], + "name" : "glibc", + "originator" : "Organization: ExampleCodeInspect (contact@example.com)", + "packageFileName" : "glibc-2.11.1.tar.gz", + "packageVerificationCode" : { + "packageVerificationCodeExcludedFiles" : [ "./package.spdx" ], + "packageVerificationCodeValue" : "d6a770ba38583ed4bb4525bd96e50461655d2758" + }, + "primaryPackagePurpose" : "SOURCE", + "hasFiles" : [ "SPDXRef-Specification", "SPDXRef-Specification", "SPDXRef-CommonsLangSrc", "SPDXRef-Specification", "SPDXRef-CommonsLangSrc", "SPDXRef-JenaLib", "SPDXRef-Specification", "SPDXRef-CommonsLangSrc", "SPDXRef-JenaLib", "SPDXRef-DoapSource", "SPDXRef-Specification", "SPDXRef-CommonsLangSrc", "SPDXRef-JenaLib", "SPDXRef-DoapSource" ], + "releaseDate" : "2012-01-29T18:30:22Z", + "sourceInfo" : "uses glibc-2_11-branch from git://sourceware.org/git/glibc.git.", + "summary" : "GNU C library.", + "supplier" : "Person: Jane Doe (jane.doe@example.com)", + "validUntilDate" : "2014-01-29T18:30:22Z", + "versionInfo" : "2.11.1" + }, { + "SPDXID" : "SPDXRef-fromDoap-1", + "copyrightText" : "NOASSERTION", + "downloadLocation" : "NOASSERTION", + "filesAnalyzed" : false, + "homepage" : "http://commons.apache.org/proper/commons-lang/", + "licenseConcluded" : "NOASSERTION", + "licenseDeclared" : "NOASSERTION", + "name" : "Apache Commons Lang" + }, { + "SPDXID" : "SPDXRef-fromDoap-0", + "downloadLocation" : "https://search.maven.org/remotecontent?filepath=org/apache/jena/apache-jena/3.12.0/apache-jena-3.12.0.tar.gz", + "externalRefs" : [ { + "referenceCategory" : "PACKAGE-MANAGER", + "referenceLocator" : "pkg:maven/org.apache.jena/apache-jena@3.12.0", + "referenceType" : "purl" + } ], + "filesAnalyzed" : false, + "homepage" : "http://www.openjena.org/", + "name" : "Jena", + "versionInfo" : "3.12.0" + }, { + "SPDXID" : "SPDXRef-Saxon", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "85ed0817af83a24ad8da68c2b5094de69833983c" + } ], + "copyrightText" : "Copyright Saxonica Ltd", + "description" : "The Saxon package is a collection of tools for processing XML documents.", + "downloadLocation" : "https://sourceforge.net/projects/saxon/files/Saxon-B/8.8.0.7/saxonb8-8-0-7j.zip/download", + "filesAnalyzed" : false, + "homepage" : "http://saxon.sourceforge.net/", + "licenseComments" : "Other versions available for a commercial license", + "licenseConcluded" : "MPL-1.0", + "licenseDeclared" : "MPL-1.0", + "name" : "Saxon", + "packageFileName" : "saxonB-8.8.zip", + "versionInfo" : "8.8" + } ], + "files" : [ { + "SPDXID" : "SPDXRef-DoapSource", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + } ], + "copyrightText" : "Copyright 2010, 2011 Source Auditor Inc.", + "fileContributors" : [ "Protecode Inc.", "SPDX Technical Team Members", "Open Logic Inc.", "Source Auditor Inc.", "Black Duck Software In.c" ], + "fileName" : "./src/org/spdx/parser/DOAPProject.java", + "fileTypes" : [ "SOURCE" ], + "licenseConcluded" : "Apache-2.0", + "licenseInfoInFiles" : [ "Apache-2.0" ] + }, { + "SPDXID" : "SPDXRef-CommonsLangSrc", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "c2b4e1c67a2d28fced849ee1bb76e7391b93f125" + } ], + "comment" : "This file is used by Jena", + "copyrightText" : "Copyright 2001-2011 The Apache Software Foundation", + "fileContributors" : [ "Apache Software Foundation" ], + "fileName" : "./lib-source/commons-lang3-3.1-sources.jar", + "fileTypes" : [ "ARCHIVE" ], + "licenseConcluded" : "Apache-2.0", + "licenseInfoInFiles" : [ "Apache-2.0" ], + "noticeText" : "Apache Commons Lang\nCopyright 2001-2011 The Apache Software Foundation\n\nThis product includes software developed by\nThe Apache Software Foundation (http://www.apache.org/).\n\nThis product includes software from the Spring Framework,\nunder the Apache License 2.0 (see: StringUtils.containsWhitespace())" + }, { + "SPDXID" : "SPDXRef-JenaLib", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" + } ], + "comment" : "This file belongs to Jena", + "copyrightText" : "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", + "fileContributors" : [ "Apache Software Foundation", "Hewlett Packard Inc." ], + "fileName" : "./lib-source/jena-2.6.3-sources.jar", + "fileTypes" : [ "ARCHIVE" ], + "licenseComments" : "This license is used by Jena", + "licenseConcluded" : "LicenseRef-1", + "licenseInfoInFiles" : [ "LicenseRef-1" ] + }, { + "SPDXID" : "SPDXRef-Specification", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "fff4e1c67a2d28fced849ee1bb76e7391b93f125" + } ], + "comment" : "Specification Documentation", + "fileName" : "./docs/myspec.pdf", + "fileTypes" : [ "DOCUMENTATION" ] + }, { + "SPDXID" : "SPDXRef-File", + "annotations" : [ { + "annotationDate" : "2011-01-29T18:30:22Z", + "annotationType" : "OTHER", + "annotator" : "Person: File Commenter", + "comment" : "File level annotation" + } ], + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "d6a770ba38583ed4bb4525bd96e50461655d2758" + }, { + "algorithm" : "MD5", + "checksumValue" : "624c1abb3664f4b35547e7c73864ad24" + } ], + "comment" : "The concluded license was taken from the package level that the file was included in.\nThis information was found in the COPYING.txt file in the xyz directory.", + "copyrightText" : "Copyright 2008-2010 John Smith", + "fileContributors" : [ "The Regents of the University of California", "Modified by Paul Mundt lethal@linux-sh.org", "IBM Corporation" ], + "fileName" : "./package/foo.c", + "fileTypes" : [ "SOURCE" ], + "licenseComments" : "The concluded license was taken from the package level that the file was included in.", + "licenseConcluded" : "(LGPL-2.0-only OR LicenseRef-2)", + "licenseInfoInFiles" : [ "GPL-2.0-only", "LicenseRef-2" ], + "noticeText" : "Copyright (c) 2001 Aaron Lehmann aaroni@vitelus.com\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: \nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." + } ], + "snippets" : [ { + "SPDXID" : "SPDXRef-Snippet", + "comment" : "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0.", + "copyrightText" : "Copyright 2008-2010 John Smith", + "licenseComments" : "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", + "licenseConcluded" : "GPL-2.0-only", + "licenseInfoInSnippets" : [ "GPL-2.0-only" ], + "name" : "from linux kernel", + "ranges" : [ { + "endPointer" : { + "offset" : 420, + "reference" : "SPDXRef-DoapSource" + }, + "startPointer" : { + "offset" : 310, + "reference" : "SPDXRef-DoapSource" + } + }, { + "endPointer" : { + "lineNumber" : 23, + "reference" : "SPDXRef-DoapSource" + }, + "startPointer" : { + "lineNumber" : 5, + "reference" : "SPDXRef-DoapSource" + } + } ], + "snippetFromFile" : "SPDXRef-DoapSource" + } ], + "relationships" : [ { + "spdxElementId" : "SPDXRef-DOCUMENT", + "relationshipType" : "CONTAINS", + "relatedSpdxElement" : "SPDXRef-Package" + }, { + "spdxElementId" : "SPDXRef-DOCUMENT", + "relationshipType" : "COPY_OF", + "relatedSpdxElement" : "DocumentRef-spdx-tool-1.2:SPDXRef-ToolsElement" + }, { + "spdxElementId" : "SPDXRef-Package", + "relationshipType" : "DYNAMIC_LINK", + "relatedSpdxElement" : "SPDXRef-Saxon" + }, { + "spdxElementId" : "SPDXRef-CommonsLangSrc", + "relationshipType" : "GENERATED_FROM", + "relatedSpdxElement" : "NOASSERTION" + }, { + "spdxElementId" : "SPDXRef-JenaLib", + "relationshipType" : "CONTAINS", + "relatedSpdxElement" : "SPDXRef-Package" + }, { + "spdxElementId" : "SPDXRef-Specification", + "relationshipType" : "SPECIFICATION_FOR", + "relatedSpdxElement" : "SPDXRef-fromDoap-0" + }, { + "spdxElementId" : "SPDXRef-File", + "relationshipType" : "GENERATED_FROM", + "relatedSpdxElement" : "SPDXRef-fromDoap-0" + } ] +} \ No newline at end of file diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index b48fd88c1..5c5af0eed 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -28,6 +28,7 @@ from scanpipe import pipes from scanpipe.models import Project from scanpipe.pipes import resolve +from scanpipe.pipes import spdx from scanpipe.pipes.input import copy_inputs from scanpipe.pipes.scancode import extract_archives from scanpipe.tests import make_package @@ -133,12 +134,20 @@ def test_scanpipe_pipes_resolve_resolve_about_packages(self): expected = {"extra_data": {}, "name": "project"} self.assertEqual([expected], package) - def test_scanpipe_pipes_resolve_spdx_package_to_discovered_package_data(self): + def test_scanpipe_pipes_resolve_get_spdx_document_from_file(self): + input_location = self.data / "spdx" / "SPDXJSONExample-v2.3.spdx.json" + spdx_document = resolve.get_spdx_document_from_file(input_location) + self.assertIsInstance(spdx_document, dict) + self.assertEqual("SPDXRef-DOCUMENT", spdx_document["SPDXID"]) + self.assertEqual("SPDX-2.3", spdx_document["spdxVersion"]) + + def test_scanpipe_pipes_resolve_spdx_package_to_package_data(self): p1 = Project.objects.create(name="Analysis") package = pipes.update_or_create_package(p1, package_data1) package_spdx = package.as_spdx() - package_data = resolve.spdx_package_to_discovered_package_data(package_spdx) + package_data = resolve.spdx_package_to_package_data(package_spdx) expected = { + "package_uid": package.spdx_id, "name": "adduser", "download_url": "https://download.url/package.zip", "declared_license_expression": "gpl-2.0 AND gpl-2.0-plus", @@ -162,6 +171,35 @@ def test_scanpipe_pipes_resolve_spdx_package_to_discovered_package_data(self): } self.assertEqual(expected, package_data) + def test_scanpipe_pipes_spdx_relationship_to_dependency_data(self): + spdx_relationship_data = { + "spdxElementId": "SPDXRef-package1", + "relatedSpdxElement": "SPDXRef-package2", + "relationshipType": "CONTAINS", + } + spdx_relationship = spdx.Relationship.from_data(spdx_relationship_data) + dependency_data = resolve.spdx_relationship_to_dependency_data( + spdx_relationship + ) + expected = { + "for_package_uid": "SPDXRef-package1", + "resolve_to_package_uid": "SPDXRef-package2", + "is_runtime": True, + "is_resolved": True, + "is_direct": True, + } + self.assertEqual(expected, dependency_data) + + def test_scanpipe_pipes_resolve_spdx_packages(self): + input_location = self.data / "spdx" / "SPDXJSONExample-v2.3.spdx.json" + packages_data = resolve.resolve_spdx_packages(input_location) + self.assertEqual(4, len(packages_data)) + + def test_scanpipe_pipes_resolve_spdx_dependencies(self): + input_location = self.data / "spdx" / "SPDXJSONExample-v2.3.spdx.json" + dependencies_data = resolve.resolve_spdx_dependencies(input_location) + self.assertEqual(7, len(dependencies_data)) + def test_scanpipe_resolve_get_manifest_resources(self): project1 = Project.objects.create(name="Analysis") input_location = self.data / "manifests" / "python-inspector-0.10.0.zip" diff --git a/scanpipe/tests/pipes/test_spdx.py b/scanpipe/tests/pipes/test_spdx.py index 1c62aeaa9..7bb68fcde 100644 --- a/scanpipe/tests/pipes/test_spdx.py +++ b/scanpipe/tests/pipes/test_spdx.py @@ -351,6 +351,12 @@ def test_spdx_relationship_from_data(self): relationship = spdx.Relationship.from_data(self.relationship_spdx_data) assert self.relationship_spdx_data == relationship.as_dict() + def test_spdx_relationship_is_dependency_relationship_property(self): + relationship = spdx.Relationship.from_data(self.relationship_spdx_data) + assert relationship.is_dependency_relationship is False + relationship.relationship = "DEPENDENCY_OF" + assert relationship.is_dependency_relationship + def test_spdx_document_as_dict(self): document = spdx.Document(**self.document_data) assert self.document_spdx_data == document.as_dict() From bcc4221f81f9d916f8b2e1e416839f716530e72c Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 31 Jul 2024 12:30:52 +0400 Subject: [PATCH 2/9] Refactor the dependency inclusion in SPDX output #1145 Signed-off-by: tdruez --- scanpipe/models.py | 78 ++++++++++--------- scanpipe/pipes/__init__.py | 1 - scanpipe/pipes/output.py | 39 +++++++--- .../data/asgiref/asgiref-3.3.0.spdx.json | 2 +- 4 files changed, 72 insertions(+), 48 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index 164cbfc87..f943a9dc7 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3579,6 +3579,15 @@ class DiscoveredDependency( system and application packages discovered in the code under analysis. Dependencies are usually collected from parsed package data such as a package manifest or lockfile. + + This class manages dependencies with the following considerations: + + 1. A dependency can be associated with a Package via the "for_package" field. + In this case, it is termed a "Package's dependency". If there is no such + association, the dependency is considered a "Project's dependency". + + 2. A dependency can also be linked to a Package through the "resolved_to_package" + field. When this link exists, the dependency is considered "resolved". """ # Overrides the `project` field to set the proper `related_name`. @@ -3729,6 +3738,18 @@ def datafile_path(self): if self.datafile_resource: return self.datafile_resource.path + @property + def is_project_dependency(self): + return not bool(self.for_package_id) + + @property + def is_for_package(self): + return bool(self.for_package_id) + + @property + def is_resolved_to_package(self): + return bool(self.resolved_to_package_id) + @classmethod def create_from_data( cls, @@ -3752,51 +3773,34 @@ def create_from_data( not stripped for `datafile_path`. """ dependency_data = dependency_data.copy() - required_fields = ["purl", "dependency_uid"] - missing_values = [ - field_name - for field_name in required_fields - if not dependency_data.get(field_name) - ] - - if missing_values: - message = ( - f"No values for the following required fields: " - f"{', '.join(missing_values)}" - ) + project_packages_qs = project.discoveredpackages - project.add_warning(description=message, model=cls, details=dependency_data) - return + if not dependency_data.get("dependency_uid"): + dependency_data["dependency_uid"] = str(uuid.uuid4()) - if not for_package: - for_package_uid = dependency_data.get("for_package_uid") - if for_package_uid: - for_package = project.discoveredpackages.get( - package_uid=for_package_uid - ) + for_package_uid = dependency_data.get("for_package_uid") + if not for_package and for_package_uid: + for_package = project_packages_qs.get(package_uid=for_package_uid) - if not resolved_to_package: - resolved_to_uid = dependency_data.get("resolved_to_uid") - if resolved_to_uid: - resolved_to_package = project.discoveredpackages.get( - package_uid=resolved_to_uid - ) + resolved_to_uid = dependency_data.get("resolved_to_uid") + if not resolved_to_package and resolved_to_uid: + resolved_to_package = project_packages_qs.get(package_uid=resolved_to_uid) - if not datafile_resource: - datafile_path = dependency_data.get("datafile_path") - if datafile_path: - if strip_datafile_path_root: - segments = datafile_path.split("/") - datafile_path = "/".join(segments[1:]) - datafile_resource = project.codebaseresources.get(path=datafile_path) + datafile_path = dependency_data.get("datafile_path") + if not datafile_resource and datafile_path: + if strip_datafile_path_root: + segments = datafile_path.split("/") + datafile_path = "/".join(segments[1:]) + datafile_resource = project.codebaseresources.get(path=datafile_path) if datasource_id: dependency_data["datasource_id"] = datasource_id - # Set purl fields from `purl` + # Set package_url fields from the ``purl`` string. purl = dependency_data.get("purl") - purl_mapping = PackageURL.from_string(purl).to_dict() - dependency_data.update(**purl_mapping) + if purl: + purl_data_dict = PackageURL.from_string(purl).to_dict() + dependency_data.update(**purl_data_dict) cleaned_data = { field_name: value @@ -3830,7 +3834,7 @@ def populate_dependency_uuid(cls, dependency_data): def spdx_id(self): return f"SPDXRef-scancodeio-{self._meta.model_name}-{self.dependency_uid}" - def as_spdx(self): + def as_spdx_package(self): """Return this Dependency as an SPDX Package entry.""" from scanpipe.pipes import spdx diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 799851007..92676f5ad 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -314,7 +314,6 @@ def get_dependencies(project, dependency_data): Given a `dependency_data` mapping, get a list of DiscoveredDependency objects for that `project` with similar dependency data. """ - dependency = None dependency_uid = dependency_data.get("dependency_uid") extracted_requirement = dependency_data.get("extracted_requirement") or "" diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 17a508504..09244eb4e 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -529,6 +529,28 @@ def _get_spdx_extracted_licenses(license_expressions): return extracted_licenses +def get_dependency_as_spdx_relationship(dependency, document_spdx_id, packages_as_spdx): + """Return a spdx.Relationship crafted from the provided ``dependency`` instance.""" + if dependency.is_for_package: # Package dependency + parent_id = dependency.for_package.spdx_id + else: # Project dependency + parent_id = document_spdx_id + + if dependency.is_resolved_to_package: # Resolved to a Package + child_id = dependency.resolved_to_package.spdx_id + else: # Not resolved to a Package (only package_url value is available) + dependency_as_package = dependency.as_spdx_package() + packages_as_spdx.append(dependency_as_package) + child_id = dependency_as_package.spdx_id + + spdx_relationship = spdx.Relationship( + spdx_id=child_id, + related_spdx_id=parent_id, + relationship="DEPENDENCY_OF", + ) + return spdx_relationship + + def to_spdx(project, include_files=False): """ Generate output for the provided ``project`` in SPDX document format. @@ -540,6 +562,7 @@ def to_spdx(project, include_files=False): discoveredpackage_qs = get_queryset(project, "discoveredpackage") discovereddependency_qs = get_queryset(project, "discovereddependency") + document_spdx_id = f"SPDXRef-DOCUMENT-{project.uuid}" packages_as_spdx = [] license_expressions = [] relationships = [] @@ -550,15 +573,12 @@ def to_spdx(project, include_files=False): license_expressions.append(license_expression) for dependency in discovereddependency_qs: - packages_as_spdx.append(dependency.as_spdx()) - if dependency.for_package: - relationships.append( - spdx.Relationship( - spdx_id=dependency.spdx_id, - related_spdx_id=dependency.for_package.spdx_id, - relationship="DEPENDENCY_OF", - ) - ) + spdx_relationship = get_dependency_as_spdx_relationship( + dependency, + document_spdx_id, + packages_as_spdx, + ) + relationships.append(spdx_relationship) files_as_spdx = [] if include_files: @@ -568,6 +588,7 @@ def to_spdx(project, include_files=False): ] document = spdx.Document( + spdx_id=document_spdx_id, name=f"scancodeio_{project.name}", namespace=f"https://scancode.io/spdxdocs/{project.uuid}", creation_info=spdx.CreationInfo(tool=f"ScanCode.io-{scancodeio_version}"), diff --git a/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json b/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json index 0554dc617..23004dab2 100644 --- a/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json +++ b/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json @@ -1,7 +1,7 @@ { "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT", + "SPDXID": "SPDXRef-DOCUMENT-2f5f5927-2cad-4ecb-9043-fda5337bd501", "name": "scancodeio_asgiref", "documentNamespace": "https://scancode.io/spdxdocs/2f5f5927-2cad-4ecb-9043-fda5337bd501", "creationInfo": { From 1a9849f8a13190f7311321c6417f41e53d3386f2 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 31 Jul 2024 17:18:37 +0400 Subject: [PATCH 3/9] Add all package and dependencies to the tree view #1145 Signed-off-by: tdruez --- scanpipe/models.py | 12 ++++++++++++ scanpipe/views.py | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/scanpipe/models.py b/scanpipe/models.py index f943a9dc7..386fafcac 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3546,6 +3546,18 @@ def as_cyclonedx(self): class DiscoveredDependencyQuerySet( PackageURLQuerySetMixin, VulnerabilityQuerySetMixin, ProjectRelatedQuerySet ): + def project_dependencies(self): + return self.filter(for_package__isnull=True) + + def package_dependencies(self): + return self.filter(for_package__isnull=False) + + def resolved(self): + return self.filter(resolved_to_package__isnull=False) + + def unresolved(self): + return self.filter(resolved_to_package__isnull=True) + def prefetch_for_serializer(self): """ Optimized prefetching for a QuerySet to be consumed by the diff --git a/scanpipe/views.py b/scanpipe/views.py index e3776f10a..8c18ffe5b 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -2303,6 +2303,11 @@ def get_dependency_tree(self, project): root_packages = project.discoveredpackages.root_packages().order_by("name") project_children = [self.get_node(package) for package in root_packages] + # Dependencies with no assigned `for_packages`. + project_dependencies = project.discovereddependencies.project_dependencies() + for dependency in project_dependencies: + project_children.append({"name": dependency.package_url}) + project_tree = { "name": project.name, "children": project_children, @@ -2312,10 +2317,17 @@ def get_dependency_tree(self, project): def get_node(self, package): node = {"name": str(package)} + # Resolved dependencies children = [ self.get_node(child_package) for child_package in package.children_packages.all() ] + + unresolved_dependencies = package.declared_dependencies.unresolved() + for dependency in unresolved_dependencies: + children.append({"name": dependency.package_url}) + if children: node["children"] = children + return node From f4afdc056e3bbe2b32106db246afea83995f90af Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 31 Jul 2024 17:52:57 +0400 Subject: [PATCH 4/9] Fix unit tests #1145 Signed-off-by: tdruez --- pyproject.toml | 1 - scanpipe/pipes/spdx.py | 27 +++++++++++++-------------- scanpipe/tests/test_models.py | 17 ----------------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a5a18bec1..22c275390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,4 +35,3 @@ max-complexity = 10 [tool.ruff.lint.per-file-ignores] # Allow the usage of assert in the test_spdx file. "**/test_spdx.py*" = ["S101"] -"scanpipe/pipes/spdx.py" = ["UP006", "UP035"] diff --git a/scanpipe/pipes/spdx.py b/scanpipe/pipes/spdx.py index 815cfa7f2..5b3024361 100644 --- a/scanpipe/pipes/spdx.py +++ b/scanpipe/pipes/spdx.py @@ -27,7 +27,6 @@ from dataclasses import field from datetime import datetime from pathlib import Path -from typing import List # Python 3.8 compatibility SPDX_SPEC_VERSION = "2.3" SPDX_LICENSE_LIST_VERSION = "3.20" @@ -271,7 +270,7 @@ class ExtractedLicensingInfo: name: str = "" comment: str = "" - see_alsos: List[str] = field(default_factory=list) + see_alsos: list[str] = field(default_factory=list) def as_dict(self): """Return the data as a serializable dict.""" @@ -331,9 +330,9 @@ class Package: comment: str = "" license_comments: str = "" - checksums: List[Checksum] = field(default_factory=list) - external_refs: List[ExternalRef] = field(default_factory=list) - attribution_texts: List[str] = field(default_factory=list) + checksums: list[Checksum] = field(default_factory=list) + external_refs: list[ExternalRef] = field(default_factory=list) + attribution_texts: list[str] = field(default_factory=list) def as_dict(self): """Return the data as a serializable dict.""" @@ -426,18 +425,18 @@ class File: spdx_id: str name: str - checksums: List[Checksum] = field(default_factory=list) + checksums: list[Checksum] = field(default_factory=list) license_concluded: str = "NOASSERTION" copyright_text: str = "NOASSERTION" - license_in_files: List[str] = field(default_factory=list) - contributors: List[str] = field(default_factory=list) + license_in_files: list[str] = field(default_factory=list) + contributors: list[str] = field(default_factory=list) notice_text: str = "" # Supported values: # SOURCE | BINARY | ARCHIVE | APPLICATION | AUDIO | IMAGE | TEXT | VIDEO | # DOCUMENTATION | SPDX | OTHER - types: List[str] = field(default_factory=list) - attribution_texts: List[str] = field(default_factory=list) + types: list[str] = field(default_factory=list) + attribution_texts: list[str] = field(default_factory=list) comment: str = "" license_comments: str = "" @@ -545,16 +544,16 @@ class Document: name: str namespace: str creation_info: CreationInfo - packages: List[Package] + packages: list[Package] spdx_id: str = "SPDXRef-DOCUMENT" version: str = SPDX_SPEC_VERSION data_license: str = "CC0-1.0" comment: str = "" - files: List[File] = field(default_factory=list) - extracted_licenses: List[ExtractedLicensingInfo] = field(default_factory=list) - relationships: List[Relationship] = field(default_factory=list) + files: list[File] = field(default_factory=list) + extracted_licenses: list[ExtractedLicensingInfo] = field(default_factory=list) + relationships: list[Relationship] = field(default_factory=list) def as_dict(self): """Return the SPDX document as a serializable dict.""" diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 3cc15b04e..168ba04a8 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2712,23 +2712,6 @@ def test_scanpipe_discovered_dependency_model_create_from_data(self): ) self.assertEqual("pypi_sdist_pkginfo", dependency.datasource_id) - # Test field validation when using create_from_data - dependency_count = DiscoveredDependency.objects.count() - incomplete_data = dict(dependency_data1) - incomplete_data["dependency_uid"] = "" - self.assertIsNone( - DiscoveredDependency.create_from_data(project1, incomplete_data) - ) - self.assertEqual(dependency_count, DiscoveredDependency.objects.count()) - message = project1.projectmessages.latest("created_date") - self.assertEqual("DiscoveredDependency", message.model) - self.assertEqual(ProjectMessage.Severity.WARNING, message.severity) - expected_message = "No values for the following required fields: dependency_uid" - self.assertEqual(expected_message, message.description) - self.assertEqual(dependency_data1["purl"], message.details["purl"]) - self.assertEqual("", message.details["dependency_uid"]) - self.assertEqual("", message.traceback) - def test_scanpipe_discovered_package_model_unique_package_uid_in_project(self): project1 = Project.objects.create(name="Analysis") From f65682982819a3462f85063c3ae44a3929865112 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 31 Jul 2024 18:44:13 +0400 Subject: [PATCH 5/9] Add unit tests #1145 Signed-off-by: tdruez --- .../tests/data/spdx/dependencies.spdx.json | 112 ++++++++++++++++++ scanpipe/tests/pipes/test_output.py | 41 +++++++ scanpipe/tests/test_models.py | 24 +++- 3 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 scanpipe/tests/data/spdx/dependencies.spdx.json diff --git a/scanpipe/tests/data/spdx/dependencies.spdx.json b/scanpipe/tests/data/spdx/dependencies.spdx.json new file mode 100644 index 000000000..bd20db1a2 --- /dev/null +++ b/scanpipe/tests/data/spdx/dependencies.spdx.json @@ -0,0 +1,112 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT-b74fe5df-e965-415e-ba65-f38421a0695d", + "name": "scancodeio_analysis", + "documentNamespace": "https://scancode.io/spdxdocs/b74fe5df-e965-415e-ba65-f38421a0695d", + "creationInfo": { + "created": "2000-01-01T01:02:03Z", + "creators": [ + "Tool: ScanCode.io" + ], + "licenseListVersion": "3.20" + }, + "packages": [ + { + "name": "a", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-a83a60de-81bc-4bf4-b48c-dc78e0e658a9", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:type/a" + } + ] + }, + { + "name": "b", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-81147701-285f-485c-ba36-9cd3742790b1", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:type/b" + } + ] + }, + { + "name": "z", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-e391c33e-d7d0-4a97-a3c3-e947375c53d5", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:type/z" + } + ] + }, + { + "name": "", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-for_package_b", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "licenseDeclared": "NOASSERTION" + }, + { + "name": "unresolved", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-unresolved", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "licenseDeclared": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:type/unresolved" + } + ] + } + ], + "documentDescribes": [ + "SPDXRef-scancodeio-discoveredpackage-a83a60de-81bc-4bf4-b48c-dc78e0e658a9", + "SPDXRef-scancodeio-discoveredpackage-81147701-285f-485c-ba36-9cd3742790b1", + "SPDXRef-scancodeio-discoveredpackage-e391c33e-d7d0-4a97-a3c3-e947375c53d5", + "SPDXRef-scancodeio-discovereddependency-for_package_b", + "SPDXRef-scancodeio-discovereddependency-unresolved" + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-scancodeio-discoveredpackage-81147701-285f-485c-ba36-9cd3742790b1", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-a83a60de-81bc-4bf4-b48c-dc78e0e658a9", + "relationshipType": "DEPENDENCY_OF" + }, + { + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-for_package_b", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-81147701-285f-485c-ba36-9cd3742790b1", + "relationshipType": "DEPENDENCY_OF" + }, + { + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-unresolved", + "relatedSpdxElement": "SPDXRef-DOCUMENT-b74fe5df-e965-415e-ba65-f38421a0695d", + "relationshipType": "DEPENDENCY_OF" + } + ], + "comment": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "files": [] +} \ No newline at end of file diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index 388a5585e..5a8fb8570 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -370,6 +370,47 @@ def test_scanpipe_pipes_outputs_to_spdx_extracted_licenses(self): self.assertEqual(expected, license_infos["seeAlsos"]) self.assertTrue(license_infos["extractedText"].startswith("License:")) + @mock.patch("uuid.uuid4") + def test_scanpipe_pipes_outputs_to_spdx_dependencies(self, mock_uuid4): + forced_uuid = "b74fe5df-e965-415e-ba65-f38421a0695d" + mock_uuid4.return_value = forced_uuid + project = Project.objects.create(name="Analysis", uuid=forced_uuid) + + a = make_package( + project, "pkg:type/a", uuid="a83a60de-81bc-4bf4-b48c-dc78e0e658a9" + ) + b = make_package( + project, "pkg:type/b", uuid="81147701-285f-485c-ba36-9cd3742790b1" + ) + # 1. Package resolved dependency + make_dependency(project, for_package=a, resolved_to_package=b) + # 2. Package unresolved dependency + make_dependency(project, for_package=b, dependency_uid="for_package_b") + # 3. Project unresolved dependency + unresolved_dependency = make_dependency(project, dependency_uid="unresolved") + unresolved_dependency.set_package_url("pkg:type/unresolved") + unresolved_dependency.save() + # 4. Project package + make_package(project, "pkg:type/z", uuid="e391c33e-d7d0-4a97-a3c3-e947375c53d5") + + self.assertEqual(3, project.discoveredpackages.count()) + self.assertEqual(3, project.discovereddependencies.count()) + + output_file = output.to_spdx(project=project) + results_json = json.loads(output_file.read_text()) + self.assertEqual(5, len(results_json["packages"])) + self.assertEqual(3, len(results_json["relationships"])) + + # Patch the `created` date and tool version + results_json["creationInfo"]["created"] = "2000-01-01T01:02:03Z" + results_json["creationInfo"]["creators"] = ["Tool: ScanCode.io"] + # Files ordering is system dependent, excluded for now + results_json["files"] = [] + results = json.dumps(results_json, indent=2) + + expected_file = self.data / "spdx" / "dependencies.spdx.json" + self.assertResultsEqual(expected_file, results) + def test_scanpipe_pipes_outputs_make_unknown_license_object(self): licensing = get_licensing() parsed_expression = licensing.parse("some-unknown-license") diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 168ba04a8..9c32db542 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -1849,8 +1849,20 @@ def test_scanpipe_discovered_package_queryset_dependency_methods(self): z = make_package(project, "pkg:type/z") # Project -> A -> B -> C # Project -> Z - make_dependency(project, for_package=a, resolved_to_package=b) - make_dependency(project, for_package=b, resolved_to_package=c) + a_to_b = make_dependency( + project, for_package=a, resolved_to_package=b, dependency_uid="a_to_b" + ) + b_to_c = make_dependency( + project, for_package=b, resolved_to_package=c, dependency_uid="b_to_c" + ) + unresolved_dependency = make_dependency(project, dependency_uid="unresolved") + + self.assertFalse(a_to_b.is_project_dependency) + self.assertTrue(a_to_b.is_for_package) + self.assertTrue(a_to_b.is_resolved_to_package) + self.assertTrue(unresolved_dependency.is_project_dependency) + self.assertFalse(unresolved_dependency.is_for_package) + self.assertFalse(unresolved_dependency.is_resolved_to_package) project_packages_qs = project.discoveredpackages.order_by("name") root_packages = project_packages_qs.root_packages() @@ -1858,6 +1870,14 @@ def test_scanpipe_discovered_package_queryset_dependency_methods(self): non_root_packages = project_packages_qs.non_root_packages() self.assertEqual([b, c], list(non_root_packages)) + dependency_qs = project.discovereddependencies + self.assertEqual( + [unresolved_dependency], list(dependency_qs.project_dependencies()) + ) + self.assertEqual([a_to_b, b_to_c], list(dependency_qs.package_dependencies())) + self.assertEqual([a_to_b, b_to_c], list(dependency_qs.resolved())) + self.assertEqual([unresolved_dependency], list(dependency_qs.unresolved())) + @skipIf(sys.platform != "linux", "Ordering differs on macOS.") def test_scanpipe_codebase_resource_model_walk_method(self): fixtures = self.data / "asgiref" / "asgiref-3.3.0_walk_test_fixtures.json" From 25e9bf131a1cb2aff84622e258413adb0b88cd6f Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 1 Aug 2024 12:30:33 +0400 Subject: [PATCH 6/9] Refactor the tree view as a pure CSS solution (drop d3.js) #1145 Signed-off-by: tdruez --- .../static/tree-views/expand-collapse.svg | 7 ++ scancodeio/static/tree-views/tree.css | 75 +++++++++++ scancodeio/static/tree-views/tree.css.ABOUT | 10 ++ scancodeio/static/tree-views/tree.css.NOTICE | 29 +++++ .../scanpipe/project_dependency_tree.html | 118 ++++++++++-------- .../templates/scanpipe/tree/children.html | 16 +++ scanpipe/templates/scanpipe/tree/node.html | 23 ++++ scanpipe/views.py | 14 ++- 8 files changed, 238 insertions(+), 54 deletions(-) create mode 100644 scancodeio/static/tree-views/expand-collapse.svg create mode 100644 scancodeio/static/tree-views/tree.css create mode 100644 scancodeio/static/tree-views/tree.css.ABOUT create mode 100644 scancodeio/static/tree-views/tree.css.NOTICE create mode 100644 scanpipe/templates/scanpipe/tree/children.html create mode 100644 scanpipe/templates/scanpipe/tree/node.html diff --git a/scancodeio/static/tree-views/expand-collapse.svg b/scancodeio/static/tree-views/expand-collapse.svg new file mode 100644 index 000000000..f34809c99 --- /dev/null +++ b/scancodeio/static/tree-views/expand-collapse.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/scancodeio/static/tree-views/tree.css b/scancodeio/static/tree-views/tree.css new file mode 100644 index 000000000..b14023615 --- /dev/null +++ b/scancodeio/static/tree-views/tree.css @@ -0,0 +1,75 @@ +.tree{ + --spacing : 1.5rem; + --radius : 10px; +} + +.tree li{ + display : block; + position : relative; + padding-left : calc(2 * var(--spacing) - var(--radius) - 2px); +} + +.tree ul{ + margin-left : calc(var(--radius) - var(--spacing)); + padding-left : 0; +} + +.tree ul li{ + border-left : 2px solid #ddd; +} + +.tree ul li:last-child{ + border-color : transparent; +} + +.tree ul li::before{ + content : ''; + display : block; + position : absolute; + top : calc(var(--spacing) / -2); + left : -2px; + width : calc(var(--spacing) + 2px); + height : calc(var(--spacing) + 1px); + border : solid #ddd; + border-width : 0 0 2px 2px; +} + +.tree summary{ + display : block; + cursor : pointer; +} + +.tree summary::marker, +.tree summary::-webkit-details-marker{ + display : none; +} + +.tree summary:focus{ + outline : none; +} + +.tree summary:focus-visible{ + outline : 1px dotted #000; +} + +.tree li::after, +.tree summary::before{ + content : ''; + display : block; + position : absolute; + top : calc(var(--spacing) / 2 - var(--radius)); + left : calc(var(--spacing) - var(--radius) - 1px); + width : calc(2 * var(--radius)); + height : calc(2 * var(--radius)); + border-radius : 50%; + background : #ddd; +} + +.tree summary::before{ + z-index : 1; + background : #696 url('expand-collapse.svg') 0 0; +} + +.tree details[open] > summary::before{ + background-position : calc(-2 * var(--radius)) 0; +} diff --git a/scancodeio/static/tree-views/tree.css.ABOUT b/scancodeio/static/tree-views/tree.css.ABOUT new file mode 100644 index 000000000..13316e526 --- /dev/null +++ b/scancodeio/static/tree-views/tree.css.ABOUT @@ -0,0 +1,10 @@ +about_resource: tree.css +name: css-tree-views +homepage_url: https://iamkate.com/code/tree-views/ +description: A tree view (collapsible list) can be created using only html and css, without + the need for JavaScript. Accessibility software will see the tree view as lists nested inside + disclosure widgets, and the standard keyboard interaction is supported automatically. +license_expression: cc0-1.0 +licenses: + - key: cc0-1.0 + name: cc0-1.0 diff --git a/scancodeio/static/tree-views/tree.css.NOTICE b/scancodeio/static/tree-views/tree.css.NOTICE new file mode 100644 index 000000000..319755b97 --- /dev/null +++ b/scancodeio/static/tree-views/tree.css.NOTICE @@ -0,0 +1,29 @@ +Free content on iamkate.com +The web was still young when I first went online in 1998. It felt like a utopian dream of free culture and free knowledge. Anyone could contribute, and within weeks I had learnt html and created my first site, hosted in the 10mb of webspace my isp included as standard. + +I’ve watched as the dream has become a nightmare of surveillance and monetisation. Companies such as Google and Facebook offer their services for free to the public because their real products are their advertising networks powered by the personal data of their visitors. + +The only concern of these companies and their shareholders is to maximise their income from advertising, regardless of the costs to society. They use dubious schemes to avoid paying tax. They encourage addiction and risk the mental health of their visitors. They threaten democratic institutions. + +I have little influence over the wider web, but I can control my small part of it, creating a haven that remains true to the original dream. This page describes my approach to copyright, my promise to protect the privacy of my visitors, and my commitment to transparency. + +Copyright +Copyright limits creativity and holds back progress by restricting our rights to build upon the works of others. Copyleft licences attempt to use copyright against itself, but “the master’s tools will never dismantle the master’s house”, as Audre Lorde remarked in a different context. + +All content on my site is released under the terms of the Creative Commons CC0 1.0 Universal Legal Code. This means I have waived all copyright and related rights to the extent possible under law, with the intention of dedicating the content to the public domain. You can use and adapt it without attribution. + +Privacy +Every site is hosted on a server, which is usually operated by a third party due to the expertise needed to manage servers securely. Most sites are accessed indirectly through the servers of a content delivery network, which protects the original server from attacks that could disable the site. + +My site is hosted on Cloudflare Pages. Cloudflare is both the host and the content delivery network, avoiding the need to trust two separate third parties. Cloudflare have a strong commitment to privacy and data protection, and frequently write about developing systems to protect visitor privacy. + +Almost every site today includes code that tracks visitors for statistical and advertising purposes. Often the site owner includes code with the deliberate aim of tracking their visitors, but sometimes they just want to include a feature provided by a third party, and that provider includes their own tracking code. + +My site doesn’t include any tracking code, and doesn’t load any code from third parties. It doesn’t have a cookie banner because it doesn’t use cookies. Instead of an invasive analytics system, Cloudflare Web Analytics gives me the most important statistics without tracking individual visitors. + +Transparency +You probably don’t know me, and shouldn’t have to trust me. Instead, you should be able to check security and privacy claims for yourself. Unfortunately most sites today use a process called code minification, which makes them faster but also makes it harder for other people to understand their code. + +The Mozilla Observatory report for my site confirms the presence of various security and privacy features, resulting in a perfect A+ rating. One of these features, the content security policy, prevents browsers from loading code and other resources from third parties. + +My site doesn’t need to use code minification in order to load quickly due to its simple design, efficient implementation, and absence of resources loaded from third parties. As a result, other software developers can easily understand how the layout, styling, and interactive features are created. diff --git a/scanpipe/templates/scanpipe/project_dependency_tree.html b/scanpipe/templates/scanpipe/project_dependency_tree.html index ff368c05e..808dd1c78 100644 --- a/scanpipe/templates/scanpipe/project_dependency_tree.html +++ b/scanpipe/templates/scanpipe/project_dependency_tree.html @@ -1,7 +1,22 @@ {% extends "scanpipe/base.html" %} +{% load static %} {% block title %}ScanCode.io: {{ project.name }} - Dependency tree{% endblock %} +{% block extrahead %} + + +{% endblock %} + {% block content %}
{% include 'scanpipe/includes/navbar_header.html' %} @@ -13,64 +28,63 @@
- {% if recursion_error %} -
-
- The dependency tree cannot be rendered as it contains circular references. - {{ message|linebreaksbr }} -
-
- {% endif %} -
+
+ {% if recursion_error %} +
+
+ The dependency tree cannot be rendered as it contains circular references. + {{ message|linebreaksbr }} +
+
+ {% endif %} + +
+ + +
+ +
    +
  • +
    + + {{ dependency_tree.name }} + + {% include 'scanpipe/tree/children.html' with children=dependency_tree.children %} +
    +
    • +
+
{% endblock %} {% block scripts %} - - - {{ dependency_tree|json_script:"dependency_tree" }} - {{ row_count|json_script:"row_count" }} - {{ max_depth|json_script:"max_depth" }} - + collapseAllButton.addEventListener('click', collapseAllDetails); + expendAllButton.addEventListener('click', expendAllDetails); + {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/children.html b/scanpipe/templates/scanpipe/tree/children.html new file mode 100644 index 000000000..e6aa6553e --- /dev/null +++ b/scanpipe/templates/scanpipe/tree/children.html @@ -0,0 +1,16 @@ +
    + {% for node in children %} +
  • + {% if node.children %} +
    + + {% include 'scanpipe/tree/node.html' with node=node only %} + + {% include 'scanpipe/tree/children.html' with children=node.children only %} +
    + {% else %} + {% include 'scanpipe/tree/node.html' with node=node only %} + {% endif %} +
    • + {% endfor %} +
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/node.html b/scanpipe/templates/scanpipe/tree/node.html new file mode 100644 index 000000000..e42f671b7 --- /dev/null +++ b/scanpipe/templates/scanpipe/tree/node.html @@ -0,0 +1,23 @@ +{% if node.name %} + {{ node.name }} +{% else %} + Missing data +{% endif %} + + + {% if node.url %} + + + + {% endif %} + {% if node.compliance_alert == "warning" or node.compliance_alert == "error" %} + + + + {% endif %} + {% if node.is_vulnerable %} + + + + {% endif %} + \ No newline at end of file diff --git a/scanpipe/views.py b/scanpipe/views.py index 8c18ffe5b..e4f4fff58 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -2316,7 +2316,12 @@ def get_dependency_tree(self, project): return project_tree def get_node(self, package): - node = {"name": str(package)} + node = { + "name": str(package), + "url": package.get_absolute_url(), + "compliance_alert": package.compliance_alert, + "is_vulnerable": package.is_vulnerable, + } # Resolved dependencies children = [ self.get_node(child_package) @@ -2325,7 +2330,12 @@ def get_node(self, package): unresolved_dependencies = package.declared_dependencies.unresolved() for dependency in unresolved_dependencies: - children.append({"name": dependency.package_url}) + children.append( + { + "name": dependency.package_url, + "is_vulnerable": dependency.is_vulnerable, + } + ) if children: node["children"] = children From f0efc3a13483740847e0e4f37ca6ab73692402c9 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 1 Aug 2024 17:39:11 +0400 Subject: [PATCH 7/9] Refine the DiscoveredDependency.create_from_data method #1145 Signed-off-by: tdruez --- scanpipe/models.py | 16 ++++++++++++---- scanpipe/tests/test_models.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index 386fafcac..ecbc7520a 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3777,6 +3777,12 @@ def create_from_data( Create and returns a DiscoveredDependency for a `project` from the `dependency_data`. + The `for_package` and `resolved_to_package` FK can be provided as args or + in the dependency_data providing the `for_package_uid` and + `resolve_to_package_uid`. + Note that a dependency without a `for_package` FK is a project dependency and + a dependency without a `resolve_to_package` is unresolved. + If `strip_datafile_path_root` is True, then `create_from_data()` will strip the root path segment from the `datafile_path` of `dependency_data` before looking up the corresponding CodebaseResource @@ -3792,11 +3798,13 @@ def create_from_data( for_package_uid = dependency_data.get("for_package_uid") if not for_package and for_package_uid: - for_package = project_packages_qs.get(package_uid=for_package_uid) + for_package = project_packages_qs.get_or_none(package_uid=for_package_uid) - resolved_to_uid = dependency_data.get("resolved_to_uid") - if not resolved_to_package and resolved_to_uid: - resolved_to_package = project_packages_qs.get(package_uid=resolved_to_uid) + resolve_to_package_uid = dependency_data.get("resolve_to_package_uid") + if not resolved_to_package and resolve_to_package_uid: + resolved_to_package = project_packages_qs.get_or_none( + package_uid=resolve_to_package_uid + ) datafile_path = dependency_data.get("datafile_path") if not datafile_resource and datafile_path: diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 9c32db542..dc2b5c83a 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2704,10 +2704,11 @@ def test_scanpipe_discovered_package_model_create_from_data_missing_type(self): def test_scanpipe_discovered_dependency_model_create_from_data(self): project1 = Project.objects.create(name="Analysis") - DiscoveredPackage.create_from_data(project1, package_data1) + package1 = DiscoveredPackage.create_from_data(project1, package_data1) CodebaseResource.objects.create( project=project1, path="daglib-0.3.2.tar.gz-extract/daglib-0.3.2/PKG-INFO" ) + # Unresolved dependency dependency = DiscoveredDependency.create_from_data( project1, dependency_data1, strip_datafile_path_root=False ) @@ -2731,6 +2732,17 @@ def test_scanpipe_discovered_dependency_model_create_from_data(self): dependency.datafile_path, ) self.assertEqual("pypi_sdist_pkginfo", dependency.datasource_id) + self.assertFalse(dependency.is_project_dependency) + self.assertTrue(dependency.is_for_package) + self.assertFalse(dependency.is_resolved_to_package) + + # Resolved project dependency, resolved_to_package provided as arg + dependency2 = DiscoveredDependency.create_from_data( + project1, dependency_data={}, resolved_to_package=package1 + ) + self.assertTrue(dependency2.is_project_dependency) + self.assertFalse(dependency2.is_for_package) + self.assertTrue(dependency2.is_resolved_to_package) def test_scanpipe_discovered_package_model_unique_package_uid_in_project(self): project1 = Project.objects.create(name="Analysis") From dd20d4d25f4394fdff5bb5305f63e6cc8e639cb6 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 1 Aug 2024 18:32:37 +0400 Subject: [PATCH 8/9] Add features to the dependency tree view #1145 Signed-off-by: tdruez --- .../scanpipe/includes/project_list_table.html | 5 ++ .../scanpipe/project_dependency_tree.html | 69 ++++++++++++++++++- .../templates/scanpipe/tree/children.html | 5 +- scanpipe/views.py | 1 + 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/scanpipe/templates/scanpipe/includes/project_list_table.html b/scanpipe/templates/scanpipe/includes/project_list_table.html index fe001d136..8403ec3af 100644 --- a/scanpipe/templates/scanpipe/includes/project_list_table.html +++ b/scanpipe/templates/scanpipe/includes/project_list_table.html @@ -28,6 +28,11 @@ {{ project.discovereddependencies_count|intcomma }} + + + + + {% else %} 0 {% endif %} diff --git a/scanpipe/templates/scanpipe/project_dependency_tree.html b/scanpipe/templates/scanpipe/project_dependency_tree.html index 808dd1c78..2f45a3025 100644 --- a/scanpipe/templates/scanpipe/project_dependency_tree.html +++ b/scanpipe/templates/scanpipe/project_dependency_tree.html @@ -10,6 +10,9 @@ line-height: 1.8rem; --spacing : 2rem; } + .tree summary { + display: inline-block; + } .tree summary::before{ background-color: rgb(72, 199, 142); background-image: url('{% static "tree-views/expand-collapse.svg" %}'); @@ -51,9 +54,21 @@ + + -
    +
    • @@ -69,6 +84,8 @@ {% block scripts %} {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/children.html b/scanpipe/templates/scanpipe/tree/children.html index e6aa6553e..ba513c745 100644 --- a/scanpipe/templates/scanpipe/tree/children.html +++ b/scanpipe/templates/scanpipe/tree/children.html @@ -1,6 +1,9 @@
        {% for node in children %} -
      • +
      • {% if node.children %}
        diff --git a/scanpipe/views.py b/scanpipe/views.py index e4f4fff58..b84969872 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1927,6 +1927,7 @@ class DiscoveredPackageDetailsView( "field_name": "other_license_expression_spdx", "label": "Other license expression (SPDX)", }, + "compliance_alert", "extracted_license_statement", "copyright", "holder", From 113fda8ea4e0e62aede4f98a6538521bfb62300d Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 1 Aug 2024 18:43:51 +0400 Subject: [PATCH 9/9] Refactor the dependency support in LoadSBOM #1145 Signed-off-by: tdruez --- scanpipe/pipelines/load_sbom.py | 16 ++++++++++++---- scanpipe/pipes/cyclonedx.py | 11 ++++++----- scanpipe/pipes/resolve.py | 32 ++++++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/scanpipe/pipelines/load_sbom.py b/scanpipe/pipelines/load_sbom.py index 617c31244..4fe9a4cd9 100644 --- a/scanpipe/pipelines/load_sbom.py +++ b/scanpipe/pipelines/load_sbom.py @@ -20,6 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. +from scanpipe.models import DiscoveredDependency from scanpipe.pipelines.scan_codebase import ScanCodebase from scanpipe.pipes import resolve @@ -44,7 +45,7 @@ def steps(cls): cls.flag_empty_files, cls.flag_ignored_resources, cls.get_sbom_inputs, - cls.get_packages_from_sboms, + cls.get_data_from_sboms, cls.create_packages_from_sboms, cls.create_dependencies_from_sboms, ) @@ -53,13 +54,13 @@ def get_sbom_inputs(self): """Locate all the SBOMs among the codebase resources.""" self.manifest_resources = resolve.get_manifest_resources(self.project) - def get_packages_from_sboms(self): + def get_data_from_sboms(self): """Get packages data from SBOMs.""" - self.packages = resolve.get_packages( + self.packages, self.dependencies = resolve.get_data_from_manifests( project=self.project, package_registry=resolve.sbom_registry, manifest_resources=self.manifest_resources, - model="get_packages_from_sboms", + model="get_data_from_sboms", ) def create_packages_from_sboms(self): @@ -71,4 +72,11 @@ def create_packages_from_sboms(self): def create_dependencies_from_sboms(self): """Create the dependency relationship declared in the SBOMs.""" + # TODO: Migrate the CycloneDX behavior too, see get_dependencies_from_manifest resolve.create_dependencies_from_packages_extra_data(project=self.project) + + for dependency_data in self.dependencies: + DiscoveredDependency.create_from_data( + project=self.project, + dependency_data=dependency_data, + ) diff --git a/scanpipe/pipes/cyclonedx.py b/scanpipe/pipes/cyclonedx.py index 4e380360f..636266b1f 100644 --- a/scanpipe/pipes/cyclonedx.py +++ b/scanpipe/pipes/cyclonedx.py @@ -155,12 +155,12 @@ def cyclonedx_component_to_package_data(cdx_component, dependencies=None): dependencies = dependencies or {} extra_data = {} - # Store the original bom_ref and dependencies for future processing. bom_ref = str(cdx_component.bom_ref) - if bom_ref: - extra_data["bom_ref"] = bom_ref - if depends_on := dependencies.get(bom_ref): - extra_data["depends_on"] = depends_on + if depends_on := dependencies.get(bom_ref): + extra_data["depends_on"] = depends_on + + # Store the original "bom_ref" as package_uid for dependencies resolution. + package_uid = bom_ref package_url_dict = {} if cdx_component.purl: @@ -176,6 +176,7 @@ def cyclonedx_component_to_package_data(cdx_component, dependencies=None): extra_data["nestedComponents"] = sorted(nested_purls) package_data = { + "package_uid": package_uid, "name": cdx_component.name, "extracted_license_statement": declared_license, "copyright": cdx_component.copyright, diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 86ef49d52..6582de443 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -57,12 +57,27 @@ def resolve_manifest_resources(resource, package_registry): return packages -def get_packages(project, package_registry, manifest_resources, model=None): +def get_dependencies_from_manifest(resource): + """Get dependency data from resource.""" + dependencies = [] + + default_package_type = get_default_package_type(resource.location) + if not default_package_type: + return [] + + if default_package_type == "spdx": + dependencies = resolve_spdx_dependencies(input_location=resource.location) + + return dependencies + + +def get_data_from_manifests(project, package_registry, manifest_resources, model=None): """ - Get package data from package manifests/lockfiles/SBOMs or - get package data for resolved packages from package requirements. + Get package and dependency data from package manifests/lockfiles/SBOMs or + for resolved packages from package requirements. """ resolved_packages = [] + resolved_dependencies = [] sboms_headers = {} if not manifest_resources.exists(): @@ -73,7 +88,8 @@ def get_packages(project, package_registry, manifest_resources, model=None): return [] for resource in manifest_resources: - if packages := resolve_manifest_resources(resource, package_registry): + packages = resolve_manifest_resources(resource, package_registry) + if packages: resolved_packages.extend(packages) if headers := get_manifest_headers(resource): sboms_headers[resource.name] = headers @@ -84,10 +100,14 @@ def get_packages(project, package_registry, manifest_resources, model=None): resource=resource, ) + dependencies = get_dependencies_from_manifest(resource) + if dependencies: + resolved_dependencies.extend(dependencies) + if sboms_headers: project.update_extra_data({"sboms_headers": sboms_headers}) - return resolved_packages + return resolved_packages, resolved_dependencies def create_packages_and_dependencies(project, packages, resolved=False): @@ -136,7 +156,7 @@ def create_dependencies_from_packages_extra_data(project): for bom_ref in for_package.extra_data.get("depends_on", []): try: - resolved_to_package = project_packages.get(extra_data__bom_ref=bom_ref) + resolved_to_package = project_packages.get(package_uid=bom_ref) except (ObjectDoesNotExist, MultipleObjectsReturned): project.add_error( description=f"Could not find resolved_to package entry: {bom_ref}.",