diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index 1e7841555..83e628a7d 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -387,15 +387,15 @@ Optional arguments: - ``--format {json,spdx,cyclonedx,attribution}``: Specify the output format. **The default format is JSON**. -For example, running the ``inspect_packages`` pipeline on a manifest file: +For example, running the ``resolve_dependencies`` pipeline on a manifest file: .. code-block:: bash - $ run inspect_packages path/to/package.json > results.json + $ run resolve_dependencies path/to/package.json > results.json .. tip:: Use the "pipeline_name:group1,group2" syntax to select steps groups:: - $ run inspect_packages:StaticResolver package.json > results.json + $ run resolve_dependencies:StaticResolver package.json > results.json In the following example, running the ``scan_codebase`` followed by the ``find_vulnerabilities`` pipelines on a codebase directory: diff --git a/docs/faq.rst b/docs/faq.rst index 29e895270..d61436103 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -58,7 +58,7 @@ Here are some general guidelines based on different input scenarios: resolve packages from their package requirements, use the :ref:`resolve_dependencies ` pipeline. - When you have application **package archives/codebases** and optionally also - their **pre-resolved dependenices** and you want to **inspect packages** + their **pre-resolved dependenices** and you want to do a fast scan to **inspect packages** present in the package manifests and dependency, use the :ref:`inspect_packages ` pipeline. - For scenarios involving both a **development and deployment codebase**, consider using diff --git a/docs/tutorial_cli_end_to_end_scanning_to_dejacode.rst b/docs/tutorial_cli_end_to_end_scanning_to_dejacode.rst index 8d1ed5675..a03d49e9c 100644 --- a/docs/tutorial_cli_end_to_end_scanning_to_dejacode.rst +++ b/docs/tutorial_cli_end_to_end_scanning_to_dejacode.rst @@ -7,7 +7,7 @@ The focus of this tutorial is to guide you through scanning a codebase end to en starting with the dependency resolution, through the scanning proper, and finally the upload of the scan in DejaCode, using DepLock and ScanCode.io. -This is designed to run a faster, simple **inspect_packages** ScanCode.io pipeline. +This is designed to run a faster, simple **resolve_dependencies** ScanCode.io pipeline. .. note:: @@ -146,7 +146,7 @@ Run ScanCode Package Detection ------------------------------- Execute the following command to run the ScanCode scanner -with **inspect_packages** pipeline with static resolution +with **resolve_dependencies** pipeline with static resolution on the codebase in our current directory. .. code-block:: shell @@ -154,7 +154,7 @@ on the codebase in our current directory. docker run --rm \ -v "$(pwd)":/code \ ghcr.io/aboutcode-org/scancode.io:latest \ - sh -c "run inspect_packages:StaticResolver /code" \ + sh -c "run resolve_dependencies:StaticResolver /code" \ > results.json diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py index 9d28c07cf..acdd1059c 100644 --- a/scanpipe/pipelines/inspect_packages.py +++ b/scanpipe/pipelines/inspect_packages.py @@ -27,14 +27,16 @@ class InspectPackages(ScanCodebase): """ - Inspect a codebase for packages and pre-resolved dependencies. + Inspect a codebase for packages and dependencies. This pipeline inspects a codebase for application packages and their dependencies using package manifests and dependency lockfiles. It does not resolve dependencies, it does instead collect already pre-resolved dependencies from lockfiles, and direct dependencies (possibly not resolved) as found in - package manifests' dependency sections. + package manifests' dependency sections. This is a fast scan to get + all the PackageUrls present from packages and dependencies, without + performing package assembly or license/copyright scans. See documentation for the list of supported package manifests and dependency lockfiles: @@ -60,16 +62,7 @@ def scan_for_application_packages(self): """ scancode.scan_for_application_packages( project=self.project, - assemble=True, + assemble=False, package_only=True, progress_logger=self.log, ) - - @group("StaticResolver") - def resolve_dependencies(self): - """ - Create packages and dependency relationships from - lockfiles or manifests containing pre-resolved - dependencies. - """ - scancode.resolve_dependencies(project=self.project) diff --git a/scanpipe/pipelines/resolve_dependencies.py b/scanpipe/pipelines/resolve_dependencies.py index 781d1d639..1990c55a4 100644 --- a/scanpipe/pipelines/resolve_dependencies.py +++ b/scanpipe/pipelines/resolve_dependencies.py @@ -34,7 +34,17 @@ class ResolveDependencies(ScanCodebase): that contain dependency requirements, and resolves these to a concrete set of package versions. - Supports resolving packages for: + Supports statically resolving packages for: + - JavaScript: Npm, Yarn, Pnpm packages + - Python: python, poetry packages + - Swift packages + - Cocoapods packages + - Nuget packages + Note: Please run deplock from https://github.com/aboutcode-org/dependency-inspector + to generate the appropriate lockfiles as a pre-requisite to statically resolve + dependencies for the supported package ecosystems. + + Supports dynamically resolving packages for: - Python: using python-inspector, using requirements.txt and setup.py manifests as inputs """ diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index ba1d2181a..b4b51f157 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -63,7 +63,7 @@ class PurlDBException(Exception): if PURLDB_API_KEY: session.headers.update({"Authorization": f"Token {PURLDB_API_KEY}"}) -DEFAULT_TIMEOUT = 60 +DEFAULT_TIMEOUT = 300 # This key can be used for filtering ENRICH_EXTRA_DATA_KEY = "enrich_with_purldb" diff --git a/setup.cfg b/setup.cfg index f4a993dfb..4378d0b34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,7 +72,7 @@ install_requires = # Docker container-inspector==33.0.0 # ScanCode-toolkit - scancode-toolkit[packages]==32.2.1 + scancode-toolkit[packages]==32.3.0 extractcode[full]==31.0.0 commoncode==32.0.0 packageurl-python==0.15.6