feat(DependencyGraphConverter): Support scope excludes

The convert() function now accepts an Excludes object. The result of the conversion contains only dependencies not defined by an excluded scope. This functionality is going to be used to implement scope exclusions centrally for package managers that do not support the dependency graph format natively, see [1]. [1] #3825 Signed-off-by: Oliver Heger <[email protected]>
oss-review-toolkit · Feb 7, 2023 · 09285cb · 09285cb
1 parent a3abf4e
commit 09285cb
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 15 deletions.
diff --git a/model/src/main/kotlin/utils/DependencyGraphConverter.kt b/model/src/main/kotlin/utils/DependencyGraphConverter.kt
@@ -27,13 +27,15 @@ import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageLinkage
 import org.ossreviewtoolkit.model.PackageReference
 import org.ossreviewtoolkit.model.Project
+import org.ossreviewtoolkit.model.config.Excludes
 
 /**
  * An object that supports the conversion of [AnalyzerResult]s to the dependency graph format.
  *
  * The main conversion function takes an [AnalyzerResult] and iterates over all projects contained in it. Each
  * project that still uses the dependency tree representation (i.e. the set of scopes) is converted, and a dependency
- * graph for the associated package manager is created.
+ * graph for the associated package manager is created. During this transformation, scope excludes are applied, so
+ * that the result contains only dependencies from included scopes.
  *
  * Via this converter, optimized results can be generated for package managers that do not yet support this format.
  * This approach is, however, not ideal, because during the conversion both the dependency tree and the dependency
@@ -42,33 +44,43 @@ import org.ossreviewtoolkit.model.Project
  */
 object DependencyGraphConverter {
     /**
-     * Convert the given [result], so that all dependencies are represented as dependency graphs. If the [result]
-     * already contains a dependency graph that covers all projects, it is returned as is.
+     * Convert the given [result], so that all dependencies are represented as dependency graphs. During conversion,
+     * apply the scope excludes defined in [excludes]. If the [result] already contains a dependency graph that covers
+     * all projects, it is returned as is.
      */
-    fun convert(result: AnalyzerResult): AnalyzerResult {
+    fun convert(result: AnalyzerResult, excludes: Excludes = Excludes.EMPTY): AnalyzerResult {
         val projectsToConvert = result.projectsWithScopes()
         if (projectsToConvert.isEmpty()) return result
 
-        val graphs = buildDependencyGraphs(projectsToConvert)
+        val graphs = buildDependencyGraphs(projectsToConvert, excludes)
+        val allGraphs = result.dependencyGraphs + graphs
+
+        val filteredPackages = if (excludes.scopes.isEmpty()) {
+            result.packages
+        } else {
+            filterExcludedPackages(graphs.values, result.packages)
+        }
 
         return result.copy(
-            dependencyGraphs = result.dependencyGraphs + graphs,
-            projects = result.projects.mapTo(mutableSetOf()) { it.convertToScopeNames() }
+            dependencyGraphs = allGraphs,
+            projects = result.projects.mapTo(mutableSetOf()) { it.convertToScopeNames(excludes) },
+            packages = filteredPackages
         )
     }
 
     /**
-     * Build [DependencyGraph]s for the given [projects]. The resulting map contains one graph for each package
-     * manager involved.
+     * Create and populate [DependencyGraphBuilder]s for the given [projects] taking [excludes] into account. The
+     * resulting map contains one fully initialized graph builder for each package manager involved which can be
+     * used to obtain the [DependencyGraph] and all packages.
      */
-    private fun buildDependencyGraphs(projects: Set<Project>): Map<String, DependencyGraph> {
+    private fun buildDependencyGraphs(projects: Set<Project>, excludes: Excludes): Map<String, DependencyGraph> {
         val graphs = mutableMapOf<String, DependencyGraph>()
 
         projects.groupBy { it.id.type }.forEach { (type, projectsForType) ->
             val builder = DependencyGraphBuilder(ScopesDependencyHandler)
 
             projectsForType.forEach { project ->
-                project.scopes.forEach { scope ->
+                project.scopes.filterNot { excludes.isScopeExcluded(it.name) }.forEach { scope ->
                     val scopeName = DependencyGraph.qualifyScope(project, scope.name)
                     scope.dependencies.forEach { dependency ->
                         builder.addDependency(scopeName, dependency)
@@ -82,6 +94,18 @@ object DependencyGraphConverter {
         return graphs
     }
 
+    /**
+     * Filter out all [packages] that are no longer referenced by one of the given [dependency graphs][graphs]. These
+     * packages have been subject of scope excludes.
+     */
+    private fun filterExcludedPackages(
+        graphs: Collection<DependencyGraph>,
+        packages: Collection<Package>
+    ): Set<Package> {
+        val includedPackages = graphs.flatMapTo(mutableSetOf()) { it.packages }
+        return packages.filterTo(mutableSetOf()) { it.id in includedPackages }
+    }
+
     /**
      * Determine the projects in this [AnalyzerResult] that require a conversion. These are the projects that manage
      * their dependencies in a scope structure.
@@ -91,11 +115,12 @@ object DependencyGraphConverter {
 
     /**
      * Convert the dependency representation used by this [Project] to the dependency graph format, i.e. a set of
-     * scope names. Return the same project if this format is already in use.
+     * scope names. Use the given [excludes] to filter out excluded scopes. Return the same project if this format is
+     * already in use.
      */
-    private fun Project.convertToScopeNames(): Project =
+    private fun Project.convertToScopeNames(excludes: Excludes): Project =
         takeIf { scopeNames != null } ?: copy(
-            scopeNames = scopes.mapTo(sortedSetOf()) { it.name },
+            scopeNames = scopes.filterNot { excludes.isScopeExcluded(it.name) }.mapTo(sortedSetOf()) { it.name },
             scopeDependencies = null
         )
 
@@ -119,4 +144,5 @@ object DependencyGraphConverter {
     }
 }
 
-fun AnalyzerResult.convertToDependencyGraph() = DependencyGraphConverter.convert(this)
+fun AnalyzerResult.convertToDependencyGraph(excludes: Excludes = Excludes.EMPTY) =
+    DependencyGraphConverter.convert(this, excludes)
diff --git a/model/src/test/kotlin/utils/DependencyGraphConverterTest.kt b/model/src/test/kotlin/utils/DependencyGraphConverterTest.kt
@@ -21,8 +21,11 @@ package org.ossreviewtoolkit.model.utils
 
 import io.kotest.assertions.fail
 import io.kotest.core.spec.style.WordSpec
+import io.kotest.inspectors.forAll
 import io.kotest.matchers.collections.beEmpty
 import io.kotest.matchers.collections.containExactlyInAnyOrder
+import io.kotest.matchers.collections.shouldContainOnly
+import io.kotest.matchers.ints.shouldBeLessThan
 import io.kotest.matchers.nulls.beNull
 import io.kotest.matchers.should
 import io.kotest.matchers.shouldBe
@@ -42,6 +45,9 @@ import org.ossreviewtoolkit.model.PackageReference
 import org.ossreviewtoolkit.model.Project
 import org.ossreviewtoolkit.model.ProjectAnalyzerResult
 import org.ossreviewtoolkit.model.Scope
+import org.ossreviewtoolkit.model.config.Excludes
+import org.ossreviewtoolkit.model.config.ScopeExclude
+import org.ossreviewtoolkit.model.config.ScopeExcludeReason
 import org.ossreviewtoolkit.model.readValue
 import org.ossreviewtoolkit.utils.test.shouldNotBeNull
 
@@ -82,6 +88,20 @@ class DependencyGraphConverterTest : WordSpec({
             convertedResult.packages should beTheSameInstanceAs(result.packages)
         }
 
+        "exclude scopes" {
+            val project = createProject("Maven", index = 1)
+
+            val result = createAnalyzerResult(project.createResult())
+
+            val scopeExclude = ScopeExclude("test", ScopeExcludeReason.TEST_DEPENDENCY_OF)
+            val excludes = Excludes(scopes = listOf(scopeExclude))
+
+            val convertedResult = DependencyGraphConverter.convert(result, excludes)
+
+            convertedResult.projects.single().scopeNames shouldContainOnly listOf("main")
+            convertedResult.packages.forAll { it.id.version.drop(2).toInt() shouldBeLessThan 110 }
+        }
+
         "convert a result with a partial dependency graph" {
             val gradleProject = createProject("Gradle", index = 1)
             val goProject1 = createProject("GoMod", index = 2)