Maven: Switch to the optimized dependency graph format

Rework the implementation to use DependencyGraphBuilder together with MavenDependencyHandler to produce the analyzer result. Adapt test classes as necessary. Rather than changing the expected test results, convert results to the classic format before they are compared with the expectations. This check is safer, as it makes sure that the content of the results has not changed, although they are now written in a different format. Add a helper function for this conversion to the test-utils module. Signed-off-by: Oliver Heger <[email protected]>
oss-review-toolkit · Apr 15, 2021 · 28eeb67 · 28eeb67
1 parent 7544866
commit 28eeb67
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 79 deletions.
diff --git a/analyzer/src/funTest/kotlin/managers/MavenFunTest.kt b/analyzer/src/funTest/kotlin/managers/MavenFunTest.kt
@@ -36,6 +36,7 @@ import org.ossreviewtoolkit.utils.test.DEFAULT_REPOSITORY_CONFIGURATION
 import org.ossreviewtoolkit.utils.test.USER_DIR
 import org.ossreviewtoolkit.utils.test.patchActualResult
 import org.ossreviewtoolkit.utils.test.patchExpectedResult
+import org.ossreviewtoolkit.utils.test.withResolvedScopes
 
 class MavenFunTest : StringSpec() {
     private val projectDir = File("src/funTest/assets/projects/synthetic/maven").absoluteFile
@@ -51,7 +52,7 @@ class MavenFunTest : StringSpec() {
 
             val result = createMaven().resolveSingleProject(pomFile)
 
-            result.toYaml() shouldBe expectedResult
+            result.withResolvedScopes().toYaml() shouldBe expectedResult
         }
 
         "jgnash-core dependencies are detected correctly" {
@@ -69,7 +70,7 @@ class MavenFunTest : StringSpec() {
 
             result.shouldNotBeNull()
             result should haveSize(1)
-            result.single().toYaml() shouldBe expectedResult
+            result.single().withResolvedScopes().toYaml() shouldBe expectedResult
         }
 
         "Root project dependencies are detected correctly" {
@@ -82,7 +83,7 @@ class MavenFunTest : StringSpec() {
 
             val result = createMaven().resolveSingleProject(pomFile)
 
-            result.toYaml() shouldBe expectedResult
+            result.withResolvedScopes().toYaml() shouldBe expectedResult
         }
 
         "Project dependencies are detected correctly" {
@@ -101,7 +102,7 @@ class MavenFunTest : StringSpec() {
 
             result.shouldNotBeNull()
             result should haveSize(1)
-            result.single().toYaml() shouldBe expectedResult
+            result.single().withResolvedScopes().toYaml() shouldBe expectedResult
         }
 
         "External dependencies are detected correctly" {
@@ -114,7 +115,7 @@ class MavenFunTest : StringSpec() {
 
             val result = createMaven().resolveSingleProject(pomFile)
 
-            result.toYaml() shouldBe expectedResult
+            result.withResolvedScopes().toYaml() shouldBe expectedResult
         }
 
         "Parent POM from Maven central can be resolved" {
@@ -133,7 +134,7 @@ class MavenFunTest : StringSpec() {
 
             val result = createMaven().resolveSingleProject(pomFile)
 
-            result.toYaml() shouldBe expectedResult
+            result.withResolvedScopes().toYaml() shouldBe expectedResult
         }
 
         "Maven Wagon extensions can be loaded" {
@@ -147,7 +148,7 @@ class MavenFunTest : StringSpec() {
 
             val result = createMaven().resolveSingleProject(pomFile)
 
-            patchActualResult(result.toYaml(), patchStartAndEndTime = true) shouldBe expectedResult
+            patchActualResult(result.withResolvedScopes().toYaml(), patchStartAndEndTime = true) shouldBe expectedResult
         }
     }
 

diff --git a/analyzer/src/main/kotlin/managers/Maven.kt b/analyzer/src/main/kotlin/managers/Maven.kt
@@ -21,34 +21,26 @@ package org.ossreviewtoolkit.analyzer.managers
 
 import java.io.File
 
-import org.apache.maven.project.ProjectBuildingException
 import org.apache.maven.project.ProjectBuildingResult
 
 import org.eclipse.aether.artifact.Artifact
-import org.eclipse.aether.graph.DependencyNode
 import org.eclipse.aether.repository.WorkspaceReader
 import org.eclipse.aether.repository.WorkspaceRepository
 
 import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
 import org.ossreviewtoolkit.analyzer.PackageManager
+import org.ossreviewtoolkit.analyzer.managers.utils.DependencyGraphBuilder
 import org.ossreviewtoolkit.analyzer.managers.utils.MavenSupport
 import org.ossreviewtoolkit.analyzer.managers.utils.identifier
 import org.ossreviewtoolkit.downloader.VersionControlSystem
 import org.ossreviewtoolkit.model.Identifier
-import org.ossreviewtoolkit.model.Package
-import org.ossreviewtoolkit.model.PackageLinkage
-import org.ossreviewtoolkit.model.PackageReference
 import org.ossreviewtoolkit.model.Project
 import org.ossreviewtoolkit.model.ProjectAnalyzerResult
-import org.ossreviewtoolkit.model.Scope
 import org.ossreviewtoolkit.model.Severity
 import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
 import org.ossreviewtoolkit.model.config.RepositoryConfiguration
 import org.ossreviewtoolkit.model.createAndLogIssue
-import org.ossreviewtoolkit.utils.collectMessagesAsString
-import org.ossreviewtoolkit.utils.log
 import org.ossreviewtoolkit.utils.searchUpwardsForSubdirectory
-import org.ossreviewtoolkit.utils.showStackTrace
 
 /**
  * The [Maven](https://maven.apache.org/) package manager for Java.
@@ -103,16 +95,12 @@ class Maven(
         val workingDir = definitionFile.parentFile
         val projectBuildingResult = mvn.buildMavenProject(definitionFile)
         val mavenProject = projectBuildingResult.project
-        val packagesById = mutableMapOf<String, Package>()
-        val scopesByName = mutableMapOf<String, Scope>()
+        val localProjects = localProjectBuildingResults.mapValues { it.value.project }
+        val dependencyHandler = MavenDependencyHandler(managerName, mvn, localProjects, sbtMode)
+        val graphBuilder = DependencyGraphBuilder(dependencyHandler)
 
         projectBuildingResult.dependencyResolutionResult.dependencyGraph.children.forEach { node ->
-            val scopeName = node.dependency.scope
-            val scope = scopesByName.getOrPut(scopeName) {
-                Scope(scopeName, sortedSetOf())
-            }
-
-            scope.dependencies += parseDependency(node, packagesById)
+            graphBuilder.addDependency(node.dependency.scope, node)
         }
 
         val declaredLicenses = MavenSupport.parseLicenses(mavenProject)
@@ -146,10 +134,11 @@ class Maven(
             vcs = vcsFromPackage,
             vcsProcessed = processProjectVcs(projectDir, vcsFromPackage, *vcsFallbackUrls),
             homepageUrl = homepageUrl.orEmpty(),
-            scopeDependencies = scopesByName.values.toSortedSet()
+            scopeDependencies = null,
+            dependencyGraph = graphBuilder.build()
         )
 
-        val packages = packagesById.values.toSortedSet()
+        val packages = graphBuilder.packages().toSortedSet()
         val issues = packages.mapNotNull { pkg ->
             if (pkg.description == "POM was created by Sonatype Nexus") {
                 createAndLogIssue(
@@ -164,57 +153,4 @@ class Maven(
 
         return listOf(ProjectAnalyzerResult(project, packages, issues))
     }
-
-    private fun parseDependency(node: DependencyNode, packages: MutableMap<String, Package>): PackageReference {
-        val identifier = node.artifact.identifier()
-
-        try {
-            val dependencies = node.children.mapNotNull { child ->
-                val toolsJarCoordinates = listOf("com.sun:tools:", "jdk.tools:jdk.tools:")
-                if (toolsJarCoordinates.any { child.artifact.identifier().startsWith(it) }) {
-                    log.info { "Omitting the Java < 1.9 system dependency on 'tools.jar'." }
-                    null
-                } else {
-                    parseDependency(child, packages)
-                }
-            }.toSortedSet()
-
-            val localProjects = localProjectBuildingResults.mapValues { it.value.project }
-
-            return if (localProjects.contains(identifier)) {
-                val id = Identifier(
-                    type = managerName,
-                    namespace = node.artifact.groupId,
-                    name = node.artifact.artifactId,
-                    version = node.artifact.version
-                )
-
-                log.info { "Dependency '${id.toCoordinates()}' refers to a local project." }
-
-                PackageReference(id, PackageLinkage.PROJECT_DYNAMIC, dependencies)
-            } else {
-                val pkg = packages.getOrPut(identifier) {
-                    // TODO: Omit the "localProjects" argument here once SBT is implemented independently of Maven as at
-                    //       this point we know already that "identifier" is not a local project.
-                    mvn.parsePackage(node.artifact, node.repositories, localProjects, sbtMode)
-                }
-
-                pkg.toReference(dependencies = dependencies)
-            }
-        } catch (e: ProjectBuildingException) {
-            e.showStackTrace()
-
-            return PackageReference(
-                Identifier(managerName, node.artifact.groupId, node.artifact.artifactId, node.artifact.version),
-                dependencies = sortedSetOf(),
-                issues = listOf(
-                    createAndLogIssue(
-                        source = managerName,
-                        message = "Could not get package information for dependency '$identifier': " +
-                                e.collectMessagesAsString()
-                    )
-                )
-            )
-        }
-    }
 }
diff --git a/test-utils/src/main/kotlin/Utils.kt b/test-utils/src/main/kotlin/Utils.kt
@@ -28,6 +28,8 @@ import java.io.File
 import java.time.Instant
 
 import org.ossreviewtoolkit.model.OrtResult
+import org.ossreviewtoolkit.model.Project
+import org.ossreviewtoolkit.model.ProjectAnalyzerResult
 import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
 import org.ossreviewtoolkit.model.config.RepositoryConfiguration
 import org.ossreviewtoolkit.model.mapper
@@ -91,3 +93,10 @@ fun patchActualResult(
 }
 
 fun readOrtResult(file: String) = File(file).let { it.mapper().readValue<OrtResult>(patchExpectedResult(it)) }
+
+/**
+ * Return a copy of this [ProjectAnalyzerResult] that has a [Project] with a resolved [Project.scopes] property.
+ * This can be used to convert results using the dependency graph format to the classic format, which may be necessary
+ * if test expectations are formulated in that way.
+ */
+fun ProjectAnalyzerResult.withResolvedScopes(): ProjectAnalyzerResult = copy(project = project.withResolvedScopes())