Merge pull request #372 from DependencyTrack/backport-endpoint-outdat…

…ed-components

Add outdated components and direct dependencies in component endpoint

src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java

@@ -72,7 +72,12 @@ public Bom create(final Component component) {
         return create(components, null, null, null);
     }
 
-    private Bom create(final List<Component>components, final List<ServiceComponent> services, final List<Finding> findings, final Project project) {
+    private Bom create(List<Component>components, final List<ServiceComponent> services, final List<Finding> findings, final Project project) {
+        if (Variant.VDR == variant) {
+            components = components.stream()
+                    .filter(component -> !component.getVulnerabilities().isEmpty())
+                    .toList();
+        }
         final List<org.cyclonedx.model.Component> cycloneComponents = (Variant.VEX != variant && components != null) ? components.stream().map(component -> ModelConverter.convert(qm, component)).collect(Collectors.toList()) : null;
         final List<org.cyclonedx.model.Service> cycloneServices = (Variant.VEX != variant && services != null) ? services.stream().map(service -> ModelConverter.convert(qm, service)).collect(Collectors.toList()) : null;
         final List<org.cyclonedx.model.vulnerability.Vulnerability> cycloneVulnerabilities = (findings != null) ? findings.stream().map(finding -> ModelConverter.convert(qm, variant, finding)).collect(Collectors.toList()) : null;

src/main/java/org/dependencytrack/parser/cyclonedx/util/ModelConverter.java

@@ -58,6 +58,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.UUID;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -681,19 +682,19 @@ public static List<Dependency> generateDependencies(final Project project, final
 
         final var dependencies = new ArrayList<Dependency>();
         final var rootDependency = new Dependency(project.getUuid().toString());
-        rootDependency.setDependencies(convertDirectDependencies(project.getDirectDependencies()));
+        rootDependency.setDependencies(convertDirectDependencies(project.getDirectDependencies(), components));
         dependencies.add(rootDependency);
 
         for (final Component component : components) {
             final var dependency = new Dependency(component.getUuid().toString());
-            dependency.setDependencies(convertDirectDependencies(component.getDirectDependencies()));
+            dependency.setDependencies(convertDirectDependencies(component.getDirectDependencies(), components));
             dependencies.add(dependency);
         }
 
         return dependencies;
     }
 
-    private static List<Dependency> convertDirectDependencies(final String directDependenciesRaw) {
+    private static List<Dependency> convertDirectDependencies(final String directDependenciesRaw, final List<Component> components) {
         if (directDependenciesRaw == null || directDependenciesRaw.isBlank()) {
             return Collections.emptyList();
         }
@@ -705,7 +706,10 @@ private static List<Dependency> convertDirectDependencies(final String directDep
         if (directDependenciesJson instanceof final JsonArray directDependenciesJsonArray) {
             for (final JsonValue directDependency : directDependenciesJsonArray) {
                 if (directDependency instanceof final JsonObject directDependencyObject) {
-                    dependencies.add(new Dependency(directDependencyObject.getString("uuid")));
+                    final String componentUuid = directDependencyObject.getString("uuid", null);
+                    if (componentUuid != null && components.stream().map(Component::getUuid).map(UUID::toString).anyMatch(componentUuid::equals)) {
+                        dependencies.add(new Dependency(directDependencyObject.getString("uuid")));
+                    }
                 }
             }
         }

src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java

@@ -137,19 +137,51 @@ public List<Component> getAllComponents(Project project) {
 
     /**
      * Returns a List of Dependency for the specified Project.
-     *
      * @param project the Project to retrieve dependencies of
+     * @param includeMetrics Optionally includes third-party metadata about the component from external repositories
      * @return a List of Dependency objects
      */
     public PaginatedResult getComponents(final Project project, final boolean includeMetrics) {
+        return getComponents(project, includeMetrics, false, false);
+    }
+
+    /**
+     * Returns a List of Dependency for the specified Project.
+     * @param project the Project to retrieve dependencies of
+     * @param includeMetrics Optionally includes third-party metadata about the component from external repositories
+     * @param onlyOutdated Optionally exclude recent components so only outdated components are shown
+     * @param onlyDirect Optionally exclude transitive dependencies so only direct dependencies are shown
+     * @return a List of Dependency objects
+     */
+    public PaginatedResult getComponents(final Project project, final boolean includeMetrics, final boolean onlyOutdated, final boolean onlyDirect) {
         final PaginatedResult result;
-        final Query<Component> query = pm.newQuery(Component.class, "project == :project");
+        String querySring ="SELECT FROM org.dependencytrack.model.Component WHERE project == :project ";
+        if (filter != null) {
+            querySring += " && (project == :project) && name.toLowerCase().matches(:name)";
+        }
+        if (onlyOutdated) {
+            // Components are considered outdated when metadata does exists, but the version is different than latestVersion
+            // Different should always mean version < latestVersion
+            // Hack JDO using % instead of .* to get the SQL LIKE clause working:
+            querySring +=
+                    " && !("+
+                            " SELECT FROM org.dependencytrack.model.RepositoryMetaComponent m " +
+                            " WHERE m.name == this.name " +
+                            " && m.namespace == this.group " +
+                            " && m.latestVersion != this.version " +
+                            " && this.purl.matches('pkg:' + m.repositoryType.toString().toLowerCase() + '/%') " +
+                            " ).isEmpty()";
+        }
+        if (onlyDirect) {
+            querySring +=
+                    " && this.project.directDependencies.matches('%\"uuid\":\"'+this.uuid+'\"%') "; // only direct dependencies
+        }
+        final Query<Component> query = pm.newQuery(querySring);
         query.getFetchPlan().setMaxFetchDepth(2);
         if (orderBy == null) {
             query.setOrdering("name asc, version desc");
         }
         if (filter != null) {
-            query.setFilter("project == :project && name.toLowerCase().matches(:name)");
             final String filterString = ".*" + filter.toLowerCase() + ".*";
             result = execute(query, project, filterString);
         } else {

src/main/java/org/dependencytrack/persistence/QueryManager.java

@@ -998,6 +998,10 @@ public PaginatedResult getComponents(final Project project, final boolean includ
         return getComponentQueryManager().getComponents(project, includeMetrics);
     }
 
+    public PaginatedResult getComponents(final Project project, final boolean includeMetrics, final boolean onlyOutdated, final boolean onlyDirect) {
+        return getComponentQueryManager().getComponents(project, includeMetrics, onlyOutdated, onlyDirect);
+    }
+
     public ServiceComponent matchServiceIdentity(final Project project, final ComponentIdentity cid) {
         return getServiceComponentQueryManager().matchServiceIdentity(project, cid);
     }

src/main/java/org/dependencytrack/resources/v1/ComponentResource.java

@@ -96,12 +96,18 @@ public class ComponentResource extends AlpineResource {
             @ApiResponse(code = 404, message = "The project could not be found")
     })
     @PermissionRequired(Permissions.Constants.VIEW_PORTFOLIO)
-    public Response getAllComponents(@PathParam("uuid") String uuid) {
+    public Response getAllComponents(
+            @ApiParam(value = "The UUID of the project to retrieve components for", required = true)
+            @PathParam("uuid") String uuid,
+            @ApiParam(value = "Optionally exclude recent components so only outdated components are returned", required = false)
+            @QueryParam("onlyOutdated") boolean onlyOutdated,
+            @ApiParam(value = "Optionally exclude transitive dependencies so only direct dependencies are returned", required = false)
+            @QueryParam("onlyDirect") boolean onlyDirect) {
         try (QueryManager qm = new QueryManager(getAlpineRequest())) {
             final Project project = qm.getObjectByUuid(Project.class, uuid);
             if (project != null) {
                 if (qm.hasAccess(super.getPrincipal(), project)) {
-                    final PaginatedResult result = qm.getComponents(project, true);
+                    final PaginatedResult result = qm.getComponents(project, true, onlyOutdated, onlyDirect);
                     return Response.ok(result.getObjects()).header(TOTAL_COUNT_HEADER, result.getTotal()).build();
                 } else {
                     return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build();

src/test/java/org/dependencytrack/policy/PolicyEngineTest.java

@@ -18,6 +18,10 @@
  */
 package org.dependencytrack.policy;
 
+import alpine.notification.Notification;
+import alpine.notification.NotificationService;
+import alpine.notification.Subscriber;
+import alpine.notification.Subscription;
 import org.dependencytrack.PersistenceCapableTest;
 import org.dependencytrack.event.kafka.KafkaTopics;
 import org.dependencytrack.model.AnalyzerIdentity;
@@ -33,14 +37,19 @@
 import org.dependencytrack.model.Vulnerability;
 import org.dependencytrack.notification.NotificationConstants;
 import org.dependencytrack.util.NotificationUtil;
+import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.dependencytrack.assertion.Assertions.assertConditionWithTimeout;
@@ -52,6 +61,37 @@
 
 public class PolicyEngineTest extends PersistenceCapableTest {
 
+    public static class NotificationSubscriber implements Subscriber {
+
+        @Override
+        public void inform(final Notification notification) {
+            NOTIFICATIONS.add(notification);
+        }
+
+    }
+
+    private static final ConcurrentLinkedQueue<Notification> NOTIFICATIONS = new ConcurrentLinkedQueue<>();
+
+    @BeforeClass
+    public static void setUpClass() {
+        NotificationService.getInstance().subscribe(new Subscription(NotificationSubscriber.class));
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+        NotificationService.getInstance().unsubscribe(new Subscription(NotificationSubscriber.class));
+    }
+
+    @Before
+    public void setup() {
+        NOTIFICATIONS.clear();
+    }
+
+    @After
+    public void tearDown() {
+        NOTIFICATIONS.clear();
+    }
+
     @Test
     public void hasTagMatchPolicyLimitedToTag() {
         Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO);

src/test/java/org/dependencytrack/resources/v1/BomResourceTest.java

@@ -540,12 +540,6 @@ public void exportProjectAsCycloneDxVdrTest() {
                         ]
                     },
                     "components": [
-                        {
-                            "type": "library",
-                            "bom-ref": "${json-unit.matches:componentWithoutVulnUuid}",
-                            "name": "acme-lib-a",
-                            "version": "1.0.0"
-                        },
                         {
                             "type": "library",
                             "bom-ref": "${json-unit.matches:componentWithVulnUuid}",
@@ -563,16 +557,9 @@ public void exportProjectAsCycloneDxVdrTest() {
                         {
                             "ref": "${json-unit.matches:projectUuid}",
                             "dependsOn": [
-                                "${json-unit.matches:componentWithoutVulnUuid}",
                                 "${json-unit.matches:componentWithVulnAndAnalysisUuid}"
                             ]
                         },
-                        {
-                            "ref": "${json-unit.matches:componentWithoutVulnUuid}",
-                            "dependsOn": [
-                                "${json-unit.matches:componentWithVulnUuid}"
-                            ]
-                        },
                         {
                             "ref": "${json-unit.matches:componentWithVulnUuid}",
                             "dependsOn": []

src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java

@@ -21,6 +21,8 @@
 import alpine.common.util.UuidUtil;
 import alpine.server.filters.ApiFilter;
 import alpine.server.filters.AuthenticationFilter;
+import com.github.packageurl.MalformedPackageURLException;
+import com.github.packageurl.PackageURL;
 import org.apache.http.HttpStatus;
 import org.dependencytrack.ResourceTest;
 import org.dependencytrack.event.kafka.KafkaTopics;
@@ -43,7 +45,9 @@
 import javax.ws.rs.client.Entity;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
 import java.util.UUID;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -69,28 +73,123 @@ public void getComponentsDefaultRequestTest() {
         Assert.assertEquals(405, response.getStatus()); // No longer prohibited in DT 4.0+
     }
 
-    @Test
-    public void getAllComponentsTest() {
+    /**
+     * Generate a project with different dependencies
+     * @return A project with 1000 dpendencies: <ul>
+     * <li>200 outdated dependencies, 75 direct and 125 transitive</li>
+     * <li>800 recent dependencies, 25 direct, 775 transitive</li>
+     * @throws MalformedPackageURLException
+     */
+    private Project prepareProject() throws MalformedPackageURLException {
         final Project project = qm.createProject("Acme Application", null, null, null, null, null, true, false);
+        final List<String> directDepencencies = new ArrayList<>();
+        // Generate 1000 dependencies
         for (int i = 0; i < 1000; i++) {
             Component component = new Component();
             component.setProject(project);
-            component.setName("Component Name");
-            component.setVersion(String.valueOf(i));
-            qm.createComponent(component, false);
+            component.setGroup("component-group");
+            component.setName("component-name-"+i);
+            component.setVersion(String.valueOf(i)+".0");
+            component.setPurl(new PackageURL(RepositoryType.MAVEN.toString(), "component-group", "component-name-"+i , String.valueOf(i)+".0", null, null));
+            component = qm.createComponent(component, false);
+            // direct depencencies
+            if (i < 100) {
+                // 100 direct depencencies, 900 transitive depencencies
+                directDepencencies.add("{\"uuid\":\"" + component.getUuid() + "\"}");
+            }
+            // Recent & Outdated
+            if ((i >= 25) && (i < 225)) {
+                // 100 outdated components, 75 of these are direct dependencies, 25 transitive
+                final var metaComponent = new RepositoryMetaComponent();
+                metaComponent.setRepositoryType(RepositoryType.MAVEN);
+                metaComponent.setNamespace("component-group");
+                metaComponent.setName("component-name-"+i);
+                metaComponent.setLatestVersion(String.valueOf(i+1)+".0");
+                metaComponent.setLastCheck(new Date());
+                qm.persist(metaComponent);
+            } else if (i<500) {
+                // 300 recent components, 25 of these are direct dependencies
+                final var metaComponent = new RepositoryMetaComponent();
+                metaComponent.setRepositoryType(RepositoryType.MAVEN);
+                metaComponent.setNamespace("component-group");
+                metaComponent.setName("component-name-"+i);
+                metaComponent.setLatestVersion(String.valueOf(i)+".0");
+                metaComponent.setLastCheck(new Date());
+                qm.persist(metaComponent);
+            } else {
+                // 500 components with no RepositoryMetaComponent containing version
+                // metadata, all transitive dependencies
+            }
         }
+        project.setDirectDependencies("[" + String.join(",", directDepencencies.toArray(new String[0])) + "]");
+        return project;
+    }
+
+    @Test
+    public void getOutdatedComponentsTest() throws MalformedPackageURLException {
+        final Project project = prepareProject();
+
+        final Response response = target(V1_COMPONENT + "/project/" + project.getUuid())
+                .queryParam("onlyOutdated", true)
+                .queryParam("onlyDirect", false)
+                .request()
+                .header(X_API_KEY, apiKey)
+                .get(Response.class);
+        assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_OK);
+        assertThat(response.getHeaderString(TOTAL_COUNT_HEADER)).isEqualTo("200"); // 200 outdated dependencies,  direct and transitive
+
+        final JsonArray json = parseJsonArray(response);
+        assertThat(json).hasSize(100); // Default page size is 100
+    }
+
+    @Test
+    public void getOutdatedDirectComponentsTest() throws MalformedPackageURLException {
+        final Project project = prepareProject();
+
+        final Response response = target(V1_COMPONENT + "/project/" + project.getUuid())
+                .queryParam("onlyOutdated", true)
+                .queryParam("onlyDirect", true)
+                .request()
+                .header(X_API_KEY, apiKey)
+                .get(Response.class);
+        assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_OK);
+        assertThat(response.getHeaderString(TOTAL_COUNT_HEADER)).isEqualTo("75"); // 75 outdated direct dependencies
+
+        final JsonArray json = parseJsonArray(response);
+        assertThat(json).hasSize(75);
+    }
+
+    @Test
+    public void getAllComponentsTest() throws MalformedPackageURLException {
+        final Project project = prepareProject();
 
         final Response response = target(V1_COMPONENT + "/project/" + project.getUuid())
                 .request()
                 .header(X_API_KEY, apiKey)
                 .get(Response.class);
         assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_OK);
-        assertThat(response.getHeaderString(TOTAL_COUNT_HEADER)).isEqualTo("1000");
+        assertThat(response.getHeaderString(TOTAL_COUNT_HEADER)).isEqualTo("1000"); // 1000 dependencies
 
         final JsonArray json = parseJsonArray(response);
         assertThat(json).hasSize(100); // Default page size is 100
     }
 
+    @Test
+    public void getAllDirectComponentsTest() throws MalformedPackageURLException {
+        final Project project = prepareProject();
+
+        final Response response = target(V1_COMPONENT + "/project/" + project.getUuid())
+                .queryParam("onlyDirect", true)
+                .request()
+                .header(X_API_KEY, apiKey)
+                .get(Response.class);
+        assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_OK);
+        assertThat(response.getHeaderString(TOTAL_COUNT_HEADER)).isEqualTo("100"); // 100 direct dependencies
+
+        final JsonArray json = parseJsonArray(response);
+        assertThat(json).hasSize(100);
+    }
+
     @Test
     public void getComponentByUuidTest() {
         Project project = qm.createProject("Acme Application", null, null, null, null, null, true, false);