Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

changes for end points with integrity meta #377

Merged
merged 7 commits into from
Oct 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/main/java/org/dependencytrack/model/Component.java
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,8 @@ public enum FetchGroup {
private transient String licenseId;
private transient DependencyMetrics metrics;
private transient RepositoryMetaComponent repositoryMeta;

private transient ComponentMetaInformation componentMetaInformation;
private transient boolean isNew;
private transient int usedBy;
private transient Set<String> dependencyGraph;
Expand Down Expand Up @@ -741,6 +743,14 @@ public void setRepositoryMeta(RepositoryMetaComponent repositoryMeta) {
this.repositoryMeta = repositoryMeta;
}

public ComponentMetaInformation getComponentMetaInformation() {
return componentMetaInformation;
}

public void setComponentMetaInformation(ComponentMetaInformation componentMetaInformation) {
this.componentMetaInformation = componentMetaInformation;
}

public boolean isNew() {
return isNew;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package org.dependencytrack.model;

import java.util.Date;

public record ComponentMetaInformation(Date publishedDate, IntegrityMatchStatus integrityMatchStatus,
Date lastFetched) {
}
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,8 @@ public List<Component> getAllComponents(Project project) {

/**
* Returns a List of Dependency for the specified Project.
* @param project the Project to retrieve dependencies of
*
* @param project the Project to retrieve dependencies of
* @param includeMetrics Optionally includes third-party metadata about the component from external repositories
* @return a List of Dependency objects
*/
Expand All @@ -147,15 +148,16 @@ public PaginatedResult getComponents(final Project project, final boolean includ

/**
* Returns a List of Dependency for the specified Project.
* @param project the Project to retrieve dependencies of
*
* @param project the Project to retrieve dependencies of
* @param includeMetrics Optionally includes third-party metadata about the component from external repositories
* @param onlyOutdated Optionally exclude recent components so only outdated components are shown
* @param onlyDirect Optionally exclude transitive dependencies so only direct dependencies are shown
* @param onlyOutdated Optionally exclude recent components so only outdated components are shown
* @param onlyDirect Optionally exclude transitive dependencies so only direct dependencies are shown
* @return a List of Dependency objects
*/
public PaginatedResult getComponents(final Project project, final boolean includeMetrics, final boolean onlyOutdated, final boolean onlyDirect) {
final PaginatedResult result;
String querySring ="SELECT FROM org.dependencytrack.model.Component WHERE project == :project ";
String querySring = "SELECT FROM org.dependencytrack.model.Component WHERE project == :project ";
if (filter != null) {
querySring += " && (project == :project) && name.toLowerCase().matches(:name)";
}
Expand All @@ -164,7 +166,7 @@ public PaginatedResult getComponents(final Project project, final boolean includ
// Different should always mean version < latestVersion
// Hack JDO using % instead of .* to get the SQL LIKE clause working:
querySring +=
" && !("+
" && !(" +
" SELECT FROM org.dependencytrack.model.RepositoryMetaComponent m " +
" WHERE m.name == this.name " +
" && m.namespace == this.group " +
Expand Down Expand Up @@ -198,6 +200,7 @@ public PaginatedResult getComponents(final Project project, final boolean includ
if (RepositoryType.UNSUPPORTED != type) {
final RepositoryMetaComponent repoMetaComponent = getRepositoryMetaComponent(type, purl.getNamespace(), purl.getName());
component.setRepositoryMeta(repoMetaComponent);
component.setComponentMetaInformation(getMetaInformation(purl, component.getUuid()));
}
}
}
Expand Down Expand Up @@ -329,6 +332,7 @@ public PaginatedResult getComponents(ComponentIdentity identity, Project project
if (RepositoryType.UNSUPPORTED != type) {
final RepositoryMetaComponent repoMetaComponent = getRepositoryMetaComponent(type, purl.getNamespace(), purl.getName());
component.setRepositoryMeta(repoMetaComponent);
component.setComponentMetaInformation(getMetaInformation(purl, component.getUuid()));
}
}
}
Expand Down Expand Up @@ -445,7 +449,7 @@ protected void deleteComponents(Project project) {
final Query<Component> query = pm.newQuery(Component.class, "project == :project");
query.setParameters(project);
List<Component> components = query.executeList();
for(Component component : components) {
for (Component component : components) {
executeAndClose(pm.newQuery(Query.JDOQL, "DELETE FROM org.dependencytrack.model.IntegrityAnalysis WHERE component == :component"), component);
}
try {
Expand Down Expand Up @@ -742,6 +746,7 @@ public Map<String, Component> getDependencyGraphForComponent(Project project, Co

/**
* Returns a list of all {@link DependencyGraphResponse} objects by {@link Component} UUID.
*
* @param uuids a list of {@link Component} UUIDs
* @return a list of {@link DependencyGraphResponse} objects
* @since 4.9.0
Expand Down
28 changes: 25 additions & 3 deletions src/main/java/org/dependencytrack/persistence/QueryManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,15 @@
import org.dependencytrack.model.Classifier;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.ComponentIdentity;
import org.dependencytrack.model.ComponentMetaInformation;
import org.dependencytrack.model.ConfigPropertyConstants;
import org.dependencytrack.model.Cpe;
import org.dependencytrack.model.Cwe;
import org.dependencytrack.model.DependencyMetrics;
import org.dependencytrack.model.Finding;
import org.dependencytrack.model.FindingAttribution;
import org.dependencytrack.model.IntegrityAnalysis;
import org.dependencytrack.model.IntegrityMatchStatus;
import org.dependencytrack.model.IntegrityMetaComponent;
import org.dependencytrack.model.License;
import org.dependencytrack.model.LicenseGroup;
Expand Down Expand Up @@ -1399,8 +1401,8 @@ public <T> T detachWithGroups(final T object, final List<String> fetchGroups) {
*
* @param clazz Class of the object to fetch
* @param uuids {@link UUID} list of uuids to fetch
* @param <T> Type of the object
* @return The list of objects found
* @param <T> Type of the object
* @since 4.9.0
*/
public <T> List<T> getObjectsByUuids(final Class<T> clazz, final List<UUID> uuids) {
Expand All @@ -1413,8 +1415,8 @@ public <T> List<T> getObjectsByUuids(final Class<T> clazz, final List<UUID> uuid
*
* @param clazz Class of the object to fetch
* @param uuids {@link UUID} list of uuids to fetch
* @param <T> Type of the object
* @return The query to execute
* @param <T> Type of the object
* @since 4.9.0
*/
public <T> Query<T> getObjectsByUuidsQuery(final Class<T> clazz, final List<UUID> uuids) {
Expand Down Expand Up @@ -1519,6 +1521,7 @@ public void recursivelyDeleteTeam(Team team) {

/**
* Returns a list of all {@link DependencyGraphResponse} objects by {@link Component} UUID.
*
* @param uuids a list of {@link Component} UUIDs
* @return a list of {@link DependencyGraphResponse} objects
* @since 4.9.0
Expand All @@ -1529,6 +1532,7 @@ public List<DependencyGraphResponse> getComponentDependencyGraphByUuids(final Li

/**
* Returns a list of all {@link DependencyGraphResponse} objects by {@link ServiceComponent} UUID.
*
* @param uuids a list of {@link ServiceComponent} UUIDs
* @return a list of {@link DependencyGraphResponse} objects
* @since 4.9.0
Expand All @@ -1539,6 +1543,7 @@ public List<DependencyGraphResponse> getServiceDependencyGraphByUuids(final List

/**
* Returns a list of all {@link RepositoryMetaComponent} objects by {@link RepositoryQueryManager.RepositoryMetaComponentSearch} with batchSize 10.
*
* @param list a list of {@link RepositoryQueryManager.RepositoryMetaComponentSearch}
* @return a list of {@link RepositoryMetaComponent} objects
* @since 4.9.0
Expand All @@ -1549,7 +1554,8 @@ public List<RepositoryMetaComponent> getRepositoryMetaComponentsBatch(final List

/**
* Returns a list of all {@link RepositoryMetaComponent} objects by {@link RepositoryQueryManager.RepositoryMetaComponentSearch} UUID.
* @param list a list of {@link RepositoryQueryManager.RepositoryMetaComponentSearch}
*
* @param list a list of {@link RepositoryQueryManager.RepositoryMetaComponentSearch}
* @param batchSize the batch size
* @return a list of {@link RepositoryMetaComponent} objects
* @since 4.9.0
Expand Down Expand Up @@ -1832,4 +1838,20 @@ public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponen
public IntegrityAnalysis getIntegrityAnalysisByComponentUuid(UUID uuid) {
return getIntegrityAnalysisQueryManager().getIntegrityAnalysisByComponentUuid(uuid);
}

public ComponentMetaInformation getMetaInformation(PackageURL purl, UUID uuid) {
Date publishedAt = null;
Date lastFetched = null;
IntegrityMatchStatus integrityMatchStatus = null;
final IntegrityMetaComponent integrityMetaComponent = getIntegrityMetaComponent(purl.toString());
final IntegrityAnalysis integrityAnalysis = getIntegrityAnalysisByComponentUuid(uuid);
if (integrityMetaComponent != null) {
publishedAt = integrityMetaComponent.getPublishedAt();
lastFetched = integrityMetaComponent.getLastFetch();
}
if (integrityAnalysis != null) {
integrityMatchStatus = integrityAnalysis.getIntegrityCheckStatus();
}
return new ComponentMetaInformation(publishedAt, integrityMatchStatus, lastFetched);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
import org.dependencytrack.event.kafka.componentmeta.HandlerFactory;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.ComponentIdentity;
import org.dependencytrack.model.IntegrityAnalysis;
import org.dependencytrack.model.IntegrityMetaComponent;
import org.dependencytrack.model.License;
import org.dependencytrack.model.Project;
import org.dependencytrack.model.RepositoryMetaComponent;
Expand Down Expand Up @@ -135,18 +137,24 @@ public Response getComponentByUuid(
@ApiParam(value = "The UUID of the component to retrieve", required = true)
@PathParam("uuid") String uuid,
@ApiParam(value = "Optionally includes third-party metadata about the component from external repositories", required = false)
@QueryParam("includeRepositoryMetaData") boolean includeRepositoryMetaData) {
@QueryParam("includeRepositoryMetaData") boolean includeRepositoryMetaData,
@QueryParam("includeIntegrityMetaData") boolean includeIntegrityMetaData) {
try (QueryManager qm = new QueryManager()) {
final Component component = qm.getObjectByUuid(Component.class, uuid);
if (component != null) {
final Project project = component.getProject();
if (qm.hasAccess(super.getPrincipal(), project)) {
final Component detachedComponent = qm.detach(Component.class, component.getId()); // TODO: Force project to be loaded. It should be anyway, but JDO seems to be having issues here.
if (includeRepositoryMetaData && detachedComponent.getPurl() != null) {
if ((includeRepositoryMetaData || includeIntegrityMetaData) && detachedComponent.getPurl() != null) {
final RepositoryType type = RepositoryType.resolve(detachedComponent.getPurl());
if (RepositoryType.UNSUPPORTED != type) {
final RepositoryMetaComponent repoMetaComponent = qm.getRepositoryMetaComponent(type, detachedComponent.getPurl().getNamespace(), detachedComponent.getPurl().getName());
detachedComponent.setRepositoryMeta(repoMetaComponent);
if (includeRepositoryMetaData) {
final RepositoryMetaComponent repoMetaComponent = qm.getRepositoryMetaComponent(type, detachedComponent.getPurl().getNamespace(), detachedComponent.getPurl().getName());
detachedComponent.setRepositoryMeta(repoMetaComponent);
}
if (includeIntegrityMetaData) {
detachedComponent.setComponentMetaInformation(qm.getMetaInformation(component.getPurl(), component.getUuid()));
}
}
}
return Response.ok(detachedComponent).build();
Expand All @@ -159,6 +167,68 @@ public Response getComponentByUuid(
}
}

@GET
@Path("/integritymetadata")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(
value = "Provides the published date and hashes of the requested version of component " +
"as received from configured repositories for integrity analysis",
response = IntegrityMetaComponent.class
)
@ApiResponses(value = {
@ApiResponse(code = 401, message = "Unauthorized"),
@ApiResponse(code = 404, message = "The integrity meta information for the specified component cannot be found"),
@ApiResponse(code = 400, message = "The package url being queried for is invalid")
})
public Response getIntegrityMetaComponent(
@ApiParam(value = "The package url of the component", required = true)
@QueryParam("purl") String purl) {
try {
final PackageURL packageURL = new PackageURL(purl);
try (QueryManager qm = new QueryManager(getAlpineRequest())) {
final RepositoryType type = RepositoryType.resolve(packageURL);
if (RepositoryType.UNSUPPORTED == type) {
return Response.noContent().build();
}
final IntegrityMetaComponent result = qm.getIntegrityMetaComponent(packageURL.toString());
if (result == null) {
return Response.status(Response.Status.NOT_FOUND).entity("The integrity metadata for the specified component cannot be found.").build();
} else {
//todo: future enhancement: provide pass-thru capability for component metadata not already present and being tracked
return Response.ok(result).build();
}
}
} catch (MalformedPackageURLException e) {
return Response.status(Response.Status.BAD_REQUEST).build();
}
}

@GET
@Path("/{uuid}/integritycheckstatus")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(
value = "Provides the integrity check status of component with provided uuid based on the configured " +
"repository for integrity analysis",
response = IntegrityAnalysis.class
)
@ApiResponses(value = {
@ApiResponse(code = 401, message = "Unauthorized"),
@ApiResponse(code = 404, message = "The integrity analysis information for the specified component cannot be found"),
})
public Response getIntegrityStatus(
@ApiParam(value = "UUID of the component for which integrity status information is needed", required = true)
@PathParam("uuid") String uuid) {
try (QueryManager qm = new QueryManager(getAlpineRequest())) {
final IntegrityAnalysis result = qm.getIntegrityAnalysisByComponentUuid(UUID.fromString(uuid));
if (result == null) {
return Response.status(Response.Status.NOT_FOUND).entity("The integrity status for the specified component cannot be found.").build();
} else {
//todo: future enhancement: provide pass-thru capability for component metadata not already present and being tracked
return Response.ok(result).build();
}
}
}

@GET
@Path("/identity")
@Produces(MediaType.APPLICATION_JSON)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@
*/
public class ComponentIdentificationUtil {

private ComponentIdentificationUtil() { }
private ComponentIdentificationUtil() {
}

@SuppressWarnings("deprecation")
public static boolean doesIdentityMatch(final Component a, final org.cyclonedx.model.Component b) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package org.dependencytrack.persistence;

import org.dependencytrack.PersistenceCapableTest;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.ComponentMetaInformation;
import org.dependencytrack.model.FetchStatus;
import org.dependencytrack.model.IntegrityAnalysis;
import org.dependencytrack.model.IntegrityMatchStatus;
import org.dependencytrack.model.IntegrityMetaComponent;
import org.dependencytrack.model.Project;
import org.junit.Assert;
import org.junit.Test;

import java.util.Date;

import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_PASSED;
import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_UNKNOWN;

public class QueryManagerTest extends PersistenceCapableTest {
@Test
public void testGetMetaInformation() {
Project project = qm.createProject("Acme Application", null, null, null, null, null, true, false);
Component component = new Component();
component.setProject(project);
component.setName("ABC");
component.setPurl("pkg:maven/org.acme/abc");
IntegrityAnalysis integrityAnalysis = new IntegrityAnalysis();
integrityAnalysis.setComponent(component);
integrityAnalysis.setIntegrityCheckStatus(IntegrityMatchStatus.HASH_MATCH_PASSED);
Date published = new Date();
integrityAnalysis.setUpdatedAt(published);
integrityAnalysis.setId(component.getId());
integrityAnalysis.setMd5HashMatchStatus(IntegrityMatchStatus.HASH_MATCH_PASSED);
integrityAnalysis.setSha1HashMatchStatus(HASH_MATCH_UNKNOWN);
integrityAnalysis.setSha256HashMatchStatus(HASH_MATCH_UNKNOWN);
integrityAnalysis.setSha512HashMatchStatus(HASH_MATCH_PASSED);
qm.persist(integrityAnalysis);
IntegrityMetaComponent integrityMetaComponent = new IntegrityMetaComponent();
integrityMetaComponent.setPurl(component.getPurl().toString());
integrityMetaComponent.setPublishedAt(published);
integrityMetaComponent.setLastFetch(published);
integrityMetaComponent.setStatus(FetchStatus.PROCESSED);
qm.createIntegrityMetaComponent(integrityMetaComponent);
component = qm.createComponent(component, false);
ComponentMetaInformation componentMetaInformation = qm.getMetaInformation(component.getPurl(), component.getUuid());
Assert.assertEquals(HASH_MATCH_PASSED, componentMetaInformation.integrityMatchStatus());
Assert.assertEquals(integrityMetaComponent.getPublishedAt(), componentMetaInformation.publishedDate());
Assert.assertEquals(integrityMetaComponent.getLastFetch(), componentMetaInformation.lastFetched());
}
}
Loading