diff --git a/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java b/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java index 6d74e38d7..335ec7ad0 100644 --- a/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java +++ b/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java @@ -1,21 +1,18 @@ package org.dependencytrack.event; import alpine.event.framework.Event; -import com.github.packageurl.PackageURL; import org.dependencytrack.model.Component; -import java.util.Optional; - /** * Defines an {@link Event} triggered when requesting a component to be analyzed for meta information. * - * @param purlCoordinates The package URL coordinates of the {@link Component} to analyze - * @param internal Whether the {@link Component} is internal + * @param purlCoordinates The package URL coordinates of the {@link Component} to analyze + * @param internal Whether the {@link Component} is internal + * @param fetchIntegrityData Whether component hash information needs to be fetched from external api + * @param fetchLatestVersion Whether to fetch latest version meta information for a component. */ -public record ComponentRepositoryMetaAnalysisEvent(String purlCoordinates, Boolean internal) implements Event { - - public ComponentRepositoryMetaAnalysisEvent(final Component component) { - this(Optional.ofNullable(component.getPurlCoordinates()).map(PackageURL::canonicalize).orElse(null), component.isInternal()); - } +public record ComponentRepositoryMetaAnalysisEvent(String purlCoordinates, Boolean internal, + boolean fetchIntegrityData, + boolean fetchLatestVersion) implements Event { } diff --git a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java index 8d0bd8c55..6b3d6c494 100644 --- a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java +++ b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java @@ -67,7 +67,7 @@ private void batchProcessPurls(QueryManager qm) { List purls = qm.fetchNextPurlsPage(offset); while (!purls.isEmpty()) { long cumulativeProcessingTime = System.currentTimeMillis() - startTime; - if(isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) { + if (isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) { LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); } dispatchPurls(qm, purls); @@ -88,7 +88,7 @@ private void updateIntegrityMetaForPurls(QueryManager qm, List purls) { private void dispatchPurls(QueryManager qm, List purls) { for (final var purl : purls) { ComponentProjection componentProjection = qm.getComponentByPurl(purl); - kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal)); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal, true, false)); } } diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java index ac11b0a45..a938d2b13 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java @@ -58,6 +58,8 @@ static KafkaEvent convert(final ComponentRepositoryMeta final var analysisCommand = AnalysisCommand.newBuilder() .setComponent(componentBuilder) + .setFetchIntegrityData(event.fetchIntegrityData()) + .setFetchLatestVersion(event.fetchLatestVersion()) .build(); return new KafkaEvent<>(KafkaTopics.REPO_META_ANALYSIS_COMMAND, event.purlCoordinates(), analysisCommand, null); diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java new file mode 100644 index 000000000..ec7dfb043 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java @@ -0,0 +1,27 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.persistence.QueryManager; + +import java.time.Instant; +import java.util.Date; + +public abstract class AbstractMetaHandler implements Handler { + + ComponentProjection componentProjection; + QueryManager queryManager; + KafkaEventDispatcher kafkaEventDispatcher; + boolean fetchLatestVersion; + + + public static IntegrityMetaComponent createIntegrityMetaComponent(String purl) { + IntegrityMetaComponent integrityMetaComponent1 = new IntegrityMetaComponent(); + integrityMetaComponent1.setStatus(FetchStatus.IN_PROGRESS); + integrityMetaComponent1.setPurl(purl); + integrityMetaComponent1.setLastFetch(Date.from(Instant.now())); + return integrityMetaComponent1; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java new file mode 100644 index 000000000..53f057cac --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java @@ -0,0 +1,4 @@ +package org.dependencytrack.event.kafka.componentmeta; + +public record ComponentProjection(String purlCoordinates, Boolean internal, String purl) { +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java new file mode 100644 index 000000000..6e77f89c8 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java @@ -0,0 +1,7 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import org.dependencytrack.model.IntegrityMetaComponent; + +public interface Handler { + IntegrityMetaComponent handle(); +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java new file mode 100644 index 000000000..1804a4a48 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java @@ -0,0 +1,19 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.persistence.QueryManager; + +public class HandlerFactory { + + public static Handler createHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, boolean fetchLatestVersion) throws MalformedPackageURLException { + PackageURL packageURL = new PackageURL(componentProjection.purl()); + boolean result = RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(packageURL.getType()); + if (result) { + return new SupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchLatestVersion); + } else { + return new UnSupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchLatestVersion); + } + } +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java new file mode 100644 index 000000000..c2054954b --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java @@ -0,0 +1,9 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import java.util.List; + +public class RepoMetaConstants { + + public static final long TIME_SPAN = 60 * 60 * 1000L; + public static final List SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK =List.of("maven", "npm", "pypi"); +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java new file mode 100644 index 000000000..494e68b99 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java @@ -0,0 +1,46 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.persistence.QueryManager; + +import java.time.Instant; +import java.util.Date; + +import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN; + +public class SupportedMetaHandler extends AbstractMetaHandler { + + public SupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, boolean fetchLatestVersion) { + this.componentProjection = componentProjection; + this.kafkaEventDispatcher = kafkaEventDispatcher; + this.queryManager = queryManager; + this.fetchLatestVersion = fetchLatestVersion; + } + + @Override + public IntegrityMetaComponent handle() { + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try (QueryManager queryManager = new QueryManager()) { + IntegrityMetaComponent integrityMetaComponent = queryManager.getIntegrityMetaComponent(componentProjection.purl()); + if (integrityMetaComponent != null) { + if (integrityMetaComponent.getStatus() == null || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime() > TIME_SPAN)) { + integrityMetaComponent.setLastFetch(Date.from(Instant.now())); + IntegrityMetaComponent integrityMetaComponent1 = queryManager.updateIntegrityMetaComponent(integrityMetaComponent); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion)); + return integrityMetaComponent1; + } else { + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion)); + return integrityMetaComponent; + } + } else { + IntegrityMetaComponent integrityMetaComponent1 = queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl())); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion)); + return integrityMetaComponent1; + } + } + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java new file mode 100644 index 000000000..c9babf3e9 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java @@ -0,0 +1,23 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.persistence.QueryManager; + +public class UnSupportedMetaHandler extends AbstractMetaHandler { + + public UnSupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher,boolean fetchLatestVersion) { + this.componentProjection = componentProjection; + this.kafkaEventDispatcher = kafkaEventDispatcher; + this.queryManager = queryManager; + this.fetchLatestVersion = fetchLatestVersion; + } + + @Override + public IntegrityMetaComponent handle() { + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion)); + return null; + } +} diff --git a/src/main/java/org/dependencytrack/model/FetchStatus.java b/src/main/java/org/dependencytrack/model/FetchStatus.java index 6824a8bff..3723c0c97 100644 --- a/src/main/java/org/dependencytrack/model/FetchStatus.java +++ b/src/main/java/org/dependencytrack/model/FetchStatus.java @@ -1,7 +1,12 @@ package org.dependencytrack.model; public enum FetchStatus { + //request processed successfully PROCESSED, - TIMED_OUT, - IN_PROGRESS + //fetching information for this component is in progress + IN_PROGRESS, + + //to be used when information is not available in source of truth so we don't go fetching this repo information again + //after first attempt + NOT_AVAILABLE } diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java index c1bb130f5..342ef2f3f 100644 --- a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java @@ -24,13 +24,14 @@ import alpine.model.UserPrincipal; import alpine.persistence.PaginatedResult; import alpine.resources.AlpineRequest; +import alpine.server.util.DbUtil; import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; -import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; import org.dependencytrack.event.IntegrityMetaInitializer; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; import org.dependencytrack.model.ConfigPropertyConstants; +import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.Project; import org.dependencytrack.model.RepositoryMetaComponent; import org.dependencytrack.model.RepositoryType; @@ -42,7 +43,11 @@ import javax.json.JsonArray; import javax.json.JsonValue; import java.io.StringReader; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.time.Instant; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -55,6 +60,7 @@ final class ComponentQueryManager extends QueryManager implements IQueryManager /** * Constructs a new QueryManager. + * * @param pm a PersistenceManager object */ ComponentQueryManager(final PersistenceManager pm) { @@ -63,7 +69,8 @@ final class ComponentQueryManager extends QueryManager implements IQueryManager /** * Constructs a new QueryManager. - * @param pm a PersistenceManager object + * + * @param pm a PersistenceManager object * @param request an AlpineRequest object */ ComponentQueryManager(final PersistenceManager pm, final AlpineRequest request) { @@ -72,6 +79,7 @@ final class ComponentQueryManager extends QueryManager implements IQueryManager /** * Returns a list of all Components defined in the datastore. + * * @return a List of Components */ public PaginatedResult getComponents(final boolean includeMetrics) { @@ -99,6 +107,7 @@ public PaginatedResult getComponents(final boolean includeMetrics) { /** * Returns a list of all Components defined in the datastore. + * * @return a List of Components */ public PaginatedResult getComponents() { @@ -108,6 +117,7 @@ public PaginatedResult getComponents() { /** * Returns a list of all components. * This method if designed NOT to provide paginated results. + * * @return a List of Components */ public List getAllComponents() { @@ -119,6 +129,7 @@ public List getAllComponents() { /** * Returns a List of all Components for the specified Project. * This method if designed NOT to provide paginated results. + * * @param project the Project to retrieve dependencies of * @return a List of Component objects */ @@ -127,11 +138,12 @@ public List getAllComponents(Project project) { final Query query = pm.newQuery(Component.class, "project == :project"); query.getFetchPlan().setMaxFetchDepth(2); query.setOrdering("name asc"); - return (List)query.execute(project); + return (List) query.execute(project); } /** * Returns a List of Dependency for the specified Project. + * * @param project the Project to retrieve dependencies of * @return a List of Dependency objects */ @@ -169,6 +181,7 @@ public PaginatedResult getComponents(final Project project, final boolean includ /** * Returns Components by their hash. + * * @param hash the hash of the component to retrieve * @return a list of components */ @@ -186,7 +199,8 @@ public PaginatedResult getComponentByHash(String hash) { default -> "(blake3 == :hash)"; }; - final Query query = pm.newQuery(Component.class);; + final Query query = pm.newQuery(Component.class); + ; final Map params = Map.of("hash", hash); preprocessACLs(query, queryFilter, params, false); return execute(query, params); @@ -194,6 +208,7 @@ public PaginatedResult getComponentByHash(String hash) { /** * Returns ComponentProjection for the purl. + * * @param purl the purl of the component to retrieve * @return associated ComponentProjection */ @@ -209,6 +224,7 @@ public IntegrityMetaInitializer.ComponentProjection getComponentByPurl(String pu /** * Returns Components by their identity. + * * @param identity the ComponentIdentity to query against * @return a list of components */ @@ -222,8 +238,9 @@ public PaginatedResult getComponents(ComponentIdentity identity, boolean include /** * Returns Components by their identity. - * @param identity the ComponentIdentity to query against - * @param project The {@link Project} the {@link Component}s shall belong to + * + * @param identity the ComponentIdentity to query against + * @param project The {@link Project} the {@link Component}s shall belong to * @param includeMetrics whether or not to include component metrics or not * @return a list of components */ @@ -313,7 +330,8 @@ private PaginatedResult loadComponents(String queryFilter, Map p /** * Creates a new Component. - * @param component the Component to persist + * + * @param component the Component to persist * @param commitIndex specifies if the search index should be committed (an expensive operation) * @return a new Component */ @@ -360,8 +378,9 @@ public Component cloneComponent(Component sourceComponent, Project destinationPr /** * Updated an existing Component. + * * @param transientComponent the component to update - * @param commitIndex specifies if the search index should be committed (an expensive operation) + * @param commitIndex specifies if the search index should be committed (an expensive operation) * @return a Component */ public Component updateComponent(Component transientComponent, boolean commitIndex) { @@ -393,6 +412,7 @@ public Component updateComponent(Component transientComponent, boolean commitInd /** * Deletes all components for the specified Project. + * * @param project the Project to delete components of */ protected void deleteComponents(Project project) { @@ -406,7 +426,8 @@ protected void deleteComponents(Project project) { /** * Deletes a Component and all objects dependant on the component. - * @param component the Component to delete + * + * @param component the Component to delete * @param commitIndex specifies if the search index should be committed (an expensive operation) */ public void recursivelyDelete(Component component, boolean commitIndex) { @@ -455,8 +476,9 @@ public void recursivelyDelete(Component component, boolean commitIndex) { /** * Returns a component by matching its identity information. + * * @param project the Project the component is a dependency of - * @param cid the identity values of the component + * @param cid the identity values of the component * @return a Component object, or null if not found */ public Component matchSingleIdentity(final Project project, final ComponentIdentity cid) { @@ -516,8 +538,9 @@ public Component matchSingleIdentity(final Project project, final ComponentIdent /** * Returns a list of components by matching its identity information. + * * @param project the Project the component is a dependency of - * @param cid the identity values of the component + * @param cid the identity values of the component * @return a List of Component objects, or null if not found */ @SuppressWarnings("unchecked") @@ -538,6 +561,7 @@ public List matchIdentity(final Project project, final ComponentIdent /** * Returns a List of components by matching identity information. + * * @param cid the identity values of the component * @return a List of Component objects */ @@ -561,17 +585,18 @@ public List matchIdentity(final ComponentIdentity cid) { * Intelligently adds dependencies for components that are not already a dependency * of the specified project and removes the dependency relationship for components * that are not in the list of specified components. - * @param project the project to bind components to + * + * @param project the project to bind components to * @param existingProjectComponents the complete list of existing dependent components - * @param components the complete list of components that should be dependencies of the project + * @param components the complete list of components that should be dependencies of the project */ public void reconcileComponents(Project project, List existingProjectComponents, List components) { // Removes components as dependencies to the project for all // components not included in the list provided List markedForDeletion = new ArrayList<>(); - for (final Component existingComponent: existingProjectComponents) { + for (final Component existingComponent : existingProjectComponents) { boolean keep = false; - for (final Component component: components) { + for (final Component component : components) { if (component.getId() == existingComponent.getId()) { keep = true; break; @@ -582,7 +607,7 @@ public void reconcileComponents(Project project, List existingProject } } if (!markedForDeletion.isEmpty()) { - for (Component c: markedForDeletion) { + for (Component c : markedForDeletion) { this.recursivelyDelete(c, false); } //this.delete(markedForDeletion); @@ -616,7 +641,7 @@ private void preprocessACLs(final Query query, final String inputFilt final Team team = super.getObjectById(Team.class, teams.get(i).getId()); sb.append(" project.accessTeams.contains(:team").append(i).append(") "); params.put("team" + i, team); - if (i < teamsSize-1) { + if (i < teamsSize - 1) { sb.append(" || "); } } @@ -651,7 +676,7 @@ public Map getDependencyGraphForComponent(Project project, Co } getParentDependenciesOfComponent(project, parentNodeComponent, dependencyGraph, component); } - if (!dependencyGraph.isEmpty() || project.getDirectDependencies().contains(component.getUuid().toString())){ + if (!dependencyGraph.isEmpty() || project.getDirectDependencies().contains(component.getUuid().toString())) { dependencyGraph.put(component.getUuid().toString(), component); getRootDependencies(dependencyGraph, project); getDirectDependenciesForPathDependencies(dependencyGraph); @@ -736,4 +761,71 @@ private void getDirectDependenciesForPathDependencies(Map dep } dependencyGraph.putAll(addToDependencyGraph); } + + /** + * Returns a IntegrityMetaComponent object from the specified purl. + * + * @param purl the Package URL string of the component + * @return a IntegrityMetaComponent object, or null if not found + */ + public IntegrityMetaComponent getIntegrityMetaComponent(String purl) { + final Query query = pm.newQuery(IntegrityMetaComponent.class); + final var params = new HashMap(); + params.put("purl", purl); + query.setParameters(params); + return query.executeUnique(); + } + + /** + * Updates a IntegrityMetaComponent record. + * + * @param transientIntegrityMetaComponent the IntegrityMetaComponent object to synchronize + * @return a synchronized IntegrityMetaComponent object + */ + public synchronized IntegrityMetaComponent updateIntegrityMetaComponent(final IntegrityMetaComponent transientIntegrityMetaComponent) { + final IntegrityMetaComponent integrityMeta = getIntegrityMetaComponent(transientIntegrityMetaComponent.getPurl()); + if (integrityMeta != null) { + integrityMeta.setMd5(transientIntegrityMetaComponent.getMd5()); + integrityMeta.setSha1(transientIntegrityMetaComponent.getSha1()); + integrityMeta.setSha256(transientIntegrityMetaComponent.getSha256()); + integrityMeta.setPublishedAt(transientIntegrityMetaComponent.getPublishedAt()); + integrityMeta.setStatus(transientIntegrityMetaComponent.getStatus()); + integrityMeta.setLastFetch(Date.from(Instant.now())); + return persist(integrityMeta); + } else { + LOGGER.debug("No record found in IntegrityMetaComponent for purl " + transientIntegrityMetaComponent.getPurl()); + return null; + } + } + + /** + * Synchronizes IntegrityMetaComponent with purls from COMPONENT. This is part of initializer. + */ + public synchronized void synchronizeIntegrityMetaComponent() { + final String PURL_SYNC_QUERY = """ + INSERT INTO "INTEGRITY_META_COMPONENT" ("PURL") + SELECT DISTINCT "PURL" + FROM "COMPONENT" + WHERE "PURL" IS NOT NULL + """; + Connection connection = null; + PreparedStatement preparedStatement = null; + try { + connection = (Connection) pm.getDataStoreConnection(); + preparedStatement = connection.prepareStatement(PURL_SYNC_QUERY); + var purlCount = preparedStatement.executeUpdate(); + LOGGER.info("Number of component purls synchronized for integrity check : " + purlCount); + } catch (Exception ex) { + LOGGER.error("Error in synchronizing component purls for integrity meta.", ex); + throw new RuntimeException(ex); + } finally { + DbUtil.close(preparedStatement); + DbUtil.close(connection); + } + } + + public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { + return persist(integrityMetaComponent); + } + } diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index aae66d6fd..9e0df5975 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -1722,4 +1722,8 @@ public List fetchNextPurlsPage(long offset) { public void batchUpdateIntegrityMetaComponent(List purls) { getIntegrityMetaQueryManager().batchUpdateIntegrityMetaComponent(purls); } + + public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { + return getComponentQueryManager().createIntegrityMetaComponent(integrityMetaComponent); + } } diff --git a/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java b/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java index 70c96c237..5615b892c 100644 --- a/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java @@ -18,6 +18,7 @@ */ package org.dependencytrack.resources.v1; +import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.persistence.PaginatedResult; import alpine.server.auth.PermissionRequired; @@ -33,10 +34,12 @@ import io.swagger.annotations.ResponseHeader; import org.apache.commons.lang3.StringUtils; import org.dependencytrack.auth.Permissions; -import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent; import org.dependencytrack.event.InternalComponentIdentificationEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.componentmeta.ComponentProjection; +import org.dependencytrack.event.kafka.componentmeta.Handler; +import org.dependencytrack.event.kafka.componentmeta.HandlerFactory; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; import org.dependencytrack.model.License; @@ -74,6 +77,7 @@ @Api(value = "component", authorizations = @Authorization(value = "X-Api-Key")) public class ComponentResource extends AlpineResource { + private static final Logger LOGGER = Logger.getLogger(ComponentResource.class); private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); @GET @@ -276,7 +280,7 @@ public Response createComponent(@PathParam("uuid") String uuid, Component jsonCo if (project == null) { return Response.status(Response.Status.NOT_FOUND).entity("The project could not be found.").build(); } - if (! qm.hasAccess(super.getPrincipal(), project)) { + if (!qm.hasAccess(super.getPrincipal(), project)) { return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build(); } final License resolvedLicense = qm.getLicense(jsonComponent.getLicense()); @@ -316,7 +320,15 @@ public Response createComponent(@PathParam("uuid") String uuid, Component jsonCo component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes())); component = qm.createComponent(component, true); - kafkaEventDispatcher.dispatchBlocking(new ComponentRepositoryMetaAnalysisEvent(component)); + ComponentProjection componentProjection = + new ComponentProjection(component.getPurlCoordinates().toString(), + component.isInternal(), component.getPurl().toString()); + try { + Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, true); + repoMetaHandler.handle(); + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Unable to process package url %s".formatted(componentProjection.purl())); + } final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), component, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, true); qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, component.getUuid(), vulnAnalysisEvent.token().toString(), 1); kafkaEventDispatcher.dispatchBlocking(vulnAnalysisEvent); @@ -361,7 +373,7 @@ public Response updateComponent(Component jsonComponent) { try (QueryManager qm = new QueryManager()) { Component component = qm.getObjectByUuid(Component.class, jsonComponent.getUuid()); if (component != null) { - if (! qm.hasAccess(super.getPrincipal(), component.getProject())) { + if (!qm.hasAccess(super.getPrincipal(), component.getProject())) { return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified component is forbidden").build(); } // Name cannot be empty or null - prevent it @@ -369,6 +381,7 @@ public Response updateComponent(Component jsonComponent) { if (name != null) { component.setName(name); } + component.setPurlCoordinates(PurlUtil.silentPurlCoordinatesOnly(component.getPurl())); component.setAuthor(StringUtils.trimToNull(jsonComponent.getAuthor())); component.setPublisher(StringUtils.trimToNull(jsonComponent.getPublisher())); component.setVersion(StringUtils.trimToNull(jsonComponent.getVersion())); @@ -402,7 +415,16 @@ public Response updateComponent(Component jsonComponent) { component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes())); component = qm.updateComponent(component, true); - kafkaEventDispatcher.dispatchBlocking(new ComponentRepositoryMetaAnalysisEvent(component)); + ComponentProjection componentProjection = + new ComponentProjection(component.getPurlCoordinates().toString(), + component.isInternal(), component.getPurl().toString()); + try { + + Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, true); + repoMetaHandler.handle(); + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Unable to determine package url type for this purl %s".formatted(component.getPurl().getType()), ex); + } final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), component, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, false); qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, component.getUuid(), vulnAnalysisEvent.token().toString(), 1); kafkaEventDispatcher.dispatchBlocking(vulnAnalysisEvent); @@ -433,7 +455,7 @@ public Response deleteComponent( try (QueryManager qm = new QueryManager()) { final Component component = qm.getObjectByUuid(Component.class, uuid, Component.FetchGroup.ALL.name()); if (component != null) { - if (! qm.hasAccess(super.getPrincipal(), component.getProject())) { + if (!qm.hasAccess(super.getPrincipal(), component.getProject())) { return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified component is forbidden").build(); } qm.recursivelyDelete(component, false); diff --git a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java index 07327424f..8d9a30892 100644 --- a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java +++ b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java @@ -40,9 +40,12 @@ import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent; import org.dependencytrack.event.ProjectMetricsUpdateEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.componentmeta.AbstractMetaHandler; import org.dependencytrack.model.Bom; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.License; import org.dependencytrack.model.Project; import org.dependencytrack.model.ServiceComponent; @@ -83,6 +86,8 @@ import static org.datanucleus.PropertyNames.PROPERTY_FLUSH_MODE; import static org.datanucleus.PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT; import static org.dependencytrack.common.ConfigKey.BOM_UPLOAD_PROCESSING_TRX_FLUSH_THRESHOLD; +import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK; +import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertComponents; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertServices; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertToProject; @@ -315,7 +320,14 @@ private void processBom(final Context ctx, final File bomFile) throws BomConsump // The constructors of ComponentRepositoryMetaAnalysisEvent and ComponentVulnerabilityAnalysisEvent // merely call a few getters on it, but the component object itself is not passed around. // Detaching would imply additional database interactions that we'd rather not do. - repoMetaAnalysisEvents.add(new ComponentRepositoryMetaAnalysisEvent(component)); + boolean result = SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(component.getPurl().getType()); + ComponentRepositoryMetaAnalysisEvent event; + if (result) { + event = collectRepoMetaAnalysisEvents(component, qm); + } else { + event = new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().toString(), component.isInternal(), false, true); + } + repoMetaAnalysisEvents.add(event); vulnAnalysisEvents.add(new ComponentVulnerabilityAnalysisEvent( ctx.uploadToken, component, VulnerabilityAnalysisLevel.BOM_UPLOAD_ANALYSIS, component.isNew())); } @@ -963,4 +975,20 @@ public String toString() { } + private ComponentRepositoryMetaAnalysisEvent collectRepoMetaAnalysisEvents(Component component, QueryManager qm) { + IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(component.getPurl().toString()); + if (integrityMetaComponent != null) { + if (integrityMetaComponent.getStatus() == null || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && (Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime()) > TIME_SPAN)) { + integrityMetaComponent.setLastFetch(Date.from(Instant.now())); + qm.updateIntegrityMetaComponent(integrityMetaComponent); + return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().toString(), component.isInternal(), true, true); + } else { + return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().toString(), component.isInternal(), false, true); + } + } else { + qm.createIntegrityMetaComponent(AbstractMetaHandler.createIntegrityMetaComponent(component.getPurl().toString())); + return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().toString(), component.isInternal(), true, true); + } + } + } diff --git a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java index 4ec75949a..bb18a5efd 100644 --- a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java +++ b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java @@ -21,13 +21,16 @@ import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; +import com.github.packageurl.MalformedPackageURLException; import net.javacrumbs.shedlock.core.LockConfiguration; import net.javacrumbs.shedlock.core.LockExtender; import net.javacrumbs.shedlock.core.LockingTaskExecutor; -import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; import org.dependencytrack.event.PortfolioRepositoryMetaAnalysisEvent; import org.dependencytrack.event.ProjectRepositoryMetaAnalysisEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.componentmeta.ComponentProjection; +import org.dependencytrack.event.kafka.componentmeta.Handler; +import org.dependencytrack.event.kafka.componentmeta.HandlerFactory; import org.dependencytrack.model.Component; import org.dependencytrack.model.Project; import org.dependencytrack.persistence.QueryManager; @@ -73,7 +76,7 @@ public void inform(final Event e) { } } else if (e instanceof PortfolioRepositoryMetaAnalysisEvent) { try { - LockProvider.executeWithLock(PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, (LockingTaskExecutor.Task)() -> processPortfolio()); + LockProvider.executeWithLock(PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, (LockingTaskExecutor.Task) () -> processPortfolio()); } catch (Throwable ex) { LOGGER.error("An unexpected error occurred while submitting components for repository meta analysis", ex); } @@ -95,7 +98,8 @@ private void processProject(final UUID projectUuid) throws Exception { long offset = 0; List components = fetchNextComponentsPage(pm, project, offset); while (!components.isEmpty()) { - dispatchComponents(components); + //latest version information needs to be fetched for project as either triggered because of fresh bom upload or individual project reanalysis + dispatchComponents(components, qm); offset += components.size(); components = fetchNextComponentsPage(pm, project, offset); @@ -118,10 +122,11 @@ private void processPortfolio() throws Exception { List components = fetchNextComponentsPage(pm, null, offset); while (!components.isEmpty()) { long cumulativeProcessingTime = System.currentTimeMillis() - startTime; - if(isLockToBeExtended(cumulativeProcessingTime, PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK)) { + if (isLockToBeExtended(cumulativeProcessingTime, PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK)) { LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); } - dispatchComponents(components); + //latest version information does not need to be fetched for project as triggered for portfolio means it is a scheduled event happening + dispatchComponents(components, qm); offset += components.size(); components = fetchNextComponentsPage(pm, null, offset); @@ -131,9 +136,14 @@ private void processPortfolio() throws Exception { LOGGER.info("All components in portfolio submitted for repository meta analysis"); } - private void dispatchComponents(final List components) { + private void dispatchComponents(final List components, QueryManager queryManager) { for (final var component : components) { - kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(component.purlCoordinates(), component.internal())); + try { + Handler repoMetaHandler = HandlerFactory.createHandler(new ComponentProjection(component.purlCoordinates(), component.internal(), component.purl()), queryManager, kafkaEventDispatcher, true); + repoMetaHandler.handle(); + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Unable to determine package url type for this purl %s".formatted(component.purl()), ex); + } } } @@ -155,7 +165,4 @@ private List fetchNextComponentsPage(final PersistenceManag } } - public record ComponentProjection(String purlCoordinates, Boolean internal) { - } - } diff --git a/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto b/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto index a58d96c27..07dab17a1 100644 --- a/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto +++ b/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto @@ -11,6 +11,9 @@ option java_package = "org.hyades.proto.repometaanalysis.v1"; message AnalysisCommand { // The component that shall be analyzed. Component component = 1; + bool fetch_integrity_data = 2; + bool fetch_latest_version = 3; + } message AnalysisResult { diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java new file mode 100644 index 000000000..c877a7d38 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java @@ -0,0 +1,46 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import alpine.common.logging.Logger; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.util.PurlUtil; +import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class HandlerFactoryTest extends PersistenceCapableTest { + + private static final Logger LOGGER = Logger.getLogger(HandlerFactoryTest.class); + + @Test + public void createHandlerForSupportedPackageTest() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + assertTrue(handler instanceof SupportedMetaHandler); + } catch (MalformedPackageURLException e) { + LOGGER.warn("Package url not formed correctly"); + } + + } + + @Test + public void createHandlerForUnSupportedPackageTest() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:golang/github.com/foo/bar@1.2.3"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + assertTrue(handler instanceof UnSupportedMetaHandler); + } catch (MalformedPackageURLException e) { + throw new RuntimeException(e); + } + + } +} diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java new file mode 100644 index 000000000..0998f8f62 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java @@ -0,0 +1,119 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import alpine.common.logging.Logger; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.util.PurlUtil; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.util.KafkaTestUtil.deserializeValue; + +public class SupportedMetaHandlerTest extends AbstractPostgresEnabledTest { + private static final Logger LOGGER = Logger.getLogger(SupportedMetaHandlerTest.class); + + @Test + public void testHandleIntegrityComponentNotInDB() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl()); + Assertions.assertNull(integrityMetaComponent); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + IntegrityMetaComponent result = handler.handle(); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); + assertThat(command.getComponent().getInternal()).isFalse(); + assertThat(command.getFetchIntegrityData()).isTrue(); + assertThat(command.getFetchLatestVersion()).isFalse(); + } + + ); + Assertions.assertEquals(FetchStatus.IN_PROGRESS, result.getStatus()); + + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Package url not formed correctly"); + } + } + + @Test + public void testHandleIntegrityComponentInDB() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + var integrityMeta = new IntegrityMetaComponent(); + integrityMeta.setPurl("pkg:maven/org.http4s/blaze-core_2.12"); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); + qm.createIntegrityMetaComponent(integrityMeta); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + IntegrityMetaComponent result = handler.handle(); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); + assertThat(command.getComponent().getInternal()).isFalse(); + assertThat(command.getFetchIntegrityData()).isFalse(); + assertThat(command.getFetchLatestVersion()).isFalse(); + } + + ); + Assertions.assertEquals(FetchStatus.IN_PROGRESS, result.getStatus()); + + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Package url not formed correctly"); + } + + } + + @Test + public void testHandleIntegrityComponentInDBForMoreThanAnHour() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + var integrityMeta = new IntegrityMetaComponent(); + integrityMeta.setPurl("pkg:maven/org.http4s/blaze-core_2.12"); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); + qm.createIntegrityMetaComponent(integrityMeta); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + IntegrityMetaComponent integrityMetaComponent = handler.handle(); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); + assertThat(command.getComponent().getInternal()).isFalse(); + assertThat(command.getFetchIntegrityData()).isTrue(); + assertThat(command.getFetchLatestVersion()).isFalse(); + } + + ); + Assertions.assertEquals(FetchStatus.IN_PROGRESS, integrityMetaComponent.getStatus()); + assertThat(integrityMetaComponent.getLastFetch()).isAfter(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); + + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Package url not formed correctly"); + } + } +} diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java new file mode 100644 index 000000000..d8fa1fa52 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java @@ -0,0 +1,50 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import alpine.common.logging.Logger; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.util.PurlUtil; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.util.KafkaTestUtil.deserializeValue; + +public class UnSupportedMetaHandlerTest extends AbstractPostgresEnabledTest { + + private static final Logger LOGGER = Logger.getLogger(SupportedMetaHandlerTest.class); + + @Test + public void testHandleComponentInDb() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:golang/foo/bar@baz?ping=pong#1/2/3"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl()); + Assertions.assertNull(integrityMetaComponent); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + handler.handle(); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:golang/foo/bar@baz"); + assertThat(command.getComponent().getInternal()).isFalse(); + assertThat(command.getFetchIntegrityData()).isFalse(); + assertThat(command.getFetchLatestVersion()).isFalse(); + } + + ); + Assertions.assertNull(integrityMetaComponent); + + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Package url not formed correctly"); + } + } +} diff --git a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java index b7a39e3ec..94de029a4 100644 --- a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java +++ b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java @@ -7,6 +7,10 @@ import org.dependencytrack.model.Project; import org.junit.Test; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; + import static org.assertj.core.api.Assertions.assertThat; public class IntegrityMetaQueryManagerTest extends PersistenceCapableTest { @@ -15,7 +19,8 @@ public class IntegrityMetaQueryManagerTest extends PersistenceCapableTest { public void testGetIntegrityMetaComponent() { var integrityMeta = new IntegrityMetaComponent(); integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.TIMED_OUT); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); var result = qm.getIntegrityMetaComponent("pkg:maven/acme/example@1.0.0?type=jar"); assertThat(result).isNull(); @@ -23,12 +28,12 @@ public void testGetIntegrityMetaComponent() { result = qm.persist(integrityMeta); assertThat(qm.getIntegrityMetaComponent(result.getPurl())).satisfies( meta -> { - assertThat(meta.getStatus()).isEqualTo(FetchStatus.TIMED_OUT); + assertThat(meta.getStatus()).isEqualTo(FetchStatus.IN_PROGRESS); assertThat(meta.getId()).isEqualTo(1L); assertThat(meta.getMd5()).isNull(); assertThat(meta.getSha1()).isNull(); assertThat(meta.getSha256()).isNull(); - assertThat(meta.getLastFetch()).isNull(); + assertThat(meta.getLastFetch()).isEqualTo(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); assertThat(meta.getPublishedAt()).isNull(); } ); @@ -38,14 +43,15 @@ public void testGetIntegrityMetaComponent() { public void testUpdateIntegrityMetaComponent() { var integrityMeta = new IntegrityMetaComponent(); integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.TIMED_OUT); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); - var result = qm.updateIntegrityMetaComponent(integrityMeta); + var result = qm.updateIntegrityMetaComponent(integrityMeta); assertThat(result).isNull(); var persisted = qm.persist(integrityMeta); persisted.setStatus(FetchStatus.PROCESSED); - result = qm.updateIntegrityMetaComponent(persisted); + result = qm.updateIntegrityMetaComponent(persisted); assertThat(result.getStatus()).isEqualTo(FetchStatus.PROCESSED); } @@ -86,7 +92,7 @@ public void testSynchronizeIntegrityMetaComponent() { public void testGetIntegrityMetaComponentCount() { var integrityMeta = new IntegrityMetaComponent(); integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.TIMED_OUT); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); qm.persist(integrityMeta); integrityMeta = new IntegrityMetaComponent(); diff --git a/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java index b9bcca5bb..072443eda 100644 --- a/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java +++ b/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java @@ -406,6 +406,7 @@ public void createComponentTest() { component.setProject(project); component.setName("My Component"); component.setVersion("1.0"); + component.setPurl("pkg:maven/org.acme/abc"); Response response = target(V1_COMPONENT + "/project/" + project.getUuid().toString()).request() .header(X_API_KEY, apiKey) .put(Entity.entity(component, MediaType.APPLICATION_JSON)); @@ -415,8 +416,13 @@ public void createComponentTest() { Assert.assertEquals("My Component", json.getString("name")); Assert.assertEquals("1.0", json.getString("version")); Assert.assertTrue(UuidUtil.isValidUUID(json.getString("uuid"))); - assertThat(kafkaMockProducer.history()).satisfiesExactly( + assertThat(kafkaMockProducer.history()).satisfiesExactlyInAnyOrder( record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = KafkaTestUtil.deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo(json.getString("purl")); + }, record -> { assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_COMMAND.name()); final var command = KafkaTestUtil.deserializeValue(KafkaTopics.VULN_ANALYSIS_COMMAND, record); @@ -432,6 +438,7 @@ public void createComponentUpperCaseHashTest() { component.setProject(project); component.setName("My Component"); component.setVersion("1.0"); + component.setPurl("pkg:maven/org.acme/abc"); component.setSha1("640ab2bae07bedc4c163f679a746f7ab7fb5d1fa".toUpperCase()); component.setSha256("532eaabd9574880dbf76b9b8cc00832c20a6ec113d682299550d7a6e0f345e25".toUpperCase()); component.setSha3_256("c0a5cca43b8aa79eb50e3464bc839dd6fd414fae0ddf928ca23dcebf8a8b8dd0".toUpperCase()); @@ -465,6 +472,7 @@ public void updateComponentTest() { Component component = new Component(); component.setProject(project); component.setName("My Component"); + component.setPurl("pkg:maven/org.acme/abc"); component.setVersion("1.0"); component = qm.createComponent(component, false); component.setDescription("Test component"); @@ -477,8 +485,13 @@ public void updateComponentTest() { Assert.assertEquals("My Component", json.getString("name")); Assert.assertEquals("1.0", json.getString("version")); Assert.assertEquals("Test component", json.getString("description")); - assertThat(kafkaMockProducer.history()).satisfiesExactly( + assertThat(kafkaMockProducer.history()).satisfiesExactlyInAnyOrder( record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = KafkaTestUtil.deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo(json.getString("purl")); + }, record -> { assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_COMMAND.name()); final var command = KafkaTestUtil.deserializeValue(KafkaTopics.VULN_ANALYSIS_COMMAND, record);