Feature/integrity analysis apiserver

src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java

@@ -1,21 +1,18 @@
 package org.dependencytrack.event;
 
 import alpine.event.framework.Event;
-import com.github.packageurl.PackageURL;
 import org.dependencytrack.model.Component;
 
-import java.util.Optional;
-
 /**
  * Defines an {@link Event} triggered when requesting a component to be analyzed for meta information.
  *
- * @param purlCoordinates The package URL coordinates of the {@link Component} to analyze
- * @param internal        Whether the {@link Component} is internal
+ * @param purlCoordinates    The package URL coordinates of the {@link Component} to analyze
+ * @param internal           Whether the {@link Component} is internal
+ * @param fetchIntegrityData Whether component hash information needs to be fetched from external api
+ * @param fetchLatestVersion Whether to fetch latest version meta information for a component.
  */
-public record ComponentRepositoryMetaAnalysisEvent(String purlCoordinates, Boolean internal) implements Event {
-
-    public ComponentRepositoryMetaAnalysisEvent(final Component component) {
-        this(Optional.ofNullable(component.getPurlCoordinates()).map(PackageURL::canonicalize).orElse(null), component.isInternal());
-    }
+public record ComponentRepositoryMetaAnalysisEvent(String purlCoordinates, Boolean internal,
+                                                   boolean fetchIntegrityData,
+                                                   boolean fetchLatestVersion) implements Event {
 
 }

src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java

@@ -67,7 +67,7 @@ private void batchProcessPurls(QueryManager qm) {
         List<String> purls = qm.fetchNextPurlsPage(offset);
         while (!purls.isEmpty()) {
             long cumulativeProcessingTime = System.currentTimeMillis() - startTime;
-            if(isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) {
+            if (isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) {
                 LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor());
             }
             dispatchPurls(qm, purls);
@@ -88,7 +88,7 @@ private void updateIntegrityMetaForPurls(QueryManager qm, List<String> purls) {
     private void dispatchPurls(QueryManager qm, List<String> purls) {
         for (final var purl : purls) {
             ComponentProjection componentProjection = qm.getComponentByPurl(purl);
-            kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal));
+            kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal, true, false));
         }
     }
 

src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java

@@ -58,6 +58,8 @@ static KafkaEvent<String, AnalysisCommand> convert(final ComponentRepositoryMeta
 
         final var analysisCommand = AnalysisCommand.newBuilder()
                 .setComponent(componentBuilder)
+                .setFetchIntegrityData(event.fetchIntegrityData())
+                .setFetchLatestVersion(event.fetchLatestVersion())
                 .build();
 
         return new KafkaEvent<>(KafkaTopics.REPO_META_ANALYSIS_COMMAND, event.purlCoordinates(), analysisCommand, null);

src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java

@@ -0,0 +1,27 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.model.FetchStatus;
+import org.dependencytrack.model.IntegrityMetaComponent;
+import org.dependencytrack.persistence.QueryManager;
+
+import java.time.Instant;
+import java.util.Date;
+
+public abstract class AbstractMetaHandler implements Handler {
+
+    ComponentProjection componentProjection;
+    QueryManager queryManager;
+    KafkaEventDispatcher kafkaEventDispatcher;
+    boolean fetchLatestVersion;
+
+
+    public static IntegrityMetaComponent createIntegrityMetaComponent(String purl) {
+        IntegrityMetaComponent integrityMetaComponent1 = new IntegrityMetaComponent();
+        integrityMetaComponent1.setStatus(FetchStatus.IN_PROGRESS);
+        integrityMetaComponent1.setPurl(purl);
+        integrityMetaComponent1.setLastFetch(Date.from(Instant.now()));
+        return integrityMetaComponent1;
+    }
+
+}

src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java

@@ -0,0 +1,4 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+public record ComponentProjection(String purlCoordinates, Boolean internal, String purl) {
+}

src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java

@@ -0,0 +1,7 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import org.dependencytrack.model.IntegrityMetaComponent;
+
+public interface Handler {
+    IntegrityMetaComponent handle();
+}

src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java

@@ -0,0 +1,19 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import com.github.packageurl.MalformedPackageURLException;
+import com.github.packageurl.PackageURL;
+import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.persistence.QueryManager;
+
+public class HandlerFactory {
+
+    public static Handler createHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, boolean fetchLatestVersion) throws MalformedPackageURLException {
+        PackageURL packageURL = new PackageURL(componentProjection.purl());
+        boolean result = RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(packageURL.getType());
+        if (result) {
+            return new SupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchLatestVersion);
+        } else {
+            return new UnSupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchLatestVersion);
+        }
+    }
+}

src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java

@@ -0,0 +1,9 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import java.util.List;
+
+public class RepoMetaConstants {
+
+    public static final long TIME_SPAN = 60 * 60 * 1000L;
+    public static final List<String> SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK =List.of("maven", "npm", "pypi");
+}

src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java

@@ -0,0 +1,46 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent;
+import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.model.FetchStatus;
+import org.dependencytrack.model.IntegrityMetaComponent;
+import org.dependencytrack.persistence.QueryManager;
+
+import java.time.Instant;
+import java.util.Date;
+
+import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN;
+
+public class SupportedMetaHandler extends AbstractMetaHandler {
+
+    public SupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, boolean fetchLatestVersion) {
+        this.componentProjection = componentProjection;
+        this.kafkaEventDispatcher = kafkaEventDispatcher;
+        this.queryManager = queryManager;
+        this.fetchLatestVersion = fetchLatestVersion;
+    }
+
+    @Override
+    public IntegrityMetaComponent handle() {
+        KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
+        try (QueryManager queryManager = new QueryManager()) {
+            IntegrityMetaComponent integrityMetaComponent = queryManager.getIntegrityMetaComponent(componentProjection.purl());
+            if (integrityMetaComponent != null) {
+                if (integrityMetaComponent.getStatus() == null || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime() > TIME_SPAN)) {
+                    integrityMetaComponent.setLastFetch(Date.from(Instant.now()));
+                    IntegrityMetaComponent integrityMetaComponent1 = queryManager.updateIntegrityMetaComponent(integrityMetaComponent);
+                    kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion));
+                    return integrityMetaComponent1;
+                } else {
+                    kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion));
+                    return integrityMetaComponent;
+                }
+            } else {
+                IntegrityMetaComponent integrityMetaComponent1 = queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl()));
+                kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion));
+                return integrityMetaComponent1;
+            }
+        }
+    }
+
+}

src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java

@@ -0,0 +1,23 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent;
+import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.model.IntegrityMetaComponent;
+import org.dependencytrack.persistence.QueryManager;
+
+public class UnSupportedMetaHandler extends AbstractMetaHandler {
+
+    public UnSupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher,boolean fetchLatestVersion) {
+        this.componentProjection = componentProjection;
+        this.kafkaEventDispatcher = kafkaEventDispatcher;
+        this.queryManager = queryManager;
+        this.fetchLatestVersion = fetchLatestVersion;
+    }
+
+    @Override
+    public IntegrityMetaComponent handle() {
+        KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
+        kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion));
+        return null;
+    }
+}

src/main/java/org/dependencytrack/model/FetchStatus.java

@@ -1,7 +1,12 @@
 package org.dependencytrack.model;
 
 public enum FetchStatus {
+    //request processed successfully
     PROCESSED,
-    TIMED_OUT,
-    IN_PROGRESS
+    //fetching information for this component is in progress
+    IN_PROGRESS,
+
+    //to be used when information is not available in source of truth so we don't go fetching this repo information again
+    //after first attempt
+    NOT_AVAILABLE
 }