Add e2e test for BOM upload storage

commons-persistence/src/main/resources/schema.sql

@@ -915,6 +915,12 @@ ALTER TABLE public."BOM" ALTER COLUMN "ID" ADD GENERATED BY DEFAULT AS IDENTITY
     CACHE 1
 );
 
+CREATE TABLE public."BOM_UPLOAD" (
+    "TOKEN" uuid NOT NULL,
+    "UPLOADED_AT" timestamp with time zone NOT NULL,
+    "BOM" bytea NOT NULL
+);
+
 CREATE TABLE public."COMPONENT" (
     "ID" bigint NOT NULL,
     "BLAKE2B_256" character varying(64),
@@ -2101,6 +2107,9 @@ ALTER TABLE ONLY public."APIKEY"
 ALTER TABLE ONLY public."BOM"
     ADD CONSTRAINT "BOM_PK" PRIMARY KEY ("ID");
 
+ALTER TABLE ONLY public."BOM_UPLOAD"
+    ADD CONSTRAINT "BOM_UPLOAD_PK" PRIMARY KEY ("TOKEN");
+
 ALTER TABLE ONLY public."BOM"
     ADD CONSTRAINT "BOM_UUID_IDX" UNIQUE ("UUID");
 

e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java

@@ -56,7 +56,7 @@ public class AbstractE2ET {
     protected static DockerImageName POSTGRES_IMAGE = DockerImageName.parse("postgres:15-alpine");
     protected static DockerImageName REDPANDA_IMAGE = DockerImageName.parse("docker.redpanda.com/vectorized/redpanda:v24.2.2");
     protected static DockerImageName API_SERVER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-apiserver")
-            .withTag(Optional.ofNullable(System.getenv("APISERVER_VERSION")).orElse("snapshot"));
+            .withTag(Optional.ofNullable(System.getenv("APISERVER_VERSION")).orElse("local"));
     protected static DockerImageName MIRROR_SERVICE_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-mirror-service")
             .withTag(Optional.ofNullable(System.getenv("HYADES_VERSION")).orElse("snapshot"));
     protected static DockerImageName NOTIFICATION_PUBLISHER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-notification-publisher")
@@ -317,7 +317,7 @@ private ApiServerClient initializeApiServerClient() {
     }
 
     @AfterEach
-    void afterEach() {
+    void afterEach() throws Exception {
         ApiServerAuthInterceptor.reset();
 
         Optional.ofNullable(vulnAnalyzerContainer).ifPresent(GenericContainer::stop);

e2e/src/test/java/org/dependencytrack/e2e/BomUploadProcessingWithLocalStorageE2ET.java

@@ -0,0 +1,84 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.e2e;
+
+import org.apache.commons.io.IOUtils;
+import org.dependencytrack.apiserver.model.BomUploadRequest;
+import org.dependencytrack.apiserver.model.EventProcessingResponse;
+import org.dependencytrack.apiserver.model.WorkflowTokenResponse;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.Container.ExecResult;
+import org.testcontainers.containers.GenericContainer;
+
+import java.time.Duration;
+import java.util.Base64;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+class BomUploadProcessingWithLocalStorageE2ET extends AbstractE2ET {
+
+    @Override
+    protected void customizeApiServerContainer(final GenericContainer<?> container) {
+        container
+                // Ensure other storage extensions are disabled.
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_DATABASE_ENABLED", "false")
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_ENABLED", "false")
+                // Enable and configure local storage extension.
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_LOCAL_ENABLED", "true")
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_LOCAL_DIRECTORY", "/tmp/bom-uploads");
+    }
+
+    @Override
+    protected void customizeVulnAnalyzerContainer(final GenericContainer<?> container) {
+        // We don't test analysis here, so don't waste any quota with the OSS Index API.
+        container.withEnv("SCANNER_OSSINDEX_ENABLED", "false");
+    }
+
+    @Test
+    void test() throws Exception {
+        // Parse and base64 encode a BOM.
+        final byte[] bomBytes = IOUtils.resourceToByteArray("/dtrack-apiserver-4.5.0.bom.json");
+        final String bomBase64 = Base64.getEncoder().encodeToString(bomBytes);
+
+        // Upload the BOM.
+        final WorkflowTokenResponse response = apiServerClient.uploadBom(new BomUploadRequest("foo", "bar", true, bomBase64));
+        assertThat(response.token()).isNotEmpty();
+
+        // Wait up to 15sec for the BOM processing to complete.
+        await("BOM processing")
+                .atMost(Duration.ofSeconds(15))
+                .pollDelay(Duration.ofMillis(250))
+                .untilAsserted(() -> {
+                    final EventProcessingResponse processingResponse = apiServerClient.isEventBeingProcessed(response.token());
+                    assertThat(processingResponse.processing()).isFalse();
+                });
+
+        verifyBomDeleted();
+    }
+
+    private void verifyBomDeleted() throws Exception {
+        final ExecResult dirExistsResult = apiServerContainer.execInContainer("test", "-d", "/tmp/bom-uploads");
+        assertThat(dirExistsResult.getExitCode()).withFailMessage("Storage directory was not created").isZero();
+
+        final ExecResult dirEmptyResult = apiServerContainer.execInContainer("ls", "/tmp/bom-uploads");
+        assertThat(dirEmptyResult.getStdout()).withFailMessage("BOM was not deleted after processing").isBlank();
+    }
+
+}

e2e/src/test/java/org/dependencytrack/e2e/BomUploadProcessingWithS3StorageE2ET.java

@@ -0,0 +1,136 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.e2e;
+
+import io.minio.ListObjectsArgs;
+import io.minio.MakeBucketArgs;
+import io.minio.MinioClient;
+import io.minio.Result;
+import io.minio.messages.Item;
+import org.apache.commons.io.IOUtils;
+import org.dependencytrack.apiserver.model.BomUploadRequest;
+import org.dependencytrack.apiserver.model.EventProcessingResponse;
+import org.dependencytrack.apiserver.model.WorkflowTokenResponse;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.MinIOContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+import java.util.Base64;
+import java.util.Optional;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+class BomUploadProcessingWithS3StorageE2ET extends AbstractE2ET {
+
+    private static final String BOM_UPLOAD_BUCKET_NAME = "bom-upload";
+
+    private MinIOContainer minioContainer;
+    private MinioClient minioClient;
+
+    @Override
+    @BeforeEach
+    void beforeEach() throws Exception {
+        minioContainer = new MinIOContainer(DockerImageName.parse("minio/minio:RELEASE.2023-12-14T18-51-57Z"))
+                .withNetworkAliases("minio")
+                .withNetwork(internalNetwork);
+        minioContainer.start();
+
+        minioClient = MinioClient.builder()
+                .endpoint(minioContainer.getS3URL())
+                .credentials(minioContainer.getUserName(), minioContainer.getPassword())
+                .build();
+        createBucket();
+
+        super.beforeEach();
+    }
+
+    @Override
+    protected void customizeApiServerContainer(final GenericContainer<?> container) {
+        container
+                // Ensure other storage extensions are disabled.
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_DATABASE_ENABLED", "false")
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_LOCAL_ENABLED", "false")
+                // Enable and configure S3 storage extension.
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_ENABLED", "true")
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_ENDPOINT", "http://minio:9000")
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_BUCKET", BOM_UPLOAD_BUCKET_NAME)
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_ACCESS_KEY", minioContainer.getUserName())
+                .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_SECRET_KEY", minioContainer.getPassword());
+    }
+
+    @Override
+    protected void customizeVulnAnalyzerContainer(final GenericContainer<?> container) {
+        // We don't test analysis here, so don't waste any quota with the OSS Index API.
+        container.withEnv("SCANNER_OSSINDEX_ENABLED", "false");
+    }
+
+    @Override
+    @AfterEach
+    void afterEach() throws Exception {
+        if (minioClient != null) {
+            minioClient.close();
+        }
+        Optional.ofNullable(minioContainer).ifPresent(GenericContainer::stop);
+
+        super.afterEach();
+    }
+
+    @Test
+    void test() throws Exception {
+        // Parse and base64 encode a BOM.
+        final byte[] bomBytes = IOUtils.resourceToByteArray("/dtrack-apiserver-4.5.0.bom.json");
+        final String bomBase64 = Base64.getEncoder().encodeToString(bomBytes);
+
+        // Upload the BOM.
+        final WorkflowTokenResponse response = apiServerClient.uploadBom(new BomUploadRequest("foo", "bar", true, bomBase64));
+        assertThat(response.token()).isNotEmpty();
+
+        // Wait up to 15sec for the BOM processing to complete.
+        await("BOM processing")
+                .atMost(Duration.ofSeconds(15))
+                .pollDelay(Duration.ofMillis(250))
+                .untilAsserted(() -> {
+                    final EventProcessingResponse processingResponse = apiServerClient.isEventBeingProcessed(response.token());
+                    assertThat(processingResponse.processing()).isFalse();
+                });
+
+        verifyBomDeleted();
+    }
+
+    private void createBucket() throws Exception {
+        minioClient.makeBucket(MakeBucketArgs.builder()
+                .bucket(BOM_UPLOAD_BUCKET_NAME)
+                .build());
+    }
+
+    private void verifyBomDeleted() {
+        final Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder()
+                .bucket(BOM_UPLOAD_BUCKET_NAME)
+                .recursive(true)
+                .build());
+
+        assertThat(results).isEmpty();
+    }
+
+}

e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java

@@ -172,7 +172,7 @@ protected void customizeVulnAnalyzerContainer(final GenericContainer<?> containe
 
     @Override
     @AfterEach
-    void afterEach() {
+    void afterEach() throws Exception {
         Optional.ofNullable(minioContainer).ifPresent(GenericContainer::stop);
 
         super.afterEach();

scripts/create-topics.sh

@@ -77,6 +77,7 @@ for topic_name in "${vuln_analysis_topics[@]}"; do
   create_topic "$topic_name" "${VULN_ANALYSIS_TOPICS_PARTITIONS:-3}" "retention.ms=${VULN_ANALYSIS_TOPICS_RETENTION_MS:-43200000}"
 done
 
+create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.event.bom-uploaded" "${EVENT_BOM_UPLOADED_TOPIC_PARTITIONS:-3}" "retention.ms=${EVENT_BOM_UPLOADED_TOPIC_RETENTION_MS:-43200000}"
 create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.mirror.command" "1" "retention.ms=${VULN_MIRROR_TOPICS_RETENTION_MS:-43200000}"
 create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.mirror.state" "1" "cleanup.policy=compact segment.bytes=67108864 max.compaction.lag.ms=1"
 create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.digest" "1" "cleanup.policy=compact segment.bytes=134217728"