Migrate DIRECT_DEPENDENCIES from TEXT to JSONB (#912)

DependencyTrack · Sep 18, 2024 · 8c5ab17 · 8c5ab17
1 parent 8a83c7f
commit 8c5ab17
Show file tree

Hide file tree

Showing 8 changed files with 50 additions and 19 deletions.
diff --git a/pom.xml b/pom.xml
@@ -93,6 +93,7 @@
         <lib.cvss-calculator.version>1.4.3</lib.cvss-calculator.version>
         <lib.owasp-rr-calculator.version>1.0.1</lib.owasp-rr-calculator.version>
         <lib.cyclonedx-java.version>9.0.5</lib.cyclonedx-java.version>
+        <lib.datanucleus-postgresql.version>0.2.0</lib.datanucleus-postgresql.version>
         <lib.jaxb.runtime.version>4.0.5</lib.jaxb.runtime.version>
         <lib.jdbi.version>3.45.4</lib.jdbi.version>
         <lib.json-unit.version>3.4.1</lib.json-unit.version>
@@ -263,6 +264,13 @@
             <artifactId>cyclonedx-core-java</artifactId>
             <version>${lib.cyclonedx-java.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>io.github.nscuro</groupId>
+            <artifactId>datanucleus-postgresql</artifactId>
+            <version>${lib.datanucleus-postgresql.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>jakarta.activation</groupId>
             <artifactId>jakarta.activation-api</artifactId>

diff --git a/src/main/java/org/dependencytrack/model/Component.java b/src/main/java/org/dependencytrack/model/Component.java
@@ -39,6 +39,7 @@
 import javax.jdo.annotations.Convert;
 import javax.jdo.annotations.Element;
 import javax.jdo.annotations.Extension;
+import javax.jdo.annotations.Extensions;
 import javax.jdo.annotations.FetchGroup;
 import javax.jdo.annotations.FetchGroups;
 import javax.jdo.annotations.IdGeneratorStrategy;
@@ -330,6 +331,10 @@ public enum FetchGroup {
 
     @Persistent(defaultFetchGroup = "true")
     @Column(name = "DIRECT_DEPENDENCIES", jdbcType = "CLOB")
+    @Extensions(value = {
+            @Extension(vendorName = "datanucleus", key = "insert-function", value = "CAST(? AS JSONB)"),
+            @Extension(vendorName = "datanucleus", key = "update-function", value = "CAST(? AS JSONB)")
+    })
     @JsonDeserialize(using = TrimmedStringDeserializer.class)
     private String directDependencies; // This will be a JSON string
 

diff --git a/src/main/java/org/dependencytrack/model/Project.java b/src/main/java/org/dependencytrack/model/Project.java
@@ -46,6 +46,7 @@
 import javax.jdo.annotations.Convert;
 import javax.jdo.annotations.Element;
 import javax.jdo.annotations.Extension;
+import javax.jdo.annotations.Extensions;
 import javax.jdo.annotations.FetchGroup;
 import javax.jdo.annotations.FetchGroups;
 import javax.jdo.annotations.IdGeneratorStrategy;
@@ -231,6 +232,10 @@ public enum FetchGroup {
 
     @Persistent(defaultFetchGroup = "true")
     @Column(name = "DIRECT_DEPENDENCIES", jdbcType = "CLOB")
+    @Extensions(value = {
+            @Extension(vendorName = "datanucleus", key = "insert-function", value = "CAST(? AS JSONB)"),
+            @Extension(vendorName = "datanucleus", key = "update-function", value = "CAST(? AS JSONB)")
+    })
     @JsonDeserialize(using = TrimmedStringDeserializer.class)
     private String directDependencies; // This will be a JSON string
 

diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java
@@ -263,7 +263,7 @@ AND NOT (NOT EXISTS (
         if (onlyDirect) {
             queryString +=
                     """
-                       AND "B0"."DIRECT_DEPENDENCIES" LIKE (('%' || "A0"."UUID") || '%') ESCAPE E'\\\\'
+                       AND "B0"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', "A0"."UUID"))
                     """;
         }
         if (orderBy == null) {
@@ -931,9 +931,8 @@ public List<DependencyGraphResponse> getDependencyGraphByUUID(final List<UUID> u
     }
 
     private void getParentDependenciesOfComponent(Project project, Component childComponent, Map<String, Component> dependencyGraph) {
-        String queryUuid = ".*" + childComponent.getUuid().toString() + ".*";
-        final Query<Component> query = pm.newQuery(Component.class, "directDependencies.matches(:queryUuid) && project == :project");
-        List<Component> parentComponents = (List<Component>) query.executeWithArray(queryUuid, project);
+        final Query<Component> query = pm.newQuery(Component.class, "directDependencies.jsonbContains(:child) && project == :project");
+        List<Component> parentComponents = (List<Component>) query.executeWithArray("[{\"uuid\":\"%s\"}]".formatted(childComponent.getUuid()), project);
         for (Component parentComponent : parentComponents) {
             parentComponent.setExpandDependencyGraph(true);
             if(parentComponent.getDependencyGraph() == null) {

diff --git a/src/main/java/org/dependencytrack/policy/cel/CelCommonPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelCommonPolicyLibrary.java
@@ -510,7 +510,7 @@ private static boolean isDependencyOf(final Component leafComponent, final Compo
                             -- Short-circuit the recursive query if we don't have any matches at all.
                             EXISTS(SELECT 1 FROM "CTE_MATCHES")
                             -- Otherwise, find components of which the given leaf component is a direct dependency.
-                            AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || :leafComponentUuid || '%')
+                            AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :leafComponentUuid))
                           UNION ALL
                           SELECT
                             "C"."UUID"                                       AS "UUID",
@@ -527,7 +527,7 @@ private static boolean isDependencyOf(final Component leafComponent, final Compo
                             -- Also, ensure we haven't seen this component before, to prevent cycles.
                             AND NOT ("C"."ID" = ANY("PREVIOUS"."PATH"))
                             -- Otherwise, the previous component must appear in the current direct dependencies.
-                            AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || "PREVIOUS"."UUID" || '%')
+                            AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', "PREVIOUS"."UUID"))
                         )
                         SELECT BOOL_OR("FOUND") FROM "CTE_DEPENDENCIES";
                         """);
@@ -586,7 +586,7 @@ AND NOT ("C"."ID" = ANY("PREVIOUS"."PATH"))
                         -- Short-circuit the recursive query if we don't have any matches at all.
                         EXISTS(SELECT 1 FROM "CTE_MATCHES")
                         -- Otherwise, find components of which the given leaf component is a direct dependency.
-                        AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || :leafComponentUuid || '%')
+                        AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :leafComponentUuid))
                       UNION ALL
                       SELECT
                         "C"."UUID"                                       AS "UUID",
@@ -612,7 +612,7 @@ AND NOT ("C"."ID" = ANY("PREVIOUS"."PATH"))
                         -- Ensure we haven't seen this component before, to prevent cycles.
                         NOT ("C"."ID" = ANY("PREVIOUS"."PATH"))
                         -- Otherwise, the previous component must appear in the current direct dependencies.
-                        AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || "PREVIOUS"."UUID" || '%')
+                        AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', "PREVIOUS"."UUID"))
                     )
                     SELECT ${selectColumnNames?join(", ")} FROM "CTE_DEPENDENCIES" WHERE "FOUND";
                     """);
@@ -700,7 +700,7 @@ private static boolean isExclusiveDependencyOf(final Component leafComponent, fi
                         -- Short-circuit the recursive query if we don't have any matches at all.
                         EXISTS(SELECT 1 FROM "CTE_MATCHES")
                         -- Otherwise, find components of which the given leaf component is a direct dependency.
-                        AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || :leafComponentUuid || '%')
+                        AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :leafComponentUuid))
                       UNION ALL
                       SELECT
                         "C"."ID"                                         AS "ID",
@@ -727,7 +727,7 @@ private static boolean isExclusiveDependencyOf(final Component leafComponent, fi
                         -- Ensure we haven't seen this component before, to prevent cycles.
                         NOT ("C"."ID" = ANY("PREVIOUS"."PATH"))
                         -- Otherwise, the previous component must appear in the current direct dependencies.
-                        AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || "PREVIOUS"."UUID" || '%')
+                        AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', "PREVIOUS"."UUID"))
                     )
                     SELECT "ID", ${selectColumnNames?join(", ", "", ", ")} "FOUND", "PATH" FROM "CTE_DEPENDENCIES";
                      """);
@@ -969,7 +969,7 @@ private static boolean isDirectDependency(final Handle jdbiHandle, final Compone
                   "PROJECT" AS "P" ON "P"."ID" = "C"."PROJECT_ID"
                 WHERE
                   "C"."UUID" = :leafComponentUuid
-                  AND "P"."DIRECT_DEPENDENCIES" LIKE ('%' || :leafComponentUuid || '%')
+                  AND "P"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :leafComponentUuid))
                 """);
 
         return query

diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java
@@ -649,14 +649,16 @@ List<UUID> getParents(final UUID uuid, final List<UUID> parents) {
     }
 
     boolean isDirectDependency(final org.dependencytrack.proto.policy.v1.Component component) {
-        String queryString = """
-                SELECT COUNT(*) FROM "COMPONENT" "C" 
-                INNER JOIN "PROJECT" "P" ON "P"."ID"="C"."PROJECT_ID" 
-                AND "P"."DIRECT_DEPENDENCIES" LIKE :wildcard WHERE "C"."UUID"= :uuid;
+        String queryString = /* language=SQL */ """
+                SELECT COUNT(*)
+                  FROM "COMPONENT" "C"
+                 INNER JOIN "PROJECT" "P"
+                    ON "P"."ID" = "C"."PROJECT_ID"
+                   AND "P"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :uuid))
+                 WHERE "C"."UUID" = :uuid
                 """;
         final Query<?> query = pm.newQuery(Query.SQL, queryString);
-        query.setNamedParameters(Map.of("uuid", UUID.fromString(component.getUuid()),
-                "wildcard", "%" + component.getUuid() + "%"));
+        query.setNamedParameters(Map.of("uuid", UUID.fromString(component.getUuid())));
         long result;
         try {
             result = query.executeResultUnique(Long.class);

diff --git a/src/main/resources/migration/changelog-v5.6.0.xml b/src/main/resources/migration/changelog-v5.6.0.xml
@@ -64,4 +64,16 @@
             <column name="AUTHOR"/>
         </dropColumn>
     </changeSet>
+
+    <changeSet id="v5.6.0-4" author="nscuro">
+        <dropIndex tableName="COMPONENT" indexName="COMPONENT_DIRECT_DEPENDENCIES_GIN_IDX"/>
+        <modifyDataType tableName="COMPONENT" columnName="DIRECT_DEPENDENCIES" newDataType="JSONB"/>
+        <modifyDataType tableName="PROJECT" columnName="DIRECT_DEPENDENCIES" newDataType="JSONB"/>
+        <sql splitStatements="true">
+            CREATE
+             INDEX "COMPONENT_DIRECT_DEPENDENCIES_JSONB_IDX"
+                ON "COMPONENT"
+             USING GIN("DIRECT_DEPENDENCIES" JSONB_PATH_OPS);
+        </sql>
+    </changeSet>
 </databaseChangeLog>
diff --git a/src/test/java/org/dependencytrack/persistence/ComponentQueryManangerPostgresTest.java b/src/test/java/org/dependencytrack/persistence/ComponentQueryManangerPostgresTest.java
@@ -146,7 +146,7 @@ public void testMappingComponentProjectionWithAllFields() {
         }});
         project.setAuthors(authors);
         project.setDescription("projectDescription");
-        project.setDirectDependencies("{7e5f6465-d2f2-424f-b1a4-68d186fa2b46}");
+        project.setDirectDependencies("[{\"uuid\":\"7e5f6465-d2f2-424f-b1a4-68d186fa2b46\"}]");
         project.setExternalReferences(List.of(new ExternalReference()));
         project.setLastBomImport(new java.util.Date());
         project.setLastBomImportFormat("projectBomFormat");
@@ -181,7 +181,7 @@ public void testMappingComponentProjectionWithAllFields() {
         component.setPurl("pkg:maven/a/[email protected]");
         component.setPublisher("componentPublisher");
         component.setPurlCoordinates("componentPurlCoordinates");
-        component.setDirectDependencies("componentDirectDependencies");
+        component.setDirectDependencies("[]");
         component.setExtension("componentExtension");
         component.setExternalReferences(List.of(new ExternalReference()));
         component.setFilename("componentFilename");