From 8c5ab177faad643286d6a44b7864cb432f010097 Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 18 Sep 2024 17:32:17 +0200 Subject: [PATCH] Migrate `DIRECT_DEPENDENCIES` from `TEXT` to `JSONB` (#912) --- pom.xml | 8 ++++++++ .../java/org/dependencytrack/model/Component.java | 5 +++++ .../java/org/dependencytrack/model/Project.java | 5 +++++ .../persistence/ComponentQueryManager.java | 7 +++---- .../policy/cel/CelCommonPolicyLibrary.java | 14 +++++++------- .../policy/cel/CelPolicyQueryManager.java | 14 ++++++++------ src/main/resources/migration/changelog-v5.6.0.xml | 12 ++++++++++++ .../ComponentQueryManangerPostgresTest.java | 4 ++-- 8 files changed, 50 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 4ca267cc9..874879f18 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,7 @@ 1.4.3 1.0.1 9.0.5 + 0.2.0 4.0.5 3.45.4 3.4.1 @@ -263,6 +264,13 @@ cyclonedx-core-java ${lib.cyclonedx-java.version} + + + io.github.nscuro + datanucleus-postgresql + ${lib.datanucleus-postgresql.version} + + jakarta.activation jakarta.activation-api diff --git a/src/main/java/org/dependencytrack/model/Component.java b/src/main/java/org/dependencytrack/model/Component.java index 907c11827..2a3c1dd36 100644 --- a/src/main/java/org/dependencytrack/model/Component.java +++ b/src/main/java/org/dependencytrack/model/Component.java @@ -39,6 +39,7 @@ import javax.jdo.annotations.Convert; import javax.jdo.annotations.Element; import javax.jdo.annotations.Extension; +import javax.jdo.annotations.Extensions; import javax.jdo.annotations.FetchGroup; import javax.jdo.annotations.FetchGroups; import javax.jdo.annotations.IdGeneratorStrategy; @@ -330,6 +331,10 @@ public enum FetchGroup { @Persistent(defaultFetchGroup = "true") @Column(name = "DIRECT_DEPENDENCIES", jdbcType = "CLOB") + @Extensions(value = { + @Extension(vendorName = "datanucleus", key = "insert-function", value = "CAST(? AS JSONB)"), + @Extension(vendorName = "datanucleus", key = "update-function", value = "CAST(? AS JSONB)") + }) @JsonDeserialize(using = TrimmedStringDeserializer.class) private String directDependencies; // This will be a JSON string diff --git a/src/main/java/org/dependencytrack/model/Project.java b/src/main/java/org/dependencytrack/model/Project.java index 4dd1d0bf2..5e5a33765 100644 --- a/src/main/java/org/dependencytrack/model/Project.java +++ b/src/main/java/org/dependencytrack/model/Project.java @@ -46,6 +46,7 @@ import javax.jdo.annotations.Convert; import javax.jdo.annotations.Element; import javax.jdo.annotations.Extension; +import javax.jdo.annotations.Extensions; import javax.jdo.annotations.FetchGroup; import javax.jdo.annotations.FetchGroups; import javax.jdo.annotations.IdGeneratorStrategy; @@ -231,6 +232,10 @@ public enum FetchGroup { @Persistent(defaultFetchGroup = "true") @Column(name = "DIRECT_DEPENDENCIES", jdbcType = "CLOB") + @Extensions(value = { + @Extension(vendorName = "datanucleus", key = "insert-function", value = "CAST(? AS JSONB)"), + @Extension(vendorName = "datanucleus", key = "update-function", value = "CAST(? AS JSONB)") + }) @JsonDeserialize(using = TrimmedStringDeserializer.class) private String directDependencies; // This will be a JSON string diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java index bca7432fe..0da71560e 100644 --- a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java @@ -263,7 +263,7 @@ AND NOT (NOT EXISTS ( if (onlyDirect) { queryString += """ - AND "B0"."DIRECT_DEPENDENCIES" LIKE (('%' || "A0"."UUID") || '%') ESCAPE E'\\\\' + AND "B0"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', "A0"."UUID")) """; } if (orderBy == null) { @@ -931,9 +931,8 @@ public List getDependencyGraphByUUID(final List u } private void getParentDependenciesOfComponent(Project project, Component childComponent, Map dependencyGraph) { - String queryUuid = ".*" + childComponent.getUuid().toString() + ".*"; - final Query query = pm.newQuery(Component.class, "directDependencies.matches(:queryUuid) && project == :project"); - List parentComponents = (List) query.executeWithArray(queryUuid, project); + final Query query = pm.newQuery(Component.class, "directDependencies.jsonbContains(:child) && project == :project"); + List parentComponents = (List) query.executeWithArray("[{\"uuid\":\"%s\"}]".formatted(childComponent.getUuid()), project); for (Component parentComponent : parentComponents) { parentComponent.setExpandDependencyGraph(true); if(parentComponent.getDependencyGraph() == null) { diff --git a/src/main/java/org/dependencytrack/policy/cel/CelCommonPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelCommonPolicyLibrary.java index 3168f8399..d479dbeb0 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelCommonPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelCommonPolicyLibrary.java @@ -510,7 +510,7 @@ private static boolean isDependencyOf(final Component leafComponent, final Compo -- Short-circuit the recursive query if we don't have any matches at all. EXISTS(SELECT 1 FROM "CTE_MATCHES") -- Otherwise, find components of which the given leaf component is a direct dependency. - AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || :leafComponentUuid || '%') + AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :leafComponentUuid)) UNION ALL SELECT "C"."UUID" AS "UUID", @@ -527,7 +527,7 @@ private static boolean isDependencyOf(final Component leafComponent, final Compo -- Also, ensure we haven't seen this component before, to prevent cycles. AND NOT ("C"."ID" = ANY("PREVIOUS"."PATH")) -- Otherwise, the previous component must appear in the current direct dependencies. - AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || "PREVIOUS"."UUID" || '%') + AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', "PREVIOUS"."UUID")) ) SELECT BOOL_OR("FOUND") FROM "CTE_DEPENDENCIES"; """); @@ -586,7 +586,7 @@ AND NOT ("C"."ID" = ANY("PREVIOUS"."PATH")) -- Short-circuit the recursive query if we don't have any matches at all. EXISTS(SELECT 1 FROM "CTE_MATCHES") -- Otherwise, find components of which the given leaf component is a direct dependency. - AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || :leafComponentUuid || '%') + AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :leafComponentUuid)) UNION ALL SELECT "C"."UUID" AS "UUID", @@ -612,7 +612,7 @@ AND NOT ("C"."ID" = ANY("PREVIOUS"."PATH")) -- Ensure we haven't seen this component before, to prevent cycles. NOT ("C"."ID" = ANY("PREVIOUS"."PATH")) -- Otherwise, the previous component must appear in the current direct dependencies. - AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || "PREVIOUS"."UUID" || '%') + AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', "PREVIOUS"."UUID")) ) SELECT ${selectColumnNames?join(", ")} FROM "CTE_DEPENDENCIES" WHERE "FOUND"; """); @@ -700,7 +700,7 @@ private static boolean isExclusiveDependencyOf(final Component leafComponent, fi -- Short-circuit the recursive query if we don't have any matches at all. EXISTS(SELECT 1 FROM "CTE_MATCHES") -- Otherwise, find components of which the given leaf component is a direct dependency. - AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || :leafComponentUuid || '%') + AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :leafComponentUuid)) UNION ALL SELECT "C"."ID" AS "ID", @@ -727,7 +727,7 @@ private static boolean isExclusiveDependencyOf(final Component leafComponent, fi -- Ensure we haven't seen this component before, to prevent cycles. NOT ("C"."ID" = ANY("PREVIOUS"."PATH")) -- Otherwise, the previous component must appear in the current direct dependencies. - AND "C"."DIRECT_DEPENDENCIES" LIKE ('%' || "PREVIOUS"."UUID" || '%') + AND "C"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', "PREVIOUS"."UUID")) ) SELECT "ID", ${selectColumnNames?join(", ", "", ", ")} "FOUND", "PATH" FROM "CTE_DEPENDENCIES"; """); @@ -969,7 +969,7 @@ private static boolean isDirectDependency(final Handle jdbiHandle, final Compone "PROJECT" AS "P" ON "P"."ID" = "C"."PROJECT_ID" WHERE "C"."UUID" = :leafComponentUuid - AND "P"."DIRECT_DEPENDENCIES" LIKE ('%' || :leafComponentUuid || '%') + AND "P"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :leafComponentUuid)) """); return query diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java index a739f5b9b..d47574ceb 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java @@ -649,14 +649,16 @@ List getParents(final UUID uuid, final List parents) { } boolean isDirectDependency(final org.dependencytrack.proto.policy.v1.Component component) { - String queryString = """ - SELECT COUNT(*) FROM "COMPONENT" "C" - INNER JOIN "PROJECT" "P" ON "P"."ID"="C"."PROJECT_ID" - AND "P"."DIRECT_DEPENDENCIES" LIKE :wildcard WHERE "C"."UUID"= :uuid; + String queryString = /* language=SQL */ """ + SELECT COUNT(*) + FROM "COMPONENT" "C" + INNER JOIN "PROJECT" "P" + ON "P"."ID" = "C"."PROJECT_ID" + AND "P"."DIRECT_DEPENDENCIES" @> JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('uuid', :uuid)) + WHERE "C"."UUID" = :uuid """; final Query query = pm.newQuery(Query.SQL, queryString); - query.setNamedParameters(Map.of("uuid", UUID.fromString(component.getUuid()), - "wildcard", "%" + component.getUuid() + "%")); + query.setNamedParameters(Map.of("uuid", UUID.fromString(component.getUuid()))); long result; try { result = query.executeResultUnique(Long.class); diff --git a/src/main/resources/migration/changelog-v5.6.0.xml b/src/main/resources/migration/changelog-v5.6.0.xml index ec9d96015..01b9aef33 100644 --- a/src/main/resources/migration/changelog-v5.6.0.xml +++ b/src/main/resources/migration/changelog-v5.6.0.xml @@ -64,4 +64,16 @@ + + + + + + + CREATE + INDEX "COMPONENT_DIRECT_DEPENDENCIES_JSONB_IDX" + ON "COMPONENT" + USING GIN("DIRECT_DEPENDENCIES" JSONB_PATH_OPS); + + \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/persistence/ComponentQueryManangerPostgresTest.java b/src/test/java/org/dependencytrack/persistence/ComponentQueryManangerPostgresTest.java index 8014c1f1c..3439c4cff 100644 --- a/src/test/java/org/dependencytrack/persistence/ComponentQueryManangerPostgresTest.java +++ b/src/test/java/org/dependencytrack/persistence/ComponentQueryManangerPostgresTest.java @@ -146,7 +146,7 @@ public void testMappingComponentProjectionWithAllFields() { }}); project.setAuthors(authors); project.setDescription("projectDescription"); - project.setDirectDependencies("{7e5f6465-d2f2-424f-b1a4-68d186fa2b46}"); + project.setDirectDependencies("[{\"uuid\":\"7e5f6465-d2f2-424f-b1a4-68d186fa2b46\"}]"); project.setExternalReferences(List.of(new ExternalReference())); project.setLastBomImport(new java.util.Date()); project.setLastBomImportFormat("projectBomFormat"); @@ -181,7 +181,7 @@ public void testMappingComponentProjectionWithAllFields() { component.setPurl("pkg:maven/a/b@1.0"); component.setPublisher("componentPublisher"); component.setPurlCoordinates("componentPurlCoordinates"); - component.setDirectDependencies("componentDirectDependencies"); + component.setDirectDependencies("[]"); component.setExtension("componentExtension"); component.setExternalReferences(List.of(new ExternalReference())); component.setFilename("componentFilename");