From 7a6c84ae20729ad3b943a4136d4b96bdc6433402 Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 27 Sep 2023 15:09:38 +0200 Subject: [PATCH 1/6] * integrity events sent from apiserver Signed-off-by: mehab Add support for CEL policy conditions (#316) * Initial commit of CEL policy work Signed-off-by: nscuro * Add a few custom CEL functions Signed-off-by: nscuro * Make policies work with legacy way of reporting violations Signed-off-by: nscuro * Implement `is_dependency_of` CEL function Signed-off-by: nscuro * Support vuln aliases in CEL policies Signed-off-by: nscuro * Few minor adjustments Signed-off-by: nscuro * Return CEL errors in API response Signed-off-by: nscuro * Fix some vulnerability fields not being fetched for policies Signed-off-by: nscuro * Bump `versatile` to `0.3.0` Signed-off-by: nscuro * Use AST visitor to determine which fields are accessed for any given type Signed-off-by: nscuro * Cleanup Signed-off-by: nscuro * Cleanup Signed-off-by: nscuro * WIP: Loading of required fields; Project policy evaluation Signed-off-by: nscuro * Improve violation reconciliation for projects Signed-off-by: nscuro * Add test with bloated BOM to debug performance bottlenecks Signed-off-by: nscuro * Disable DataNucleus L1 cache for policy reconciliation Signed-off-by: nscuro * Add field mapping tests Signed-off-by: nscuro * Handle implicit policy script requirements for custom functions Signed-off-by: nscuro * Minor readability and code documentation improvements Signed-off-by: nscuro * Fetch data for policy violation notifications in a single query DataNucleus on its own loads too much data, and does so using too many queries. Signed-off-by: nscuro * Perform violation reconciliation using direct JDBC access Signed-off-by: nscuro * Include strings library in CEL policy environment Signed-off-by: nscuro * Cleanup; Support project properties, tags, and vulnerability aliases Signed-off-by: nscuro * Add test to verify that all fields can be loaded Signed-off-by: nscuro * Add remaining fields to `testWithAllFields` Signed-off-by: nscuro * Add test for vuln severity evaluation Signed-off-by: nscuro * Remove un-implemented `depends_on` function; Add proper logging for custom functions Signed-off-by: nscuro * Handle invalid scripts and script runtime failures Signed-off-by: nscuro * Add `escapeQuotes` for CEL script builders Using `escapeJson` doesn't work quite right when special characters / regular expressions are provided. All we need is prevention of "breaking out" of strings, so escaping double quotes alone is sufficient. Signed-off-by: nscuro * Add tests for some legacy conditions Signed-off-by: nscuro * More tests for `CelPolicyEngine` Signed-off-by: nscuro * Add more tests; Implement script cache bypass for REST API interactions Signed-off-by: nscuro * Add tests for hash policy (#326) * added tests for hash policy Signed-off-by: mehab * updated tests Signed-off-by: mehab --------- Signed-off-by: mehab * Add version cel policy script builder (#324) * Add version cel policy script builder Signed-off-by: vithikashukla * add version support for coordinates cel policy Signed-off-by: vithikashukla * Added unit test for version policy script builder Signed-off-by: vithikashukla * added coordninates condition test Signed-off-by: vithikashukla * added coordinates condition test Signed-off-by: vithikashukla --------- Signed-off-by: vithikashukla Co-authored-by: vithikashukla * Fix new UNIQUE constraint breaking existing behavior Signed-off-by: nscuro * Add feature flag for CEL policy engine Signed-off-by: nscuro * Add `UpgradeItem` to update type of `"POLICYCONDITION"."VALUE"` to `TEXT` Signed-off-by: nscuro * Handle policy evaluation for individual components Signed-off-by: nscuro * added unit tests for cwe cel policy Signed-off-by: mehab * Add license condition test (#332) * Add version cel policy script builder Signed-off-by: vithikashukla * add version support for coordinates cel policy Signed-off-by: vithikashukla * Added unit test for version policy script builder Signed-off-by: vithikashukla * added coordninates condition test Signed-off-by: vithikashukla * added coordinates condition test Signed-off-by: vithikashukla * added more conditions to test Signed-off-by: vithikashukla * Added license condition test Signed-off-by: vithikashukla * Update src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java Co-authored-by: Niklas Signed-off-by: VithikaS <122881935+VithikaS@users.noreply.github.com> * Added license group condition test Signed-off-by: vithikashukla * updated comment Signed-off-by: vithikashukla --------- Signed-off-by: vithikashukla Signed-off-by: VithikaS <122881935+VithikaS@users.noreply.github.com> Co-authored-by: vithikashukla Co-authored-by: Niklas * Fix projection mapping for `Double` / `BigDecimal` fields Signed-off-by: nscuro * support wildcard Signed-off-by: vithikashukla * Add `buf` config and workflow Signed-off-by: nscuro * Change Proto package from `hyades` to `dependencytrack` As this feature will be backported, we need to make sure policies will be compatible once folks start upgrading to Hyades. Signed-off-by: nscuro * Fix failing tests due to Proto package change Signed-off-by: nscuro * Un-ignore `cyclonedx.proto` from breaking changes check Signed-off-by: Niklas --------- Signed-off-by: nscuro Signed-off-by: mehab Signed-off-by: vithikashukla Signed-off-by: VithikaS <122881935+VithikaS@users.noreply.github.com> Signed-off-by: Niklas Co-authored-by: meha Co-authored-by: VithikaS <122881935+VithikaS@users.noreply.github.com> Co-authored-by: vithikashukla Co-authored-by: mehab changes for sending repo meta analysis events from apiserver Signed-off-by: mehab initial refactoring Signed-off-by: mehab refactored code Signed-off-by: mehab code changes completed for sending Signed-off-by: mehab fixed component resource unit tests Signed-off-by: mehab --- .github/workflows/buf.yml | 29 + pom.xml | 13 + .../org/dependencytrack/common/ConfigKey.java | 3 +- .../ComponentRepositoryMetaAnalysisEvent.java | 17 +- .../event/kafka/KafkaEventConverter.java | 2 + .../componentmeta/AbstractMetaHandler.java | 27 + .../componentmeta/ComponentProjection.java | 4 + .../event/kafka/componentmeta/Handler.java | 5 + .../kafka/componentmeta/HandlerFactory.java | 19 + .../componentmeta/RepoMetaConstants.java | 9 + .../componentmeta/SupportedMetaHandler.java | 43 + .../componentmeta/UnSupportedMetaHandler.java | 21 + .../dependencytrack/model/FetchStatus.java | 9 +- .../org/dependencytrack/model/Policy.java | 2 +- .../model/PolicyCondition.java | 53 +- .../model/PolicyViolation.java | 3 + .../model/VulnerabilityAlias.java | 1 + .../parser/cyclonedx/CycloneDXExporter.java | 2 +- .../persistence/ComponentQueryManager.java | 9 +- .../persistence/PolicyQueryManager.java | 24 +- .../persistence/QueryManager.java | 12 +- .../dependencytrack/policy/PolicyEngine.java | 3 + .../policy/cel/CelPolicyEngine.java | 561 +++++++++++ .../policy/cel/CelPolicyLibrary.java | 378 ++++++++ .../policy/cel/CelPolicyQueryManager.java | 600 ++++++++++++ .../policy/cel/CelPolicyScript.java | 37 + .../policy/cel/CelPolicyScriptHost.java | 177 ++++ .../policy/cel/CelPolicyScriptVisitor.java | 122 +++ .../compat/CelPolicyScriptSourceBuilder.java | 16 + ...onentHashCelPolicyScriptSourceBuilder.java | 45 + ...ordinatesCelPolicyScriptSourceBuilder.java | 92 ++ .../CpeCelPolicyScriptSourceBuilder.java | 24 + .../CweCelPolicyScriptSourceBuilder.java | 54 ++ .../LicenseCelPolicyScriptSourceBuilder.java | 55 ++ ...enseGroupCelPolicyScriptSourceBuilder.java | 24 + ...ackageUrlCelPolicyScriptSourceBuilder.java | 24 + .../SeverityCelPolicyScriptSourceBuilder.java | 24 + ...SwidTagIdCelPolicyScriptSourceBuilder.java | 24 + .../VersionCelPolicyScriptSourceBuilder.java | 61 ++ ...abilityIdCelPolicyScriptSourceBuilder.java | 41 + .../cel/mapping/ComponentProjection.java | 82 ++ .../ComponentsVulnerabilitiesProjection.java | 9 + .../policy/cel/mapping/FieldMapping.java | 4 + .../policy/cel/mapping/FieldMappingUtil.java | 41 + .../cel/mapping/LicenseGroupProjection.java | 11 + .../policy/cel/mapping/LicenseProjection.java | 32 + .../policy/cel/mapping/MappedField.java | 28 + .../mapping/PolicyViolationProjection.java | 4 + .../policy/cel/mapping/ProjectProjection.java | 44 + .../mapping/ProjectPropertyProjection.java | 17 + .../cel/mapping/VulnerabilityProjection.java | 80 ++ .../resources/v1/ComponentResource.java | 34 +- .../resources/v1/PolicyConditionResource.java | 35 +- .../resources/v1/vo/CelExpressionError.java | 4 + .../tasks/BomUploadProcessingTask.java | 30 +- .../tasks/PolicyEvaluationTask.java | 39 +- .../tasks/RepositoryMetaAnalyzerTask.java | 27 +- .../dependencytrack/upgrade/UpgradeItems.java | 5 + .../upgrade/v510/v510Updater.java | 33 + .../util/NotificationUtil.java | 177 +++- src/main/proto/buf.yaml | 5 + .../dependencytrack/policy/v1/policy.proto | 135 +++ .../v1/repo_meta_analysis.proto | 3 + src/main/resources/application.properties | 4 + .../java/org/dependencytrack/TestUtil.java | 1 + .../ComponentQueryManagerTest.java | 12 +- .../policy/cel/CelPolicyEngineTest.java | 901 ++++++++++++++++++ .../policy/cel/CelPolicyScriptHostTest.java | 84 ++ .../CelPolicyScriptSourceBuilderTest.java | 15 + .../cel/compat/CoordinatesConditionTest.java | 127 +++ .../policy/cel/compat/CpeConditionTest.java | 63 ++ .../policy/cel/compat/CweConditionTest.java | 68 ++ .../policy/cel/compat/HashConditionTest.java | 93 ++ .../cel/compat/LicenseConditionTest.java | 122 +++ .../cel/compat/LicenseGroupConditionTest.java | 152 +++ .../cel/compat/SeverityConditionTest.java | 128 +++ .../cel/compat/SwidTagIdConditionTest.java | 61 ++ .../cel/compat/VersionConditionTest.java | 62 ++ .../compat/VulnerabilityIdConditionTest.java | 74 ++ .../cel/mapping/FieldMappingUtilTest.java | 88 ++ .../resources/v1/ComponentResourceTest.java | 17 +- .../v1/PolicyConditionResourceTest.java | 154 +++ 82 files changed, 5702 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/buf.yml create mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/VersionCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/ComponentProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/ComponentsVulnerabilitiesProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/FieldMapping.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtil.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/LicenseGroupProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/LicenseProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/MappedField.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/PolicyViolationProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/ProjectPropertyProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java create mode 100644 src/main/java/org/dependencytrack/resources/v1/vo/CelExpressionError.java create mode 100644 src/main/java/org/dependencytrack/upgrade/v510/v510Updater.java create mode 100644 src/main/proto/buf.yaml create mode 100644 src/main/proto/org/dependencytrack/policy/v1/policy.proto create mode 100644 src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilderTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/CpeConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/CweConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/HashConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/LicenseConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/LicenseGroupConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/SeverityConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/SwidTagIdConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/VersionConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java create mode 100644 src/test/java/org/dependencytrack/resources/v1/PolicyConditionResourceTest.java diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml new file mode 100644 index 000000000..93ce5da13 --- /dev/null +++ b/.github/workflows/buf.yml @@ -0,0 +1,29 @@ +name: Buf + +on: + pull_request: + branches: [ "main" ] + +permissions: { } + +jobs: + buf: + name: Buf + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout Repository + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # tag=v4.1.0 + - name: Setup buf + uses: bufbuild/buf-setup-action@eb60cd0de4f14f1f57cf346916b8cd69a9e7ed0b # tag=v1.26.1 + with: + github_token: ${{ github.token }} + - name: Lint Protobuf + uses: bufbuild/buf-lint-action@bd48f53224baaaf0fc55de9a913e7680ca6dbea4 # tag=v1.0.3 + with: + input: src/main/proto + - name: Detect Breaking Changes + uses: bufbuild/buf-breaking-action@a074e988ee34efcd4927079e79c611f428354c01 # tag=v1.1.3 + with: + input: src/main/proto + against: https://github.com/${{ github.repository }}.git#branch=main,subdir=src/main/proto \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7c0264bf7..e9108b4a6 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,7 @@ 4.7.1 ${project.parent.version} 4.2.0 + 0.3.21 2.0.2 1.4.1 1.0.1 @@ -102,6 +103,7 @@ 1.18.3 2.0.1 1.1.10.1 + 0.3.0 6.4.0 1.1.1 2.0.9 @@ -193,6 +195,12 @@ + + + org.projectnessie.cel + cel-tools + ${lib.cel-tools.version} + us.springett @@ -326,6 +334,11 @@ woodstox-core ${lib.woodstox.version} + + io.github.nscuro + versatile + ${lib.versatile.version} + org.apache.maven maven-artifact diff --git a/src/main/java/org/dependencytrack/common/ConfigKey.java b/src/main/java/org/dependencytrack/common/ConfigKey.java index f2351b2ef..a78107457 100644 --- a/src/main/java/org/dependencytrack/common/ConfigKey.java +++ b/src/main/java/org/dependencytrack/common/ConfigKey.java @@ -70,7 +70,8 @@ public enum ConfigKey implements Config.Key { BOM_UPLOAD_PROCESSING_TRX_FLUSH_THRESHOLD("bom.upload.processing.trx.flush.threshold", "10000"), WORKFLOW_RETENTION_DURATION("workflow.retention.duration", "P3D"), WORKFLOW_STEP_TIMEOUT_DURATION("workflow.step.timeout.duration", "PT1H"), - TMP_DELAY_BOM_PROCESSED_NOTIFICATION("tmp.delay.bom.processed.notification", "false"); + TMP_DELAY_BOM_PROCESSED_NOTIFICATION("tmp.delay.bom.processed.notification", "false"), + CEL_POLICY_ENGINE_ENABLED("cel.policy.engine.enabled", "false"); private final String propertyName; private final Object defaultValue; diff --git a/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java b/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java index 6d74e38d7..335ec7ad0 100644 --- a/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java +++ b/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java @@ -1,21 +1,18 @@ package org.dependencytrack.event; import alpine.event.framework.Event; -import com.github.packageurl.PackageURL; import org.dependencytrack.model.Component; -import java.util.Optional; - /** * Defines an {@link Event} triggered when requesting a component to be analyzed for meta information. * - * @param purlCoordinates The package URL coordinates of the {@link Component} to analyze - * @param internal Whether the {@link Component} is internal + * @param purlCoordinates The package URL coordinates of the {@link Component} to analyze + * @param internal Whether the {@link Component} is internal + * @param fetchIntegrityData Whether component hash information needs to be fetched from external api + * @param fetchLatestVersion Whether to fetch latest version meta information for a component. */ -public record ComponentRepositoryMetaAnalysisEvent(String purlCoordinates, Boolean internal) implements Event { - - public ComponentRepositoryMetaAnalysisEvent(final Component component) { - this(Optional.ofNullable(component.getPurlCoordinates()).map(PackageURL::canonicalize).orElse(null), component.isInternal()); - } +public record ComponentRepositoryMetaAnalysisEvent(String purlCoordinates, Boolean internal, + boolean fetchIntegrityData, + boolean fetchLatestVersion) implements Event { } diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java index ac11b0a45..a938d2b13 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java @@ -58,6 +58,8 @@ static KafkaEvent convert(final ComponentRepositoryMeta final var analysisCommand = AnalysisCommand.newBuilder() .setComponent(componentBuilder) + .setFetchIntegrityData(event.fetchIntegrityData()) + .setFetchLatestVersion(event.fetchLatestVersion()) .build(); return new KafkaEvent<>(KafkaTopics.REPO_META_ANALYSIS_COMMAND, event.purlCoordinates(), analysisCommand, null); diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java new file mode 100644 index 000000000..ec7dfb043 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java @@ -0,0 +1,27 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.persistence.QueryManager; + +import java.time.Instant; +import java.util.Date; + +public abstract class AbstractMetaHandler implements Handler { + + ComponentProjection componentProjection; + QueryManager queryManager; + KafkaEventDispatcher kafkaEventDispatcher; + boolean fetchLatestVersion; + + + public static IntegrityMetaComponent createIntegrityMetaComponent(String purl) { + IntegrityMetaComponent integrityMetaComponent1 = new IntegrityMetaComponent(); + integrityMetaComponent1.setStatus(FetchStatus.IN_PROGRESS); + integrityMetaComponent1.setPurl(purl); + integrityMetaComponent1.setLastFetch(Date.from(Instant.now())); + return integrityMetaComponent1; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java new file mode 100644 index 000000000..53f057cac --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java @@ -0,0 +1,4 @@ +package org.dependencytrack.event.kafka.componentmeta; + +public record ComponentProjection(String purlCoordinates, Boolean internal, String purl) { +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java new file mode 100644 index 000000000..b02c7a724 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java @@ -0,0 +1,5 @@ +package org.dependencytrack.event.kafka.componentmeta; + +public interface Handler { + void handle(); +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java new file mode 100644 index 000000000..1804a4a48 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java @@ -0,0 +1,19 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.persistence.QueryManager; + +public class HandlerFactory { + + public static Handler createHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, boolean fetchLatestVersion) throws MalformedPackageURLException { + PackageURL packageURL = new PackageURL(componentProjection.purl()); + boolean result = RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(packageURL.getType()); + if (result) { + return new SupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchLatestVersion); + } else { + return new UnSupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchLatestVersion); + } + } +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java new file mode 100644 index 000000000..c2054954b --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java @@ -0,0 +1,9 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import java.util.List; + +public class RepoMetaConstants { + + public static final long TIME_SPAN = 60 * 60 * 1000L; + public static final List SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK =List.of("maven", "npm", "pypi"); +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java new file mode 100644 index 000000000..9a539d74c --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java @@ -0,0 +1,43 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.persistence.QueryManager; + +import java.time.Instant; +import java.util.Date; + +import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN; + +public class SupportedMetaHandler extends AbstractMetaHandler { + + public SupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, boolean fetchLatestVersion) { + this.componentProjection = componentProjection; + this.kafkaEventDispatcher = kafkaEventDispatcher; + this.queryManager = queryManager; + this.fetchLatestVersion = fetchLatestVersion; + } + + @Override + public void handle() { + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try (QueryManager queryManager = new QueryManager()) { + IntegrityMetaComponent integrityMetaComponent = queryManager.getIntegrityMetaComponent(componentProjection.purl()); + if (integrityMetaComponent != null) { + if (integrityMetaComponent.getStatus() == null || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime() > TIME_SPAN)) { + integrityMetaComponent.setLastFetch(Date.from(Instant.now())); + queryManager.updateIntegrityMetaComponent(integrityMetaComponent); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion)); + } else { + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion)); + } + } else { + queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl())); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion)); + } + } + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java new file mode 100644 index 000000000..09aa1d2ac --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java @@ -0,0 +1,21 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.persistence.QueryManager; + +public class UnSupportedMetaHandler extends AbstractMetaHandler { + + public UnSupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher,boolean fetchLatestVersion) { + this.componentProjection = componentProjection; + this.kafkaEventDispatcher = kafkaEventDispatcher; + this.queryManager = queryManager; + this.fetchLatestVersion = fetchLatestVersion; + } + + @Override + public void handle() { + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion)); + } +} diff --git a/src/main/java/org/dependencytrack/model/FetchStatus.java b/src/main/java/org/dependencytrack/model/FetchStatus.java index 099b45783..c376fbfc6 100644 --- a/src/main/java/org/dependencytrack/model/FetchStatus.java +++ b/src/main/java/org/dependencytrack/model/FetchStatus.java @@ -1,6 +1,13 @@ package org.dependencytrack.model; public enum FetchStatus { + //request processed successfully PROCESSED, - TIMED_OUT + + //fetching information for this component is in progress + IN_PROGRESS, + + //to be used when information is not available in source of truth so we don't go fetching this repo information again + //after first attempt + NOT_AVAILABLE } diff --git a/src/main/java/org/dependencytrack/model/Policy.java b/src/main/java/org/dependencytrack/model/Policy.java index 34685353d..9d3dad806 100644 --- a/src/main/java/org/dependencytrack/model/Policy.java +++ b/src/main/java/org/dependencytrack/model/Policy.java @@ -98,7 +98,7 @@ public enum ViolationState { @Column(name = "VIOLATIONSTATE", allowsNull = "false") @NotBlank @Size(min = 1, max = 255) - @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The operator may only contain printable characters") + @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The violation state may only contain printable characters") private ViolationState violationState; /** diff --git a/src/main/java/org/dependencytrack/model/PolicyCondition.java b/src/main/java/org/dependencytrack/model/PolicyCondition.java index 9fdd144fa..4a492d440 100644 --- a/src/main/java/org/dependencytrack/model/PolicyCondition.java +++ b/src/main/java/org/dependencytrack/model/PolicyCondition.java @@ -63,21 +63,28 @@ public enum Operator { } public enum Subject { - AGE, + AGE(PolicyViolation.Type.OPERATIONAL), //ANALYZER, //BOM, - COORDINATES, - CPE, + COORDINATES(PolicyViolation.Type.OPERATIONAL), + CPE(PolicyViolation.Type.OPERATIONAL), //INHERITED_RISK_SCORE, - LICENSE, - LICENSE_GROUP, - PACKAGE_URL, - SEVERITY, - SWID_TAGID, - VERSION, - COMPONENT_HASH, - CWE, - VULNERABILITY_ID + EXPRESSION(null), + LICENSE(PolicyViolation.Type.LICENSE), + LICENSE_GROUP(PolicyViolation.Type.LICENSE), + PACKAGE_URL(PolicyViolation.Type.OPERATIONAL), + SEVERITY(PolicyViolation.Type.SECURITY), + SWID_TAGID(PolicyViolation.Type.OPERATIONAL), + VERSION(PolicyViolation.Type.OPERATIONAL), + COMPONENT_HASH(PolicyViolation.Type.OPERATIONAL), + CWE(PolicyViolation.Type.SECURITY), + VULNERABILITY_ID(PolicyViolation.Type.SECURITY); + + private final PolicyViolation.Type violationType; + + Subject(final PolicyViolation.Type violationType) { + this.violationType = violationType; + } } @PrimaryKey @@ -104,12 +111,18 @@ public enum Subject { private Subject subject; @Persistent - @Column(name = "VALUE", allowsNull = "false") + @Column(name = "VALUE", allowsNull = "false", jdbcType = "CLOB") @NotBlank - @Size(min = 1, max = 255) + @Size(min = 1) @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The value may only contain printable characters") private String value; + @Persistent + @Column(name = "VIOLATIONTYPE", allowsNull = "true") + @Size(min = 1, max = 255) + @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The violation type may only contain printable characters") + private PolicyViolation.Type violationType; + /** * The unique identifier of the object. */ @@ -159,6 +172,18 @@ public void setValue(String value) { this.value = value; } + public PolicyViolation.Type getViolationType() { + if (subject != null && subject.violationType != null) { + return subject.violationType; + } + + return violationType; + } + + public void setViolationType(PolicyViolation.Type violationType) { + this.violationType = violationType; + } + public UUID getUuid() { return uuid; } diff --git a/src/main/java/org/dependencytrack/model/PolicyViolation.java b/src/main/java/org/dependencytrack/model/PolicyViolation.java index 317b52b0a..2820da971 100644 --- a/src/main/java/org/dependencytrack/model/PolicyViolation.java +++ b/src/main/java/org/dependencytrack/model/PolicyViolation.java @@ -46,6 +46,9 @@ @PersistenceCapable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) +// TODO: Add @Unique composite constraint on the fields component, policyCondition, and type. +// The legacy PolicyEngine erroneously allows for duplicates on those fields, but CelPolicyEngine +// will never produce such duplicates. Until we remove the legacy engine, we can't add this constraint. public class PolicyViolation implements Serializable { public enum Type { diff --git a/src/main/java/org/dependencytrack/model/VulnerabilityAlias.java b/src/main/java/org/dependencytrack/model/VulnerabilityAlias.java index 40124a0dd..487eb7629 100644 --- a/src/main/java/org/dependencytrack/model/VulnerabilityAlias.java +++ b/src/main/java/org/dependencytrack/model/VulnerabilityAlias.java @@ -200,6 +200,7 @@ private String getBySource(final Vulnerability.Source source) { case NVD -> getCveId(); case OSSINDEX -> getSonatypeId(); case OSV -> getOsvId(); + case SNYK -> getSnykId(); case VULNDB -> getVulnDbId(); default -> null; }; diff --git a/src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java b/src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java index df743fe77..48cda1fdf 100644 --- a/src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java +++ b/src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java @@ -83,7 +83,7 @@ private Bom create(final Listcomponents, final List bom.setComponents(cycloneComponents); bom.setServices(cycloneServices); bom.setVulnerabilities(cycloneVulnerabilities); - if (components != null) { + if (cycloneComponents != null) { bom.setDependencies(ModelConverter.generateDependencies(project, components)); } return bom; diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java index e08f31401..1432c86b6 100644 --- a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java @@ -44,7 +44,9 @@ import java.io.StringReader; import java.sql.Connection; import java.sql.PreparedStatement; +import java.time.Instant; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -752,7 +754,7 @@ public synchronized IntegrityMetaComponent updateIntegrityMetaComponent(final In integrityMeta.setSha256(transientIntegrityMetaComponent.getSha256()); integrityMeta.setPublishedAt(transientIntegrityMetaComponent.getPublishedAt()); integrityMeta.setStatus(transientIntegrityMetaComponent.getStatus()); - integrityMeta.setLastFetch(transientIntegrityMetaComponent.getLastFetch()); + integrityMeta.setLastFetch(Date.from(Instant.now())); return persist(integrityMeta); } else { LOGGER.debug("No record found in IntegrityMetaComponent for purl " + transientIntegrityMetaComponent.getPurl()); @@ -785,4 +787,9 @@ public synchronized void synchronizeIntegrityMetaComponent() { DbUtil.close(connection); } } + + public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { + return persist(integrityMetaComponent); + } + } diff --git a/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java b/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java index f2af10162..d9aec4e75 100644 --- a/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java @@ -185,11 +185,26 @@ public Policy createPolicy(String name, Policy.Operator operator, Policy.Violati */ public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCondition.Subject subject, final PolicyCondition.Operator operator, final String value) { + return createPolicyCondition(policy, subject, operator, value, null); + } + + /** + * Creates a policy condition for the specified Project. + * @return the created PolicyCondition object + */ + public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCondition.Subject subject, + final PolicyCondition.Operator operator, final String value, + final PolicyViolation.Type violationType) { final PolicyCondition pc = new PolicyCondition(); pc.setPolicy(policy); pc.setSubject(subject); - pc.setOperator(operator); + if (subject == PolicyCondition.Subject.EXPRESSION) { + pc.setOperator(PolicyCondition.Operator.MATCHES); + } else { + pc.setOperator(operator); + } pc.setValue(value); + pc.setViolationType(violationType); return persist(pc); } @@ -200,8 +215,13 @@ public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCo public PolicyCondition updatePolicyCondition(final PolicyCondition policyCondition) { final PolicyCondition pc = getObjectByUuid(PolicyCondition.class, policyCondition.getUuid()); pc.setSubject(policyCondition.getSubject()); - pc.setOperator(policyCondition.getOperator()); + if (policyCondition.getSubject() == PolicyCondition.Subject.EXPRESSION) { + pc.setOperator(PolicyCondition.Operator.MATCHES); + } else { + pc.setOperator(policyCondition.getOperator()); + } pc.setValue(policyCondition.getValue()); + pc.setViolationType(policyCondition.getViolationType()); return persist(pc); } diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index 50269fdcd..f392b7af2 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -44,13 +44,13 @@ import org.dependencytrack.model.Classifier; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; -import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.ConfigPropertyConstants; import org.dependencytrack.model.Cpe; import org.dependencytrack.model.Cwe; import org.dependencytrack.model.DependencyMetrics; import org.dependencytrack.model.Finding; import org.dependencytrack.model.FindingAttribution; +import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.License; import org.dependencytrack.model.LicenseGroup; import org.dependencytrack.model.NotificationPublisher; @@ -655,6 +655,12 @@ public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCo return getPolicyQueryManager().createPolicyCondition(policy, subject, operator, value); } + public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCondition.Subject subject, + final PolicyCondition.Operator operator, final String value, + final PolicyViolation.Type violationType) { + return getPolicyQueryManager().createPolicyCondition(policy, subject, operator, value, violationType); + } + public PolicyCondition updatePolicyCondition(final PolicyCondition policyCondition) { return getPolicyQueryManager().updatePolicyCondition(policyCondition); } @@ -1691,4 +1697,8 @@ public IntegrityMetaComponent updateIntegrityMetaComponent(IntegrityMetaComponen public void synchronizeIntegrityMetaComponent() { getComponentQueryManager().synchronizeIntegrityMetaComponent(); } + + public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { + return getComponentQueryManager().createIntegrityMetaComponent(integrityMetaComponent); + } } diff --git a/src/main/java/org/dependencytrack/policy/PolicyEngine.java b/src/main/java/org/dependencytrack/policy/PolicyEngine.java index 7140db559..69e022a1a 100644 --- a/src/main/java/org/dependencytrack/policy/PolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/PolicyEngine.java @@ -213,6 +213,9 @@ public PolicyViolation.Type determineViolationType(final PolicyCondition.Subject case AGE, COORDINATES, PACKAGE_URL, CPE, SWID_TAGID, COMPONENT_HASH, VERSION -> PolicyViolation.Type.OPERATIONAL; case LICENSE, LICENSE_GROUP -> PolicyViolation.Type.LICENSE; + // Just here to satisfy the switch exhaustiveness. Conditions with subject EXPRESSION + // will never yield any violations, because there's no evaluator supporting it. + case EXPRESSION -> null; }; } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java new file mode 100644 index 000000000..6fe736823 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -0,0 +1,561 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import alpine.common.metrics.Metrics; +import com.fasterxml.jackson.core.JacksonException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.google.api.expr.v1alpha1.Type; +import com.google.protobuf.util.Timestamps; +import io.micrometer.core.instrument.Timer; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; +import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyCondition.Subject; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Severity; +import org.dependencytrack.model.VulnerabilityAlias; +import org.dependencytrack.persistence.CollectionIntegerConverter; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.policy.cel.CelPolicyScriptHost.CacheMode; +import org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.ComponentHashCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.CoordinatesCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.CpeCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.CweCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.LicenseCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.LicenseGroupCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.PackageUrlCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.SeverityCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.VersionCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.mapping.ComponentProjection; +import org.dependencytrack.policy.cel.mapping.LicenseProjection; +import org.dependencytrack.policy.cel.mapping.ProjectProjection; +import org.dependencytrack.policy.cel.mapping.ProjectPropertyProjection; +import org.dependencytrack.policy.cel.mapping.VulnerabilityProjection; +import org.dependencytrack.proto.policy.v1.Vulnerability; +import org.dependencytrack.util.NotificationUtil; +import org.dependencytrack.util.VulnerabilityUtil; +import org.projectnessie.cel.tools.ScriptCreateException; +import org.projectnessie.cel.tools.ScriptException; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.Collections.emptyList; +import static org.apache.commons.collections4.MultiMapUtils.emptyMultiValuedMap; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE_GROUP; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT_PROPERTY; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_COMPONENT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_PROJECT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_VULNERABILITIES; + +/** + * A policy engine powered by the Common Expression Language (CEL). + * + * @since 5.1.0 + */ +public class CelPolicyEngine { + + private static final Logger LOGGER = Logger.getLogger(CelPolicyEngine.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Map SCRIPT_BUILDERS; + + static { + SCRIPT_BUILDERS = new HashMap<>(); + SCRIPT_BUILDERS.put(Subject.CPE, new CpeCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.COMPONENT_HASH, new ComponentHashCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.COORDINATES, new CoordinatesCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.CWE, new CweCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.EXPRESSION, PolicyCondition::getValue); + SCRIPT_BUILDERS.put(Subject.LICENSE, new LicenseCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.LICENSE_GROUP, new LicenseGroupCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.PACKAGE_URL, new PackageUrlCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.SEVERITY, new SeverityCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.SWID_TAGID, new SwidTagIdCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.VULNERABILITY_ID, new VulnerabilityIdCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.VERSION, new VersionCelPolicyScriptSourceBuilder()); + } + + private final CelPolicyScriptHost scriptHost; + + public CelPolicyEngine() { + this(CelPolicyScriptHost.getInstance()); + } + + CelPolicyEngine(final CelPolicyScriptHost scriptHost) { + this.scriptHost = scriptHost; + } + + /** + * Evaluate {@link Policy}s for a {@link Project}. + * + * @param uuid The {@link UUID} of the {@link Project} + */ + public void evaluateProject(final UUID uuid) { + final Timer.Sample timerSample = Timer.start(); + + try (final var qm = new QueryManager(); + final var celQm = new CelPolicyQueryManager(qm)) { + // TODO: Should this entire procedure run in a single DB transaction? + // Would be better for atomicity, but could block DB connections for prolonged + // period of time for larger projects with many violations. + + final Project project = qm.getObjectByUuid(Project.class, uuid, List.of(Project.FetchGroup.IDENTIFIERS.name())); + if (project == null) { + LOGGER.warn("Project with UUID %s does not exist; Skipping".formatted(uuid)); + return; + } + + LOGGER.debug("Compiling policy scripts for project %s".formatted(uuid)); + final List> conditionScriptPairs = getApplicableConditionScriptPairs(celQm, project); + if (conditionScriptPairs.isEmpty()) { + LOGGER.info("No applicable policies found for project %s".formatted(uuid)); + celQm.reconcileViolations(project.getId(), emptyMultiValuedMap()); + return; + } + + final MultiValuedMap requirements = determineScriptRequirements(conditionScriptPairs); + LOGGER.debug("Requirements for project %s and %d policy conditions: %s" + .formatted(uuid, conditionScriptPairs.size(), requirements)); + + final org.dependencytrack.proto.policy.v1.Project protoProject; + if (requirements.containsKey(TYPE_PROJECT)) { + protoProject = mapToProto(celQm.fetchProject(project.getId(), requirements.get(TYPE_PROJECT), requirements.get(TYPE_PROJECT_PROPERTY))); + } else { + protoProject = org.dependencytrack.proto.policy.v1.Project.getDefaultInstance(); + } + + // Preload components for the entire project, to avoid excessive queries. + final List components = celQm.fetchAllComponents(project.getId(), requirements.get(TYPE_COMPONENT)); + + // Preload licenses for the entire project, as chances are high that they will be used by multiple components. + final Map licenseById; + if (requirements.containsKey(TYPE_LICENSE) || (requirements.containsKey(TYPE_COMPONENT) && requirements.get(TYPE_COMPONENT).contains("resolved_license"))) { + licenseById = celQm.fetchAllLicenses(project.getId(), requirements.get(TYPE_LICENSE), requirements.get(TYPE_LICENSE_GROUP)).stream() + .collect(Collectors.toMap( + projection -> projection.id, + CelPolicyEngine::mapToProto + )); + } else { + licenseById = Collections.emptyMap(); + } + + // Preload vulnerabilities for the entire project, as chances are high that they will be used by multiple components. + final Map protoVulnById; + final Map> vulnIdsByComponentId; + if (requirements.containsKey(TYPE_VULNERABILITY)) { + protoVulnById = celQm.fetchAllVulnerabilities(project.getId(), requirements.get(TYPE_VULNERABILITY)).stream() + .collect(Collectors.toMap( + projection -> projection.id, + CelPolicyEngine::mapToProto + )); + + vulnIdsByComponentId = celQm.fetchAllComponentsVulnerabilities(project.getId()).stream() + .collect(Collectors.groupingBy( + projection -> projection.componentId, + Collectors.mapping(projection -> projection.vulnerabilityId, Collectors.toList()) + )); + } else { + protoVulnById = Collections.emptyMap(); + vulnIdsByComponentId = Collections.emptyMap(); + } + + // Evaluate all policy conditions against all components. + final var conditionsViolated = new HashSetValuedHashMap(); + for (final ComponentProjection component : components) { + final org.dependencytrack.proto.policy.v1.Component protoComponent = mapToProto(component, licenseById); + final List protoVulns = + vulnIdsByComponentId.getOrDefault(component.id, emptyList()).stream() + .map(protoVulnById::get) + .toList(); + + conditionsViolated.putAll(component.id, evaluateConditions(conditionScriptPairs, Map.of( + VAR_COMPONENT, protoComponent, + VAR_PROJECT, protoProject, + VAR_VULNERABILITIES, protoVulns + ))); + } + + final var violationsByComponentId = new ArrayListValuedHashMap(); + for (final long componentId : conditionsViolated.keySet()) { + violationsByComponentId.putAll(componentId, evaluatePolicyOperators(conditionsViolated.get(componentId))); + } + + final List newViolationIds = celQm.reconcileViolations(project.getId(), violationsByComponentId); + LOGGER.info("Identified %d new violations for project %s".formatted(newViolationIds.size(), uuid)); + + for (final Long newViolationId : newViolationIds) { + NotificationUtil.analyzeNotificationCriteria(qm, newViolationId); + } + } finally { + final long durationNs = timerSample.stop(Timer + .builder("policy_evaluation") + .tag("target", "project") + .register(Metrics.getRegistry())); + LOGGER.info("Evaluation of project %s completed in %s" + .formatted(uuid, Duration.ofNanos(durationNs))); + } + } + + public void evaluateComponent(final UUID uuid) { + // Evaluation of individual components is only triggered when they are added or modified + // manually. As this happens very rarely, in low frequencies (due to being manual actions), + // and because CEL policy evaluation is so efficient, it's not worth it to maintain extra + // logic to handle component evaluation. Instead, re-purpose to project evaluation. + + final UUID projectUuid; + try (final var qm = new QueryManager(); + final var celQm = new CelPolicyQueryManager(qm)) { + projectUuid = celQm.getProjectUuidForComponentUuid(uuid); + } + + if (projectUuid == null) { + LOGGER.warn("Component with UUID %s does not exist; Skipping".formatted(uuid)); + return; + } + + evaluateProject(projectUuid); + } + + + /** + * Pre-compile the CEL scripts for all conditions of all applicable policies. + * Compiled scripts are cached in-memory by CelPolicyScriptHost, so if the same script + * is encountered for multiple components (possibly concurrently), the compilation is + * a one-time effort. + * + * @param celQm The {@link CelPolicyQueryManager} instance to use + * @param project The {@link Project} to get applicable conditions for + * @return {@link Pair}s of {@link PolicyCondition}s and {@link CelPolicyScript}s + */ + private List> getApplicableConditionScriptPairs(final CelPolicyQueryManager celQm, final Project project) { + final List policies = celQm.getApplicablePolicies(project); + if (policies.isEmpty()) { + return emptyList(); + } + + return policies.stream() + .map(Policy::getPolicyConditions) + .flatMap(Collection::stream) + .map(this::buildConditionScriptSrc) + .filter(Objects::nonNull) + .map(this::compileConditionScript) + .filter(Objects::nonNull) + .toList(); + } + + /** + * Check what kind of data we need to evaluate all policy conditions. + *

+ * Some conditions will be very simple and won't require us to load additional data (e.g. "component PURL matches 'XYZ'"), + * whereas other conditions can span across multiple models, forcing us to load more data + * (e.g. "project has tag 'public-facing' and component has a vulnerability with severity 'critical'"). + *

+ * What we want to avoid is loading data we don't need, and loading it multiple times. + * Instead, only load what's really needed, and only do so once. + * + * @param conditionScriptPairs {@link Pair}s of {@link PolicyCondition}s and corresponding {@link CelPolicyScript}s + * @return A {@link MultiValuedMap} containing all fields accessed on any {@link Type}, across all {@link CelPolicyScript}s + */ + private static MultiValuedMap determineScriptRequirements(final Collection> conditionScriptPairs) { + return conditionScriptPairs.stream() + .map(Pair::getRight) + .map(CelPolicyScript::getRequirements) + .reduce(new HashSetValuedHashMap<>(), (lhs, rhs) -> { + lhs.putAll(rhs); + return lhs; + }); + } + + private Pair buildConditionScriptSrc(final PolicyCondition policyCondition) { + final CelPolicyScriptSourceBuilder scriptBuilder = SCRIPT_BUILDERS.get(policyCondition.getSubject()); + if (scriptBuilder == null) { + LOGGER.warn(""" + No script builder found that is capable of handling subjects of type %s;\ + Condition will be skipped""".formatted(policyCondition.getSubject())); + return null; + } + + final String scriptSrc = scriptBuilder.apply(policyCondition); + if (scriptSrc == null) { + LOGGER.warn("Unable to create CEL script for condition %s; Condition will be skipped".formatted(policyCondition.getUuid())); + return null; + } + + return Pair.of(policyCondition, scriptSrc); + } + + private Pair compileConditionScript(final Pair conditionScriptSrcPair) { + final CelPolicyScript script; + try { + script = scriptHost.compile(conditionScriptSrcPair.getRight(), CacheMode.CACHE); + } catch (ScriptCreateException e) { + LOGGER.warn("Failed to compile script for condition %s; Condition will be skipped" + .formatted(conditionScriptSrcPair.getLeft().getUuid()), e); + return null; + } + + return Pair.of(conditionScriptSrcPair.getLeft(), script); + } + + private static List evaluateConditions(final Collection> conditionScriptPairs, + final Map scriptArguments) { + final var conditionsViolated = new ArrayList(); + + for (final Pair conditionScriptPair : conditionScriptPairs) { + final PolicyCondition condition = conditionScriptPair.getLeft(); + final CelPolicyScript script = conditionScriptPair.getRight(); + + try { + if (script.execute(scriptArguments)) { + conditionsViolated.add(condition); + } + } catch (ScriptException e) { + LOGGER.warn("Failed to execute script for condition %s with arguments %s" + .formatted(condition.getUuid(), scriptArguments), e); + } + } + + return conditionsViolated; + } + + private static List evaluatePolicyOperators(final Collection conditionsViolated) { + final Map> violatedConditionsByPolicy = conditionsViolated.stream() + .collect(Collectors.groupingBy(PolicyCondition::getPolicy)); + + return violatedConditionsByPolicy.entrySet().stream() + .flatMap(policyAndViolatedConditions -> { + final Policy policy = policyAndViolatedConditions.getKey(); + final List violatedConditions = policyAndViolatedConditions.getValue(); + + if ((policy.getOperator() == Policy.Operator.ANY && !violatedConditions.isEmpty()) + || (policy.getOperator() == Policy.Operator.ALL && violatedConditions.size() == policy.getPolicyConditions().size())) { + // TODO: Only a single violation should be raised, and instead multiple matched conditions + // should be associated with it. Keeping the existing behavior in order to avoid having to + // touch too much persistence and REST API code. + return violatedConditions.stream() + .map(condition -> { + final var violation = new PolicyViolation(); + violation.setType(condition.getViolationType()); + // Note: violation.setComponent is intentionally omitted here, + // because the component must be an object attached to the persistence + // context. We don't have that at this point, we'll add it later. + violation.setPolicyCondition(condition); + violation.setTimestamp(new Date()); + return violation; + }); + } + + return Stream.empty(); + }) + .filter(Objects::nonNull) + .toList(); + } + + private static org.dependencytrack.proto.policy.v1.Project mapToProto(final ProjectProjection projection) { + final org.dependencytrack.proto.policy.v1.Project.Builder builder = org.dependencytrack.proto.policy.v1.Project.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setGroup(trimToEmpty(projection.group)) + .setName(trimToEmpty(projection.name)) + .setVersion(trimToEmpty(projection.version)) + .setClassifier(trimToEmpty(projection.classifier)) + .setCpe(trimToEmpty(projection.cpe)) + .setPurl(trimToEmpty(projection.purl)) + .setSwidTagId(trimToEmpty(projection.swidTagId)); + Optional.ofNullable(projection.isActive).ifPresent(builder::setIsActive); + Optional.ofNullable(projection.lastBomImport).map(Timestamps::fromDate).ifPresent(builder::setLastBomImport); + + if (projection.propertiesJson != null) { + try { + final List properties = + OBJECT_MAPPER.readValue(projection.propertiesJson, new TypeReference<>() { + }); + for (final ProjectPropertyProjection property : properties) { + builder.addProperties(org.dependencytrack.proto.policy.v1.Project.Property.newBuilder() + .setGroup(trimToEmpty(property.group)) + .setName(trimToEmpty(property.name)) + .setValue(trimToEmpty(property.value)) + .setType(trimToEmpty(property.type)) + .build()); + } + } catch (JacksonException e) { + LOGGER.warn("Failed to parse properties from %s for project %s" + .formatted(projection.propertiesJson, projection.id), e); + } + } + + if (projection.tagsJson != null) { + try { + final List tags = OBJECT_MAPPER.readValue(projection.tagsJson, new TypeReference<>() { + }); + builder.addAllTags(tags); + } catch (JacksonException e) { + LOGGER.warn("Failed to parse tags from %s for project %s" + .formatted(projection.tagsJson, projection.id), e); + } + } + + return builder.build(); + } + + private static org.dependencytrack.proto.policy.v1.Component mapToProto(final ComponentProjection projection, + final Map protoLicenseById) { + final org.dependencytrack.proto.policy.v1.Component.Builder componentBuilder = + org.dependencytrack.proto.policy.v1.Component.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setGroup(trimToEmpty(projection.group)) + .setName(trimToEmpty(projection.name)) + .setVersion(trimToEmpty(projection.version)) + .setClassifier(trimToEmpty(projection.classifier)) + .setCpe(trimToEmpty(projection.cpe)) + .setPurl(trimToEmpty(projection.purl)) + .setSwidTagId(trimToEmpty(projection.swidTagId)) + .setIsInternal(Optional.ofNullable(projection.internal).orElse(false)) + .setLicenseName(trimToEmpty(projection.licenseName)) + .setMd5(trimToEmpty(projection.md5)) + .setSha1(trimToEmpty(projection.sha1)) + .setSha256(trimToEmpty(projection.sha256)) + .setSha384(trimToEmpty(projection.sha384)) + .setSha512(trimToEmpty(projection.sha512)) + .setSha3256(trimToEmpty(projection.sha3_256)) + .setSha3384(trimToEmpty(projection.sha3_384)) + .setSha3512(trimToEmpty(projection.sha3_512)) + .setBlake2B256(trimToEmpty(projection.blake2b_256)) + .setBlake2B384(trimToEmpty(projection.blake2b_384)) + .setBlake2B512(trimToEmpty(projection.blake2b_512)) + .setBlake3(trimToEmpty(projection.blake3)); + + if (projection.resolvedLicenseId != null && projection.resolvedLicenseId > 0) { + final org.dependencytrack.proto.policy.v1.License protoLicense = protoLicenseById.get(projection.resolvedLicenseId); + if (protoLicense != null) { + componentBuilder.setResolvedLicense(protoLicenseById.get(projection.resolvedLicenseId)); + } else { + LOGGER.warn("Component with ID %d refers to license with ID %d, but no license with that ID was found" + .formatted(projection.id, projection.resolvedLicenseId)); + } + } + + return componentBuilder.build(); + } + + private static org.dependencytrack.proto.policy.v1.License mapToProto(final LicenseProjection projection) { + final org.dependencytrack.proto.policy.v1.License.Builder licenseBuilder = + org.dependencytrack.proto.policy.v1.License.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setId(trimToEmpty(projection.licenseId)) + .setName(trimToEmpty(projection.name)); + Optional.ofNullable(projection.isOsiApproved).ifPresent(licenseBuilder::setIsOsiApproved); + Optional.ofNullable(projection.isFsfLibre).ifPresent(licenseBuilder::setIsFsfLibre); + Optional.ofNullable(projection.isDeprecatedId).ifPresent(licenseBuilder::setIsDeprecatedId); + Optional.ofNullable(projection.isCustomLicense).ifPresent(licenseBuilder::setIsCustom); + + if (projection.licenseGroupsJson != null) { + try { + final ArrayNode groupsArray = OBJECT_MAPPER.readValue(projection.licenseGroupsJson, ArrayNode.class); + for (final JsonNode groupNode : groupsArray) { + licenseBuilder.addGroups(org.dependencytrack.proto.policy.v1.License.Group.newBuilder() + .setUuid(Optional.ofNullable(groupNode.get("uuid")).map(JsonNode::asText).orElse("")) + .setName(Optional.ofNullable(groupNode.get("name")).map(JsonNode::asText).orElse("")) + .build()); + } + } catch (JacksonException e) { + LOGGER.warn("Failed to parse license groups from %s for license %s" + .formatted(projection.licenseGroupsJson, projection.id), e); + } + } + + return licenseBuilder.build(); + } + + private static final TypeReference> VULNERABILITY_ALIASES_TYPE_REF = new TypeReference<>() { + }; + + private static org.dependencytrack.proto.policy.v1.Vulnerability mapToProto(final VulnerabilityProjection projection) { + final org.dependencytrack.proto.policy.v1.Vulnerability.Builder builder = + org.dependencytrack.proto.policy.v1.Vulnerability.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setId(trimToEmpty(projection.vulnId)) + .setSource(trimToEmpty(projection.source)) + .setCvssv2Vector(trimToEmpty(projection.cvssV2Vector)) + .setCvssv3Vector(trimToEmpty(projection.cvssV3Vector)) + .setOwaspRrVector(trimToEmpty(projection.owaspRrVector)); + Optional.ofNullable(projection.cvssV2BaseScore).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2BaseScore); + Optional.ofNullable(projection.cvssV2ImpactSubScore).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2ImpactSubscore); + Optional.ofNullable(projection.cvssV2ExploitabilitySubScore).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2ExploitabilitySubscore); + Optional.ofNullable(projection.cvssV3BaseScore).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3BaseScore); + Optional.ofNullable(projection.cvssV3ImpactSubScore).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3ImpactSubscore); + Optional.ofNullable(projection.cvssV3ExploitabilitySubScore).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3ExploitabilitySubscore); + Optional.ofNullable(projection.owaspRrLikelihoodScore).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrLikelihoodScore); + Optional.ofNullable(projection.owaspRrTechnicalImpactScore).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrTechnicalImpactScore); + Optional.ofNullable(projection.owaspRrBusinessImpactScore).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrBusinessImpactScore); + Optional.ofNullable(projection.epssScore).map(BigDecimal::doubleValue).ifPresent(builder::setEpssScore); + Optional.ofNullable(projection.epssPercentile).map(BigDecimal::doubleValue).ifPresent(builder::setEpssPercentile); + Optional.ofNullable(projection.created).map(Timestamps::fromDate).ifPresent(builder::setCreated); + Optional.ofNullable(projection.published).map(Timestamps::fromDate).ifPresent(builder::setPublished); + Optional.ofNullable(projection.updated).map(Timestamps::fromDate).ifPresent(builder::setUpdated); + Optional.ofNullable(projection.cwes) + .map(StringUtils::trimToNull) + .filter(Objects::nonNull) + .map(new CollectionIntegerConverter()::convertToAttribute) + .ifPresent(builder::addAllCwes); + + // Workaround for https://github.com/DependencyTrack/dependency-track/issues/2474. + final Severity severity = VulnerabilityUtil.getSeverity(projection.severity, + projection.cvssV2BaseScore, + projection.cvssV3BaseScore, + projection.owaspRrLikelihoodScore, + projection.owaspRrTechnicalImpactScore, + projection.owaspRrBusinessImpactScore); + builder.setSeverity(severity.name()); + + if (projection.aliasesJson != null) { + try { + OBJECT_MAPPER.readValue(projection.aliasesJson, VULNERABILITY_ALIASES_TYPE_REF).stream() + .flatMap(CelPolicyEngine::mapToProto) + .distinct() + .forEach(builder::addAliases); + } catch (JacksonException e) { + LOGGER.warn("Failed to parse aliases from %s for vulnerability %d" + .formatted(projection.aliasesJson, projection.id), e); + } + } + + return builder.build(); + } + + private static Stream mapToProto(final VulnerabilityAlias alias) { + return alias.getAllBySource().entrySet().stream() + .map(aliasEntry -> Vulnerability.Alias.newBuilder() + .setSource(aliasEntry.getKey().name()) + .setId(aliasEntry.getValue()) + .build()); + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java new file mode 100644 index 000000000..acb2721e3 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -0,0 +1,378 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import com.google.api.expr.v1alpha1.Type; +import io.github.nscuro.versatile.Vers; +import io.github.nscuro.versatile.VersException; +import org.apache.commons.lang3.tuple.Pair; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.proto.policy.v1.Component; +import org.dependencytrack.proto.policy.v1.License; +import org.dependencytrack.proto.policy.v1.Project; +import org.dependencytrack.proto.policy.v1.Vulnerability; +import org.projectnessie.cel.EnvOption; +import org.projectnessie.cel.Library; +import org.projectnessie.cel.ProgramOption; +import org.projectnessie.cel.checker.Decls; +import org.projectnessie.cel.common.types.Err; +import org.projectnessie.cel.common.types.Types; +import org.projectnessie.cel.common.types.ref.Val; +import org.projectnessie.cel.interpreter.functions.Overload; + +import javax.jdo.Query; +import javax.jdo.datastore.JDOConnection; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +class CelPolicyLibrary implements Library { + + private static final Logger LOGGER = Logger.getLogger(CelPolicyLibrary.class); + + static final String VAR_COMPONENT = "component"; + static final String VAR_PROJECT = "project"; + static final String VAR_VULNERABILITIES = "vulns"; + + static final Type TYPE_COMPONENT = Decls.newObjectType(Component.getDescriptor().getFullName()); + static final Type TYPE_LICENSE = Decls.newObjectType(License.getDescriptor().getFullName()); + static final Type TYPE_LICENSE_GROUP = Decls.newObjectType(License.Group.getDescriptor().getFullName()); + static final Type TYPE_PROJECT = Decls.newObjectType(Project.getDescriptor().getFullName()); + static final Type TYPE_PROJECT_PROPERTY = Decls.newObjectType(Project.Property.getDescriptor().getFullName()); + static final Type TYPE_VULNERABILITY = Decls.newObjectType(Vulnerability.getDescriptor().getFullName()); + static final Type TYPE_VULNERABILITIES = Decls.newListType(TYPE_VULNERABILITY); + static final Type TYPE_VULNERABILITY_ALIAS = Decls.newObjectType(Vulnerability.Alias.getDescriptor().getFullName()); + + static final String FUNC_DEPENDS_ON = "depends_on"; + static final String FUNC_IS_DEPENDENCY_OF = "is_dependency_of"; + static final String FUNC_MATCHES_RANGE = "matches_range"; + + @Override + public List getCompileOptions() { + return List.of( + EnvOption.declarations( + Decls.newVar( + VAR_COMPONENT, + TYPE_COMPONENT + ), + Decls.newVar( + VAR_PROJECT, + TYPE_PROJECT + ), + Decls.newVar( + VAR_VULNERABILITIES, + TYPE_VULNERABILITIES + ), + Decls.newFunction( + FUNC_DEPENDS_ON, + // project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) + Decls.newInstanceOverload( + "project_depends_on_component_bool", + List.of(TYPE_PROJECT, TYPE_COMPONENT), + Decls.Bool + ) + ), + Decls.newFunction( + FUNC_IS_DEPENDENCY_OF, + // component.is_dependency_of(org.hyades.policy.v1.Component{name: "foo"}) + Decls.newInstanceOverload( + "component_is_dependency_of_component_bool", + List.of(TYPE_COMPONENT, TYPE_COMPONENT), + Decls.Bool + ) + ), + Decls.newFunction( + FUNC_MATCHES_RANGE, + // component.matches_range("vers:golang/>0|!=v3.2.1") + Decls.newInstanceOverload( + "component_matches_range_bool", + List.of(TYPE_COMPONENT, Decls.String), + Decls.Bool + ), + // project.matches_range("vers:golang/>0|!=v3.2.1") + Decls.newInstanceOverload( + "project_matches_range_bool", + List.of(TYPE_PROJECT, Decls.String), + Decls.Bool + ) + ) + ), + EnvOption.types( + Component.getDefaultInstance(), + License.getDefaultInstance(), + License.Group.getDefaultInstance(), + Project.getDefaultInstance(), + Project.Property.getDefaultInstance(), + Vulnerability.getDefaultInstance(), + Vulnerability.Alias.getDefaultInstance() + ) + ); + } + + @Override + public List getProgramOptions() { + return List.of( + ProgramOption.functions( + Overload.binary( + FUNC_DEPENDS_ON, + CelPolicyLibrary::dependsOnFunc + ), + Overload.binary( + FUNC_IS_DEPENDENCY_OF, + CelPolicyLibrary::isDependencyOfFunc + ), + Overload.binary( + FUNC_MATCHES_RANGE, + CelPolicyLibrary::matchesRangeFunc + ) + ) + ); + } + + private static Val dependsOnFunc(final Val lhs, final Val rhs) { + final Component leafComponent; + if (rhs.value() instanceof final Component rhsValue) { + leafComponent = rhsValue; + } else { + return Err.maybeNoSuchOverloadErr(rhs); + } + + if (lhs.value() instanceof final Project project) { + // project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) + return Types.boolOf(dependsOn(project, leafComponent)); + } + + return Err.maybeNoSuchOverloadErr(lhs); + } + + private static Val isDependencyOfFunc(final Val lhs, final Val rhs) { + final Component leafComponent; + if (lhs.value() instanceof final Component lhsValue) { + leafComponent = lhsValue; + } else { + return Err.maybeNoSuchOverloadErr(lhs); + } + + if (rhs.value() instanceof final Component rootComponent) { + return Types.boolOf(isDependencyOf(leafComponent, rootComponent)); + } + + return Err.maybeNoSuchOverloadErr(rhs); + } + + private static Val matchesRangeFunc(final Val lhs, final Val rhs) { + final String version; + if (lhs.value() instanceof final Component lhsValue) { + // component.matches_range("vers:golang/>0|!=v3.2.1") + version = lhsValue.getVersion(); + } else if (lhs.value() instanceof final Project lhsValue) { + // project.matches_range("vers:golang/>0|!=v3.2.1") + version = lhsValue.getVersion(); + } else { + return Err.maybeNoSuchOverloadErr(lhs); + } + + final String versStr; + if (rhs.value() instanceof final String rhsValue) { + versStr = rhsValue; + } else { + return Err.maybeNoSuchOverloadErr(rhs); + } + + return Types.boolOf(matchesRange(version, versStr)); + } + + private static boolean dependsOn(final Project project, final Component component) { + if (project.getUuid().isBlank()) { + // Need a UUID for our starting point. + LOGGER.warn("%s: project does not have a UUID; Unable to evaluate, returning false" + .formatted(FUNC_DEPENDS_ON)); + return false; + } + + final Pair> filterAndParams = toFilterAndParams(component); + if (filterAndParams == null) { + LOGGER.warn(""" + %s: Unable to construct filter expression from component %s; \ + Unable to evaluate, returning false""".formatted(FUNC_DEPENDS_ON, component)); + return false; + } + + final String filter = "project.uuid == :projectUuid && " + filterAndParams.getLeft(); + final Map params = filterAndParams.getRight(); + params.put("projectUuid", UUID.fromString(project.getUuid())); + + // TODO: Result can / should likely be cached based on filter and params. + + try (final var qm = new QueryManager()) { + final Query query = + qm.getPersistenceManager().newQuery(org.dependencytrack.model.Component.class); + query.setFilter(filter); + query.setNamedParameters(params); + query.setResult("count(this)"); + try { + return query.executeResultUnique(Long.class) > 0; + } finally { + query.closeAll(); + } + } + } + + private static boolean dependsOn(final Component rootComponent, final Component leafComponent) { + // TODO + return false; + } + + private static boolean isDependencyOf(final Component leafComponent, final Component rootComponent) { + if (leafComponent.getUuid().isBlank()) { + // Need a UUID for our starting point. + LOGGER.warn("%s: leaf component does not have a UUID; Unable to evaluate, returning false" + .formatted(FUNC_IS_DEPENDENCY_OF)); + return false; + } + + final var filters = new ArrayList(); + final var params = new HashMap(); + var paramPosition = 1; + if (!rootComponent.getUuid().isBlank()) { + filters.add("\"C\".\"UUID\" = ?"); + params.put(paramPosition++, rootComponent.getUuid()); + } + if (!rootComponent.getGroup().isBlank()) { + filters.add("\"C\".\"GROUP\" = ?"); + params.put(paramPosition++, rootComponent.getGroup()); + } + if (!rootComponent.getName().isBlank()) { + filters.add("\"C\".\"NAME\" = ?"); + params.put(paramPosition++, rootComponent.getName()); + } + if (!rootComponent.getVersion().isBlank()) { + filters.add("\"C\".\"VERSION\" = ?"); + params.put(paramPosition++, rootComponent.getVersion()); + } + if (!rootComponent.getPurl().isBlank()) { + filters.add("\"C\".\"PURL\" = ?"); + params.put(paramPosition++, rootComponent.getPurl()); + } + + if (filters.isEmpty()) { + LOGGER.warn(""" + %s: Unable to construct filter expression from root component %s; \ + Unable to evaluate, returning false""".formatted(FUNC_IS_DEPENDENCY_OF, rootComponent)); + return false; + } + + final String sqlFilter = String.join(" AND ", filters); + + final var query = """ + WITH RECURSIVE + "CTE_DEPENDENCIES" ("UUID", "PROJECT_ID", "FOUND", "COMPONENTS_SEEN") AS ( + SELECT + "C"."UUID", + "C"."PROJECT_ID", + CASE WHEN (%s) THEN TRUE ELSE FALSE END AS "FOUND", + ARRAY []::BIGINT[] AS "COMPONENTS_SEEN" + FROM + "COMPONENT" AS "C" + WHERE + -- TODO: Need to get project ID from somewhere to speed up + -- this initial query for the CTE. + -- "PROJECT_ID" = ? + "C"."DIRECT_DEPENDENCIES" IS NOT NULL + AND "C"."DIRECT_DEPENDENCIES" LIKE ? + UNION ALL + SELECT + "C"."UUID" AS "UUID", + "C"."PROJECT_ID" AS "PROJECT_ID", + CASE WHEN (%s) THEN TRUE ELSE FALSE END AS "FOUND", + ARRAY_APPEND("COMPONENTS_SEEN", "C"."ID") + FROM + "COMPONENT" AS "C" + INNER JOIN + "CTE_DEPENDENCIES" + ON "C"."PROJECT_ID" = "CTE_DEPENDENCIES"."PROJECT_ID" + AND "C"."DIRECT_DEPENDENCIES" LIKE ('%%"' || "CTE_DEPENDENCIES"."UUID" || '"%%') + WHERE + "C"."PROJECT_ID" = "CTE_DEPENDENCIES"."PROJECT_ID" + AND ( + "FOUND" OR "C"."DIRECT_DEPENDENCIES" IS NOT NULL + ) + ) + SELECT BOOL_OR("FOUND") FROM "CTE_DEPENDENCIES"; + """.formatted(sqlFilter, sqlFilter); + + try (final var qm = new QueryManager()) { + final JDOConnection jdoConnection = qm.getPersistenceManager().getDataStoreConnection(); + try { + final var connection = (Connection) jdoConnection.getNativeConnection(); + final var preparedStatement = connection.prepareStatement(query); + // Params need to be set twice because the rootComponent filter + // appears twice in the query... This needs improvement. + for (final Map.Entry param : params.entrySet()) { + preparedStatement.setObject(param.getKey(), param.getValue()); + } + preparedStatement.setString(params.size() + 1, "%" + leafComponent.getUuid() + "%"); + for (final Map.Entry param : params.entrySet()) { + preparedStatement.setObject((params.size() + 1) + param.getKey(), param.getValue()); + } + + try (final ResultSet rs = preparedStatement.executeQuery()) { + if (rs.next()) { + return rs.getBoolean(1); + } + } + } catch (SQLException e) { + LOGGER.warn("%s: Failed to execute query: %s".formatted(FUNC_IS_DEPENDENCY_OF, query), e); + } finally { + jdoConnection.close(); + } + } + + return false; + } + + private static boolean matchesRange(final String version, final String versStr) { + try { + return Vers.parse(versStr).contains(version); + } catch (VersException e) { + LOGGER.warn("%s: Failed to check if version %s matches range %s" + .formatted(FUNC_MATCHES_RANGE, version, versStr), e); + return false; + } + } + + private static Pair> toFilterAndParams(final Component component) { + var filters = new ArrayList(); + var params = new HashMap(); + + if (!component.getUuid().isBlank()) { + filters.add("uuid == :uuid"); + params.put("uuid", component.getUuid()); + } + if (!component.getGroup().isBlank()) { + filters.add("group == :group"); + params.put("group", component.getGroup()); + } + if (!component.getName().isBlank()) { + filters.add("name == :name"); + params.put("name", component.getName()); + } + if (!component.getVersion().isBlank()) { + filters.add("version"); + params.put("version", component.getVersion()); + } + + // TODO: Add more fields + + if (filters.isEmpty()) { + return null; + } + + return Pair.of(String.join(" && ", filters), params); + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java new file mode 100644 index 000000000..d3598e72f --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java @@ -0,0 +1,600 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.policy.cel.mapping.ComponentProjection; +import org.dependencytrack.policy.cel.mapping.ComponentsVulnerabilitiesProjection; +import org.dependencytrack.policy.cel.mapping.LicenseGroupProjection; +import org.dependencytrack.policy.cel.mapping.LicenseProjection; +import org.dependencytrack.policy.cel.mapping.PolicyViolationProjection; +import org.dependencytrack.policy.cel.mapping.ProjectProjection; +import org.dependencytrack.policy.cel.mapping.ProjectPropertyProjection; +import org.dependencytrack.policy.cel.mapping.VulnerabilityProjection; + +import javax.jdo.PersistenceManager; +import javax.jdo.Query; +import javax.jdo.datastore.JDOConnection; +import java.sql.Array; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.sql.Connection.TRANSACTION_READ_COMMITTED; +import static org.dependencytrack.policy.cel.mapping.FieldMappingUtil.getFieldMappings; + +class CelPolicyQueryManager implements AutoCloseable { + + private static final Logger LOGGER = Logger.getLogger(CelPolicyQueryManager.class); + + private final PersistenceManager pm; + + CelPolicyQueryManager(final QueryManager qm) { + this.pm = qm.getPersistenceManager(); + } + + UUID getProjectUuidForComponentUuid(final UUID componentUuid) { + try (final var qm = new QueryManager()) { + final Query query = qm.getPersistenceManager().newQuery(Component.class); + query.setFilter("uuid == :uuid"); + query.setParameters(componentUuid); + query.setResult("project.uuid"); + try { + return query.executeResultUnique(UUID.class); + } finally { + query.closeAll(); + } + } + } + + ProjectProjection fetchProject(final long projectId, + final Collection projectProtoFieldNames, + final Collection projectPropertyProtoFieldNames) { + // Determine the columns to select from the PROJECT (P) table. + String sqlProjectSelectColumns = Stream.concat( + Stream.of(ProjectProjection.ID_FIELD_MAPPING), + getFieldMappings(ProjectProjection.class).stream() + .filter(mapping -> projectProtoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"P\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + // Determine the columns to select from the PROJECT_PROPERTY (PP) table. + // The resulting expression will be used to populate JSON objects, using + // the JSONB_BUILD_OBJECT(name1, value1, name2, value2) notation. + String sqlPropertySelectColumns = ""; + if (projectPropertyProtoFieldNames != null) { + sqlPropertySelectColumns = getFieldMappings(ProjectPropertyProjection.class).stream() + .filter(mapping -> projectPropertyProtoFieldNames.contains(mapping.protoFieldName())) + .map(mapping -> "'%s', \"PP\".\"%s\"".formatted(mapping.javaFieldName(), mapping.sqlColumnName())) + .collect(Collectors.joining(", ")); + } + + // Properties will be selected into propertiesJson, tags into tagsJson. + // Both these fields are not part of the Project proto, thus their selection + // must be added manually. + if (!sqlPropertySelectColumns.isBlank()) { + sqlProjectSelectColumns += ", \"propertiesJson\""; + } + if (projectProtoFieldNames.contains("tags")) { + sqlProjectSelectColumns += ", \"tagsJson\""; + } + + final Query query = pm.newQuery(Query.SQL, """ + SELECT + %s + FROM + "PROJECT" AS "P" + LEFT JOIN LATERAL ( + SELECT + CAST(JSONB_AGG(DISTINCT JSONB_BUILD_OBJECT(%s)) AS TEXT) AS "propertiesJson" + FROM + "PROJECT_PROPERTY" AS "PP" + WHERE + "PP"."PROJECT_ID" = "P"."ID" + ) AS "properties" ON :shouldFetchProperties + LEFT JOIN LATERAL ( + SELECT + CAST(JSONB_AGG(DISTINCT "T"."NAME") AS TEXT) AS "tagsJson" + FROM + "TAG" AS "T" + INNER JOIN + "PROJECTS_TAGS" AS "PT" ON "PT"."TAG_ID" = "T"."ID" + WHERE + "PT"."PROJECT_ID" = "P"."ID" + ) AS "tags" ON :shouldFetchTags + WHERE + "ID" = :projectId + """.formatted(sqlProjectSelectColumns, sqlPropertySelectColumns)); + query.setNamedParameters(Map.of( + "shouldFetchProperties", !sqlPropertySelectColumns.isBlank(), + "shouldFetchTags", projectProtoFieldNames.contains("tags"), + "projectId", projectId + )); + try { + return query.executeResultUnique(ProjectProjection.class); + } finally { + query.closeAll(); + } + } + + List fetchAllComponents(final long projectId, final Collection protoFieldNames) { + final String sqlSelectColumns = Stream.concat( + Stream.of(ComponentProjection.ID_FIELD_MAPPING), + getFieldMappings(ComponentProjection.class).stream() + .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + final Query query = pm.newQuery(Query.SQL, """ + SELECT %s FROM "COMPONENT" WHERE "PROJECT_ID" = ? + """.formatted(sqlSelectColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(ComponentProjection.class)); + } finally { + query.closeAll(); + } + } + + /** + * Fetch all {@link org.dependencytrack.model.Component} {@code <->} {@link org.dependencytrack.model.Vulnerability} + * relationships for a given {@link Project}. + * + * @param projectId ID of the {@link Project} to fetch relationships for + * @return A {@link List} of {@link ComponentsVulnerabilitiesProjection} + */ + List fetchAllComponentsVulnerabilities(final long projectId) { + final Query query = pm.newQuery(Query.SQL, """ + SELECT + "CV"."COMPONENT_ID" AS "componentId", + "CV"."VULNERABILITY_ID" AS "vulnerabilityId" + FROM + "COMPONENTS_VULNERABILITIES" AS "CV" + INNER JOIN + "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" + WHERE + "C"."PROJECT_ID" = ? + """); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(ComponentsVulnerabilitiesProjection.class)); + } finally { + query.closeAll(); + } + } + + List fetchAllLicenses(final long projectId, + final Collection licenseProtoFieldNames, + final Collection licenseGroupProtoFieldNames) { + final String licenseSqlSelectColumns = Stream.concat( + Stream.of(LicenseProjection.ID_FIELD_MAPPING), + getFieldMappings(LicenseProjection.class).stream() + .filter(mapping -> licenseProtoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"L\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + // If fetching license groups is not necessary, we can just query for licenses and be done with it. + if (!licenseProtoFieldNames.contains("groups")) { + final Query query = pm.newQuery(Query.SQL, """ + SELECT DISTINCT + %s + FROM + "LICENSE" AS "L" + INNER JOIN + "COMPONENT" AS "C" ON "C"."LICENSE_ID" = "L"."ID" + WHERE + "C"."PROJECT_ID" = ? + """.formatted(licenseSqlSelectColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(LicenseProjection.class)); + } finally { + query.closeAll(); + } + } + + // If groups are required, include them in the license query in order to avoid the 1+N problem. + // Licenses may or may not be assigned to a group. Licenses can be in multiple groups. + // + // Using a simple LEFT JOIN would result in duplicate license data being fetched, e.g.: + // + // | "L"."ID" | "L"."NAME" | "LG"."NAME" | + // | :------- | :--------- | :---------- | + // | 1 | foo | groupA | + // | 1 | foo | groupB | + // | 1 | foo | groupC | + // | 2 | bar | NULL | + // + // To avoid this, we instead aggregate license group fields for each license, and return them as JSON. + // The reason for choosing JSON over native arrays, is that DataNucleus can't deal with arrays cleanly. + // + // | "L"."ID" | "L"."NAME" | "licenseGroupsJson" | + // | :------- | :--------- | :------------------------------------------------------ | + // | 1 | foo | [{"name":"groupA"},{"name":"groupB"},{"name":"groupC"}] | + // | 2 | bar | [] | + + final String licenseSqlGroupByColumns = Stream.concat( + Stream.of(LicenseProjection.ID_FIELD_MAPPING), + getFieldMappings(LicenseProjection.class).stream() + .filter(mapping -> licenseProtoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"L\".\"%s\"".formatted(mapping.sqlColumnName())) + .collect(Collectors.joining(", ")); + + final String licenseGroupSqlSelectColumns = getFieldMappings(LicenseGroupProjection.class).stream() + .filter(mapping -> licenseGroupProtoFieldNames.contains(mapping.protoFieldName())) + .map(mapping -> "'%s', \"LG\".\"%s\"".formatted(mapping.javaFieldName(), mapping.sqlColumnName())) + .collect(Collectors.joining(", ")); + + final Query query = pm.newQuery(Query.SQL, """ + SELECT DISTINCT + "L"."ID" AS "id", + %s, + CAST(JSONB_AGG(DISTINCT JSONB_BUILD_OBJECT(%s)) AS TEXT) AS "licenseGroupsJson" + FROM + "LICENSE" AS "L" + INNER JOIN + "COMPONENT" AS "C" ON "C"."LICENSE_ID" = "L"."ID" + LEFT JOIN + "LICENSEGROUP_LICENSE" AS "LGL" ON "LGL"."LICENSE_ID" = "L"."ID" + LEFT JOIN + "LICENSEGROUP" AS "LG" ON "LG"."ID" = "LGL"."LICENSEGROUP_ID" + WHERE + "C"."PROJECT_ID" = ? + GROUP BY + %s + """.formatted(licenseSqlSelectColumns, licenseGroupSqlSelectColumns, licenseSqlGroupByColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(LicenseProjection.class)); + } finally { + query.closeAll(); + } + } + + List fetchAllVulnerabilities(final long projectId, final Collection protoFieldNames) { + String sqlSelectColumns = Stream.concat( + Stream.of(VulnerabilityProjection.ID_FIELD_MAPPING), + getFieldMappings(VulnerabilityProjection.class).stream() + .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"V\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + if (protoFieldNames.contains("aliases")) { + sqlSelectColumns += ", \"aliasesJson\""; + } + + final Query query = pm.newQuery(Query.SQL, """ + SELECT DISTINCT + %s + FROM + "VULNERABILITY" AS "V" + INNER JOIN + "COMPONENTS_VULNERABILITIES" AS "CV" ON "CV"."VULNERABILITY_ID" = "V"."ID" + INNER JOIN + "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" + LEFT JOIN LATERAL ( + SELECT + CAST(JSONB_AGG(DISTINCT JSONB_STRIP_NULLS(JSONB_BUILD_OBJECT( + 'cveId', "VA"."CVE_ID", + 'ghsaId', "VA"."GHSA_ID", + 'gsdId', "VA"."GSD_ID", + 'internalId', "VA"."INTERNAL_ID", + 'osvId', "VA"."OSV_ID", + 'sonatypeId', "VA"."SONATYPE_ID", + 'snykId', "VA"."SNYK_ID", + 'vulnDbId', "VA"."VULNDB_ID" + ))) AS TEXT) AS "aliasesJson" + FROM + "VULNERABILITYALIAS" AS "VA" + WHERE + ("V"."SOURCE" = 'NVD' AND "VA"."CVE_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'GITHUB' AND "VA"."GHSA_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'GSD' AND "VA"."GSD_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'INTERNAL' AND "VA"."INTERNAL_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'OSV' AND "VA"."OSV_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'SONATYPE' AND "VA"."SONATYPE_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'SNYK' AND "VA"."SNYK_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'VULNDB' AND "VA"."VULNDB_ID" = "V"."VULNID") + ) AS "aliases" ON :shouldFetchAliases + WHERE + "C"."PROJECT_ID" = :projectId + """.formatted(sqlSelectColumns)); + query.setNamedParameters(Map.of( + "shouldFetchAliases", protoFieldNames.contains("aliases"), + "projectId", projectId + )); + try { + return List.copyOf(query.executeResultList(VulnerabilityProjection.class)); + } finally { + query.closeAll(); + } + } + + List reconcileViolations(final long projectId, final MultiValuedMap reportedViolationsByComponentId) { + // We want to send notifications for newly identified policy violations, + // so need to keep track of which violations we created. + final var newViolationIds = new ArrayList(); + + // DataNucleus does not support batch inserts, which is something we need in order to + // create new violations efficiently. Falling back to "raw" JDBC for the sake of efficiency. + final JDOConnection jdoConnection = pm.getDataStoreConnection(); + final var nativeConnection = (Connection) jdoConnection.getNativeConnection(); + Boolean originalAutoCommit = null; + Integer originalTrxIsolation = null; + + try { + // JDBC connections default to autocommit. + // We'll do multiple write operations here, and want to commit them all in a single transaction. + originalAutoCommit = nativeConnection.getAutoCommit(); + originalTrxIsolation = nativeConnection.getTransactionIsolation(); + nativeConnection.setAutoCommit(false); + nativeConnection.setTransactionIsolation(TRANSACTION_READ_COMMITTED); + + // First, query for all existing policy violations of the project, grouping them by component ID. + final var existingViolationsByComponentId = new HashSetValuedHashMap(); + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + SELECT + "ID" AS "id", + "COMPONENT_ID" AS "componentId", + "POLICYCONDITION_ID" AS "policyConditionId" + FROM + "POLICYVIOLATION" + WHERE + "PROJECT_ID" = ? + """)) { + ps.setLong(1, projectId); + + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + existingViolationsByComponentId.put( + rs.getLong("componentId"), + new PolicyViolationProjection( + rs.getLong("id"), + rs.getLong("policyConditionId") + )); + } + } + + // For each component that has existing and / or reported violations... + final Set componentIds = new HashSet<>(reportedViolationsByComponentId.keySet().size() + existingViolationsByComponentId.keySet().size()); + componentIds.addAll(reportedViolationsByComponentId.keySet()); + componentIds.addAll(existingViolationsByComponentId.keySet()); + + // ... determine which existing violations should be deleted (because they're no longer reported), + // and which reported violations should be created (because they have not been reported before). + // + // Violations not belonging to either of those buckets are reported, but already exist, + // meaning no action needs to be taken for them. + final var violationIdsToDelete = new ArrayList(); + final var violationsToCreate = new HashSetValuedHashMap(); + for (final Long componentId : componentIds) { + final Collection existingViolations = existingViolationsByComponentId.get(componentId); + final Collection reportedViolations = reportedViolationsByComponentId.get(componentId); + + if (reportedViolations == null || reportedViolations.isEmpty()) { + // Component has been removed, or does not have any violations anymore. + // All of its existing violations can be deleted. + violationIdsToDelete.addAll(existingViolations.stream().map(PolicyViolationProjection::id).toList()); + continue; + } + + if (existingViolations == null || existingViolations.isEmpty()) { + // Component did not have any violations before, but has some now. + // All reported violations must be newly created. + violationsToCreate.putAll(componentId, reportedViolations); + continue; + } + + // To determine which violations shall be deleted, find occurrences of violations appearing + // in the collection of existing violations, but not in the collection of reported violations. + existingViolations.stream() + .filter(existingViolation -> reportedViolations.stream().noneMatch(newViolation -> + newViolation.getPolicyCondition().getId() == existingViolation.policyConditionId())) + .map(PolicyViolationProjection::id) + .forEach(violationIdsToDelete::add); + + // To determine which violations shall be created, find occurrences of violations appearing + // in the collection of reported violations, but not in the collection of existing violations. + reportedViolations.stream() + .filter(reportedViolation -> existingViolations.stream().noneMatch(existingViolation -> + existingViolation.policyConditionId() == reportedViolation.getPolicyCondition().getId())) + .forEach(reportedViolation -> violationsToCreate.put(componentId, reportedViolation)); + } + + if (!violationsToCreate.isEmpty()) { + // For violations that need to be created, utilize batch inserts to limit database round-trips. + // Keep note of the IDs that were generated as part of the insert; For those we'll need to send + // notifications later. + + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + INSERT INTO "POLICYVIOLATION" + ("UUID", "TIMESTAMP", "COMPONENT_ID", "PROJECT_ID", "POLICYCONDITION_ID", "TYPE") + VALUES + (?, ?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + RETURNING "ID" + """, Statement.RETURN_GENERATED_KEYS)) { + for (final Map.Entry entry : violationsToCreate.entries()) { + ps.setString(1, UUID.randomUUID().toString()); + ps.setTimestamp(2, new Timestamp(entry.getValue().getTimestamp().getTime())); + ps.setLong(3, entry.getKey()); + ps.setLong(4, projectId); + ps.setLong(5, entry.getValue().getPolicyCondition().getId()); + ps.setString(6, entry.getValue().getType().name()); + ps.addBatch(); + } + ps.executeBatch(); + + final ResultSet rs = ps.getGeneratedKeys(); + while (rs.next()) { + newViolationIds.add(rs.getLong(1)); + } + } + } + + if (!violationIdsToDelete.isEmpty()) { + final Array violationIdsToDeleteArray = + nativeConnection.createArrayOf("BIGINT", violationIdsToDelete.toArray(new Long[0])); + + // First, bulk-delete any analyses attached to the violations first. + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + DELETE FROM + "VIOLATIONANALYSIS" + WHERE + "POLICYVIOLATION_ID" = ANY(?) + """)) { + ps.setArray(1, violationIdsToDeleteArray); + ps.execute(); + } + + // Finally, bulk-delete the actual violations. + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + DELETE FROM + "POLICYVIOLATION" + WHERE + "ID" = ANY(?) + """)) { + ps.setArray(1, violationIdsToDeleteArray); + ps.execute(); + } + } + + nativeConnection.commit(); + } catch (Exception e) { + try { + nativeConnection.rollback(); + } catch (SQLException ex) { + throw new RuntimeException(ex); + } + + throw new RuntimeException(e); + } finally { + try { + if (originalAutoCommit != null) { + nativeConnection.setAutoCommit(originalAutoCommit); + } + if (originalTrxIsolation != null) { + nativeConnection.setTransactionIsolation(originalTrxIsolation); + } + } catch (SQLException e) { + LOGGER.error("Failed to restore original connection settings (autoCommit=%s, trxIsolation=%d)" + .formatted(originalAutoCommit, originalTrxIsolation), e); + } + + jdoConnection.close(); + } + + return newViolationIds; + } + + List getApplicablePolicies(final Project project) { + var filter = """ + (this.projects.isEmpty() && this.tags.isEmpty()) + || (this.projects.contains(:project) + """; + var params = new HashMap(); + params.put("project", project); + + // To compensate for missing support for recursion of Common Table Expressions (CTEs) + // in JDO, we have to fetch the UUIDs of all parent projects upfront. Otherwise, we'll + // not be able to evaluate whether the policy is inherited from parent projects. + var variables = ""; + final List parentUuids = getParents(project); + if (!parentUuids.isEmpty()) { + filter += """ + || (this.includeChildren + && this.projects.contains(parentVar) + && :parentUuids.contains(parentVar.uuid)) + """; + variables += "org.dependencytrack.model.Project parentVar"; + params.put("parentUuids", parentUuids); + } + filter += ")"; + + // DataNucleus generates an invalid SQL query when using the idiomatic solution. + // The following works, but it's ugly and likely doesn't perform well if the project + // has many tags. Worth trying the idiomatic way again once DN has been updated to > 6.0.4. + // + // filter += " || (this.tags.contains(commonTag) && :project.tags.contains(commonTag))"; + // variables += "org.dependencytrack.model.Tag commonTag"; + if (project.getTags() != null && !project.getTags().isEmpty()) { + filter += " || ("; + for (int i = 0; i < project.getTags().size(); i++) { + filter += "this.tags.contains(:tag" + i + ")"; + params.put("tag" + i, project.getTags().get(i)); + if (i < (project.getTags().size() - 1)) { + filter += " || "; + } + } + filter += ")"; + } + + final List policies; + final Query query = pm.newQuery(Policy.class); + try { + query.setFilter(filter); + query.setNamedParameters(params); + if (!variables.isEmpty()) { + query.declareVariables(variables); + } + policies = List.copyOf(query.executeList()); + } finally { + query.closeAll(); + } + + return policies; + } + + List getParents(final Project project) { + return getParents(project.getUuid(), new ArrayList<>()); + } + + List getParents(final UUID uuid, final List parents) { + final UUID parentUuid; + final Query query = pm.newQuery(Project.class); + try { + query.setFilter("uuid == :uuid && parent != null"); + query.setParameters(uuid); + query.setResult("parent.uuid"); + parentUuid = query.executeResultUnique(UUID.class); + } finally { + query.closeAll(); + } + + if (parentUuid == null) { + return parents; + } + + parents.add(parentUuid); + return getParents(parentUuid, parents); + } + + @Override + public void close() { + // Noop + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java new file mode 100644 index 000000000..28ae2612b --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java @@ -0,0 +1,37 @@ +package org.dependencytrack.policy.cel; + +import com.google.api.expr.v1alpha1.Type; +import org.apache.commons.collections4.MultiValuedMap; +import org.projectnessie.cel.Program; +import org.projectnessie.cel.common.types.Err; +import org.projectnessie.cel.common.types.ref.Val; +import org.projectnessie.cel.tools.ScriptExecutionException; + +import java.util.Map; + +public class CelPolicyScript { + + private final Program program; + private final MultiValuedMap requirements; + + CelPolicyScript(final Program program, final MultiValuedMap requirements) { + this.program = program; + this.requirements = requirements; + } + + MultiValuedMap getRequirements() { + return requirements; + } + + boolean execute(final Map arguments) throws ScriptExecutionException { + final Val result = program.eval(arguments).getVal(); + + if (Err.isError(result)) { + final Err error = (Err) result; + throw new ScriptExecutionException(error.toString(), error.getCause()); + } + + return result.convertToNative(Boolean.class); + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java new file mode 100644 index 000000000..0721469cb --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -0,0 +1,177 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import alpine.server.cache.AbstractCacheManager; +import alpine.server.cache.CacheManager; +import com.google.api.expr.v1alpha1.CheckedExpr; +import com.google.api.expr.v1alpha1.Type; +import com.google.common.util.concurrent.Striped; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.collections4.MultiValuedMap; +import org.dependencytrack.policy.cel.CelPolicyScriptVisitor.FunctionSignature; +import org.projectnessie.cel.Ast; +import org.projectnessie.cel.CEL; +import org.projectnessie.cel.Env; +import org.projectnessie.cel.Env.AstIssuesTuple; +import org.projectnessie.cel.Library; +import org.projectnessie.cel.Program; +import org.projectnessie.cel.common.CELError; +import org.projectnessie.cel.common.Errors; +import org.projectnessie.cel.common.Location; +import org.projectnessie.cel.common.types.Err.ErrException; +import org.projectnessie.cel.common.types.pb.ProtoTypeRegistry; +import org.projectnessie.cel.extension.StringsLib; +import org.projectnessie.cel.tools.ScriptCreateException; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.locks.Lock; + +import static org.dependencytrack.policy.cel.CelPolicyLibrary.FUNC_DEPENDS_ON; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.FUNC_IS_DEPENDENCY_OF; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.FUNC_MATCHES_RANGE; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY; +import static org.projectnessie.cel.Issues.newIssues; +import static org.projectnessie.cel.common.Source.newTextSource; + +public class CelPolicyScriptHost { + + public enum CacheMode { + CACHE, + NO_CACHE + } + + private static final Logger LOGGER = Logger.getLogger(CelPolicyScriptHost.class); + private static CelPolicyScriptHost INSTANCE; + + private final Striped locks; + private final AbstractCacheManager cacheManager; + private final Env environment; + + CelPolicyScriptHost(final AbstractCacheManager cacheManager) { + this.locks = Striped.lock(128); + this.cacheManager = cacheManager; + this.environment = Env.newCustomEnv( + ProtoTypeRegistry.newRegistry(), + List.of( + Library.StdLib(), + Library.Lib(new StringsLib()), + Library.Lib(new CelPolicyLibrary()) + ) + ); + } + + public static synchronized CelPolicyScriptHost getInstance() { + if (INSTANCE == null) { + INSTANCE = new CelPolicyScriptHost(CacheManager.getInstance()); + } + + return INSTANCE; + } + + /** + * Compile, type-check, ana analyze a given CEL script. + * + * @param scriptSrc Source of the script to compile + * @param cacheMode Whether the {@link CelPolicyScript} shall be cached upon successful compilation + * @return The compiled {@link CelPolicyScript} + * @throws ScriptCreateException When compilation, type checking, or analysis failed + */ + public CelPolicyScript compile(final String scriptSrc, final CacheMode cacheMode) throws ScriptCreateException { + final String scriptDigest = DigestUtils.sha256Hex(scriptSrc); + + // Acquire a lock for the SHA256 digest of the script source. + // It is possible that compilation of the same script will be attempted multiple + // times concurrently. + final Lock lock = locks.get(scriptDigest); + lock.lock(); + + try { + CelPolicyScript script = cacheManager.get(CelPolicyScript.class, scriptDigest); + if (script != null) { + return script; + } + + LOGGER.debug("Compiling script: %s".formatted(scriptSrc)); + AstIssuesTuple astIssuesTuple = environment.parse(scriptSrc); + if (astIssuesTuple.hasIssues()) { + throw new ScriptCreateException("Failed to parse script", astIssuesTuple.getIssues()); + } + + try { + astIssuesTuple = environment.check(astIssuesTuple.getAst()); + } catch (ErrException e) { + // TODO: Bring error message in a more digestible form. + throw new ScriptCreateException("Failed to check script", newIssues(new Errors(newTextSource(scriptSrc)) + .append(Collections.singletonList( + new CELError(e, Location.newLocation(1, 1), e.getMessage()) + )) + )); + } + if (astIssuesTuple.hasIssues()) { + throw new ScriptCreateException("Failed to check script", astIssuesTuple.getIssues()); + } + + final Ast ast = astIssuesTuple.getAst(); + final Program program = environment.program(ast); + final MultiValuedMap requirements = analyzeRequirements(CEL.astToCheckedExpr(ast)); + + script = new CelPolicyScript(program, requirements); + if (cacheMode == CacheMode.CACHE) { + cacheManager.put(scriptDigest, script); + } + return script; + } finally { + lock.unlock(); + } + } + + private static MultiValuedMap analyzeRequirements(final CheckedExpr expr) { + final var visitor = new CelPolicyScriptVisitor(expr.getTypeMapMap()); + visitor.visit(expr.getExpr()); + + // Fields that are accessed directly are always a requirement. + final MultiValuedMap requirements = visitor.getAccessedFieldsByType(); + + // Special case for vulnerability severity: The "true" severity may or may not be persisted + // in the SEVERITY database column. To compute the actual severity, CVSSv2, CVSSv3, and OWASP RR + // scores may be required. See https://github.com/DependencyTrack/dependency-track/issues/2474 + if (requirements.containsKey(TYPE_VULNERABILITY) + && requirements.get(TYPE_VULNERABILITY).contains("severity")) { + requirements.putAll(TYPE_VULNERABILITY, List.of( + "cvssv2_base_score", + "cvssv3_base_score", + "owasp_rr_likelihood_score", + "owasp_rr_technical_impact_score", + "owasp_rr_business_impact_score" + )); + } + + // Custom functions may access certain fields implicitly, in a way that is not visible + // to the AST visitor. To compensate, we hardcode the functions' requirements here. + // TODO: This should be restructured to be more generic. + for (final FunctionSignature functionSignature : visitor.getUsedFunctionSignatures()) { + switch (functionSignature.function()) { + case FUNC_DEPENDS_ON, FUNC_IS_DEPENDENCY_OF -> { + if (TYPE_PROJECT.equals(functionSignature.targetType())) { + requirements.put(TYPE_PROJECT, "uuid"); + } else if (TYPE_COMPONENT.equals(functionSignature.targetType())) { + requirements.put(TYPE_COMPONENT, "uuid"); + } + } + case FUNC_MATCHES_RANGE -> { + if (TYPE_PROJECT.equals(functionSignature.targetType())) { + requirements.put(TYPE_PROJECT, "version"); + } else if (TYPE_COMPONENT.equals(functionSignature.targetType())) { + requirements.put(TYPE_COMPONENT, "version"); + } + } + } + } + + return requirements; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java new file mode 100644 index 000000000..2ae141947 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java @@ -0,0 +1,122 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import com.google.api.expr.v1alpha1.Expr; +import com.google.api.expr.v1alpha1.Type; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.HashSetValuedHashMap; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +class CelPolicyScriptVisitor { + + private static final Logger LOGGER = Logger.getLogger(CelPolicyScriptVisitor.class); + + record FunctionSignature(String function, Type targetType, List argumentTypes) { + } + + private final Map types; + private final MultiValuedMap accessedFieldsByType; + private final Set usedFunctionSignatures; + private final Deque callFunctionStack; + private final Deque selectFieldStack; + private final Deque selectOperandTypeStack; + + CelPolicyScriptVisitor(final Map types) { + this.types = types; + this.accessedFieldsByType = new HashSetValuedHashMap<>(); + this.usedFunctionSignatures = new HashSet<>(); + this.callFunctionStack = new ArrayDeque<>(); + this.selectFieldStack = new ArrayDeque<>(); + this.selectOperandTypeStack = new ArrayDeque<>(); + } + + void visit(final Expr expr) { + switch (expr.getExprKindCase()) { + case CALL_EXPR -> visitCall(expr); + case COMPREHENSION_EXPR -> visitComprehension(expr); + case CONST_EXPR -> visitConst(expr); + case IDENT_EXPR -> visitIdent(expr); + case LIST_EXPR -> visitList(expr); + case SELECT_EXPR -> visitSelect(expr); + case STRUCT_EXPR -> visitStruct(expr); + case EXPRKIND_NOT_SET -> LOGGER.debug("Unknown expression: %s".formatted(expr)); + } + } + + private void visitCall(final Expr expr) { + logExpr(expr); + final Expr.Call callExpr = expr.getCallExpr(); + + final Type targetType = types.get(callExpr.getTarget().getId()); + final List argumentTypes = callExpr.getArgsList().stream() + .map(Expr::getId) + .map(types::get) + .toList(); + usedFunctionSignatures.add(new FunctionSignature(callExpr.getFunction(), targetType, argumentTypes)); + + callFunctionStack.push(callExpr.getFunction()); + visit(callExpr.getTarget()); + for (final Expr argExpr : callExpr.getArgsList()) { + visit(argExpr); + } + callFunctionStack.pop(); + } + + private void visitComprehension(final Expr expr) { + logExpr(expr); + final Expr.Comprehension comprehensionExpr = expr.getComprehensionExpr(); + + visit(comprehensionExpr.getAccuInit()); + visit(comprehensionExpr.getIterRange()); + visit(comprehensionExpr.getLoopStep()); + visit(comprehensionExpr.getLoopCondition()); + visit(comprehensionExpr.getResult()); + } + + private void visitConst(final Expr expr) { + logExpr(expr); + } + + private void visitIdent(final Expr expr) { + logExpr(expr); + selectOperandTypeStack.push(types.get(expr.getId())); + } + + private void visitList(final Expr expr) { + logExpr(expr); + } + + private void visitSelect(final Expr expr) { + logExpr(expr); + final Expr.Select selectExpr = expr.getSelectExpr(); + + selectFieldStack.push(selectExpr.getField()); + selectOperandTypeStack.push(types.get(expr.getId())); + visit(selectExpr.getOperand()); + accessedFieldsByType.put(selectOperandTypeStack.pop(), selectFieldStack.pop()); + } + + private void visitStruct(final Expr expr) { + logExpr(expr); + } + + private void logExpr(final Expr expr) { + LOGGER.debug("Visiting %s (id=%d, fieldStack=%s, fieldTypeStack=%s, functionStack=%s)" + .formatted(expr.getExprKindCase(), expr.getId(), selectFieldStack, selectOperandTypeStack, callFunctionStack)); + } + + MultiValuedMap getAccessedFieldsByType() { + return this.accessedFieldsByType; + } + + Set getUsedFunctionSignatures() { + return this.usedFunctionSignatures; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..2d814f34f --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java @@ -0,0 +1,16 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import java.util.function.Function; +import java.util.regex.Pattern; + +public interface CelPolicyScriptSourceBuilder extends Function { + + Pattern QUOTES_PATTERN = Pattern.compile("\""); + + static String escapeQuotes(final String value) { + return QUOTES_PATTERN.matcher(value).replaceAll("\\\\\""); + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..34e9dde97 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java @@ -0,0 +1,45 @@ +package org.dependencytrack.policy.cel.compat; + +import alpine.common.logging.Logger; +import org.cyclonedx.model.Hash; +import org.dependencytrack.model.PolicyCondition; +import org.json.JSONObject; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class ComponentHashCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + private static final Logger LOGGER = Logger.getLogger(ComponentHashCelPolicyScriptSourceBuilder.class); + + @Override + public String apply(final PolicyCondition policyCondition) { + final Hash hash = extractHashValues(policyCondition); + if (hash.getAlgorithm() == null || hash.getValue() == null || hash.getAlgorithm().isEmpty() || hash.getValue().isEmpty()) { + return null; + } + + final String fieldName = hash.getAlgorithm().toLowerCase().replaceAll("-", "_"); + if (org.dependencytrack.proto.policy.v1.Component.getDescriptor().findFieldByName(fieldName) == null) { + LOGGER.warn("Component does not have a field named %s".formatted(fieldName)); + return null; + } + if (policyCondition.getOperator().equals(PolicyCondition.Operator.IS)) { + return """ + component.%s == "%s" + """.formatted(fieldName, escapeJson(hash.getValue())); + } else { + LOGGER.warn("Policy operator %s is not allowed with this policy".formatted(policyCondition.getOperator().toString())); + return null; + } + } + + private static Hash extractHashValues(PolicyCondition condition) { + //Policy condition received here will never be null + final JSONObject def = new JSONObject(condition.getValue()); + return new Hash( + def.optString("algorithm", null), + def.optString("value", null) + ); + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..721f1fe03 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java @@ -0,0 +1,92 @@ +package org.dependencytrack.policy.cel.compat; + +import alpine.common.logging.Logger; +import io.github.nscuro.versatile.Comparator; +import io.github.nscuro.versatile.Vers; +import io.github.nscuro.versatile.version.VersioningScheme; +import org.dependencytrack.model.PolicyCondition; +import org.json.JSONObject; + +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class CoordinatesCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + private static final Logger LOGGER = Logger.getLogger(CoordinatesCelPolicyScriptSourceBuilder.class); + private static final Pattern VERSION_OPERATOR_PATTERN = Pattern.compile("^(?[<>]=?|[!=]=)\\s*"); + + @Override + public String apply(final PolicyCondition condition) { + if (condition.getValue() == null) { + return null; + } + + final JSONObject def = new JSONObject(condition.getValue()); + final String group = Optional.ofNullable(def.optString("group", null)).orElse(""); + final String name = Optional.ofNullable(def.optString("name", null)).orElse(""); + final String version = Optional.ofNullable(def.optString("version")).orElse(""); + + final var scriptSrc = evaluateScript(group, name, version); + if (condition.getOperator() == PolicyCondition.Operator.MATCHES) { + return scriptSrc; + } else if (condition.getOperator() == PolicyCondition.Operator.NO_MATCH) { + return "!(%s)".formatted(scriptSrc); + } + + return null; + } + + private static String evaluateScript(final String conditionGroupPart, final String conditionNamePart, final String conditionVersionPart) { + final String group = replace(conditionGroupPart); + final String name = replace(conditionNamePart); + + final Matcher versionOperatorMatcher = VERSION_OPERATOR_PATTERN.matcher(conditionVersionPart); + //Do an exact match if no operator found + if (!versionOperatorMatcher.find()) { + + Vers conditionVers = Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.EQUAL, conditionVersionPart) + .build(); + return """ + component.group.matches("%s") && component.name.matches("%s") && component.matches_range("%s") + """.formatted(escapeJson(group), escapeJson(name), conditionVers.toString()); + } + + io.github.nscuro.versatile.Comparator versionComparator = switch (versionOperatorMatcher.group(1)) { + case "==" -> Comparator.EQUAL; + case "!=" -> Comparator.NOT_EQUAL; + case "<" -> Comparator.LESS_THAN; + case "<=" -> Comparator.LESS_THAN_OR_EQUAL; + case ">" -> Comparator.GREATER_THAN; + case ">=" -> Comparator.GREATER_THAN_OR_EQUAL; + default -> null; + }; + if (versionComparator == null) { + // Shouldn't ever happen because the regex won't match anything else + LOGGER.error("Failed to infer version operator from " + versionOperatorMatcher.group(1)); + return null; + } + String condition = VERSION_OPERATOR_PATTERN.split(conditionVersionPart)[1]; + Vers conditionVers = Vers.builder(VersioningScheme.GENERIC) + .withConstraint(versionComparator, condition) + .build(); + + return """ + component.group.matches("%s") && component.name.matches("%s") && component.matches_range("%s") + """.formatted(escapeJson(group), escapeJson(name), conditionVers.toString()); + } + + private static String replace(String conditionString) { + conditionString = conditionString.replace("*", ".*").replace("..*", ".*"); + if (!conditionString.startsWith("^") && !conditionString.startsWith(".*")) { + conditionString = ".*" + conditionString; + } + if (!conditionString.endsWith("$") && !conditionString.endsWith(".*")) { + conditionString += ".*"; + } + return conditionString; + } +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..23b673f87 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; + +public class CpeCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + component.cpe.matches("%s") + """.formatted(escapeQuotes(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.NO_MATCH) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..7e6e9e1a3 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java @@ -0,0 +1,54 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.parser.common.resolver.CweResolver; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class CweCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final List conditionCwes = Arrays.stream(policyCondition.getValue().split(",")) + .map(String::trim) + .map(CweResolver.getInstance()::parseCweString) + .filter(Objects::nonNull) + .sorted() + .toList(); + if (conditionCwes.isEmpty()) { + return null; + } + + final String celCweListLiteral = "[%s]".formatted(conditionCwes.stream() + .map(String::valueOf) + .collect(Collectors.joining(", "))); + + if (policyCondition.getOperator() == PolicyCondition.Operator.CONTAINS_ANY) { + // ANY of the vulnerabilities affecting the component have ANY of the + // CWEs defined in the policy condition assigned to them. + return """ + %s.exists(policyCwe, + vulns.exists(vuln, + vuln.cwes.exists(vulnCwe, vulnCwe == policyCwe) + ) + ) + """.formatted(celCweListLiteral); + } else if (policyCondition.getOperator() == PolicyCondition.Operator.CONTAINS_ALL) { + // ANY of the vulnerabilities affecting the component have ALL the + // CWEs defined in the policy condition assigned to them. + return """ + vulns.exists(vuln, + %s.all(policyCwe, + vuln.cwes.exists(vulnCwe, vulnCwe == policyCwe) + ) + ) + """.formatted(celCweListLiteral); + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..1927a7253 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java @@ -0,0 +1,55 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class LicenseCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + if ("unresolved".equals(policyCondition.getValue())) { + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return """ + !has(component.resolved_license) + """; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return """ + has(component.resolved_license) + """; + } + } else { + final String escapedLicenseUuid = escapeJson(policyCondition.getValue()); + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return """ + component.resolved_license.uuid == "%s" + """.formatted(escapedLicenseUuid); + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return """ + component.resolved_license.uuid != "%s" + """.formatted(escapedLicenseUuid); + } + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..87b8bf540 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class LicenseGroupCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + component.resolved_license.groups.exists(group, group.uuid == "%s") + """.formatted(escapeJson(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..45f9b0e92 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class PackageUrlCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + component.purl.matches("%s") + """.formatted(escapeJson(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.NO_MATCH) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..c10c0d1a5 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; + +public class SeverityCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return """ + vulns.exists(vuln, vuln.severity == "%s") + """.formatted(escapeQuotes(policyCondition.getValue())); + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return """ + vulns.exists(vuln, vuln.severity != "%s") + """.formatted(escapeQuotes(policyCondition.getValue())); + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..3053f0594 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; + +public class SwidTagIdCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + component.swid_tag_id.matches("%s") + """.formatted(escapeQuotes(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.NO_MATCH) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/VersionCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/VersionCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..8384cf53d --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/VersionCelPolicyScriptSourceBuilder.java @@ -0,0 +1,61 @@ +package org.dependencytrack.policy.cel.compat; + +import alpine.common.logging.Logger; +import io.github.nscuro.versatile.Comparator; +import io.github.nscuro.versatile.Vers; +import io.github.nscuro.versatile.VersException; +import io.github.nscuro.versatile.version.VersioningScheme; +import org.dependencytrack.model.PolicyCondition; + +public class VersionCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + private static final Logger LOGGER = Logger.getLogger(VersionCelPolicyScriptSourceBuilder.class); + + @Override + public String apply(PolicyCondition policyCondition) { + + Vers conditionVers = evaluateVers(policyCondition); + if (conditionVers == null) { + return null; + } + return """ + component.matches_range("%s") + """.formatted(conditionVers.toString()); + } + + private static Vers evaluateVers(final PolicyCondition policyCondition) { + try { + switch (policyCondition.getOperator()) { + case NUMERIC_EQUAL: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.EQUAL, policyCondition.getValue()) + .build(); + case NUMERIC_NOT_EQUAL: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.NOT_EQUAL, policyCondition.getValue()) + .build(); + case NUMERIC_LESS_THAN: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.LESS_THAN, policyCondition.getValue()) + .build(); + case NUMERIC_LESSER_THAN_OR_EQUAL: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.LESS_THAN_OR_EQUAL, policyCondition.getValue()) + .build(); + case NUMERIC_GREATER_THAN: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.GREATER_THAN, policyCondition.getValue()) + .build(); + case NUMERIC_GREATER_THAN_OR_EQUAL: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.GREATER_THAN_OR_EQUAL, policyCondition.getValue()) + .build(); + default: + LOGGER.warn("Unsupported operation " + policyCondition.getOperator()); + return null; + } + } catch (VersException versException) { + LOGGER.warn("Unable to parse version range in policy condition", versException); + return null; + } + } +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..43ac2f8af --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java @@ -0,0 +1,41 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; + +public class VulnerabilityIdCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + vulns.exists(v, v.id == "%s") + """.formatted(escapeQuotes(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentProjection.java new file mode 100644 index 000000000..36603b9ca --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentProjection.java @@ -0,0 +1,82 @@ +package org.dependencytrack.policy.cel.mapping; + +public class ComponentProjection { + + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", /* protoFieldName */ null, "ID"); + + public long id; + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(sqlColumnName = "GROUP") + public String group; + + @MappedField(sqlColumnName = "NAME") + public String name; + + @MappedField(sqlColumnName = "VERSION") + public String version; + + @MappedField(sqlColumnName = "CLASSIFIER") + public String classifier; + + @MappedField(sqlColumnName = "CPE") + public String cpe; + + @MappedField(sqlColumnName = "PURL") + public String purl; + + @MappedField(protoFieldName = "swid_tag_id", sqlColumnName = "SWIDTAGID") + public String swidTagId; + + @MappedField(protoFieldName = "is_internal", sqlColumnName = "INTERNAL") + public Boolean internal; + + @MappedField(sqlColumnName = "MD5") + public String md5; + + @MappedField(sqlColumnName = "SHA1") + public String sha1; + + @MappedField(sqlColumnName = "SHA_256") + public String sha256; + + @MappedField(sqlColumnName = "SHA_384") + public String sha384; + + @MappedField(sqlColumnName = "SHA_512") + public String sha512; + + @MappedField(sqlColumnName = "SHA3_256") + public String sha3_256; + + @MappedField(sqlColumnName = "SHA3_384") + public String sha3_384; + + @MappedField(sqlColumnName = "SHA3_512") + public String sha3_512; + + @MappedField(sqlColumnName = "BLAKE2B_256") + public String blake2b_256; + + @MappedField(sqlColumnName = "BLAKE2B_384") + public String blake2b_384; + + @MappedField(sqlColumnName = "BLAKE2B_512") + public String blake2b_512; + + @MappedField(sqlColumnName = "BLAKE3") + public String blake3; + + @MappedField(protoFieldName = "resolved_license", sqlColumnName = "LICENSE_ID") + public Long resolvedLicenseId; + + @MappedField(protoFieldName = "license_name", sqlColumnName = "LICENSE") + public String licenseName; + + // Requires https://github.com/DependencyTrack/dependency-track/pull/2400 to be ported to Hyades. + // @MappedField(protoFieldName = "license_expression", sqlColumnName = "LICENSE_EXPRESSION") + // public String licenseExpression; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentsVulnerabilitiesProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentsVulnerabilitiesProjection.java new file mode 100644 index 000000000..530bab03e --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentsVulnerabilitiesProjection.java @@ -0,0 +1,9 @@ +package org.dependencytrack.policy.cel.mapping; + +public class ComponentsVulnerabilitiesProjection { + + public Long componentId; + + public Long vulnerabilityId; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMapping.java b/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMapping.java new file mode 100644 index 000000000..b4eb6d3f7 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMapping.java @@ -0,0 +1,4 @@ +package org.dependencytrack.policy.cel.mapping; + +public record FieldMapping(String javaFieldName, String protoFieldName, String sqlColumnName) { +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtil.java b/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtil.java new file mode 100644 index 000000000..6efd2ffba --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtil.java @@ -0,0 +1,41 @@ +package org.dependencytrack.policy.cel.mapping; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.commons.lang3.StringUtils.trimToNull; + +public final class FieldMappingUtil { + + private static final Map, List> FIELD_MAPPINGS_BY_CLASS = new ConcurrentHashMap<>(); + + private FieldMappingUtil() { + } + + public static List getFieldMappings(final Class clazz) { + return FIELD_MAPPINGS_BY_CLASS.computeIfAbsent(clazz, FieldMappingUtil::createFieldMappings); + } + + private static List createFieldMappings(final Class clazz) { + final var fieldMappings = new ArrayList(); + + for (final Field field : clazz.getDeclaredFields()) { + final MappedField mappedFieldAnnotation = field.getAnnotation(MappedField.class); + if (mappedFieldAnnotation == null) { + continue; + } + + final String javaFieldName = field.getName(); + final String protoFieldName = Optional.ofNullable(trimToNull(mappedFieldAnnotation.protoFieldName())).orElse(javaFieldName); + final String sqlColumnName = Optional.ofNullable(trimToNull(mappedFieldAnnotation.sqlColumnName())).orElseThrow(); + fieldMappings.add(new FieldMapping(javaFieldName, protoFieldName, sqlColumnName)); + } + + return fieldMappings; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseGroupProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseGroupProjection.java new file mode 100644 index 000000000..d226be831 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseGroupProjection.java @@ -0,0 +1,11 @@ +package org.dependencytrack.policy.cel.mapping; + +public class LicenseGroupProjection { + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(sqlColumnName = "NAME") + public String name; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseProjection.java new file mode 100644 index 000000000..71e20ddc0 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseProjection.java @@ -0,0 +1,32 @@ +package org.dependencytrack.policy.cel.mapping; + +public class LicenseProjection { + + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", /* protoFieldName */ null, "ID"); + + public long id; + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(protoFieldName = "id", sqlColumnName = "LICENSEID") + public String licenseId; + + @MappedField(sqlColumnName = "NAME") + public String name; + + @MappedField(protoFieldName = "is_osi_approved", sqlColumnName = "ISOSIAPPROVED") + public Boolean isOsiApproved; + + @MappedField(protoFieldName = "is_fsf_libre", sqlColumnName = "FSFLIBRE") + public Boolean isFsfLibre; + + @MappedField(protoFieldName = "is_deprecated_id", sqlColumnName = "ISDEPRECATED") + public Boolean isDeprecatedId; + + @MappedField(protoFieldName = "is_custom", sqlColumnName = "ISCUSTOMLICENSE") + public Boolean isCustomLicense; + + public String licenseGroupsJson; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/MappedField.java b/src/main/java/org/dependencytrack/policy/cel/mapping/MappedField.java new file mode 100644 index 000000000..89f1a4b44 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/MappedField.java @@ -0,0 +1,28 @@ +package org.dependencytrack.policy.cel.mapping; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface MappedField { + + /** + * Name of the field in the Protobuf schema. + *

+ * If empty string (the default), the name of the annotated field will be assumed. + * + * @return Name of the Protobuf field + */ + String protoFieldName() default ""; + + /** + * Name of the SQL column corresponding to this field. + * + * @return Name of the SQL column + */ + String sqlColumnName(); + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/PolicyViolationProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/PolicyViolationProjection.java new file mode 100644 index 000000000..0a4c75ac7 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/PolicyViolationProjection.java @@ -0,0 +1,4 @@ +package org.dependencytrack.policy.cel.mapping; + +public record PolicyViolationProjection(Long id, Long policyConditionId) { +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java new file mode 100644 index 000000000..40f9ab980 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java @@ -0,0 +1,44 @@ +package org.dependencytrack.policy.cel.mapping; + +import java.util.Date; + +public class ProjectProjection { + + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", /* protoFieldName */ null, "ID"); + + public long id; + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(sqlColumnName = "GROUP") + public String group; + + @MappedField(sqlColumnName = "NAME") + public String name; + + @MappedField(sqlColumnName = "VERSION") + public String version; + + @MappedField(sqlColumnName = "CLASSIFIER") + public String classifier; + + @MappedField(protoFieldName = "is_active", sqlColumnName = "ACTIVE") + public Boolean isActive; + + @MappedField(sqlColumnName = "CPE") + public String cpe; + + @MappedField(sqlColumnName = "PURL") + public String purl; + + @MappedField(protoFieldName = "swid_tag_id", sqlColumnName = "SWIDTAGID") + public String swidTagId; + + @MappedField(protoFieldName = "last_bom_import", sqlColumnName = "LAST_BOM_IMPORTED") + public Date lastBomImport; + + public String propertiesJson; + public String tagsJson; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectPropertyProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectPropertyProjection.java new file mode 100644 index 000000000..522181f6d --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectPropertyProjection.java @@ -0,0 +1,17 @@ +package org.dependencytrack.policy.cel.mapping; + +public class ProjectPropertyProjection { + + @MappedField(sqlColumnName = "GROUPNAME") + public String group; + + @MappedField(sqlColumnName = "PROPERTYNAME") + public String name; + + @MappedField(sqlColumnName = "PROPERTYVALUE") + public String value; + + @MappedField(sqlColumnName = "PROPERTYTYPE") + public String type; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java new file mode 100644 index 000000000..5c50c7fbd --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java @@ -0,0 +1,80 @@ +package org.dependencytrack.policy.cel.mapping; + +import java.math.BigDecimal; +import java.util.Date; + +public class VulnerabilityProjection { + + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", /* protoFieldName */ null, "ID"); + + public long id; + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(protoFieldName = "id", sqlColumnName = "VULNID") + public String vulnId; + + @MappedField(sqlColumnName = "SOURCE") + public String source; + + @MappedField(sqlColumnName = "CWES") + public String cwes; + + @MappedField(sqlColumnName = "CREATED") + public Date created; + + @MappedField(sqlColumnName = "PUBLISHED") + public Date published; + + @MappedField(sqlColumnName = "UPDATED") + public Date updated; + + @MappedField(sqlColumnName = "SEVERITY") + public String severity; + + @MappedField(protoFieldName = "cvssv2_base_score", sqlColumnName = "CVSSV2BASESCORE") + public BigDecimal cvssV2BaseScore; + + @MappedField(protoFieldName = "cvssv2_impact_subscore", sqlColumnName = "CVSSV2IMPACTSCORE") + public BigDecimal cvssV2ImpactSubScore; + + @MappedField(protoFieldName = "cvssv2_exploitability_subscore", sqlColumnName = "CVSSV2EXPLOITSCORE") + public BigDecimal cvssV2ExploitabilitySubScore; + + @MappedField(protoFieldName = "cvssv2_vector", sqlColumnName = "CVSSV2VECTOR") + public String cvssV2Vector; + + @MappedField(protoFieldName = "cvssv3_base_score", sqlColumnName = "CVSSV3BASESCORE") + public BigDecimal cvssV3BaseScore; + + @MappedField(protoFieldName = "cvssv3_impact_subscore", sqlColumnName = "CVSSV3IMPACTSCORE") + public BigDecimal cvssV3ImpactSubScore; + + @MappedField(protoFieldName = "cvssv3_exploitability_subscore", sqlColumnName = "CVSSV3EXPLOITSCORE") + public BigDecimal cvssV3ExploitabilitySubScore; + + @MappedField(protoFieldName = "cvssv3_vector", sqlColumnName = "CVSSV3VECTOR") + public String cvssV3Vector; + + @MappedField(protoFieldName = "owasp_rr_likelihood_score", sqlColumnName = "OWASPRRLIKELIHOODSCORE") + public BigDecimal owaspRrLikelihoodScore; + + @MappedField(protoFieldName = "owasp_rr_technical_impact_score", sqlColumnName = "OWASPRRTECHNICALIMPACTSCORE") + public BigDecimal owaspRrTechnicalImpactScore; + + @MappedField(protoFieldName = "owasp_rr_business_impact_score", sqlColumnName = "OWASPRRBUSINESSIMPACTSCORE") + public BigDecimal owaspRrBusinessImpactScore; + + @MappedField(protoFieldName = "owasp_rr_vector", sqlColumnName = "OWASPRRVECTOR") + public String owaspRrVector; + + @MappedField(protoFieldName = "epss_score", sqlColumnName = "EPSSSCORE") + public BigDecimal epssScore; + + @MappedField(protoFieldName = "epss_percentile", sqlColumnName = "EPSSPERCENTILE") + public BigDecimal epssPercentile; + + public String aliasesJson; + +} diff --git a/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java b/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java index 70c96c237..5615b892c 100644 --- a/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java @@ -18,6 +18,7 @@ */ package org.dependencytrack.resources.v1; +import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.persistence.PaginatedResult; import alpine.server.auth.PermissionRequired; @@ -33,10 +34,12 @@ import io.swagger.annotations.ResponseHeader; import org.apache.commons.lang3.StringUtils; import org.dependencytrack.auth.Permissions; -import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent; import org.dependencytrack.event.InternalComponentIdentificationEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.componentmeta.ComponentProjection; +import org.dependencytrack.event.kafka.componentmeta.Handler; +import org.dependencytrack.event.kafka.componentmeta.HandlerFactory; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; import org.dependencytrack.model.License; @@ -74,6 +77,7 @@ @Api(value = "component", authorizations = @Authorization(value = "X-Api-Key")) public class ComponentResource extends AlpineResource { + private static final Logger LOGGER = Logger.getLogger(ComponentResource.class); private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); @GET @@ -276,7 +280,7 @@ public Response createComponent(@PathParam("uuid") String uuid, Component jsonCo if (project == null) { return Response.status(Response.Status.NOT_FOUND).entity("The project could not be found.").build(); } - if (! qm.hasAccess(super.getPrincipal(), project)) { + if (!qm.hasAccess(super.getPrincipal(), project)) { return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build(); } final License resolvedLicense = qm.getLicense(jsonComponent.getLicense()); @@ -316,7 +320,15 @@ public Response createComponent(@PathParam("uuid") String uuid, Component jsonCo component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes())); component = qm.createComponent(component, true); - kafkaEventDispatcher.dispatchBlocking(new ComponentRepositoryMetaAnalysisEvent(component)); + ComponentProjection componentProjection = + new ComponentProjection(component.getPurlCoordinates().toString(), + component.isInternal(), component.getPurl().toString()); + try { + Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, true); + repoMetaHandler.handle(); + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Unable to process package url %s".formatted(componentProjection.purl())); + } final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), component, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, true); qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, component.getUuid(), vulnAnalysisEvent.token().toString(), 1); kafkaEventDispatcher.dispatchBlocking(vulnAnalysisEvent); @@ -361,7 +373,7 @@ public Response updateComponent(Component jsonComponent) { try (QueryManager qm = new QueryManager()) { Component component = qm.getObjectByUuid(Component.class, jsonComponent.getUuid()); if (component != null) { - if (! qm.hasAccess(super.getPrincipal(), component.getProject())) { + if (!qm.hasAccess(super.getPrincipal(), component.getProject())) { return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified component is forbidden").build(); } // Name cannot be empty or null - prevent it @@ -369,6 +381,7 @@ public Response updateComponent(Component jsonComponent) { if (name != null) { component.setName(name); } + component.setPurlCoordinates(PurlUtil.silentPurlCoordinatesOnly(component.getPurl())); component.setAuthor(StringUtils.trimToNull(jsonComponent.getAuthor())); component.setPublisher(StringUtils.trimToNull(jsonComponent.getPublisher())); component.setVersion(StringUtils.trimToNull(jsonComponent.getVersion())); @@ -402,7 +415,16 @@ public Response updateComponent(Component jsonComponent) { component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes())); component = qm.updateComponent(component, true); - kafkaEventDispatcher.dispatchBlocking(new ComponentRepositoryMetaAnalysisEvent(component)); + ComponentProjection componentProjection = + new ComponentProjection(component.getPurlCoordinates().toString(), + component.isInternal(), component.getPurl().toString()); + try { + + Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, true); + repoMetaHandler.handle(); + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Unable to determine package url type for this purl %s".formatted(component.getPurl().getType()), ex); + } final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), component, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, false); qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, component.getUuid(), vulnAnalysisEvent.token().toString(), 1); kafkaEventDispatcher.dispatchBlocking(vulnAnalysisEvent); @@ -433,7 +455,7 @@ public Response deleteComponent( try (QueryManager qm = new QueryManager()) { final Component component = qm.getObjectByUuid(Component.class, uuid, Component.FetchGroup.ALL.name()); if (component != null) { - if (! qm.hasAccess(super.getPrincipal(), component.getProject())) { + if (!qm.hasAccess(super.getPrincipal(), component.getProject())) { return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified component is forbidden").build(); } qm.recursivelyDelete(component, false); diff --git a/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java b/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java index 6d30a684b..acd3b3eae 100644 --- a/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java @@ -32,8 +32,14 @@ import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.policy.cel.CelPolicyScriptHost; +import org.dependencytrack.policy.cel.CelPolicyScriptHost.CacheMode; +import org.dependencytrack.resources.v1.vo.CelExpressionError; +import org.projectnessie.cel.common.CELError; +import org.projectnessie.cel.tools.ScriptCreateException; import javax.validation.Validator; +import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.POST; @@ -43,6 +49,8 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.Map; /** * JAX-RS resources for processing policies. @@ -79,8 +87,10 @@ public Response createPolicyCondition( try (QueryManager qm = new QueryManager()) { Policy policy = qm.getObjectByUuid(Policy.class, uuid); if (policy != null) { + maybeValidateExpression(jsonPolicyCondition); final PolicyCondition pc = qm.createPolicyCondition(policy, jsonPolicyCondition.getSubject(), - jsonPolicyCondition.getOperator(), StringUtils.trimToNull(jsonPolicyCondition.getValue())); + jsonPolicyCondition.getOperator(), StringUtils.trimToNull(jsonPolicyCondition.getValue()), + jsonPolicyCondition.getViolationType()); return Response.status(Response.Status.CREATED).entity(pc).build(); } else { return Response.status(Response.Status.NOT_FOUND).entity("The UUID of the policy could not be found.").build(); @@ -110,6 +120,7 @@ public Response updatePolicyCondition(PolicyCondition jsonPolicyCondition) { try (QueryManager qm = new QueryManager()) { PolicyCondition pc = qm.getObjectByUuid(PolicyCondition.class, jsonPolicyCondition.getUuid()); if (pc != null) { + maybeValidateExpression(jsonPolicyCondition); pc = qm.updatePolicyCondition(jsonPolicyCondition); return Response.status(Response.Status.CREATED).entity(pc).build(); } else { @@ -144,4 +155,26 @@ public Response deletePolicyCondition( } } } + + private void maybeValidateExpression(final PolicyCondition policyCondition) { + if (policyCondition.getSubject() != PolicyCondition.Subject.EXPRESSION) { + return; + } + + if (policyCondition.getViolationType() == null) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity("Expression conditions must define a violation type").build()); + } + + try { + CelPolicyScriptHost.getInstance().compile(policyCondition.getValue(), CacheMode.NO_CACHE); + } catch (ScriptCreateException e) { + final var celErrors = new ArrayList(); + for (final CELError error : e.getIssues().getErrors()) { + celErrors.add(new CelExpressionError(error.getLocation().line(), error.getLocation().column(), error.getMessage())); + } + + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(Map.of("celErrors", celErrors)).build()); + } + } + } diff --git a/src/main/java/org/dependencytrack/resources/v1/vo/CelExpressionError.java b/src/main/java/org/dependencytrack/resources/v1/vo/CelExpressionError.java new file mode 100644 index 000000000..d7693fc95 --- /dev/null +++ b/src/main/java/org/dependencytrack/resources/v1/vo/CelExpressionError.java @@ -0,0 +1,4 @@ +package org.dependencytrack.resources.v1.vo; + +public record CelExpressionError(Integer line, Integer column, String message) { +} diff --git a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java index 07327424f..8d9a30892 100644 --- a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java +++ b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java @@ -40,9 +40,12 @@ import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent; import org.dependencytrack.event.ProjectMetricsUpdateEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.componentmeta.AbstractMetaHandler; import org.dependencytrack.model.Bom; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.License; import org.dependencytrack.model.Project; import org.dependencytrack.model.ServiceComponent; @@ -83,6 +86,8 @@ import static org.datanucleus.PropertyNames.PROPERTY_FLUSH_MODE; import static org.datanucleus.PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT; import static org.dependencytrack.common.ConfigKey.BOM_UPLOAD_PROCESSING_TRX_FLUSH_THRESHOLD; +import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK; +import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertComponents; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertServices; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertToProject; @@ -315,7 +320,14 @@ private void processBom(final Context ctx, final File bomFile) throws BomConsump // The constructors of ComponentRepositoryMetaAnalysisEvent and ComponentVulnerabilityAnalysisEvent // merely call a few getters on it, but the component object itself is not passed around. // Detaching would imply additional database interactions that we'd rather not do. - repoMetaAnalysisEvents.add(new ComponentRepositoryMetaAnalysisEvent(component)); + boolean result = SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(component.getPurl().getType()); + ComponentRepositoryMetaAnalysisEvent event; + if (result) { + event = collectRepoMetaAnalysisEvents(component, qm); + } else { + event = new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().toString(), component.isInternal(), false, true); + } + repoMetaAnalysisEvents.add(event); vulnAnalysisEvents.add(new ComponentVulnerabilityAnalysisEvent( ctx.uploadToken, component, VulnerabilityAnalysisLevel.BOM_UPLOAD_ANALYSIS, component.isNew())); } @@ -963,4 +975,20 @@ public String toString() { } + private ComponentRepositoryMetaAnalysisEvent collectRepoMetaAnalysisEvents(Component component, QueryManager qm) { + IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(component.getPurl().toString()); + if (integrityMetaComponent != null) { + if (integrityMetaComponent.getStatus() == null || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && (Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime()) > TIME_SPAN)) { + integrityMetaComponent.setLastFetch(Date.from(Instant.now())); + qm.updateIntegrityMetaComponent(integrityMetaComponent); + return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().toString(), component.isInternal(), true, true); + } else { + return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().toString(), component.isInternal(), false, true); + } + } else { + qm.createIntegrityMetaComponent(AbstractMetaHandler.createIntegrityMetaComponent(component.getPurl().toString())); + return new ComponentRepositoryMetaAnalysisEvent(component.getPurlCoordinates().toString(), component.isInternal(), true, true); + } + } + } diff --git a/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java b/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java index 03626b051..7bc48f038 100644 --- a/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java +++ b/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java @@ -1,21 +1,21 @@ package org.dependencytrack.tasks; +import alpine.Config; import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; +import org.dependencytrack.common.ConfigKey; import org.dependencytrack.event.ComponentPolicyEvaluationEvent; import org.dependencytrack.event.ProjectPolicyEvaluationEvent; import org.dependencytrack.model.Component; import org.dependencytrack.model.Project; import org.dependencytrack.model.WorkflowState; -import org.dependencytrack.model.WorkflowStatus; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.policy.PolicyEngine; +import org.dependencytrack.policy.cel.CelPolicyEngine; -import java.time.Instant; -import java.util.Date; +import java.util.UUID; -import static org.dependencytrack.model.WorkflowStep.METRICS_UPDATE; import static org.dependencytrack.model.WorkflowStep.POLICY_EVALUATION; /** @@ -27,6 +27,16 @@ public class PolicyEvaluationTask implements Subscriber { private static final Logger LOGGER = Logger.getLogger(PolicyEvaluationTask.class); + private final boolean celPolicyEngineEnabled; + + public PolicyEvaluationTask() { + this(Config.getInstance().getPropertyAsBoolean(ConfigKey.CEL_POLICY_ENGINE_ENABLED)); + } + + PolicyEvaluationTask(final boolean celPolicyEngineEnabled) { + this.celPolicyEngineEnabled = celPolicyEngineEnabled; + } + @Override public void inform(final Event e) { if (e instanceof final ProjectPolicyEvaluationEvent event) { @@ -34,7 +44,7 @@ public void inform(final Event e) { try (final var qm = new QueryManager()) { projectPolicyEvaluationState = qm.updateStartTimeIfWorkflowStateExists(event.getChainIdentifier(), POLICY_EVALUATION); try { - new PolicyEngine().evaluateProject(event.getUuid()); + evaluateProject(event.getUuid()); qm.updateWorkflowStateToComplete(projectPolicyEvaluationState); } catch (Exception ex) { qm.updateWorkflowStateToFailed(projectPolicyEvaluationState, ex.getMessage()); @@ -46,7 +56,7 @@ public void inform(final Event e) { try (final var qm = new QueryManager()) { componentMetricsEvaluationState = qm.updateStartTimeIfWorkflowStateExists(event.getChainIdentifier(), POLICY_EVALUATION); try { - new PolicyEngine().evaluate(event.getUuid()); + evaluateComponent(event.getUuid()); qm.updateWorkflowStateToComplete(componentMetricsEvaluationState); } catch (Exception ex) { qm.updateWorkflowStateToFailed(componentMetricsEvaluationState, ex.getMessage()); @@ -55,4 +65,21 @@ public void inform(final Event e) { } } } + + private void evaluateProject(final UUID uuid) { + if (celPolicyEngineEnabled) { + new CelPolicyEngine().evaluateProject(uuid); + } else { + new PolicyEngine().evaluateProject(uuid); + } + } + + private void evaluateComponent(final UUID uuid) { + if (celPolicyEngineEnabled) { + new CelPolicyEngine().evaluateComponent(uuid); + } else { + new PolicyEngine().evaluate(uuid); + } + } + } diff --git a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java index 987189af6..50ec0bc00 100644 --- a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java +++ b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java @@ -21,13 +21,16 @@ import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; +import com.github.packageurl.MalformedPackageURLException; import net.javacrumbs.shedlock.core.LockConfiguration; import net.javacrumbs.shedlock.core.LockExtender; import net.javacrumbs.shedlock.core.LockingTaskExecutor; -import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; import org.dependencytrack.event.PortfolioRepositoryMetaAnalysisEvent; import org.dependencytrack.event.ProjectRepositoryMetaAnalysisEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.componentmeta.ComponentProjection; +import org.dependencytrack.event.kafka.componentmeta.Handler; +import org.dependencytrack.event.kafka.componentmeta.HandlerFactory; import org.dependencytrack.model.Component; import org.dependencytrack.model.Project; import org.dependencytrack.persistence.QueryManager; @@ -73,7 +76,7 @@ public void inform(final Event e) { } } else if (e instanceof PortfolioRepositoryMetaAnalysisEvent) { try { - LockProvider.executeWithLock(PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, (LockingTaskExecutor.Task)() -> processPortfolio()); + LockProvider.executeWithLock(PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, (LockingTaskExecutor.Task) () -> processPortfolio()); } catch (Throwable ex) { LOGGER.error("An unexpected error occurred while submitting components for repository meta analysis", ex); } @@ -95,7 +98,8 @@ private void processProject(final UUID projectUuid) throws Exception { long offset = 0; List components = fetchNextComponentsPage(pm, project, offset); while (!components.isEmpty()) { - dispatchComponents(components); + //latest version information needs to be fetched for project as either triggered because of fresh bom upload or individual project reanalysis + dispatchComponents(components, qm); offset += components.size(); components = fetchNextComponentsPage(qm.getPersistenceManager(), project, offset); @@ -118,10 +122,11 @@ private void processPortfolio() throws Exception { List components = fetchNextComponentsPage(pm, null, offset); while (!components.isEmpty()) { long cumulativeProcessingTime = System.currentTimeMillis() - startTime; - if(isLockToBeExtended(cumulativeProcessingTime, PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK)) { + if (isLockToBeExtended(cumulativeProcessingTime, PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK)) { LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); } - dispatchComponents(components); + //latest version information does not need to be fetched for project as triggered for portfolio means it is a scheduled event happening + dispatchComponents(components, qm); offset += components.size(); components = fetchNextComponentsPage(qm.getPersistenceManager(), null, offset); @@ -131,9 +136,14 @@ private void processPortfolio() throws Exception { LOGGER.info("All components in portfolio submitted for repository meta analysis"); } - private void dispatchComponents(final List components) { + private void dispatchComponents(final List components, QueryManager queryManager) { for (final var component : components) { - kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(component.purlCoordinates(), component.internal())); + try { + Handler repoMetaHandler = HandlerFactory.createHandler(new ComponentProjection(component.purlCoordinates(), component.internal(), component.purl()), queryManager, kafkaEventDispatcher, true); + repoMetaHandler.handle(); + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Unable to determine package url type for this purl %s".formatted(component.purl()), ex); + } } } @@ -155,7 +165,4 @@ private List fetchNextComponentsPage(final PersistenceManag } } - public record ComponentProjection(String purlCoordinates, Boolean internal) { - } - } diff --git a/src/main/java/org/dependencytrack/upgrade/UpgradeItems.java b/src/main/java/org/dependencytrack/upgrade/UpgradeItems.java index f035f4601..c6d660bd3 100644 --- a/src/main/java/org/dependencytrack/upgrade/UpgradeItems.java +++ b/src/main/java/org/dependencytrack/upgrade/UpgradeItems.java @@ -19,6 +19,7 @@ package org.dependencytrack.upgrade; import alpine.server.upgrade.UpgradeItem; +import org.dependencytrack.upgrade.v510.v510Updater; import java.util.ArrayList; import java.util.List; @@ -27,6 +28,10 @@ class UpgradeItems { private static final List> UPGRADE_ITEMS = new ArrayList<>(); + static { + UPGRADE_ITEMS.add(v510Updater.class); + } + static List> getUpgradeItems() { return UPGRADE_ITEMS; } diff --git a/src/main/java/org/dependencytrack/upgrade/v510/v510Updater.java b/src/main/java/org/dependencytrack/upgrade/v510/v510Updater.java new file mode 100644 index 000000000..ba16ab9c6 --- /dev/null +++ b/src/main/java/org/dependencytrack/upgrade/v510/v510Updater.java @@ -0,0 +1,33 @@ +package org.dependencytrack.upgrade.v510; + +import alpine.common.logging.Logger; +import alpine.persistence.AlpineQueryManager; +import alpine.server.upgrade.AbstractUpgradeItem; + +import java.sql.Connection; +import java.sql.PreparedStatement; + +public class v510Updater extends AbstractUpgradeItem { + + private static final Logger LOGGER = Logger.getLogger(v510Updater.class); + + @Override + public String getSchemaVersion() { + return "5.1.0"; + } + + @Override + public void executeUpgrade(final AlpineQueryManager qm, final Connection connection) throws Exception { + changePolicyConditionValueTypeToText(connection); + } + + private static void changePolicyConditionValueTypeToText(final Connection connection) throws Exception { + LOGGER.info("Changing type of \"POLICYCONDITION\".\"VALUE\" from VARCHAR(255) to TEXT"); + try (final PreparedStatement ps = connection.prepareStatement(""" + ALTER TABLE "POLICYCONDITION" ALTER COLUMN "VALUE" TYPE TEXT; + """)) { + ps.execute(); + } + } + +} diff --git a/src/main/java/org/dependencytrack/util/NotificationUtil.java b/src/main/java/org/dependencytrack/util/NotificationUtil.java index 053979416..580c9a6db 100644 --- a/src/main/java/org/dependencytrack/util/NotificationUtil.java +++ b/src/main/java/org/dependencytrack/util/NotificationUtil.java @@ -22,14 +22,18 @@ import alpine.notification.Notification; import alpine.notification.NotificationLevel; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.dependencytrack.event.kafka.KafkaEventDispatcher; import org.dependencytrack.model.Analysis; import org.dependencytrack.model.Component; import org.dependencytrack.model.ConfigPropertyConstants; import org.dependencytrack.model.Finding; import org.dependencytrack.model.NotificationPublisher; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.model.PolicyViolation; import org.dependencytrack.model.Project; +import org.dependencytrack.model.Tag; import org.dependencytrack.model.ViolationAnalysis; import org.dependencytrack.model.ViolationAnalysisState; import org.dependencytrack.model.Vulnerability; @@ -48,14 +52,19 @@ import org.hyades.proto.notification.v1.ProjectVulnAnalysisStatus; import javax.jdo.FetchPlan; +import javax.jdo.Query; import java.io.File; import java.io.IOException; import java.net.URLDecoder; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -93,6 +102,8 @@ public static void dispatchNotificationsWithSubject(UUID projectUuid, Notificati public static void analyzeNotificationCriteria(final QueryManager qm, Analysis analysis, final boolean analysisStateChange, final boolean suppressionChange) { + // TODO: Convert data loading to raw SQL to avoid loading unneeded data and excessive queries. + // See #analyzeNotificationCriteria(QueryManager, PolicyViolation) for an example. if (analysisStateChange || suppressionChange) { final NotificationGroup notificationGroup; notificationGroup = NotificationGroup.PROJECT_AUDIT_CHANGE; @@ -150,6 +161,8 @@ public static void analyzeNotificationCriteria(final QueryManager qm, Analysis a public static void analyzeNotificationCriteria(final QueryManager qm, ViolationAnalysis violationAnalysis, final boolean analysisStateChange, final boolean suppressionChange) { + // TODO: Convert data loading to raw SQL to avoid loading unneeded data and excessive queries. + // See #analyzeNotificationCriteria(QueryManager, PolicyViolation) for an example. if (analysisStateChange || suppressionChange) { final NotificationGroup notificationGroup; notificationGroup = NotificationGroup.PROJECT_AUDIT_CHANGE; @@ -211,21 +224,132 @@ public static void analyzeNotificationCriteria(final QueryManager qm, ViolationA } public static void analyzeNotificationCriteria(final QueryManager qm, final PolicyViolation policyViolation) { - final ViolationAnalysis violationAnalysis = qm.getViolationAnalysis(policyViolation.getComponent(), policyViolation); - if (violationAnalysis != null && (violationAnalysis.isSuppressed() || ViolationAnalysisState.APPROVED == violationAnalysis.getAnalysisState())) + analyzeNotificationCriteria(qm, policyViolation.getId()); + } + + public static void analyzeNotificationCriteria(final QueryManager qm, final Long violationId) { + final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ + SELECT + "PV"."UUID" AS "violationUuid", + "PV"."TYPE" AS "violationType", + "PV"."TIMESTAMP" AS "violationTimestamp", + "PC"."UUID" AS "conditionUuid", + "PC"."SUBJECT" AS "conditionSubject", + "PC"."OPERATOR" AS "conditionOperator", + "PC"."VALUE" AS "conditionValue", + "P"."UUID" AS "policyUuid", + "P"."NAME" AS "policyName", + "P"."VIOLATIONSTATE" AS "policyViolationState", + "VA"."SUPPRESSED" AS "analysisSuppressed", + "VA"."STATE" AS "analysisState", + "C"."UUID" AS "componentUuid", + "C"."GROUP" AS "componentGroup", + "C"."NAME" AS "componentName", + "C"."VERSION" AS "componentVersion", + "C"."PURL" AS "componentPurl", + "C"."MD5" AS "componentMd5", + "C"."SHA1" AS "componentSha1", + "C"."SHA_256" AS "componentSha256", + "C"."SHA_512" AS "componentSha512", + "PR"."UUID" AS "projectUuid", + "PR"."NAME" AS "projectName", + "PR"."VERSION" AS "projectVersion", + "PR"."DESCRIPTION" AS "projectDescription", + "PR"."PURL" AS "projectPurl", + (SELECT + STRING_AGG("T"."NAME", ',') + FROM + "TAG" AS "T" + INNER JOIN + "PROJECTS_TAGS" AS "PT" ON "PT"."TAG_ID" = "T"."ID" + WHERE + "PT"."PROJECT_ID" = "PR"."ID" + ) AS "projectTags" + FROM + "POLICYVIOLATION" AS "PV" + INNER JOIN + "POLICYCONDITION" AS "PC" ON "PC"."ID" = "PV"."POLICYCONDITION_ID" + INNER JOIN + "POLICY" AS "P" ON "P"."ID" = "PC"."POLICY_ID" + INNER JOIN + "COMPONENT" AS "C" ON "C"."ID" = "PV"."COMPONENT_ID" + INNER JOIN + "PROJECT" AS "PR" ON "PR"."ID" = "PV"."PROJECT_ID" + LEFT JOIN + "VIOLATIONANALYSIS" AS "VA" ON "VA"."POLICYVIOLATION_ID" = "PV"."ID" + WHERE + "PV"."ID" = ? + """); + query.setParameters(violationId); + final PolicyViolationNotificationProjection projection; + try { + projection = query.executeResultUnique(PolicyViolationNotificationProjection.class); + } finally { + query.closeAll(); + } + + if (projection == null) { + return; + } + + if ((projection.analysisSuppressed != null && projection.analysisSuppressed) + || ViolationAnalysisState.APPROVED.name().equals(projection.analysisState)) { return; - policyViolation.getPolicyCondition().getPolicy(); // Force loading of policy - qm.getPersistenceManager().getFetchPlan().setMaxFetchDepth(2); // Ensure policy is included - qm.getPersistenceManager().getFetchPlan().setDetachmentOptions(FetchPlan.DETACH_LOAD_FIELDS); - final PolicyViolation pv = qm.getPersistenceManager().detachCopy(policyViolation); - Project project = policyViolation.getComponent().getProject(); + } + + final var project = new Project(); + project.setUuid(UUID.fromString(projection.projectUuid)); + project.setName(projection.projectName); + project.setVersion(projection.projectVersion); + project.setDescription(projection.projectDescription); + project.setPurl(projection.projectPurl); + project.setTags(Optional.ofNullable(projection.projectTags).stream() + .flatMap(tagNames -> Arrays.stream(tagNames.split(","))) + .map(StringUtils::trimToNull) + .filter(Objects::nonNull) + .map(tagName -> { + final var tag = new Tag(); + tag.setName(tagName); + return tag; + }) + .toList()); + + final var component = new Component(); + component.setUuid(UUID.fromString(projection.componentUuid)); + component.setGroup(projection.componentGroup); + component.setName(projection.componentName); + component.setVersion(projection.componentVersion); + component.setPurl(projection.componentPurl); + component.setMd5(projection.componentMd5); + component.setSha1(projection.componentSha1); + component.setSha256(projection.componentSha256); + component.setSha512(projection.componentSha512); + + final var policy = new Policy(); + policy.setUuid(UUID.fromString(projection.policyUuid)); + policy.setName(projection.policyName); + policy.setViolationState(Policy.ViolationState.valueOf(projection.policyViolationState)); + + final var policyCondition = new PolicyCondition(); + policyCondition.setPolicy(policy); + policyCondition.setUuid(UUID.fromString(projection.conditionUuid)); + policyCondition.setSubject(PolicyCondition.Subject.valueOf(projection.conditionSubject)); + policyCondition.setOperator(PolicyCondition.Operator.valueOf(projection.conditionOperator)); + policyCondition.setValue(projection.conditionValue); + + final var violation = new PolicyViolation(); + violation.setPolicyCondition(policyCondition); + violation.setUuid(UUID.fromString(projection.violationUuid)); + violation.setType(PolicyViolation.Type.valueOf(projection.violationType)); + violation.setTimestamp(projection.violationTimestamp); + sendNotificationToKafka(project.getUuid(), new Notification() .scope(NotificationScope.PORTFOLIO) .group(NotificationGroup.POLICY_VIOLATION) - .title(generateNotificationTitle(NotificationConstants.Title.POLICY_VIOLATION, policyViolation.getComponent().getProject())) + .title(generateNotificationTitle(NotificationConstants.Title.POLICY_VIOLATION, project)) .level(NotificationLevel.INFORMATIONAL) - .content(generateNotificationContent(pv)) - .subject(new PolicyViolationIdentified(pv, pv.getComponent(), pv.getProject())) + .content(generateNotificationContent(violation)) + .subject(new PolicyViolationIdentified(violation, component, project)) ); } @@ -312,6 +436,8 @@ private static void sendNotificationToKafka(UUID projectUuid, Notification notif } public static Notification createProjectVulnerabilityAnalysisCompleteNotification(VulnerabilityScan vulnScan, UUID token, ProjectVulnAnalysisStatus status) { + // TODO: Convert data loading to raw SQL to avoid loading unneeded data and excessive queries. + // See #analyzeNotificationCriteria(QueryManager, PolicyViolation) for an example. try (QueryManager qm = new QueryManager()) { Project project = qm.getObjectByUuid(Project.class, vulnScan.getTargetIdentifier()); if (project == null) { @@ -395,4 +521,35 @@ public static List createList(List com } return componentAnalysisCompleteList; } + + public static class PolicyViolationNotificationProjection { + public String projectUuid; + public String projectName; + public String projectVersion; + public String projectDescription; + public String projectPurl; + public String projectTags; + public String componentUuid; + public String componentGroup; + public String componentName; + public String componentVersion; + public String componentPurl; + public String componentMd5; + public String componentSha1; + public String componentSha256; + public String componentSha512; + public String violationUuid; + public String violationType; + public Date violationTimestamp; + public String conditionUuid; + public String conditionSubject; + public String conditionOperator; + public String conditionValue; + public String policyUuid; + public String policyName; + public String policyViolationState; + public Boolean analysisSuppressed; + public String analysisState; + } + } diff --git a/src/main/proto/buf.yaml b/src/main/proto/buf.yaml new file mode 100644 index 000000000..eb14abb5f --- /dev/null +++ b/src/main/proto/buf.yaml @@ -0,0 +1,5 @@ +version: v1 +name: github.com/DependencyTrack/hyades-apiserver +lint: + ignore: + - org/cyclonedx/v1_4/cyclonedx.proto diff --git a/src/main/proto/org/dependencytrack/policy/v1/policy.proto b/src/main/proto/org/dependencytrack/policy/v1/policy.proto new file mode 100644 index 000000000..71a861171 --- /dev/null +++ b/src/main/proto/org/dependencytrack/policy/v1/policy.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; + +package org.dependencytrack.policy.v1; + +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_package = "org.dependencytrack.proto.policy.v1"; + +message Component { + // UUID of the component. + string uuid = 1; + + // Group / namespace of the component. + optional string group = 2; + + // Name of the component. + string name = 3; + + // Version of the component. + string version = 4; + + // Classifier / type of the component. + // May be any of: + // - APPLICATION + // - CONTAINER + // - DEVICE + // - FILE + // - FIRMWARE + // - FRAMEWORK + // - LIBRARY + // - OPERATING_SYSTEM + optional string classifier = 5; + + // CPE of the component. + // https://csrc.nist.gov/projects/security-content-automation-protocol/specifications/cpe + optional string cpe = 6; + + // Package URL of the component. + // https://github.com/package-url/purl-spec + optional string purl = 7; + + // SWID tag ID of the component. + // https://csrc.nist.gov/projects/Software-Identification-SWID + optional string swid_tag_id = 8; + + // Whether the component is internal to the organization. + optional bool is_internal = 9; + + optional string md5 = 20; + optional string sha1 = 21; + optional string sha256 = 22; + optional string sha384 = 23; + optional string sha512 = 24; + optional string sha3_256 = 25; + optional string sha3_384 = 26; + optional string sha3_512 = 27; + optional string blake2b_256 = 28; + optional string blake2b_384 = 29; + optional string blake2b_512 = 30; + optional string blake3 = 31; + + optional string license_name = 50; + optional string license_expression = 51; + optional License resolved_license = 52; +} + +message License { + string uuid = 1; + string id = 2; + string name = 3; + repeated Group groups = 4; + bool is_osi_approved = 5; + bool is_fsf_libre = 6; + bool is_deprecated_id = 7; + bool is_custom = 8; + + message Group { + string uuid = 1; + string name = 2; + } +} + +message Project { + string uuid = 1; + optional string group = 2; + string name = 3; + optional string version = 4; + optional string classifier = 5; + bool is_active = 6; + repeated string tags = 7; + repeated Property properties = 8; + optional string cpe = 9; + optional string purl = 10; + optional string swid_tag_id = 11; + optional google.protobuf.Timestamp last_bom_import = 12; + + message Property { + string group = 1; + string name = 2; + optional string value = 3; + string type = 4; + } +} + +message Vulnerability { + string uuid = 1; + string id = 2; + string source = 3; + repeated Alias aliases = 4; + repeated int32 cwes = 5; + optional google.protobuf.Timestamp created = 6; + optional google.protobuf.Timestamp published = 7; + optional google.protobuf.Timestamp updated = 8; + string severity = 20; + optional double cvssv2_base_score = 21; + optional double cvssv2_impact_subscore = 22; + optional double cvssv2_exploitability_subscore = 23; + optional string cvssv2_vector = 24; + optional double cvssv3_base_score = 25; + optional double cvssv3_impact_subscore = 26; + optional double cvssv3_exploitability_subscore = 27; + optional string cvssv3_vector = 28; + optional double owasp_rr_likelihood_score = 29; + optional double owasp_rr_technical_impact_score = 30; + optional double owasp_rr_business_impact_score = 31; + optional string owasp_rr_vector = 32; + optional double epss_score = 33; + optional double epss_percentile = 34; + + message Alias { + string id = 1; + string source = 2; + } +} diff --git a/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto b/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto index a58d96c27..07dab17a1 100644 --- a/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto +++ b/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto @@ -11,6 +11,9 @@ option java_package = "org.hyades.proto.repometaanalysis.v1"; message AnalysisCommand { // The component that shall be analyzed. Component component = 1; + bool fetch_integrity_data = 2; + bool fetch_latest_version = 3; + } message AnalysisResult { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index a24091023..bc636af72 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -513,3 +513,7 @@ workflow.retention.duration=P3D # This is specifically for cases where polling the /api/v1/bom/token/ endpoint is not feasible. # THIS IS A TEMPORARY FUNCTIONALITY AND MAY BE REMOVED IN FUTURE RELEASES WITHOUT FURTHER NOTICE. tmp.delay.bom.processed.notification=false + +# Optional +# Specifies whether the Common Expression Language (CEL) based policy engine shall be enabled. +cel.policy.engine.enabled=false \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/TestUtil.java b/src/test/java/org/dependencytrack/TestUtil.java index 8e1559e1f..3966bef81 100644 --- a/src/test/java/org/dependencytrack/TestUtil.java +++ b/src/test/java/org/dependencytrack/TestUtil.java @@ -8,6 +8,7 @@ public class TestUtil { public static Properties getDatanucleusProperties(String jdbcUrl, String driverName, String username, String pwd) { final var dnProps = new Properties(); + dnProps.put(PropertyNames.PROPERTY_PERSISTENCE_UNIT_NAME, "Alpine"); dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_DATABASE, "true"); dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_TABLES, "true"); dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_COLUMNS, "true"); diff --git a/src/test/java/org/dependencytrack/persistence/ComponentQueryManagerTest.java b/src/test/java/org/dependencytrack/persistence/ComponentQueryManagerTest.java index d541b4783..de2063503 100644 --- a/src/test/java/org/dependencytrack/persistence/ComponentQueryManagerTest.java +++ b/src/test/java/org/dependencytrack/persistence/ComponentQueryManagerTest.java @@ -20,6 +20,8 @@ import org.junit.Test; import javax.jdo.JDOObjectNotFoundException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Date; import static org.assertj.core.api.Assertions.assertThat; @@ -112,7 +114,8 @@ public void recursivelyDeleteTest() { public void testGetIntegrityMetaComponent() { var integrityMeta = new IntegrityMetaComponent(); integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.TIMED_OUT); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); var result = qm.getIntegrityMetaComponent("pkg:maven/acme/example@1.0.0?type=jar"); assertThat(result).isNull(); @@ -120,12 +123,12 @@ public void testGetIntegrityMetaComponent() { result = qm.persist(integrityMeta); assertThat(qm.getIntegrityMetaComponent(result.getPurl())).satisfies( meta -> { - assertThat(meta.getStatus()).isEqualTo(FetchStatus.TIMED_OUT); + assertThat(meta.getStatus()).isEqualTo(FetchStatus.IN_PROGRESS); assertThat(meta.getId()).isEqualTo(1L); assertThat(meta.getMd5()).isNull(); assertThat(meta.getSha1()).isNull(); assertThat(meta.getSha256()).isNull(); - assertThat(meta.getLastFetch()).isNull(); + assertThat(meta.getLastFetch()).isEqualTo(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); assertThat(meta.getPublishedAt()).isNull(); } ); @@ -135,7 +138,8 @@ public void testGetIntegrityMetaComponent() { public void testUpdateIntegrityMetaComponent() { var integrityMeta = new IntegrityMetaComponent(); integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.TIMED_OUT); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); var result = qm.updateIntegrityMetaComponent(integrityMeta); assertThat(result).isNull(); diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java new file mode 100644 index 000000000..a08d60447 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -0,0 +1,901 @@ +package org.dependencytrack.policy.cel; + +import alpine.model.IConfigProperty; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.BomUploadEvent; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Classifier; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.ComponentIdentity; +import org.dependencytrack.model.ConfigPropertyConstants; +import org.dependencytrack.model.License; +import org.dependencytrack.model.LicenseGroup; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Severity; +import org.dependencytrack.model.Tag; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.model.VulnerabilityAlias; +import org.dependencytrack.persistence.DefaultObjectGenerator; +import org.dependencytrack.tasks.BomUploadProcessingTask; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; +import java.math.BigDecimal; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.apache.commons.io.IOUtils.resourceToURL; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; + +public class CelPolicyEngineTest extends AbstractPostgresEnabledTest { + + @Before + public void setUp() throws Exception { + super.setUp(); + + // Enable processing of CycloneDX BOMs + qm.createConfigProperty(ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getGroupName(), + ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getPropertyName(), "true", + ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getPropertyType(), + ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getDescription()); + } + + /** + * (Regression-)Test for ensuring that all data available in the policy expression context + * can be accessed in the expression at runtime. + *

+ * Data being available means: + *

    + *
  • Expression requirements were analyzed correctly
    • + *
  • Database was retrieved from the database correctly
    • + *
  • The mapping from DB data to CEL Protobuf models worked as expected
    • + *
+ */ + @Test + public void testEvaluateProjectWithAllFields() { + final var project = new Project(); + project.setUuid(UUID.fromString("d7173786-60aa-4a4f-a950-c92fe6422307")); + project.setGroup("projectGroup"); + project.setName("projectName"); + project.setVersion("projectVersion"); + project.setClassifier(Classifier.APPLICATION); + project.setActive(true); + project.setCpe("projectCpe"); + project.setPurl("projectPurl"); + project.setSwidTagId("projectSwidTagId"); + project.setLastBomImport(new java.util.Date()); + qm.persist(project); + + qm.createProjectProperty(project, "propertyGroup", "propertyName", "propertyValue", IConfigProperty.PropertyType.STRING, null); + + qm.bind(project, List.of( + qm.createTag("projectTagA"), + qm.createTag("projectTagB") + )); + + final var licenseGroup = new LicenseGroup(); + licenseGroup.setUuid(UUID.fromString("bbdb62f8-d854-4e43-a9ed-36481545c201")); + licenseGroup.setName("licenseGroupName"); + qm.persist(licenseGroup); + + final var license = new License(); + license.setUuid(UUID.fromString("dc9876c2-0adc-422b-9f71-3ca78285f138")); + license.setLicenseId("resolvedLicenseId"); + license.setName("resolvedLicenseName"); + license.setOsiApproved(true); + license.setFsfLibre(true); + license.setDeprecatedLicenseId(true); + license.setCustomLicense(true); + license.setLicenseGroups(List.of(licenseGroup)); + qm.persist(license); + + final var component = new Component(); + component.setProject(project); + component.setUuid(UUID.fromString("7e5f6465-d2f2-424f-b1a4-68d186fa2b46")); + component.setGroup("componentGroup"); + component.setName("componentName"); + component.setVersion("componentVersion"); + component.setClassifier(Classifier.LIBRARY); + component.setCpe("componentCpe"); + component.setPurl("componentPurl"); + component.setSwidTagId("componentSwidTagId"); + component.setInternal(true); + component.setMd5("componentMd5"); + component.setSha1("componentSha1"); + component.setSha256("componentSha256"); + component.setSha384("componentSha384"); + component.setSha512("componentSha512"); + component.setSha3_256("componentSha3_256"); + component.setSha3_384("componentSha3_384"); + component.setSha3_512("componentSha3_512"); + component.setBlake2b_256("componentBlake2b_256"); + component.setBlake2b_384("componentBlake2b_384"); + component.setBlake2b_512("componentBlake2b_512"); + component.setBlake3("componentBlake3"); + component.setLicense("componentLicenseName"); + component.setResolvedLicense(license); + qm.persist(component); + + final var vuln = new Vulnerability(); + vuln.setUuid(UUID.fromString("ffe9743f-b916-431e-8a68-9b3ac56db72c")); + vuln.setVulnId("CVE-001"); + vuln.setSource(Vulnerability.Source.NVD); + vuln.setCwes(List.of(666, 777)); + vuln.setCreated(new java.util.Date(666)); + vuln.setPublished(new java.util.Date(777)); + vuln.setUpdated(new java.util.Date(888)); + vuln.setSeverity(Severity.INFO); + vuln.setCvssV2BaseScore(BigDecimal.valueOf(6.0)); + vuln.setCvssV2ImpactSubScore(BigDecimal.valueOf(6.4)); + vuln.setCvssV2ExploitabilitySubScore(BigDecimal.valueOf(6.8)); + vuln.setCvssV2Vector("(AV:N/AC:M/Au:S/C:P/I:P/A:P)"); + vuln.setCvssV3BaseScore(BigDecimal.valueOf(9.1)); + vuln.setCvssV3ImpactSubScore(BigDecimal.valueOf(5.3)); + vuln.setCvssV3ExploitabilitySubScore(BigDecimal.valueOf(3.1)); + vuln.setCvssV3Vector("CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:H/A:L"); + vuln.setOwaspRRLikelihoodScore(BigDecimal.valueOf(4.5)); + vuln.setOwaspRRTechnicalImpactScore(BigDecimal.valueOf(5.0)); + vuln.setOwaspRRBusinessImpactScore(BigDecimal.valueOf(3.75)); + vuln.setOwaspRRVector("(SL:5/M:5/O:2/S:9/ED:4/EE:2/A:7/ID:2/LC:2/LI:2/LAV:7/LAC:9/FD:3/RD:5/NC:0/PV:7)"); + vuln.setEpssScore(BigDecimal.valueOf(0.6)); + vuln.setEpssPercentile(BigDecimal.valueOf(0.2)); + qm.persist(vuln); + + qm.addVulnerability(vuln, component, AnalyzerIdentity.INTERNAL_ANALYZER); + + final var vulnAlias = new VulnerabilityAlias(); + vulnAlias.setCveId("CVE-001"); + vulnAlias.setGhsaId("GHSA-001"); + vulnAlias.setGsdId("GSD-001"); + vulnAlias.setInternalId("INT-001"); + vulnAlias.setOsvId("OSV-001"); + vulnAlias.setSnykId("SNYK-001"); + vulnAlias.setSonatypeId("SONATYPE-001"); + vulnAlias.setVulnDbId("VULNDB-001"); + qm.synchronizeVulnerabilityAlias(vulnAlias); + + final Policy policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.uuid == "__COMPONENT_UUID__" + && component.group == "componentGroup" + && component.name == "componentName" + && component.version == "componentVersion" + && component.classifier == "LIBRARY" + && component.cpe == "componentCpe" + && component.purl == "componentPurl" + && component.swid_tag_id == "componentSwidTagId" + && component.is_internal + && component.md5 == "componentmd5" + && component.sha1 == "componentsha1" + && component.sha256 == "componentsha256" + && component.sha384 == "componentsha384" + && component.sha512 == "componentsha512" + && component.sha3_256 == "componentsha3_256" + && component.sha3_384 == "componentsha3_384" + && component.sha3_512 == "componentsha3_512" + && component.blake2b_256 == "componentBlake2b_256" + && component.blake2b_384 == "componentBlake2b_384" + && component.blake2b_512 == "componentBlake2b_512" + && component.blake3 == "componentBlake3" + && component.license_name == "componentLicenseName" + && !has(component.license_expression) // Requires https://github.com/DependencyTrack/dependency-track/pull/2400 to be ported to Hyades + && component.resolved_license.uuid == "__RESOLVED_LICENSE_UUID__" + && component.resolved_license.id == "resolvedLicenseId" + && component.resolved_license.name == "resolvedLicenseName" + && component.resolved_license.is_osi_approved + && component.resolved_license.is_fsf_libre + && component.resolved_license.is_deprecated_id + && component.resolved_license.is_custom + && component.resolved_license.groups.all(licenseGroup, + licenseGroup.uuid == "__LICENSE_GROUP_UUID__" + && licenseGroup.name == "licenseGroupName" + ) + && project.uuid == "__PROJECT_UUID__" + && project.group == "projectGroup" + && project.name == "projectName" + && project.version == "projectVersion" + && project.classifier == "APPLICATION" + && project.is_active + && project.cpe == "projectCpe" + && project.purl == "projectPurl" + && project.swid_tag_id == "projectSwidTagId" + && has(project.last_bom_import) + && "projecttaga" in project.tags + && project.properties.all(property, + property.group == "propertyGroup" + && property.name == "propertyName" + && property.value == "propertyValue" + && property.type == "STRING" + ) + && vulns.all(vuln, + vuln.uuid == "__VULN_UUID__" + && vuln.id == "CVE-001" + && vuln.source == "NVD" + && 666 in vuln.cwes + && vuln.aliases + .map(alias, alias.source + ":" + alias.id) + .all(alias, alias in [ + "NVD:CVE-001", + "GITHUB:GHSA-001", + "GSD:GSD-001", + "INTERNAL:INT-001", + "OSV:OSV-001", + "SNYK:SNYK-001", + "OSSINDEX:SONATYPE-001", + "VULNDB:VULNDB-001" + ]) + && vuln.created == timestamp("1970-01-01T00:00:00.666Z") + && vuln.published == timestamp("1970-01-01T00:00:00.777Z") + && vuln.updated == timestamp("1970-01-01T00:00:00.888Z") + && vuln.severity == "INFO" + && vuln.cvssv2_base_score == 6.0 + && vuln.cvssv2_impact_subscore == 6.4 + && vuln.cvssv2_exploitability_subscore == 6.8 + && vuln.cvssv2_vector == "(AV:N/AC:M/Au:S/C:P/I:P/A:P)" + && vuln.cvssv3_base_score == 9.1 + && vuln.cvssv3_impact_subscore == 5.3 + && vuln.cvssv3_exploitability_subscore == 3.1 + && vuln.cvssv3_vector == "CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:H/A:L" + && vuln.owasp_rr_likelihood_score == 4.5 + && vuln.owasp_rr_technical_impact_score == 5.0 + && vuln.owasp_rr_business_impact_score == 3.75 + && vuln.owasp_rr_vector == "(SL:5/M:5/O:2/S:9/ED:4/EE:2/A:7/ID:2/LC:2/LI:2/LAV:7/LAC:9/FD:3/RD:5/NC:0/PV:7)" + && vuln.epss_score == 0.6 + && vuln.epss_percentile == 0.2 + ) + """ + .replace("__COMPONENT_UUID__", component.getUuid().toString()) + .replace("__PROJECT_UUID__", project.getUuid().toString()) + .replace("__RESOLVED_LICENSE_UUID__", license.getUuid().toString()) + .replace("__LICENSE_GROUP_UUID__", licenseGroup.getUuid().toString()) + .replace("__VULN_UUID__", vuln.getUuid().toString()), PolicyViolation.Type.OPERATIONAL); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(project)).hasSize(1); + } + + @Test + public void testEvaluateProjectWithPolicyOperatorAnyAndAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "acme-lib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(2); + } + + @Test + public void testEvaluateProjectWithPolicyOperatorAnyAndNotAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "someOtherComponentThatIsNotAcmeLib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void testEvaluateProjectWithPolicyOperatorAnyAndNoConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "someOtherProjectThatIsNotAcmeApp" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "someOtherComponentThatIsNotAcmeLib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void testEvaluateProjectWithPolicyOperatorAllAndAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "acme-lib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(2); + } + + @Test + public void testEvaluateProjectWithPolicyOperatorAllAndNotAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "someOtherComponentThatIsNotAcmeLib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void testEvaluateProjectWithPolicyOperatorAllAndNoConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "someOtherProjectThatIsNotAcmeApp" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "someOtherComponentThatIsNotAcmeLib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void testEvaluateProjectWithPolicyAssignedToProject() { + final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyA, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + final var policyB = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyB, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + + final var projectA = new Project(); + projectA.setName("acme-app-a"); + qm.persist(projectA); + final var componentA = new Component(); + componentA.setProject(projectA); + componentA.setName("acme-lib"); + qm.persist(componentA); + + final var projectB = new Project(); + projectB.setName("acme-app-b"); + qm.persist(projectB); + final var componentB = new Component(); + componentB.setProject(projectB); + componentB.setName("acme-lib"); + qm.persist(componentB); + + policyB.setProjects(List.of(projectB)); + qm.persist(policyB); + + new CelPolicyEngine().evaluateProject(projectA.getUuid()); + new CelPolicyEngine().evaluateProject(projectB.getUuid()); + + assertThat(qm.getAllPolicyViolations(projectA)).hasSize(1); + assertThat(qm.getAllPolicyViolations(projectB)).hasSize(2); + } + + @Test + public void testEvaluateProjectWithPolicyAssignedToProjectParent() { + final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyA, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + final var policyB = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyB, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + + final var projectA = new Project(); + projectA.setName("acme-app-a"); + qm.persist(projectA); + final var componentA = new Component(); + componentA.setProject(projectA); + componentA.setName("acme-lib"); + qm.persist(componentA); + + final var projectParentB = new Project(); + projectParentB.setName("acme-app-parent-b"); + qm.persist(projectParentB); + + policyB.setProjects(List.of(projectParentB)); + policyB.setIncludeChildren(true); + qm.persist(policyB); + + final var projectB = new Project(); + projectB.setParent(projectParentB); + projectB.setName("acme-app-b"); + qm.persist(projectB); + final var componentB = new Component(); + componentB.setProject(projectB); + componentB.setName("acme-lib"); + qm.persist(componentB); + + new CelPolicyEngine().evaluateProject(projectA.getUuid()); + new CelPolicyEngine().evaluateProject(projectB.getUuid()); + + assertThat(qm.getAllPolicyViolations(projectA)).hasSize(1); + assertThat(qm.getAllPolicyViolations(projectB)).hasSize(2); + } + + @Test + public void testEvaluateProjectWithPolicyAssignedToTag() { + final Tag tag = qm.createTag("foo"); + + final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyA, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + final var policyB = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyB, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + policyB.setTags(List.of(tag)); + qm.persist(policyB); + + final var projectA = new Project(); + projectA.setName("acme-app-a"); + qm.persist(projectA); + final var componentA = new Component(); + componentA.setProject(projectA); + componentA.setName("acme-lib"); + qm.persist(componentA); + + final var projectB = new Project(); + projectB.setName("acme-app-b"); + qm.persist(projectB); + final var componentB = new Component(); + componentB.setProject(projectB); + componentB.setName("acme-lib"); + qm.persist(componentB); + + qm.bind(projectB, List.of(tag)); + + new CelPolicyEngine().evaluateProject(projectA.getUuid()); + new CelPolicyEngine().evaluateProject(projectB.getUuid()); + + assertThat(qm.getAllPolicyViolations(projectA)).hasSize(1); + assertThat(qm.getAllPolicyViolations(projectB)).hasSize(2); + } + + @Test + public void testEvaluateProjectWithInvalidScript() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.doesNotExist == "foo" + """, PolicyViolation.Type.OPERATIONAL); + final PolicyCondition validCondition = qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, + PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + assertThatNoException().isThrownBy(() -> new CelPolicyEngine().evaluateProject(project.getUuid())); + assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> + assertThat(violation.getPolicyCondition()).isEqualTo(validCondition) + ); + } + + @Test + public void testEvaluateProjectWithScriptExecutionException() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.last_bom_import == timestamp("invalid") + """, PolicyViolation.Type.OPERATIONAL); + final PolicyCondition validCondition = qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, + PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + assertThatNoException().isThrownBy(() -> new CelPolicyEngine().evaluateProject(project.getUuid())); + assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> + assertThat(violation.getPolicyCondition()).isEqualTo(validCondition) + ); + } + + @Test + public void testEvaluateProjectWithFuncProjectDependsOnComponent() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.depends_on(org.dependencytrack.policy.v1.Component{name: "acme-lib-a"}) + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var componentA = new Component(); + componentA.setProject(project); + componentA.setName("acme-lib-a"); + qm.persist(componentA); + + final var componentB = new Component(); + componentB.setProject(project); + componentB.setName("acme-lib-b"); + qm.persist(componentB); + + project.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentA).toJSON())); + qm.persist(project); + componentA.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentB).toJSON())); + qm.persist(componentA); + + final var policyEngine = new CelPolicyEngine(); + + policyEngine.evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); + assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); + } + + @Test + public void testEvaluateProjectWithFuncComponentIsDependencyOfComponent() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.is_dependency_of(org.dependencytrack.policy.v1.Component{name: "acme-lib-a"}) + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var componentA = new Component(); + componentA.setProject(project); + componentA.setName("acme-lib-a"); + qm.persist(componentA); + + final var componentB = new Component(); + componentB.setProject(project); + componentB.setName("acme-lib-b"); + qm.persist(componentB); + + project.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentA).toJSON())); + qm.persist(project); + componentA.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentB).toJSON())); + qm.persist(componentA); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + + assertThat(qm.getAllPolicyViolations(componentA)).isEmpty(); + assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); + } + + @Test + public void testEvaluateProjectWithFuncMatchesRange() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.matches_range("vers:generic/<1") + && component.matches_range("vers:golang/>0| new CelPolicyEngine().evaluateProject(project.getUuid())); + assertThat(qm.getAllPolicyViolations(componentA)).isEmpty(); + assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); + } + + @Test + public void testEvaluateProjectWhenProjectDoesNotExist() { + assertThatNoException().isThrownBy(() -> new CelPolicyEngine().evaluateProject(UUID.randomUUID())); + } + + @Test + public void testEvaluateComponent() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "acme-lib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateComponent(component.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void testEvaluateComponentWhenComponentDoesNotExist() { + assertThatNoException().isThrownBy(() -> new CelPolicyEngine().evaluateComponent(UUID.randomUUID())); + } + + @Test + public void issue1924() { + Policy policy = qm.createPolicy("Policy 1924", Policy.Operator.ALL, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.SEVERITY, PolicyCondition.Operator.IS, Severity.CRITICAL.name()); + qm.createPolicyCondition(policy, PolicyCondition.Subject.PACKAGE_URL, PolicyCondition.Operator.NO_MATCH, "pkg:deb"); + Project project = qm.createProject("My Project", null, "1", null, null, null, true, false); + qm.persist(project); + ArrayList components = new ArrayList<>(); + Component component = new Component(); + component.setName("OpenSSL"); + component.setVersion("3.0.2-0ubuntu1.6"); + component.setPurl("pkg:deb/openssl@3.0.2-0ubuntu1.6"); + component.setProject(project); + components.add(component); + qm.persist(component); + Vulnerability vulnerability = new Vulnerability(); + vulnerability.setVulnId("1"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.setSeverity(Severity.CRITICAL); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + vulnerability = new Vulnerability(); + vulnerability.setVulnId("2"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.setSeverity(Severity.CRITICAL); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + component = new Component(); + component.setName("Log4J"); + component.setVersion("1.2.16"); + component.setPurl("pkg:mvn/log4j/log4j@1.2.16"); + component.setProject(project); + components.add(component); + qm.persist(component); + vulnerability = new Vulnerability(); + vulnerability.setVulnId("3"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.setSeverity(Severity.CRITICAL); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + vulnerability = new Vulnerability(); + vulnerability.setVulnId("4"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.setSeverity(Severity.CRITICAL); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + CelPolicyEngine policyEngine = new CelPolicyEngine(); + policyEngine.evaluateProject(project.getUuid()); + final List violations = qm.getAllPolicyViolations(); + // NOTE: This behavior changed in CelPolicyEngine over the legacy PolicyEngine. + // A matched PolicyCondition can now only yield a single PolicyViolation, whereas + // with the legacy PolicyEngine, multiple PolicyViolations could be raised. +// Assert.assertEquals(3, violations.size()); +// PolicyViolation policyViolation = violations.get(0); +// Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); +// Assert.assertEquals(PolicyCondition.Subject.SEVERITY, policyViolation.getPolicyCondition().getSubject()); +// policyViolation = violations.get(1); +// Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); +// Assert.assertEquals(PolicyCondition.Subject.SEVERITY, policyViolation.getPolicyCondition().getSubject()); +// policyViolation = violations.get(2); +// Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); +// Assert.assertEquals(PolicyCondition.Subject.PACKAGE_URL, policyViolation.getPolicyCondition().getSubject()); + assertThat(violations).satisfiesExactlyInAnyOrder( + violation -> { + assertThat(violation.getComponent().getName()).isEqualTo("Log4J"); + assertThat(violation.getPolicyCondition().getSubject()).isEqualTo(PolicyCondition.Subject.SEVERITY); + }, + violation -> { + assertThat(violation.getComponent().getName()).isEqualTo("Log4J"); + assertThat(violation.getPolicyCondition().getSubject()).isEqualTo(PolicyCondition.Subject.PACKAGE_URL); + } + ); + } + + @Test + public void issue2455() { + Policy policy = qm.createPolicy("Policy 1924", Policy.Operator.ALL, Policy.ViolationState.INFO); + + License license = new License(); + license.setName("Apache 2.0"); + license.setLicenseId("Apache-2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + LicenseGroup lg = qm.createLicenseGroup("Test License Group 1"); + lg.setLicenses(Collections.singletonList(license)); + lg = qm.persist(lg); + lg = qm.detach(LicenseGroup.class, lg.getId()); + license = qm.detach(License.class, license.getId()); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS_NOT, lg.getUuid().toString()); + + license = new License(); + license.setName("MIT"); + license.setLicenseId("MIT"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + lg = qm.createLicenseGroup("Test License Group 2"); + lg.setLicenses(Collections.singletonList(license)); + lg = qm.persist(lg); + lg = qm.detach(LicenseGroup.class, lg.getId()); + license = qm.detach(License.class, license.getId()); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS_NOT, lg.getUuid().toString()); + + Project project = qm.createProject("My Project", null, "1", null, null, null, true, false); + qm.persist(project); + + license = new License(); + license.setName("LGPL"); + license.setLicenseId("LGPL"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + ArrayList components = new ArrayList<>(); + Component component = new Component(); + component.setName("Log4J"); + component.setVersion("2.0.0"); + component.setProject(project); + component.setResolvedLicense(license); + components.add(component); + qm.persist(component); + + CelPolicyEngine policyEngine = new CelPolicyEngine(); + policyEngine.evaluateProject(project.getUuid()); + final List violations = qm.getAllPolicyViolations(); + Assert.assertEquals(2, violations.size()); + PolicyViolation policyViolation = violations.get(0); + Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); + Assert.assertEquals(PolicyCondition.Subject.LICENSE_GROUP, policyViolation.getPolicyCondition().getSubject()); + policyViolation = violations.get(1); + Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); + Assert.assertEquals(PolicyCondition.Subject.LICENSE_GROUP, policyViolation.getPolicyCondition().getSubject()); + } + + @Test + @Ignore // Un-ignore for manual profiling purposes. + public void testWithBloatedBom() throws Exception { + // Import all default objects (includes licenses and license groups). + new DefaultObjectGenerator().contextInitialized(null); + + final var project = new Project(); + project.setName("acme-app"); + project.setVersion("1.2.3"); + qm.persist(project); + + // Create a policy that will be violated by the vast majority (>8000) components. + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.resolved_license.groups.exists(lg, lg.name == "Permissive") + """); + policyConditionA.setViolationType(PolicyViolation.Type.OPERATIONAL); + qm.persist(policyConditionA); + + // Import the bloated BOM. + new BomUploadProcessingTask().inform(new BomUploadEvent(qm.detach(Project.class, project.getId()), createTempBomFile("bom-bloated.json"))); + + // Evaluate policies on the project. + new CelPolicyEngine().evaluateProject(project.getUuid()); + } + + private static File createTempBomFile(final String testFileName) throws Exception { + // The task will delete the input file after processing it, + // so create a temporary copy to not impact other tests. + final Path bomFilePath = Files.createTempFile(null, null); + Files.copy(Paths.get(resourceToURL("/unit/" + testFileName).toURI()), bomFilePath, StandardCopyOption.REPLACE_EXISTING); + return bomFilePath.toFile(); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java new file mode 100644 index 000000000..11fae9b64 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java @@ -0,0 +1,84 @@ +package org.dependencytrack.policy.cel; + +import alpine.server.cache.AbstractCacheManager; +import com.google.api.expr.v1alpha1.Type; +import org.apache.commons.codec.digest.DigestUtils; +import org.dependencytrack.policy.cel.CelPolicyScriptHost.CacheMode; +import org.junit.Test; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE_GROUP; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY; + +public class CelPolicyScriptHostTest { + + private static class TestCacheManager extends AbstractCacheManager { + + private TestCacheManager() { + super(30, TimeUnit.SECONDS, 5); + } + + } + + @Test + public void testCompileWithCache() throws Exception { + final var scriptSrc = """ + component.name == "foo" + """; + + final var cacheManager = new TestCacheManager(); + final CelPolicyScript script = new CelPolicyScriptHost(cacheManager).compile(""" + component.name == "foo" + """, CacheMode.CACHE); + + assertThat((Object) cacheManager.get(CelPolicyScript.class, DigestUtils.sha256Hex(scriptSrc))).isEqualTo(script); + } + + @Test + public void testCompileWithoutCache() throws Exception { + final var scriptSrc = """ + component.name == "foo" + """; + + final var cacheManager = new TestCacheManager(); + new CelPolicyScriptHost(cacheManager).compile(""" + component.name == "foo" + """, CacheMode.NO_CACHE); + + assertThat((Object) cacheManager.get(CelPolicyScript.class, DigestUtils.sha256Hex(scriptSrc))).isNull(); + } + + @Test + public void testRequirementsAnalysis() throws Exception { + final CelPolicyScript compiledScript = CelPolicyScriptHost.getInstance().compile(""" + component.resolved_license.groups.exists(licenseGroup, licenseGroup.name == "Permissive") + && vulns.exists(vuln, vuln.severity in ["HIGH", "CRITICAL"] && has(vuln.aliases)) + && project.depends_on(org.dependencytrack.policy.v1.Component{name: "foo"}) + """, CacheMode.NO_CACHE); + + final Map> requirements = compiledScript.getRequirements().asMap(); + assertThat(requirements).containsOnlyKeys(TYPE_COMPONENT, TYPE_LICENSE, TYPE_LICENSE_GROUP, TYPE_PROJECT, TYPE_VULNERABILITY); + + assertThat(requirements.get(TYPE_COMPONENT)).containsOnly("resolved_license"); + assertThat(requirements.get(TYPE_LICENSE)).containsOnly("groups"); + assertThat(requirements.get(TYPE_LICENSE_GROUP)).containsOnly("name"); + assertThat(requirements.get(TYPE_PROJECT)).containsOnly("uuid"); // Implicit through project.depends_on + assertThat(requirements.get(TYPE_VULNERABILITY)).containsOnly( + "aliases", + // Scores are necessary to calculate severity... + "cvssv2_base_score", + "cvssv3_base_score", + "owasp_rr_likelihood_score", + "owasp_rr_technical_impact_score", + "owasp_rr_business_impact_score", + "severity"); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilderTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilderTest.java new file mode 100644 index 000000000..f7d2fa634 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilderTest.java @@ -0,0 +1,15 @@ +package org.dependencytrack.policy.cel.compat; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; + +public class CelPolicyScriptSourceBuilderTest { + + @Test + public void testEscapeQuotes() { + assertThat(escapeQuotes("\"foobar")).isEqualTo("\\\"foobar"); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java new file mode 100644 index 000000000..ef46208f0 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java @@ -0,0 +1,127 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.json.JSONObject; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class CoordinatesConditionTest extends AbstractPostgresEnabledTest { + private Object[] parameters() { + return new Object[]{ + //MATCHES group regex + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'acme*','name': 'acme*','version': '>=v1.2*'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, + //Exact matches + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, + //Exact group does not match + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'org.hippo','name': 'acme-lib','version': 'v1.2.3'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", false}, + //Name does not match regex + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'acme-app','name': '*acme-lib*','version': 'v1.2.3'}", "{'group': 'acme-app','name': 'good-foo-lib','version': 'v1.2.3'}", false}, + //Version regex does not match + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'acme-app','name': '*acme-lib*','version': 'v1.*'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v2.2.3'}", false}, + //Does not match on exact group + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'group': 'diff-group','name': 'acme-lib','version': 'v1.2.3'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, + //Does not match on version range greater than or equal + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'group': 'acme-app','name': '*acme-lib*','version': '>=v2.2.2'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, + //Matches without group + new Object[]{PolicyCondition.Operator.MATCHES, "{'name': 'Test Component','version': '1.0.0'}", "{'name': 'Test Component','version': '1.0.0'}", true}, + //Matches on wild card group - uncomment after fixing script builder + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'group': '*', 'name': 'Test Component', 'version': '1.0.0' }", "{ 'group': 'Anything', 'name': 'Test Component', 'version': '1.0.0' }", true}, + //Matches on wild card name + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'group': 'acme-app', 'name': '*', 'version': '1.0.0' }", "{ 'group': 'acme-app', 'name': 'Anything', 'version': '1.0.0' }", true}, + //Matches on wild card version + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'group': 'acme-app', 'name': 'Test Component', 'version': '>=*' }", "{ 'group': 'acme-app', 'name': 'Test Component', 'version': '4.4.4' }", true}, + //Matches on empty policy - uncomment after fixing script builder + //new Object[]{PolicyCondition.Operator.MATCHES, "{}", "{}", true}, + //Does not match on lower version + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '== 1.1.1' }", "{'version': '0.1.1'}", true}, + //Matches on equal version + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '== 1.1.1' }", "{'version': '1.1.1'}", true}, + //Does not match on higher version + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '== 1.1.1' }", "{'version': '2.1.1'}", true}, + //No match with version not equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '!= 1.1.1' }", "{ 'version': '1.1.1' }", false}, + //Matches with version not equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '!= 1.1.1' }", "{'version': '2.1.1'}", true}, + //Matches with version not equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '!= 1.1.1' }", "{'version': '0.1.1'}", true}, + //Matches with version greater than + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '> 1.1.1' }", "{'version': '2.1.1'}", true}, + //Does not match on version greater than + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '> 1.1.1' }", "{'version': '0.1.1'}", false}, + //Does not match on version equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '> 1.1.1' }", "{'version': '1.1.1'}", false}, + //No match with version greater than + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '> 1.1.1' }", "{'version': '0.1.1'}", true}, + //No match with version equal to + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '> 1.1.1' }", "{'version': '1.1.1'}", true}, + //No match with version greater than + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '> 1.1.1' }", "{'version': '2.1.1'}", false}, + //Matches on version less than + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<1.1.1'}", "{'version': '0.1.1'}", true}, + //Does not match on version less than + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<1.1.1'}", "{'version': '2.1.1'}", false}, + //Does not match on equal version + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<1.1.1'}", "{'version': '1.1.1'}", false}, + //No match on version less than + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<1.1.1'}", "{'version': '0.1.1'}", false}, + //No match on version less than + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<1.1.1'}", "{'version': '2.1.1'}", true}, + //No match on equal version + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<1.1.1'}", "{'version': '1.1.1'}", true}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<=1.1.1'}", "{'version': '0.1.1'}", true}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<=1.1.1'}", "{'version': '2.1.1'}", false}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<=1.1.1'}", "{'version': '1.1.1'}", true}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<=1.1.1'}", "{'version': '0.1.1'}", false}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<=1.1.1'}", "{'version': '2.1.1'}", true}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<=1.1.1'}", "{'version': '1.1.1'}", false}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final PolicyCondition.Operator operator, final String conditionCoordinates, final String componentCoordinates, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.COORDINATES, operator, conditionCoordinates); + + final JSONObject def = new JSONObject(componentCoordinates); + final String group = Optional.ofNullable(def.optString("group", null)).orElse(""); + final String name = Optional.ofNullable(def.optString("name", null)).orElse(""); + final String version = Optional.ofNullable(def.optString("version")).orElse(""); + + final var project = new Project(); + project.setName(group); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setGroup(group); + component.setName(name); + component.setVersion(version); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CpeConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CpeConditionTest.java new file mode 100644 index 000000000..ed65a5026 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CpeConditionTest.java @@ -0,0 +1,63 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyCondition.Operator; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.model.PolicyCondition.Operator.MATCHES; + +@RunWith(JUnitParamsRunner.class) +public class CpeConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // MATCHES with exact match + new Object[]{MATCHES, "cpe:/a:acme:application:1.0.0", "cpe:/a:acme:application:1.0.0", true}, + // MATCHES with regex match + new Object[]{MATCHES, "cpe:/a:acme:\\\\w+:[0-9].0.0", "cpe:/a:acme:application:1.0.0", true}, + // MATCHES with no match + new Object[]{MATCHES, "cpe:/a:acme:application:1.0.0", "cpe:/a:acme:application:9.9.9", false}, + // NO_MATCH with no match + new Object[]{Operator.NO_MATCH, "cpe:/a:acme:application:1.0.0", "cpe:/a:acme:application:9.9.9", true}, + // NO_MATCH with exact match + new Object[]{Operator.NO_MATCH, "cpe:/a:acme:application:1.0.0", "cpe:/a:acme:application:1.0.0", false}, + // MATCHES with quotes + new Object[]{MATCHES, "\"cpe:/a:acme:application:1.0.0", "\"cpe:/a:acme:application:1.0.0", true} + }; + } + + + @Test + @Parameters(method = "parameters") + public void testCondition(final Operator operator, final String conditionCpe, final String componentCpe, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.CPE, operator, conditionCpe); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setCpe(componentCpe); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CweConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CweConditionTest.java new file mode 100644 index 000000000..87388e9cf --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CweConditionTest.java @@ -0,0 +1,68 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class CweConditionTest extends AbstractPostgresEnabledTest { + private Object[] parameters() { + return new Object[]{ + new Object[]{Policy.Operator.ANY, Policy.ViolationState.INFO, PolicyCondition.Operator.CONTAINS_ANY, + "CWE-123", 123, 0, true, PolicyViolation.Type.SECURITY, Policy.ViolationState.INFO}, + new Object[]{Policy.Operator.ANY, Policy.ViolationState.FAIL, PolicyCondition.Operator.CONTAINS_ALL, + "CWE-123, CWE-786", 123, 786, true, PolicyViolation.Type.SECURITY, Policy.ViolationState.FAIL}, + new Object[]{Policy.Operator.ANY, Policy.ViolationState.FAIL, PolicyCondition.Operator.IS, + "CWE-123, CWE-786", 123, 786, false, null, null}, + new Object[]{Policy.Operator.ANY, Policy.ViolationState.FAIL, PolicyCondition.Operator.CONTAINS_ALL, + "CWE-123.565, CWE-786.67", 123, 786, false, null, null}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testSingleCwe(Policy.Operator policyOperator, Policy.ViolationState violationState, + PolicyCondition.Operator conditionOperator, String inputConditionCwe, int inputCweId, int inputCweId2, + boolean expectViolation, PolicyViolation.Type actualType, Policy.ViolationState actualViolationState) { + Policy policy = qm.createPolicy("Test Policy", policyOperator, violationState); + qm.createPolicyCondition(policy, PolicyCondition.Subject.CWE, conditionOperator, inputConditionCwe); + final var project = new Project(); + project.setName("acme-app"); + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + Vulnerability vulnerability = new Vulnerability(); + vulnerability.setVulnId("12345"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.addCwe(inputCweId); + if (inputCweId2 != 0) { + vulnerability.addCwe(inputCweId2); + } + qm.persist(project); + qm.persist(component); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getSubject()).isEqualTo(PolicyCondition.Subject.CWE); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getViolationType()).isEqualTo(actualType); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getPolicy().getViolationState()).isEqualTo(actualViolationState); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/HashConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/HashConditionTest.java new file mode 100644 index 000000000..ad0cee6f7 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/HashConditionTest.java @@ -0,0 +1,93 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.Policy.ViolationState; +import org.dependencytrack.model.PolicyCondition.Operator; +import org.dependencytrack.model.PolicyCondition.Subject; +import org.dependencytrack.model.PolicyViolation.Type; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class HashConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash", true, ViolationState.FAIL, Type.OPERATIONAL, ViolationState.FAIL}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash", true, ViolationState.WARN, Type.OPERATIONAL, ViolationState.WARN}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash_false", false, ViolationState.INFO, Type.OPERATIONAL, ViolationState.INFO}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'test', 'value': 'test_hash' }", + "test_hash", false, ViolationState.INFO, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS_NOT, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash20", false, ViolationState.INFO, null, null}, + new Object[]{Policy.Operator.ANY, Operator.MATCHES, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash", false, ViolationState.INFO, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': null, 'value': 'test_hash' }", + "test_hash", false, ViolationState.FAIL, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'MD5', 'value': null }", + "test_hash", false, ViolationState.FAIL, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'SHA256', 'value': '' }", + "test_hash", false, ViolationState.FAIL, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': '', 'value': 'test_hash' }", + "test_hash", false, ViolationState.FAIL, null, null}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(Policy.Operator policyOperator, final Operator condition, final String conditionHash, + final String actualHash, final boolean expectViolation, ViolationState violationState, + Type actualType, ViolationState actualViolationState) { + final Policy policy = qm.createPolicy("policy", policyOperator, violationState); + qm.createPolicyCondition(policy, Subject.COMPONENT_HASH, condition, conditionHash); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setSha256(actualHash); + qm.persist(component); + + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getViolationType()).isEqualTo(actualType); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getPolicy().getViolationState()).isEqualTo(actualViolationState); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + + + @Test + public void testWithNullPolicyCondition() { + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setSha256("actualHash"); + qm.persist(component); + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/LicenseConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/LicenseConditionTest.java new file mode 100644 index 000000000..4ec271338 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/LicenseConditionTest.java @@ -0,0 +1,122 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.License; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class LicenseConditionTest extends AbstractPostgresEnabledTest { + + @Test + public void hasMatch() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE, PolicyCondition.Operator.IS, license.getUuid().toString()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void noMatch() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE, PolicyCondition.Operator.IS, UUID.randomUUID().toString()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void wrongOperator() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE, PolicyCondition.Operator.MATCHES, license.getUuid().toString()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setProject(project); + component.setResolvedLicense(license); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void valueIsUnresolved() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE, PolicyCondition.Operator.IS, "unresolved"); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + Component componentWithoutLicense = new Component(); + componentWithoutLicense.setName("second-component"); + componentWithoutLicense.setProject(project); + qm.persist(componentWithoutLicense); + + CelPolicyEngine policyEngine = new CelPolicyEngine(); + + policyEngine.evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(componentWithoutLicense)).hasSize(1); + + final var componentWithLicense = new Component(); + componentWithLicense.setName("acme-app"); + componentWithLicense.setProject(project); + componentWithLicense.setResolvedLicense(license); + qm.persist(componentWithLicense); + + policyEngine.evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(componentWithLicense)).hasSize(0); + } +} + diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/LicenseGroupConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/LicenseGroupConditionTest.java new file mode 100644 index 000000000..2712d9afe --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/LicenseGroupConditionTest.java @@ -0,0 +1,152 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.License; +import org.dependencytrack.model.LicenseGroup; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; + +import java.util.Collections; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class LicenseGroupConditionTest extends AbstractPostgresEnabledTest { + + @Test + public void hasMatch() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + LicenseGroup lg = qm.createLicenseGroup("Test License Group"); + lg.setLicenses(Collections.singletonList(license)); + lg = qm.persist(lg); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS, lg.getUuid().toString()); + + qm.detach(Policy.class, policy.getId()); + qm.detach(PolicyCondition.class, condition.getId()); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void noMatch() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + LicenseGroup lg = qm.createLicenseGroup("Test License Group"); + lg = qm.persist(lg); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS, lg.getUuid().toString()); + qm.detach(Policy.class, policy.getId()); + qm.detach(PolicyCondition.class, condition.getId()); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void unknownLicenseViolateWhitelist() { + LicenseGroup lg = qm.createLicenseGroup("Test License Group"); + lg = qm.persist(lg); + lg = qm.detach(LicenseGroup.class, lg.getId()); + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS_NOT, lg.getUuid().toString()); + qm.detach(Policy.class, policy.getId()); + qm.detach(PolicyCondition.class, condition.getId()); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(null); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void wrongOperator() { + License license = new License(); + license.setName("Apache 2.0"); + license.setLicenseId("Apache-2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + LicenseGroup lg = qm.createLicenseGroup("Test License Group"); + lg.setLicenses(Collections.singletonList(license)); + lg = qm.persist(lg); + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.MATCHES, lg.getUuid().toString()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void licenseGroupDoesNotExist() { + License license = new License(); + license.setName("Apache 2.0"); + license.setLicenseId("Apache-2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS, UUID.randomUUID().toString()); + qm.detach(Policy.class, policy.getId()); + qm.detach(PolicyCondition.class, condition.getId()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/SeverityConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/SeverityConditionTest.java new file mode 100644 index 000000000..dc8d62afb --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/SeverityConditionTest.java @@ -0,0 +1,128 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.Policy.ViolationState; +import org.dependencytrack.model.PolicyCondition.Operator; +import org.dependencytrack.model.PolicyCondition.Subject; +import org.dependencytrack.model.PolicyViolation.Type; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Severity; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class SeverityConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // IS with exact match + new Object[]{Operator.IS, "CRITICAL", "CRITICAL", true}, + // IS with regex match (regex is not supported by this condition) + new Object[]{Operator.IS, "CRI[A-Z]+", "CRITICAL", false}, + // IS with no match + new Object[]{Operator.IS, "CRITICAL", "LOW", false}, + // IS_NOT with no match + new Object[]{Operator.IS_NOT, "CRITICAL", "LOW", true}, + // IS_NOT with exact match + new Object[]{Operator.IS_NOT, "UNASSIGNED", "UNASSIGNED", false}, + // IS with quotes (actualSeverity can't have quotes because it's an enum) + new Object[]{Operator.IS, "\"CRITICAL", "CRITICAL", false} + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final Operator operator, final String conditionSeverity, final String actualSeverity, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, ViolationState.INFO); + qm.createPolicyCondition(policy, Subject.SEVERITY, operator, conditionSeverity); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + final var vulnA = new Vulnerability(); + vulnA.setVulnId("INT-123"); + vulnA.setSource(Vulnerability.Source.INTERNAL); + vulnA.setSeverity(Severity.valueOf(actualSeverity)); + qm.persist(vulnA); + + final var vulnB = new Vulnerability(); + vulnB.setVulnId("INT-666"); + vulnB.setSource(Vulnerability.Source.INTERNAL); + qm.persist(vulnB); + + qm.addVulnerability(vulnA, component, AnalyzerIdentity.INTERNAL_ANALYZER); + qm.addVulnerability(vulnB, component, AnalyzerIdentity.INTERNAL_ANALYZER); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + + @Test + public void testSeverityCalculation() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, ViolationState.FAIL); + qm.createPolicyCondition(policy, Subject.SEVERITY, Operator.IS, Severity.CRITICAL.name(), Type.SECURITY); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + // Create a vulnerability that has all scores (CVSSv2, CVSSv3, OWASP RR) + // available, but no severity is set explicitly. + // + // Even though the expression only accesses the `severity` field, the policy + // engine should fetch all scores in order to derive the severity from them. + // Note that when multiple scores are available, the highest severity wins. + // + // The highest severity among the scores below is CRITICAL from CVSSv3. + + final var vuln = new Vulnerability(); + vuln.setVulnId("CVE-123"); + vuln.setSource(Vulnerability.Source.NVD); + // vuln.setSeverity(Severity.INFO); + vuln.setCvssV2BaseScore(BigDecimal.valueOf(6.0)); + vuln.setCvssV2ImpactSubScore(BigDecimal.valueOf(6.4)); + vuln.setCvssV2ExploitabilitySubScore(BigDecimal.valueOf(6.8)); + vuln.setCvssV2Vector("(AV:N/AC:M/Au:S/C:P/I:P/A:P)"); + vuln.setCvssV3BaseScore(BigDecimal.valueOf(9.1)); + vuln.setCvssV3ImpactSubScore(BigDecimal.valueOf(5.3)); + vuln.setCvssV3ExploitabilitySubScore(BigDecimal.valueOf(3.1)); + vuln.setCvssV3Vector("CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:H/A:L"); + vuln.setOwaspRRLikelihoodScore(BigDecimal.valueOf(4.5)); + vuln.setOwaspRRTechnicalImpactScore(BigDecimal.valueOf(5.0)); + vuln.setOwaspRRBusinessImpactScore(BigDecimal.valueOf(3.75)); + vuln.setOwaspRRVector("(SL:5/M:5/O:2/S:9/ED:4/EE:2/A:7/ID:2/LC:2/LI:2/LAV:7/LAC:9/FD:3/RD:5/NC:0/PV:7)"); + qm.persist(vuln); + + qm.addVulnerability(vuln, component, AnalyzerIdentity.INTERNAL_ANALYZER); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/SwidTagIdConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/SwidTagIdConditionTest.java new file mode 100644 index 000000000..d94541adc --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/SwidTagIdConditionTest.java @@ -0,0 +1,61 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyCondition.Operator; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class SwidTagIdConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // MATCHES with exact match + new Object[]{Operator.MATCHES, "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", true}, + // MATCHES with regex match + new Object[]{Operator.MATCHES, "swidgen-242eb18a-[a-z0-9]{4}-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", true}, + // MATCHES with no match + new Object[]{Operator.MATCHES, "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_6.6.6", false}, + // NO_MATCH with no match + new Object[]{Operator.NO_MATCH, "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_6.6.6", true}, + // NO_MATCH with exact match + new Object[]{Operator.NO_MATCH, "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", false}, + // MATCHES with quotes + new Object[]{Operator.MATCHES, "\"swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "\"swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", true} + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final Operator operator, final String conditionSwidTagId, final String componentSwidTagId, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.SWID_TAGID, operator, conditionSwidTagId); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setSwidTagId(componentSwidTagId); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/VersionConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/VersionConditionTest.java new file mode 100644 index 000000000..1f1a3ab7e --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/VersionConditionTest.java @@ -0,0 +1,62 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class VersionConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // MATCHES with exact match + new Object[]{PolicyCondition.Operator.NUMERIC_EQUAL, "v1.2.3", "v1.2.3", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_EQUAL, "v1.2.3", "v1.2.4", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_NOT_EQUAL, "0.4.5-SNAPSHOT", "0.4.5", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_NOT_EQUAL, "0.4.5", "0.4.5", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN, "0.4.5", "0.5.5", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN, "0.4.4", "0.4.4", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN_OR_EQUAL, "0.4.4", "0.4.4", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN_OR_EQUAL, "v0.4.5-SNAPSHOT", "z0.4.5", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN_OR_EQUAL, "v0.4.5-SNAPSHOT", "0.4.5", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN_OR_EQUAL, "v0.4.*", "v0.4.1", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_LESS_THAN, "v0.4.*", "v0.4.1", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_LESS_THAN, "v0.4.*", "v0.3.1", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_LESSER_THAN_OR_EQUAL, "v0.4.*", "v0.4.0", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_LESSER_THAN_OR_EQUAL, "v0.4.*", "v0.4.2", false}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final PolicyCondition.Operator operator, final String conditionVersion, final String componentVersion, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.VERSION, operator, conditionVersion); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setVersion(componentVersion); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdConditionTest.java new file mode 100644 index 000000000..63f27958d --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdConditionTest.java @@ -0,0 +1,74 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class VulnerabilityIdConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // IS with exact match + new Object[]{PolicyCondition.Operator.IS, "CVE-123", "CVE-123", true}, + // IS with regex match (regex is not supported by this condition) + new Object[]{PolicyCondition.Operator.IS, "CVE-[0-9]+", "CVE-123", false}, + // IS with no match + new Object[]{PolicyCondition.Operator.IS, "CVE-123", "CVE-666", false}, + // IS_NOT with no match + new Object[]{PolicyCondition.Operator.IS_NOT, "CVE-123", "CVE-666", true}, + // IS_NOT with exact match + new Object[]{PolicyCondition.Operator.IS_NOT, "CVE-123", "CVE-123", false}, + // IS with quotes + new Object[]{PolicyCondition.Operator.IS, "\"CVE-123", "\"CVE-123", true} + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final PolicyCondition.Operator operator, final String conditionVulnId, final String actualVulnId, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.VULNERABILITY_ID, operator, conditionVulnId); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + final var vulnA = new Vulnerability(); + vulnA.setVulnId(actualVulnId); + vulnA.setSource(Vulnerability.Source.INTERNAL); + qm.persist(vulnA); + + final var vulnB = new Vulnerability(); + vulnB.setVulnId("INT-666"); + vulnB.setSource(Vulnerability.Source.INTERNAL); + qm.persist(vulnB); + + qm.addVulnerability(vulnA, component, AnalyzerIdentity.INTERNAL_ANALYZER); + qm.addVulnerability(vulnB, component, AnalyzerIdentity.INTERNAL_ANALYZER); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java b/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java new file mode 100644 index 000000000..f6143264f --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java @@ -0,0 +1,88 @@ +package org.dependencytrack.policy.cel.mapping; + +import com.google.protobuf.Descriptors.Descriptor; +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.proto.policy.v1.Component; +import org.dependencytrack.proto.policy.v1.License; +import org.dependencytrack.proto.policy.v1.Project; +import org.dependencytrack.proto.policy.v1.Vulnerability; +import org.junit.Test; + +import javax.jdo.PersistenceManagerFactory; +import javax.jdo.metadata.ColumnMetadata; +import javax.jdo.metadata.MemberMetadata; +import javax.jdo.metadata.TypeMetadata; + +import static org.assertj.core.api.Assertions.assertThat; + +public class FieldMappingUtilTest extends PersistenceCapableTest { + + @Test + public void testGetFieldMappingsForComponentProjection() { + assertValidProtoFieldsAndColumns(ComponentProjection.class, Component.getDescriptor(), org.dependencytrack.model.Component.class); + } + + @Test + public void testGetFieldMappingsForLicenseProjection() { + assertValidProtoFieldsAndColumns(LicenseProjection.class, License.getDescriptor(), org.dependencytrack.model.License.class); + } + + @Test + public void testGetFieldMappingsForLicenseGroupProjection() { + assertValidProtoFieldsAndColumns(LicenseGroupProjection.class, License.Group.getDescriptor(), org.dependencytrack.model.LicenseGroup.class); + } + + @Test + public void testGetFieldMappingsForProjectProjection() { + assertValidProtoFieldsAndColumns(ProjectProjection.class, Project.getDescriptor(), org.dependencytrack.model.Project.class); + } + + @Test + public void testGetFieldMappingsForProjectPropertyProjection() { + assertValidProtoFieldsAndColumns(ProjectPropertyProjection.class, Project.Property.getDescriptor(), org.dependencytrack.model.ProjectProperty.class); + } + + @Test + public void testGetFieldMappingsForVulnerabilityProjection() { + assertValidProtoFieldsAndColumns(VulnerabilityProjection.class, Vulnerability.getDescriptor(), org.dependencytrack.model.Vulnerability.class); + } + + private void assertValidProtoFieldsAndColumns(final Class projectionClazz, + final Descriptor protoDescriptor, + final Class persistenceClass) { + assertThat(FieldMappingUtil.getFieldMappings(projectionClazz)).allSatisfy( + fieldMapping -> { + assertHasProtoField(protoDescriptor, fieldMapping.protoFieldName()); + assertHasSqlColumn(persistenceClass, fieldMapping.sqlColumnName()); + } + ); + } + + private void assertHasProtoField(final Descriptor protoDescriptor, final String fieldName) { + assertThat(protoDescriptor.findFieldByName(fieldName)).isNotNull(); + } + + private void assertHasSqlColumn(final Class clazz, final String columnName) { + final PersistenceManagerFactory pmf = qm.getPersistenceManager().getPersistenceManagerFactory(); + + final TypeMetadata typeMetadata = pmf.getMetadata(clazz.getName()); + assertThat(typeMetadata).isNotNull(); + + var found = false; + for (final MemberMetadata memberMetadata : typeMetadata.getMembers()) { + if (memberMetadata.getColumns() == null) { + continue; + } + + for (final ColumnMetadata columnMetadata : memberMetadata.getColumns()) { + if (columnName.equals(columnMetadata.getName())) { + found = true; + break; + } + } + } + + assertThat(found).isTrue(); + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java index b9bcca5bb..072443eda 100644 --- a/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java +++ b/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java @@ -406,6 +406,7 @@ public void createComponentTest() { component.setProject(project); component.setName("My Component"); component.setVersion("1.0"); + component.setPurl("pkg:maven/org.acme/abc"); Response response = target(V1_COMPONENT + "/project/" + project.getUuid().toString()).request() .header(X_API_KEY, apiKey) .put(Entity.entity(component, MediaType.APPLICATION_JSON)); @@ -415,8 +416,13 @@ public void createComponentTest() { Assert.assertEquals("My Component", json.getString("name")); Assert.assertEquals("1.0", json.getString("version")); Assert.assertTrue(UuidUtil.isValidUUID(json.getString("uuid"))); - assertThat(kafkaMockProducer.history()).satisfiesExactly( + assertThat(kafkaMockProducer.history()).satisfiesExactlyInAnyOrder( record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = KafkaTestUtil.deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo(json.getString("purl")); + }, record -> { assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_COMMAND.name()); final var command = KafkaTestUtil.deserializeValue(KafkaTopics.VULN_ANALYSIS_COMMAND, record); @@ -432,6 +438,7 @@ public void createComponentUpperCaseHashTest() { component.setProject(project); component.setName("My Component"); component.setVersion("1.0"); + component.setPurl("pkg:maven/org.acme/abc"); component.setSha1("640ab2bae07bedc4c163f679a746f7ab7fb5d1fa".toUpperCase()); component.setSha256("532eaabd9574880dbf76b9b8cc00832c20a6ec113d682299550d7a6e0f345e25".toUpperCase()); component.setSha3_256("c0a5cca43b8aa79eb50e3464bc839dd6fd414fae0ddf928ca23dcebf8a8b8dd0".toUpperCase()); @@ -465,6 +472,7 @@ public void updateComponentTest() { Component component = new Component(); component.setProject(project); component.setName("My Component"); + component.setPurl("pkg:maven/org.acme/abc"); component.setVersion("1.0"); component = qm.createComponent(component, false); component.setDescription("Test component"); @@ -477,8 +485,13 @@ public void updateComponentTest() { Assert.assertEquals("My Component", json.getString("name")); Assert.assertEquals("1.0", json.getString("version")); Assert.assertEquals("Test component", json.getString("description")); - assertThat(kafkaMockProducer.history()).satisfiesExactly( + assertThat(kafkaMockProducer.history()).satisfiesExactlyInAnyOrder( record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = KafkaTestUtil.deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo(json.getString("purl")); + }, record -> { assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_COMMAND.name()); final var command = KafkaTestUtil.deserializeValue(KafkaTopics.VULN_ANALYSIS_COMMAND, record); diff --git a/src/test/java/org/dependencytrack/resources/v1/PolicyConditionResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/PolicyConditionResourceTest.java new file mode 100644 index 000000000..ff4e57bb6 --- /dev/null +++ b/src/test/java/org/dependencytrack/resources/v1/PolicyConditionResourceTest.java @@ -0,0 +1,154 @@ +package org.dependencytrack.resources.v1; + +import alpine.server.filters.ApiFilter; +import alpine.server.filters.AuthenticationFilter; +import org.dependencytrack.ResourceTest; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; +import org.glassfish.jersey.test.DeploymentContext; +import org.glassfish.jersey.test.ServletDeploymentContext; +import org.junit.Test; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; +import static org.assertj.core.api.Assertions.assertThat; + +public class PolicyConditionResourceTest extends ResourceTest { + + @Override + protected DeploymentContext configureDeployment() { + return ServletDeploymentContext.forServlet(new ServletContainer( + new ResourceConfig(PolicyConditionResource.class) + .register(ApiFilter.class) + .register(AuthenticationFilter.class))) + .build(); + } + + @Test + public void testCreateExpressionCondition() { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + + final Response response = target("%s/%s/condition".formatted(V1_POLICY, policy.getUuid())) + .request() + .header(X_API_KEY, apiKey) + .put(Entity.entity(""" + { + "subject": "EXPRESSION", + "value": "component.name == \\"foo\\"", + "violationType": "SECURITY" + } + """, MediaType.APPLICATION_JSON)); + + assertThat(response.getStatus()).isEqualTo(201); + assertThatJson(getPlainTextBody(response)) + .isEqualTo(""" + { + "uuid": "${json-unit.any-string}", + "subject": "EXPRESSION", + "operator": "MATCHES", + "value": "component.name == \\"foo\\"", + "violationType": "SECURITY" + } + """); + } + + @Test + public void testCreateExpressionConditionWithError() { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + + final Response response = target("%s/%s/condition".formatted(V1_POLICY, policy.getUuid())) + .request() + .header(X_API_KEY, apiKey) + .put(Entity.entity(""" + { + "subject": "EXPRESSION", + "value": "component.doesNotExist == \\"foo\\"", + "violationType": "SECURITY" + } + """, MediaType.APPLICATION_JSON)); + + assertThat(response.getStatus()).isEqualTo(400); + assertThatJson(getPlainTextBody(response)) + .isEqualTo(""" + { + "celErrors": [ + { + "line": 1, + "column": 9, + "message": "undefined field 'doesNotExist'" + } + ] + } + """); + } + + @Test + public void testUpdateExpressionCondition() { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition condition = qm.createPolicyCondition(policy, + PolicyCondition.Subject.VULNERABILITY_ID, PolicyCondition.Operator.IS, "foobar"); + + final Response response = target("%s/condition".formatted(V1_POLICY)) + .request() + .header(X_API_KEY, apiKey) + .post(Entity.entity(""" + { + "uuid": "%s", + "subject": "EXPRESSION", + "value": "component.name == \\"foo\\"", + "violationType": "OPERATIONAL" + } + """.formatted(condition.getUuid()), MediaType.APPLICATION_JSON)); + + assertThat(response.getStatus()).isEqualTo(201); + assertThatJson(getPlainTextBody(response)) + .isEqualTo(""" + { + "uuid": "${json-unit.any-string}", + "subject": "EXPRESSION", + "operator": "MATCHES", + "value": "component.name == \\"foo\\"", + "violationType": "OPERATIONAL" + } + """); + } + + @Test + public void testUpdateExpressionConditionWithError() { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition condition = qm.createPolicyCondition(policy, + PolicyCondition.Subject.VULNERABILITY_ID, PolicyCondition.Operator.IS, "foobar"); + + final Response response = target("%s/condition".formatted(V1_POLICY)) + .request() + .header(X_API_KEY, apiKey) + .post(Entity.entity(""" + { + "uuid": "%s", + "subject": "EXPRESSION", + "value": "component.doesNotExist == \\"foo\\"", + "violationType": "SECURITY" + } + """.formatted(condition.getUuid()), MediaType.APPLICATION_JSON)); + + assertThat(response.getStatus()).isEqualTo(400); + assertThatJson(getPlainTextBody(response)) + .isEqualTo(""" + { + "celErrors": [ + { + "line": 1, + "column": 9, + "message": "undefined field 'doesNotExist'" + } + ] + } + """); + } + +} \ No newline at end of file From 13e217a131b405ebfd8c4535ef84ffe6d0fc75dd Mon Sep 17 00:00:00 2001 From: vithikashukla Date: Thu, 28 Sep 2023 18:28:27 +0100 Subject: [PATCH 2/6] Add tests for Purl CelPolicy Signed-off-by: vithikashukla --- ...ackageUrlCelPolicyScriptSourceBuilder.java | 4 +- .../cel/compat/PackageUrlConditionTest.java | 79 +++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/PackageUrlConditionTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java index 45f9b0e92..20e1d83cb 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java @@ -2,7 +2,7 @@ import org.dependencytrack.model.PolicyCondition; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class PackageUrlCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @@ -10,7 +10,7 @@ public class PackageUrlCelPolicyScriptSourceBuilder implements CelPolicyScriptSo public String apply(final PolicyCondition policyCondition) { final String scriptSrc = """ component.purl.matches("%s") - """.formatted(escapeJson(policyCondition.getValue())); + """.formatted(escapeQuotes(policyCondition.getValue())); if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { return scriptSrc; diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/PackageUrlConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/PackageUrlConditionTest.java new file mode 100644 index 000000000..4b1f1988e --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/PackageUrlConditionTest.java @@ -0,0 +1,79 @@ +package org.dependencytrack.policy.cel.compat; + +import com.github.packageurl.PackageURL; +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@RunWith(JUnitParamsRunner.class) +public class PackageUrlConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + //Matches with exact match + new Object[]{PolicyCondition.Operator.MATCHES, "pkg:generic/acme/example-component@1.0", "pkg:generic/acme/example-component@1.0", true}, + //matching on null purl - invalid. We cannot pass null for purl + //new Object[]{PolicyCondition.Operator.NO_MATCH, ".+", "", true}, + //No Match exact + new Object[]{PolicyCondition.Operator.NO_MATCH, "pkg:generic/acme/example-component@1.0", "pkg:generic/acme/web-component@6.9", true}, + //Wrong operator + new Object[]{PolicyCondition.Operator.IS, "pkg:generic/acme/example-component@1.0", "pkg:generic/acme/example-component@1.0", false}, + //Exact match + new Object[]{PolicyCondition.Operator.MATCHES, "pkg:generic/acme/example-component@1.0", "pkg:generic/acme/example-component@1.0", true}, + //Matches with qualifier also + new Object[]{PolicyCondition.Operator.MATCHES, "pkg:generic/acme/example-component@1.0", "pkg:generic/acme/example-component@1.0?type=jar", true}, + //Partial match + new Object[]{PolicyCondition.Operator.MATCHES, "/com/acme/", "pkg:generic/com/acme/example-component@1.0?type=jar", true}, + //Partial match + new Object[]{PolicyCondition.Operator.MATCHES, "/com.acme/", "pkg:generic/com/acme/example-component@1.0?type=jar", true}, + //Matches on wild card + new Object[]{PolicyCondition.Operator.MATCHES, ".*com.acme.*", "pkg:generic/com/acme/example-component@1.0?type=jar", true}, + //Matches on wild card + new Object[]{PolicyCondition.Operator.MATCHES, ".*acme.*myCompany.*", "pkg:generic/com/acme/example-component@1.0-myCompanyFix-1?type=jar", true}, + //Matches on wild card + new Object[]{PolicyCondition.Operator.MATCHES, ".*(a|b|c)cme.*", "pkg:generic/com/acme/example-component@1.0?type=jar", true}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final PolicyCondition.Operator operator, final String conditionPurl, final String componentPurl, final boolean expectViolation) throws Exception{ + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.PACKAGE_URL, operator, conditionPurl); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setPurl(new PackageURL(componentPurl)); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + List violations = qm.getAllPolicyViolations(component); + assertThat(violations).hasSize(1); + PolicyViolation violation = violations.get(0); + assertEquals(component, violation.getComponent()); + assertEquals(condition, violation.getPolicyCondition()); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} From edce7463cc736b749db6ed93dda40faeafca8deb Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 28 Sep 2023 15:52:32 +0200 Subject: [PATCH 3/6] Bump version to `5.1.0-SNAPSHOT` Signed-off-by: Niklas --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e9108b4a6..ceb5b986e 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.dependencytrack dependency-track war - 5.0.4 + 5.1.0-SNAPSHOT Dependency-Track https://dependencytrack.org/ From 94c8d2769d5d1f73fc41315d3a78f47fe7f75b50 Mon Sep 17 00:00:00 2001 From: nscuro Date: Fri, 29 Sep 2023 11:47:23 +0200 Subject: [PATCH 4/6] Use `escapeQuotes` over `escapeJson` for values passed to CEL policy script sources Continuation of #337 Signed-off-by: nscuro --- .../compat/ComponentHashCelPolicyScriptSourceBuilder.java | 4 ++-- .../cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java | 6 +++--- .../cel/compat/LicenseCelPolicyScriptSourceBuilder.java | 4 ++-- .../compat/LicenseGroupCelPolicyScriptSourceBuilder.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java index 34e9dde97..c75faf798 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java @@ -5,7 +5,7 @@ import org.dependencytrack.model.PolicyCondition; import org.json.JSONObject; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class ComponentHashCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @@ -26,7 +26,7 @@ public String apply(final PolicyCondition policyCondition) { if (policyCondition.getOperator().equals(PolicyCondition.Operator.IS)) { return """ component.%s == "%s" - """.formatted(fieldName, escapeJson(hash.getValue())); + """.formatted(fieldName, escapeQuotes(hash.getValue())); } else { LOGGER.warn("Policy operator %s is not allowed with this policy".formatted(policyCondition.getOperator().toString())); return null; diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java index 721f1fe03..76fa18469 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java @@ -11,7 +11,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class CoordinatesCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @@ -52,7 +52,7 @@ private static String evaluateScript(final String conditionGroupPart, final Stri .build(); return """ component.group.matches("%s") && component.name.matches("%s") && component.matches_range("%s") - """.formatted(escapeJson(group), escapeJson(name), conditionVers.toString()); + """.formatted(escapeQuotes(group), escapeQuotes(name), conditionVers.toString()); } io.github.nscuro.versatile.Comparator versionComparator = switch (versionOperatorMatcher.group(1)) { @@ -76,7 +76,7 @@ private static String evaluateScript(final String conditionGroupPart, final Stri return """ component.group.matches("%s") && component.name.matches("%s") && component.matches_range("%s") - """.formatted(escapeJson(group), escapeJson(name), conditionVers.toString()); + """.formatted(escapeQuotes(group), escapeQuotes(name), conditionVers.toString()); } private static String replace(String conditionString) { diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java index 1927a7253..d66b36bf4 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java @@ -20,7 +20,7 @@ import org.dependencytrack.model.PolicyCondition; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class LicenseCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @@ -37,7 +37,7 @@ public String apply(final PolicyCondition policyCondition) { """; } } else { - final String escapedLicenseUuid = escapeJson(policyCondition.getValue()); + final String escapedLicenseUuid = escapeQuotes(policyCondition.getValue()); if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { return """ component.resolved_license.uuid == "%s" diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java index 87b8bf540..617458658 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java @@ -2,7 +2,7 @@ import org.dependencytrack.model.PolicyCondition; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class LicenseGroupCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @@ -10,7 +10,7 @@ public class LicenseGroupCelPolicyScriptSourceBuilder implements CelPolicyScript public String apply(final PolicyCondition policyCondition) { final String scriptSrc = """ component.resolved_license.groups.exists(group, group.uuid == "%s") - """.formatted(escapeJson(policyCondition.getValue())); + """.formatted(escapeQuotes(policyCondition.getValue())); if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { return scriptSrc; From 6fb17ce128d36865124d03a1611c70f983db38d0 Mon Sep 17 00:00:00 2001 From: mehab Date: Sat, 30 Sep 2023 07:54:35 +0100 Subject: [PATCH 5/6] merged changes from main Signed-off-by: mehab --- .../org/dependencytrack/common/ConfigKey.java | 5 +- .../event/IntegrityMetaInitializer.java | 97 +++++++++++ .../dependencytrack/model/FetchStatus.java | 1 - .../persistence/ComponentQueryManager.java | 25 ++- .../IntegrityMetaQueryManager.java | 162 ++++++++++++++++++ .../persistence/QueryManager.java | 31 +++- .../org/dependencytrack/tasks/LockName.java | 3 +- .../tasks/RepositoryMetaAnalyzerTask.java | 4 +- .../dependencytrack/util/LockProvider.java | 7 + src/main/resources/application.properties | 8 +- src/main/webapp/WEB-INF/web.xml | 3 + .../event/IntegrityMetaInitializerTest.java | 121 +++++++++++++ .../ComponentQueryManagerTest.java | 76 -------- .../IntegrityMetaQueryManagerTest.java | 106 ++++++++++++ 14 files changed, 562 insertions(+), 87 deletions(-) create mode 100644 src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java create mode 100644 src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java create mode 100644 src/test/java/org/dependencytrack/event/IntegrityMetaInitializerTest.java create mode 100644 src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java diff --git a/src/main/java/org/dependencytrack/common/ConfigKey.java b/src/main/java/org/dependencytrack/common/ConfigKey.java index a78107457..8afdf2875 100644 --- a/src/main/java/org/dependencytrack/common/ConfigKey.java +++ b/src/main/java/org/dependencytrack/common/ConfigKey.java @@ -71,7 +71,10 @@ public enum ConfigKey implements Config.Key { WORKFLOW_RETENTION_DURATION("workflow.retention.duration", "P3D"), WORKFLOW_STEP_TIMEOUT_DURATION("workflow.step.timeout.duration", "PT1H"), TMP_DELAY_BOM_PROCESSED_NOTIFICATION("tmp.delay.bom.processed.notification", "false"), - CEL_POLICY_ENGINE_ENABLED("cel.policy.engine.enabled", "false"); + CEL_POLICY_ENGINE_ENABLED("cel.policy.engine.enabled", "false"), + TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR("task.integrity.integrityMetaInitializer.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), + TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR("task.integrity.integrityMetaInitializer.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())), + INTEGRITY_INITIALIZER_ENABLED("integrity.initializer.enabled", "false"); private final String propertyName; private final Object defaultValue; diff --git a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java new file mode 100644 index 000000000..8d0bd8c55 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java @@ -0,0 +1,97 @@ +package org.dependencytrack.event; + +import alpine.Config; +import alpine.common.logging.Logger; +import net.javacrumbs.shedlock.core.LockConfiguration; +import net.javacrumbs.shedlock.core.LockExtender; +import net.javacrumbs.shedlock.core.LockingTaskExecutor; +import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.util.LockProvider; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_TASK_LOCK; +import static org.dependencytrack.util.LockProvider.isLockToBeExtended; + +public class IntegrityMetaInitializer implements ServletContextListener { + + private static final Logger LOGGER = Logger.getLogger(IntegrityMetaInitializer.class); + private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + private final boolean integrityInitializerEnabled; + + public IntegrityMetaInitializer() { + this(Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_INITIALIZER_ENABLED)); + } + + IntegrityMetaInitializer(final boolean integrityInitializerEnabled) { + this.integrityInitializerEnabled = integrityInitializerEnabled; + } + + + @Override + public void contextInitialized(final ServletContextEvent event) { + if (integrityInitializerEnabled) { + try { + LockProvider.executeWithLock(INTEGRITY_META_INITIALIZER_TASK_LOCK, (LockingTaskExecutor.Task) () -> process()); + } catch (Throwable e) { + throw new RuntimeException("An unexpected error occurred while running Initializer for integrity meta", e); + } + } else { + LOGGER.info("Component integrity initializer is disabled."); + } + } + + private void process() { + LOGGER.info("Initializing integrity meta component sync"); + try (final var qm = new QueryManager()) { + if (qm.getIntegrityMetaComponentCount() == 0) { + // Sync purls from Component only if IntegrityMetaComponent is empty + qm.synchronizeIntegrityMetaComponent(); + } + // dispatch purls not processed yet + batchProcessPurls(qm); + } + } + + private void batchProcessPurls(QueryManager qm) { + LockConfiguration lockConfiguration = LockProvider.getLockConfigurationByLockName(INTEGRITY_META_INITIALIZER_TASK_LOCK); + long offset = 0; + long startTime = System.currentTimeMillis(); + List purls = qm.fetchNextPurlsPage(offset); + while (!purls.isEmpty()) { + long cumulativeProcessingTime = System.currentTimeMillis() - startTime; + if(isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) { + LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); + } + dispatchPurls(qm, purls); + updateIntegrityMetaForPurls(qm, purls); + offset += purls.size(); + purls = qm.fetchNextPurlsPage(offset); + } + } + + private void updateIntegrityMetaForPurls(QueryManager qm, List purls) { + List purlRecords = new ArrayList<>(); + for (var purl : purls) { + purlRecords.add(qm.getIntegrityMetaComponent(purl)); + } + qm.batchUpdateIntegrityMetaComponent(purlRecords); + } + + private void dispatchPurls(QueryManager qm, List purls) { + for (final var purl : purls) { + ComponentProjection componentProjection = qm.getComponentByPurl(purl); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal)); + } + } + + public record ComponentProjection(String purlCoordinates, Boolean internal) { + } +} diff --git a/src/main/java/org/dependencytrack/model/FetchStatus.java b/src/main/java/org/dependencytrack/model/FetchStatus.java index c376fbfc6..3723c0c97 100644 --- a/src/main/java/org/dependencytrack/model/FetchStatus.java +++ b/src/main/java/org/dependencytrack/model/FetchStatus.java @@ -3,7 +3,6 @@ public enum FetchStatus { //request processed successfully PROCESSED, - //fetching information for this component is in progress IN_PROGRESS, diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java index 1432c86b6..238443fb0 100644 --- a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java @@ -24,13 +24,13 @@ import alpine.model.UserPrincipal; import alpine.persistence.PaginatedResult; import alpine.resources.AlpineRequest; -import alpine.server.util.DbUtil; import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; +import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; +import org.dependencytrack.event.IntegrityMetaInitializer; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; import org.dependencytrack.model.ConfigPropertyConstants; -import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.Project; import org.dependencytrack.model.RepositoryMetaComponent; import org.dependencytrack.model.RepositoryType; @@ -42,9 +42,12 @@ import javax.json.JsonArray; import javax.json.JsonValue; import java.io.StringReader; +<<<<<<< HEAD import java.sql.Connection; import java.sql.PreparedStatement; import java.time.Instant; +======= +>>>>>>> a0c2364d (Add initialiser for integrity meta component sync (#334)) import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -196,6 +199,21 @@ public PaginatedResult getComponentByHash(String hash) { return execute(query, params); } + /** + * Returns ComponentProjection for the purl. + * @param purl the purl of the component to retrieve + * @return associated ComponentProjection + */ + public IntegrityMetaInitializer.ComponentProjection getComponentByPurl(String purl) { + if (purl == null) { + return null; + } + final Query query = pm.newQuery(Component.class, "purl == :purl"); + query.setParameters(purl); + query.setResult("DISTINCT purlCoordinates, internal"); + return query.executeResultUnique(IntegrityMetaInitializer.ComponentProjection.class); + } + /** * Returns Components by their identity. * @param identity the ComponentIdentity to query against @@ -725,6 +743,7 @@ private void getDirectDependenciesForPathDependencies(Map dep } dependencyGraph.putAll(addToDependencyGraph); } +<<<<<<< HEAD /** * Returns a IntegrityMetaComponent object from the specified purl. @@ -792,4 +811,6 @@ public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponen return persist(integrityMetaComponent); } +======= +>>>>>>> a0c2364d (Add initialiser for integrity meta component sync (#334)) } diff --git a/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java b/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java new file mode 100644 index 000000000..841d50741 --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java @@ -0,0 +1,162 @@ +package org.dependencytrack.persistence; + +import alpine.common.logging.Logger; +import alpine.resources.AlpineRequest; +import alpine.server.util.DbUtil; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; + +import javax.jdo.PersistenceManager; +import javax.jdo.Query; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; +import java.util.List; + +public class IntegrityMetaQueryManager extends QueryManager implements IQueryManager { + + private static final Logger LOGGER = Logger.getLogger(IntegrityMetaQueryManager.class); + + /** + * Constructs a new QueryManager. + * @param pm a PersistenceManager object + */ + IntegrityMetaQueryManager(final PersistenceManager pm) { + super(pm); + } + + /** + * Constructs a new QueryManager. + * @param pm a PersistenceManager object + * @param request an AlpineRequest object + */ + IntegrityMetaQueryManager(final PersistenceManager pm, final AlpineRequest request) { + super(pm, request); + } + + /** + * Returns a IntegrityMetaComponent object from the specified purl. + * + * @param purl the Package URL string of the component + * @return a IntegrityMetaComponent object, or null if not found + */ + public IntegrityMetaComponent getIntegrityMetaComponent(String purl) { + final Query query = pm.newQuery(IntegrityMetaComponent.class, "purl == :purl"); + query.setParameters(purl); + return query.executeUnique(); + } + + /** + * Updates a IntegrityMetaComponent record. + * + * @param transientIntegrityMetaComponent the IntegrityMetaComponent object to synchronize + * @return a synchronized IntegrityMetaComponent object + */ + public synchronized IntegrityMetaComponent updateIntegrityMetaComponent(final IntegrityMetaComponent transientIntegrityMetaComponent) { + final IntegrityMetaComponent integrityMeta = getIntegrityMetaComponent(transientIntegrityMetaComponent.getPurl()); + if (integrityMeta != null) { + integrityMeta.setMd5(transientIntegrityMetaComponent.getMd5()); + integrityMeta.setSha1(transientIntegrityMetaComponent.getSha1()); + integrityMeta.setSha256(transientIntegrityMetaComponent.getSha256()); + integrityMeta.setPublishedAt(transientIntegrityMetaComponent.getPublishedAt()); + integrityMeta.setStatus(transientIntegrityMetaComponent.getStatus()); + integrityMeta.setLastFetch(Date.from(Instant.now())); + return persist(integrityMeta); + } else { + LOGGER.debug("No record found in IntegrityMetaComponent for purl " + transientIntegrityMetaComponent.getPurl()); + return null; + } + } + + /** + * Synchronizes IntegrityMetaComponent with purls from COMPONENT. This is part of initializer. + */ + public synchronized void synchronizeIntegrityMetaComponent() { + final String purlSyncQuery = """ + INSERT INTO "INTEGRITY_META_COMPONENT" ("PURL") + SELECT DISTINCT "PURL" + FROM "COMPONENT" + WHERE "PURL" IS NOT NULL + """; + Connection connection = null; + PreparedStatement preparedStatement = null; + try { + connection = (Connection) pm.getDataStoreConnection(); + preparedStatement = connection.prepareStatement(purlSyncQuery); + var purlCount = preparedStatement.executeUpdate(); + LOGGER.info("Number of component purls synchronized for integrity check : " + purlCount); + } catch (Exception ex) { + LOGGER.error("Error in synchronizing component purls for integrity meta.", ex); + throw new RuntimeException(ex); + } finally { + DbUtil.close(preparedStatement); + DbUtil.close(connection); + } + } + + /** + * Returns the count of records in IntegrityMetaComponent. + * + * @return the count of records + */ + public long getIntegrityMetaComponentCount() { + try (final Query query = pm.newQuery(IntegrityMetaComponent.class)) { + query.setResult("count(this)"); + return query.executeResultUnique(Long.class); + } catch (Exception e) { + LOGGER.error("Error in getting count of integrity meta.", e); + throw new RuntimeException(e); + } + } + + /** + * Returns the list of purls in IntegrityMetaComponent which are not yet processed. + * + * @return the list of purls + */ + public List fetchNextPurlsPage(long offset) { + try (final Query query = + pm.newQuery(IntegrityMetaComponent.class, "status == null || (status == :inProgress && lastFetch < :latest)")) { + query.setParameters(FetchStatus.IN_PROGRESS, Date.from(Instant.now().minus(1, ChronoUnit.HOURS))); + query.setRange(offset, offset + 5000); + query.setResult("purl"); + return List.copyOf(query.executeResultList(String.class)); + } catch (Exception e) { + LOGGER.error("Error in getting purls from integrity meta.", e); + throw new RuntimeException(e); + } + } + + /** + * Batch updates IntegrityMetaComponent records + */ + public void batchUpdateIntegrityMetaComponent(List purls) { + final String updateQuery = """ + UPDATE "INTEGRITY_META_COMPONENT" + SET "LAST_FETCH" = ?, "STATUS" = ? + WHERE "ID" = ? + """; + Connection connection = null; + PreparedStatement preparedStatement = null; + try { + connection = (Connection) pm.getDataStoreConnection(); + preparedStatement = connection.prepareStatement(updateQuery); + for (var purlRecord : purls) { + preparedStatement.setTimestamp(1, new Timestamp(Date.from(Instant.now()).getTime())); + preparedStatement.setString(2, FetchStatus.IN_PROGRESS.toString()); + preparedStatement.setLong(3, purlRecord.getId()); + preparedStatement.addBatch(); + } + preparedStatement.executeBatch(); + } catch (Exception ex) { + LOGGER.error("Error in batch updating integrity meta.", ex); + throw new RuntimeException(ex); + } finally { + DbUtil.close(preparedStatement); + DbUtil.close(connection); + } + } +} diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index f392b7af2..9e0df5975 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -33,6 +33,7 @@ import org.apache.commons.lang3.ClassUtils; import org.datanucleus.PropertyNames; import org.datanucleus.api.jdo.JDOQuery; +import org.dependencytrack.event.IntegrityMetaInitializer; import org.dependencytrack.model.AffectedVersionAttribution; import org.dependencytrack.model.Analysis; import org.dependencytrack.model.AnalysisComment; @@ -132,6 +133,7 @@ public class QueryManager extends AlpineQueryManager { private VulnerabilityQueryManager vulnerabilityQueryManager; private VulnerableSoftwareQueryManager vulnerableSoftwareQueryManager; private WorkflowStateQueryManager workflowStateQueryManager; + private IntegrityMetaQueryManager integrityMetaQueryManager; private TagQueryManager tagQueryManager; @@ -350,6 +352,13 @@ private WorkflowStateQueryManager getWorkflowStateQueryManager() { return workflowStateQueryManager; } + private IntegrityMetaQueryManager getIntegrityMetaQueryManager() { + if (integrityMetaQueryManager == null) { + integrityMetaQueryManager = (request == null) ? new IntegrityMetaQueryManager(getPersistenceManager()) : new IntegrityMetaQueryManager(getPersistenceManager(), request); + } + return integrityMetaQueryManager; + } + /** * Get the IDs of the {@link Team}s a given {@link Principal} is a member of. * @@ -565,6 +574,10 @@ public PaginatedResult getComponentByHash(String hash) { return getComponentQueryManager().getComponentByHash(hash); } + public IntegrityMetaInitializer.ComponentProjection getComponentByPurl(String purl) { + return getComponentQueryManager().getComponentByPurl(purl); + } + public PaginatedResult getComponents(ComponentIdentity identity) { return getComponentQueryManager().getComponents(identity); } @@ -1687,15 +1700,27 @@ public boolean hasWorkflowStepWithStatus(final UUID token, final WorkflowStep st } public IntegrityMetaComponent getIntegrityMetaComponent(String purl) { - return getComponentQueryManager().getIntegrityMetaComponent(purl); + return getIntegrityMetaQueryManager().getIntegrityMetaComponent(purl); } public IntegrityMetaComponent updateIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { - return getComponentQueryManager().updateIntegrityMetaComponent(integrityMetaComponent); + return getIntegrityMetaQueryManager().updateIntegrityMetaComponent(integrityMetaComponent); } public void synchronizeIntegrityMetaComponent() { - getComponentQueryManager().synchronizeIntegrityMetaComponent(); + getIntegrityMetaQueryManager().synchronizeIntegrityMetaComponent(); + } + + public long getIntegrityMetaComponentCount() { + return getIntegrityMetaQueryManager().getIntegrityMetaComponentCount(); + } + + public List fetchNextPurlsPage(long offset) { + return getIntegrityMetaQueryManager().fetchNextPurlsPage(offset); + } + + public void batchUpdateIntegrityMetaComponent(List purls) { + getIntegrityMetaQueryManager().batchUpdateIntegrityMetaComponent(purls); } public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { diff --git a/src/main/java/org/dependencytrack/tasks/LockName.java b/src/main/java/org/dependencytrack/tasks/LockName.java index 4b34044c8..9b25cee2c 100644 --- a/src/main/java/org/dependencytrack/tasks/LockName.java +++ b/src/main/java/org/dependencytrack/tasks/LockName.java @@ -8,5 +8,6 @@ public enum LockName { INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK, WORKFLOW_STEP_CLEANUP_TASK_LOCK, PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, - PORTFOLIO_VULN_ANALYSIS_TASK_LOCK + PORTFOLIO_VULN_ANALYSIS_TASK_LOCK, + INTEGRITY_META_INITIALIZER_TASK_LOCK } diff --git a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java index 50ec0bc00..bb18a5efd 100644 --- a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java +++ b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java @@ -102,7 +102,7 @@ private void processProject(final UUID projectUuid) throws Exception { dispatchComponents(components, qm); offset += components.size(); - components = fetchNextComponentsPage(qm.getPersistenceManager(), project, offset); + components = fetchNextComponentsPage(pm, project, offset); } } @@ -129,7 +129,7 @@ private void processPortfolio() throws Exception { dispatchComponents(components, qm); offset += components.size(); - components = fetchNextComponentsPage(qm.getPersistenceManager(), null, offset); + components = fetchNextComponentsPage(pm, null, offset); } } diff --git a/src/main/java/org/dependencytrack/util/LockProvider.java b/src/main/java/org/dependencytrack/util/LockProvider.java index e03a254ea..86cbd2d4e 100644 --- a/src/main/java/org/dependencytrack/util/LockProvider.java +++ b/src/main/java/org/dependencytrack/util/LockProvider.java @@ -20,6 +20,8 @@ import static org.dependencytrack.common.ConfigKey.TASK_COMPONENT_IDENTIFICATION_LOCK_AT_LEAST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_COMPONENT_IDENTIFICATION_LOCK_AT_MOST_FOR; +import static org.dependencytrack.common.ConfigKey.TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR; +import static org.dependencytrack.common.ConfigKey.TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_LDAP_SYNC_LOCK_AT_MOST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_METRICS_VULNERABILITY_LOCK_AT_LEAST_FOR; @@ -35,6 +37,7 @@ import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_LEAST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_MOST_FOR; import static org.dependencytrack.tasks.LockName.EPSS_MIRROR_TASK_LOCK; +import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_TASK_LOCK; import static org.dependencytrack.tasks.LockName.INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK; import static org.dependencytrack.tasks.LockName.LDAP_SYNC_TASK_LOCK; import static org.dependencytrack.tasks.LockName.PORTFOLIO_METRICS_TASK_LOCK; @@ -134,6 +137,10 @@ public static LockConfiguration getLockConfigurationByLockName(LockName lockName PORTFOLIO_VULN_ANALYSIS_TASK_LOCK.name(), Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_MOST_FOR)), Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_LEAST_FOR))); + case INTEGRITY_META_INITIALIZER_TASK_LOCK -> new LockConfiguration(Instant.now(), + INTEGRITY_META_INITIALIZER_TASK_LOCK.name(), + Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR)), + Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR))); }; } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index bc636af72..220dd688d 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -456,6 +456,8 @@ task.portfolio.repoMetaAnalysis.lockAtMostForInMillis=900000 task.portfolio.repoMetaAnalysis.lockAtLeastForInMillis=90000 task.portfolio.vulnAnalysis.lockAtMostForInMillis=900000 task.portfolio.vulnAnalysis.lockAtLeastForInMillis=90000 +task.integrity.integrityMetaInitializer.lockAtMostForInMillis=900000 +task.integrity.integrityMetaInitializer.lockAtLeastForInMillis=90000 #schedule task for 10th minute of every hour task.cron.metrics.portfolio=10 * * * * @@ -516,4 +518,8 @@ tmp.delay.bom.processed.notification=false # Optional # Specifies whether the Common Expression Language (CEL) based policy engine shall be enabled. -cel.policy.engine.enabled=false \ No newline at end of file +cel.policy.engine.enabled=false + +# Optional +# Specifies whether the Integrity Initializer shall be enabled. +integrity.initializer.enabled=false \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 1522a2589..9431d0f1c 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -59,6 +59,9 @@ org.dependencytrack.persistence.H2WebConsoleInitializer + + org.dependencytrack.event.IntegrityMetaInitializer + WhitelistUrlFilter diff --git a/src/test/java/org/dependencytrack/event/IntegrityMetaInitializerTest.java b/src/test/java/org/dependencytrack/event/IntegrityMetaInitializerTest.java new file mode 100644 index 000000000..b7432b25d --- /dev/null +++ b/src/test/java/org/dependencytrack/event/IntegrityMetaInitializerTest.java @@ -0,0 +1,121 @@ +package org.dependencytrack.event; + +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Date; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.model.FetchStatus.IN_PROGRESS; +import static org.dependencytrack.model.FetchStatus.PROCESSED; + +public class IntegrityMetaInitializerTest extends AbstractPostgresEnabledTest { + + final Component componentPersisted = new Component(); + + @Before + public void persistComponentData() { + final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); + componentPersisted.setProject(projectA); + componentPersisted.setName("acme-lib-a"); + componentPersisted.setInternal(false); + componentPersisted.setPurlCoordinates("pkg:maven/acme/acme-lib-a@1.0.1"); + componentPersisted.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); + qm.persist(componentPersisted); + kafkaMockProducer.clear(); + } + + @Test + public void testIntegrityMetaInitializerWhenDisabledByDefault() { + IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(); + initializer.contextInitialized(null); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(0); + assertThat(kafkaMockProducer.history().size()).isEqualTo(0); + } + + @Test + public void testIntegrityMetaInitializer() { + IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); + initializer.contextInitialized(null); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()) + ); + assertThat(qm.getIntegrityMetaComponent(componentPersisted.getPurl().toString())).satisfies( + meta -> { + assertThat(meta.getStatus()).isEqualTo(IN_PROGRESS); + assertThat(meta.getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); + assertThat(meta.getId()).isEqualTo(1L); + assertThat(meta.getMd5()).isNull(); + assertThat(meta.getSha1()).isNull(); + assertThat(meta.getSha256()).isNull(); + assertThat(meta.getLastFetch()).isNotNull(); + assertThat(meta.getPublishedAt()).isNull(); + } + ); + } + + @Test + public void testIntegrityMetaInitializerWithExistingDataProcessed() { + var integrityMetaExisting = new IntegrityMetaComponent(); + integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); + integrityMetaExisting.setStatus(PROCESSED); + qm.persist(integrityMetaExisting); + // data exists in IntegrityMetaComponent so sync will be skipped + IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); + initializer.contextInitialized(null); + // kafka event is not dispatched + assertThat(kafkaMockProducer.history().size()).isEqualTo(0); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + } + + @Test + public void testIntegrityMetaInitializerWithExistingDataNotProcessed() { + // data exists in IntegrityMetaComponent but not processed yet + var integrityMetaExisting = new IntegrityMetaComponent(); + integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); + qm.persist(integrityMetaExisting); + IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); + initializer.contextInitialized(null); + // kafka event is dispatched + assertThat(kafkaMockProducer.history().size()).isEqualTo(1); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + } + + @Test + public void testIntegrityMetaInitializerWithExistingDataFetchedNotRecently() { + // data exists in IntegrityMetaComponent but last fetched 3 hours ago > 1 hour wait time + var integrityMetaExisting = new IntegrityMetaComponent(); + integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); + integrityMetaExisting.setStatus(IN_PROGRESS); + integrityMetaExisting.setLastFetch(Date.from(Instant.now().minus(3, ChronoUnit.HOURS))); + qm.persist(integrityMetaExisting); + IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); + initializer.contextInitialized(null); + // kafka event is dispatched + assertThat(kafkaMockProducer.history().size()).isEqualTo(1); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + } + + @Test + public void testIntegrityMetaInitializerWithExistingDataFetchedRecently() { + // data exists in IntegrityMetaComponent but last fetched 30 min ago < 1 hour wait time + var integrityMetaExisting = new IntegrityMetaComponent(); + integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); + integrityMetaExisting.setStatus(IN_PROGRESS); + integrityMetaExisting.setLastFetch(Date.from(Instant.now().minus(30, ChronoUnit.MINUTES))); + qm.persist(integrityMetaExisting); + + IntegrityMetaInitializer initializer = new IntegrityMetaInitializer(true); + initializer.contextInitialized(null); + // kafka event is dispatched + assertThat(kafkaMockProducer.history().size()).isEqualTo(0); + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); + } +} diff --git a/src/test/java/org/dependencytrack/persistence/ComponentQueryManagerTest.java b/src/test/java/org/dependencytrack/persistence/ComponentQueryManagerTest.java index de2063503..1a5086733 100644 --- a/src/test/java/org/dependencytrack/persistence/ComponentQueryManagerTest.java +++ b/src/test/java/org/dependencytrack/persistence/ComponentQueryManagerTest.java @@ -8,8 +8,6 @@ import org.dependencytrack.model.AnalyzerIdentity; import org.dependencytrack.model.Component; import org.dependencytrack.model.DependencyMetrics; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.model.PolicyViolation; @@ -20,8 +18,6 @@ import org.junit.Test; import javax.jdo.JDOObjectNotFoundException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.Date; import static org.assertj.core.api.Assertions.assertThat; @@ -110,77 +106,5 @@ public void recursivelyDeleteTest() { assertThatNoException().isThrownBy(() -> qm.getObjectById(Policy.class, policy.getId())); } - @Test - public void testGetIntegrityMetaComponent() { - var integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.IN_PROGRESS); - integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); - - var result = qm.getIntegrityMetaComponent("pkg:maven/acme/example@1.0.0?type=jar"); - assertThat(result).isNull(); - - result = qm.persist(integrityMeta); - assertThat(qm.getIntegrityMetaComponent(result.getPurl())).satisfies( - meta -> { - assertThat(meta.getStatus()).isEqualTo(FetchStatus.IN_PROGRESS); - assertThat(meta.getId()).isEqualTo(1L); - assertThat(meta.getMd5()).isNull(); - assertThat(meta.getSha1()).isNull(); - assertThat(meta.getSha256()).isNull(); - assertThat(meta.getLastFetch()).isEqualTo(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); - assertThat(meta.getPublishedAt()).isNull(); - } - ); - } - - @Test - public void testUpdateIntegrityMetaComponent() { - var integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.IN_PROGRESS); - integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); - var result = qm.updateIntegrityMetaComponent(integrityMeta); - assertThat(result).isNull(); - - var persisted = qm.persist(integrityMeta); - persisted.setStatus(FetchStatus.PROCESSED); - result = qm.updateIntegrityMetaComponent(persisted); - assertThat(result.getStatus()).isEqualTo(FetchStatus.PROCESSED); - } - - @Test - public void testSynchronizeIntegrityMetaComponent() { - final var project = new Project(); - project.setName("acme-app"); - project.setVersion("1.0.0"); - qm.persist(project); - - final var component = new Component(); - component.setProject(project); - component.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - component.setName("acme-lib"); - - // without any component in database - qm.synchronizeIntegrityMetaComponent(); - assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).isNull(); - - // with existing component in database - qm.persist(component); - qm.synchronizeIntegrityMetaComponent(); - - assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).satisfies( - meta -> { - assertThat(meta.getStatus()).isNull(); - assertThat(meta.getPurl()).isEqualTo("pkg:maven/acme/example@1.0.0?type=jar"); - assertThat(meta.getId()).isEqualTo(1L); - assertThat(meta.getMd5()).isNull(); - assertThat(meta.getSha1()).isNull(); - assertThat(meta.getSha256()).isNull(); - assertThat(meta.getLastFetch()).isNull(); - assertThat(meta.getPublishedAt()).isNull(); - } - ); - } } \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java new file mode 100644 index 000000000..3269f3a9b --- /dev/null +++ b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java @@ -0,0 +1,106 @@ +package org.dependencytrack.persistence; + +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.model.Project; +import org.junit.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; + +import static org.assertj.core.api.Assertions.assertThat; + +public class IntegrityMetaQueryManagerTest extends PersistenceCapableTest { + + @Test + public void testGetIntegrityMetaComponent() { + var integrityMeta = new IntegrityMetaComponent(); + integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); + + var result = qm.getIntegrityMetaComponent("pkg:maven/acme/example@1.0.0?type=jar"); + assertThat(result).isNull(); + + result = qm.persist(integrityMeta); + assertThat(qm.getIntegrityMetaComponent(result.getPurl())).satisfies( + meta -> { + assertThat(meta.getStatus()).isEqualTo(FetchStatus.IN_PROGRESS); + assertThat(meta.getId()).isEqualTo(1L); + assertThat(meta.getMd5()).isNull(); + assertThat(meta.getSha1()).isNull(); + assertThat(meta.getSha256()).isNull(); + assertThat(meta.getLastFetch()).isEqualTo(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); + assertThat(meta.getPublishedAt()).isNull(); + } + ); + } + + @Test + public void testUpdateIntegrityMetaComponent() { + var integrityMeta = new IntegrityMetaComponent(); + integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); + + var result = qm.updateIntegrityMetaComponent(integrityMeta); + assertThat(result).isNull(); + + var persisted = qm.persist(integrityMeta); + persisted.setStatus(FetchStatus.PROCESSED); + result = qm.updateIntegrityMetaComponent(persisted); + assertThat(result.getStatus()).isEqualTo(FetchStatus.PROCESSED); + } + + @Test + public void testSynchronizeIntegrityMetaComponent() { + final var project = new Project(); + project.setName("acme-app"); + project.setVersion("1.0.0"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); + component.setName("acme-lib"); + + // without any component in database + qm.synchronizeIntegrityMetaComponent(); + assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).isNull(); + + // with existing component in database + qm.persist(component); + qm.synchronizeIntegrityMetaComponent(); + + assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).satisfies( + meta -> { + assertThat(meta.getStatus()).isNull(); + assertThat(meta.getPurl()).isEqualTo("pkg:maven/acme/example@1.0.0?type=jar"); + assertThat(meta.getId()).isEqualTo(1L); + assertThat(meta.getMd5()).isNull(); + assertThat(meta.getSha1()).isNull(); + assertThat(meta.getSha256()).isNull(); + assertThat(meta.getLastFetch()).isNull(); + assertThat(meta.getPublishedAt()).isNull(); + } + ); + } + + @Test + public void testGetIntegrityMetaComponentCount() { + var integrityMeta = new IntegrityMetaComponent(); + integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + qm.persist(integrityMeta); + + integrityMeta = new IntegrityMetaComponent(); + integrityMeta.setPurl("pkg:npm/acme/example@2.0.0"); + integrityMeta.setStatus(FetchStatus.PROCESSED); + qm.persist(integrityMeta); + + assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(2); + } +} From 0c1d184d62307d3902bb24a82718e474c8a00e52 Mon Sep 17 00:00:00 2001 From: mehab Date: Mon, 2 Oct 2023 07:24:00 +0100 Subject: [PATCH 6/6] unit tests added Signed-off-by: mehab --- .../event/IntegrityMetaInitializer.java | 4 +- .../event/kafka/componentmeta/Handler.java | 4 +- .../componentmeta/SupportedMetaHandler.java | 9 +- .../componentmeta/UnSupportedMetaHandler.java | 4 +- .../persistence/QueryManager.java | 3 - .../componentmeta/HandlerFactoryTest.java | 46 +++++++ .../SupportedMetaHandlerTest.java | 119 ++++++++++++++++++ .../UnSupportedMetaHandlerTest.java | 50 ++++++++ 8 files changed, 229 insertions(+), 10 deletions(-) create mode 100644 src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java create mode 100644 src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java create mode 100644 src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java diff --git a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java index 8d0bd8c55..6b3d6c494 100644 --- a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java +++ b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java @@ -67,7 +67,7 @@ private void batchProcessPurls(QueryManager qm) { List purls = qm.fetchNextPurlsPage(offset); while (!purls.isEmpty()) { long cumulativeProcessingTime = System.currentTimeMillis() - startTime; - if(isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) { + if (isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) { LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor()); } dispatchPurls(qm, purls); @@ -88,7 +88,7 @@ private void updateIntegrityMetaForPurls(QueryManager qm, List purls) { private void dispatchPurls(QueryManager qm, List purls) { for (final var purl : purls) { ComponentProjection componentProjection = qm.getComponentByPurl(purl); - kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal)); + kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal, true, false)); } } diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java index b02c7a724..6e77f89c8 100644 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java @@ -1,5 +1,7 @@ package org.dependencytrack.event.kafka.componentmeta; +import org.dependencytrack.model.IntegrityMetaComponent; + public interface Handler { - void handle(); + IntegrityMetaComponent handle(); } diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java index 9a539d74c..494e68b99 100644 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java @@ -21,21 +21,24 @@ public SupportedMetaHandler(ComponentProjection componentProjection, QueryManage } @Override - public void handle() { + public IntegrityMetaComponent handle() { KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); try (QueryManager queryManager = new QueryManager()) { IntegrityMetaComponent integrityMetaComponent = queryManager.getIntegrityMetaComponent(componentProjection.purl()); if (integrityMetaComponent != null) { if (integrityMetaComponent.getStatus() == null || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime() > TIME_SPAN)) { integrityMetaComponent.setLastFetch(Date.from(Instant.now())); - queryManager.updateIntegrityMetaComponent(integrityMetaComponent); + IntegrityMetaComponent integrityMetaComponent1 = queryManager.updateIntegrityMetaComponent(integrityMetaComponent); kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion)); + return integrityMetaComponent1; } else { kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion)); + return integrityMetaComponent; } } else { - queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl())); + IntegrityMetaComponent integrityMetaComponent1 = queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl())); kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion)); + return integrityMetaComponent1; } } } diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java index 09aa1d2ac..c9babf3e9 100644 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java +++ b/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java @@ -2,6 +2,7 @@ import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.persistence.QueryManager; public class UnSupportedMetaHandler extends AbstractMetaHandler { @@ -14,8 +15,9 @@ public UnSupportedMetaHandler(ComponentProjection componentProjection, QueryMana } @Override - public void handle() { + public IntegrityMetaComponent handle() { KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion)); + return null; } } diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index 7d3d6465c..9e0df5975 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -1721,12 +1721,9 @@ public List fetchNextPurlsPage(long offset) { public void batchUpdateIntegrityMetaComponent(List purls) { getIntegrityMetaQueryManager().batchUpdateIntegrityMetaComponent(purls); -<<<<<<< HEAD } public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { return getComponentQueryManager().createIntegrityMetaComponent(integrityMetaComponent); -======= ->>>>>>> a0c2364d6f47ccf6d44f3c44724d1fae65604740 } } diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java new file mode 100644 index 000000000..c877a7d38 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java @@ -0,0 +1,46 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import alpine.common.logging.Logger; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.util.PurlUtil; +import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class HandlerFactoryTest extends PersistenceCapableTest { + + private static final Logger LOGGER = Logger.getLogger(HandlerFactoryTest.class); + + @Test + public void createHandlerForSupportedPackageTest() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + assertTrue(handler instanceof SupportedMetaHandler); + } catch (MalformedPackageURLException e) { + LOGGER.warn("Package url not formed correctly"); + } + + } + + @Test + public void createHandlerForUnSupportedPackageTest() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:golang/github.com/foo/bar@1.2.3"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + assertTrue(handler instanceof UnSupportedMetaHandler); + } catch (MalformedPackageURLException e) { + throw new RuntimeException(e); + } + + } +} diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java new file mode 100644 index 000000000..0998f8f62 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java @@ -0,0 +1,119 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import alpine.common.logging.Logger; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.util.PurlUtil; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.util.KafkaTestUtil.deserializeValue; + +public class SupportedMetaHandlerTest extends AbstractPostgresEnabledTest { + private static final Logger LOGGER = Logger.getLogger(SupportedMetaHandlerTest.class); + + @Test + public void testHandleIntegrityComponentNotInDB() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl()); + Assertions.assertNull(integrityMetaComponent); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + IntegrityMetaComponent result = handler.handle(); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); + assertThat(command.getComponent().getInternal()).isFalse(); + assertThat(command.getFetchIntegrityData()).isTrue(); + assertThat(command.getFetchLatestVersion()).isFalse(); + } + + ); + Assertions.assertEquals(FetchStatus.IN_PROGRESS, result.getStatus()); + + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Package url not formed correctly"); + } + } + + @Test + public void testHandleIntegrityComponentInDB() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + var integrityMeta = new IntegrityMetaComponent(); + integrityMeta.setPurl("pkg:maven/org.http4s/blaze-core_2.12"); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); + qm.createIntegrityMetaComponent(integrityMeta); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + IntegrityMetaComponent result = handler.handle(); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); + assertThat(command.getComponent().getInternal()).isFalse(); + assertThat(command.getFetchIntegrityData()).isFalse(); + assertThat(command.getFetchLatestVersion()).isFalse(); + } + + ); + Assertions.assertEquals(FetchStatus.IN_PROGRESS, result.getStatus()); + + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Package url not formed correctly"); + } + + } + + @Test + public void testHandleIntegrityComponentInDBForMoreThanAnHour() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + var integrityMeta = new IntegrityMetaComponent(); + integrityMeta.setPurl("pkg:maven/org.http4s/blaze-core_2.12"); + integrityMeta.setStatus(FetchStatus.IN_PROGRESS); + integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); + qm.createIntegrityMetaComponent(integrityMeta); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + IntegrityMetaComponent integrityMetaComponent = handler.handle(); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); + assertThat(command.getComponent().getInternal()).isFalse(); + assertThat(command.getFetchIntegrityData()).isTrue(); + assertThat(command.getFetchLatestVersion()).isFalse(); + } + + ); + Assertions.assertEquals(FetchStatus.IN_PROGRESS, integrityMetaComponent.getStatus()); + assertThat(integrityMetaComponent.getLastFetch()).isAfter(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); + + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Package url not formed correctly"); + } + } +} diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java new file mode 100644 index 000000000..d8fa1fa52 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java @@ -0,0 +1,50 @@ +package org.dependencytrack.event.kafka.componentmeta; + +import alpine.common.logging.Logger; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.util.PurlUtil; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.util.KafkaTestUtil.deserializeValue; + +public class UnSupportedMetaHandlerTest extends AbstractPostgresEnabledTest { + + private static final Logger LOGGER = Logger.getLogger(SupportedMetaHandlerTest.class); + + @Test + public void testHandleComponentInDb() { + Handler handler; + KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); + try { + PackageURL packageUrl = new PackageURL("pkg:golang/foo/bar@baz?ping=pong#1/2/3"); + ComponentProjection componentProjection = new ComponentProjection(PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl.toString()); + IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl()); + Assertions.assertNull(integrityMetaComponent); + handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, false); + handler.handle(); + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:golang/foo/bar@baz"); + assertThat(command.getComponent().getInternal()).isFalse(); + assertThat(command.getFetchIntegrityData()).isFalse(); + assertThat(command.getFetchLatestVersion()).isFalse(); + } + + ); + Assertions.assertNull(integrityMetaComponent); + + } catch (MalformedPackageURLException ex) { + LOGGER.warn("Package url not formed correctly"); + } + } +}