From f98e6007d739c7ecdeddfaa35fda5266ec452ea0 Mon Sep 17 00:00:00 2001 From: nscuro Date: Wed, 6 Sep 2023 15:51:29 +0200 Subject: [PATCH 01/46] Initial commit of CEL policy work Signed-off-by: nscuro --- pom.xml | 7 + .../org/dependencytrack/model/Policy.java | 31 +- .../model/PolicyCondition.java | 1 + .../dependencytrack/model/PolicyScope.java | 9 + .../model/PolicyViolation.java | 48 +- .../dependencytrack/policy/PolicyEngine.java | 1 + .../policy/cel/CelPolicyEngine.java | 578 ++++++++++++++++++ .../policy/cel/CelPolicyLibrary.java | 59 ++ .../policy/cel/CelPolicyScript.java | 45 ++ .../policy/cel/CelPolicyScriptHost.java | 142 +++++ .../compat/CelPolicyScriptSourceBuilder.java | 8 + ...onentHashCelPolicyScriptSourceBuilder.java | 43 ++ .../CpeCelPolicyScriptSourceBuilder.java | 24 + .../CweCelPolicyScriptSourceBuilder.java | 52 ++ .../LicenseCelPolicyScriptSourceBuilder.java | 55 ++ ...enseGroupCelPolicyScriptSourceBuilder.java | 24 + ...ackageUrlCelPolicyScriptSourceBuilder.java | 24 + .../SeverityCelPolicyScriptSourceBuilder.java | 26 + ...SwidTagIdCelPolicyScriptSourceBuilder.java | 24 + ...abilityIdCelPolicyScriptSourceBuilder.java | 41 ++ .../tasks/PolicyEvaluationTask.java | 5 +- .../proto/org/hyades/policy/v1/policy.proto | 140 +++++ .../policy/cel/CelPolicyEngineTest.java | 378 ++++++++++++ 23 files changed, 1757 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/dependencytrack/model/PolicyScope.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java create mode 100644 src/main/proto/org/hyades/policy/v1/policy.proto create mode 100644 src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java diff --git a/pom.xml b/pom.xml index 7c0264bf7..b61d164fd 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,7 @@ 4.7.1 ${project.parent.version} 4.2.0 + 0.3.21 2.0.2 1.4.1 1.0.1 @@ -193,6 +194,12 @@ + + + org.projectnessie.cel + cel-tools + ${lib.cel-tools.version} + us.springett diff --git a/src/main/java/org/dependencytrack/model/Policy.java b/src/main/java/org/dependencytrack/model/Policy.java index 34685353d..bbe8837cd 100644 --- a/src/main/java/org/dependencytrack/model/Policy.java +++ b/src/main/java/org/dependencytrack/model/Policy.java @@ -81,6 +81,10 @@ public enum ViolationState { @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The name may only contain printable characters") private String name; + @Persistent + @Column(name = "SCOPE", allowsNull = "true") + private PolicyScope scope = PolicyScope.COMPONENT; // TODO: Remove default, handle this for existing policies with DB migration. + /** * The operator to use when evaluating conditions. */ @@ -98,8 +102,15 @@ public enum ViolationState { @Column(name = "VIOLATIONSTATE", allowsNull = "false") @NotBlank @Size(min = 1, max = 255) - @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The operator may only contain printable characters") - private ViolationState violationState; + @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The violation state may only contain printable characters") + private ViolationState violationState; // TODO: New column, possibly needs migration. + + @Persistent + @Column(name = "VIOLATIONTYPE", allowsNull = "true") + @NotBlank + @Size(min = 1, max = 255) + @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The violation type may only contain printable characters") + private PolicyViolation.Type violationType; /** * A list of zero-to-n policy conditions. @@ -158,6 +169,14 @@ public void setName(String name) { this.name = name; } + public PolicyScope getScope() { + return scope; + } + + public void setScope(final PolicyScope scope) { + this.scope = scope; + } + public Operator getOperator() { return operator; } @@ -174,6 +193,14 @@ public void setViolationState(ViolationState violationState) { this.violationState = violationState; } + public PolicyViolation.Type getViolationType() { + return violationType; + } + + public void setViolationType(final PolicyViolation.Type violationType) { + this.violationType = violationType; + } + public List getPolicyConditions() { return policyConditions; } diff --git a/src/main/java/org/dependencytrack/model/PolicyCondition.java b/src/main/java/org/dependencytrack/model/PolicyCondition.java index 9fdd144fa..502f364b0 100644 --- a/src/main/java/org/dependencytrack/model/PolicyCondition.java +++ b/src/main/java/org/dependencytrack/model/PolicyCondition.java @@ -68,6 +68,7 @@ public enum Subject { //BOM, COORDINATES, CPE, + EXPRESSION, //INHERITED_RISK_SCORE, LICENSE, LICENSE_GROUP, diff --git a/src/main/java/org/dependencytrack/model/PolicyScope.java b/src/main/java/org/dependencytrack/model/PolicyScope.java new file mode 100644 index 000000000..e7de34dc3 --- /dev/null +++ b/src/main/java/org/dependencytrack/model/PolicyScope.java @@ -0,0 +1,9 @@ +package org.dependencytrack.model; + +public enum PolicyScope { + + COMPONENT, + + PROJECT + +} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/model/PolicyViolation.java b/src/main/java/org/dependencytrack/model/PolicyViolation.java index 317b52b0a..b857100e1 100644 --- a/src/main/java/org/dependencytrack/model/PolicyViolation.java +++ b/src/main/java/org/dependencytrack/model/PolicyViolation.java @@ -24,8 +24,12 @@ import com.fasterxml.jackson.annotation.JsonInclude; import javax.jdo.annotations.Column; +import javax.jdo.annotations.Element; +import javax.jdo.annotations.Extension; import javax.jdo.annotations.IdGeneratorStrategy; import javax.jdo.annotations.Index; +import javax.jdo.annotations.Join; +import javax.jdo.annotations.Order; import javax.jdo.annotations.PersistenceCapable; import javax.jdo.annotations.Persistent; import javax.jdo.annotations.PrimaryKey; @@ -35,6 +39,7 @@ import javax.validation.constraints.Size; import java.io.Serializable; import java.util.Date; +import java.util.List; import java.util.UUID; /** @@ -69,14 +74,26 @@ public enum Type { private Project project; @Persistent(defaultFetchGroup = "true") - @Column(name = "COMPONENT_ID", allowsNull = "false") + @Column(name = "COMPONENT_ID", allowsNull = "true") @Index(name = "POLICYVIOLATION_COMPONENT_IDX") private Component component; + @Persistent + @Column(name = "POLICY_ID", allowsNull = "true") + private Policy policy; + + // TODO: Remove + @Deprecated(forRemoval = true) @Persistent(defaultFetchGroup = "true") - @Column(name = "POLICYCONDITION_ID", allowsNull = "false") + @Column(name = "POLICYCONDITION_ID", allowsNull = "true") private PolicyCondition policyCondition; + @Persistent(table = "POLICYVIOLATIONS_MATCHED_POLICYCONDITIONS") + @Join(column = "POLICYVIOLATION_ID") + @Element(column = "POLICYCONDITION_ID") + @Order(extensions = @Extension(vendorName = "datanucleus", key = "list-ordering", value = "id ASC")) + private List matchedConditions; + @Persistent @Column(name = "TIMESTAMP", allowsNull = "false") private Date timestamp; @@ -88,7 +105,7 @@ public enum Type { private String text; @Persistent(mappedBy="policyViolation", defaultFetchGroup = "true") - private ViolationAnalysis analysis; + private ViolationAnalysis analysis; /** * The unique identifier of the object. @@ -121,21 +138,44 @@ public Component getComponent() { public void setComponent(Component component) { this.component = component; - this.project = component.getProject(); } public Project getProject() { return project; } + public void setProject(final Project project) { + this.project = project; + } + + public Policy getPolicy() { + return policy; + } + + public void setPolicy(final Policy policy) { + this.policy = policy; + } + + // TODO: Remove + @Deprecated(forRemoval = true) public PolicyCondition getPolicyCondition() { return policyCondition; } + // TODO: Remove + @Deprecated(forRemoval = true) public void setPolicyCondition(PolicyCondition policyCondition) { this.policyCondition = policyCondition; } + public List getMatchedConditions() { + return matchedConditions; + } + + public void setMatchedConditions(final List matchedConditions) { + this.matchedConditions = matchedConditions; + } + public Date getTimestamp() { return timestamp; } diff --git a/src/main/java/org/dependencytrack/policy/PolicyEngine.java b/src/main/java/org/dependencytrack/policy/PolicyEngine.java index 7140db559..3a4754735 100644 --- a/src/main/java/org/dependencytrack/policy/PolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/PolicyEngine.java @@ -213,6 +213,7 @@ public PolicyViolation.Type determineViolationType(final PolicyCondition.Subject case AGE, COORDINATES, PACKAGE_URL, CPE, SWID_TAGID, COMPONENT_HASH, VERSION -> PolicyViolation.Type.OPERATIONAL; case LICENSE, LICENSE_GROUP -> PolicyViolation.Type.LICENSE; + case EXPRESSION -> null; // TODO: Just here to satisfy the switch exhaustiveness }; } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java new file mode 100644 index 000000000..07b83276b --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -0,0 +1,578 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import alpine.common.metrics.Metrics; +import com.fasterxml.jackson.core.JacksonException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.github.packageurl.PackageURL; +import com.google.protobuf.util.Timestamps; +import io.micrometer.core.instrument.Timer; +import org.apache.commons.lang3.tuple.Pair; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.LicenseGroup; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyCondition.Subject; +import org.dependencytrack.model.PolicyScope; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Tag; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.policy.cel.CelPolicyScript.Requirement; +import org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.ComponentHashCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.CpeCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.CweCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.LicenseCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.LicenseGroupCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.PackageUrlCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.SeverityCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; +import org.dependencytrack.proto.policy.v1.License; +import org.dependencytrack.proto.policy.v1.Vulnerability; +import org.projectnessie.cel.tools.ScriptCreateException; +import org.projectnessie.cel.tools.ScriptException; + +import javax.jdo.Query; +import javax.jdo.Transaction; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_COMPONENT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_PROJECT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_VULNERABILITIES; + +public class CelPolicyEngine { + + private static final Logger LOGGER = Logger.getLogger(CelPolicyEngine.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Map SCRIPT_BUILDERS = Map.of( + Subject.CPE, new CpeCelPolicyScriptSourceBuilder(), + Subject.COMPONENT_HASH, new ComponentHashCelPolicyScriptSourceBuilder(), + Subject.CWE, new CweCelPolicyScriptSourceBuilder(), + Subject.EXPRESSION, PolicyCondition::getValue, + Subject.LICENSE, new LicenseCelPolicyScriptSourceBuilder(), + Subject.LICENSE_GROUP, new LicenseGroupCelPolicyScriptSourceBuilder(), + Subject.PACKAGE_URL, new PackageUrlCelPolicyScriptSourceBuilder(), + Subject.SEVERITY, new SeverityCelPolicyScriptSourceBuilder(), + Subject.SWID_TAGID, new SwidTagIdCelPolicyScriptSourceBuilder(), + Subject.VULNERABILITY_ID, new VulnerabilityIdCelPolicyScriptSourceBuilder() + ); + + private final CelPolicyScriptHost scriptHost; + + public CelPolicyEngine() { + this(CelPolicyScriptHost.getInstance()); + } + + CelPolicyEngine(CelPolicyScriptHost scriptHost) { + this.scriptHost = scriptHost; + } + + public void evaluateProject(final UUID projectUuid) { + final Timer.Sample timerSample = Timer.start(); + + try { + // TODO + } finally { + timerSample.stop(Timer + .builder("dtrack_policy_eval") + .tag("target", "project") + .register(Metrics.getRegistry())); + } + } + + // TODO: Just here to satisfy contract with legacy PolicyEngine; Remove after testing + public void evaluate(final List components) { + components.stream().map(Component::getUuid).forEach(this::evaluateComponent); + } + + public void evaluateComponent(final UUID componentUuid) { + final Timer.Sample timerSample = Timer.start(); + + try (final var qm = new QueryManager()) { + final Component component = qm.getObjectByUuid(Component.class, componentUuid); + if (component == null) { + LOGGER.warn("Component with UUID %s does not exist".formatted(componentUuid)); + return; + } + + final List policies = getApplicablePolicies(qm, component.getProject(), Set.of(PolicyScope.COMPONENT)); + if (policies.isEmpty()) { + // With no applicable policies, there's no way to resolve violations. + // As a compensation, simply delete all violations associated with the component. + reconcileViolations(qm, component, Collections.emptyList()); + return; + } + + // Pre-compile the CEL scripts for all conditions of all applicable policies. + // Compiled scripts are cached in-memory by CelPolicyScriptHost, so if the same script + // is encountered for multiple components (possibly concurrently), the compilation is + // a one-time effort. + LOGGER.debug("Compiling policy scripts for component %s".formatted(componentUuid)); + final List> conditionScriptPairs = policies.stream() + .map(Policy::getPolicyConditions) + .flatMap(Collection::stream) + .map(policyCondition -> { + final CelPolicyScriptSourceBuilder scriptBuilder = SCRIPT_BUILDERS.get(policyCondition.getSubject()); + if (scriptBuilder == null) { + LOGGER.warn("No script builder found that is capable of handling subjects of type %s".formatted(policyCondition.getSubject())); + return null; + } + + final String scriptSrc = scriptBuilder.apply(policyCondition); + if (scriptSrc == null) { + LOGGER.warn("Script builder was unable to create a script for condition %s".formatted(policyCondition.getUuid())); + return null; + } + + return Pair.of(policyCondition, scriptSrc); + }) + .filter(Objects::nonNull) + .map(conditionScriptSrcPair -> { + final CelPolicyScript script; + try { + script = scriptHost.compile(conditionScriptSrcPair.getRight()); + } catch (ScriptCreateException e) { + throw new RuntimeException(e); + } + + return Pair.of(conditionScriptSrcPair.getLeft(), script); + }) + .toList(); + + // Check what kind of data we need to evaluate all policy conditions. + // + // Some conditions will be very simple and won't require us to load additional data (e.g. "component PURL matches 'XYZ'"), + // whereas other conditions can span across multiple models, forcing us to load more data + // (e.g. "project has tag 'public-facing' and component has a vulnerability with severity 'critical'"). + // + // What we want to avoid is loading data we don't need, and loading it multiple times. + // Instead, only load what's really needed, and only do so once. + LOGGER.debug("Determining evaluation requirements for component %s and %d policy conditions" + .formatted(componentUuid, conditionScriptPairs.size())); + final Set requirements = conditionScriptPairs.stream() + .map(Pair::getRight) + .map(CelPolicyScript::getRequirements) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + + // Prepare the script arguments according to the requirements gathered before. + LOGGER.debug("Building script arguments for component %s and requirements %s" + .formatted(componentUuid, requirements)); + final Map scriptArgs = Map.of( + VAR_COMPONENT, mapComponent(qm, component, requirements), + VAR_PROJECT, mapProject(component.getProject(), requirements), + VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) + ); + + LOGGER.debug("Evaluating component %s against %d applicable policy conditions" + .formatted(componentUuid, conditionScriptPairs.size())); + final var conditionsViolated = new HashSet(); + for (final Pair conditionScriptPair : conditionScriptPairs) { + final PolicyCondition condition = conditionScriptPair.getLeft(); + final CelPolicyScript script = conditionScriptPair.getRight(); + LOGGER.debug("Executing script for policy condition %s with arguments: %s" + .formatted(condition.getUuid(), scriptArgs)); + + try { + if (script.execute(scriptArgs)) { + conditionsViolated.add(condition); + } + } catch (ScriptException e) { + throw new RuntimeException("Failed to evaluate script", e); + } + } + + // Group the detected condition violations by policy. Necessary to be able to evaluate + // each policy's operator (ANY, ALL). + LOGGER.debug("Detected violation of %d policy conditions for component %s; Evaluating policy operators" + .formatted(conditionsViolated.size(), componentUuid)); + final Map> violatedConditionsByPolicy = conditionsViolated.stream() + .collect(Collectors.groupingBy(PolicyCondition::getPolicy)); + + // Create policy violations, but only do so when the detected condition violations + // match the configured policy operator. When the operator is ALL, and not all conditions + // of the policy were violated, we don't want to create any violations. + final List violations = violatedConditionsByPolicy.entrySet().stream() + .map(policyAndViolatedConditions -> { + final Policy policy = policyAndViolatedConditions.getKey(); + final List violatedConditions = policyAndViolatedConditions.getValue(); + + if ((policy.getOperator() == Policy.Operator.ANY && !violatedConditions.isEmpty()) + || (policy.getOperator() == Policy.Operator.ALL && violatedConditions.size() == policy.getPolicyConditions().size())) { + final var violation = new PolicyViolation(); + violation.setProject(component.getProject()); + violation.setComponent(component); + violation.setPolicy(policy); + violation.setType(PolicyViolation.Type.OPERATIONAL); // TODO: We need violationType at policy level + violation.setMatchedConditions(violatedConditions); + violation.setTimestamp(new Date()); + return violation; + } + + return null; + }) + .filter(Objects::nonNull) + .toList(); + + // Reconcile the violations created above with what's already in the database. + // Create new records if necessary, and delete records that are no longer current. + final List newViolations = reconcileViolations(qm, component, violations); + + // Notify users about any new violations. + for (final PolicyViolation newViolation : newViolations) { + // TODO: Handle switch from policyCondition to matchedConditions in PolicyViolation + // NotificationUtil.analyzeNotificationCriteria(qm, newViolation); + } + } finally { + timerSample.stop(Timer + .builder("dtrack_policy_eval") + .tag("target", "component") + .register(Metrics.getRegistry())); + } + + LOGGER.debug("Policy evaluation completed for component %s".formatted(componentUuid)); + } + + // TODO: Move to PolicyQueryManager + private static List getApplicablePolicies(final QueryManager qm, final Project project, final Set scopes) { + var filter = """ + (this.projects.isEmpty() && this.tags.isEmpty()) + || (this.projects.contains(:project) + """; + var params = new HashMap(); + params.put("project", project); + + // To compensate for missing support for recursion of Common Table Expressions (CTEs) + // in JDO, we have to fetch the UUIDs of all parent projects upfront. Otherwise, we'll + // not be able to evaluate whether the policy is inherited from parent projects. + var variables = ""; + final List parentUuids = getParents(qm, project); + if (!parentUuids.isEmpty()) { + filter += """ + || (this.includeChildren + && this.projects.contains(parentVar) + && :parentUuids.contains(parentVar.uuid)) + """; + variables += "org.dependencytrack.model.Project parentVar"; + params.put("parentUuids", parentUuids); + } + filter += ")"; + + // DataNucleus generates an invalid SQL query when using the idiomatic solution. + // The following works, but it's ugly and likely doesn't perform well if the project + // has many tags. Worth trying the idiomatic way again once DN has been updated to > 6.0.4. + // + // filter += " || (this.tags.contains(commonTag) && :project.tags.contains(commonTag))"; + // variables += "org.dependencytrack.model.Tag commonTag"; + if (project.getTags() != null && !project.getTags().isEmpty()) { + filter += " || ("; + for (int i = 0; i < project.getTags().size(); i++) { + filter += "this.tags.contains(:tag" + i + ")"; + params.put("tag" + i, project.getTags().get(i)); + if (i < (project.getTags().size() - 1)) { + filter += " || "; + } + } + filter += ")"; + } + + filter = "(%s) && :scopes.contains(this.scope)".formatted(filter); + params.put("scopes", scopes); + + final List policies; + final Query query = qm.getPersistenceManager().newQuery(Policy.class); + try { + query.setFilter(filter); + query.setNamedParameters(params); + if (!variables.isEmpty()) { + query.declareVariables(variables); + } + policies = List.copyOf(query.executeList()); + } finally { + query.closeAll(); + } + + return policies; + } + + // TODO: Move to ProjectQueryManager + private static List getParents(final QueryManager qm, final Project project) { + return getParents(qm, project.getUuid(), new ArrayList<>()); + } + + // TODO: Move to ProjectQueryManager + private static List getParents(final QueryManager qm, final UUID uuid, final List parents) { + final UUID parentUuid; + final Query query = qm.getPersistenceManager().newQuery(Project.class); + try { + query.setFilter("uuid == :uuid && parent != null"); + query.setParameters(uuid); + query.setResult("parent.uuid"); + parentUuid = query.executeResultUnique(UUID.class); + } finally { + query.closeAll(); + } + + if (parentUuid == null) { + return parents; + } + + parents.add(parentUuid); + return getParents(qm, parentUuid, parents); + } + + private static org.dependencytrack.proto.policy.v1.Component mapComponent(final QueryManager qm, + final Component component, + final Set requirements) { + final org.dependencytrack.proto.policy.v1.Component.Builder builder = + org.dependencytrack.proto.policy.v1.Component.newBuilder() + .setUuid(Optional.ofNullable(component.getUuid()).map(UUID::toString).orElse("")) + .setGroup(trimToEmpty(component.getGroup())) + .setName(trimToEmpty(component.getName())) + .setVersion(trimToEmpty(component.getVersion())) + .setClassifier(Optional.ofNullable(component.getClassifier()).map(Enum::name).orElse("")) + .setCpe(trimToEmpty(component.getCpe())) + .setPurl(Optional.ofNullable(component.getPurl()).map(PackageURL::canonicalize).orElse("")) + .setSwidTagId(trimToEmpty(component.getSwidTagId())) + .setIsInternal(component.isInternal()) + .setMd5(trimToEmpty(component.getMd5())) + .setSha1(trimToEmpty(component.getSha1())) + .setSha256(trimToEmpty(component.getSha256())) + .setSha384(trimToEmpty(component.getSha384())) + .setSha512(trimToEmpty(component.getSha512())) + .setSha3256(trimToEmpty(component.getSha3_256())) + .setSha3384(trimToEmpty(component.getSha3_384())) + .setSha3512(trimToEmpty(component.getSha3_512())) + .setBlake2B256(trimToEmpty(component.getBlake2b_256())) + .setBlake2B384(trimToEmpty(component.getBlake2b_384())) + .setBlake2B512(trimToEmpty(component.getBlake2b_512())) + .setBlake3(trimToEmpty(component.getBlake3())); + + if (component.getProject().getDirectDependencies() != null) { + try { + final ArrayNode dependencyArray = OBJECT_MAPPER.readValue(component.getProject().getDirectDependencies(), ArrayNode.class); + for (final JsonNode dependencyNode : dependencyArray) { + if (dependencyNode.get("uuid") != null && dependencyNode.get("uuid").asText().equals(component.getUuid().toString())) { + builder.setIsDirectDependency(true); + break; + } + } + } catch (JacksonException | RuntimeException e) { + LOGGER.warn("Failed to parse direct dependencies of project %s".formatted(component.getProject().getUuid()), e); + } + } + + if (requirements.contains(Requirement.LICENSE) && component.getResolvedLicense() != null) { + final License.Builder licenseBuilder = License.newBuilder() + .setUuid(Optional.ofNullable(component.getResolvedLicense().getUuid()).map(UUID::toString).orElse("")) + .setId(trimToEmpty(component.getResolvedLicense().getLicenseId())) + .setName(trimToEmpty(component.getResolvedLicense().getName())) + .setIsOsiApproved(component.getResolvedLicense().isOsiApproved()) + .setIsFsfLibre(component.getResolvedLicense().isFsfLibre()) + .setIsDeprecatedId(component.getResolvedLicense().isDeprecatedLicenseId()) + .setIsCustom(component.getResolvedLicense().isCustomLicense()); + + if (requirements.contains(Requirement.LICENSE_GROUPS)) { + final Query licenseGroupQuery = qm.getPersistenceManager().newQuery(LicenseGroup.class); + licenseGroupQuery.setFilter("licenses.contains(:license)"); + licenseGroupQuery.setNamedParameters(Map.of("license", component.getResolvedLicense())); + licenseGroupQuery.setResult("uuid, name"); + try { + licenseGroupQuery.executeResultList(LicenseGroup.class).stream() + .map(licenseGroup -> License.Group.newBuilder() + .setUuid(Optional.ofNullable(licenseGroup.getUuid()).map(UUID::toString).orElse("")) + .setName(trimToEmpty(licenseGroup.getName()))) + .forEach(licenseBuilder::addGroups); + } finally { + licenseGroupQuery.closeAll(); + } + } + + builder.setResolvedLicense(licenseBuilder); + } + + return builder.build(); + } + + private static org.dependencytrack.proto.policy.v1.Project mapProject(final Project project, + final Set requirements) { + if (!requirements.contains(Requirement.PROJECT)) { + return org.dependencytrack.proto.policy.v1.Project.newBuilder().build(); + } + + final org.dependencytrack.proto.policy.v1.Project.Builder builder = + org.dependencytrack.proto.policy.v1.Project.newBuilder() + .setUuid(Optional.ofNullable(project.getUuid()).map(UUID::toString).orElse("")) + .setGroup(trimToEmpty(project.getGroup())) + .setName(trimToEmpty(project.getName())) + .setVersion(trimToEmpty(project.getVersion())) + .addAllTags(project.getTags().stream().map(Tag::getName).toList()) + .setCpe(trimToEmpty(project.getCpe())) + .setPurl(Optional.ofNullable(project.getPurl()).map(PackageURL::canonicalize).orElse("")) + .setSwidTagId(trimToEmpty(project.getSwidTagId())); + + if (requirements.contains(Requirement.PROJECT_PROPERTIES)) { + // TODO + } + + return builder.build(); + } + + private static List loadVulnerabilities(final QueryManager qm, + final Component component, + final Set requirements) { + if (!requirements.contains(Requirement.VULNERABILITIES)) { + return Collections.emptyList(); + } + + final Query query = + qm.getPersistenceManager().newQuery(org.dependencytrack.model.Vulnerability.class); + query.setFilter("components.contains(:component)"); + query.setParameters(component); + // Avoid some ORM overhead by explicitly specifying the fields we want + // to fetch, and load them into a result class, rather than a candidate + // class. The returned Vulnerability objects are thus just dumb POJOs and + // not attached to the persistence context. + query.setResult(""" + uuid, + vulnId, + source, + cwes, + created, + published, + updated, + cvssV2BaseScore, + cvssV2ImpactSubScore, + cvssV2ExploitabilitySubScore, + cvssV2Vector, + cvssV3BaseScore, + cvssV3ImpactSubScore, + cvssV3ExploitabilitySubScore, + cvssV3Vector, + owaspRRLikelihoodScore, + owaspRRTechnicalImpactScore, + owaspRRBusinessImpactScore, + owaspRRVector, + severity, + epssScore, + epssPercentile + """); + final List vulns; + try { + vulns = List.copyOf(query.executeResultList(org.dependencytrack.model.Vulnerability.class)); + } finally { + query.closeAll(); + } + + final List vulnBuilders = vulns.stream() + .map(v -> { + final Vulnerability.Builder builder = Vulnerability.newBuilder() + .setUuid(v.getUuid().toString()) + .setId(trimToEmpty(v.getVulnId())) + .setSource(trimToEmpty(v.getSource())) + .setCvssv2Vector(trimToEmpty(v.getCvssV2Vector())) + .setCvssv3Vector(trimToEmpty(v.getCvssV3Vector())) + .setOwaspRrVector(trimToEmpty(v.getOwaspRRVector())) + .setSeverity(v.getSeverity().name()); + Optional.ofNullable(v.getCwes()).ifPresent(builder::addAllCwes); + Optional.ofNullable(v.getCvssV2BaseScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2BaseScore); + Optional.ofNullable(v.getCvssV2ImpactSubScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2ImpactSubscore); + Optional.ofNullable(v.getCvssV2ExploitabilitySubScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2ExploitabilitySubscore); + Optional.ofNullable(v.getCvssV3BaseScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3BaseScore); + Optional.ofNullable(v.getCvssV3ImpactSubScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3ImpactSubscore); + Optional.ofNullable(v.getCvssV3ExploitabilitySubScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3ExploitabilitySubscore); + Optional.ofNullable(v.getOwaspRRLikelihoodScore()).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrLikelihoodScore); + Optional.ofNullable(v.getOwaspRRTechnicalImpactScore()).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrTechnicalImpactScore); + Optional.ofNullable(v.getOwaspRRBusinessImpactScore()).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrBusinessImpactScore); + Optional.ofNullable(v.getEpssScore()).map(BigDecimal::doubleValue).ifPresent(builder::setEpssScore); + Optional.ofNullable(v.getEpssPercentile()).map(BigDecimal::doubleValue).ifPresent(builder::setEpssPercentile); + Optional.ofNullable(v.getCreated()).map(Timestamps::fromDate).ifPresent(builder::setCreated); + Optional.ofNullable(v.getPublished()).map(Timestamps::fromDate).ifPresent(builder::setPublished); + Optional.ofNullable(v.getUpdated()).map(Timestamps::fromDate).ifPresent(builder::setUpdated); + return builder; + }) + .toList(); + + if (requirements.contains(Requirement.VULNERABILITY_ALIASES)) { + // TODO: Fetch aliases + } + + return vulnBuilders.stream() + .map(Vulnerability.Builder::build) + .toList(); + } + + // TODO: Move to PolicyQueryManager + private static List reconcileViolations(final QueryManager qm, final Component component, final List violations) { + final var newViolations = new ArrayList(); + + final Transaction trx = qm.getPersistenceManager().currentTransaction(); + try { + trx.begin(); + + final var violationIdsToKeep = new HashSet(); + + for (final PolicyViolation violation : violations) { + final Query query = qm.getPersistenceManager().newQuery(PolicyViolation.class); + query.setFilter("component == :component && policy == :policy && type == :type"); + query.setNamedParameters(Map.of( + "component", violation.getComponent(), + "policy", violation.getPolicy(), + "type", violation.getType() + )); + query.setResult("id"); + + final Long existingViolationId; + try { + existingViolationId = query.executeResultUnique(Long.class); + } finally { + query.closeAll(); + } + + if (existingViolationId != null) { + violationIdsToKeep.add(existingViolationId); + } else { + qm.getPersistenceManager().makePersistent(violation); + violationIdsToKeep.add(violation.getId()); + newViolations.add(violation); + } + } + + final Query deleteQuery = qm.getPersistenceManager().newQuery(PolicyViolation.class); + deleteQuery.setFilter("component == :component && !:ids.contains(id)"); + try { + final long violationsDeleted = deleteQuery.deletePersistentAll(component, violationIdsToKeep); + LOGGER.debug("Deleted %s outdated violations".formatted(violationsDeleted)); // TODO: Add component UUID + } finally { + deleteQuery.closeAll(); + } + + trx.commit(); + } finally { + if (trx.isActive()) { + trx.rollback(); + } + } + + return newViolations; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java new file mode 100644 index 000000000..566a31022 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -0,0 +1,59 @@ +package org.dependencytrack.policy.cel; + +import com.google.api.expr.v1alpha1.Type; +import org.dependencytrack.proto.policy.v1.Component; +import org.dependencytrack.proto.policy.v1.License; +import org.dependencytrack.proto.policy.v1.Project; +import org.dependencytrack.proto.policy.v1.Vulnerability; +import org.projectnessie.cel.EnvOption; +import org.projectnessie.cel.Library; +import org.projectnessie.cel.ProgramOption; +import org.projectnessie.cel.checker.Decls; + +import java.util.Collections; +import java.util.List; + +public class CelPolicyLibrary implements Library { + + static final String VAR_COMPONENT = "component"; + static final String VAR_PROJECT = "project"; + static final String VAR_VULNERABILITIES = "vulns"; + + private static final Type TYPE_COMPONENT = Decls.newObjectType(Component.getDescriptor().getFullName()); + private static final Type TYPE_PROJECT = Decls.newObjectType(Project.getDescriptor().getFullName()); + private static final Type TYPE_VULNERABILITY = Decls.newObjectType(Vulnerability.getDescriptor().getFullName()); + private static final Type TYPE_VULNERABILITIES = Decls.newListType(TYPE_VULNERABILITY); + + @Override + public List getCompileOptions() { + return List.of( + EnvOption.declarations( + Decls.newVar( + VAR_COMPONENT, + TYPE_COMPONENT + ), + Decls.newVar( + VAR_PROJECT, + TYPE_PROJECT + ), + Decls.newVar( + VAR_VULNERABILITIES, + TYPE_VULNERABILITIES + ) + ), + EnvOption.types( + Component.getDefaultInstance(), + License.getDefaultInstance(), + Project.getDefaultInstance(), + Vulnerability.getDefaultInstance(), + Vulnerability.Alias.getDefaultInstance() + ) + ); + } + + @Override + public List getProgramOptions() { + return Collections.emptyList(); + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java new file mode 100644 index 000000000..2db8dc1a4 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java @@ -0,0 +1,45 @@ +package org.dependencytrack.policy.cel; + +import org.projectnessie.cel.Program; +import org.projectnessie.cel.common.types.Err; +import org.projectnessie.cel.common.types.ref.Val; +import org.projectnessie.cel.tools.ScriptExecutionException; + +import java.util.Map; +import java.util.Set; + +public class CelPolicyScript { + + public enum Requirement { + LICENSE, + LICENSE_GROUPS, + PROJECT, + PROJECT_PROPERTIES, + VULNERABILITIES, + VULNERABILITY_ALIASES + } + + private final Program program; + private final Set requirements; + + public CelPolicyScript(final Program program, final Set requirements) { + this.program = program; + this.requirements = requirements; + } + + public Set getRequirements() { + return requirements; + } + + public boolean execute(final Map arguments) throws ScriptExecutionException { + final Val result = program.eval(arguments).getVal(); + + if (Err.isError(result)) { + final Err error = (Err) result; + throw new ScriptExecutionException(error.toString(), error.getCause()); + } + + return result.convertToNative(Boolean.class); + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java new file mode 100644 index 000000000..0da367981 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -0,0 +1,142 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import alpine.server.cache.CacheManager; +import com.google.api.expr.v1alpha1.CheckedExpr; +import com.google.api.expr.v1alpha1.Type; +import com.google.common.util.concurrent.Striped; +import org.apache.commons.codec.digest.DigestUtils; +import org.dependencytrack.policy.cel.CelPolicyScript.Requirement; +import org.dependencytrack.proto.policy.v1.License; +import org.dependencytrack.proto.policy.v1.Project; +import org.dependencytrack.proto.policy.v1.Vulnerability; +import org.projectnessie.cel.Ast; +import org.projectnessie.cel.CEL; +import org.projectnessie.cel.Env; +import org.projectnessie.cel.Env.AstIssuesTuple; +import org.projectnessie.cel.Library; +import org.projectnessie.cel.Program; +import org.projectnessie.cel.common.types.pb.ProtoTypeRegistry; +import org.projectnessie.cel.tools.ScriptCreateException; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.stream.Collectors; + +public class CelPolicyScriptHost { + + private static final Logger LOGGER = Logger.getLogger(CelPolicyScriptHost.class); + private static CelPolicyScriptHost INSTANCE; + + private final Striped locks; + private final CacheManager cacheManager; + private final Env environment; + + CelPolicyScriptHost(final CacheManager cacheManager) { + this.locks = Striped.lock(128); + this.cacheManager = cacheManager; + this.environment = Env.newCustomEnv( + ProtoTypeRegistry.newRegistry(), + List.of( + Library.StdLib(), + Library.Lib(new CelPolicyLibrary()) + ) + ); + } + + public static synchronized CelPolicyScriptHost getInstance() { + if (INSTANCE == null) { + INSTANCE = new CelPolicyScriptHost(CacheManager.getInstance()); + } + + return INSTANCE; + } + + public CelPolicyScript compile(final String scriptSrc) throws ScriptCreateException { + final String scriptDigest = DigestUtils.sha256Hex(scriptSrc); + + // Acquire a lock for the SHA256 digest of the script source. + // It is possible that compilation of the same script will be attempted multiple + // times concurrently. + final Lock lock = locks.get(scriptDigest); + lock.lock(); + + try { + CelPolicyScript script = cacheManager.get(CelPolicyScript.class, scriptDigest); + if (script != null) { + return script; + } + + LOGGER.debug("Compiling script: %s".formatted(scriptSrc)); + AstIssuesTuple astIssuesTuple = environment.parse(scriptSrc); + if (astIssuesTuple.hasIssues()) { + throw new ScriptCreateException("Failed to parse script", astIssuesTuple.getIssues()); + } + + astIssuesTuple = environment.check(astIssuesTuple.getAst()); + if (astIssuesTuple.hasIssues()) { + throw new ScriptCreateException("Failed to check script", astIssuesTuple.getIssues()); + } + + final Ast ast = astIssuesTuple.getAst(); + final Program program = environment.program(ast); + final Set requirements = analyzeRequirements(CEL.astToCheckedExpr(ast)); + + script = new CelPolicyScript(program, requirements); + cacheManager.put(scriptDigest, script); + return script; + } finally { + lock.unlock(); + } + } + + private static Set analyzeRequirements(final CheckedExpr expr) { + final var requirements = new HashSet(); + + final Set typeNames = expr.getTypeMapMap().values().stream() + .map(Type::getMessageType) + .collect(Collectors.toSet()); + + // For the majority of cases, it is sufficient to check whether a given type + // is present in the type map constructed by the type checker. This works as long + // as a field of the respective type is accessed in the script, e.g. + // + // component.license.name + // + // will result in the License type being present in the type map. However, it does NOT + // work when only the presence of a field is checked in the script, e.g. + // + // has(component.license) + // + // will result in the License type NOT being present in the type map. + // + // To cover this limitation, we could implement a visitor that traverses the AST + // and keeps track of which fields are accessed for which type. + if (typeNames.contains(Project.getDescriptor().getFullName())) { + requirements.add(Requirement.PROJECT); + + if (typeNames.contains(Project.Property.getDescriptor().getFullName())) { + requirements.add(Requirement.PROJECT_PROPERTIES); + } + } + if (typeNames.contains(License.getDescriptor().getFullName())) { + requirements.add(Requirement.LICENSE); + + if (typeNames.contains(License.Group.getDescriptor().getFullName())) { + requirements.add(Requirement.LICENSE_GROUPS); + } + } + if (typeNames.contains(Vulnerability.getDescriptor().getFullName())) { + requirements.add(Requirement.VULNERABILITIES); + + if (typeNames.contains(Vulnerability.Alias.getDescriptor().getFullName())) { + requirements.add(Requirement.VULNERABILITY_ALIASES); + } + } + + return requirements; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..fdef88f15 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java @@ -0,0 +1,8 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import java.util.function.Function; + +public interface CelPolicyScriptSourceBuilder extends Function { +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..6137ee536 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java @@ -0,0 +1,43 @@ +package org.dependencytrack.policy.cel.compat; + +import alpine.common.logging.Logger; +import org.cyclonedx.model.Hash; +import org.dependencytrack.model.PolicyCondition; +import org.json.JSONObject; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class ComponentHashCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + private static final Logger LOGGER = Logger.getLogger(ComponentHashCelPolicyScriptSourceBuilder.class); + + @Override + public String apply(final PolicyCondition policyCondition) { + final Hash hash = extractHashValues(policyCondition); + if (hash == null) { + return null; + } + + final String fieldName = hash.getAlgorithm().toLowerCase().replaceAll("-", "_"); + if (org.dependencytrack.proto.policy.v1.Component.getDescriptor().findFieldByName(fieldName) == null) { + LOGGER.warn("Component does not have a field named %s".formatted(fieldName)); + return null; + } + + return """ + component.%s == "%s" + """.formatted(fieldName, escapeJson(hash.getValue())); + } + + private static Hash extractHashValues(PolicyCondition condition) { + if (condition.getValue() == null) { + return null; + } + final JSONObject def = new JSONObject(condition.getValue()); + return new Hash( + def.optString("algorithm", null), + def.optString("value", null) + ); + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..7a101567b --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class CpeCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + component.cpe.matches("%s") + """.formatted(escapeJson(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.NO_MATCH) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..1d27d108a --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java @@ -0,0 +1,52 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.parser.common.resolver.CweResolver; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class CweCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final List conditionCwes = Arrays.stream(policyCondition.getValue().split(",")) + .map(String::trim) + .map(CweResolver.getInstance()::parseCweString) + .sorted() + .toList(); + if (conditionCwes.isEmpty()) { + return null; + } + + final String celCweListLiteral = "[%s]".formatted(conditionCwes.stream() + .map(String::valueOf) + .collect(Collectors.joining(", "))); + + if (policyCondition.getOperator() == PolicyCondition.Operator.CONTAINS_ANY) { + // ANY of the vulnerabilities affecting the component have ANY of the + // CWEs defined in the policy condition assigned to them. + return """ + %s.exists(policyCwe, + vulns.exists(vuln, + vuln.cwes.exists(vulnCwe, vulnCwe == policyCwe) + ) + ) + """.formatted(celCweListLiteral); + } else if (policyCondition.getOperator() == PolicyCondition.Operator.CONTAINS_ALL) { + // ANY of the vulnerabilities affecting the component have ALL the + // CWEs defined in the policy condition assigned to them. + return """ + vulns.exists(vuln, + %s.all(policyCwe, + vuln.cwes.exists(vulnCwe, vulnCwe == policyCwe) + ) + ) + """.formatted(celCweListLiteral); + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..1927a7253 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseCelPolicyScriptSourceBuilder.java @@ -0,0 +1,55 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class LicenseCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + if ("unresolved".equals(policyCondition.getValue())) { + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return """ + !has(component.resolved_license) + """; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return """ + has(component.resolved_license) + """; + } + } else { + final String escapedLicenseUuid = escapeJson(policyCondition.getValue()); + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return """ + component.resolved_license.uuid == "%s" + """.formatted(escapedLicenseUuid); + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return """ + component.resolved_license.uuid != "%s" + """.formatted(escapedLicenseUuid); + } + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..87b8bf540 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/LicenseGroupCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class LicenseGroupCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + component.resolved_license.groups.exists(group, group.uuid == "%s") + """.formatted(escapeJson(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..45f9b0e92 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/PackageUrlCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class PackageUrlCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + component.purl.matches("%s") + """.formatted(escapeJson(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.NO_MATCH) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..58b01ec43 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java @@ -0,0 +1,26 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class SeverityCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String escapedPolicyValue = escapeJson(policyCondition.getValue()); + + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return """ + vulns.exists(vuln, vuln.severity == "%s") + """.formatted(escapedPolicyValue); + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return """ + vulns.exists(vuln, vuln.severity != "%s") + """.formatted(escapedPolicyValue); + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..f5e46af6c --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java @@ -0,0 +1,24 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class SwidTagIdCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + component.swid_tag_id.matches("%s") + """.formatted(escapeJson(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.NO_MATCH) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..508a2c276 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java @@ -0,0 +1,41 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class VulnerabilityIdCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + @Override + public String apply(final PolicyCondition policyCondition) { + final String scriptSrc = """ + vulns.exists(v, v.id == "%s") + """.formatted(escapeJson(policyCondition.getValue())); + + if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { + return scriptSrc; + } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { + return "!" + scriptSrc; + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java b/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java index 03626b051..eade881ee 100644 --- a/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java +++ b/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java @@ -11,6 +11,7 @@ import org.dependencytrack.model.WorkflowStatus; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.policy.PolicyEngine; +import org.dependencytrack.policy.cel.CelPolicyEngine; import java.time.Instant; import java.util.Date; @@ -34,7 +35,7 @@ public void inform(final Event e) { try (final var qm = new QueryManager()) { projectPolicyEvaluationState = qm.updateStartTimeIfWorkflowStateExists(event.getChainIdentifier(), POLICY_EVALUATION); try { - new PolicyEngine().evaluateProject(event.getUuid()); + new CelPolicyEngine().evaluateProject(event.getUuid()); qm.updateWorkflowStateToComplete(projectPolicyEvaluationState); } catch (Exception ex) { qm.updateWorkflowStateToFailed(projectPolicyEvaluationState, ex.getMessage()); @@ -46,7 +47,7 @@ public void inform(final Event e) { try (final var qm = new QueryManager()) { componentMetricsEvaluationState = qm.updateStartTimeIfWorkflowStateExists(event.getChainIdentifier(), POLICY_EVALUATION); try { - new PolicyEngine().evaluate(event.getUuid()); + new CelPolicyEngine().evaluateComponent(event.getUuid()); qm.updateWorkflowStateToComplete(componentMetricsEvaluationState); } catch (Exception ex) { qm.updateWorkflowStateToFailed(componentMetricsEvaluationState, ex.getMessage()); diff --git a/src/main/proto/org/hyades/policy/v1/policy.proto b/src/main/proto/org/hyades/policy/v1/policy.proto new file mode 100644 index 000000000..5d1caeae4 --- /dev/null +++ b/src/main/proto/org/hyades/policy/v1/policy.proto @@ -0,0 +1,140 @@ +syntax = "proto3"; + +package org.dependencytrack.policy.v1; + +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_package = "org.dependencytrack.proto.policy.v1"; + +message Component { + + // UUID of the component. + string uuid = 1; + + // Group / namespace of the component. + optional string group = 2; + + // Name of the component. + string name = 3; + + // Version of the component. + string version = 4; + + // Classifier / type of the component. + // May be any of: + // - APPLICATION + // - CONTAINER + // - DEVICE + // - FILE + // - FIRMWARE + // - FRAMEWORK + // - LIBRARY + // - OPERATING_SYSTEM + optional string classifier = 5; + + // CPE of the component. + // https://csrc.nist.gov/projects/security-content-automation-protocol/specifications/cpe + optional string cpe = 6; + + // Package URL of the component. + // https://github.com/package-url/purl-spec + optional string purl = 7; + + // SWID tag ID of the component. + // https://csrc.nist.gov/projects/Software-Identification-SWID + optional string swid_tag_id = 8; + + // Whether the component is internal to the organization. + optional bool is_internal = 9; + + // Whether the component is a direct dependency of the project. + optional bool is_direct_dependency = 10; + + optional string md5 = 20; + optional string sha1 = 21; + optional string sha256 = 22; + optional string sha384 = 23; + optional string sha512 = 24; + optional string sha3_256 = 25; + optional string sha3_384 = 26; + optional string sha3_512 = 27; + optional string blake2b_256 = 28; + optional string blake2b_384 = 29; + optional string blake2b_512 = 30; + optional string blake3 = 31; + + optional string license_name = 50; + optional string license_expression = 51; + optional License resolved_license = 52; +} + +message License { + string uuid = 1; + string id = 2; + string name = 3; + repeated Group groups = 4; + bool is_osi_approved = 5; + bool is_fsf_libre = 6; + bool is_deprecated_id = 7; + bool is_custom = 8; + + message Group { + string uuid = 1; + string name = 2; + } +} + +message Project { + string uuid = 1; + optional string group = 2; + string name = 3; + optional string version = 4; + optional string classifier = 5; + bool is_active = 6; + repeated string tags = 7; + repeated Property properties = 8; + optional string cpe = 9; + optional string purl = 10; + optional string swid_tag_id = 11; + optional google.protobuf.Timestamp last_bom_import = 12; + + message Property { + string group = 1; + string name = 2; + optional string value = 3; + string type = 4; + } +} + +message Vulnerability { + string uuid = 1; + string id = 2; + string source = 3; + repeated Alias aliases = 4; + repeated int32 cwes = 5; + optional google.protobuf.Timestamp created = 6; + optional google.protobuf.Timestamp published = 7; + optional google.protobuf.Timestamp updated = 8; + string severity = 20; + optional double cvssv2_base_score = 21; + optional double cvssv2_impact_subscore = 22; + optional double cvssv2_exploitability_subscore = 23; + optional string cvssv2_vector = 24; + optional double cvssv3_base_score = 25; + optional double cvssv3_impact_subscore = 26; + optional double cvssv3_exploitability_subscore = 27; + optional string cvssv3_vector = 28; + optional double owasp_rr_likelihood_score = 29; + optional double owasp_rr_technical_impact_score = 30; + optional double owasp_rr_business_impact_score = 31; + optional string owasp_rr_vector = 32; + optional double epss_score = 33; + optional double epss_percentile = 34; + + message Alias { + string id = 1; + string source = 2; + } + +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java new file mode 100644 index 000000000..d6ff8fcd2 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -0,0 +1,378 @@ +package org.dependencytrack.policy.cel; + +import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.ComponentIdentity; +import org.dependencytrack.model.License; +import org.dependencytrack.model.LicenseGroup; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Severity; +import org.dependencytrack.model.Vulnerability; +import org.junit.Assert; +import org.junit.Test; + +import java.sql.Date; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CelPolicyEngineTest extends PersistenceCapableTest { + + @Test + public void test() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + "critical" in project.tags + && component.name == "bar" + && vulns.exists(v, v.source == "SNYK") + && component.resolved_license.groups.exists(lg, lg.name == "Permissive") + """); + + final var policy2 = qm.createPolicy("policy2", Policy.Operator.ALL, Policy.ViolationState.WARN); + qm.createPolicyCondition(policy2, PolicyCondition.Subject.VULNERABILITY_ID, PolicyCondition.Operator.IS, "CVE-123"); + + final var policy3 = qm.createPolicy("policy3", Policy.Operator.ALL, Policy.ViolationState.INFO); + final PolicyCondition condition3 = qm.createPolicyCondition(policy3, PolicyCondition.Subject.SWID_TAGID, PolicyCondition.Operator.IS, "foo"); + + final var policy4 = qm.createPolicy("policy4", Policy.Operator.ALL, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy4, PolicyCondition.Subject.CWE, PolicyCondition.Operator.CONTAINS_ALL, "CWE-666, CWE-123, 555"); + + final var project = new Project(); + project.setName("foo"); + qm.persist(project); + qm.bind(project, List.of( + qm.createTag("public-facing"), + qm.createTag("critical") + )); + + final var license = new License(); + license.setName("MIT"); + license.setLicenseId("MIT"); + qm.persist(license); + + final var licenseGroup = new LicenseGroup(); + licenseGroup.setName("Permissive"); + licenseGroup.setLicenses(List.of(license)); + qm.persist(licenseGroup); + + final var component = new Component(); + component.setProject(project); + component.setName("bar"); + component.setResolvedLicense(license); + qm.persist(component); + + final var vulnA = new Vulnerability(); + vulnA.setVulnId("CVE-123"); + vulnA.setSource(Vulnerability.Source.NVD); + vulnA.setCreated(Date.from(LocalDateTime.now().minusYears(1).toInstant(ZoneOffset.UTC))); + qm.persist(vulnA); + + final var vulnB = new Vulnerability(); + vulnB.setVulnId("SNYK-123"); + vulnB.setSource(Vulnerability.Source.SNYK); + vulnB.setCwes(List.of(555, 666, 123)); + qm.persist(vulnB); + + qm.addVulnerability(vulnA, component, AnalyzerIdentity.INTERNAL_ANALYZER); + qm.addVulnerability(vulnB, component, AnalyzerIdentity.SNYK_ANALYZER); + + final var existingViolation = new PolicyViolation(); + existingViolation.setProject(project); + existingViolation.setComponent(component); + existingViolation.setPolicyCondition(condition3); + existingViolation.setType(PolicyViolation.Type.OPERATIONAL); + existingViolation.setTimestamp(new java.util.Date()); + qm.persist(existingViolation); + + final var policyEngine = new CelPolicyEngine(); + policyEngine.evaluateComponent(component.getUuid()); + + final List violations = qm.getAllPolicyViolations(component); + assertThat(violations).isNotEmpty(); + } + + @Test + public void testPolicyOperatorAnyWithOneConditionMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "foo" + """); + final PolicyCondition policyConditionB = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "bar" + """); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("foo"); + qm.persist(component); + + final var policyEngine = new CelPolicyEngine(); + policyEngine.evaluateComponent(component.getUuid()); + + assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> { + assertThat(violation.getPolicy().getUuid()).isEqualTo(policy.getUuid()); + assertThat(violation.getMatchedConditions()).hasSize(1); + assertThat(violation.getMatchedConditions().get(0).getUuid()).isEqualTo(policyConditionA.getUuid()); + }); + } + + @Test + public void testPolicyOperatorAnyWithAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "foo" + """); + final PolicyCondition policyConditionB = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name != "bar" + """); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("foo"); + qm.persist(component); + + final var policyEngine = new CelPolicyEngine(); + policyEngine.evaluateComponent(component.getUuid()); + + assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> { + assertThat(violation.getPolicy().getUuid()).isEqualTo(policy.getUuid()); + assertThat(violation.getMatchedConditions()).hasSize(2); + assertThat(violation.getMatchedConditions().get(0).getUuid()).isEqualTo(policyConditionA.getUuid()); + assertThat(violation.getMatchedConditions().get(1).getUuid()).isEqualTo(policyConditionB.getUuid()); + }); + } + + @Test + public void testPolicyOperatorAllWithOneConditionMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "foo" + """); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "bar" + """); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("foo"); + qm.persist(component); + + final var policyEngine = new CelPolicyEngine(); + policyEngine.evaluateComponent(component.getUuid()); + + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void testPolicyOperatorAllWithAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "foo" + """); + final PolicyCondition policyConditionB = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name != "bar" + """); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("foo"); + qm.persist(component); + + final var policyEngine = new CelPolicyEngine(); + policyEngine.evaluateComponent(component.getUuid()); + + assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> { + assertThat(violation.getPolicy().getUuid()).isEqualTo(policy.getUuid()); + assertThat(violation.getMatchedConditions()).hasSize(2); + assertThat(violation.getMatchedConditions().get(0).getUuid()).isEqualTo(policyConditionA.getUuid()); + assertThat(violation.getMatchedConditions().get(1).getUuid()).isEqualTo(policyConditionB.getUuid()); + }); + } + + @Test + public void testIsDirectDependency() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition policyCondition = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.is_direct_dependency + """); + + final var project = new Project(); + project.setName("foo"); + qm.persist(project); + + final var componentA = new Component(); + componentA.setProject(project); + componentA.setName("bar"); + qm.persist(componentA); + + final var componentB = new Component(); + componentB.setProject(project); + componentB.setName("baz"); + qm.persist(componentB); + + project.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentA).toJSON())); + qm.persist(project); + componentA.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentB).toJSON())); + qm.persist(componentA); + + final var policyEngine = new CelPolicyEngine(); + + policyEngine.evaluateComponent(componentA.getUuid()); + assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); + + policyEngine.evaluateComponent(componentB.getUuid()); + assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); + } + + @Test + public void issue1924() { + Policy policy = qm.createPolicy("Policy 1924", Policy.Operator.ALL, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.SEVERITY, PolicyCondition.Operator.IS, Severity.CRITICAL.name()); + qm.createPolicyCondition(policy, PolicyCondition.Subject.PACKAGE_URL, PolicyCondition.Operator.NO_MATCH, "pkg:deb"); + Project project = qm.createProject("My Project", null, "1", null, null, null, true, false); + qm.persist(project); + ArrayList components = new ArrayList<>(); + Component component = new Component(); + component.setName("OpenSSL"); + component.setVersion("3.0.2-0ubuntu1.6"); + component.setPurl("pkg:deb/openssl@3.0.2-0ubuntu1.6"); + component.setProject(project); + components.add(component); + qm.persist(component); + Vulnerability vulnerability = new Vulnerability(); + vulnerability.setVulnId("1"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.setSeverity(Severity.CRITICAL); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + vulnerability = new Vulnerability(); + vulnerability.setVulnId("2"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.setSeverity(Severity.CRITICAL); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + component = new Component(); + component.setName("Log4J"); + component.setVersion("1.2.16"); + component.setPurl("pkg:mvn/log4j/log4j@1.2.16"); + component.setProject(project); + components.add(component); + qm.persist(component); + vulnerability = new Vulnerability(); + vulnerability.setVulnId("3"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.setSeverity(Severity.CRITICAL); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + vulnerability = new Vulnerability(); + vulnerability.setVulnId("4"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.setSeverity(Severity.CRITICAL); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + CelPolicyEngine policyEngine = new CelPolicyEngine(); + policyEngine.evaluate(components); + final List violations = qm.getAllPolicyViolations(); + Assert.assertEquals(3, violations.size()); + PolicyViolation policyViolation = violations.get(0); + Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); + Assert.assertEquals(PolicyCondition.Subject.SEVERITY, policyViolation.getPolicyCondition().getSubject()); + policyViolation = violations.get(1); + Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); + Assert.assertEquals(PolicyCondition.Subject.SEVERITY, policyViolation.getPolicyCondition().getSubject()); + policyViolation = violations.get(2); + Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); + Assert.assertEquals(PolicyCondition.Subject.PACKAGE_URL, policyViolation.getPolicyCondition().getSubject()); + } + + @Test + public void issue2455() { + Policy policy = qm.createPolicy("Policy 1924", Policy.Operator.ALL, Policy.ViolationState.INFO); + + License license = new License(); + license.setName("Apache 2.0"); + license.setLicenseId("Apache-2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + LicenseGroup lg = qm.createLicenseGroup("Test License Group 1"); + lg.setLicenses(Collections.singletonList(license)); + lg = qm.persist(lg); + lg = qm.detach(LicenseGroup.class, lg.getId()); + license = qm.detach(License.class, license.getId()); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS_NOT, lg.getUuid().toString()); + + license = new License(); + license.setName("MIT"); + license.setLicenseId("MIT"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + lg = qm.createLicenseGroup("Test License Group 2"); + lg.setLicenses(Collections.singletonList(license)); + lg = qm.persist(lg); + lg = qm.detach(LicenseGroup.class, lg.getId()); + license = qm.detach(License.class, license.getId()); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS_NOT, lg.getUuid().toString()); + + Project project = qm.createProject("My Project", null, "1", null, null, null, true, false); + qm.persist(project); + + license = new License(); + license.setName("LGPL"); + license.setLicenseId("LGPL"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + ArrayList components = new ArrayList<>(); + Component component = new Component(); + component.setName("Log4J"); + component.setVersion("2.0.0"); + component.setProject(project); + component.setResolvedLicense(license); + components.add(component); + qm.persist(component); + + CelPolicyEngine policyEngine = new CelPolicyEngine(); + policyEngine.evaluate(components); + final List violations = qm.getAllPolicyViolations(); + Assert.assertEquals(2, violations.size()); + PolicyViolation policyViolation = violations.get(0); + Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); + Assert.assertEquals(PolicyCondition.Subject.LICENSE_GROUP, policyViolation.getPolicyCondition().getSubject()); + policyViolation = violations.get(1); + Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); + Assert.assertEquals(PolicyCondition.Subject.LICENSE_GROUP, policyViolation.getPolicyCondition().getSubject()); + } + +} \ No newline at end of file From e0d35d0fd643fa1f1a518c7d184f8f3ab283d03e Mon Sep 17 00:00:00 2001 From: nscuro Date: Wed, 6 Sep 2023 20:20:10 +0200 Subject: [PATCH 02/46] Add a few custom CEL functions Signed-off-by: nscuro --- pom.xml | 6 + .../policy/cel/CelPolicyEngine.java | 40 +++- .../policy/cel/CelPolicyLibrary.java | 214 +++++++++++++++++- .../policy/cel/CelPolicyScriptHost.java | 6 +- ...onentHashCelPolicyScriptSourceBuilder.java | 2 +- .../proto/org/hyades/policy/v1/policy.proto | 4 +- .../policy/cel/CelPolicyEngineTest.java | 91 ++++++++ 7 files changed, 340 insertions(+), 23 deletions(-) diff --git a/pom.xml b/pom.xml index b61d164fd..cae48fea3 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,7 @@ 1.18.3 2.0.1 1.1.10.1 + 0.2.0 6.4.0 1.1.1 2.0.9 @@ -333,6 +334,11 @@ woodstox-core ${lib.woodstox.version} + + io.github.nscuro + versatile + ${lib.versatile.version} + org.apache.maven maven-artifact diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 07b83276b..d618532af 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -31,8 +31,8 @@ import org.dependencytrack.policy.cel.compat.SeverityCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; -import org.dependencytrack.proto.policy.v1.License; -import org.dependencytrack.proto.policy.v1.Vulnerability; +import org.hyades.proto.policy.v1.License; +import org.hyades.proto.policy.v1.Vulnerability; import org.projectnessie.cel.tools.ScriptCreateException; import org.projectnessie.cel.tools.ScriptException; @@ -81,7 +81,7 @@ public CelPolicyEngine() { this(CelPolicyScriptHost.getInstance()); } - CelPolicyEngine(CelPolicyScriptHost scriptHost) { + CelPolicyEngine(final CelPolicyScriptHost scriptHost) { this.scriptHost = scriptHost; } @@ -89,7 +89,25 @@ public void evaluateProject(final UUID projectUuid) { final Timer.Sample timerSample = Timer.start(); try { - // TODO + // TODO: Temporary solution for demonstration purposes. Super inefficient. + // When evaluating a project, policies can be pre-compiled, and input data pre-loaded + // for all components. + final List componentUuids; + try (final var qm = new QueryManager()) { + final Query query = qm.getPersistenceManager().newQuery(Component.class); + query.setFilter("project.uuid == :projectUuid"); + query.setParameters(projectUuid); + query.setResult("uuid"); + try { + componentUuids = query.executeResultList(UUID.class); + } finally { + query.closeAll(); + } + } + + for (final UUID componentUuid : componentUuids) { + evaluateComponent(componentUuid); + } } finally { timerSample.stop(Timer .builder("dtrack_policy_eval") @@ -339,11 +357,11 @@ private static List getParents(final QueryManager qm, final UUID uuid, fin return getParents(qm, parentUuid, parents); } - private static org.dependencytrack.proto.policy.v1.Component mapComponent(final QueryManager qm, + private static org.hyades.proto.policy.v1.Component mapComponent(final QueryManager qm, final Component component, final Set requirements) { - final org.dependencytrack.proto.policy.v1.Component.Builder builder = - org.dependencytrack.proto.policy.v1.Component.newBuilder() + final org.hyades.proto.policy.v1.Component.Builder builder = + org.hyades.proto.policy.v1.Component.newBuilder() .setUuid(Optional.ofNullable(component.getUuid()).map(UUID::toString).orElse("")) .setGroup(trimToEmpty(component.getGroup())) .setName(trimToEmpty(component.getName())) @@ -412,14 +430,14 @@ private static org.dependencytrack.proto.policy.v1.Component mapComponent(final return builder.build(); } - private static org.dependencytrack.proto.policy.v1.Project mapProject(final Project project, + private static org.hyades.proto.policy.v1.Project mapProject(final Project project, final Set requirements) { if (!requirements.contains(Requirement.PROJECT)) { - return org.dependencytrack.proto.policy.v1.Project.newBuilder().build(); + return org.hyades.proto.policy.v1.Project.newBuilder().build(); } - final org.dependencytrack.proto.policy.v1.Project.Builder builder = - org.dependencytrack.proto.policy.v1.Project.newBuilder() + final org.hyades.proto.policy.v1.Project.Builder builder = + org.hyades.proto.policy.v1.Project.newBuilder() .setUuid(Optional.ofNullable(project.getUuid()).map(UUID::toString).orElse("")) .setGroup(trimToEmpty(project.getGroup())) .setName(trimToEmpty(project.getName())) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index 566a31022..26088b379 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -1,17 +1,28 @@ package org.dependencytrack.policy.cel; import com.google.api.expr.v1alpha1.Type; -import org.dependencytrack.proto.policy.v1.Component; -import org.dependencytrack.proto.policy.v1.License; -import org.dependencytrack.proto.policy.v1.Project; -import org.dependencytrack.proto.policy.v1.Vulnerability; +import io.github.nscuro.versatile.Vers; +import io.github.nscuro.versatile.VersException; +import org.apache.commons.lang3.tuple.Pair; +import org.dependencytrack.persistence.QueryManager; +import org.hyades.proto.policy.v1.Component; +import org.hyades.proto.policy.v1.License; +import org.hyades.proto.policy.v1.Project; +import org.hyades.proto.policy.v1.Vulnerability; import org.projectnessie.cel.EnvOption; import org.projectnessie.cel.Library; import org.projectnessie.cel.ProgramOption; import org.projectnessie.cel.checker.Decls; +import org.projectnessie.cel.common.types.Err; +import org.projectnessie.cel.common.types.Types; +import org.projectnessie.cel.interpreter.functions.Overload; -import java.util.Collections; +import javax.jdo.Query; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.UUID; public class CelPolicyLibrary implements Library { @@ -24,6 +35,10 @@ public class CelPolicyLibrary implements Library { private static final Type TYPE_VULNERABILITY = Decls.newObjectType(Vulnerability.getDescriptor().getFullName()); private static final Type TYPE_VULNERABILITIES = Decls.newListType(TYPE_VULNERABILITY); + private static final String FUNC_DEPENDS_ON = "depends_on"; + private static final String FUNC_IS_DEPENDENCY_OF = "is_dependency_of"; + private static final String FUNC_MATCHES_RANGE = "matches_range"; + @Override public List getCompileOptions() { return List.of( @@ -39,6 +54,45 @@ public List getCompileOptions() { Decls.newVar( VAR_VULNERABILITIES, TYPE_VULNERABILITIES + ), + Decls.newFunction( + FUNC_DEPENDS_ON, + // component.depends_on(org.hyades.policy.v1.Component{"name": "foo"}) + Decls.newInstanceOverload( + "component_depends_on_component_bool", + List.of(TYPE_COMPONENT, TYPE_COMPONENT), + Decls.Bool + ), + // project.depends_on(org.hyades.policy.v1.Component{"name": "foo"}) + Decls.newInstanceOverload( + "project_depends_on_component_bool", + List.of(TYPE_PROJECT, TYPE_COMPONENT), + Decls.Bool + ) + ), + Decls.newFunction( + FUNC_IS_DEPENDENCY_OF, + // component.is_dependency_of(org.hyades.policy.v1.Component{"name": "foo"}) + Decls.newInstanceOverload( + "component_is_dependency_of_component_bool", + List.of(TYPE_COMPONENT, TYPE_COMPONENT), + Decls.Bool + ) + ), + Decls.newFunction( + FUNC_MATCHES_RANGE, + // component.matches_range("vers:golang/>0|!=v3.2.1") + Decls.newInstanceOverload( + "component_matches_range_bool", + List.of(TYPE_COMPONENT, Decls.String), + Decls.Bool + ), + // project.matches_range("vers:golang/>0|!=v3.2.1") + Decls.newInstanceOverload( + "project_matches_range_bool", + List.of(TYPE_PROJECT, Decls.String), + Decls.Bool + ) ) ), EnvOption.types( @@ -53,7 +107,155 @@ public List getCompileOptions() { @Override public List getProgramOptions() { - return Collections.emptyList(); + return List.of( + ProgramOption.functions( + Overload.binary( + FUNC_DEPENDS_ON, + (lhs, rhs) -> { + final Component leafComponent; + if (rhs.value() instanceof final Component rhsValue) { + leafComponent = rhsValue; + } else { + return Err.maybeNoSuchOverloadErr(rhs); + } + + if (lhs.value() instanceof final Project project) { + return Types.boolOf(dependsOn(project, leafComponent)); + } else if (lhs.value() instanceof final Component rootComponent) { + // TODO: Traverse dep graph from rootComponent downwards and look for leafComponent + return Types.boolOf(dependsOn(rootComponent, leafComponent)); + } + + return Err.maybeNoSuchOverloadErr(lhs); + } + ), + Overload.binary( + FUNC_IS_DEPENDENCY_OF, + (lhs, rhs) -> { + final Component leafComponent; + if (lhs.value() instanceof final Component lhsValue) { + leafComponent = lhsValue; + } else { + return Err.maybeNoSuchOverloadErr(lhs); + } + + if (rhs.value() instanceof final Component rootComponent) { + // TODO: traverse dep graph from lhsComponent upwards and look for rhsComponent + return Types.boolOf(isDependencyOf(leafComponent, rootComponent)); + } + + return Err.maybeNoSuchOverloadErr(rhs); + } + ), + Overload.binary( + FUNC_MATCHES_RANGE, + (lhs, rhs) -> { + final String version; + if (lhs.value() instanceof final Component lhsValue) { + version = lhsValue.getVersion(); + } else if (lhs.value() instanceof final Project lhsValue) { + version = lhsValue.getVersion(); + } else { + return Err.maybeNoSuchOverloadErr(lhs); + } + + final String versStr; + if (rhs.value() instanceof final String rhsValue) { + versStr = rhsValue; + } else { + return Err.maybeNoSuchOverloadErr(rhs); + } + + return Types.boolOf(matchesRange(version, versStr)); + } + ) + ) + ); + } + + private static boolean dependsOn(final Project project, final Component component) { + if (project.getUuid().isBlank()) { + // Need a UUID for our starting point. + return false; + } + + final Pair> filterAndParams = toFilterAndParams(component); + if (filterAndParams == null) { + return false; + } + + final String filter = "project.uuid == :projectUuid && " + filterAndParams.getLeft(); + final Map params = filterAndParams.getRight(); + params.put("projectUuid", UUID.fromString(project.getUuid())); + + // TODO: Result can / should likely be cached based on filter and params. + + try (final var qm = new QueryManager()) { + final Query query = + qm.getPersistenceManager().newQuery(org.dependencytrack.model.Component.class); + query.setFilter(filter); + query.setNamedParameters(params); + query.setResult("count(this)"); + try { + return query.executeResultUnique(Long.class) > 0; + } finally { + query.closeAll(); + } + } + } + + private static boolean dependsOn(final Component rootComponent, final Component leafComponent) { + // TODO + return false; + } + + private static boolean isDependencyOf(final Component leafComponent, final Component rootComponent) { + if (leafComponent.getUuid().isBlank()) { + // Need a UUID for our starting point. + return false; + } + + // TODO + + return false; + } + + private static boolean matchesRange(final String version, final String versStr) { + try { + return Vers.parse(versStr).contains(version); + } catch (VersException e) { + return false; + } + } + + private static Pair> toFilterAndParams(final Component component) { + var filters = new ArrayList(); + var params = new HashMap(); + + if (!component.getUuid().isBlank()) { + filters.add("uuid == :uuid"); + params.put("uuid", component.getUuid()); + } + if (!component.getGroup().isBlank()) { + filters.add("group == :group"); + params.put("group", component.getGroup()); + } + if (!component.getName().isBlank()) { + filters.add("name == :name"); + params.put("name", component.getName()); + } + if (!component.getVersion().isBlank()) { + filters.add("version"); + params.put("version", component.getVersion()); + } + + // TODO: Add more fields + + if (filters.isEmpty()) { + return null; + } + + return Pair.of(String.join(" && ", filters), params); } } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java index 0da367981..f66a9fc9b 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -7,9 +7,9 @@ import com.google.common.util.concurrent.Striped; import org.apache.commons.codec.digest.DigestUtils; import org.dependencytrack.policy.cel.CelPolicyScript.Requirement; -import org.dependencytrack.proto.policy.v1.License; -import org.dependencytrack.proto.policy.v1.Project; -import org.dependencytrack.proto.policy.v1.Vulnerability; +import org.hyades.proto.policy.v1.License; +import org.hyades.proto.policy.v1.Project; +import org.hyades.proto.policy.v1.Vulnerability; import org.projectnessie.cel.Ast; import org.projectnessie.cel.CEL; import org.projectnessie.cel.Env; diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java index 6137ee536..14fb4cd05 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java @@ -19,7 +19,7 @@ public String apply(final PolicyCondition policyCondition) { } final String fieldName = hash.getAlgorithm().toLowerCase().replaceAll("-", "_"); - if (org.dependencytrack.proto.policy.v1.Component.getDescriptor().findFieldByName(fieldName) == null) { + if (org.hyades.proto.policy.v1.Component.getDescriptor().findFieldByName(fieldName) == null) { LOGGER.warn("Component does not have a field named %s".formatted(fieldName)); return null; } diff --git a/src/main/proto/org/hyades/policy/v1/policy.proto b/src/main/proto/org/hyades/policy/v1/policy.proto index 5d1caeae4..3718d0e63 100644 --- a/src/main/proto/org/hyades/policy/v1/policy.proto +++ b/src/main/proto/org/hyades/policy/v1/policy.proto @@ -1,11 +1,11 @@ syntax = "proto3"; -package org.dependencytrack.policy.v1; +package org.hyades.policy.v1; import "google/protobuf/timestamp.proto"; option java_multiple_files = true; -option java_package = "org.dependencytrack.proto.policy.v1"; +option java_package = "org.hyades.proto.policy.v1"; message Component { diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index d6ff8fcd2..26ca201b4 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -27,6 +27,21 @@ public class CelPolicyEngineTest extends PersistenceCapableTest { + // Raise a violation when the component is affected by a certain vulnerability, + // which is only exploitable when Apache Tomcat is present in the project as well. + // vulns.exists(v, v.id == "CVE-123) + // && project.depends_on(org.hyades.policy.v1.Component{"name": "tomcat-embed-core"}) + + // Raise a violation when the component is SnakeYAML, affected by CVE-2022-1471, + // and introduced to the project as a dependency of Spring Framework. + // vulns.exists(v, v.id == "CVE-2022-1471") + // && component.purl.startsWith("pkg:maven/org.snakeyaml/snakeyaml") + // && component.is_dependency_of(org.hyades.policy.v1.Component{"group": "org.springframework"}) + + // Raise a violation when the component is affected by CVE-2022-229665, + // and the project was packaged as WAR: + // vulns.exists(v, v.id == "CVE-2022-229665") && project.purl.contains("type=war") + @Test public void test() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); @@ -257,6 +272,82 @@ public void testIsDirectDependency() { assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); } + @Test + public void testProjectDependsOnComponent() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) + """); + + final var project = new Project(); + project.setName("foo"); + qm.persist(project); + + final var componentA = new Component(); + componentA.setProject(project); + componentA.setName("bar"); + qm.persist(componentA); + + final var componentB = new Component(); + componentB.setProject(project); + componentB.setName("baz"); + qm.persist(componentB); + + project.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentA).toJSON())); + qm.persist(project); + componentA.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentB).toJSON())); + qm.persist(componentA); + + final var policyEngine = new CelPolicyEngine(); + + policyEngine.evaluateComponent(componentA.getUuid()); + assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); + + policyEngine.evaluateComponent(componentB.getUuid()); + assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); + } + + @Test + public void testMatchesRange() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.matches_range("vers:generic/<1") + && component.matches_range("vers:golang/>0| Date: Thu, 7 Sep 2023 15:40:37 +0200 Subject: [PATCH 03/46] Make policies work with legacy way of reporting violations Signed-off-by: nscuro --- .../model/PolicyCondition.java | 50 +++++++++++++----- .../persistence/PolicyQueryManager.java | 12 +++++ .../persistence/QueryManager.java | 6 +++ .../policy/cel/CelPolicyEngine.java | 51 +++++++++++-------- .../resources/v1/PolicyConditionResource.java | 35 ++++++++++++- 5 files changed, 119 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/dependencytrack/model/PolicyCondition.java b/src/main/java/org/dependencytrack/model/PolicyCondition.java index 502f364b0..113644c7c 100644 --- a/src/main/java/org/dependencytrack/model/PolicyCondition.java +++ b/src/main/java/org/dependencytrack/model/PolicyCondition.java @@ -63,22 +63,28 @@ public enum Operator { } public enum Subject { - AGE, + AGE(PolicyViolation.Type.OPERATIONAL), //ANALYZER, //BOM, - COORDINATES, - CPE, - EXPRESSION, + COORDINATES(PolicyViolation.Type.OPERATIONAL), + CPE(PolicyViolation.Type.OPERATIONAL), //INHERITED_RISK_SCORE, - LICENSE, - LICENSE_GROUP, - PACKAGE_URL, - SEVERITY, - SWID_TAGID, - VERSION, - COMPONENT_HASH, - CWE, - VULNERABILITY_ID + EXPRESSION(null), + LICENSE(PolicyViolation.Type.LICENSE), + LICENSE_GROUP(PolicyViolation.Type.LICENSE), + PACKAGE_URL(PolicyViolation.Type.OPERATIONAL), + SEVERITY(PolicyViolation.Type.SECURITY), + SWID_TAGID(PolicyViolation.Type.OPERATIONAL), + VERSION(PolicyViolation.Type.OPERATIONAL), + COMPONENT_HASH(PolicyViolation.Type.OPERATIONAL), + CWE(PolicyViolation.Type.SECURITY), + VULNERABILITY_ID(PolicyViolation.Type.SECURITY); + + private final PolicyViolation.Type violationType; + + Subject(final PolicyViolation.Type violationType) { + this.violationType = violationType; + } } @PrimaryKey @@ -111,6 +117,12 @@ public enum Subject { @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The value may only contain printable characters") private String value; + @Persistent + @Column(name = "VIOLATIONTYPE", allowsNull = "true") + @Size(min = 1, max = 255) + @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The violation type may only contain printable characters") + private PolicyViolation.Type violationType; + /** * The unique identifier of the object. */ @@ -160,6 +172,18 @@ public void setValue(String value) { this.value = value; } + public PolicyViolation.Type getViolationType() { + if (subject != null && subject.violationType != null) { + return subject.violationType; + } + + return violationType; + } + + public void setViolationType(PolicyViolation.Type violationType) { + this.violationType = violationType; + } + public UUID getUuid() { return uuid; } diff --git a/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java b/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java index f2af10162..ac7d406f5 100644 --- a/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java @@ -185,11 +185,22 @@ public Policy createPolicy(String name, Policy.Operator operator, Policy.Violati */ public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCondition.Subject subject, final PolicyCondition.Operator operator, final String value) { + return createPolicyCondition(policy, subject, operator, value, null); + } + + /** + * Creates a policy condition for the specified Project. + * @return the created PolicyCondition object + */ + public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCondition.Subject subject, + final PolicyCondition.Operator operator, final String value, + final PolicyViolation.Type violationType) { final PolicyCondition pc = new PolicyCondition(); pc.setPolicy(policy); pc.setSubject(subject); pc.setOperator(operator); pc.setValue(value); + pc.setViolationType(violationType); return persist(pc); } @@ -202,6 +213,7 @@ public PolicyCondition updatePolicyCondition(final PolicyCondition policyConditi pc.setSubject(policyCondition.getSubject()); pc.setOperator(policyCondition.getOperator()); pc.setValue(policyCondition.getValue()); + pc.setViolationType(policyCondition.getViolationType()); return persist(pc); } diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index 1df482fb2..78ae84d1b 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -654,6 +654,12 @@ public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCo return getPolicyQueryManager().createPolicyCondition(policy, subject, operator, value); } + public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCondition.Subject subject, + final PolicyCondition.Operator operator, final String value, + final PolicyViolation.Type violationType) { + return getPolicyQueryManager().createPolicyCondition(policy, subject, operator, value, violationType); + } + public PolicyCondition updatePolicyCondition(final PolicyCondition policyCondition) { return getPolicyQueryManager().updatePolicyCondition(policyCondition); } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index d618532af..b0bd0a749 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -52,6 +52,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_COMPONENT; @@ -99,7 +100,7 @@ public void evaluateProject(final UUID projectUuid) { query.setParameters(projectUuid); query.setResult("uuid"); try { - componentUuids = query.executeResultList(UUID.class); + componentUuids = List.copyOf(query.executeResultList(UUID.class)); } finally { query.closeAll(); } @@ -135,6 +136,7 @@ public void evaluateComponent(final UUID componentUuid) { if (policies.isEmpty()) { // With no applicable policies, there's no way to resolve violations. // As a compensation, simply delete all violations associated with the component. + LOGGER.info("No applicable policies found for component %s".formatted(componentUuid)); reconcileViolations(qm, component, Collections.emptyList()); return; } @@ -143,7 +145,7 @@ public void evaluateComponent(final UUID componentUuid) { // Compiled scripts are cached in-memory by CelPolicyScriptHost, so if the same script // is encountered for multiple components (possibly concurrently), the compilation is // a one-time effort. - LOGGER.debug("Compiling policy scripts for component %s".formatted(componentUuid)); + LOGGER.info("Compiling policy scripts for component %s".formatted(componentUuid)); final List> conditionScriptPairs = policies.stream() .map(Policy::getPolicyConditions) .flatMap(Collection::stream) @@ -183,7 +185,7 @@ public void evaluateComponent(final UUID componentUuid) { // // What we want to avoid is loading data we don't need, and loading it multiple times. // Instead, only load what's really needed, and only do so once. - LOGGER.debug("Determining evaluation requirements for component %s and %d policy conditions" + LOGGER.info("Determining evaluation requirements for component %s and %d policy conditions" .formatted(componentUuid, conditionScriptPairs.size())); final Set requirements = conditionScriptPairs.stream() .map(Pair::getRight) @@ -192,7 +194,7 @@ public void evaluateComponent(final UUID componentUuid) { .collect(Collectors.toSet()); // Prepare the script arguments according to the requirements gathered before. - LOGGER.debug("Building script arguments for component %s and requirements %s" + LOGGER.info("Building script arguments for component %s and requirements %s" .formatted(componentUuid, requirements)); final Map scriptArgs = Map.of( VAR_COMPONENT, mapComponent(qm, component, requirements), @@ -200,13 +202,13 @@ VAR_PROJECT, mapProject(component.getProject(), requirements), VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) ); - LOGGER.debug("Evaluating component %s against %d applicable policy conditions" + LOGGER.info("Evaluating component %s against %d applicable policy conditions" .formatted(componentUuid, conditionScriptPairs.size())); final var conditionsViolated = new HashSet(); for (final Pair conditionScriptPair : conditionScriptPairs) { final PolicyCondition condition = conditionScriptPair.getLeft(); final CelPolicyScript script = conditionScriptPair.getRight(); - LOGGER.debug("Executing script for policy condition %s with arguments: %s" + LOGGER.info("Executing script for policy condition %s with arguments: %s" .formatted(condition.getUuid(), scriptArgs)); try { @@ -220,7 +222,7 @@ VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) // Group the detected condition violations by policy. Necessary to be able to evaluate // each policy's operator (ANY, ALL). - LOGGER.debug("Detected violation of %d policy conditions for component %s; Evaluating policy operators" + LOGGER.info("Detected violation of %d policy conditions for component %s; Evaluating policy operators" .formatted(conditionsViolated.size(), componentUuid)); final Map> violatedConditionsByPolicy = conditionsViolated.stream() .collect(Collectors.groupingBy(PolicyCondition::getPolicy)); @@ -229,23 +231,30 @@ VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) // match the configured policy operator. When the operator is ALL, and not all conditions // of the policy were violated, we don't want to create any violations. final List violations = violatedConditionsByPolicy.entrySet().stream() - .map(policyAndViolatedConditions -> { + .flatMap(policyAndViolatedConditions -> { final Policy policy = policyAndViolatedConditions.getKey(); final List violatedConditions = policyAndViolatedConditions.getValue(); if ((policy.getOperator() == Policy.Operator.ANY && !violatedConditions.isEmpty()) || (policy.getOperator() == Policy.Operator.ALL && violatedConditions.size() == policy.getPolicyConditions().size())) { - final var violation = new PolicyViolation(); - violation.setProject(component.getProject()); - violation.setComponent(component); - violation.setPolicy(policy); - violation.setType(PolicyViolation.Type.OPERATIONAL); // TODO: We need violationType at policy level - violation.setMatchedConditions(violatedConditions); - violation.setTimestamp(new Date()); - return violation; + // TODO: Only a single violation should be raised, and instead multiple matched conditions + // should be associated with it. Keeping the existing behavior in order to avoid having to + // touch too much persistence and REST API code. + return violatedConditions.stream() + .map(condition -> { + final var violation = new PolicyViolation(); + violation.setProject(component.getProject()); + violation.setComponent(component); + violation.setPolicy(policy); + violation.setType(condition.getViolationType()); // TODO: We need violationType at policy level + violation.setPolicyCondition(condition); + // violation.setMatchedConditions(violatedConditions); + violation.setTimestamp(new Date()); + return violation; + }); } - return null; + return Stream.empty(); }) .filter(Objects::nonNull) .toList(); @@ -266,7 +275,7 @@ VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) .register(Metrics.getRegistry())); } - LOGGER.debug("Policy evaluation completed for component %s".formatted(componentUuid)); + LOGGER.info("Policy evaluation completed for component %s".formatted(componentUuid)); } // TODO: Move to PolicyQueryManager @@ -358,8 +367,8 @@ private static List getParents(final QueryManager qm, final UUID uuid, fin } private static org.hyades.proto.policy.v1.Component mapComponent(final QueryManager qm, - final Component component, - final Set requirements) { + final Component component, + final Set requirements) { final org.hyades.proto.policy.v1.Component.Builder builder = org.hyades.proto.policy.v1.Component.newBuilder() .setUuid(Optional.ofNullable(component.getUuid()).map(UUID::toString).orElse("")) @@ -431,7 +440,7 @@ private static org.hyades.proto.policy.v1.Component mapComponent(final QueryMana } private static org.hyades.proto.policy.v1.Project mapProject(final Project project, - final Set requirements) { + final Set requirements) { if (!requirements.contains(Requirement.PROJECT)) { return org.hyades.proto.policy.v1.Project.newBuilder().build(); } diff --git a/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java b/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java index 6d30a684b..2efa8319f 100644 --- a/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java @@ -32,6 +32,8 @@ import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.policy.cel.CelPolicyScriptHost; +import org.projectnessie.cel.tools.ScriptCreateException; import javax.validation.Validator; import javax.ws.rs.Consumes; @@ -79,8 +81,24 @@ public Response createPolicyCondition( try (QueryManager qm = new QueryManager()) { Policy policy = qm.getObjectByUuid(Policy.class, uuid); if (policy != null) { + if (jsonPolicyCondition.getSubject() == PolicyCondition.Subject.EXPRESSION) { + try { + CelPolicyScriptHost.getInstance().compile(jsonPolicyCondition.getValue()); + } catch (ScriptCreateException e) { + // TODO: Bring this in a format that is digestible by the frontend. + // It'd be great if we could give visual hints to users as to *where* + // in their script the errors were found. The exception provides that info. + return Response.status(Response.Status.BAD_REQUEST).entity("The provided CEL expression is invalid: %s".formatted(e.getMessage())).build(); + } + + if (jsonPolicyCondition.getViolationType() == null) { + return Response.status(Response.Status.BAD_REQUEST).entity("Expression conditions must define a violation type").build(); + } + } + final PolicyCondition pc = qm.createPolicyCondition(policy, jsonPolicyCondition.getSubject(), - jsonPolicyCondition.getOperator(), StringUtils.trimToNull(jsonPolicyCondition.getValue())); + jsonPolicyCondition.getOperator(), StringUtils.trimToNull(jsonPolicyCondition.getValue()), + jsonPolicyCondition.getViolationType()); return Response.status(Response.Status.CREATED).entity(pc).build(); } else { return Response.status(Response.Status.NOT_FOUND).entity("The UUID of the policy could not be found.").build(); @@ -110,6 +128,21 @@ public Response updatePolicyCondition(PolicyCondition jsonPolicyCondition) { try (QueryManager qm = new QueryManager()) { PolicyCondition pc = qm.getObjectByUuid(PolicyCondition.class, jsonPolicyCondition.getUuid()); if (pc != null) { + if (jsonPolicyCondition.getSubject() == PolicyCondition.Subject.EXPRESSION) { + try { + CelPolicyScriptHost.getInstance().compile(jsonPolicyCondition.getValue()); + } catch (ScriptCreateException e) { + // TODO: Bring this in a format that is digestible by the frontend. + // It'd be great if we could give visual hints to users as to *where* + // in their script the errors were found. The exception provides that info. + return Response.status(Response.Status.BAD_REQUEST).entity("The provided CEL expression is invalid: %s".formatted(e.getMessage())).build(); + } + + if (jsonPolicyCondition.getViolationType() == null) { + return Response.status(Response.Status.BAD_REQUEST).entity("Expression conditions must define a violation type").build(); + } + } + pc = qm.updatePolicyCondition(jsonPolicyCondition); return Response.status(Response.Status.CREATED).entity(pc).build(); } else { From 6703e62599e80ce8a39d49cf0824f770be265e71 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 7 Sep 2023 15:41:01 +0200 Subject: [PATCH 04/46] Implement `is_dependency_of` CEL function Signed-off-by: nscuro --- .../policy/cel/CelPolicyLibrary.java | 101 +++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index 26088b379..ee114090f 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -1,5 +1,6 @@ package org.dependencytrack.policy.cel; +import alpine.common.logging.Logger; import com.google.api.expr.v1alpha1.Type; import io.github.nscuro.versatile.Vers; import io.github.nscuro.versatile.VersException; @@ -18,6 +19,10 @@ import org.projectnessie.cel.interpreter.functions.Overload; import javax.jdo.Query; +import javax.jdo.datastore.JDOConnection; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -26,6 +31,8 @@ public class CelPolicyLibrary implements Library { + private static final Logger LOGGER = Logger.getLogger(CelPolicyLibrary.class); + static final String VAR_COMPONENT = "component"; static final String VAR_PROJECT = "project"; static final String VAR_VULNERABILITIES = "vulns"; @@ -215,7 +222,99 @@ private static boolean isDependencyOf(final Component leafComponent, final Compo return false; } - // TODO + final var filters = new ArrayList(); + final var params = new HashMap(); + var paramPosition = 1; + if (!rootComponent.getUuid().isBlank()) { + filters.add("\"C\".\"UUID\" = ?"); + params.put(paramPosition++, rootComponent.getUuid()); + } + if (!rootComponent.getGroup().isBlank()) { + filters.add("\"C\".\"GROUP\" = ?"); + params.put(paramPosition++, rootComponent.getGroup()); + } + if (!rootComponent.getName().isBlank()) { + filters.add("\"C\".\"NAME\" = ?"); + params.put(paramPosition++, rootComponent.getName()); + } + if (!rootComponent.getVersion().isBlank()) { + filters.add("\"C\".\"VERSION\" = ?"); + params.put(paramPosition++, rootComponent.getVersion()); + } + if (!rootComponent.getPurl().isBlank()) { + filters.add("\"C\".\"PURL\" = ?"); + params.put(paramPosition++, rootComponent.getPurl()); + } + + if (filters.isEmpty()) { + return false; + } + + final String sqlFilter = String.join(" AND ", filters); + + final var query = """ + WITH RECURSIVE + "CTE_DEPENDENCIES" ("UUID", "PROJECT_ID", "FOUND", "COMPONENTS_SEEN") AS ( + SELECT + "C"."UUID", + "C"."PROJECT_ID", + CASE WHEN (%s) THEN TRUE ELSE FALSE END AS "FOUND", + ARRAY []::BIGINT[] AS "COMPONENTS_SEEN" + FROM + "COMPONENT" AS "C" + WHERE + -- TODO: Need to get project ID from somewhere to speed up + -- this initial query for the CTE. + -- "PROJECT_ID" = ? + "C"."DIRECT_DEPENDENCIES" IS NOT NULL + AND "C"."DIRECT_DEPENDENCIES" LIKE ? + UNION ALL + SELECT + "C"."UUID" AS "UUID", + "C"."PROJECT_ID" AS "PROJECT_ID", + CASE WHEN (%s) THEN TRUE ELSE FALSE END AS "FOUND", + ARRAY_APPEND("COMPONENTS_SEEN", "C"."ID") + FROM + "COMPONENT" AS "C" + INNER JOIN + "CTE_DEPENDENCIES" + ON "C"."PROJECT_ID" = "CTE_DEPENDENCIES"."PROJECT_ID" + AND "C"."DIRECT_DEPENDENCIES" LIKE ('%%"' || "CTE_DEPENDENCIES"."UUID" || '"%%') + WHERE + "C"."PROJECT_ID" = "CTE_DEPENDENCIES"."PROJECT_ID" + AND ( + "FOUND" OR ("C"."DIRECT_DEPENDENCIES" IS NOT NULL AND "C"."ID" != ANY ("COMPONENTS_SEEN")) + ) + ) + SELECT BOOL_OR("FOUND") FROM "CTE_DEPENDENCIES"; + """.formatted(sqlFilter, sqlFilter); + + try (final var qm = new QueryManager()) { + final JDOConnection jdoConnection = qm.getPersistenceManager().getDataStoreConnection(); + try { + final var connection = (Connection) jdoConnection.getNativeConnection(); + final var preparedStatement = connection.prepareStatement(query); + // Params need to be set twice because the rootComponent filter + // appears twice in the query... This needs improvement. + for (final Map.Entry param : params.entrySet()) { + preparedStatement.setObject(param.getKey(), param.getValue()); + } + preparedStatement.setString(params.size() + 1, "%" + leafComponent.getUuid() + "%"); + for (final Map.Entry param : params.entrySet()) { + preparedStatement.setObject((params.size() + 1) + param.getKey(), param.getValue()); + } + + try (final ResultSet rs = preparedStatement.executeQuery()) { + if (rs.next()) { + return rs.getBoolean(1); + } + } + } catch (SQLException e) { + LOGGER.warn("Failed to execute query", e); + } finally { + jdoConnection.close(); + } + } return false; } From fe0a3ed8ef9a19aa92ad24aee78c211b22eb9a3c Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 7 Sep 2023 16:00:09 +0200 Subject: [PATCH 05/46] Support vuln aliases in CEL policies Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index b0bd0a749..d5882ce40 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -31,6 +31,7 @@ import org.dependencytrack.policy.cel.compat.SeverityCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; +import org.dependencytrack.util.VulnerabilityUtil; import org.hyades.proto.policy.v1.License; import org.hyades.proto.policy.v1.Vulnerability; import org.projectnessie.cel.tools.ScriptCreateException; @@ -534,14 +535,25 @@ private static List loadVulnerabilities(final QueryManager qm, Optional.ofNullable(v.getCreated()).map(Timestamps::fromDate).ifPresent(builder::setCreated); Optional.ofNullable(v.getPublished()).map(Timestamps::fromDate).ifPresent(builder::setPublished); Optional.ofNullable(v.getUpdated()).map(Timestamps::fromDate).ifPresent(builder::setUpdated); + + if (requirements.contains(Requirement.VULNERABILITY_ALIASES)) { + // TODO: Dirty hack, create a proper solution. Likely needs caching, too. + final var tmpVuln = new org.dependencytrack.model.Vulnerability(); + tmpVuln.setVulnId(builder.getId()); + tmpVuln.setSource(builder.getSource()); + tmpVuln.setAliases(qm.getVulnerabilityAliases(tmpVuln)); + VulnerabilityUtil.getUniqueAliases(tmpVuln).stream() + .map(alias -> Vulnerability.Alias.newBuilder() + .setId(alias.getKey().name()) + .setSource(alias.getValue()) + .build()) + .forEach(builder::addAliases); + } + return builder; }) .toList(); - if (requirements.contains(Requirement.VULNERABILITY_ALIASES)) { - // TODO: Fetch aliases - } - return vulnBuilders.stream() .map(Vulnerability.Builder::build) .toList(); From fee1f461f95118810809e8cf61c46fa580c55166 Mon Sep 17 00:00:00 2001 From: nscuro Date: Fri, 8 Sep 2023 11:56:03 +0200 Subject: [PATCH 06/46] Few minor adjustments Signed-off-by: nscuro --- .../model/PolicyCondition.java | 4 +-- .../parser/cyclonedx/CycloneDXExporter.java | 2 +- .../persistence/PolicyQueryManager.java | 6 +++- .../policy/cel/CelPolicyEngine.java | 29 ++++++++------- .../policy/cel/CelPolicyLibrary.java | 2 +- ...ordinatesCelPolicyScriptSourceBuilder.java | 35 +++++++++++++++++++ 6 files changed, 61 insertions(+), 17 deletions(-) create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java diff --git a/src/main/java/org/dependencytrack/model/PolicyCondition.java b/src/main/java/org/dependencytrack/model/PolicyCondition.java index 113644c7c..71d4bbb80 100644 --- a/src/main/java/org/dependencytrack/model/PolicyCondition.java +++ b/src/main/java/org/dependencytrack/model/PolicyCondition.java @@ -111,9 +111,9 @@ public enum Subject { private Subject subject; @Persistent - @Column(name = "VALUE", allowsNull = "false") + @Column(name = "VALUE", allowsNull = "false", jdbcType = "VARCHAR", length = 1024) @NotBlank - @Size(min = 1, max = 255) + @Size(min = 1, max = 1024) @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The value may only contain printable characters") private String value; diff --git a/src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java b/src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java index df743fe77..48cda1fdf 100644 --- a/src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java +++ b/src/main/java/org/dependencytrack/parser/cyclonedx/CycloneDXExporter.java @@ -83,7 +83,7 @@ private Bom create(final Listcomponents, final List bom.setComponents(cycloneComponents); bom.setServices(cycloneServices); bom.setVulnerabilities(cycloneVulnerabilities); - if (components != null) { + if (cycloneComponents != null) { bom.setDependencies(ModelConverter.generateDependencies(project, components)); } return bom; diff --git a/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java b/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java index ac7d406f5..44965630d 100644 --- a/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java @@ -211,7 +211,11 @@ public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCo public PolicyCondition updatePolicyCondition(final PolicyCondition policyCondition) { final PolicyCondition pc = getObjectByUuid(PolicyCondition.class, policyCondition.getUuid()); pc.setSubject(policyCondition.getSubject()); - pc.setOperator(policyCondition.getOperator()); + if (policyCondition.getSubject() == PolicyCondition.Subject.EXPRESSION) { + pc.setOperator(PolicyCondition.Operator.MATCHES); + } else { + pc.setOperator(policyCondition.getOperator()); + } pc.setValue(policyCondition.getValue()); pc.setViolationType(policyCondition.getViolationType()); return persist(pc); diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index d5882ce40..65a427d98 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -23,6 +23,7 @@ import org.dependencytrack.policy.cel.CelPolicyScript.Requirement; import org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.ComponentHashCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.CoordinatesCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.CpeCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.CweCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.LicenseCelPolicyScriptSourceBuilder; @@ -64,18 +65,22 @@ public class CelPolicyEngine { private static final Logger LOGGER = Logger.getLogger(CelPolicyEngine.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final Map SCRIPT_BUILDERS = Map.of( - Subject.CPE, new CpeCelPolicyScriptSourceBuilder(), - Subject.COMPONENT_HASH, new ComponentHashCelPolicyScriptSourceBuilder(), - Subject.CWE, new CweCelPolicyScriptSourceBuilder(), - Subject.EXPRESSION, PolicyCondition::getValue, - Subject.LICENSE, new LicenseCelPolicyScriptSourceBuilder(), - Subject.LICENSE_GROUP, new LicenseGroupCelPolicyScriptSourceBuilder(), - Subject.PACKAGE_URL, new PackageUrlCelPolicyScriptSourceBuilder(), - Subject.SEVERITY, new SeverityCelPolicyScriptSourceBuilder(), - Subject.SWID_TAGID, new SwidTagIdCelPolicyScriptSourceBuilder(), - Subject.VULNERABILITY_ID, new VulnerabilityIdCelPolicyScriptSourceBuilder() - ); + private static final Map SCRIPT_BUILDERS; + + static { + SCRIPT_BUILDERS = new HashMap<>(); + SCRIPT_BUILDERS.put(Subject.CPE, new CpeCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.COMPONENT_HASH, new ComponentHashCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.COORDINATES, new CoordinatesCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.CWE, new CweCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.EXPRESSION, PolicyCondition::getValue); + SCRIPT_BUILDERS.put(Subject.LICENSE, new LicenseCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.LICENSE_GROUP, new LicenseGroupCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.PACKAGE_URL, new PackageUrlCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.SEVERITY, new SeverityCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.SWID_TAGID, new SwidTagIdCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.VULNERABILITY_ID, new VulnerabilityIdCelPolicyScriptSourceBuilder()); + } private final CelPolicyScriptHost scriptHost; diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index ee114090f..33d85ddfe 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -283,7 +283,7 @@ CASE WHEN (%s) THEN TRUE ELSE FALSE END AS "FOUND", WHERE "C"."PROJECT_ID" = "CTE_DEPENDENCIES"."PROJECT_ID" AND ( - "FOUND" OR ("C"."DIRECT_DEPENDENCIES" IS NOT NULL AND "C"."ID" != ANY ("COMPONENTS_SEEN")) + "FOUND" OR "C"."DIRECT_DEPENDENCIES" IS NOT NULL ) ) SELECT BOOL_OR("FOUND") FROM "CTE_DEPENDENCIES"; diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..a28ce3e00 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java @@ -0,0 +1,35 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.model.PolicyCondition; +import org.json.JSONObject; + +import java.util.Optional; + +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; + +public class CoordinatesCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + + @Override + public String apply(final PolicyCondition condition) { + if (condition.getValue() == null) { + return null; + } + + final JSONObject def = new JSONObject(condition.getValue()); + final String group = Optional.ofNullable(def.optString("group", null)).orElse(".*"); + final String name = Optional.ofNullable(def.optString("name", null)).orElse(".*"); + final String version = Optional.ofNullable(def.optString("version")).orElse(".*"); + + final var scriptSrc = """ + component.group.matches("%s") && component.name.matches("%s") && component.version.matches("%s") + """.formatted(escapeJson(group), escapeJson(name), escapeJson(version)); + if (condition.getOperator() == PolicyCondition.Operator.MATCHES) { + return scriptSrc; + } else if (condition.getOperator() == PolicyCondition.Operator.NO_MATCH) { + return "!(%s)".formatted(scriptSrc); + } + + return null; + } + +} From b95d72aea5eb700fa078048dbe0a04df80ff3ba5 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 11 Sep 2023 20:16:26 +0200 Subject: [PATCH 07/46] Return CEL errors in API response Signed-off-by: nscuro --- .../policy/cel/CelPolicyScriptHost.java | 19 ++++++++++++++++++- .../resources/v1/PolicyConditionResource.java | 17 +++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java index f66a9fc9b..7d8c064c8 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -16,15 +16,23 @@ import org.projectnessie.cel.Env.AstIssuesTuple; import org.projectnessie.cel.Library; import org.projectnessie.cel.Program; +import org.projectnessie.cel.common.CELError; +import org.projectnessie.cel.common.Errors; +import org.projectnessie.cel.common.Location; +import org.projectnessie.cel.common.types.Err.ErrException; import org.projectnessie.cel.common.types.pb.ProtoTypeRegistry; import org.projectnessie.cel.tools.ScriptCreateException; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.stream.Collectors; +import static org.projectnessie.cel.Issues.newIssues; +import static org.projectnessie.cel.common.Source.newTextSource; + public class CelPolicyScriptHost { private static final Logger LOGGER = Logger.getLogger(CelPolicyScriptHost.class); @@ -75,7 +83,16 @@ public CelPolicyScript compile(final String scriptSrc) throws ScriptCreateExcept throw new ScriptCreateException("Failed to parse script", astIssuesTuple.getIssues()); } - astIssuesTuple = environment.check(astIssuesTuple.getAst()); + try { + astIssuesTuple = environment.check(astIssuesTuple.getAst()); + } catch (ErrException e) { + // TODO: Bring error message in a more digestible form. + throw new ScriptCreateException("Failed to check script", newIssues(new Errors(newTextSource(scriptSrc)) + .append(Collections.singletonList( + new CELError(e, Location.newLocation(1, 1), e.getMessage()) + )) + )); + } if (astIssuesTuple.hasIssues()) { throw new ScriptCreateException("Failed to check script", astIssuesTuple.getIssues()); } diff --git a/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java b/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java index 2efa8319f..d0f109345 100644 --- a/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java @@ -33,6 +33,7 @@ import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.policy.cel.CelPolicyScriptHost; +import org.projectnessie.cel.common.CELError; import org.projectnessie.cel.tools.ScriptCreateException; import javax.validation.Validator; @@ -45,6 +46,8 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.Map; /** * JAX-RS resources for processing policies. @@ -132,10 +135,16 @@ public Response updatePolicyCondition(PolicyCondition jsonPolicyCondition) { try { CelPolicyScriptHost.getInstance().compile(jsonPolicyCondition.getValue()); } catch (ScriptCreateException e) { - // TODO: Bring this in a format that is digestible by the frontend. - // It'd be great if we could give visual hints to users as to *where* - // in their script the errors were found. The exception provides that info. - return Response.status(Response.Status.BAD_REQUEST).entity("The provided CEL expression is invalid: %s".formatted(e.getMessage())).build(); + final var errors = new ArrayList>(); + for (final CELError error : e.getIssues().getErrors()) { + errors.add(Map.of( + "line", error.getLocation().line(), + "column", error.getLocation().column(), + "message", error.getMessage() + )); + } + + return Response.status(Response.Status.BAD_REQUEST).entity(Map.of("celErrors", errors)).build(); } if (jsonPolicyCondition.getViolationType() == null) { From 2f6c2e3bd232ddb2b91443587c81b8c5ddde480b Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 12 Sep 2023 12:33:31 +0200 Subject: [PATCH 08/46] Fix some vulnerability fields not being fetched for policies Signed-off-by: nscuro --- .../dependencytrack/model/Vulnerability.java | 25 +++++++++++++++ .../policy/cel/CelPolicyEngine.java | 32 ++----------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/dependencytrack/model/Vulnerability.java b/src/main/java/org/dependencytrack/model/Vulnerability.java index 4fedf1bc5..ae4eeeb79 100644 --- a/src/main/java/org/dependencytrack/model/Vulnerability.java +++ b/src/main/java/org/dependencytrack/model/Vulnerability.java @@ -92,6 +92,30 @@ @Persistent(name = "cvssV2BaseScore"), @Persistent(name = "cvssV3BaseScore") }), + @FetchGroup(name = "POLICY", members = { + @Persistent(name = "uuid"), + @Persistent(name = "vulnId"), + @Persistent(name = "source"), + @Persistent(name = "cwes"), + @Persistent(name = "created"), + @Persistent(name = "published"), + @Persistent(name = "updated"), + @Persistent(name = "cvssV2BaseScore"), + @Persistent(name = "cvssV2ImpactSubScore"), + @Persistent(name = "cvssV2ExploitabilitySubScore"), + @Persistent(name = "cvssV2Vector"), + @Persistent(name = "cvssV3BaseScore"), + @Persistent(name = "cvssV3ImpactSubScore"), + @Persistent(name = "cvssV3ExploitabilitySubScore"), + @Persistent(name = "cvssV3Vector"), + @Persistent(name = "owaspRRLikelihoodScore"), + @Persistent(name = "owaspRRTechnicalImpactScore"), + @Persistent(name = "owaspRRBusinessImpactScore"), + @Persistent(name = "owaspRRVector"), + @Persistent(name = "severity"), + @Persistent(name = "epssScore"), + @Persistent(name = "epssPercentile") + }), @FetchGroup(name = "VULNERABLE_SOFTWARE", members = { @Persistent(name = "vulnerableSoftware") }) @@ -109,6 +133,7 @@ public enum FetchGroup { BASIC, COMPONENTS, METRICS_UPDATE, + POLICY, VULNERABLE_SOFTWARE, } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 65a427d98..d4006d3e0 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -478,39 +478,13 @@ private static List loadVulnerabilities(final QueryManager qm, final Query query = qm.getPersistenceManager().newQuery(org.dependencytrack.model.Vulnerability.class); + query.getFetchPlan().clearGroups(); + query.getFetchPlan().setGroup(org.dependencytrack.model.Vulnerability.FetchGroup.POLICY.name()); query.setFilter("components.contains(:component)"); query.setParameters(component); - // Avoid some ORM overhead by explicitly specifying the fields we want - // to fetch, and load them into a result class, rather than a candidate - // class. The returned Vulnerability objects are thus just dumb POJOs and - // not attached to the persistence context. - query.setResult(""" - uuid, - vulnId, - source, - cwes, - created, - published, - updated, - cvssV2BaseScore, - cvssV2ImpactSubScore, - cvssV2ExploitabilitySubScore, - cvssV2Vector, - cvssV3BaseScore, - cvssV3ImpactSubScore, - cvssV3ExploitabilitySubScore, - cvssV3Vector, - owaspRRLikelihoodScore, - owaspRRTechnicalImpactScore, - owaspRRBusinessImpactScore, - owaspRRVector, - severity, - epssScore, - epssPercentile - """); final List vulns; try { - vulns = List.copyOf(query.executeResultList(org.dependencytrack.model.Vulnerability.class)); + vulns = (List) qm.getPersistenceManager().detachCopyAll(query.executeList()); } finally { query.closeAll(); } From 07c42fa2e8ac53701be3992031ce397001b961c2 Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 12 Sep 2023 14:46:47 +0200 Subject: [PATCH 09/46] Bump `versatile` to `0.3.0` Signed-off-by: nscuro --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cae48fea3..e9108b4a6 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ 1.18.3 2.0.1 1.1.10.1 - 0.2.0 + 0.3.0 6.4.0 1.1.1 2.0.9 From 86345634c50623c1d21ddc323db1a0b7651eadad Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 12 Sep 2023 23:40:23 +0200 Subject: [PATCH 10/46] Use AST visitor to determine which fields are accessed for any given type Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 48 +++++--- .../policy/cel/CelPolicyLibrary.java | 14 ++- .../policy/cel/CelPolicyScript.java | 18 +-- .../policy/cel/CelPolicyScriptHost.java | 59 +--------- .../policy/cel/CelPolicyScriptVisitor.java | 103 ++++++++++++++++++ .../policy/cel/CelPolicyEngineTest.java | 6 +- .../policy/cel/CelPolicyScriptHostTest.java | 33 ++++++ 7 files changed, 194 insertions(+), 87 deletions(-) create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index d4006d3e0..7f336f89e 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -7,8 +7,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.packageurl.PackageURL; +import com.google.api.expr.v1alpha1.Type; import com.google.protobuf.util.Timestamps; import io.micrometer.core.instrument.Timer; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.HashSetValuedHashMap; import org.apache.commons.lang3.tuple.Pair; import org.dependencytrack.model.Component; import org.dependencytrack.model.LicenseGroup; @@ -20,7 +23,6 @@ import org.dependencytrack.model.Project; import org.dependencytrack.model.Tag; import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.policy.cel.CelPolicyScript.Requirement; import org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.ComponentHashCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.CoordinatesCelPolicyScriptSourceBuilder; @@ -57,6 +59,13 @@ import java.util.stream.Stream; import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE_GROUP; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT_PROPERTY; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY_ALIAS; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_PROJECT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_VULNERABILITIES; @@ -193,11 +202,13 @@ public void evaluateComponent(final UUID componentUuid) { // Instead, only load what's really needed, and only do so once. LOGGER.info("Determining evaluation requirements for component %s and %d policy conditions" .formatted(componentUuid, conditionScriptPairs.size())); - final Set requirements = conditionScriptPairs.stream() + final MultiValuedMap requirements = conditionScriptPairs.stream() .map(Pair::getRight) .map(CelPolicyScript::getRequirements) - .flatMap(Collection::stream) - .collect(Collectors.toSet()); + .reduce(new HashSetValuedHashMap<>(), (a, b) -> { + a.putAll(b); + return a; + }); // Prepare the script arguments according to the requirements gathered before. LOGGER.info("Building script arguments for component %s and requirements %s" @@ -374,7 +385,8 @@ private static List getParents(final QueryManager qm, final UUID uuid, fin private static org.hyades.proto.policy.v1.Component mapComponent(final QueryManager qm, final Component component, - final Set requirements) { + final MultiValuedMap requirements) { + // TODO: Load only required fields final org.hyades.proto.policy.v1.Component.Builder builder = org.hyades.proto.policy.v1.Component.newBuilder() .setUuid(Optional.ofNullable(component.getUuid()).map(UUID::toString).orElse("")) @@ -399,7 +411,8 @@ private static org.hyades.proto.policy.v1.Component mapComponent(final QueryMana .setBlake2B512(trimToEmpty(component.getBlake2b_512())) .setBlake3(trimToEmpty(component.getBlake3())); - if (component.getProject().getDirectDependencies() != null) { + if (requirements.get(TYPE_COMPONENT).contains("is_direct_dependency") + && component.getProject().getDirectDependencies() != null) { try { final ArrayNode dependencyArray = OBJECT_MAPPER.readValue(component.getProject().getDirectDependencies(), ArrayNode.class); for (final JsonNode dependencyNode : dependencyArray) { @@ -413,7 +426,7 @@ private static org.hyades.proto.policy.v1.Component mapComponent(final QueryMana } } - if (requirements.contains(Requirement.LICENSE) && component.getResolvedLicense() != null) { + if (requirements.containsKey(TYPE_LICENSE) && component.getResolvedLicense() != null) { final License.Builder licenseBuilder = License.newBuilder() .setUuid(Optional.ofNullable(component.getResolvedLicense().getUuid()).map(UUID::toString).orElse("")) .setId(trimToEmpty(component.getResolvedLicense().getLicenseId())) @@ -423,7 +436,8 @@ private static org.hyades.proto.policy.v1.Component mapComponent(final QueryMana .setIsDeprecatedId(component.getResolvedLicense().isDeprecatedLicenseId()) .setIsCustom(component.getResolvedLicense().isCustomLicense()); - if (requirements.contains(Requirement.LICENSE_GROUPS)) { + if (requirements.containsKey(TYPE_LICENSE_GROUP) + || requirements.get(TYPE_LICENSE).contains("groups")) { final Query licenseGroupQuery = qm.getPersistenceManager().newQuery(LicenseGroup.class); licenseGroupQuery.setFilter("licenses.contains(:license)"); licenseGroupQuery.setNamedParameters(Map.of("license", component.getResolvedLicense())); @@ -446,11 +460,12 @@ private static org.hyades.proto.policy.v1.Component mapComponent(final QueryMana } private static org.hyades.proto.policy.v1.Project mapProject(final Project project, - final Set requirements) { - if (!requirements.contains(Requirement.PROJECT)) { - return org.hyades.proto.policy.v1.Project.newBuilder().build(); + final MultiValuedMap requirements) { + if (!requirements.containsKey(TYPE_PROJECT)) { + return org.hyades.proto.policy.v1.Project.getDefaultInstance(); } + // TODO: Load only required fields final org.hyades.proto.policy.v1.Project.Builder builder = org.hyades.proto.policy.v1.Project.newBuilder() .setUuid(Optional.ofNullable(project.getUuid()).map(UUID::toString).orElse("")) @@ -462,7 +477,8 @@ private static org.hyades.proto.policy.v1.Project mapProject(final Project proje .setPurl(Optional.ofNullable(project.getPurl()).map(PackageURL::canonicalize).orElse("")) .setSwidTagId(trimToEmpty(project.getSwidTagId())); - if (requirements.contains(Requirement.PROJECT_PROPERTIES)) { + if (requirements.containsKey(TYPE_PROJECT_PROPERTY) + || requirements.get(TYPE_PROJECT).contains("properties")) { // TODO } @@ -471,11 +487,12 @@ private static org.hyades.proto.policy.v1.Project mapProject(final Project proje private static List loadVulnerabilities(final QueryManager qm, final Component component, - final Set requirements) { - if (!requirements.contains(Requirement.VULNERABILITIES)) { + final MultiValuedMap requirements) { + if (!requirements.containsKey(TYPE_VULNERABILITY)) { return Collections.emptyList(); } + // TODO: Load only required fields final Query query = qm.getPersistenceManager().newQuery(org.dependencytrack.model.Vulnerability.class); query.getFetchPlan().clearGroups(); @@ -515,7 +532,8 @@ private static List loadVulnerabilities(final QueryManager qm, Optional.ofNullable(v.getPublished()).map(Timestamps::fromDate).ifPresent(builder::setPublished); Optional.ofNullable(v.getUpdated()).map(Timestamps::fromDate).ifPresent(builder::setUpdated); - if (requirements.contains(Requirement.VULNERABILITY_ALIASES)) { + if (requirements.containsKey(TYPE_VULNERABILITY_ALIAS) + || requirements.get(TYPE_VULNERABILITY).contains("aliases")) { // TODO: Dirty hack, create a proper solution. Likely needs caching, too. final var tmpVuln = new org.dependencytrack.model.Vulnerability(); tmpVuln.setVulnId(builder.getId()); diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index 33d85ddfe..b746e777c 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -37,10 +37,14 @@ public class CelPolicyLibrary implements Library { static final String VAR_PROJECT = "project"; static final String VAR_VULNERABILITIES = "vulns"; - private static final Type TYPE_COMPONENT = Decls.newObjectType(Component.getDescriptor().getFullName()); - private static final Type TYPE_PROJECT = Decls.newObjectType(Project.getDescriptor().getFullName()); - private static final Type TYPE_VULNERABILITY = Decls.newObjectType(Vulnerability.getDescriptor().getFullName()); - private static final Type TYPE_VULNERABILITIES = Decls.newListType(TYPE_VULNERABILITY); + public static final Type TYPE_COMPONENT = Decls.newObjectType(Component.getDescriptor().getFullName()); + public static final Type TYPE_LICENSE = Decls.newObjectType(License.getDescriptor().getFullName()); + public static final Type TYPE_LICENSE_GROUP = Decls.newObjectType(License.Group.getDescriptor().getFullName()); + public static final Type TYPE_PROJECT = Decls.newObjectType(Project.getDescriptor().getFullName()); + public static final Type TYPE_PROJECT_PROPERTY = Decls.newObjectType(Project.Property.getDescriptor().getFullName()); + public static final Type TYPE_VULNERABILITY = Decls.newObjectType(Vulnerability.getDescriptor().getFullName()); + public static final Type TYPE_VULNERABILITIES = Decls.newListType(TYPE_VULNERABILITY); + public static final Type TYPE_VULNERABILITY_ALIAS = Decls.newObjectType(Vulnerability.Alias.getDescriptor().getFullName()); private static final String FUNC_DEPENDS_ON = "depends_on"; private static final String FUNC_IS_DEPENDENCY_OF = "is_dependency_of"; @@ -105,7 +109,9 @@ public List getCompileOptions() { EnvOption.types( Component.getDefaultInstance(), License.getDefaultInstance(), + License.Group.getDefaultInstance(), Project.getDefaultInstance(), + Project.Property.getDefaultInstance(), Vulnerability.getDefaultInstance(), Vulnerability.Alias.getDefaultInstance() ) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java index 2db8dc1a4..0525e0e8c 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java @@ -1,33 +1,25 @@ package org.dependencytrack.policy.cel; +import com.google.api.expr.v1alpha1.Type; +import org.apache.commons.collections4.MultiValuedMap; import org.projectnessie.cel.Program; import org.projectnessie.cel.common.types.Err; import org.projectnessie.cel.common.types.ref.Val; import org.projectnessie.cel.tools.ScriptExecutionException; import java.util.Map; -import java.util.Set; public class CelPolicyScript { - public enum Requirement { - LICENSE, - LICENSE_GROUPS, - PROJECT, - PROJECT_PROPERTIES, - VULNERABILITIES, - VULNERABILITY_ALIASES - } - private final Program program; - private final Set requirements; + private final MultiValuedMap requirements; - public CelPolicyScript(final Program program, final Set requirements) { + public CelPolicyScript(final Program program, final MultiValuedMap requirements) { this.program = program; this.requirements = requirements; } - public Set getRequirements() { + public MultiValuedMap getRequirements() { return requirements; } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java index 7d8c064c8..5f01c2167 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -6,10 +6,7 @@ import com.google.api.expr.v1alpha1.Type; import com.google.common.util.concurrent.Striped; import org.apache.commons.codec.digest.DigestUtils; -import org.dependencytrack.policy.cel.CelPolicyScript.Requirement; -import org.hyades.proto.policy.v1.License; -import org.hyades.proto.policy.v1.Project; -import org.hyades.proto.policy.v1.Vulnerability; +import org.apache.commons.collections4.MultiValuedMap; import org.projectnessie.cel.Ast; import org.projectnessie.cel.CEL; import org.projectnessie.cel.Env; @@ -24,11 +21,8 @@ import org.projectnessie.cel.tools.ScriptCreateException; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.locks.Lock; -import java.util.stream.Collectors; import static org.projectnessie.cel.Issues.newIssues; import static org.projectnessie.cel.common.Source.newTextSource; @@ -99,7 +93,7 @@ public CelPolicyScript compile(final String scriptSrc) throws ScriptCreateExcept final Ast ast = astIssuesTuple.getAst(); final Program program = environment.program(ast); - final Set requirements = analyzeRequirements(CEL.astToCheckedExpr(ast)); + final MultiValuedMap requirements = analyzeRequirements(CEL.astToCheckedExpr(ast)); script = new CelPolicyScript(program, requirements); cacheManager.put(scriptDigest, script); @@ -109,51 +103,10 @@ public CelPolicyScript compile(final String scriptSrc) throws ScriptCreateExcept } } - private static Set analyzeRequirements(final CheckedExpr expr) { - final var requirements = new HashSet(); - - final Set typeNames = expr.getTypeMapMap().values().stream() - .map(Type::getMessageType) - .collect(Collectors.toSet()); - - // For the majority of cases, it is sufficient to check whether a given type - // is present in the type map constructed by the type checker. This works as long - // as a field of the respective type is accessed in the script, e.g. - // - // component.license.name - // - // will result in the License type being present in the type map. However, it does NOT - // work when only the presence of a field is checked in the script, e.g. - // - // has(component.license) - // - // will result in the License type NOT being present in the type map. - // - // To cover this limitation, we could implement a visitor that traverses the AST - // and keeps track of which fields are accessed for which type. - if (typeNames.contains(Project.getDescriptor().getFullName())) { - requirements.add(Requirement.PROJECT); - - if (typeNames.contains(Project.Property.getDescriptor().getFullName())) { - requirements.add(Requirement.PROJECT_PROPERTIES); - } - } - if (typeNames.contains(License.getDescriptor().getFullName())) { - requirements.add(Requirement.LICENSE); - - if (typeNames.contains(License.Group.getDescriptor().getFullName())) { - requirements.add(Requirement.LICENSE_GROUPS); - } - } - if (typeNames.contains(Vulnerability.getDescriptor().getFullName())) { - requirements.add(Requirement.VULNERABILITIES); - - if (typeNames.contains(Vulnerability.Alias.getDescriptor().getFullName())) { - requirements.add(Requirement.VULNERABILITY_ALIASES); - } - } - - return requirements; + private static MultiValuedMap analyzeRequirements(final CheckedExpr expr) { + final var visitor = new CelPolicyScriptVisitor(expr.getTypeMapMap()); + visitor.visit(expr.getExpr()); + return visitor.getAccessedFieldsByType(); } } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java new file mode 100644 index 000000000..c7284e6d4 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java @@ -0,0 +1,103 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import com.google.api.expr.v1alpha1.Expr; +import com.google.api.expr.v1alpha1.Type; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.HashSetValuedHashMap; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Map; + +public class CelPolicyScriptVisitor { + + private static final Logger LOGGER = Logger.getLogger(CelPolicyScriptVisitor.class); + + private final Map types; + private final MultiValuedMap accessedFieldsByType; + private final Deque callFunctionStack; + private final Deque selectFieldStack; + private final Deque selectOperandTypeStack; + + public CelPolicyScriptVisitor(final Map types) { + this.types = types; + this.accessedFieldsByType = new HashSetValuedHashMap<>(); + this.callFunctionStack = new ArrayDeque<>(); + this.selectFieldStack = new ArrayDeque<>(); + this.selectOperandTypeStack = new ArrayDeque<>(); + } + + public void visit(final Expr expr) { + switch (expr.getExprKindCase()) { + case CALL_EXPR -> visitCall(expr); + case COMPREHENSION_EXPR -> visitComprehension(expr); + case CONST_EXPR -> visitConst(expr); + case IDENT_EXPR -> visitIdent(expr); + case LIST_EXPR -> visitList(expr); + case SELECT_EXPR -> visitSelect(expr); + case STRUCT_EXPR -> visitStruct(expr); + case EXPRKIND_NOT_SET -> LOGGER.debug("Unknown expression: %s".formatted(expr)); + } + } + + private void visitCall(final Expr expr) { + logExpr(expr); + final Expr.Call callExpr = expr.getCallExpr(); + + callFunctionStack.push(callExpr.getFunction()); + visit(callExpr.getTarget()); + for (final Expr argExpr : callExpr.getArgsList()) { + visit(argExpr); + } + callFunctionStack.pop(); + } + + private void visitComprehension(final Expr expr) { + logExpr(expr); + final Expr.Comprehension comprehensionExpr = expr.getComprehensionExpr(); + + visit(comprehensionExpr.getAccuInit()); + visit(comprehensionExpr.getIterRange()); + visit(comprehensionExpr.getLoopStep()); + visit(comprehensionExpr.getLoopCondition()); + visit(comprehensionExpr.getResult()); + } + + private void visitConst(final Expr expr) { + logExpr(expr); + } + + private void visitIdent(final Expr expr) { + logExpr(expr); + selectOperandTypeStack.push(types.get(expr.getId())); + } + + private void visitList(final Expr expr) { + logExpr(expr); + } + + private void visitSelect(final Expr expr) { + logExpr(expr); + final Expr.Select selectExpr = expr.getSelectExpr(); + + selectFieldStack.push(selectExpr.getField()); + selectOperandTypeStack.push(types.get(expr.getId())); + visit(selectExpr.getOperand()); + accessedFieldsByType.put(selectOperandTypeStack.pop(), selectFieldStack.pop()); + } + + private void visitStruct(final Expr expr) { + logExpr(expr); + } + + private void logExpr(final Expr expr) { + LOGGER.debug("Visiting %s (id=%d, fieldStack=%s, fieldTypeStack=%s, functionStack=%s)" + .formatted(expr.getExprKindCase(), expr.getId(), selectFieldStack, selectOperandTypeStack, callFunctionStack)); + } + + public MultiValuedMap getAccessedFieldsByType() { + return this.accessedFieldsByType; + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index 26ca201b4..c5fe712e5 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -45,19 +45,21 @@ public class CelPolicyEngineTest extends PersistenceCapableTest { @Test public void test() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - qm.createPolicyCondition(policy, + final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ "critical" in project.tags && component.name == "bar" && vulns.exists(v, v.source == "SNYK") && component.resolved_license.groups.exists(lg, lg.name == "Permissive") """); + policyConditionA.setViolationType(PolicyViolation.Type.OPERATIONAL); + qm.persist(policyConditionA); final var policy2 = qm.createPolicy("policy2", Policy.Operator.ALL, Policy.ViolationState.WARN); qm.createPolicyCondition(policy2, PolicyCondition.Subject.VULNERABILITY_ID, PolicyCondition.Operator.IS, "CVE-123"); final var policy3 = qm.createPolicy("policy3", Policy.Operator.ALL, Policy.ViolationState.INFO); - final PolicyCondition condition3 = qm.createPolicyCondition(policy3, PolicyCondition.Subject.SWID_TAGID, PolicyCondition.Operator.IS, "foo"); + final PolicyCondition condition3 = qm.createPolicyCondition(policy3, PolicyCondition.Subject.SWID_TAGID, PolicyCondition.Operator.MATCHES, "foo"); final var policy4 = qm.createPolicy("policy4", Policy.Operator.ALL, Policy.ViolationState.INFO); qm.createPolicyCondition(policy4, PolicyCondition.Subject.CWE, PolicyCondition.Operator.CONTAINS_ALL, "CWE-666, CWE-123, 555"); diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java new file mode 100644 index 000000000..394210c5c --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java @@ -0,0 +1,33 @@ +package org.dependencytrack.policy.cel; + +import com.google.api.expr.v1alpha1.Type; +import org.junit.Test; + +import java.util.Collection; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE_GROUP; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY; + +public class CelPolicyScriptHostTest { + + @Test + public void testRequirements() throws Exception { + final CelPolicyScript compiledScript = CelPolicyScriptHost.getInstance().compile(""" + component.resolved_license.groups.exists(licenseGroup, licenseGroup.name == "Permissive") + && vulns.filter(vuln, vuln.severity in ["HIGH", "CRITICAL"]).size() > 1 + """); + + final Map> requirements = compiledScript.getRequirements().asMap(); + assertThat(requirements).containsKeys(TYPE_COMPONENT, TYPE_LICENSE, TYPE_LICENSE_GROUP, TYPE_VULNERABILITY); + + assertThat(requirements.get(TYPE_COMPONENT)).containsOnly("resolved_license"); + assertThat(requirements.get(TYPE_LICENSE)).containsOnly("groups"); + assertThat(requirements.get(TYPE_LICENSE_GROUP)).containsOnly("name"); + assertThat(requirements.get(TYPE_VULNERABILITY)).containsOnly("severity"); + } + +} \ No newline at end of file From 086f70ae14e0dea0b552cd825a20fa393df225c2 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 14 Sep 2023 14:02:05 +0200 Subject: [PATCH 11/46] Cleanup Signed-off-by: nscuro --- .../policy/cel/CelPolicyLibrary.java | 114 +++++++++--------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index b746e777c..5569679fb 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -16,6 +16,7 @@ import org.projectnessie.cel.checker.Decls; import org.projectnessie.cel.common.types.Err; import org.projectnessie.cel.common.types.Types; +import org.projectnessie.cel.common.types.ref.Val; import org.projectnessie.cel.interpreter.functions.Overload; import javax.jdo.Query; @@ -68,13 +69,13 @@ public List getCompileOptions() { ), Decls.newFunction( FUNC_DEPENDS_ON, - // component.depends_on(org.hyades.policy.v1.Component{"name": "foo"}) + // component.depends_on(org.hyades.policy.v1.Component{name: "foo"}) Decls.newInstanceOverload( "component_depends_on_component_bool", List.of(TYPE_COMPONENT, TYPE_COMPONENT), Decls.Bool ), - // project.depends_on(org.hyades.policy.v1.Component{"name": "foo"}) + // project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) Decls.newInstanceOverload( "project_depends_on_component_bool", List.of(TYPE_PROJECT, TYPE_COMPONENT), @@ -83,7 +84,7 @@ public List getCompileOptions() { ), Decls.newFunction( FUNC_IS_DEPENDENCY_OF, - // component.is_dependency_of(org.hyades.policy.v1.Component{"name": "foo"}) + // component.is_dependency_of(org.hyades.policy.v1.Component{name: "foo"}) Decls.newInstanceOverload( "component_is_dependency_of_component_bool", List.of(TYPE_COMPONENT, TYPE_COMPONENT), @@ -124,68 +125,73 @@ public List getProgramOptions() { ProgramOption.functions( Overload.binary( FUNC_DEPENDS_ON, - (lhs, rhs) -> { - final Component leafComponent; - if (rhs.value() instanceof final Component rhsValue) { - leafComponent = rhsValue; - } else { - return Err.maybeNoSuchOverloadErr(rhs); - } - - if (lhs.value() instanceof final Project project) { - return Types.boolOf(dependsOn(project, leafComponent)); - } else if (lhs.value() instanceof final Component rootComponent) { - // TODO: Traverse dep graph from rootComponent downwards and look for leafComponent - return Types.boolOf(dependsOn(rootComponent, leafComponent)); - } - - return Err.maybeNoSuchOverloadErr(lhs); - } + CelPolicyLibrary::dependsOnFunc ), Overload.binary( FUNC_IS_DEPENDENCY_OF, - (lhs, rhs) -> { - final Component leafComponent; - if (lhs.value() instanceof final Component lhsValue) { - leafComponent = lhsValue; - } else { - return Err.maybeNoSuchOverloadErr(lhs); - } - - if (rhs.value() instanceof final Component rootComponent) { - // TODO: traverse dep graph from lhsComponent upwards and look for rhsComponent - return Types.boolOf(isDependencyOf(leafComponent, rootComponent)); - } - - return Err.maybeNoSuchOverloadErr(rhs); - } + CelPolicyLibrary::isDependencyOfFunc ), Overload.binary( FUNC_MATCHES_RANGE, - (lhs, rhs) -> { - final String version; - if (lhs.value() instanceof final Component lhsValue) { - version = lhsValue.getVersion(); - } else if (lhs.value() instanceof final Project lhsValue) { - version = lhsValue.getVersion(); - } else { - return Err.maybeNoSuchOverloadErr(lhs); - } - - final String versStr; - if (rhs.value() instanceof final String rhsValue) { - versStr = rhsValue; - } else { - return Err.maybeNoSuchOverloadErr(rhs); - } - - return Types.boolOf(matchesRange(version, versStr)); - } + CelPolicyLibrary::matchesRangeFunc ) ) ); } + private static Val dependsOnFunc(final Val lhs, final Val rhs) { + final Component leafComponent; + if (rhs.value() instanceof final Component rhsValue) { + leafComponent = rhsValue; + } else { + return Err.maybeNoSuchOverloadErr(rhs); + } + + if (lhs.value() instanceof final Project project) { + return Types.boolOf(dependsOn(project, leafComponent)); + } else if (lhs.value() instanceof final Component rootComponent) { + // TODO: Traverse dep graph from rootComponent downwards and look for leafComponent + return Types.boolOf(dependsOn(rootComponent, leafComponent)); + } + + return Err.maybeNoSuchOverloadErr(lhs); + } + + private static Val isDependencyOfFunc(final Val lhs, final Val rhs) { + final Component leafComponent; + if (lhs.value() instanceof final Component lhsValue) { + leafComponent = lhsValue; + } else { + return Err.maybeNoSuchOverloadErr(lhs); + } + + if (rhs.value() instanceof final Component rootComponent) { + return Types.boolOf(isDependencyOf(leafComponent, rootComponent)); + } + + return Err.maybeNoSuchOverloadErr(rhs); + } + + private static Val matchesRangeFunc(final Val lhs, final Val rhs) { + final String version; + if (lhs.value() instanceof final Component lhsValue) { + version = lhsValue.getVersion(); + } else if (lhs.value() instanceof final Project lhsValue) { + version = lhsValue.getVersion(); + } else { + return Err.maybeNoSuchOverloadErr(lhs); + } + + final String versStr; + if (rhs.value() instanceof final String rhsValue) { + versStr = rhsValue; + } else { + return Err.maybeNoSuchOverloadErr(rhs); + } + + return Types.boolOf(matchesRange(version, versStr)); + } + private static boolean dependsOn(final Project project, final Component component) { if (project.getUuid().isBlank()) { // Need a UUID for our starting point. From e956ca46880a50b03b82b709a28539028eb5dd94 Mon Sep 17 00:00:00 2001 From: nscuro Date: Fri, 15 Sep 2023 12:31:25 +0200 Subject: [PATCH 12/46] Cleanup Signed-off-by: nscuro --- .../org/dependencytrack/model/Policy.java | 12 -- .../dependencytrack/model/PolicyScope.java | 9 -- .../model/PolicyViolation.java | 50 +----- .../policy/cel/CelPolicyEngine.java | 22 +-- .../policy/cel/CelPolicyEngineTest.java | 146 +----------------- 5 files changed, 20 insertions(+), 219 deletions(-) delete mode 100644 src/main/java/org/dependencytrack/model/PolicyScope.java diff --git a/src/main/java/org/dependencytrack/model/Policy.java b/src/main/java/org/dependencytrack/model/Policy.java index bbe8837cd..8b93f546c 100644 --- a/src/main/java/org/dependencytrack/model/Policy.java +++ b/src/main/java/org/dependencytrack/model/Policy.java @@ -81,10 +81,6 @@ public enum ViolationState { @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The name may only contain printable characters") private String name; - @Persistent - @Column(name = "SCOPE", allowsNull = "true") - private PolicyScope scope = PolicyScope.COMPONENT; // TODO: Remove default, handle this for existing policies with DB migration. - /** * The operator to use when evaluating conditions. */ @@ -169,14 +165,6 @@ public void setName(String name) { this.name = name; } - public PolicyScope getScope() { - return scope; - } - - public void setScope(final PolicyScope scope) { - this.scope = scope; - } - public Operator getOperator() { return operator; } diff --git a/src/main/java/org/dependencytrack/model/PolicyScope.java b/src/main/java/org/dependencytrack/model/PolicyScope.java deleted file mode 100644 index e7de34dc3..000000000 --- a/src/main/java/org/dependencytrack/model/PolicyScope.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.dependencytrack.model; - -public enum PolicyScope { - - COMPONENT, - - PROJECT - -} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/model/PolicyViolation.java b/src/main/java/org/dependencytrack/model/PolicyViolation.java index b857100e1..f76de9cc5 100644 --- a/src/main/java/org/dependencytrack/model/PolicyViolation.java +++ b/src/main/java/org/dependencytrack/model/PolicyViolation.java @@ -24,12 +24,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import javax.jdo.annotations.Column; -import javax.jdo.annotations.Element; -import javax.jdo.annotations.Extension; import javax.jdo.annotations.IdGeneratorStrategy; import javax.jdo.annotations.Index; -import javax.jdo.annotations.Join; -import javax.jdo.annotations.Order; import javax.jdo.annotations.PersistenceCapable; import javax.jdo.annotations.Persistent; import javax.jdo.annotations.PrimaryKey; @@ -39,7 +35,6 @@ import javax.validation.constraints.Size; import java.io.Serializable; import java.util.Date; -import java.util.List; import java.util.UUID; /** @@ -74,26 +69,14 @@ public enum Type { private Project project; @Persistent(defaultFetchGroup = "true") - @Column(name = "COMPONENT_ID", allowsNull = "true") + @Column(name = "COMPONENT_ID", allowsNull = "false") @Index(name = "POLICYVIOLATION_COMPONENT_IDX") private Component component; - @Persistent - @Column(name = "POLICY_ID", allowsNull = "true") - private Policy policy; - - // TODO: Remove - @Deprecated(forRemoval = true) @Persistent(defaultFetchGroup = "true") - @Column(name = "POLICYCONDITION_ID", allowsNull = "true") + @Column(name = "POLICYCONDITION_ID", allowsNull = "false") private PolicyCondition policyCondition; - @Persistent(table = "POLICYVIOLATIONS_MATCHED_POLICYCONDITIONS") - @Join(column = "POLICYVIOLATION_ID") - @Element(column = "POLICYCONDITION_ID") - @Order(extensions = @Extension(vendorName = "datanucleus", key = "list-ordering", value = "id ASC")) - private List matchedConditions; - @Persistent @Column(name = "TIMESTAMP", allowsNull = "false") private Date timestamp; @@ -104,8 +87,8 @@ public enum Type { @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The text may only contain printable characters") private String text; - @Persistent(mappedBy="policyViolation", defaultFetchGroup = "true") - private ViolationAnalysis analysis; + @Persistent(mappedBy = "policyViolation", defaultFetchGroup = "true") + private ViolationAnalysis analysis; /** * The unique identifier of the object. @@ -138,44 +121,21 @@ public Component getComponent() { public void setComponent(Component component) { this.component = component; + this.project = component.getProject(); } public Project getProject() { return project; } - public void setProject(final Project project) { - this.project = project; - } - - public Policy getPolicy() { - return policy; - } - - public void setPolicy(final Policy policy) { - this.policy = policy; - } - - // TODO: Remove - @Deprecated(forRemoval = true) public PolicyCondition getPolicyCondition() { return policyCondition; } - // TODO: Remove - @Deprecated(forRemoval = true) public void setPolicyCondition(PolicyCondition policyCondition) { this.policyCondition = policyCondition; } - public List getMatchedConditions() { - return matchedConditions; - } - - public void setMatchedConditions(final List matchedConditions) { - this.matchedConditions = matchedConditions; - } - public Date getTimestamp() { return timestamp; } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 7f336f89e..0a7f77956 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -18,7 +18,6 @@ import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.model.PolicyCondition.Subject; -import org.dependencytrack.model.PolicyScope; import org.dependencytrack.model.PolicyViolation; import org.dependencytrack.model.Project; import org.dependencytrack.model.Tag; @@ -34,6 +33,7 @@ import org.dependencytrack.policy.cel.compat.SeverityCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; +import org.dependencytrack.util.NotificationUtil; import org.dependencytrack.util.VulnerabilityUtil; import org.hyades.proto.policy.v1.License; import org.hyades.proto.policy.v1.Vulnerability; @@ -53,7 +53,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -147,7 +146,7 @@ public void evaluateComponent(final UUID componentUuid) { return; } - final List policies = getApplicablePolicies(qm, component.getProject(), Set.of(PolicyScope.COMPONENT)); + final List policies = getApplicablePolicies(qm, component.getProject()); if (policies.isEmpty()) { // With no applicable policies, there's no way to resolve violations. // As a compensation, simply delete all violations associated with the component. @@ -260,12 +259,9 @@ VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) return violatedConditions.stream() .map(condition -> { final var violation = new PolicyViolation(); - violation.setProject(component.getProject()); violation.setComponent(component); - violation.setPolicy(policy); - violation.setType(condition.getViolationType()); // TODO: We need violationType at policy level + violation.setType(condition.getViolationType()); violation.setPolicyCondition(condition); - // violation.setMatchedConditions(violatedConditions); violation.setTimestamp(new Date()); return violation; }); @@ -282,8 +278,7 @@ VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) // Notify users about any new violations. for (final PolicyViolation newViolation : newViolations) { - // TODO: Handle switch from policyCondition to matchedConditions in PolicyViolation - // NotificationUtil.analyzeNotificationCriteria(qm, newViolation); + NotificationUtil.analyzeNotificationCriteria(qm, newViolation); } } finally { timerSample.stop(Timer @@ -296,7 +291,7 @@ VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) } // TODO: Move to PolicyQueryManager - private static List getApplicablePolicies(final QueryManager qm, final Project project, final Set scopes) { + private static List getApplicablePolicies(final QueryManager qm, final Project project) { var filter = """ (this.projects.isEmpty() && this.tags.isEmpty()) || (this.projects.contains(:project) @@ -338,9 +333,6 @@ private static List getApplicablePolicies(final QueryManager qm, final P filter += ")"; } - filter = "(%s) && :scopes.contains(this.scope)".formatted(filter); - params.put("scopes", scopes); - final List policies; final Query query = qm.getPersistenceManager().newQuery(Policy.class); try { @@ -568,10 +560,10 @@ private static List reconcileViolations(final QueryManager qm, for (final PolicyViolation violation : violations) { final Query query = qm.getPersistenceManager().newQuery(PolicyViolation.class); - query.setFilter("component == :component && policy == :policy && type == :type"); + query.setFilter("component == :component && policyCondition == :policyCondition && type == :type"); query.setNamedParameters(Map.of( "component", violation.getComponent(), - "policy", violation.getPolicy(), + "policyCondition", violation.getPolicyCondition(), "type", violation.getType() )); query.setResult("id"); diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index c5fe712e5..ce9173309 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -27,21 +27,6 @@ public class CelPolicyEngineTest extends PersistenceCapableTest { - // Raise a violation when the component is affected by a certain vulnerability, - // which is only exploitable when Apache Tomcat is present in the project as well. - // vulns.exists(v, v.id == "CVE-123) - // && project.depends_on(org.hyades.policy.v1.Component{"name": "tomcat-embed-core"}) - - // Raise a violation when the component is SnakeYAML, affected by CVE-2022-1471, - // and introduced to the project as a dependency of Spring Framework. - // vulns.exists(v, v.id == "CVE-2022-1471") - // && component.purl.startsWith("pkg:maven/org.snakeyaml/snakeyaml") - // && component.is_dependency_of(org.hyades.policy.v1.Component{"group": "org.springframework"}) - - // Raise a violation when the component is affected by CVE-2022-229665, - // and the project was packaged as WAR: - // vulns.exists(v, v.id == "CVE-2022-229665") && project.purl.contains("type=war") - @Test public void test() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); @@ -104,7 +89,6 @@ public void test() { qm.addVulnerability(vulnB, component, AnalyzerIdentity.SNYK_ANALYZER); final var existingViolation = new PolicyViolation(); - existingViolation.setProject(project); existingViolation.setComponent(component); existingViolation.setPolicyCondition(condition3); existingViolation.setType(PolicyViolation.Type.OPERATIONAL); @@ -118,126 +102,6 @@ public void test() { assertThat(violations).isNotEmpty(); } - @Test - public void testPolicyOperatorAnyWithOneConditionMatching() { - final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.name == "foo" - """); - final PolicyCondition policyConditionB = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.name == "bar" - """); - - final var project = new Project(); - project.setName("acme-app"); - qm.persist(project); - - final var component = new Component(); - component.setProject(project); - component.setName("foo"); - qm.persist(component); - - final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateComponent(component.getUuid()); - - assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> { - assertThat(violation.getPolicy().getUuid()).isEqualTo(policy.getUuid()); - assertThat(violation.getMatchedConditions()).hasSize(1); - assertThat(violation.getMatchedConditions().get(0).getUuid()).isEqualTo(policyConditionA.getUuid()); - }); - } - - @Test - public void testPolicyOperatorAnyWithAllConditionsMatching() { - final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.name == "foo" - """); - final PolicyCondition policyConditionB = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.name != "bar" - """); - - final var project = new Project(); - project.setName("acme-app"); - qm.persist(project); - - final var component = new Component(); - component.setProject(project); - component.setName("foo"); - qm.persist(component); - - final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateComponent(component.getUuid()); - - assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> { - assertThat(violation.getPolicy().getUuid()).isEqualTo(policy.getUuid()); - assertThat(violation.getMatchedConditions()).hasSize(2); - assertThat(violation.getMatchedConditions().get(0).getUuid()).isEqualTo(policyConditionA.getUuid()); - assertThat(violation.getMatchedConditions().get(1).getUuid()).isEqualTo(policyConditionB.getUuid()); - }); - } - - @Test - public void testPolicyOperatorAllWithOneConditionMatching() { - final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); - qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.name == "foo" - """); - qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.name == "bar" - """); - - final var project = new Project(); - project.setName("acme-app"); - qm.persist(project); - - final var component = new Component(); - component.setProject(project); - component.setName("foo"); - qm.persist(component); - - final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateComponent(component.getUuid()); - - assertThat(qm.getAllPolicyViolations(component)).isEmpty(); - } - - @Test - public void testPolicyOperatorAllWithAllConditionsMatching() { - final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.name == "foo" - """); - final PolicyCondition policyConditionB = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.name != "bar" - """); - - final var project = new Project(); - project.setName("acme-app"); - qm.persist(project); - - final var component = new Component(); - component.setProject(project); - component.setName("foo"); - qm.persist(component); - - final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateComponent(component.getUuid()); - - assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> { - assertThat(violation.getPolicy().getUuid()).isEqualTo(policy.getUuid()); - assertThat(violation.getMatchedConditions()).hasSize(2); - assertThat(violation.getMatchedConditions().get(0).getUuid()).isEqualTo(policyConditionA.getUuid()); - assertThat(violation.getMatchedConditions().get(1).getUuid()).isEqualTo(policyConditionB.getUuid()); - }); - } - @Test public void testIsDirectDependency() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); @@ -245,6 +109,8 @@ public void testIsDirectDependency() { PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ component.is_direct_dependency """); + policyCondition.setViolationType(PolicyViolation.Type.OPERATIONAL); + qm.persist(policyCondition); final var project = new Project(); project.setName("foo"); @@ -277,10 +143,12 @@ public void testIsDirectDependency() { @Test public void testProjectDependsOnComponent() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - qm.createPolicyCondition(policy, + final PolicyCondition policyCondition = qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) """); + policyCondition.setViolationType(PolicyViolation.Type.OPERATIONAL); + qm.persist(policyCondition); final var project = new Project(); project.setName("foo"); @@ -313,11 +181,13 @@ public void testProjectDependsOnComponent() { @Test public void testMatchesRange() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - qm.createPolicyCondition(policy, + final PolicyCondition policyCondition = qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.matches_range("vers:generic/<1") && component.matches_range("vers:golang/>0| Date: Sun, 17 Sep 2023 20:55:12 +0200 Subject: [PATCH 13/46] WIP: Loading of required fields; Project policy evaluation Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 467 +++++++++++++++++- .../cel/persistence/ComponentProjection.java | 81 +++ .../ComponentsVulnerabilitiesProjection.java | 9 + .../policy/cel/persistence/FieldMapping.java | 4 + .../cel/persistence/FieldMappingUtil.java | 38 ++ .../persistence/LicenseGroupProjection.java | 11 + .../cel/persistence/LicenseProjection.java | 32 ++ .../policy/cel/persistence/MappedField.java | 16 + .../cel/persistence/ProjectProjection.java | 39 ++ .../persistence/VulnerabilityProjection.java | 39 ++ .../cel/persistence/FieldMappingUtilTest.java | 20 + 11 files changed, 736 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/ComponentsVulnerabilitiesProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/FieldMapping.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtil.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/LicenseGroupProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/MappedField.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 0a7f77956..52f0e2e17 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -33,6 +33,12 @@ import org.dependencytrack.policy.cel.compat.SeverityCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.persistence.ComponentProjection; +import org.dependencytrack.policy.cel.persistence.ComponentsVulnerabilitiesProjection; +import org.dependencytrack.policy.cel.persistence.LicenseGroupProjection; +import org.dependencytrack.policy.cel.persistence.LicenseProjection; +import org.dependencytrack.policy.cel.persistence.ProjectProjection; +import org.dependencytrack.policy.cel.persistence.VulnerabilityProjection; import org.dependencytrack.util.NotificationUtil; import org.dependencytrack.util.VulnerabilityUtil; import org.hyades.proto.policy.v1.License; @@ -43,6 +49,7 @@ import javax.jdo.Query; import javax.jdo.Transaction; import java.math.BigDecimal; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -54,9 +61,11 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Collections.emptyList; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE; @@ -68,6 +77,7 @@ import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_PROJECT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_VULNERABILITIES; +import static org.dependencytrack.policy.cel.persistence.FieldMappingUtil.getFieldMappings; public class CelPolicyEngine { @@ -100,34 +110,193 @@ public CelPolicyEngine() { this.scriptHost = scriptHost; } - public void evaluateProject(final UUID projectUuid) { + public void evaluateProject(final UUID uuid) { final Timer.Sample timerSample = Timer.start(); - try { - // TODO: Temporary solution for demonstration purposes. Super inefficient. - // When evaluating a project, policies can be pre-compiled, and input data pre-loaded - // for all components. - final List componentUuids; - try (final var qm = new QueryManager()) { - final Query query = qm.getPersistenceManager().newQuery(Component.class); - query.setFilter("project.uuid == :projectUuid"); - query.setParameters(projectUuid); - query.setResult("uuid"); - try { - componentUuids = List.copyOf(query.executeResultList(UUID.class)); - } finally { - query.closeAll(); + try (final var qm = new QueryManager()) { + final Project project = qm.getObjectByUuid(Project.class, uuid, List.of(Project.FetchGroup.IDENTIFIERS.name())); + if (project == null) { + LOGGER.warn("Project with UUID %s does not exist".formatted(uuid)); + return; + } + + final List policies = getApplicablePolicies(qm, project); + if (policies.isEmpty()) { + // With no applicable policies, there's no way to resolve violations. + // As a compensation, simply delete all violations associated with the component. + LOGGER.info("No applicable policies found for component %s".formatted(uuid)); + // reconcileViolations(qm, component, Collections.emptyList()); + return; + } + + LOGGER.info("Compiling policy scripts for project %s".formatted(uuid)); + final List> conditionScriptPairs = policies.stream() + .map(Policy::getPolicyConditions) + .flatMap(Collection::stream) + .map(policyCondition -> { + final CelPolicyScriptSourceBuilder scriptBuilder = SCRIPT_BUILDERS.get(policyCondition.getSubject()); + if (scriptBuilder == null) { + LOGGER.warn("No script builder found that is capable of handling subjects of type %s".formatted(policyCondition.getSubject())); + return null; + } + + final String scriptSrc = scriptBuilder.apply(policyCondition); + if (scriptSrc == null) { + LOGGER.warn("Script builder was unable to create a script for condition %s".formatted(policyCondition.getUuid())); + return null; + } + + return Pair.of(policyCondition, scriptSrc); + }) + .filter(Objects::nonNull) + .map(conditionScriptSrcPair -> { + final CelPolicyScript script; + try { + script = scriptHost.compile(conditionScriptSrcPair.getRight()); + } catch (ScriptCreateException e) { + throw new RuntimeException(e); + } + + return Pair.of(conditionScriptSrcPair.getLeft(), script); + }) + .toList(); + + LOGGER.info("Determining evaluation requirements for project %s and %d policy conditions" + .formatted(uuid, conditionScriptPairs.size())); + final MultiValuedMap requirements = conditionScriptPairs.stream() + .map(Pair::getRight) + .map(CelPolicyScript::getRequirements) + .reduce(new HashSetValuedHashMap<>(), (a, b) -> { + a.putAll(b); + return a; + }); + LOGGER.info("Requirements for project %s and %d policy conditions: %s" + .formatted(uuid, conditionScriptPairs.size(), requirements)); + + final org.hyades.proto.policy.v1.Project protoProject; + if (requirements.containsKey(TYPE_PROJECT)) { + protoProject = mapProject(fetchProject(qm, project.getId(), requirements.get(TYPE_PROJECT))); + } else { + protoProject = org.hyades.proto.policy.v1.Project.getDefaultInstance(); + } + + // Preload components for the entire project, to avoid excessive queries. + final List components = fetchComponents(qm, project.getId(), requirements.get(TYPE_COMPONENT)); + + // Preload licenses for the entire project, as chances are high that they will be used by multiple components. + final Map licenseById; + if (requirements.containsKey(TYPE_LICENSE)) { + licenseById = fetchLicenses(qm, project.getId(), requirements.get(TYPE_LICENSE), requirements.get(TYPE_LICENSE_GROUP)).stream() + .map(projection -> Pair.of(projection.id, mapLicense(projection))) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); + } else { + licenseById = Collections.emptyMap(); + } + + // Preload vulnerabilities for the entire project, as chances are high that they will be used by multiple components. + final Map protoVulnById; + final Map> vulnIdsByComponentId; + if (requirements.containsKey(TYPE_VULNERABILITY)) { + protoVulnById = fetchVulnerabilities(qm, project.getId(), requirements.get(TYPE_VULNERABILITY)).stream() + .map(vulnProjection -> Pair.of(vulnProjection.id, Vulnerability.getDefaultInstance())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); + + vulnIdsByComponentId = fetchComponentsVulnerabilities(qm, project.getId()).stream() + .collect(Collectors.groupingBy( + projection -> projection.componentId, + Collectors.mapping(projection -> projection.vulnerabilityId, Collectors.toList()) + )); + } else { + protoVulnById = Collections.emptyMap(); + vulnIdsByComponentId = Collections.emptyMap(); + } + + final var conditionsViolated = new HashSetValuedHashMap(); + for (final ComponentProjection component : components) { + final org.hyades.proto.policy.v1.Component protoComponent = mapComponent(component, licenseById); + + final List vulns = vulnIdsByComponentId.getOrDefault(component.id, emptyList()).stream() + .map(protoVulnById::get) + .toList(); + + for (final Pair conditionScriptPair : conditionScriptPairs) { + final PolicyCondition condition = conditionScriptPair.getLeft(); + final CelPolicyScript script = conditionScriptPair.getRight(); + final Map scriptArgs = Map.of( + VAR_COMPONENT, protoComponent, + VAR_PROJECT, protoProject, + VAR_VULNERABILITIES, vulns + ); + + try { + if (script.execute(scriptArgs)) { + conditionsViolated.put(component.id, condition); + } + } catch (ScriptException e) { + throw new RuntimeException("Failed to evaluate script", e); + } + } + + if (!conditionsViolated.containsKey(component.id)) { + conditionsViolated.putAll(component.id, Collections.emptySet()); } } - for (final UUID componentUuid : componentUuids) { - evaluateComponent(componentUuid); + // In order to create policy violations, we need Component objects that are attached to the + // persistence context. + final Query componentQuery = qm.getPersistenceManager().newQuery(Component.class); + componentQuery.getFetchPlan().setGroup(Component.FetchGroup.IDENTITY.name()); + componentQuery.setFilter(":ids.contains(id)"); + componentQuery.setParameters(conditionsViolated.keySet()); + final Map persistentComponentById; + try { + persistentComponentById = componentQuery.executeList().stream() + .collect(Collectors.toMap(Component::getId, Function.identity())); + } finally { + componentQuery.closeAll(); + } + + // TODO: Short-circuit for components for which no violations were detected. + + for (final long componentId : conditionsViolated.keySet()) { + final Map> violatedConditionsByPolicy = conditionsViolated.get(componentId).stream() + .collect(Collectors.groupingBy(PolicyCondition::getPolicy)); + + final List violations = violatedConditionsByPolicy.entrySet().stream() + .flatMap(policyAndViolatedConditions -> { + final Policy policy = policyAndViolatedConditions.getKey(); + final List violatedConditions = policyAndViolatedConditions.getValue(); + + if ((policy.getOperator() == Policy.Operator.ANY && !violatedConditions.isEmpty()) + || (policy.getOperator() == Policy.Operator.ALL && violatedConditions.size() == policy.getPolicyConditions().size())) { + // TODO: Only a single violation should be raised, and instead multiple matched conditions + // should be associated with it. Keeping the existing behavior in order to avoid having to + // touch too much persistence and REST API code. + return violatedConditions.stream() + .map(condition -> { + final var violation = new PolicyViolation(); + violation.setComponent(persistentComponentById.get(componentId)); + violation.setType(condition.getViolationType()); + violation.setPolicyCondition(condition); + violation.setTimestamp(new Date()); + return violation; + }); + } + + return Stream.empty(); + }) + .filter(Objects::nonNull) + .toList(); + + final List newViolations = reconcileViolations(qm, persistentComponentById.get(componentId), violations); } } finally { - timerSample.stop(Timer + final long durationNs = timerSample.stop(Timer .builder("dtrack_policy_eval") .tag("target", "project") .register(Metrics.getRegistry())); + LOGGER.info("Evaluation of project %s completed in %s" + .formatted(uuid, Duration.ofNanos(durationNs))); } } @@ -151,7 +320,7 @@ public void evaluateComponent(final UUID componentUuid) { // With no applicable policies, there's no way to resolve violations. // As a compensation, simply delete all violations associated with the component. LOGGER.info("No applicable policies found for component %s".formatted(componentUuid)); - reconcileViolations(qm, component, Collections.emptyList()); + reconcileViolations(qm, component, emptyList()); return; } @@ -375,6 +544,264 @@ private static List getParents(final QueryManager qm, final UUID uuid, fin return getParents(qm, parentUuid, parents); } + private static ProjectProjection fetchProject(final QueryManager qm, final long projectId, final Collection protoFieldNames) { + final String sqlSelectColumns = getFieldMappings(ProjectProjection.class).stream() + .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) + .map(mapping -> "\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ + SELECT %s FROM "PROJECT" WHERE "ID" = ? + """.formatted(sqlSelectColumns)); + query.setParameters(projectId); + try { + return query.executeResultUnique(ProjectProjection.class); + } finally { + query.closeAll(); + } + } + + private static org.hyades.proto.policy.v1.Project mapProject(final ProjectProjection projection) { + return org.hyades.proto.policy.v1.Project.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setGroup(trimToEmpty(projection.group)) + .setName(trimToEmpty(projection.name)) + .setVersion(trimToEmpty(projection.version)) + // .addAllTags(project.getTags().stream().map(Tag::getName).toList()) + .setCpe(trimToEmpty(projection.cpe)) + .setPurl(trimToEmpty(projection.purl)) + .setSwidTagId(trimToEmpty(projection.swidTagId)) + .build(); + } + + private static List fetchComponents(final QueryManager qm, final long projectId, final Collection protoFieldNames) { + final String sqlSelectColumns = Stream.concat( + Stream.of(ComponentProjection.ID_FIELD_MAPPING), + getFieldMappings(ComponentProjection.class).stream() + .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ + SELECT %s FROM "COMPONENT" WHERE "PROJECT_ID" = ? + """.formatted(sqlSelectColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(ComponentProjection.class)); + } finally { + query.closeAll(); + } + } + + private static org.hyades.proto.policy.v1.Component mapComponent(final ComponentProjection projection, + final Map licensesById) { + final org.hyades.proto.policy.v1.Component.Builder componentBuilder = + org.hyades.proto.policy.v1.Component.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setGroup(trimToEmpty(projection.group)) + .setName(trimToEmpty(projection.name)) + .setVersion(trimToEmpty(projection.name)) + .setClassifier(trimToEmpty(projection.classifier)) + .setCpe(trimToEmpty(projection.cpe)) + .setPurl(trimToEmpty(projection.purl)) + .setSwidTagId(trimToEmpty(projection.swidTagId)) + .setIsInternal(Optional.ofNullable(projection.internal).orElse(false)) + .setMd5(trimToEmpty(projection.md5)) + .setSha1(trimToEmpty(projection.sha1)) + .setSha256(trimToEmpty(projection.sha256)) + .setSha384(trimToEmpty(projection.sha384)) + .setSha512(trimToEmpty(projection.sha512)) + .setSha3256(trimToEmpty(projection.sha3_256)) + .setSha3384(trimToEmpty(projection.sha3_384)) + .setSha3512(trimToEmpty(projection.sha3_512)) + .setBlake2B256(trimToEmpty(projection.blake2b_256)) + .setBlake2B384(trimToEmpty(projection.blake2b_384)) + .setBlake2B512(trimToEmpty(projection.blake2b_512)) + .setBlake3(trimToEmpty(projection.blake3)); + + if (projection.resolvedLicenseId != null && projection.resolvedLicenseId > 0) { + componentBuilder.setResolvedLicense(licensesById.get(projection.resolvedLicenseId)); + } + + return componentBuilder.build(); + } + + private static List fetchComponentsVulnerabilities(final QueryManager qm, final long projectId) { + final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ + SELECT + "CV"."COMPONENT_ID" AS "componentId", + "CV"."VULNERABILITY_ID" AS "vulnerabilityId" + FROM + "COMPONENTS_VULNERABILITIES" AS "CV" + INNER JOIN + "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" + WHERE + "C"."PROJECT_ID" = ? + """); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(ComponentsVulnerabilitiesProjection.class)); + } finally { + query.closeAll(); + } + } + + private static List fetchLicenses(final QueryManager qm, final long projectId, + final Collection licenseProtoFieldNames, + final Collection licenseGroupProtoFieldNames) { + final String licenseSqlSelectColumns = Stream.concat( + Stream.of(LicenseProjection.ID_FIELD_MAPPING), + getFieldMappings(LicenseProjection.class).stream() + .filter(mapping -> licenseProtoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"L\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + if (!licenseProtoFieldNames.contains("groups")) { + final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ + SELECT DISTINCT + %s + FROM + "LICENSE" AS "L" + INNER JOIN + "COMPONENT" AS "C" ON "C"."LICENSE_ID" = "L"."ID" + WHERE + "C"."PROJECT_ID" = ? + """.formatted(licenseSqlSelectColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(LicenseProjection.class)); + } finally { + query.closeAll(); + } + } + + final String licenseSqlGroupByColumns = Stream.concat( + Stream.of(LicenseProjection.ID_FIELD_MAPPING), + getFieldMappings(LicenseProjection.class).stream() + .filter(mapping -> licenseProtoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"L\".\"%s\"".formatted(mapping.sqlColumnName())) + .collect(Collectors.joining(", ")); + + final String licenseGroupSqlSelectColumns = getFieldMappings(LicenseGroupProjection.class).stream() + .filter(mapping -> licenseGroupProtoFieldNames.contains(mapping.protoFieldName())) + .map(mapping -> "'%s', \"LG\".\"%s\"".formatted(mapping.javaFieldName(), mapping.sqlColumnName())) + .collect(Collectors.joining(", ")); + + final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ + SELECT DISTINCT + "L"."ID" AS "id", + %s, + CAST(JSONB_AGG(DISTINCT JSONB_BUILD_OBJECT(%s)) AS TEXT) AS "licenseGroupsJson" + FROM + "LICENSE" AS "L" + INNER JOIN + "COMPONENT" AS "C" ON "C"."LICENSE_ID" = "L"."ID" + LEFT JOIN + "LICENSEGROUP_LICENSE" AS "LGL" ON "LGL"."LICENSE_ID" = "L"."ID" + LEFT JOIN + "LICENSEGROUP" AS "LG" ON "LG"."ID" = "LGL"."LICENSEGROUP_ID" + WHERE + "C"."PROJECT_ID" = ? + GROUP BY + %s + """.formatted(licenseSqlSelectColumns, licenseGroupSqlSelectColumns, licenseSqlGroupByColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(LicenseProjection.class)); + } finally { + query.closeAll(); + } + } + + private static License mapLicense(final LicenseProjection licenseProjection) { + final License.Builder licenseBuilder = License.newBuilder() + .setUuid(trimToEmpty(licenseProjection.uuid)) + .setId(trimToEmpty(licenseProjection.licenseId)) + .setName(trimToEmpty(licenseProjection.name)); + Optional.ofNullable(licenseProjection.isOsiApproved).ifPresent(licenseBuilder::setIsOsiApproved); + Optional.ofNullable(licenseProjection.isFsfLibre).ifPresent(licenseBuilder::setIsFsfLibre); + Optional.ofNullable(licenseProjection.isDeprecatedId).ifPresent(licenseBuilder::setIsDeprecatedId); + Optional.ofNullable(licenseProjection.isCustomLicense).ifPresent(licenseBuilder::setIsCustom); + + if (licenseProjection.licenseGroupsJson != null) { + try { + final ArrayNode groupsArray = OBJECT_MAPPER.readValue(licenseProjection.licenseGroupsJson, ArrayNode.class); + for (final JsonNode groupNode : groupsArray) { + licenseBuilder.addGroups(License.Group.newBuilder() + .setUuid(Optional.ofNullable(groupNode.get("uuid")).map(JsonNode::asText).orElse("")) + .setName(Optional.ofNullable(groupNode.get("name")).map(JsonNode::asText).orElse("")) + .build()); + } + } catch (JacksonException e) { + LOGGER.warn("Failed to parse license groups JSON", e); + } + } + + return licenseBuilder.build(); + } + + private static List fetchVulnerabilities(final QueryManager qm, final long projectId, final Collection protoFieldNames) { + final String sqlSelectColumns = getFieldMappings(VulnerabilityProjection.class).stream() + .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) + .map(mapping -> "\"V\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + // TODO: Aliases could be fetched in the same query, using a JSONB aggregate. + // SELECT DISTINCT + // "V"."ID" AS "id", + // "V"."VULNID" AS "vulnId", + // "V"."SOURCE" AS "source", + // (SELECT + // JSONB_AGG(DISTINCT JSONB_STRIP_NULLS(JSONB_BUILD_OBJECT( + // 'cveId', "VA"."CVE_ID", + // 'ghsaId', "VA"."GHSA_ID", + // 'gsdId', "VA"."GSD_ID", + // 'internalId', "VA"."INTERNAL_ID", + // 'osvId', "VA"."OSV_ID", + // 'sonatypeId', "VA"."SONATYPE_ID", + // 'snykId', "VA"."SNYK_ID", + // 'vulnDbId', "VA"."VULNDB_ID" + // )))::TEXT + // FROM + // "VULNERABILITYALIAS" AS "VA" + // WHERE + // ("V"."SOURCE" = 'NVD' AND "VA"."CVE_ID" = "V"."VULNID") + // OR ("V"."SOURCE" = 'SNYK' AND "VA"."SNYK_ID" = "V"."VULNID") + // -- OR ... + // ) AS "aliasesJson" + // FROM + // "VULNERABILITY" AS "V" + // INNER JOIN + // "COMPONENTS_VULNERABILITIES" AS "CV" ON "CV"."VULNERABILITY_ID" = "V"."ID" + // INNER JOIN + // "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" + // WHERE + // "C"."PROJECT_ID" = ?; + + final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ + SELECT DISTINCT + "V"."ID" AS "id", + %s + FROM + "VULNERABILITY" AS "V" + INNER JOIN + "COMPONENTS_VULNERABILITIES" AS "CV" ON "CV"."VULNERABILITY_ID" = "V"."ID" + INNER JOIN + "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" + WHERE + "C"."PROJECT_ID" = ? + """.formatted(sqlSelectColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(VulnerabilityProjection.class)); + } finally { + query.closeAll(); + } + } + private static org.hyades.proto.policy.v1.Component mapComponent(final QueryManager qm, final Component component, final MultiValuedMap requirements) { @@ -481,7 +908,7 @@ private static List loadVulnerabilities(final QueryManager qm, final Component component, final MultiValuedMap requirements) { if (!requirements.containsKey(TYPE_VULNERABILITY)) { - return Collections.emptyList(); + return emptyList(); } // TODO: Load only required fields diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java new file mode 100644 index 000000000..b88405bb5 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java @@ -0,0 +1,81 @@ +package org.dependencytrack.policy.cel.persistence; + +public class ComponentProjection { + + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", null, "ID"); + + public long id; + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(sqlColumnName = "GROUP") + public String group; + + @MappedField(sqlColumnName = "NAME") + public String name; + + @MappedField(sqlColumnName = "VERSION") + public String version; + + @MappedField(sqlColumnName = "CLASSIFIER") + public String classifier; + + @MappedField(sqlColumnName = "CPE") + public String cpe; + + @MappedField(sqlColumnName = "PURL") + public String purl; + + @MappedField(protoFieldName = "swid_tag_id", sqlColumnName = "SWIDTAGID") + public String swidTagId; + + @MappedField(protoFieldName = "is_internal", sqlColumnName = "INTERNAL") + public Boolean internal; + + @MappedField(sqlColumnName = "MD5") + public String md5; + + @MappedField(sqlColumnName = "SHA1") + public String sha1; + + @MappedField(sqlColumnName = "SHA_256") + public String sha256; + + @MappedField(sqlColumnName = "SHA_384") + public String sha384; + + @MappedField(sqlColumnName = "SHA_512") + public String sha512; + + @MappedField(sqlColumnName = "SHA3_256") + public String sha3_256; + + @MappedField(sqlColumnName = "SHA3_384") + public String sha3_384; + + @MappedField(sqlColumnName = "SHA3_512") + public String sha3_512; + + @MappedField(sqlColumnName = "BLAKE2B_256") + public String blake2b_256; + + @MappedField(sqlColumnName = "BLAKE2B_384") + public String blake2b_384; + + @MappedField(sqlColumnName = "BLAKE2B_512") + public String blake2b_512; + + @MappedField(sqlColumnName = "BLAKE3") + public String blake3; + + @MappedField(protoFieldName = "resolved_license", sqlColumnName = "LICENSE_ID") + public Long resolvedLicenseId; + + @MappedField(protoFieldName = "license_name", sqlColumnName = "LICENSE") + public String licenseName; + + @MappedField(protoFieldName = "license_expression", sqlColumnName = "LICENSE_EXPRESSION") + public String licenseExpression; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentsVulnerabilitiesProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentsVulnerabilitiesProjection.java new file mode 100644 index 000000000..2918ef85e --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentsVulnerabilitiesProjection.java @@ -0,0 +1,9 @@ +package org.dependencytrack.policy.cel.persistence; + +public class ComponentsVulnerabilitiesProjection { + + public Long componentId; + + public Long vulnerabilityId; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMapping.java b/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMapping.java new file mode 100644 index 000000000..7498c1980 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMapping.java @@ -0,0 +1,4 @@ +package org.dependencytrack.policy.cel.persistence; + +public record FieldMapping(String javaFieldName, String protoFieldName, String sqlColumnName) { +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtil.java b/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtil.java new file mode 100644 index 000000000..971ab7824 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtil.java @@ -0,0 +1,38 @@ +package org.dependencytrack.policy.cel.persistence; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.commons.lang3.StringUtils.trimToNull; + +public final class FieldMappingUtil { + + private static final Map, List> FIELD_MAPPINGS_BY_CLASS = new ConcurrentHashMap<>(); + + public static List getFieldMappings(final Class clazz) { + return FIELD_MAPPINGS_BY_CLASS.computeIfAbsent(clazz, FieldMappingUtil::createFieldMappings); + } + + private static List createFieldMappings(final Class clazz) { + final var fieldMappings = new ArrayList(); + + for (final Field field : clazz.getDeclaredFields()) { + final MappedField mappedFieldAnnotation = field.getAnnotation(MappedField.class); + if (mappedFieldAnnotation == null) { + continue; + } + + final String javaFieldName = field.getName(); + final String protoFieldName = Optional.ofNullable(trimToNull(mappedFieldAnnotation.protoFieldName())).orElse(javaFieldName); + final String sqlColumnName = Optional.ofNullable(trimToNull(mappedFieldAnnotation.sqlColumnName())).orElseThrow(); + fieldMappings.add(new FieldMapping(javaFieldName, protoFieldName, sqlColumnName)); + } + + return fieldMappings; + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseGroupProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseGroupProjection.java new file mode 100644 index 000000000..f91c48adc --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseGroupProjection.java @@ -0,0 +1,11 @@ +package org.dependencytrack.policy.cel.persistence; + +public class LicenseGroupProjection { + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(sqlColumnName = "NAME") + public String name; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java new file mode 100644 index 000000000..8885b66dc --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java @@ -0,0 +1,32 @@ +package org.dependencytrack.policy.cel.persistence; + +public class LicenseProjection { + + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", null, "ID"); + + public long id; + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(protoFieldName = "id", sqlColumnName = "LICENSEID") + public String licenseId; + + @MappedField(sqlColumnName = "NAME") + public String name; + + @MappedField(protoFieldName = "is_osi_approved", sqlColumnName = "OSIAPPROVED") + public Boolean isOsiApproved; + + @MappedField(protoFieldName = "is_fsf_libre", sqlColumnName = "FSFLIBRE") + public Boolean isFsfLibre; + + @MappedField(protoFieldName = "is_deprecated_id", sqlColumnName = "ISDEPRECATED") + public Boolean isDeprecatedId; + + @MappedField(protoFieldName = "is_custom", sqlColumnName = "ISCUSTOMLICENSE") + public Boolean isCustomLicense; + + public String licenseGroupsJson; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/MappedField.java b/src/main/java/org/dependencytrack/policy/cel/persistence/MappedField.java new file mode 100644 index 000000000..5ee0823d8 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/MappedField.java @@ -0,0 +1,16 @@ +package org.dependencytrack.policy.cel.persistence; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface MappedField { + + String protoFieldName() default ""; + + String sqlColumnName(); + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java new file mode 100644 index 000000000..36477292c --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java @@ -0,0 +1,39 @@ +package org.dependencytrack.policy.cel.persistence; + +import java.util.Date; + +public class ProjectProjection { + + public long id; + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(sqlColumnName = "GROUP") + public String group; + + @MappedField(sqlColumnName = "NAME") + public String name; + + @MappedField(sqlColumnName = "VERSION") + public String version; + + @MappedField(sqlColumnName = "CLASSIFIER") + public String classifier; + + @MappedField(protoFieldName = "is_active", sqlColumnName = "ACTIVE") + public Boolean isActive; + + @MappedField(sqlColumnName = "CPE") + public String cpe; + + @MappedField(sqlColumnName = "PURL") + public String purl; + + @MappedField(protoFieldName = "swid_tag_id", sqlColumnName = "SWIDTAGID") + public String swidTagId; + + @MappedField(protoFieldName = "last_bom_import", sqlColumnName = "LAST_BOM_IMPORT") + public Date lastBomImport; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java new file mode 100644 index 000000000..53d93d80a --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java @@ -0,0 +1,39 @@ +package org.dependencytrack.policy.cel.persistence; + +public class VulnerabilityProjection { + + public long id; + + @MappedField(sqlColumnName = "UUID") + public String uuid; + + @MappedField(protoFieldName = "id", sqlColumnName = "VULNID") + public String vulnId; + + @MappedField(sqlColumnName = "SOURCE") + public String source; + + @MappedField(sqlColumnName = "CWES") + public String cwes; + + @MappedField(sqlColumnName = "CREATED") + public String created; + + @MappedField(sqlColumnName = "PUBLISHED") + public String published; + + @MappedField(sqlColumnName = "UPDATED") + public String updated; + + @MappedField(sqlColumnName = "SEVERITY") + public String severity; + + @MappedField(protoFieldName = "cvssv2_base_score", sqlColumnName = "CVSSV2_BASE_SCORE") + public Double cvssV2BaseScore; + + @MappedField(protoFieldName = "cvssv2_impact_subscore", sqlColumnName = "CVSSV2_IMPACT_SUBSCORE") + public Double cvssV2ImpactSubScore; + + public String aliasesJson; + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java b/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java new file mode 100644 index 000000000..22f60b972 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java @@ -0,0 +1,20 @@ +package org.dependencytrack.policy.cel.persistence; + +import com.google.protobuf.Descriptors.Descriptor; +import org.hyades.proto.policy.v1.Component; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class FieldMappingUtilTest { + + @Test + public void testGetFieldMappingsForComponentProjection() { + final Descriptor protoDescriptor = Component.getDescriptor(); + + assertThat(FieldMappingUtil.getFieldMappings(ComponentProjection.class)).allSatisfy( + fieldMapping -> assertThat(protoDescriptor.findFieldByName(fieldMapping.protoFieldName())).isNotNull() + ); + } + +} \ No newline at end of file From 26c1a4bd82536212961aea41c3252b6c1d884590 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 18 Sep 2023 00:32:12 +0200 Subject: [PATCH 14/46] Improve violation reconciliation for projects Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 197 +++++++++++------- .../persistence/VulnerabilityProjection.java | 48 ++++- .../java/org/dependencytrack/TestUtil.java | 1 + .../policy/cel/CelPolicyEngineTest.java | 6 +- 4 files changed, 165 insertions(+), 87 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 52f0e2e17..90d65684a 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -11,6 +11,7 @@ import com.google.protobuf.util.Timestamps; import io.micrometer.core.instrument.Timer; import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; import org.apache.commons.lang3.tuple.Pair; import org.dependencytrack.model.Component; @@ -66,6 +67,7 @@ import java.util.stream.Stream; import static java.util.Collections.emptyList; +import static java.util.function.Predicate.not; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE; @@ -114,6 +116,10 @@ public void evaluateProject(final UUID uuid) { final Timer.Sample timerSample = Timer.start(); try (final var qm = new QueryManager()) { + // TODO: Should this entire procedure run in a single DB transaction? + // Would be better for atomicity, but could block DB connections for prolonged + // period of time for larger projects with many violations. + final Project project = qm.getObjectByUuid(Project.class, uuid, List.of(Project.FetchGroup.IDENTIFIERS.name())); if (project == null) { LOGGER.warn("Project with UUID %s does not exist".formatted(uuid)); @@ -133,32 +139,9 @@ public void evaluateProject(final UUID uuid) { final List> conditionScriptPairs = policies.stream() .map(Policy::getPolicyConditions) .flatMap(Collection::stream) - .map(policyCondition -> { - final CelPolicyScriptSourceBuilder scriptBuilder = SCRIPT_BUILDERS.get(policyCondition.getSubject()); - if (scriptBuilder == null) { - LOGGER.warn("No script builder found that is capable of handling subjects of type %s".formatted(policyCondition.getSubject())); - return null; - } - - final String scriptSrc = scriptBuilder.apply(policyCondition); - if (scriptSrc == null) { - LOGGER.warn("Script builder was unable to create a script for condition %s".formatted(policyCondition.getUuid())); - return null; - } - - return Pair.of(policyCondition, scriptSrc); - }) + .map(this::buildConditionScriptSrc) .filter(Objects::nonNull) - .map(conditionScriptSrcPair -> { - final CelPolicyScript script; - try { - script = scriptHost.compile(conditionScriptSrcPair.getRight()); - } catch (ScriptCreateException e) { - throw new RuntimeException(e); - } - - return Pair.of(conditionScriptSrcPair.getLeft(), script); - }) + .map(this::compileConditionScript) .toList(); LOGGER.info("Determining evaluation requirements for project %s and %d policy conditions" @@ -187,8 +170,10 @@ public void evaluateProject(final UUID uuid) { final Map licenseById; if (requirements.containsKey(TYPE_LICENSE)) { licenseById = fetchLicenses(qm, project.getId(), requirements.get(TYPE_LICENSE), requirements.get(TYPE_LICENSE_GROUP)).stream() - .map(projection -> Pair.of(projection.id, mapLicense(projection))) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); + .collect(Collectors.toMap( + projection -> projection.id, + CelPolicyEngine::mapLicense + )); } else { licenseById = Collections.emptyMap(); } @@ -198,8 +183,10 @@ public void evaluateProject(final UUID uuid) { final Map> vulnIdsByComponentId; if (requirements.containsKey(TYPE_VULNERABILITY)) { protoVulnById = fetchVulnerabilities(qm, project.getId(), requirements.get(TYPE_VULNERABILITY)).stream() - .map(vulnProjection -> Pair.of(vulnProjection.id, Vulnerability.getDefaultInstance())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); + .collect(Collectors.toMap( + projection -> projection.id, + CelPolicyEngine::mapVulnerability + )); vulnIdsByComponentId = fetchComponentsVulnerabilities(qm, project.getId()).stream() .collect(Collectors.groupingBy( @@ -214,8 +201,7 @@ public void evaluateProject(final UUID uuid) { final var conditionsViolated = new HashSetValuedHashMap(); for (final ComponentProjection component : components) { final org.hyades.proto.policy.v1.Component protoComponent = mapComponent(component, licenseById); - - final List vulns = vulnIdsByComponentId.getOrDefault(component.id, emptyList()).stream() + final List protoVulns = vulnIdsByComponentId.getOrDefault(component.id, emptyList()).stream() .map(protoVulnById::get) .toList(); @@ -225,7 +211,7 @@ public void evaluateProject(final UUID uuid) { final Map scriptArgs = Map.of( VAR_COMPONENT, protoComponent, VAR_PROJECT, protoProject, - VAR_VULNERABILITIES, vulns + VAR_VULNERABILITIES, protoVulns ); try { @@ -236,28 +222,13 @@ public void evaluateProject(final UUID uuid) { throw new RuntimeException("Failed to evaluate script", e); } } - - if (!conditionsViolated.containsKey(component.id)) { - conditionsViolated.putAll(component.id, Collections.emptySet()); - } } - // In order to create policy violations, we need Component objects that are attached to the - // persistence context. - final Query componentQuery = qm.getPersistenceManager().newQuery(Component.class); - componentQuery.getFetchPlan().setGroup(Component.FetchGroup.IDENTITY.name()); - componentQuery.setFilter(":ids.contains(id)"); - componentQuery.setParameters(conditionsViolated.keySet()); - final Map persistentComponentById; - try { - persistentComponentById = componentQuery.executeList().stream() - .collect(Collectors.toMap(Component::getId, Function.identity())); - } finally { - componentQuery.closeAll(); - } - - // TODO: Short-circuit for components for which no violations were detected. - + // Evaluate policy operators. + // Determines whether violations must be reported in the first place. + // i.e. if not all conditions matched for a policy with ALL operator, + // then no violations are to be raised. + final var violationsByComponentId = new ArrayListValuedHashMap(); for (final long componentId : conditionsViolated.keySet()) { final Map> violatedConditionsByPolicy = conditionsViolated.get(componentId).stream() .collect(Collectors.groupingBy(PolicyCondition::getPolicy)); @@ -275,8 +246,10 @@ public void evaluateProject(final UUID uuid) { return violatedConditions.stream() .map(condition -> { final var violation = new PolicyViolation(); - violation.setComponent(persistentComponentById.get(componentId)); violation.setType(condition.getViolationType()); + // Note: violation.setComponent is intentionally omitted here, + // because the component must be an object attached to the persistence + // context. We don't have that at this point, we'll add it later. violation.setPolicyCondition(condition); violation.setTimestamp(new Date()); return violation; @@ -288,7 +261,40 @@ public void evaluateProject(final UUID uuid) { .filter(Objects::nonNull) .toList(); - final List newViolations = reconcileViolations(qm, persistentComponentById.get(componentId), violations); + violationsByComponentId.putAll(componentId, violations); + } + + final List componentIdsWithoutViolations = components.stream() + .map(projection -> projection.id) + .filter(not(violationsByComponentId.keySet()::contains)) + .toList(); + // TODO: Delete all existing violations for all elements of componentIdsWithoutViolations. + + // In order to create policy violations, we unfortunately need Component objects that + // are attached to the persistence context. We don't need any other fields beside their + // identity (i.e. ID) though. + // And because they are only required for CREATING violations, it's sufficient to only + // fetch them for components that violate at least one policy. + final Query componentQuery = qm.getPersistenceManager().newQuery(Component.class); + componentQuery.getFetchPlan().setGroup(Component.FetchGroup.IDENTITY.name()); + componentQuery.setFilter(":ids.contains(id)"); + componentQuery.setParameters(violationsByComponentId.keySet()); + final Map persistentComponentById; + try { + persistentComponentById = componentQuery.executeList().stream() + .collect(Collectors.toMap(Component::getId, Function.identity())); + } finally { + componentQuery.closeAll(); + } + + final var newViolations = new ArrayList(); + for (final Long componentId : violationsByComponentId.keySet()) { + final Component persistentComponent = persistentComponentById.get(componentId); + newViolations.addAll(reconcileViolations(qm, persistentComponent, violationsByComponentId.get(componentId))); + } + + for (final PolicyViolation newViolation : newViolations) { + NotificationUtil.analyzeNotificationCriteria(qm, newViolation); } } finally { final long durationNs = timerSample.stop(Timer @@ -332,32 +338,9 @@ public void evaluateComponent(final UUID componentUuid) { final List> conditionScriptPairs = policies.stream() .map(Policy::getPolicyConditions) .flatMap(Collection::stream) - .map(policyCondition -> { - final CelPolicyScriptSourceBuilder scriptBuilder = SCRIPT_BUILDERS.get(policyCondition.getSubject()); - if (scriptBuilder == null) { - LOGGER.warn("No script builder found that is capable of handling subjects of type %s".formatted(policyCondition.getSubject())); - return null; - } - - final String scriptSrc = scriptBuilder.apply(policyCondition); - if (scriptSrc == null) { - LOGGER.warn("Script builder was unable to create a script for condition %s".formatted(policyCondition.getUuid())); - return null; - } - - return Pair.of(policyCondition, scriptSrc); - }) + .map(this::buildConditionScriptSrc) .filter(Objects::nonNull) - .map(conditionScriptSrcPair -> { - final CelPolicyScript script; - try { - script = scriptHost.compile(conditionScriptSrcPair.getRight()); - } catch (ScriptCreateException e) { - throw new RuntimeException(e); - } - - return Pair.of(conditionScriptSrcPair.getLeft(), script); - }) + .map(this::compileConditionScript) .toList(); // Check what kind of data we need to evaluate all policy conditions. @@ -544,6 +527,33 @@ private static List getParents(final QueryManager qm, final UUID uuid, fin return getParents(qm, parentUuid, parents); } + private Pair buildConditionScriptSrc(final PolicyCondition policyCondition) { + final CelPolicyScriptSourceBuilder scriptBuilder = SCRIPT_BUILDERS.get(policyCondition.getSubject()); + if (scriptBuilder == null) { + LOGGER.warn("No script builder found that is capable of handling subjects of type %s".formatted(policyCondition.getSubject())); + return null; + } + + final String scriptSrc = scriptBuilder.apply(policyCondition); + if (scriptSrc == null) { + LOGGER.warn("Script builder was unable to create a script for condition %s".formatted(policyCondition.getUuid())); + return null; + } + + return Pair.of(policyCondition, scriptSrc); + } + + private Pair compileConditionScript(final Pair conditionScriptSrcPair) { + final CelPolicyScript script; + try { + script = scriptHost.compile(conditionScriptSrcPair.getRight()); + } catch (ScriptCreateException e) { + throw new RuntimeException(e); + } + + return Pair.of(conditionScriptSrcPair.getLeft(), script); + } + private static ProjectProjection fetchProject(final QueryManager qm, final long projectId, final Collection protoFieldNames) { final String sqlSelectColumns = getFieldMappings(ProjectProjection.class).stream() .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) @@ -802,6 +812,33 @@ private static List fetchVulnerabilities(final QueryMan } } + private static Vulnerability mapVulnerability(final VulnerabilityProjection projection) { + final Vulnerability.Builder builder = Vulnerability.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setId(trimToEmpty(projection.vulnId)) + .setSource(trimToEmpty(projection.source)) + .setCvssv2Vector(trimToEmpty(projection.cvssV2Vector)) + .setCvssv3Vector(trimToEmpty(projection.cvssV3Vector)) + .setOwaspRrVector(trimToEmpty(projection.owaspRrVector)) + .setSeverity(trimToEmpty(projection.severity)); + // Optional.ofNullable(v.getCwes()).ifPresent(builder::addAllCwes); + Optional.ofNullable(projection.cvssV2BaseScore).ifPresent(builder::setCvssv2BaseScore); + Optional.ofNullable(projection.cvssV2ImpactSubScore).ifPresent(builder::setCvssv2ImpactSubscore); + Optional.ofNullable(projection.cvssV2ExploitabilitySubScore).ifPresent(builder::setCvssv2ExploitabilitySubscore); + Optional.ofNullable(projection.cvssV3BaseScore).ifPresent(builder::setCvssv3BaseScore); + Optional.ofNullable(projection.cvssV3ImpactSubScore).ifPresent(builder::setCvssv3ImpactSubscore); + Optional.ofNullable(projection.cvssV3ExploitabilitySubScore).ifPresent(builder::setCvssv3ExploitabilitySubscore); + Optional.ofNullable(projection.owaspRrLikelihoodScore).ifPresent(builder::setOwaspRrLikelihoodScore); + Optional.ofNullable(projection.owaspRrTechnicalImpactScore).ifPresent(builder::setOwaspRrTechnicalImpactScore); + Optional.ofNullable(projection.owaspRrBusinessImpactScore).ifPresent(builder::setOwaspRrBusinessImpactScore); + Optional.ofNullable(projection.epssScore).ifPresent(builder::setEpssScore); + Optional.ofNullable(projection.epssPercentile).ifPresent(builder::setEpssPercentile); + Optional.ofNullable(projection.created).map(Timestamps::fromDate).ifPresent(builder::setCreated); + Optional.ofNullable(projection.published).map(Timestamps::fromDate).ifPresent(builder::setPublished); + Optional.ofNullable(projection.updated).map(Timestamps::fromDate).ifPresent(builder::setUpdated); + return builder.build(); + } + private static org.hyades.proto.policy.v1.Component mapComponent(final QueryManager qm, final Component component, final MultiValuedMap requirements) { @@ -986,6 +1023,8 @@ private static List reconcileViolations(final QueryManager qm, final var violationIdsToKeep = new HashSet(); for (final PolicyViolation violation : violations) { + violation.setComponent(component); + final Query query = qm.getPersistenceManager().newQuery(PolicyViolation.class); query.setFilter("component == :component && policyCondition == :policyCondition && type == :type"); query.setNamedParameters(Map.of( diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java index 53d93d80a..f41f9dcef 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java @@ -1,5 +1,7 @@ package org.dependencytrack.policy.cel.persistence; +import java.util.Date; + public class VulnerabilityProjection { public long id; @@ -17,23 +19,59 @@ public class VulnerabilityProjection { public String cwes; @MappedField(sqlColumnName = "CREATED") - public String created; + public Date created; @MappedField(sqlColumnName = "PUBLISHED") - public String published; + public Date published; @MappedField(sqlColumnName = "UPDATED") - public String updated; + public Date updated; @MappedField(sqlColumnName = "SEVERITY") public String severity; - @MappedField(protoFieldName = "cvssv2_base_score", sqlColumnName = "CVSSV2_BASE_SCORE") + @MappedField(protoFieldName = "cvssv2_base_score", sqlColumnName = "CVSSV2BASESCORE") public Double cvssV2BaseScore; - @MappedField(protoFieldName = "cvssv2_impact_subscore", sqlColumnName = "CVSSV2_IMPACT_SUBSCORE") + @MappedField(protoFieldName = "cvssv2_impact_subscore", sqlColumnName = "CVSSV2IMPACTSCORE") public Double cvssV2ImpactSubScore; + @MappedField(protoFieldName = "cvssv2_exploitability_subscore", sqlColumnName = "CVSSV2EXPLOITSCORE") + public Double cvssV2ExploitabilitySubScore; + + @MappedField(protoFieldName = "cvssv2_vector", sqlColumnName = "CVSSV2VECTOR") + public String cvssV2Vector; + + @MappedField(protoFieldName = "cvssv3_base_score", sqlColumnName = "CVSSV3BASESCORE") + public Double cvssV3BaseScore; + + @MappedField(protoFieldName = "cvssv3_impact_subscore", sqlColumnName = "CVSSV3IMPACTSCORE") + public Double cvssV3ImpactSubScore; + + @MappedField(protoFieldName = "cvssv3_exploitability_subscore", sqlColumnName = "CVSSV3EXPLOITSCORE") + public Double cvssV3ExploitabilitySubScore; + + @MappedField(protoFieldName = "cvssv3_vector", sqlColumnName = "CVSSV3VECTOR") + public String cvssV3Vector; + + @MappedField(protoFieldName = "owasp_rr_likelihood_score", sqlColumnName = "OWASPRRLIKELIHOODSCORE") + public Double owaspRrLikelihoodScore; + + @MappedField(protoFieldName = "owasp_rr_technical_impact_score", sqlColumnName = "OWASPRRTECHNICALIMPACTSCORE") + public Double owaspRrTechnicalImpactScore; + + @MappedField(protoFieldName = "owasp_rr_business_impact_score", sqlColumnName = "OWASPRRBUSINESSIMPACTSCORE") + public Double owaspRrBusinessImpactScore; + + @MappedField(protoFieldName = "owasp_rr_vector", sqlColumnName = "OWASPRRVECTOR") + public String owaspRrVector; + + @MappedField(protoFieldName = "epss_score", sqlColumnName = "EPSSSCORE") + public Double epssScore; + + @MappedField(protoFieldName = "epss_percentile", sqlColumnName = "EPSSPERCENTILE") + public Double epssPercentile; + public String aliasesJson; } diff --git a/src/test/java/org/dependencytrack/TestUtil.java b/src/test/java/org/dependencytrack/TestUtil.java index 8e1559e1f..3966bef81 100644 --- a/src/test/java/org/dependencytrack/TestUtil.java +++ b/src/test/java/org/dependencytrack/TestUtil.java @@ -8,6 +8,7 @@ public class TestUtil { public static Properties getDatanucleusProperties(String jdbcUrl, String driverName, String username, String pwd) { final var dnProps = new Properties(); + dnProps.put(PropertyNames.PROPERTY_PERSISTENCE_UNIT_NAME, "Alpine"); dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_DATABASE, "true"); dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_TABLES, "true"); dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_COLUMNS, "true"); diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index ce9173309..8ec4ba242 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -1,6 +1,6 @@ package org.dependencytrack.policy.cel; -import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.AbstractPostgresEnabledTest; import org.dependencytrack.model.AnalyzerIdentity; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; @@ -25,7 +25,7 @@ import static org.assertj.core.api.Assertions.assertThat; -public class CelPolicyEngineTest extends PersistenceCapableTest { +public class CelPolicyEngineTest extends AbstractPostgresEnabledTest { @Test public void test() { @@ -96,7 +96,7 @@ public void test() { qm.persist(existingViolation); final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateComponent(component.getUuid()); + policyEngine.evaluateProject(project.getUuid()); final List violations = qm.getAllPolicyViolations(component); assertThat(violations).isNotEmpty(); From 5f563843b417dbc7786fa572290c729a93987c50 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 18 Sep 2023 11:38:51 +0200 Subject: [PATCH 15/46] Add test with bloated BOM to debug performance bottlenecks Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngineTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index 8ec4ba242..1b51ccb39 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -1,9 +1,11 @@ package org.dependencytrack.policy.cel; import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.BomUploadEvent; import org.dependencytrack.model.AnalyzerIdentity; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; +import org.dependencytrack.model.ConfigPropertyConstants; import org.dependencytrack.model.License; import org.dependencytrack.model.LicenseGroup; import org.dependencytrack.model.Policy; @@ -12,9 +14,17 @@ import org.dependencytrack.model.Project; import org.dependencytrack.model.Severity; import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.persistence.DefaultObjectGenerator; +import org.dependencytrack.tasks.BomUploadProcessingTask; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.sql.Date; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -23,10 +33,22 @@ import java.util.List; import java.util.UUID; +import static org.apache.commons.io.IOUtils.resourceToURL; import static org.assertj.core.api.Assertions.assertThat; public class CelPolicyEngineTest extends AbstractPostgresEnabledTest { + @Before + public void setUp() throws Exception { + super.setUp(); + + // Enable processing of CycloneDX BOMs + qm.createConfigProperty(ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getGroupName(), + ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getPropertyName(), "true", + ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getPropertyType(), + ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getDescription()); + } + @Test public void test() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); @@ -338,4 +360,38 @@ public void issue2455() { Assert.assertEquals(PolicyCondition.Subject.LICENSE_GROUP, policyViolation.getPolicyCondition().getSubject()); } + @Test + public void testWithBloatedBom() throws Exception { + // Import all default objects (includes licenses and license groups). + new DefaultObjectGenerator().contextInitialized(null); + + final var project = new Project(); + project.setName("acme-app"); + project.setVersion("1.2.3"); + qm.persist(project); + + // Create a policy that will be violated by the vast majority (>8000) components. + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, + PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.resolved_license.groups.exists(lg, lg.name == "Permissive") + """); + policyConditionA.setViolationType(PolicyViolation.Type.OPERATIONAL); + qm.persist(policyConditionA); + + // Import the bloated BOM. + new BomUploadProcessingTask().inform(new BomUploadEvent(qm.detach(Project.class, project.getId()), createTempBomFile("bom-bloated.json"))); + + // Evaluate policies on the project. + new CelPolicyEngine().evaluateProject(project.getUuid()); + } + + private static File createTempBomFile(final String testFileName) throws Exception { + // The task will delete the input file after processing it, + // so create a temporary copy to not impact other tests. + final Path bomFilePath = Files.createTempFile(null, null); + Files.copy(Paths.get(resourceToURL("/unit/" + testFileName).toURI()), bomFilePath, StandardCopyOption.REPLACE_EXISTING); + return bomFilePath.toFile(); + } + } \ No newline at end of file From 2016e0eb79ae942abc0696e2c3ea0efe13fad3d8 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 18 Sep 2023 12:32:15 +0200 Subject: [PATCH 16/46] Disable DataNucleus L1 cache for policy reconciliation Signed-off-by: nscuro --- .../org/dependencytrack/policy/cel/CelPolicyEngine.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 90d65684a..516800f92 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -14,6 +14,7 @@ import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; import org.apache.commons.lang3.tuple.Pair; +import org.datanucleus.PropertyNames; import org.dependencytrack.model.Component; import org.dependencytrack.model.LicenseGroup; import org.dependencytrack.model.Policy; @@ -116,6 +117,12 @@ public void evaluateProject(final UUID uuid) { final Timer.Sample timerSample = Timer.start(); try (final var qm = new QueryManager()) { + // Disable L1 cache. We don't need the results of INSERT or UPDATE operations to be present in the cache. + // We do potentially perform lots of INSERTS during violation reconciliation, maintenance of L1 cache + // will have a huge overhead we'd rather not have to deal with. + qm.getPersistenceManager().setProperty(PropertyNames.PROPERTY_CACHE_L1_TYPE, "none"); + qm.getPersistenceManager().setProperty(PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false"); + // TODO: Should this entire procedure run in a single DB transaction? // Would be better for atomicity, but could block DB connections for prolonged // period of time for larger projects with many violations. From 40e92182410e57943eb87690fc877325805775fe Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 18 Sep 2023 12:32:40 +0200 Subject: [PATCH 17/46] Add field mapping tests Signed-off-by: nscuro --- .../cel/persistence/ComponentProjection.java | 7 +- .../cel/persistence/LicenseProjection.java | 2 +- .../cel/persistence/ProjectProjection.java | 2 +- .../cel/persistence/FieldMappingUtilTest.java | 71 +++++++++++++++++-- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java index b88405bb5..7e4ddf7b2 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java @@ -2,7 +2,7 @@ public class ComponentProjection { - public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", null, "ID"); + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", /* protoFieldName */ null, "ID"); public long id; @@ -75,7 +75,8 @@ public class ComponentProjection { @MappedField(protoFieldName = "license_name", sqlColumnName = "LICENSE") public String licenseName; - @MappedField(protoFieldName = "license_expression", sqlColumnName = "LICENSE_EXPRESSION") - public String licenseExpression; + // Requires https://github.com/DependencyTrack/dependency-track/pull/2400 to be ported to Hyades. + // @MappedField(protoFieldName = "license_expression", sqlColumnName = "LICENSE_EXPRESSION") + // public String licenseExpression; } diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java index 8885b66dc..d587558fa 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java @@ -15,7 +15,7 @@ public class LicenseProjection { @MappedField(sqlColumnName = "NAME") public String name; - @MappedField(protoFieldName = "is_osi_approved", sqlColumnName = "OSIAPPROVED") + @MappedField(protoFieldName = "is_osi_approved", sqlColumnName = "ISOSIAPPROVED") public Boolean isOsiApproved; @MappedField(protoFieldName = "is_fsf_libre", sqlColumnName = "FSFLIBRE") diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java b/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java index 36477292c..a41dfec52 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java @@ -33,7 +33,7 @@ public class ProjectProjection { @MappedField(protoFieldName = "swid_tag_id", sqlColumnName = "SWIDTAGID") public String swidTagId; - @MappedField(protoFieldName = "last_bom_import", sqlColumnName = "LAST_BOM_IMPORT") + @MappedField(protoFieldName = "last_bom_import", sqlColumnName = "LAST_BOM_IMPORTED") public Date lastBomImport; } diff --git a/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java b/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java index 22f60b972..0603b2add 100644 --- a/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java @@ -1,20 +1,83 @@ package org.dependencytrack.policy.cel.persistence; import com.google.protobuf.Descriptors.Descriptor; +import org.dependencytrack.PersistenceCapableTest; import org.hyades.proto.policy.v1.Component; +import org.hyades.proto.policy.v1.License; +import org.hyades.proto.policy.v1.Project; +import org.hyades.proto.policy.v1.Vulnerability; import org.junit.Test; +import javax.jdo.PersistenceManagerFactory; +import javax.jdo.metadata.ColumnMetadata; +import javax.jdo.metadata.MemberMetadata; +import javax.jdo.metadata.TypeMetadata; + import static org.assertj.core.api.Assertions.assertThat; -public class FieldMappingUtilTest { +public class FieldMappingUtilTest extends PersistenceCapableTest { @Test public void testGetFieldMappingsForComponentProjection() { - final Descriptor protoDescriptor = Component.getDescriptor(); + assertValidProtoFieldsAndColumns(ComponentProjection.class, Component.getDescriptor(), org.dependencytrack.model.Component.class); + } + + @Test + public void testGetFieldMappingsForLicenseProjection() { + assertValidProtoFieldsAndColumns(LicenseProjection.class, License.getDescriptor(), org.dependencytrack.model.License.class); + } + + @Test + public void testGetFieldMappingsForLicenseGroupProjection() { + assertValidProtoFieldsAndColumns(LicenseGroupProjection.class, License.Group.getDescriptor(), org.dependencytrack.model.LicenseGroup.class); + } + + @Test + public void testGetFieldMappingsForProjectProjection() { + assertValidProtoFieldsAndColumns(ProjectProjection.class, Project.getDescriptor(), org.dependencytrack.model.Project.class); + } - assertThat(FieldMappingUtil.getFieldMappings(ComponentProjection.class)).allSatisfy( - fieldMapping -> assertThat(protoDescriptor.findFieldByName(fieldMapping.protoFieldName())).isNotNull() + @Test + public void testGetFieldMappingsForVulnerabilityProjection() { + assertValidProtoFieldsAndColumns(VulnerabilityProjection.class, Vulnerability.getDescriptor(), org.dependencytrack.model.Vulnerability.class); + } + + private void assertValidProtoFieldsAndColumns(final Class projectionClazz, + final Descriptor protoDescriptor, + final Class persistenceClass) { + assertThat(FieldMappingUtil.getFieldMappings(projectionClazz)).allSatisfy( + fieldMapping -> { + assertHasProtoField(protoDescriptor, fieldMapping.protoFieldName()); + assertHasSqlColumn(persistenceClass, fieldMapping.sqlColumnName()); + } ); } + private void assertHasProtoField(final Descriptor protoDescriptor, final String fieldName) { + assertThat(protoDescriptor.findFieldByName(fieldName)).isNotNull(); + } + + private void assertHasSqlColumn(final Class clazz, final String columnName) { + final PersistenceManagerFactory pmf = qm.getPersistenceManager().getPersistenceManagerFactory(); + + final TypeMetadata typeMetadata = pmf.getMetadata(clazz.getName()); + assertThat(typeMetadata).isNotNull(); + + var found = false; + for (final MemberMetadata memberMetadata : typeMetadata.getMembers()) { + if (memberMetadata.getColumns() == null) { + continue; + } + + for (final ColumnMetadata columnMetadata : memberMetadata.getColumns()) { + if (columnName.equals(columnMetadata.getName())) { + found = true; + break; + } + } + } + + assertThat(found).isTrue(); + } + } \ No newline at end of file From 199498893b399c122b4c511162c4958d710906ba Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 18 Sep 2023 13:52:05 +0200 Subject: [PATCH 18/46] Handle implicit policy script requirements for custom functions Signed-off-by: nscuro --- .../policy/cel/CelPolicyLibrary.java | 6 ++-- .../policy/cel/CelPolicyScriptHost.java | 32 +++++++++++++++++++ .../policy/cel/CelPolicyScriptVisitor.java | 27 +++++++++++++--- .../policy/cel/CelPolicyScriptHostTest.java | 5 ++- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index 5569679fb..5bba55afb 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -47,9 +47,9 @@ public class CelPolicyLibrary implements Library { public static final Type TYPE_VULNERABILITIES = Decls.newListType(TYPE_VULNERABILITY); public static final Type TYPE_VULNERABILITY_ALIAS = Decls.newObjectType(Vulnerability.Alias.getDescriptor().getFullName()); - private static final String FUNC_DEPENDS_ON = "depends_on"; - private static final String FUNC_IS_DEPENDENCY_OF = "is_dependency_of"; - private static final String FUNC_MATCHES_RANGE = "matches_range"; + public static final String FUNC_DEPENDS_ON = "depends_on"; + public static final String FUNC_IS_DEPENDENCY_OF = "is_dependency_of"; + public static final String FUNC_MATCHES_RANGE = "matches_range"; @Override public List getCompileOptions() { diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java index 5f01c2167..03f38919c 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -7,6 +7,7 @@ import com.google.common.util.concurrent.Striped; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.MultiValuedMap; +import org.dependencytrack.policy.cel.CelPolicyScriptVisitor.FunctionSignature; import org.projectnessie.cel.Ast; import org.projectnessie.cel.CEL; import org.projectnessie.cel.Env; @@ -24,6 +25,11 @@ import java.util.List; import java.util.concurrent.locks.Lock; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.FUNC_DEPENDS_ON; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.FUNC_IS_DEPENDENCY_OF; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.FUNC_MATCHES_RANGE; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT; import static org.projectnessie.cel.Issues.newIssues; import static org.projectnessie.cel.common.Source.newTextSource; @@ -106,6 +112,32 @@ public CelPolicyScript compile(final String scriptSrc) throws ScriptCreateExcept private static MultiValuedMap analyzeRequirements(final CheckedExpr expr) { final var visitor = new CelPolicyScriptVisitor(expr.getTypeMapMap()); visitor.visit(expr.getExpr()); + + // Fields that are accessed directly are always a requirement. + final MultiValuedMap requirements = visitor.getAccessedFieldsByType(); + + // Custom functions may access certain fields implicitly, in a way that is not visible + // to the AST visitor. To compensate, we hardcode the functions' requirements here. + // TODO: This should be restructured to be more generic. + for (final FunctionSignature functionSignature : visitor.getUsedFunctionSignatures()) { + switch (functionSignature.function()) { + case FUNC_DEPENDS_ON, FUNC_IS_DEPENDENCY_OF -> { + if (TYPE_PROJECT.equals(functionSignature.targetType())) { + requirements.put(TYPE_PROJECT, "uuid"); + } else if (TYPE_COMPONENT.equals(functionSignature.targetType())) { + requirements.put(TYPE_COMPONENT, "uuid"); + } + } + case FUNC_MATCHES_RANGE -> { + if (TYPE_PROJECT.equals(functionSignature.targetType())) { + requirements.put(TYPE_PROJECT, "version"); + } else if (TYPE_COMPONENT.equals(functionSignature.targetType())) { + requirements.put(TYPE_COMPONENT, "version"); + } + } + } + } + return visitor.getAccessedFieldsByType(); } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java index c7284e6d4..2ae141947 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptVisitor.java @@ -8,27 +8,35 @@ import java.util.ArrayDeque; import java.util.Deque; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; -public class CelPolicyScriptVisitor { +class CelPolicyScriptVisitor { private static final Logger LOGGER = Logger.getLogger(CelPolicyScriptVisitor.class); + record FunctionSignature(String function, Type targetType, List argumentTypes) { + } + private final Map types; private final MultiValuedMap accessedFieldsByType; + private final Set usedFunctionSignatures; private final Deque callFunctionStack; private final Deque selectFieldStack; private final Deque selectOperandTypeStack; - public CelPolicyScriptVisitor(final Map types) { + CelPolicyScriptVisitor(final Map types) { this.types = types; this.accessedFieldsByType = new HashSetValuedHashMap<>(); + this.usedFunctionSignatures = new HashSet<>(); this.callFunctionStack = new ArrayDeque<>(); this.selectFieldStack = new ArrayDeque<>(); this.selectOperandTypeStack = new ArrayDeque<>(); } - public void visit(final Expr expr) { + void visit(final Expr expr) { switch (expr.getExprKindCase()) { case CALL_EXPR -> visitCall(expr); case COMPREHENSION_EXPR -> visitComprehension(expr); @@ -45,6 +53,13 @@ private void visitCall(final Expr expr) { logExpr(expr); final Expr.Call callExpr = expr.getCallExpr(); + final Type targetType = types.get(callExpr.getTarget().getId()); + final List argumentTypes = callExpr.getArgsList().stream() + .map(Expr::getId) + .map(types::get) + .toList(); + usedFunctionSignatures.add(new FunctionSignature(callExpr.getFunction(), targetType, argumentTypes)); + callFunctionStack.push(callExpr.getFunction()); visit(callExpr.getTarget()); for (final Expr argExpr : callExpr.getArgsList()) { @@ -96,8 +111,12 @@ private void logExpr(final Expr expr) { .formatted(expr.getExprKindCase(), expr.getId(), selectFieldStack, selectOperandTypeStack, callFunctionStack)); } - public MultiValuedMap getAccessedFieldsByType() { + MultiValuedMap getAccessedFieldsByType() { return this.accessedFieldsByType; } + Set getUsedFunctionSignatures() { + return this.usedFunctionSignatures; + } + } diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java index 394210c5c..4a6daadf3 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java @@ -10,6 +10,7 @@ import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE_GROUP; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY; public class CelPolicyScriptHostTest { @@ -19,14 +20,16 @@ public void testRequirements() throws Exception { final CelPolicyScript compiledScript = CelPolicyScriptHost.getInstance().compile(""" component.resolved_license.groups.exists(licenseGroup, licenseGroup.name == "Permissive") && vulns.filter(vuln, vuln.severity in ["HIGH", "CRITICAL"]).size() > 1 + && project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) """); final Map> requirements = compiledScript.getRequirements().asMap(); - assertThat(requirements).containsKeys(TYPE_COMPONENT, TYPE_LICENSE, TYPE_LICENSE_GROUP, TYPE_VULNERABILITY); + assertThat(requirements).containsOnlyKeys(TYPE_COMPONENT, TYPE_LICENSE, TYPE_LICENSE_GROUP, TYPE_PROJECT, TYPE_VULNERABILITY); assertThat(requirements.get(TYPE_COMPONENT)).containsOnly("resolved_license"); assertThat(requirements.get(TYPE_LICENSE)).containsOnly("groups"); assertThat(requirements.get(TYPE_LICENSE_GROUP)).containsOnly("name"); + assertThat(requirements.get(TYPE_PROJECT)).containsOnly("uuid"); // Implicit through project.depends_on assertThat(requirements.get(TYPE_VULNERABILITY)).containsOnly("severity"); } From 3ec5f3b591b28abbdd8942c8eac04f31fe4d6a7b Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 18 Sep 2023 13:53:08 +0200 Subject: [PATCH 19/46] Minor readability and code documentation improvements Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 72 ++++++++++++++----- 1 file changed, 54 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 516800f92..ad821e176 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -13,6 +13,7 @@ import org.apache.commons.collections4.MultiValuedMap; import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.datanucleus.PropertyNames; import org.dependencytrack.model.Component; @@ -23,6 +24,7 @@ import org.dependencytrack.model.PolicyViolation; import org.dependencytrack.model.Project; import org.dependencytrack.model.Tag; +import org.dependencytrack.persistence.CollectionIntegerConverter; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.ComponentHashCelPolicyScriptSourceBuilder; @@ -226,6 +228,8 @@ public void evaluateProject(final UUID uuid) { conditionsViolated.put(component.id, condition); } } catch (ScriptException e) { + // TODO: Should we really fail the entire run for ALL components, + // if only one condition failed to evaluate? throw new RuntimeException("Failed to evaluate script", e); } } @@ -579,7 +583,7 @@ private static ProjectProjection fetchProject(final QueryManager qm, final long } private static org.hyades.proto.policy.v1.Project mapProject(final ProjectProjection projection) { - return org.hyades.proto.policy.v1.Project.newBuilder() + final org.hyades.proto.policy.v1.Project.Builder builder = org.hyades.proto.policy.v1.Project.newBuilder() .setUuid(trimToEmpty(projection.uuid)) .setGroup(trimToEmpty(projection.group)) .setName(trimToEmpty(projection.name)) @@ -587,8 +591,9 @@ private static org.hyades.proto.policy.v1.Project mapProject(final ProjectProjec // .addAllTags(project.getTags().stream().map(Tag::getName).toList()) .setCpe(trimToEmpty(projection.cpe)) .setPurl(trimToEmpty(projection.purl)) - .setSwidTagId(trimToEmpty(projection.swidTagId)) - .build(); + .setSwidTagId(trimToEmpty(projection.swidTagId)); + Optional.ofNullable(projection.lastBomImport).map(Timestamps::fromDate).ifPresent(builder::setLastBomImport); + return builder.build(); } private static List fetchComponents(final QueryManager qm, final long projectId, final Collection protoFieldNames) { @@ -612,7 +617,7 @@ private static List fetchComponents(final QueryManager qm, } private static org.hyades.proto.policy.v1.Component mapComponent(final ComponentProjection projection, - final Map licensesById) { + final Map protoLicenseById) { final org.hyades.proto.policy.v1.Component.Builder componentBuilder = org.hyades.proto.policy.v1.Component.newBuilder() .setUuid(trimToEmpty(projection.uuid)) @@ -638,7 +643,13 @@ private static org.hyades.proto.policy.v1.Component mapComponent(final Component .setBlake3(trimToEmpty(projection.blake3)); if (projection.resolvedLicenseId != null && projection.resolvedLicenseId > 0) { - componentBuilder.setResolvedLicense(licensesById.get(projection.resolvedLicenseId)); + final License protoLicense = protoLicenseById.get(projection.resolvedLicenseId); + if (protoLicense != null) { + componentBuilder.setResolvedLicense(protoLicenseById.get(projection.resolvedLicenseId)); + } else { + LOGGER.warn("Component with ID %d refers to license %d, but no license with that ID was found" + .formatted(projection.id, projection.resolvedLicenseId)); + } } return componentBuilder.build(); @@ -675,6 +686,7 @@ private static List fetchLicenses(final QueryManager qm, fina .map(mapping -> "\"L\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) .collect(Collectors.joining(", ")); + // If fetching license groups is not necessary, we can just query for licenses and be done with it. if (!licenseProtoFieldNames.contains("groups")) { final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ SELECT DISTINCT @@ -694,6 +706,26 @@ private static List fetchLicenses(final QueryManager qm, fina } } + // If groups are required, include them in the license query in order to avoid the 1+N problem. + // Licenses may or may not be assigned to a group. Licenses can be in multiple groups. + // + // Using a simple LEFT JOIN would result in duplicate license data being fetched, e.g.: + // + // | "L"."ID" | "L"."NAME" | "LG"."NAME" | + // | :------- | :--------- | :---------- | + // | 1 | foo | groupA | + // | 1 | foo | groupB | + // | 1 | foo | groupC | + // | 2 | bar | NULL | + // + // To avoid this, we instead aggregate license group fields for each license, and return them as JSON. + // The reason for choosing JSON over native arrays, is that DataNucleus can't deal with arrays cleanly. + // + // | "L"."ID" | "L"."NAME" | "licenseGroupsJson" | + // | :------- | :--------- | :------------------------------------------------------ | + // | 1 | foo | [{"name":"groupA"},{"name":"groupB"},{"name":"groupC"}] | + // | 2 | bar | [] | + final String licenseSqlGroupByColumns = Stream.concat( Stream.of(LicenseProjection.ID_FIELD_MAPPING), getFieldMappings(LicenseProjection.class).stream() @@ -733,19 +765,19 @@ private static List fetchLicenses(final QueryManager qm, fina } } - private static License mapLicense(final LicenseProjection licenseProjection) { + private static License mapLicense(final LicenseProjection projection) { final License.Builder licenseBuilder = License.newBuilder() - .setUuid(trimToEmpty(licenseProjection.uuid)) - .setId(trimToEmpty(licenseProjection.licenseId)) - .setName(trimToEmpty(licenseProjection.name)); - Optional.ofNullable(licenseProjection.isOsiApproved).ifPresent(licenseBuilder::setIsOsiApproved); - Optional.ofNullable(licenseProjection.isFsfLibre).ifPresent(licenseBuilder::setIsFsfLibre); - Optional.ofNullable(licenseProjection.isDeprecatedId).ifPresent(licenseBuilder::setIsDeprecatedId); - Optional.ofNullable(licenseProjection.isCustomLicense).ifPresent(licenseBuilder::setIsCustom); - - if (licenseProjection.licenseGroupsJson != null) { + .setUuid(trimToEmpty(projection.uuid)) + .setId(trimToEmpty(projection.licenseId)) + .setName(trimToEmpty(projection.name)); + Optional.ofNullable(projection.isOsiApproved).ifPresent(licenseBuilder::setIsOsiApproved); + Optional.ofNullable(projection.isFsfLibre).ifPresent(licenseBuilder::setIsFsfLibre); + Optional.ofNullable(projection.isDeprecatedId).ifPresent(licenseBuilder::setIsDeprecatedId); + Optional.ofNullable(projection.isCustomLicense).ifPresent(licenseBuilder::setIsCustom); + + if (projection.licenseGroupsJson != null) { try { - final ArrayNode groupsArray = OBJECT_MAPPER.readValue(licenseProjection.licenseGroupsJson, ArrayNode.class); + final ArrayNode groupsArray = OBJECT_MAPPER.readValue(projection.licenseGroupsJson, ArrayNode.class); for (final JsonNode groupNode : groupsArray) { licenseBuilder.addGroups(License.Group.newBuilder() .setUuid(Optional.ofNullable(groupNode.get("uuid")).map(JsonNode::asText).orElse("")) @@ -753,7 +785,7 @@ private static License mapLicense(final LicenseProjection licenseProjection) { .build()); } } catch (JacksonException e) { - LOGGER.warn("Failed to parse license groups JSON", e); + LOGGER.warn("Failed to parse license groups JSON for license %s".formatted(projection.id), e); } } @@ -828,7 +860,6 @@ private static Vulnerability mapVulnerability(final VulnerabilityProjection proj .setCvssv3Vector(trimToEmpty(projection.cvssV3Vector)) .setOwaspRrVector(trimToEmpty(projection.owaspRrVector)) .setSeverity(trimToEmpty(projection.severity)); - // Optional.ofNullable(v.getCwes()).ifPresent(builder::addAllCwes); Optional.ofNullable(projection.cvssV2BaseScore).ifPresent(builder::setCvssv2BaseScore); Optional.ofNullable(projection.cvssV2ImpactSubScore).ifPresent(builder::setCvssv2ImpactSubscore); Optional.ofNullable(projection.cvssV2ExploitabilitySubScore).ifPresent(builder::setCvssv2ExploitabilitySubscore); @@ -843,6 +874,11 @@ private static Vulnerability mapVulnerability(final VulnerabilityProjection proj Optional.ofNullable(projection.created).map(Timestamps::fromDate).ifPresent(builder::setCreated); Optional.ofNullable(projection.published).map(Timestamps::fromDate).ifPresent(builder::setPublished); Optional.ofNullable(projection.updated).map(Timestamps::fromDate).ifPresent(builder::setUpdated); + Optional.ofNullable(projection.cwes) + .map(StringUtils::trimToNull) + .filter(Objects::nonNull) + .map(new CollectionIntegerConverter()::convertToAttribute) + .ifPresent(builder::addAllCwes); return builder.build(); } From 489d1c52bff908ea468e256b6b9c806f3beb4781 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 18 Sep 2023 19:11:56 +0200 Subject: [PATCH 20/46] Fetch data for policy violation notifications in a single query DataNucleus on its own loads too much data, and does so using too many queries. Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 15 +- .../policy/cel/CelPolicyLibrary.java | 24 +-- .../policy/cel/CelPolicyScriptHost.java | 2 +- .../ComponentProjection.java | 2 +- .../ComponentsVulnerabilitiesProjection.java | 2 +- .../FieldMapping.java | 2 +- .../FieldMappingUtil.java | 5 +- .../LicenseGroupProjection.java | 2 +- .../LicenseProjection.java | 4 +- .../policy/cel/mapping/MappedField.java | 28 +++ .../ProjectProjection.java | 2 +- .../VulnerabilityProjection.java | 2 +- .../policy/cel/persistence/MappedField.java | 16 -- .../util/NotificationUtil.java | 167 ++++++++++++++++-- .../FieldMappingUtilTest.java | 2 +- 15 files changed, 219 insertions(+), 56 deletions(-) rename src/main/java/org/dependencytrack/policy/cel/{persistence => mapping}/ComponentProjection.java (97%) rename src/main/java/org/dependencytrack/policy/cel/{persistence => mapping}/ComponentsVulnerabilitiesProjection.java (69%) rename src/main/java/org/dependencytrack/policy/cel/{persistence => mapping}/FieldMapping.java (65%) rename src/main/java/org/dependencytrack/policy/cel/{persistence => mapping}/FieldMappingUtil.java (94%) rename src/main/java/org/dependencytrack/policy/cel/{persistence => mapping}/LicenseGroupProjection.java (76%) rename src/main/java/org/dependencytrack/policy/cel/{persistence => mapping}/LicenseProjection.java (90%) create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/MappedField.java rename src/main/java/org/dependencytrack/policy/cel/{persistence => mapping}/ProjectProjection.java (94%) rename src/main/java/org/dependencytrack/policy/cel/{persistence => mapping}/VulnerabilityProjection.java (98%) delete mode 100644 src/main/java/org/dependencytrack/policy/cel/persistence/MappedField.java rename src/test/java/org/dependencytrack/policy/cel/{persistence => mapping}/FieldMappingUtilTest.java (98%) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index ad821e176..afe9bf7ff 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -37,12 +37,12 @@ import org.dependencytrack.policy.cel.compat.SeverityCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; -import org.dependencytrack.policy.cel.persistence.ComponentProjection; -import org.dependencytrack.policy.cel.persistence.ComponentsVulnerabilitiesProjection; -import org.dependencytrack.policy.cel.persistence.LicenseGroupProjection; -import org.dependencytrack.policy.cel.persistence.LicenseProjection; -import org.dependencytrack.policy.cel.persistence.ProjectProjection; -import org.dependencytrack.policy.cel.persistence.VulnerabilityProjection; +import org.dependencytrack.policy.cel.mapping.ComponentProjection; +import org.dependencytrack.policy.cel.mapping.ComponentsVulnerabilitiesProjection; +import org.dependencytrack.policy.cel.mapping.LicenseGroupProjection; +import org.dependencytrack.policy.cel.mapping.LicenseProjection; +import org.dependencytrack.policy.cel.mapping.ProjectProjection; +import org.dependencytrack.policy.cel.mapping.VulnerabilityProjection; import org.dependencytrack.util.NotificationUtil; import org.dependencytrack.util.VulnerabilityUtil; import org.hyades.proto.policy.v1.License; @@ -82,7 +82,7 @@ import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_PROJECT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_VULNERABILITIES; -import static org.dependencytrack.policy.cel.persistence.FieldMappingUtil.getFieldMappings; +import static org.dependencytrack.policy.cel.mapping.FieldMappingUtil.getFieldMappings; public class CelPolicyEngine { @@ -207,6 +207,7 @@ public void evaluateProject(final UUID uuid) { vulnIdsByComponentId = Collections.emptyMap(); } + // Evaluate all policy conditions against all components. final var conditionsViolated = new HashSetValuedHashMap(); for (final ComponentProjection component : components) { final org.hyades.proto.policy.v1.Component protoComponent = mapComponent(component, licenseById); diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index 5bba55afb..da1e65f25 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -38,18 +38,18 @@ public class CelPolicyLibrary implements Library { static final String VAR_PROJECT = "project"; static final String VAR_VULNERABILITIES = "vulns"; - public static final Type TYPE_COMPONENT = Decls.newObjectType(Component.getDescriptor().getFullName()); - public static final Type TYPE_LICENSE = Decls.newObjectType(License.getDescriptor().getFullName()); - public static final Type TYPE_LICENSE_GROUP = Decls.newObjectType(License.Group.getDescriptor().getFullName()); - public static final Type TYPE_PROJECT = Decls.newObjectType(Project.getDescriptor().getFullName()); - public static final Type TYPE_PROJECT_PROPERTY = Decls.newObjectType(Project.Property.getDescriptor().getFullName()); - public static final Type TYPE_VULNERABILITY = Decls.newObjectType(Vulnerability.getDescriptor().getFullName()); - public static final Type TYPE_VULNERABILITIES = Decls.newListType(TYPE_VULNERABILITY); - public static final Type TYPE_VULNERABILITY_ALIAS = Decls.newObjectType(Vulnerability.Alias.getDescriptor().getFullName()); - - public static final String FUNC_DEPENDS_ON = "depends_on"; - public static final String FUNC_IS_DEPENDENCY_OF = "is_dependency_of"; - public static final String FUNC_MATCHES_RANGE = "matches_range"; + static final Type TYPE_COMPONENT = Decls.newObjectType(Component.getDescriptor().getFullName()); + static final Type TYPE_LICENSE = Decls.newObjectType(License.getDescriptor().getFullName()); + static final Type TYPE_LICENSE_GROUP = Decls.newObjectType(License.Group.getDescriptor().getFullName()); + static final Type TYPE_PROJECT = Decls.newObjectType(Project.getDescriptor().getFullName()); + static final Type TYPE_PROJECT_PROPERTY = Decls.newObjectType(Project.Property.getDescriptor().getFullName()); + static final Type TYPE_VULNERABILITY = Decls.newObjectType(Vulnerability.getDescriptor().getFullName()); + static final Type TYPE_VULNERABILITIES = Decls.newListType(TYPE_VULNERABILITY); + static final Type TYPE_VULNERABILITY_ALIAS = Decls.newObjectType(Vulnerability.Alias.getDescriptor().getFullName()); + + static final String FUNC_DEPENDS_ON = "depends_on"; + static final String FUNC_IS_DEPENDENCY_OF = "is_dependency_of"; + static final String FUNC_MATCHES_RANGE = "matches_range"; @Override public List getCompileOptions() { diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java index 03f38919c..be5668a94 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -138,7 +138,7 @@ private static MultiValuedMap analyzeRequirements(final CheckedExp } } - return visitor.getAccessedFieldsByType(); + return requirements; } } diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentProjection.java similarity index 97% rename from src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java rename to src/main/java/org/dependencytrack/policy/cel/mapping/ComponentProjection.java index 7e4ddf7b2..36603b9ca 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentProjection.java @@ -1,4 +1,4 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; public class ComponentProjection { diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentsVulnerabilitiesProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentsVulnerabilitiesProjection.java similarity index 69% rename from src/main/java/org/dependencytrack/policy/cel/persistence/ComponentsVulnerabilitiesProjection.java rename to src/main/java/org/dependencytrack/policy/cel/mapping/ComponentsVulnerabilitiesProjection.java index 2918ef85e..530bab03e 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/ComponentsVulnerabilitiesProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ComponentsVulnerabilitiesProjection.java @@ -1,4 +1,4 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; public class ComponentsVulnerabilitiesProjection { diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMapping.java b/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMapping.java similarity index 65% rename from src/main/java/org/dependencytrack/policy/cel/persistence/FieldMapping.java rename to src/main/java/org/dependencytrack/policy/cel/mapping/FieldMapping.java index 7498c1980..b4eb6d3f7 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMapping.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMapping.java @@ -1,4 +1,4 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; public record FieldMapping(String javaFieldName, String protoFieldName, String sqlColumnName) { } diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtil.java b/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtil.java similarity index 94% rename from src/main/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtil.java rename to src/main/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtil.java index 971ab7824..6efd2ffba 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtil.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtil.java @@ -1,4 +1,4 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; import java.lang.reflect.Field; import java.util.ArrayList; @@ -13,6 +13,9 @@ public final class FieldMappingUtil { private static final Map, List> FIELD_MAPPINGS_BY_CLASS = new ConcurrentHashMap<>(); + private FieldMappingUtil() { + } + public static List getFieldMappings(final Class clazz) { return FIELD_MAPPINGS_BY_CLASS.computeIfAbsent(clazz, FieldMappingUtil::createFieldMappings); } diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseGroupProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseGroupProjection.java similarity index 76% rename from src/main/java/org/dependencytrack/policy/cel/persistence/LicenseGroupProjection.java rename to src/main/java/org/dependencytrack/policy/cel/mapping/LicenseGroupProjection.java index f91c48adc..d226be831 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseGroupProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseGroupProjection.java @@ -1,4 +1,4 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; public class LicenseGroupProjection { diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseProjection.java similarity index 90% rename from src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java rename to src/main/java/org/dependencytrack/policy/cel/mapping/LicenseProjection.java index d587558fa..71e20ddc0 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/LicenseProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/LicenseProjection.java @@ -1,8 +1,8 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; public class LicenseProjection { - public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", null, "ID"); + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", /* protoFieldName */ null, "ID"); public long id; diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/MappedField.java b/src/main/java/org/dependencytrack/policy/cel/mapping/MappedField.java new file mode 100644 index 000000000..89f1a4b44 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/MappedField.java @@ -0,0 +1,28 @@ +package org.dependencytrack.policy.cel.mapping; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface MappedField { + + /** + * Name of the field in the Protobuf schema. + *

+ * If empty string (the default), the name of the annotated field will be assumed. + * + * @return Name of the Protobuf field + */ + String protoFieldName() default ""; + + /** + * Name of the SQL column corresponding to this field. + * + * @return Name of the SQL column + */ + String sqlColumnName(); + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java similarity index 94% rename from src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java rename to src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java index a41dfec52..f6c680335 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/ProjectProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java @@ -1,4 +1,4 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; import java.util.Date; diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java similarity index 98% rename from src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java rename to src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java index f41f9dcef..a41ed4b11 100644 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/VulnerabilityProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java @@ -1,4 +1,4 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; import java.util.Date; diff --git a/src/main/java/org/dependencytrack/policy/cel/persistence/MappedField.java b/src/main/java/org/dependencytrack/policy/cel/persistence/MappedField.java deleted file mode 100644 index 5ee0823d8..000000000 --- a/src/main/java/org/dependencytrack/policy/cel/persistence/MappedField.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.dependencytrack.policy.cel.persistence; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.FIELD) -public @interface MappedField { - - String protoFieldName() default ""; - - String sqlColumnName(); - -} diff --git a/src/main/java/org/dependencytrack/util/NotificationUtil.java b/src/main/java/org/dependencytrack/util/NotificationUtil.java index 053979416..215861257 100644 --- a/src/main/java/org/dependencytrack/util/NotificationUtil.java +++ b/src/main/java/org/dependencytrack/util/NotificationUtil.java @@ -22,14 +22,18 @@ import alpine.notification.Notification; import alpine.notification.NotificationLevel; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.dependencytrack.event.kafka.KafkaEventDispatcher; import org.dependencytrack.model.Analysis; import org.dependencytrack.model.Component; import org.dependencytrack.model.ConfigPropertyConstants; import org.dependencytrack.model.Finding; import org.dependencytrack.model.NotificationPublisher; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.model.PolicyViolation; import org.dependencytrack.model.Project; +import org.dependencytrack.model.Tag; import org.dependencytrack.model.ViolationAnalysis; import org.dependencytrack.model.ViolationAnalysisState; import org.dependencytrack.model.Vulnerability; @@ -48,14 +52,19 @@ import org.hyades.proto.notification.v1.ProjectVulnAnalysisStatus; import javax.jdo.FetchPlan; +import javax.jdo.Query; import java.io.File; import java.io.IOException; import java.net.URLDecoder; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -211,21 +220,128 @@ public static void analyzeNotificationCriteria(final QueryManager qm, ViolationA } public static void analyzeNotificationCriteria(final QueryManager qm, final PolicyViolation policyViolation) { - final ViolationAnalysis violationAnalysis = qm.getViolationAnalysis(policyViolation.getComponent(), policyViolation); - if (violationAnalysis != null && (violationAnalysis.isSuppressed() || ViolationAnalysisState.APPROVED == violationAnalysis.getAnalysisState())) + final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ + SELECT + "PV"."UUID" AS "violationUuid", + "PV"."TYPE" AS "violationType", + "PV"."TIMESTAMP" AS "violationTimestamp", + "PC"."UUID" AS "conditionUuid", + "PC"."SUBJECT" AS "conditionSubject", + "PC"."OPERATOR" AS "conditionOperator", + "PC"."VALUE" AS "conditionValue", + "P"."UUID" AS "policyUuid", + "P"."NAME" AS "policyName", + "P"."VIOLATIONSTATE" AS "policyViolationState", + "VA"."SUPPRESSED" AS "analysisSuppressed", + "VA"."STATE" AS "analysisState", + "C"."UUID" AS "componentUuid", + "C"."GROUP" AS "componentGroup", + "C"."NAME" AS "componentName", + "C"."VERSION" AS "componentVersion", + "C"."PURL" AS "componentPurl", + "C"."MD5" AS "componentMd5", + "C"."SHA1" AS "componentSha1", + "C"."SHA_256" AS "componentSha256", + "C"."SHA_512" AS "componentSha512", + "PR"."UUID" AS "projectUuid", + "PR"."NAME" AS "projectName", + "PR"."VERSION" AS "projectVersion", + "PR"."DESCRIPTION" AS "projectDescription", + "PR"."PURL" AS "projectPurl", + (SELECT + STRING_AGG("T"."NAME", ',') + FROM + "TAG" AS "T" + INNER JOIN + "PROJECTS_TAGS" AS "PT" ON "PT"."TAG_ID" = "T"."ID" + WHERE + "PT"."PROJECT_ID" = "PR"."ID" + ) AS "projectTags" + FROM + "POLICYVIOLATION" AS "PV" + INNER JOIN + "POLICYCONDITION" AS "PC" ON "PC"."ID" = "PV"."POLICYCONDITION_ID" + INNER JOIN + "POLICY" AS "P" ON "P"."ID" = "PC"."POLICY_ID" + INNER JOIN + "COMPONENT" AS "C" ON "C"."ID" = "PV"."COMPONENT_ID" + INNER JOIN + "PROJECT" AS "PR" ON "PR"."ID" = "PV"."PROJECT_ID" + LEFT JOIN + "VIOLATIONANALYSIS" AS "VA" ON "VA"."POLICYVIOLATION_ID" = "PV"."ID" + WHERE + "PV"."ID" = ? + """); + query.setParameters(policyViolation.getId()); + final PolicyViolationNotificationProjection projection; + try { + projection = query.executeResultUnique(PolicyViolationNotificationProjection.class); + } finally { + query.closeAll(); + } + + if (projection == null) { + return; + } + + if ((projection.analysisSuppressed != null && projection.analysisSuppressed) + || ViolationAnalysisState.APPROVED.name().equals(projection.analysisState)) { return; - policyViolation.getPolicyCondition().getPolicy(); // Force loading of policy - qm.getPersistenceManager().getFetchPlan().setMaxFetchDepth(2); // Ensure policy is included - qm.getPersistenceManager().getFetchPlan().setDetachmentOptions(FetchPlan.DETACH_LOAD_FIELDS); - final PolicyViolation pv = qm.getPersistenceManager().detachCopy(policyViolation); - Project project = policyViolation.getComponent().getProject(); + } + + final var project = new Project(); + project.setUuid(UUID.fromString(projection.projectUuid)); + project.setName(projection.projectName); + project.setVersion(projection.projectVersion); + project.setDescription(projection.projectDescription); + project.setPurl(projection.projectPurl); + project.setTags(Optional.ofNullable(projection.projectTags).stream() + .flatMap(tagNames -> Arrays.stream(tagNames.split(","))) + .map(StringUtils::trimToNull) + .filter(Objects::nonNull) + .map(tagName -> { + final var tag = new Tag(); + tag.setName(tagName); + return tag; + }) + .toList()); + + final var component = new Component(); + component.setUuid(UUID.fromString(projection.componentUuid)); + component.setGroup(projection.componentGroup); + component.setName(projection.componentName); + component.setVersion(projection.componentVersion); + component.setPurl(projection.componentPurl); + component.setMd5(projection.componentMd5); + component.setSha1(projection.componentSha1); + component.setSha256(projection.componentSha256); + component.setSha512(projection.componentSha512); + + final var policy = new Policy(); + policy.setUuid(UUID.fromString(projection.policyUuid)); + policy.setName(projection.policyName); + policy.setViolationState(Policy.ViolationState.valueOf(projection.policyViolationState)); + + final var policyCondition = new PolicyCondition(); + policyCondition.setPolicy(policy); + policyCondition.setUuid(UUID.fromString(projection.conditionUuid)); + policyCondition.setSubject(PolicyCondition.Subject.valueOf(projection.conditionSubject)); + policyCondition.setOperator(PolicyCondition.Operator.valueOf(projection.conditionOperator)); + policyCondition.setValue(projection.conditionValue); + + final var violation = new PolicyViolation(); + violation.setPolicyCondition(policyCondition); + violation.setUuid(UUID.fromString(projection.violationUuid)); + violation.setType(PolicyViolation.Type.valueOf(projection.violationType)); + violation.setTimestamp(projection.violationTimestamp); + sendNotificationToKafka(project.getUuid(), new Notification() .scope(NotificationScope.PORTFOLIO) .group(NotificationGroup.POLICY_VIOLATION) - .title(generateNotificationTitle(NotificationConstants.Title.POLICY_VIOLATION, policyViolation.getComponent().getProject())) + .title(generateNotificationTitle(NotificationConstants.Title.POLICY_VIOLATION, project)) .level(NotificationLevel.INFORMATIONAL) - .content(generateNotificationContent(pv)) - .subject(new PolicyViolationIdentified(pv, pv.getComponent(), pv.getProject())) + .content(generateNotificationContent(violation)) + .subject(new PolicyViolationIdentified(violation, component, project)) ); } @@ -395,4 +511,35 @@ public static List createList(List com } return componentAnalysisCompleteList; } + + public static class PolicyViolationNotificationProjection { + public String projectUuid; + public String projectName; + public String projectVersion; + public String projectDescription; + public String projectPurl; + public String projectTags; + public String componentUuid; + public String componentGroup; + public String componentName; + public String componentVersion; + public String componentPurl; + public String componentMd5; + public String componentSha1; + public String componentSha256; + public String componentSha512; + public String violationUuid; + public String violationType; + public Date violationTimestamp; + public String conditionUuid; + public String conditionSubject; + public String conditionOperator; + public String conditionValue; + public String policyUuid; + public String policyName; + public String policyViolationState; + public Boolean analysisSuppressed; + public String analysisState; + } + } diff --git a/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java b/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java similarity index 98% rename from src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java rename to src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java index 0603b2add..1cda78ff6 100644 --- a/src/test/java/org/dependencytrack/policy/cel/persistence/FieldMappingUtilTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.policy.cel.persistence; +package org.dependencytrack.policy.cel.mapping; import com.google.protobuf.Descriptors.Descriptor; import org.dependencytrack.PersistenceCapableTest; From 68d54a9eeb91787ef958717e7e5d95e8d7127ef1 Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 19 Sep 2023 20:41:31 +0200 Subject: [PATCH 21/46] Perform violation reconciliation using direct JDBC access Signed-off-by: nscuro --- .../model/PolicyViolation.java | 4 + .../policy/cel/CelPolicyEngine.java | 198 ++++++++++++++---- .../mapping/PolicyViolationProjection.java | 4 + .../util/NotificationUtil.java | 12 +- 4 files changed, 181 insertions(+), 37 deletions(-) create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/PolicyViolationProjection.java diff --git a/src/main/java/org/dependencytrack/model/PolicyViolation.java b/src/main/java/org/dependencytrack/model/PolicyViolation.java index f76de9cc5..a4d298793 100644 --- a/src/main/java/org/dependencytrack/model/PolicyViolation.java +++ b/src/main/java/org/dependencytrack/model/PolicyViolation.java @@ -46,6 +46,10 @@ @PersistenceCapable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) +@Unique(name = "foo", members = { + "component", + "policyCondition", +}) public class PolicyViolation implements Serializable { public enum Type { diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index afe9bf7ff..f4ac34102 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -41,6 +41,7 @@ import org.dependencytrack.policy.cel.mapping.ComponentsVulnerabilitiesProjection; import org.dependencytrack.policy.cel.mapping.LicenseGroupProjection; import org.dependencytrack.policy.cel.mapping.LicenseProjection; +import org.dependencytrack.policy.cel.mapping.PolicyViolationProjection; import org.dependencytrack.policy.cel.mapping.ProjectProjection; import org.dependencytrack.policy.cel.mapping.VulnerabilityProjection; import org.dependencytrack.util.NotificationUtil; @@ -52,7 +53,14 @@ import javax.jdo.Query; import javax.jdo.Transaction; +import javax.jdo.datastore.JDOConnection; import java.math.BigDecimal; +import java.sql.Array; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -64,13 +72,14 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.UUID; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.sql.Connection.TRANSACTION_READ_COMMITTED; import static java.util.Collections.emptyList; -import static java.util.function.Predicate.not; +import static org.apache.commons.collections4.MultiMapUtils.emptyMultiValuedMap; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_LICENSE; @@ -137,10 +146,8 @@ public void evaluateProject(final UUID uuid) { final List policies = getApplicablePolicies(qm, project); if (policies.isEmpty()) { - // With no applicable policies, there's no way to resolve violations. - // As a compensation, simply delete all violations associated with the component. - LOGGER.info("No applicable policies found for component %s".formatted(uuid)); - // reconcileViolations(qm, component, Collections.emptyList()); + LOGGER.info("No applicable policies found for project %s".formatted(uuid)); + reconcileViolations(qm, project.getId(), emptyMultiValuedMap()); return; } @@ -276,37 +283,11 @@ public void evaluateProject(final UUID uuid) { violationsByComponentId.putAll(componentId, violations); } - final List componentIdsWithoutViolations = components.stream() - .map(projection -> projection.id) - .filter(not(violationsByComponentId.keySet()::contains)) - .toList(); - // TODO: Delete all existing violations for all elements of componentIdsWithoutViolations. - - // In order to create policy violations, we unfortunately need Component objects that - // are attached to the persistence context. We don't need any other fields beside their - // identity (i.e. ID) though. - // And because they are only required for CREATING violations, it's sufficient to only - // fetch them for components that violate at least one policy. - final Query componentQuery = qm.getPersistenceManager().newQuery(Component.class); - componentQuery.getFetchPlan().setGroup(Component.FetchGroup.IDENTITY.name()); - componentQuery.setFilter(":ids.contains(id)"); - componentQuery.setParameters(violationsByComponentId.keySet()); - final Map persistentComponentById; - try { - persistentComponentById = componentQuery.executeList().stream() - .collect(Collectors.toMap(Component::getId, Function.identity())); - } finally { - componentQuery.closeAll(); - } + final List newViolationIds = reconcileViolations(qm, project.getId(), violationsByComponentId); + LOGGER.info("Identified %d new violations for project %s".formatted(newViolationIds.size(), uuid)); - final var newViolations = new ArrayList(); - for (final Long componentId : violationsByComponentId.keySet()) { - final Component persistentComponent = persistentComponentById.get(componentId); - newViolations.addAll(reconcileViolations(qm, persistentComponent, violationsByComponentId.get(componentId))); - } - - for (final PolicyViolation newViolation : newViolations) { - NotificationUtil.analyzeNotificationCriteria(qm, newViolation); + for (final Long newViolationId : newViolationIds) { + NotificationUtil.analyzeNotificationCriteria(qm, newViolationId); } } finally { final long durationNs = timerSample.stop(Timer @@ -1113,4 +1094,149 @@ private static List reconcileViolations(final QueryManager qm, return newViolations; } + private static List reconcileViolations(final QueryManager qm, final long projectId, + final MultiValuedMap reportedViolationsByComponentId) { + // We want to send notifications for newly identified policy violations, + // so need to keep track of which violations we created. + final var newViolationIds = new ArrayList(); + + // DataNucleus does not support batch inserts, which is something we need to create + // new violations efficiently. Falling back to "raw" JDBC for the sake of efficiency. + final JDOConnection jdoConnection = qm.getPersistenceManager().getDataStoreConnection(); + final var nativeConnection = (Connection) jdoConnection.getNativeConnection(); + + try { + // JDBC connections default to autocommit. + // We'll do multiple write operations here, and want to commit them all in a single transaction. + nativeConnection.setAutoCommit(false); + nativeConnection.setTransactionIsolation(TRANSACTION_READ_COMMITTED); + + // First, query for all existing policy violations of the project, grouping them by component ID. + final var existingViolationsByComponentId = new HashSetValuedHashMap(); + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + SELECT + "ID" AS "id", + "COMPONENT_ID" AS "componentId", + "POLICYCONDITION_ID" AS "policyConditionId" + FROM + "POLICYVIOLATION" + WHERE + "PROJECT_ID" = ? + """)) { + ps.setLong(1, projectId); + + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + existingViolationsByComponentId.put( + rs.getLong("componentId"), + new PolicyViolationProjection( + rs.getLong("id"), + rs.getLong("policyConditionId") + )); + } + } + + // For each component that has existing and / or reported violations... + final Set componentIds = new HashSet<>(reportedViolationsByComponentId.keySet().size() + existingViolationsByComponentId.keySet().size()); + componentIds.addAll(reportedViolationsByComponentId.keySet()); + componentIds.addAll(existingViolationsByComponentId.keySet()); + + // ... determine which existing violations should be deleted (because they're no longer reported), + // and which reported violations should be created (because they have not been reported before). + // + // Violations not belonging to either of those buckets are reported, but already exist, + // meaning no action needs to be taken for them. + final var violationIdsToDelete = new ArrayList(); + final var violationsToCreate = new HashSetValuedHashMap(); + for (final Long componentId : componentIds) { + final Collection existingViolations = existingViolationsByComponentId.get(componentId); + final Collection reportedViolations = reportedViolationsByComponentId.get(componentId); + + if (reportedViolations == null || reportedViolations.isEmpty()) { + // Component has been removed, or does not have any violations anymore. + // All of its existing violations can be deleted. + violationIdsToDelete.addAll(existingViolations.stream().map(PolicyViolationProjection::id).toList()); + continue; + } + + if (existingViolations == null || existingViolations.isEmpty()) { + // Component did not have any violations before, but has some now. + // All reported violations must be newly created. + violationsToCreate.putAll(componentId, reportedViolations); + continue; + } + + // To determine which violations shall be deleted, find occurrences of violations appearing + // in the collection of existing violations, but not in the collection of reported violations. + existingViolations.stream() + .filter(existingViolation -> reportedViolations.stream().noneMatch(newViolation -> + newViolation.getPolicyCondition().getId() == existingViolation.policyConditionId())) + .map(PolicyViolationProjection::id) + .forEach(violationIdsToDelete::add); + + // To determine which violations shall be created, find occurrences of violations appearing + // in the collection of reported violations, but not in the collection of existing violations. + reportedViolations.stream() + .filter(reportedViolation -> existingViolations.stream().noneMatch(existingViolation -> + existingViolation.policyConditionId() == reportedViolation.getPolicyCondition().getId())) + .forEach(reportedViolation -> violationsToCreate.put(componentId, reportedViolation)); + } + + if (!violationsToCreate.isEmpty()) { + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + INSERT INTO "POLICYVIOLATION" + ("UUID", "TIMESTAMP", "COMPONENT_ID", "PROJECT_ID", "POLICYCONDITION_ID", "TYPE") + VALUES + (?, NOW(), ?, ?, ?, ?) + ON CONFLICT DO NOTHING + RETURNING "ID" + """, Statement.RETURN_GENERATED_KEYS)) { + for (final Map.Entry entry : violationsToCreate.entries()) { + ps.setString(1, UUID.randomUUID().toString()); + ps.setLong(2, entry.getKey()); + ps.setLong(3, projectId); + ps.setLong(4, entry.getValue().getPolicyCondition().getId()); + ps.setString(5, entry.getValue().getType().name()); + ps.addBatch(); + } + ps.executeBatch(); + + final ResultSet rs = ps.getGeneratedKeys(); + while (rs.next()) { + newViolationIds.add(rs.getLong(1)); + } + } + } + + if (!violationIdsToDelete.isEmpty()) { + final Array violationIdsToDeleteArray = + nativeConnection.createArrayOf("BIGINT", violationIdsToDelete.toArray(new Long[0])); + + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + DELETE FROM + "POLICYVIOLATION" + WHERE + "ID" = ANY(?) + """)) { + ps.setArray(1, violationIdsToDeleteArray); + ps.execute(); + } + } + + nativeConnection.commit(); + } catch (Exception e) { + try { + nativeConnection.rollback(); + } catch (SQLException ex) { + throw new RuntimeException(ex); + } + + throw new RuntimeException(e); + } finally { + jdoConnection.close(); + } + + return newViolationIds; + } + } diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/PolicyViolationProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/PolicyViolationProjection.java new file mode 100644 index 000000000..0a4c75ac7 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/PolicyViolationProjection.java @@ -0,0 +1,4 @@ +package org.dependencytrack.policy.cel.mapping; + +public record PolicyViolationProjection(Long id, Long policyConditionId) { +} diff --git a/src/main/java/org/dependencytrack/util/NotificationUtil.java b/src/main/java/org/dependencytrack/util/NotificationUtil.java index 215861257..580c9a6db 100644 --- a/src/main/java/org/dependencytrack/util/NotificationUtil.java +++ b/src/main/java/org/dependencytrack/util/NotificationUtil.java @@ -102,6 +102,8 @@ public static void dispatchNotificationsWithSubject(UUID projectUuid, Notificati public static void analyzeNotificationCriteria(final QueryManager qm, Analysis analysis, final boolean analysisStateChange, final boolean suppressionChange) { + // TODO: Convert data loading to raw SQL to avoid loading unneeded data and excessive queries. + // See #analyzeNotificationCriteria(QueryManager, PolicyViolation) for an example. if (analysisStateChange || suppressionChange) { final NotificationGroup notificationGroup; notificationGroup = NotificationGroup.PROJECT_AUDIT_CHANGE; @@ -159,6 +161,8 @@ public static void analyzeNotificationCriteria(final QueryManager qm, Analysis a public static void analyzeNotificationCriteria(final QueryManager qm, ViolationAnalysis violationAnalysis, final boolean analysisStateChange, final boolean suppressionChange) { + // TODO: Convert data loading to raw SQL to avoid loading unneeded data and excessive queries. + // See #analyzeNotificationCriteria(QueryManager, PolicyViolation) for an example. if (analysisStateChange || suppressionChange) { final NotificationGroup notificationGroup; notificationGroup = NotificationGroup.PROJECT_AUDIT_CHANGE; @@ -220,6 +224,10 @@ public static void analyzeNotificationCriteria(final QueryManager qm, ViolationA } public static void analyzeNotificationCriteria(final QueryManager qm, final PolicyViolation policyViolation) { + analyzeNotificationCriteria(qm, policyViolation.getId()); + } + + public static void analyzeNotificationCriteria(final QueryManager qm, final Long violationId) { final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ SELECT "PV"."UUID" AS "violationUuid", @@ -272,7 +280,7 @@ public static void analyzeNotificationCriteria(final QueryManager qm, final Poli WHERE "PV"."ID" = ? """); - query.setParameters(policyViolation.getId()); + query.setParameters(violationId); final PolicyViolationNotificationProjection projection; try { projection = query.executeResultUnique(PolicyViolationNotificationProjection.class); @@ -428,6 +436,8 @@ private static void sendNotificationToKafka(UUID projectUuid, Notification notif } public static Notification createProjectVulnerabilityAnalysisCompleteNotification(VulnerabilityScan vulnScan, UUID token, ProjectVulnAnalysisStatus status) { + // TODO: Convert data loading to raw SQL to avoid loading unneeded data and excessive queries. + // See #analyzeNotificationCriteria(QueryManager, PolicyViolation) for an example. try (QueryManager qm = new QueryManager()) { Project project = qm.getObjectByUuid(Project.class, vulnScan.getTargetIdentifier()); if (project == null) { From eb8c4588aec6983be605e7d63577b644f4266631 Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 19 Sep 2023 23:17:43 +0200 Subject: [PATCH 22/46] Include strings library in CEL policy environment Signed-off-by: nscuro --- .../org/dependencytrack/policy/cel/CelPolicyScriptHost.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java index be5668a94..7a061cd3e 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -19,6 +19,7 @@ import org.projectnessie.cel.common.Location; import org.projectnessie.cel.common.types.Err.ErrException; import org.projectnessie.cel.common.types.pb.ProtoTypeRegistry; +import org.projectnessie.cel.extension.StringsLib; import org.projectnessie.cel.tools.ScriptCreateException; import java.util.Collections; @@ -49,6 +50,7 @@ public class CelPolicyScriptHost { ProtoTypeRegistry.newRegistry(), List.of( Library.StdLib(), + Library.Lib(new StringsLib()), Library.Lib(new CelPolicyLibrary()) ) ); From 3d1a4362cd466d62fa9aa7af22bbb8f0b2c4b9f6 Mon Sep 17 00:00:00 2001 From: nscuro Date: Wed, 20 Sep 2023 15:59:48 +0200 Subject: [PATCH 23/46] Cleanup; Support project properties, tags, and vulnerability aliases Signed-off-by: nscuro --- .../dependencytrack/model/Vulnerability.java | 25 - .../policy/cel/CelPolicyEngine.java | 1132 ++++------------- .../policy/cel/CelPolicyQueryManager.java | 586 +++++++++ .../policy/cel/mapping/ProjectProjection.java | 5 + .../mapping/ProjectPropertyProjection.java | 17 + .../cel/mapping/VulnerabilityProjection.java | 2 + .../policy/cel/CelPolicyEngineTest.java | 16 +- .../cel/mapping/FieldMappingUtilTest.java | 5 + 8 files changed, 853 insertions(+), 935 deletions(-) create mode 100644 src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java create mode 100644 src/main/java/org/dependencytrack/policy/cel/mapping/ProjectPropertyProjection.java diff --git a/src/main/java/org/dependencytrack/model/Vulnerability.java b/src/main/java/org/dependencytrack/model/Vulnerability.java index ae4eeeb79..4fedf1bc5 100644 --- a/src/main/java/org/dependencytrack/model/Vulnerability.java +++ b/src/main/java/org/dependencytrack/model/Vulnerability.java @@ -92,30 +92,6 @@ @Persistent(name = "cvssV2BaseScore"), @Persistent(name = "cvssV3BaseScore") }), - @FetchGroup(name = "POLICY", members = { - @Persistent(name = "uuid"), - @Persistent(name = "vulnId"), - @Persistent(name = "source"), - @Persistent(name = "cwes"), - @Persistent(name = "created"), - @Persistent(name = "published"), - @Persistent(name = "updated"), - @Persistent(name = "cvssV2BaseScore"), - @Persistent(name = "cvssV2ImpactSubScore"), - @Persistent(name = "cvssV2ExploitabilitySubScore"), - @Persistent(name = "cvssV2Vector"), - @Persistent(name = "cvssV3BaseScore"), - @Persistent(name = "cvssV3ImpactSubScore"), - @Persistent(name = "cvssV3ExploitabilitySubScore"), - @Persistent(name = "cvssV3Vector"), - @Persistent(name = "owaspRRLikelihoodScore"), - @Persistent(name = "owaspRRTechnicalImpactScore"), - @Persistent(name = "owaspRRBusinessImpactScore"), - @Persistent(name = "owaspRRVector"), - @Persistent(name = "severity"), - @Persistent(name = "epssScore"), - @Persistent(name = "epssPercentile") - }), @FetchGroup(name = "VULNERABLE_SOFTWARE", members = { @Persistent(name = "vulnerableSoftware") }) @@ -133,7 +109,6 @@ public enum FetchGroup { BASIC, COMPONENTS, METRICS_UPDATE, - POLICY, VULNERABLE_SOFTWARE, } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index f4ac34102..0339cbe4c 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -3,10 +3,10 @@ import alpine.common.logging.Logger; import alpine.common.metrics.Metrics; import com.fasterxml.jackson.core.JacksonException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; -import com.github.packageurl.PackageURL; import com.google.api.expr.v1alpha1.Type; import com.google.protobuf.util.Timestamps; import io.micrometer.core.instrument.Timer; @@ -15,15 +15,13 @@ import org.apache.commons.collections4.multimap.HashSetValuedHashMap; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; -import org.datanucleus.PropertyNames; import org.dependencytrack.model.Component; -import org.dependencytrack.model.LicenseGroup; import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.model.PolicyCondition.Subject; import org.dependencytrack.model.PolicyViolation; import org.dependencytrack.model.Project; -import org.dependencytrack.model.Tag; +import org.dependencytrack.model.VulnerabilityAlias; import org.dependencytrack.persistence.CollectionIntegerConverter; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder; @@ -38,46 +36,29 @@ import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.mapping.ComponentProjection; -import org.dependencytrack.policy.cel.mapping.ComponentsVulnerabilitiesProjection; -import org.dependencytrack.policy.cel.mapping.LicenseGroupProjection; import org.dependencytrack.policy.cel.mapping.LicenseProjection; -import org.dependencytrack.policy.cel.mapping.PolicyViolationProjection; import org.dependencytrack.policy.cel.mapping.ProjectProjection; +import org.dependencytrack.policy.cel.mapping.ProjectPropertyProjection; import org.dependencytrack.policy.cel.mapping.VulnerabilityProjection; import org.dependencytrack.util.NotificationUtil; -import org.dependencytrack.util.VulnerabilityUtil; -import org.hyades.proto.policy.v1.License; import org.hyades.proto.policy.v1.Vulnerability; import org.projectnessie.cel.tools.ScriptCreateException; import org.projectnessie.cel.tools.ScriptException; -import javax.jdo.Query; -import javax.jdo.Transaction; -import javax.jdo.datastore.JDOConnection; -import java.math.BigDecimal; -import java.sql.Array; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; -import static java.sql.Connection.TRANSACTION_READ_COMMITTED; import static java.util.Collections.emptyList; import static org.apache.commons.collections4.MultiMapUtils.emptyMultiValuedMap; import static org.apache.commons.lang3.StringUtils.trimToEmpty; @@ -87,12 +68,15 @@ import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT_PROPERTY; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY; -import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY_ALIAS; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_PROJECT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.VAR_VULNERABILITIES; -import static org.dependencytrack.policy.cel.mapping.FieldMappingUtil.getFieldMappings; +/** + * A policy engine powered by the Common Expression Language (CEL). + * + * @since 5.1.0 + */ public class CelPolicyEngine { private static final Logger LOGGER = Logger.getLogger(CelPolicyEngine.class); @@ -124,87 +108,71 @@ public CelPolicyEngine() { this.scriptHost = scriptHost; } + /** + * Evaluate {@link Policy}s for a {@link Project}. + * + * @param uuid The {@link UUID} of the {@link Project} + */ public void evaluateProject(final UUID uuid) { final Timer.Sample timerSample = Timer.start(); - try (final var qm = new QueryManager()) { - // Disable L1 cache. We don't need the results of INSERT or UPDATE operations to be present in the cache. - // We do potentially perform lots of INSERTS during violation reconciliation, maintenance of L1 cache - // will have a huge overhead we'd rather not have to deal with. - qm.getPersistenceManager().setProperty(PropertyNames.PROPERTY_CACHE_L1_TYPE, "none"); - qm.getPersistenceManager().setProperty(PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false"); - + try (final var qm = new QueryManager(); + final var celQm = new CelPolicyQueryManager(qm)) { // TODO: Should this entire procedure run in a single DB transaction? // Would be better for atomicity, but could block DB connections for prolonged // period of time for larger projects with many violations. final Project project = qm.getObjectByUuid(Project.class, uuid, List.of(Project.FetchGroup.IDENTIFIERS.name())); if (project == null) { - LOGGER.warn("Project with UUID %s does not exist".formatted(uuid)); + LOGGER.warn("Project with UUID %s does not exist; Skipping".formatted(uuid)); return; } - final List policies = getApplicablePolicies(qm, project); - if (policies.isEmpty()) { + LOGGER.debug("Compiling policy scripts for project %s".formatted(uuid)); + final List> conditionScriptPairs = getApplicableConditionScriptPairs(celQm, project); + if (conditionScriptPairs.isEmpty()) { LOGGER.info("No applicable policies found for project %s".formatted(uuid)); - reconcileViolations(qm, project.getId(), emptyMultiValuedMap()); + celQm.reconcileViolations(project.getId(), emptyMultiValuedMap()); return; } - LOGGER.info("Compiling policy scripts for project %s".formatted(uuid)); - final List> conditionScriptPairs = policies.stream() - .map(Policy::getPolicyConditions) - .flatMap(Collection::stream) - .map(this::buildConditionScriptSrc) - .filter(Objects::nonNull) - .map(this::compileConditionScript) - .toList(); - - LOGGER.info("Determining evaluation requirements for project %s and %d policy conditions" - .formatted(uuid, conditionScriptPairs.size())); - final MultiValuedMap requirements = conditionScriptPairs.stream() - .map(Pair::getRight) - .map(CelPolicyScript::getRequirements) - .reduce(new HashSetValuedHashMap<>(), (a, b) -> { - a.putAll(b); - return a; - }); - LOGGER.info("Requirements for project %s and %d policy conditions: %s" + final MultiValuedMap requirements = determineScriptRequirements(conditionScriptPairs); + LOGGER.debug("Requirements for project %s and %d policy conditions: %s" .formatted(uuid, conditionScriptPairs.size(), requirements)); final org.hyades.proto.policy.v1.Project protoProject; if (requirements.containsKey(TYPE_PROJECT)) { - protoProject = mapProject(fetchProject(qm, project.getId(), requirements.get(TYPE_PROJECT))); + protoProject = mapToProto(celQm.fetchProject(project.getId(), requirements.get(TYPE_PROJECT), requirements.get(TYPE_PROJECT_PROPERTY))); } else { protoProject = org.hyades.proto.policy.v1.Project.getDefaultInstance(); } // Preload components for the entire project, to avoid excessive queries. - final List components = fetchComponents(qm, project.getId(), requirements.get(TYPE_COMPONENT)); + final List components = celQm.fetchAllComponents(project.getId(), requirements.get(TYPE_COMPONENT)); // Preload licenses for the entire project, as chances are high that they will be used by multiple components. - final Map licenseById; + final Map licenseById; if (requirements.containsKey(TYPE_LICENSE)) { - licenseById = fetchLicenses(qm, project.getId(), requirements.get(TYPE_LICENSE), requirements.get(TYPE_LICENSE_GROUP)).stream() + licenseById = celQm.fetchAllLicenses(project.getId(), requirements.get(TYPE_LICENSE), requirements.get(TYPE_LICENSE_GROUP)).stream() .collect(Collectors.toMap( projection -> projection.id, - CelPolicyEngine::mapLicense + CelPolicyEngine::mapToProto )); } else { licenseById = Collections.emptyMap(); } // Preload vulnerabilities for the entire project, as chances are high that they will be used by multiple components. - final Map protoVulnById; + final Map protoVulnById; final Map> vulnIdsByComponentId; if (requirements.containsKey(TYPE_VULNERABILITY)) { - protoVulnById = fetchVulnerabilities(qm, project.getId(), requirements.get(TYPE_VULNERABILITY)).stream() + protoVulnById = celQm.fetchAllVulnerabilities(project.getId(), requirements.get(TYPE_VULNERABILITY)).stream() .collect(Collectors.toMap( projection -> projection.id, - CelPolicyEngine::mapVulnerability + CelPolicyEngine::mapToProto )); - vulnIdsByComponentId = fetchComponentsVulnerabilities(qm, project.getId()).stream() + vulnIdsByComponentId = celQm.fetchAllComponentsVulnerabilities(project.getId()).stream() .collect(Collectors.groupingBy( projection -> projection.componentId, Collectors.mapping(projection -> projection.vulnerabilityId, Collectors.toList()) @@ -217,73 +185,25 @@ public void evaluateProject(final UUID uuid) { // Evaluate all policy conditions against all components. final var conditionsViolated = new HashSetValuedHashMap(); for (final ComponentProjection component : components) { - final org.hyades.proto.policy.v1.Component protoComponent = mapComponent(component, licenseById); - final List protoVulns = vulnIdsByComponentId.getOrDefault(component.id, emptyList()).stream() - .map(protoVulnById::get) - .toList(); - - for (final Pair conditionScriptPair : conditionScriptPairs) { - final PolicyCondition condition = conditionScriptPair.getLeft(); - final CelPolicyScript script = conditionScriptPair.getRight(); - final Map scriptArgs = Map.of( - VAR_COMPONENT, protoComponent, - VAR_PROJECT, protoProject, - VAR_VULNERABILITIES, protoVulns - ); - - try { - if (script.execute(scriptArgs)) { - conditionsViolated.put(component.id, condition); - } - } catch (ScriptException e) { - // TODO: Should we really fail the entire run for ALL components, - // if only one condition failed to evaluate? - throw new RuntimeException("Failed to evaluate script", e); - } - } + final org.hyades.proto.policy.v1.Component protoComponent = mapToProto(component, licenseById); + final List protoVulns = + vulnIdsByComponentId.getOrDefault(component.id, emptyList()).stream() + .map(protoVulnById::get) + .toList(); + + conditionsViolated.putAll(component.id, evaluateConditions(conditionScriptPairs, Map.of( + VAR_COMPONENT, protoComponent, + VAR_PROJECT, protoProject, + VAR_VULNERABILITIES, protoVulns + ))); } - // Evaluate policy operators. - // Determines whether violations must be reported in the first place. - // i.e. if not all conditions matched for a policy with ALL operator, - // then no violations are to be raised. final var violationsByComponentId = new ArrayListValuedHashMap(); for (final long componentId : conditionsViolated.keySet()) { - final Map> violatedConditionsByPolicy = conditionsViolated.get(componentId).stream() - .collect(Collectors.groupingBy(PolicyCondition::getPolicy)); - - final List violations = violatedConditionsByPolicy.entrySet().stream() - .flatMap(policyAndViolatedConditions -> { - final Policy policy = policyAndViolatedConditions.getKey(); - final List violatedConditions = policyAndViolatedConditions.getValue(); - - if ((policy.getOperator() == Policy.Operator.ANY && !violatedConditions.isEmpty()) - || (policy.getOperator() == Policy.Operator.ALL && violatedConditions.size() == policy.getPolicyConditions().size())) { - // TODO: Only a single violation should be raised, and instead multiple matched conditions - // should be associated with it. Keeping the existing behavior in order to avoid having to - // touch too much persistence and REST API code. - return violatedConditions.stream() - .map(condition -> { - final var violation = new PolicyViolation(); - violation.setType(condition.getViolationType()); - // Note: violation.setComponent is intentionally omitted here, - // because the component must be an object attached to the persistence - // context. We don't have that at this point, we'll add it later. - violation.setPolicyCondition(condition); - violation.setTimestamp(new Date()); - return violation; - }); - } - - return Stream.empty(); - }) - .filter(Objects::nonNull) - .toList(); - - violationsByComponentId.putAll(componentId, violations); + violationsByComponentId.putAll(componentId, evaluatePolicyOperators(conditionsViolated.get(componentId))); } - final List newViolationIds = reconcileViolations(qm, project.getId(), violationsByComponentId); + final List newViolationIds = celQm.reconcileViolations(project.getId(), violationsByComponentId); LOGGER.info("Identified %d new violations for project %s".formatted(newViolationIds.size(), uuid)); for (final Long newViolationId : newViolationIds) { @@ -299,225 +219,108 @@ public void evaluateProject(final UUID uuid) { } } - // TODO: Just here to satisfy contract with legacy PolicyEngine; Remove after testing - public void evaluate(final List components) { - components.stream().map(Component::getUuid).forEach(this::evaluateComponent); - } - - public void evaluateComponent(final UUID componentUuid) { + public void evaluateComponent(final UUID uuid) { final Timer.Sample timerSample = Timer.start(); - try (final var qm = new QueryManager()) { - final Component component = qm.getObjectByUuid(Component.class, componentUuid); + try (final var qm = new QueryManager(); + final var celQm = new CelPolicyQueryManager(qm)) { + final Component component = qm.getObjectByUuid(Component.class, uuid); if (component == null) { - LOGGER.warn("Component with UUID %s does not exist".formatted(componentUuid)); + LOGGER.warn("Component with UUID %s does not exist".formatted(uuid)); return; } - final List policies = getApplicablePolicies(qm, component.getProject()); - if (policies.isEmpty()) { - // With no applicable policies, there's no way to resolve violations. - // As a compensation, simply delete all violations associated with the component. - LOGGER.info("No applicable policies found for component %s".formatted(componentUuid)); - reconcileViolations(qm, component, emptyList()); + LOGGER.debug("Compiling policy scripts for project %s".formatted(uuid)); + final List> conditionScriptPairs = getApplicableConditionScriptPairs(celQm, component.getProject()); + if (conditionScriptPairs.isEmpty()) { + LOGGER.info("No applicable policies found for project %s".formatted(uuid)); + // reconcileViolations(qm, project.getId(), emptyMultiValuedMap()); return; } - // Pre-compile the CEL scripts for all conditions of all applicable policies. - // Compiled scripts are cached in-memory by CelPolicyScriptHost, so if the same script - // is encountered for multiple components (possibly concurrently), the compilation is - // a one-time effort. - LOGGER.info("Compiling policy scripts for component %s".formatted(componentUuid)); - final List> conditionScriptPairs = policies.stream() - .map(Policy::getPolicyConditions) - .flatMap(Collection::stream) - .map(this::buildConditionScriptSrc) - .filter(Objects::nonNull) - .map(this::compileConditionScript) - .toList(); - - // Check what kind of data we need to evaluate all policy conditions. - // - // Some conditions will be very simple and won't require us to load additional data (e.g. "component PURL matches 'XYZ'"), - // whereas other conditions can span across multiple models, forcing us to load more data - // (e.g. "project has tag 'public-facing' and component has a vulnerability with severity 'critical'"). - // - // What we want to avoid is loading data we don't need, and loading it multiple times. - // Instead, only load what's really needed, and only do so once. LOGGER.info("Determining evaluation requirements for component %s and %d policy conditions" - .formatted(componentUuid, conditionScriptPairs.size())); - final MultiValuedMap requirements = conditionScriptPairs.stream() - .map(Pair::getRight) - .map(CelPolicyScript::getRequirements) - .reduce(new HashSetValuedHashMap<>(), (a, b) -> { - a.putAll(b); - return a; - }); - - // Prepare the script arguments according to the requirements gathered before. - LOGGER.info("Building script arguments for component %s and requirements %s" - .formatted(componentUuid, requirements)); - final Map scriptArgs = Map.of( - VAR_COMPONENT, mapComponent(qm, component, requirements), - VAR_PROJECT, mapProject(component.getProject(), requirements), - VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) - ); + .formatted(uuid, conditionScriptPairs.size())); + final MultiValuedMap requirements = determineScriptRequirements(conditionScriptPairs); LOGGER.info("Evaluating component %s against %d applicable policy conditions" - .formatted(componentUuid, conditionScriptPairs.size())); - final var conditionsViolated = new HashSet(); - for (final Pair conditionScriptPair : conditionScriptPairs) { - final PolicyCondition condition = conditionScriptPair.getLeft(); - final CelPolicyScript script = conditionScriptPair.getRight(); - LOGGER.info("Executing script for policy condition %s with arguments: %s" - .formatted(condition.getUuid(), scriptArgs)); - - try { - if (script.execute(scriptArgs)) { - conditionsViolated.add(condition); - } - } catch (ScriptException e) { - throw new RuntimeException("Failed to evaluate script", e); - } - } + .formatted(uuid, conditionScriptPairs.size())); + final List conditionsViolated = evaluateConditions(conditionScriptPairs, Map.of( + // VAR_COMPONENT, mapComponent(qm, component, requirements), + // VAR_PROJECT, mapProject(component.getProject(), requirements), + // VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) + )); // Group the detected condition violations by policy. Necessary to be able to evaluate // each policy's operator (ANY, ALL). LOGGER.info("Detected violation of %d policy conditions for component %s; Evaluating policy operators" - .formatted(conditionsViolated.size(), componentUuid)); - final Map> violatedConditionsByPolicy = conditionsViolated.stream() - .collect(Collectors.groupingBy(PolicyCondition::getPolicy)); - - // Create policy violations, but only do so when the detected condition violations - // match the configured policy operator. When the operator is ALL, and not all conditions - // of the policy were violated, we don't want to create any violations. - final List violations = violatedConditionsByPolicy.entrySet().stream() - .flatMap(policyAndViolatedConditions -> { - final Policy policy = policyAndViolatedConditions.getKey(); - final List violatedConditions = policyAndViolatedConditions.getValue(); - - if ((policy.getOperator() == Policy.Operator.ANY && !violatedConditions.isEmpty()) - || (policy.getOperator() == Policy.Operator.ALL && violatedConditions.size() == policy.getPolicyConditions().size())) { - // TODO: Only a single violation should be raised, and instead multiple matched conditions - // should be associated with it. Keeping the existing behavior in order to avoid having to - // touch too much persistence and REST API code. - return violatedConditions.stream() - .map(condition -> { - final var violation = new PolicyViolation(); - violation.setComponent(component); - violation.setType(condition.getViolationType()); - violation.setPolicyCondition(condition); - violation.setTimestamp(new Date()); - return violation; - }); - } - - return Stream.empty(); - }) - .filter(Objects::nonNull) - .toList(); + .formatted(conditionsViolated.size(), uuid)); + final List violations = evaluatePolicyOperators(conditionsViolated); // Reconcile the violations created above with what's already in the database. // Create new records if necessary, and delete records that are no longer current. - final List newViolations = reconcileViolations(qm, component, violations); + // final List newViolations = reconcileViolations(qm, component, violations); // Notify users about any new violations. - for (final PolicyViolation newViolation : newViolations) { - NotificationUtil.analyzeNotificationCriteria(qm, newViolation); - } + // for (final PolicyViolation newViolation : newViolations) { + // NotificationUtil.analyzeNotificationCriteria(qm, newViolation); + // } } finally { - timerSample.stop(Timer + final long durationNs = timerSample.stop(Timer .builder("dtrack_policy_eval") .tag("target", "component") .register(Metrics.getRegistry())); + LOGGER.info("Evaluation of component %s completed in %s" + .formatted(uuid, Duration.ofNanos(durationNs))); } - - LOGGER.info("Policy evaluation completed for component %s".formatted(componentUuid)); } - // TODO: Move to PolicyQueryManager - private static List getApplicablePolicies(final QueryManager qm, final Project project) { - var filter = """ - (this.projects.isEmpty() && this.tags.isEmpty()) - || (this.projects.contains(:project) - """; - var params = new HashMap(); - params.put("project", project); - - // To compensate for missing support for recursion of Common Table Expressions (CTEs) - // in JDO, we have to fetch the UUIDs of all parent projects upfront. Otherwise, we'll - // not be able to evaluate whether the policy is inherited from parent projects. - var variables = ""; - final List parentUuids = getParents(qm, project); - if (!parentUuids.isEmpty()) { - filter += """ - || (this.includeChildren - && this.projects.contains(parentVar) - && :parentUuids.contains(parentVar.uuid)) - """; - variables += "org.dependencytrack.model.Project parentVar"; - params.put("parentUuids", parentUuids); - } - filter += ")"; - - // DataNucleus generates an invalid SQL query when using the idiomatic solution. - // The following works, but it's ugly and likely doesn't perform well if the project - // has many tags. Worth trying the idiomatic way again once DN has been updated to > 6.0.4. - // - // filter += " || (this.tags.contains(commonTag) && :project.tags.contains(commonTag))"; - // variables += "org.dependencytrack.model.Tag commonTag"; - if (project.getTags() != null && !project.getTags().isEmpty()) { - filter += " || ("; - for (int i = 0; i < project.getTags().size(); i++) { - filter += "this.tags.contains(:tag" + i + ")"; - params.put("tag" + i, project.getTags().get(i)); - if (i < (project.getTags().size() - 1)) { - filter += " || "; - } - } - filter += ")"; - } - final List policies; - final Query query = qm.getPersistenceManager().newQuery(Policy.class); - try { - query.setFilter(filter); - query.setNamedParameters(params); - if (!variables.isEmpty()) { - query.declareVariables(variables); - } - policies = List.copyOf(query.executeList()); - } finally { - query.closeAll(); + /** + * Pre-compile the CEL scripts for all conditions of all applicable policies. + * Compiled scripts are cached in-memory by CelPolicyScriptHost, so if the same script + * is encountered for multiple components (possibly concurrently), the compilation is + * a one-time effort. + * + * @param celQm The {@link CelPolicyQueryManager} instance to use + * @param project The {@link Project} to get applicable conditions for + * @return {@link Pair}s of {@link PolicyCondition}s and {@link CelPolicyScript}s + */ + private List> getApplicableConditionScriptPairs(final CelPolicyQueryManager celQm, final Project project) { + final List policies = celQm.getApplicablePolicies(project); + if (policies.isEmpty()) { + return emptyList(); } - return policies; - } - - // TODO: Move to ProjectQueryManager - private static List getParents(final QueryManager qm, final Project project) { - return getParents(qm, project.getUuid(), new ArrayList<>()); + return policies.stream() + .map(Policy::getPolicyConditions) + .flatMap(Collection::stream) + .map(this::buildConditionScriptSrc) + .filter(Objects::nonNull) + .map(this::compileConditionScript) + .toList(); } - // TODO: Move to ProjectQueryManager - private static List getParents(final QueryManager qm, final UUID uuid, final List parents) { - final UUID parentUuid; - final Query query = qm.getPersistenceManager().newQuery(Project.class); - try { - query.setFilter("uuid == :uuid && parent != null"); - query.setParameters(uuid); - query.setResult("parent.uuid"); - parentUuid = query.executeResultUnique(UUID.class); - } finally { - query.closeAll(); - } - - if (parentUuid == null) { - return parents; - } - - parents.add(parentUuid); - return getParents(qm, parentUuid, parents); + /** + * Check what kind of data we need to evaluate all policy conditions. + *

+ * Some conditions will be very simple and won't require us to load additional data (e.g. "component PURL matches 'XYZ'"), + * whereas other conditions can span across multiple models, forcing us to load more data + * (e.g. "project has tag 'public-facing' and component has a vulnerability with severity 'critical'"). + *

+ * What we want to avoid is loading data we don't need, and loading it multiple times. + * Instead, only load what's really needed, and only do so once. + * + * @param conditionScriptPairs {@link Pair}s of {@link PolicyCondition}s and corresponding {@link CelPolicyScript}s + * @return A {@link MultiValuedMap} containing all fields accessed on any {@link Type}, across all {@link CelPolicyScript}s + */ + private static MultiValuedMap determineScriptRequirements(final Collection> conditionScriptPairs) { + return conditionScriptPairs.stream() + .map(Pair::getRight) + .map(CelPolicyScript::getRequirements) + .reduce(new HashSetValuedHashMap<>(), (lhs, rhs) -> { + lhs.putAll(rhs); + return lhs; + }); } private Pair buildConditionScriptSrc(final PolicyCondition policyCondition) { @@ -547,70 +350,119 @@ private Pair compileConditionScript(final Pair return Pair.of(conditionScriptSrcPair.getLeft(), script); } - private static ProjectProjection fetchProject(final QueryManager qm, final long projectId, final Collection protoFieldNames) { - final String sqlSelectColumns = getFieldMappings(ProjectProjection.class).stream() - .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) - .map(mapping -> "\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) - .collect(Collectors.joining(", ")); + private static List evaluateConditions(final Collection> conditionScriptPairs, + final Map scriptArguments) { + final var conditionsViolated = new ArrayList(); - final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ - SELECT %s FROM "PROJECT" WHERE "ID" = ? - """.formatted(sqlSelectColumns)); - query.setParameters(projectId); - try { - return query.executeResultUnique(ProjectProjection.class); - } finally { - query.closeAll(); + for (final Pair conditionScriptPair : conditionScriptPairs) { + final PolicyCondition condition = conditionScriptPair.getLeft(); + final CelPolicyScript script = conditionScriptPair.getRight(); + + try { + if (script.execute(scriptArguments)) { + conditionsViolated.add(condition); + } + } catch (ScriptException e) { + // TODO: Should we really fail the entire run for ALL components, + // if only one condition failed to evaluate? + throw new RuntimeException("Failed to evaluate script", e); + } } + + return conditionsViolated; } - private static org.hyades.proto.policy.v1.Project mapProject(final ProjectProjection projection) { + private static List evaluatePolicyOperators(final Collection conditionsViolated) { + final Map> violatedConditionsByPolicy = conditionsViolated.stream() + .collect(Collectors.groupingBy(PolicyCondition::getPolicy)); + + return violatedConditionsByPolicy.entrySet().stream() + .flatMap(policyAndViolatedConditions -> { + final Policy policy = policyAndViolatedConditions.getKey(); + final List violatedConditions = policyAndViolatedConditions.getValue(); + + if ((policy.getOperator() == Policy.Operator.ANY && !violatedConditions.isEmpty()) + || (policy.getOperator() == Policy.Operator.ALL && violatedConditions.size() == policy.getPolicyConditions().size())) { + // TODO: Only a single violation should be raised, and instead multiple matched conditions + // should be associated with it. Keeping the existing behavior in order to avoid having to + // touch too much persistence and REST API code. + return violatedConditions.stream() + .map(condition -> { + final var violation = new PolicyViolation(); + violation.setType(condition.getViolationType()); + // Note: violation.setComponent is intentionally omitted here, + // because the component must be an object attached to the persistence + // context. We don't have that at this point, we'll add it later. + violation.setPolicyCondition(condition); + violation.setTimestamp(new Date()); + return violation; + }); + } + + return Stream.empty(); + }) + .filter(Objects::nonNull) + .toList(); + } + + private static org.hyades.proto.policy.v1.Project mapToProto(final ProjectProjection projection) { final org.hyades.proto.policy.v1.Project.Builder builder = org.hyades.proto.policy.v1.Project.newBuilder() .setUuid(trimToEmpty(projection.uuid)) .setGroup(trimToEmpty(projection.group)) .setName(trimToEmpty(projection.name)) .setVersion(trimToEmpty(projection.version)) - // .addAllTags(project.getTags().stream().map(Tag::getName).toList()) .setCpe(trimToEmpty(projection.cpe)) .setPurl(trimToEmpty(projection.purl)) .setSwidTagId(trimToEmpty(projection.swidTagId)); Optional.ofNullable(projection.lastBomImport).map(Timestamps::fromDate).ifPresent(builder::setLastBomImport); - return builder.build(); - } - private static List fetchComponents(final QueryManager qm, final long projectId, final Collection protoFieldNames) { - final String sqlSelectColumns = Stream.concat( - Stream.of(ComponentProjection.ID_FIELD_MAPPING), - getFieldMappings(ComponentProjection.class).stream() - .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) - ) - .map(mapping -> "\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) - .collect(Collectors.joining(", ")); - - final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ - SELECT %s FROM "COMPONENT" WHERE "PROJECT_ID" = ? - """.formatted(sqlSelectColumns)); - query.setParameters(projectId); - try { - return List.copyOf(query.executeResultList(ComponentProjection.class)); - } finally { - query.closeAll(); + if (projection.propertiesJson != null) { + try { + final List properties = + OBJECT_MAPPER.readValue(projection.propertiesJson, new TypeReference<>() { + }); + for (final ProjectPropertyProjection property : properties) { + builder.addProperties(org.hyades.proto.policy.v1.Project.Property.newBuilder() + .setGroup(trimToEmpty(property.group)) + .setName(trimToEmpty(property.name)) + .setValue(trimToEmpty(property.value)) + .setType(trimToEmpty(property.type)) + .build()); + } + } catch (JacksonException e) { + LOGGER.warn("Failed to parse properties from %s for project %s" + .formatted(projection.propertiesJson, projection.id), e); + } } + + if (projection.tagsJson != null) { + try { + final List tags = OBJECT_MAPPER.readValue(projection.tagsJson, new TypeReference<>() { + }); + builder.addAllTags(tags); + } catch (JacksonException e) { + LOGGER.warn("Failed to parse tags from %s for project %s" + .formatted(projection.tagsJson, projection.id), e); + } + } + + return builder.build(); } - private static org.hyades.proto.policy.v1.Component mapComponent(final ComponentProjection projection, - final Map protoLicenseById) { + private static org.hyades.proto.policy.v1.Component mapToProto(final ComponentProjection projection, + final Map protoLicenseById) { final org.hyades.proto.policy.v1.Component.Builder componentBuilder = org.hyades.proto.policy.v1.Component.newBuilder() .setUuid(trimToEmpty(projection.uuid)) .setGroup(trimToEmpty(projection.group)) .setName(trimToEmpty(projection.name)) - .setVersion(trimToEmpty(projection.name)) + .setVersion(trimToEmpty(projection.version)) .setClassifier(trimToEmpty(projection.classifier)) .setCpe(trimToEmpty(projection.cpe)) .setPurl(trimToEmpty(projection.purl)) .setSwidTagId(trimToEmpty(projection.swidTagId)) .setIsInternal(Optional.ofNullable(projection.internal).orElse(false)) + .setLicenseName(trimToEmpty(projection.licenseName)) .setMd5(trimToEmpty(projection.md5)) .setSha1(trimToEmpty(projection.sha1)) .setSha256(trimToEmpty(projection.sha256)) @@ -625,7 +477,7 @@ private static org.hyades.proto.policy.v1.Component mapComponent(final Component .setBlake3(trimToEmpty(projection.blake3)); if (projection.resolvedLicenseId != null && projection.resolvedLicenseId > 0) { - final License protoLicense = protoLicenseById.get(projection.resolvedLicenseId); + final org.hyades.proto.policy.v1.License protoLicense = protoLicenseById.get(projection.resolvedLicenseId); if (protoLicense != null) { componentBuilder.setResolvedLicense(protoLicenseById.get(projection.resolvedLicenseId)); } else { @@ -637,121 +489,12 @@ private static org.hyades.proto.policy.v1.Component mapComponent(final Component return componentBuilder.build(); } - private static List fetchComponentsVulnerabilities(final QueryManager qm, final long projectId) { - final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ - SELECT - "CV"."COMPONENT_ID" AS "componentId", - "CV"."VULNERABILITY_ID" AS "vulnerabilityId" - FROM - "COMPONENTS_VULNERABILITIES" AS "CV" - INNER JOIN - "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" - WHERE - "C"."PROJECT_ID" = ? - """); - query.setParameters(projectId); - try { - return List.copyOf(query.executeResultList(ComponentsVulnerabilitiesProjection.class)); - } finally { - query.closeAll(); - } - } - - private static List fetchLicenses(final QueryManager qm, final long projectId, - final Collection licenseProtoFieldNames, - final Collection licenseGroupProtoFieldNames) { - final String licenseSqlSelectColumns = Stream.concat( - Stream.of(LicenseProjection.ID_FIELD_MAPPING), - getFieldMappings(LicenseProjection.class).stream() - .filter(mapping -> licenseProtoFieldNames.contains(mapping.protoFieldName())) - ) - .map(mapping -> "\"L\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) - .collect(Collectors.joining(", ")); - - // If fetching license groups is not necessary, we can just query for licenses and be done with it. - if (!licenseProtoFieldNames.contains("groups")) { - final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ - SELECT DISTINCT - %s - FROM - "LICENSE" AS "L" - INNER JOIN - "COMPONENT" AS "C" ON "C"."LICENSE_ID" = "L"."ID" - WHERE - "C"."PROJECT_ID" = ? - """.formatted(licenseSqlSelectColumns)); - query.setParameters(projectId); - try { - return List.copyOf(query.executeResultList(LicenseProjection.class)); - } finally { - query.closeAll(); - } - } - - // If groups are required, include them in the license query in order to avoid the 1+N problem. - // Licenses may or may not be assigned to a group. Licenses can be in multiple groups. - // - // Using a simple LEFT JOIN would result in duplicate license data being fetched, e.g.: - // - // | "L"."ID" | "L"."NAME" | "LG"."NAME" | - // | :------- | :--------- | :---------- | - // | 1 | foo | groupA | - // | 1 | foo | groupB | - // | 1 | foo | groupC | - // | 2 | bar | NULL | - // - // To avoid this, we instead aggregate license group fields for each license, and return them as JSON. - // The reason for choosing JSON over native arrays, is that DataNucleus can't deal with arrays cleanly. - // - // | "L"."ID" | "L"."NAME" | "licenseGroupsJson" | - // | :------- | :--------- | :------------------------------------------------------ | - // | 1 | foo | [{"name":"groupA"},{"name":"groupB"},{"name":"groupC"}] | - // | 2 | bar | [] | - - final String licenseSqlGroupByColumns = Stream.concat( - Stream.of(LicenseProjection.ID_FIELD_MAPPING), - getFieldMappings(LicenseProjection.class).stream() - .filter(mapping -> licenseProtoFieldNames.contains(mapping.protoFieldName())) - ) - .map(mapping -> "\"L\".\"%s\"".formatted(mapping.sqlColumnName())) - .collect(Collectors.joining(", ")); - - final String licenseGroupSqlSelectColumns = getFieldMappings(LicenseGroupProjection.class).stream() - .filter(mapping -> licenseGroupProtoFieldNames.contains(mapping.protoFieldName())) - .map(mapping -> "'%s', \"LG\".\"%s\"".formatted(mapping.javaFieldName(), mapping.sqlColumnName())) - .collect(Collectors.joining(", ")); - - final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ - SELECT DISTINCT - "L"."ID" AS "id", - %s, - CAST(JSONB_AGG(DISTINCT JSONB_BUILD_OBJECT(%s)) AS TEXT) AS "licenseGroupsJson" - FROM - "LICENSE" AS "L" - INNER JOIN - "COMPONENT" AS "C" ON "C"."LICENSE_ID" = "L"."ID" - LEFT JOIN - "LICENSEGROUP_LICENSE" AS "LGL" ON "LGL"."LICENSE_ID" = "L"."ID" - LEFT JOIN - "LICENSEGROUP" AS "LG" ON "LG"."ID" = "LGL"."LICENSEGROUP_ID" - WHERE - "C"."PROJECT_ID" = ? - GROUP BY - %s - """.formatted(licenseSqlSelectColumns, licenseGroupSqlSelectColumns, licenseSqlGroupByColumns)); - query.setParameters(projectId); - try { - return List.copyOf(query.executeResultList(LicenseProjection.class)); - } finally { - query.closeAll(); - } - } - - private static License mapLicense(final LicenseProjection projection) { - final License.Builder licenseBuilder = License.newBuilder() - .setUuid(trimToEmpty(projection.uuid)) - .setId(trimToEmpty(projection.licenseId)) - .setName(trimToEmpty(projection.name)); + private static org.hyades.proto.policy.v1.License mapToProto(final LicenseProjection projection) { + final org.hyades.proto.policy.v1.License.Builder licenseBuilder = + org.hyades.proto.policy.v1.License.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setId(trimToEmpty(projection.licenseId)) + .setName(trimToEmpty(projection.name)); Optional.ofNullable(projection.isOsiApproved).ifPresent(licenseBuilder::setIsOsiApproved); Optional.ofNullable(projection.isFsfLibre).ifPresent(licenseBuilder::setIsFsfLibre); Optional.ofNullable(projection.isDeprecatedId).ifPresent(licenseBuilder::setIsDeprecatedId); @@ -761,7 +504,7 @@ private static License mapLicense(final LicenseProjection projection) { try { final ArrayNode groupsArray = OBJECT_MAPPER.readValue(projection.licenseGroupsJson, ArrayNode.class); for (final JsonNode groupNode : groupsArray) { - licenseBuilder.addGroups(License.Group.newBuilder() + licenseBuilder.addGroups(org.hyades.proto.policy.v1.License.Group.newBuilder() .setUuid(Optional.ofNullable(groupNode.get("uuid")).map(JsonNode::asText).orElse("")) .setName(Optional.ofNullable(groupNode.get("name")).map(JsonNode::asText).orElse("")) .build()); @@ -774,74 +517,19 @@ private static License mapLicense(final LicenseProjection projection) { return licenseBuilder.build(); } - private static List fetchVulnerabilities(final QueryManager qm, final long projectId, final Collection protoFieldNames) { - final String sqlSelectColumns = getFieldMappings(VulnerabilityProjection.class).stream() - .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) - .map(mapping -> "\"V\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) - .collect(Collectors.joining(", ")); - - // TODO: Aliases could be fetched in the same query, using a JSONB aggregate. - // SELECT DISTINCT - // "V"."ID" AS "id", - // "V"."VULNID" AS "vulnId", - // "V"."SOURCE" AS "source", - // (SELECT - // JSONB_AGG(DISTINCT JSONB_STRIP_NULLS(JSONB_BUILD_OBJECT( - // 'cveId', "VA"."CVE_ID", - // 'ghsaId', "VA"."GHSA_ID", - // 'gsdId', "VA"."GSD_ID", - // 'internalId', "VA"."INTERNAL_ID", - // 'osvId', "VA"."OSV_ID", - // 'sonatypeId', "VA"."SONATYPE_ID", - // 'snykId', "VA"."SNYK_ID", - // 'vulnDbId', "VA"."VULNDB_ID" - // )))::TEXT - // FROM - // "VULNERABILITYALIAS" AS "VA" - // WHERE - // ("V"."SOURCE" = 'NVD' AND "VA"."CVE_ID" = "V"."VULNID") - // OR ("V"."SOURCE" = 'SNYK' AND "VA"."SNYK_ID" = "V"."VULNID") - // -- OR ... - // ) AS "aliasesJson" - // FROM - // "VULNERABILITY" AS "V" - // INNER JOIN - // "COMPONENTS_VULNERABILITIES" AS "CV" ON "CV"."VULNERABILITY_ID" = "V"."ID" - // INNER JOIN - // "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" - // WHERE - // "C"."PROJECT_ID" = ?; - - final Query query = qm.getPersistenceManager().newQuery(Query.SQL, """ - SELECT DISTINCT - "V"."ID" AS "id", - %s - FROM - "VULNERABILITY" AS "V" - INNER JOIN - "COMPONENTS_VULNERABILITIES" AS "CV" ON "CV"."VULNERABILITY_ID" = "V"."ID" - INNER JOIN - "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" - WHERE - "C"."PROJECT_ID" = ? - """.formatted(sqlSelectColumns)); - query.setParameters(projectId); - try { - return List.copyOf(query.executeResultList(VulnerabilityProjection.class)); - } finally { - query.closeAll(); - } - } + private static final TypeReference> VULNERABILITY_ALIASES_TYPE_REF = new TypeReference<>() { + }; - private static Vulnerability mapVulnerability(final VulnerabilityProjection projection) { - final Vulnerability.Builder builder = Vulnerability.newBuilder() - .setUuid(trimToEmpty(projection.uuid)) - .setId(trimToEmpty(projection.vulnId)) - .setSource(trimToEmpty(projection.source)) - .setCvssv2Vector(trimToEmpty(projection.cvssV2Vector)) - .setCvssv3Vector(trimToEmpty(projection.cvssV3Vector)) - .setOwaspRrVector(trimToEmpty(projection.owaspRrVector)) - .setSeverity(trimToEmpty(projection.severity)); + private static org.hyades.proto.policy.v1.Vulnerability mapToProto(final VulnerabilityProjection projection) { + final org.hyades.proto.policy.v1.Vulnerability.Builder builder = + org.hyades.proto.policy.v1.Vulnerability.newBuilder() + .setUuid(trimToEmpty(projection.uuid)) + .setId(trimToEmpty(projection.vulnId)) + .setSource(trimToEmpty(projection.source)) + .setCvssv2Vector(trimToEmpty(projection.cvssV2Vector)) + .setCvssv3Vector(trimToEmpty(projection.cvssV3Vector)) + .setOwaspRrVector(trimToEmpty(projection.owaspRrVector)) + .setSeverity(trimToEmpty(projection.severity)); Optional.ofNullable(projection.cvssV2BaseScore).ifPresent(builder::setCvssv2BaseScore); Optional.ofNullable(projection.cvssV2ImpactSubScore).ifPresent(builder::setCvssv2ImpactSubscore); Optional.ofNullable(projection.cvssV2ExploitabilitySubScore).ifPresent(builder::setCvssv2ExploitabilitySubscore); @@ -861,382 +549,28 @@ private static Vulnerability mapVulnerability(final VulnerabilityProjection proj .filter(Objects::nonNull) .map(new CollectionIntegerConverter()::convertToAttribute) .ifPresent(builder::addAllCwes); - return builder.build(); - } - private static org.hyades.proto.policy.v1.Component mapComponent(final QueryManager qm, - final Component component, - final MultiValuedMap requirements) { - // TODO: Load only required fields - final org.hyades.proto.policy.v1.Component.Builder builder = - org.hyades.proto.policy.v1.Component.newBuilder() - .setUuid(Optional.ofNullable(component.getUuid()).map(UUID::toString).orElse("")) - .setGroup(trimToEmpty(component.getGroup())) - .setName(trimToEmpty(component.getName())) - .setVersion(trimToEmpty(component.getVersion())) - .setClassifier(Optional.ofNullable(component.getClassifier()).map(Enum::name).orElse("")) - .setCpe(trimToEmpty(component.getCpe())) - .setPurl(Optional.ofNullable(component.getPurl()).map(PackageURL::canonicalize).orElse("")) - .setSwidTagId(trimToEmpty(component.getSwidTagId())) - .setIsInternal(component.isInternal()) - .setMd5(trimToEmpty(component.getMd5())) - .setSha1(trimToEmpty(component.getSha1())) - .setSha256(trimToEmpty(component.getSha256())) - .setSha384(trimToEmpty(component.getSha384())) - .setSha512(trimToEmpty(component.getSha512())) - .setSha3256(trimToEmpty(component.getSha3_256())) - .setSha3384(trimToEmpty(component.getSha3_384())) - .setSha3512(trimToEmpty(component.getSha3_512())) - .setBlake2B256(trimToEmpty(component.getBlake2b_256())) - .setBlake2B384(trimToEmpty(component.getBlake2b_384())) - .setBlake2B512(trimToEmpty(component.getBlake2b_512())) - .setBlake3(trimToEmpty(component.getBlake3())); - - if (requirements.get(TYPE_COMPONENT).contains("is_direct_dependency") - && component.getProject().getDirectDependencies() != null) { + if (projection.aliasesJson != null) { try { - final ArrayNode dependencyArray = OBJECT_MAPPER.readValue(component.getProject().getDirectDependencies(), ArrayNode.class); - for (final JsonNode dependencyNode : dependencyArray) { - if (dependencyNode.get("uuid") != null && dependencyNode.get("uuid").asText().equals(component.getUuid().toString())) { - builder.setIsDirectDependency(true); - break; - } - } - } catch (JacksonException | RuntimeException e) { - LOGGER.warn("Failed to parse direct dependencies of project %s".formatted(component.getProject().getUuid()), e); - } - } - - if (requirements.containsKey(TYPE_LICENSE) && component.getResolvedLicense() != null) { - final License.Builder licenseBuilder = License.newBuilder() - .setUuid(Optional.ofNullable(component.getResolvedLicense().getUuid()).map(UUID::toString).orElse("")) - .setId(trimToEmpty(component.getResolvedLicense().getLicenseId())) - .setName(trimToEmpty(component.getResolvedLicense().getName())) - .setIsOsiApproved(component.getResolvedLicense().isOsiApproved()) - .setIsFsfLibre(component.getResolvedLicense().isFsfLibre()) - .setIsDeprecatedId(component.getResolvedLicense().isDeprecatedLicenseId()) - .setIsCustom(component.getResolvedLicense().isCustomLicense()); - - if (requirements.containsKey(TYPE_LICENSE_GROUP) - || requirements.get(TYPE_LICENSE).contains("groups")) { - final Query licenseGroupQuery = qm.getPersistenceManager().newQuery(LicenseGroup.class); - licenseGroupQuery.setFilter("licenses.contains(:license)"); - licenseGroupQuery.setNamedParameters(Map.of("license", component.getResolvedLicense())); - licenseGroupQuery.setResult("uuid, name"); - try { - licenseGroupQuery.executeResultList(LicenseGroup.class).stream() - .map(licenseGroup -> License.Group.newBuilder() - .setUuid(Optional.ofNullable(licenseGroup.getUuid()).map(UUID::toString).orElse("")) - .setName(trimToEmpty(licenseGroup.getName()))) - .forEach(licenseBuilder::addGroups); - } finally { - licenseGroupQuery.closeAll(); - } + OBJECT_MAPPER.readValue(projection.aliasesJson, VULNERABILITY_ALIASES_TYPE_REF).stream() + .flatMap(CelPolicyEngine::mapToProto) + .distinct() + .forEach(builder::addAliases); + } catch (JacksonException e) { + LOGGER.warn("Failed to parse aliases from %s for vulnerability %d" + .formatted(projection.aliasesJson, projection.id), e); } - - builder.setResolvedLicense(licenseBuilder); } return builder.build(); } - private static org.hyades.proto.policy.v1.Project mapProject(final Project project, - final MultiValuedMap requirements) { - if (!requirements.containsKey(TYPE_PROJECT)) { - return org.hyades.proto.policy.v1.Project.getDefaultInstance(); - } - - // TODO: Load only required fields - final org.hyades.proto.policy.v1.Project.Builder builder = - org.hyades.proto.policy.v1.Project.newBuilder() - .setUuid(Optional.ofNullable(project.getUuid()).map(UUID::toString).orElse("")) - .setGroup(trimToEmpty(project.getGroup())) - .setName(trimToEmpty(project.getName())) - .setVersion(trimToEmpty(project.getVersion())) - .addAllTags(project.getTags().stream().map(Tag::getName).toList()) - .setCpe(trimToEmpty(project.getCpe())) - .setPurl(Optional.ofNullable(project.getPurl()).map(PackageURL::canonicalize).orElse("")) - .setSwidTagId(trimToEmpty(project.getSwidTagId())); - - if (requirements.containsKey(TYPE_PROJECT_PROPERTY) - || requirements.get(TYPE_PROJECT).contains("properties")) { - // TODO - } - - return builder.build(); - } - - private static List loadVulnerabilities(final QueryManager qm, - final Component component, - final MultiValuedMap requirements) { - if (!requirements.containsKey(TYPE_VULNERABILITY)) { - return emptyList(); - } - - // TODO: Load only required fields - final Query query = - qm.getPersistenceManager().newQuery(org.dependencytrack.model.Vulnerability.class); - query.getFetchPlan().clearGroups(); - query.getFetchPlan().setGroup(org.dependencytrack.model.Vulnerability.FetchGroup.POLICY.name()); - query.setFilter("components.contains(:component)"); - query.setParameters(component); - final List vulns; - try { - vulns = (List) qm.getPersistenceManager().detachCopyAll(query.executeList()); - } finally { - query.closeAll(); - } - - final List vulnBuilders = vulns.stream() - .map(v -> { - final Vulnerability.Builder builder = Vulnerability.newBuilder() - .setUuid(v.getUuid().toString()) - .setId(trimToEmpty(v.getVulnId())) - .setSource(trimToEmpty(v.getSource())) - .setCvssv2Vector(trimToEmpty(v.getCvssV2Vector())) - .setCvssv3Vector(trimToEmpty(v.getCvssV3Vector())) - .setOwaspRrVector(trimToEmpty(v.getOwaspRRVector())) - .setSeverity(v.getSeverity().name()); - Optional.ofNullable(v.getCwes()).ifPresent(builder::addAllCwes); - Optional.ofNullable(v.getCvssV2BaseScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2BaseScore); - Optional.ofNullable(v.getCvssV2ImpactSubScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2ImpactSubscore); - Optional.ofNullable(v.getCvssV2ExploitabilitySubScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv2ExploitabilitySubscore); - Optional.ofNullable(v.getCvssV3BaseScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3BaseScore); - Optional.ofNullable(v.getCvssV3ImpactSubScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3ImpactSubscore); - Optional.ofNullable(v.getCvssV3ExploitabilitySubScore()).map(BigDecimal::doubleValue).ifPresent(builder::setCvssv3ExploitabilitySubscore); - Optional.ofNullable(v.getOwaspRRLikelihoodScore()).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrLikelihoodScore); - Optional.ofNullable(v.getOwaspRRTechnicalImpactScore()).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrTechnicalImpactScore); - Optional.ofNullable(v.getOwaspRRBusinessImpactScore()).map(BigDecimal::doubleValue).ifPresent(builder::setOwaspRrBusinessImpactScore); - Optional.ofNullable(v.getEpssScore()).map(BigDecimal::doubleValue).ifPresent(builder::setEpssScore); - Optional.ofNullable(v.getEpssPercentile()).map(BigDecimal::doubleValue).ifPresent(builder::setEpssPercentile); - Optional.ofNullable(v.getCreated()).map(Timestamps::fromDate).ifPresent(builder::setCreated); - Optional.ofNullable(v.getPublished()).map(Timestamps::fromDate).ifPresent(builder::setPublished); - Optional.ofNullable(v.getUpdated()).map(Timestamps::fromDate).ifPresent(builder::setUpdated); - - if (requirements.containsKey(TYPE_VULNERABILITY_ALIAS) - || requirements.get(TYPE_VULNERABILITY).contains("aliases")) { - // TODO: Dirty hack, create a proper solution. Likely needs caching, too. - final var tmpVuln = new org.dependencytrack.model.Vulnerability(); - tmpVuln.setVulnId(builder.getId()); - tmpVuln.setSource(builder.getSource()); - tmpVuln.setAliases(qm.getVulnerabilityAliases(tmpVuln)); - VulnerabilityUtil.getUniqueAliases(tmpVuln).stream() - .map(alias -> Vulnerability.Alias.newBuilder() - .setId(alias.getKey().name()) - .setSource(alias.getValue()) - .build()) - .forEach(builder::addAliases); - } - - return builder; - }) - .toList(); - - return vulnBuilders.stream() - .map(Vulnerability.Builder::build) - .toList(); - } - - // TODO: Move to PolicyQueryManager - private static List reconcileViolations(final QueryManager qm, final Component component, final List violations) { - final var newViolations = new ArrayList(); - - final Transaction trx = qm.getPersistenceManager().currentTransaction(); - try { - trx.begin(); - - final var violationIdsToKeep = new HashSet(); - - for (final PolicyViolation violation : violations) { - violation.setComponent(component); - - final Query query = qm.getPersistenceManager().newQuery(PolicyViolation.class); - query.setFilter("component == :component && policyCondition == :policyCondition && type == :type"); - query.setNamedParameters(Map.of( - "component", violation.getComponent(), - "policyCondition", violation.getPolicyCondition(), - "type", violation.getType() - )); - query.setResult("id"); - - final Long existingViolationId; - try { - existingViolationId = query.executeResultUnique(Long.class); - } finally { - query.closeAll(); - } - - if (existingViolationId != null) { - violationIdsToKeep.add(existingViolationId); - } else { - qm.getPersistenceManager().makePersistent(violation); - violationIdsToKeep.add(violation.getId()); - newViolations.add(violation); - } - } - - final Query deleteQuery = qm.getPersistenceManager().newQuery(PolicyViolation.class); - deleteQuery.setFilter("component == :component && !:ids.contains(id)"); - try { - final long violationsDeleted = deleteQuery.deletePersistentAll(component, violationIdsToKeep); - LOGGER.debug("Deleted %s outdated violations".formatted(violationsDeleted)); // TODO: Add component UUID - } finally { - deleteQuery.closeAll(); - } - - trx.commit(); - } finally { - if (trx.isActive()) { - trx.rollback(); - } - } - - return newViolations; - } - - private static List reconcileViolations(final QueryManager qm, final long projectId, - final MultiValuedMap reportedViolationsByComponentId) { - // We want to send notifications for newly identified policy violations, - // so need to keep track of which violations we created. - final var newViolationIds = new ArrayList(); - - // DataNucleus does not support batch inserts, which is something we need to create - // new violations efficiently. Falling back to "raw" JDBC for the sake of efficiency. - final JDOConnection jdoConnection = qm.getPersistenceManager().getDataStoreConnection(); - final var nativeConnection = (Connection) jdoConnection.getNativeConnection(); - - try { - // JDBC connections default to autocommit. - // We'll do multiple write operations here, and want to commit them all in a single transaction. - nativeConnection.setAutoCommit(false); - nativeConnection.setTransactionIsolation(TRANSACTION_READ_COMMITTED); - - // First, query for all existing policy violations of the project, grouping them by component ID. - final var existingViolationsByComponentId = new HashSetValuedHashMap(); - try (final PreparedStatement ps = nativeConnection.prepareStatement(""" - SELECT - "ID" AS "id", - "COMPONENT_ID" AS "componentId", - "POLICYCONDITION_ID" AS "policyConditionId" - FROM - "POLICYVIOLATION" - WHERE - "PROJECT_ID" = ? - """)) { - ps.setLong(1, projectId); - - final ResultSet rs = ps.executeQuery(); - while (rs.next()) { - existingViolationsByComponentId.put( - rs.getLong("componentId"), - new PolicyViolationProjection( - rs.getLong("id"), - rs.getLong("policyConditionId") - )); - } - } - - // For each component that has existing and / or reported violations... - final Set componentIds = new HashSet<>(reportedViolationsByComponentId.keySet().size() + existingViolationsByComponentId.keySet().size()); - componentIds.addAll(reportedViolationsByComponentId.keySet()); - componentIds.addAll(existingViolationsByComponentId.keySet()); - - // ... determine which existing violations should be deleted (because they're no longer reported), - // and which reported violations should be created (because they have not been reported before). - // - // Violations not belonging to either of those buckets are reported, but already exist, - // meaning no action needs to be taken for them. - final var violationIdsToDelete = new ArrayList(); - final var violationsToCreate = new HashSetValuedHashMap(); - for (final Long componentId : componentIds) { - final Collection existingViolations = existingViolationsByComponentId.get(componentId); - final Collection reportedViolations = reportedViolationsByComponentId.get(componentId); - - if (reportedViolations == null || reportedViolations.isEmpty()) { - // Component has been removed, or does not have any violations anymore. - // All of its existing violations can be deleted. - violationIdsToDelete.addAll(existingViolations.stream().map(PolicyViolationProjection::id).toList()); - continue; - } - - if (existingViolations == null || existingViolations.isEmpty()) { - // Component did not have any violations before, but has some now. - // All reported violations must be newly created. - violationsToCreate.putAll(componentId, reportedViolations); - continue; - } - - // To determine which violations shall be deleted, find occurrences of violations appearing - // in the collection of existing violations, but not in the collection of reported violations. - existingViolations.stream() - .filter(existingViolation -> reportedViolations.stream().noneMatch(newViolation -> - newViolation.getPolicyCondition().getId() == existingViolation.policyConditionId())) - .map(PolicyViolationProjection::id) - .forEach(violationIdsToDelete::add); - - // To determine which violations shall be created, find occurrences of violations appearing - // in the collection of reported violations, but not in the collection of existing violations. - reportedViolations.stream() - .filter(reportedViolation -> existingViolations.stream().noneMatch(existingViolation -> - existingViolation.policyConditionId() == reportedViolation.getPolicyCondition().getId())) - .forEach(reportedViolation -> violationsToCreate.put(componentId, reportedViolation)); - } - - if (!violationsToCreate.isEmpty()) { - try (final PreparedStatement ps = nativeConnection.prepareStatement(""" - INSERT INTO "POLICYVIOLATION" - ("UUID", "TIMESTAMP", "COMPONENT_ID", "PROJECT_ID", "POLICYCONDITION_ID", "TYPE") - VALUES - (?, NOW(), ?, ?, ?, ?) - ON CONFLICT DO NOTHING - RETURNING "ID" - """, Statement.RETURN_GENERATED_KEYS)) { - for (final Map.Entry entry : violationsToCreate.entries()) { - ps.setString(1, UUID.randomUUID().toString()); - ps.setLong(2, entry.getKey()); - ps.setLong(3, projectId); - ps.setLong(4, entry.getValue().getPolicyCondition().getId()); - ps.setString(5, entry.getValue().getType().name()); - ps.addBatch(); - } - ps.executeBatch(); - - final ResultSet rs = ps.getGeneratedKeys(); - while (rs.next()) { - newViolationIds.add(rs.getLong(1)); - } - } - } - - if (!violationIdsToDelete.isEmpty()) { - final Array violationIdsToDeleteArray = - nativeConnection.createArrayOf("BIGINT", violationIdsToDelete.toArray(new Long[0])); - - try (final PreparedStatement ps = nativeConnection.prepareStatement(""" - DELETE FROM - "POLICYVIOLATION" - WHERE - "ID" = ANY(?) - """)) { - ps.setArray(1, violationIdsToDeleteArray); - ps.execute(); - } - } - - nativeConnection.commit(); - } catch (Exception e) { - try { - nativeConnection.rollback(); - } catch (SQLException ex) { - throw new RuntimeException(ex); - } - - throw new RuntimeException(e); - } finally { - jdoConnection.close(); - } - - return newViolationIds; + private static Stream mapToProto(final VulnerabilityAlias alias) { + return alias.getAllBySource().entrySet().stream() + .map(aliasEntry -> Vulnerability.Alias.newBuilder() + .setSource(aliasEntry.getKey().name()) + .setId(aliasEntry.getValue()) + .build()); } } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java new file mode 100644 index 000000000..4dc4a39e3 --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java @@ -0,0 +1,586 @@ +package org.dependencytrack.policy.cel; + +import alpine.common.logging.Logger; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.policy.cel.mapping.ComponentProjection; +import org.dependencytrack.policy.cel.mapping.ComponentsVulnerabilitiesProjection; +import org.dependencytrack.policy.cel.mapping.LicenseGroupProjection; +import org.dependencytrack.policy.cel.mapping.LicenseProjection; +import org.dependencytrack.policy.cel.mapping.PolicyViolationProjection; +import org.dependencytrack.policy.cel.mapping.ProjectProjection; +import org.dependencytrack.policy.cel.mapping.ProjectPropertyProjection; +import org.dependencytrack.policy.cel.mapping.VulnerabilityProjection; + +import javax.jdo.PersistenceManager; +import javax.jdo.Query; +import javax.jdo.datastore.JDOConnection; +import java.sql.Array; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.sql.Connection.TRANSACTION_READ_COMMITTED; +import static org.dependencytrack.policy.cel.mapping.FieldMappingUtil.getFieldMappings; + +class CelPolicyQueryManager implements AutoCloseable { + + private static final Logger LOGGER = Logger.getLogger(CelPolicyQueryManager.class); + + private final PersistenceManager pm; + + CelPolicyQueryManager(final QueryManager qm) { + this.pm = qm.getPersistenceManager(); + } + + ProjectProjection fetchProject(final long projectId, + final Collection projectProtoFieldNames, + final Collection projectPropertyProtoFieldNames) { + // Determine the columns to select from the PROJECT (P) table. + String sqlProjectSelectColumns = Stream.concat( + Stream.of(ProjectProjection.ID_FIELD_MAPPING), + getFieldMappings(ProjectProjection.class).stream() + .filter(mapping -> projectProtoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"P\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + // Determine the columns to select from the PROJECT_PROPERTY (PP) table. + // The resulting expression will be used to populate JSON objects, using + // the JSONB_BUILD_OBJECT(name1, value1, name2, value2) notation. + String sqlPropertySelectColumns = ""; + if (projectPropertyProtoFieldNames != null) { + sqlPropertySelectColumns = getFieldMappings(ProjectPropertyProjection.class).stream() + .filter(mapping -> projectPropertyProtoFieldNames.contains(mapping.protoFieldName())) + .map(mapping -> "'%s', \"PP\".\"%s\"".formatted(mapping.javaFieldName(), mapping.sqlColumnName())) + .collect(Collectors.joining(", ")); + } + + // Properties will be selected into propertiesJson, tags into tagsJson. + // Both these fields are not part of the Project proto, thus their selection + // must be added manually. + if (!sqlPropertySelectColumns.isBlank()) { + sqlProjectSelectColumns += ", \"propertiesJson\""; + } + if (projectProtoFieldNames.contains("tags")) { + sqlProjectSelectColumns += ", \"tagsJson\""; + } + + final Query query = pm.newQuery(Query.SQL, """ + SELECT + %s + FROM + "PROJECT" AS "P" + LEFT JOIN LATERAL ( + SELECT + CAST(JSONB_AGG(DISTINCT JSONB_BUILD_OBJECT(%s)) AS TEXT) AS "propertiesJson" + FROM + "PROJECT_PROPERTY" AS "PP" + WHERE + "PP"."PROJECT_ID" = "P"."ID" + ) AS "properties" ON :shouldFetchProperties + LEFT JOIN LATERAL ( + SELECT + CAST(JSONB_AGG(DISTINCT "T"."NAME") AS TEXT) AS "tagsJson" + FROM + "TAG" AS "T" + INNER JOIN + "PROJECTS_TAGS" AS "PT" ON "PT"."TAG_ID" = "T"."ID" + WHERE + "PT"."PROJECT_ID" = "P"."ID" + ) AS "tags" ON :shouldFetchTags + WHERE + "ID" = :projectId + """.formatted(sqlProjectSelectColumns, sqlPropertySelectColumns)); + query.setNamedParameters(Map.of( + "shouldFetchProperties", !sqlPropertySelectColumns.isBlank(), + "shouldFetchTags", projectProtoFieldNames.contains("tags"), + "projectId", projectId + )); + try { + return query.executeResultUnique(ProjectProjection.class); + } finally { + query.closeAll(); + } + } + + List fetchAllComponents(final long projectId, final Collection protoFieldNames) { + final String sqlSelectColumns = Stream.concat( + Stream.of(ComponentProjection.ID_FIELD_MAPPING), + getFieldMappings(ComponentProjection.class).stream() + .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + final Query query = pm.newQuery(Query.SQL, """ + SELECT %s FROM "COMPONENT" WHERE "PROJECT_ID" = ? + """.formatted(sqlSelectColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(ComponentProjection.class)); + } finally { + query.closeAll(); + } + } + + /** + * Fetch all {@link org.dependencytrack.model.Component} {@code <->} {@link org.dependencytrack.model.Vulnerability} + * relationships for a given {@link Project}. + * + * @param projectId ID of the {@link Project} to fetch relationships for + * @return A {@link List} of {@link ComponentsVulnerabilitiesProjection} + */ + List fetchAllComponentsVulnerabilities(final long projectId) { + final Query query = pm.newQuery(Query.SQL, """ + SELECT + "CV"."COMPONENT_ID" AS "componentId", + "CV"."VULNERABILITY_ID" AS "vulnerabilityId" + FROM + "COMPONENTS_VULNERABILITIES" AS "CV" + INNER JOIN + "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" + WHERE + "C"."PROJECT_ID" = ? + """); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(ComponentsVulnerabilitiesProjection.class)); + } finally { + query.closeAll(); + } + } + + List fetchAllLicenses(final long projectId, + final Collection licenseProtoFieldNames, + final Collection licenseGroupProtoFieldNames) { + final String licenseSqlSelectColumns = Stream.concat( + Stream.of(LicenseProjection.ID_FIELD_MAPPING), + getFieldMappings(LicenseProjection.class).stream() + .filter(mapping -> licenseProtoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"L\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + // If fetching license groups is not necessary, we can just query for licenses and be done with it. + if (!licenseProtoFieldNames.contains("groups")) { + final Query query = pm.newQuery(Query.SQL, """ + SELECT DISTINCT + %s + FROM + "LICENSE" AS "L" + INNER JOIN + "COMPONENT" AS "C" ON "C"."LICENSE_ID" = "L"."ID" + WHERE + "C"."PROJECT_ID" = ? + """.formatted(licenseSqlSelectColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(LicenseProjection.class)); + } finally { + query.closeAll(); + } + } + + // If groups are required, include them in the license query in order to avoid the 1+N problem. + // Licenses may or may not be assigned to a group. Licenses can be in multiple groups. + // + // Using a simple LEFT JOIN would result in duplicate license data being fetched, e.g.: + // + // | "L"."ID" | "L"."NAME" | "LG"."NAME" | + // | :------- | :--------- | :---------- | + // | 1 | foo | groupA | + // | 1 | foo | groupB | + // | 1 | foo | groupC | + // | 2 | bar | NULL | + // + // To avoid this, we instead aggregate license group fields for each license, and return them as JSON. + // The reason for choosing JSON over native arrays, is that DataNucleus can't deal with arrays cleanly. + // + // | "L"."ID" | "L"."NAME" | "licenseGroupsJson" | + // | :------- | :--------- | :------------------------------------------------------ | + // | 1 | foo | [{"name":"groupA"},{"name":"groupB"},{"name":"groupC"}] | + // | 2 | bar | [] | + + final String licenseSqlGroupByColumns = Stream.concat( + Stream.of(LicenseProjection.ID_FIELD_MAPPING), + getFieldMappings(LicenseProjection.class).stream() + .filter(mapping -> licenseProtoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"L\".\"%s\"".formatted(mapping.sqlColumnName())) + .collect(Collectors.joining(", ")); + + final String licenseGroupSqlSelectColumns = getFieldMappings(LicenseGroupProjection.class).stream() + .filter(mapping -> licenseGroupProtoFieldNames.contains(mapping.protoFieldName())) + .map(mapping -> "'%s', \"LG\".\"%s\"".formatted(mapping.javaFieldName(), mapping.sqlColumnName())) + .collect(Collectors.joining(", ")); + + final Query query = pm.newQuery(Query.SQL, """ + SELECT DISTINCT + "L"."ID" AS "id", + %s, + CAST(JSONB_AGG(DISTINCT JSONB_BUILD_OBJECT(%s)) AS TEXT) AS "licenseGroupsJson" + FROM + "LICENSE" AS "L" + INNER JOIN + "COMPONENT" AS "C" ON "C"."LICENSE_ID" = "L"."ID" + LEFT JOIN + "LICENSEGROUP_LICENSE" AS "LGL" ON "LGL"."LICENSE_ID" = "L"."ID" + LEFT JOIN + "LICENSEGROUP" AS "LG" ON "LG"."ID" = "LGL"."LICENSEGROUP_ID" + WHERE + "C"."PROJECT_ID" = ? + GROUP BY + %s + """.formatted(licenseSqlSelectColumns, licenseGroupSqlSelectColumns, licenseSqlGroupByColumns)); + query.setParameters(projectId); + try { + return List.copyOf(query.executeResultList(LicenseProjection.class)); + } finally { + query.closeAll(); + } + } + + List fetchAllVulnerabilities(final long projectId, final Collection protoFieldNames) { + String sqlSelectColumns = Stream.concat( + Stream.of(VulnerabilityProjection.ID_FIELD_MAPPING), + getFieldMappings(VulnerabilityProjection.class).stream() + .filter(mapping -> protoFieldNames.contains(mapping.protoFieldName())) + ) + .map(mapping -> "\"V\".\"%s\" AS \"%s\"".formatted(mapping.sqlColumnName(), mapping.javaFieldName())) + .collect(Collectors.joining(", ")); + + if (protoFieldNames.contains("aliases")) { + sqlSelectColumns += ", \"aliasesJson\""; + } + + final Query query = pm.newQuery(Query.SQL, """ + SELECT DISTINCT + %s + FROM + "VULNERABILITY" AS "V" + INNER JOIN + "COMPONENTS_VULNERABILITIES" AS "CV" ON "CV"."VULNERABILITY_ID" = "V"."ID" + INNER JOIN "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" + LEFT JOIN LATERAL ( + SELECT + CAST(JSONB_AGG(DISTINCT JSONB_STRIP_NULLS(JSONB_BUILD_OBJECT( + 'cveId', "VA"."CVE_ID", + 'ghsaId', "VA"."GHSA_ID", + 'gsdId', "VA"."GSD_ID", + 'internalId', "VA"."INTERNAL_ID", + 'osvId', "VA"."OSV_ID", + 'sonatypeId', "VA"."SONATYPE_ID", + 'snykId', "VA"."SNYK_ID", + 'vulnDbId', "VA"."VULNDB_ID" + ))) AS TEXT) AS "aliasesJson" + FROM + "VULNERABILITYALIAS" AS "VA" + WHERE + ("V"."SOURCE" = 'NVD' AND "VA"."CVE_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'GITHUB' AND "VA"."GHSA_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'GSD' AND "VA"."GSD_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'INTERNAL' AND "VA"."INTERNAL_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'OSV' AND "VA"."OSV_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'SONATYPE' AND "VA"."SONATYPE_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'SNYK' AND "VA"."SNYK_ID" = "V"."VULNID") + OR ("V"."SOURCE" = 'VULNDB' AND "VA"."VULNDB_ID" = "V"."VULNID") + ) AS "aliases" ON :shouldFetchAliases + WHERE + "C"."PROJECT_ID" = :projectId + """.formatted(sqlSelectColumns)); + query.setNamedParameters(Map.of( + "shouldFetchAliases", protoFieldNames.contains("aliases"), + "projectId", projectId + )); + try { + return List.copyOf(query.executeResultList(VulnerabilityProjection.class)); + } finally { + query.closeAll(); + } + } + + List reconcileViolations(final long projectId, final MultiValuedMap reportedViolationsByComponentId) { + // We want to send notifications for newly identified policy violations, + // so need to keep track of which violations we created. + final var newViolationIds = new ArrayList(); + + // DataNucleus does not support batch inserts, which is something we need in order to + // create new violations efficiently. Falling back to "raw" JDBC for the sake of efficiency. + final JDOConnection jdoConnection = pm.getDataStoreConnection(); + final var nativeConnection = (Connection) jdoConnection.getNativeConnection(); + Boolean originalAutoCommit = null; + Integer originalTrxIsolation = null; + + try { + // JDBC connections default to autocommit. + // We'll do multiple write operations here, and want to commit them all in a single transaction. + originalAutoCommit = nativeConnection.getAutoCommit(); + originalTrxIsolation = nativeConnection.getTransactionIsolation(); + nativeConnection.setAutoCommit(false); + nativeConnection.setTransactionIsolation(TRANSACTION_READ_COMMITTED); + + // First, query for all existing policy violations of the project, grouping them by component ID. + final var existingViolationsByComponentId = new HashSetValuedHashMap(); + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + SELECT + "ID" AS "id", + "COMPONENT_ID" AS "componentId", + "POLICYCONDITION_ID" AS "policyConditionId" + FROM + "POLICYVIOLATION" + WHERE + "PROJECT_ID" = ? + """)) { + ps.setLong(1, projectId); + + final ResultSet rs = ps.executeQuery(); + while (rs.next()) { + existingViolationsByComponentId.put( + rs.getLong("componentId"), + new PolicyViolationProjection( + rs.getLong("id"), + rs.getLong("policyConditionId") + )); + } + } + + // For each component that has existing and / or reported violations... + final Set componentIds = new HashSet<>(reportedViolationsByComponentId.keySet().size() + existingViolationsByComponentId.keySet().size()); + componentIds.addAll(reportedViolationsByComponentId.keySet()); + componentIds.addAll(existingViolationsByComponentId.keySet()); + + // ... determine which existing violations should be deleted (because they're no longer reported), + // and which reported violations should be created (because they have not been reported before). + // + // Violations not belonging to either of those buckets are reported, but already exist, + // meaning no action needs to be taken for them. + final var violationIdsToDelete = new ArrayList(); + final var violationsToCreate = new HashSetValuedHashMap(); + for (final Long componentId : componentIds) { + final Collection existingViolations = existingViolationsByComponentId.get(componentId); + final Collection reportedViolations = reportedViolationsByComponentId.get(componentId); + + if (reportedViolations == null || reportedViolations.isEmpty()) { + // Component has been removed, or does not have any violations anymore. + // All of its existing violations can be deleted. + violationIdsToDelete.addAll(existingViolations.stream().map(PolicyViolationProjection::id).toList()); + continue; + } + + if (existingViolations == null || existingViolations.isEmpty()) { + // Component did not have any violations before, but has some now. + // All reported violations must be newly created. + violationsToCreate.putAll(componentId, reportedViolations); + continue; + } + + // To determine which violations shall be deleted, find occurrences of violations appearing + // in the collection of existing violations, but not in the collection of reported violations. + existingViolations.stream() + .filter(existingViolation -> reportedViolations.stream().noneMatch(newViolation -> + newViolation.getPolicyCondition().getId() == existingViolation.policyConditionId())) + .map(PolicyViolationProjection::id) + .forEach(violationIdsToDelete::add); + + // To determine which violations shall be created, find occurrences of violations appearing + // in the collection of reported violations, but not in the collection of existing violations. + reportedViolations.stream() + .filter(reportedViolation -> existingViolations.stream().noneMatch(existingViolation -> + existingViolation.policyConditionId() == reportedViolation.getPolicyCondition().getId())) + .forEach(reportedViolation -> violationsToCreate.put(componentId, reportedViolation)); + } + + if (!violationsToCreate.isEmpty()) { + // For violations that need to be created, utilize batch inserts to limit database round-trips. + // Keep note of the IDs that were generated as part of the insert; For those we'll need to send + // notifications later. + + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + INSERT INTO "POLICYVIOLATION" + ("UUID", "TIMESTAMP", "COMPONENT_ID", "PROJECT_ID", "POLICYCONDITION_ID", "TYPE") + VALUES + (?, ?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + RETURNING "ID" + """, Statement.RETURN_GENERATED_KEYS)) { + for (final Map.Entry entry : violationsToCreate.entries()) { + ps.setString(1, UUID.randomUUID().toString()); + ps.setTimestamp(2, new Timestamp(entry.getValue().getTimestamp().getTime())); + ps.setLong(3, entry.getKey()); + ps.setLong(4, projectId); + ps.setLong(5, entry.getValue().getPolicyCondition().getId()); + ps.setString(6, entry.getValue().getType().name()); + ps.addBatch(); + } + ps.executeBatch(); + + final ResultSet rs = ps.getGeneratedKeys(); + while (rs.next()) { + newViolationIds.add(rs.getLong(1)); + } + } + } + + if (!violationIdsToDelete.isEmpty()) { + final Array violationIdsToDeleteArray = + nativeConnection.createArrayOf("BIGINT", violationIdsToDelete.toArray(new Long[0])); + + // First, bulk-delete any analyses attached to the violations first. + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + DELETE FROM + "VIOLATIONANALYSIS" + WHERE + "POLICYVIOLATION_ID" = ANY(?) + """)) { + ps.setArray(1, violationIdsToDeleteArray); + ps.execute(); + } + + // Finally, bulk-delete the actual violations. + try (final PreparedStatement ps = nativeConnection.prepareStatement(""" + DELETE FROM + "POLICYVIOLATION" + WHERE + "ID" = ANY(?) + """)) { + ps.setArray(1, violationIdsToDeleteArray); + ps.execute(); + } + } + + nativeConnection.commit(); + } catch (Exception e) { + try { + nativeConnection.rollback(); + } catch (SQLException ex) { + throw new RuntimeException(ex); + } + + throw new RuntimeException(e); + } finally { + try { + if (originalAutoCommit != null) { + nativeConnection.setAutoCommit(originalAutoCommit); + } + if (originalTrxIsolation != null) { + nativeConnection.setTransactionIsolation(originalTrxIsolation); + } + } catch (SQLException e) { + LOGGER.error("Failed to restore original connection settings (autoCommit=%s, trxIsolation=%d)" + .formatted(originalAutoCommit, originalTrxIsolation), e); + } + + jdoConnection.close(); + } + + return newViolationIds; + } + + List getApplicablePolicies(final Project project) { + var filter = """ + (this.projects.isEmpty() && this.tags.isEmpty()) + || (this.projects.contains(:project) + """; + var params = new HashMap(); + params.put("project", project); + + // To compensate for missing support for recursion of Common Table Expressions (CTEs) + // in JDO, we have to fetch the UUIDs of all parent projects upfront. Otherwise, we'll + // not be able to evaluate whether the policy is inherited from parent projects. + var variables = ""; + final List parentUuids = getParents(project); + if (!parentUuids.isEmpty()) { + filter += """ + || (this.includeChildren + && this.projects.contains(parentVar) + && :parentUuids.contains(parentVar.uuid)) + """; + variables += "org.dependencytrack.model.Project parentVar"; + params.put("parentUuids", parentUuids); + } + filter += ")"; + + // DataNucleus generates an invalid SQL query when using the idiomatic solution. + // The following works, but it's ugly and likely doesn't perform well if the project + // has many tags. Worth trying the idiomatic way again once DN has been updated to > 6.0.4. + // + // filter += " || (this.tags.contains(commonTag) && :project.tags.contains(commonTag))"; + // variables += "org.dependencytrack.model.Tag commonTag"; + if (project.getTags() != null && !project.getTags().isEmpty()) { + filter += " || ("; + for (int i = 0; i < project.getTags().size(); i++) { + filter += "this.tags.contains(:tag" + i + ")"; + params.put("tag" + i, project.getTags().get(i)); + if (i < (project.getTags().size() - 1)) { + filter += " || "; + } + } + filter += ")"; + } + + final List policies; + final Query query = pm.newQuery(Policy.class); + try { + query.setFilter(filter); + query.setNamedParameters(params); + if (!variables.isEmpty()) { + query.declareVariables(variables); + } + policies = List.copyOf(query.executeList()); + } finally { + query.closeAll(); + } + + return policies; + } + + // TODO: Move to ProjectQueryManager + List getParents(final Project project) { + return getParents(project.getUuid(), new ArrayList<>()); + } + + // TODO: Move to ProjectQueryManager + List getParents(final UUID uuid, final List parents) { + final UUID parentUuid; + final Query query = pm.newQuery(Project.class); + try { + query.setFilter("uuid == :uuid && parent != null"); + query.setParameters(uuid); + query.setResult("parent.uuid"); + parentUuid = query.executeResultUnique(UUID.class); + } finally { + query.closeAll(); + } + + if (parentUuid == null) { + return parents; + } + + parents.add(parentUuid); + return getParents(parentUuid, parents); + } + + @Override + public void close() { + // Noop + } + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java index f6c680335..40f9ab980 100644 --- a/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectProjection.java @@ -4,6 +4,8 @@ public class ProjectProjection { + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", /* protoFieldName */ null, "ID"); + public long id; @MappedField(sqlColumnName = "UUID") @@ -36,4 +38,7 @@ public class ProjectProjection { @MappedField(protoFieldName = "last_bom_import", sqlColumnName = "LAST_BOM_IMPORTED") public Date lastBomImport; + public String propertiesJson; + public String tagsJson; + } diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectPropertyProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectPropertyProjection.java new file mode 100644 index 000000000..522181f6d --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/ProjectPropertyProjection.java @@ -0,0 +1,17 @@ +package org.dependencytrack.policy.cel.mapping; + +public class ProjectPropertyProjection { + + @MappedField(sqlColumnName = "GROUPNAME") + public String group; + + @MappedField(sqlColumnName = "PROPERTYNAME") + public String name; + + @MappedField(sqlColumnName = "PROPERTYVALUE") + public String value; + + @MappedField(sqlColumnName = "PROPERTYTYPE") + public String type; + +} diff --git a/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java b/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java index a41ed4b11..76bd2f829 100644 --- a/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java +++ b/src/main/java/org/dependencytrack/policy/cel/mapping/VulnerabilityProjection.java @@ -4,6 +4,8 @@ public class VulnerabilityProjection { + public static FieldMapping ID_FIELD_MAPPING = new FieldMapping("id", /* protoFieldName */ null, "ID"); + public long id; @MappedField(sqlColumnName = "UUID") diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index 1b51ccb39..d9bc560fe 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -155,10 +155,8 @@ public void testIsDirectDependency() { final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateComponent(componentA.getUuid()); + policyEngine.evaluateProject(project.getUuid()); assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); - - policyEngine.evaluateComponent(componentB.getUuid()); assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); } @@ -193,10 +191,8 @@ public void testProjectDependsOnComponent() { final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateComponent(componentA.getUuid()); + policyEngine.evaluateProject(project.getUuid()); assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); - - policyEngine.evaluateComponent(componentB.getUuid()); assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); } @@ -235,10 +231,8 @@ public void testMatchesRange() { final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateComponent(componentA.getUuid()); + policyEngine.evaluateProject(project.getUuid()); assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); - - policyEngine.evaluateComponent(componentB.getUuid()); assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); } @@ -289,7 +283,7 @@ public void issue1924() { qm.persist(vulnerability); qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); CelPolicyEngine policyEngine = new CelPolicyEngine(); - policyEngine.evaluate(components); + policyEngine.evaluateProject(project.getUuid()); final List violations = qm.getAllPolicyViolations(); Assert.assertEquals(3, violations.size()); PolicyViolation policyViolation = violations.get(0); @@ -349,7 +343,7 @@ public void issue2455() { qm.persist(component); CelPolicyEngine policyEngine = new CelPolicyEngine(); - policyEngine.evaluate(components); + policyEngine.evaluateProject(project.getUuid()); final List violations = qm.getAllPolicyViolations(); Assert.assertEquals(2, violations.size()); PolicyViolation policyViolation = violations.get(0); diff --git a/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java b/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java index 1cda78ff6..f2b480391 100644 --- a/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java @@ -37,6 +37,11 @@ public void testGetFieldMappingsForProjectProjection() { assertValidProtoFieldsAndColumns(ProjectProjection.class, Project.getDescriptor(), org.dependencytrack.model.Project.class); } + @Test + public void testGetFieldMappingsForProjectPropertyProjection() { + assertValidProtoFieldsAndColumns(ProjectPropertyProjection.class, Project.Property.getDescriptor(), org.dependencytrack.model.ProjectProperty.class); + } + @Test public void testGetFieldMappingsForVulnerabilityProjection() { assertValidProtoFieldsAndColumns(VulnerabilityProjection.class, Vulnerability.getDescriptor(), org.dependencytrack.model.Vulnerability.class); From e1f3419a5b6f641a4b95698b6b98c31e9d247438 Mon Sep 17 00:00:00 2001 From: nscuro Date: Wed, 20 Sep 2023 18:41:03 +0200 Subject: [PATCH 24/46] Add test to verify that all fields can be loaded Signed-off-by: nscuro --- .../model/PolicyCondition.java | 4 +- .../model/VulnerabilityAlias.java | 1 + .../policy/cel/CelPolicyEngine.java | 2 + .../policy/cel/CelPolicyQueryManager.java | 12 +- .../proto/org/hyades/policy/v1/policy.proto | 3 - .../policy/cel/CelPolicyEngineTest.java | 176 ++++++++++++++++++ 6 files changed, 187 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/dependencytrack/model/PolicyCondition.java b/src/main/java/org/dependencytrack/model/PolicyCondition.java index 71d4bbb80..4a492d440 100644 --- a/src/main/java/org/dependencytrack/model/PolicyCondition.java +++ b/src/main/java/org/dependencytrack/model/PolicyCondition.java @@ -111,9 +111,9 @@ public enum Subject { private Subject subject; @Persistent - @Column(name = "VALUE", allowsNull = "false", jdbcType = "VARCHAR", length = 1024) + @Column(name = "VALUE", allowsNull = "false", jdbcType = "CLOB") @NotBlank - @Size(min = 1, max = 1024) + @Size(min = 1) @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The value may only contain printable characters") private String value; diff --git a/src/main/java/org/dependencytrack/model/VulnerabilityAlias.java b/src/main/java/org/dependencytrack/model/VulnerabilityAlias.java index 40124a0dd..487eb7629 100644 --- a/src/main/java/org/dependencytrack/model/VulnerabilityAlias.java +++ b/src/main/java/org/dependencytrack/model/VulnerabilityAlias.java @@ -200,6 +200,7 @@ private String getBySource(final Vulnerability.Source source) { case NVD -> getCveId(); case OSSINDEX -> getSonatypeId(); case OSV -> getOsvId(); + case SNYK -> getSnykId(); case VULNDB -> getVulnDbId(); default -> null; }; diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 0339cbe4c..b4c8183a4 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -411,9 +411,11 @@ private static org.hyades.proto.policy.v1.Project mapToProto(final ProjectProjec .setGroup(trimToEmpty(projection.group)) .setName(trimToEmpty(projection.name)) .setVersion(trimToEmpty(projection.version)) + .setClassifier(trimToEmpty(projection.classifier)) .setCpe(trimToEmpty(projection.cpe)) .setPurl(trimToEmpty(projection.purl)) .setSwidTagId(trimToEmpty(projection.swidTagId)); + Optional.ofNullable(projection.isActive).ifPresent(builder::setIsActive); Optional.ofNullable(projection.lastBomImport).map(Timestamps::fromDate).ifPresent(builder::setLastBomImport); if (projection.propertiesJson != null) { diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java index 4dc4a39e3..3c2fe362b 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java @@ -282,14 +282,14 @@ List fetchAllVulnerabilities(final long projectId, fina LEFT JOIN LATERAL ( SELECT CAST(JSONB_AGG(DISTINCT JSONB_STRIP_NULLS(JSONB_BUILD_OBJECT( - 'cveId', "VA"."CVE_ID", - 'ghsaId', "VA"."GHSA_ID", - 'gsdId', "VA"."GSD_ID", + 'cveId', "VA"."CVE_ID", + 'ghsaId', "VA"."GHSA_ID", + 'gsdId', "VA"."GSD_ID", 'internalId', "VA"."INTERNAL_ID", - 'osvId', "VA"."OSV_ID", + 'osvId', "VA"."OSV_ID", 'sonatypeId', "VA"."SONATYPE_ID", - 'snykId', "VA"."SNYK_ID", - 'vulnDbId', "VA"."VULNDB_ID" + 'snykId', "VA"."SNYK_ID", + 'vulnDbId', "VA"."VULNDB_ID" ))) AS TEXT) AS "aliasesJson" FROM "VULNERABILITYALIAS" AS "VA" diff --git a/src/main/proto/org/hyades/policy/v1/policy.proto b/src/main/proto/org/hyades/policy/v1/policy.proto index 3718d0e63..88f64f8ee 100644 --- a/src/main/proto/org/hyades/policy/v1/policy.proto +++ b/src/main/proto/org/hyades/policy/v1/policy.proto @@ -48,9 +48,6 @@ message Component { // Whether the component is internal to the organization. optional bool is_internal = 9; - // Whether the component is a direct dependency of the project. - optional bool is_direct_dependency = 10; - optional string md5 = 20; optional string sha1 = 21; optional string sha256 = 22; diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index d9bc560fe..9c13718f4 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -1,8 +1,10 @@ package org.dependencytrack.policy.cel; +import alpine.model.IConfigProperty; import org.dependencytrack.AbstractPostgresEnabledTest; import org.dependencytrack.event.BomUploadEvent; import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Classifier; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; import org.dependencytrack.model.ConfigPropertyConstants; @@ -14,6 +16,7 @@ import org.dependencytrack.model.Project; import org.dependencytrack.model.Severity; import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.model.VulnerabilityAlias; import org.dependencytrack.persistence.DefaultObjectGenerator; import org.dependencytrack.tasks.BomUploadProcessingTask; import org.junit.Assert; @@ -124,6 +127,179 @@ public void test() { assertThat(violations).isNotEmpty(); } + @Test + public void testWithAllFields() { + final var project = new Project(); + project.setUuid(UUID.fromString("d7173786-60aa-4a4f-a950-c92fe6422307")); + project.setGroup("projectGroup"); + project.setName("projectName"); + project.setVersion("projectVersion"); + project.setClassifier(Classifier.APPLICATION); + project.setActive(true); + project.setCpe("projectCpe"); + project.setPurl("projectPurl"); + project.setSwidTagId("projectSwidTagId"); + project.setLastBomImport(new java.util.Date()); + qm.persist(project); + + qm.createProjectProperty(project, "propertyGroup", "propertyName", "propertyValue", IConfigProperty.PropertyType.STRING, null); + + qm.bind(project, List.of( + qm.createTag("projectTagA"), + qm.createTag("projectTagB") + )); + + final var licenseGroup = new LicenseGroup(); + licenseGroup.setUuid(UUID.fromString("bbdb62f8-d854-4e43-a9ed-36481545c201")); + licenseGroup.setName("licenseGroupName"); + qm.persist(licenseGroup); + + final var license = new License(); + license.setUuid(UUID.fromString("dc9876c2-0adc-422b-9f71-3ca78285f138")); + license.setLicenseId("resolvedLicenseId"); + license.setName("resolvedLicenseName"); + license.setOsiApproved(true); + license.setFsfLibre(true); + license.setDeprecatedLicenseId(true); + license.setCustomLicense(true); + license.setLicenseGroups(List.of(licenseGroup)); + qm.persist(license); + + final var component = new Component(); + component.setProject(project); + component.setUuid(UUID.fromString("7e5f6465-d2f2-424f-b1a4-68d186fa2b46")); + component.setGroup("componentGroup"); + component.setName("componentName"); + component.setVersion("componentVersion"); + component.setClassifier(Classifier.LIBRARY); + component.setCpe("componentCpe"); + component.setPurl("componentPurl"); + component.setSwidTagId("componentSwidTagId"); + component.setInternal(true); + component.setMd5("componentMd5"); + component.setSha1("componentSha1"); + component.setSha256("componentSha256"); + component.setSha384("componentSha384"); + component.setSha512("componentSha512"); + component.setSha3_256("componentSha3_256"); + component.setSha3_384("componentSha3_384"); + component.setSha3_512("componentSha3_512"); + component.setBlake2b_256("componentBlake2b_256"); + component.setBlake2b_384("componentBlake2b_384"); + component.setBlake2b_512("componentBlake2b_512"); + component.setBlake3("componentBlake3"); + component.setLicense("componentLicenseName"); + component.setResolvedLicense(license); + qm.persist(component); + + final var vuln = new Vulnerability(); + vuln.setUuid(UUID.fromString("ffe9743f-b916-431e-8a68-9b3ac56db72c")); + vuln.setVulnId("CVE-001"); + vuln.setSource(Vulnerability.Source.NVD); + vuln.setCwes(List.of(666, 777)); + vuln.setCreated(new java.util.Date(666)); + vuln.setPublished(new java.util.Date(777)); + vuln.setUpdated(new java.util.Date(888)); + qm.persist(vuln); + + qm.addVulnerability(vuln, component, AnalyzerIdentity.INTERNAL_ANALYZER); + + final var vulnAlias = new VulnerabilityAlias(); + vulnAlias.setCveId("CVE-001"); + vulnAlias.setGhsaId("GHSA-001"); + vulnAlias.setGsdId("GSD-001"); + vulnAlias.setInternalId("INT-001"); + vulnAlias.setOsvId("OSV-001"); + vulnAlias.setSnykId("SNYK-001"); + vulnAlias.setSonatypeId("SONATYPE-001"); + vulnAlias.setVulnDbId("VULNDB-001"); + qm.synchronizeVulnerabilityAlias(vulnAlias); + + final Policy policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.uuid == "__COMPONENT_UUID__" + && component.group == "componentGroup" + && component.name == "componentName" + && component.version == "componentVersion" + && component.classifier == "LIBRARY" + && component.cpe == "componentCpe" + && component.purl == "componentPurl" + && component.swid_tag_id == "componentSwidTagId" + && component.is_internal + && component.md5 == "componentmd5" + && component.sha1 == "componentsha1" + && component.sha256 == "componentsha256" + && component.sha384 == "componentsha384" + && component.sha512 == "componentsha512" + && component.sha3_256 == "componentsha3_256" + && component.sha3_384 == "componentsha3_384" + && component.sha3_512 == "componentsha3_512" + && component.blake2b_256 == "componentBlake2b_256" + && component.blake2b_384 == "componentBlake2b_384" + && component.blake2b_512 == "componentBlake2b_512" + && component.blake3 == "componentBlake3" + && component.license_name == "componentLicenseName" + && !has(component.license_expression) + && component.resolved_license.uuid == "__RESOLVED_LICENSE_UUID__" + && component.resolved_license.id == "resolvedLicenseId" + && component.resolved_license.name == "resolvedLicenseName" + && component.resolved_license.is_osi_approved + && component.resolved_license.is_fsf_libre + && component.resolved_license.is_deprecated_id + && component.resolved_license.is_custom + && component.resolved_license.groups.all(licenseGroup, + licenseGroup.uuid == "__LICENSE_GROUP_UUID__" + && licenseGroup.name == "licenseGroupName" + ) + && project.uuid == "__PROJECT_UUID__" + && project.group == "projectGroup" + && project.name == "projectName" + && project.version == "projectVersion" + && project.classifier == "APPLICATION" + && project.is_active + && project.cpe == "projectCpe" + && project.purl == "projectPurl" + && project.swid_tag_id == "projectSwidTagId" + && has(project.last_bom_import) + && "projecttaga" in project.tags + && project.properties.all(property, + property.group == "propertyGroup" + && property.name == "propertyName" + && property.value == "propertyValue" + && property.type == "STRING" + ) + && vulns.all(vuln, + vuln.uuid == "__VULN_UUID__" + && vuln.id == "CVE-001" + && vuln.source == "NVD" + && 666 in vuln.cwes + && vuln.aliases + .map(alias, alias.source + ":" + alias.id) + .all(alias, alias in [ + "NVD:CVE-001", + "GITHUB:GHSA-001", + "GSD:GSD-001", + "INTERNAL:INT-001", + "OSV:OSV-001", + "SNYK:SNYK-001", + "OSSINDEX:SONATYPE-001", + "VULNDB:VULNDB-001" + ]) + && vuln.created == timestamp("1970-01-01T00:00:00.666Z") + && vuln.published == timestamp("1970-01-01T00:00:00.777Z") + && vuln.updated == timestamp("1970-01-01T00:00:00.888Z") + ) + """ + .replace("__COMPONENT_UUID__", component.getUuid().toString()) + .replace("__PROJECT_UUID__", project.getUuid().toString()) + .replace("__RESOLVED_LICENSE_UUID__", license.getUuid().toString()) + .replace("__LICENSE_GROUP_UUID__", licenseGroup.getUuid().toString()) + .replace("__VULN_UUID__", vuln.getUuid().toString()), PolicyViolation.Type.OPERATIONAL); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(project)).hasSize(1); + } + @Test public void testIsDirectDependency() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); From 499e97df01ca6c4756602775cd981e51844f8445 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 21 Sep 2023 13:42:13 +0200 Subject: [PATCH 25/46] Add remaining fields to `testWithAllFields` Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 15 +++++++-- .../policy/cel/CelPolicyScriptHost.java | 15 +++++++++ .../policy/cel/CelPolicyEngineTest.java | 31 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index b4c8183a4..ef503d529 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -21,6 +21,7 @@ import org.dependencytrack.model.PolicyCondition.Subject; import org.dependencytrack.model.PolicyViolation; import org.dependencytrack.model.Project; +import org.dependencytrack.model.Severity; import org.dependencytrack.model.VulnerabilityAlias; import org.dependencytrack.persistence.CollectionIntegerConverter; import org.dependencytrack.persistence.QueryManager; @@ -41,10 +42,12 @@ import org.dependencytrack.policy.cel.mapping.ProjectPropertyProjection; import org.dependencytrack.policy.cel.mapping.VulnerabilityProjection; import org.dependencytrack.util.NotificationUtil; +import org.dependencytrack.util.VulnerabilityUtil; import org.hyades.proto.policy.v1.Vulnerability; import org.projectnessie.cel.tools.ScriptCreateException; import org.projectnessie.cel.tools.ScriptException; +import java.math.BigDecimal; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -530,8 +533,7 @@ private static org.hyades.proto.policy.v1.Vulnerability mapToProto(final Vulnera .setSource(trimToEmpty(projection.source)) .setCvssv2Vector(trimToEmpty(projection.cvssV2Vector)) .setCvssv3Vector(trimToEmpty(projection.cvssV3Vector)) - .setOwaspRrVector(trimToEmpty(projection.owaspRrVector)) - .setSeverity(trimToEmpty(projection.severity)); + .setOwaspRrVector(trimToEmpty(projection.owaspRrVector)); Optional.ofNullable(projection.cvssV2BaseScore).ifPresent(builder::setCvssv2BaseScore); Optional.ofNullable(projection.cvssV2ImpactSubScore).ifPresent(builder::setCvssv2ImpactSubscore); Optional.ofNullable(projection.cvssV2ExploitabilitySubScore).ifPresent(builder::setCvssv2ExploitabilitySubscore); @@ -552,6 +554,15 @@ private static org.hyades.proto.policy.v1.Vulnerability mapToProto(final Vulnera .map(new CollectionIntegerConverter()::convertToAttribute) .ifPresent(builder::addAllCwes); + // Workaround for https://github.com/DependencyTrack/dependency-track/issues/2474. + final Severity severity = VulnerabilityUtil.getSeverity(projection.severity, + Optional.ofNullable(projection.cvssV2BaseScore).map(BigDecimal::valueOf).orElse(null), + Optional.ofNullable(projection.cvssV3BaseScore).map(BigDecimal::valueOf).orElse(null), + Optional.ofNullable(projection.owaspRrLikelihoodScore).map(BigDecimal::valueOf).orElse(null), + Optional.ofNullable(projection.owaspRrTechnicalImpactScore).map(BigDecimal::valueOf).orElse(null), + Optional.ofNullable(projection.owaspRrBusinessImpactScore).map(BigDecimal::valueOf).orElse(null)); + builder.setSeverity(severity.name()); + if (projection.aliasesJson != null) { try { OBJECT_MAPPER.readValue(projection.aliasesJson, VULNERABILITY_ALIASES_TYPE_REF).stream() diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java index 7a061cd3e..c95fab2ff 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -31,6 +31,7 @@ import static org.dependencytrack.policy.cel.CelPolicyLibrary.FUNC_MATCHES_RANGE; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_PROJECT; +import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_VULNERABILITY; import static org.projectnessie.cel.Issues.newIssues; import static org.projectnessie.cel.common.Source.newTextSource; @@ -118,6 +119,20 @@ private static MultiValuedMap analyzeRequirements(final CheckedExp // Fields that are accessed directly are always a requirement. final MultiValuedMap requirements = visitor.getAccessedFieldsByType(); + // Special case for vulnerability severity: The "true" severity may or may not be persisted + // in the SEVERITY database column. To compute the actual severity, CVSSv2, CVSSv3, and OWASP RR + // scores may be required. See https://github.com/DependencyTrack/dependency-track/issues/2474 + if (requirements.containsKey(TYPE_VULNERABILITY) + && requirements.get(TYPE_VULNERABILITY).contains("severity")) { + requirements.putAll(TYPE_VULNERABILITY, List.of( + "cvssv2_base_score", + "cvssv3_base_score", + "owasp_rr_likelihood_score", + "owasp_rr_technical_impact_score", + "owasp_rr_business_impact_score" + )); + } + // Custom functions may access certain fields implicitly, in a way that is not visible // to the AST visitor. To compensate, we hardcode the functions' requirements here. // TODO: This should be restructured to be more generic. diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index 9c13718f4..bd7258978 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import java.io.File; +import java.math.BigDecimal; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -200,6 +201,21 @@ public void testWithAllFields() { vuln.setCreated(new java.util.Date(666)); vuln.setPublished(new java.util.Date(777)); vuln.setUpdated(new java.util.Date(888)); + vuln.setSeverity(Severity.INFO); + vuln.setCvssV2BaseScore(BigDecimal.valueOf(6.0)); + vuln.setCvssV2ImpactSubScore(BigDecimal.valueOf(6.4)); + vuln.setCvssV2ExploitabilitySubScore(BigDecimal.valueOf(6.8)); + vuln.setCvssV2Vector("(AV:N/AC:M/Au:S/C:P/I:P/A:P)"); + vuln.setCvssV3BaseScore(BigDecimal.valueOf(9.1)); + vuln.setCvssV3ImpactSubScore(BigDecimal.valueOf(5.3)); + vuln.setCvssV3ExploitabilitySubScore(BigDecimal.valueOf(3.1)); + vuln.setCvssV3Vector("CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:H/A:L"); + vuln.setOwaspRRLikelihoodScore(BigDecimal.valueOf(4.5)); + vuln.setOwaspRRTechnicalImpactScore(BigDecimal.valueOf(5.0)); + vuln.setOwaspRRBusinessImpactScore(BigDecimal.valueOf(3.75)); + vuln.setOwaspRRVector("(SL:5/M:5/O:2/S:9/ED:4/EE:2/A:7/ID:2/LC:2/LI:2/LAV:7/LAC:9/FD:3/RD:5/NC:0/PV:7)"); + vuln.setEpssScore(BigDecimal.valueOf(0.6)); + vuln.setEpssPercentile(BigDecimal.valueOf(0.2)); qm.persist(vuln); qm.addVulnerability(vuln, component, AnalyzerIdentity.INTERNAL_ANALYZER); @@ -288,6 +304,21 @@ && has(project.last_bom_import) && vuln.created == timestamp("1970-01-01T00:00:00.666Z") && vuln.published == timestamp("1970-01-01T00:00:00.777Z") && vuln.updated == timestamp("1970-01-01T00:00:00.888Z") + && vuln.severity == "INFO" + && vuln.cvssv2_base_score == 6.0 + && vuln.cvssv2_impact_subscore == 6.4 + && vuln.cvssv2_exploitability_subscore == 6.8 + && vuln.cvssv2_vector == "(AV:N/AC:M/Au:S/C:P/I:P/A:P)" + && vuln.cvssv3_base_score == 9.1 + && vuln.cvssv3_impact_subscore == 5.3 + && vuln.cvssv3_exploitability_subscore == 3.1 + && vuln.cvssv3_vector == "CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:H/A:L" + && vuln.owasp_rr_likelihood_score == 4.5 + && vuln.owasp_rr_technical_impact_score == 5.0 + && vuln.owasp_rr_business_impact_score == 3.75 + && vuln.owasp_rr_vector == "(SL:5/M:5/O:2/S:9/ED:4/EE:2/A:7/ID:2/LC:2/LI:2/LAV:7/LAC:9/FD:3/RD:5/NC:0/PV:7)" + && vuln.epss_score == 0.6 + && vuln.epss_percentile == 0.2 ) """ .replace("__COMPONENT_UUID__", component.getUuid().toString()) From 047d5a5c601f647e7ebee6bb2be53e3eb315697c Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 21 Sep 2023 14:31:54 +0200 Subject: [PATCH 26/46] Add test for vuln severity evaluation Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngineTest.java | 92 +++++++++++-------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index bd7258978..b92c94315 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -121,13 +121,23 @@ public void test() { existingViolation.setTimestamp(new java.util.Date()); qm.persist(existingViolation); - final var policyEngine = new CelPolicyEngine(); - policyEngine.evaluateProject(project.getUuid()); + new CelPolicyEngine().evaluateProject(project.getUuid()); final List violations = qm.getAllPolicyViolations(component); assertThat(violations).isNotEmpty(); } + /** + * (Regression-)Test for ensuring that all data available in the policy expression context + * can be accessed in the expression at runtime. + *

+ * Data being available means: + *

    + *
  • Expression requirements were analyzed correctly
    • + *
  • Database was retrieved from the database correctly
    • + *
  • The mapping from DB data to CEL Protobuf models worked as expected
    • + *
+ */ @Test public void testWithAllFields() { final var project = new Project(); @@ -255,7 +265,7 @@ public void testWithAllFields() { && component.blake2b_512 == "componentBlake2b_512" && component.blake3 == "componentBlake3" && component.license_name == "componentLicenseName" - && !has(component.license_expression) + && !has(component.license_expression) // Requires https://github.com/DependencyTrack/dependency-track/pull/2400 to be ported to Hyades && component.resolved_license.uuid == "__RESOLVED_LICENSE_UUID__" && component.resolved_license.id == "resolvedLicenseId" && component.resolved_license.name == "resolvedLicenseName" @@ -332,11 +342,11 @@ && has(project.last_bom_import) } @Test - public void testIsDirectDependency() { + public void testProjectDependsOnComponent() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); final PolicyCondition policyCondition = qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.is_direct_dependency + project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) """); policyCondition.setViolationType(PolicyViolation.Type.OPERATIONAL); qm.persist(policyCondition); @@ -364,43 +374,56 @@ public void testIsDirectDependency() { policyEngine.evaluateProject(project.getUuid()); assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); - assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); + assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); } @Test - public void testProjectDependsOnComponent() { + public void testVulnerabilitySeverity() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - final PolicyCondition policyCondition = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) - """); - policyCondition.setViolationType(PolicyViolation.Type.OPERATIONAL); - qm.persist(policyCondition); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + vulns.exists(vuln, vuln.severity == "CRITICAL") + """, PolicyViolation.Type.SECURITY); final var project = new Project(); - project.setName("foo"); + project.setName("acme-app"); qm.persist(project); - final var componentA = new Component(); - componentA.setProject(project); - componentA.setName("bar"); - qm.persist(componentA); + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); - final var componentB = new Component(); - componentB.setProject(project); - componentB.setName("baz"); - qm.persist(componentB); + // Create a vulnerability that has all scores (CVSSv2, CVSSv3, OWASP RR) + // available, but no severity is set explicitly. + // + // Even though the expression only accesses the `severity` field, the policy + // engine should fetch all scores in order to derive the severity from them. + // Note that when multiple scores are available, the highest severity wins. + // + // The highest severity among the scores below is CRITICAL from CVSSv3. - project.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentA).toJSON())); - qm.persist(project); - componentA.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentB).toJSON())); - qm.persist(componentA); + final var vuln = new Vulnerability(); + vuln.setVulnId("CVE-123"); + vuln.setSource(Vulnerability.Source.NVD); + // vuln.setSeverity(Severity.INFO); + vuln.setCvssV2BaseScore(BigDecimal.valueOf(6.0)); + vuln.setCvssV2ImpactSubScore(BigDecimal.valueOf(6.4)); + vuln.setCvssV2ExploitabilitySubScore(BigDecimal.valueOf(6.8)); + vuln.setCvssV2Vector("(AV:N/AC:M/Au:S/C:P/I:P/A:P)"); + vuln.setCvssV3BaseScore(BigDecimal.valueOf(9.1)); + vuln.setCvssV3ImpactSubScore(BigDecimal.valueOf(5.3)); + vuln.setCvssV3ExploitabilitySubScore(BigDecimal.valueOf(3.1)); + vuln.setCvssV3Vector("CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:H/A:L"); + vuln.setOwaspRRLikelihoodScore(BigDecimal.valueOf(4.5)); + vuln.setOwaspRRTechnicalImpactScore(BigDecimal.valueOf(5.0)); + vuln.setOwaspRRBusinessImpactScore(BigDecimal.valueOf(3.75)); + vuln.setOwaspRRVector("(SL:5/M:5/O:2/S:9/ED:4/EE:2/A:7/ID:2/LC:2/LI:2/LAV:7/LAC:9/FD:3/RD:5/NC:0/PV:7)"); + qm.persist(vuln); - final var policyEngine = new CelPolicyEngine(); + qm.addVulnerability(vuln, component, AnalyzerIdentity.INTERNAL_ANALYZER); - policyEngine.evaluateProject(project.getUuid()); - assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); - assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); } @Test @@ -431,14 +454,7 @@ public void testMatchesRange() { componentB.setVersion("v2.0.0"); qm.persist(componentB); - project.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentA).toJSON())); - qm.persist(project); - componentA.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentB).toJSON())); - qm.persist(componentA); - - final var policyEngine = new CelPolicyEngine(); - - policyEngine.evaluateProject(project.getUuid()); + new CelPolicyEngine().evaluateProject(project.getUuid()); assertThat(qm.getAllPolicyViolations(componentA)).hasSize(1); assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); } From e824c5c9238f5e0843932229933c7a6b5328c57d Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 21 Sep 2023 15:04:32 +0200 Subject: [PATCH 27/46] Remove un-implemented `depends_on` function; Add proper logging for custom functions Signed-off-by: nscuro --- .../policy/cel/CelPolicyLibrary.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index da1e65f25..688063e41 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -69,12 +69,6 @@ public List getCompileOptions() { ), Decls.newFunction( FUNC_DEPENDS_ON, - // component.depends_on(org.hyades.policy.v1.Component{name: "foo"}) - Decls.newInstanceOverload( - "component_depends_on_component_bool", - List.of(TYPE_COMPONENT, TYPE_COMPONENT), - Decls.Bool - ), // project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) Decls.newInstanceOverload( "project_depends_on_component_bool", @@ -148,10 +142,8 @@ private static Val dependsOnFunc(final Val lhs, final Val rhs) { } if (lhs.value() instanceof final Project project) { + // project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) return Types.boolOf(dependsOn(project, leafComponent)); - } else if (lhs.value() instanceof final Component rootComponent) { - // TODO: Traverse dep graph from rootComponent downwards and look for leafComponent - return Types.boolOf(dependsOn(rootComponent, leafComponent)); } return Err.maybeNoSuchOverloadErr(lhs); @@ -175,8 +167,10 @@ private static Val isDependencyOfFunc(final Val lhs, final Val rhs) { private static Val matchesRangeFunc(final Val lhs, final Val rhs) { final String version; if (lhs.value() instanceof final Component lhsValue) { + // component.matches_range("vers:golang/>0|!=v3.2.1") version = lhsValue.getVersion(); } else if (lhs.value() instanceof final Project lhsValue) { + // project.matches_range("vers:golang/>0|!=v3.2.1") version = lhsValue.getVersion(); } else { return Err.maybeNoSuchOverloadErr(lhs); @@ -195,11 +189,16 @@ private static Val matchesRangeFunc(final Val lhs, final Val rhs) { private static boolean dependsOn(final Project project, final Component component) { if (project.getUuid().isBlank()) { // Need a UUID for our starting point. + LOGGER.warn("%s: project does not have a UUID; Unable to evaluate, returning false" + .formatted(FUNC_DEPENDS_ON)); return false; } final Pair> filterAndParams = toFilterAndParams(component); if (filterAndParams == null) { + LOGGER.warn(""" + %s: Unable to construct filter expression from component %s; \ + Unable to evaluate, returning false""".formatted(FUNC_DEPENDS_ON, component)); return false; } @@ -231,6 +230,8 @@ private static boolean dependsOn(final Component rootComponent, final Component private static boolean isDependencyOf(final Component leafComponent, final Component rootComponent) { if (leafComponent.getUuid().isBlank()) { // Need a UUID for our starting point. + LOGGER.warn("%s: leaf component does not have a UUID; Unable to evaluate, returning false" + .formatted(FUNC_IS_DEPENDENCY_OF)); return false; } @@ -259,6 +260,9 @@ private static boolean isDependencyOf(final Component leafComponent, final Compo } if (filters.isEmpty()) { + LOGGER.warn(""" + %s: Unable to construct filter expression from root component %s; \ + Unable to evaluate, returning false""".formatted(FUNC_IS_DEPENDENCY_OF, rootComponent)); return false; } @@ -322,7 +326,7 @@ CASE WHEN (%s) THEN TRUE ELSE FALSE END AS "FOUND", } } } catch (SQLException e) { - LOGGER.warn("Failed to execute query", e); + LOGGER.warn("%s: Failed to execute query: %s".formatted(FUNC_IS_DEPENDENCY_OF, query), e); } finally { jdoConnection.close(); } @@ -335,6 +339,8 @@ private static boolean matchesRange(final String version, final String versStr) try { return Vers.parse(versStr).contains(version); } catch (VersException e) { + LOGGER.warn("%s: Failed to check if version %s matches range %s" + .formatted(FUNC_MATCHES_RANGE, version, versStr), e); return false; } } From 7978c1e512b2978406065fab4161a04b1e1a90d8 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 21 Sep 2023 15:05:10 +0200 Subject: [PATCH 28/46] Handle invalid scripts and script runtime failures Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 21 ++- .../policy/cel/CelPolicyEngineTest.java | 147 +++++++++++++++--- 2 files changed, 141 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index ef503d529..3d85fdb28 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -300,6 +300,7 @@ private List> getApplicableConditionScrip .map(this::buildConditionScriptSrc) .filter(Objects::nonNull) .map(this::compileConditionScript) + .filter(Objects::nonNull) .toList(); } @@ -329,13 +330,15 @@ private static MultiValuedMap determineScriptRequirements(final Co private Pair buildConditionScriptSrc(final PolicyCondition policyCondition) { final CelPolicyScriptSourceBuilder scriptBuilder = SCRIPT_BUILDERS.get(policyCondition.getSubject()); if (scriptBuilder == null) { - LOGGER.warn("No script builder found that is capable of handling subjects of type %s".formatted(policyCondition.getSubject())); + LOGGER.warn(""" + No script builder found that is capable of handling subjects of type %s;\ + Condition will be skipped""".formatted(policyCondition.getSubject())); return null; } final String scriptSrc = scriptBuilder.apply(policyCondition); if (scriptSrc == null) { - LOGGER.warn("Script builder was unable to create a script for condition %s".formatted(policyCondition.getUuid())); + LOGGER.warn("Unable to create CEL script for condition %s; Condition will be skipped".formatted(policyCondition.getUuid())); return null; } @@ -347,7 +350,9 @@ private Pair compileConditionScript(final Pair try { script = scriptHost.compile(conditionScriptSrcPair.getRight()); } catch (ScriptCreateException e) { - throw new RuntimeException(e); + LOGGER.warn("Failed to compile script for condition %s; Condition will be skipped" + .formatted(conditionScriptSrcPair.getLeft().getUuid()), e); + return null; } return Pair.of(conditionScriptSrcPair.getLeft(), script); @@ -366,9 +371,8 @@ private static List evaluateConditions(final Collection new CelPolicyEngine().evaluateProject(project.getUuid())); + assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> + assertThat(violation.getPolicyCondition()).isEqualTo(validCondition) + ); + } + + @Test + public void testWithScriptExecutionException() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.last_bom_import == timestamp("invalid") + """, PolicyViolation.Type.OPERATIONAL); + final PolicyCondition validCondition = qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, + PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + assertThatNoException().isThrownBy(() -> new CelPolicyEngine().evaluateProject(project.getUuid())); + assertThat(qm.getAllPolicyViolations(component)).satisfiesExactly(violation -> + assertThat(violation.getPolicyCondition()).isEqualTo(validCondition) + ); + } + @Test public void testProjectDependsOnComponent() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - final PolicyCondition policyCondition = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) - """); - policyCondition.setViolationType(PolicyViolation.Type.OPERATIONAL); - qm.persist(policyCondition); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) + """, PolicyViolation.Type.OPERATIONAL); final var project = new Project(); - project.setName("foo"); + project.setName("acme-app"); qm.persist(project); final var componentA = new Component(); componentA.setProject(project); - componentA.setName("bar"); + componentA.setName("acme-lib-a"); qm.persist(componentA); final var componentB = new Component(); componentB.setProject(project); - componentB.setName("baz"); + componentB.setName("acme-lib-b"); qm.persist(componentB); project.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentA).toJSON())); @@ -377,6 +427,38 @@ public void testProjectDependsOnComponent() { assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); } + @Test + public void testComponentIsDependencyOfComponent() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.is_dependency_of(org.hyades.policy.v1.Component{name: "acme-lib-a"}) + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var componentA = new Component(); + componentA.setProject(project); + componentA.setName("acme-lib-a"); + qm.persist(componentA); + + final var componentB = new Component(); + componentB.setProject(project); + componentB.setName("acme-lib-b"); + qm.persist(componentB); + + project.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentA).toJSON())); + qm.persist(project); + componentA.setDirectDependencies("[%s]".formatted(new ComponentIdentity(componentB).toJSON())); + qm.persist(componentA); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + + assertThat(qm.getAllPolicyViolations(componentA)).isEmpty(); + assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); + } + @Test public void testVulnerabilitySeverity() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); @@ -429,28 +511,25 @@ public void testVulnerabilitySeverity() { @Test public void testMatchesRange() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - final PolicyCondition policyCondition = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - project.matches_range("vers:generic/<1") - && component.matches_range("vers:golang/>0|0| new CelPolicyEngine().evaluateProject(project.getUuid())); + assertThat(qm.getAllPolicyViolations(componentA)).isEmpty(); + assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); + } + @Test public void issue1924() { Policy policy = qm.createPolicy("Policy 1924", Policy.Operator.ALL, Policy.ViolationState.INFO); From 2f37004e80acb35ca4aecea2306d69940733f871 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 21 Sep 2023 16:42:32 +0200 Subject: [PATCH 29/46] Add `escapeQuotes` for CEL script builders Using `escapeJson` doesn't work quite right when special characters / regular expressions are provided. All we need is prevention of "breaking out" of strings, so escaping double quotes alone is sufficient. Signed-off-by: nscuro --- .../compat/CpeCelPolicyScriptSourceBuilder.java | 4 ++-- .../compat/CelPolicyScriptSourceBuilderTest.java | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilderTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java index 7a101567b..23b673f87 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CpeCelPolicyScriptSourceBuilder.java @@ -2,7 +2,7 @@ import org.dependencytrack.model.PolicyCondition; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class CpeCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @@ -10,7 +10,7 @@ public class CpeCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBui public String apply(final PolicyCondition policyCondition) { final String scriptSrc = """ component.cpe.matches("%s") - """.formatted(escapeJson(policyCondition.getValue())); + """.formatted(escapeQuotes(policyCondition.getValue())); if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { return scriptSrc; diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilderTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilderTest.java new file mode 100644 index 000000000..f7d2fa634 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilderTest.java @@ -0,0 +1,15 @@ +package org.dependencytrack.policy.cel.compat; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; + +public class CelPolicyScriptSourceBuilderTest { + + @Test + public void testEscapeQuotes() { + assertThat(escapeQuotes("\"foobar")).isEqualTo("\\\"foobar"); + } + +} \ No newline at end of file From ff9d52f214a2d1bad1cdbf6d4677cd7f907559b2 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 21 Sep 2023 16:43:17 +0200 Subject: [PATCH 30/46] Add tests for some legacy conditions Signed-off-by: nscuro --- .../compat/CelPolicyScriptSourceBuilder.java | 8 ++ .../SeverityCelPolicyScriptSourceBuilder.java | 8 +- ...SwidTagIdCelPolicyScriptSourceBuilder.java | 4 +- ...abilityIdCelPolicyScriptSourceBuilder.java | 4 +- .../policy/cel/CelPolicyEngineTest.java | 49 ------- .../policy/cel/compat/CpeConditionTest.java | 63 +++++++++ .../cel/compat/SeverityConditionTest.java | 128 ++++++++++++++++++ .../cel/compat/SwidTagIdConditionTest.java | 61 +++++++++ .../compat/VulnerabilityIdConditionTest.java | 74 ++++++++++ 9 files changed, 341 insertions(+), 58 deletions(-) create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/CpeConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/SeverityConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/SwidTagIdConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdConditionTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java index fdef88f15..2d814f34f 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CelPolicyScriptSourceBuilder.java @@ -3,6 +3,14 @@ import org.dependencytrack.model.PolicyCondition; import java.util.function.Function; +import java.util.regex.Pattern; public interface CelPolicyScriptSourceBuilder extends Function { + + Pattern QUOTES_PATTERN = Pattern.compile("\""); + + static String escapeQuotes(final String value) { + return QUOTES_PATTERN.matcher(value).replaceAll("\\\\\""); + } + } diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java index 58b01ec43..c10c0d1a5 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/SeverityCelPolicyScriptSourceBuilder.java @@ -2,22 +2,20 @@ import org.dependencytrack.model.PolicyCondition; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class SeverityCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @Override public String apply(final PolicyCondition policyCondition) { - final String escapedPolicyValue = escapeJson(policyCondition.getValue()); - if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { return """ vulns.exists(vuln, vuln.severity == "%s") - """.formatted(escapedPolicyValue); + """.formatted(escapeQuotes(policyCondition.getValue())); } else if (policyCondition.getOperator() == PolicyCondition.Operator.IS_NOT) { return """ vulns.exists(vuln, vuln.severity != "%s") - """.formatted(escapedPolicyValue); + """.formatted(escapeQuotes(policyCondition.getValue())); } return null; diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java index f5e46af6c..3053f0594 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/SwidTagIdCelPolicyScriptSourceBuilder.java @@ -2,7 +2,7 @@ import org.dependencytrack.model.PolicyCondition; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class SwidTagIdCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @@ -10,7 +10,7 @@ public class SwidTagIdCelPolicyScriptSourceBuilder implements CelPolicyScriptSou public String apply(final PolicyCondition policyCondition) { final String scriptSrc = """ component.swid_tag_id.matches("%s") - """.formatted(escapeJson(policyCondition.getValue())); + """.formatted(escapeQuotes(policyCondition.getValue())); if (policyCondition.getOperator() == PolicyCondition.Operator.MATCHES) { return scriptSrc; diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java index 508a2c276..43ac2f8af 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdCelPolicyScriptSourceBuilder.java @@ -20,14 +20,14 @@ import org.dependencytrack.model.PolicyCondition; -import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder.escapeQuotes; public class VulnerabilityIdCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @Override public String apply(final PolicyCondition policyCondition) { final String scriptSrc = """ vulns.exists(v, v.id == "%s") - """.formatted(escapeJson(policyCondition.getValue())); + """.formatted(escapeQuotes(policyCondition.getValue())); if (policyCondition.getOperator() == PolicyCondition.Operator.IS) { return scriptSrc; diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index 8452c6394..807b4d846 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -459,55 +459,6 @@ public void testComponentIsDependencyOfComponent() { assertThat(qm.getAllPolicyViolations(componentB)).hasSize(1); } - @Test - public void testVulnerabilitySeverity() { - final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - vulns.exists(vuln, vuln.severity == "CRITICAL") - """, PolicyViolation.Type.SECURITY); - - final var project = new Project(); - project.setName("acme-app"); - qm.persist(project); - - final var component = new Component(); - component.setProject(project); - component.setName("acme-lib"); - qm.persist(component); - - // Create a vulnerability that has all scores (CVSSv2, CVSSv3, OWASP RR) - // available, but no severity is set explicitly. - // - // Even though the expression only accesses the `severity` field, the policy - // engine should fetch all scores in order to derive the severity from them. - // Note that when multiple scores are available, the highest severity wins. - // - // The highest severity among the scores below is CRITICAL from CVSSv3. - - final var vuln = new Vulnerability(); - vuln.setVulnId("CVE-123"); - vuln.setSource(Vulnerability.Source.NVD); - // vuln.setSeverity(Severity.INFO); - vuln.setCvssV2BaseScore(BigDecimal.valueOf(6.0)); - vuln.setCvssV2ImpactSubScore(BigDecimal.valueOf(6.4)); - vuln.setCvssV2ExploitabilitySubScore(BigDecimal.valueOf(6.8)); - vuln.setCvssV2Vector("(AV:N/AC:M/Au:S/C:P/I:P/A:P)"); - vuln.setCvssV3BaseScore(BigDecimal.valueOf(9.1)); - vuln.setCvssV3ImpactSubScore(BigDecimal.valueOf(5.3)); - vuln.setCvssV3ExploitabilitySubScore(BigDecimal.valueOf(3.1)); - vuln.setCvssV3Vector("CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:H/A:L"); - vuln.setOwaspRRLikelihoodScore(BigDecimal.valueOf(4.5)); - vuln.setOwaspRRTechnicalImpactScore(BigDecimal.valueOf(5.0)); - vuln.setOwaspRRBusinessImpactScore(BigDecimal.valueOf(3.75)); - vuln.setOwaspRRVector("(SL:5/M:5/O:2/S:9/ED:4/EE:2/A:7/ID:2/LC:2/LI:2/LAV:7/LAC:9/FD:3/RD:5/NC:0/PV:7)"); - qm.persist(vuln); - - qm.addVulnerability(vuln, component, AnalyzerIdentity.INTERNAL_ANALYZER); - - new CelPolicyEngine().evaluateProject(project.getUuid()); - assertThat(qm.getAllPolicyViolations(component)).hasSize(1); - } - @Test public void testMatchesRange() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CpeConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CpeConditionTest.java new file mode 100644 index 000000000..ed65a5026 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CpeConditionTest.java @@ -0,0 +1,63 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyCondition.Operator; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.dependencytrack.model.PolicyCondition.Operator.MATCHES; + +@RunWith(JUnitParamsRunner.class) +public class CpeConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // MATCHES with exact match + new Object[]{MATCHES, "cpe:/a:acme:application:1.0.0", "cpe:/a:acme:application:1.0.0", true}, + // MATCHES with regex match + new Object[]{MATCHES, "cpe:/a:acme:\\\\w+:[0-9].0.0", "cpe:/a:acme:application:1.0.0", true}, + // MATCHES with no match + new Object[]{MATCHES, "cpe:/a:acme:application:1.0.0", "cpe:/a:acme:application:9.9.9", false}, + // NO_MATCH with no match + new Object[]{Operator.NO_MATCH, "cpe:/a:acme:application:1.0.0", "cpe:/a:acme:application:9.9.9", true}, + // NO_MATCH with exact match + new Object[]{Operator.NO_MATCH, "cpe:/a:acme:application:1.0.0", "cpe:/a:acme:application:1.0.0", false}, + // MATCHES with quotes + new Object[]{MATCHES, "\"cpe:/a:acme:application:1.0.0", "\"cpe:/a:acme:application:1.0.0", true} + }; + } + + + @Test + @Parameters(method = "parameters") + public void testCondition(final Operator operator, final String conditionCpe, final String componentCpe, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.CPE, operator, conditionCpe); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setCpe(componentCpe); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/SeverityConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/SeverityConditionTest.java new file mode 100644 index 000000000..dc8d62afb --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/SeverityConditionTest.java @@ -0,0 +1,128 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.Policy.ViolationState; +import org.dependencytrack.model.PolicyCondition.Operator; +import org.dependencytrack.model.PolicyCondition.Subject; +import org.dependencytrack.model.PolicyViolation.Type; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Severity; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class SeverityConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // IS with exact match + new Object[]{Operator.IS, "CRITICAL", "CRITICAL", true}, + // IS with regex match (regex is not supported by this condition) + new Object[]{Operator.IS, "CRI[A-Z]+", "CRITICAL", false}, + // IS with no match + new Object[]{Operator.IS, "CRITICAL", "LOW", false}, + // IS_NOT with no match + new Object[]{Operator.IS_NOT, "CRITICAL", "LOW", true}, + // IS_NOT with exact match + new Object[]{Operator.IS_NOT, "UNASSIGNED", "UNASSIGNED", false}, + // IS with quotes (actualSeverity can't have quotes because it's an enum) + new Object[]{Operator.IS, "\"CRITICAL", "CRITICAL", false} + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final Operator operator, final String conditionSeverity, final String actualSeverity, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, ViolationState.INFO); + qm.createPolicyCondition(policy, Subject.SEVERITY, operator, conditionSeverity); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + final var vulnA = new Vulnerability(); + vulnA.setVulnId("INT-123"); + vulnA.setSource(Vulnerability.Source.INTERNAL); + vulnA.setSeverity(Severity.valueOf(actualSeverity)); + qm.persist(vulnA); + + final var vulnB = new Vulnerability(); + vulnB.setVulnId("INT-666"); + vulnB.setSource(Vulnerability.Source.INTERNAL); + qm.persist(vulnB); + + qm.addVulnerability(vulnA, component, AnalyzerIdentity.INTERNAL_ANALYZER); + qm.addVulnerability(vulnB, component, AnalyzerIdentity.INTERNAL_ANALYZER); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + + @Test + public void testSeverityCalculation() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, ViolationState.FAIL); + qm.createPolicyCondition(policy, Subject.SEVERITY, Operator.IS, Severity.CRITICAL.name(), Type.SECURITY); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + // Create a vulnerability that has all scores (CVSSv2, CVSSv3, OWASP RR) + // available, but no severity is set explicitly. + // + // Even though the expression only accesses the `severity` field, the policy + // engine should fetch all scores in order to derive the severity from them. + // Note that when multiple scores are available, the highest severity wins. + // + // The highest severity among the scores below is CRITICAL from CVSSv3. + + final var vuln = new Vulnerability(); + vuln.setVulnId("CVE-123"); + vuln.setSource(Vulnerability.Source.NVD); + // vuln.setSeverity(Severity.INFO); + vuln.setCvssV2BaseScore(BigDecimal.valueOf(6.0)); + vuln.setCvssV2ImpactSubScore(BigDecimal.valueOf(6.4)); + vuln.setCvssV2ExploitabilitySubScore(BigDecimal.valueOf(6.8)); + vuln.setCvssV2Vector("(AV:N/AC:M/Au:S/C:P/I:P/A:P)"); + vuln.setCvssV3BaseScore(BigDecimal.valueOf(9.1)); + vuln.setCvssV3ImpactSubScore(BigDecimal.valueOf(5.3)); + vuln.setCvssV3ExploitabilitySubScore(BigDecimal.valueOf(3.1)); + vuln.setCvssV3Vector("CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:C/C:L/I:H/A:L"); + vuln.setOwaspRRLikelihoodScore(BigDecimal.valueOf(4.5)); + vuln.setOwaspRRTechnicalImpactScore(BigDecimal.valueOf(5.0)); + vuln.setOwaspRRBusinessImpactScore(BigDecimal.valueOf(3.75)); + vuln.setOwaspRRVector("(SL:5/M:5/O:2/S:9/ED:4/EE:2/A:7/ID:2/LC:2/LI:2/LAV:7/LAC:9/FD:3/RD:5/NC:0/PV:7)"); + qm.persist(vuln); + + qm.addVulnerability(vuln, component, AnalyzerIdentity.INTERNAL_ANALYZER); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/SwidTagIdConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/SwidTagIdConditionTest.java new file mode 100644 index 000000000..d94541adc --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/SwidTagIdConditionTest.java @@ -0,0 +1,61 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyCondition.Operator; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class SwidTagIdConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // MATCHES with exact match + new Object[]{Operator.MATCHES, "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", true}, + // MATCHES with regex match + new Object[]{Operator.MATCHES, "swidgen-242eb18a-[a-z0-9]{4}-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", true}, + // MATCHES with no match + new Object[]{Operator.MATCHES, "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_6.6.6", false}, + // NO_MATCH with no match + new Object[]{Operator.NO_MATCH, "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_6.6.6", true}, + // NO_MATCH with exact match + new Object[]{Operator.NO_MATCH, "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", false}, + // MATCHES with quotes + new Object[]{Operator.MATCHES, "\"swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", "\"swidgen-242eb18a-503e-ca37-393b-cf156ef09691_9.1.1", true} + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final Operator operator, final String conditionSwidTagId, final String componentSwidTagId, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.SWID_TAGID, operator, conditionSwidTagId); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setSwidTagId(componentSwidTagId); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdConditionTest.java new file mode 100644 index 000000000..63f27958d --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/VulnerabilityIdConditionTest.java @@ -0,0 +1,74 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class VulnerabilityIdConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // IS with exact match + new Object[]{PolicyCondition.Operator.IS, "CVE-123", "CVE-123", true}, + // IS with regex match (regex is not supported by this condition) + new Object[]{PolicyCondition.Operator.IS, "CVE-[0-9]+", "CVE-123", false}, + // IS with no match + new Object[]{PolicyCondition.Operator.IS, "CVE-123", "CVE-666", false}, + // IS_NOT with no match + new Object[]{PolicyCondition.Operator.IS_NOT, "CVE-123", "CVE-666", true}, + // IS_NOT with exact match + new Object[]{PolicyCondition.Operator.IS_NOT, "CVE-123", "CVE-123", false}, + // IS with quotes + new Object[]{PolicyCondition.Operator.IS, "\"CVE-123", "\"CVE-123", true} + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final PolicyCondition.Operator operator, final String conditionVulnId, final String actualVulnId, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.VULNERABILITY_ID, operator, conditionVulnId); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + final var vulnA = new Vulnerability(); + vulnA.setVulnId(actualVulnId); + vulnA.setSource(Vulnerability.Source.INTERNAL); + qm.persist(vulnA); + + final var vulnB = new Vulnerability(); + vulnB.setVulnId("INT-666"); + vulnB.setSource(Vulnerability.Source.INTERNAL); + qm.persist(vulnB); + + qm.addVulnerability(vulnA, component, AnalyzerIdentity.INTERNAL_ANALYZER); + qm.addVulnerability(vulnB, component, AnalyzerIdentity.INTERNAL_ANALYZER); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} From 706922c03041b7b6a7bbd86d89d9834ec961f302 Mon Sep 17 00:00:00 2001 From: nscuro Date: Fri, 22 Sep 2023 16:21:27 +0200 Subject: [PATCH 31/46] More tests for `CelPolicyEngine` Signed-off-by: nscuro --- .../policy/cel/CelPolicyQueryManager.java | 2 - .../policy/cel/CelPolicyEngineTest.java | 371 +++++++++++++----- 2 files changed, 283 insertions(+), 90 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java index 3c2fe362b..4d68b00f9 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java @@ -552,12 +552,10 @@ List getApplicablePolicies(final Project project) { return policies; } - // TODO: Move to ProjectQueryManager List getParents(final Project project) { return getParents(project.getUuid(), new ArrayList<>()); } - // TODO: Move to ProjectQueryManager List getParents(final UUID uuid, final List parents) { final UUID parentUuid; final Query query = pm.newQuery(Project.class); diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index 807b4d846..b93330ada 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -15,6 +15,7 @@ import org.dependencytrack.model.PolicyViolation; import org.dependencytrack.model.Project; import org.dependencytrack.model.Severity; +import org.dependencytrack.model.Tag; import org.dependencytrack.model.Vulnerability; import org.dependencytrack.model.VulnerabilityAlias; import org.dependencytrack.persistence.DefaultObjectGenerator; @@ -29,9 +30,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.sql.Date; -import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -54,80 +52,6 @@ public void setUp() throws Exception { ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getDescription()); } - @Test - public void test() { - final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); - final PolicyCondition policyConditionA = qm.createPolicyCondition(policy, - PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - "critical" in project.tags - && component.name == "bar" - && vulns.exists(v, v.source == "SNYK") - && component.resolved_license.groups.exists(lg, lg.name == "Permissive") - """); - policyConditionA.setViolationType(PolicyViolation.Type.OPERATIONAL); - qm.persist(policyConditionA); - - final var policy2 = qm.createPolicy("policy2", Policy.Operator.ALL, Policy.ViolationState.WARN); - qm.createPolicyCondition(policy2, PolicyCondition.Subject.VULNERABILITY_ID, PolicyCondition.Operator.IS, "CVE-123"); - - final var policy3 = qm.createPolicy("policy3", Policy.Operator.ALL, Policy.ViolationState.INFO); - final PolicyCondition condition3 = qm.createPolicyCondition(policy3, PolicyCondition.Subject.SWID_TAGID, PolicyCondition.Operator.MATCHES, "foo"); - - final var policy4 = qm.createPolicy("policy4", Policy.Operator.ALL, Policy.ViolationState.INFO); - qm.createPolicyCondition(policy4, PolicyCondition.Subject.CWE, PolicyCondition.Operator.CONTAINS_ALL, "CWE-666, CWE-123, 555"); - - final var project = new Project(); - project.setName("foo"); - qm.persist(project); - qm.bind(project, List.of( - qm.createTag("public-facing"), - qm.createTag("critical") - )); - - final var license = new License(); - license.setName("MIT"); - license.setLicenseId("MIT"); - qm.persist(license); - - final var licenseGroup = new LicenseGroup(); - licenseGroup.setName("Permissive"); - licenseGroup.setLicenses(List.of(license)); - qm.persist(licenseGroup); - - final var component = new Component(); - component.setProject(project); - component.setName("bar"); - component.setResolvedLicense(license); - qm.persist(component); - - final var vulnA = new Vulnerability(); - vulnA.setVulnId("CVE-123"); - vulnA.setSource(Vulnerability.Source.NVD); - vulnA.setCreated(Date.from(LocalDateTime.now().minusYears(1).toInstant(ZoneOffset.UTC))); - qm.persist(vulnA); - - final var vulnB = new Vulnerability(); - vulnB.setVulnId("SNYK-123"); - vulnB.setSource(Vulnerability.Source.SNYK); - vulnB.setCwes(List.of(555, 666, 123)); - qm.persist(vulnB); - - qm.addVulnerability(vulnA, component, AnalyzerIdentity.INTERNAL_ANALYZER); - qm.addVulnerability(vulnB, component, AnalyzerIdentity.SNYK_ANALYZER); - - final var existingViolation = new PolicyViolation(); - existingViolation.setComponent(component); - existingViolation.setPolicyCondition(condition3); - existingViolation.setType(PolicyViolation.Type.OPERATIONAL); - existingViolation.setTimestamp(new java.util.Date()); - qm.persist(existingViolation); - - new CelPolicyEngine().evaluateProject(project.getUuid()); - - final List violations = qm.getAllPolicyViolations(component); - assertThat(violations).isNotEmpty(); - } - /** * (Regression-)Test for ensuring that all data available in the policy expression context * can be accessed in the expression at runtime. @@ -342,6 +266,264 @@ && has(project.last_bom_import) assertThat(qm.getAllPolicyViolations(project)).hasSize(1); } + @Test + public void testWithPolicyOperatorAnyAndAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "acme-lib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(2); + } + + @Test + public void testWithPolicyOperatorAnyAndNotAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "someOtherComponentThatIsNotAcmeLib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void testWithPolicyOperatorAnyAndNoConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "someOtherProjectThatIsNotAcmeApp" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "someOtherComponentThatIsNotAcmeLib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void testWithPolicyOperatorAllAndAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "acme-lib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(2); + } + + @Test + public void testWithPolicyOperatorAllAndNotAllConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "acme-app" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "someOtherComponentThatIsNotAcmeLib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void testWithPolicyOperatorAllAndNoConditionsMatching() { + final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + project.name == "someOtherProjectThatIsNotAcmeApp" + """, PolicyViolation.Type.OPERATIONAL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "someOtherComponentThatIsNotAcmeLib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void testWithPolicyAssignedToProject() { + final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyA, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + final var policyB = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyB, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + + final var projectA = new Project(); + projectA.setName("acme-app-a"); + qm.persist(projectA); + final var componentA = new Component(); + componentA.setProject(projectA); + componentA.setName("acme-lib"); + qm.persist(componentA); + + final var projectB = new Project(); + projectB.setName("acme-app-b"); + qm.persist(projectB); + final var componentB = new Component(); + componentB.setProject(projectB); + componentB.setName("acme-lib"); + qm.persist(componentB); + + policyB.setProjects(List.of(projectB)); + qm.persist(policyB); + + new CelPolicyEngine().evaluateProject(projectA.getUuid()); + new CelPolicyEngine().evaluateProject(projectB.getUuid()); + + assertThat(qm.getAllPolicyViolations(projectA)).hasSize(1); + assertThat(qm.getAllPolicyViolations(projectB)).hasSize(2); + } + + @Test + public void testWithPolicyAssignedToProjectParent() { + final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyA, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + final var policyB = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyB, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + + final var projectA = new Project(); + projectA.setName("acme-app-a"); + qm.persist(projectA); + final var componentA = new Component(); + componentA.setProject(projectA); + componentA.setName("acme-lib"); + qm.persist(componentA); + + final var projectParentB = new Project(); + projectParentB.setName("acme-app-parent-b"); + qm.persist(projectParentB); + + policyB.setProjects(List.of(projectParentB)); + policyB.setIncludeChildren(true); + qm.persist(policyB); + + final var projectB = new Project(); + projectB.setParent(projectParentB); + projectB.setName("acme-app-b"); + qm.persist(projectB); + final var componentB = new Component(); + componentB.setProject(projectB); + componentB.setName("acme-lib"); + qm.persist(componentB); + + new CelPolicyEngine().evaluateProject(projectA.getUuid()); + new CelPolicyEngine().evaluateProject(projectB.getUuid()); + + assertThat(qm.getAllPolicyViolations(projectA)).hasSize(1); + assertThat(qm.getAllPolicyViolations(projectB)).hasSize(2); + } + + @Test + public void testWithPolicyAssignedToTag() { + final Tag tag = qm.createTag("foo"); + + final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyA, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + final var policyB = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policyB, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name.startsWith("acme-lib") + """, PolicyViolation.Type.OPERATIONAL); + policyB.setTags(List.of(tag)); + qm.persist(policyB); + + final var projectA = new Project(); + projectA.setName("acme-app-a"); + qm.persist(projectA); + final var componentA = new Component(); + componentA.setProject(projectA); + componentA.setName("acme-lib"); + qm.persist(componentA); + + final var projectB = new Project(); + projectB.setName("acme-app-b"); + qm.persist(projectB); + final var componentB = new Component(); + componentB.setProject(projectB); + componentB.setName("acme-lib"); + qm.persist(componentB); + + qm.bind(projectB, List.of(tag)); + + new CelPolicyEngine().evaluateProject(projectA.getUuid()); + new CelPolicyEngine().evaluateProject(projectB.getUuid()); + + assertThat(qm.getAllPolicyViolations(projectA)).hasSize(1); + assertThat(qm.getAllPolicyViolations(projectB)).hasSize(2); + } + @Test public void testWithInvalidScript() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); @@ -398,7 +580,7 @@ public void testWithScriptExecutionException() { public void testProjectDependsOnComponent() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) + project.depends_on(org.hyades.policy.v1.Component{name: "acme-lib-a"}) """, PolicyViolation.Type.OPERATIONAL); final var project = new Project(); @@ -568,16 +750,29 @@ public void issue1924() { CelPolicyEngine policyEngine = new CelPolicyEngine(); policyEngine.evaluateProject(project.getUuid()); final List violations = qm.getAllPolicyViolations(); - Assert.assertEquals(3, violations.size()); - PolicyViolation policyViolation = violations.get(0); - Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); - Assert.assertEquals(PolicyCondition.Subject.SEVERITY, policyViolation.getPolicyCondition().getSubject()); - policyViolation = violations.get(1); - Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); - Assert.assertEquals(PolicyCondition.Subject.SEVERITY, policyViolation.getPolicyCondition().getSubject()); - policyViolation = violations.get(2); - Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); - Assert.assertEquals(PolicyCondition.Subject.PACKAGE_URL, policyViolation.getPolicyCondition().getSubject()); + // NOTE: This behavior changed in CelPolicyEngine over the legacy PolicyEngine. + // A matched PolicyCondition can now only yield a single PolicyViolation, whereas + // with the legacy PolicyEngine, multiple PolicyViolations could be raised. +// Assert.assertEquals(3, violations.size()); +// PolicyViolation policyViolation = violations.get(0); +// Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); +// Assert.assertEquals(PolicyCondition.Subject.SEVERITY, policyViolation.getPolicyCondition().getSubject()); +// policyViolation = violations.get(1); +// Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); +// Assert.assertEquals(PolicyCondition.Subject.SEVERITY, policyViolation.getPolicyCondition().getSubject()); +// policyViolation = violations.get(2); +// Assert.assertEquals("Log4J", policyViolation.getComponent().getName()); +// Assert.assertEquals(PolicyCondition.Subject.PACKAGE_URL, policyViolation.getPolicyCondition().getSubject()); + assertThat(violations).satisfiesExactlyInAnyOrder( + violation -> { + assertThat(violation.getComponent().getName()).isEqualTo("Log4J"); + assertThat(violation.getPolicyCondition().getSubject()).isEqualTo(PolicyCondition.Subject.SEVERITY); + }, + violation -> { + assertThat(violation.getComponent().getName()).isEqualTo("Log4J"); + assertThat(violation.getPolicyCondition().getSubject()).isEqualTo(PolicyCondition.Subject.PACKAGE_URL); + } + ); } @Test From 0452d00a793b10b589f9a11ac4658ad62eae1a37 Mon Sep 17 00:00:00 2001 From: nscuro Date: Fri, 22 Sep 2023 17:49:41 +0200 Subject: [PATCH 32/46] Add more tests; Implement script cache bypass for REST API interactions Signed-off-by: nscuro --- .../persistence/PolicyQueryManager.java | 6 +- .../policy/cel/CelPolicyEngine.java | 3 +- .../policy/cel/CelPolicyQueryManager.java | 3 +- .../policy/cel/CelPolicyScriptHost.java | 24 ++- .../resources/v1/PolicyConditionResource.java | 63 +++---- .../resources/v1/vo/CelExpressionError.java | 4 + .../policy/cel/CelPolicyScriptHostTest.java | 56 ++++++- .../v1/PolicyConditionResourceTest.java | 154 ++++++++++++++++++ 8 files changed, 266 insertions(+), 47 deletions(-) create mode 100644 src/main/java/org/dependencytrack/resources/v1/vo/CelExpressionError.java create mode 100644 src/test/java/org/dependencytrack/resources/v1/PolicyConditionResourceTest.java diff --git a/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java b/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java index 44965630d..d9aec4e75 100644 --- a/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/PolicyQueryManager.java @@ -198,7 +198,11 @@ public PolicyCondition createPolicyCondition(final Policy policy, final PolicyCo final PolicyCondition pc = new PolicyCondition(); pc.setPolicy(policy); pc.setSubject(subject); - pc.setOperator(operator); + if (subject == PolicyCondition.Subject.EXPRESSION) { + pc.setOperator(PolicyCondition.Operator.MATCHES); + } else { + pc.setOperator(operator); + } pc.setValue(value); pc.setViolationType(violationType); return persist(pc); diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 3d85fdb28..626951c8f 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -25,6 +25,7 @@ import org.dependencytrack.model.VulnerabilityAlias; import org.dependencytrack.persistence.CollectionIntegerConverter; import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.policy.cel.CelPolicyScriptHost.CacheMode; import org.dependencytrack.policy.cel.compat.CelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.ComponentHashCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.CoordinatesCelPolicyScriptSourceBuilder; @@ -348,7 +349,7 @@ private Pair buildConditionScriptSrc(final PolicyCondit private Pair compileConditionScript(final Pair conditionScriptSrcPair) { final CelPolicyScript script; try { - script = scriptHost.compile(conditionScriptSrcPair.getRight()); + script = scriptHost.compile(conditionScriptSrcPair.getRight(), CacheMode.CACHE); } catch (ScriptCreateException e) { LOGGER.warn("Failed to compile script for condition %s; Condition will be skipped" .formatted(conditionScriptSrcPair.getLeft().getUuid()), e); diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java index 4d68b00f9..6c7a72b47 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java @@ -278,7 +278,8 @@ List fetchAllVulnerabilities(final long projectId, fina "VULNERABILITY" AS "V" INNER JOIN "COMPONENTS_VULNERABILITIES" AS "CV" ON "CV"."VULNERABILITY_ID" = "V"."ID" - INNER JOIN "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" + INNER JOIN + "COMPONENT" AS "C" ON "C"."ID" = "CV"."COMPONENT_ID" LEFT JOIN LATERAL ( SELECT CAST(JSONB_AGG(DISTINCT JSONB_STRIP_NULLS(JSONB_BUILD_OBJECT( diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java index c95fab2ff..0721469cb 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScriptHost.java @@ -1,6 +1,7 @@ package org.dependencytrack.policy.cel; import alpine.common.logging.Logger; +import alpine.server.cache.AbstractCacheManager; import alpine.server.cache.CacheManager; import com.google.api.expr.v1alpha1.CheckedExpr; import com.google.api.expr.v1alpha1.Type; @@ -37,14 +38,19 @@ public class CelPolicyScriptHost { + public enum CacheMode { + CACHE, + NO_CACHE + } + private static final Logger LOGGER = Logger.getLogger(CelPolicyScriptHost.class); private static CelPolicyScriptHost INSTANCE; private final Striped locks; - private final CacheManager cacheManager; + private final AbstractCacheManager cacheManager; private final Env environment; - CelPolicyScriptHost(final CacheManager cacheManager) { + CelPolicyScriptHost(final AbstractCacheManager cacheManager) { this.locks = Striped.lock(128); this.cacheManager = cacheManager; this.environment = Env.newCustomEnv( @@ -65,7 +71,15 @@ public static synchronized CelPolicyScriptHost getInstance() { return INSTANCE; } - public CelPolicyScript compile(final String scriptSrc) throws ScriptCreateException { + /** + * Compile, type-check, ana analyze a given CEL script. + * + * @param scriptSrc Source of the script to compile + * @param cacheMode Whether the {@link CelPolicyScript} shall be cached upon successful compilation + * @return The compiled {@link CelPolicyScript} + * @throws ScriptCreateException When compilation, type checking, or analysis failed + */ + public CelPolicyScript compile(final String scriptSrc, final CacheMode cacheMode) throws ScriptCreateException { final String scriptDigest = DigestUtils.sha256Hex(scriptSrc); // Acquire a lock for the SHA256 digest of the script source. @@ -105,7 +119,9 @@ public CelPolicyScript compile(final String scriptSrc) throws ScriptCreateExcept final MultiValuedMap requirements = analyzeRequirements(CEL.astToCheckedExpr(ast)); script = new CelPolicyScript(program, requirements); - cacheManager.put(scriptDigest, script); + if (cacheMode == CacheMode.CACHE) { + cacheManager.put(scriptDigest, script); + } return script; } finally { lock.unlock(); diff --git a/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java b/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java index d0f109345..acd3b3eae 100644 --- a/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/PolicyConditionResource.java @@ -33,10 +33,13 @@ import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.policy.cel.CelPolicyScriptHost; +import org.dependencytrack.policy.cel.CelPolicyScriptHost.CacheMode; +import org.dependencytrack.resources.v1.vo.CelExpressionError; import org.projectnessie.cel.common.CELError; import org.projectnessie.cel.tools.ScriptCreateException; import javax.validation.Validator; +import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.POST; @@ -84,21 +87,7 @@ public Response createPolicyCondition( try (QueryManager qm = new QueryManager()) { Policy policy = qm.getObjectByUuid(Policy.class, uuid); if (policy != null) { - if (jsonPolicyCondition.getSubject() == PolicyCondition.Subject.EXPRESSION) { - try { - CelPolicyScriptHost.getInstance().compile(jsonPolicyCondition.getValue()); - } catch (ScriptCreateException e) { - // TODO: Bring this in a format that is digestible by the frontend. - // It'd be great if we could give visual hints to users as to *where* - // in their script the errors were found. The exception provides that info. - return Response.status(Response.Status.BAD_REQUEST).entity("The provided CEL expression is invalid: %s".formatted(e.getMessage())).build(); - } - - if (jsonPolicyCondition.getViolationType() == null) { - return Response.status(Response.Status.BAD_REQUEST).entity("Expression conditions must define a violation type").build(); - } - } - + maybeValidateExpression(jsonPolicyCondition); final PolicyCondition pc = qm.createPolicyCondition(policy, jsonPolicyCondition.getSubject(), jsonPolicyCondition.getOperator(), StringUtils.trimToNull(jsonPolicyCondition.getValue()), jsonPolicyCondition.getViolationType()); @@ -131,27 +120,7 @@ public Response updatePolicyCondition(PolicyCondition jsonPolicyCondition) { try (QueryManager qm = new QueryManager()) { PolicyCondition pc = qm.getObjectByUuid(PolicyCondition.class, jsonPolicyCondition.getUuid()); if (pc != null) { - if (jsonPolicyCondition.getSubject() == PolicyCondition.Subject.EXPRESSION) { - try { - CelPolicyScriptHost.getInstance().compile(jsonPolicyCondition.getValue()); - } catch (ScriptCreateException e) { - final var errors = new ArrayList>(); - for (final CELError error : e.getIssues().getErrors()) { - errors.add(Map.of( - "line", error.getLocation().line(), - "column", error.getLocation().column(), - "message", error.getMessage() - )); - } - - return Response.status(Response.Status.BAD_REQUEST).entity(Map.of("celErrors", errors)).build(); - } - - if (jsonPolicyCondition.getViolationType() == null) { - return Response.status(Response.Status.BAD_REQUEST).entity("Expression conditions must define a violation type").build(); - } - } - + maybeValidateExpression(jsonPolicyCondition); pc = qm.updatePolicyCondition(jsonPolicyCondition); return Response.status(Response.Status.CREATED).entity(pc).build(); } else { @@ -186,4 +155,26 @@ public Response deletePolicyCondition( } } } + + private void maybeValidateExpression(final PolicyCondition policyCondition) { + if (policyCondition.getSubject() != PolicyCondition.Subject.EXPRESSION) { + return; + } + + if (policyCondition.getViolationType() == null) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity("Expression conditions must define a violation type").build()); + } + + try { + CelPolicyScriptHost.getInstance().compile(policyCondition.getValue(), CacheMode.NO_CACHE); + } catch (ScriptCreateException e) { + final var celErrors = new ArrayList(); + for (final CELError error : e.getIssues().getErrors()) { + celErrors.add(new CelExpressionError(error.getLocation().line(), error.getLocation().column(), error.getMessage())); + } + + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(Map.of("celErrors", celErrors)).build()); + } + } + } diff --git a/src/main/java/org/dependencytrack/resources/v1/vo/CelExpressionError.java b/src/main/java/org/dependencytrack/resources/v1/vo/CelExpressionError.java new file mode 100644 index 000000000..d7693fc95 --- /dev/null +++ b/src/main/java/org/dependencytrack/resources/v1/vo/CelExpressionError.java @@ -0,0 +1,4 @@ +package org.dependencytrack.resources.v1.vo; + +public record CelExpressionError(Integer line, Integer column, String message) { +} diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java index 4a6daadf3..ee8a687a4 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java @@ -1,10 +1,14 @@ package org.dependencytrack.policy.cel; +import alpine.server.cache.AbstractCacheManager; import com.google.api.expr.v1alpha1.Type; +import org.apache.commons.codec.digest.DigestUtils; +import org.dependencytrack.policy.cel.CelPolicyScriptHost.CacheMode; import org.junit.Test; import java.util.Collection; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.dependencytrack.policy.cel.CelPolicyLibrary.TYPE_COMPONENT; @@ -15,13 +19,49 @@ public class CelPolicyScriptHostTest { + private static class TestCacheManager extends AbstractCacheManager { + + private TestCacheManager() { + super(30, TimeUnit.SECONDS, 5); + } + + } + + @Test + public void testCompileWithCache() throws Exception { + final var scriptSrc = """ + component.name == "foo" + """; + + final var cacheManager = new TestCacheManager(); + final CelPolicyScript script = new CelPolicyScriptHost(cacheManager).compile(""" + component.name == "foo" + """, CacheMode.CACHE); + + assertThat((Object) cacheManager.get(CelPolicyScript.class, DigestUtils.sha256Hex(scriptSrc))).isEqualTo(script); + } + + @Test + public void testCompileWithoutCache() throws Exception { + final var scriptSrc = """ + component.name == "foo" + """; + + final var cacheManager = new TestCacheManager(); + new CelPolicyScriptHost(cacheManager).compile(""" + component.name == "foo" + """, CacheMode.NO_CACHE); + + assertThat((Object) cacheManager.get(CelPolicyScript.class, DigestUtils.sha256Hex(scriptSrc))).isNull(); + } + @Test - public void testRequirements() throws Exception { + public void testRequirementsAnalysis() throws Exception { final CelPolicyScript compiledScript = CelPolicyScriptHost.getInstance().compile(""" component.resolved_license.groups.exists(licenseGroup, licenseGroup.name == "Permissive") - && vulns.filter(vuln, vuln.severity in ["HIGH", "CRITICAL"]).size() > 1 + && vulns.exists(vuln, vuln.severity in ["HIGH", "CRITICAL"] && has(vuln.aliases)) && project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) - """); + """, CacheMode.NO_CACHE); final Map> requirements = compiledScript.getRequirements().asMap(); assertThat(requirements).containsOnlyKeys(TYPE_COMPONENT, TYPE_LICENSE, TYPE_LICENSE_GROUP, TYPE_PROJECT, TYPE_VULNERABILITY); @@ -30,7 +70,15 @@ public void testRequirements() throws Exception { assertThat(requirements.get(TYPE_LICENSE)).containsOnly("groups"); assertThat(requirements.get(TYPE_LICENSE_GROUP)).containsOnly("name"); assertThat(requirements.get(TYPE_PROJECT)).containsOnly("uuid"); // Implicit through project.depends_on - assertThat(requirements.get(TYPE_VULNERABILITY)).containsOnly("severity"); + assertThat(requirements.get(TYPE_VULNERABILITY)).containsOnly( + "aliases", + // Scores are necessary to calculate severity... + "cvssv2_base_score", + "cvssv3_base_score", + "owasp_rr_likelihood_score", + "owasp_rr_technical_impact_score", + "owasp_rr_business_impact_score", + "severity"); } } \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/resources/v1/PolicyConditionResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/PolicyConditionResourceTest.java new file mode 100644 index 000000000..ff4e57bb6 --- /dev/null +++ b/src/test/java/org/dependencytrack/resources/v1/PolicyConditionResourceTest.java @@ -0,0 +1,154 @@ +package org.dependencytrack.resources.v1; + +import alpine.server.filters.ApiFilter; +import alpine.server.filters.AuthenticationFilter; +import org.dependencytrack.ResourceTest; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; +import org.glassfish.jersey.test.DeploymentContext; +import org.glassfish.jersey.test.ServletDeploymentContext; +import org.junit.Test; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; +import static org.assertj.core.api.Assertions.assertThat; + +public class PolicyConditionResourceTest extends ResourceTest { + + @Override + protected DeploymentContext configureDeployment() { + return ServletDeploymentContext.forServlet(new ServletContainer( + new ResourceConfig(PolicyConditionResource.class) + .register(ApiFilter.class) + .register(AuthenticationFilter.class))) + .build(); + } + + @Test + public void testCreateExpressionCondition() { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + + final Response response = target("%s/%s/condition".formatted(V1_POLICY, policy.getUuid())) + .request() + .header(X_API_KEY, apiKey) + .put(Entity.entity(""" + { + "subject": "EXPRESSION", + "value": "component.name == \\"foo\\"", + "violationType": "SECURITY" + } + """, MediaType.APPLICATION_JSON)); + + assertThat(response.getStatus()).isEqualTo(201); + assertThatJson(getPlainTextBody(response)) + .isEqualTo(""" + { + "uuid": "${json-unit.any-string}", + "subject": "EXPRESSION", + "operator": "MATCHES", + "value": "component.name == \\"foo\\"", + "violationType": "SECURITY" + } + """); + } + + @Test + public void testCreateExpressionConditionWithError() { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + + final Response response = target("%s/%s/condition".formatted(V1_POLICY, policy.getUuid())) + .request() + .header(X_API_KEY, apiKey) + .put(Entity.entity(""" + { + "subject": "EXPRESSION", + "value": "component.doesNotExist == \\"foo\\"", + "violationType": "SECURITY" + } + """, MediaType.APPLICATION_JSON)); + + assertThat(response.getStatus()).isEqualTo(400); + assertThatJson(getPlainTextBody(response)) + .isEqualTo(""" + { + "celErrors": [ + { + "line": 1, + "column": 9, + "message": "undefined field 'doesNotExist'" + } + ] + } + """); + } + + @Test + public void testUpdateExpressionCondition() { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition condition = qm.createPolicyCondition(policy, + PolicyCondition.Subject.VULNERABILITY_ID, PolicyCondition.Operator.IS, "foobar"); + + final Response response = target("%s/condition".formatted(V1_POLICY)) + .request() + .header(X_API_KEY, apiKey) + .post(Entity.entity(""" + { + "uuid": "%s", + "subject": "EXPRESSION", + "value": "component.name == \\"foo\\"", + "violationType": "OPERATIONAL" + } + """.formatted(condition.getUuid()), MediaType.APPLICATION_JSON)); + + assertThat(response.getStatus()).isEqualTo(201); + assertThatJson(getPlainTextBody(response)) + .isEqualTo(""" + { + "uuid": "${json-unit.any-string}", + "subject": "EXPRESSION", + "operator": "MATCHES", + "value": "component.name == \\"foo\\"", + "violationType": "OPERATIONAL" + } + """); + } + + @Test + public void testUpdateExpressionConditionWithError() { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + final PolicyCondition condition = qm.createPolicyCondition(policy, + PolicyCondition.Subject.VULNERABILITY_ID, PolicyCondition.Operator.IS, "foobar"); + + final Response response = target("%s/condition".formatted(V1_POLICY)) + .request() + .header(X_API_KEY, apiKey) + .post(Entity.entity(""" + { + "uuid": "%s", + "subject": "EXPRESSION", + "value": "component.doesNotExist == \\"foo\\"", + "violationType": "SECURITY" + } + """.formatted(condition.getUuid()), MediaType.APPLICATION_JSON)); + + assertThat(response.getStatus()).isEqualTo(400); + assertThatJson(getPlainTextBody(response)) + .isEqualTo(""" + { + "celErrors": [ + { + "line": 1, + "column": 9, + "message": "undefined field 'doesNotExist'" + } + ] + } + """); + } + +} \ No newline at end of file From b1c4b0a1447f7401395aa221a80e7e1a81028b61 Mon Sep 17 00:00:00 2001 From: meha Date: Mon, 25 Sep 2023 10:46:31 +0100 Subject: [PATCH 33/46] Add tests for hash policy (#326) * added tests for hash policy Signed-off-by: mehab * updated tests Signed-off-by: mehab --------- Signed-off-by: mehab --- ...onentHashCelPolicyScriptSourceBuilder.java | 18 ++-- .../policy/cel/compat/HashConditionTest.java | 93 +++++++++++++++++++ 2 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/HashConditionTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java index 14fb4cd05..339c0c311 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java @@ -14,7 +14,7 @@ public class ComponentHashCelPolicyScriptSourceBuilder implements CelPolicyScrip @Override public String apply(final PolicyCondition policyCondition) { final Hash hash = extractHashValues(policyCondition); - if (hash == null) { + if (hash.getAlgorithm() == null || hash.getValue() == null || hash.getAlgorithm().isEmpty() || hash.getValue().isEmpty()) { return null; } @@ -23,16 +23,18 @@ public String apply(final PolicyCondition policyCondition) { LOGGER.warn("Component does not have a field named %s".formatted(fieldName)); return null; } - - return """ - component.%s == "%s" - """.formatted(fieldName, escapeJson(hash.getValue())); + if (policyCondition.getOperator().equals(PolicyCondition.Operator.IS)) { + return """ + component.%s == "%s" + """.formatted(fieldName, escapeJson(hash.getValue())); + } else { + LOGGER.warn("Policy operator %s is not allowed with this policy".formatted(policyCondition.getOperator().toString())); + return null; + } } private static Hash extractHashValues(PolicyCondition condition) { - if (condition.getValue() == null) { - return null; - } + //Policy condition received here will never be null final JSONObject def = new JSONObject(condition.getValue()); return new Hash( def.optString("algorithm", null), diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/HashConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/HashConditionTest.java new file mode 100644 index 000000000..ad0cee6f7 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/HashConditionTest.java @@ -0,0 +1,93 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.Policy.ViolationState; +import org.dependencytrack.model.PolicyCondition.Operator; +import org.dependencytrack.model.PolicyCondition.Subject; +import org.dependencytrack.model.PolicyViolation.Type; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class HashConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash", true, ViolationState.FAIL, Type.OPERATIONAL, ViolationState.FAIL}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash", true, ViolationState.WARN, Type.OPERATIONAL, ViolationState.WARN}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash_false", false, ViolationState.INFO, Type.OPERATIONAL, ViolationState.INFO}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'test', 'value': 'test_hash' }", + "test_hash", false, ViolationState.INFO, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS_NOT, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash20", false, ViolationState.INFO, null, null}, + new Object[]{Policy.Operator.ANY, Operator.MATCHES, "{ 'algorithm': 'SHA256', 'value': 'test_hash' }", + "test_hash", false, ViolationState.INFO, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': null, 'value': 'test_hash' }", + "test_hash", false, ViolationState.FAIL, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'MD5', 'value': null }", + "test_hash", false, ViolationState.FAIL, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': 'SHA256', 'value': '' }", + "test_hash", false, ViolationState.FAIL, null, null}, + new Object[]{Policy.Operator.ANY, Operator.IS, "{ 'algorithm': '', 'value': 'test_hash' }", + "test_hash", false, ViolationState.FAIL, null, null}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(Policy.Operator policyOperator, final Operator condition, final String conditionHash, + final String actualHash, final boolean expectViolation, ViolationState violationState, + Type actualType, ViolationState actualViolationState) { + final Policy policy = qm.createPolicy("policy", policyOperator, violationState); + qm.createPolicyCondition(policy, Subject.COMPONENT_HASH, condition, conditionHash); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setSha256(actualHash); + qm.persist(component); + + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getViolationType()).isEqualTo(actualType); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getPolicy().getViolationState()).isEqualTo(actualViolationState); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + + + @Test + public void testWithNullPolicyCondition() { + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setSha256("actualHash"); + qm.persist(component); + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + +} From 9487edbbdd3c1396722b21c9fb9363aa3914a368 Mon Sep 17 00:00:00 2001 From: VithikaS <122881935+VithikaS@users.noreply.github.com> Date: Mon, 25 Sep 2023 10:47:11 +0100 Subject: [PATCH 34/46] Add version cel policy script builder (#324) * Add version cel policy script builder Signed-off-by: vithikashukla * add version support for coordinates cel policy Signed-off-by: vithikashukla * Added unit test for version policy script builder Signed-off-by: vithikashukla * added coordninates condition test Signed-off-by: vithikashukla * added coordinates condition test Signed-off-by: vithikashukla --------- Signed-off-by: vithikashukla Co-authored-by: vithikashukla --- .../policy/cel/CelPolicyEngine.java | 2 + ...ordinatesCelPolicyScriptSourceBuilder.java | 56 +++++++++++++-- .../VersionCelPolicyScriptSourceBuilder.java | 61 ++++++++++++++++ .../cel/compat/CoordinatesConditionTest.java | 69 +++++++++++++++++++ .../cel/compat/VersionConditionTest.java | 62 +++++++++++++++++ 5 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/dependencytrack/policy/cel/compat/VersionCelPolicyScriptSourceBuilder.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/VersionConditionTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 626951c8f..81d9e1db6 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -36,6 +36,7 @@ import org.dependencytrack.policy.cel.compat.PackageUrlCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.SeverityCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.SwidTagIdCelPolicyScriptSourceBuilder; +import org.dependencytrack.policy.cel.compat.VersionCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.compat.VulnerabilityIdCelPolicyScriptSourceBuilder; import org.dependencytrack.policy.cel.mapping.ComponentProjection; import org.dependencytrack.policy.cel.mapping.LicenseProjection; @@ -100,6 +101,7 @@ public class CelPolicyEngine { SCRIPT_BUILDERS.put(Subject.SEVERITY, new SeverityCelPolicyScriptSourceBuilder()); SCRIPT_BUILDERS.put(Subject.SWID_TAGID, new SwidTagIdCelPolicyScriptSourceBuilder()); SCRIPT_BUILDERS.put(Subject.VULNERABILITY_ID, new VulnerabilityIdCelPolicyScriptSourceBuilder()); + SCRIPT_BUILDERS.put(Subject.VERSION, new VersionCelPolicyScriptSourceBuilder()); } private final CelPolicyScriptHost scriptHost; diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java index a28ce3e00..8406363aa 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java @@ -1,14 +1,23 @@ package org.dependencytrack.policy.cel.compat; +import alpine.common.logging.Logger; +import io.github.nscuro.versatile.Comparator; +import io.github.nscuro.versatile.Vers; +import io.github.nscuro.versatile.version.VersioningScheme; import org.dependencytrack.model.PolicyCondition; import org.json.JSONObject; import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; public class CoordinatesCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + private static final Logger LOGGER = Logger.getLogger(CoordinatesCelPolicyScriptSourceBuilder.class); + private static final Pattern VERSION_OPERATOR_PATTERN = Pattern.compile("^(?[<>]=?|[!=]=)\\s*"); + @Override public String apply(final PolicyCondition condition) { if (condition.getValue() == null) { @@ -16,13 +25,11 @@ public String apply(final PolicyCondition condition) { } final JSONObject def = new JSONObject(condition.getValue()); - final String group = Optional.ofNullable(def.optString("group", null)).orElse(".*"); - final String name = Optional.ofNullable(def.optString("name", null)).orElse(".*"); - final String version = Optional.ofNullable(def.optString("version")).orElse(".*"); + final String group = Optional.ofNullable(def.optString("group", null)).orElse(""); + final String name = Optional.ofNullable(def.optString("name", null)).orElse(""); + final String version = Optional.ofNullable(def.optString("version")).orElse(""); - final var scriptSrc = """ - component.group.matches("%s") && component.name.matches("%s") && component.version.matches("%s") - """.formatted(escapeJson(group), escapeJson(name), escapeJson(version)); + final var scriptSrc = evaluateScript(group, name, version); if (condition.getOperator() == PolicyCondition.Operator.MATCHES) { return scriptSrc; } else if (condition.getOperator() == PolicyCondition.Operator.NO_MATCH) { @@ -32,4 +39,41 @@ public String apply(final PolicyCondition condition) { return null; } + private static String evaluateScript(final String group, final String name, final String version) { + + final Matcher versionOperatorMatcher = VERSION_OPERATOR_PATTERN.matcher(version); + //Do an exact match if no operator found + if (!versionOperatorMatcher.find()) { + + Vers conditionVers = Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.EQUAL, version) + .build(); + return """ + component.group.matches("%s") && component.name.matches("%s") && component.matches_range("%s") + """.formatted(escapeJson(group), escapeJson(name), conditionVers.toString()); + } + + io.github.nscuro.versatile.Comparator versionComparator = switch (versionOperatorMatcher.group(1)) { + case "==" -> Comparator.EQUAL; + case "!=" -> Comparator.NOT_EQUAL; + case "<" -> Comparator.LESS_THAN; + case "<=" -> Comparator.LESS_THAN_OR_EQUAL; + case ">" -> Comparator.GREATER_THAN; + case ">=" -> Comparator.GREATER_THAN_OR_EQUAL; + default -> null; + }; + if (versionComparator == null) { + // Shouldn't ever happen because the regex won't match anything else + LOGGER.error("Failed to infer version operator from " + versionOperatorMatcher.group(1)); + return null; + } + String condition = VERSION_OPERATOR_PATTERN.split(version)[1]; + Vers conditionVers = Vers.builder(VersioningScheme.GENERIC) + .withConstraint(versionComparator, condition) + .build(); + + return """ + component.group.matches("%s") && component.name.matches("%s") && component.matches_range("%s") + """.formatted(escapeJson(group), escapeJson(name), conditionVers.toString()); + } } diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/VersionCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/VersionCelPolicyScriptSourceBuilder.java new file mode 100644 index 000000000..8384cf53d --- /dev/null +++ b/src/main/java/org/dependencytrack/policy/cel/compat/VersionCelPolicyScriptSourceBuilder.java @@ -0,0 +1,61 @@ +package org.dependencytrack.policy.cel.compat; + +import alpine.common.logging.Logger; +import io.github.nscuro.versatile.Comparator; +import io.github.nscuro.versatile.Vers; +import io.github.nscuro.versatile.VersException; +import io.github.nscuro.versatile.version.VersioningScheme; +import org.dependencytrack.model.PolicyCondition; + +public class VersionCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { + private static final Logger LOGGER = Logger.getLogger(VersionCelPolicyScriptSourceBuilder.class); + + @Override + public String apply(PolicyCondition policyCondition) { + + Vers conditionVers = evaluateVers(policyCondition); + if (conditionVers == null) { + return null; + } + return """ + component.matches_range("%s") + """.formatted(conditionVers.toString()); + } + + private static Vers evaluateVers(final PolicyCondition policyCondition) { + try { + switch (policyCondition.getOperator()) { + case NUMERIC_EQUAL: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.EQUAL, policyCondition.getValue()) + .build(); + case NUMERIC_NOT_EQUAL: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.NOT_EQUAL, policyCondition.getValue()) + .build(); + case NUMERIC_LESS_THAN: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.LESS_THAN, policyCondition.getValue()) + .build(); + case NUMERIC_LESSER_THAN_OR_EQUAL: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.LESS_THAN_OR_EQUAL, policyCondition.getValue()) + .build(); + case NUMERIC_GREATER_THAN: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.GREATER_THAN, policyCondition.getValue()) + .build(); + case NUMERIC_GREATER_THAN_OR_EQUAL: + return Vers.builder(VersioningScheme.GENERIC) + .withConstraint(Comparator.GREATER_THAN_OR_EQUAL, policyCondition.getValue()) + .build(); + default: + LOGGER.warn("Unsupported operation " + policyCondition.getOperator()); + return null; + } + } catch (VersException versException) { + LOGGER.warn("Unable to parse version range in policy condition", versException); + return null; + } + } +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java new file mode 100644 index 000000000..4ef544c29 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java @@ -0,0 +1,69 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.json.JSONObject; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class CoordinatesConditionTest extends AbstractPostgresEnabledTest { + private Object[] parameters() { + return new Object[]{ + // MATCHES regex + new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"acme*\",\"name\": \"acme*\",\"version\": \">=v1.2*\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", true}, + //Exact match + new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", true}, + //Group does not match + new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"org.hippo\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", false}, + //Name does not match regex + new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"acme-app\",\"name\": \"*acme-lib*\",\"version\": \"v1.2.3\"}", "{\"group\": \"acme-app\",\"name\": \"good-foo-lib\",\"version\": \"v1.2.3\"}", false}, + //Version does not match + new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"acme-app\",\"name\": \"*acme-lib*\",\"version\": \"v1.*\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v2.2.3\"}", false}, + //Does not match on group + new Object[]{PolicyCondition.Operator.NO_MATCH, "{\"group\": \"diff-group\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", true}, + //Does not match on version + new Object[]{PolicyCondition.Operator.NO_MATCH, "{\"group\": \"acme-app\",\"name\": \"*acme-lib*\",\"version\": \">=v2.2.2\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", true}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final PolicyCondition.Operator operator, final String conditionCoordinates, final String componentCoordinates, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.COORDINATES, operator, conditionCoordinates); + + final JSONObject def = new JSONObject(componentCoordinates); + final String group = Optional.ofNullable(def.optString("group", null)).orElse(""); + final String name = Optional.ofNullable(def.optString("name", null)).orElse(""); + final String version = Optional.ofNullable(def.optString("version")).orElse(""); + + final var project = new Project(); + project.setName(group); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setGroup(group); + component.setName(name); + component.setVersion(version); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } +} diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/VersionConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/VersionConditionTest.java new file mode 100644 index 000000000..1f1a3ab7e --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/VersionConditionTest.java @@ -0,0 +1,62 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class VersionConditionTest extends AbstractPostgresEnabledTest { + + private Object[] parameters() { + return new Object[]{ + // MATCHES with exact match + new Object[]{PolicyCondition.Operator.NUMERIC_EQUAL, "v1.2.3", "v1.2.3", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_EQUAL, "v1.2.3", "v1.2.4", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_NOT_EQUAL, "0.4.5-SNAPSHOT", "0.4.5", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_NOT_EQUAL, "0.4.5", "0.4.5", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN, "0.4.5", "0.5.5", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN, "0.4.4", "0.4.4", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN_OR_EQUAL, "0.4.4", "0.4.4", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN_OR_EQUAL, "v0.4.5-SNAPSHOT", "z0.4.5", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN_OR_EQUAL, "v0.4.5-SNAPSHOT", "0.4.5", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_GREATER_THAN_OR_EQUAL, "v0.4.*", "v0.4.1", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_LESS_THAN, "v0.4.*", "v0.4.1", false}, + new Object[]{PolicyCondition.Operator.NUMERIC_LESS_THAN, "v0.4.*", "v0.3.1", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_LESSER_THAN_OR_EQUAL, "v0.4.*", "v0.4.0", true}, + new Object[]{PolicyCondition.Operator.NUMERIC_LESSER_THAN_OR_EQUAL, "v0.4.*", "v0.4.2", false}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testCondition(final PolicyCondition.Operator operator, final String conditionVersion, final String componentVersion, final boolean expectViolation) { + final Policy policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.VERSION, operator, conditionVersion); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + component.setVersion(componentVersion); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } +} From f012cfa4f81ebebacd76787a9eed28287546dac6 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 25 Sep 2023 12:04:33 +0200 Subject: [PATCH 35/46] Fix new UNIQUE constraint breaking existing behavior Signed-off-by: nscuro --- .../java/org/dependencytrack/model/Policy.java | 17 +---------------- .../dependencytrack/model/PolicyViolation.java | 9 ++++----- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/dependencytrack/model/Policy.java b/src/main/java/org/dependencytrack/model/Policy.java index 8b93f546c..9d3dad806 100644 --- a/src/main/java/org/dependencytrack/model/Policy.java +++ b/src/main/java/org/dependencytrack/model/Policy.java @@ -99,14 +99,7 @@ public enum ViolationState { @NotBlank @Size(min = 1, max = 255) @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The violation state may only contain printable characters") - private ViolationState violationState; // TODO: New column, possibly needs migration. - - @Persistent - @Column(name = "VIOLATIONTYPE", allowsNull = "true") - @NotBlank - @Size(min = 1, max = 255) - @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The violation type may only contain printable characters") - private PolicyViolation.Type violationType; + private ViolationState violationState; /** * A list of zero-to-n policy conditions. @@ -181,14 +174,6 @@ public void setViolationState(ViolationState violationState) { this.violationState = violationState; } - public PolicyViolation.Type getViolationType() { - return violationType; - } - - public void setViolationType(final PolicyViolation.Type violationType) { - this.violationType = violationType; - } - public List getPolicyConditions() { return policyConditions; } diff --git a/src/main/java/org/dependencytrack/model/PolicyViolation.java b/src/main/java/org/dependencytrack/model/PolicyViolation.java index a4d298793..2820da971 100644 --- a/src/main/java/org/dependencytrack/model/PolicyViolation.java +++ b/src/main/java/org/dependencytrack/model/PolicyViolation.java @@ -46,10 +46,9 @@ @PersistenceCapable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -@Unique(name = "foo", members = { - "component", - "policyCondition", -}) +// TODO: Add @Unique composite constraint on the fields component, policyCondition, and type. +// The legacy PolicyEngine erroneously allows for duplicates on those fields, but CelPolicyEngine +// will never produce such duplicates. Until we remove the legacy engine, we can't add this constraint. public class PolicyViolation implements Serializable { public enum Type { @@ -91,7 +90,7 @@ public enum Type { @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The text may only contain printable characters") private String text; - @Persistent(mappedBy = "policyViolation", defaultFetchGroup = "true") + @Persistent(mappedBy="policyViolation", defaultFetchGroup = "true") private ViolationAnalysis analysis; /** From 4310a80ce4622c95e9e65d2cc09c7ff81d1daf70 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 25 Sep 2023 12:18:32 +0200 Subject: [PATCH 36/46] Add feature flag for CEL policy engine Signed-off-by: nscuro --- .../org/dependencytrack/common/ConfigKey.java | 3 +- .../dependencytrack/policy/PolicyEngine.java | 4 +- .../tasks/PolicyEvaluationTask.java | 38 ++++++++++++++++--- src/main/resources/application.properties | 4 ++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/dependencytrack/common/ConfigKey.java b/src/main/java/org/dependencytrack/common/ConfigKey.java index f2351b2ef..a78107457 100644 --- a/src/main/java/org/dependencytrack/common/ConfigKey.java +++ b/src/main/java/org/dependencytrack/common/ConfigKey.java @@ -70,7 +70,8 @@ public enum ConfigKey implements Config.Key { BOM_UPLOAD_PROCESSING_TRX_FLUSH_THRESHOLD("bom.upload.processing.trx.flush.threshold", "10000"), WORKFLOW_RETENTION_DURATION("workflow.retention.duration", "P3D"), WORKFLOW_STEP_TIMEOUT_DURATION("workflow.step.timeout.duration", "PT1H"), - TMP_DELAY_BOM_PROCESSED_NOTIFICATION("tmp.delay.bom.processed.notification", "false"); + TMP_DELAY_BOM_PROCESSED_NOTIFICATION("tmp.delay.bom.processed.notification", "false"), + CEL_POLICY_ENGINE_ENABLED("cel.policy.engine.enabled", "false"); private final String propertyName; private final Object defaultValue; diff --git a/src/main/java/org/dependencytrack/policy/PolicyEngine.java b/src/main/java/org/dependencytrack/policy/PolicyEngine.java index 3a4754735..69e022a1a 100644 --- a/src/main/java/org/dependencytrack/policy/PolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/PolicyEngine.java @@ -213,7 +213,9 @@ public PolicyViolation.Type determineViolationType(final PolicyCondition.Subject case AGE, COORDINATES, PACKAGE_URL, CPE, SWID_TAGID, COMPONENT_HASH, VERSION -> PolicyViolation.Type.OPERATIONAL; case LICENSE, LICENSE_GROUP -> PolicyViolation.Type.LICENSE; - case EXPRESSION -> null; // TODO: Just here to satisfy the switch exhaustiveness + // Just here to satisfy the switch exhaustiveness. Conditions with subject EXPRESSION + // will never yield any violations, because there's no evaluator supporting it. + case EXPRESSION -> null; }; } diff --git a/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java b/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java index eade881ee..7bc48f038 100644 --- a/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java +++ b/src/main/java/org/dependencytrack/tasks/PolicyEvaluationTask.java @@ -1,22 +1,21 @@ package org.dependencytrack.tasks; +import alpine.Config; import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; +import org.dependencytrack.common.ConfigKey; import org.dependencytrack.event.ComponentPolicyEvaluationEvent; import org.dependencytrack.event.ProjectPolicyEvaluationEvent; import org.dependencytrack.model.Component; import org.dependencytrack.model.Project; import org.dependencytrack.model.WorkflowState; -import org.dependencytrack.model.WorkflowStatus; import org.dependencytrack.persistence.QueryManager; import org.dependencytrack.policy.PolicyEngine; import org.dependencytrack.policy.cel.CelPolicyEngine; -import java.time.Instant; -import java.util.Date; +import java.util.UUID; -import static org.dependencytrack.model.WorkflowStep.METRICS_UPDATE; import static org.dependencytrack.model.WorkflowStep.POLICY_EVALUATION; /** @@ -28,6 +27,16 @@ public class PolicyEvaluationTask implements Subscriber { private static final Logger LOGGER = Logger.getLogger(PolicyEvaluationTask.class); + private final boolean celPolicyEngineEnabled; + + public PolicyEvaluationTask() { + this(Config.getInstance().getPropertyAsBoolean(ConfigKey.CEL_POLICY_ENGINE_ENABLED)); + } + + PolicyEvaluationTask(final boolean celPolicyEngineEnabled) { + this.celPolicyEngineEnabled = celPolicyEngineEnabled; + } + @Override public void inform(final Event e) { if (e instanceof final ProjectPolicyEvaluationEvent event) { @@ -35,7 +44,7 @@ public void inform(final Event e) { try (final var qm = new QueryManager()) { projectPolicyEvaluationState = qm.updateStartTimeIfWorkflowStateExists(event.getChainIdentifier(), POLICY_EVALUATION); try { - new CelPolicyEngine().evaluateProject(event.getUuid()); + evaluateProject(event.getUuid()); qm.updateWorkflowStateToComplete(projectPolicyEvaluationState); } catch (Exception ex) { qm.updateWorkflowStateToFailed(projectPolicyEvaluationState, ex.getMessage()); @@ -47,7 +56,7 @@ public void inform(final Event e) { try (final var qm = new QueryManager()) { componentMetricsEvaluationState = qm.updateStartTimeIfWorkflowStateExists(event.getChainIdentifier(), POLICY_EVALUATION); try { - new CelPolicyEngine().evaluateComponent(event.getUuid()); + evaluateComponent(event.getUuid()); qm.updateWorkflowStateToComplete(componentMetricsEvaluationState); } catch (Exception ex) { qm.updateWorkflowStateToFailed(componentMetricsEvaluationState, ex.getMessage()); @@ -56,4 +65,21 @@ public void inform(final Event e) { } } } + + private void evaluateProject(final UUID uuid) { + if (celPolicyEngineEnabled) { + new CelPolicyEngine().evaluateProject(uuid); + } else { + new PolicyEngine().evaluateProject(uuid); + } + } + + private void evaluateComponent(final UUID uuid) { + if (celPolicyEngineEnabled) { + new CelPolicyEngine().evaluateComponent(uuid); + } else { + new PolicyEngine().evaluate(uuid); + } + } + } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index a24091023..bc636af72 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -513,3 +513,7 @@ workflow.retention.duration=P3D # This is specifically for cases where polling the /api/v1/bom/token/ endpoint is not feasible. # THIS IS A TEMPORARY FUNCTIONALITY AND MAY BE REMOVED IN FUTURE RELEASES WITHOUT FURTHER NOTICE. tmp.delay.bom.processed.notification=false + +# Optional +# Specifies whether the Common Expression Language (CEL) based policy engine shall be enabled. +cel.policy.engine.enabled=false \ No newline at end of file From ae9f0678c8a3f246da9a1cae7f8c1363b4c4cedc Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 25 Sep 2023 12:25:56 +0200 Subject: [PATCH 37/46] Add `UpgradeItem` to update type of `"POLICYCONDITION"."VALUE"` to `TEXT` Signed-off-by: nscuro --- .../dependencytrack/upgrade/UpgradeItems.java | 5 +++ .../upgrade/v510/v510Updater.java | 33 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 src/main/java/org/dependencytrack/upgrade/v510/v510Updater.java diff --git a/src/main/java/org/dependencytrack/upgrade/UpgradeItems.java b/src/main/java/org/dependencytrack/upgrade/UpgradeItems.java index f035f4601..c6d660bd3 100644 --- a/src/main/java/org/dependencytrack/upgrade/UpgradeItems.java +++ b/src/main/java/org/dependencytrack/upgrade/UpgradeItems.java @@ -19,6 +19,7 @@ package org.dependencytrack.upgrade; import alpine.server.upgrade.UpgradeItem; +import org.dependencytrack.upgrade.v510.v510Updater; import java.util.ArrayList; import java.util.List; @@ -27,6 +28,10 @@ class UpgradeItems { private static final List> UPGRADE_ITEMS = new ArrayList<>(); + static { + UPGRADE_ITEMS.add(v510Updater.class); + } + static List> getUpgradeItems() { return UPGRADE_ITEMS; } diff --git a/src/main/java/org/dependencytrack/upgrade/v510/v510Updater.java b/src/main/java/org/dependencytrack/upgrade/v510/v510Updater.java new file mode 100644 index 000000000..ba16ab9c6 --- /dev/null +++ b/src/main/java/org/dependencytrack/upgrade/v510/v510Updater.java @@ -0,0 +1,33 @@ +package org.dependencytrack.upgrade.v510; + +import alpine.common.logging.Logger; +import alpine.persistence.AlpineQueryManager; +import alpine.server.upgrade.AbstractUpgradeItem; + +import java.sql.Connection; +import java.sql.PreparedStatement; + +public class v510Updater extends AbstractUpgradeItem { + + private static final Logger LOGGER = Logger.getLogger(v510Updater.class); + + @Override + public String getSchemaVersion() { + return "5.1.0"; + } + + @Override + public void executeUpgrade(final AlpineQueryManager qm, final Connection connection) throws Exception { + changePolicyConditionValueTypeToText(connection); + } + + private static void changePolicyConditionValueTypeToText(final Connection connection) throws Exception { + LOGGER.info("Changing type of \"POLICYCONDITION\".\"VALUE\" from VARCHAR(255) to TEXT"); + try (final PreparedStatement ps = connection.prepareStatement(""" + ALTER TABLE "POLICYCONDITION" ALTER COLUMN "VALUE" TYPE TEXT; + """)) { + ps.execute(); + } + } + +} From 5a5dcf733410fc2ffa4519b0238a965575159880 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 25 Sep 2023 12:50:34 +0200 Subject: [PATCH 38/46] Handle policy evaluation for individual components Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 62 ++++--------------- .../policy/cel/CelPolicyLibrary.java | 2 +- .../policy/cel/CelPolicyQueryManager.java | 15 +++++ .../policy/cel/CelPolicyScript.java | 6 +- .../policy/cel/CelPolicyEngineTest.java | 62 ++++++++++++++----- 5 files changed, 78 insertions(+), 69 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 81d9e1db6..e6d0004e1 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -15,7 +15,6 @@ import org.apache.commons.collections4.multimap.HashSetValuedHashMap; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; -import org.dependencytrack.model.Component; import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.model.PolicyCondition.Subject; @@ -217,7 +216,7 @@ public void evaluateProject(final UUID uuid) { } } finally { final long durationNs = timerSample.stop(Timer - .builder("dtrack_policy_eval") + .builder("policy_evaluation") .tag("target", "project") .register(Metrics.getRegistry())); LOGGER.info("Evaluation of project %s completed in %s" @@ -226,58 +225,23 @@ public void evaluateProject(final UUID uuid) { } public void evaluateComponent(final UUID uuid) { - final Timer.Sample timerSample = Timer.start(); + // Evaluation of individual components is only triggered when they are added or modified + // manually. As this happens very rarely, in low frequencies (due to being manual actions), + // and because CEL policy evaluation is so efficient, it's not worth it to maintain extra + // logic to handle component evaluation. Instead, re-purpose to project evaluation. + final UUID projectUuid; try (final var qm = new QueryManager(); final var celQm = new CelPolicyQueryManager(qm)) { - final Component component = qm.getObjectByUuid(Component.class, uuid); - if (component == null) { - LOGGER.warn("Component with UUID %s does not exist".formatted(uuid)); - return; - } - - LOGGER.debug("Compiling policy scripts for project %s".formatted(uuid)); - final List> conditionScriptPairs = getApplicableConditionScriptPairs(celQm, component.getProject()); - if (conditionScriptPairs.isEmpty()) { - LOGGER.info("No applicable policies found for project %s".formatted(uuid)); - // reconcileViolations(qm, project.getId(), emptyMultiValuedMap()); - return; - } - - LOGGER.info("Determining evaluation requirements for component %s and %d policy conditions" - .formatted(uuid, conditionScriptPairs.size())); - final MultiValuedMap requirements = determineScriptRequirements(conditionScriptPairs); + projectUuid = celQm.getProjectUuidForComponentUuid(uuid); + } - LOGGER.info("Evaluating component %s against %d applicable policy conditions" - .formatted(uuid, conditionScriptPairs.size())); - final List conditionsViolated = evaluateConditions(conditionScriptPairs, Map.of( - // VAR_COMPONENT, mapComponent(qm, component, requirements), - // VAR_PROJECT, mapProject(component.getProject(), requirements), - // VAR_VULNERABILITIES, loadVulnerabilities(qm, component, requirements) - )); - - // Group the detected condition violations by policy. Necessary to be able to evaluate - // each policy's operator (ANY, ALL). - LOGGER.info("Detected violation of %d policy conditions for component %s; Evaluating policy operators" - .formatted(conditionsViolated.size(), uuid)); - final List violations = evaluatePolicyOperators(conditionsViolated); - - // Reconcile the violations created above with what's already in the database. - // Create new records if necessary, and delete records that are no longer current. - // final List newViolations = reconcileViolations(qm, component, violations); - - // Notify users about any new violations. - // for (final PolicyViolation newViolation : newViolations) { - // NotificationUtil.analyzeNotificationCriteria(qm, newViolation); - // } - } finally { - final long durationNs = timerSample.stop(Timer - .builder("dtrack_policy_eval") - .tag("target", "component") - .register(Metrics.getRegistry())); - LOGGER.info("Evaluation of component %s completed in %s" - .formatted(uuid, Duration.ofNanos(durationNs))); + if (projectUuid == null) { + LOGGER.warn("Component with UUID %s does not exist; Skipping".formatted(uuid)); + return; } + + evaluateProject(projectUuid); } diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index 688063e41..562714ace 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.UUID; -public class CelPolicyLibrary implements Library { +class CelPolicyLibrary implements Library { private static final Logger LOGGER = Logger.getLogger(CelPolicyLibrary.class); diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java index 6c7a72b47..d3598e72f 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyQueryManager.java @@ -3,6 +3,7 @@ import alpine.common.logging.Logger; import org.apache.commons.collections4.MultiValuedMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.dependencytrack.model.Component; import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyViolation; import org.dependencytrack.model.Project; @@ -50,6 +51,20 @@ class CelPolicyQueryManager implements AutoCloseable { this.pm = qm.getPersistenceManager(); } + UUID getProjectUuidForComponentUuid(final UUID componentUuid) { + try (final var qm = new QueryManager()) { + final Query query = qm.getPersistenceManager().newQuery(Component.class); + query.setFilter("uuid == :uuid"); + query.setParameters(componentUuid); + query.setResult("project.uuid"); + try { + return query.executeResultUnique(UUID.class); + } finally { + query.closeAll(); + } + } + } + ProjectProjection fetchProject(final long projectId, final Collection projectProtoFieldNames, final Collection projectPropertyProtoFieldNames) { diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java index 0525e0e8c..28ae2612b 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyScript.java @@ -14,16 +14,16 @@ public class CelPolicyScript { private final Program program; private final MultiValuedMap requirements; - public CelPolicyScript(final Program program, final MultiValuedMap requirements) { + CelPolicyScript(final Program program, final MultiValuedMap requirements) { this.program = program; this.requirements = requirements; } - public MultiValuedMap getRequirements() { + MultiValuedMap getRequirements() { return requirements; } - public boolean execute(final Map arguments) throws ScriptExecutionException { + boolean execute(final Map arguments) throws ScriptExecutionException { final Val result = program.eval(arguments).getVal(); if (Err.isError(result)) { diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index b93330ada..91b4d93e0 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -64,7 +64,7 @@ public void setUp() throws Exception { * */ @Test - public void testWithAllFields() { + public void testEvaluateProjectWithAllFields() { final var project = new Project(); project.setUuid(UUID.fromString("d7173786-60aa-4a4f-a950-c92fe6422307")); project.setGroup("projectGroup"); @@ -267,7 +267,7 @@ && has(project.last_bom_import) } @Test - public void testWithPolicyOperatorAnyAndAllConditionsMatching() { + public void testEvaluateProjectWithPolicyOperatorAnyAndAllConditionsMatching() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.name == "acme-app" @@ -290,7 +290,7 @@ public void testWithPolicyOperatorAnyAndAllConditionsMatching() { } @Test - public void testWithPolicyOperatorAnyAndNotAllConditionsMatching() { + public void testEvaluateProjectWithPolicyOperatorAnyAndNotAllConditionsMatching() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.name == "acme-app" @@ -313,7 +313,7 @@ public void testWithPolicyOperatorAnyAndNotAllConditionsMatching() { } @Test - public void testWithPolicyOperatorAnyAndNoConditionsMatching() { + public void testEvaluateProjectWithPolicyOperatorAnyAndNoConditionsMatching() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.name == "someOtherProjectThatIsNotAcmeApp" @@ -336,7 +336,7 @@ public void testWithPolicyOperatorAnyAndNoConditionsMatching() { } @Test - public void testWithPolicyOperatorAllAndAllConditionsMatching() { + public void testEvaluateProjectWithPolicyOperatorAllAndAllConditionsMatching() { final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.name == "acme-app" @@ -359,7 +359,7 @@ public void testWithPolicyOperatorAllAndAllConditionsMatching() { } @Test - public void testWithPolicyOperatorAllAndNotAllConditionsMatching() { + public void testEvaluateProjectWithPolicyOperatorAllAndNotAllConditionsMatching() { final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.name == "acme-app" @@ -382,7 +382,7 @@ public void testWithPolicyOperatorAllAndNotAllConditionsMatching() { } @Test - public void testWithPolicyOperatorAllAndNoConditionsMatching() { + public void testEvaluateProjectWithPolicyOperatorAllAndNoConditionsMatching() { final var policy = qm.createPolicy("policy", Policy.Operator.ALL, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.name == "someOtherProjectThatIsNotAcmeApp" @@ -405,7 +405,7 @@ public void testWithPolicyOperatorAllAndNoConditionsMatching() { } @Test - public void testWithPolicyAssignedToProject() { + public void testEvaluateProjectWithPolicyAssignedToProject() { final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policyA, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ component.name.startsWith("acme-lib") @@ -442,7 +442,7 @@ public void testWithPolicyAssignedToProject() { } @Test - public void testWithPolicyAssignedToProjectParent() { + public void testEvaluateProjectWithPolicyAssignedToProjectParent() { final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policyA, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ component.name.startsWith("acme-lib") @@ -485,7 +485,7 @@ public void testWithPolicyAssignedToProjectParent() { } @Test - public void testWithPolicyAssignedToTag() { + public void testEvaluateProjectWithPolicyAssignedToTag() { final Tag tag = qm.createTag("foo"); final var policyA = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); @@ -525,7 +525,7 @@ public void testWithPolicyAssignedToTag() { } @Test - public void testWithInvalidScript() { + public void testEvaluateProjectWithInvalidScript() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ component.doesNotExist == "foo" @@ -551,7 +551,7 @@ public void testWithInvalidScript() { } @Test - public void testWithScriptExecutionException() { + public void testEvaluateProjectWithScriptExecutionException() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.last_bom_import == timestamp("invalid") @@ -577,7 +577,7 @@ public void testWithScriptExecutionException() { } @Test - public void testProjectDependsOnComponent() { + public void testEvaluateProjectWithFuncProjectDependsOnComponent() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.depends_on(org.hyades.policy.v1.Component{name: "acme-lib-a"}) @@ -610,7 +610,7 @@ public void testProjectDependsOnComponent() { } @Test - public void testComponentIsDependencyOfComponent() { + public void testEvaluateProjectWithFuncComponentIsDependencyOfComponent() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ component.is_dependency_of(org.hyades.policy.v1.Component{name: "acme-lib-a"}) @@ -642,7 +642,7 @@ public void testComponentIsDependencyOfComponent() { } @Test - public void testMatchesRange() { + public void testEvaluateProjectWithFuncMatchesRange() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.matches_range("vers:generic/<1") @@ -672,7 +672,7 @@ public void testMatchesRange() { } @Test - public void testMatchesRangeWithInvalidRange() { + public void testEvaluateProjectWithFuncMatchesRangeWithInvalidRange() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ project.matches_range("foo") @@ -701,6 +701,36 @@ public void testMatchesRangeWithInvalidRange() { assertThat(qm.getAllPolicyViolations(componentB)).isEmpty(); } + @Test + public void testEvaluateProjectWhenProjectDoesNotExist() { + assertThatNoException().isThrownBy(() -> new CelPolicyEngine().evaluateProject(UUID.randomUUID())); + } + + @Test + public void testEvaluateComponent() { + final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); + qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ + component.name == "acme-lib" + """, PolicyViolation.Type.OPERATIONAL); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + qm.persist(component); + + new CelPolicyEngine().evaluateComponent(component.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void testEvaluateComponentWhenComponentDoesNotExist() { + assertThatNoException().isThrownBy(() -> new CelPolicyEngine().evaluateComponent(UUID.randomUUID())); + } + @Test public void issue1924() { Policy policy = qm.createPolicy("Policy 1924", Policy.Operator.ALL, Policy.ViolationState.INFO); From 88ad93ee1e22e2a48afda1464406dfebbbdc3a62 Mon Sep 17 00:00:00 2001 From: mehab Date: Mon, 25 Sep 2023 17:10:03 +0100 Subject: [PATCH 39/46] added unit tests for cwe cel policy Signed-off-by: mehab --- .../CweCelPolicyScriptSourceBuilder.java | 2 + .../policy/cel/compat/CweConditionTest.java | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/CweConditionTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java index 1d27d108a..7e6e9e1a3 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CweCelPolicyScriptSourceBuilder.java @@ -5,6 +5,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; public class CweCelPolicyScriptSourceBuilder implements CelPolicyScriptSourceBuilder { @@ -14,6 +15,7 @@ public String apply(final PolicyCondition policyCondition) { final List conditionCwes = Arrays.stream(policyCondition.getValue().split(",")) .map(String::trim) .map(CweResolver.getInstance()::parseCweString) + .filter(Objects::nonNull) .sorted() .toList(); if (conditionCwes.isEmpty()) { diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CweConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CweConditionTest.java new file mode 100644 index 000000000..87388e9cf --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CweConditionTest.java @@ -0,0 +1,68 @@ +package org.dependencytrack.policy.cel.compat; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.PolicyViolation; +import org.dependencytrack.model.Project; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(JUnitParamsRunner.class) +public class CweConditionTest extends AbstractPostgresEnabledTest { + private Object[] parameters() { + return new Object[]{ + new Object[]{Policy.Operator.ANY, Policy.ViolationState.INFO, PolicyCondition.Operator.CONTAINS_ANY, + "CWE-123", 123, 0, true, PolicyViolation.Type.SECURITY, Policy.ViolationState.INFO}, + new Object[]{Policy.Operator.ANY, Policy.ViolationState.FAIL, PolicyCondition.Operator.CONTAINS_ALL, + "CWE-123, CWE-786", 123, 786, true, PolicyViolation.Type.SECURITY, Policy.ViolationState.FAIL}, + new Object[]{Policy.Operator.ANY, Policy.ViolationState.FAIL, PolicyCondition.Operator.IS, + "CWE-123, CWE-786", 123, 786, false, null, null}, + new Object[]{Policy.Operator.ANY, Policy.ViolationState.FAIL, PolicyCondition.Operator.CONTAINS_ALL, + "CWE-123.565, CWE-786.67", 123, 786, false, null, null}, + }; + } + + @Test + @Parameters(method = "parameters") + public void testSingleCwe(Policy.Operator policyOperator, Policy.ViolationState violationState, + PolicyCondition.Operator conditionOperator, String inputConditionCwe, int inputCweId, int inputCweId2, + boolean expectViolation, PolicyViolation.Type actualType, Policy.ViolationState actualViolationState) { + Policy policy = qm.createPolicy("Test Policy", policyOperator, violationState); + qm.createPolicyCondition(policy, PolicyCondition.Subject.CWE, conditionOperator, inputConditionCwe); + final var project = new Project(); + project.setName("acme-app"); + final var component = new Component(); + component.setProject(project); + component.setName("acme-lib"); + Vulnerability vulnerability = new Vulnerability(); + vulnerability.setVulnId("12345"); + vulnerability.setSource(Vulnerability.Source.INTERNAL); + vulnerability.addCwe(inputCweId); + if (inputCweId2 != 0) { + vulnerability.addCwe(inputCweId2); + } + qm.persist(project); + qm.persist(component); + qm.persist(vulnerability); + qm.addVulnerability(vulnerability, component, AnalyzerIdentity.INTERNAL_ANALYZER); + new CelPolicyEngine().evaluateProject(project.getUuid()); + if (expectViolation) { + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getSubject()).isEqualTo(PolicyCondition.Subject.CWE); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getViolationType()).isEqualTo(actualType); + assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getPolicy().getViolationState()).isEqualTo(actualViolationState); + } else { + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + } + +} From d6eb71b0b62917d914aec3a9049a6d43d2059db4 Mon Sep 17 00:00:00 2001 From: VithikaS <122881935+VithikaS@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:23:25 +0100 Subject: [PATCH 40/46] Add license condition test (#332) * Add version cel policy script builder Signed-off-by: vithikashukla * add version support for coordinates cel policy Signed-off-by: vithikashukla * Added unit test for version policy script builder Signed-off-by: vithikashukla * added coordninates condition test Signed-off-by: vithikashukla * added coordinates condition test Signed-off-by: vithikashukla * added more conditions to test Signed-off-by: vithikashukla * Added license condition test Signed-off-by: vithikashukla * Update src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java Co-authored-by: Niklas Signed-off-by: VithikaS <122881935+VithikaS@users.noreply.github.com> * Added license group condition test Signed-off-by: vithikashukla * updated comment Signed-off-by: vithikashukla --------- Signed-off-by: vithikashukla Signed-off-by: VithikaS <122881935+VithikaS@users.noreply.github.com> Co-authored-by: vithikashukla Co-authored-by: Niklas --- .../policy/cel/CelPolicyEngine.java | 2 +- .../cel/compat/CoordinatesConditionTest.java | 80 +++++++-- .../cel/compat/LicenseConditionTest.java | 122 ++++++++++++++ .../cel/compat/LicenseGroupConditionTest.java | 152 ++++++++++++++++++ 4 files changed, 342 insertions(+), 14 deletions(-) create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/LicenseConditionTest.java create mode 100644 src/test/java/org/dependencytrack/policy/cel/compat/LicenseGroupConditionTest.java diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index e6d0004e1..a19b1075d 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -157,7 +157,7 @@ public void evaluateProject(final UUID uuid) { // Preload licenses for the entire project, as chances are high that they will be used by multiple components. final Map licenseById; - if (requirements.containsKey(TYPE_LICENSE)) { + if (requirements.containsKey(TYPE_LICENSE) || (requirements.containsKey(TYPE_COMPONENT) && requirements.get(TYPE_COMPONENT).contains("resolved_license"))) { licenseById = celQm.fetchAllLicenses(project.getId(), requirements.get(TYPE_LICENSE), requirements.get(TYPE_LICENSE_GROUP)).stream() .collect(Collectors.toMap( projection -> projection.id, diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java index 4ef544c29..2cd16514d 100644 --- a/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java @@ -20,20 +20,74 @@ public class CoordinatesConditionTest extends AbstractPostgresEnabledTest { private Object[] parameters() { return new Object[]{ - // MATCHES regex - new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"acme*\",\"name\": \"acme*\",\"version\": \">=v1.2*\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", true}, - //Exact match - new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", true}, - //Group does not match - new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"org.hippo\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", false}, + //MATCHES group regex + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'acme*','name': 'acme*','version': '>=v1.2*'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, + //Exact matches + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, + //Exact group does not match + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'org.hippo','name': 'acme-lib','version': 'v1.2.3'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", false}, //Name does not match regex - new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"acme-app\",\"name\": \"*acme-lib*\",\"version\": \"v1.2.3\"}", "{\"group\": \"acme-app\",\"name\": \"good-foo-lib\",\"version\": \"v1.2.3\"}", false}, - //Version does not match - new Object[]{PolicyCondition.Operator.MATCHES, "{\"group\": \"acme-app\",\"name\": \"*acme-lib*\",\"version\": \"v1.*\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v2.2.3\"}", false}, - //Does not match on group - new Object[]{PolicyCondition.Operator.NO_MATCH, "{\"group\": \"diff-group\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", true}, - //Does not match on version - new Object[]{PolicyCondition.Operator.NO_MATCH, "{\"group\": \"acme-app\",\"name\": \"*acme-lib*\",\"version\": \">=v2.2.2\"}", "{\"group\": \"acme-app\",\"name\": \"acme-lib\",\"version\": \"v1.2.3\"}", true}, + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'acme-app','name': '*acme-lib*','version': 'v1.2.3'}", "{'group': 'acme-app','name': 'good-foo-lib','version': 'v1.2.3'}", false}, + //Version regex does not match + new Object[]{PolicyCondition.Operator.MATCHES, "{'group': 'acme-app','name': '*acme-lib*','version': 'v1.*'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v2.2.3'}", false}, + //Does not match on exact group + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'group': 'diff-group','name': 'acme-lib','version': 'v1.2.3'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, + //Does not match on version range greater than or equal + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'group': 'acme-app','name': '*acme-lib*','version': '>=v2.2.2'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, + //Matches without group + new Object[]{PolicyCondition.Operator.MATCHES, "{'name': 'Test Component','version': '1.0.0'}", "{'name': 'Test Component','version': '1.0.0'}", true}, + //Matches on wild card - uncomment after fixing script builder + //new Object[]{PolicyCondition.Operator.MATCHES, "{ 'group': '*', 'name': 'Test Component', 'version': '1.0.0' }", "{ 'group': 'Anything', 'name': 'Test Component', 'version': '1.0.0' }", false}, + //Matches on empty policy - uncomment after fixing script builder + //new Object[]{PolicyCondition.Operator.MATCHES, "{}", "{}", true}, + //Does not match on lower version + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '== 1.1.1' }", "{'version': '0.1.1'}", true}, + //Matches on equal version + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '== 1.1.1' }", "{'version': '1.1.1'}", true}, + //Does not match on higher version + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '== 1.1.1' }", "{'version': '2.1.1'}", true}, + //No match with version not equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '!= 1.1.1' }", "{ 'version': '1.1.1' }", false}, + //Matches with version not equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '!= 1.1.1' }", "{'version': '2.1.1'}", true}, + //Matches with version not equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '!= 1.1.1' }", "{'version': '0.1.1'}", true}, + //Matches with version greater than + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '> 1.1.1' }", "{'version': '2.1.1'}", true}, + //Does not match on version greater than + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '> 1.1.1' }", "{'version': '0.1.1'}", false}, + //Does not match on version equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'version': '> 1.1.1' }", "{'version': '1.1.1'}", false}, + //No match with version greater than + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '> 1.1.1' }", "{'version': '0.1.1'}", true}, + //No match with version equal to + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '> 1.1.1' }", "{'version': '1.1.1'}", true}, + //No match with version greater than + new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '> 1.1.1' }", "{'version': '2.1.1'}", false}, + //Matches on version less than + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<1.1.1'}", "{'version': '0.1.1'}", true}, + //Does not match on version less than + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<1.1.1'}", "{'version': '2.1.1'}", false}, + //Does not match on equal version + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<1.1.1'}", "{'version': '1.1.1'}", false}, + //No match on version less than + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<1.1.1'}", "{'version': '0.1.1'}", false}, + //No match on version less than + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<1.1.1'}", "{'version': '2.1.1'}", true}, + //No match on equal version + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<1.1.1'}", "{'version': '1.1.1'}", true}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<=1.1.1'}", "{'version': '0.1.1'}", true}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<=1.1.1'}", "{'version': '2.1.1'}", false}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.MATCHES, "{'version': '<=1.1.1'}", "{'version': '1.1.1'}", true}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<=1.1.1'}", "{'version': '0.1.1'}", false}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<=1.1.1'}", "{'version': '2.1.1'}", true}, + //Matches on version less than equal to + new Object[]{PolicyCondition.Operator.NO_MATCH, "{'version': '<=1.1.1'}", "{'version': '1.1.1'}", false}, }; } diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/LicenseConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/LicenseConditionTest.java new file mode 100644 index 000000000..4ec271338 --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/LicenseConditionTest.java @@ -0,0 +1,122 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.License; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class LicenseConditionTest extends AbstractPostgresEnabledTest { + + @Test + public void hasMatch() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE, PolicyCondition.Operator.IS, license.getUuid().toString()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void noMatch() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE, PolicyCondition.Operator.IS, UUID.randomUUID().toString()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void wrongOperator() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE, PolicyCondition.Operator.MATCHES, license.getUuid().toString()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setProject(project); + component.setResolvedLicense(license); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void valueIsUnresolved() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE, PolicyCondition.Operator.IS, "unresolved"); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + Component componentWithoutLicense = new Component(); + componentWithoutLicense.setName("second-component"); + componentWithoutLicense.setProject(project); + qm.persist(componentWithoutLicense); + + CelPolicyEngine policyEngine = new CelPolicyEngine(); + + policyEngine.evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(componentWithoutLicense)).hasSize(1); + + final var componentWithLicense = new Component(); + componentWithLicense.setName("acme-app"); + componentWithLicense.setProject(project); + componentWithLicense.setResolvedLicense(license); + qm.persist(componentWithLicense); + + policyEngine.evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(componentWithLicense)).hasSize(0); + } +} + diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/LicenseGroupConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/LicenseGroupConditionTest.java new file mode 100644 index 000000000..2712d9afe --- /dev/null +++ b/src/test/java/org/dependencytrack/policy/cel/compat/LicenseGroupConditionTest.java @@ -0,0 +1,152 @@ +package org.dependencytrack.policy.cel.compat; + +import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.License; +import org.dependencytrack.model.LicenseGroup; +import org.dependencytrack.model.Policy; +import org.dependencytrack.model.PolicyCondition; +import org.dependencytrack.model.Project; +import org.dependencytrack.policy.cel.CelPolicyEngine; +import org.junit.Test; + +import java.util.Collections; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class LicenseGroupConditionTest extends AbstractPostgresEnabledTest { + + @Test + public void hasMatch() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + LicenseGroup lg = qm.createLicenseGroup("Test License Group"); + lg.setLicenses(Collections.singletonList(license)); + lg = qm.persist(lg); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS, lg.getUuid().toString()); + + qm.detach(Policy.class, policy.getId()); + qm.detach(PolicyCondition.class, condition.getId()); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void noMatch() { + License license = new License(); + license.setName("Apache 2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + LicenseGroup lg = qm.createLicenseGroup("Test License Group"); + lg = qm.persist(lg); + + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS, lg.getUuid().toString()); + qm.detach(Policy.class, policy.getId()); + qm.detach(PolicyCondition.class, condition.getId()); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void unknownLicenseViolateWhitelist() { + LicenseGroup lg = qm.createLicenseGroup("Test License Group"); + lg = qm.persist(lg); + lg = qm.detach(LicenseGroup.class, lg.getId()); + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS_NOT, lg.getUuid().toString()); + qm.detach(Policy.class, policy.getId()); + qm.detach(PolicyCondition.class, condition.getId()); + + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(null); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).hasSize(1); + } + + @Test + public void wrongOperator() { + License license = new License(); + license.setName("Apache 2.0"); + license.setLicenseId("Apache-2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + LicenseGroup lg = qm.createLicenseGroup("Test License Group"); + lg.setLicenses(Collections.singletonList(license)); + lg = qm.persist(lg); + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.MATCHES, lg.getUuid().toString()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } + + @Test + public void licenseGroupDoesNotExist() { + License license = new License(); + license.setName("Apache 2.0"); + license.setLicenseId("Apache-2.0"); + license.setUuid(UUID.randomUUID()); + license = qm.persist(license); + Policy policy = qm.createPolicy("Test Policy", Policy.Operator.ANY, Policy.ViolationState.INFO); + PolicyCondition condition = qm.createPolicyCondition(policy, PolicyCondition.Subject.LICENSE_GROUP, PolicyCondition.Operator.IS, UUID.randomUUID().toString()); + qm.detach(Policy.class, policy.getId()); + qm.detach(PolicyCondition.class, condition.getId()); + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final var component = new Component(); + component.setName("acme-app"); + component.setResolvedLicense(license); + component.setProject(project); + qm.persist(component); + + new CelPolicyEngine().evaluateProject(project.getUuid()); + assertThat(qm.getAllPolicyViolations(component)).isEmpty(); + } +} From 0f4c0a8d503ce30fb2a3bf0641960a7011cdd643 Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 26 Sep 2023 11:52:23 +0200 Subject: [PATCH 41/46] Fix projection mapping for `Double` / `BigDecimal` fields Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 34 +++++++++---------- .../cel/mapping/VulnerabilityProjection.java | 23 +++++++------ 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index a19b1075d..51ec8d606 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -339,7 +339,7 @@ private static List evaluateConditions(final Collection Date: Tue, 26 Sep 2023 12:55:02 +0100 Subject: [PATCH 42/46] support wildcard Signed-off-by: vithikashukla --- ...ordinatesCelPolicyScriptSourceBuilder.java | 21 +++++++++++++++---- .../cel/compat/CoordinatesConditionTest.java | 10 ++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java index 8406363aa..721f1fe03 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/CoordinatesCelPolicyScriptSourceBuilder.java @@ -39,14 +39,16 @@ public String apply(final PolicyCondition condition) { return null; } - private static String evaluateScript(final String group, final String name, final String version) { + private static String evaluateScript(final String conditionGroupPart, final String conditionNamePart, final String conditionVersionPart) { + final String group = replace(conditionGroupPart); + final String name = replace(conditionNamePart); - final Matcher versionOperatorMatcher = VERSION_OPERATOR_PATTERN.matcher(version); + final Matcher versionOperatorMatcher = VERSION_OPERATOR_PATTERN.matcher(conditionVersionPart); //Do an exact match if no operator found if (!versionOperatorMatcher.find()) { Vers conditionVers = Vers.builder(VersioningScheme.GENERIC) - .withConstraint(Comparator.EQUAL, version) + .withConstraint(Comparator.EQUAL, conditionVersionPart) .build(); return """ component.group.matches("%s") && component.name.matches("%s") && component.matches_range("%s") @@ -67,7 +69,7 @@ private static String evaluateScript(final String group, final String name, fina LOGGER.error("Failed to infer version operator from " + versionOperatorMatcher.group(1)); return null; } - String condition = VERSION_OPERATOR_PATTERN.split(version)[1]; + String condition = VERSION_OPERATOR_PATTERN.split(conditionVersionPart)[1]; Vers conditionVers = Vers.builder(VersioningScheme.GENERIC) .withConstraint(versionComparator, condition) .build(); @@ -76,4 +78,15 @@ private static String evaluateScript(final String group, final String name, fina component.group.matches("%s") && component.name.matches("%s") && component.matches_range("%s") """.formatted(escapeJson(group), escapeJson(name), conditionVers.toString()); } + + private static String replace(String conditionString) { + conditionString = conditionString.replace("*", ".*").replace("..*", ".*"); + if (!conditionString.startsWith("^") && !conditionString.startsWith(".*")) { + conditionString = ".*" + conditionString; + } + if (!conditionString.endsWith("$") && !conditionString.endsWith(".*")) { + conditionString += ".*"; + } + return conditionString; + } } diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java index 2cd16514d..ef46208f0 100644 --- a/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/compat/CoordinatesConditionTest.java @@ -36,9 +36,13 @@ private Object[] parameters() { new Object[]{PolicyCondition.Operator.NO_MATCH, "{'group': 'acme-app','name': '*acme-lib*','version': '>=v2.2.2'}", "{'group': 'acme-app','name': 'acme-lib','version': 'v1.2.3'}", true}, //Matches without group new Object[]{PolicyCondition.Operator.MATCHES, "{'name': 'Test Component','version': '1.0.0'}", "{'name': 'Test Component','version': '1.0.0'}", true}, - //Matches on wild card - uncomment after fixing script builder - //new Object[]{PolicyCondition.Operator.MATCHES, "{ 'group': '*', 'name': 'Test Component', 'version': '1.0.0' }", "{ 'group': 'Anything', 'name': 'Test Component', 'version': '1.0.0' }", false}, - //Matches on empty policy - uncomment after fixing script builder + //Matches on wild card group - uncomment after fixing script builder + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'group': '*', 'name': 'Test Component', 'version': '1.0.0' }", "{ 'group': 'Anything', 'name': 'Test Component', 'version': '1.0.0' }", true}, + //Matches on wild card name + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'group': 'acme-app', 'name': '*', 'version': '1.0.0' }", "{ 'group': 'acme-app', 'name': 'Anything', 'version': '1.0.0' }", true}, + //Matches on wild card version + new Object[]{PolicyCondition.Operator.MATCHES, "{ 'group': 'acme-app', 'name': 'Test Component', 'version': '>=*' }", "{ 'group': 'acme-app', 'name': 'Test Component', 'version': '4.4.4' }", true}, + //Matches on empty policy - uncomment after fixing script builder //new Object[]{PolicyCondition.Operator.MATCHES, "{}", "{}", true}, //Does not match on lower version new Object[]{PolicyCondition.Operator.NO_MATCH, "{ 'version': '== 1.1.1' }", "{'version': '0.1.1'}", true}, From 94eb8fcaa1128adc82118bc79d3cbbbce91d0d6e Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 26 Sep 2023 18:22:03 +0200 Subject: [PATCH 43/46] Add `buf` config and workflow Signed-off-by: nscuro --- .github/workflows/buf.yml | 29 +++++++++++++++++++++++++++++ src/main/proto/buf.yaml | 8 ++++++++ 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/buf.yml create mode 100644 src/main/proto/buf.yaml diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml new file mode 100644 index 000000000..93ce5da13 --- /dev/null +++ b/.github/workflows/buf.yml @@ -0,0 +1,29 @@ +name: Buf + +on: + pull_request: + branches: [ "main" ] + +permissions: { } + +jobs: + buf: + name: Buf + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout Repository + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # tag=v4.1.0 + - name: Setup buf + uses: bufbuild/buf-setup-action@eb60cd0de4f14f1f57cf346916b8cd69a9e7ed0b # tag=v1.26.1 + with: + github_token: ${{ github.token }} + - name: Lint Protobuf + uses: bufbuild/buf-lint-action@bd48f53224baaaf0fc55de9a913e7680ca6dbea4 # tag=v1.0.3 + with: + input: src/main/proto + - name: Detect Breaking Changes + uses: bufbuild/buf-breaking-action@a074e988ee34efcd4927079e79c611f428354c01 # tag=v1.1.3 + with: + input: src/main/proto + against: https://github.com/${{ github.repository }}.git#branch=main,subdir=src/main/proto \ No newline at end of file diff --git a/src/main/proto/buf.yaml b/src/main/proto/buf.yaml new file mode 100644 index 000000000..022c724b6 --- /dev/null +++ b/src/main/proto/buf.yaml @@ -0,0 +1,8 @@ +version: v1 +name: github.com/DependencyTrack/hyades-apiserver +lint: + ignore: + - org/cyclonedx/v1_4/cyclonedx.proto +breaking: + ignore: + - org/cyclonedx/v1_4/cyclonedx.proto \ No newline at end of file From 1d78e4bbb0bbc76377b3defb2babd9f08919c5cb Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 26 Sep 2023 18:23:33 +0200 Subject: [PATCH 44/46] Change Proto package from `hyades` to `dependencytrack` As this feature will be backported, we need to make sure policies will be compatible once folks start upgrading to Hyades. Signed-off-by: nscuro --- .../policy/cel/CelPolicyEngine.java | 44 +++++++++---------- .../policy/cel/CelPolicyLibrary.java | 8 ++-- ...onentHashCelPolicyScriptSourceBuilder.java | 2 +- .../policy/v1/policy.proto | 8 ++-- .../policy/cel/CelPolicyScriptHostTest.java | 2 +- .../cel/mapping/FieldMappingUtilTest.java | 8 ++-- 6 files changed, 35 insertions(+), 37 deletions(-) rename src/main/proto/org/{hyades => dependencytrack}/policy/v1/policy.proto (97%) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java index 51ec8d606..6fe736823 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyEngine.java @@ -42,9 +42,9 @@ import org.dependencytrack.policy.cel.mapping.ProjectProjection; import org.dependencytrack.policy.cel.mapping.ProjectPropertyProjection; import org.dependencytrack.policy.cel.mapping.VulnerabilityProjection; +import org.dependencytrack.proto.policy.v1.Vulnerability; import org.dependencytrack.util.NotificationUtil; import org.dependencytrack.util.VulnerabilityUtil; -import org.hyades.proto.policy.v1.Vulnerability; import org.projectnessie.cel.tools.ScriptCreateException; import org.projectnessie.cel.tools.ScriptException; @@ -145,18 +145,18 @@ public void evaluateProject(final UUID uuid) { LOGGER.debug("Requirements for project %s and %d policy conditions: %s" .formatted(uuid, conditionScriptPairs.size(), requirements)); - final org.hyades.proto.policy.v1.Project protoProject; + final org.dependencytrack.proto.policy.v1.Project protoProject; if (requirements.containsKey(TYPE_PROJECT)) { protoProject = mapToProto(celQm.fetchProject(project.getId(), requirements.get(TYPE_PROJECT), requirements.get(TYPE_PROJECT_PROPERTY))); } else { - protoProject = org.hyades.proto.policy.v1.Project.getDefaultInstance(); + protoProject = org.dependencytrack.proto.policy.v1.Project.getDefaultInstance(); } // Preload components for the entire project, to avoid excessive queries. final List components = celQm.fetchAllComponents(project.getId(), requirements.get(TYPE_COMPONENT)); // Preload licenses for the entire project, as chances are high that they will be used by multiple components. - final Map licenseById; + final Map licenseById; if (requirements.containsKey(TYPE_LICENSE) || (requirements.containsKey(TYPE_COMPONENT) && requirements.get(TYPE_COMPONENT).contains("resolved_license"))) { licenseById = celQm.fetchAllLicenses(project.getId(), requirements.get(TYPE_LICENSE), requirements.get(TYPE_LICENSE_GROUP)).stream() .collect(Collectors.toMap( @@ -168,7 +168,7 @@ public void evaluateProject(final UUID uuid) { } // Preload vulnerabilities for the entire project, as chances are high that they will be used by multiple components. - final Map protoVulnById; + final Map protoVulnById; final Map> vulnIdsByComponentId; if (requirements.containsKey(TYPE_VULNERABILITY)) { protoVulnById = celQm.fetchAllVulnerabilities(project.getId(), requirements.get(TYPE_VULNERABILITY)).stream() @@ -190,8 +190,8 @@ public void evaluateProject(final UUID uuid) { // Evaluate all policy conditions against all components. final var conditionsViolated = new HashSetValuedHashMap(); for (final ComponentProjection component : components) { - final org.hyades.proto.policy.v1.Component protoComponent = mapToProto(component, licenseById); - final List protoVulns = + final org.dependencytrack.proto.policy.v1.Component protoComponent = mapToProto(component, licenseById); + final List protoVulns = vulnIdsByComponentId.getOrDefault(component.id, emptyList()).stream() .map(protoVulnById::get) .toList(); @@ -379,8 +379,8 @@ private static List evaluatePolicyOperators(final Collection() { }); for (final ProjectPropertyProjection property : properties) { - builder.addProperties(org.hyades.proto.policy.v1.Project.Property.newBuilder() + builder.addProperties(org.dependencytrack.proto.policy.v1.Project.Property.newBuilder() .setGroup(trimToEmpty(property.group)) .setName(trimToEmpty(property.name)) .setValue(trimToEmpty(property.value)) @@ -425,10 +425,10 @@ private static org.hyades.proto.policy.v1.Project mapToProto(final ProjectProjec return builder.build(); } - private static org.hyades.proto.policy.v1.Component mapToProto(final ComponentProjection projection, - final Map protoLicenseById) { - final org.hyades.proto.policy.v1.Component.Builder componentBuilder = - org.hyades.proto.policy.v1.Component.newBuilder() + private static org.dependencytrack.proto.policy.v1.Component mapToProto(final ComponentProjection projection, + final Map protoLicenseById) { + final org.dependencytrack.proto.policy.v1.Component.Builder componentBuilder = + org.dependencytrack.proto.policy.v1.Component.newBuilder() .setUuid(trimToEmpty(projection.uuid)) .setGroup(trimToEmpty(projection.group)) .setName(trimToEmpty(projection.name)) @@ -453,7 +453,7 @@ private static org.hyades.proto.policy.v1.Component mapToProto(final ComponentPr .setBlake3(trimToEmpty(projection.blake3)); if (projection.resolvedLicenseId != null && projection.resolvedLicenseId > 0) { - final org.hyades.proto.policy.v1.License protoLicense = protoLicenseById.get(projection.resolvedLicenseId); + final org.dependencytrack.proto.policy.v1.License protoLicense = protoLicenseById.get(projection.resolvedLicenseId); if (protoLicense != null) { componentBuilder.setResolvedLicense(protoLicenseById.get(projection.resolvedLicenseId)); } else { @@ -465,9 +465,9 @@ private static org.hyades.proto.policy.v1.Component mapToProto(final ComponentPr return componentBuilder.build(); } - private static org.hyades.proto.policy.v1.License mapToProto(final LicenseProjection projection) { - final org.hyades.proto.policy.v1.License.Builder licenseBuilder = - org.hyades.proto.policy.v1.License.newBuilder() + private static org.dependencytrack.proto.policy.v1.License mapToProto(final LicenseProjection projection) { + final org.dependencytrack.proto.policy.v1.License.Builder licenseBuilder = + org.dependencytrack.proto.policy.v1.License.newBuilder() .setUuid(trimToEmpty(projection.uuid)) .setId(trimToEmpty(projection.licenseId)) .setName(trimToEmpty(projection.name)); @@ -480,7 +480,7 @@ private static org.hyades.proto.policy.v1.License mapToProto(final LicenseProjec try { final ArrayNode groupsArray = OBJECT_MAPPER.readValue(projection.licenseGroupsJson, ArrayNode.class); for (final JsonNode groupNode : groupsArray) { - licenseBuilder.addGroups(org.hyades.proto.policy.v1.License.Group.newBuilder() + licenseBuilder.addGroups(org.dependencytrack.proto.policy.v1.License.Group.newBuilder() .setUuid(Optional.ofNullable(groupNode.get("uuid")).map(JsonNode::asText).orElse("")) .setName(Optional.ofNullable(groupNode.get("name")).map(JsonNode::asText).orElse("")) .build()); @@ -497,9 +497,9 @@ private static org.hyades.proto.policy.v1.License mapToProto(final LicenseProjec private static final TypeReference> VULNERABILITY_ALIASES_TYPE_REF = new TypeReference<>() { }; - private static org.hyades.proto.policy.v1.Vulnerability mapToProto(final VulnerabilityProjection projection) { - final org.hyades.proto.policy.v1.Vulnerability.Builder builder = - org.hyades.proto.policy.v1.Vulnerability.newBuilder() + private static org.dependencytrack.proto.policy.v1.Vulnerability mapToProto(final VulnerabilityProjection projection) { + final org.dependencytrack.proto.policy.v1.Vulnerability.Builder builder = + org.dependencytrack.proto.policy.v1.Vulnerability.newBuilder() .setUuid(trimToEmpty(projection.uuid)) .setId(trimToEmpty(projection.vulnId)) .setSource(trimToEmpty(projection.source)) diff --git a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java index 562714ace..acb2721e3 100644 --- a/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java +++ b/src/main/java/org/dependencytrack/policy/cel/CelPolicyLibrary.java @@ -6,10 +6,10 @@ import io.github.nscuro.versatile.VersException; import org.apache.commons.lang3.tuple.Pair; import org.dependencytrack.persistence.QueryManager; -import org.hyades.proto.policy.v1.Component; -import org.hyades.proto.policy.v1.License; -import org.hyades.proto.policy.v1.Project; -import org.hyades.proto.policy.v1.Vulnerability; +import org.dependencytrack.proto.policy.v1.Component; +import org.dependencytrack.proto.policy.v1.License; +import org.dependencytrack.proto.policy.v1.Project; +import org.dependencytrack.proto.policy.v1.Vulnerability; import org.projectnessie.cel.EnvOption; import org.projectnessie.cel.Library; import org.projectnessie.cel.ProgramOption; diff --git a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java index 339c0c311..34e9dde97 100644 --- a/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java +++ b/src/main/java/org/dependencytrack/policy/cel/compat/ComponentHashCelPolicyScriptSourceBuilder.java @@ -19,7 +19,7 @@ public String apply(final PolicyCondition policyCondition) { } final String fieldName = hash.getAlgorithm().toLowerCase().replaceAll("-", "_"); - if (org.hyades.proto.policy.v1.Component.getDescriptor().findFieldByName(fieldName) == null) { + if (org.dependencytrack.proto.policy.v1.Component.getDescriptor().findFieldByName(fieldName) == null) { LOGGER.warn("Component does not have a field named %s".formatted(fieldName)); return null; } diff --git a/src/main/proto/org/hyades/policy/v1/policy.proto b/src/main/proto/org/dependencytrack/policy/v1/policy.proto similarity index 97% rename from src/main/proto/org/hyades/policy/v1/policy.proto rename to src/main/proto/org/dependencytrack/policy/v1/policy.proto index 88f64f8ee..71a861171 100644 --- a/src/main/proto/org/hyades/policy/v1/policy.proto +++ b/src/main/proto/org/dependencytrack/policy/v1/policy.proto @@ -1,14 +1,13 @@ syntax = "proto3"; -package org.hyades.policy.v1; +package org.dependencytrack.policy.v1; import "google/protobuf/timestamp.proto"; option java_multiple_files = true; -option java_package = "org.hyades.proto.policy.v1"; +option java_package = "org.dependencytrack.proto.policy.v1"; message Component { - // UUID of the component. string uuid = 1; @@ -133,5 +132,4 @@ message Vulnerability { string id = 1; string source = 2; } - -} \ No newline at end of file +} diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java index ee8a687a4..11fae9b64 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyScriptHostTest.java @@ -60,7 +60,7 @@ public void testRequirementsAnalysis() throws Exception { final CelPolicyScript compiledScript = CelPolicyScriptHost.getInstance().compile(""" component.resolved_license.groups.exists(licenseGroup, licenseGroup.name == "Permissive") && vulns.exists(vuln, vuln.severity in ["HIGH", "CRITICAL"] && has(vuln.aliases)) - && project.depends_on(org.hyades.policy.v1.Component{name: "foo"}) + && project.depends_on(org.dependencytrack.policy.v1.Component{name: "foo"}) """, CacheMode.NO_CACHE); final Map> requirements = compiledScript.getRequirements().asMap(); diff --git a/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java b/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java index f2b480391..f6143264f 100644 --- a/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/mapping/FieldMappingUtilTest.java @@ -2,10 +2,10 @@ import com.google.protobuf.Descriptors.Descriptor; import org.dependencytrack.PersistenceCapableTest; -import org.hyades.proto.policy.v1.Component; -import org.hyades.proto.policy.v1.License; -import org.hyades.proto.policy.v1.Project; -import org.hyades.proto.policy.v1.Vulnerability; +import org.dependencytrack.proto.policy.v1.Component; +import org.dependencytrack.proto.policy.v1.License; +import org.dependencytrack.proto.policy.v1.Project; +import org.dependencytrack.proto.policy.v1.Vulnerability; import org.junit.Test; import javax.jdo.PersistenceManagerFactory; From 2b035e4affe68a4f80be2783a7f727d0accc9043 Mon Sep 17 00:00:00 2001 From: nscuro Date: Wed, 27 Sep 2023 11:02:36 +0200 Subject: [PATCH 45/46] Fix failing tests due to Proto package change Signed-off-by: nscuro --- .../org/dependencytrack/policy/cel/CelPolicyEngineTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index 91b4d93e0..a08d60447 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -22,6 +22,7 @@ import org.dependencytrack.tasks.BomUploadProcessingTask; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -580,7 +581,7 @@ public void testEvaluateProjectWithScriptExecutionException() { public void testEvaluateProjectWithFuncProjectDependsOnComponent() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - project.depends_on(org.hyades.policy.v1.Component{name: "acme-lib-a"}) + project.depends_on(org.dependencytrack.policy.v1.Component{name: "acme-lib-a"}) """, PolicyViolation.Type.OPERATIONAL); final var project = new Project(); @@ -613,7 +614,7 @@ public void testEvaluateProjectWithFuncProjectDependsOnComponent() { public void testEvaluateProjectWithFuncComponentIsDependencyOfComponent() { final var policy = qm.createPolicy("policy", Policy.Operator.ANY, Policy.ViolationState.FAIL); qm.createPolicyCondition(policy, PolicyCondition.Subject.EXPRESSION, PolicyCondition.Operator.MATCHES, """ - component.is_dependency_of(org.hyades.policy.v1.Component{name: "acme-lib-a"}) + component.is_dependency_of(org.dependencytrack.policy.v1.Component{name: "acme-lib-a"}) """, PolicyViolation.Type.OPERATIONAL); final var project = new Project(); @@ -863,6 +864,7 @@ public void issue2455() { } @Test + @Ignore // Un-ignore for manual profiling purposes. public void testWithBloatedBom() throws Exception { // Import all default objects (includes licenses and license groups). new DefaultObjectGenerator().contextInitialized(null); From 39664d5e0872d2f8cae710f926e36952735eed6b Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 27 Sep 2023 13:02:49 +0200 Subject: [PATCH 46/46] Un-ignore `cyclonedx.proto` from breaking changes check Signed-off-by: Niklas --- src/main/proto/buf.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/proto/buf.yaml b/src/main/proto/buf.yaml index 022c724b6..eb14abb5f 100644 --- a/src/main/proto/buf.yaml +++ b/src/main/proto/buf.yaml @@ -3,6 +3,3 @@ name: github.com/DependencyTrack/hyades-apiserver lint: ignore: - org/cyclonedx/v1_4/cyclonedx.proto -breaking: - ignore: - - org/cyclonedx/v1_4/cyclonedx.proto \ No newline at end of file