Skip to content

Commit

Permalink
Implement SPDX expressions
Browse files Browse the repository at this point in the history
Ported from DependencyTrack/dependency-track#2400

Note: Evaluation of SPDX expressions is not implemented for CEL policies. This will be done separately in DependencyTrack/hyades#872.

Co-authored-by: Hendrik Borchardt <[email protected]>
Signed-off-by: nscuro <[email protected]>
  • Loading branch information
nscuro and hborchardt committed Oct 24, 2023
1 parent 53f6e7d commit 0be59a9
Show file tree
Hide file tree
Showing 17 changed files with 877 additions and 71 deletions.
23 changes: 18 additions & 5 deletions src/main/java/org/dependencytrack/model/Component.java
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,11 @@ public enum FetchGroup {
@Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The license may only contain printable characters")
private String license;

@Persistent
@Column(name = "LICENSE_EXPRESSION", jdbcType = "CLOB", allowsNull = "true")
@Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The license expression may only contain printable characters")
private String licenseExpression;

@Persistent
@Column(name = "LICENSE_URL", jdbcType = "VARCHAR")
@Size(max = 255)
Expand Down Expand Up @@ -352,7 +357,7 @@ public enum FetchGroup {
private UUID uuid;

private transient String bomRef;
private transient String licenseId;
private transient List<org.cyclonedx.model.License> licenseCandidates;
private transient DependencyMetrics metrics;
private transient RepositoryMetaComponent repositoryMeta;

Expand Down Expand Up @@ -629,6 +634,14 @@ public void setLicense(String license) {
this.license = StringUtils.abbreviate(license, 255);
}

public String getLicenseExpression() {
return licenseExpression;
}

public void setLicenseExpression(String licenseExpression) {
this.licenseExpression = licenseExpression;
}

public String getLicenseUrl() {
return licenseUrl;
}
Expand Down Expand Up @@ -775,12 +788,12 @@ public void setBomRef(String bomRef) {
this.bomRef = bomRef;
}

public String getLicenseId() {
return licenseId;
public List<org.cyclonedx.model.License> getLicenseCandidates() {
return licenseCandidates;
}

public void setLicenseId(final String licenseId) {
this.licenseId = licenseId;
public void setLicenseCandidates(final List<org.cyclonedx.model.License> licenseCandidates) {
this.licenseCandidates = licenseCandidates;
}

public int getUsedBy() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
import org.dependencytrack.model.Vulnerability;
import org.dependencytrack.parser.common.resolver.CweResolver;
import org.dependencytrack.parser.cyclonedx.CycloneDXExporter;
import org.dependencytrack.parser.spdx.expression.SpdxExpressionParser;
import org.dependencytrack.parser.spdx.expression.model.SpdxExpression;
import org.dependencytrack.persistence.QueryManager;
import org.dependencytrack.util.VulnerabilityUtil;

Expand All @@ -56,13 +58,16 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;

import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.trim;
import static org.apache.commons.lang3.StringUtils.trimToNull;
import static org.dependencytrack.util.PurlUtil.silentPurlCoordinatesOnly;

Expand Down Expand Up @@ -162,18 +167,33 @@ public static Component convertComponent(final org.cyclonedx.model.Component cdx
}
}

if (cdxComponent.getLicenseChoice() != null
&& cdxComponent.getLicenseChoice().getLicenses() != null
&& !cdxComponent.getLicenseChoice().getLicenses().isEmpty()) {
for (final org.cyclonedx.model.License cdxLicense : cdxComponent.getLicenseChoice().getLicenses()) {
if (cdxLicense != null) {
component.setLicenseId(trimToNull(cdxLicense.getId()));
component.setLicense(trimToNull(cdxLicense.getName()));
component.setLicenseUrl(trimToNull(cdxLicense.getUrl()));
break; // Components in CDX can have multiple licenses, but DT supports only one
final var licenseCandidates = new ArrayList<org.cyclonedx.model.License>();
if (cdxComponent.getLicenseChoice() != null) {
if (cdxComponent.getLicenseChoice().getLicenses() != null) {
cdxComponent.getLicenseChoice().getLicenses().stream()
.filter(license -> isNotBlank(license.getId()) || isNotBlank(license.getName()))
.peek(license -> {
// License text can be large, but we don't need it for further processing. Drop it.
license.setLicenseText(null);
})
.forEach(licenseCandidates::add);
}

if (isNotBlank(cdxComponent.getLicenseChoice().getExpression())) {
component.setLicenseExpression(trim(cdxComponent.getLicenseChoice().getExpression()));

// If the expression consists of just one license ID, add it as another option.
final var expressionParser = new SpdxExpressionParser();
final SpdxExpression expression = expressionParser.parse(component.getLicenseExpression());
if (expression.getSpdxLicenseId() != null) {
final var expressionLicense = new org.cyclonedx.model.License();
expressionLicense.setId(expression.getSpdxLicenseId());
expressionLicense.setName(expression.getSpdxLicenseId());
licenseCandidates.add(expressionLicense);
}
}
}
component.setLicenseCandidates(licenseCandidates);

if (cdxComponent.getComponents() != null && !cdxComponent.getComponents().isEmpty()) {
final var children = new ArrayList<Component>();
Expand Down Expand Up @@ -358,32 +378,33 @@ public static org.cyclonedx.model.Component convert(final QueryManager qm, final
cycloneComponent.addHash(new Hash(Hash.Algorithm.SHA3_512, component.getSha3_512()));
}

final LicenseChoice licenseChoice = new LicenseChoice();
if (component.getResolvedLicense() != null) {
final org.cyclonedx.model.License license = new org.cyclonedx.model.License();
license.setId(component.getResolvedLicense().getLicenseId());
license.setUrl(component.getLicenseUrl());
final LicenseChoice licenseChoice = new LicenseChoice();
licenseChoice.addLicense(license);
cycloneComponent.setLicenseChoice(licenseChoice);
} else if (component.getLicense() != null) {
final org.cyclonedx.model.License license = new org.cyclonedx.model.License();
license.setName(component.getLicense());
license.setUrl(component.getLicenseUrl());
final LicenseChoice licenseChoice = new LicenseChoice();
licenseChoice.addLicense(license);
cycloneComponent.setLicenseChoice(licenseChoice);
} else if (StringUtils.isNotEmpty(component.getLicenseUrl())) {
final org.cyclonedx.model.License license = new org.cyclonedx.model.License();
license.setUrl(component.getLicenseUrl());
final LicenseChoice licenseChoice = new LicenseChoice();
licenseChoice.addLicense(license);
cycloneComponent.setLicenseChoice(licenseChoice);
}

if (component.getLicenseExpression() != null) {
licenseChoice.setExpression(component.getLicenseExpression());
cycloneComponent.setLicenseChoice(licenseChoice);
}

if (component.getExternalReferences() != null && component.getExternalReferences().size() > 0) {
List<org.cyclonedx.model.ExternalReference> references = new ArrayList<>();
for (ExternalReference ref: component.getExternalReferences()) {
for (ExternalReference ref : component.getExternalReferences()) {
org.cyclonedx.model.ExternalReference cdxRef = new org.cyclonedx.model.ExternalReference();
cdxRef.setType(ref.getType());
cdxRef.setUrl(ref.getUrl());
Expand Down Expand Up @@ -737,17 +758,23 @@ private static org.cyclonedx.model.vulnerability.Vulnerability.Source convertDtV
cdxSource.setName(vulnSource.name());
switch (vulnSource) {
case NVD:
cdxSource.setUrl("https://nvd.nist.gov/"); break;
cdxSource.setUrl("https://nvd.nist.gov/");
break;
case NPM:
cdxSource.setUrl("https://www.npmjs.com/"); break;
cdxSource.setUrl("https://www.npmjs.com/");
break;
case GITHUB:
cdxSource.setUrl("https://github.com/advisories"); break;
cdxSource.setUrl("https://github.com/advisories");
break;
case VULNDB:
cdxSource.setUrl("https://vulndb.cyberriskanalytics.com/"); break;
cdxSource.setUrl("https://vulndb.cyberriskanalytics.com/");
break;
case OSSINDEX:
cdxSource.setUrl("https://ossindex.sonatype.org/"); break;
cdxSource.setUrl("https://ossindex.sonatype.org/");
break;
case RETIREJS:
cdxSource.setUrl("https://github.com/RetireJS/retire.js"); break;
cdxSource.setUrl("https://github.com/RetireJS/retire.js");
break;
}
return cdxSource;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) Steve Springett. All Rights Reserved.
*/
package org.dependencytrack.parser.spdx.expression;

import java.util.ArrayDeque;
import java.util.Iterator;
import java.util.List;

import org.dependencytrack.parser.spdx.expression.model.SpdxOperator;
import org.dependencytrack.parser.spdx.expression.model.SpdxExpression;

/**
* This class parses SPDX expressions according to
* https://spdx.github.io/spdx-spec/v2-draft/SPDX-license-expressions/ into a tree of
* SpdxExpressions and SpdxExpressionOperations
*
* @author hborchardt
* @since 4.8.0
*/
public class SpdxExpressionParser {

/**
* Reads in a SPDX expression and returns a parsed tree of SpdxExpressionOperators and license
* ids.
*
* @param spdxExpression
* spdx expression string
* @return parsed SpdxExpression tree, or SpdxExpression.INVALID if an error has occurred during
* parsing
*/
public SpdxExpression parse(final String spdxExpression) {
// operators are surrounded by spaces or brackets. Let's make our life easier and surround brackets by spaces.
var _spdxExpression = spdxExpression.replace("(", " ( ").replace(")", " ) ").split(" ");
if (_spdxExpression.length == 1) {
return new SpdxExpression(spdxExpression);
}

// Shunting yard algorithm to convert SPDX expression to reverse polish notation
// specify list of infix operators
List<String> infixOperators = List.of(SpdxOperator.OR.getToken(), SpdxOperator.AND.getToken(),
SpdxOperator.WITH.getToken());

ArrayDeque<String> operatorStack = new ArrayDeque<>();
ArrayDeque<String> outputQueue = new ArrayDeque<>();
Iterator<String> it = List.of(_spdxExpression).iterator();
while(it.hasNext()) {
var token = it.next();
if (token.length() == 0) {
continue;
}
if (infixOperators.contains(token)) {
int opPrecedence = SpdxOperator.valueOf(token).getPrecedence();
for (String o2; (o2 = operatorStack.peek()) != null && !o2.equals("(")
&& SpdxOperator.valueOf(o2).getPrecedence() > opPrecedence;) {
outputQueue.push(operatorStack.pop());
}
;
operatorStack.push(token);
} else if (token.equals("(")) {
operatorStack.push(token);
} else if (token.equals(")")) {
for (String o2; (o2 = operatorStack.peek()) == null || !o2.equals("(");) {
if (o2 == null) {
// Mismatched parentheses
return SpdxExpression.INVALID;
}
outputQueue.push(operatorStack.pop());
}
;
String leftParens = operatorStack.pop();

if (!"(".equals(leftParens)) {
// Mismatched parentheses
return SpdxExpression.INVALID;
}
// no function tokens implemented
} else {
outputQueue.push(token);
}
}
for (String o2; (o2 = operatorStack.peek()) != null;) {
if ("(".equals(o2)) {
// Mismatched parentheses
return SpdxExpression.INVALID;
}
outputQueue.push(operatorStack.pop());
}

// convert RPN stack into tree
// this is easy because all infix operators have two arguments
ArrayDeque<SpdxExpression> expressions = new ArrayDeque<>();
SpdxExpression expr = null;
while (!outputQueue.isEmpty()) {
var token = outputQueue.pollLast();
if (infixOperators.contains(token)) {
var rhs = expressions.pop();
var lhs = expressions.pop();
expr = new SpdxExpression(SpdxOperator.valueOf(token), List.of(lhs, rhs));
} else {
if (token.endsWith("+")) {
// trailing `+` is not a whitespace-delimited operator - process it separately
expr = new SpdxExpression(SpdxOperator.PLUS,
List.of(new SpdxExpression(token.substring(0, token.length() - 1))));
} else {
expr = new SpdxExpression(token);
}
}
expressions.push(expr);
}
return expr;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) Steve Springett. All Rights Reserved.
*/
package org.dependencytrack.parser.spdx.expression.model;

import java.util.List;

/**
* A node of an SPDX expression tree. If it is a leaf node, it contains a spdxLicenseId. If it is an
* inner node, containss an operation.
*
* @author hborchardt
* @since 4.8.0
*/
public class SpdxExpression {
public static final SpdxExpression INVALID = new SpdxExpression(null);
public SpdxExpression(String spdxLicenseId) {
this.spdxLicenseId = spdxLicenseId;
}

public SpdxExpression(SpdxOperator operator, List<SpdxExpression> arguments) {
this.operation = new SpdxExpressionOperation(operator, arguments);
}

private SpdxExpressionOperation operation;
private String spdxLicenseId;

public SpdxExpressionOperation getOperation() {
return operation;
}

public void setOperation(SpdxExpressionOperation operation) {
this.operation = operation;
}

public String getSpdxLicenseId() {
return spdxLicenseId;
}

public void setSpdxLicenseId(String spdxLicenseId) {
this.spdxLicenseId = spdxLicenseId;
}

@Override
public String toString() {
if (this == INVALID) {
return "INVALID";
}
if (spdxLicenseId != null) {
return spdxLicenseId;
}
return operation.toString();
}
}
Loading

0 comments on commit 0be59a9

Please sign in to comment.