From f8018650de342af2a676998f58ccee4c45af9e7c Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Fri, 14 Jun 2024 14:07:36 +0100 Subject: [PATCH 1/2] add hackage and nixpkgs analyzers Co-Authored-By: Mango The Fourth <40720523+mangoiv@users.noreply.github.com> --- .gitignore | 6 +- .../persistence/model/RepositoryType.java | 6 + .../repositories/HackageMetaAnalyzer.java | 101 ++++++++++++++++ .../repositories/NixpkgsMetaAnalyzer.java | 112 ++++++++++++++++++ .../repositories/HackageMetaAnalyzerTest.java | 49 ++++++++ .../repositories/NixpkgsMetaAnalyzerTest.java | 101 ++++++++++++++++ 6 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzer.java create mode 100644 repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzer.java create mode 100644 repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzerTest.java create mode 100644 repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java diff --git a/.gitignore b/.gitignore index 5b2a28299..db23953a1 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,8 @@ secret.key apiserver-data/ # Generated test fixtures -testdata/boms/generated/*.cdx.json \ No newline at end of file +testdata/boms/generated/*.cdx.json + +# nix +.direnv +.pre-commit-config.yaml \ No newline at end of file diff --git a/commons-persistence/src/main/java/org/dependencytrack/persistence/model/RepositoryType.java b/commons-persistence/src/main/java/org/dependencytrack/persistence/model/RepositoryType.java index c9397304a..f8bf5c5c0 100644 --- a/commons-persistence/src/main/java/org/dependencytrack/persistence/model/RepositoryType.java +++ b/commons-persistence/src/main/java/org/dependencytrack/persistence/model/RepositoryType.java @@ -39,6 +39,8 @@ public enum RepositoryType { GO_MODULES, CPAN, GITHUB, + HACKAGE, + NIXPKGS, UNSUPPORTED; /** @@ -70,6 +72,10 @@ public static RepositoryType resolve(PackageURL packageURL) { return CPAN; } else if (PackageURL.StandardTypes.GITHUB.equals(type)) { return GITHUB; + } else if ("hackage".equals(type)) { + return HACKAGE; + } else if ("nixpkgs".equals(type)) { + return NIXPKGS; } return UNSUPPORTED; } diff --git a/repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzer.java new file mode 100644 index 000000000..db49c495c --- /dev/null +++ b/repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzer.java @@ -0,0 +1,101 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.repometaanalyzer.repositories; + +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.util.EntityUtils; +import org.dependencytrack.persistence.model.Component; +import org.dependencytrack.persistence.model.RepositoryType; +import org.dependencytrack.repometaanalyzer.model.MetaAnalyzerException; +import org.dependencytrack.repometaanalyzer.model.MetaModel; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * An IMetaAnalyzer implementation that supports Hackage. + */ +public class HackageMetaAnalyzer extends AbstractMetaAnalyzer { + private static final Logger LOGGER = LoggerFactory.getLogger(HackageMetaAnalyzer.class); + private static final String DEFAULT_BASE_URL = "https://hackage.haskell.org/"; + + HackageMetaAnalyzer() { + this.baseUrl = DEFAULT_BASE_URL; + } + + /** + * {@inheritDoc} + */ + public RepositoryType supportedRepositoryType() { + return RepositoryType.HACKAGE; + } + + /** + * {@inheritDoc} + */ + public boolean isApplicable(Component component) { + final var purl = component.getPurl(); + return purl != null && "hackage".equals(purl.getType()); + } + + /** + * {@inheritDoc} + */ + public MetaModel analyze(final Component component) { + final var meta = new MetaModel(component); + final var purl = component.getPurl(); + if (purl != null) { + final var url = baseUrl + "/package/" + purl.getName() + "/preferred"; + try (final CloseableHttpResponse response = processHttpRequest(url)) { + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + final var entity = response.getEntity(); + if (entity != null) { + String responseString = EntityUtils.toString(entity); + final var deserialized = new JSONObject(responseString); + final var preferred = deserialized.getJSONArray("normal-version"); + // the latest version is the first in the list + if (preferred != null) { + final var latest = preferred.getString(0); + meta.setLatestVersion(latest); + } + // the hackage API doesn't expose the "published_at" information + // we could use https://flora.pm/experimental/packages/{namespace}/{packageName} + // but it appears this isn't reliable yet + } + } else { + var statusLine = response.getStatusLine(); + handleUnexpectedHttpResponse(LOGGER, url, statusLine.getStatusCode(), statusLine.getReasonPhrase(), component); + } + } catch (IOException ex) { + handleRequestException(LOGGER, ex); + } catch (Exception ex) { + throw new MetaAnalyzerException(ex); + } + } + return meta; + } + + @Override + public String getName() { + return this.getClass().getSimpleName(); + } +} \ No newline at end of file diff --git a/repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzer.java new file mode 100644 index 000000000..2ec148edc --- /dev/null +++ b/repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzer.java @@ -0,0 +1,112 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.repometaanalyzer.repositories; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.dependencytrack.persistence.model.Component; +import org.dependencytrack.persistence.model.RepositoryType; +import org.dependencytrack.repometaanalyzer.model.MetaAnalyzerException; +import org.dependencytrack.repometaanalyzer.model.MetaModel; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class NixpkgsMetaAnalyzer extends AbstractMetaAnalyzer { + private static final Logger LOGGER = LoggerFactory.getLogger(NixpkgsMetaAnalyzer.class); + private static final String DEFAULT_CHANNEL_URL = "https://channels.nixos.org/nixpkgs-unstable/packages.json.br"; + private static final Cache> CACHE = Caffeine.newBuilder() + .expireAfterWrite(60, TimeUnit.MINUTES) + .maximumSize(1) + .build(); + + NixpkgsMetaAnalyzer() { + this.baseUrl = DEFAULT_CHANNEL_URL; + } + + /** + * {@inheritDoc} + */ + public MetaModel analyze(Component component) { + Map latestVersions = CACHE.get("nixpkgs", cacheKey -> { + final var versions = new HashMap(); + try (final CloseableHttpResponse response = processHttpRequest(baseUrl)) { + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + var reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); + var packages = new JSONObject(new JSONTokener(reader)).getJSONObject("packages").toMap().values(); + packages.forEach(pkg -> { + // FUTUREWORK(mangoiv): there are potentially packages with the same pname + if (pkg instanceof HashMap jsonPkg) { + final var pname = jsonPkg.get("pname"); + final var version = jsonPkg.get("version"); + versions.putIfAbsent((String) pname, (String) version); + } + }); + } + } catch (IOException ex) { + LOGGER.debug(ex.toString()); + handleRequestException(LOGGER, ex); + } catch (Exception ex) { + LOGGER.debug(ex.toString()); + throw new MetaAnalyzerException(ex); + } + return versions; + }); + + final var meta = new MetaModel(component); + final var purl = component.getPurl(); + if (purl != null) { + final var newerVersion = latestVersions.get(purl.getName()); + if (newerVersion != null) { + meta.setLatestVersion(newerVersion); + } + } + return meta; + } + + @Override + public String getName() { + return this.getClass().getSimpleName(); + } + + /** + * {@inheritDoc} + */ + public RepositoryType supportedRepositoryType() { + return RepositoryType.NIXPKGS; + } + + /** + * {@inheritDoc} + */ + public boolean isApplicable(Component component) { + final var purl = component.getPurl(); + return purl != null && "nixpkgs".equals(purl.getType()); + } +} diff --git a/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzerTest.java new file mode 100644 index 000000000..e2f5330ca --- /dev/null +++ b/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzerTest.java @@ -0,0 +1,49 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.repometaanalyzer.repositories; + +import com.github.packageurl.PackageURL; +import org.apache.http.impl.client.HttpClients; +import org.dependencytrack.persistence.model.Component; +import org.dependencytrack.persistence.model.RepositoryType; +import org.dependencytrack.repometaanalyzer.model.MetaModel; +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class HackageMetaAnalyzerTest { + + private IMetaAnalyzer analyzer; + + @BeforeEach + void beforeEach() { + analyzer = new HackageMetaAnalyzer(); + analyzer.setHttpClient(HttpClients.createDefault()); + } + + @Test + void testAnalyzer() throws Exception { + Component component = new Component(); + component.setPurl(new PackageURL("pkg:hackage/singletons-th@3.1")); + Assert.assertTrue(analyzer.isApplicable(component)); + Assert.assertEquals(RepositoryType.HACKAGE, analyzer.supportedRepositoryType()); + MetaModel metaModel = analyzer.analyze(component); + Assert.assertNotNull(metaModel.getLatestVersion()); + } +} diff --git a/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java new file mode 100644 index 000000000..0c3ef65fd --- /dev/null +++ b/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java @@ -0,0 +1,101 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.repometaanalyzer.repositories; + +import com.github.packageurl.PackageURL; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.http.Body; +import com.github.tomakehurst.wiremock.http.ContentTypeHeader; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.impl.client.HttpClients; +import org.dependencytrack.persistence.model.Component; +import org.dependencytrack.persistence.model.RepositoryType; +import org.junit.Assert; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; +import static com.github.tomakehurst.wiremock.client.WireMock.get; + +public class NixpkgsMetaAnalyzerTest { + + private IMetaAnalyzer analyzer; + + static WireMockServer wireMockServer; + + @BeforeEach + void beforeEach() { + analyzer = new NixpkgsMetaAnalyzer(); + analyzer.setHttpClient(HttpClients.createDefault()); + wireMockServer = new WireMockServer(1080); + wireMockServer.start(); + } + + @AfterEach + void afterEach() { + wireMockServer.stop(); + wireMockServer.resetAll(); + } + + @Test + public void testAnalyzerWithEmptyPackageResponse() throws Exception { + final var component = new Component(); + component.setPurl(new PackageURL("pkg:nixpkgs/SDL_sound@1.0.3")); + analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", wireMockServer.port())); + wireMockServer.stubFor(get(anyUrl()) + .willReturn(aResponse().withHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .withResponseBody(Body.ofBinaryOrText(""" + { + "packages": {} + } + """.getBytes(), + new ContentTypeHeader("application/json"))).withStatus(HttpStatus.SC_OK))); + Assert.assertTrue(analyzer.isApplicable(component)); + Assert.assertEquals(RepositoryType.NIXPKGS, analyzer.supportedRepositoryType()); + Assert.assertNotNull(analyzer.analyze(component).getComponent()); + } + + @Test + public void testAnalyzerWithPackageResponse() throws Exception { + final var component = new Component(); + component.setPurl(new PackageURL("pkg:nixpkgs/SDL_sound@1.0.3")); + analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", wireMockServer.port())); + wireMockServer.stubFor(get(anyUrl()) + .willReturn(aResponse().withHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .withResponseBody(Body.ofBinaryOrText(""" + { + "packages": { + "p1": { + "pname": "SDL_sound", + "version": "1.0.5" + } + } + } + """.getBytes(), + new ContentTypeHeader("application/json"))).withStatus(HttpStatus.SC_OK))); + Assert.assertTrue(analyzer.isApplicable(component)); + Assert.assertEquals(RepositoryType.NIXPKGS, analyzer.supportedRepositoryType()); + var metaModel = analyzer.analyze(component); + Assert.assertNotNull(metaModel.getComponent()); + Assert.assertEquals("1.0.5", metaModel.getLatestVersion()); + } +} From 06cee8bd5beb15b822792209e9b2de6e5d5981b2 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Fri, 14 Jun 2024 14:32:00 +0100 Subject: [PATCH 2/2] Update NixpkgsMetaAnalyzerTest.java --- .../repositories/NixpkgsMetaAnalyzerTest.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java index 0c3ef65fd..0110e4f37 100644 --- a/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java @@ -56,24 +56,6 @@ void afterEach() { wireMockServer.resetAll(); } - @Test - public void testAnalyzerWithEmptyPackageResponse() throws Exception { - final var component = new Component(); - component.setPurl(new PackageURL("pkg:nixpkgs/SDL_sound@1.0.3")); - analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", wireMockServer.port())); - wireMockServer.stubFor(get(anyUrl()) - .willReturn(aResponse().withHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .withResponseBody(Body.ofBinaryOrText(""" - { - "packages": {} - } - """.getBytes(), - new ContentTypeHeader("application/json"))).withStatus(HttpStatus.SC_OK))); - Assert.assertTrue(analyzer.isApplicable(component)); - Assert.assertEquals(RepositoryType.NIXPKGS, analyzer.supportedRepositoryType()); - Assert.assertNotNull(analyzer.analyze(component).getComponent()); - } - @Test public void testAnalyzerWithPackageResponse() throws Exception { final var component = new Component();