Merge pull request #1332 from DependencyTrack/backport-hackage-and-ni…

…xpkgs-meta-analyzers

Port: add hackage and nixpkgs analyzers

.gitignore

@@ -49,4 +49,8 @@ secret.key
 apiserver-data/
 
 # Generated test fixtures
-testdata/boms/generated/*.cdx.json
+testdata/boms/generated/*.cdx.json
+
+# nix
+.direnv
+.pre-commit-config.yaml

commons-persistence/src/main/java/org/dependencytrack/persistence/model/RepositoryType.java

@@ -39,6 +39,8 @@ public enum RepositoryType {
     GO_MODULES,
     CPAN,
     GITHUB,
+    HACKAGE,
+    NIXPKGS,
     UNSUPPORTED;
 
     /**
@@ -70,6 +72,10 @@ public static RepositoryType resolve(PackageURL packageURL) {
             return CPAN;
         } else if (PackageURL.StandardTypes.GITHUB.equals(type)) {
             return GITHUB;
+        } else if ("hackage".equals(type)) {
+            return HACKAGE;
+        } else if ("nixpkgs".equals(type)) {
+            return NIXPKGS;
         }
         return UNSUPPORTED;
     }

repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzer.java

@@ -0,0 +1,101 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.repometaanalyzer.repositories;
+
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.util.EntityUtils;
+import org.dependencytrack.persistence.model.Component;
+import org.dependencytrack.persistence.model.RepositoryType;
+import org.dependencytrack.repometaanalyzer.model.MetaAnalyzerException;
+import org.dependencytrack.repometaanalyzer.model.MetaModel;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * An IMetaAnalyzer implementation that supports Hackage.
+ */
+public class HackageMetaAnalyzer extends AbstractMetaAnalyzer {
+    private static final Logger LOGGER = LoggerFactory.getLogger(HackageMetaAnalyzer.class);
+    private static final String DEFAULT_BASE_URL = "https://hackage.haskell.org/";
+
+    HackageMetaAnalyzer() {
+        this.baseUrl = DEFAULT_BASE_URL;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public RepositoryType supportedRepositoryType() {
+        return RepositoryType.HACKAGE;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public boolean isApplicable(Component component) {
+        final var purl = component.getPurl();
+        return purl != null && "hackage".equals(purl.getType());
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public MetaModel analyze(final Component component) {
+        final var meta = new MetaModel(component);
+        final var purl = component.getPurl();
+        if (purl != null) {
+            final var url = baseUrl + "/package/" + purl.getName() + "/preferred";
+            try (final CloseableHttpResponse response = processHttpRequest(url)) {
+                if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                    final var entity = response.getEntity();
+                    if (entity != null) {
+                        String responseString = EntityUtils.toString(entity);
+                        final var deserialized = new JSONObject(responseString);
+                        final var preferred = deserialized.getJSONArray("normal-version");
+                        // the latest version is the first in the list
+                        if (preferred != null) {
+                            final var latest = preferred.getString(0);
+                            meta.setLatestVersion(latest);
+                        }
+                        // the hackage API doesn't expose the "published_at" information
+                        // we could use https://flora.pm/experimental/packages/{namespace}/{packageName}
+                        // but it appears this isn't reliable yet
+                    }
+                } else {
+                    var statusLine = response.getStatusLine();
+                    handleUnexpectedHttpResponse(LOGGER, url, statusLine.getStatusCode(), statusLine.getReasonPhrase(), component);
+                }
+            } catch (IOException ex) {
+                handleRequestException(LOGGER, ex);
+            } catch (Exception ex) {
+                throw new MetaAnalyzerException(ex);
+            }
+        }
+        return meta;
+    }
+
+    @Override
+    public String getName() {
+        return this.getClass().getSimpleName();
+    }
+}

repository-meta-analyzer/src/main/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzer.java

@@ -0,0 +1,112 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.repometaanalyzer.repositories;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.dependencytrack.persistence.model.Component;
+import org.dependencytrack.persistence.model.RepositoryType;
+import org.dependencytrack.repometaanalyzer.model.MetaAnalyzerException;
+import org.dependencytrack.repometaanalyzer.model.MetaModel;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public class NixpkgsMetaAnalyzer extends AbstractMetaAnalyzer {
+    private static final Logger LOGGER = LoggerFactory.getLogger(NixpkgsMetaAnalyzer.class);
+    private static final String DEFAULT_CHANNEL_URL = "https://channels.nixos.org/nixpkgs-unstable/packages.json.br";
+    private static final Cache<String, Map<String, String>> CACHE = Caffeine.newBuilder()
+            .expireAfterWrite(60, TimeUnit.MINUTES)
+            .maximumSize(1)
+            .build();
+
+    NixpkgsMetaAnalyzer() {
+        this.baseUrl = DEFAULT_CHANNEL_URL;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public MetaModel analyze(Component component) {
+        Map<String, String> latestVersions = CACHE.get("nixpkgs", cacheKey -> {
+            final var versions = new HashMap<String, String>();
+            try (final CloseableHttpResponse response = processHttpRequest(baseUrl)) {
+                if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                    var reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
+                    var packages = new JSONObject(new JSONTokener(reader)).getJSONObject("packages").toMap().values();
+                    packages.forEach(pkg -> {
+                        // FUTUREWORK(mangoiv): there are potentially packages with the same pname
+                        if (pkg instanceof HashMap jsonPkg) {
+                            final var pname = jsonPkg.get("pname");
+                            final var version = jsonPkg.get("version");
+                            versions.putIfAbsent((String) pname, (String) version);
+                        }
+                    });
+                }
+            } catch (IOException ex) {
+                LOGGER.debug(ex.toString());
+                handleRequestException(LOGGER, ex);
+            } catch (Exception ex) {
+                LOGGER.debug(ex.toString());
+                throw new MetaAnalyzerException(ex);
+            }
+            return versions;
+        });
+
+        final var meta = new MetaModel(component);
+        final var purl = component.getPurl();
+        if (purl != null) {
+            final var newerVersion = latestVersions.get(purl.getName());
+            if (newerVersion != null) {
+                meta.setLatestVersion(newerVersion);
+            }
+        }
+        return meta;
+    }
+
+    @Override
+    public String getName() {
+        return this.getClass().getSimpleName();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public RepositoryType supportedRepositoryType() {
+        return RepositoryType.NIXPKGS;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public boolean isApplicable(Component component) {
+        final var purl = component.getPurl();
+        return purl != null && "nixpkgs".equals(purl.getType());
+    }
+}

repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/HackageMetaAnalyzerTest.java

@@ -0,0 +1,49 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.repometaanalyzer.repositories;
+
+import com.github.packageurl.PackageURL;
+import org.apache.http.impl.client.HttpClients;
+import org.dependencytrack.persistence.model.Component;
+import org.dependencytrack.persistence.model.RepositoryType;
+import org.dependencytrack.repometaanalyzer.model.MetaModel;
+import org.junit.Assert;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class HackageMetaAnalyzerTest {
+
+    private IMetaAnalyzer analyzer;
+
+    @BeforeEach
+    void beforeEach() {
+        analyzer = new HackageMetaAnalyzer();
+        analyzer.setHttpClient(HttpClients.createDefault());
+    }
+
+    @Test
+    void testAnalyzer() throws Exception {
+        Component component = new Component();
+        component.setPurl(new PackageURL("pkg:hackage/[email protected]"));
+        Assert.assertTrue(analyzer.isApplicable(component));
+        Assert.assertEquals(RepositoryType.HACKAGE, analyzer.supportedRepositoryType());
+        MetaModel metaModel = analyzer.analyze(component);
+        Assert.assertNotNull(metaModel.getLatestVersion());
+    }
+}

repository-meta-analyzer/src/test/java/org/dependencytrack/repometaanalyzer/repositories/NixpkgsMetaAnalyzerTest.java

@@ -0,0 +1,83 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.repometaanalyzer.repositories;
+
+import com.github.packageurl.PackageURL;
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.http.Body;
+import com.github.tomakehurst.wiremock.http.ContentTypeHeader;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.impl.client.HttpClients;
+import org.dependencytrack.persistence.model.Component;
+import org.dependencytrack.persistence.model.RepositoryType;
+import org.junit.Assert;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+
+public class NixpkgsMetaAnalyzerTest {
+
+    private IMetaAnalyzer analyzer;
+
+    static WireMockServer wireMockServer;
+
+    @BeforeEach
+    void beforeEach() {
+        analyzer = new NixpkgsMetaAnalyzer();
+        analyzer.setHttpClient(HttpClients.createDefault());
+        wireMockServer = new WireMockServer(1080);
+        wireMockServer.start();
+    }
+
+    @AfterEach
+    void afterEach() {
+        wireMockServer.stop();
+        wireMockServer.resetAll();
+    }
+
+    @Test
+    public void testAnalyzerWithPackageResponse() throws Exception {
+        final var component = new Component();
+        component.setPurl(new PackageURL("pkg:nixpkgs/[email protected]"));
+        analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", wireMockServer.port()));
+        wireMockServer.stubFor(get(anyUrl())
+                .willReturn(aResponse().withHeader(HttpHeaders.CONTENT_TYPE, "application/json")
+                        .withResponseBody(Body.ofBinaryOrText("""
+                                {
+                                    "packages": {
+                                        "p1": {
+                                            "pname": "SDL_sound",
+                                            "version": "1.0.5"
+                                        }
+                                    }
+                                }
+                                """.getBytes(),
+                                new ContentTypeHeader("application/json"))).withStatus(HttpStatus.SC_OK)));
+        Assert.assertTrue(analyzer.isApplicable(component));
+        Assert.assertEquals(RepositoryType.NIXPKGS, analyzer.supportedRepositoryType());
+        var metaModel = analyzer.analyze(component);
+        Assert.assertNotNull(metaModel.getComponent());
+        Assert.assertEquals("1.0.5", metaModel.getLatestVersion());
+    }
+}