Support Cargo Non-Flat Dependency Graph Using cargo tree

detectable/src/main/java/com/blackduck/integration/detectable/detectable/executable/resolver/CargoResolver.java

@@ -0,0 +1,10 @@
+
+package com.blackduck.integration.detectable.detectable.executable.resolver;
+
+import com.blackduck.integration.detectable.DetectableEnvironment;
+import com.blackduck.integration.detectable.ExecutableTarget;
+import com.blackduck.integration.detectable.detectable.exception.DetectableException;
+
+public interface CargoResolver {
+    ExecutableTarget resolveCargo(DetectableEnvironment environment) throws DetectableException;
+}

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoCliDetectable.java

@@ -0,0 +1,70 @@
+package com.blackduck.integration.detectable.detectables.cargo;
+
+import java.io.File;
+import java.io.IOException;
+
+import com.blackduck.integration.bdio.graph.builder.MissingExternalIdException;
+import com.blackduck.integration.common.util.finder.FileFinder;
+import com.blackduck.integration.detectable.Detectable;
+import com.blackduck.integration.detectable.DetectableEnvironment;
+import com.blackduck.integration.detectable.ExecutableTarget;
+import com.blackduck.integration.detectable.detectable.DetectableAccuracyType;
+import com.blackduck.integration.detectable.detectable.Requirements;
+import com.blackduck.integration.detectable.detectable.annotation.DetectableInfo;
+import com.blackduck.integration.detectable.detectable.exception.DetectableException;
+import com.blackduck.integration.detectable.detectable.executable.ExecutableFailedException;
+import com.blackduck.integration.detectable.detectable.result.DetectableResult;
+import com.blackduck.integration.detectable.detectables.cargo.parse.CargoTomlParser;
+import com.blackduck.integration.detectable.extraction.Extraction;
+import com.blackduck.integration.detectable.extraction.ExtractionEnvironment;
+import com.blackduck.integration.detectable.detectable.executable.resolver.CargoResolver;
+import com.blackduck.integration.executable.ExecutableRunner;
+import com.blackduck.integration.executable.ExecutableRunnerException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@DetectableInfo(name = "Cargo CLI", language = "Rust", forge = "crates", accuracy = DetectableAccuracyType.HIGH, requirementsMarkdown = "Command: cargo tree")
+public class CargoCliDetectable extends Detectable {
+    private static final Logger logger = LoggerFactory.getLogger(CargoCliDetectable.class);
+
+    public static final String CARGO_LOCK_FILENAME = "Cargo.lock";
+    public static final String CARGO_TOML_FILENAME = "Cargo.toml";
+
+    private final FileFinder fileFinder;
+
+    private final CargoResolver cargoResolver;
+    private final CargoCliExtractor cargoCliExtractor;
+    private ExecutableTarget cargoExe;
+    private File cargoToml;
+
+    public CargoCliDetectable(DetectableEnvironment environment, FileFinder fileFinder, CargoResolver cargoResolver, CargoCliExtractor cargoCliExtractor) {
+        super(environment);
+        this.fileFinder = fileFinder;
+        this.cargoResolver = cargoResolver;
+        this.cargoCliExtractor = cargoCliExtractor;
+    }
+
+    @Override
+    public DetectableResult applicable() {
+        Requirements requirements = new Requirements(fileFinder, environment);
+        cargoToml = requirements.file(CARGO_TOML_FILENAME);
+        return requirements.result();
+    }
+
+    @Override
+    public DetectableResult extractable() throws DetectableException {
+        Requirements requirements = new Requirements(fileFinder, environment);
+        cargoExe = requirements.executable(() -> cargoResolver.resolveCargo(environment), "cargo");
+        return requirements.result();
+    }
+
+    @Override
+    public Extraction extract(ExtractionEnvironment extractionEnvironment) throws IOException, DetectableException, MissingExternalIdException, ExecutableRunnerException {
+        try {
+            return cargoCliExtractor.extract(environment.getDirectory(), cargoExe);
+        } catch (Exception e) {
+            logger.error("Failed to extract Cargo dependencies.", e);
+            return new Extraction.Builder().failure("Cargo extraction failed due to an exception: " + e.getMessage()).build();
+        }
+    }
+}

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoCliExtractor.java

@@ -0,0 +1,45 @@
+package com.blackduck.integration.detectable.detectables.cargo;
+
+import com.blackduck.integration.bdio.graph.DependencyGraph;
+import com.blackduck.integration.detectable.ExecutableTarget;
+import com.blackduck.integration.detectable.ExecutableUtils;
+import com.blackduck.integration.detectable.detectable.codelocation.CodeLocation;
+import com.blackduck.integration.detectable.detectable.executable.DetectableExecutableRunner;
+import com.blackduck.integration.detectable.detectable.executable.ExecutableFailedException;
+import com.blackduck.integration.detectable.extraction.Extraction;
+import com.blackduck.integration.detectable.util.ToolVersionLogger;
+import com.blackduck.integration.executable.ExecutableOutput;
+import com.google.gson.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+
+public class CargoCliExtractor {
+    private static final Logger logger = LoggerFactory.getLogger(CargoCliExtractor.class.getName());
+    private final DetectableExecutableRunner executableRunner;
+    private final CargoMetadataParser cargoMetadataParser;
+    private final CargoDependencyTransformer cargoDependencyTransformer;
+
+    public CargoCliExtractor(DetectableExecutableRunner executableRunner, CargoMetadataParser cargoMetadataParser, CargoDependencyTransformer cargoDependencyTransformer) {
+        this.executableRunner = executableRunner;
+        this.cargoMetadataParser = cargoMetadataParser;
+        this.cargoDependencyTransformer = cargoDependencyTransformer;
+    }
+
+    public Extraction extract(File directory, ExecutableTarget cargoExe) throws ExecutableFailedException {
+        List<String> commandArguments = Arrays.asList("metadata", "--format-version=1");
+        ExecutableOutput cargoOutput = executableRunner.executeSuccessfully(ExecutableUtils.createFromTarget(directory, cargoExe, commandArguments));
+        String cargoMetadataJson = cargoOutput.getStandardOutput();
+
+        JsonObject jsonObject = cargoMetadataParser.parseMetadata(cargoMetadataJson);
+        DependencyGraph graph = cargoDependencyTransformer.transform(jsonObject);
+
+        CodeLocation codeLocation = new CodeLocation(graph);
+        return new Extraction.Builder()
+            .success(codeLocation)
+            .build();
+    }
+}

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoDependencyTransformer.java

@@ -0,0 +1,92 @@
+package com.blackduck.integration.detectable.detectables.cargo;
+
+import com.blackduck.integration.bdio.graph.BasicDependencyGraph;
+import com.blackduck.integration.bdio.graph.DependencyGraph;
+import com.blackduck.integration.bdio.model.Forge;
+import com.blackduck.integration.bdio.model.dependency.Dependency;
+
+import com.blackduck.integration.bdio.model.dependency.ProjectDependency;
+import com.blackduck.integration.bdio.model.externalid.ExternalId;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+//TODO: Proposed CargoDependency Parser, Need to be implemented to parser
+
+public class CargoDependencyTransformer {
+
+    public DependencyGraph transform(JsonObject jsonObject) {
+        BasicDependencyGraph graph = new BasicDependencyGraph();
+        Map<String, Dependency> packageIdToDependency = new HashMap<>();
+        Set<String> allDependencies = new HashSet<>();
+
+        JsonArray packages = jsonObject.getAsJsonArray("packages");
+        for (JsonElement pkg : packages) {
+            JsonObject packageObj = pkg.getAsJsonObject();
+            String packageId = packageObj.get("id").getAsString();
+            String name = packageObj.get("name").getAsString();
+            String version = packageObj.get("version").getAsString();
+
+            ExternalId externalId = new ExternalId(Forge.CRATES);
+            externalId.setName(name);
+            externalId.setVersion(version);
+
+            Dependency dep = new ProjectDependency(name, version, externalId);
+            packageIdToDependency.put(packageId, dep);
+        }
+
+        for (JsonElement pkg : packages) {
+            JsonObject packageObj = pkg.getAsJsonObject();
+            String currentPackageId = packageObj.get("id").getAsString();
+            Dependency currentDep = packageIdToDependency.get(currentPackageId);
+
+            JsonArray deps = packageObj.getAsJsonArray("dependencies");
+            for (JsonElement dep : deps) {
+                JsonObject depObj = dep.getAsJsonObject();
+                String depName = depObj.get("name").getAsString();
+                String depReq = depObj.get("req").getAsString();
+
+                String depPackageId = findPackageId(packages, depName, depReq);
+                if (depPackageId != null) {
+                    Dependency childDep = packageIdToDependency.get(depPackageId);
+                    graph.addParentWithChild(currentDep, childDep);
+                    allDependencies.add(depPackageId);
+                }
+            }
+        }
+
+        for (JsonElement pkg : packages) {
+            JsonObject packageObj = pkg.getAsJsonObject();
+            String packageId = packageObj.get("id").getAsString();
+            Dependency dependency = packageIdToDependency.get(packageId);
+
+            if (!allDependencies.contains(packageId)) {
+                graph.addDirectDependency(dependency);
+            }
+        }
+
+        return graph;
+    }
+
+    private String findPackageId(JsonArray packages, String name, String req) {
+        for (JsonElement pkg : packages) {
+            JsonObject packageObj = pkg.getAsJsonObject();
+            String packageName = packageObj.get("name").getAsString();
+            String packageVersion = packageObj.get("version").getAsString();
+
+            if (packageName.equals(name) && satisfiesVersionRequirement(packageVersion, req)) {
+                return packageObj.get("id").getAsString();
+            }
+        }
+        return null;
+    }
+
+    private boolean satisfiesVersionRequirement(String version, String req) {
+        return version.startsWith(req.replace("^", "").replace("~", ""));
+    }
+}

detectable/src/main/java/com/blackduck/integration/detectable/detectables/cargo/CargoMetadataParser.java

@@ -0,0 +1,13 @@
+package com.blackduck.integration.detectable.detectables.cargo;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+public class CargoMetadataParser {
+    private final Gson gson = new Gson();
+
+    public JsonObject parseMetadata(String cargoMetadataJson) {
+        return JsonParser.parseString(cargoMetadataJson).getAsJsonObject();
+    }
+}

detectable/src/main/java/com/blackduck/integration/detectable/factory/DetectableFactory.java

@@ -5,6 +5,9 @@
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;
+
+import com.blackduck.integration.detectable.detectable.executable.resolver.*;
+import com.blackduck.integration.detectable.detectables.cargo.*;
 import org.xml.sax.SAXException;
 
 import com.google.gson.Gson;
@@ -14,29 +17,6 @@
 import com.blackduck.integration.common.util.parse.CommandParser;
 import com.blackduck.integration.detectable.DetectableEnvironment;
 import com.blackduck.integration.detectable.detectable.executable.DetectableExecutableRunner;
-import com.blackduck.integration.detectable.detectable.executable.resolver.BashResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.BazelResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.CondaResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.CpanResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.CpanmResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.DartResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.DockerResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.FlutterResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.GitResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.GoResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.GradleResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.JavaResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.LernaResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.MavenResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.NpmResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.PearResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.PipResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.PipenvResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.PythonResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.Rebar3Resolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.SbtResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.SwiftResolver;
-import com.blackduck.integration.detectable.detectable.executable.resolver.OpamResolver;
 import com.blackduck.integration.detectable.detectable.inspector.GradleInspectorResolver;
 import com.blackduck.integration.detectable.detectable.inspector.PipInspectorResolver;
 import com.blackduck.integration.detectable.detectable.inspector.ProjectInspectorResolver;
@@ -61,8 +41,6 @@
 import com.blackduck.integration.detectable.detectables.bitbake.parse.PwdOutputParser;
 import com.blackduck.integration.detectable.detectables.bitbake.transform.BitbakeDependencyGraphTransformer;
 import com.blackduck.integration.detectable.detectables.bitbake.transform.BitbakeGraphTransformer;
-import com.blackduck.integration.detectable.detectables.cargo.CargoExtractor;
-import com.blackduck.integration.detectable.detectables.cargo.CargoLockDetectable;
 import com.blackduck.integration.detectable.detectables.cargo.parse.CargoDependencyLineParser;
 import com.blackduck.integration.detectable.detectables.cargo.parse.CargoTomlParser;
 import com.blackduck.integration.detectable.detectables.cargo.transform.CargoLockPackageDataTransformer;
@@ -369,6 +347,16 @@ public CargoLockDetectable createCargoDetectable(DetectableEnvironment environme
         return new CargoLockDetectable(environment, fileFinder, cargoExtractor);
     }
 
+    public CargoCliDetectable createCargoCliDetectable(DetectableEnvironment environment, CargoResolver cargoResolver) {
+        CargoMetadataParser cargoMetadataParser = new CargoMetadataParser();
+        CargoDependencyTransformer cargoDependencyTransformer = new CargoDependencyTransformer();
+        CargoCliExtractor cargoCliExtractor = new CargoCliExtractor(executableRunner, cargoMetadataParser, cargoDependencyTransformer);
+
+        return new CargoCliDetectable(environment, fileFinder, cargoResolver, cargoCliExtractor);
+    }
+
+
+
     public CarthageLockDetectable createCarthageDetectable(DetectableEnvironment environment) {
         CartfileResolvedParser cartfileResolvedParser = new CartfileResolvedParser();
         CarthageDeclarationTransformer carthageDeclarationTransformer = new CarthageDeclarationTransformer();

src/main/java/com/blackduck/integration/detect/configuration/DetectConfigurationFactory.java

@@ -476,6 +476,7 @@ public DetectExecutableOptions createDetectExecutableOptions() {
             detectConfiguration.getPathOrNull(DetectProperties.DETECT_BAZEL_PATH),
             detectConfiguration.getPathOrNull(DetectProperties.DETECT_CONAN_PATH),
             detectConfiguration.getPathOrNull(DetectProperties.DETECT_CONDA_PATH),
+            detectConfiguration.getPathOrNull(DetectProperties.DETECT_CARGO_PATH),
             detectConfiguration.getPathOrNull(DetectProperties.DETECT_CPAN_PATH),
             detectConfiguration.getPathOrNull(DetectProperties.DETECT_CPANM_PATH),
             detectConfiguration.getPathOrNull(DetectProperties.DETECT_DART_PATH),

src/main/java/com/blackduck/integration/detect/configuration/DetectProperties.java

@@ -580,6 +580,13 @@ private DetectProperties() {
             .setGroups(DetectGroup.CONDA, DetectGroup.GLOBAL)
             .build();
 
+    public static final NullablePathProperty DETECT_CARGO_PATH =
+        NullablePathProperty.newBuilder("detect.cargo.path")
+            .setInfo("Cargo Executable", DetectPropertyFromVersion.VERSION_10_3_0)
+            .setHelp("The path to the cargo executable.")
+            .setGroups(DetectGroup.CARGO, DetectGroup.GLOBAL)
+            .build();
+
     public static final NullablePathProperty DETECT_CPAN_PATH =
         NullablePathProperty.newBuilder("detect.cpan.path")
             .setInfo("cpan Executable", DetectPropertyFromVersion.VERSION_3_0_0)

src/main/java/com/blackduck/integration/detect/tool/detector/DetectorRuleFactory.java

@@ -2,6 +2,7 @@
 
 import com.blackduck.integration.detect.tool.detector.factory.DetectDetectableFactory;
 import com.blackduck.integration.detectable.detectables.bitbake.BitbakeDetectable;
+import com.blackduck.integration.detectable.detectables.cargo.CargoCliDetectable;
 import com.blackduck.integration.detectable.detectables.cargo.CargoLockDetectable;
 import com.blackduck.integration.detectable.detectables.carthage.CarthageLockDetectable;
 import com.blackduck.integration.detectable.detectables.clang.ClangDetectable;
@@ -67,9 +68,11 @@ public DetectorRuleSet createRules(DetectDetectableFactory detectableFactory) {
         DetectorRuleSetBuilder rules = new DetectorRuleSetBuilder(detectableFactory);
 
         rules.addDetector(DetectorType.CARGO, detector -> {
+            detector.entryPoint(CargoCliDetectable.class)
+                .search().defaults();
             detector.entryPoint(CargoLockDetectable.class)
                 .search().defaults();
-        });
+        }).allEntryPointsFallbackToNext();
 
         rules.addDetector(DetectorType.CARTHAGE, detector -> {
             detector.entryPoint(CarthageLockDetectable.class)

src/main/java/com/blackduck/integration/detect/tool/detector/executable/DetectExecutableOptions.java

@@ -27,6 +27,7 @@ public class DetectExecutableOptions {
     private final Path sbtUserPath;
     private final Path lernaUserPath;
     private final Path opamUserPath;
+    private final Path cargoUserPath;
 
     public DetectExecutableOptions(
         Path bashUserPath,
@@ -52,7 +53,8 @@ public DetectExecutableOptions(
         Path swiftUserPath,
         Path sbtUserPath,
         Path lernaUserPath,
-        Path opamUserPath
+        Path opamUserPath,
+        Path cargoUserPath
     ) {
         this.bashUserPath = bashUserPath;
         this.bazelUserPath = bazelUserPath;
@@ -78,6 +80,7 @@ public DetectExecutableOptions(
         this.sbtUserPath = sbtUserPath;
         this.lernaUserPath = lernaUserPath;
         this.opamUserPath = opamUserPath;
+        this.cargoUserPath = cargoUserPath;
     }
 
     public Path getBashUserPath() {
@@ -175,4 +178,6 @@ public Path getSbtUserPath() {
     public Path getOpamUserPath() {
         return opamUserPath;
     }
+
+    public Path getCargoUserPath() { return cargoUserPath; }
 }