Merge pull request #1282 from blackducksoftware/dev/devm/gradle-issue-fix

Fix IDETECT-4533: Update project name mechanism to avoid infinite loop

Author: devmehtabd

detectable/src/main/java/com/blackduck/integration/detectable/detectables/gradle/inspection/GradleInspectorExtractor.java

@@ -63,6 +63,8 @@ public Extraction extract(File directory, ExecutableTarget gradleExe, @Nullable
             List<File> reportFiles = fileFinder.findFiles(outputDirectory,"*_dependencyGraph.txt");
             List<CodeLocation> codeLocations = new ArrayList<>();
 
+            //Get all the extraction files and sort all the files by depth number in ascending order starting from root to the deepest submodule,
+            // this will help in getting all the rich versions for parents, and then we move on to parse child submodules to see usage of those files
             File[] files = new File[reportFiles.size()];
             reportFiles.toArray(files);
             List<File> reportFilesSorted = Arrays.asList(sortFilesByDepth(files));
@@ -109,6 +111,7 @@ private Optional<NameVersion> parseRootProjectMetadataFile(File rootProjectMetad
         }
     }
 
+    // Sort all the files containing dependency extractions in ascending-order
     private File[] sortFilesByDepth(File[] files) {
         Arrays.sort(files, new Comparator<File>() {
             @Override
@@ -121,7 +124,7 @@ public int compare(File o1, File o2) {
             private int extractDepthNumber(String name) {
                 int i;
                 try {
-                    int s = name.indexOf("depth") + 5;
+                    int s = name.lastIndexOf("depth") + 5; // File name is like project__projectname__depth3_dependencyGraph.txt, we extract the number after depth
                     int e = name.indexOf("_dependencyGraph");
                     String number = name.substring(s, e);
                     i = Integer.parseInt(number);

detectable/src/main/java/com/blackduck/integration/detectable/detectables/gradle/inspection/parse/GradleReportLineParser.java

@@ -1,10 +1,11 @@
 package com.blackduck.integration.detectable.detectables.gradle.inspection.parse;
 
-import java.util.Map;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Arrays;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.HashMap;
+
 
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -36,19 +37,19 @@ public class GradleReportLineParser {
     // The nested map has dependencies with their versions and each
     // module will have its own nested map.
     private final Map<String, Map<String, String>> transitiveRichVersions = new HashMap<>();
-
-    // This map is handling all the child-parent relationships which are found in the entire project
-    // with each child as a key and their respective parent as the value.
-    private final Map<String, String> relationsMap = new HashMap<>();
+    // tracks the project where rich version was declared
     private String richVersionProject = null;
+    // tracks if rich version was declared while parsing the previous line
     private boolean richVersionDeclared = false;
     private String projectName;
     private String rootProjectName;
-    private String projectParent;
+    private String projectPath;
     private int level;
+    private String depthNumber;
     public static final String PROJECT_NAME_PREFIX = "projectName:";
     public static final String ROOT_PROJECT_NAME_PREFIX = "rootProjectName:";
-    public static final String PROJECT_PARENT_PREFIX = "projectParent:";
+    public static final String PROJECT_PATH_PREFIX = "projectPath:";
+    public static final String FILE_NAME_PREFIX = "fileName:";
 
     public GradleTreeNode parseLine(String line, Map<String, String> metadata) {
         level = parseTreeLevel(line);
@@ -129,15 +130,27 @@ private List<String> parseGav(String line, Map<String, String> metadata) {
             }
         }
 
-        projectName = metadata.getOrDefault(PROJECT_NAME_PREFIX, "orphanProject");
-        rootProjectName = metadata.getOrDefault(ROOT_PROJECT_NAME_PREFIX, "");
-        projectParent = metadata.getOrDefault(PROJECT_PARENT_PREFIX, "null");
+        projectName = metadata.getOrDefault(PROJECT_NAME_PREFIX, "orphanProject"); // get project name from metadata
+        rootProjectName = metadata.getOrDefault(ROOT_PROJECT_NAME_PREFIX, "")+"_0"; // get root project name
+        String fileName = metadata.getOrDefault(FILE_NAME_PREFIX, "");
+        projectPath = metadata.getOrDefault(PROJECT_PATH_PREFIX, ""); // get project path Eg: :sub:foo
+
 
-        addRelation();
+        // To avoid a bug caused by an edge case where child and parent modules have the same name causing the loop for checking rich version to stuck
+        // in an infinite state, we are going to suffix the name of the project with the depth number
+        int s = fileName.lastIndexOf("depth") + 5; // File name is like project__projectname__depth3_dependencyGraph.txt, we extract the number after depth
+        int e = fileName.indexOf("_dependencyGraph");
+        depthNumber = fileName.substring(s, e);
+        projectName = projectName+"_"+depthNumber;
+
+        // Example of dependency using rich version:
+        // --- com.graphql-java:graphql-java:{strictly [21.2, 21.3]; prefer 21.3; reject [20.6, 19.5, 18.2]} -> 21.3 direct depenendency, will be stored in rich versions, richVersionProject value will be current project
+        //        +--- com.graphql-java:java-dataloader:3.2.1 transitive needs to be stored
+        //          |    \--- org.slf4j:slf4j-api:1.7.30 -> 2.0.4 transitive needs to be stored
 
         if(gavPieces.size() == 3) {
             String dependencyGroupName = gavPieces.get(0) + ":" + gavPieces.get(1);
-            if(level == 0 && checkRichVersionUse(cleanedOutput)) {
+            if(level == 0 && checkRichVersionUse(cleanedOutput)) { // we only track rich versions if they are declared in direct dependencies
                 storeDirectRichVersion(dependencyGroupName, gavPieces);
             } else {
                 storeOrUpdateRichVersion(dependencyGroupName, gavPieces);
@@ -147,46 +160,51 @@ private List<String> parseGav(String line, Map<String, String> metadata) {
         return gavPieces;
     }
 
+    // store the dependency where rich version was declared and update the global tracking values
     private void storeDirectRichVersion(String dependencyGroupName, List<String> gavPieces) {
         gradleRichVersions.computeIfAbsent(projectName, value -> new HashMap<>()).putIfAbsent(dependencyGroupName, gavPieces.get(2));
         richVersionProject = projectName;
         richVersionDeclared = true;
     }
 
     private void storeOrUpdateRichVersion(String dependencyGroupName, List<String> gavPieces) {
+        // this condition is checking for rich version use for current direct dependency in one of the parent submodule of the current module and updates the current version
         if (checkParentRichVersion(dependencyGroupName)) {
             gavPieces.set(2, gradleRichVersions.get(richVersionProject).get(dependencyGroupName));
         } else if(checkIfTransitiveRichVersion() && transitiveRichVersions.containsKey(richVersionProject) && transitiveRichVersions.get(richVersionProject).containsKey(dependencyGroupName)) {
+            // this is checking if we are parsing a transitive dependency and that transitive
+            // dependency has already been memoized for the use of rich version
             gavPieces.set(2, transitiveRichVersions.get(richVersionProject).get(dependencyGroupName));
         } else if (checkIfTransitiveRichVersion() && richVersionDeclared) {
+            // if while parsing the last direct dependency, we found the use of rich version, we store the version resolved for this transitive dependency
             transitiveRichVersions.computeIfAbsent(richVersionProject, value -> new HashMap<>()).putIfAbsent(dependencyGroupName, gavPieces.get(2));
         } else {
+            // no use of rich versions found
             richVersionDeclared = false;
             richVersionProject = null;
         }
     }
 
-    private void addRelation() {
-        if (!projectParent.equals("null") && !projectParent.contains("root project")) {
-            String parentString = projectParent.substring(projectParent.lastIndexOf(":") + 1, projectParent.lastIndexOf("'"));
-            relationsMap.putIfAbsent(projectName, parentString);
-        } else if (!projectParent.equals("null") && !projectName.equals(rootProjectName)) {
-            relationsMap.putIfAbsent(projectName, rootProjectName);
-        } else {
-            relationsMap.putIfAbsent(rootProjectName, null);
+    private boolean checkParentRichVersion(String dependencyGroupName) {
+        // this method checks all the parent modules for the current submodule upto rootProject for the use of the rich version for the current dependency
+        // if the rich version is used return true and update the richVersionProject
+        // We will check if rich version was declared in root project, if yes immediately apply it, otherwise parse the whole project path for the current submodule
+        // path will start from level 1 Eg: :sub:foo, we will check dependency in :sub_1 first foo_2 next where the name is similar to project name we put in the gradle Rich versions map.
+        //Eg: if sub declares rich version and foo is child of both sub and subtwo, we change version if :sub:foo is the path we are parsing and do not change if we are parsing :subtwo:foo
+
+        if(gradleRichVersions.containsKey(rootProjectName) && gradleRichVersions.get(rootProjectName).containsKey(dependencyGroupName)) {
+            richVersionProject = rootProjectName;
+            return true;
         }
-    }
 
-    private boolean checkParentRichVersion(String dependencyGroupName) {
-        String currentProject = projectName;
-        while (currentProject != null) {
-            if (gradleRichVersions.containsKey(currentProject) && gradleRichVersions.get(currentProject).containsKey(dependencyGroupName)) {
-                richVersionProject = currentProject;
+        String[] pathParts = projectPath.split(":");
+        for(int depth = 1; depth < pathParts.length; depth++) { // Since path is like :sub:foo we start at the first index which will be the parent at first level
+            if(gradleRichVersions.containsKey(pathParts[depth]+"_"+depth) && gradleRichVersions.get(pathParts[depth]+"_"+depth).containsKey(dependencyGroupName)) {
+                richVersionProject = pathParts[depth]+"_"+depth;
                 return true;
             }
-            currentProject = relationsMap.getOrDefault(currentProject, null);
         }
-       return false;
+        return false;
     }
 
     private boolean checkRichVersionUse(String dependencyLine) {

detectable/src/main/java/com/blackduck/integration/detectable/detectables/gradle/inspection/parse/GradleReportParser.java

@@ -26,13 +26,13 @@ public class GradleReportParser {
     public static final String PROJECT_DIRECTORY_PREFIX = "projectDirectory:";
     public static final String PROJECT_GROUP_PREFIX = "projectGroup:";
     public static final String PROJECT_NAME_PREFIX = "projectName:";
-    public static final String PROJECT_PARENT_PREFIX = "projectParent:";
     public static final String PROJECT_VERSION_PREFIX = "projectVersion:";
+    public static final String PROJECT_PATH_PREFIX = "projectPath:";
     public static final String ROOT_PROJECT_NAME_PREFIX = "rootProjectName:";
     public static final String ROOT_PROJECT_VERSION_PREFIX = "rootProjectVersion:";
     public static final String DETECT_META_DATA_HEADER = "DETECT META DATA START";
     public static final String DETECT_META_DATA_FOOTER = "DETECT META DATA END";
-    private final Map<String, String> metadata = new HashMap<>();
+    private final Map<String, String> metadata = new HashMap<>(); // important metadata to pass to line parsers for rich versions
     private final GradleReportConfigurationParser gradleReportConfigurationParser = new GradleReportConfigurationParser();
 
     public Optional<GradleReport> parseReport(File reportFile) {
@@ -44,10 +44,14 @@ public Optional<GradleReport> parseReport(File reportFile) {
 
             List<String> reportLines = reader.lines().collect(Collectors.toList());
 
+            // we parse the last few lines of the extraction file, as we have the metadata information at the end of the file
+            // such as project name, project parent etc. This information is helpful in getting the rich version information by storing parent information
+            // and then parse childs to see if the rich versions declared are used.
             for(int i = reportLines.size()-1; i>=0; i--) {
                 if (reportLines.get(i).startsWith(DETECT_META_DATA_FOOTER)) {
                     processingMetaData = true;
                 } else if (reportLines.get(i).startsWith(DETECT_META_DATA_HEADER)) {
+                    metadata.put("fileName:", reportFile.getName());
                     break;
                 } else if (processingMetaData) {
                     setGradleReportInfo(gradleReport, reportLines.get(i));
@@ -80,17 +84,17 @@ private void setGradleReportInfo(GradleReport gradleReport, String line) {
         } else if (line.startsWith(PROJECT_GROUP_PREFIX)) {
             gradleReport.setProjectGroup(line.substring(PROJECT_GROUP_PREFIX.length()).trim());
         } else if (line.startsWith(PROJECT_NAME_PREFIX)) {
-            String projectName = line.substring(PROJECT_NAME_PREFIX.length()).trim();
+            String projectName = line.substring(PROJECT_NAME_PREFIX.length()).trim(); // get project name
             gradleReport.setProjectName(projectName);
             metadata.put(PROJECT_NAME_PREFIX, projectName);
         } else if (line.startsWith(PROJECT_VERSION_PREFIX)) {
             gradleReport.setProjectVersionName(line.substring(PROJECT_VERSION_PREFIX.length()).trim());
         } else if (line.startsWith(ROOT_PROJECT_NAME_PREFIX)) {
             String rootProjectName = line.substring(ROOT_PROJECT_NAME_PREFIX.length()).trim();
             metadata.put(ROOT_PROJECT_NAME_PREFIX, rootProjectName);
-        } else if (line.startsWith(PROJECT_PARENT_PREFIX)) {
-            String projectParent = line.substring(PROJECT_PARENT_PREFIX.length()).trim();
-            metadata.put(PROJECT_PARENT_PREFIX, projectParent);
+        } else if (line.startsWith(PROJECT_PATH_PREFIX)) {
+            String projectParent = line.substring(PROJECT_PATH_PREFIX.length()).trim(); // get current project's path Eg: :subtwo:foo
+            metadata.put(PROJECT_PATH_PREFIX, projectParent);
         }
     }
 

documentation/src/main/markdown/currentreleasenotes.md

@@ -23,6 +23,7 @@
 ### Resolved issues
 
 * (IDETECT-4447) - ID strings of detected Yarn project dependencies are now correctly formed. Related warning messages have been improved to identify entries in the yarn.lock file that have not been resolved through package.json files and could not be resolved with any standard NPM packages.
+* (IDETECT-4533) - Resolved an issue with [detect_product_short] Gradle Native Inspector causing scans to hang indefinitely when submodule has the same name as the parent module.
 
 ### Dependency updates