Skip to content

Commit 33b6b99

Browse files
andrian-sevastyanovandrian-sevastyanov
authored
Option to exclude Poetry dependencies based on group names (#1085)
* Move Poetry test classes to the correct directory * Option to exclude Poetry dev dependencies * New property documentation * More verbose property documentation * Fix casing in documentation * Change exluded groups type from list to set for faster lookups
1 parent 1dbcd80 commit 33b6b99

File tree

17 files changed

+393
-70
lines changed

17 files changed

+393
-70
lines changed

‎detectable/src/main/java/com/synopsys/integration/detectable/detectables/poetry/PoetryDetectable.java

+12-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
package com.synopsys.integration.detectable.detectables.poetry;
22

33
import java.io.File;
4+
import java.util.Set;
45

56
import com.synopsys.integration.common.util.finder.FileFinder;
67
import com.synopsys.integration.detectable.Detectable;
78
import com.synopsys.integration.detectable.DetectableEnvironment;
89
import com.synopsys.integration.detectable.detectable.DetectableAccuracyType;
910
import com.synopsys.integration.detectable.detectable.annotation.DetectableInfo;
1011
import com.synopsys.integration.detectable.detectable.result.DetectableResult;
12+
import com.synopsys.integration.detectable.detectable.result.FileNotFoundDetectableResult;
1113
import com.synopsys.integration.detectable.detectable.result.FilesNotFoundDetectableResult;
1214
import com.synopsys.integration.detectable.detectable.result.PassedDetectableResult;
1315
import com.synopsys.integration.detectable.detectable.result.PoetryLockfileNotFoundDetectableResult;
@@ -25,17 +27,19 @@ public class PoetryDetectable extends Detectable {
2527
private final FileFinder fileFinder;
2628
private final PoetryExtractor poetryExtractor;
2729
private final ToolPoetrySectionParser poetrySectionParser;
30+
private final PoetryOptions poetryOptions;
2831

2932
private File pyprojectToml;
3033
private File poetryLock;
3134
private ToolPoetrySectionResult toolPoetrySectionResult;
3235

33-
public PoetryDetectable(DetectableEnvironment environment, FileFinder fileFinder, PoetryExtractor poetryExtractor, ToolPoetrySectionParser tomlPoetrySectionParser) {
36+
public PoetryDetectable(DetectableEnvironment environment, FileFinder fileFinder, PoetryExtractor poetryExtractor, ToolPoetrySectionParser tomlPoetrySectionParser, PoetryOptions options) {
3437
super(environment);
3538
this.fileFinder = fileFinder;
3639
this.poetryExtractor = poetryExtractor;
3740
this.toolPoetrySectionResult = null;
3841
this.poetrySectionParser = tomlPoetrySectionParser;
42+
this.poetryOptions = options;
3943
}
4044

4145
@Override
@@ -59,11 +63,17 @@ public DetectableResult extractable() {
5963
if (poetryLock == null && pyprojectToml != null) {
6064
return new PoetryLockfileNotFoundDetectableResult(environment.getDirectory().getAbsolutePath());
6165
}
66+
67+
if (!poetryOptions.getExcludedGroups().isEmpty() && pyprojectToml == null) {
68+
return new FileNotFoundDetectableResult(PYPROJECT_TOML_FILE_NAME);
69+
}
70+
6271
return new PassedDetectableResult();
6372
}
6473

6574
@Override
6675
public Extraction extract(ExtractionEnvironment extractionEnvironment) {
67-
return poetryExtractor.extract(poetryLock, toolPoetrySectionResult.getToolPoetrySection().orElse(null));
76+
Set<String> rootPackages = poetrySectionParser.parseRootPackages(pyprojectToml, poetryOptions);
77+
return poetryExtractor.extract(poetryLock, toolPoetrySectionResult.getToolPoetrySection().orElse(null), rootPackages);
6878
}
6979
}

‎detectable/src/main/java/com/synopsys/integration/detectable/detectables/poetry/PoetryExtractor.java

+6-10
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@
22

33
import java.io.File;
44
import java.io.IOException;
5-
import java.nio.charset.Charset;
65
import java.nio.charset.StandardCharsets;
7-
import java.nio.file.Files;
8-
import java.util.List;
96
import java.util.Optional;
7+
import java.util.Set;
108

119
import org.apache.commons.io.FileUtils;
1210
import org.jetbrains.annotations.Nullable;
@@ -28,9 +26,12 @@ public PoetryExtractor(PoetryLockParser poetryLockParser) {
2826
this.poetryLockParser = poetryLockParser;
2927
}
3028

31-
public Extraction extract(File poetryLock, @Nullable TomlTable toolDotPoetrySection) {
29+
public Extraction extract(File poetryLock, @Nullable TomlTable toolDotPoetrySection, Set<String> rootPackages) {
3230
try {
33-
DependencyGraph graph = poetryLockParser.parseLockFile(FileUtils.readFileToString(poetryLock, StandardCharsets.UTF_8));
31+
DependencyGraph graph = poetryLockParser.parseLockFile(
32+
FileUtils.readFileToString(poetryLock, StandardCharsets.UTF_8),
33+
rootPackages
34+
);
3435
CodeLocation codeLocation = new CodeLocation(graph);
3536

3637
Optional<NameVersion> poetryNameVersion = extractNameVersionFromToolDotPoetrySection(toolDotPoetrySection);
@@ -47,11 +48,6 @@ public Extraction extract(File poetryLock, @Nullable TomlTable toolDotPoetrySect
4748
}
4849
}
4950

50-
private String getFileAsString(File cargoLock, Charset encoding) throws IOException {
51-
List<String> goLockAsList = Files.readAllLines(cargoLock.toPath(), encoding);
52-
return String.join(System.lineSeparator(), goLockAsList);
53-
}
54-
5551
private Optional<NameVersion> extractNameVersionFromToolDotPoetrySection(@Nullable TomlTable toolDotPoetry) {
5652
if (toolDotPoetry != null) {
5753
if (toolDotPoetry.get(NAME_KEY) != null && toolDotPoetry.get(VERSION_KEY) != null) {

‎detectable/src/main/java/com/synopsys/integration/detectable/detectables/poetry/PoetryOptions.java

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package com.synopsys.integration.detectable.detectables.poetry;
2+
3+
import java.util.HashSet;
4+
import java.util.List;
5+
import java.util.Set;
6+
7+
public class PoetryOptions {
8+
private final Set<String> excludedGroups;
9+
10+
public PoetryOptions(List<String> excludedGroups) {
11+
this.excludedGroups = new HashSet<>(excludedGroups);
12+
}
13+
14+
public Set<String> getExcludedGroups() {
15+
return excludedGroups;
16+
}
17+
}

‎detectable/src/main/java/com/synopsys/integration/detectable/detectables/poetry/parser/PoetryLockParser.java

+21-8
Original file line numberDiff line numberDiff line change
@@ -26,25 +26,26 @@ public class PoetryLockParser {
2626

2727
private final Map<String, Dependency> packageMap = new HashMap<>();
2828

29-
public DependencyGraph parseLockFile(String lockFile) {
29+
public DependencyGraph parseLockFile(String lockFile, Set<String> rootPackages) {
3030
TomlParseResult result = Toml.parse(lockFile);
3131
if (result.get(PACKAGE_KEY) != null) {
3232
TomlArray lockPackages = result.getArray(PACKAGE_KEY);
33-
return parseDependencies(lockPackages);
33+
return parseDependencies(lockPackages, rootPackages);
3434
}
3535

3636
return new BasicDependencyGraph();
3737
}
3838

39-
private DependencyGraph parseDependencies(TomlArray lockPackages) {
39+
private DependencyGraph parseDependencies(TomlArray lockPackages, Set<String> rootPackages) {
4040
DependencyGraph graph = new BasicDependencyGraph();
4141

42-
Set<String> rootPackages = determineRootPackages(lockPackages);
43-
44-
for (String rootPackage : rootPackages) {
45-
graph.addChildToRoot(packageMap.get(rootPackage));
42+
Set<String> lockFileRootPackages = populatePackageMapAndGetRootPackages(lockPackages);
43+
if (rootPackages == null) {
44+
rootPackages = lockFileRootPackages;
4645
}
4746

47+
populateDirectDependencies(graph, rootPackages);
48+
4849
for (int i = 0; i < lockPackages.size(); i++) {
4950
TomlTable lockPackage = lockPackages.getTable(i);
5051
List<String> dependencies = extractFromDependencyList(lockPackage.getTable(DEPENDENCIES_KEY));
@@ -63,7 +64,19 @@ private DependencyGraph parseDependencies(TomlArray lockPackages) {
6364
return graph;
6465
}
6566

66-
private Set<String> determineRootPackages(TomlArray lockPackages) {
67+
private void populateDirectDependencies(DependencyGraph graph, Set<String> rootPackages) {
68+
for (String rootPackage : rootPackages) {
69+
Dependency dependency = packageMap.get(rootPackage);
70+
if (dependency == null) {
71+
throw new RuntimeException(
72+
"Likely pyproject.toml and poetry.lock mismatch. A root package could not be found in the lockfile: " + rootPackage
73+
);
74+
}
75+
graph.addDirectDependency(dependency);
76+
}
77+
}
78+
79+
private Set<String> populatePackageMapAndGetRootPackages(TomlArray lockPackages) {
6780
Set<String> rootPackages = new HashSet<>();
6881
Set<String> dependencyPackages = new HashSet<>();
6982

‎detectable/src/main/java/com/synopsys/integration/detectable/detectables/poetry/parser/ToolPoetrySectionParser.java

+57
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,28 @@
22

33
import java.io.File;
44
import java.io.IOException;
5+
import java.util.HashSet;
6+
import java.util.Set;
57

68
import org.jetbrains.annotations.Nullable;
79
import org.tomlj.TomlParseResult;
810
import org.tomlj.TomlTable;
911

1012
import com.synopsys.integration.detectable.detectable.util.TomlFileUtils;
13+
import com.synopsys.integration.detectable.detectables.poetry.PoetryOptions;
1114

1215
public class ToolPoetrySectionParser {
1316
public static final String TOOL_POETRY_KEY = "tool.poetry";
17+
public static final String MAIN_DEPENDENCY_GROUP_KEY = "tool.poetry.dependencies";
18+
19+
public static final String LEGACY_DEV_DEPENDENCY_GROUP_KEY = "tool.poetry.dev-dependencies";
20+
21+
public static final String DEPENDENCY_GROUP_KEY_PREFIX = "tool.poetry.group.";
22+
public static final String DEPENDENCY_GROUP_KEY_SUFFIX = ".dependencies";
23+
24+
public static final String DEFAULT_DEV_GROUP_NAME = "dev";
25+
26+
public static final String PYTHON_COMPONENT_NAME = "python";
1427

1528
public ToolPoetrySectionResult parseToolPoetrySection(@Nullable File pyprojectToml) {
1629
if (pyprojectToml != null) {
@@ -26,4 +39,48 @@ public ToolPoetrySectionResult parseToolPoetrySection(@Nullable File pyprojectTo
2639
}
2740
return ToolPoetrySectionResult.NOT_FOUND();
2841
}
42+
43+
public Set<String> parseRootPackages(File pyprojectToml, PoetryOptions options) {
44+
if (options.getExcludedGroups().isEmpty() || pyprojectToml == null) {
45+
return null;
46+
}
47+
48+
Set<String> result = new HashSet<>();
49+
50+
TomlParseResult parseResult;
51+
try {
52+
parseResult = TomlFileUtils.parseFile(pyprojectToml);
53+
} catch (IOException e) {
54+
throw new RuntimeException("Unable to read pyproject.toml file");
55+
}
56+
57+
for (String key : parseResult.dottedKeySet(true)) {
58+
if (!parseResult.isTable(key)) {
59+
continue;
60+
}
61+
62+
TomlTable table = parseResult.getTable(key);
63+
64+
if (key.equals(MAIN_DEPENDENCY_GROUP_KEY)) {
65+
addAllTableKeysToSet(result, table);
66+
} else if (key.equals(LEGACY_DEV_DEPENDENCY_GROUP_KEY)) { // in Poetry 1.0 to 1.2 this was the way of specifying dev dependencies
67+
if (!options.getExcludedGroups().contains(DEFAULT_DEV_GROUP_NAME)) {
68+
addAllTableKeysToSet(result, table);
69+
}
70+
} else if (key.startsWith(DEPENDENCY_GROUP_KEY_PREFIX) && key.endsWith(DEPENDENCY_GROUP_KEY_SUFFIX)) {
71+
String group = key.substring(DEPENDENCY_GROUP_KEY_PREFIX.length(), key.length() - DEPENDENCY_GROUP_KEY_SUFFIX.length());
72+
73+
if (!options.getExcludedGroups().contains(group)) {
74+
addAllTableKeysToSet(result, table);
75+
}
76+
}
77+
}
78+
79+
result.remove(PYTHON_COMPONENT_NAME);
80+
return result;
81+
}
82+
83+
private void addAllTableKeysToSet(Set<String> set, TomlTable table) {
84+
set.addAll(table.dottedKeySet());
85+
}
2986
}

‎detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@
237237
import com.synopsys.integration.detectable.detectables.pnpm.lockfile.process.PnpmLockYamlParser;
238238
import com.synopsys.integration.detectable.detectables.poetry.PoetryDetectable;
239239
import com.synopsys.integration.detectable.detectables.poetry.PoetryExtractor;
240+
import com.synopsys.integration.detectable.detectables.poetry.PoetryOptions;
240241
import com.synopsys.integration.detectable.detectables.poetry.parser.PoetryLockParser;
241242
import com.synopsys.integration.detectable.detectables.poetry.parser.ToolPoetrySectionParser;
242243
import com.synopsys.integration.detectable.detectables.projectinspector.ProjectInspectorExtractor;
@@ -628,8 +629,8 @@ public PodlockDetectable createPodLockDetectable(DetectableEnvironment environme
628629
return new PodlockDetectable(environment, fileFinder, podlockExtractor());
629630
}
630631

631-
public PoetryDetectable createPoetryDetectable(DetectableEnvironment environment) {
632-
return new PoetryDetectable(environment, fileFinder, poetryExtractor(), toolPoetrySectionParser());
632+
public PoetryDetectable createPoetryDetectable(DetectableEnvironment environment, PoetryOptions poetryOptions) {
633+
return new PoetryDetectable(environment, fileFinder, poetryExtractor(), toolPoetrySectionParser(), poetryOptions);
633634
}
634635

635636
public RebarDetectable createRebarDetectable(DetectableEnvironment environment, Rebar3Resolver rebar3Resolver) {

‎detectable/src/test/java/com/synopsys/integration/detectable/detectables/pip/inspector/unit/PoetryFalsePositiveTest.java

-33
This file was deleted.

0 commit comments

Comments
 (0)