From 22a88b52afcd1890a91f7481cfc325473673d3e6 Mon Sep 17 00:00:00 2001 From: Fernando Rojo Date: Fri, 14 Jun 2024 13:10:26 -0700 Subject: [PATCH] Add python version conditional dependency check (#973) * Add python version conditional dependency check * add sys_platform condition, and some tests * add comments / string comparison --------- Co-authored-by: Coby Allred Co-authored-by: Paul Dorsch --- .../pip/Contracts/IPythonCommandService.cs | 2 + .../pip/Contracts/IPythonResolver.cs | 13 +++ .../Contracts/PipDependencySpecification.cs | 75 +++++++++++++++ .../pip/PipComponentDetector.cs | 14 +++ .../pip/PythonCommandService.cs | 9 ++ .../pip/PythonResolver.cs | 13 ++- .../PipDependencySpecifierTests.cs | 92 ++++++++++++++++++- 7 files changed, 213 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonCommandService.cs index a8486c666..a4d2b8828 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonCommandService.cs @@ -9,4 +9,6 @@ public interface IPythonCommandService Task PythonExistsAsync(string pythonPath = null); Task> ParseFileAsync(string path, string pythonPath = null); + + Task GetPythonVersionAsync(string pythonPath = null); } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonResolver.cs index 86b59d8b3..8a34b4bc8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonResolver.cs @@ -13,4 +13,17 @@ public interface IPythonResolver /// The initial list of packages. /// The root packages, with dependencies associated as children. Task> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList initialPackages); + + /// + /// Sets a python environment variable used for conditional dependency checks. + /// + /// The key for a variable to be stored. + /// the value to be stored for that key. + void SetPythonEnvironmentVariable(string key, string value); + + /// + /// Retrieves a the dictionary of python environment variables used for conditional dependency checks. + /// + /// the dictionary of stored python environment variables else null if not stored. + Dictionary GetPythonEnvironmentVariables(); } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs index 93f35236d..3e5059c03 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs @@ -16,6 +16,11 @@ public class PipDependencySpecification @"Requires-Dist:\s*([^\s;\[<>=!~]+)(?:\[[^\]]+\])?(?:\s*\(([^)]+)\))?([^;]*)", RegexOptions.Compiled); + // Extracts name and version from a Requires-Dist string that is found in a metadata file + public static readonly Regex RequiresDistConditionalDependenciesMatch = new Regex( + @"(?<=.*;.*\s*)(?:and |or )?(?:\S+)\s*(?:<=|>=|<|>|===|==|!=|~=)\s*(?:\S+)", + RegexOptions.Compiled); + /// /// These are packages that we don't want to evaluate in our graph as they are generally python builtins. /// @@ -74,6 +79,16 @@ public PipDependencySpecification(string packageString, bool requiresDist = fals this.DependencySpecifiers = distMatch.Groups[i].Value.Split(','); } } + + var conditionalDependenciesMatches = RequiresDistConditionalDependenciesMatch.Matches(packageString); + + for (var i = 0; i < conditionalDependenciesMatches.Count; i++) + { + if (!string.IsNullOrWhiteSpace(conditionalDependenciesMatches[i].Value)) + { + this.ConditionalDependencySpecifiers.Add(conditionalDependenciesMatches[i].Value); + } + } } else { @@ -110,6 +125,8 @@ public PipDependencySpecification(string packageString, bool requiresDist = fals /// public IList DependencySpecifiers { get; set; } = new List(); + public IList ConditionalDependencySpecifiers { get; set; } = new List(); + private string DebuggerDisplay => $"{this.Name} ({string.Join(';', this.DependencySpecifiers)})"; /// @@ -120,4 +137,62 @@ public bool PackageIsUnsafe() { return PackagesToIgnore.Contains(this.Name); } + + /// + /// Whether or not the package is safe to resolve based on the packagesToIgnore. + /// + /// True if the package is unsafe, otherwise false. + public bool PackageConditionsMet(Dictionary pythonEnvironmentVariables) + { + var conditionalRegex = new Regex(@"(and|or)?\s*(\S+)\s*(<=|>=|<|>|===|==|!=|~=)\s*['""]?([^'""]+)['""]?", RegexOptions.Compiled); + var conditionsMet = true; + foreach (var conditional in this.ConditionalDependencySpecifiers) + { + var conditionMet = true; + var conditionalMatch = conditionalRegex.Match(conditional); + var conditionalJoinOperator = conditionalMatch.Groups[1].Value; + var conditionalVar = conditionalMatch.Groups[2].Value; + var conditionalOperator = conditionalMatch.Groups[3].Value; + var conditionalValue = conditionalMatch.Groups[4].Value; + if (!pythonEnvironmentVariables.ContainsKey(conditionalVar)) + { + continue; // If the variable isn't in the environment, we can't evaluate it. + } + + if (string.Equals(conditionalVar, "python_version", System.StringComparison.OrdinalIgnoreCase)) + { + var pythonVersion = PythonVersion.Create(conditionalValue); + if (pythonVersion.Valid) + { + var conditionalSpec = $"{conditionalOperator}{conditionalValue}"; + conditionMet = PythonVersionUtilities.VersionValidForSpec(pythonEnvironmentVariables[conditionalVar], new List { conditionalSpec }); + } + else + { + conditionMet = pythonEnvironmentVariables[conditionalVar] == conditionalValue; + } + } + else if (string.Equals(conditionalVar, "sys_platform", System.StringComparison.OrdinalIgnoreCase)) + { + // if the platform is not windows or linux (empty string in env var), allow the package to be added. Otherwise, ensure it matches the python condition + conditionMet = string.IsNullOrEmpty(pythonEnvironmentVariables[conditionalVar]) || string.Equals(pythonEnvironmentVariables[conditionalVar], conditionalValue, System.StringComparison.OrdinalIgnoreCase); + } + else + { + // we don't know how to handle cases besides python_version or sys_platform, so allow the package + continue; + } + + if (conditionalJoinOperator == "or") + { + conditionsMet = conditionsMet || conditionMet; + } + else if (conditionalJoinOperator == "and" || string.IsNullOrEmpty(conditionalJoinOperator)) + { + conditionsMet = conditionsMet && conditionMet; + } + } + + return conditionsMet; + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs index 40a0cd34d..b4ba5ebc2 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs @@ -4,6 +4,7 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; using System.Collections.Generic; using System.Linq; using System.Reactive.Linq; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; @@ -49,6 +50,18 @@ protected override async Task> OnPrepareDetectionAsy return Enumerable.Empty().ToObservable(); } + else + { + var pythonVersion = await this.pythonCommandService.GetPythonVersionAsync(pythonExePath); + this.pythonResolver.SetPythonEnvironmentVariable("python_version", pythonVersion); + + var pythonPlatformString = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? "win32" + : RuntimeInformation.IsOSPlatform(OSPlatform.Linux) + ? "linux" + : string.Empty; + this.pythonResolver.SetPythonEnvironmentVariable("sys_platform", pythonPlatformString); + } return processRequests; } @@ -67,6 +80,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID .Where(x => !string.IsNullOrWhiteSpace(x)) .Select(x => new PipDependencySpecification(x)) .Where(x => !x.PackageIsUnsafe()) + .Where(x => x.PackageConditionsMet(this.pythonResolver.GetPythonEnvironmentVariables())) .ToList(); var roots = await this.pythonResolver.ResolveRootsAsync(singleFileComponentRecorder, listedPackage); diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs index a60bacb09..19ba82d1c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs @@ -172,4 +172,13 @@ private async Task CanCommandBeLocatedAsync(string pythonPath) { return await this.commandLineInvocationService.CanCommandBeLocatedAsync(pythonPath, new List { "python3", "python2" }, "--version"); } + + public async Task GetPythonVersionAsync(string pythonPath) + { + var pythonCommand = await this.ResolvePythonAsync(pythonPath); + var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, new List { "python3", "python2" }, "--version"); + var version = new Regex("Python ([\\d.]+)"); + var match = version.Match(versionResult.StdOut); + return match.Success ? match.Groups[1].Value : null; + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs index 5c166b7e5..292f64a86 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs @@ -14,6 +14,7 @@ public class PythonResolver : PythonResolverBase, IPythonResolver { private readonly IPyPiClient pypiClient; private readonly ILogger logger; + private readonly Dictionary pythonEnvironmentVariables = new Dictionary(); private readonly int maxLicenseFieldLength = 100; private readonly string classifierFieldSeparator = " :: "; @@ -84,7 +85,7 @@ private async Task> ProcessQueueAsync(ISingleFileComponentRe var (root, currentNode) = state.ProcessingQueue.Dequeue(); // gather all dependencies for the current node - var dependencies = (await this.FetchPackageDependenciesAsync(state, currentNode)).Where(x => !x.PackageIsUnsafe()); + var dependencies = (await this.FetchPackageDependenciesAsync(state, currentNode)).Where(x => !x.PackageIsUnsafe()).Where(x => x.PackageConditionsMet(this.pythonEnvironmentVariables)).ToList(); foreach (var dependencyNode in dependencies) { @@ -230,4 +231,14 @@ private string GetLicenseFromProject(PythonProject project) return null; } + + public void SetPythonEnvironmentVariable(string key, string value) + { + this.pythonEnvironmentVariables[key] = value; + } + + public Dictionary GetPythonEnvironmentVariables() + { + return this.pythonEnvironmentVariables; + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs index 2e55fbd0c..eb0351aa3 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Detectors.Tests; +namespace Microsoft.ComponentDetection.Detectors.Tests; using System.Collections.Generic; using FluentAssertions; @@ -17,12 +17,47 @@ private static void VerifyPipDependencyParsing( var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); + dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) + { + dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.DependencySpecifiers[i]); + } + + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) + { + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); + } + } + } + private static void VerifyPipConditionalDependencyParsing( + List<(string SpecString, bool ShouldBeIncluded, PipDependencySpecification ReferenceDependencySpecification)> testCases, + Dictionary pythonEnvironmentVariables, + bool requiresDist = false) + { + foreach (var (specString, shouldBeIncluded, referenceDependencySpecification) in testCases) + { + var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); + + dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); + dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) { dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( i, referenceDependencySpecification.DependencySpecifiers[i]); } + + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) + { + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); + } + + dependencySpecifier.PackageConditionsMet(pythonEnvironmentVariables).Should().Be(shouldBeIncluded, string.Join(',', dependencySpecifier.ConditionalDependencySpecifiers)); } } @@ -46,13 +81,62 @@ public void TestPipDependencyRequireDist() var specs = new List<(string, PipDependencySpecification)> { ("Requires-Dist: TestPackage<1.27.0,>=1.19.5", new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { "<1.27.0", ">=1.19.5" } }), - ("Requires-Dist: TestPackage (>=1.0.0) ; sys_platform == \"win32\"", new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=1.0.0" } }), + ("Requires-Dist: TestPackage (>=1.0.0) ; sys_platform == \"win32\"", new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=1.0.0" }, ConditionalDependencySpecifiers = new List { "sys_platform == \"win32\"" } }), ("Requires-Dist: OtherPackage[Optional] (<3,>=1.0.0)", new PipDependencySpecification { Name = "OtherPackage", DependencySpecifiers = new List { "<3", ">=1.0.0" } }), - ("Requires-Dist: TestPackage (>=3.7.4.3) ; python_version < \"3.8\"", new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=3.7.4.3" } }), - ("Requires-Dist: TestPackage ; python_version < \"3.8\"", new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List() }), + ("Requires-Dist: TestPackage (>=3.7.4.3) ; python_version < \"3.8\"", new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=3.7.4.3" }, ConditionalDependencySpecifiers = new List { "python_version < \"3.8\"" } }), + ("Requires-Dist: TestPackage ; python_version < \"3.8\"", new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List(), ConditionalDependencySpecifiers = new List { "python_version < \"3.8\"" } }), ("Requires-Dist: SpacePackage >=1.16.0", new PipDependencySpecification() { Name = "SpacePackage", DependencySpecifiers = new List() { ">=1.16.0" } }), }; VerifyPipDependencyParsing(specs, true); } + + [TestMethod] + public void TestPipDependencyRequireDistConditionalDependenciesMet() + { + var specs = new List<(string, bool, PipDependencySpecification)> + { + ("Requires-Dist: TestPackage (>=1.0.0) ; sys_platform == \"win32\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=1.0.0" }, ConditionalDependencySpecifiers = new List { "sys_platform == \"win32\"" } }), + ("Requires-Dist: TestPackage (>=3.7.4.3) ; python_version < \"3.8\"", false, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=3.7.4.3" }, ConditionalDependencySpecifiers = new List { "python_version < \"3.8\"" } }), + ("Requires-Dist: TestPackage ; python_version == \"3.8\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List(), ConditionalDependencySpecifiers = new List { "python_version == \"3.8\"" } }), + ("Requires-Dist: TestPackage (>=3.0.1) ; python_version < \"3.5\" or sys_platform == \"win32\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=3.0.1" }, ConditionalDependencySpecifiers = new List { "python_version < \"3.5\"", "or sys_platform == \"win32\"" } }), + ("Requires-Dist: TestPackage (>=2.0.1) ; python_version == \"3.8\" and sys_platform == \"win32\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=2.0.1" }, ConditionalDependencySpecifiers = new List { "python_version == \"3.8\"", "and sys_platform == \"win32\"" } }), + ("Requires-Dist: TestPackage (>=2.0.1) ; python_version == \"3.8\" and sys_platform == \"linux\"", false, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=2.0.1" }, ConditionalDependencySpecifiers = new List { "python_version == \"3.8\"", "and sys_platform == \"linux\"" } }), + ("Requires-Dist: TestPackage (>=4.0.1) ; python_version < \"3.6\" and sys_platform == \"win32\"", false, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=4.0.1" }, ConditionalDependencySpecifiers = new List { "python_version < \"3.6\"", "and sys_platform == \"win32\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; python_version > \"3.7\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "python_version > \"3.7\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; python_version >= \"3.7\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "python_version >= \"3.7\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; python_version < \"3.9\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "python_version < \"3.9\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; python_version <= \"3.9\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "python_version <= \"3.9\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; python_version == \"3.8\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "python_version == \"3.8\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; python_version === \"3.8\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "python_version === \"3.8\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; python_version ~= \"3.8\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "python_version ~= \"3.8\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; python_version != \"3.8\"", false, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "python_version != \"3.8\"" } }), + }; + var pythonEnvironmentVariables = new Dictionary + { + { "python_version", "3.8" }, + { "sys_platform", "win32" }, + }; + + VerifyPipConditionalDependencyParsing(specs, pythonEnvironmentVariables, true); + } + + [TestMethod] + public void TestPipDependencyRequireDistConditionalDependenciesMet_Linux() + { + var specs = new List<(string, bool, PipDependencySpecification)> + { + ("Requires-Dist: TestPackage (>=2.0.1) ; python_version == \"3.8\" and sys_platform == \"linux\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=2.0.1" }, ConditionalDependencySpecifiers = new List { "python_version == \"3.8\"", "and sys_platform == \"linux\"" } }), + ("Requires-Dist: TestPackage (>=4.0.1) ; python_version == \"3.6\" and sys_platform == \"win32\"", false, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=4.0.1" }, ConditionalDependencySpecifiers = new List { "python_version == \"3.6\"", "and sys_platform == \"win32\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; sys_platform == \"linux\"", true, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "sys_platform == \"linux\"" } }), + ("Requires-Dist: TestPackage (>=5.0.1) ; sys_platform == \"win32\"", false, new PipDependencySpecification { Name = "TestPackage", DependencySpecifiers = new List { ">=5.0.1" }, ConditionalDependencySpecifiers = new List { "sys_platform == \"win32\"" } }), + }; + var pythonEnvironmentVariables = new Dictionary + { + { "python_version", "3.8" }, + { "sys_platform", "linux" }, + }; + + VerifyPipConditionalDependencyParsing(specs, pythonEnvironmentVariables, true); + } }