Add python version conditional dependency check (#973)

* Add python version conditional dependency check * add sys_platform condition, and some tests * add comments / string comparison --------- Co-authored-by: Coby Allred <[email protected]> Co-authored-by: Paul Dorsch <[email protected]>
microsoft · Jun 14, 2024 · 22a88b5 · 22a88b5
1 parent e626211
commit 22a88b5
Show file tree

Hide file tree

Showing 7 changed files with 213 additions and 5 deletions.
diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonCommandService.cs
@@ -9,4 +9,6 @@ public interface IPythonCommandService
     Task<bool> PythonExistsAsync(string pythonPath = null);
 
     Task<IList<(string PackageString, GitComponent Component)>> ParseFileAsync(string path, string pythonPath = null);
+
+    Task<string> GetPythonVersionAsync(string pythonPath = null);
 }
diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPythonResolver.cs
@@ -13,4 +13,17 @@ public interface IPythonResolver
     /// <param name="initialPackages">The initial list of packages.</param>
     /// <returns>The root packages, with dependencies associated as children.</returns>
     Task<IList<PipGraphNode>> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList<PipDependencySpecification> initialPackages);
+
+    /// <summary>
+    /// Sets a python environment variable used for conditional dependency checks.
+    /// </summary>
+    /// <param name="key">The key for a variable to be stored.</param>
+    /// <param name="value">the value to be stored for that key.</param>
+    void SetPythonEnvironmentVariable(string key, string value);
+
+    /// <summary>
+    /// Retrieves a the dictionary of python environment variables used for conditional dependency checks.
+    /// </summary>
+    /// <returns> the dictionary of stored python environment variables else null if not stored.</returns>
+    Dictionary<string, string> GetPythonEnvironmentVariables();
 }
diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs
@@ -16,6 +16,11 @@ public class PipDependencySpecification
         @"Requires-Dist:\s*([^\s;\[<>=!~]+)(?:\[[^\]]+\])?(?:\s*\(([^)]+)\))?([^;]*)",
         RegexOptions.Compiled);
 
+    // Extracts name and version from a Requires-Dist string that is found in a metadata file
+    public static readonly Regex RequiresDistConditionalDependenciesMatch = new Regex(
+        @"(?<=.*;.*\s*)(?:and |or )?(?:\S+)\s*(?:<=|>=|<|>|===|==|!=|~=)\s*(?:\S+)",
+        RegexOptions.Compiled);
+
     /// <summary>
     /// These are packages that we don't want to evaluate in our graph as they are generally python builtins.
     /// </summary>
@@ -74,6 +79,16 @@ public PipDependencySpecification(string packageString, bool requiresDist = fals
                     this.DependencySpecifiers = distMatch.Groups[i].Value.Split(',');
                 }
             }
+
+            var conditionalDependenciesMatches = RequiresDistConditionalDependenciesMatch.Matches(packageString);
+
+            for (var i = 0; i < conditionalDependenciesMatches.Count; i++)
+            {
+                if (!string.IsNullOrWhiteSpace(conditionalDependenciesMatches[i].Value))
+                {
+                    this.ConditionalDependencySpecifiers.Add(conditionalDependenciesMatches[i].Value);
+                }
+            }
         }
         else
         {
@@ -110,6 +125,8 @@ public PipDependencySpecification(string packageString, bool requiresDist = fals
     /// </summary>
     public IList<string> DependencySpecifiers { get; set; } = new List<string>();
 
+    public IList<string> ConditionalDependencySpecifiers { get; set; } = new List<string>();
+
     private string DebuggerDisplay => $"{this.Name} ({string.Join(';', this.DependencySpecifiers)})";
 
     /// <summary>
@@ -120,4 +137,62 @@ public bool PackageIsUnsafe()
     {
         return PackagesToIgnore.Contains(this.Name);
     }
+
+    /// <summary>
+    /// Whether or not the package is safe to resolve based on the packagesToIgnore.
+    /// </summary>
+    /// <returns> True if the package is unsafe, otherwise false. </returns>
+    public bool PackageConditionsMet(Dictionary<string, string> pythonEnvironmentVariables)
+    {
+        var conditionalRegex = new Regex(@"(and|or)?\s*(\S+)\s*(<=|>=|<|>|===|==|!=|~=)\s*['""]?([^'""]+)['""]?", RegexOptions.Compiled);
+        var conditionsMet = true;
+        foreach (var conditional in this.ConditionalDependencySpecifiers)
+        {
+            var conditionMet = true;
+            var conditionalMatch = conditionalRegex.Match(conditional);
+            var conditionalJoinOperator = conditionalMatch.Groups[1].Value;
+            var conditionalVar = conditionalMatch.Groups[2].Value;
+            var conditionalOperator = conditionalMatch.Groups[3].Value;
+            var conditionalValue = conditionalMatch.Groups[4].Value;
+            if (!pythonEnvironmentVariables.ContainsKey(conditionalVar))
+            {
+                continue; // If the variable isn't in the environment, we can't evaluate it.
+            }
+
+            if (string.Equals(conditionalVar, "python_version", System.StringComparison.OrdinalIgnoreCase))
+            {
+                var pythonVersion = PythonVersion.Create(conditionalValue);
+                if (pythonVersion.Valid)
+                {
+                    var conditionalSpec = $"{conditionalOperator}{conditionalValue}";
+                    conditionMet = PythonVersionUtilities.VersionValidForSpec(pythonEnvironmentVariables[conditionalVar], new List<string> { conditionalSpec });
+                }
+                else
+                {
+                    conditionMet = pythonEnvironmentVariables[conditionalVar] == conditionalValue;
+                }
+            }
+            else if (string.Equals(conditionalVar, "sys_platform", System.StringComparison.OrdinalIgnoreCase))
+            {
+                // if the platform is not windows or linux (empty string in env var), allow the package to be added. Otherwise, ensure it matches the python condition
+                conditionMet = string.IsNullOrEmpty(pythonEnvironmentVariables[conditionalVar]) || string.Equals(pythonEnvironmentVariables[conditionalVar], conditionalValue, System.StringComparison.OrdinalIgnoreCase);
+            }
+            else
+            {
+                // we don't know how to handle cases besides python_version or sys_platform, so allow the package
+                continue;
+            }
+
+            if (conditionalJoinOperator == "or")
+            {
+                conditionsMet = conditionsMet || conditionMet;
+            }
+            else if (conditionalJoinOperator == "and" || string.IsNullOrEmpty(conditionalJoinOperator))
+            {
+                conditionsMet = conditionsMet && conditionMet;
+            }
+        }
+
+        return conditionsMet;
+    }
 }
diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs
@@ -4,6 +4,7 @@ namespace Microsoft.ComponentDetection.Detectors.Pip;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
@@ -49,6 +50,18 @@ protected override async Task<IObservable<ProcessRequest>> OnPrepareDetectionAsy
 
             return Enumerable.Empty<ProcessRequest>().ToObservable();
         }
+        else
+        {
+            var pythonVersion = await this.pythonCommandService.GetPythonVersionAsync(pythonExePath);
+            this.pythonResolver.SetPythonEnvironmentVariable("python_version", pythonVersion);
+
+            var pythonPlatformString = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+                ? "win32"
+                : RuntimeInformation.IsOSPlatform(OSPlatform.Linux)
+                    ? "linux"
+                    : string.Empty;
+            this.pythonResolver.SetPythonEnvironmentVariable("sys_platform", pythonPlatformString);
+        }
 
         return processRequests;
     }
@@ -67,6 +80,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
                 .Where(x => !string.IsNullOrWhiteSpace(x))
                 .Select(x => new PipDependencySpecification(x))
                 .Where(x => !x.PackageIsUnsafe())
+                .Where(x => x.PackageConditionsMet(this.pythonResolver.GetPythonEnvironmentVariables()))
                 .ToList();
 
             var roots = await this.pythonResolver.ResolveRootsAsync(singleFileComponentRecorder, listedPackage);

diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs
@@ -172,4 +172,13 @@ private async Task<bool> CanCommandBeLocatedAsync(string pythonPath)
     {
         return await this.commandLineInvocationService.CanCommandBeLocatedAsync(pythonPath, new List<string> { "python3", "python2" }, "--version");
     }
+
+    public async Task<string> GetPythonVersionAsync(string pythonPath)
+    {
+        var pythonCommand = await this.ResolvePythonAsync(pythonPath);
+        var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, new List<string> { "python3", "python2" }, "--version");
+        var version = new Regex("Python ([\\d.]+)");
+        var match = version.Match(versionResult.StdOut);
+        return match.Success ? match.Groups[1].Value : null;
+    }
 }
diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs
@@ -14,6 +14,7 @@ public class PythonResolver : PythonResolverBase, IPythonResolver
 {
     private readonly IPyPiClient pypiClient;
     private readonly ILogger<PythonResolver> logger;
+    private readonly Dictionary<string, string> pythonEnvironmentVariables = new Dictionary<string, string>();
 
     private readonly int maxLicenseFieldLength = 100;
     private readonly string classifierFieldSeparator = " :: ";
@@ -84,7 +85,7 @@ private async Task<IList<PipGraphNode>> ProcessQueueAsync(ISingleFileComponentRe
             var (root, currentNode) = state.ProcessingQueue.Dequeue();
 
             // gather all dependencies for the current node
-            var dependencies = (await this.FetchPackageDependenciesAsync(state, currentNode)).Where(x => !x.PackageIsUnsafe());
+            var dependencies = (await this.FetchPackageDependenciesAsync(state, currentNode)).Where(x => !x.PackageIsUnsafe()).Where(x => x.PackageConditionsMet(this.pythonEnvironmentVariables)).ToList();
 
             foreach (var dependencyNode in dependencies)
             {
@@ -230,4 +231,14 @@ private string GetLicenseFromProject(PythonProject project)
 
         return null;
     }
+
+    public void SetPythonEnvironmentVariable(string key, string value)
+    {
+        this.pythonEnvironmentVariables[key] = value;
+    }
+
+    public Dictionary<string, string> GetPythonEnvironmentVariables()
+    {
+        return this.pythonEnvironmentVariables;
+    }
 }