Merged pnpm6 experiment into pnpm detector (#1145)

* merged pnpm6 into pnpm detector

* bump version and make the factory method private

* name of type

* improved logging

* added telemetry record for pnpm, and other minor updates to methods for conciseness

* standardize the invalid version telemetry object

* removed invalid version file

src/Microsoft.ComponentDetection.Common/Telemetry/Records/InvalidParseVersionTelemetryRecord.cs

@@ -0,0 +1,14 @@
+namespace Microsoft.ComponentDetection.Common.Telemetry.Records;
+
+public class InvalidParseVersionTelemetryRecord : BaseDetectionTelemetryRecord
+{
+    public override string RecordName => "InvalidParseVersion";
+
+    public string DetectorId { get; set; }
+
+    public string FilePath { get; set; }
+
+    public string Version { get; set; }
+
+    public string MaxVersion { get; set; }
+}

src/Microsoft.ComponentDetection.Detectors/pip/PipReportComponentDetector.cs

@@ -96,8 +96,10 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
                     report.Version,
                     MaxReportVersion);
 
-                using var versionRecord = new PipReportVersionTelemetryRecord
+                using var versionRecord = new InvalidParseVersionTelemetryRecord
                 {
+                    DetectorId = this.Id,
+                    FilePath = file.Location,
                     Version = report.Version,
                     MaxVersion = MaxReportVersion.ToString(),
                 };

src/Microsoft.ComponentDetection.Detectors/pnpm/IPnpmDetector.cs

@@ -0,0 +1,16 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using Microsoft.ComponentDetection.Contracts;
+
+/// <summary>
+/// Interface that represents a version of the pnpm detector.
+/// </summary>
+public interface IPnpmDetector
+{
+    /// <summary>
+    /// Parses a yaml file content in pnmp format into the dependecy graph.
+    /// </summary>
+    /// <param name="yamlFileContent">Content of the yaml file that contains the pnpm dependencies.</param>
+    /// <param name="singleFileComponentRecorder">Component recorder to which to write the dependency graph.</param>
+    public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder);
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs

@@ -0,0 +1,58 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ComponentDetection.Contracts;
+
+public class Pnpm5Detector : IPnpmDetector
+{
+    public const string MajorVersion = "5";
+
+    public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder)
+    {
+        var yaml = PnpmParsingUtilities.DeserializePnpmYamlV5File(yamlFileContent);
+
+        foreach (var packageKeyValue in yaml.Packages ?? Enumerable.Empty<KeyValuePair<string, Package>>())
+        {
+            // Ignore file: as these are local packages.
+            if (packageKeyValue.Key.StartsWith("file:"))
+            {
+                continue;
+            }
+
+            var parentDetectedComponent = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5(pnpmPackagePath: packageKeyValue.Key);
+            var isDevDependency = packageKeyValue.Value != null && PnpmParsingUtilities.IsPnpmPackageDevDependency(packageKeyValue.Value);
+            singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: isDevDependency);
+            parentDetectedComponent = singleFileComponentRecorder.GetComponent(parentDetectedComponent.Component.Id);
+
+            if (packageKeyValue.Value.Dependencies != null)
+            {
+                foreach (var dependency in packageKeyValue.Value.Dependencies)
+                {
+                    // Ignore local packages.
+                    if (PnpmParsingUtilities.IsLocalDependency(dependency))
+                    {
+                        continue;
+                    }
+
+                    var childDetectedComponent = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5(
+                        pnpmPackagePath: PnpmParsingUtilities.CreatePnpmPackagePathFromDependencyV5(dependency.Key, dependency.Value));
+
+                    // Older code used the root's dev dependency value. We're leaving this null until we do a second pass to look at each components' top level referrers.
+                    singleFileComponentRecorder.RegisterUsage(childDetectedComponent, parentComponentId: parentDetectedComponent.Component.Id, isDevelopmentDependency: null);
+                }
+            }
+        }
+
+        // PNPM doesn't know at the time of RegisterUsage being called for a dependency whether something is a dev dependency or not, so after building up the graph we look at top level referrers.
+        foreach (var component in singleFileComponentRecorder.GetDetectedComponents())
+        {
+            var graph = singleFileComponentRecorder.DependencyGraph;
+            var explicitReferences = graph.GetExplicitReferencedDependencyIds(component.Key);
+            foreach (var explicitReference in explicitReferences)
+            {
+                singleFileComponentRecorder.RegisterUsage(component.Value, isDevelopmentDependency: graph.IsDevelopmentDependency(explicitReference));
+            }
+        }
+    }
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6ComponentDetector.cs → src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs

@@ -1,82 +1,17 @@
 namespace Microsoft.ComponentDetection.Detectors.Pnpm;
 
-using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
-using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
-using Microsoft.ComponentDetection.Contracts.Internal;
-using Microsoft.ComponentDetection.Contracts.TypedComponent;
-using Microsoft.Extensions.Logging;
 
-public class Pnpm6ComponentDetector : FileComponentDetector, IExperimentalDetector
+public class Pnpm6Detector : IPnpmDetector
 {
-    public Pnpm6ComponentDetector(
-        IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
-        IObservableDirectoryWalkerFactory walkerFactory,
-        ILogger<Pnpm6ComponentDetector> logger)
-    {
-        this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
-        this.Scanner = walkerFactory;
-        this.Logger = logger;
-    }
-
-    public override string Id { get; } = "Pnpm6";
-
-    public override IEnumerable<string> Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.Npm) };
-
-    public override IList<string> SearchPatterns { get; } = new List<string> { "shrinkwrap.yaml", "pnpm-lock.yaml" };
-
-    public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = new[] { ComponentType.Npm };
+    public const string MajorVersion = "6";
 
-    public override int Version { get; } = 1;
-
-    public override bool NeedsAutomaticRootDependencyCalculation => true;
-
-    /// <inheritdoc />
-    protected override IList<string> SkippedFolders => new List<string> { "node_modules", "pnpm-store" };
-
-    protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs)
+    public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder)
     {
-        var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder;
-        var file = processRequest.ComponentStream;
+        var yaml = PnpmParsingUtilities.DeserializePnpmYamlV6File(yamlFileContent);
 
-        var skippedFolder = this.SkippedFolders.FirstOrDefault(folder => file.Location.Contains(folder));
-        if (!string.IsNullOrEmpty(skippedFolder))
-        {
-            this.Logger.LogDebug("Skipping found file, it was detected as being within a {SkippedFolder} folder.", skippedFolder);
-        }
-
-        try
-        {
-            var fileContent = await new StreamReader(file.Stream).ReadToEndAsync();
-            var version = PnpmParsingUtilities.DeserializePnpmYamlFileVersion(fileContent);
-            this.RecordLockfileVersion(version);
-            var majorVersion = version?.Split(".")[0];
-            switch (majorVersion)
-            {
-                case null:
-                case "5":
-                    // Handled in the non-experimental detector. No-op here.
-                    break;
-                case "6":
-                    var pnpmYamlV6 = PnpmParsingUtilities.DeserializePnpmYamlV6File(fileContent);
-                    this.RecordDependencyGraphFromFileV6(pnpmYamlV6, singleFileComponentRecorder);
-                    break;
-                default:
-                    // Handled in the non-experimental detector. No-op here.
-                    break;
-            }
-        }
-        catch (Exception e)
-        {
-            this.Logger.LogError(e, "Failed to read pnpm yaml file {File}", file.Location);
-        }
-    }
-
-    private void RecordDependencyGraphFromFileV6(PnpmYamlV6 yaml, ISingleFileComponentRecorder singleFileComponentRecorder)
-    {
         // There may be multiple instance of the same package (even at the same version) in pnpm differentiated by other aspects of the pnpm dependency path.
         // Therefor all DetectedComponents are tracked by the same full string pnpm uses, the pnpm dependency path, which is used as the key in this dictionary.
         // Some documentation about pnpm dependency paths can be found at https://github.com/pnpm/spec/blob/master/dependency-path.md.