diff --git a/src/LCT.Common/Constants/Dataconstant.cs b/src/LCT.Common/Constants/Dataconstant.cs index ab09556c..7f910600 100644 --- a/src/LCT.Common/Constants/Dataconstant.cs +++ b/src/LCT.Common/Constants/Dataconstant.cs @@ -32,6 +32,8 @@ public static class Dataconstant public const char ForwardSlash = '/'; public const string SourceURLSuffix = "/srcfiles?fileinfo=1"; public const string DebianPackage = "pkg:deb/debian"; + public const string NPMPackage = "pkg:npm"; + public const string NUGETPackage = "pkg:pypi"; public const string MavenPackage = "pkg:maven"; public const string Cdx_ArtifactoryRepoUrl = "internal:siemens:clearing:repo-url"; public const string Cdx_ProjectType = "internal:siemens:clearing:project-type"; diff --git a/src/LCT.Common/Constants/FileConstant.cs b/src/LCT.Common/Constants/FileConstant.cs index 5d4707b0..caffa0ae 100644 --- a/src/LCT.Common/Constants/FileConstant.cs +++ b/src/LCT.Common/Constants/FileConstant.cs @@ -47,5 +47,6 @@ public static class FileConstant public const string DockerImage = "clearingautomationtool"; public static readonly string DockerCMDTool = Path.Combine(@"/bin/bash"); public const string appSettingFileName = "appSettings.json"; + public const string CycloneDXFileExtension = ".cdx.json"; } } diff --git a/src/LCT.PackageIdentifier/DebianProcessor.cs b/src/LCT.PackageIdentifier/DebianProcessor.cs index 19e52d59..9dab146b 100644 --- a/src/LCT.PackageIdentifier/DebianProcessor.cs +++ b/src/LCT.PackageIdentifier/DebianProcessor.cs @@ -25,7 +25,7 @@ namespace LCT.PackageIdentifier /// /// The DebianProcessor class /// - public class DebianProcessor : IParser + public class DebianProcessor : CycloneDXBomParser, IParser { static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); @@ -33,18 +33,21 @@ public class DebianProcessor : IParser public Bom ParsePackageFile(CommonAppSettings appSettings) { - List configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Debian); + List configFiles; List listofComponents = new List(); Bom bom = new Bom(); List listComponentForBOM; + + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Debian); + foreach (string filepath in configFiles) { - if (filepath.EndsWith(".xml") || filepath.EndsWith(".json")) - { - listofComponents.AddRange(ParseCycloneDX(filepath)); - } + Logger.Debug($"ParsePackageFile():FileName: " + filepath); + listofComponents.AddRange(ParseCycloneDX(filepath)); } + //todo:testing is pending for the new logic addition + int initialCount = listofComponents.Count; GetDistinctComponentList(ref listofComponents); listComponentForBOM = FormComponentReleaseExternalID(listofComponents); @@ -94,121 +97,39 @@ public async Task IdentificationOfInternalComponents(Co #region private methods - private static List ParseCycloneDX(string filePath) + public List ParseCycloneDX(string filePath) { List debianPackages = new List(); - try - { - if (filePath.EndsWith(".xml")) - { - XmlDocument doc = new XmlDocument(); - doc.Load(filePath); - XmlNodeList PackageNodes = doc.GetElementsByTagName("components"); - - foreach (XmlNode node in PackageNodes) - { - ExtractDetailsForXML(node.ChildNodes, ref debianPackages); - } - } - else if (filePath.EndsWith(".json")) - { - ExtractDetailsForJson(filePath, ref debianPackages); - } - else - { - // do nothing - } - } - catch (XmlException ex) - { - Logger.Debug($"ParseCycloneDX", ex); - } + ExtractDetailsForJson(filePath, ref debianPackages); return debianPackages; } - private static void ExtractDetailsForXML(XmlNodeList packageNodes, ref List debianPackages) + private void ExtractDetailsForJson(string filePath, ref List debianPackages) { - foreach (XmlNode packageinfo in packageNodes) + Bom bom = ParseCycloneDXBom(filePath); + + foreach (var componentsInfo in bom.Components) { - if (packageinfo.Name == "component") + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++; + DebianPackage package = new DebianPackage { - DebianPackage package = GetPackageDetails(packageinfo); - BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++; - - if (!string.IsNullOrEmpty(package.Name) && !string.IsNullOrEmpty(package.Version) && !string.IsNullOrEmpty(package.PurlID) && package.PurlID.Contains(Dataconstant.DebianPackage)) - { - BomCreator.bomKpiData.DebianComponents++; - debianPackages.Add(package); - Logger.Debug($"ExtractDetailsForXML():ValidComponent:Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); - } - else - { - BomCreator.bomKpiData.ComponentsExcluded++; - Logger.Debug($"ExtractDetailsForXML():InvalidComponent:Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); - } - } - } - } - - private static void ExtractDetailsForJson(string filePath, ref List debianPackages) - { - Model.CycloneDxBomData cycloneDxBomData; - string json = File.ReadAllText(filePath); - cycloneDxBomData = JsonConvert.DeserializeObject(json); - - if (cycloneDxBomData != null && cycloneDxBomData.ComponentsInfo != null) - { - foreach (var componentsInfo in cycloneDxBomData.ComponentsInfo) - { - if (componentsInfo.Type == "library") - { - BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++; - DebianPackage package = new DebianPackage - { - Name = componentsInfo.Name, - Version = componentsInfo.Version, - PurlID = componentsInfo.ReleaseExternalId, - }; - - if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.ReleaseExternalId) && componentsInfo.ReleaseExternalId.Contains(Dataconstant.DebianPackage)) - { - BomCreator.bomKpiData.DebianComponents++; - debianPackages.Add(package); - Logger.Debug($"ExtractDetailsForJson():ValidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); - } - else - { - BomCreator.bomKpiData.ComponentsExcluded++; - Logger.Debug($"ExtractDetailsForJson():InvalidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); - } - } - } - } - else - { - Logger.Debug($"ExtractDetailsForJson():NoComponenstFound!!"); - } - } + Name = componentsInfo.Name, + Version = componentsInfo.Version, + PurlID = componentsInfo.Purl, + }; - private static DebianPackage GetPackageDetails(XmlNode packageinfo) - { - DebianPackage package = new DebianPackage(); - foreach (XmlNode mainNode in packageinfo.ChildNodes) - { - if (mainNode.Name == "name") - { - package.Name = mainNode.InnerText; - } - if (mainNode.Name == "version") + if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.Purl) && componentsInfo.Purl.Contains(Dataconstant.DebianPackage)) { - package.Version = mainNode.InnerText; + BomCreator.bomKpiData.DebianComponents++; + debianPackages.Add(package); + Logger.Debug($"ExtractDetailsForJson():ValidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); } - if (mainNode.Name == "purl") + else { - package.PurlID = mainNode.InnerText; + BomCreator.bomKpiData.ComponentsExcluded++; + Logger.Debug($"ExtractDetailsForJson():InvalidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); } } - return package; } private static void GetDistinctComponentList(ref List listofComponents) diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index ddd6ae8a..32139ffb 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -13,6 +13,7 @@ using LCT.PackageIdentifier.Model; using LCT.Services.Interface; using log4net; +using Microsoft.CodeAnalysis.CSharp.Syntax; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System; @@ -45,9 +46,10 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) int totalComponentsIdentified = 0; - ParsingInputFileForBOM(appSettings, ref componentsForBOM, ref bom); + componentsForBOM = GetExcludedComponentsList(componentsForBOM); + totalComponentsIdentified = componentsForBOM.Count; componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList(); @@ -89,6 +91,17 @@ public List ParsePackageLockJson(string filepath, CommonAppSettings a GetComponentsForBom(filepath, appSettings, ref bundledComponents, ref lstComponentForBOM, ref noOfDevDependent, depencyComponentList); } + // the below logic for angular 16+version due to package-lock.json file format change + if (dependencies == null) + { + var pacakages = jsonDeserialized["packages"]; + if (pacakages?.Children() != null) + { + IEnumerable depencyComponentList = pacakages?.Children().OfType(); + GetPackagesForBom(filepath, ref bundledComponents, ref lstComponentForBOM, ref noOfDevDependent, depencyComponentList); + } + } + if (appSettings.Npm.ExcludedComponents != null) { lstComponentForBOM = CommonHelper.RemoveExcludedComponents(lstComponentForBOM, appSettings.Npm.ExcludedComponents, ref noOfExcludedComponents); @@ -117,6 +130,64 @@ public List ParsePackageLockJson(string filepath, CommonAppSettings a return lstComponentForBOM; } + private static void GetPackagesForBom(string filepath, ref List bundledComponents, ref List lstComponentForBOM, ref int noOfDevDependent, IEnumerable depencyComponentList) + { + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile += depencyComponentList.Count(); + + foreach (JProperty prop in depencyComponentList) + { + if (string.IsNullOrEmpty(prop.Name)) + { + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile--; + continue; + } + + Component components = new Component(); + var properties = JObject.Parse(Convert.ToString(prop.Value)); + + // ignoring the dev= true components, because they are not needed in clearing + if (IsDevDependency(prop.Value[Dev], ref noOfDevDependent)) + { + continue; + } + + string folderPath = CommonHelper.TrimEndOfString(filepath, $"\\{FileConstant.PackageLockFileName}"); + string packageName = CommonHelper.GetSubstringOfLastOccurance(prop.Name, $"node_modules/"); + string componentName = packageName.StartsWith('@') ? packageName.Replace("@", "%40") : packageName; + + if (packageName.Contains('@')) + { + components.Group = packageName.Split('/')[0]; + components.Name = packageName.Split('/')[1]; + } + else + { + components.Name = packageName; + } + + components.Description = folderPath; + components.Version = Convert.ToString(properties[Version]); + components.Purl = $"{ApiConstant.NPMExternalID}{componentName}@{components.Version}"; + components.BomRef = $"{ApiConstant.NPMExternalID}{componentName}@{components.Version}"; + + CheckAndAddToBundleComponents(bundledComponents, prop, components); + + lstComponentForBOM.Add(components); + lstComponentForBOM = RemoveBundledComponentFromList(bundledComponents, lstComponentForBOM); + } + } + + private static void CheckAndAddToBundleComponents(List bundledComponents, JProperty prop, Component components) + { + if (prop.Value[Bundled] != null && + !(bundledComponents.Any(x => x.Name == components.Name && x.Version.ToLowerInvariant() == components.Version))) + { + BundledComponents component = new() { Name = components.Name, Version = components.Version }; + bundledComponents.Add(component); + } + } + + private void GetComponentsForBom(string filepath, CommonAppSettings appSettings, ref List bundledComponents, ref List lstComponentForBOM, ref int noOfDevDependent, IEnumerable depencyComponentList) @@ -226,7 +297,7 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List(); @@ -258,30 +329,34 @@ public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cy private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List componentsForBOM, ref Bom bom) { List configFiles; + int count = 0; - if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) - { - Logger.Debug($"ParsePackageFile():Start"); - - configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm); + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm); + foreach (string filepath in configFiles) + { + Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); + if (filepath.EndsWith(FileConstant.CycloneDXFileExtension)) + { + Logger.Debug($"ParsingInputFileForBOM():Found as CycloneDXFile"); + bom = ParseCycloneDXBom(filepath); + count += bom.Components.Count; + bom = RemoveExcludedComponents(appSettings, bom); - foreach (string filepath in configFiles) + componentsForBOM.AddRange(bom.Components); + } + else { - componentsForBOM.AddRange(ParsePackageLockJson(filepath, appSettings)); + Logger.Debug($"ParsingInputFileForBOM():Found as Package File"); + var lst = ParsePackageLockJson(filepath, appSettings); + count += lst.Count; + componentsForBOM.AddRange(lst); } } - else - { - bom = ParseCycloneDXBom(appSettings.CycloneDxBomFilePath); - BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; - bom = RemoveExcludedComponents(appSettings, bom); - - componentsForBOM = bom.Components; - } + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = count; } - private static bool IsDevDependency( JToken devValue, ref int noOfDevDependent) + private static bool IsDevDependency(JToken devValue, ref int noOfDevDependent) { if (devValue != null) { @@ -368,5 +443,24 @@ private static string GetArtifactoryRepoName(List aqlResultList, Comp return repoName; } + + private static List GetExcludedComponentsList(List componentsForBOM) + { + List components = new List(); + foreach (Component componentsInfo in componentsForBOM) + { + if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.Purl) && componentsInfo.Purl.Contains(Dataconstant.NPMPackage)) + { + components.Add(componentsInfo); + Logger.Debug($"GetExcludedComponentsList():ValidComponent For NPM : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}"); + } + else + { + BomCreator.bomKpiData.ComponentsExcluded++; + Logger.Debug($"GetExcludedComponentsList():InvalidComponent For NPM : Component Details : {componentsInfo?.Name} @ {componentsInfo?.Version} @ {componentsInfo?.Purl}"); + } + } + return components; + } } }