From c73df6decab00526b2195b15c8a7a9fd2ed31ae9 Mon Sep 17 00:00:00 2001 From: Viji Date: Mon, 19 Jun 2023 11:32:15 +0530 Subject: [PATCH 1/3] sbom_import debian --- src/LCT.PackageIdentifier/DebianProcessor.cs | 27 ++++++-- src/LCT.PackageIdentifier/NpmProcessor.cs | 69 ++++++++++++++++++++ 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/src/LCT.PackageIdentifier/DebianProcessor.cs b/src/LCT.PackageIdentifier/DebianProcessor.cs index 19e52d59..c41afeaf 100644 --- a/src/LCT.PackageIdentifier/DebianProcessor.cs +++ b/src/LCT.PackageIdentifier/DebianProcessor.cs @@ -33,17 +33,36 @@ public class DebianProcessor : IParser public Bom ParsePackageFile(CommonAppSettings appSettings) { - List configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Debian); + List configFiles = new(); List listofComponents = new List(); Bom bom = new Bom(); List listComponentForBOM; - foreach (string filepath in configFiles) + + if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) { - if (filepath.EndsWith(".xml") || filepath.EndsWith(".json")) + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Debian); + + foreach (string filepath in configFiles) { - listofComponents.AddRange(ParseCycloneDX(filepath)); + if (filepath.EndsWith(".xml") || filepath.EndsWith(".json")) + { + listofComponents.AddRange(ParseCycloneDX(filepath)); + } + } + } + else if (!string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) //todo: need to add a folder in the docker , to mount this + { + + configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Debian); + foreach (string filepath in configFiles) + { + if (filepath.EndsWith(".xml") || filepath.EndsWith(".json")) //todo: decide the end string . is it only .json or _Bom.cdx.json + { + listofComponents.AddRange(ParseCycloneDX(filepath)); + } } } + //todo:testing is pending for the new logic addition int initialCount = listofComponents.Count; GetDistinctComponentList(ref listofComponents); diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index ddd6ae8a..c3d49313 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -89,6 +89,17 @@ public List ParsePackageLockJson(string filepath, CommonAppSettings a GetComponentsForBom(filepath, appSettings, ref bundledComponents, ref lstComponentForBOM, ref noOfDevDependent, depencyComponentList); } + // the below logic for angular 16+version due to package-lock.json file format change + if (dependencies == null) + { + var pacakages = jsonDeserialized["packages"]; + if (pacakages?.Children() != null) + { + IEnumerable depencyComponentList = pacakages?.Children().OfType(); + GetPackagesForBom(filepath, ref bundledComponents, ref lstComponentForBOM, ref noOfDevDependent, depencyComponentList); + } + } + if (appSettings.Npm.ExcludedComponents != null) { lstComponentForBOM = CommonHelper.RemoveExcludedComponents(lstComponentForBOM, appSettings.Npm.ExcludedComponents, ref noOfExcludedComponents); @@ -117,6 +128,64 @@ public List ParsePackageLockJson(string filepath, CommonAppSettings a return lstComponentForBOM; } + private static void GetPackagesForBom(string filepath, ref List bundledComponents, ref List lstComponentForBOM, ref int noOfDevDependent, IEnumerable depencyComponentList) + { + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile += depencyComponentList.Count(); + + foreach (JProperty prop in depencyComponentList) + { + if (string.IsNullOrEmpty(prop.Name)) + { + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile--; + continue; + } + + Component components = new Component(); + var properties = JObject.Parse(Convert.ToString(prop.Value)); + + // ignoring the dev= true components, because they are not needed in clearing + if (IsDevDependency(prop.Value[Dev], ref noOfDevDependent)) + { + continue; + } + + string folderPath = CommonHelper.TrimEndOfString(filepath, $"\\{FileConstant.PackageLockFileName}"); + string packageName = CommonHelper.GetSubstringOfLastOccurance(prop.Name, $"node_modules/"); + string componentName = packageName.StartsWith('@') ? packageName.Replace("@", "%40") : packageName; + + if (packageName.Contains('@')) + { + components.Group = packageName.Split('/')[0]; + components.Name = packageName.Split('/')[1]; + } + else + { + components.Name = packageName; + } + + components.Description = folderPath; + components.Version = Convert.ToString(properties[Version]); + components.Purl = $"{ApiConstant.NPMExternalID}{componentName}@{components.Version}"; + components.BomRef = $"{ApiConstant.NPMExternalID}{componentName}@{components.Version}"; + + CheckAndAddToBundleComponents(bundledComponents, prop, components); + + lstComponentForBOM.Add(components); + lstComponentForBOM = RemoveBundledComponentFromList(bundledComponents, lstComponentForBOM); + } + } + + private static void CheckAndAddToBundleComponents(List bundledComponents, JProperty prop, Component components) + { + if (prop.Value[Bundled] != null && + !(bundledComponents.Any(x => x.Name == components.Name && x.Version.ToLowerInvariant() == components.Version))) + { + BundledComponents component = new() { Name = components.Name, Version = components.Version }; + bundledComponents.Add(component); + } + } + + private void GetComponentsForBom(string filepath, CommonAppSettings appSettings, ref List bundledComponents, ref List lstComponentForBOM, ref int noOfDevDependent, IEnumerable depencyComponentList) From 737be26c8e85aa100f6720251aeee1e81d660252 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Wed, 21 Jun 2023 18:24:15 +0530 Subject: [PATCH 2/3] Multiple Sbom reading changes for NPM,DEBIAN --- src/LCT.Common/Constants/FileConstant.cs | 1 + src/LCT.PackageIdentifier/DebianProcessor.cs | 150 ++++--------------- src/LCT.PackageIdentifier/NpmProcessor.cs | 33 ++-- 3 files changed, 44 insertions(+), 140 deletions(-) diff --git a/src/LCT.Common/Constants/FileConstant.cs b/src/LCT.Common/Constants/FileConstant.cs index 5d4707b0..caffa0ae 100644 --- a/src/LCT.Common/Constants/FileConstant.cs +++ b/src/LCT.Common/Constants/FileConstant.cs @@ -47,5 +47,6 @@ public static class FileConstant public const string DockerImage = "clearingautomationtool"; public static readonly string DockerCMDTool = Path.Combine(@"/bin/bash"); public const string appSettingFileName = "appSettings.json"; + public const string CycloneDXFileExtension = ".cdx.json"; } } diff --git a/src/LCT.PackageIdentifier/DebianProcessor.cs b/src/LCT.PackageIdentifier/DebianProcessor.cs index c41afeaf..9dab146b 100644 --- a/src/LCT.PackageIdentifier/DebianProcessor.cs +++ b/src/LCT.PackageIdentifier/DebianProcessor.cs @@ -25,7 +25,7 @@ namespace LCT.PackageIdentifier /// /// The DebianProcessor class /// - public class DebianProcessor : IParser + public class DebianProcessor : CycloneDXBomParser, IParser { static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); @@ -33,35 +33,19 @@ public class DebianProcessor : IParser public Bom ParsePackageFile(CommonAppSettings appSettings) { - List configFiles = new(); + List configFiles; List listofComponents = new List(); Bom bom = new Bom(); List listComponentForBOM; - if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) - { - configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Debian); + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Debian); - foreach (string filepath in configFiles) - { - if (filepath.EndsWith(".xml") || filepath.EndsWith(".json")) - { - listofComponents.AddRange(ParseCycloneDX(filepath)); - } - } - } - else if (!string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) //todo: need to add a folder in the docker , to mount this + foreach (string filepath in configFiles) { - - configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Debian); - foreach (string filepath in configFiles) - { - if (filepath.EndsWith(".xml") || filepath.EndsWith(".json")) //todo: decide the end string . is it only .json or _Bom.cdx.json - { - listofComponents.AddRange(ParseCycloneDX(filepath)); - } - } + Logger.Debug($"ParsePackageFile():FileName: " + filepath); + listofComponents.AddRange(ParseCycloneDX(filepath)); } + //todo:testing is pending for the new logic addition int initialCount = listofComponents.Count; @@ -113,121 +97,39 @@ public async Task IdentificationOfInternalComponents(Co #region private methods - private static List ParseCycloneDX(string filePath) + public List ParseCycloneDX(string filePath) { List debianPackages = new List(); - try - { - if (filePath.EndsWith(".xml")) - { - XmlDocument doc = new XmlDocument(); - doc.Load(filePath); - XmlNodeList PackageNodes = doc.GetElementsByTagName("components"); - - foreach (XmlNode node in PackageNodes) - { - ExtractDetailsForXML(node.ChildNodes, ref debianPackages); - } - } - else if (filePath.EndsWith(".json")) - { - ExtractDetailsForJson(filePath, ref debianPackages); - } - else - { - // do nothing - } - } - catch (XmlException ex) - { - Logger.Debug($"ParseCycloneDX", ex); - } + ExtractDetailsForJson(filePath, ref debianPackages); return debianPackages; } - private static void ExtractDetailsForXML(XmlNodeList packageNodes, ref List debianPackages) + private void ExtractDetailsForJson(string filePath, ref List debianPackages) { - foreach (XmlNode packageinfo in packageNodes) + Bom bom = ParseCycloneDXBom(filePath); + + foreach (var componentsInfo in bom.Components) { - if (packageinfo.Name == "component") + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++; + DebianPackage package = new DebianPackage { - DebianPackage package = GetPackageDetails(packageinfo); - BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++; - - if (!string.IsNullOrEmpty(package.Name) && !string.IsNullOrEmpty(package.Version) && !string.IsNullOrEmpty(package.PurlID) && package.PurlID.Contains(Dataconstant.DebianPackage)) - { - BomCreator.bomKpiData.DebianComponents++; - debianPackages.Add(package); - Logger.Debug($"ExtractDetailsForXML():ValidComponent:Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); - } - else - { - BomCreator.bomKpiData.ComponentsExcluded++; - Logger.Debug($"ExtractDetailsForXML():InvalidComponent:Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); - } - } - } - } - - private static void ExtractDetailsForJson(string filePath, ref List debianPackages) - { - Model.CycloneDxBomData cycloneDxBomData; - string json = File.ReadAllText(filePath); - cycloneDxBomData = JsonConvert.DeserializeObject(json); - - if (cycloneDxBomData != null && cycloneDxBomData.ComponentsInfo != null) - { - foreach (var componentsInfo in cycloneDxBomData.ComponentsInfo) - { - if (componentsInfo.Type == "library") - { - BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++; - DebianPackage package = new DebianPackage - { - Name = componentsInfo.Name, - Version = componentsInfo.Version, - PurlID = componentsInfo.ReleaseExternalId, - }; - - if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.ReleaseExternalId) && componentsInfo.ReleaseExternalId.Contains(Dataconstant.DebianPackage)) - { - BomCreator.bomKpiData.DebianComponents++; - debianPackages.Add(package); - Logger.Debug($"ExtractDetailsForJson():ValidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); - } - else - { - BomCreator.bomKpiData.ComponentsExcluded++; - Logger.Debug($"ExtractDetailsForJson():InvalidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); - } - } - } - } - else - { - Logger.Debug($"ExtractDetailsForJson():NoComponenstFound!!"); - } - } + Name = componentsInfo.Name, + Version = componentsInfo.Version, + PurlID = componentsInfo.Purl, + }; - private static DebianPackage GetPackageDetails(XmlNode packageinfo) - { - DebianPackage package = new DebianPackage(); - foreach (XmlNode mainNode in packageinfo.ChildNodes) - { - if (mainNode.Name == "name") + if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.Purl) && componentsInfo.Purl.Contains(Dataconstant.DebianPackage)) { - package.Name = mainNode.InnerText; + BomCreator.bomKpiData.DebianComponents++; + debianPackages.Add(package); + Logger.Debug($"ExtractDetailsForJson():ValidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); } - if (mainNode.Name == "version") - { - package.Version = mainNode.InnerText; - } - if (mainNode.Name == "purl") + else { - package.PurlID = mainNode.InnerText; + BomCreator.bomKpiData.ComponentsExcluded++; + Logger.Debug($"ExtractDetailsForJson():InvalidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}"); } } - return package; } private static void GetDistinctComponentList(ref List listofComponents) diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index c3d49313..34d812aa 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -13,6 +13,7 @@ using LCT.PackageIdentifier.Model; using LCT.Services.Interface; using log4net; +using Microsoft.CodeAnalysis.CSharp.Syntax; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System; @@ -295,7 +296,7 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List(); @@ -328,29 +329,29 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List configFiles; - if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) - { - Logger.Debug($"ParsePackageFile():Start"); - - configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm); + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm); + foreach (string filepath in configFiles) + { + Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); + if (filepath.EndsWith(FileConstant.CycloneDXFileExtension)) + { + Logger.Debug($"ParsingInputFileForBOM():Found as CycloneDXFile"); + bom = ParseCycloneDXBom(filepath); + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; + bom = RemoveExcludedComponents(appSettings, bom); - foreach (string filepath in configFiles) + componentsForBOM.AddRange(bom.Components); + } + else { + Logger.Debug($"ParsingInputFileForBOM():Found as Package File"); componentsForBOM.AddRange(ParsePackageLockJson(filepath, appSettings)); } } - else - { - bom = ParseCycloneDXBom(appSettings.CycloneDxBomFilePath); - BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; - bom = RemoveExcludedComponents(appSettings, bom); - - componentsForBOM = bom.Components; - } } - private static bool IsDevDependency( JToken devValue, ref int noOfDevDependent) + private static bool IsDevDependency(JToken devValue, ref int noOfDevDependent) { if (devValue != null) { From 76b00c9c23f547c09802dbcfb6558fc10060da74 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Thu, 22 Jun 2023 12:17:21 +0530 Subject: [PATCH 3/3] Unique Package reading for NPM --- src/LCT.Common/Constants/Dataconstant.cs | 2 ++ src/LCT.PackageIdentifier/NpmProcessor.cs | 30 ++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/LCT.Common/Constants/Dataconstant.cs b/src/LCT.Common/Constants/Dataconstant.cs index ab09556c..7f910600 100644 --- a/src/LCT.Common/Constants/Dataconstant.cs +++ b/src/LCT.Common/Constants/Dataconstant.cs @@ -32,6 +32,8 @@ public static class Dataconstant public const char ForwardSlash = '/'; public const string SourceURLSuffix = "/srcfiles?fileinfo=1"; public const string DebianPackage = "pkg:deb/debian"; + public const string NPMPackage = "pkg:npm"; + public const string NUGETPackage = "pkg:pypi"; public const string MavenPackage = "pkg:maven"; public const string Cdx_ArtifactoryRepoUrl = "internal:siemens:clearing:repo-url"; public const string Cdx_ProjectType = "internal:siemens:clearing:project-type"; diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index 34d812aa..32139ffb 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -46,9 +46,10 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) int totalComponentsIdentified = 0; - ParsingInputFileForBOM(appSettings, ref componentsForBOM, ref bom); + componentsForBOM = GetExcludedComponentsList(componentsForBOM); + totalComponentsIdentified = componentsForBOM.Count; componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList(); @@ -328,6 +329,7 @@ public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cy private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List componentsForBOM, ref Bom bom) { List configFiles; + int count = 0; configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm); @@ -338,7 +340,7 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List aqlResultList, Comp return repoName; } + + private static List GetExcludedComponentsList(List componentsForBOM) + { + List components = new List(); + foreach (Component componentsInfo in componentsForBOM) + { + if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.Purl) && componentsInfo.Purl.Contains(Dataconstant.NPMPackage)) + { + components.Add(componentsInfo); + Logger.Debug($"GetExcludedComponentsList():ValidComponent For NPM : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}"); + } + else + { + BomCreator.bomKpiData.ComponentsExcluded++; + Logger.Debug($"GetExcludedComponentsList():InvalidComponent For NPM : Component Details : {componentsInfo?.Name} @ {componentsInfo?.Version} @ {componentsInfo?.Purl}"); + } + } + return components; + } } }