Skip to content

Commit

Permalink
merge with latest dev SBOM
Browse files Browse the repository at this point in the history
  • Loading branch information
@karthika-g
karthika-g committed Jun 26, 2023
2 parents 52ca996 + 30b2caf commit c4ca719
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 117 deletions.
2 changes: 2 additions & 0 deletions src/LCT.Common/Constants/Dataconstant.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ public static class Dataconstant
public const char ForwardSlash = '/';
public const string SourceURLSuffix = "/srcfiles?fileinfo=1";
public const string DebianPackage = "pkg:deb/debian";
public const string NPMPackage = "pkg:npm";
public const string NUGETPackage = "pkg:pypi";
public const string MavenPackage = "pkg:maven";
public const string Cdx_ArtifactoryRepoUrl = "internal:siemens:clearing:repo-url";
public const string Cdx_ProjectType = "internal:siemens:clearing:project-type";
Expand Down
1 change: 1 addition & 0 deletions src/LCT.Common/Constants/FileConstant.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,5 +47,6 @@ public static class FileConstant
public const string DockerImage = "clearingautomationtool";
public static readonly string DockerCMDTool = Path.Combine(@"/bin/bash");
public const string appSettingFileName = "appSettings.json";
public const string CycloneDXFileExtension = ".cdx.json";
}
}
135 changes: 28 additions & 107 deletions src/LCT.PackageIdentifier/DebianProcessor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,26 +25,29 @@ namespace LCT.PackageIdentifier
/// <summary>
/// The DebianProcessor class
/// </summary>
public class DebianProcessor : IParser
public class DebianProcessor : CycloneDXBomParser, IParser
{
static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

#region public method

public Bom ParsePackageFile(CommonAppSettings appSettings)
{
List<string> configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Debian);
List<string> configFiles;
List<DebianPackage> listofComponents = new List<DebianPackage>();
Bom bom = new Bom();
List<Component> listComponentForBOM;

configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Debian);

foreach (string filepath in configFiles)
{
if (filepath.EndsWith(".xml") || filepath.EndsWith(".json"))
{
listofComponents.AddRange(ParseCycloneDX(filepath));
}
Logger.Debug($"ParsePackageFile():FileName: " + filepath);
listofComponents.AddRange(ParseCycloneDX(filepath));
}



int initialCount = listofComponents.Count;
GetDistinctComponentList(ref listofComponents);
listComponentForBOM = FormComponentReleaseExternalID(listofComponents);
Expand Down Expand Up @@ -94,121 +97,39 @@ public async Task<ComponentIdentification> IdentificationOfInternalComponents(Co

#region private methods

private static List<DebianPackage> ParseCycloneDX(string filePath)
public List<DebianPackage> ParseCycloneDX(string filePath)
{
List<DebianPackage> debianPackages = new List<DebianPackage>();
try
{
if (filePath.EndsWith(".xml"))
{
XmlDocument doc = new XmlDocument();
doc.Load(filePath);
XmlNodeList PackageNodes = doc.GetElementsByTagName("components");

foreach (XmlNode node in PackageNodes)
{
ExtractDetailsForXML(node.ChildNodes, ref debianPackages);
}
}
else if (filePath.EndsWith(".json"))
{
ExtractDetailsForJson(filePath, ref debianPackages);
}
else
{
// do nothing
}
}
catch (XmlException ex)
{
Logger.Debug($"ParseCycloneDX", ex);
}
ExtractDetailsForJson(filePath, ref debianPackages);
return debianPackages;
}

private static void ExtractDetailsForXML(XmlNodeList packageNodes, ref List<DebianPackage> debianPackages)
private void ExtractDetailsForJson(string filePath, ref List<DebianPackage> debianPackages)
{
foreach (XmlNode packageinfo in packageNodes)
Bom bom = ParseCycloneDXBom(filePath);

foreach (var componentsInfo in bom.Components)
{
if (packageinfo.Name == "component")
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++;
DebianPackage package = new DebianPackage
{
DebianPackage package = GetPackageDetails(packageinfo);
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++;

if (!string.IsNullOrEmpty(package.Name) && !string.IsNullOrEmpty(package.Version) && !string.IsNullOrEmpty(package.PurlID) && package.PurlID.Contains(Dataconstant.DebianPackage))
{
BomCreator.bomKpiData.DebianComponents++;
debianPackages.Add(package);
Logger.Debug($"ExtractDetailsForXML():ValidComponent:Component Details : {package.Name} @ {package.Version} @ {package.PurlID}");
}
else
{
BomCreator.bomKpiData.ComponentsExcluded++;
Logger.Debug($"ExtractDetailsForXML():InvalidComponent:Component Details : {package.Name} @ {package.Version} @ {package.PurlID}");
}
}
}
}

private static void ExtractDetailsForJson(string filePath, ref List<DebianPackage> debianPackages)
{
Model.CycloneDxBomData cycloneDxBomData;
string json = File.ReadAllText(filePath);
cycloneDxBomData = JsonConvert.DeserializeObject<CycloneDxBomData>(json);

if (cycloneDxBomData != null && cycloneDxBomData.ComponentsInfo != null)
{
foreach (var componentsInfo in cycloneDxBomData.ComponentsInfo)
{
if (componentsInfo.Type == "library")
{
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++;
DebianPackage package = new DebianPackage
{
Name = componentsInfo.Name,
Version = componentsInfo.Version,
PurlID = componentsInfo.ReleaseExternalId,
};

if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.ReleaseExternalId) && componentsInfo.ReleaseExternalId.Contains(Dataconstant.DebianPackage))
{
BomCreator.bomKpiData.DebianComponents++;
debianPackages.Add(package);
Logger.Debug($"ExtractDetailsForJson():ValidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}");
}
else
{
BomCreator.bomKpiData.ComponentsExcluded++;
Logger.Debug($"ExtractDetailsForJson():InvalidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}");
}
}
}
}
else
{
Logger.Debug($"ExtractDetailsForJson():NoComponenstFound!!");
}
}
Name = componentsInfo.Name,
Version = componentsInfo.Version,
PurlID = componentsInfo.Purl,
};

private static DebianPackage GetPackageDetails(XmlNode packageinfo)
{
DebianPackage package = new DebianPackage();
foreach (XmlNode mainNode in packageinfo.ChildNodes)
{
if (mainNode.Name == "name")
{
package.Name = mainNode.InnerText;
}
if (mainNode.Name == "version")
if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.Purl) && componentsInfo.Purl.Contains(Dataconstant.DebianPackage))
{
package.Version = mainNode.InnerText;
BomCreator.bomKpiData.DebianComponents++;
debianPackages.Add(package);
Logger.Debug($"ExtractDetailsForJson():ValidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}");
}
if (mainNode.Name == "purl")
else
{
package.PurlID = mainNode.InnerText;
BomCreator.bomKpiData.ComponentsExcluded++;
Logger.Debug($"ExtractDetailsForJson():InvalidComponent : Component Details : {package.Name} @ {package.Version} @ {package.PurlID}");
}
}
return package;
}

private static void GetDistinctComponentList(ref List<DebianPackage> listofComponents)
Expand Down
121 changes: 111 additions & 10 deletions src/LCT.PackageIdentifier/NpmProcessor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using LCT.PackageIdentifier.Model;
using LCT.Services.Interface;
using log4net;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
Expand Down Expand Up @@ -45,9 +46,10 @@ public Bom ParsePackageFile(CommonAppSettings appSettings)

int totalComponentsIdentified = 0;


ParsingInputFileForBOM(appSettings, ref componentsForBOM, ref bom);

componentsForBOM = GetExcludedComponentsList(componentsForBOM);

totalComponentsIdentified = componentsForBOM.Count;

componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList();
Expand Down Expand Up @@ -89,6 +91,17 @@ public List<Component> ParsePackageLockJson(string filepath, CommonAppSettings a
GetComponentsForBom(filepath, appSettings, ref bundledComponents, ref lstComponentForBOM, ref noOfDevDependent, depencyComponentList);
}

// the below logic for angular 16+version due to package-lock.json file format change
if (dependencies == null)
{
var pacakages = jsonDeserialized["packages"];
if (pacakages?.Children() != null)
{
IEnumerable<JProperty> depencyComponentList = pacakages?.Children().OfType<JProperty>();
GetPackagesForBom(filepath, ref bundledComponents, ref lstComponentForBOM, ref noOfDevDependent, depencyComponentList);
}
}

if (appSettings.Npm.ExcludedComponents != null)
{
lstComponentForBOM = CommonHelper.RemoveExcludedComponents(lstComponentForBOM, appSettings.Npm.ExcludedComponents, ref noOfExcludedComponents);
Expand Down Expand Up @@ -117,6 +130,64 @@ public List<Component> ParsePackageLockJson(string filepath, CommonAppSettings a
return lstComponentForBOM;
}

private static void GetPackagesForBom(string filepath, ref List<BundledComponents> bundledComponents, ref List<Component> lstComponentForBOM, ref int noOfDevDependent, IEnumerable<JProperty> depencyComponentList)
{
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile += depencyComponentList.Count();

foreach (JProperty prop in depencyComponentList)
{
if (string.IsNullOrEmpty(prop.Name))
{
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile--;
continue;
}

Component components = new Component();
var properties = JObject.Parse(Convert.ToString(prop.Value));

// ignoring the dev= true components, because they are not needed in clearing
if (IsDevDependency(prop.Value[Dev], ref noOfDevDependent))
{
continue;
}

string folderPath = CommonHelper.TrimEndOfString(filepath, $"\\{FileConstant.PackageLockFileName}");
string packageName = CommonHelper.GetSubstringOfLastOccurance(prop.Name, $"node_modules/");
string componentName = packageName.StartsWith('@') ? packageName.Replace("@", "%40") : packageName;

if (packageName.Contains('@'))
{
components.Group = packageName.Split('/')[0];
components.Name = packageName.Split('/')[1];
}
else
{
components.Name = packageName;
}

components.Description = folderPath;
components.Version = Convert.ToString(properties[Version]);
components.Purl = $"{ApiConstant.NPMExternalID}{componentName}@{components.Version}";
components.BomRef = $"{ApiConstant.NPMExternalID}{componentName}@{components.Version}";

CheckAndAddToBundleComponents(bundledComponents, prop, components);

lstComponentForBOM.Add(components);
lstComponentForBOM = RemoveBundledComponentFromList(bundledComponents, lstComponentForBOM);
}
}

private static void CheckAndAddToBundleComponents(List<BundledComponents> bundledComponents, JProperty prop, Component components)
{
if (prop.Value[Bundled] != null &&
!(bundledComponents.Any(x => x.Name == components.Name && x.Version.ToLowerInvariant() == components.Version)))
{
BundledComponents component = new() { Name = components.Name, Version = components.Version };
bundledComponents.Add(component);
}
}


private void GetComponentsForBom(string filepath, CommonAppSettings appSettings,
ref List<BundledComponents> bundledComponents, ref List<Component> lstComponentForBOM,
ref int noOfDevDependent, IEnumerable<JProperty> depencyComponentList)
Expand Down Expand Up @@ -226,7 +297,7 @@ public async Task<List<Component>> GetJfrogRepoDetailsOfAComponent(List<Componen
string repoName = GetArtifactoryRepoName(aqlResultList, component, bomhelper);
Property artifactoryrepo = new() { Name = Dataconstant.Cdx_ArtifactoryRepoUrl, Value = repoName };
Component componentVal = component;

if (componentVal.Properties?.Count == null || componentVal.Properties?.Count <= 0)
{
componentVal.Properties = new List<Property>();
Expand Down Expand Up @@ -258,17 +329,28 @@ public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cy
private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List<Component> componentsForBOM, ref Bom bom)
{
List<string> configFiles;
int count = 0;

if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath))
{
Logger.Debug($"ParsePackageFile():Start");

configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm);
configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm);

foreach (string filepath in configFiles)
{
Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath);
if (filepath.EndsWith(FileConstant.CycloneDXFileExtension))
{
Logger.Debug($"ParsingInputFileForBOM():Found as CycloneDXFile");
bom = ParseCycloneDXBom(filepath);
count += bom.Components.Count;
bom = RemoveExcludedComponents(appSettings, bom);

foreach (string filepath in configFiles)
componentsForBOM.AddRange(bom.Components);
}
else
{
componentsForBOM.AddRange(ParsePackageLockJson(filepath, appSettings));
Logger.Debug($"ParsingInputFileForBOM():Found as Package File");
var lst = ParsePackageLockJson(filepath, appSettings);
count += lst.Count;
componentsForBOM.AddRange(lst);
}
}
else
Expand All @@ -287,7 +369,7 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List<Comp
}
}

private static bool IsDevDependency( JToken devValue, ref int noOfDevDependent)
private static bool IsDevDependency(JToken devValue, ref int noOfDevDependent)
{
if (devValue != null)
{
Expand Down Expand Up @@ -374,5 +456,24 @@ private static string GetArtifactoryRepoName(List<AqlResult> aqlResultList, Comp

return repoName;
}

private static List<Component> GetExcludedComponentsList(List<Component> componentsForBOM)
{
List<Component> components = new List<Component>();
foreach (Component componentsInfo in componentsForBOM)
{
if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.Purl) && componentsInfo.Purl.Contains(Dataconstant.NPMPackage))
{
components.Add(componentsInfo);
Logger.Debug($"GetExcludedComponentsList():ValidComponent For NPM : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}");
}
else
{
BomCreator.bomKpiData.ComponentsExcluded++;
Logger.Debug($"GetExcludedComponentsList():InvalidComponent For NPM : Component Details : {componentsInfo?.Name} @ {componentsInfo?.Version} @ {componentsInfo?.Purl}");
}
}
return components;
}
}
}

0 comments on commit c4ca719

Please sign in to comment.