Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SBOM multiple customer file import #50

Merged
merged 9 commits into from
Jun 26, 2023
Merged

SBOM multiple customer file import #50

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
1c916e7
Import multiple SBOM files from customer maven ,nuget,npm
karthika-g Jun 14, 2023
1f1caff
Merge remote-tracking branch 'remotes/origin/development_SBOM_14JUN' …
karthika-g Jun 15, 2023
e53ee9b
SBOM creation
karthika-g Jun 15, 2023
4e92704
SBOM parsing For nuget and Maven
karthika-g Jun 16, 2023
f3c6ecc
identifier type added
karthika-g Jun 19, 2023
c158df6
New pattern added for input filetype
karthika-g Jun 19, 2023
7a7bc43
Review comment implementation
karthika-g Jun 21, 2023
52ca996
Build failure fix
karthika-g Jun 21, 2023
c4ca719
merge with latest dev SBOM
karthika-g Jun 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/LCT.Common/CycloneDXBomParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@

using CycloneDX.Json;
using CycloneDX.Models;
using LCT.Common.Model;
using log4net;
using log4net.Core;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;

Expand All @@ -18,7 +20,7 @@ namespace LCT.Common
public class CycloneDXBomParser : ICycloneDXBomParser
{
static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
public Bom ParseCycloneDXBom(string filePath)
public List<Component> ParseCycloneDXBom(string filePath)
{
Bom bom = new Bom();
string json = string.Empty;
Expand All @@ -45,7 +47,7 @@ public Bom ParseCycloneDXBom(string filePath)
{
Logger.Error("Exception in reading cycloneDx bom", ex);
}
return bom;
return bom.Components;
karthika-g marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
3 changes: 2 additions & 1 deletion src/LCT.Common/Interface/ICycloneDXBomParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
// --------------------------------------------------------------------------------------------------------------------

using CycloneDX.Models;
using System.Collections.Generic;

namespace LCT.Common
{
Expand All @@ -13,6 +14,6 @@ namespace LCT.Common
/// </summary>
public interface ICycloneDXBomParser
{
public Bom ParseCycloneDXBom(string filePath);
public List<Component> ParseCycloneDXBom(string filePath);
}
}
4 changes: 2 additions & 2 deletions src/LCT.Common/appSettings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
"ExcludedComponents": []
},
"Nuget": {
"Include": [ "pack*.config", "p*.lock.json" ],
"Include": [ "pack*.config", "p*.lock.json","*.cdx.json" ],
"Exclude": [],
"JfrogNugetRepoList": [
"<Nuget Remote Cache Repo Name>", //This is a mirror repo for nuget.org in JFrog
Expand All @@ -51,7 +51,7 @@
"ExcludedComponents": []
},
"Maven": {
"Include": [ "pom.xml" ],
"Include": [ "pom.xml","*.cdx.json" ],
"Exclude": [],
"JfrogMavenRepoList": [
"<Maven Remote Cache Repo Name>", //This is a mirror repo for repo.maven in JFrog
Expand Down
12 changes: 6 additions & 6 deletions src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,10 @@ public void ParseCycloneDXBom_GivenBOMFilePath_ReturnsBOM()

//Act
CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser();
Bom files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile);
List<Component> files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile);

//Assert
Assert.That(4, Is.EqualTo(files.Components.Count), "Returns components in BOM");
Assert.That(4, Is.EqualTo(files.Count), "Returns components in BOM");

}

Expand All @@ -145,10 +145,10 @@ public void ParseCycloneDXBom_GivenInvlidBOMFilePath_ReturnsZeroComponents()

//Act
CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser();
Bom files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile);
List<Component> files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile);

//Assert
Assert.IsNull(files.Components, "Returns Zero components in BOM");
Assert.IsNull(files, "Returns Zero components in BOM");

}

Expand All @@ -161,10 +161,10 @@ public void ParseCycloneDXBom_GivenInCorrectJsonFile_ReturnsZeroComponents()

//Act
CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser();
Bom files = cycloneBomProcessor.ParseCycloneDXBom(sourcePath + "/output.json");
List<Component> files = cycloneBomProcessor.ParseCycloneDXBom(sourcePath + "/output.json");

//Assert
Assert.IsNull(files.Components, "Returns Zero components in BOM");
Assert.IsNull(files, "Returns Zero components in BOM");

}
}
Expand Down
2 changes: 1 addition & 1 deletion src/LCT.PackageIdentifier.UTest/MavenParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public void IsDevDependent_GivenListOfMavenDevComponents_ReturnsNonDevComponents
Bom bom = MavenProcessor.ParsePackageFile(appSettings);

//Assert
Assert.That(bom.Components.Count, Is.EqualTo(1), "Returns the count of NON Dev Dependency components");
Assert.That(bom.Components.Count-BomCreator.bomKpiData.DevDependentComponents, Is.EqualTo(1), "Returns the count of NON Dev Dependency components");
}

[Test]
Expand Down
2 changes: 1 addition & 1 deletion src/LCT.PackageIdentifier.UTest/NugetParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public void ParsePackageConfig_GivenAInputFilePath_ReturnsSuccess()
public void ParsePackageLockJson_GivenAInputFilePath_ReturnsSuccess()
{
//Arrange
int expectednoofcomponents = 152;
int expectednoofcomponents = 153;
string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location;
string outFolder = Path.GetDirectoryName(exePath);
string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles\packages.lock.json";
Expand Down
13 changes: 4 additions & 9 deletions src/LCT.PackageIdentifier/BomCreator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,14 +80,9 @@ public async Task GenerateBom(CommonAppSettings appSettings, IBomHelper bomHelpe

private static void WritecontentsToBOM(CommonAppSettings appSettings, BomKpiData bomKpiData, Bom listOfComponentsToBom)
{
if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath))
{

WriteContentToComparisonBOM(appSettings, listOfComponentsToBom, ref bomKpiData);
}
else
{
WriteContentToCycloneDxBOM(appSettings, listOfComponentsToBom, ref bomKpiData);
}


}

Expand All @@ -96,13 +91,13 @@ private static void WriteContentToCycloneDxBOM(CommonAppSettings appSettings, Bo
IFileOperations fileOperations = new FileOperations();
if (string.IsNullOrEmpty(appSettings.IdentifierBomFilePath))
{
fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.CycloneDxBomFilePath);
fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.SW360ProjectName);
}
else
{
listOfComponentsToBom = fileOperations.CombineComponentsFromExistingBOM(listOfComponentsToBom, appSettings.IdentifierBomFilePath);
bomKpiData.ComponentsInComparisonBOM = listOfComponentsToBom.Components.Count;
fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.CycloneDxBomFilePath);
fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.SW360ProjectName);
}

}
Expand Down
39 changes: 29 additions & 10 deletions src/LCT.PackageIdentifier/MavenProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings)
}
}

ParseConfigFile(depFilePath, appSettings, ref componentsForBOM);
ParseDependencyTextFile(depFilePath, appSettings, ref componentsForBOM);

totalComponentsIdentified = componentsForBOM.Count;

Expand All @@ -76,14 +76,28 @@ public Bom ParsePackageFile(CommonAppSettings appSettings)
}
else
{
bom = ParseCycloneDXBom(appSettings.CycloneDxBomFilePath);
configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Npm);
foreach (string filepath in configFiles)
{
componentsForBOM.AddRange(ParseCycloneDXBom(filepath));
}
foreach (var component in componentsForBOM)
{
component.Properties = new List<Property>();
Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" };
Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Manually Added" };
component.Properties.Add(isDev);
component.Properties.Add(identifierType);

}
bom.Components = componentsForBOM;
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count;
}
Logger.Debug($"ParsePackageFile():End");
return bom;
}

private static void ParseConfigFile(string depFilePath, CommonAppSettings appSettings, ref List<Component> foundPackages)
private static void ParseDependencyTextFile(string depFilePath, CommonAppSettings appSettings, ref List<Component> foundPackages)
{
string[] lines = File.ReadAllLines(depFilePath);
int noOfExcludedComponents = 0;
Expand All @@ -101,18 +115,23 @@ private static void ParseConfigFile(string depFilePath, CommonAppSettings appSet
string scope = "";
bool isDevelopmentComponent;

Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" };
Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Discovered" };
scope = GetPackageDetails(parts, out component);

component.Properties = new List<Property>();
isDevelopmentComponent = GetDevDependentScopeList(appSettings, scope);

if (!component.Version.Contains("win") && !isDevelopmentComponent)
{
foundPackages.Add(component);
}
if (isDevelopmentComponent)
{
isDev.Value = "true";
BomCreator.bomKpiData.DevDependentComponents++;
}
component.Properties.Add(isDev);
component.Properties.Add(identifierType);
if (!component.Version.Contains("win"))
{
foundPackages.Add(component);
}

}
}
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = totalComponenstinInputFile;
Expand Down Expand Up @@ -174,7 +193,7 @@ public async Task<ComponentIdentification> IdentificationOfInternalComponents(
{
currentIterationItem.Properties = new List<Property>();
}

Property isInternal = new() { Name = Dataconstant.Cdx_IsInternal, Value = "false" };
if (isTrue)
{
Expand Down
1 change: 1 addition & 0 deletions src/LCT.PackageIdentifier/Model/NugetPackage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public class NugetPackage
public string Version { get; set; }

public string Filepath { get; set; }
public string IsDev { get; set; }

}
}
7 changes: 6 additions & 1 deletion src/LCT.PackageIdentifier/NpmProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,12 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List<Comp
}
else
{
bom = ParseCycloneDXBom(appSettings.CycloneDxBomFilePath);
configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Npm);
foreach (string filepath in configFiles)
{
componentsForBOM.AddRange(ParseCycloneDXBom(filepath));
}
bom.Components = componentsForBOM;
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count;
bom = RemoveExcludedComponents(appSettings, bom);

Expand Down
45 changes: 39 additions & 6 deletions src/LCT.PackageIdentifier/NugetProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings)
public static List<NugetPackage> ParsePackageConfig(string packagesFilePath, CommonAppSettings appSettings)
{
List<NugetPackage> nugetPackages = new List<NugetPackage>();
string isDev = "false";
try
{
List<ReferenceDetails> referenceList = Parsecsproj(appSettings);
Expand All @@ -79,7 +80,7 @@ public static List<NugetPackage> ParsePackageConfig(string packagesFilePath, Com
{

BomCreator.bomKpiData.DevDependentComponents++;
continue;
isDev = "true";
}

if (idAttribute?.Value == null)
Expand All @@ -97,7 +98,8 @@ public static List<NugetPackage> ParsePackageConfig(string packagesFilePath, Com
{
ID = idAttribute.Value,
Version = versionAttribute.Value,
Filepath = packagesFilePath
Filepath = packagesFilePath,
IsDev= isDev
};
nugetPackages.Add(package);
}
Expand All @@ -116,6 +118,7 @@ public static List<NugetPackage> ParsePackageConfig(string packagesFilePath, Com
public static List<NugetPackage> ParsePackageLock(string packagesFilePath, CommonAppSettings appSettings)
{
List<NugetPackage> packageList = new List<NugetPackage>();
string isDev = "false";
try
{
List<ReferenceDetails> referenceList = Parsecsproj(appSettings);
Expand All @@ -134,8 +137,8 @@ public static List<NugetPackage> ParsePackageLock(string packagesFilePath, Commo
string version = dependencyToken.First.Value<string>("resolved");
if (dependencyToken.First.Value<string>("type") == "Dev" || IsDevDependent(referenceList, id, version))
{
BomCreator.bomKpiData.DevDependentComponents++;
continue;
BomCreator.bomKpiData.DevDependentComponents++;
isDev = "true";
}
if (dependencyToken.First.Value<string>("type") == "Project" || string.IsNullOrEmpty(version) && string.IsNullOrEmpty(id))
{
Expand All @@ -149,7 +152,9 @@ public static List<NugetPackage> ParsePackageLock(string packagesFilePath, Commo
{
ID = id,
Version = version,
Filepath = packagesFilePath
Filepath = packagesFilePath,
IsDev= isDev

};
packageList.Add(package);
}
Expand Down Expand Up @@ -396,6 +401,8 @@ public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cy
private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List<Component> listComponentForBOM, ref Bom bom)
{
List<string> configFiles;
List<Component> componentsForBOM=new List<Component>();

if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath))
{
configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Nuget);
Expand All @@ -407,7 +414,20 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List<Comp
}
else
{
bom = ParseCycloneDXBom(appSettings.CycloneDxBomFilePath);
configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Nuget);
foreach (string filepath in configFiles)
{
componentsForBOM.AddRange(ParseCycloneDXBom(filepath));
}
foreach(var component in componentsForBOM)
{
component.Properties = new List<Property>();
Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" };
Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Manually Added" };
component.Properties.Add(isDev);
component.Properties.Add(identifierType);
}
bom.Components = componentsForBOM;
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count;
bom = RemoveExcludedComponents(appSettings, bom);
listComponentForBOM = bom.Components;
Expand All @@ -427,6 +447,19 @@ private static void ConvertToCycloneDXModel(List<Component> listComponentForBOM,
components.Purl = $"{ApiConstant.NugetExternalID}{prop.ID}@{components.Version}";
components.BomRef = $"{ApiConstant.NugetExternalID}{prop.ID}@{components.Version}";
components.Description = prop.Filepath;
components.Properties = new List<Property>()
{
new()
{
Name = Dataconstant.Cdx_IsDevelopment, Value = prop.IsDev
},
new Property()
{
Name=Dataconstant.Cdx_IdentifierType,Value="Discovered"
}
};


listComponentForBOM.Add(components);
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/LCT.SW360PackageCreator.UTest/ComponentCreatorTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ public async Task CycloneDxBomParser_PassingFilePath_ReturnsSuccess()
var sw360Service = new Mock<ISW360Service>();
var creatorHelper = new Mock<ICreatorHelper>();
var parser = new Mock<ICycloneDXBomParser>();
parser.Setup(x => x.ParseCycloneDXBom(It.IsAny<string>())).Returns(bom);
parser.Setup(x => x.ParseCycloneDXBom(It.IsAny<string>())).Returns(bom.Components);
creatorHelper.Setup(x => x.SetContentsForComparisonBOM(It.IsAny<List<Components>>(), sw360Service.Object)).ReturnsAsync(comparisonBomData);
var cycloneDXBomParser = new ComponentCreator();

Expand Down
2 changes: 1 addition & 1 deletion src/LCT.SW360PackageCreator/ComponentCreator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public class ComponentCreator : IComponentCreator
public async Task<List<ComparisonBomData>> CycloneDxBomParser(CommonAppSettings appSettings,
ISW360Service sw360Service, ICycloneDXBomParser cycloneDXBomParser, ICreatorHelper creatorHelper)
{
bom = cycloneDXBomParser.ParseCycloneDXBom(appSettings.BomFilePath);
bom.Components = cycloneDXBomParser.ParseCycloneDXBom(appSettings.BomFilePath);
TotalComponentsFromPackageIdentifier = bom != null ? bom.Components.Count : 0;
ListofBomComponents = await GetListOfBomData(bom?.Components ?? new List<Component>());

Expand Down