From 1c916e7ea29ba2ff7e850fb0b5c829cabd15f6cc Mon Sep 17 00:00:00 2001 From: karthika Date: Wed, 14 Jun 2023 18:13:50 +0530 Subject: [PATCH 01/14] Import multiple SBOM files from customer maven ,nuget,npm --- src/LCT.Common/CycloneDXBomParser.cs | 6 ++++-- src/LCT.Common/Interface/ICycloneDXBomParser.cs | 3 ++- .../CycloneBomProcessorTests.cs | 12 ++++++------ src/LCT.PackageIdentifier/MavenProcessor.cs | 7 ++++++- src/LCT.PackageIdentifier/NpmProcessor.cs | 7 ++++++- src/LCT.PackageIdentifier/NugetProcessor.cs | 8 +++++++- .../ComponentCreatorTest.cs | 2 +- src/LCT.SW360PackageCreator/ComponentCreator.cs | 2 +- 8 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/LCT.Common/CycloneDXBomParser.cs b/src/LCT.Common/CycloneDXBomParser.cs index b4ec0449..0b85ceb2 100644 --- a/src/LCT.Common/CycloneDXBomParser.cs +++ b/src/LCT.Common/CycloneDXBomParser.cs @@ -6,10 +6,12 @@ using CycloneDX.Json; using CycloneDX.Models; +using LCT.Common.Model; using log4net; using log4net.Core; using Newtonsoft.Json; using System; +using System.Collections.Generic; using System.IO; using System.Reflection; @@ -18,7 +20,7 @@ namespace LCT.Common public class CycloneDXBomParser : ICycloneDXBomParser { static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - public Bom ParseCycloneDXBom(string filePath) + public List ParseCycloneDXBom(string filePath) { Bom bom = new Bom(); string json = string.Empty; @@ -45,7 +47,7 @@ public Bom ParseCycloneDXBom(string filePath) { Logger.Error("Exception in reading cycloneDx bom", ex); } - return bom; + return bom.Components; } } } diff --git a/src/LCT.Common/Interface/ICycloneDXBomParser.cs b/src/LCT.Common/Interface/ICycloneDXBomParser.cs index 23cd62da..4cb9ef74 100644 --- a/src/LCT.Common/Interface/ICycloneDXBomParser.cs +++ b/src/LCT.Common/Interface/ICycloneDXBomParser.cs @@ -5,6 +5,7 @@ // -------------------------------------------------------------------------------------------------------------------- using CycloneDX.Models; +using System.Collections.Generic; namespace LCT.Common { @@ -13,6 +14,6 @@ namespace LCT.Common /// public interface ICycloneDXBomParser { - public Bom ParseCycloneDXBom(string filePath); + public List ParseCycloneDXBom(string filePath); } } diff --git a/src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs b/src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs index dd22627a..a821b26a 100644 --- a/src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs +++ b/src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs @@ -128,10 +128,10 @@ public void ParseCycloneDXBom_GivenBOMFilePath_ReturnsBOM() //Act CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser(); - Bom files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile); + List files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile); //Assert - Assert.That(4, Is.EqualTo(files.Components.Count), "Returns components in BOM"); + Assert.That(4, Is.EqualTo(files.Count), "Returns components in BOM"); } @@ -145,10 +145,10 @@ public void ParseCycloneDXBom_GivenInvlidBOMFilePath_ReturnsZeroComponents() //Act CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser(); - Bom files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile); + List files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile); //Assert - Assert.IsNull(files.Components, "Returns Zero components in BOM"); + Assert.IsNull(files, "Returns Zero components in BOM"); } @@ -161,10 +161,10 @@ public void ParseCycloneDXBom_GivenInCorrectJsonFile_ReturnsZeroComponents() //Act CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser(); - Bom files = cycloneBomProcessor.ParseCycloneDXBom(sourcePath + "/output.json"); + List files = cycloneBomProcessor.ParseCycloneDXBom(sourcePath + "/output.json"); //Assert - Assert.IsNull(files.Components, "Returns Zero components in BOM"); + Assert.IsNull(files, "Returns Zero components in BOM"); } } diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 244ae8ce..962c545a 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -76,7 +76,12 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) } else { - bom = ParseCycloneDXBom(appSettings.CycloneDxBomFilePath); + configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Npm); + foreach (string filepath in configFiles) + { + componentsForBOM.AddRange(ParseCycloneDXBom(filepath)); + } + bom.Components = componentsForBOM; BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; } Logger.Debug($"ParsePackageFile():End"); diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index 777b5dc4..04945939 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -265,7 +265,12 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List listComponentForBOM, ref Bom bom) { List configFiles; + List componentsForBOM=new List(); if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) { configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Nuget); @@ -403,7 +404,12 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List(); var creatorHelper = new Mock(); var parser = new Mock(); - parser.Setup(x => x.ParseCycloneDXBom(It.IsAny())).Returns(bom); + parser.Setup(x => x.ParseCycloneDXBom(It.IsAny())).Returns(bom.Components); creatorHelper.Setup(x => x.SetContentsForComparisonBOM(It.IsAny>(), sw360Service.Object)).ReturnsAsync(comparisonBomData); var cycloneDXBomParser = new ComponentCreator(); diff --git a/src/LCT.SW360PackageCreator/ComponentCreator.cs b/src/LCT.SW360PackageCreator/ComponentCreator.cs index ecda2fd7..2856832f 100644 --- a/src/LCT.SW360PackageCreator/ComponentCreator.cs +++ b/src/LCT.SW360PackageCreator/ComponentCreator.cs @@ -43,7 +43,7 @@ public class ComponentCreator : IComponentCreator public async Task> CycloneDxBomParser(CommonAppSettings appSettings, ISW360Service sw360Service, ICycloneDXBomParser cycloneDXBomParser, ICreatorHelper creatorHelper) { - bom = cycloneDXBomParser.ParseCycloneDXBom(appSettings.BomFilePath); + bom.Components = cycloneDXBomParser.ParseCycloneDXBom(appSettings.BomFilePath); TotalComponentsFromPackageIdentifier = bom != null ? bom.Components.Count : 0; ListofBomComponents = await GetListOfBomData(bom?.Components ?? new List()); From e53ee9b5ed97e19432b815fcdd4ce6e85d84b2bf Mon Sep 17 00:00:00 2001 From: karthika Date: Thu, 15 Jun 2023 18:12:00 +0530 Subject: [PATCH 02/14] SBOM creation --- src/LCT.PackageIdentifier/BomCreator.cs | 13 ++++--------- src/LCT.PackageIdentifier/MavenProcessor.cs | 20 ++++++++++++++------ src/LCT.PackageIdentifier/NugetProcessor.cs | 9 ++++++++- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/LCT.PackageIdentifier/BomCreator.cs b/src/LCT.PackageIdentifier/BomCreator.cs index db1101df..d04f690e 100644 --- a/src/LCT.PackageIdentifier/BomCreator.cs +++ b/src/LCT.PackageIdentifier/BomCreator.cs @@ -80,14 +80,9 @@ public async Task GenerateBom(CommonAppSettings appSettings, IBomHelper bomHelpe private static void WritecontentsToBOM(CommonAppSettings appSettings, BomKpiData bomKpiData, Bom listOfComponentsToBom) { - if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) - { + WriteContentToComparisonBOM(appSettings, listOfComponentsToBom, ref bomKpiData); - } - else - { - WriteContentToCycloneDxBOM(appSettings, listOfComponentsToBom, ref bomKpiData); - } + } @@ -96,13 +91,13 @@ private static void WriteContentToCycloneDxBOM(CommonAppSettings appSettings, Bo IFileOperations fileOperations = new FileOperations(); if (string.IsNullOrEmpty(appSettings.IdentifierBomFilePath)) { - fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.CycloneDxBomFilePath); + fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.SW360ProjectName); } else { listOfComponentsToBom = fileOperations.CombineComponentsFromExistingBOM(listOfComponentsToBom, appSettings.IdentifierBomFilePath); bomKpiData.ComponentsInComparisonBOM = listOfComponentsToBom.Components.Count; - fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.CycloneDxBomFilePath); + fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.SW360ProjectName); } } diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index efad07b6..94b11c8d 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -81,6 +81,12 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) { componentsForBOM.AddRange(ParseCycloneDXBom(filepath)); } + foreach (var component in componentsForBOM) + { + component.Properties = new List(); + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + component.Properties.Add(isDev); + } bom.Components = componentsForBOM; BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; } @@ -105,19 +111,21 @@ private static void ParseConfigFile(string depFilePath, CommonAppSettings appSet string[] parts = trimmedLine.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries); string scope = ""; bool isDevelopmentComponent; - + Property isInternal = new() { Name = Dataconstant.Cdx_IsInternal, Value = "false" }; scope = GetPackageDetails(parts, out component); isDevelopmentComponent = GetDevDependentScopeList(appSettings, scope); - - if (!component.Version.Contains("win") && !isDevelopmentComponent) - { - foundPackages.Add(component); - } if (isDevelopmentComponent) { + isInternal.Value = "true"; BomCreator.bomKpiData.DevDependentComponents++; } + component.Properties.Add(isInternal); + if (!component.Version.Contains("win")) + { + foundPackages.Add(component); + } + } } BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = totalComponenstinInputFile; diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index a4de57bf..0fe9caa1 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -397,6 +397,7 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List configFiles; List componentsForBOM=new List(); + if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) { configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Nuget); @@ -408,11 +409,17 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List(); + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + component.Properties.Add(isDev); + } bom.Components = componentsForBOM; BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; bom = RemoveExcludedComponents(appSettings, bom); From 4e92704836ff743ee4fda04b9831c9053affb1c7 Mon Sep 17 00:00:00 2001 From: karthika Date: Fri, 16 Jun 2023 18:57:35 +0530 Subject: [PATCH 03/14] SBOM parsing For nuget and Maven --- .../MavenParserTests.cs | 2 +- .../NugetParserTests.cs | 2 +- src/LCT.PackageIdentifier/MavenProcessor.cs | 15 ++++++----- .../Model/NugetPackage.cs | 1 + src/LCT.PackageIdentifier/NugetProcessor.cs | 26 ++++++++++++++----- 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/LCT.PackageIdentifier.UTest/MavenParserTests.cs b/src/LCT.PackageIdentifier.UTest/MavenParserTests.cs index e531690c..f15ab249 100644 --- a/src/LCT.PackageIdentifier.UTest/MavenParserTests.cs +++ b/src/LCT.PackageIdentifier.UTest/MavenParserTests.cs @@ -73,7 +73,7 @@ public void IsDevDependent_GivenListOfMavenDevComponents_ReturnsNonDevComponents Bom bom = MavenProcessor.ParsePackageFile(appSettings); //Assert - Assert.That(bom.Components.Count, Is.EqualTo(1), "Returns the count of NON Dev Dependency components"); + Assert.That(bom.Components.Count-BomCreator.bomKpiData.DevDependentComponents, Is.EqualTo(1), "Returns the count of NON Dev Dependency components"); } [Test] diff --git a/src/LCT.PackageIdentifier.UTest/NugetParserTests.cs b/src/LCT.PackageIdentifier.UTest/NugetParserTests.cs index 83cb93da..65629c3c 100644 --- a/src/LCT.PackageIdentifier.UTest/NugetParserTests.cs +++ b/src/LCT.PackageIdentifier.UTest/NugetParserTests.cs @@ -52,7 +52,7 @@ public void ParsePackageConfig_GivenAInputFilePath_ReturnsSuccess() public void ParsePackageLockJson_GivenAInputFilePath_ReturnsSuccess() { //Arrange - int expectednoofcomponents = 152; + int expectednoofcomponents = 153; string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; string outFolder = Path.GetDirectoryName(exePath); string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles\packages.lock.json"; diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 94b11c8d..be0114a8 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -53,7 +53,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) } } - ParseConfigFile(depFilePath, appSettings, ref componentsForBOM); + ParseDependencyTextFile(depFilePath, appSettings, ref componentsForBOM); totalComponentsIdentified = componentsForBOM.Count; @@ -94,7 +94,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) return bom; } - private static void ParseConfigFile(string depFilePath, CommonAppSettings appSettings, ref List foundPackages) + private static void ParseDependencyTextFile(string depFilePath, CommonAppSettings appSettings, ref List foundPackages) { string[] lines = File.ReadAllLines(depFilePath); int noOfExcludedComponents = 0; @@ -111,16 +111,19 @@ private static void ParseConfigFile(string depFilePath, CommonAppSettings appSet string[] parts = trimmedLine.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries); string scope = ""; bool isDevelopmentComponent; - Property isInternal = new() { Name = Dataconstant.Cdx_IsInternal, Value = "false" }; - scope = GetPackageDetails(parts, out component); + + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + scope = GetPackageDetails(parts, out component); + isDevelopmentComponent = GetDevDependentScopeList(appSettings, scope); if (isDevelopmentComponent) { - isInternal.Value = "true"; + component.Properties = new List(); + isDev.Value = "true"; BomCreator.bomKpiData.DevDependentComponents++; } - component.Properties.Add(isInternal); + component.Properties.Add(isDev); if (!component.Version.Contains("win")) { foundPackages.Add(component); diff --git a/src/LCT.PackageIdentifier/Model/NugetPackage.cs b/src/LCT.PackageIdentifier/Model/NugetPackage.cs index ec9f7112..c2717113 100644 --- a/src/LCT.PackageIdentifier/Model/NugetPackage.cs +++ b/src/LCT.PackageIdentifier/Model/NugetPackage.cs @@ -19,6 +19,7 @@ public class NugetPackage public string Version { get; set; } public string Filepath { get; set; } + public string IsDev { get; set; } } } diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 0fe9caa1..bfb6a046 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -61,6 +61,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) public static List ParsePackageConfig(string packagesFilePath, CommonAppSettings appSettings) { List nugetPackages = new List(); + string isDev = "false"; try { List referenceList = Parsecsproj(appSettings); @@ -79,7 +80,7 @@ public static List ParsePackageConfig(string packagesFilePath, Com { BomCreator.bomKpiData.DevDependentComponents++; - continue; + isDev = "true"; } if (idAttribute?.Value == null) @@ -97,7 +98,8 @@ public static List ParsePackageConfig(string packagesFilePath, Com { ID = idAttribute.Value, Version = versionAttribute.Value, - Filepath = packagesFilePath + Filepath = packagesFilePath, + IsDev= isDev }; nugetPackages.Add(package); } @@ -116,6 +118,7 @@ public static List ParsePackageConfig(string packagesFilePath, Com public static List ParsePackageLock(string packagesFilePath, CommonAppSettings appSettings) { List packageList = new List(); + string isDev = "false"; try { List referenceList = Parsecsproj(appSettings); @@ -134,8 +137,8 @@ public static List ParsePackageLock(string packagesFilePath, Commo string version = dependencyToken.First.Value("resolved"); if (dependencyToken.First.Value("type") == "Dev" || IsDevDependent(referenceList, id, version)) { - BomCreator.bomKpiData.DevDependentComponents++; - continue; + BomCreator.bomKpiData.DevDependentComponents++; + isDev = "true"; } if (dependencyToken.First.Value("type") == "Project" || string.IsNullOrEmpty(version) && string.IsNullOrEmpty(id)) { @@ -149,7 +152,9 @@ public static List ParsePackageLock(string packagesFilePath, Commo { ID = id, Version = version, - Filepath = packagesFilePath + Filepath = packagesFilePath, + IsDev= isDev + }; packageList.Add(package); } @@ -417,7 +422,7 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List(); - Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; component.Properties.Add(isDev); } bom.Components = componentsForBOM; @@ -440,6 +445,15 @@ private static void ConvertToCycloneDXModel(List listComponentForBOM, components.Purl = $"{ApiConstant.NugetExternalID}{prop.ID}@{components.Version}"; components.BomRef = $"{ApiConstant.NugetExternalID}{prop.ID}@{components.Version}"; components.Description = prop.Filepath; + components.Properties = new List() + { + new() + { + Name = Dataconstant.Cdx_IsDevelopment, Value = prop.IsDev + } + }; + + listComponentForBOM.Add(components); } } From f3c6eccade06e1cfa2bc8990fdca0163ef23daa0 Mon Sep 17 00:00:00 2001 From: karthika Date: Mon, 19 Jun 2023 10:52:50 +0530 Subject: [PATCH 04/14] identifier type added --- src/LCT.PackageIdentifier/MavenProcessor.cs | 15 +++++++++------ src/LCT.PackageIdentifier/NugetProcessor.cs | 6 ++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index be0114a8..63ad2c62 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -85,7 +85,10 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) { component.Properties = new List(); Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Manually Added" }; component.Properties.Add(isDev); + component.Properties.Add(identifierType); + } bom.Components = componentsForBOM; BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; @@ -111,24 +114,24 @@ private static void ParseDependencyTextFile(string depFilePath, CommonAppSetting string[] parts = trimmedLine.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries); string scope = ""; bool isDevelopmentComponent; - - Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Discovered" }; scope = GetPackageDetails(parts, out component); - + component.Properties = new List(); isDevelopmentComponent = GetDevDependentScopeList(appSettings, scope); if (isDevelopmentComponent) { - component.Properties = new List(); isDev.Value = "true"; BomCreator.bomKpiData.DevDependentComponents++; } component.Properties.Add(isDev); + component.Properties.Add(identifierType); if (!component.Version.Contains("win")) { foundPackages.Add(component); } - + } } BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = totalComponenstinInputFile; @@ -190,7 +193,7 @@ public async Task IdentificationOfInternalComponents( { currentIterationItem.Properties = new List(); } - + Property isInternal = new() { Name = Dataconstant.Cdx_IsInternal, Value = "false" }; if (isTrue) { diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index bfb6a046..be067ba0 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -423,7 +423,9 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List(); Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Manually Added" }; component.Properties.Add(isDev); + component.Properties.Add(identifierType); } bom.Components = componentsForBOM; BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; @@ -450,6 +452,10 @@ private static void ConvertToCycloneDXModel(List listComponentForBOM, new() { Name = Dataconstant.Cdx_IsDevelopment, Value = prop.IsDev + }, + new Property() + { + Name=Dataconstant.Cdx_IdentifierType,Value="Discovered" } }; From c158df67e6ac4a64d6a900024e8fc9e0848e4ab4 Mon Sep 17 00:00:00 2001 From: karthika Date: Mon, 19 Jun 2023 16:51:08 +0530 Subject: [PATCH 05/14] New pattern added for input filetype --- src/LCT.Common/appSettings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LCT.Common/appSettings.json b/src/LCT.Common/appSettings.json index 0c2831c9..082e0aec 100644 --- a/src/LCT.Common/appSettings.json +++ b/src/LCT.Common/appSettings.json @@ -42,7 +42,7 @@ "ExcludedComponents": [] }, "Nuget": { - "Include": [ "pack*.config", "p*.lock.json" ], + "Include": [ "pack*.config", "p*.lock.json","*.cdx.json" ], "Exclude": [], "JfrogNugetRepoList": [ "", //This is a mirror repo for nuget.org in JFrog @@ -51,7 +51,7 @@ "ExcludedComponents": [] }, "Maven": { - "Include": [ "pom.xml" ], + "Include": [ "pom.xml","*.cdx.json" ], "Exclude": [], "JfrogMavenRepoList": [ "", //This is a mirror repo for repo.maven in JFrog From 7a7bc4324b0fb1c7047a5a6ccabe8a581025dc4f Mon Sep 17 00:00:00 2001 From: karthika Date: Wed, 21 Jun 2023 11:19:29 +0530 Subject: [PATCH 06/14] Review comment implementation --- src/LCT.Common/CycloneDXBomParser.cs | 4 ++-- src/LCT.Common/Interface/ICycloneDXBomParser.cs | 2 +- src/LCT.PackageIdentifier/MavenProcessor.cs | 3 ++- src/LCT.PackageIdentifier/NugetProcessor.cs | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/LCT.Common/CycloneDXBomParser.cs b/src/LCT.Common/CycloneDXBomParser.cs index 0b85ceb2..af407172 100644 --- a/src/LCT.Common/CycloneDXBomParser.cs +++ b/src/LCT.Common/CycloneDXBomParser.cs @@ -20,7 +20,7 @@ namespace LCT.Common public class CycloneDXBomParser : ICycloneDXBomParser { static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - public List ParseCycloneDXBom(string filePath) + public Bom ParseCycloneDXBom(string filePath) { Bom bom = new Bom(); string json = string.Empty; @@ -47,7 +47,7 @@ public List ParseCycloneDXBom(string filePath) { Logger.Error("Exception in reading cycloneDx bom", ex); } - return bom.Components; + return bom; } } } diff --git a/src/LCT.Common/Interface/ICycloneDXBomParser.cs b/src/LCT.Common/Interface/ICycloneDXBomParser.cs index 4cb9ef74..f5250f4f 100644 --- a/src/LCT.Common/Interface/ICycloneDXBomParser.cs +++ b/src/LCT.Common/Interface/ICycloneDXBomParser.cs @@ -14,6 +14,6 @@ namespace LCT.Common /// public interface ICycloneDXBomParser { - public List ParseCycloneDXBom(string filePath); + public Bom ParseCycloneDXBom(string filePath); } } diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 63ad2c62..dc71ac48 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -79,7 +79,8 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Npm); foreach (string filepath in configFiles) { - componentsForBOM.AddRange(ParseCycloneDXBom(filepath)); + Bom bomList=ParseCycloneDXBom(filepath); + componentsForBOM.AddRange(bomList.Components); } foreach (var component in componentsForBOM) { diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index be067ba0..22f3553d 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -417,7 +417,8 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List Date: Wed, 21 Jun 2023 11:25:44 +0530 Subject: [PATCH 07/14] Build failure fix --- .../CycloneBomProcessorTests.cs | 12 ++++++------ src/LCT.PackageIdentifier/NpmProcessor.cs | 3 ++- .../ComponentCreatorTest.cs | 2 +- src/LCT.SW360PackageCreator/ComponentCreator.cs | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs b/src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs index a821b26a..dd22627a 100644 --- a/src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs +++ b/src/LCT.PackageIdentifier.UTest/CycloneBomProcessorTests.cs @@ -128,10 +128,10 @@ public void ParseCycloneDXBom_GivenBOMFilePath_ReturnsBOM() //Act CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser(); - List files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile); + Bom files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile); //Assert - Assert.That(4, Is.EqualTo(files.Count), "Returns components in BOM"); + Assert.That(4, Is.EqualTo(files.Components.Count), "Returns components in BOM"); } @@ -145,10 +145,10 @@ public void ParseCycloneDXBom_GivenInvlidBOMFilePath_ReturnsZeroComponents() //Act CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser(); - List files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile); + Bom files = cycloneBomProcessor.ParseCycloneDXBom(BomTestFile); //Assert - Assert.IsNull(files, "Returns Zero components in BOM"); + Assert.IsNull(files.Components, "Returns Zero components in BOM"); } @@ -161,10 +161,10 @@ public void ParseCycloneDXBom_GivenInCorrectJsonFile_ReturnsZeroComponents() //Act CycloneDXBomParser cycloneBomProcessor = new CycloneDXBomParser(); - List files = cycloneBomProcessor.ParseCycloneDXBom(sourcePath + "/output.json"); + Bom files = cycloneBomProcessor.ParseCycloneDXBom(sourcePath + "/output.json"); //Assert - Assert.IsNull(files, "Returns Zero components in BOM"); + Assert.IsNull(files.Components, "Returns Zero components in BOM"); } } diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index 2fba4445..fc0e8ad3 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -276,7 +276,8 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List(); var creatorHelper = new Mock(); var parser = new Mock(); - parser.Setup(x => x.ParseCycloneDXBom(It.IsAny())).Returns(bom.Components); + parser.Setup(x => x.ParseCycloneDXBom(It.IsAny())).Returns(bom); creatorHelper.Setup(x => x.SetContentsForComparisonBOM(It.IsAny>(), sw360Service.Object)).ReturnsAsync(comparisonBomData); var cycloneDXBomParser = new ComponentCreator(); diff --git a/src/LCT.SW360PackageCreator/ComponentCreator.cs b/src/LCT.SW360PackageCreator/ComponentCreator.cs index cccad7a3..1d9f2b7a 100644 --- a/src/LCT.SW360PackageCreator/ComponentCreator.cs +++ b/src/LCT.SW360PackageCreator/ComponentCreator.cs @@ -43,7 +43,7 @@ public class ComponentCreator : IComponentCreator public async Task> CycloneDxBomParser(CommonAppSettings appSettings, ISW360Service sw360Service, ICycloneDXBomParser cycloneDXBomParser, ICreatorHelper creatorHelper) { - bom.Components = cycloneDXBomParser.ParseCycloneDXBom(appSettings.BomFilePath); + bom = cycloneDXBomParser.ParseCycloneDXBom(appSettings.BomFilePath); TotalComponentsFromPackageIdentifier = bom != null ? bom.Components.Count : 0; ListofBomComponents = await GetListOfBomData(bom?.Components ?? new List()); From b854ad9818f513bcd213e46dc89f220777d60064 Mon Sep 17 00:00:00 2001 From: karthika Date: Wed, 28 Jun 2023 18:28:58 +0530 Subject: [PATCH 08/14] Check in for bom reading --- .../Cyclonedx2.json | 6 -- .../Duplicate_Cyclonedx.json | 6 -- .../SourceDetails_Cyclonedx.json | 6 -- .../SourceDetails_Cyclonedx2.json | 6 -- src/LCT.PackageIdentifier/DebianProcessor.cs | 4 +- src/LCT.PackageIdentifier/MavenProcessor.cs | 77 +++++-------------- src/LCT.PackageIdentifier/NpmProcessor.cs | 34 ++++---- 7 files changed, 39 insertions(+), 100 deletions(-) diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/Cyclonedx2.json b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/Cyclonedx2.json index 95e3e1a0..71200273 100644 --- a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/Cyclonedx2.json +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/Cyclonedx2.json @@ -1,9 +1,3 @@ -// -------------------------------------------------------------------------------------------------------------------- -// SPDX-FileCopyrightText: 2023 Siemens AG -// -// SPDX-License-Identifier: MIT - -// -------------------------------------------------------------------------------------------------------------------- { "bomFormat": "CycloneDX", "specVersion": "1.3", diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/Duplicate_Cyclonedx.json b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/Duplicate_Cyclonedx.json index e31ee157..8ff4ce7c 100644 --- a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/Duplicate_Cyclonedx.json +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/Duplicate_Cyclonedx.json @@ -1,9 +1,3 @@ -// -------------------------------------------------------------------------------------------------------------------- -// SPDX-FileCopyrightText: 2023 Siemens AG -// -// SPDX-License-Identifier: MIT - -// -------------------------------------------------------------------------------------------------------------------- { "bomFormat": "CycloneDX", "specVersion": "1.3", diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/SourceDetails_Cyclonedx.json b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/SourceDetails_Cyclonedx.json index fc503dc8..578d1734 100644 --- a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/SourceDetails_Cyclonedx.json +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/SourceDetails_Cyclonedx.json @@ -1,9 +1,3 @@ -// -------------------------------------------------------------------------------------------------------------------- -// SPDX-FileCopyrightText: 2023 Siemens AG -// -// SPDX-License-Identifier: MIT - -// -------------------------------------------------------------------------------------------------------------------- { "bomFormat": "CycloneDX", "specVersion": "1.3", diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/SourceDetails_Cyclonedx2.json b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/SourceDetails_Cyclonedx2.json index 5e50af65..1680937d 100644 --- a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/SourceDetails_Cyclonedx2.json +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/SourceDetails_Cyclonedx2.json @@ -1,9 +1,3 @@ -// -------------------------------------------------------------------------------------------------------------------- -// SPDX-FileCopyrightText: 2023 Siemens AG -// -// SPDX-License-Identifier: MIT - -// -------------------------------------------------------------------------------------------------------------------- { "bomFormat": "CycloneDX", "specVersion": "1.3", diff --git a/src/LCT.PackageIdentifier/DebianProcessor.cs b/src/LCT.PackageIdentifier/DebianProcessor.cs index d3e83d07..464c661f 100644 --- a/src/LCT.PackageIdentifier/DebianProcessor.cs +++ b/src/LCT.PackageIdentifier/DebianProcessor.cs @@ -44,9 +44,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) { Logger.Debug($"ParsePackageFile():FileName: " + filepath); listofComponents.AddRange(ParseCycloneDX(filepath)); - } - - + } int initialCount = listofComponents.Count; GetDistinctComponentList(ref listofComponents); diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index dc71ac48..95cdcdcb 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -30,70 +30,33 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) { List componentsForBOM = new(); Bom bom = new(); - - string depFilePath = ""; - int totalComponentsIdentified = 0; - List configFiles = new(); + List configFiles=new(); if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) { - //Create empty dependency list file - if (!string.IsNullOrEmpty(appSettings.PackageFilePath)) - { - configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Maven); - depFilePath = Path.Combine(appSettings.PackageFilePath, "POMDependencies.txt"); - File.Create(depFilePath).Close(); - } - - foreach (var bomFilePath in configFiles) - { - Result result = BomHelper.GetDependencyList(bomFilePath, depFilePath); - if (result.ExitCode != 0) - { - Logger.Debug("Error in downloading maven packages"); - } - } - - ParseDependencyTextFile(depFilePath, appSettings, ref componentsForBOM); - - totalComponentsIdentified = componentsForBOM.Count; - - componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList(); - - BomCreator.bomKpiData.DuplicateComponents = totalComponentsIdentified - componentsForBOM.Count; - - var componentsWithMultipleVersions = componentsForBOM.GroupBy(s => s.Name) - .Where(g => g.Count() > 1).SelectMany(g => g).ToList(); - - if (componentsWithMultipleVersions.Count != 0) - { - Logger.Warn($"Multiple versions detected :\n"); - foreach (var item in componentsWithMultipleVersions) - { - Logger.Warn($"Component Name : {item.Name}\nComponent Version : {item.Version}\nPackage Found in : {appSettings.PackageFilePath}\n"); - } - } - bom.Components = componentsForBOM; + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm); } else { - configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Npm); - foreach (string filepath in configFiles) - { - Bom bomList=ParseCycloneDXBom(filepath); - componentsForBOM.AddRange(bomList.Components); - } - foreach (var component in componentsForBOM) - { - component.Properties = new List(); - Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; - Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Manually Added" }; - component.Properties.Add(isDev); - component.Properties.Add(identifierType); + configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Npm); + } + + foreach (string filepath in configFiles) + { + Bom bomList = ParseCycloneDXBom(filepath); + componentsForBOM.AddRange(bomList.Components); + } + foreach (var component in componentsForBOM) + { + component.Properties = new List(); + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Manually Added" }; + component.Properties.Add(isDev); + component.Properties.Add(identifierType); - } - bom.Components = componentsForBOM; - BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; } + bom.Components = componentsForBOM; + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = bom.Components.Count; + BomCreator.bomKpiData.ComponentsInComparisonBOM = bom.Components.Count; Logger.Debug($"ParsePackageFile():End"); return bom; } diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index 039d1d7f..a0a6ec89 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -332,25 +332,27 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List Date: Fri, 30 Jun 2023 15:44:44 +0530 Subject: [PATCH 09/14] maven cycloneDx parsing logic --- .../LCT.PackageIdentifier.UTest.csproj | 6 + .../MavenParserTests.cs | 6 +- .../PackageIdentifierUTTestFiles/POM.xml | 5 + .../PackageIdentifierUTTestFiles/bom.cdx.json | 147 ++++++++++++++++++ .../bom1.cdx.json | 147 ++++++++++++++++++ .../dependency-reduced-pom.xml | 53 +++++++ src/LCT.PackageIdentifier/MavenProcessor.cs | 6 +- 7 files changed, 364 insertions(+), 6 deletions(-) create mode 100644 src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/bom.cdx.json create mode 100644 src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/bom1.cdx.json create mode 100644 src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/dependency-reduced-pom.xml diff --git a/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj b/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj index dbe62222..aef93ce9 100644 --- a/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj +++ b/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj @@ -53,6 +53,12 @@ + + PreserveNewest + + + Always + Always diff --git a/src/LCT.PackageIdentifier.UTest/MavenParserTests.cs b/src/LCT.PackageIdentifier.UTest/MavenParserTests.cs index f15ab249..636ad66d 100644 --- a/src/LCT.PackageIdentifier.UTest/MavenParserTests.cs +++ b/src/LCT.PackageIdentifier.UTest/MavenParserTests.cs @@ -28,7 +28,7 @@ public void ParsePackageFile_PackageLockWithDuplicateComponents_ReturnsCountOfDu string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; string outFolder = Path.GetDirectoryName(exePath); string filepath = outFolder + @"\PackageIdentifierUTTestFiles"; - string[] Includes = { "POM.xml" }; + string[] Includes = { "*.cdx.json" }; string[] Excludes = { "lol" }; CommonAppSettings appSettings = new CommonAppSettings() @@ -45,7 +45,7 @@ public void ParsePackageFile_PackageLockWithDuplicateComponents_ReturnsCountOfDu Bom bom = MavenProcessor.ParsePackageFile(appSettings); //Assert - Assert.That(bom.Components.Count, Is.EqualTo(3), "Returns the count of components"); + Assert.That(bom.Components.Count, Is.EqualTo(2), "Returns the count of components"); } @@ -56,7 +56,7 @@ public void IsDevDependent_GivenListOfMavenDevComponents_ReturnsNonDevComponents string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; string outFolder = Path.GetDirectoryName(exePath); string filepath = outFolder + @"\PackageIdentifierUTTestFiles"; - string[] Includes = { "POM.xml" }; + string[] Includes = { "*.cdx.json" }; string[] Excludes = { "lol" }; CommonAppSettings appSettings = new CommonAppSettings() diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/POM.xml b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/POM.xml index fb80d2b5..87fbb69b 100644 --- a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/POM.xml +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/POM.xml @@ -29,6 +29,11 @@ + + org.cyclonedx + cyclonedx-maven-plugin + 2.5.3 + org.apache.maven.plugins maven-shade-plugin diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/bom.cdx.json b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/bom.cdx.json new file mode 100644 index 00000000..ef8163bc --- /dev/null +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/bom.cdx.json @@ -0,0 +1,147 @@ +{ + "bomFormat" : "CycloneDX", + "specVersion" : "1.3", + "serialNumber" : "urn:uuid:cf9dd7ef-4b1b-4343-be5a-44837cfa5005", + "version" : 1, + "metadata" : { + "timestamp" : "2023-06-30T05:34:37Z", + "tools" : [ + { + "vendor" : "CycloneDX", + "name" : "CycloneDX Maven plugin", + "version" : "2.5.3", + "hashes" : [ + { + "alg" : "MD5", + "content" : "4f7d894200ad695fc9f0aad66d7da40a" + }, + { + "alg" : "SHA-1", + "content" : "c044d9b726650cbea3adeb5cc1715c67d8356c0a" + }, + { + "alg" : "SHA-256", + "content" : "b9a385e430e1f5efd9b835a084c195dde4d5e1bc79e469a8187ec58275c15313" + }, + { + "alg" : "SHA-384", + "content" : "d96f68ef4b8830d70dc2eb5f2de5211d96b70dd1169da641f34474265c06a5321b63d2c80fe2d82d74c767391225e480" + }, + { + "alg" : "SHA-512", + "content" : "1d7d1129cdc8604772b3c454d8dff98d936f85af705c95705e3263a038c0bb58fdd58c0b90efa3f56b4ce8ef9c84d3154b74b8451e0470f856f4d688489704b0" + }, + { + "alg" : "SHA3-256", + "content" : "44231962fe0c1e5501ca38ad3320f9223ea5e8d62aa8aad170577818801ce349" + }, + { + "alg" : "SHA3-384", + "content" : "99659ce3e58d8416f9e28d6b87c800442a79c4a5703fb657f6a9da87495d1d9d3b9788e06a3d6ea0e1b659a4681a4c92" + }, + { + "alg" : "SHA3-512", + "content" : "8d5c3f0ee5a53cc714c4d829ccc07688f951a6b6655ad1e6435b8ab1c281bc38a78073b329bdaaf4887114b6843723ac8b5176a5f954581960a43662c688a95a" + } + ] + } + ], + "component" : { + "group" : "org.springframework", + "name" : "gs-maven", + "version" : "0.1.0", + "licenses" : [ ], + "purl" : "pkg:maven/org.springframework/gs-maven@0.1.0?type=jar", + "type" : "library", + "bom-ref" : "pkg:maven/org.springframework/gs-maven@0.1.0?type=jar" + } + }, + "components" : [ + { + "publisher" : "Joda.org", + "group" : "joda-time", + "name" : "joda-time", + "version" : "2.9.2", + "description" : "Date and time library to replace JDK date handling", + "scope" : "optional", + "hashes" : [ + { + "alg" : "MD5", + "content" : "32a794b6a820daf3fad92e59988df64c" + }, + { + "alg" : "SHA-1", + "content" : "36d6e77a419cb455e6fd5909f6f96b168e21e9d0" + }, + { + "alg" : "SHA-256", + "content" : "0be5c40e8cdce9ec0643d76be99f276db17c45d7616a217fd1b19b7ef73ca7b1" + }, + { + "alg" : "SHA-384", + "content" : "fe4d61fa8c2ae6bfe94b897fb100a23678bbd172b5c939531197c5566c5836f9a719484b5cf2f70960996bd397c0025c" + }, + { + "alg" : "SHA-512", + "content" : "52bf64e32ae5303ecf78510f78acfdce46b1654214a106f4d92f7c8e09ab4214790567198dd4c54b0f6e2b75765ad0c7b4a2d2cb3483e2782f16faed5546a8da" + }, + { + "alg" : "SHA3-256", + "content" : "361583e31c9add8f66af3220979a7a96aea0f2886644cd40e15e90ac5da0ca24" + }, + { + "alg" : "SHA3-384", + "content" : "4aaa49db59997ce580609dfb0142ed91656cb2f8db667e9fc7d8e206f4480e379601c8c16ee3e7a8870048b7da8209f0" + }, + { + "alg" : "SHA3-512", + "content" : "047292bca529cf8e9702041982348af816dbcec95917df377197eb22d798c3ac3d09a70591d21cea16a4e5e55ec491c74e0a9d062994303a7715548a9b122454" + } + ], + "licenses" : [ + { + "license" : { + "id" : "Apache-2.0" + } + } + ], + "purl" : "pkg:maven/joda-time/joda-time@2.9.2?type=jar", + "externalReferences" : [ + { + "type" : "website", + "url" : "http://www.joda.org" + }, + { + "type" : "distribution", + "url" : "http://oss.sonatype.org/content/repositories/joda-releases" + }, + { + "type" : "issue-tracker", + "url" : "https://github.com/JodaOrg/joda-time/issues" + }, + { + "type" : "mailing-list", + "url" : "http://sourceforge.net/mailarchive/forum.php?forum_name=joda-interest" + }, + { + "type" : "vcs", + "url" : "https://github.com/JodaOrg/joda-time" + } + ], + "type" : "library", + "bom-ref" : "pkg:maven/joda-time/joda-time@2.9.2?type=jar" + } + ], + "dependencies" : [ + { + "ref" : "pkg:maven/org.springframework/gs-maven@0.1.0?type=jar", + "dependsOn" : [ + "pkg:maven/joda-time/joda-time@2.9.2?type=jar" + ] + }, + { + "ref" : "pkg:maven/joda-time/joda-time@2.9.2?type=jar", + "dependsOn" : [ ] + } + ] +} \ No newline at end of file diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/bom1.cdx.json b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/bom1.cdx.json new file mode 100644 index 00000000..8597f6a3 --- /dev/null +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/bom1.cdx.json @@ -0,0 +1,147 @@ +{ + "bomFormat" : "CycloneDX", + "specVersion" : "1.3", + "serialNumber" : "urn:uuid:f33a1063-8d60-45e0-aa93-ae7352b6fe86", + "version" : 1, + "metadata" : { + "timestamp" : "2023-06-30T05:40:11Z", + "tools" : [ + { + "vendor" : "CycloneDX", + "name" : "CycloneDX Maven plugin", + "version" : "2.5.3", + "hashes" : [ + { + "alg" : "MD5", + "content" : "4f7d894200ad695fc9f0aad66d7da40a" + }, + { + "alg" : "SHA-1", + "content" : "c044d9b726650cbea3adeb5cc1715c67d8356c0a" + }, + { + "alg" : "SHA-256", + "content" : "b9a385e430e1f5efd9b835a084c195dde4d5e1bc79e469a8187ec58275c15313" + }, + { + "alg" : "SHA-384", + "content" : "d96f68ef4b8830d70dc2eb5f2de5211d96b70dd1169da641f34474265c06a5321b63d2c80fe2d82d74c767391225e480" + }, + { + "alg" : "SHA-512", + "content" : "1d7d1129cdc8604772b3c454d8dff98d936f85af705c95705e3263a038c0bb58fdd58c0b90efa3f56b4ce8ef9c84d3154b74b8451e0470f856f4d688489704b0" + }, + { + "alg" : "SHA3-256", + "content" : "44231962fe0c1e5501ca38ad3320f9223ea5e8d62aa8aad170577818801ce349" + }, + { + "alg" : "SHA3-384", + "content" : "99659ce3e58d8416f9e28d6b87c800442a79c4a5703fb657f6a9da87495d1d9d3b9788e06a3d6ea0e1b659a4681a4c92" + }, + { + "alg" : "SHA3-512", + "content" : "8d5c3f0ee5a53cc714c4d829ccc07688f951a6b6655ad1e6435b8ab1c281bc38a78073b329bdaaf4887114b6843723ac8b5176a5f954581960a43662c688a95a" + } + ] + } + ], + "component" : { + "group" : "org.springframework", + "name" : "gs-maven", + "version" : "0.1.0", + "licenses" : [ ], + "purl" : "pkg:maven/org.springframework/gs-maven@0.1.0?type=jar", + "type" : "library", + "bom-ref" : "pkg:maven/org.springframework/gs-maven@0.1.0?type=jar" + } + }, + "components" : [ + { + "publisher" : "Joda.org", + "group" : "joda-time", + "name" : "joda-time", + "version" : "2.9.2", + "description" : "Date and time library to replace JDK date handling", + "scope" : "optional", + "hashes" : [ + { + "alg" : "MD5", + "content" : "32a794b6a820daf3fad92e59988df64c" + }, + { + "alg" : "SHA-1", + "content" : "36d6e77a419cb455e6fd5909f6f96b168e21e9d0" + }, + { + "alg" : "SHA-256", + "content" : "0be5c40e8cdce9ec0643d76be99f276db17c45d7616a217fd1b19b7ef73ca7b1" + }, + { + "alg" : "SHA-384", + "content" : "fe4d61fa8c2ae6bfe94b897fb100a23678bbd172b5c939531197c5566c5836f9a719484b5cf2f70960996bd397c0025c" + }, + { + "alg" : "SHA-512", + "content" : "52bf64e32ae5303ecf78510f78acfdce46b1654214a106f4d92f7c8e09ab4214790567198dd4c54b0f6e2b75765ad0c7b4a2d2cb3483e2782f16faed5546a8da" + }, + { + "alg" : "SHA3-256", + "content" : "361583e31c9add8f66af3220979a7a96aea0f2886644cd40e15e90ac5da0ca24" + }, + { + "alg" : "SHA3-384", + "content" : "4aaa49db59997ce580609dfb0142ed91656cb2f8db667e9fc7d8e206f4480e379601c8c16ee3e7a8870048b7da8209f0" + }, + { + "alg" : "SHA3-512", + "content" : "047292bca529cf8e9702041982348af816dbcec95917df377197eb22d798c3ac3d09a70591d21cea16a4e5e55ec491c74e0a9d062994303a7715548a9b122454" + } + ], + "licenses" : [ + { + "license" : { + "id" : "Apache-2.0" + } + } + ], + "purl" : "pkg:maven/joda-time/joda-time@2.9.2?type=jar", + "externalReferences" : [ + { + "type" : "website", + "url" : "http://www.joda.org" + }, + { + "type" : "distribution", + "url" : "http://oss.sonatype.org/content/repositories/joda-releases" + }, + { + "type" : "issue-tracker", + "url" : "https://github.com/JodaOrg/joda-time/issues" + }, + { + "type" : "mailing-list", + "url" : "http://sourceforge.net/mailarchive/forum.php?forum_name=joda-interest" + }, + { + "type" : "vcs", + "url" : "https://github.com/JodaOrg/joda-time" + } + ], + "type" : "library", + "bom-ref" : "pkg:maven/joda-time/joda-time@2.9.2?type=jar" + } + ], + "dependencies" : [ + { + "ref" : "pkg:maven/org.springframework/gs-maven@0.1.0?type=jar", + "dependsOn" : [ + "pkg:maven/joda-time/joda-time@2.9.2?type=jar" + ] + }, + { + "ref" : "pkg:maven/joda-time/joda-time@2.9.2?type=jar", + "dependsOn" : [ ] + } + ] +} \ No newline at end of file diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/dependency-reduced-pom.xml b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/dependency-reduced-pom.xml new file mode 100644 index 00000000..991e7487 --- /dev/null +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/dependency-reduced-pom.xml @@ -0,0 +1,53 @@ + + + 4.0.0 + org.springframework + gs-maven + 0.1.0 + + + + org.cyclonedx + cyclonedx-maven-plugin + 2.5.3 + + + maven-shade-plugin + 3.2.4 + + + package + + shade + + + + + hello.HelloWorld + + + + + + + + + + + junit + junit + 4.12 + test + + + hamcrest-core + org.hamcrest + + + + + + 1.8 + 1.8 + + diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 95cdcdcb..3347caae 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -33,11 +33,11 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) List configFiles=new(); if (string.IsNullOrEmpty(appSettings.CycloneDxBomFilePath)) { - configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Npm); + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Maven); } else { - configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Npm); + configFiles = FolderScanner.FileScanner(appSettings.CycloneDxBomFilePath, appSettings.Maven); } foreach (string filepath in configFiles) @@ -49,7 +49,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) { component.Properties = new List(); Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; - Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Manually Added" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = "Manually" }; component.Properties.Add(isDev); component.Properties.Add(identifierType); From 5f145b335009e85118a25fb28ad1abeed9bb7134 Mon Sep 17 00:00:00 2001 From: karthika Date: Tue, 4 Jul 2023 16:24:08 +0530 Subject: [PATCH 10/14] ununsed method removal --- src/LCT.PackageIdentifier/BomCreator.cs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/LCT.PackageIdentifier/BomCreator.cs b/src/LCT.PackageIdentifier/BomCreator.cs index d04f690e..d82a59a9 100644 --- a/src/LCT.PackageIdentifier/BomCreator.cs +++ b/src/LCT.PackageIdentifier/BomCreator.cs @@ -86,21 +86,7 @@ private static void WritecontentsToBOM(CommonAppSettings appSettings, BomKpiData } - private static void WriteContentToCycloneDxBOM(CommonAppSettings appSettings, Bom listOfComponentsToBom, ref BomKpiData bomKpiData) - { - IFileOperations fileOperations = new FileOperations(); - if (string.IsNullOrEmpty(appSettings.IdentifierBomFilePath)) - { - fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.SW360ProjectName); - } - else - { - listOfComponentsToBom = fileOperations.CombineComponentsFromExistingBOM(listOfComponentsToBom, appSettings.IdentifierBomFilePath); - bomKpiData.ComponentsInComparisonBOM = listOfComponentsToBom.Components.Count; - fileOperations.WriteContentToCycloneDXFile(listOfComponentsToBom, appSettings.BomFolderPath, appSettings.SW360ProjectName); - } - } private static void WriteContentToComparisonBOM(CommonAppSettings appSettings, Bom listOfComponentsToBom, ref BomKpiData bomKpiData) { From 2e4266305ef3b09e5742d5ea8736f620c391c4ca Mon Sep 17 00:00:00 2001 From: sumanthkb44 <84563853+sumanthkb44@users.noreply.github.com> Date: Thu, 6 Jul 2023 12:24:07 +0530 Subject: [PATCH 11/14] Update CA_UsageDocument.md --- doc/UsageDoc/CA_UsageDocument.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/UsageDoc/CA_UsageDocument.md b/doc/UsageDoc/CA_UsageDocument.md index 29fa1ebf..4a20f714 100644 --- a/doc/UsageDoc/CA_UsageDocument.md +++ b/doc/UsageDoc/CA_UsageDocument.md @@ -46,7 +46,7 @@ # Introduction -The Continuous Clearing Tool helps the Project Manager/Developer to automate the sw360 clearing process of 3rd party components. This tool scans and identifies the third-party components used in a NPM, NUGET, MAVEN and Debian projects and makes an entry in SW360, if it is not present. Continuous Clearing Tool links the components to the respective project and creates job for code scan in FOSSology. +The Continuous Clearing Tool helps the Project Manager/Developer to automate the sw360 clearing process of 3rd party components. This tool scans and identifies the third-party components used in a NPM, NUGET, MAVEN and Debian projects and makes an entry in SW360, if it is not present. Continuous Clearing Tool links the components to the respective project and creates job for code scan in FOSSology.. Continuous Clearing Tool reduces the effort in creating components in SW360 and identifying the matching source codes from the public repository. Tool eliminates the manual error while creating component and identifying correct version of source code from public repository. Continuous Clearing Tool harmonize the creation of 3P components in SW360 by filling necessary information. From 7c2e0f46b5eb532ecde76a2785f479304d044295 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Thu, 6 Jul 2023 12:35:33 +0530 Subject: [PATCH 12/14] Npm Change --- src/LCT.PackageIdentifier/NpmProcessor.cs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index 039d1d7f..32139ffb 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -353,20 +353,7 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List Date: Thu, 6 Jul 2023 12:37:58 +0530 Subject: [PATCH 13/14] Update CA_UsageDocument.md --- doc/UsageDoc/CA_UsageDocument.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/UsageDoc/CA_UsageDocument.md b/doc/UsageDoc/CA_UsageDocument.md index 4a20f714..29fa1ebf 100644 --- a/doc/UsageDoc/CA_UsageDocument.md +++ b/doc/UsageDoc/CA_UsageDocument.md @@ -46,7 +46,7 @@ # Introduction -The Continuous Clearing Tool helps the Project Manager/Developer to automate the sw360 clearing process of 3rd party components. This tool scans and identifies the third-party components used in a NPM, NUGET, MAVEN and Debian projects and makes an entry in SW360, if it is not present. Continuous Clearing Tool links the components to the respective project and creates job for code scan in FOSSology.. +The Continuous Clearing Tool helps the Project Manager/Developer to automate the sw360 clearing process of 3rd party components. This tool scans and identifies the third-party components used in a NPM, NUGET, MAVEN and Debian projects and makes an entry in SW360, if it is not present. Continuous Clearing Tool links the components to the respective project and creates job for code scan in FOSSology. Continuous Clearing Tool reduces the effort in creating components in SW360 and identifying the matching source codes from the public repository. Tool eliminates the manual error while creating component and identifying correct version of source code from public repository. Continuous Clearing Tool harmonize the creation of 3P components in SW360 by filling necessary information. From e240dcb300efda01da9632a949949ca314573b43 Mon Sep 17 00:00:00 2001 From: karthika Date: Fri, 7 Jul 2023 16:24:54 +0530 Subject: [PATCH 14/14] Updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 4fc1c884..057a3740 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ - # Introduction The Continuous Clearing Tool scans and collects the 3rd party OSS components used in a NPM/NuGet/Maven/Debian project and uploads it to SW360 and Fossology by accepting respective project ID for license clearing.