From 2c3c20e0f9f8e7d0220923cb938b2195d7a5c0d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Spie=C3=9F?= Date: Tue, 26 Mar 2024 19:13:45 +0000 Subject: [PATCH 1/7] Update expression normalization to support LicenseRef --- lib/utils.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 30f9cd3c2..12fcc0492 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -416,20 +416,20 @@ function joinExpressions(expressions) { return SPDX.normalize(joinedExpressionString) } -function normalizeLicenseExpression(licenseExpression, logger) { - if (!licenseExpression) return null +function normalizeLicenseExpression(rawLicenseExpression, logger) { + if (!rawLicenseExpression) return null - const licenseVisitor = rawLicenseExpression => { - const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression) - const licenseExpression = mappedLicenseExpression ? mappedLicenseExpression : rawLicenseExpression - - return SPDX.normalizeSingle(licenseExpression) + const licenseVisitor = licenseExpression => { + return scancodeMap.get(licenseExpression) || SPDX.normalizeSingle(licenseExpression) } - const parsed = SPDX.parse(licenseExpression, licenseVisitor) + // parse() checks for LicenseRef- and other special types of expressions before calling the visitor + // therefore use the mapped license expression as an argument if it was found + const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression) + const parsed = SPDX.parse(mappedLicenseExpression || rawLicenseExpression || '', licenseVisitor) const result = SPDX.stringify(parsed) - if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${licenseExpression}`) + if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${rawLicenseExpression}`) return result } From af5d8a44312d4acf29e54887522cd741db5ced96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Spie=C3=9F?= Date: Thu, 30 May 2024 14:42:23 +0000 Subject: [PATCH 2/7] Update expected test result --- test/providers/summary/scancode/new-summarizer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/providers/summary/scancode/new-summarizer.js b/test/providers/summary/scancode/new-summarizer.js index a2ac10b60..273039207 100644 --- a/test/providers/summary/scancode/new-summarizer.js +++ b/test/providers/summary/scancode/new-summarizer.js @@ -58,7 +58,7 @@ describe('ScanCodeNewSummarizer basic compatability', () => { const coordinates = { type: 'pypi', provider: 'pypi' } const harvestData = getHarvestData(scancodeVersion, 'pypi-complex-declared-license') const result = summarizer.summarize(coordinates, harvestData) - assert.equal(result.licensed.declared, 'HPND') + assert.equal(result.licensed.declared, 'LicenseRef-scancode-secret-labs-2011') } }) From ace7e0526623b0b8af82e71225ce1f44a34aad07 Mon Sep 17 00:00:00 2001 From: "E. Lynette Rayle" Date: Thu, 8 Aug 2024 17:13:42 -0400 Subject: [PATCH 3/7] handle complex licenses with scancode LicenseRefs --- lib/utils.js | 27 ++++++++++++++++++++++++++- test/lib/util.js | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/lib/utils.js b/lib/utils.js index 12fcc0492..4c641b4c5 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -427,13 +427,38 @@ function normalizeLicenseExpression(rawLicenseExpression, logger) { // therefore use the mapped license expression as an argument if it was found const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression) const parsed = SPDX.parse(mappedLicenseExpression || rawLicenseExpression || '', licenseVisitor) - const result = SPDX.stringify(parsed) + // normalize the parsed license expression will recursively normalize the parsed license expression + const normalizedParsed = _normalizeParsedLicenseExpression(parsed, logger) + + const result = SPDX.stringify(normalizedParsed) if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${rawLicenseExpression}`) return result } +function _normalizeParsedLicenseExpression(parsedLicenseExpression, logger) { + if (parsedLicenseExpression.left) { + if (parsedLicenseExpression.left.hasOwnProperty('left')) { + parsedLicenseExpression.left = _normalizeParsedLicenseExpression(parsedLicenseExpression.left, logger) + } else if (parsedLicenseExpression.left.hasOwnProperty('noassertion')) { + const new_left = normalizeLicenseExpression(parsedLicenseExpression.left['noassertion'], logger) + if (new_left.toLowerCase() === 'noassertion') parsedLicenseExpression.right = { 'noassertion': new_left } + else parsedLicenseExpression.left = { license: new_left } + } + } + if (parsedLicenseExpression.right) { + if (parsedLicenseExpression.right.hasOwnProperty('left')) { + parsedLicenseExpression.right = _normalizeParsedLicenseExpression(parsedLicenseExpression.right, logger) + } else if (parsedLicenseExpression.right.hasOwnProperty('noassertion')) { + const new_right = normalizeLicenseExpression(parsedLicenseExpression.right['noassertion'], logger) + if (new_right.toLowerCase() === 'noassertion') parsedLicenseExpression.right = { 'noassertion': new_right } + else parsedLicenseExpression.right = { license: new_right } + } + } + return parsedLicenseExpression +} + function _normalizeVersion(version) { if (version == '1') return '1.0.0' // version '1' is not semver valid see https://github.com/clearlydefined/crawler/issues/124 return semver.valid(version) ? version : null diff --git a/test/lib/util.js b/test/lib/util.js index f4bb1d3b7..2ab773ccb 100644 --- a/test/lib/util.js +++ b/test/lib/util.js @@ -880,3 +880,45 @@ describe('Utils buildSourceUrl', () => { expect(result).to.eq('https://pypi.org/project/zuul/3.3.0/') }) }) + +describe('normalizeLicenseExpression', () => { + it('should normalize license', () => { + const expression = 'MIT AND GPL-3.0' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('MIT AND GPL-3.0') + }) + it('should normalize single licenseRef', () => { + const expression = 'afpl-9.0' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('LicenseRef-scancode-afpl-9.0') + }) + it('should normalize license and licenseRef', () => { + const expression = 'afl-1.1 AND afpl-9.0' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('AFL-1.1 AND LicenseRef-scancode-afpl-9.0') + }) + it('should normalize licenseRef and license', () => { + const expression = 'afpl-9.0 AND MIT' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND MIT') + }) + it('should normalize licenseRef and licenseRef', () => { + const expression = 'afpl-9.0 AND activestate-community' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community') + }) + it('should normalize licenseRef and licenseRef or licenseRef', () => { + const expression = 'afpl-9.0 AND activestate-community OR ac3filter' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community OR LicenseRef-scancode-ac3filter') + }) + it('should normalize INVALID to NOASSERTION', () => { + const mockLogger = { + info: (message) => { + console.log(message); + } + }; const expression = 'INVALID' + const result = utils.normalizeLicenseExpression(expression, mockLogger) + expect(result).to.eq('NOASSERTION') + }) +}) From 4fbdc2c50e6d77a6c82414f2b6f8762db3ef0cac Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Wed, 16 Oct 2024 16:24:02 -0600 Subject: [PATCH 4/7] Use licenseRefLookup to normalize license expression (#1202) --- lib/utils.js | 45 ++++--------------- .../summary/scancode/legacy-summarizer.js | 4 +- 2 files changed, 11 insertions(+), 38 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 4c641b4c5..8b2709368 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -416,49 +416,22 @@ function joinExpressions(expressions) { return SPDX.normalize(joinedExpressionString) } -function normalizeLicenseExpression(rawLicenseExpression, logger) { +function normalizeLicenseExpression( + rawLicenseExpression, + logger, + licenseRefLookup = token => token && scancodeMap.get(token) +) { if (!rawLicenseExpression) return null - const licenseVisitor = licenseExpression => { - return scancodeMap.get(licenseExpression) || SPDX.normalizeSingle(licenseExpression) - } - - // parse() checks for LicenseRef- and other special types of expressions before calling the visitor - // therefore use the mapped license expression as an argument if it was found - const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression) - const parsed = SPDX.parse(mappedLicenseExpression || rawLicenseExpression || '', licenseVisitor) - - // normalize the parsed license expression will recursively normalize the parsed license expression - const normalizedParsed = _normalizeParsedLicenseExpression(parsed, logger) - - const result = SPDX.stringify(normalizedParsed) + const licenseVisitor = licenseExpression => + scancodeMap.get(licenseExpression) || SPDX.normalizeSingle(licenseExpression) + const parsed = SPDX.parse(rawLicenseExpression, licenseVisitor, licenseRefLookup) + const result = SPDX.stringify(parsed) if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${rawLicenseExpression}`) return result } -function _normalizeParsedLicenseExpression(parsedLicenseExpression, logger) { - if (parsedLicenseExpression.left) { - if (parsedLicenseExpression.left.hasOwnProperty('left')) { - parsedLicenseExpression.left = _normalizeParsedLicenseExpression(parsedLicenseExpression.left, logger) - } else if (parsedLicenseExpression.left.hasOwnProperty('noassertion')) { - const new_left = normalizeLicenseExpression(parsedLicenseExpression.left['noassertion'], logger) - if (new_left.toLowerCase() === 'noassertion') parsedLicenseExpression.right = { 'noassertion': new_left } - else parsedLicenseExpression.left = { license: new_left } - } - } - if (parsedLicenseExpression.right) { - if (parsedLicenseExpression.right.hasOwnProperty('left')) { - parsedLicenseExpression.right = _normalizeParsedLicenseExpression(parsedLicenseExpression.right, logger) - } else if (parsedLicenseExpression.right.hasOwnProperty('noassertion')) { - const new_right = normalizeLicenseExpression(parsedLicenseExpression.right['noassertion'], logger) - if (new_right.toLowerCase() === 'noassertion') parsedLicenseExpression.right = { 'noassertion': new_right } - else parsedLicenseExpression.right = { license: new_right } - } - } - return parsedLicenseExpression -} - function _normalizeVersion(version) { if (version == '1') return '1.0.0' // version '1' is not semver valid see https://github.com/clearlydefined/crawler/issues/124 return semver.valid(version) ? version : null diff --git a/providers/summary/scancode/legacy-summarizer.js b/providers/summary/scancode/legacy-summarizer.js index a0e74c481..82322c31f 100644 --- a/providers/summary/scancode/legacy-summarizer.js +++ b/providers/summary/scancode/legacy-summarizer.js @@ -83,7 +83,7 @@ class ScanCodeLegacySummarizer { _readLicenseExpressionFromSummary(harvested) { const licenseExpression = get(harvested, 'content.summary.packages[0].license_expression') - const result = licenseExpression && normalizeLicenseExpression(licenseExpression, this.logger) + const result = licenseExpression && normalizeLicenseExpression(licenseExpression, this.logger, null) return result?.includes('NOASSERTION') ? null : result } @@ -196,7 +196,7 @@ class ScanCodeLegacySummarizer { _createExpressionFromLicense(license) { const rule = license.matched_rule if (!rule || !rule.license_expression) return SPDX.normalize(license.spdx_license_key) - return normalizeLicenseExpression(rule.license_expression, this.logger) + return normalizeLicenseExpression(rule.license_expression, this.logger, null) } } From 8942c16caf287925ebfeafeb0bf2e1269055049c Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 17 Oct 2024 16:07:02 -0600 Subject: [PATCH 5/7] Update to use @clearlydefined/spdx version v0.1.9 Also updated spdx-license-list to the most recent version, 6.9.0, to be consistent with @clearlydefined/spdx --- package-lock.json | 15 +++++++-------- package.json | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7bb45a650..31771f271 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "MIT", "dependencies": { - "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.8", + "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.9", "@gitbeaker/node": "^29.2.4", "@octokit/rest": "^14.0.9", "ajv": "^6.12.3", @@ -55,7 +55,7 @@ "semver": "7.6.0", "serialize-error": "^2.1.0", "spdx-expression-parse": "github:clearlydefined/spdx-expression-parse.js#fork", - "spdx-license-list": "^6.6.0", + "spdx-license-list": "^6.9.0", "swagger-ui-express": "^4.0.1", "throat": "^4.1.0", "tiny-attribution-generator": "0.1.2", @@ -1622,12 +1622,12 @@ } }, "node_modules/@clearlydefined/spdx": { - "version": "0.1.8", - "resolved": "git+ssh://git@github.com/clearlydefined/spdx.git#b1cf4be4733ef148fe8abf9df77c614bf06fdc73", + "version": "0.1.9", + "resolved": "git+ssh://git@github.com/clearlydefined/spdx.git#93916093259bc8593400948b679b0dc32a5a12dd", "license": "MIT", "dependencies": { "spdx-expression-parse": "github:clearlydefined/spdx-expression-parse.js#fork", - "spdx-license-ids": "^3.0.18", + "spdx-license-ids": "^3.0.20", "spdx-license-list": "^6.9.0", "spdx-satisfies": "github:clearlydefined/spdx-satisfies.js#parse-override" } @@ -10101,12 +10101,11 @@ }, "node_modules/spdx-expression-parse": { "version": "3.0.0", - "resolved": "git+ssh://git@github.com/clearlydefined/spdx-expression-parse.js.git#cf496b97100705445e7cfb48b65a0c7587ea4672", - "integrity": "sha512-iZZpzTTKhXPYiXaXgFaUxm0WvZOr3q7GiUuKXbYuwlbjXpBXoNqbVYEhlt5Q66k+9cg63aGRXawP7tE6OtvOyg==", + "resolved": "git+ssh://git@github.com/clearlydefined/spdx-expression-parse.js.git#86fda6a3f84bb9da35d3536822804ac3fd1e587d", "license": "MIT", "dependencies": { "spdx-exceptions": "^2.1.0", - "spdx-license-ids": "^3.0.0" + "spdx-license-ids": "^3.0.13" } }, "node_modules/spdx-license-ids": { diff --git a/package.json b/package.json index 437d3a5c2..dc17f5211 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "url": "git+https://github.com/clearlydefined/service.git" }, "dependencies": { - "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.8", + "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.9", "@gitbeaker/node": "^29.2.4", "@octokit/rest": "^14.0.9", "ajv": "^6.12.3", @@ -66,7 +66,7 @@ "semver": "7.6.0", "serialize-error": "^2.1.0", "spdx-expression-parse": "github:clearlydefined/spdx-expression-parse.js#fork", - "spdx-license-list": "^6.6.0", + "spdx-license-list": "^6.9.0", "swagger-ui-express": "^4.0.1", "throat": "^4.1.0", "tiny-attribution-generator": "0.1.2", From 8335fba4e2d3b13893411bdfc2598c58da0bf510 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 17 Oct 2024 16:29:13 -0600 Subject: [PATCH 6/7] Fix formatting --- test/lib/util.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/lib/util.js b/test/lib/util.js index 2ab773ccb..7623643f0 100644 --- a/test/lib/util.js +++ b/test/lib/util.js @@ -910,14 +910,17 @@ describe('normalizeLicenseExpression', () => { it('should normalize licenseRef and licenseRef or licenseRef', () => { const expression = 'afpl-9.0 AND activestate-community OR ac3filter' const result = utils.normalizeLicenseExpression(expression) - expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community OR LicenseRef-scancode-ac3filter') + expect(result).to.eq( + 'LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community OR LicenseRef-scancode-ac3filter' + ) }) it('should normalize INVALID to NOASSERTION', () => { const mockLogger = { - info: (message) => { - console.log(message); + info: message => { + console.log(message) } - }; const expression = 'INVALID' + } + const expression = 'INVALID' const result = utils.normalizeLicenseExpression(expression, mockLogger) expect(result).to.eq('NOASSERTION') }) From bddf4a682d6da70159ae39cc21123149f853d0ce Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Fri, 25 Oct 2024 11:27:19 -0600 Subject: [PATCH 7/7] Update test/lib/util.js Co-authored-by: E. Lynette Rayle --- test/lib/util.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/lib/util.js b/test/lib/util.js index 7623643f0..12f62b58e 100644 --- a/test/lib/util.js +++ b/test/lib/util.js @@ -887,6 +887,15 @@ describe('normalizeLicenseExpression', () => { const result = utils.normalizeLicenseExpression(expression) expect(result).to.eq('MIT AND GPL-3.0') }) + it('should normalize license to SPDX equivalent', () => { + /* + NOTE: If this fails in tests for generated scancode map workflow PR, it is incorrect if it is expecting a LicenseRef. + There is an SPDX valid license which does not require a LicenseRef meaning this test is correct as is. + */ + const expression = 'net-snmp' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('Net-SNMP') + }) it('should normalize single licenseRef', () => { const expression = 'afpl-9.0' const result = utils.normalizeLicenseExpression(expression)