diff --git a/biome.json b/biome.json index 6edc6f97..f7c5fce4 100644 --- a/biome.json +++ b/biome.json @@ -7,7 +7,8 @@ }, "files": { "ignoreUnknown": false, - "ignore": ["dist", "build"] + "ignore": ["dist", "build"], + "maxSize": 10000000 }, "formatter": { "enabled": true, diff --git a/package-lock.json b/package-lock.json index c5423a56..e1ee02d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7171,6 +7171,7 @@ "@license-auditor/data": "*", "@total-typescript/ts-reset": "0.6.1", "detect-package-manager": "3.0.2", + "env-paths": "3.0.0", "fast-glob": "3.3.2", "lodash.flattendeep": "4.4.0", "spdx-expression-parse": "4.0.0", @@ -7187,6 +7188,18 @@ "vitest": "2.1.5" } }, + "packages/core/node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "packages/core/node_modules/spdx-expression-parse": { "version": "4.0.0", "license": "MIT", @@ -7211,6 +7224,7 @@ "version": "2.0.0-beta", "license": "MIT", "dependencies": { + "env-paths": "3.0.0", "zod": "3.23.8" }, "devDependencies": { @@ -7219,6 +7233,18 @@ "typescript": "5.6.2" } }, + "tooling/data/node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "tooling/typescript-config": { "name": "@license-auditor/typescript-config", "version": "0.1.0", diff --git a/packages/cli/src/commands/index.tsx b/packages/cli/src/commands/index.tsx index 3029c313..59e9ada7 100644 --- a/packages/cli/src/commands/index.tsx +++ b/packages/cli/src/commands/index.tsx @@ -22,7 +22,7 @@ export const options = z.object({ production: z .boolean() .describe(`Don't check licenses in development dependencies`), - defaultConfig: z // pacsalCase options are converted to kebab-case, so the flag is actually --default-config + defaultConfig: z // camelCase options are converted to kebab-case, so the flag is actually --default-config .boolean() .describe("Run audit with default whitelist/blacklist configuration"), filterRegex: z diff --git a/packages/cli/src/commands/update-licenses.tsx b/packages/cli/src/commands/update-licenses.tsx new file mode 100644 index 00000000..a507d3be --- /dev/null +++ b/packages/cli/src/commands/update-licenses.tsx @@ -0,0 +1,39 @@ +import { deleteLicenses, updateLicenses } from "@license-auditor/data"; +import { Text, useApp } from "ink"; +import { useEffect, useState } from "react"; +import { z } from "zod"; +import { SpinnerWithLabel } from "../components/spinner-with-label.js"; + +export const options = z.object({ + clearCache: z.boolean().describe("Compress output"), +}); + +type Props = { + options: z.infer; +}; + +export default function UpdateLicenses({ options }: Props) { + const { exit } = useApp(); + const [working, setWorking] = useState(false); + useEffect(() => { + setWorking(true); + + const runUpdate = async () => { + if (options.clearCache) { + deleteLicenses(); + } else { + await updateLicenses({ fetchAllLicenseTexts: true }); + } + setWorking(false); + exit(); + }; + + void runUpdate(); + }, [options, exit]); + + if (working) { + return ; + } + + return Licenses updated!; +} diff --git a/packages/core/package.json b/packages/core/package.json index 7be30b22..536bbd60 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -26,6 +26,7 @@ "@license-auditor/data": "*", "@total-typescript/ts-reset": "0.6.1", "detect-package-manager": "3.0.2", + "env-paths": "3.0.0", "fast-glob": "3.3.2", "lodash.flattendeep": "4.4.0", "spdx-expression-parse": "4.0.0", diff --git a/packages/core/src/license-finder/detect-from-license-content.test.ts b/packages/core/src/license-finder/detect-from-license-content.test.ts index bdcab6e4..f47c1ed2 100644 --- a/packages/core/src/license-finder/detect-from-license-content.test.ts +++ b/packages/core/src/license-finder/detect-from-license-content.test.ts @@ -10,13 +10,13 @@ describe("detectFromLicenseContent", () => { }); }); describe("detectFromLicenseContent", () => { - it("detects license from license content", () => { + it("detects license from license content", async () => { const licenseContents = licenseMap.get("MIT")?.licenseText; if (!licenseContents) { throw new Error("MIT doesn't have license text"); } - expect(detectLicenses(licenseContents)[0]?.licenseId).toBe("MIT"); + expect((await detectLicenses(licenseContents))[0]?.licenseId).toBe("MIT"); }); }); }); diff --git a/packages/core/src/license-finder/detect-from-license-content.ts b/packages/core/src/license-finder/detect-from-license-content.ts index c92c8fe4..aaafcf46 100644 --- a/packages/core/src/license-finder/detect-from-license-content.ts +++ b/packages/core/src/license-finder/detect-from-license-content.ts @@ -1,4 +1,28 @@ -import { type LicenseId, licenses } from "@license-auditor/data"; +import { existsSync } from "node:fs"; +import type { + LicenseId, + // licenses as defaultLicenses, +} from "@license-auditor/data"; +import envPaths from "env-paths"; + +const resolveLicenses = async (): Promise< + { licenseId: string; licenseText: string }[] +> => { + const paths = envPaths("license-auditor"); + const licensesPath = `${paths.cache}/licenses.js`; + + if (existsSync(licensesPath)) { + const fullLicenses = (await import(licensesPath)) as { + licenses: { licenseId: string; licenseText: string }[]; + }; + + return fullLicenses.licenses; + } + + throw Error("Licenses not found"); + + // return defaultLicenses; +}; /** * Tokenizes text into words, removes punctuation and whitespaces @@ -49,19 +73,22 @@ const createLibrary = ( ); }; -const licensesLibrary = createLibrary( - licenses.reduce>((acc, license) => { - if (license.licenseText) { - acc[license.licenseId as LicenseId] = license.licenseText; - } - return acc; - }, {}), -); -const calculateSimilarity = getCalculateSimilarity(licensesLibrary); - -export function detectLicenses( +export async function detectLicenses( licenseContent: string, -): { licenseId: LicenseId; similarity: number }[] { +): Promise<{ licenseId: LicenseId; similarity: number }[]> { + const licenses = await resolveLicenses(); + + const licensesLibrary = createLibrary( + licenses.reduce>((acc, license) => { + if (license.licenseText) { + acc[license.licenseId as LicenseId] = license.licenseText; + } + return acc; + }, {}), + ); + + const calculateSimilarity = getCalculateSimilarity(licensesLibrary); + const similarities = calculateSimilarity(licenseContent); return similarities.sort((a, b) => b.similarity - a.similarity); } diff --git a/packages/core/src/license-finder/find-license-in-license-file.test.ts b/packages/core/src/license-finder/find-license-in-license-file.test.ts index eed4676d..54f8eab3 100644 --- a/packages/core/src/license-finder/find-license-in-license-file.test.ts +++ b/packages/core/src/license-finder/find-license-in-license-file.test.ts @@ -3,19 +3,19 @@ import { describe, expect, it } from "vitest"; import { retrieveLicenseFromLicenseFileContent } from "./find-license-in-license-file.js"; describe("retrieveLicenseFromLicenseFileContent", () => { - it("should return an empty array when content does not match any licenses", () => { + it("should return an empty array when content does not match any licenses", async () => { const content = "This is some random content without any license keywords."; - const result = retrieveLicenseFromLicenseFileContent( + const result = await retrieveLicenseFromLicenseFileContent( content, "/path/to/LICENSE", ); expect(result.licenses).toEqual([]); }); - it("should return the correct license when content matches a license key", () => { + it("should return the correct license when content matches a license key", async () => { const content = "MIT"; const expectedLicense = LicenseSchema.parse(licenseMap.get("MIT")); - const result = retrieveLicenseFromLicenseFileContent( + const result = await retrieveLicenseFromLicenseFileContent( content, "/path/to/LICENSE", ); @@ -28,10 +28,10 @@ describe("retrieveLicenseFromLicenseFileContent", () => { ]); }); - it("should return the correct license when content matches a license name", () => { + it("should return the correct license when content matches a license name", async () => { const content = "MIT License"; const expectedLicense = LicenseSchema.parse(licenseMap.get("MIT")); - const result = retrieveLicenseFromLicenseFileContent( + const result = await retrieveLicenseFromLicenseFileContent( content, "/path/to/LICENSE", ); @@ -44,7 +44,7 @@ describe("retrieveLicenseFromLicenseFileContent", () => { ]); }); - it("should return multiple licenses when content matches multiple license keys or names", () => { + it("should return multiple licenses when content matches multiple license keys or names", async () => { const content = "MIT, Apache-2.0"; const expectedLicenses = [ { @@ -58,7 +58,7 @@ describe("retrieveLicenseFromLicenseFileContent", () => { licensePath: "/path/to/LICENSE", }, ].sort((a, b) => a.name.localeCompare(b.name)); - const result = retrieveLicenseFromLicenseFileContent( + const result = await retrieveLicenseFromLicenseFileContent( content, "/path/to/LICENSE", ); diff --git a/packages/core/src/license-finder/find-license-in-license-file.ts b/packages/core/src/license-finder/find-license-in-license-file.ts index 8a61434f..809b5b1c 100644 --- a/packages/core/src/license-finder/find-license-in-license-file.ts +++ b/packages/core/src/license-finder/find-license-in-license-file.ts @@ -10,13 +10,13 @@ import { addLicenseSource } from "./add-license-source.js"; import { detectLicenses } from "./detect-from-license-content.js"; import type { LicensesWithPathAndStatus } from "./licenses-with-path.js"; -export function retrieveLicenseFromLicenseFileContent( +export async function retrieveLicenseFromLicenseFileContent( content: string, licensePath: string, -): { +): Promise<{ licenses: LicenseWithSource[]; -} { - const detectedLicenses = detectLicenses(content); +}> { + const detectedLicenses = await detectLicenses(content); const detectedLicense = detectedLicenses[0]; if (detectedLicense && (detectedLicense.similarity ?? 0) > 0.75) { // threshold selected empirically based on our tests @@ -62,7 +62,10 @@ export async function findLicenseInLicenseFile(filePath: string): Promise<{ }; } - const result = retrieveLicenseFromLicenseFileContent(content, filePath); + const result = await retrieveLicenseFromLicenseFileContent( + content, + filePath, + ); return { licenses: result.licenses, diff --git a/tooling/data/package.json b/tooling/data/package.json index 04060a02..5079248b 100644 --- a/tooling/data/package.json +++ b/tooling/data/package.json @@ -13,6 +13,7 @@ "update-licenses": "tsx src/licenses/update-licenses.ts" }, "dependencies": { + "env-paths": "3.0.0", "zod": "3.23.8" }, "devDependencies": { diff --git a/tooling/data/src/licenses/index.ts b/tooling/data/src/licenses/index.ts index ae4246d2..88e13c85 100644 --- a/tooling/data/src/licenses/index.ts +++ b/tooling/data/src/licenses/index.ts @@ -1,3 +1,4 @@ export * from "./schemas.js"; export * from "./types.js"; export * from "./constants.js"; +export * from "./update-licenses.js"; diff --git a/tooling/data/src/licenses/update-licenses.ts b/tooling/data/src/licenses/update-licenses.ts index 48d936ac..665ebf97 100644 --- a/tooling/data/src/licenses/update-licenses.ts +++ b/tooling/data/src/licenses/update-licenses.ts @@ -1,8 +1,11 @@ -import { writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, unlinkSync, writeFileSync } from "node:fs"; +import envPaths from "env-paths"; +const paths = envPaths("license-auditor"); +const licensesFilePath = `${paths.cache}/licenses.js`; const url = "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json"; -const outputFile = "./src/licenses/licenses.ts"; +// const defaultOutputFile = "./src/licenses/licenses.ts"; // licenses are chosen arbitrarily, based on their popularity in our projects const licensesToFetchContentFor = [ @@ -26,9 +29,13 @@ const licensesToFetchContentFor = [ "MPL-2.0", ]; -// pulls the licenses from spdx and transforms them into an object -// needed so TS can properly infer types in union types -(async () => { +export async function updateLicenses({ + outputFile, + fetchAllLicenseTexts, +}: { + outputFile?: string; + fetchAllLicenseTexts?: boolean; +}) { try { console.log("Fetching license list..."); const response = await fetch(url); @@ -47,6 +54,7 @@ const licensesToFetchContentFor = [ ); try { if ( + fetchAllLicenseTexts || licensesToFetchContentFor.includes( // biome-ignore lint/style/noNonNullAssertion: we can be sure that the licenses field is a dense array licensesData.licenses[i]!.licenseId, @@ -65,21 +73,43 @@ const licensesToFetchContentFor = [ } } catch (error) { console.log( - // biome-ignore lint/style/noNonNullAssertion: we can be sure that the licenses field is a dense array - `Failed to fetch license contents for "${licensesData.licenses[i]!.licenseId}"`, + `Failed to fetch license contents for "${ + // biome-ignore lint/style/noNonNullAssertion: we can be sure that the licenses field is a dense array + licensesData.licenses[i]!.licenseId + }"`, ); failedFetches++; } } - const content = `export const licensesData = ${JSON.stringify(licensesData, null, 2)} as const;`; + const content = `export const licensesData = ${JSON.stringify( + licensesData, + null, + 2, + )};`; + + if (!(outputFile || existsSync(paths.cache))) { + mkdirSync(paths.cache, { recursive: true }); + } - writeFileSync(outputFile, content); + writeFileSync(outputFile || licensesFilePath, content); console.log( - `licenses.ts has been updated.${failedFetches ? ` ${failedFetches} licenses failed to fetch.` : ""}`, + `licenses.ts has been updated.${ + failedFetches ? ` ${failedFetches} licenses failed to fetch.` : "" + }`, ); } catch (error) { console.error("Error fetching licenses:", error); process.exit(1); } -})(); +} + +export function deleteLicenses() { + if (existsSync(licensesFilePath)) { + unlinkSync(licensesFilePath); + } +} + +// pulls the licenses from spdx and transforms them into an object +// needed so TS can properly infer types in union types +// (async () => updateLicenses({ outputFile: defaultOutputFile }))();