From 457b0309cefb58fae0dcb2e4c06de9bd7582967f Mon Sep 17 00:00:00 2001 From: nicole mazzuca <83086508+strega-nil-ms@users.noreply.github.com> Date: Tue, 1 Feb 2022 14:43:06 -0800 Subject: [PATCH] [manifest license] fully implement SPDX, plus general parsing stuff (#334) * [ParserBase] get ready for parsing SPDX license expressions * parse SPDX license expressions * add error message tests * more tests, plus minor error change * format Co-authored-by: nicole mazzuca --- Generate-SpdxLicenseList.ps1 | 9 +- include/vcpkg/base/messages.h | 15 + include/vcpkg/base/optional.h | 1 + include/vcpkg/base/parse.h | 51 +- include/vcpkg/base/unicode.h | 41 +- include/vcpkg/base/util.h | 12 +- include/vcpkg/fwd/configuration.h | 1 + include/vcpkg/platform-expression.h | 2 +- include/vcpkg/sourceparagraph.h | 13 +- locales/messages.json | 23 + src/vcpkg-test/json.cpp | 40 +- src/vcpkg-test/manifests.cpp | 305 ++++++--- src/vcpkg/base/parse.cpp | 86 ++- src/vcpkg/base/unicode.cpp | 169 +++-- src/vcpkg/binarycaching.cpp | 7 +- src/vcpkg/sourceparagraph.cpp | 412 +++++++++--- src/vcpkg/spdx-exceptions.inc | 86 ++- src/vcpkg/spdx-licenses.inc | 954 ++++++++++++++-------------- 18 files changed, 1389 insertions(+), 838 deletions(-) diff --git a/Generate-SpdxLicenseList.ps1 b/Generate-SpdxLicenseList.ps1 index 92542bd418..2e8892c3ed 100644 --- a/Generate-SpdxLicenseList.ps1 +++ b/Generate-SpdxLicenseList.ps1 @@ -37,16 +37,15 @@ function Transform-JsonFile { $fileContent = @( "// Data downloaded from $Uri", - "// Generated by scripts/Generate-SpdxLicenseList.ps1", - "{") + "// Generated by Generate-SpdxLicenseList.ps1") $json.$OuterName | Sort-Object -Property $Id -Culture '' | ForEach-Object { - $fileContent += " `"$($_.$Id)`"," + $fileContent += "`"$($_.$Id)`"," } - $fileContent += "}" - $fileContent -join "`n" | Out-File -FilePath $OutFile -Encoding 'utf8' + ($fileContent -join "`n") + "`n" ` + | Out-File -FilePath $OutFile -Encoding 'utf8' -NoNewline } $baseUrl = "https://raw.githubusercontent.com/$GithubRepository/$Commit/json" diff --git a/include/vcpkg/base/messages.h b/include/vcpkg/base/messages.h index b1a66ee7e6..103a7378d6 100644 --- a/include/vcpkg/base/messages.h +++ b/include/vcpkg/base/messages.h @@ -55,6 +55,7 @@ namespace vcpkg::msg LocalizedString() = default; operator StringView() const { return m_data; } const std::string& data() const { return m_data; } + std::string extract_data() { return std::exchange(m_data, ""); } static LocalizedString from_string_unchecked(std::string&& s) { @@ -87,6 +88,8 @@ namespace vcpkg::msg } }; + inline const char* to_printf_arg(const msg::LocalizedString& s) { return s.data().c_str(); } + struct LocalizedStringMapLess { using is_transparent = void; @@ -110,6 +113,16 @@ namespace vcpkg::msg inline void print(Color c, const LocalizedString& s) { write_unlocalized_text_to_stdout(c, s); } inline void print(const LocalizedString& s) { write_unlocalized_text_to_stdout(Color::none, s); } + inline void println(Color c, const LocalizedString& s) + { + write_unlocalized_text_to_stdout(c, s); + write_unlocalized_text_to_stdout(Color::none, "\n"); + } + inline void println(const LocalizedString& s) + { + write_unlocalized_text_to_stdout(Color::none, s); + write_unlocalized_text_to_stdout(Color::none, "\n"); + } template void print(Message m, Ts... args) @@ -157,6 +170,8 @@ namespace vcpkg::msg DECLARE_MSG_ARG(version); DECLARE_MSG_ARG(list); DECLARE_MSG_ARG(output); + DECLARE_MSG_ARG(row); + DECLARE_MSG_ARG(column); #undef DECLARE_MSG_ARG // These are `...` instead of diff --git a/include/vcpkg/base/optional.h b/include/vcpkg/base/optional.h index d998f684f8..be49907b9d 100644 --- a/include/vcpkg/base/optional.h +++ b/include/vcpkg/base/optional.h @@ -400,6 +400,7 @@ namespace vcpkg return !rhs.m_base.has_value(); } + friend bool operator!=(const Optional& lhs, const Optional& rhs) noexcept { return !(lhs == rhs); } private: details::OptionalStorage m_base; diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index 9cb24680e2..9cbd890b7e 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -42,16 +43,36 @@ namespace vcpkg::Parse virtual const std::string& get_message() const override; }; - struct ParserBase + struct SourceLoc { - struct SourceLoc - { - Unicode::Utf8Decoder it; - Unicode::Utf8Decoder start_of_line; - int row; - int column; - }; + Unicode::Utf8Decoder it; + Unicode::Utf8Decoder start_of_line; + int row; + int column; + }; + enum class MessageKind + { + Warning, + Error, + }; + + struct ParseMessage + { + SourceLoc location = {}; + msg::LocalizedString message; + + msg::LocalizedString format(StringView origin, MessageKind kind) const; + }; + + struct ParseMessages + { + std::unique_ptr error; + std::vector warnings; + }; + + struct ParserBase + { ParserBase(StringView text, StringView origin, TextRowCol init_rowcol = {}); static constexpr bool is_whitespace(char32_t ch) { return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n'; } @@ -110,9 +131,17 @@ namespace vcpkg::Parse void add_error(std::string message) { add_error(std::move(message), cur_loc()); } void add_error(std::string message, const SourceLoc& loc); + void add_error(msg::LocalizedString&& message) { add_error(message.extract_data(), cur_loc()); } + void add_error(msg::LocalizedString&& message, const SourceLoc& loc) { add_error(message.extract_data(), loc); } + + void add_warning(msg::LocalizedString&& message) { add_warning(std::move(message), cur_loc()); } + void add_warning(msg::LocalizedString&& message, const SourceLoc& loc); + + const IParseError* get_error() const { return m_messages.error.get(); } + std::unique_ptr extract_error() { return std::move(m_messages.error); } - const Parse::IParseError* get_error() const { return m_err.get(); } - std::unique_ptr extract_error() { return std::move(m_err); } + const ParseMessages& messages() const { return m_messages; } + ParseMessages extract_messages() { return std::move(m_messages); } private: Unicode::Utf8Decoder m_it; @@ -123,6 +152,6 @@ namespace vcpkg::Parse StringView m_text; StringView m_origin; - std::unique_ptr m_err; + ParseMessages m_messages; }; } diff --git a/include/vcpkg/base/unicode.h b/include/vcpkg/base/unicode.h index bfde0bcf07..80a4a507d3 100644 --- a/include/vcpkg/base/unicode.h +++ b/include/vcpkg/base/unicode.h @@ -14,12 +14,37 @@ namespace vcpkg::Unicode StartFour = 4, }; + constexpr static char32_t end_of_file = 0xFFFF'FFFF; + + enum class utf8_errc + { + NoError = 0, + InvalidCodeUnit = 1, + InvalidCodePoint = 2, + PairedSurrogates = 3, + UnexpectedContinue = 4, + UnexpectedStart = 5, + UnexpectedEof = 6, + }; + + const std::error_category& utf8_category() noexcept; + Utf8CodeUnitKind utf8_code_unit_kind(unsigned char code_unit) noexcept; int utf8_code_unit_count(Utf8CodeUnitKind kind) noexcept; int utf8_code_unit_count(char code_unit) noexcept; int utf8_encode_code_point(char (&array)[4], char32_t code_point) noexcept; + // returns {after-current-code-point, error}, + // and if error = NoError, then out = parsed code point. + // else, out = end_of_file. + std::pair utf8_decode_code_point(const char* first, + const char* last, + char32_t& out) noexcept; + + // uses the C++20 definition + bool is_double_width_code_point(char32_t ch) noexcept; + inline std::string& utf8_append_code_point(std::string& str, char32_t code_point) { if (static_cast(code_point) < 0x80) @@ -52,21 +77,6 @@ namespace vcpkg::Unicode char32_t utf16_surrogates_to_code_point(char32_t leading, char32_t trailing); - constexpr static char32_t end_of_file = 0xFFFF'FFFF; - - enum class utf8_errc - { - NoError = 0, - InvalidCodeUnit = 1, - InvalidCodePoint = 2, - PairedSurrogates = 3, - UnexpectedContinue = 4, - UnexpectedStart = 5, - UnexpectedEof = 6, - }; - - const std::error_category& utf8_category() noexcept; - inline std::error_code make_error_code(utf8_errc err) noexcept { return std::error_code(static_cast(err), utf8_category()); @@ -89,6 +99,7 @@ namespace vcpkg::Unicode struct Utf8Decoder { Utf8Decoder() noexcept; + explicit Utf8Decoder(StringView sv) : Utf8Decoder(sv.begin(), sv.end()) { } Utf8Decoder(const char* first, const char* last) noexcept; struct sentinel diff --git a/include/vcpkg/base/util.h b/include/vcpkg/base/util.h index 9c522910f8..88fd16b025 100644 --- a/include/vcpkg/base/util.h +++ b/include/vcpkg/base/util.h @@ -147,7 +147,17 @@ namespace vcpkg::Util { using std::begin; using std::end; - return std::find_if(begin(cont), end(cont), pred); + // allow cont.begin() to not have the same type as cont.end() + auto it = begin(cont); + auto last = end(cont); + for (; it != last; ++it) + { + if (pred(*it)) + { + break; + } + } + return it; } template diff --git a/include/vcpkg/fwd/configuration.h b/include/vcpkg/fwd/configuration.h index cddbbf024a..6be58b363c 100644 --- a/include/vcpkg/fwd/configuration.h +++ b/include/vcpkg/fwd/configuration.h @@ -4,4 +4,5 @@ namespace vcpkg { struct Configuration; struct RegistryConfig; + struct ManifestConfiguration; } diff --git a/include/vcpkg/platform-expression.h b/include/vcpkg/platform-expression.h index 5de13b7a24..423b60f4b1 100644 --- a/include/vcpkg/platform-expression.h +++ b/include/vcpkg/platform-expression.h @@ -77,6 +77,6 @@ namespace vcpkg::PlatformExpression }; // platform expression parses a platform expression; the EBNF of such is defined in - // /docs/maintainers/manifest-files.md#supports + // https://github.com/microsoft/vcpkg/blob/master/docs/maintainers/manifest-files.md#supports ExpectedS parse_platform_expression(StringView expression, MultipleBinaryOperators multiple_binary_operators); } diff --git a/include/vcpkg/sourceparagraph.h b/include/vcpkg/sourceparagraph.h index a9eae7c1be..825247c5a9 100644 --- a/include/vcpkg/sourceparagraph.h +++ b/include/vcpkg/sourceparagraph.h @@ -2,6 +2,7 @@ #include +#include #include #include @@ -71,7 +72,12 @@ namespace vcpkg std::vector dependencies; std::vector overrides; std::vector default_features; - std::string license; // SPDX license expression + + // there are two distinct "empty" states here + // "user did not provide a license" -> nullopt + // "user provided license = null" -> {""} + Optional license; // SPDX license expression + Optional builtin_baseline; Optional vcpkg_configuration; // Currently contacts is only a Json::Object but it will eventually be unified with maintainers @@ -127,8 +133,7 @@ namespace vcpkg Json::Object serialize_manifest(const SourceControlFile& scf); Json::Object serialize_debug_manifest(const SourceControlFile& scf); - ExpectedS parse_manifest_configuration(StringView origin, - const Json::Object& manifest); + ExpectedS parse_manifest_configuration(StringView origin, const Json::Object& manifest); /// /// Named pair of a SourceControlFile and the location of this file @@ -146,4 +151,6 @@ namespace vcpkg { return print_error_message({&error_info_list, 1}); } + + std::string parse_spdx_license_expression(StringView sv, Parse::ParseMessages& messages); } diff --git a/locales/messages.json b/locales/messages.json index 42f51634b0..5423c776f7 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -5,6 +5,7 @@ "AwsFailedToDownload": "aws failed to download with exit code: {value}\n{output}", "AwsRestoredPackages": "Restored {value} packages from AWS servers in {elapsed}s", "AwsUploadedPackages": "Uploaded binaries to {value} AWS servers", + "EmptyLicenseExpression": "SPDX license expression was empty.", "ErrorIndividualPackagesUnsupported": "Error: In manifest mode, `vcpkg install` does not support individual package arguments.\nTo install additional packages, edit vcpkg.json and then run `vcpkg install` without any package arguments.", "ErrorInvalidClassicModeOption": "Error: The option {value} is not supported in classic mode and no manifest was found.", "ErrorInvalidManifestModeOption": "Error: The option {value} is not supported in manifest mode.", @@ -21,8 +22,30 @@ "ErrorRequirePackagesToInstall": "Error: No packages were listed for installation and no manifest was found.", "ErrorVcvarsUnsupported": "Error: in triplet {triplet}: Use of Visual Studio's Developer Prompt is unsupported on non-Windows hosts.\nDefine 'VCPKG_CMAKE_SYSTEM_NAME' or 'VCPKG_CHAINLOAD_TOOLCHAIN_FILE' in the triplet file.", "ForceSystemBinariesOnWeirdPlatforms": "Environment variable VCPKG_FORCE_SYSTEM_BINARIES must be set on arm, s390x, and ppc64le platforms.", + "FormattedParseError": "error: {value}", + "FormattedParseMessageExpression": " on expression: {value}", + "FormattedParseMessageLocation": "{path}:{row}:{column}: ", + "_FormattedParseMessageLocation.comment": "{LOCKED}", + "FormattedParseWarning": "warning: {value}", "IllegalFeatures": "Error: List of features is not allowed in this contect", "IllegalPlatformSpec": "Error: Platform qualifier is not allowed in this context", + "LicenseExpressionContainsExtraPlus": "SPDX license expression contains an extra '+'. These are only allowed directly after a license identifier.", + "LicenseExpressionContainsInvalidCharacter": "SPDX license expression contains an invalid character (0x{value:02x} '{value}').", + "LicenseExpressionContainsUnicode": "SPDX license expression contains a unicode character (U+{value:04x} '{pretty_value}'), but these expressions are ASCII-only.", + "LicenseExpressionDocumentRefUnsupported": "The current implementation does not support DocumentRef- SPDX references.", + "LicenseExpressionExpectCompoundFoundParen": "Expected a compound or the end of the string, found a parenthesis.", + "LicenseExpressionExpectCompoundFoundWith": "Expected either AND or OR, found WITH (WITH is only allowed after license names, not parenthesized expressions).", + "LicenseExpressionExpectCompoundFoundWord": "Expected either AND or OR, found a license or exception name: '{value}'.", + "LicenseExpressionExpectCompoundOrWithFoundWord": "Expected either AND, OR, or WITH, found a license or exception name: '{value}'.", + "LicenseExpressionExpectExceptionFoundCompound": "Expected an exception name, found the compound {value}.", + "LicenseExpressionExpectExceptionFoundEof": "Expected an exception name, found the end of the string.", + "LicenseExpressionExpectExceptionFoundParen": "Expected an exception name, found a parenthesis.", + "LicenseExpressionExpectLicenseFoundCompound": "Expected a license name, found the compound {value}.", + "LicenseExpressionExpectLicenseFoundEof": "Expected a license name, found the end of the string.", + "LicenseExpressionExpectLicenseFoundParen": "Expected a license name, found a parenthesis.", + "LicenseExpressionImbalancedParens": "There was a close parenthesis without an opening parenthesis.", + "LicenseExpressionUnknownException": "Unknown license exception identifier '{value}'. Known values are listed at https://spdx.org/licenses/exceptions-index.html", + "LicenseExpressionUnknownLicense": "Unknown license identifier '{value}'. Known values are listed at https://spdx.org/licenses/", "NoLocalizationForMessages": "No localization for the following messages:", "ProcessorArchitectureMalformed": "Failed to parse %PROCESSOR_ARCHITECTURE% ({value}) as a valid CPU architecture.", "ProcessorArchitectureMissing": "The required environment variable %PROCESSOR_ARCHITECTURE% is missing.", diff --git a/src/vcpkg-test/json.cpp b/src/vcpkg-test/json.cpp index 71039b0d59..09c4c0819a 100644 --- a/src/vcpkg-test/json.cpp +++ b/src/vcpkg-test/json.cpp @@ -236,8 +236,8 @@ TEST_CASE ("JSON track newlines", "[json]") REQUIRE(!res); REQUIRE(res.error()->format() == R"(filename:2:1: error: Unexpected character; expected property name - on expression: , - ^ + on expression: , + ^ )"); } @@ -247,7 +247,39 @@ TEST_CASE ("JSON duplicated object keys", "[json]") REQUIRE(!res); REQUIRE(res.error()->format() == R"(filename:1:13: error: Duplicated key "name" in an object - on expression: {"name": 1, "name": 2} - ^ + on expression: {"name": 1, "name": 2} + ^ +)"); +} + +TEST_CASE ("JSON support unicode characters in errors", "[json]") +{ + // unicode characters w/ bytes >1 + auto res = Json::parse(R"json("Δx/Δt" "")json", "filename"); + REQUIRE(!res); + CHECK(res.error()->format() == + R"(filename:1:9: error: Unexpected character; expected EOF + on expression: "Δx/Δt" "" + ^ +)"); + + // full width unicode characters + // note that the A is full width + res = Json::parse(R"json("姐姐aＡ" "")json", "filename"); + REQUIRE(!res); + CHECK(res.error()->format() == + R"(filename:1:8: error: Unexpected character; expected EOF + on expression: "姐姐aＡ" "" + ^ +)"); + + // incorrect errors in the face of combining characters + // (this test should be fixed once the underlying bug is fixed) + res = Json::parse(R"json("é" "")json", "filename"); + REQUIRE(!res); + CHECK(res.error()->format() == + R"(filename:1:6: error: Unexpected character; expected EOF + on expression: "é" "" + ^ )"); } diff --git a/src/vcpkg-test/manifests.cpp b/src/vcpkg-test/manifests.cpp index db199649f2..460bac0557 100644 --- a/src/vcpkg-test/manifests.cpp +++ b/src/vcpkg-test/manifests.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -28,21 +29,40 @@ static Json::Object parse_json_object(StringView sv) } else { + vcpkg::print2("Error found while parsing JSON document:\n", sv, '\n'); Checks::exit_with_message(VCPKG_LINE_INFO, json.error()->format()); } } -static Parse::ParseExpected test_parse_manifest(StringView sv, bool expect_fail = false) +enum class PrintErrors : bool { - auto object = parse_json_object(sv); - auto res = SourceControlFile::parse_manifest_object("", object); - if (!res.has_value() && !expect_fail) + No, + Yes, +}; + +static Parse::ParseExpected test_parse_manifest(const Json::Object& obj, + PrintErrors print = PrintErrors::Yes) +{ + auto res = SourceControlFile::parse_manifest_object("", obj); + if (!res.has_value() && print == PrintErrors::Yes) { print_error_message(res.error()); } - REQUIRE(res.has_value() == !expect_fail); return res; } +static Parse::ParseExpected test_parse_manifest(StringView obj, PrintErrors print = PrintErrors::Yes) +{ + return test_parse_manifest(parse_json_object(obj), print); +} + +static bool manifest_is_parseable(const Json::Object& obj) +{ + return test_parse_manifest(obj, PrintErrors::No).has_value(); +} +static bool manifest_is_parseable(StringView obj) +{ + return test_parse_manifest(parse_json_object(obj), PrintErrors::No).has_value(); +} static const FeatureFlagSettings feature_flags_with_versioning{false, false, false, true}; static const FeatureFlagSettings feature_flags_without_versioning{false, false, false, false}; @@ -113,57 +133,49 @@ TEST_CASE ("manifest versioning", "[manifests]") CHECK(pgh.core_paragraph->port_version == 0); } - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "version-semver": "1.2.3-rc3" - })json", - true); + })json")); - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd#1" - })json", - true); - test_parse_manifest(R"json({ + })json")); + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version": "abcd#1" - })json", - true); - test_parse_manifest(R"json({ + })json")); + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-date": "abcd#1" - })json", - true); - test_parse_manifest(R"json({ + })json")); + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-semver": "abcd#1" - })json", - true); + })json")); SECTION ("version syntax") { - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-semver": "2020-01-01" - })json", - true); - test_parse_manifest(R"json({ + })json")); + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-date": "1.1.1" - })json", - true); - test_parse_manifest(R"json({ + })json")); + REQUIRE(manifest_is_parseable(R"json({ "name": "zlib", "version": "1.2.3-rc3" - })json", - false); + })json")); } } TEST_CASE ("manifest constraints hash", "[manifests]") { - auto p = unwrap(test_parse_manifest(R"json({ + auto m_pgh = test_parse_manifest(R"json({ "name": "zlib", "version-string": "abcd", "dependencies": [ @@ -172,11 +184,13 @@ TEST_CASE ("manifest constraints hash", "[manifests]") "version>=": "2018-09-01#1" } ] -})json")); +})json"); + REQUIRE(m_pgh.has_value()); + const auto& p = *m_pgh.get(); REQUIRE(p->core_paragraph->dependencies.at(0).constraint.value == "2018-09-01"); REQUIRE(p->core_paragraph->dependencies.at(0).constraint.port_version == 1); - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "dependencies": [ @@ -185,10 +199,9 @@ TEST_CASE ("manifest constraints hash", "[manifests]") "version>=": "2018-09-01#0" } ] -})json", - true); +})json")); - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "dependencies": [ @@ -197,10 +210,9 @@ TEST_CASE ("manifest constraints hash", "[manifests]") "version>=": "2018-09-01#-1" } ] -})json", - true); +})json")); - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "dependencies": [ @@ -210,13 +222,12 @@ TEST_CASE ("manifest constraints hash", "[manifests]") "port-version": 1 } ] -})json", - true); +})json")); } TEST_CASE ("manifest overrides embedded port version", "[manifests]") { - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "overrides": [ @@ -226,9 +237,8 @@ TEST_CASE ("manifest overrides embedded port version", "[manifests]") "port-version": 1 } ] -})json", - true); - test_parse_manifest(R"json({ +})json")); + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "overrides": [ @@ -238,9 +248,8 @@ TEST_CASE ("manifest overrides embedded port version", "[manifests]") "port-version": 1 } ] -})json", - true); - test_parse_manifest(R"json({ +})json")); + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "overrides": [ @@ -250,9 +259,8 @@ TEST_CASE ("manifest overrides embedded port version", "[manifests]") "port-version": 1 } ] -})json", - true); - test_parse_manifest(R"json({ +})json")); + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "overrides": [ @@ -262,10 +270,9 @@ TEST_CASE ("manifest overrides embedded port version", "[manifests]") "port-version": 1 } ] -})json", - true); +})json")); - CHECK(unwrap(test_parse_manifest(R"json({ + auto parsed = test_parse_manifest(R"json({ "name": "zlib", "version-string": "abcd", "overrides": [ @@ -274,11 +281,11 @@ TEST_CASE ("manifest overrides embedded port version", "[manifests]") "version-string": "abcd#1" } ] -})json", - false)) - ->core_paragraph->overrides.at(0) - .port_version == 1); - CHECK(unwrap(test_parse_manifest(R"json({ +})json"); + REQUIRE(parsed.has_value()); + CHECK((*parsed.get())->core_paragraph->overrides.at(0).port_version == 1); + + parsed = test_parse_manifest(R"json({ "name": "zlib", "version-string": "abcd", "overrides": [ @@ -287,11 +294,11 @@ TEST_CASE ("manifest overrides embedded port version", "[manifests]") "version-date": "2018-01-01#1" } ] -})json", - false)) - ->core_paragraph->overrides.at(0) - .port_version == 1); - CHECK(unwrap(test_parse_manifest(R"json({ +})json"); + REQUIRE(parsed.has_value()); + CHECK((*parsed.get())->core_paragraph->overrides.at(0).port_version == 1); + + parsed = test_parse_manifest(R"json({ "name": "zlib", "version-string": "abcd", "overrides": [ @@ -300,11 +307,11 @@ TEST_CASE ("manifest overrides embedded port version", "[manifests]") "version": "1.2#1" } ] -})json", - false)) - ->core_paragraph->overrides.at(0) - .port_version == 1); - CHECK(unwrap(test_parse_manifest(R"json({ +})json"); + REQUIRE(parsed.has_value()); + CHECK((*parsed.get())->core_paragraph->overrides.at(0).port_version == 1); + + parsed = test_parse_manifest(R"json({ "name": "zlib", "version-string": "abcd", "overrides": [ @@ -313,10 +320,9 @@ TEST_CASE ("manifest overrides embedded port version", "[manifests]") "version-semver": "1.2.0#1" } ] -})json", - false)) - ->core_paragraph->overrides.at(0) - .port_version == 1); +})json"); + REQUIRE(parsed.has_value()); + CHECK((*parsed.get())->core_paragraph->overrides.at(0).port_version == 1); } TEST_CASE ("manifest constraints", "[manifests]") @@ -355,7 +361,7 @@ TEST_CASE ("manifest constraints", "[manifests]") DependencyConstraint{VersionConstraintKind::Minimum, "2018-09-01", 0}); REQUIRE(pgh.core_paragraph->builtin_baseline == "089fa4de7dca22c67dcab631f618d5cd0697c8d4"); - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "dependencies": [ @@ -364,8 +370,7 @@ TEST_CASE ("manifest constraints", "[manifests]") "port-version": 5 } ] - })json", - true); + })json")); } TEST_CASE ("manifest builtin-baseline", "[manifests]") @@ -555,7 +560,7 @@ TEST_CASE ("manifest overrides", "[manifests]") REQUIRE(!pgh.check_against_feature_flags({}, feature_flags_with_versioning)); } - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "builtin-baseline": "089fa4de7dca22c67dcab631f618d5cd0697c8d4", @@ -565,10 +570,9 @@ TEST_CASE ("manifest overrides", "[manifests]") "version-semver": "1.2.3-rc3", "version-string": "1.2.3-rc3" } - ]})json", - true); + ]})json")); - test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "zlib", "version-string": "abcd", "builtin-baseline": "089fa4de7dca22c67dcab631f618d5cd0697c8d4", @@ -577,8 +581,7 @@ TEST_CASE ("manifest overrides", "[manifests]") "name": "abc", "port-version": 5 } - ]})json", - true); + ]})json")); std::string raw = R"json({ "name": "zlib", @@ -946,22 +949,146 @@ TEST_CASE ("SourceParagraph manifest supports", "[manifests]") TEST_CASE ("SourceParagraph manifest empty supports", "[manifests]") { - auto m_pgh = test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "a", "version-string": "1.0", "supports": "" - })json", - true); - REQUIRE_FALSE(m_pgh.has_value()); + })json")); } TEST_CASE ("SourceParagraph manifest non-string supports", "[manifests]") { - auto m_pgh = test_parse_manifest(R"json({ + REQUIRE_FALSE(manifest_is_parseable(R"json({ "name": "a", "version-string": "1.0", "supports": true - })json", - true); - REQUIRE_FALSE(m_pgh.has_value()); + })json")); +} + +static Json::Object manifest_with_license(Json::Value&& license) +{ + Json::Object res; + res.insert("name", Json::Value::string("foo")); + res.insert("version", Json::Value::string("0")); + res.insert("license", std::move(license)); + return res; +} +static Json::Object manifest_with_license(StringView license) +{ + return manifest_with_license(Json::Value::string(license.to_string())); +} +static std::string test_serialized_license(StringView license) +{ + auto m_pgh = test_parse_manifest(manifest_with_license(license)); + REQUIRE(m_pgh.has_value()); + + return serialize_manifest(**m_pgh.get())["license"].string().to_string(); +} + +static bool license_is_parseable(StringView license) +{ + Parse::ParseMessages messages; + parse_spdx_license_expression(license, messages); + return messages.error == nullptr; +} +static bool license_is_strict(StringView license) +{ + Parse::ParseMessages messages; + parse_spdx_license_expression(license, messages); + return messages.error == nullptr && messages.warnings.empty(); +} + +static std::string test_format_parse_warning(const Parse::ParseMessage& msg) +{ + return msg.format("", Parse::MessageKind::Warning).extract_data(); +} + +TEST_CASE ("simple license in manifest", "[manifests][license]") +{ + CHECK(manifest_is_parseable(manifest_with_license(Json::Value::null(nullptr)))); + CHECK_FALSE(manifest_is_parseable(manifest_with_license(""))); + CHECK(manifest_is_parseable(manifest_with_license("MIT"))); +} + +TEST_CASE ("valid and invalid licenses", "[manifests][license]") +{ + CHECK(license_is_strict("mIt")); + CHECK(license_is_strict("Apache-2.0")); + CHECK(license_is_strict("GPL-2.0+")); + CHECK_FALSE(license_is_parseable("GPL-2.0++")); + CHECK(license_is_strict("LicenseRef-blah")); + CHECK_FALSE(license_is_strict("unknownlicense")); + CHECK(license_is_parseable("unknownlicense")); +} + +TEST_CASE ("licenses with compounds", "[manifests][license]") +{ + CHECK(license_is_strict("GPL-3.0+ WITH GCC-exception-3.1")); + CHECK(license_is_strict("Apache-2.0 WITH LLVM-exception")); + CHECK_FALSE(license_is_parseable("(Apache-2.0) WITH LLVM-exception")); + CHECK(license_is_strict("(Apache-2.0 OR MIT) AND GPL-3.0+ WITH GCC-exception-3.1")); + CHECK_FALSE(license_is_parseable("Apache-2.0 WITH")); + CHECK_FALSE(license_is_parseable("GPL-3.0+ AND")); + CHECK_FALSE(license_is_parseable("MIT and Apache-2.0")); + CHECK_FALSE(license_is_parseable("GPL-3.0 WITH GCC-exception+")); + CHECK_FALSE(license_is_parseable("(GPL-3.0 WITH GCC-exception)+")); +} + +TEST_CASE ("license serialization", "[manifests][license]") +{ + auto m_pgh = test_parse_manifest(manifest_with_license(Json::Value::null(nullptr))); + REQUIRE(m_pgh); + auto manifest = serialize_manifest(**m_pgh.get()); + REQUIRE(manifest.contains("license")); + CHECK(manifest["license"].is_null()); + + CHECK(test_serialized_license("MIT") == "MIT"); + CHECK(test_serialized_license("mit") == "MIT"); + CHECK(test_serialized_license("MiT AND (aPACHe-2.0 \tOR \n gpl-2.0+)") == "MIT AND (Apache-2.0 OR GPL-2.0+)"); + CHECK(test_serialized_license("uNkNoWnLiCeNsE") == "uNkNoWnLiCeNsE"); +} + +TEST_CASE ("license error messages", "[manifests][license]") +{ + Parse::ParseMessages messages; + parse_spdx_license_expression("", messages); + REQUIRE(messages.error); + CHECK(messages.error->format() == R"(:1:1: error: SPDX license expression was empty. + on expression: + ^ +)"); + + parse_spdx_license_expression("MIT ()", messages); + REQUIRE(messages.error); + CHECK(messages.error->format() == + R"(:1:5: error: Expected a compound or the end of the string, found a parenthesis. + on expression: MIT () + ^ +)"); + + parse_spdx_license_expression("MIT +", messages); + REQUIRE(messages.error); + CHECK( + messages.error->format() == + R"(:1:5: error: SPDX license expression contains an extra '+'. These are only allowed directly after a license identifier. + on expression: MIT + + ^ +)"); + + parse_spdx_license_expression("MIT AND", messages); + REQUIRE(messages.error); + CHECK(messages.error->format() == + R"(:1:8: error: Expected a license name, found the end of the string. + on expression: MIT AND + ^ +)"); + + parse_spdx_license_expression("MIT AND unknownlicense", messages); + CHECK(!messages.error); + REQUIRE(messages.warnings.size() == 1); + CHECK( + test_format_parse_warning(messages.warnings[0]) == + R"(:1:9: warning: Unknown license identifier 'unknownlicense'. Known values are listed at https://spdx.org/licenses/ + on expression: MIT AND unknownlicense + ^)"); } diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index fb1b4c3bfa..73e7f216a2 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -1,7 +1,9 @@ +#include #include #include #include +#include #include using namespace vcpkg; @@ -25,28 +27,67 @@ namespace vcpkg::Parse std::string ParseError::format() const { - auto caret_spacing = std::string(18, ' '); auto decoder = Unicode::Utf8Decoder(line.data(), line.data() + line.size()); - for (int i = 0; i < caret_col; ++i, ++decoder) + ParseMessage as_message; + as_message.location = SourceLoc{std::next(decoder, caret_col), decoder, row, column}; + as_message.message = msg::LocalizedString::from_string_unchecked(std::string(message)); + + auto res = as_message.format(origin, MessageKind::Error).extract_data(); + res.push_back('\n'); + return res; + } + + DECLARE_AND_REGISTER_MESSAGE(FormattedParseMessageLocation, + (msg::path, msg::row, msg::column), + "{LOCKED}", + "{path}:{row}:{column}: "); + DECLARE_AND_REGISTER_MESSAGE(FormattedParseError, (msg::value), "", "error: {value}"); + DECLARE_AND_REGISTER_MESSAGE(FormattedParseWarning, (msg::value), "", "warning: {value}"); + DECLARE_AND_REGISTER_MESSAGE(FormattedParseMessageExpression, (msg::value), "", " on expression: {value}"); + + msg::LocalizedString ParseMessage::format(StringView origin, MessageKind kind) const + { + msg::LocalizedString res = msg::format(msgFormattedParseMessageLocation, + msg::path = origin, + msg::row = location.row, + msg::column = location.column); + if (kind == MessageKind::Warning) { - const char32_t cp = *decoder; - // this may eventually want to check for full-width characters and grapheme clusters as well - caret_spacing.push_back(cp == '\t' ? '\t' : ' '); + res.append(msg::format(msgFormattedParseWarning, msg::value = message)); } + else + { + res.append(msg::format(msgFormattedParseError, msg::value = message)); + } + res.appendnl(); + + auto line_end = Util::find_if(location.it, Parse::ParserBase::is_lineend); + StringView line = StringView{ + location.start_of_line.pointer_to_current(), + line_end.pointer_to_current(), + }; + res.append(msg::format(msgFormattedParseMessageExpression, msg::value = line)); + res.appendnl(); - return Strings::concat(origin, - ":", - row, - ":", - column, - ": error: ", - message, - "\n" - " on expression: ", // 18 columns - line, - "\n", - caret_spacing, - "^\n"); + auto caret_point = StringView{location.start_of_line.pointer_to_current(), location.it.pointer_to_current()}; + auto formatted_caret_point = msg::format(msgFormattedParseMessageExpression, msg::value = caret_point); + + std::string caret_string; + caret_string.reserve(formatted_caret_point.data().size()); + for (char32_t ch : Unicode::Utf8Decoder(formatted_caret_point)) + { + if (ch == '\t') + caret_string.push_back('\t'); + else if (Unicode::is_double_width_code_point(ch)) + caret_string.append(" "); + else + caret_string.push_back(' '); + } + caret_string.push_back('^'); + + res.append(msg::LocalizedString::from_string_unchecked(std::move(caret_string))); + + return res; } const std::string& ParseError::get_message() const { return this->message; } @@ -84,10 +125,15 @@ namespace vcpkg::Parse return cur(); } + void ParserBase::add_warning(msg::LocalizedString&& message, const SourceLoc& loc) + { + m_messages.warnings.push_back(ParseMessage{loc, std::move(message)}); + } + void ParserBase::add_error(std::string message, const SourceLoc& loc) { // avoid cascading errors by only saving the first - if (!m_err) + if (!m_messages.error) { // find end of line auto line_end = loc.it; @@ -95,7 +141,7 @@ namespace vcpkg::Parse { ++line_end; } - m_err = std::make_unique( + m_messages.error = std::make_unique( m_origin.to_string(), loc.row, loc.column, diff --git a/src/vcpkg/base/unicode.cpp b/src/vcpkg/base/unicode.cpp index d0758e45b4..bb81b24f72 100644 --- a/src/vcpkg/base/unicode.cpp +++ b/src/vcpkg/base/unicode.cpp @@ -92,6 +92,103 @@ namespace vcpkg::Unicode return count; } + std::pair utf8_decode_code_point(const char* first, + const char* last, + char32_t& out) noexcept + { + out = end_of_file; + if (first == last) + { + return {last, utf8_errc::NoError}; + } + + auto code_unit = *first; + auto kind = utf8_code_unit_kind(code_unit); + const int count = utf8_code_unit_count(kind); + + const char* it = first + 1; + + if (kind == Utf8CodeUnitKind::Invalid) + { + return {it, utf8_errc::InvalidCodeUnit}; + } + else if (kind == Utf8CodeUnitKind::Continue) + { + return {it, utf8_errc::UnexpectedContinue}; + } + else if (count > last - first) + { + return {last, utf8_errc::UnexpectedEof}; + } + + if (count == 1) + { + out = static_cast(code_unit); + return {it, utf8_errc::NoError}; + } + + // 2 -> 0b0001'1111, 6 + // 3 -> 0b0000'1111, 12 + // 4 -> 0b0000'0111, 18 + const auto start_mask = static_cast(0xFF >> (count + 1)); + const int start_shift = 6 * (count - 1); + char32_t code_point = static_cast(code_unit & start_mask) << start_shift; + + constexpr unsigned char continue_mask = 0b0011'1111; + for (int byte = 1; byte < count; ++byte) + { + code_unit = static_cast(*it++); + + kind = utf8_code_unit_kind(code_unit); + if (kind == Utf8CodeUnitKind::Invalid) + { + return {it, utf8_errc::InvalidCodeUnit}; + } + else if (kind != Utf8CodeUnitKind::Continue) + { + return {it, utf8_errc::UnexpectedStart}; + } + + const int shift = 6 * (count - byte - 1); + code_point |= (code_unit & continue_mask) << shift; + } + + if (code_point > 0x10'FFFF) + { + return {it, utf8_errc::InvalidCodePoint}; + } + + out = code_point; + return {it, utf8_errc::NoError}; + } + + // uses the C++20 definition + /* + [format.string.std] + * U+1100 - U+115F + * U+2329 - U+232A + * U+2E80 - U+303E + * U+3040 - U+A4CF + * U+AC00 - U+D7A3 + * U+F900 - U+FAFF + * U+FE10 - U+FE19 + * U+FE30 - U+FE6F + * U+FF00 - U+FF60 + * U+FFE0 - U+FFE6 + * U+1F300 - U+1F64F + * U+1F900 - U+1F9FF + * U+20000 - U+2FFFD + * U+30000 - U+3FFFD + */ + bool is_double_width_code_point(char32_t ch) noexcept + { + return (ch >= 0x1100 && ch <= 0x115F) || (ch >= 0x2329 && ch <= 0x232A) || (ch >= 0x2E80 && ch <= 0x303E) || + (ch >= 0x3040 && ch <= 0xA4CF) || (ch >= 0xAC00 && ch <= 0xD7A3) || (ch >= 0xF900 && ch <= 0xFAFF) || + (ch >= 0xFE10 && ch <= 0xFE19) || (ch >= 0xFE30 && ch <= 0xFE6F) || (ch >= 0xFF00 && ch <= 0xFF60) || + (ch >= 0xFFE0 && ch <= 0xFFE6) || (ch >= 0x1F300 && ch <= 0x1F64F) || (ch >= 0x1F900 && ch <= 0x1F9FF) || + (ch >= 0x20000 && ch <= 0x2FFFD) || (ch >= 0x30000 && ch <= 0x3FFFD); + } + bool utf8_is_valid_string(const char* first, const char* last) noexcept { utf8_errc err = utf8_errc::NoError; @@ -179,76 +276,22 @@ namespace vcpkg::Unicode return utf8_errc::NoError; } - unsigned char code_unit = static_cast(*next_++); - - auto kind = utf8_code_unit_kind(code_unit); - if (kind == Utf8CodeUnitKind::Invalid) - { - *this = sentinel(); - return utf8_errc::InvalidCodeUnit; - } - else if (kind == Utf8CodeUnitKind::Continue) + char32_t code_point; + auto new_next = utf8_decode_code_point(next_, last_, code_point); + if (new_next.second != utf8_errc::NoError) { *this = sentinel(); - return utf8_errc::UnexpectedContinue; + return new_next.second; } - const int count = utf8_code_unit_count(kind); - if (count == 1) + if (utf16_is_trailing_surrogate_code_point(code_point) && utf16_is_leading_surrogate_code_point(current_)) { - current_ = static_cast(code_unit); + *this = sentinel(); + return utf8_errc::PairedSurrogates; } - else - { - // 2 -> 0b0001'1111, 6 - // 3 -> 0b0000'1111, 12 - // 4 -> 0b0000'0111, 18 - const auto start_mask = static_cast(0xFF >> (count + 1)); - const int start_shift = 6 * (count - 1); - auto code_point = static_cast(code_unit & start_mask) << start_shift; - - constexpr unsigned char continue_mask = 0b0011'1111; - for (int byte = 1; byte < count; ++byte) - { - if (next_ == last_) - { - *this = sentinel(); - return utf8_errc::UnexpectedContinue; - } - code_unit = static_cast(*next_++); - - kind = utf8_code_unit_kind(code_unit); - if (kind == Utf8CodeUnitKind::Invalid) - { - *this = sentinel(); - return utf8_errc::InvalidCodeUnit; - } - else if (kind != Utf8CodeUnitKind::Continue) - { - *this = sentinel(); - return utf8_errc::UnexpectedStart; - } - - const int shift = 6 * (count - byte - 1); - code_point |= (code_unit & continue_mask) << shift; - } - if (code_point > 0x10'FFFF) - { - *this = sentinel(); - return utf8_errc::InvalidCodePoint; - } - else if (utf16_is_trailing_surrogate_code_point(code_point) && - utf16_is_leading_surrogate_code_point(current_)) - { - *this = sentinel(); - return utf8_errc::PairedSurrogates; - } - else - { - current_ = code_point; - } - } + next_ = new_next.first; + current_ = code_point; return utf8_errc::NoError; } diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index d541cb72ff..5b0937a773 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -38,6 +38,8 @@ namespace "Restored {value} packages from AWS servers in {elapsed}s"); DECLARE_AND_REGISTER_MESSAGE(AwsUploadedPackages, (msg::value), "", "Uploaded binaries to {value} AWS servers"); + using Parse::SourceLoc; + struct ConfigSegmentsParser : Parse::ParserBase { using Parse::ParserBase::ParserBase; @@ -134,10 +136,9 @@ namespace } } - std::vector>> ConfigSegmentsParser:: - parse_all_segments() + std::vector>> ConfigSegmentsParser::parse_all_segments() { - std::vector>> ret; + std::vector>> ret; while (!at_eof()) { std::vector> segments; diff --git a/src/vcpkg/sourceparagraph.cpp b/src/vcpkg/sourceparagraph.cpp index c025b7dc78..a5e14805f8 100644 --- a/src/vcpkg/sourceparagraph.cpp +++ b/src/vcpkg/sourceparagraph.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,85 @@ #include #include +namespace +{ + namespace msg = vcpkg::msg; + DECLARE_AND_REGISTER_MESSAGE(EmptyLicenseExpression, (), "", "SPDX license expression was empty."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionContainsUnicode, + (msg::value, msg::pretty_value), + "", + "SPDX license expression contains a unicode character (U+{value:04x} " + "'{pretty_value}'), but these expressions are ASCII-only."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionContainsInvalidCharacter, + (msg::value), + "", + "SPDX license expression contains an invalid character (0x{value:02x} '{value}')."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionContainsExtraPlus, + (), + "", + "SPDX license expression contains an extra '+'. These are only allowed directly " + "after a license identifier."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionDocumentRefUnsupported, + (), + "", + "The current implementation does not support DocumentRef- SPDX references."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectLicenseFoundEof, + (), + "", + "Expected a license name, found the end of the string."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectExceptionFoundEof, + (), + "", + "Expected an exception name, found the end of the string."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectCompoundFoundParen, + (), + "", + "Expected a compound or the end of the string, found a parenthesis."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectLicenseFoundParen, + (), + "", + "Expected a license name, found a parenthesis."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectExceptionFoundParen, + (), + "", + "Expected an exception name, found a parenthesis."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionImbalancedParens, + (), + "", + "There was a close parenthesis without an opening parenthesis."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectLicenseFoundCompound, + (msg::value), + "", + "Expected a license name, found the compound {value}."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectExceptionFoundCompound, + (msg::value), + "", + "Expected an exception name, found the compound {value}."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectCompoundFoundWith, + (), + "", + "Expected either AND or OR, found WITH (WITH is only allowed after license names, not " + "parenthesized expressions)."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectCompoundOrWithFoundWord, + (msg::value), + "", + "Expected either AND, OR, or WITH, found a license or exception name: '{value}'."); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionExpectCompoundFoundWord, + (msg::value), + "", + "Expected either AND or OR, found a license or exception name: '{value}'."); + DECLARE_AND_REGISTER_MESSAGE( + LicenseExpressionUnknownLicense, + (msg::value), + "", + "Unknown license identifier '{value}'. Known values are listed at https://spdx.org/licenses/"); + DECLARE_AND_REGISTER_MESSAGE(LicenseExpressionUnknownException, + (msg::value), + "", + "Unknown license exception identifier '{value}'. Known values are listed at " + "https://spdx.org/licenses/exceptions-index.html"); +} // anonymous namespace + namespace vcpkg { using namespace vcpkg::Parse; @@ -708,143 +788,253 @@ namespace vcpkg }; ContactsDeserializer ContactsDeserializer::instance; - static constexpr StringLiteral EXPRESSION_WORDS[] = { - "WITH", - "AND", - "OR", - }; - static constexpr StringLiteral VALID_LICENSES[] = -#include "spdx-licenses.inc" - ; - static constexpr StringLiteral VALID_EXCEPTIONS[] = + static constexpr StringLiteral VALID_LICENSES[] = { #include "spdx-licenses.inc" - ; + }; + static constexpr StringLiteral VALID_EXCEPTIONS[] = { +#include "spdx-exceptions.inc" + }; - // We "parse" this so that we can add actual license parsing at some point in the future - // without breaking anyone - struct LicenseExpressionDeserializer : Json::IDeserializer + // The "license" field; either: + // * a string, which must be an SPDX license expression. + // EBNF located at: https://github.com/microsoft/vcpkg/blob/master/docs/maintainers/manifest-files.md#license + // * `null`, for when the license of the package cannot be described by an SPDX expression + struct SpdxLicenseExpressionParser : Parse::ParserBase { - virtual StringView type_name() const override { return "an SPDX license expression"; } + SpdxLicenseExpressionParser(StringView sv, StringView origin) : Parse::ParserBase(sv, origin) { } - enum class Mode + static const StringLiteral* case_insensitive_find(View lst, StringView id) { - ExpectExpression, - ExpectContinue, - ExpectException, + return Util::find_if(lst, + [id](StringLiteral el) { return Strings::case_insensitive_ascii_equals(id, el); }); + } + static constexpr bool is_idstring_element(char32_t ch) { return is_alphanumdash(ch) || ch == '.'; } + + enum class Expecting + { + License, // at the beginning, or after a compound (AND, OR) + Exception, // after a WITH + CompoundOrWith, // after a license + Compound, // after an exception (only one WITH is allowed), or after a close paren }; - virtual Optional visit_string(Json::Reader&, StringView sv) override + void eat_idstring(std::string& result, Expecting& expecting) { - Mode mode = Mode::ExpectExpression; - size_t open_parens = 0; - std::string current_word; + auto loc = cur_loc(); + auto token = match_zero_or_more(is_idstring_element); - const auto check_current_word = [¤t_word, &mode] { - if (current_word.empty()) + if (Strings::starts_with(token, "DocumentRef-")) + { + add_error(msg::format(msgLicenseExpressionDocumentRefUnsupported), loc); + if (cur() == ':') { - return true; + next(); } - - Span valid_ids; - bool case_sensitive = false; - switch (mode) + return; + } + else if (token == "AND" || token == "OR" || token == "WITH") + { + if (expecting == Expecting::License) { - case Mode::ExpectExpression: - valid_ids = VALID_LICENSES; - mode = Mode::ExpectContinue; - // a single + is allowed on the end of licenses - if (current_word.back() == '+') - { - current_word.pop_back(); - } - break; - case Mode::ExpectContinue: - valid_ids = EXPRESSION_WORDS; - mode = Mode::ExpectExpression; - case_sensitive = true; - break; - case Mode::ExpectException: - valid_ids = VALID_EXCEPTIONS; - mode = Mode::ExpectContinue; - break; + add_error(msg::format(msgLicenseExpressionExpectLicenseFoundCompound, msg::value = token), loc); + } + if (expecting == Expecting::Exception) + { + add_error(msg::format(msgLicenseExpressionExpectExceptionFoundCompound, msg::value = token), loc); } - const auto equal = [&](StringView sv) { - if (case_sensitive) - { - return sv == current_word; - } - else + if (token == "WITH") + { + if (expecting == Expecting::Compound) { - return Strings::case_insensitive_ascii_equals(sv, current_word); + add_error(msg::format(msgLicenseExpressionExpectCompoundFoundWith), loc); } - }; - - if (std::find_if(valid_ids.begin(), valid_ids.end(), equal) == valid_ids.end()) - { - return false; + expecting = Expecting::Exception; } - - if (current_word == "WITH") + else { - mode = Mode::ExpectException; + expecting = Expecting::License; } - current_word.clear(); - return true; - }; + result.push_back(' '); + result.append(token.begin(), token.end()); + result.push_back(' '); + return; + } - for (const auto& ch : sv) + switch (expecting) { - if (ch == ' ' || ch == '\t') - { - if (!check_current_word()) + case Expecting::Compound: + add_error(msg::format(msgLicenseExpressionExpectCompoundFoundWord, msg::value = token), loc); + break; + case Expecting::CompoundOrWith: + add_error(msg::format(msgLicenseExpressionExpectCompoundOrWithFoundWord, msg::value = token), loc); + break; + case Expecting::License: + if (Strings::starts_with(token, "LicenseRef-")) { - return nullopt; + result.append(token.begin(), token.end()); } - } - else if (ch == '(') - { - if (!check_current_word()) - { - return nullopt; - } - if (mode != Mode::ExpectExpression) - { - return nullopt; - } - ++open_parens; - } - else if (ch == ')') - { - if (!check_current_word()) + else { - return nullopt; + auto it = case_insensitive_find(VALID_LICENSES, token); + if (it != std::end(VALID_LICENSES)) + { + result.append(it->begin(), it->end()); + } + else + { + add_warning(msg::format(msgLicenseExpressionUnknownLicense, msg::value = token), loc); + result.append(token.begin(), token.end()); + } + + if (cur() == '+') + { + next(); + result.push_back('+'); + } } - if (mode != Mode::ExpectContinue) + expecting = Expecting::CompoundOrWith; + break; + case Expecting::Exception: + auto it = case_insensitive_find(VALID_EXCEPTIONS, token); + if (it != std::end(VALID_EXCEPTIONS)) { - return nullopt; + // case normalization + result.append(it->begin(), it->end()); } - if (open_parens == 0) + else { - return nullopt; + add_warning(msg::format(msgLicenseExpressionUnknownException, msg::value = token), loc); + result.append(token.begin(), token.end()); } - --open_parens; - } - else + expecting = Expecting::Compound; + break; + } + } + + std::string parse() + { + if (cur() == Unicode::end_of_file) + { + add_error(msg::format(msgEmptyLicenseExpression)); + return ""; + } + + Expecting expecting = Expecting::License; + std::string result; + + size_t open_parens = 0; + while (!at_eof()) + { + skip_whitespace(); + switch (cur()) { - current_word.push_back(ch); + case '(': + if (expecting == Expecting::Compound || expecting == Expecting::CompoundOrWith) + { + add_error(msg::format(msgLicenseExpressionExpectCompoundFoundParen)); + } + if (expecting == Expecting::Exception) + { + add_error(msg::format(msgLicenseExpressionExpectExceptionFoundParen)); + } + result.push_back('('); + expecting = Expecting::License; + ++open_parens; + next(); + break; + case ')': + if (expecting == Expecting::License) + { + add_error(msg::format(msgLicenseExpressionExpectLicenseFoundParen)); + } + else if (expecting == Expecting::Exception) + { + add_error(msg::format(msgLicenseExpressionExpectExceptionFoundParen)); + } + if (open_parens == 0) + { + add_error(msg::format(msgLicenseExpressionImbalancedParens)); + } + result.push_back(')'); + expecting = Expecting::Compound; + --open_parens; + next(); + break; + case '+': + add_error(msg::format(msgLicenseExpressionContainsExtraPlus)); + next(); + break; + default: + if (cur() > 0x7F) + { + auto ch = cur(); + auto first = it().pointer_to_current(); + next(); + auto last = it().pointer_to_current(); + add_error(msg::format(msgLicenseExpressionContainsUnicode, + msg::value = ch, + msg::pretty_value = StringView{first, last})); + break; + } + if (!is_idstring_element(cur())) + { + add_error(msg::format(msgLicenseExpressionContainsInvalidCharacter, + msg::value = static_cast(cur()))); + next(); + break; + } + eat_idstring(result, expecting); + break; } } - if (!check_current_word()) + if (expecting == Expecting::License) { - return nullopt; + add_error(msg::format(msgLicenseExpressionExpectLicenseFoundEof)); } - else + if (expecting == Expecting::Exception) + { + add_error(msg::format(msgLicenseExpressionExpectExceptionFoundEof)); + } + + return result; + } + }; + + std::string parse_spdx_license_expression(StringView sv, Parse::ParseMessages& messages) + { + auto parser = SpdxLicenseExpressionParser(sv, ""); + auto result = parser.parse(); + messages = parser.extract_messages(); + return result; + } + + struct LicenseExpressionDeserializer : Json::IDeserializer + { + virtual StringView type_name() const override { return "an SPDX license expression"; } + + virtual Optional visit_null(Json::Reader&) override { return {std::string()}; } + + // if `sv` is a valid SPDX license expression, returns sv, + // but with whitespace normalized + virtual Optional visit_string(Json::Reader& r, StringView sv) override + { + auto parser = SpdxLicenseExpressionParser(sv, ""); + auto res = parser.parse(); + + for (const auto& warning : parser.messages().warnings) + { + msg::println(Color::warning, warning.format("", Parse::MessageKind::Warning)); + } + if (auto err = parser.get_error()) { - return sv.to_string(); + r.add_generic_error(type_name(), err->format()); + return std::string(); } + + return res; } static LicenseExpressionDeserializer instance; @@ -941,7 +1131,13 @@ namespace vcpkg r.optional_object_field(obj, DESCRIPTION, spgh->description, Json::ParagraphDeserializer::instance); r.optional_object_field(obj, HOMEPAGE, spgh->homepage, url_deserializer); r.optional_object_field(obj, DOCUMENTATION, spgh->documentation, url_deserializer); - r.optional_object_field(obj, LICENSE, spgh->license, LicenseExpressionDeserializer::instance); + + std::string license; + if (r.optional_object_field(obj, LICENSE, license, LicenseExpressionDeserializer::instance)) + { + spgh->license = {std::move(license)}; + } + r.optional_object_field(obj, DEPENDENCIES, spgh->dependencies, DependencyArrayDeserializer::instance); static Json::ArrayDeserializer overrides_deserializer{ "an array of overrides"}; @@ -1464,7 +1660,21 @@ namespace vcpkg serialize_optional_string(obj, ManifestDeserializer::HOMEPAGE, scf.core_paragraph->homepage); serialize_optional_string(obj, ManifestDeserializer::DOCUMENTATION, scf.core_paragraph->documentation); - serialize_optional_string(obj, ManifestDeserializer::LICENSE, scf.core_paragraph->license); + if (auto license = scf.core_paragraph->license.get()) + { + if (license->empty()) + { + obj.insert(ManifestDeserializer::LICENSE, Json::Value::null(nullptr)); + } + else + { + obj.insert(ManifestDeserializer::LICENSE, Json::Value::string(*license)); + } + } + else if (debug) + { + obj.insert(ManifestDeserializer::LICENSE, Json::Value::string("")); + } serialize_optional_string( obj, ManifestDeserializer::SUPPORTS, to_string(scf.core_paragraph->supports_expression)); if (scf.core_paragraph->builtin_baseline.has_value()) diff --git a/src/vcpkg/spdx-exceptions.inc b/src/vcpkg/spdx-exceptions.inc index c89465c5e0..36d4ec9670 100644 --- a/src/vcpkg/spdx-exceptions.inc +++ b/src/vcpkg/spdx-exceptions.inc @@ -1,45 +1,43 @@ // Data downloaded from https://raw.githubusercontent.com/spdx/license-list-data/0af22f869f8b0906097cfac90ee0516992e8939f/json/exceptions.json -// Generated by scripts/Generate-SpdxLicenseList.ps1 -{ - "389-exception", - "Autoconf-exception-2.0", - "Autoconf-exception-3.0", - "Bison-exception-2.2", - "Bootloader-exception", - "Classpath-exception-2.0", - "CLISP-exception-2.0", - "DigiRule-FOSS-exception", - "eCos-exception-2.0", - "Fawkes-Runtime-exception", - "FLTK-exception", - "Font-exception-2.0", - "freertos-exception-2.0", - "GCC-exception-2.0", - "GCC-exception-3.1", - "gnu-javamail-exception", - "GPL-3.0-linking-exception", - "GPL-3.0-linking-source-exception", - "GPL-CC-1.0", - "i2p-gpl-java-exception", - "LGPL-3.0-linking-exception", - "Libtool-exception", - "Linux-syscall-note", - "LLVM-exception", - "LZMA-exception", - "mif-exception", - "Nokia-Qt-exception-1.1", - "OCaml-LGPL-linking-exception", - "OCCT-exception-1.0", - "OpenJDK-assembly-exception-1.0", - "openvpn-openssl-exception", - "PS-or-PDF-font-exception-20170817", - "Qt-GPL-exception-1.0", - "Qt-LGPL-exception-1.1", - "Qwt-exception-1.0", - "SHL-2.0", - "SHL-2.1", - "Swift-exception", - "u-boot-exception-2.0", - "Universal-FOSS-exception-1.0", - "WxWindows-exception-3.1", -} +// Generated by Generate-SpdxLicenseList.ps1 +"389-exception", +"Autoconf-exception-2.0", +"Autoconf-exception-3.0", +"Bison-exception-2.2", +"Bootloader-exception", +"Classpath-exception-2.0", +"CLISP-exception-2.0", +"DigiRule-FOSS-exception", +"eCos-exception-2.0", +"Fawkes-Runtime-exception", +"FLTK-exception", +"Font-exception-2.0", +"freertos-exception-2.0", +"GCC-exception-2.0", +"GCC-exception-3.1", +"gnu-javamail-exception", +"GPL-3.0-linking-exception", +"GPL-3.0-linking-source-exception", +"GPL-CC-1.0", +"i2p-gpl-java-exception", +"LGPL-3.0-linking-exception", +"Libtool-exception", +"Linux-syscall-note", +"LLVM-exception", +"LZMA-exception", +"mif-exception", +"Nokia-Qt-exception-1.1", +"OCaml-LGPL-linking-exception", +"OCCT-exception-1.0", +"OpenJDK-assembly-exception-1.0", +"openvpn-openssl-exception", +"PS-or-PDF-font-exception-20170817", +"Qt-GPL-exception-1.0", +"Qt-LGPL-exception-1.1", +"Qwt-exception-1.0", +"SHL-2.0", +"SHL-2.1", +"Swift-exception", +"u-boot-exception-2.0", +"Universal-FOSS-exception-1.0", +"WxWindows-exception-3.1", diff --git a/src/vcpkg/spdx-licenses.inc b/src/vcpkg/spdx-licenses.inc index 76ff65a1c5..c947223589 100644 --- a/src/vcpkg/spdx-licenses.inc +++ b/src/vcpkg/spdx-licenses.inc @@ -1,479 +1,477 @@ // Data downloaded from https://raw.githubusercontent.com/spdx/license-list-data/0af22f869f8b0906097cfac90ee0516992e8939f/json/licenses.json -// Generated by scripts/Generate-SpdxLicenseList.ps1 -{ - "0BSD", - "AAL", - "Abstyles", - "Adobe-2006", - "Adobe-Glyph", - "ADSL", - "AFL-1.1", - "AFL-1.2", - "AFL-2.0", - "AFL-2.1", - "AFL-3.0", - "Afmparse", - "AGPL-1.0", - "AGPL-1.0-only", - "AGPL-1.0-or-later", - "AGPL-3.0", - "AGPL-3.0-only", - "AGPL-3.0-or-later", - "Aladdin", - "AMDPLPA", - "AML", - "AMPAS", - "ANTLR-PD", - "ANTLR-PD-fallback", - "Apache-1.0", - "Apache-1.1", - "Apache-2.0", - "APAFML", - "APL-1.0", - "APSL-1.0", - "APSL-1.1", - "APSL-1.2", - "APSL-2.0", - "Artistic-1.0", - "Artistic-1.0-cl8", - "Artistic-1.0-Perl", - "Artistic-2.0", - "Bahyph", - "Barr", - "Beerware", - "BitTorrent-1.0", - "BitTorrent-1.1", - "blessing", - "BlueOak-1.0.0", - "Borceux", - "BSD-1-Clause", - "BSD-2-Clause", - "BSD-2-Clause-FreeBSD", - "BSD-2-Clause-NetBSD", - "BSD-2-Clause-Patent", - "BSD-2-Clause-Views", - "BSD-3-Clause", - "BSD-3-Clause-Attribution", - "BSD-3-Clause-Clear", - "BSD-3-Clause-LBNL", - "BSD-3-Clause-Modification", - "BSD-3-Clause-No-Military-License", - "BSD-3-Clause-No-Nuclear-License", - "BSD-3-Clause-No-Nuclear-License-2014", - "BSD-3-Clause-No-Nuclear-Warranty", - "BSD-3-Clause-Open-MPI", - "BSD-4-Clause", - "BSD-4-Clause-Shortened", - "BSD-4-Clause-UC", - "BSD-Protection", - "BSD-Source-Code", - "BSL-1.0", - "BUSL-1.1", - "bzip2-1.0.5", - "bzip2-1.0.6", - "C-UDA-1.0", - "CAL-1.0", - "CAL-1.0-Combined-Work-Exception", - "Caldera", - "CATOSL-1.1", - "CC-BY-1.0", - "CC-BY-2.0", - "CC-BY-2.5", - "CC-BY-2.5-AU", - "CC-BY-3.0", - "CC-BY-3.0-AT", - "CC-BY-3.0-DE", - "CC-BY-3.0-NL", - "CC-BY-3.0-US", - "CC-BY-4.0", - "CC-BY-NC-1.0", - "CC-BY-NC-2.0", - "CC-BY-NC-2.5", - "CC-BY-NC-3.0", - "CC-BY-NC-3.0-DE", - "CC-BY-NC-4.0", - "CC-BY-NC-ND-1.0", - "CC-BY-NC-ND-2.0", - "CC-BY-NC-ND-2.5", - "CC-BY-NC-ND-3.0", - "CC-BY-NC-ND-3.0-DE", - "CC-BY-NC-ND-3.0-IGO", - "CC-BY-NC-ND-4.0", - "CC-BY-NC-SA-1.0", - "CC-BY-NC-SA-2.0", - "CC-BY-NC-SA-2.0-FR", - "CC-BY-NC-SA-2.0-UK", - "CC-BY-NC-SA-2.5", - "CC-BY-NC-SA-3.0", - "CC-BY-NC-SA-3.0-DE", - "CC-BY-NC-SA-3.0-IGO", - "CC-BY-NC-SA-4.0", - "CC-BY-ND-1.0", - "CC-BY-ND-2.0", - "CC-BY-ND-2.5", - "CC-BY-ND-3.0", - "CC-BY-ND-3.0-DE", - "CC-BY-ND-4.0", - "CC-BY-SA-1.0", - "CC-BY-SA-2.0", - "CC-BY-SA-2.0-UK", - "CC-BY-SA-2.1-JP", - "CC-BY-SA-2.5", - "CC-BY-SA-3.0", - "CC-BY-SA-3.0-AT", - "CC-BY-SA-3.0-DE", - "CC-BY-SA-4.0", - "CC-PDDC", - "CC0-1.0", - "CDDL-1.0", - "CDDL-1.1", - "CDL-1.0", - "CDLA-Permissive-1.0", - "CDLA-Permissive-2.0", - "CDLA-Sharing-1.0", - "CECILL-1.0", - "CECILL-1.1", - "CECILL-2.0", - "CECILL-2.1", - "CECILL-B", - "CECILL-C", - "CERN-OHL-1.1", - "CERN-OHL-1.2", - "CERN-OHL-P-2.0", - "CERN-OHL-S-2.0", - "CERN-OHL-W-2.0", - "ClArtistic", - "CNRI-Jython", - "CNRI-Python", - "CNRI-Python-GPL-Compatible", - "Condor-1.1", - "copyleft-next-0.3.0", - "copyleft-next-0.3.1", - "CPAL-1.0", - "CPL-1.0", - "CPOL-1.02", - "Crossword", - "CrystalStacker", - "CUA-OPL-1.0", - "Cube", - "curl", - "D-FSL-1.0", - "diffmark", - "DOC", - "Dotseqn", - "DRL-1.0", - "DSDP", - "dvipdfm", - "ECL-1.0", - "ECL-2.0", - "eCos-2.0", - "EFL-1.0", - "EFL-2.0", - "eGenix", - "Entessa", - "EPICS", - "EPL-1.0", - "EPL-2.0", - "ErlPL-1.1", - "etalab-2.0", - "EUDatagrid", - "EUPL-1.0", - "EUPL-1.1", - "EUPL-1.2", - "Eurosym", - "Fair", - "Frameworx-1.0", - "FreeBSD-DOC", - "FreeImage", - "FSFAP", - "FSFUL", - "FSFULLR", - "FTL", - "GD", - "GFDL-1.1", - "GFDL-1.1-invariants-only", - "GFDL-1.1-invariants-or-later", - "GFDL-1.1-no-invariants-only", - "GFDL-1.1-no-invariants-or-later", - "GFDL-1.1-only", - "GFDL-1.1-or-later", - "GFDL-1.2", - "GFDL-1.2-invariants-only", - "GFDL-1.2-invariants-or-later", - "GFDL-1.2-no-invariants-only", - "GFDL-1.2-no-invariants-or-later", - "GFDL-1.2-only", - "GFDL-1.2-or-later", - "GFDL-1.3", - "GFDL-1.3-invariants-only", - "GFDL-1.3-invariants-or-later", - "GFDL-1.3-no-invariants-only", - "GFDL-1.3-no-invariants-or-later", - "GFDL-1.3-only", - "GFDL-1.3-or-later", - "Giftware", - "GL2PS", - "Glide", - "Glulxe", - "GLWTPL", - "gnuplot", - "GPL-1.0", - "GPL-1.0-only", - "GPL-1.0-or-later", - "GPL-1.0+", - "GPL-2.0", - "GPL-2.0-only", - "GPL-2.0-or-later", - "GPL-2.0-with-autoconf-exception", - "GPL-2.0-with-bison-exception", - "GPL-2.0-with-classpath-exception", - "GPL-2.0-with-font-exception", - "GPL-2.0-with-GCC-exception", - "GPL-2.0+", - "GPL-3.0", - "GPL-3.0-only", - "GPL-3.0-or-later", - "GPL-3.0-with-autoconf-exception", - "GPL-3.0-with-GCC-exception", - "GPL-3.0+", - "gSOAP-1.3b", - "HaskellReport", - "Hippocratic-2.1", - "HPND", - "HPND-sell-variant", - "HTMLTIDY", - "IBM-pibs", - "ICU", - "IJG", - "ImageMagick", - "iMatix", - "Imlib2", - "Info-ZIP", - "Intel", - "Intel-ACPI", - "Interbase-1.0", - "IPA", - "IPL-1.0", - "ISC", - "JasPer-2.0", - "JPNIC", - "JSON", - "LAL-1.2", - "LAL-1.3", - "Latex2e", - "Leptonica", - "LGPL-2.0", - "LGPL-2.0-only", - "LGPL-2.0-or-later", - "LGPL-2.0+", - "LGPL-2.1", - "LGPL-2.1-only", - "LGPL-2.1-or-later", - "LGPL-2.1+", - "LGPL-3.0", - "LGPL-3.0-only", - "LGPL-3.0-or-later", - "LGPL-3.0+", - "LGPLLR", - "Libpng", - "libpng-2.0", - "libselinux-1.0", - "libtiff", - "LiLiQ-P-1.1", - "LiLiQ-R-1.1", - "LiLiQ-Rplus-1.1", - "Linux-OpenIB", - "LPL-1.0", - "LPL-1.02", - "LPPL-1.0", - "LPPL-1.1", - "LPPL-1.2", - "LPPL-1.3a", - "LPPL-1.3c", - "MakeIndex", - "MirOS", - "MIT", - "MIT-0", - "MIT-advertising", - "MIT-CMU", - "MIT-enna", - "MIT-feh", - "MIT-Modern-Variant", - "MIT-open-group", - "MITNFA", - "Motosoto", - "mpich2", - "MPL-1.0", - "MPL-1.1", - "MPL-2.0", - "MPL-2.0-no-copyleft-exception", - "MS-PL", - "MS-RL", - "MTLL", - "MulanPSL-1.0", - "MulanPSL-2.0", - "Multics", - "Mup", - "NAIST-2003", - "NASA-1.3", - "Naumen", - "NBPL-1.0", - "NCGL-UK-2.0", - "NCSA", - "Net-SNMP", - "NetCDF", - "Newsletr", - "NGPL", - "NIST-PD", - "NIST-PD-fallback", - "NLOD-1.0", - "NLOD-2.0", - "NLPL", - "Nokia", - "NOSL", - "Noweb", - "NPL-1.0", - "NPL-1.1", - "NPOSL-3.0", - "NRL", - "NTP", - "NTP-0", - "Nunit", - "O-UDA-1.0", - "OCCT-PL", - "OCLC-2.0", - "ODbL-1.0", - "ODC-By-1.0", - "OFL-1.0", - "OFL-1.0-no-RFN", - "OFL-1.0-RFN", - "OFL-1.1", - "OFL-1.1-no-RFN", - "OFL-1.1-RFN", - "OGC-1.0", - "OGDL-Taiwan-1.0", - "OGL-Canada-2.0", - "OGL-UK-1.0", - "OGL-UK-2.0", - "OGL-UK-3.0", - "OGTSL", - "OLDAP-1.1", - "OLDAP-1.2", - "OLDAP-1.3", - "OLDAP-1.4", - "OLDAP-2.0", - "OLDAP-2.0.1", - "OLDAP-2.1", - "OLDAP-2.2", - "OLDAP-2.2.1", - "OLDAP-2.2.2", - "OLDAP-2.3", - "OLDAP-2.4", - "OLDAP-2.5", - "OLDAP-2.6", - "OLDAP-2.7", - "OLDAP-2.8", - "OML", - "OpenSSL", - "OPL-1.0", - "OPUBL-1.0", - "OSET-PL-2.1", - "OSL-1.0", - "OSL-1.1", - "OSL-2.0", - "OSL-2.1", - "OSL-3.0", - "Parity-6.0.0", - "Parity-7.0.0", - "PDDL-1.0", - "PHP-3.0", - "PHP-3.01", - "Plexus", - "PolyForm-Noncommercial-1.0.0", - "PolyForm-Small-Business-1.0.0", - "PostgreSQL", - "PSF-2.0", - "psfrag", - "psutils", - "Python-2.0", - "Qhull", - "QPL-1.0", - "Rdisc", - "RHeCos-1.1", - "RPL-1.1", - "RPL-1.5", - "RPSL-1.0", - "RSA-MD", - "RSCPL", - "Ruby", - "SAX-PD", - "Saxpath", - "SCEA", - "Sendmail", - "Sendmail-8.23", - "SGI-B-1.0", - "SGI-B-1.1", - "SGI-B-2.0", - "SHL-0.5", - "SHL-0.51", - "SimPL-2.0", - "SISSL", - "SISSL-1.2", - "Sleepycat", - "SMLNJ", - "SMPPL", - "SNIA", - "Spencer-86", - "Spencer-94", - "Spencer-99", - "SPL-1.0", - "SSH-OpenSSH", - "SSH-short", - "SSPL-1.0", - "StandardML-NJ", - "SugarCRM-1.1.3", - "SWL", - "TAPR-OHL-1.0", - "TCL", - "TCP-wrappers", - "TMate", - "TORQUE-1.1", - "TOSL", - "TU-Berlin-1.0", - "TU-Berlin-2.0", - "UCL-1.0", - "Unicode-DFS-2015", - "Unicode-DFS-2016", - "Unicode-TOU", - "Unlicense", - "UPL-1.0", - "Verbatim-man-pages", - "Vim", - "VOSTROM", - "VSL-1.0", - "W3C", - "W3C-19980720", - "W3C-20150513", - "Watcom-1.0", - "Wsuipa", - "WTFPL", - "wxWindows", - "X11", - "Xerox", - "XFree86-1.1", - "xinetd", - "Xnet", - "xpp", - "XSkat", - "YPL-1.0", - "YPL-1.1", - "Zed", - "Zend-2.0", - "Zimbra-1.3", - "Zimbra-1.4", - "Zlib", - "zlib-acknowledgement", - "ZPL-1.1", - "ZPL-2.0", - "ZPL-2.1", -} +// Generated by Generate-SpdxLicenseList.ps1 +"0BSD", +"AAL", +"Abstyles", +"Adobe-2006", +"Adobe-Glyph", +"ADSL", +"AFL-1.1", +"AFL-1.2", +"AFL-2.0", +"AFL-2.1", +"AFL-3.0", +"Afmparse", +"AGPL-1.0", +"AGPL-1.0-only", +"AGPL-1.0-or-later", +"AGPL-3.0", +"AGPL-3.0-only", +"AGPL-3.0-or-later", +"Aladdin", +"AMDPLPA", +"AML", +"AMPAS", +"ANTLR-PD", +"ANTLR-PD-fallback", +"Apache-1.0", +"Apache-1.1", +"Apache-2.0", +"APAFML", +"APL-1.0", +"APSL-1.0", +"APSL-1.1", +"APSL-1.2", +"APSL-2.0", +"Artistic-1.0", +"Artistic-1.0-cl8", +"Artistic-1.0-Perl", +"Artistic-2.0", +"Bahyph", +"Barr", +"Beerware", +"BitTorrent-1.0", +"BitTorrent-1.1", +"blessing", +"BlueOak-1.0.0", +"Borceux", +"BSD-1-Clause", +"BSD-2-Clause", +"BSD-2-Clause-FreeBSD", +"BSD-2-Clause-NetBSD", +"BSD-2-Clause-Patent", +"BSD-2-Clause-Views", +"BSD-3-Clause", +"BSD-3-Clause-Attribution", +"BSD-3-Clause-Clear", +"BSD-3-Clause-LBNL", +"BSD-3-Clause-Modification", +"BSD-3-Clause-No-Military-License", +"BSD-3-Clause-No-Nuclear-License", +"BSD-3-Clause-No-Nuclear-License-2014", +"BSD-3-Clause-No-Nuclear-Warranty", +"BSD-3-Clause-Open-MPI", +"BSD-4-Clause", +"BSD-4-Clause-Shortened", +"BSD-4-Clause-UC", +"BSD-Protection", +"BSD-Source-Code", +"BSL-1.0", +"BUSL-1.1", +"bzip2-1.0.5", +"bzip2-1.0.6", +"C-UDA-1.0", +"CAL-1.0", +"CAL-1.0-Combined-Work-Exception", +"Caldera", +"CATOSL-1.1", +"CC-BY-1.0", +"CC-BY-2.0", +"CC-BY-2.5", +"CC-BY-2.5-AU", +"CC-BY-3.0", +"CC-BY-3.0-AT", +"CC-BY-3.0-DE", +"CC-BY-3.0-NL", +"CC-BY-3.0-US", +"CC-BY-4.0", +"CC-BY-NC-1.0", +"CC-BY-NC-2.0", +"CC-BY-NC-2.5", +"CC-BY-NC-3.0", +"CC-BY-NC-3.0-DE", +"CC-BY-NC-4.0", +"CC-BY-NC-ND-1.0", +"CC-BY-NC-ND-2.0", +"CC-BY-NC-ND-2.5", +"CC-BY-NC-ND-3.0", +"CC-BY-NC-ND-3.0-DE", +"CC-BY-NC-ND-3.0-IGO", +"CC-BY-NC-ND-4.0", +"CC-BY-NC-SA-1.0", +"CC-BY-NC-SA-2.0", +"CC-BY-NC-SA-2.0-FR", +"CC-BY-NC-SA-2.0-UK", +"CC-BY-NC-SA-2.5", +"CC-BY-NC-SA-3.0", +"CC-BY-NC-SA-3.0-DE", +"CC-BY-NC-SA-3.0-IGO", +"CC-BY-NC-SA-4.0", +"CC-BY-ND-1.0", +"CC-BY-ND-2.0", +"CC-BY-ND-2.5", +"CC-BY-ND-3.0", +"CC-BY-ND-3.0-DE", +"CC-BY-ND-4.0", +"CC-BY-SA-1.0", +"CC-BY-SA-2.0", +"CC-BY-SA-2.0-UK", +"CC-BY-SA-2.1-JP", +"CC-BY-SA-2.5", +"CC-BY-SA-3.0", +"CC-BY-SA-3.0-AT", +"CC-BY-SA-3.0-DE", +"CC-BY-SA-4.0", +"CC-PDDC", +"CC0-1.0", +"CDDL-1.0", +"CDDL-1.1", +"CDL-1.0", +"CDLA-Permissive-1.0", +"CDLA-Permissive-2.0", +"CDLA-Sharing-1.0", +"CECILL-1.0", +"CECILL-1.1", +"CECILL-2.0", +"CECILL-2.1", +"CECILL-B", +"CECILL-C", +"CERN-OHL-1.1", +"CERN-OHL-1.2", +"CERN-OHL-P-2.0", +"CERN-OHL-S-2.0", +"CERN-OHL-W-2.0", +"ClArtistic", +"CNRI-Jython", +"CNRI-Python", +"CNRI-Python-GPL-Compatible", +"Condor-1.1", +"copyleft-next-0.3.0", +"copyleft-next-0.3.1", +"CPAL-1.0", +"CPL-1.0", +"CPOL-1.02", +"Crossword", +"CrystalStacker", +"CUA-OPL-1.0", +"Cube", +"curl", +"D-FSL-1.0", +"diffmark", +"DOC", +"Dotseqn", +"DRL-1.0", +"DSDP", +"dvipdfm", +"ECL-1.0", +"ECL-2.0", +"eCos-2.0", +"EFL-1.0", +"EFL-2.0", +"eGenix", +"Entessa", +"EPICS", +"EPL-1.0", +"EPL-2.0", +"ErlPL-1.1", +"etalab-2.0", +"EUDatagrid", +"EUPL-1.0", +"EUPL-1.1", +"EUPL-1.2", +"Eurosym", +"Fair", +"Frameworx-1.0", +"FreeBSD-DOC", +"FreeImage", +"FSFAP", +"FSFUL", +"FSFULLR", +"FTL", +"GD", +"GFDL-1.1", +"GFDL-1.1-invariants-only", +"GFDL-1.1-invariants-or-later", +"GFDL-1.1-no-invariants-only", +"GFDL-1.1-no-invariants-or-later", +"GFDL-1.1-only", +"GFDL-1.1-or-later", +"GFDL-1.2", +"GFDL-1.2-invariants-only", +"GFDL-1.2-invariants-or-later", +"GFDL-1.2-no-invariants-only", +"GFDL-1.2-no-invariants-or-later", +"GFDL-1.2-only", +"GFDL-1.2-or-later", +"GFDL-1.3", +"GFDL-1.3-invariants-only", +"GFDL-1.3-invariants-or-later", +"GFDL-1.3-no-invariants-only", +"GFDL-1.3-no-invariants-or-later", +"GFDL-1.3-only", +"GFDL-1.3-or-later", +"Giftware", +"GL2PS", +"Glide", +"Glulxe", +"GLWTPL", +"gnuplot", +"GPL-1.0", +"GPL-1.0-only", +"GPL-1.0-or-later", +"GPL-1.0+", +"GPL-2.0", +"GPL-2.0-only", +"GPL-2.0-or-later", +"GPL-2.0-with-autoconf-exception", +"GPL-2.0-with-bison-exception", +"GPL-2.0-with-classpath-exception", +"GPL-2.0-with-font-exception", +"GPL-2.0-with-GCC-exception", +"GPL-2.0+", +"GPL-3.0", +"GPL-3.0-only", +"GPL-3.0-or-later", +"GPL-3.0-with-autoconf-exception", +"GPL-3.0-with-GCC-exception", +"GPL-3.0+", +"gSOAP-1.3b", +"HaskellReport", +"Hippocratic-2.1", +"HPND", +"HPND-sell-variant", +"HTMLTIDY", +"IBM-pibs", +"ICU", +"IJG", +"ImageMagick", +"iMatix", +"Imlib2", +"Info-ZIP", +"Intel", +"Intel-ACPI", +"Interbase-1.0", +"IPA", +"IPL-1.0", +"ISC", +"JasPer-2.0", +"JPNIC", +"JSON", +"LAL-1.2", +"LAL-1.3", +"Latex2e", +"Leptonica", +"LGPL-2.0", +"LGPL-2.0-only", +"LGPL-2.0-or-later", +"LGPL-2.0+", +"LGPL-2.1", +"LGPL-2.1-only", +"LGPL-2.1-or-later", +"LGPL-2.1+", +"LGPL-3.0", +"LGPL-3.0-only", +"LGPL-3.0-or-later", +"LGPL-3.0+", +"LGPLLR", +"Libpng", +"libpng-2.0", +"libselinux-1.0", +"libtiff", +"LiLiQ-P-1.1", +"LiLiQ-R-1.1", +"LiLiQ-Rplus-1.1", +"Linux-OpenIB", +"LPL-1.0", +"LPL-1.02", +"LPPL-1.0", +"LPPL-1.1", +"LPPL-1.2", +"LPPL-1.3a", +"LPPL-1.3c", +"MakeIndex", +"MirOS", +"MIT", +"MIT-0", +"MIT-advertising", +"MIT-CMU", +"MIT-enna", +"MIT-feh", +"MIT-Modern-Variant", +"MIT-open-group", +"MITNFA", +"Motosoto", +"mpich2", +"MPL-1.0", +"MPL-1.1", +"MPL-2.0", +"MPL-2.0-no-copyleft-exception", +"MS-PL", +"MS-RL", +"MTLL", +"MulanPSL-1.0", +"MulanPSL-2.0", +"Multics", +"Mup", +"NAIST-2003", +"NASA-1.3", +"Naumen", +"NBPL-1.0", +"NCGL-UK-2.0", +"NCSA", +"Net-SNMP", +"NetCDF", +"Newsletr", +"NGPL", +"NIST-PD", +"NIST-PD-fallback", +"NLOD-1.0", +"NLOD-2.0", +"NLPL", +"Nokia", +"NOSL", +"Noweb", +"NPL-1.0", +"NPL-1.1", +"NPOSL-3.0", +"NRL", +"NTP", +"NTP-0", +"Nunit", +"O-UDA-1.0", +"OCCT-PL", +"OCLC-2.0", +"ODbL-1.0", +"ODC-By-1.0", +"OFL-1.0", +"OFL-1.0-no-RFN", +"OFL-1.0-RFN", +"OFL-1.1", +"OFL-1.1-no-RFN", +"OFL-1.1-RFN", +"OGC-1.0", +"OGDL-Taiwan-1.0", +"OGL-Canada-2.0", +"OGL-UK-1.0", +"OGL-UK-2.0", +"OGL-UK-3.0", +"OGTSL", +"OLDAP-1.1", +"OLDAP-1.2", +"OLDAP-1.3", +"OLDAP-1.4", +"OLDAP-2.0", +"OLDAP-2.0.1", +"OLDAP-2.1", +"OLDAP-2.2", +"OLDAP-2.2.1", +"OLDAP-2.2.2", +"OLDAP-2.3", +"OLDAP-2.4", +"OLDAP-2.5", +"OLDAP-2.6", +"OLDAP-2.7", +"OLDAP-2.8", +"OML", +"OpenSSL", +"OPL-1.0", +"OPUBL-1.0", +"OSET-PL-2.1", +"OSL-1.0", +"OSL-1.1", +"OSL-2.0", +"OSL-2.1", +"OSL-3.0", +"Parity-6.0.0", +"Parity-7.0.0", +"PDDL-1.0", +"PHP-3.0", +"PHP-3.01", +"Plexus", +"PolyForm-Noncommercial-1.0.0", +"PolyForm-Small-Business-1.0.0", +"PostgreSQL", +"PSF-2.0", +"psfrag", +"psutils", +"Python-2.0", +"Qhull", +"QPL-1.0", +"Rdisc", +"RHeCos-1.1", +"RPL-1.1", +"RPL-1.5", +"RPSL-1.0", +"RSA-MD", +"RSCPL", +"Ruby", +"SAX-PD", +"Saxpath", +"SCEA", +"Sendmail", +"Sendmail-8.23", +"SGI-B-1.0", +"SGI-B-1.1", +"SGI-B-2.0", +"SHL-0.5", +"SHL-0.51", +"SimPL-2.0", +"SISSL", +"SISSL-1.2", +"Sleepycat", +"SMLNJ", +"SMPPL", +"SNIA", +"Spencer-86", +"Spencer-94", +"Spencer-99", +"SPL-1.0", +"SSH-OpenSSH", +"SSH-short", +"SSPL-1.0", +"StandardML-NJ", +"SugarCRM-1.1.3", +"SWL", +"TAPR-OHL-1.0", +"TCL", +"TCP-wrappers", +"TMate", +"TORQUE-1.1", +"TOSL", +"TU-Berlin-1.0", +"TU-Berlin-2.0", +"UCL-1.0", +"Unicode-DFS-2015", +"Unicode-DFS-2016", +"Unicode-TOU", +"Unlicense", +"UPL-1.0", +"Verbatim-man-pages", +"Vim", +"VOSTROM", +"VSL-1.0", +"W3C", +"W3C-19980720", +"W3C-20150513", +"Watcom-1.0", +"Wsuipa", +"WTFPL", +"wxWindows", +"X11", +"Xerox", +"XFree86-1.1", +"xinetd", +"Xnet", +"xpp", +"XSkat", +"YPL-1.0", +"YPL-1.1", +"Zed", +"Zend-2.0", +"Zimbra-1.3", +"Zimbra-1.4", +"Zlib", +"zlib-acknowledgement", +"ZPL-1.1", +"ZPL-2.0", +"ZPL-2.1",