diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6065715868..608ff9310f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -80,7 +80,6 @@ jobs: shell: pwsh run: | cd out/build/${{ matrix.preset }} - Import-Module Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99' ${{ github.workspace }}/azure-pipelines/end-to-end-tests.ps1 -RunArtifactsTests env: VCPKG_ROOT: ${{ github.workspace }}/vcpkg-root diff --git a/azure-pipelines/end-to-end-tests-dir/fetch.ps1 b/azure-pipelines/end-to-end-tests-dir/fetch.ps1 index 831b70f1cb..1a0e332be6 100644 --- a/azure-pipelines/end-to-end-tests-dir/fetch.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/fetch.ps1 @@ -1,95 +1,123 @@ . $PSScriptRoot/../end-to-end-tests-prelude.ps1 +$VcpkgToolsJsonSchemaFile = (Get-Item "$PSScriptRoot/../../docs/vcpkg-tools.schema.json").FullName +if (-not (Test-Json -ea:0 -LiteralPath "$VcpkgRoot/scripts/vcpkg-tools.json" -SchemaFile $VcpkgToolsJsonSchemaFile)) { + throw "real vcpkg-tools.json doesn't conform to schema" +} + if (-not $IsMacOS -and -not $IsLinux) { "" | Out-File -enc ascii $(Join-Path $TestingRoot .vcpkg-root) $Scripts = Join-Path $TestingRoot "scripts" mkdir $Scripts | Out-Null + + $VcpkgToolsJson = Join-Path $Scripts "vcpkg-tools.json" + + $7zip_version = "19.00" + $ninja_version = "1.10.2" -@" - - - - 19.00 - Files\7-Zip\7z.exe - https://www.7-zip.org/a/7z1900-x64.msi - 7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8 - 7z1900-x64.msi - - - 1.10.2 - ninja.exe - https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip - 6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3 - ninja-win-1.10.2.zip - - - 1.10.2 - ninja.exe - https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip - 6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3 - ninja-win-1.10.2.zip - - - 3.22.2 - cmake-3.22.2-windows-i386\bin\cmake.exe - https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip - 969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79 - cmake-3.22.2-windows-i386.zip - - - 3.22.2 - cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake - https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz - 08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac - cmake-3.22.2-macos-universal.tar.gz - - - 3.22.2 - cmake-3.22.2-linux-x86_64/bin/cmake - https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz - 579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b - cmake-3.22.2linux-x86_64.tar.gz - - - 3.20.4 - usr/local/bin/cmake - https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz - 3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475 - cmake-3.20.4.txz - - -"@ | % { $_ -replace "`r","" } | Out-File -enc ascii $(Join-Path $Scripts "vcpkgTools.xml") + @' +{ + "schema-version": 1, + "tools": [{ + "name": "7zip", + "os": "windows", + "version": "19.00", + "executable": "Files\\7-Zip\\7z.exe", + "url": "https://www.7-zip.org/a/7z1900-x64.msi", + "sha512": "7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8", + "archive": "7z1900-x64.msi" + }, + { + "name": "ninja-testing", + "os": "windows", + "version": "1.10.2", + "executable": "ninja.exe", + "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip", + "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3", + "archive": "ninja-win-1.10.2.zip" + }, + { + "name": "ninja", + "os": "windows", + "version": "1.10.2", + "executable": "ninja.exe", + "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip", + "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3", + "archive": "ninja-win-1.10.2.zip" + }, + { + "name": "cmake", + "os": "windows", + "version": "3.22.2", + "executable": "cmake-3.22.2-windows-i386\\bin\\cmake.exe", + "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip", + "sha512": "969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79", + "archive": "cmake-3.22.2-windows-i386.zip" + }, + { + "name": "cmake", + "os": "osx", + "version": "3.22.2", + "executable": "cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake", + "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz", + "sha512": "08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac", + "archive": "cmake-3.22.2-macos-universal.tar.gz" + }, + { + "name": "cmake", + "os": "linux", + "version": "3.22.2", + "executable": "cmake-3.22.2-linux-x86_64/bin/cmake", + "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz", + "sha512": "579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b", + "archive": "cmake-3.22.2linux-x86_64.tar.gz" + }, + { + "name": "cmake", + "os": "freebsd", + "version": "3.20.4", + "executable": "/usr/local/bin/cmake", + "url": "https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz", + "sha512": "3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475", + "archive": "cmake-3.20.4.txz" + }] +} +'@ | % { $_ -replace "`r","" } | Out-File -enc ascii $VcpkgToolsJson + + if (-not (Test-Json -ea:0 -LiteralPath $VcpkgToolsJson -SchemaFile $VcpkgToolsJsonSchemaFile)) { + throw "testing vcpkg-tools.json doesn't conform to schema" + } $env:VCPKG_DOWNLOADS = Join-Path $TestingRoot 'down loads' Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "7zip", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileExists "$TestingRoot/down loads/tools/7zip-19.00-windows/Files/7-Zip/7z.exe" + Require-FileExists "$env:VCPKG_DOWNLOADS/tools/7zip-19.00-windows/Files/7-Zip/7z.exe" Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja-testing", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileExists "$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows/ninja.exe" + Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows/ninja.exe" $path = $env:PATH - $env:PATH = "$path;$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows" + $env:PATH = "$path;$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows" Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe" + Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe" $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1" Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe" + Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe" - Remove-Item -Recurse -Force "$TestingRoot/down loads/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES $env:VCPKG_FORCE_SYSTEM_BINARIES = "1" $env:PATH = "$PSScriptRoot\..\e2e-assets\fetch;$path" Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe" + Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe" Remove-Item env:VCPKG_FORCE_SYSTEM_BINARIES $out = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot", "--x-stderr-status")) diff --git a/azure-pipelines/end-to-end-tests.ps1 b/azure-pipelines/end-to-end-tests.ps1 index 8bafa8d563..88aefaa76c 100755 --- a/azure-pipelines/end-to-end-tests.ps1 +++ b/azure-pipelines/end-to-end-tests.ps1 @@ -41,6 +41,10 @@ if ($PSVersionTable.PSVersion.Major -lt 7) { Write-Error "vcpkg end to end tests must use pwsh rather than Windows PowerShell" } +# If you get an error on the next line, install Pester from an administrative command prompt with: +# Install-Module -Name Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99' -Scope AllUsers +Import-Module Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99' + if ($IsLinux) { $Triplet = 'x64-linux' } elseif ($IsMacOS) { diff --git a/docs/vcpkg-tools.schema.json b/docs/vcpkg-tools.schema.json new file mode 100644 index 0000000000..f211e3c424 --- /dev/null +++ b/docs/vcpkg-tools.schema.json @@ -0,0 +1,69 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg-tools.schema.json", + "type": "object", + "properties": { + "schema-version": { + "description": "Identifies the version of the tools database format; currently always 1.", + "type": "number", + "minimum": 1, + "maximum": 1 + }, + "tools": { + "type": "array", + "description": "The set of known tools.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the tool as exposed to `vcpkg fetch`." + }, + "os": { + "type": "string", + "description": "The platform where the record is valid.", + "enum": [ "windows", "osx", "linux", "freebsd", "openbsd" ] + }, + "version": { + "type": "string", + "description": "The version to match against against a system provided copy of this tool.", + "pattern": "\\d+(\\.\\d+(\\.\\d+)?)?" + }, + "arch": { + "type": "string", + "description": "The architecture where the record is valid.", + "enum": [ "x86", "x64", "amd64", "arm", "arm64", "arm64ec", "s390x", "ppc64le", "riscv32", "riscv64", "loongarch32", "loongarch64", "mips64" ] + }, + "executable": { + "type": "string", + "description": "The relative path to the executable for the tool within the downloaded archive, if any." + }, + "url": { + "type": "string", + "description": "The URL to download the tool from.", + "format": "uri" + }, + "sha512": { + "type": "string", + "description": "The SHA-512 hash of the downloaded archive.", + "pattern": "^[0-9a-fA-F]{128}$" + }, + "archive": { + "type": "string", + "description": "The name of the downloaded archive." + } + }, + "patternProperties": { + "^\\$": {} + }, + "required": ["name", "os", "version"], + "additionalProperties": false + } + } + }, + "patternProperties": { + "^\\$": {} + }, + "additionalProperties": false, + "required": [ "schema-version", "tools" ] +} diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h index ac6bee38f2..fdb5c6cbba 100644 --- a/include/vcpkg/base/contractual-constants.h +++ b/include/vcpkg/base/contractual-constants.h @@ -12,9 +12,11 @@ namespace vcpkg inline constexpr StringLiteral JsonIdAcquiredArtifacts = "acquired-artifacts"; inline constexpr StringLiteral JsonIdActivatedArtifacts = "activated-artifacts"; inline constexpr StringLiteral JsonIdAlgorithm = "algorithm"; + inline constexpr StringLiteral JsonIdArchive = "archive"; inline constexpr StringLiteral JsonIdAllCapsSHA256 = "SHA256"; inline constexpr StringLiteral JsonIdAllCapsSHA512 = "SHA512"; inline constexpr StringLiteral JsonIdApply = "apply"; + inline constexpr StringLiteral JsonIdArch = "arch"; inline constexpr StringLiteral JsonIdArchiveCapitalLocation = "archiveLocation"; inline constexpr StringLiteral JsonIdArtifact = "artifact"; inline constexpr StringLiteral JsonIdBaseline = "baseline"; @@ -46,6 +48,7 @@ namespace vcpkg inline constexpr StringLiteral JsonIdDollarSchema = "$schema"; inline constexpr StringLiteral JsonIdDownloads = "downloads"; inline constexpr StringLiteral JsonIdError = "error"; + inline constexpr StringLiteral JsonIdExecutable = "executable"; inline constexpr StringLiteral JsonIdFeatures = "features"; inline constexpr StringLiteral JsonIdFiles = "files"; inline constexpr StringLiteral JsonIdFilesystem = "filesystem"; @@ -68,6 +71,7 @@ namespace vcpkg inline constexpr StringLiteral JsonIdMessage = "message"; inline constexpr StringLiteral JsonIdMicrosoft = "microsoft"; inline constexpr StringLiteral JsonIdName = "name"; + inline constexpr StringLiteral JsonIdOS = "os"; inline constexpr StringLiteral JsonIdOverlayPorts = "overlay-ports"; inline constexpr StringLiteral JsonIdOverlayTriplets = "overlay-triplets"; inline constexpr StringLiteral JsonIdOverrides = "overrides"; @@ -87,8 +91,10 @@ namespace vcpkg inline constexpr StringLiteral JsonIdRequires = "requires"; inline constexpr StringLiteral JsonIdResolved = "resolved"; inline constexpr StringLiteral JsonIdScanned = "scanned"; + inline constexpr StringLiteral JsonIdSchemaVersion = "schema-version"; inline constexpr StringLiteral JsonIdSettings = "settings"; inline constexpr StringLiteral JsonIdSha = "sha"; + inline constexpr StringLiteral JsonIdSha512 = "sha512"; inline constexpr StringLiteral JsonIdState = "state"; inline constexpr StringLiteral JsonIdSummary = "summary"; inline constexpr StringLiteral JsonIdSupports = "supports"; @@ -290,6 +296,7 @@ namespace vcpkg inline constexpr StringLiteral SwitchTargetX86 = "target:x86"; inline constexpr StringLiteral SwitchTLogFile = "tlog-file"; inline constexpr StringLiteral SwitchTools = "tools"; + inline constexpr StringLiteral SwitchToolDataFile = "tool-data-file"; inline constexpr StringLiteral SwitchTriplet = "triplet"; inline constexpr StringLiteral SwitchUrl = "url"; inline constexpr StringLiteral SwitchVcpkgRoot = "vcpkg-root"; diff --git a/include/vcpkg/base/jsonreader.h b/include/vcpkg/base/jsonreader.h index e4c3522ed7..542cdb741e 100644 --- a/include/vcpkg/base/jsonreader.h +++ b/include/vcpkg/base/jsonreader.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -30,7 +31,7 @@ namespace vcpkg::Json virtual Optional visit_string(Reader&, StringView) const; virtual Optional visit_array(Reader&, const Array&) const; virtual Optional visit_object(Reader&, const Object&) const; - virtual View valid_fields() const; + virtual View valid_fields() const noexcept; virtual ~IDeserializer() = default; @@ -100,7 +101,7 @@ namespace vcpkg::Json // * are not in `valid_fields` // if known_fields.empty(), then it's treated as if all field names are valid void check_for_unexpected_fields(const Object& obj, - View valid_fields, + View valid_fields, const LocalizedString& type_name); template @@ -238,7 +239,7 @@ namespace vcpkg::Json } template - View IDeserializer::valid_fields() const + View IDeserializer::valid_fields() const noexcept { return {}; } @@ -361,4 +362,18 @@ namespace vcpkg::Json virtual Optional visit_string(Json::Reader&, StringView sv) const override; static const FeatureNameDeserializer instance; }; + + struct ArchitectureDeserializer final : Json::IDeserializer> + { + virtual LocalizedString type_name() const override; + virtual Optional> visit_string(Json::Reader&, StringView sv) const override; + static const ArchitectureDeserializer instance; + }; + + struct Sha512Deserializer final : Json::IDeserializer + { + virtual LocalizedString type_name() const override; + virtual Optional visit_string(Json::Reader&, StringView sv) const override; + static const Sha512Deserializer instance; + }; } diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 123a0c1fd8..7357d2a412 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -4,6 +4,7 @@ DECLARE_MESSAGE(ADefaultFeature, (), "", "a default feature") DECLARE_MESSAGE(ABoolean, (), "", "a boolean") DECLARE_MESSAGE(ABuiltinRegistry, (), "", "a builtin registry") DECLARE_MESSAGE(AConfigurationObject, (), "", "a configuration object") +DECLARE_MESSAGE(ACpuArchitecture, (), "", "a CPU architecture") DECLARE_MESSAGE(ADependency, (), "", "a dependency") DECLARE_MESSAGE(ADependencyFeature, (), "", "a feature of a dependency") DECLARE_MESSAGE(ADemandObject, @@ -11,6 +12,7 @@ DECLARE_MESSAGE(ADemandObject, "'demands' are a concept in the schema of a JSON file the user can edit", "a demand object") DECLARE_MESSAGE(AString, (), "", "a string") +DECLARE_MESSAGE(ASha512, (), "", "a SHA-512 hash") DECLARE_MESSAGE(ADateVersionString, (), "", "a date version string") DECLARE_MESSAGE(AddArtifactOnlyOne, (msg::command_line), "", "'{command_line}' can only add one artifact at a time.") DECLARE_MESSAGE(AddCommandFirstArg, (), "", "The first parameter to add must be 'artifact' or 'port'.") @@ -264,6 +266,15 @@ DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") DECLARE_MESSAGE(AStringOrArrayOfStrings, (), "", "a string or array of strings") DECLARE_MESSAGE(AStringStringDictionary, (), "", "a \"string\": \"string\" dictionary") +DECLARE_MESSAGE(AToolDataObject, (), "", "tool metadata") +DECLARE_MESSAGE(AToolDataArray, (), "", "an array of tool metadata") +DECLARE_MESSAGE(AToolDataFile, (), "", "a tool data file") +DECLARE_MESSAGE(AToolDataOS, (), "", "a tool data operating system") +DECLARE_MESSAGE(AToolDataVersion, (), "", "a tool data version") +DECLARE_MESSAGE(ToolDataFileSchemaVersionNotSupported, + (msg::version), + "", + "document schema version {version} is not supported by this version of vcpkg") DECLARE_MESSAGE(AttemptingToSetBuiltInBaseline, (), "", @@ -923,10 +934,6 @@ DECLARE_MESSAGE(CouldNotFindGitTreeAtCommit, (msg::package_name, msg::commit_sha), "", "could not find the git tree for `versions` in repo {package_name} at commit {commit_sha}") -DECLARE_MESSAGE(CouldNotFindToolVersion, - (msg::version, msg::path), - "", - "Could not find in {path}") DECLARE_MESSAGE(CouldNotFindVersionDatabaseFile, (msg::path), "", "Couldn't find the versions database file: {path}") DECLARE_MESSAGE(CreatedNuGetPackage, (msg::path), "", "Created nupkg: {path}") DECLARE_MESSAGE(CreateFailureLogsDir, (msg::path), "", "Creating failure logs output directory {path}.") @@ -1296,10 +1303,6 @@ DECLARE_MESSAGE(FailedToParseSerializedBinParagraph, "[sanity check] Failed to parse a serialized binary paragraph.\nPlease open an issue at " "https://github.com/microsoft/vcpkg, " "with the following output:\n{error_msg}\nSerialized Binary Paragraph:") -DECLARE_MESSAGE(FailedToParseVersionXML, - (msg::tool_name, msg::version), - "", - "Could not parse version for tool {tool_name}. Version string was: {version}") DECLARE_MESSAGE(FailedToRunToolToDetermineVersion, (msg::tool_name, msg::path), "Additional information, such as the command line output, if any, will be appended on " @@ -1824,10 +1827,11 @@ DECLARE_MESSAGE( "Please open an issue at " "https://github.com/microsoft/vcpkg/issues/new?template=other-type-of-bug-report.md&labels=category:vcpkg-bug " "with detailed steps to reproduce the problem.") -DECLARE_MESSAGE(InvalidArchitecture, - (msg::value), - "{value} is what the user entered that we did not understand", - "invalid architecture: {value}") +DECLARE_MESSAGE( + InvalidArchitectureValue, + (msg::value, msg::expected), + "{value} is an unknown CPU architecture type, {expected} is the list of accepted CPU architecture values", + "Invalid architecture: {value}. Expected one of: {expected}") DECLARE_MESSAGE(InvalidArgument, (), "", "invalid argument") DECLARE_MESSAGE( InvalidArgumentRequiresAbsolutePath, @@ -1950,12 +1954,26 @@ DECLARE_MESSAGE(InvalidOptionForRemove, "'remove' is a command that should not be changed.", "'remove' accepts either libraries or '--outdated'") DECLARE_MESSAGE(InvalidPortVersonName, (msg::path), "", "Found invalid port version file name: `{path}`.") +DECLARE_MESSAGE(InvalidSha512, + (msg::sha), + "", + "invalid SHA-512 hash: {sha}\n" + "SHA-512 hash must be 128 characters long and contain only hexadecimal digits") DECLARE_MESSAGE(InvalidSharpInVersion, (), "", "invalid character '#' in version text") DECLARE_MESSAGE(InvalidSharpInVersionDidYouMean, (msg::value), "{value} is an integer. `\"port-version\":' is JSON syntax and should be unlocalized", "invalid character '#' in version text. Did you mean \"port-version\": {value}?") DECLARE_MESSAGE(InvalidString, (), "", "Invalid utf8 passed to Value::string(std::string)") +DECLARE_MESSAGE(InvalidToolOSValue, + (msg::value, msg::expected), + "{value} is an unknown operating system, {expected} is the list of accepted operating system values", + "Invalid tool operating system: {value}. Expected one of: {expected}") +DECLARE_MESSAGE( + InvalidToolVersion, + (), + "", + "Invalid tool version; expected a string containing a substring of between 1 and 3 numbers separated by dots.") DECLARE_MESSAGE(InvalidTriplet, (msg::triplet), "", "Invalid triplet: {triplet}") DECLARE_MESSAGE(InvalidValueHashAdditionalFiles, (msg::path), @@ -3014,11 +3032,6 @@ DECLARE_MESSAGE(VersionCommandHeader, (msg::version), "", "vcpkg package management program version {version}\n\nSee LICENSE.txt for license information.") -DECLARE_MESSAGE( - VersionConflictXML, - (msg::path, msg::expected_version, msg::actual_version), - "", - "Expected {path} version: [{expected_version}], but was [{actual_version}]. Please re-run bootstrap-vcpkg.") DECLARE_MESSAGE(VersionConstraintNotInDatabase1, (msg::package_name, msg::version), "", diff --git a/include/vcpkg/base/strings.h b/include/vcpkg/base/strings.h index 2d27d27b29..af7ca77acf 100644 --- a/include/vcpkg/base/strings.h +++ b/include/vcpkg/base/strings.h @@ -122,7 +122,7 @@ namespace vcpkg::Strings const char* case_insensitive_ascii_search(StringView s, StringView pattern); bool case_insensitive_ascii_contains(StringView s, StringView pattern); - bool case_insensitive_ascii_equals(StringView left, StringView right); + bool case_insensitive_ascii_equals(StringView left, StringView right) noexcept; bool case_insensitive_ascii_less(StringView left, StringView right); void inplace_ascii_to_lowercase(char* first, char* last); diff --git a/include/vcpkg/base/system.h b/include/vcpkg/base/system.h index 1d1c334845..aaa4ec8b04 100644 --- a/include/vcpkg/base/system.h +++ b/include/vcpkg/base/system.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -44,9 +45,11 @@ namespace vcpkg long get_process_id(); - Optional to_cpu_architecture(StringView arch); + Optional to_cpu_architecture(StringView arch) noexcept; - ZStringView to_zstring_view(CPUArchitecture arch) noexcept; + StringLiteral to_string_literal(CPUArchitecture arch) noexcept; + + LocalizedString all_comma_separated_cpu_architectures(); CPUArchitecture get_host_processor(); @@ -62,3 +65,5 @@ namespace vcpkg Optional guess_visual_studio_prompt_target_architecture(); } + +VCPKG_FORMAT_WITH_TO_STRING_LITERAL_NONMEMBER(vcpkg::CPUArchitecture); diff --git a/include/vcpkg/tools.h b/include/vcpkg/tools.h index 80c64c1100..6542f324e3 100644 --- a/include/vcpkg/tools.h +++ b/include/vcpkg/tools.h @@ -62,7 +62,7 @@ namespace vcpkg std::unique_ptr get_tool_cache(const Filesystem& fs, std::shared_ptr downloader, Path downloads, - Path xml_config, + Path config_path, Path tools, RequireExactVersions abiToolVersionHandling); } diff --git a/include/vcpkg/tools.test.h b/include/vcpkg/tools.test.h index 212d41ba65..ce4fa47cd5 100644 --- a/include/vcpkg/tools.test.h +++ b/include/vcpkg/tools.test.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include @@ -11,6 +13,12 @@ namespace vcpkg { + struct ToolVersion + { + std::array cooked; // e.g. 24.8 or 2.7.4 or 1.0.0 + std::string raw; // e.g. 24.08 or 2.7.4 or 1.0 + }; + struct ToolData { std::string name; @@ -29,7 +37,69 @@ namespace vcpkg Path exe_path(const Path& tools_base_path) const { return tools_base_path / tool_dir_subpath / exe_subpath; } }; - Optional parse_tool_data_from_xml(StringView XML, StringView XML_PATH, StringView tool, StringView os); - Optional> parse_tool_version_string(StringView string_version); + + enum class ToolOs + { + Windows, + Osx, + Linux, + FreeBsd, + OpenBsd, + }; + + Optional to_tool_os(StringView os) noexcept; + StringLiteral to_string_literal(ToolOs os) noexcept; + LocalizedString all_comma_separated_tool_oses(); + + struct ToolDataEntry + { + std::string tool; + ToolOs os; + Optional arch; + ToolVersion version; + std::string exeRelativePath; + std::string url; + std::string sha512; + std::string archiveName; + }; + + ExpectedL> parse_tool_data(StringView contents, StringView origin); + + const ToolDataEntry* get_raw_tool_data(const std::vector& tool_data_table, + StringView toolname, + const CPUArchitecture arch, + const ToolOs os); + + struct ToolDataFileDeserializer final : Json::IDeserializer> + { + virtual LocalizedString type_name() const override; + + virtual View valid_fields() const noexcept override; + + virtual Optional> visit_object(Json::Reader& r, + const Json::Object& obj) const override; + + static const ToolDataFileDeserializer instance; + }; + + struct ToolOsDeserializer final : Json::IDeserializer + { + virtual LocalizedString type_name() const override; + + virtual Optional visit_string(Json::Reader& r, StringView str) const override; + + static const ToolOsDeserializer instance; + }; + + struct ToolVersionDeserializer final : Json::IDeserializer + { + virtual LocalizedString type_name() const override; + + virtual Optional visit_string(Json::Reader& r, StringView str) const override; + + static const ToolVersionDeserializer instance; + }; } + +VCPKG_FORMAT_WITH_TO_STRING_LITERAL_NONMEMBER(vcpkg::ToolOs); diff --git a/include/vcpkg/vcpkgcmdarguments.h b/include/vcpkg/vcpkgcmdarguments.h index 37109d3056..12b2452dd1 100644 --- a/include/vcpkg/vcpkgcmdarguments.h +++ b/include/vcpkg/vcpkgcmdarguments.h @@ -223,6 +223,7 @@ namespace vcpkg Optional builtin_ports_root_dir; Optional builtin_registry_versions_dir; Optional registries_cache_dir; + Optional tools_data_file; Optional default_visual_studio_path; diff --git a/include/vcpkg/versiondeserializers.h b/include/vcpkg/versiondeserializers.h index 82a5e8d29d..c68951ed25 100644 --- a/include/vcpkg/versiondeserializers.h +++ b/include/vcpkg/versiondeserializers.h @@ -4,6 +4,7 @@ #include +#include #include #include @@ -23,7 +24,8 @@ namespace vcpkg Json::Reader& r, const Json::Object& obj); - View schemed_deserializer_fields(); +#define VCPKG_SCHEMED_DESERIALIZER_FIELDS \ + JsonIdVersion, JsonIdVersionSemver, JsonIdVersionString, JsonIdVersionDate, JsonIdPortVersion void serialize_schemed_version(Json::Object& out_obj, VersionScheme scheme, const Version& version); diff --git a/locales/messages.json b/locales/messages.json index 27df37aee1..779bc18bd9 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -4,6 +4,7 @@ "ABoolean": "a boolean", "ABuiltinRegistry": "a builtin registry", "AConfigurationObject": "a configuration object", + "ACpuArchitecture": "a CPU architecture", "ADateVersionString": "a date version string", "ADefaultFeature": "a default feature", "ADemandObject": "a demand object", @@ -36,9 +37,15 @@ "ARelaxedVersionString": "a relaxed version string", "ASemanticVersionString": "a semantic version string", "ASetOfFeatures": "a set of features", + "ASha512": "a SHA-512 hash", "AString": "a string", "AStringOrArrayOfStrings": "a string or array of strings", "AStringStringDictionary": "a \"string\": \"string\" dictionary", + "AToolDataArray": "an array of tool metadata", + "AToolDataFile": "a tool data file", + "AToolDataOS": "a tool data operating system", + "AToolDataObject": "tool metadata", + "AToolDataVersion": "a tool data version", "AUrl": "a url", "AVcpkgRepositoryCommit": "a vcpkg repository commit", "AVersionConstraint": "a version constraint", @@ -529,8 +536,6 @@ "_CouldNotFindBaselineInCommit.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949. An example of {package_name} is zlib.", "CouldNotFindGitTreeAtCommit": "could not find the git tree for `versions` in repo {package_name} at commit {commit_sha}", "_CouldNotFindGitTreeAtCommit.comment": "An example of {package_name} is zlib. An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.", - "CouldNotFindToolVersion": "Could not find in {path}", - "_CouldNotFindToolVersion.comment": "An example of {version} is 1.3.8. An example of {path} is /foo/bar.", "CouldNotFindVersionDatabaseFile": "Couldn't find the versions database file: {path}", "_CouldNotFindVersionDatabaseFile.comment": "An example of {path} is /foo/bar.", "CreateFailureLogsDir": "Creating failure logs output directory {path}.", @@ -764,8 +769,6 @@ "_FailedToParseSerializedBinParagraph.comment": "'{error_msg}' is the error message for failing to parse the Binary Paragraph. An example of {error_msg} is File Not Found.", "FailedToParseVersionFile": "Failed to parse version file: {path}", "_FailedToParseVersionFile.comment": "An example of {path} is /foo/bar.", - "FailedToParseVersionXML": "Could not parse version for tool {tool_name}. Version string was: {version}", - "_FailedToParseVersionXML.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", "FailedToRunToolToDetermineVersion": "Failed to run \"{path}\" to determine the {tool_name} version.", "_FailedToRunToolToDetermineVersion.comment": "Additional information, such as the command line output, if any, will be appended on the line after this message An example of {tool_name} is aria2. An example of {path} is /foo/bar.", "FailedToStoreBackToMirror": "Failed to store {path} to {url}.", @@ -999,8 +1002,8 @@ "IntegrationFailedVS2015": "Integration was not applied for Visual Studio 2015.", "InternalCICommand": "vcpkg ci is an internal command which will change incompatibly or be removed at any time.", "InternalErrorMessageContact": "Please open an issue at https://github.com/microsoft/vcpkg/issues/new?template=other-type-of-bug-report.md&labels=category:vcpkg-bug with detailed steps to reproduce the problem.", - "InvalidArchitecture": "invalid architecture: {value}", - "_InvalidArchitecture.comment": "{value} is what the user entered that we did not understand", + "InvalidArchitectureValue": "Invalid architecture: {value}. Expected one of: {expected}", + "_InvalidArchitectureValue.comment": "{value} is an unknown CPU architecture type, {expected} is the list of accepted CPU architecture values", "InvalidArgument": "invalid argument", "InvalidArgumentRequiresAbsolutePath": "invalid argument: binary config '{binary_source}' path arguments for binary config strings must be absolute", "_InvalidArgumentRequiresAbsolutePath.comment": "An example of {binary_source} is azblob.", @@ -1068,10 +1071,15 @@ "_InvalidOptionForRemove.comment": "'remove' is a command that should not be changed.", "InvalidPortVersonName": "Found invalid port version file name: `{path}`.", "_InvalidPortVersonName.comment": "An example of {path} is /foo/bar.", + "InvalidSha512": "invalid SHA-512 hash: {sha}\nSHA-512 hash must be 128 characters long and contain only hexadecimal digits", + "_InvalidSha512.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", "InvalidSharpInVersion": "invalid character '#' in version text", "InvalidSharpInVersionDidYouMean": "invalid character '#' in version text. Did you mean \"port-version\": {value}?", "_InvalidSharpInVersionDidYouMean.comment": "{value} is an integer. `\"port-version\":' is JSON syntax and should be unlocalized", "InvalidString": "Invalid utf8 passed to Value::string(std::string)", + "InvalidToolOSValue": "Invalid tool operating system: {value}. Expected one of: {expected}", + "_InvalidToolOSValue.comment": "{value} is an unknown operating system, {expected} is the list of accepted operating system values", + "InvalidToolVersion": "Invalid tool version; expected a string containing a substring of between 1 and 3 numbers separated by dots.", "InvalidTriplet": "Invalid triplet: {triplet}", "_InvalidTriplet.comment": "An example of {triplet} is x64-windows.", "InvalidUri": "unable to parse uri: {value}", @@ -1436,6 +1444,8 @@ "_ToRemovePackages.comment": "An example of {command_name} is install.", "ToUpdatePackages": "To update these packages and all dependencies, run\n{command_name} upgrade'", "_ToUpdatePackages.comment": "An example of {command_name} is install.", + "ToolDataFileSchemaVersionNotSupported": "document schema version {version} is not supported by this version of vcpkg", + "_ToolDataFileSchemaVersionNotSupported.comment": "An example of {version} is 1.3.8.", "ToolFetchFailed": "Could not fetch {tool_name}.", "_ToolFetchFailed.comment": "An example of {tool_name} is aria2.", "ToolInWin10": "This utility is bundled with Windows 10 or later.", @@ -1596,8 +1606,6 @@ "_VersionBuiltinPortTreeEntryMissing.comment": "{expected} and {actual} are versions like 1.0. An example of {package_name} is zlib.", "VersionCommandHeader": "vcpkg package management program version {version}\n\nSee LICENSE.txt for license information.", "_VersionCommandHeader.comment": "An example of {version} is 1.3.8.", - "VersionConflictXML": "Expected {path} version: [{expected_version}], but was [{actual_version}]. Please re-run bootstrap-vcpkg.", - "_VersionConflictXML.comment": "An example of {path} is /foo/bar. An example of {expected_version} is 1.3.8. An example of {actual_version} is 1.3.8.", "VersionConstraintNotInDatabase1": "the \"version>=\" constraint to {package_name} names version {version} which does not exist in the version database. All versions must exist in the version database to be interpreted by vcpkg.", "_VersionConstraintNotInDatabase1.comment": "An example of {package_name} is zlib. An example of {version} is 1.3.8.", "VersionConstraintNotInDatabase2": "consider removing the version constraint or choosing a value declared here", diff --git a/src/vcpkg-test/system.cpp b/src/vcpkg-test/system.cpp index 4d0e771d87..9e50027d0c 100644 --- a/src/vcpkg-test/system.cpp +++ b/src/vcpkg-test/system.cpp @@ -78,7 +78,7 @@ TEST_CASE ("from_cpu_architecture", "[system]") for (auto&& instance : test_cases) { - CHECK(to_zstring_view(instance.input) == instance.expected); + CHECK(to_string_literal(instance.input) == instance.expected); } } diff --git a/src/vcpkg-test/tools.cpp b/src/vcpkg-test/tools.cpp index 431309af4c..1dfe52b993 100644 --- a/src/vcpkg-test/tools.cpp +++ b/src/vcpkg-test/tools.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include @@ -41,81 +43,6 @@ Copyright (C) 2006, 2019 Tatsuhiro Tsujikawa)"); CHECK_FALSE(result.has_value()); } -TEST_CASE ("parse_tool_data_from_xml", "[tools]") -{ - const StringView tool_doc = R"( - - - - 2.7.4 - - - - - - 5.11.0 - nuget.exe - https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe - 06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9ebfda83b64c03ce14c8f35ca9957a17a8c02b8c4b7 - - - 16.12.0 - node-v16.12.0-win-x64\node.exe - https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z - 0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023fb7068ed1f8a7678e446260c3db3537fa888 - node-v16.12.0-win-x64.7z - - -)"; - - { - auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "tool1", "windows"); - REQUIRE(!data.has_value()); - } - { - auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "node", "unknown"); - REQUIRE(!data.has_value()); - } - { - auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "node", "windows"); - REQUIRE(data.has_value()); - auto& p = *data.get(); - CHECK(p.is_archive); - CHECK(p.version == decltype(p.version){16, 12, 0}); - CHECK(p.tool_dir_subpath == "node-16.12.0-windows"); - CHECK(p.exe_subpath == "node-v16.12.0-win-x64\\node.exe"); - CHECK(p.download_subpath == "node-v16.12.0-win-x64.7z"); - CHECK(p.sha512 == "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023" - "fb7068ed1f8a7678e446260c3db3537fa888"); - CHECK(p.url == "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z"); - } - { - auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "nuget", "osx"); - REQUIRE(data.has_value()); - auto& p = *data.get(); - CHECK_FALSE(p.is_archive); - CHECK(p.version == decltype(p.version){5, 11, 0}); - CHECK(p.tool_dir_subpath == "nuget-5.11.0-osx"); - CHECK(p.exe_subpath == "nuget.exe"); - CHECK(p.download_subpath == "06a337c9-nuget.exe"); - CHECK(p.sha512 == "06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9ebfda8" - "3b64c03ce14c8f35ca9957a17a8c02b8c4b7"); - CHECK(p.url == "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe"); - } - { - auto data = parse_tool_data_from_xml(tool_doc, "vcpkgTools.xml", "git", "linux"); - REQUIRE(data.has_value()); - auto& p = *data.get(); - CHECK_FALSE(p.is_archive); - CHECK(p.version == decltype(p.version){2, 7, 4}); - CHECK(p.tool_dir_subpath == "git-2.7.4-linux"); - CHECK(p.exe_subpath == ""); - CHECK(p.download_subpath == ""); - CHECK(p.sha512 == ""); - CHECK(p.url == ""); - } -} - TEST_CASE ("extract_prefixed_nonwhitespace", "[tools]") { CHECK(extract_prefixed_nonwhitespace("fooutil version ", "fooutil", "fooutil version 1.2", "fooutil.exe") @@ -140,3 +67,240 @@ TEST_CASE ("extract_prefixed_nonquote", "[tools]") CHECK(error_result.error() == "error: fooutil (fooutil.exe) produced unexpected output when attempting to " "determine the version:\nmalformed output"); } + +TEST_CASE ("parse_tool_data", "[tools]") +{ + const StringView tool_doc = R"( +{ + "$comment": "This is a comment", + "schema-version": 1, + "tools": [ + { + "$comment": "This is a comment", + "name": "git", + "os": "linux", + "version": "2.7.4", + "executable": "git" + }, + { + "name": "git", + "os": "linux", + "arch": "arm64", + "version": "2.7.4", + "executable": "git-arm64" + }, + { + "name": "nuget", + "os": "osx", + "version": "5.11.0", + "executable": "nuget.exe", + "url": "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe", + "sha512": "06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9ebfda83b64c03ce14c8f35ca9957a17a8c02b8c4b7" + }, + { + "name": "node", + "os": "windows", + "version": "node version 16.12.0.windows.2", + "executable": "node-v16.12.0-win-x64\\node.exe", + "url": "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z", + "sha512": "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023fb7068ed1f8a7678e446260c3db3537fa888", + "archive": "node-v16.12.0-win-x64.7z" + } + ] +})"; + + auto maybe_data = parse_tool_data(tool_doc, "vcpkgTools.json"); + REQUIRE(maybe_data.has_value()); + + auto data = maybe_data.value_or_exit(VCPKG_LINE_INFO); + REQUIRE(data.size() == 4); + + auto& git_linux = data[0]; + CHECK(git_linux.tool == "git"); + CHECK(git_linux.os == ToolOs::Linux); + CHECK_FALSE(git_linux.arch.has_value()); + CHECK(git_linux.version.cooked == std::array{2, 7, 4}); + CHECK(git_linux.version.raw == "2.7.4"); + CHECK(git_linux.exeRelativePath == "git"); + CHECK(git_linux.url == ""); + CHECK(git_linux.sha512 == ""); + + auto& git_arm64 = data[1]; + CHECK(git_arm64.tool == "git"); + CHECK(git_arm64.os == ToolOs::Linux); + CHECK(git_arm64.arch.has_value()); + CHECK(*git_arm64.arch.get() == CPUArchitecture::ARM64); + CHECK(git_linux.version.cooked == std::array{2, 7, 4}); + CHECK(git_linux.version.raw == "2.7.4"); + CHECK(git_arm64.exeRelativePath == "git-arm64"); + CHECK(git_arm64.url == ""); + CHECK(git_arm64.sha512 == ""); + + auto& nuget_osx = data[2]; + CHECK(nuget_osx.tool == "nuget"); + CHECK(nuget_osx.os == ToolOs::Osx); + CHECK_FALSE(nuget_osx.arch.has_value()); + CHECK(nuget_osx.version.cooked == std::array{5, 11, 0}); + CHECK(nuget_osx.version.raw == "5.11.0"); + CHECK(nuget_osx.exeRelativePath == "nuget.exe"); + CHECK(nuget_osx.url == "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe"); + CHECK(nuget_osx.sha512 == "06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9eb" + "fda83b64c03ce14c8f35ca9957a17a8c02b8c4b7"); + + auto& node_windows = data[3]; + CHECK(node_windows.tool == "node"); + CHECK(node_windows.os == ToolOs::Windows); + CHECK_FALSE(node_windows.arch.has_value()); + CHECK(node_windows.version.cooked == std::array{16, 12, 0}); + CHECK(node_windows.version.raw == "node version 16.12.0.windows.2"); + CHECK(node_windows.exeRelativePath == "node-v16.12.0-win-x64\\node.exe"); + CHECK(node_windows.url == "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z"); + CHECK(node_windows.sha512 == + "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023" + "fb7068ed1f8a7678e446260c3db3537fa888"); + CHECK(node_windows.archiveName == "node-v16.12.0-win-x64.7z"); + + auto* tool_git_linux = get_raw_tool_data(data, "git", CPUArchitecture::X64, ToolOs::Linux); + REQUIRE(tool_git_linux != nullptr); + CHECK(tool_git_linux->tool == "git"); + CHECK(tool_git_linux->os == ToolOs::Linux); + CHECK_FALSE(tool_git_linux->arch.has_value()); + CHECK(tool_git_linux->version.cooked == std::array{2, 7, 4}); + CHECK(tool_git_linux->version.raw == "2.7.4"); + CHECK(tool_git_linux->exeRelativePath == "git"); + CHECK(tool_git_linux->url == ""); + CHECK(tool_git_linux->sha512 == ""); + + auto* tool_git_arm64 = get_raw_tool_data(data, "git", CPUArchitecture::ARM64, ToolOs::Linux); + REQUIRE(tool_git_arm64 != nullptr); + CHECK(tool_git_arm64->tool == "git"); + CHECK(tool_git_arm64->os == ToolOs::Linux); + CHECK(tool_git_arm64->arch.has_value()); + CHECK(*tool_git_arm64->arch.get() == CPUArchitecture::ARM64); + CHECK(tool_git_arm64->version.cooked == std::array{2, 7, 4}); + CHECK(tool_git_arm64->version.raw == "2.7.4"); + CHECK(tool_git_arm64->exeRelativePath == "git-arm64"); + CHECK(tool_git_arm64->url == ""); + CHECK(tool_git_arm64->sha512 == ""); + + auto* tool_nuget_osx = get_raw_tool_data(data, "nuget", CPUArchitecture::X64, ToolOs::Osx); + REQUIRE(tool_nuget_osx != nullptr); + CHECK(tool_nuget_osx->tool == "nuget"); + CHECK(tool_nuget_osx->os == ToolOs::Osx); + CHECK_FALSE(tool_nuget_osx->arch.has_value()); + CHECK(tool_nuget_osx->version.cooked == std::array{5, 11, 0}); + CHECK(tool_nuget_osx->version.raw == "5.11.0"); + CHECK(tool_nuget_osx->exeRelativePath == "nuget.exe"); + CHECK(tool_nuget_osx->url == "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe"); + + auto* tool_node_windows = get_raw_tool_data(data, "node", CPUArchitecture::X64, ToolOs::Windows); + REQUIRE(tool_node_windows != nullptr); + CHECK(tool_node_windows->tool == "node"); + CHECK(tool_node_windows->os == ToolOs::Windows); + CHECK_FALSE(tool_node_windows->arch.has_value()); + CHECK(tool_node_windows->version.cooked == std::array{16, 12, 0}); + CHECK(tool_node_windows->version.raw == "node version 16.12.0.windows.2"); + CHECK(tool_node_windows->exeRelativePath == "node-v16.12.0-win-x64\\node.exe"); + CHECK(tool_node_windows->url == "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z"); + CHECK(tool_node_windows->sha512 == + "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023" + "fb7068ed1f8a7678e446260c3db3537fa888"); + CHECK(tool_node_windows->archiveName == "node-v16.12.0-win-x64.7z"); +} + +TEST_CASE ("parse_tool_data errors", "[tools]") +{ + auto empty = parse_tool_data("", "empty.json"); + REQUIRE(!empty.has_value()); + CHECK(Strings::starts_with(empty.error(), "empty.json:1:1: error: Unexpected EOF")); + + auto top_level_json = parse_tool_data("[]", "top_level.json"); + REQUIRE(!top_level_json.has_value()); + CHECK("Expected \"top_level.json\" to be an object." == top_level_json.error()); + + auto missing_required = + parse_tool_data(R"({ "schema-version": 1, "tools": [{ "executable": "git.exe" }]})", "missing_required.json"); + REQUIRE(!missing_required.has_value()); + CHECK("missing_required.json: error: $.tools[0] (tool metadata): missing required field 'name' (a string)\n" + "missing_required.json: error: $.tools[0] (tool metadata): missing required field 'os' (a tool data " + "operating system)\n" + "missing_required.json: error: $.tools[0] (tool metadata): missing required field 'version' (a tool data " + "version)" == missing_required.error()); + + auto uexpected_field = parse_tool_data(R"( +{ + "schema-version": 1, + "tools": [{ + "name": "git", + "os": "linux", + "version": "2.7.4", + "arc": "x64" + }] +})", + "uexpected_field.json"); + REQUIRE(!uexpected_field.has_value()); + CHECK("uexpected_field.json: error: $.tools[0] (tool metadata): unexpected field 'arc', did you mean 'arch'?" == + uexpected_field.error()); + + auto invalid_os = parse_tool_data(R"( +{ + "schema-version": 1, + "tools": [{ + "name": "git", + "os": "notanos", + "version": "2.7.4" + }] +})", + "invalid_os.json"); + REQUIRE(!invalid_os.has_value()); + CHECK( + "invalid_os.json: error: $.tools[0].os (a tool data operating system): Invalid tool operating system: notanos. " + "Expected one of: windows, osx, linux, freebsd, openbsd" == invalid_os.error()); + + auto invalid_version = parse_tool_data(R"( +{ + "schema-version": 1, + "tools": [{ + "name": "git", + "os": "windows", + "version": "abc" + }] +})", + "invalid_version.json"); + REQUIRE(!invalid_version.has_value()); + CHECK("invalid_version.json: error: $.tools[0].version (a tool data version): Invalid tool version; expected a " + "string containing a substring of between 1 and 3 numbers separated by dots." == invalid_version.error()); + + auto invalid_arch = parse_tool_data(R"( +{ + "schema-version": 1, + "tools": [{ + "name": "git", + "os": "linux", + "version": "2.7.4", + "arch": "notanarchitecture" + }] +})", + "invalid_arch.json"); + REQUIRE(!invalid_arch.has_value()); + CHECK("invalid_arch.json: error: $.tools[0].arch (a CPU architecture): Invalid architecture: notanarchitecture. " + "Expected one of: x86, x64, amd64, arm, arm64, arm64ec, s390x, ppc64le, riscv32, riscv64, loongarch32, " + "loongarch64, mips64" == invalid_arch.error()); + + auto invalid_sha512 = parse_tool_data(R"( +{ + "schema-version": 1, + "tools": [{ + "name": "git", + "os": "linux", + "version": "2.7.4", + "executable": "git", + "sha512": "notasha512" + }] +})", + "invalid_sha512.json"); + + REQUIRE(!invalid_sha512.has_value()); + CHECK("invalid_sha512.json: error: $.tools[0].sha512 (a SHA-512 hash): invalid SHA-512 hash: notasha512\n" + "SHA-512 hash must be 128 characters long and contain only hexadecimal digits" == invalid_sha512.error()); +} diff --git a/src/vcpkg.cpp b/src/vcpkg.cpp index ec28ec992d..8691c5022d 100644 --- a/src/vcpkg.cpp +++ b/src/vcpkg.cpp @@ -270,7 +270,7 @@ int main(const int argc, const char* const* const argv) register_console_ctrl_handler(); -#if (defined(__aarch64__) || defined(__arm__) || defined(__s390x__) || defined(__riscv) || \ +#if (defined(__arm__) || defined(__s390x__) || defined(__riscv) || \ ((defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || defined(__PPC64LE__)) && \ defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) || \ defined(_M_ARM) || defined(_M_ARM64)) && \ diff --git a/src/vcpkg/base/json.cpp b/src/vcpkg/base/json.cpp index bab3ccff65..b4066b0688 100644 --- a/src/vcpkg/base/json.cpp +++ b/src/vcpkg/base/json.cpp @@ -4,7 +4,9 @@ #include #include #include +#include #include +#include #include @@ -1425,7 +1427,7 @@ namespace vcpkg::Json uint64_t get_json_parsing_stats() { return g_json_parsing_stats.load(); } static std::vector invalid_json_fields(const Json::Object& obj, - Span known_fields) noexcept + View known_fields) noexcept { const auto field_is_unknown = [known_fields](StringView sv) { // allow directives @@ -1490,7 +1492,7 @@ namespace vcpkg::Json } void Reader::check_for_unexpected_fields(const Object& obj, - View valid_fields, + View valid_fields, const LocalizedString& type_name) { if (valid_fields.size() == 0) @@ -1632,4 +1634,38 @@ namespace vcpkg::Json } const FeatureNameDeserializer FeatureNameDeserializer::instance; + + LocalizedString ArchitectureDeserializer::type_name() const { return msg::format(msgACpuArchitecture); } + + Optional> ArchitectureDeserializer::visit_string(Json::Reader& r, StringView sv) const + { + auto maybe_cpu_architecture = to_cpu_architecture(sv); + if (maybe_cpu_architecture.has_value()) + { + return maybe_cpu_architecture; + } + + r.add_generic_error(type_name(), + msg::format(msgInvalidArchitectureValue, + msg::value = sv, + msg::expected = all_comma_separated_cpu_architectures())); + return Optional{nullopt}; + } + + const ArchitectureDeserializer ArchitectureDeserializer::instance; + + LocalizedString Sha512Deserializer::type_name() const { return msg::format(msgASha512); } + + Optional Sha512Deserializer::visit_string(Json::Reader& r, StringView sv) const + { + if (sv.size() == 128 && std::all_of(sv.begin(), sv.end(), ParserBase::is_hex_digit)) + { + return sv.to_string(); + } + + r.add_generic_error(type_name(), msg::format(msgInvalidSha512, msg::sha = sv)); + return std::string(); + } + + const Sha512Deserializer Sha512Deserializer::instance; } diff --git a/src/vcpkg/base/strings.cpp b/src/vcpkg/base/strings.cpp index 34633102c7..16d6a686b3 100644 --- a/src/vcpkg/base/strings.cpp +++ b/src/vcpkg/base/strings.cpp @@ -173,7 +173,7 @@ bool Strings::case_insensitive_ascii_contains(StringView s, StringView pattern) return case_insensitive_ascii_search(s, pattern) != s.end(); } -bool Strings::case_insensitive_ascii_equals(StringView left, StringView right) +bool Strings::case_insensitive_ascii_equals(StringView left, StringView right) noexcept { return std::equal(left.begin(), left.end(), right.begin(), right.end(), icase_eq); } diff --git a/src/vcpkg/base/system.cpp b/src/vcpkg/base/system.cpp index 3df622e156..7592d18ed0 100644 --- a/src/vcpkg/base/system.cpp +++ b/src/vcpkg/base/system.cpp @@ -134,6 +134,29 @@ namespace return result; } #endif // ^^^ _WIN32 + + struct CPUArchitectureEntry + { + StringLiteral name; + CPUArchitecture arch; + }; + + // keep this in sync with vcpkg-tools.schema.json + static constexpr CPUArchitectureEntry cpu_architecture_table[] = { + {"x86", CPUArchitecture::X86}, + {"x64", CPUArchitecture::X64}, + {"amd64", CPUArchitecture::X64}, + {"arm", CPUArchitecture::ARM}, + {"arm64", CPUArchitecture::ARM64}, + {"arm64ec", CPUArchitecture::ARM64EC}, + {"s390x", CPUArchitecture::S390X}, + {"ppc64le", CPUArchitecture::PPC64LE}, + {"riscv32", CPUArchitecture::RISCV32}, + {"riscv64", CPUArchitecture::RISCV64}, + {"loongarch32", CPUArchitecture::LOONGARCH32}, + {"loongarch64", CPUArchitecture::LOONGARCH64}, + {"mips64", CPUArchitecture::MIPS64}, + }; } namespace vcpkg @@ -147,43 +170,36 @@ namespace vcpkg #endif // ^^^ !_WIN32 } - Optional to_cpu_architecture(StringView arch) - { - if (Strings::case_insensitive_ascii_equals(arch, "x86")) return CPUArchitecture::X86; - if (Strings::case_insensitive_ascii_equals(arch, "x64")) return CPUArchitecture::X64; - if (Strings::case_insensitive_ascii_equals(arch, "amd64")) return CPUArchitecture::X64; - if (Strings::case_insensitive_ascii_equals(arch, "arm")) return CPUArchitecture::ARM; - if (Strings::case_insensitive_ascii_equals(arch, "arm64")) return CPUArchitecture::ARM64; - if (Strings::case_insensitive_ascii_equals(arch, "arm64ec")) return CPUArchitecture::ARM64EC; - if (Strings::case_insensitive_ascii_equals(arch, "s390x")) return CPUArchitecture::S390X; - if (Strings::case_insensitive_ascii_equals(arch, "ppc64le")) return CPUArchitecture::PPC64LE; - if (Strings::case_insensitive_ascii_equals(arch, "riscv32")) return CPUArchitecture::RISCV32; - if (Strings::case_insensitive_ascii_equals(arch, "riscv64")) return CPUArchitecture::RISCV64; - if (Strings::case_insensitive_ascii_equals(arch, "loongarch32")) return CPUArchitecture::LOONGARCH32; - if (Strings::case_insensitive_ascii_equals(arch, "loongarch64")) return CPUArchitecture::LOONGARCH64; - if (Strings::case_insensitive_ascii_equals(arch, "mips64")) return CPUArchitecture::MIPS64; + Optional to_cpu_architecture(StringView arch) noexcept + { + for (auto&& entry : cpu_architecture_table) + { + if (Strings::case_insensitive_ascii_equals(arch, entry.name)) + { + return entry.arch; + } + } return nullopt; } - ZStringView to_zstring_view(CPUArchitecture arch) noexcept + StringLiteral to_string_literal(CPUArchitecture arch) noexcept { - switch (arch) + for (auto&& entry : cpu_architecture_table) { - case CPUArchitecture::X86: return "x86"; - case CPUArchitecture::X64: return "x64"; - case CPUArchitecture::ARM: return "arm"; - case CPUArchitecture::ARM64: return "arm64"; - case CPUArchitecture::ARM64EC: return "arm64ec"; - case CPUArchitecture::S390X: return "s390x"; - case CPUArchitecture::PPC64LE: return "ppc64le"; - case CPUArchitecture::RISCV32: return "riscv32"; - case CPUArchitecture::RISCV64: return "riscv64"; - case CPUArchitecture::LOONGARCH32: return "loongarch32"; - case CPUArchitecture::LOONGARCH64: return "loongarch64"; - case CPUArchitecture::MIPS64: return "mips64"; - default: Checks::exit_with_message(VCPKG_LINE_INFO, "unexpected vcpkg::CPUArchitecture"); + if (entry.arch == arch) + { + return entry.name; + } } + + Checks::unreachable(VCPKG_LINE_INFO, "unexpected vcpkg::CPUArchitecture"); + } + + LocalizedString all_comma_separated_cpu_architectures() + { + return LocalizedString::from_raw( + Strings::join(", ", cpu_architecture_table, [](const CPUArchitectureEntry& entry) { return entry.name; })); } CPUArchitecture get_host_processor() @@ -624,8 +640,7 @@ namespace vcpkg case CPUArchitecture::ARM: value = L"ARM"; break; case CPUArchitecture::ARM64: value = L"ARM64"; break; default: - Checks::msg_exit_with_error( - VCPKG_LINE_INFO, msgUnexpectedWindowsArchitecture, msg::actual = to_zstring_view(proc)); + Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgUnexpectedWindowsArchitecture, msg::actual = proc); break; } diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 16029b5612..63b344e610 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -323,7 +323,9 @@ namespace vcpkg auto maybe_target_arch = to_cpu_architecture(target_architecture); if (!maybe_target_arch.has_value()) { - msg::println_error(msgInvalidArchitecture, msg::value = target_architecture); + msg::println_error(msgInvalidArchitectureValue, + msg::value = target_architecture, + msg::expected = all_comma_separated_cpu_architectures()); Checks::exit_maybe_upgrade(VCPKG_LINE_INFO); } @@ -1636,7 +1638,7 @@ namespace vcpkg fmt::format_to(std::back_inserter(issue_body), "Package: {}\n\n**Host Environment**\n\n- Host: {}-{}\n", action.display_name(), - to_zstring_view(get_host_processor()), + get_host_processor(), get_host_os_name()); if (const auto* abi_info = action.abi_info.get()) diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 5c1abf9c30..037d6742f2 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -1316,9 +1316,7 @@ namespace vcpkg const auto vs_prompt = maybe_vs_prompt.value_or_exit(VCPKG_LINE_INFO); if (common_arch != vs_prompt) { - const auto vs_prompt_view = to_zstring_view(vs_prompt); - msg::println_warning( - msgVcpkgInVsPrompt, msg::value = vs_prompt_view, msg::triplet = common_triplet); + msg::println_warning(msgVcpkgInVsPrompt, msg::value = vs_prompt, msg::triplet = common_triplet); } } } diff --git a/src/vcpkg/configuration.cpp b/src/vcpkg/configuration.cpp index 9aa32cd370..8f48babe18 100644 --- a/src/vcpkg/configuration.cpp +++ b/src/vcpkg/configuration.cpp @@ -99,7 +99,7 @@ namespace struct RegistryConfigDeserializer final : Json::IDeserializer { virtual LocalizedString type_name() const override { return msg::format(msgARegistry); } - virtual View valid_fields() const override; + virtual View valid_fields() const noexcept override; virtual Optional visit_null(Json::Reader&) const override; virtual Optional visit_object(Json::Reader&, const Json::Object&) const override; @@ -147,7 +147,7 @@ namespace struct RegistryDeserializer final : Json::IDeserializer { virtual LocalizedString type_name() const override { return msg::format(msgARegistry); } - virtual View valid_fields() const override; + virtual View valid_fields() const noexcept override; virtual Optional visit_object(Json::Reader&, const Json::Object&) const override; @@ -164,9 +164,9 @@ namespace const RegistriesArrayDeserializer RegistriesArrayDeserializer::instance; - View RegistryConfigDeserializer::valid_fields() const + View RegistryConfigDeserializer::valid_fields() const noexcept { - static constexpr StringView t[] = { + static constexpr StringLiteral t[] = { JsonIdKind, JsonIdBaseline, JsonIdPath, @@ -177,20 +177,20 @@ namespace }; return t; } - static constexpr StringView valid_builtin_fields[] = { + static constexpr StringLiteral valid_builtin_fields[] = { JsonIdKind, JsonIdBaseline, JsonIdPackages, }; - static constexpr StringView valid_filesystem_fields[] = { + static constexpr StringLiteral valid_filesystem_fields[] = { JsonIdKind, JsonIdBaseline, JsonIdPath, JsonIdPackages, }; - static constexpr StringView valid_git_fields[] = { + static constexpr StringLiteral valid_git_fields[] = { JsonIdKind, JsonIdBaseline, JsonIdRepository, @@ -198,7 +198,7 @@ namespace JsonIdPackages, }; - static constexpr StringView valid_artifact_fields[] = { + static constexpr StringLiteral valid_artifact_fields[] = { JsonIdKind, JsonIdName, JsonIdLocation, @@ -283,9 +283,9 @@ namespace return std::move(res); // gcc-7 bug workaround redundant move } - View RegistryDeserializer::valid_fields() const + View RegistryDeserializer::valid_fields() const noexcept { - static constexpr StringView t[] = { + static constexpr StringLiteral t[] = { JsonIdKind, JsonIdBaseline, JsonIdPath, diff --git a/src/vcpkg/registries.cpp b/src/vcpkg/registries.cpp index c77c0f75dc..2bac907db7 100644 --- a/src/vcpkg/registries.cpp +++ b/src/vcpkg/registries.cpp @@ -46,17 +46,15 @@ namespace struct GitVersionDbEntryDeserializer final : Json::IDeserializer { LocalizedString type_name() const override; - View valid_fields() const override; + View valid_fields() const noexcept override; Optional visit_object(Json::Reader& r, const Json::Object& obj) const override; }; LocalizedString GitVersionDbEntryDeserializer::type_name() const { return msg::format(msgAVersionDatabaseEntry); } - View GitVersionDbEntryDeserializer::valid_fields() const + View GitVersionDbEntryDeserializer::valid_fields() const noexcept { - static constexpr StringView u_git[] = {JsonIdGitTree}; - static const auto t_git = vcpkg::Util::Vectors::concat(schemed_deserializer_fields(), u_git); - - return t_git; + static constexpr StringLiteral fields[] = {VCPKG_SCHEMED_DESERIALIZER_FIELDS, JsonIdGitTree}; + return fields; } Optional GitVersionDbEntryDeserializer::visit_object(Json::Reader& r, @@ -88,7 +86,7 @@ namespace struct FilesystemVersionDbEntryDeserializer final : Json::IDeserializer { LocalizedString type_name() const override; - View valid_fields() const override; + View valid_fields() const noexcept override; Optional visit_object(Json::Reader& r, const Json::Object& obj) const override; FilesystemVersionDbEntryDeserializer(const Path& root) : registry_root(root) { } @@ -100,11 +98,10 @@ namespace { return msg::format(msgAVersionDatabaseEntry); } - View FilesystemVersionDbEntryDeserializer::valid_fields() const + View FilesystemVersionDbEntryDeserializer::valid_fields() const noexcept { - static constexpr StringView u_path[] = {JsonIdPath}; - static const auto t_path = vcpkg::Util::Vectors::concat(schemed_deserializer_fields(), u_path); - return t_path; + static constexpr StringLiteral fields[] = {VCPKG_SCHEMED_DESERIALIZER_FIELDS, JsonIdPath}; + return fields; } Optional FilesystemVersionDbEntryDeserializer::visit_object(Json::Reader& r, diff --git a/src/vcpkg/sourceparagraph.cpp b/src/vcpkg/sourceparagraph.cpp index a250c4c4b6..3f17507780 100644 --- a/src/vcpkg/sourceparagraph.cpp +++ b/src/vcpkg/sourceparagraph.cpp @@ -516,9 +516,9 @@ namespace vcpkg { LocalizedString type_name() const override { return msg::format(msgADefaultFeature); } - Span valid_fields() const override + View valid_fields() const noexcept override { - static const StringView t[] = { + static const StringLiteral t[] = { JsonIdName, JsonIdPlatform, }; @@ -580,9 +580,9 @@ namespace vcpkg { LocalizedString type_name() const override { return msg::format(msgADependencyFeature); } - Span valid_fields() const override + View valid_fields() const noexcept override { - static const StringView t[] = { + static const StringLiteral t[] = { JsonIdName, JsonIdPlatform, }; @@ -620,9 +620,9 @@ namespace vcpkg { virtual LocalizedString type_name() const override { return msg::format(msgADependency); } - virtual Span valid_fields() const override + virtual View valid_fields() const noexcept override { - static constexpr StringView t[] = { + static constexpr StringLiteral t[] = { JsonIdName, JsonIdHost, JsonIdFeatures, @@ -701,11 +701,10 @@ namespace vcpkg struct DependencyOverrideDeserializer final : Json::IDeserializer { virtual LocalizedString type_name() const override { return msg::format(msgAnOverride); } - virtual Span valid_fields() const override + virtual View valid_fields() const noexcept override { - static constexpr StringView u[] = {JsonIdName}; - static const auto t = Util::Vectors::concat(schemed_deserializer_fields(), u); - return t; + static constexpr StringLiteral fields[] = {VCPKG_SCHEMED_DESERIALIZER_FIELDS, JsonIdName}; + return fields; } virtual Optional visit_object(Json::Reader& r, const Json::Object& obj) const override @@ -1004,9 +1003,9 @@ namespace vcpkg { virtual LocalizedString type_name() const override { return msg::format(msgAFeature); } - virtual Span valid_fields() const override + virtual View valid_fields() const noexcept override { - static constexpr StringView t[] = {JsonIdDescription, JsonIdDependencies, JsonIdSupports, JsonIdLicense}; + static constexpr StringLiteral t[] = {JsonIdDescription, JsonIdDependencies, JsonIdSupports, JsonIdLicense}; return t; } @@ -1052,8 +1051,6 @@ namespace vcpkg { virtual LocalizedString type_name() const override { return msg::format(msgASetOfFeatures); } - virtual Span valid_fields() const override { return {}; } - virtual Optional visit_object(Json::Reader& r, const Json::Object& obj) const override { FeaturesObject res; @@ -1122,9 +1119,10 @@ namespace vcpkg { virtual LocalizedString type_name() const override { return msg::format(msgAManifest); } - virtual Span valid_fields() const override + virtual View valid_fields() const noexcept override { - static constexpr StringView u[] = { + static constexpr StringLiteral fields[] = { + VCPKG_SCHEMED_DESERIALIZER_FIELDS, JsonIdName, JsonIdMaintainers, JsonIdContacts, @@ -1141,9 +1139,8 @@ namespace vcpkg JsonIdBuiltinBaseline, JsonIdVcpkgConfiguration, }; - static const auto t = Util::Vectors::concat(schemed_deserializer_fields(), u); - return t; + return fields; } vcpkg::Optional> visit_object_common( diff --git a/src/vcpkg/tools.cpp b/src/vcpkg/tools.cpp index 0f88301576..9836d269d5 100644 --- a/src/vcpkg/tools.cpp +++ b/src/vcpkg/tools.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -18,11 +19,30 @@ #include #include -#include +#include + +namespace +{ + using namespace vcpkg; + struct ToolOsEntry + { + StringLiteral name; + ToolOs os; + }; + + // keep this in sync with vcpkg-tools.schema.json + static constexpr ToolOsEntry all_tool_oses[] = { + {"windows", ToolOs::Windows}, + {"osx", ToolOs::Osx}, + {"linux", ToolOs::Linux}, + {"freebsd", ToolOs::FreeBsd}, + {"openbsd", ToolOs::OpenBsd}, + }; +} namespace vcpkg { - // /\d+\.\d+(\.\d+)?/ + // /\d+(\.\d+(\.\d+)?)?/ Optional> parse_tool_version_string(StringView string_version) { // first, find the beginning of the version @@ -62,82 +82,126 @@ namespace vcpkg return std::array{*d1.get(), *d2.get(), *d3.get()}; } - static Optional parse_tool_data_from_xml(StringView XML, StringView XML_PATH, StringView tool) + Optional to_tool_os(StringView os) noexcept + { + for (auto&& entry : all_tool_oses) + { + if (os == entry.name) + { + return entry.os; + } + } + + return nullopt; + } + + StringLiteral to_string_literal(ToolOs os) noexcept + { + for (auto&& entry : all_tool_oses) + { + if (os == entry.os) + { + return entry.name; + } + } + + Checks::unreachable(VCPKG_LINE_INFO, "Unexpected ToolOs"); + } + + LocalizedString all_comma_separated_tool_oses() + { + return LocalizedString::from_raw( + Strings::join(", ", all_tool_oses, [](const ToolOsEntry& entry) { return entry.name; })); + } + + ExpectedL> parse_tool_data(StringView contents, StringView origin) { + return Json::parse_object(contents, origin) + .then([&](Json::Object&& as_object) -> ExpectedL> { + Json::Reader r(origin); + auto maybe_tool_data = r.visit(as_object, ToolDataFileDeserializer::instance); + if (!r.errors().empty() || !r.warnings().empty()) + { + return r.join(); + } + + return maybe_tool_data.value_or_exit(VCPKG_LINE_INFO); + }); + } + + static ExpectedL> parse_tool_data_file(const Filesystem& fs, Path path) + { + return fs.try_read_contents(path).then([](FileContents&& fc) -> ExpectedL> { + return parse_tool_data(fc.content, Path{fc.origin}); + }); + } + + const ToolDataEntry* get_raw_tool_data(const std::vector& tool_data_table, + StringView toolname, + const CPUArchitecture arch, + const ToolOs os) + { + const ToolDataEntry* default_tool = nullptr; + for (auto&& tool_candidate : tool_data_table) + { + if (tool_candidate.tool == toolname && tool_candidate.os == os) + { + if (!tool_candidate.arch) + { + if (!default_tool) + { + default_tool = &tool_candidate; + } + } + else if (arch == *tool_candidate.arch.get()) + { + return &tool_candidate; + } + } + } + return default_tool; + } + + static Optional get_tool_data(const std::vector& tool_data_table, StringView tool) + { + auto hp = get_host_processor(); #if defined(_WIN32) - return parse_tool_data_from_xml(XML, XML_PATH, tool, "windows"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, ToolOs::Windows); #elif defined(__APPLE__) - return parse_tool_data_from_xml(XML, XML_PATH, tool, "osx"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, ToolOs::Osx); #elif defined(__linux__) - return parse_tool_data_from_xml(XML, XML_PATH, tool, "linux"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, ToolOs::Linux); #elif defined(__FreeBSD__) - return parse_tool_data_from_xml(XML, XML_PATH, tool, "freebsd"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, ToolOs::FreeBsd); #elif defined(__OpenBSD__) - return parse_tool_data_from_xml(XML, XML_PATH, tool, "openbsd"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, ToolOs::OpenBsd); #else return nullopt; #endif - } + if (!data) + { + return nullopt; + } - Optional parse_tool_data_from_xml(StringView XML, StringView XML_PATH, StringView tool, StringView os) - { - static const char* XML_VERSION = "2"; - static const std::regex XML_VERSION_REGEX{R"###()###"}; - std::cmatch match_xml_version; - const bool has_xml_version = std::regex_search(XML.begin(), XML.end(), match_xml_version, XML_VERSION_REGEX); - Checks::msg_check_exit(VCPKG_LINE_INFO, - has_xml_version, - msgCouldNotFindToolVersion, - msg::version = XML_VERSION, - msg::path = XML_PATH); - Checks::msg_check_exit(VCPKG_LINE_INFO, - XML_VERSION == match_xml_version[1], - msgVersionConflictXML, - msg::path = XML_PATH, - msg::expected_version = XML_VERSION, - msg::actual_version = match_xml_version[1].str()); - - const std::regex tool_regex{fmt::format(R"###()###", tool, os)}; - std::cmatch match_tool_entry; - const bool has_tool_entry = std::regex_search(XML.begin(), XML.end(), match_tool_entry, tool_regex); - if (!has_tool_entry) return nullopt; - - const std::string tool_data = - Strings::find_exactly_one_enclosed(XML, match_tool_entry[0].str(), "").to_string(); - const std::string version_as_string = - Strings::find_exactly_one_enclosed(tool_data, "", "").to_string(); - const std::string exe_relative_path = - Strings::find_exactly_one_enclosed(tool_data, "", "").to_string(); - const std::string url = Strings::find_exactly_one_enclosed(tool_data, "", "").to_string(); - const std::string sha512 = Strings::find_exactly_one_enclosed(tool_data, "", "").to_string(); - auto archive_name = Strings::find_at_most_one_enclosed(tool_data, "", ""); - - const Optional> version = parse_tool_version_string(version_as_string); - Checks::msg_check_exit(VCPKG_LINE_INFO, - version.has_value(), - msgFailedToParseVersionXML, - msg::tool_name = tool, - msg::version = version_as_string); - - Path tool_dir_name = fmt::format("{}-{}-{}", tool, version_as_string, os); + Path tool_dir_name = fmt::format("{}-{}-{}", tool, data->version.raw, data->os); Path download_subpath; - if (auto a = archive_name.get()) + if (!data->archiveName.empty()) { - download_subpath = a->to_string(); + download_subpath = data->archiveName; } - else if (!exe_relative_path.empty()) + else if (!data->exeRelativePath.empty()) { - download_subpath = Strings::concat(sha512.substr(0, 8), '-', exe_relative_path); + download_subpath = Strings::concat(StringView{data->sha512}.substr(0, 8), '-', data->exeRelativePath); } return ToolData{tool.to_string(), - *version.get(), - exe_relative_path, - url, + data->version.cooked, + data->exeRelativePath, + data->url, download_subpath, - archive_name.has_value(), + !data->archiveName.empty(), tool_dir_name, - sha512}; + data->sha512}; } struct PathAndVersion @@ -664,23 +728,23 @@ namespace vcpkg const Filesystem& fs; const std::shared_ptr downloader; const Path downloads; - const Path xml_config; + const Path config_path; const Path tools; const RequireExactVersions abiToolVersionHandling; - vcpkg::Lazy xml_config_contents; vcpkg::Cache path_version_cache; + vcpkg::Lazy> m_tool_data_cache; ToolCacheImpl(const Filesystem& fs, const std::shared_ptr& downloader, Path downloads, - Path xml_config, + Path config_path, Path tools, RequireExactVersions abiToolVersionHandling) : fs(fs) , downloader(downloader) , downloads(std::move(downloads)) - , xml_config(std::move(xml_config)) + , config_path(std::move(config_path)) , tools(std::move(tools)) , abiToolVersionHandling(abiToolVersionHandling) { @@ -763,12 +827,6 @@ namespace vcpkg return exe_path; } - const std::string& get_config_contents() const - { - return xml_config_contents.get_lazy( - [this]() { return this->fs.read_contents(this->xml_config, VCPKG_LINE_INFO); }); - } - virtual const Path& get_tool_path(StringView tool, MessageSink& status_sink) const override { return get_tool_pathversion(tool, status_sink).p; @@ -780,9 +838,7 @@ namespace vcpkg get_environment_variable(EnvironmentVariableVcpkgForceSystemBinaries).has_value(); const bool env_force_download_binaries = get_environment_variable(EnvironmentVariableVcpkgForceDownloadedBinaries).has_value(); - const auto maybe_tool_data = - parse_tool_data_from_xml(get_config_contents(), xml_config, tool.tool_data_name()); - + const auto maybe_tool_data = get_tool_data(load_tool_data(), tool.tool_data_name()); const bool download_available = maybe_tool_data.has_value() && !maybe_tool_data.get()->url.empty(); // search for system searchable tools unless forcing downloads and download available const auto system_exe_stems = tool.system_exe_stems(); @@ -800,7 +856,7 @@ namespace vcpkg if (auto tool_data = maybe_tool_data.get()) { - // If there is an entry for the tool in vcpkgTools.xml, use that version as the minimum + // If there is an entry for the tool in vcpkg-tools.json, use that version as the minimum min_version = tool_data->version; if (consider_downloads) @@ -941,6 +997,18 @@ namespace vcpkg { return get_tool_pathversion(tool, status_sink).version; } + + std::vector load_tool_data() const + { + return m_tool_data_cache.get_lazy([&]() { + auto maybe_tool_data = parse_tool_data_file(fs, config_path); + if (auto tool_data = maybe_tool_data.get()) + { + return std::move(*tool_data); + } + Checks::msg_exit_with_error(VCPKG_LINE_INFO, maybe_tool_data.error()); + }); + } }; ExpectedL find_system_tar(const ReadOnlyFilesystem& fs) @@ -1013,11 +1081,125 @@ namespace vcpkg std::unique_ptr get_tool_cache(const Filesystem& fs, std::shared_ptr downloader, Path downloads, - Path xml_config, + Path config_path, Path tools, RequireExactVersions abiToolVersionHandling) { return std::make_unique( - fs, std::move(downloader), downloads, xml_config, tools, abiToolVersionHandling); + fs, std::move(downloader), downloads, config_path, tools, abiToolVersionHandling); + } + + struct ToolDataEntryDeserializer final : Json::IDeserializer + { + virtual LocalizedString type_name() const override { return msg::format(msgAToolDataObject); } + + virtual View valid_fields() const noexcept override + { + static const StringLiteral fields[] = { + JsonIdName, + JsonIdOS, + JsonIdVersion, + JsonIdArch, + JsonIdExecutable, + JsonIdUrl, + JsonIdSha512, + JsonIdArchive, + }; + return fields; + } + + virtual Optional visit_object(Json::Reader& r, const Json::Object& obj) const override + { + ToolDataEntry value; + + r.required_object_field( + type_name(), obj, JsonIdName, value.tool, Json::UntypedStringDeserializer::instance); + r.required_object_field(type_name(), obj, JsonIdOS, value.os, ToolOsDeserializer::instance); + r.required_object_field(type_name(), obj, JsonIdVersion, value.version, ToolVersionDeserializer::instance); + + r.optional_object_field(obj, JsonIdArch, value.arch, Json::ArchitectureDeserializer::instance); + r.optional_object_field( + obj, JsonIdExecutable, value.exeRelativePath, Json::UntypedStringDeserializer::instance); + r.optional_object_field(obj, JsonIdUrl, value.url, Json::UntypedStringDeserializer::instance); + r.optional_object_field(obj, JsonIdSha512, value.sha512, Json::Sha512Deserializer::instance); + r.optional_object_field(obj, JsonIdArchive, value.archiveName, Json::UntypedStringDeserializer::instance); + return value; + } + + static const ToolDataEntryDeserializer instance; + }; + const ToolDataEntryDeserializer ToolDataEntryDeserializer::instance; + + struct ToolDataArrayDeserializer final : Json::ArrayDeserializer + { + virtual LocalizedString type_name() const override { return msg::format(msgAToolDataArray); } + + static const ToolDataArrayDeserializer instance; + }; + const ToolDataArrayDeserializer ToolDataArrayDeserializer::instance; + + LocalizedString ToolDataFileDeserializer::type_name() const { return msg::format(msgAToolDataFile); } + + View ToolDataFileDeserializer::valid_fields() const noexcept + { + static constexpr StringLiteral valid_fields[] = {JsonIdSchemaVersion, JsonIdTools}; + return valid_fields; } + + Optional> ToolDataFileDeserializer::visit_object(Json::Reader& r, + const Json::Object& obj) const + { + int schema_version = -1; + r.required_object_field( + type_name(), obj, JsonIdSchemaVersion, schema_version, Json::NaturalNumberDeserializer::instance); + + std::vector value; + if (schema_version == 1) + { + r.required_object_field(type_name(), obj, JsonIdTools, value, ToolDataArrayDeserializer::instance); + } + else + { + r.add_generic_error(type_name(), + msg::format(msgToolDataFileSchemaVersionNotSupported, msg::version = schema_version)); + } + + return value; + } + + const ToolDataFileDeserializer ToolDataFileDeserializer::instance; + + LocalizedString ToolOsDeserializer::type_name() const { return msg::format(msgAToolDataOS); } + + Optional ToolOsDeserializer::visit_string(Json::Reader& r, StringView str) const + { + auto maybe_tool_os = to_tool_os(str); + if (auto tool_os = maybe_tool_os.get()) + { + return *tool_os; + } + + r.add_generic_error( + type_name(), + msg::format(msgInvalidToolOSValue, msg::value = str, msg::expected = all_comma_separated_tool_oses())); + return ToolOs::Windows; + } + + const ToolOsDeserializer ToolOsDeserializer::instance; + + LocalizedString ToolVersionDeserializer::type_name() const { return msg::format(msgAToolDataVersion); } + + Optional ToolVersionDeserializer::visit_string(Json::Reader& r, StringView str) const + { + auto maybe_parsed = parse_tool_version_string(str); + if (auto parsed = maybe_parsed.get()) + { + return ToolVersion{*parsed, str.to_string()}; + } + + r.add_generic_error(type_name(), msg::format(msgInvalidToolVersion)); + return ToolVersion{}; + } + + const ToolVersionDeserializer ToolVersionDeserializer::instance; } diff --git a/src/vcpkg/triplet.cpp b/src/vcpkg/triplet.cpp index ba5060eccd..efc3aeebaf 100644 --- a/src/vcpkg/triplet.cpp +++ b/src/vcpkg/triplet.cpp @@ -47,62 +47,19 @@ namespace vcpkg Optional Triplet::guess_architecture() const noexcept { - if (Strings::starts_with(this->canonical_name(), "x86-")) + StringView canonical = this->canonical_name(); + auto dash = std::find(canonical.begin(), canonical.end(), '-'); + if (dash != canonical.end()) { - return CPUArchitecture::X86; - } - if (Strings::starts_with(this->canonical_name(), "x64-")) - { - return CPUArchitecture::X64; - } - if (Strings::starts_with(this->canonical_name(), "arm-")) - { - return CPUArchitecture::ARM; - } - if (Strings::starts_with(this->canonical_name(), "arm64-")) - { - return CPUArchitecture::ARM64; - } - if (Strings::starts_with(this->canonical_name(), "arm64ec-")) - { - return CPUArchitecture::ARM64EC; - } - if (Strings::starts_with(this->canonical_name(), "s390x-")) - { - return CPUArchitecture::S390X; - } - if (Strings::starts_with(this->canonical_name(), "ppc64le-")) - { - return CPUArchitecture::PPC64LE; - } - if (Strings::starts_with(this->canonical_name(), "riscv32-")) - { - return CPUArchitecture::RISCV32; - } - if (Strings::starts_with(this->canonical_name(), "riscv64-")) - { - return CPUArchitecture::RISCV64; - } - if (Strings::starts_with(this->canonical_name(), "loongarch32-")) - { - return CPUArchitecture::LOONGARCH32; - } - if (Strings::starts_with(this->canonical_name(), "loongarch64-")) - { - return CPUArchitecture::LOONGARCH64; - } - if (Strings::starts_with(this->canonical_name(), "mips64-")) - { - return CPUArchitecture::MIPS64; + canonical = StringView{canonical.begin(), dash}; } - return nullopt; + return to_cpu_architecture(canonical); } static std::string system_triplet_canonical_name() { - auto host_proc = get_host_processor(); - return fmt::format("{}-{}", to_zstring_view(host_proc), get_host_os_name()); + return fmt::format("{}-{}", get_host_processor(), get_host_os_name()); } Triplet default_triplet(const VcpkgCmdArguments& args, const TripletDatabase& database) diff --git a/src/vcpkg/vcpkgcmdarguments.cpp b/src/vcpkg/vcpkgcmdarguments.cpp index 446466079d..afd3c148c3 100644 --- a/src/vcpkg/vcpkgcmdarguments.cpp +++ b/src/vcpkg/vcpkgcmdarguments.cpp @@ -358,6 +358,7 @@ namespace vcpkg args.parser.parse_option( SwitchBuiltinRegistryVersionsDir, StabilityTag::Experimental, args.builtin_registry_versions_dir); args.parser.parse_option(SwitchRegistriesCache, StabilityTag::Experimental, args.registries_cache_dir); + args.parser.parse_option(SwitchToolDataFile, StabilityTag::ImplementationDetail, args.tools_data_file); args.parser.parse_option(SwitchAssetSources, StabilityTag::Experimental, args.asset_sources_template_arg, diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index fe7748d6ca..06f52c5801 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -570,13 +570,13 @@ namespace vcpkg "packages", "pkgs", VCPKG_LINE_INFO)) - , m_tool_cache(get_tool_cache(fs, - m_download_manager, - downloads, - scripts / "vcpkgTools.xml", - tools, - args.exact_abi_tools_versions.value_or(false) ? RequireExactVersions::YES - : RequireExactVersions::NO)) + , m_tool_cache(get_tool_cache( + fs, + m_download_manager, + downloads, + args.tools_data_file.has_value() ? Path{*args.tools_data_file.get()} : scripts / "vcpkg-tools.json", + tools, + args.exact_abi_tools_versions.value_or(false) ? RequireExactVersions::YES : RequireExactVersions::NO)) , m_env_cache(m_ff_settings.compiler_tracking) , triplets_dirs() , m_artifacts_dir(downloads / "artifacts") diff --git a/src/vcpkg/versiondeserializers.cpp b/src/vcpkg/versiondeserializers.cpp index b14cde2c74..32c3be8720 100644 --- a/src/vcpkg/versiondeserializers.cpp +++ b/src/vcpkg/versiondeserializers.cpp @@ -261,13 +261,6 @@ namespace vcpkg return Version(std::move(proto_version.first), port_version); } - View schemed_deserializer_fields() - { - static constexpr StringView t[] = { - JsonIdVersion, JsonIdVersionSemver, JsonIdVersionString, JsonIdVersionDate, JsonIdPortVersion}; - return t; - } - void serialize_schemed_version(Json::Object& out_obj, VersionScheme scheme, const Version& version) { auto version_field = [](VersionScheme version_scheme) { diff --git a/vcpkg-init/vcpkg-scripts-sha.txt b/vcpkg-init/vcpkg-scripts-sha.txt index 7ecebbfbfe..f30e9b1027 100644 --- a/vcpkg-init/vcpkg-scripts-sha.txt +++ b/vcpkg-init/vcpkg-scripts-sha.txt @@ -1 +1 @@ -0c4cf19224a049cf82f4521e29e39f7bd680440c +12393e114e2acb75ba14e52ace6e90c4c8ab2d04