[RFC][registries] Allow pattern matching in packages declarations (#778)

* Resolve prefix patterns in packages declarations

* Allow prefixes in configuration parser

* A few more test cases

* Add warning for ignored patterns/packages

* Print warnings on vcpkg configuration

* Localize warning messages

* Clean up

* Test registry parsing

* Apply Billy's patch

* Apply Billy's suggestions from code review

Co-authored-by: Billy O'Neal <[email protected]>

* Format

* Collect all package pattern warnings into one

* Improve warning message

* Format messages

* Disallow period separated segments in port names

* Error messages on invalid patterns/names

* Cleanup

* Disallow registries without setting a baseline

* Fix for artifact registries

* Billy's PR comments

* Add e2e test registry

* End-to-end tests

* Fix e2e tests

* Make tests success case silent

* Accept prefixes ending in '-'

Co-authored-by: Billy O'Neal <[email protected]>

azure-pipelines/e2e_projects/registries-package-patterns/no-patterns.json.in

@@ -0,0 +1,29 @@
+{
+  "vcpkg-configuration": {
+    "default-registry": null,
+    "registries": [
+      {
+        "kind": "git",
+        "repository": "$E2ERegistryPath",
+        "baseline": "$E2ERegistryBaseline",
+        "packages": [
+          "foo",
+          "bar"
+        ]
+      },
+      {
+        "kind": "git",
+        "repository": "$E2ERegistryPath",
+        "baseline": "$E2ERegistryBaseline",
+        "packages": [
+          "baz"
+        ]
+      }
+    ]
+  },
+  "dependencies": [
+    "bar",
+    "baz",
+    "foo"
+  ]
+}

azure-pipelines/e2e_projects/registries-package-patterns/only-patterns.json.in

@@ -0,0 +1,28 @@
+{
+  "vcpkg-configuration": {
+    "default-registry": null,
+    "registries": [
+      {
+        "kind": "git",
+        "repository": "$E2ERegistryPath",
+        "baseline": "$E2ERegistryBaseline",
+        "packages": [
+          "*"
+        ]
+      },
+      {
+        "kind": "git",
+        "repository": "$E2ERegistryPath",
+        "baseline": "$E2ERegistryBaseline",
+        "packages": [
+          "b*"
+        ]
+      }
+    ]
+  },
+  "dependencies": [
+    "bar",
+    "baz",
+    "foo"
+  ]
+}

azure-pipelines/e2e_projects/registries-package-patterns/with-default.json.in

@@ -0,0 +1,24 @@
+{
+  "vcpkg-configuration": {
+    "default-registry": {
+      "kind": "git",
+      "repository": "$E2ERegistryPath",
+      "baseline": "$E2ERegistryBaseline"
+    },
+    "registries": [
+      {
+        "kind": "git",
+        "repository": "$E2ERegistryPath",
+        "baseline": "$E2ERegistryBaseline",
+        "packages": [
+          "b*"
+        ]
+      }
+    ]
+  },
+  "dependencies": [
+    "bar",
+    "baz",
+    "foo"
+  ]
+}

azure-pipelines/e2e_projects/registries-package-patterns/with-redeclaration.json.in

@@ -0,0 +1,32 @@
+{
+  "vcpkg-configuration": {
+    "default-registry": {
+      "kind": "git",
+      "repository": "$E2ERegistryPath",
+      "baseline": "$E2ERegistryBaseline"
+    },
+    "registries": [
+      {
+        "kind": "git",
+        "repository": "$E2ERegistryPath",
+        "baseline": "$E2ERegistryBaseline",
+        "packages": [
+          "b*"
+        ]
+      },
+      {
+        "kind": "git",
+        "repository": "$E2ERegistryPath",
+        "baseline": "$E2ERegistryBaseline",
+        "packages": [
+          "b*"
+        ]
+      }
+    ]
+  },
+  "dependencies": [
+    "bar",
+    "baz",
+    "foo"
+  ]
+}

azure-pipelines/e2e_registry/extra-ports/foo/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_registry/extra-ports/foo/vcpkg.json

@@ -0,0 +1,5 @@
+{
+  "name": "foo",
+  "version": "1.1.0",
+  "description": "e2e test port"
+}

azure-pipelines/e2e_registry/ports/bar/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_registry/ports/bar/vcpkg.json

@@ -0,0 +1,5 @@
+{
+  "name": "bar",
+  "version": "1.0.0",
+  "description": "e2e test port"
+}

azure-pipelines/e2e_registry/ports/baz/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_registry/ports/baz/vcpkg.json

@@ -0,0 +1,5 @@
+{
+  "name": "baz",
+  "version": "1.0.0",
+  "description": "e2e test port"
+}

azure-pipelines/e2e_registry/ports/foo/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_registry/ports/foo/vcpkg.json

@@ -0,0 +1,5 @@
+{
+  "name": "foo",
+  "version": "1.0.0",
+  "description": "e2e test port"
+}

azure-pipelines/e2e_registry/versions/b-/bar.json

@@ -0,0 +1,9 @@
+{
+  "versions": [
+    {
+      "git-tree": "51562fff2b6db21a115564871c7174d59e77e637",
+      "version": "1.0.0",
+      "port-version": 0
+    }
+  ]
+}

azure-pipelines/e2e_registry/versions/b-/baz.json

@@ -0,0 +1,9 @@
+{
+  "versions": [
+    {
+      "git-tree": "85462e55644fe3c9cb42928545e01784dda3e8d7",
+      "version": "1.0.0",
+      "port-version": 0
+    }
+  ]
+}

azure-pipelines/e2e_registry/versions/baseline.json

@@ -0,0 +1,16 @@
+{
+  "default": {
+    "bar": {
+      "baseline": "1.0.0",
+      "port-version": 0
+    },
+    "baz": {
+      "baseline": "1.0.0",
+      "port-version": 0
+    },
+    "foo": {
+      "baseline": "1.1.0",
+      "port-version": 0
+    }
+  }
+}

azure-pipelines/e2e_registry/versions/f-/foo.json

@@ -0,0 +1,14 @@
+{
+  "versions": [
+    {
+      "git-tree": "a1626920f5bbc6a6ccd007ba5a194f38eb8e6222",
+      "version": "1.1.0",
+      "port-version": 0
+    },
+    {
+      "git-tree": "011346ce22c43b3312e32a18c751ca6dd35c4578",
+      "version": "1.0.0",
+      "port-version": 0
+    }
+  ]
+}

azure-pipelines/end-to-end-tests-dir/patterns.ps1

@@ -0,0 +1,84 @@
+. $PSScriptRoot/../end-to-end-tests-prelude.ps1
+
+### <Initialize registry>
+# Creates a git registry to run the e2e tests on
+$e2eProjects = "$PSScriptRoot/../e2e_projects"
+$manifestRoot = "$e2eProjects/registries-package-patterns"
+$e2eRegistryPath = "$PSScriptRoot/../e2e_registry".Replace('\', '\\')
+Push-Location $e2eRegistryPath
+try
+{
+	Write-Host "Initializing test registry"
+	if (Test-Path "$e2eRegistryPath/.git")
+	{
+		Remove-Item -Recurse -Force "$e2eRegistryPath/.git"
+	}
+
+
+	$gitConfig = @(
+		'-c', 'user.name=Nobody',
+		'-c', '[email protected]',
+		'-c', 'core.autocrlf=false'
+	)
+
+	git @gitConfig init . | Out-Null
+	Throw-IfFailed
+	git @gitConfig add -A | Out-Null
+	Throw-IfFailed
+	git @gitConfig commit -m "initial commit" | Out-Null
+	Throw-IfFailed
+	$e2eRegistryBaseline = git rev-parse HEAD
+	Throw-IfFailed
+}
+finally 
+{
+	Pop-Location
+}
+### </Initialize Registry>
+
+$commonArgs += @("--x-manifest-root=$manifestRoot")
+
+# [patterns] No patterns (no default)
+Write-Host "[patterns] No patterns (no default)"
+$inFile = "$manifestRoot/no-patterns.json.in"
+(Get-Content -Path "$inFile").Replace("`$E2ERegistryPath", $e2eRegistryPath).Replace("`$E2ERegistryBaseline", $e2eRegistryBaseline) `
+| Out-File "$manifestRoot/vcpkg.json"
+
+Run-Vcpkg -EndToEndTestSilent @commonArgs install | Out-Null
+Throw-IfFailed
+Refresh-TestRoot
+
+# [patterns] Patterns only (no default)
+Write-Host "[patterns] Patterns only (no default)"
+$inFile = "$manifestRoot/only-patterns.json.in"
+(Get-Content -Path "$inFile").Replace("`$E2ERegistryPath", $e2eRegistryPath).Replace("`$E2ERegistryBaseline", $e2eRegistryBaseline) `
+| Out-File "$manifestRoot/vcpkg.json"
+
+Run-Vcpkg -EndToEndTestSilent @commonArgs install | Out-Null
+Throw-IfFailed
+Refresh-TestRoot
+
+# [patterns] Patterns with default
+Write-Host "[patterns] Patterns with default"
+$inFile = "$manifestRoot/with-default.json.in"
+(Get-Content -Path "$inFile").Replace("`$E2ERegistryPath", $e2eRegistryPath).Replace("`$E2ERegistryBaseline", $e2eRegistryBaseline) `
+| Out-File "$manifestRoot/vcpkg.json"
+
+Run-Vcpkg -EndToEndTestSilent @commonArgs install | Out-Null
+Throw-IfFailed
+Refresh-TestRoot
+
+# [patterns] Repeated patterns
+Write-Host "[patterns] Repeated patterns"
+$inFile = "$manifestRoot/with-redeclaration.json.in"
+(Get-Content -Path "$inFile").Replace("`$E2ERegistryPath", $e2eRegistryPath).Replace("`$E2ERegistryBaseline", $e2eRegistryBaseline) `
+| Out-File "$manifestRoot/vcpkg.json"
+
+$out = Run-VcpkgAndCaptureOutput -EndToEndTestSilent @commonArgs install
+Throw-IfFailed
+if ($out -notmatch "redeclarations will be ignored")
+{
+	$out
+	throw "Expected warning about redeclaration"
+}
+Refresh-TestRoot

include/vcpkg/base/jsonreader.h

@@ -378,10 +378,29 @@ namespace vcpkg::Json
     {
         virtual StringView type_name() const override { return "a package name"; }
 
-        static bool is_package_name(StringView sv);
-
         virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) override;
 
         static PackageNameDeserializer instance;
     };
+
+    /// <summary>
+    /// A registry package pattern (e.g.: boost*) and the in-file location where it was declared.
+    /// </summary>
+    struct PackagePatternDeclaration
+    {
+        std::string pattern;
+        std::string location;
+    };
+
+    /// <summary>
+    /// Deserializes a list of package names and patterns along with their respective in-file declaration locations.
+    /// </summary>
+    struct PackagePatternDeserializer final : Json::IDeserializer<PackagePatternDeclaration>
+    {
+        virtual StringView type_name() const override { return "a package pattern"; }
+
+        static bool is_package_pattern(StringView sv);
+
+        virtual Optional<PackagePatternDeclaration> visit_string(Json::Reader&, StringView sv) override;
+    };
 }

include/vcpkg/base/messages.h

@@ -947,6 +947,13 @@ namespace vcpkg
                     "Comparing Utf8Decoders with different provenance; this is always an error");
     DECLARE_MESSAGE(CompressFolderFailed, (msg::path), "", "Failed to compress folder \"{path}\":");
     DECLARE_MESSAGE(ComputingInstallPlan, (), "", "Computing installation plan...");
+    DECLARE_MESSAGE(ConfigurationErrorRegistriesWithoutBaseline,
+                    (msg::path, msg::url),
+                    "",
+                    "The configuration defined in {path} is invalid.\n\n"
+                    "Using registries requires that a baseline is set for the default registry or that the default "
+                    "registry is null.\n\n"
+                    "See {url} for more details.");
     DECLARE_MESSAGE(ConflictingFiles,
                     (msg::path, msg::spec),
                     "",
@@ -1101,6 +1108,11 @@ namespace vcpkg
                     (msg::value),
                     "'{value}' is a command line option.",
                     "'--{value}' specified multiple times.");
+    DECLARE_MESSAGE(DuplicatePackagePattern, (msg::package_name), "", "Package \"{package_name}\" is duplicated.");
+    DECLARE_MESSAGE(DuplicatePackagePatternFirstOcurrence, (), "", "First declared in:");
+    DECLARE_MESSAGE(DuplicatePackagePatternIgnoredLocations, (), "", "The following redeclarations will be ignored:");
+    DECLARE_MESSAGE(DuplicatePackagePatternLocation, (msg::path), "", "location: {path}");
+    DECLARE_MESSAGE(DuplicatePackagePatternRegistry, (msg::url), "", "registry: {url}");
     DECLARE_MESSAGE(ElapsedInstallTime, (msg::count), "", "Total elapsed time: {count}");
     DECLARE_MESSAGE(ElapsedTimeForChecks, (msg::elapsed), "", "Time to determine pass/fail: {elapsed}");
     DECLARE_MESSAGE(EmailVcpkgTeam, (msg::url), "", "Send an email to {url} with any feedback.");
@@ -1965,6 +1977,24 @@ namespace vcpkg
                     "Error messages are is printed after this.",
                     "while loading {path}:");
     DECLARE_MESSAGE(ParseControlErrorInfoWrongTypeFields, (), "", "The following fields had the wrong types:");
+    DECLARE_MESSAGE(
+        ParseIdentifierError,
+        (msg::value, msg::url),
+        "{value} is a lowercase identifier like 'boost'",
+        "\"{value}\" is not a valid identifier. "
+        "Identifiers must be lowercase alphanumeric+hypens and not reserved (see {url} for more information)");
+    DECLARE_MESSAGE(
+        ParsePackageNameError,
+        (msg::package_name, msg::url),
+        "",
+        "\"{package_name}\" is not a valid package name. "
+        "Package names must be lowercase alphanumeric+hypens and not reserved (see {url} for more information)");
+    DECLARE_MESSAGE(ParsePackagePatternError,
+                    (msg::package_name, msg::url),
+                    "",
+                    "\"{package_name}\" is not a valid package pattern. "
+                    "Package patterns must use only one wildcard character (*) and it must be the last character in "
+                    "the pattern (see {url} for more information)");
     DECLARE_MESSAGE(PathMustBeAbsolute,
                     (msg::path),
                     "",
@@ -2544,6 +2574,7 @@ namespace vcpkg
                     "The message named {value} starts with warning:, it must be changed to prepend "
                     "WarningMessage in code instead.");
     DECLARE_MESSAGE(WarningsTreatedAsErrors, (), "", "previous warnings being interpreted as errors");
+    DECLARE_MESSAGE(WarnOnParseConfig, (msg::path), "", "Found the following warnings in configuration {path}:");
     DECLARE_MESSAGE(WhileLookingForSpec, (msg::spec), "", "while looking for {spec}:");
     DECLARE_MESSAGE(WindowsOnlyCommand, (), "", "This command only supports Windows.");
     DECLARE_MESSAGE(WroteNuGetPkgConfInfo, (msg::path), "", "Wrote NuGet package config information to {path}");