Binary cache: async push_success

include/vcpkg/binarycaching.h

@@ -9,10 +9,14 @@
 #include <vcpkg/base/files.h>
 
 #include <vcpkg/packagespec.h>
+#include <vcpkg/sourceparagraph.h>
 
+#include <condition_variable>
 #include <iterator>
+#include <queue>
 #include <set>
 #include <string>
+#include <thread>
 #include <unordered_map>
 #include <vector>
 
@@ -42,6 +46,21 @@ namespace vcpkg
         const IBinaryProvider* m_available_provider = nullptr; // meaningful iff m_status == available
     };
 
+    struct BinaryPackageInformation
+    {
+        explicit BinaryPackageInformation(const InstallPlanAction& action);
+        std::string package_abi;
+        // The following fields are only needed for the nuget binary cache
+        PackageSpec spec;
+        SourceControlFile& source_control_file;
+        std::string& raw_version;
+        std::string compiler_id;
+        std::string compiler_version;
+        std::string triplet_abi;
+        InternalFeatureSet feature_list;
+        std::vector<PackageSpec> package_dependencies;
+    };
+
     struct IBinaryProvider
     {
         virtual ~IBinaryProvider() = default;
@@ -52,7 +71,9 @@ namespace vcpkg
 
         /// Called upon a successful build of `action` to store those contents in the binary cache.
         /// Prerequisite: action has a package_abi()
-        virtual void push_success(const InstallPlanAction& action) const = 0;
+        virtual void push_success(const BinaryPackageInformation& info,
+                                  const Path& packages_dir,
+                                  MessageSink& msg_sink) = 0;
 
         /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for
         /// executing `actions`.
@@ -77,7 +98,7 @@ namespace vcpkg
         std::vector<std::string> headers_for_get;
 
         LocalizedString valid() const;
-        std::string instantiate_variables(const InstallPlanAction& action) const;
+        std::string instantiate_variables(const BinaryPackageInformation& info) const;
     };
 
     struct BinaryConfigParserState
@@ -121,17 +142,21 @@ namespace vcpkg
 
     struct BinaryCache
     {
-        BinaryCache() = default;
+        static BinaryCache* current_instance;
+        static void wait_for_async_complete();
+        BinaryCache(Filesystem& filesystem);
         explicit BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths);
 
+        ~BinaryCache();
+
         void install_providers(std::vector<std::unique_ptr<IBinaryProvider>>&& providers);
         void install_providers_for(const VcpkgCmdArguments& args, const VcpkgPaths& paths);
 
         /// Attempts to restore the package referenced by `action` into the packages directory.
         RestoreResult try_restore(const InstallPlanAction& action);
 
         /// Called upon a successful build of `action` to store those contents in the binary cache.
-        void push_success(const InstallPlanAction& action);
+        void push_success(const InstallPlanAction& action, Path package_dir);
 
         /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for
         /// executing `actions`.
@@ -143,8 +168,22 @@ namespace vcpkg
         std::vector<CacheAvailability> precheck(View<InstallPlanAction> actions);
 
     private:
+        struct ActionToPush
+        {
+            BinaryPackageInformation info;
+            bool clean_after_push;
+            Path packages_dir;
+        };
+        void push_thread_main();
+
         std::unordered_map<std::string, CacheStatus> m_status;
         std::vector<std::unique_ptr<IBinaryProvider>> m_providers;
+        std::condition_variable actions_to_push_notifier;
+        std::mutex actions_to_push_mutex;
+        std::queue<ActionToPush> actions_to_push;
+        std::thread push_thread;
+        std::atomic_bool end_push_thread;
+        Filesystem& filesystem;
     };
 
     ExpectedL<DownloadManagerConfig> parse_download_configuration(const Optional<std::string>& arg);

include/vcpkg/binarycaching.private.h

@@ -5,6 +5,7 @@
 
 #include <vcpkg/base/strings.h>
 
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/dependencies.h>
 
 namespace vcpkg
@@ -35,6 +36,10 @@ namespace vcpkg
     {
         return {Strings::concat(prefix, spec.dir()), format_version_for_nugetref(raw_version, abi_tag)};
     }
+    inline NugetReference make_nugetref(const BinaryPackageInformation& info, const std::string& prefix)
+    {
+        return make_nugetref(info.spec, info.raw_version, info.package_abi, prefix);
+    }
     inline NugetReference make_nugetref(const InstallPlanAction& action, const std::string& prefix)
     {
         return make_nugetref(action.spec,
@@ -57,7 +62,7 @@ namespace vcpkg
     }
 
     std::string generate_nuspec(const Path& package_dir,
-                                const InstallPlanAction& action,
+                                const BinaryPackageInformation& info,
                                 const NugetReference& ref,
                                 details::NuGetRepoInfo rinfo = details::get_nuget_repo_info_from_env());
 }

include/vcpkg/build.h

@@ -205,7 +205,6 @@ namespace vcpkg
     ExtendedBuildResult build_package(const VcpkgCmdArguments& args,
                                       const VcpkgPaths& paths,
                                       const InstallPlanAction& config,
-                                      BinaryCache& binary_cache,
                                       const IBuildLogsRecorder& build_logs_recorder,
                                       const StatusParagraphs& status_db);
 

include/vcpkg/fwd/binarycaching.h

@@ -25,4 +25,5 @@ namespace vcpkg
     struct IBinaryProvider;
     struct BinaryCache;
     struct BinaryConfigParserState;
+    struct BinaryPackageInformation;
 }

src/vcpkg-test/binarycaching.cpp

@@ -24,17 +24,20 @@ struct KnowNothingBinaryProvider : IBinaryProvider
         return RestoreResult::unavailable;
     }
 
-    virtual void push_success(const InstallPlanAction& action) const override { CHECK(action.has_package_abi()); }
+    void push_success(const BinaryPackageInformation& info, const Path&, MessageSink&) override
+    {
+        CHECK_FALSE(info.package_abi.empty());
+    }
 
-    virtual void prefetch(View<InstallPlanAction> actions, View<CacheStatus* const> cache_status) const override
+    void prefetch(View<InstallPlanAction> actions, View<CacheStatus* const> cache_status) const override
     {
         REQUIRE(actions.size() == cache_status.size());
         for (size_t idx = 0; idx < cache_status.size(); ++idx)
         {
             CHECK(actions[idx].has_package_abi() == (cache_status[idx] != nullptr));
         }
     }
-    virtual void precheck(View<InstallPlanAction> actions, View<CacheStatus* const> cache_status) const override
+    void precheck(View<InstallPlanAction> actions, View<CacheStatus* const> cache_status) const override
     {
         REQUIRE(actions.size() == cache_status.size());
         for (const auto c : cache_status)
@@ -277,7 +280,7 @@ Build-Depends: bzip
     REQUIRE(ref.nupkg_filename() == "zlib2_x64-windows.1.5.0-vcpkgpackageabi.nupkg");
 
     {
-        auto nuspec = generate_nuspec(pkgPath, ipa, ref, {});
+        auto nuspec = generate_nuspec(pkgPath, BinaryPackageInformation{ipa}, ref, {});
         std::string expected = R"(<package>
   <metadata>
     <id>zlib2_x64-windows</id>
@@ -305,7 +308,7 @@ Features: a, b
     }
 
     {
-        auto nuspec = generate_nuspec(pkgPath, ipa, ref, {"urlvalue"});
+        auto nuspec = generate_nuspec(pkgPath, BinaryPackageInformation{ipa}, ref, {"urlvalue"});
         std::string expected = R"(<package>
   <metadata>
     <id>zlib2_x64-windows</id>
@@ -333,7 +336,8 @@ Features: a, b
         REQUIRE_EQUAL_TEXT(nuspec, expected);
     }
     {
-        auto nuspec = generate_nuspec(pkgPath, ipa, ref, {"urlvalue", "branchvalue", "commitvalue"});
+        auto nuspec =
+            generate_nuspec(pkgPath, BinaryPackageInformation{ipa}, ref, {"urlvalue", "branchvalue", "commitvalue"});
         std::string expected = R"(<package>
   <metadata>
     <id>zlib2_x64-windows</id>
@@ -365,7 +369,7 @@ Features: a, b
 TEST_CASE ("Provider nullptr checks", "[BinaryCache]")
 {
     // create a binary cache to test
-    BinaryCache uut;
+    BinaryCache uut(get_real_filesystem());
     std::vector<std::unique_ptr<IBinaryProvider>> providers;
     providers.emplace_back(std::make_unique<KnowNothingBinaryProvider>());
     uut.install_providers(std::move(providers));
@@ -391,7 +395,7 @@ Version: 1.5
     InstallPlanAction& ipa_without_abi = install_plan.back();
 
     // test that the binary cache does the right thing. See also CHECKs etc. in KnowNothingBinaryProvider
-    uut.push_success(ipa_without_abi); // should have no effects
+    uut.push_success(ipa_without_abi, {}); // should have no effects
     CHECK(uut.try_restore(ipa_without_abi) == RestoreResult::unavailable);
     uut.prefetch(install_plan); // should have no effects
 }

src/vcpkg.cpp

@@ -10,6 +10,7 @@
 #include <vcpkg/base/system.h>
 #include <vcpkg/base/system.process.h>
 
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/bundlesettings.h>
 #include <vcpkg/cgroup-parser.h>
 #include <vcpkg/commands.contact.h>
@@ -156,6 +157,7 @@ namespace vcpkg::Checks
     // Implements link seam from basic_checks.h
     void on_final_cleanup_and_exit()
     {
+        BinaryCache::wait_for_async_complete();
         const auto elapsed_us_inner = g_total_time.microseconds();
         bool debugging = Debug::g_debugging;