From ee6dc8e8969ac790c250a8d1dbea0002c701af27 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 6 Sep 2024 16:54:25 -0700 Subject: [PATCH] Fix read operations like list on readonly filesystems. Resolves https://github.com/microsoft/vcpkg/issues/10812 --- include/vcpkg/vcpkglib.h | 8 +- src/vcpkg/commands.build.cpp | 2 +- src/vcpkg/commands.ci.cpp | 2 +- src/vcpkg/commands.export.cpp | 2 +- src/vcpkg/commands.install.cpp | 5 +- src/vcpkg/commands.list.cpp | 3 +- src/vcpkg/commands.owns.cpp | 6 +- src/vcpkg/commands.package-info.cpp | 2 +- src/vcpkg/commands.remove.cpp | 5 +- src/vcpkg/commands.set-installed.cpp | 2 +- src/vcpkg/commands.update.cpp | 2 +- src/vcpkg/commands.upgrade.cpp | 2 +- src/vcpkg/vcpkglib.cpp | 189 ++++++++++++++++++--------- 13 files changed, 149 insertions(+), 81 deletions(-) diff --git a/include/vcpkg/vcpkglib.h b/include/vcpkg/vcpkglib.h index 7621d8526b..6eeffbf4c3 100644 --- a/include/vcpkg/vcpkglib.h +++ b/include/vcpkg/vcpkglib.h @@ -13,7 +13,8 @@ namespace vcpkg { - StatusParagraphs database_load_check(const Filesystem& fs, const InstalledPaths& installed); + StatusParagraphs database_load(const ReadOnlyFilesystem& fs, const InstalledPaths& installed); + StatusParagraphs database_load_collapse(const Filesystem& fs, const InstalledPaths& installed); void write_update(const Filesystem& fs, const InstalledPaths& installed, const StatusParagraph& p); @@ -24,9 +25,12 @@ namespace vcpkg }; std::vector get_installed_ports(const StatusParagraphs& status_db); - std::vector get_installed_files(const Filesystem& fs, + std::vector get_installed_files(const ReadOnlyFilesystem& fs, const InstalledPaths& installed, const StatusParagraphs& status_db); + std::vector get_installed_files_and_upgrade(const Filesystem& fs, + const InstalledPaths& installed, + const StatusParagraphs& status_db); std::string shorten_text(StringView desc, const size_t length); } // namespace vcpkg diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index d1a8a8d494..5c6c60119a 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -132,7 +132,7 @@ namespace vcpkg auto& var_provider = *var_provider_storage; var_provider.load_dep_info_vars({{spec}}, host_triplet); - StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + StatusParagraphs status_db = database_load_collapse(paths.get_filesystem(), paths.installed()); auto action_plan = create_feature_install_plan( provider, var_provider, diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 24581927f5..e000d5dde7 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -502,7 +502,7 @@ namespace vcpkg } else { - StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + StatusParagraphs status_db = database_load_collapse(paths.get_filesystem(), paths.installed()); auto already_installed = adjust_action_plan_to_status_db(action_plan, status_db); Util::erase_if(already_installed, [&](auto& spec) { return Util::Sets::contains(split_specs->known, spec); }); diff --git a/src/vcpkg/commands.export.cpp b/src/vcpkg/commands.export.cpp index 4032519205..8546687ae1 100644 --- a/src/vcpkg/commands.export.cpp +++ b/src/vcpkg/commands.export.cpp @@ -596,7 +596,7 @@ namespace vcpkg Triplet host_triplet) { (void)host_triplet; - const StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + const StatusParagraphs status_db = database_load(paths.get_filesystem(), paths.installed()); const auto opts = handle_export_command_arguments(paths, args, default_triplet, status_db); // Load ports from ports dirs diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 82c91158d8..dd714e1b3b 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -224,7 +224,7 @@ namespace vcpkg const auto package_dir = paths.package_dir(bcf.core_paragraph.spec); Triplet triplet = bcf.core_paragraph.spec.triplet(); const std::vector pgh_and_files = - get_installed_files(fs, installed, *status_db); + get_installed_files_and_upgrade(fs, installed, *status_db); const SortedVector package_files = build_list_of_package_files(fs, package_dir); const SortedVector installed_files = build_list_of_installed_files(pgh_and_files, triplet); @@ -617,6 +617,7 @@ namespace vcpkg this_install.current_summary.build_result.emplace(std::move(result)); } + database_load_collapse(fs, paths.installed()); msg::println(msgTotalInstallTime, msg::elapsed = timer.to_string()); return InstallSummary{std::move(results)}; } @@ -1286,7 +1287,7 @@ namespace vcpkg // create the plan msg::println(msgComputingInstallPlan); - StatusParagraphs status_db = database_load_check(fs, paths.installed()); + StatusParagraphs status_db = database_load_collapse(fs, paths.installed()); // Note: action_plan will hold raw pointers to SourceControlFileLocations from this map auto action_plan = create_feature_install_plan(provider, var_provider, specs, status_db, create_options); diff --git a/src/vcpkg/commands.list.cpp b/src/vcpkg/commands.list.cpp index a7ff89e78b..0d4f4956b2 100644 --- a/src/vcpkg/commands.list.cpp +++ b/src/vcpkg/commands.list.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -103,7 +104,7 @@ namespace vcpkg msg::default_output_stream = OutputStream::StdErr; const ParsedArguments options = args.parse_arguments(CommandListMetadata); - const StatusParagraphs status_paragraphs = database_load_check(paths.get_filesystem(), paths.installed()); + const StatusParagraphs status_paragraphs = database_load(paths.get_filesystem(), paths.installed()); auto installed_ipv = get_installed_ports(status_paragraphs); const auto output_json = Util::Sets::contains(options.switches, SwitchXJson); diff --git a/src/vcpkg/commands.owns.cpp b/src/vcpkg/commands.owns.cpp index f960371c84..ca3e47c5ea 100644 --- a/src/vcpkg/commands.owns.cpp +++ b/src/vcpkg/commands.owns.cpp @@ -1,3 +1,5 @@ +#include + #include #include #include @@ -8,7 +10,7 @@ using namespace vcpkg; namespace { - void search_file(const Filesystem& fs, + void search_file(const ReadOnlyFilesystem& fs, const InstalledPaths& installed, const std::string& file_substr, const StatusParagraphs& status_db) @@ -46,7 +48,7 @@ namespace vcpkg void command_owns_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) { const auto parsed = args.parse_arguments(CommandOwnsMetadata); - const StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + const StatusParagraphs status_db = database_load(paths.get_filesystem(), paths.installed()); search_file(paths.get_filesystem(), paths.installed(), parsed.command_arguments[0], status_db); Checks::exit_success(VCPKG_LINE_INFO); } diff --git a/src/vcpkg/commands.package-info.cpp b/src/vcpkg/commands.package-info.cpp index 70608eb521..f7c9597905 100644 --- a/src/vcpkg/commands.package-info.cpp +++ b/src/vcpkg/commands.package-info.cpp @@ -61,7 +61,7 @@ namespace vcpkg auto& fs = paths.get_filesystem(); if (installed) { - const StatusParagraphs status_paragraphs = database_load_check(fs, paths.installed()); + const StatusParagraphs status_paragraphs = database_load(fs, paths.installed()); std::set specs_written; std::vector specs_to_write; for (auto&& arg : options.command_arguments) diff --git a/src/vcpkg/commands.remove.cpp b/src/vcpkg/commands.remove.cpp index 9d3dbe6e5f..835b9bdf9e 100644 --- a/src/vcpkg/commands.remove.cpp +++ b/src/vcpkg/commands.remove.cpp @@ -148,7 +148,7 @@ namespace std::vector valid_arguments(const VcpkgPaths& paths) { - const StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + const StatusParagraphs status_db = database_load(paths.get_filesystem(), paths.installed()); auto installed_packages = get_installed_ports(status_db); return Util::fmap(installed_packages, [](auto&& pgh) -> std::string { return pgh.spec().to_string(); }); @@ -181,7 +181,7 @@ namespace vcpkg } const ParsedArguments options = args.parse_arguments(CommandRemoveMetadata); - StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + StatusParagraphs status_db = database_load_collapse(paths.get_filesystem(), paths.installed()); std::vector specs; if (Util::Sets::contains(options.switches, SwitchOutdated)) { @@ -299,6 +299,7 @@ namespace vcpkg } } + database_load_collapse(fs, paths.installed()); Checks::exit_success(VCPKG_LINE_INFO); } } diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 7bc701e519..0d38a0f171 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -211,7 +211,7 @@ namespace vcpkg } // currently (or once) installed specifications - auto status_db = database_load_check(fs, paths.installed()); + auto status_db = database_load_collapse(fs, paths.installed()); adjust_action_plan_to_status_db(action_plan, status_db); print_plan(action_plan, paths.builtin_ports_directory()); diff --git a/src/vcpkg/commands.update.cpp b/src/vcpkg/commands.update.cpp index 3aec063456..e2b7b44109 100644 --- a/src/vcpkg/commands.update.cpp +++ b/src/vcpkg/commands.update.cpp @@ -65,7 +65,7 @@ namespace vcpkg msg::println(msgLocalPortfileVersion); auto& fs = paths.get_filesystem(); - const StatusParagraphs status_db = database_load_check(fs, paths.installed()); + const StatusParagraphs status_db = database_load(fs, paths.installed()); auto registry_set = paths.make_registry_set(); PathsPortFileProvider provider(*registry_set, diff --git a/src/vcpkg/commands.upgrade.cpp b/src/vcpkg/commands.upgrade.cpp index a113231a56..a59ff797d5 100644 --- a/src/vcpkg/commands.upgrade.cpp +++ b/src/vcpkg/commands.upgrade.cpp @@ -76,7 +76,7 @@ namespace vcpkg const CreateUpgradePlanOptions create_upgrade_plan_options{ nullptr, host_triplet, paths.packages(), unsupported_port_action}; - StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + StatusParagraphs status_db = database_load_collapse(paths.get_filesystem(), paths.installed()); // Load ports from ports dirs auto& fs = paths.get_filesystem(); diff --git a/src/vcpkg/vcpkglib.cpp b/src/vcpkg/vcpkglib.cpp index eda0c1f641..339a518ed5 100644 --- a/src/vcpkg/vcpkglib.cpp +++ b/src/vcpkg/vcpkglib.cpp @@ -10,21 +10,8 @@ namespace vcpkg { - static StatusParagraphs load_current_database(const Filesystem& fs, - const Path& vcpkg_dir_status_file, - const Path& vcpkg_dir_status_file_old) + static StatusParagraphs load_current_database(const ReadOnlyFilesystem& fs, const Path& vcpkg_dir_status_file) { - if (!fs.exists(vcpkg_dir_status_file, IgnoreErrors{})) - { - if (!fs.exists(vcpkg_dir_status_file_old, IgnoreErrors{})) - { - // no status file, use empty db - return StatusParagraphs(); - } - - fs.rename(vcpkg_dir_status_file_old, vcpkg_dir_status_file, VCPKG_LINE_INFO); - } - auto pghs = Paragraphs::get_paragraphs(fs, vcpkg_dir_status_file).value_or_exit(VCPKG_LINE_INFO); std::vector> status_pghs; @@ -37,7 +24,76 @@ namespace vcpkg return StatusParagraphs(std::move(status_pghs)); } - StatusParagraphs database_load_check(const Filesystem& fs, const InstalledPaths& installed) + static std::vector apply_database_updates(const ReadOnlyFilesystem& fs, + StatusParagraphs& current_status_db, + const Path& updates_dir) + { + auto update_files = fs.get_regular_files_non_recursive(updates_dir, VCPKG_LINE_INFO); + Util::sort(update_files); + if (!update_files.empty()) + { + for (auto&& file : update_files) + { + if (file.filename() == "incomplete") continue; + + auto pghs = Paragraphs::get_paragraphs(fs, file).value_or_exit(VCPKG_LINE_INFO); + for (auto&& p : pghs) + { + current_status_db.insert(std::make_unique(file, std::move(p))); + } + } + } + + return update_files; + } + + static void apply_database_updates_on_disk(const Filesystem& fs, + const InstalledPaths& installed, + StatusParagraphs& current_status_db) + { + auto update_files = apply_database_updates(fs, current_status_db, installed.vcpkg_dir_updates()); + if (!update_files.empty()) + { + const auto status_file = installed.vcpkg_dir_status_file(); + const auto status_file_new = Path(status_file.parent_path()) / "status-new"; + fs.write_contents(status_file_new, Strings::serialize(current_status_db), VCPKG_LINE_INFO); + + fs.rename(status_file_new, status_file, VCPKG_LINE_INFO); + + for (auto&& file : update_files) + { + fs.remove(file, VCPKG_LINE_INFO); + } + } + } + + StatusParagraphs database_load(const ReadOnlyFilesystem& fs, const InstalledPaths& installed) + { + const auto maybe_status_file = installed.vcpkg_dir_status_file(); + const auto status_parent = Path(maybe_status_file.parent_path()); + const auto status_file_old = status_parent / "status-old"; + + auto status_file = &maybe_status_file; + + if (!fs.exists(maybe_status_file, IgnoreErrors{})) + { + if (!fs.exists(status_file_old, IgnoreErrors{})) + { + // no status file, use empty db + StatusParagraphs current_status_db; + (void)apply_database_updates(fs, current_status_db, installed.vcpkg_dir_updates()); + return current_status_db; + } + + status_file = &status_file_old; + } + + StatusParagraphs current_status_db = load_current_database(fs, *status_file); + (void)apply_database_updates(fs, current_status_db, installed.vcpkg_dir_updates()); + return current_status_db; + } + + StatusParagraphs database_load_collapse(const Filesystem& fs, const InstalledPaths& installed) { const auto updates_dir = installed.vcpkg_dir_updates(); @@ -49,37 +105,22 @@ namespace vcpkg const auto status_file = installed.vcpkg_dir_status_file(); const auto status_parent = Path(status_file.parent_path()); const auto status_file_old = status_parent / "status-old"; - const auto status_file_new = status_parent / "status-new"; - - StatusParagraphs current_status_db = load_current_database(fs, status_file, status_file_old); - auto update_files = fs.get_regular_files_non_recursive(updates_dir, VCPKG_LINE_INFO); - Util::sort(update_files); - if (update_files.empty()) - { - // updates directory is empty, control file is up-to-date. - return current_status_db; - } - for (auto&& file : update_files) + if (!fs.exists(status_file, IgnoreErrors{})) { - if (file.filename() == "incomplete") continue; - - auto pghs = Paragraphs::get_paragraphs(fs, file).value_or_exit(VCPKG_LINE_INFO); - for (auto&& p : pghs) + if (!fs.exists(status_file_old, IgnoreErrors{})) { - current_status_db.insert(std::make_unique(file, std::move(p))); + // no status file, use empty db + StatusParagraphs current_status_db; + apply_database_updates_on_disk(fs, installed, current_status_db); + return current_status_db; } - } - - fs.write_contents(status_file_new, Strings::serialize(current_status_db), VCPKG_LINE_INFO); - fs.rename(status_file_new, status_file, VCPKG_LINE_INFO); - - for (auto&& file : update_files) - { - fs.remove(file, VCPKG_LINE_INFO); + fs.rename(status_file_old, status_file, VCPKG_LINE_INFO); } + StatusParagraphs current_status_db = load_current_database(fs, status_file); + apply_database_updates_on_disk(fs, installed, current_status_db); return current_status_db; } @@ -93,25 +134,22 @@ namespace vcpkg fs.write_rename_contents(update_path, "incomplete", Strings::serialize(p), VCPKG_LINE_INFO); } - static void upgrade_to_slash_terminated_sorted_format(const Filesystem& fs, - std::vector* lines, - const Path& listfile_path) + static bool upgrade_to_slash_terminated_sorted_format(std::vector& lines) { - static bool was_tracked = false; + static std::atomic was_tracked = false; - if (lines->empty()) + if (lines.empty()) { - return; + return false; } - if (lines->at(0).back() == '/') + if (lines.front().back() == '/') { - return; // File already in the new format + return false; // File already in the new format } - if (!was_tracked) + if (was_tracked.exchange(true)) { - was_tracked = true; get_global_metrics_collector().track_string(StringMetric::ListFile, "update to new format"); } @@ -119,14 +157,15 @@ namespace vcpkg // (They are not necessarily sorted alphabetically, e.g. libflac) // Therefore we can detect the entries that represent directories by comparing every element with the next one // and checking if the next has a slash immediately after the current one's length - for (size_t i = 0; i < lines->size() - 1; i++) + const size_t end = lines.size() - 1; + for (size_t i = 0; i < end; i++) { - std::string& current_string = lines->at(i); - const std::string& next_string = lines->at(i + 1); - + std::string& current_string = lines[i]; + const std::string& next_string = lines[i + 1]; + // check if the next line is the same as this one with a slash after; that indicates that this one + // represents a directory const size_t potential_slash_char_index = current_string.length(); - // Make sure the index exists first - if (next_string.size() > potential_slash_char_index && next_string.at(potential_slash_char_index) == '/') + if (next_string.size() > potential_slash_char_index && next_string[potential_slash_char_index] == '/') { current_string += '/'; // Mark as a directory } @@ -155,12 +194,8 @@ namespace vcpkg */ // Note that after sorting, the FLAC++/ group will be placed before the FLAC/ group // The new format is lexicographically sorted - std::sort(lines->begin(), lines->end()); - - // Replace the listfile on disk - const auto updated_listfile_path = listfile_path + "_updated"; - fs.write_lines(updated_listfile_path, *lines, VCPKG_LINE_INFO); - fs.rename(updated_listfile_path, listfile_path, VCPKG_LINE_INFO); + Util::sort(lines); + return true; } std::vector get_installed_ports(const StatusParagraphs& status_db) @@ -189,9 +224,10 @@ namespace vcpkg return Util::fmap(ipv_map, [](auto&& p) -> InstalledPackageView { return std::move(p.second); }); } - std::vector get_installed_files(const Filesystem& fs, - const InstalledPaths& installed, - const StatusParagraphs& status_db) + template + static std::vector get_installed_files_impl(const FilesystemLike& fs, + const InstalledPaths& installed, + const StatusParagraphs& status_db) { std::vector installed_files; @@ -206,7 +242,16 @@ namespace vcpkg std::vector installed_files_of_current_pgh = fs.read_lines(listfile_path).value_or_exit(VCPKG_LINE_INFO); Strings::inplace_trim_all_and_remove_whitespace_strings(installed_files_of_current_pgh); - upgrade_to_slash_terminated_sorted_format(fs, &installed_files_of_current_pgh, listfile_path); + if (upgrade_to_slash_terminated_sorted_format(installed_files_of_current_pgh)) + { + if constexpr (AndUpdate) + { + // Replace the listfile on disk + const auto updated_listfile_path = listfile_path + "_updated"; + fs.write_lines(updated_listfile_path, installed_files_of_current_pgh, VCPKG_LINE_INFO); + fs.rename(updated_listfile_path, listfile_path, VCPKG_LINE_INFO); + } + } // Remove the directories Util::erase_remove_if(installed_files_of_current_pgh, @@ -220,6 +265,20 @@ namespace vcpkg return installed_files; } + std::vector get_installed_files(const ReadOnlyFilesystem& fs, + const InstalledPaths& installed, + const StatusParagraphs& status_db) + { + return get_installed_files_impl(fs, installed, status_db); + } + + std::vector get_installed_files_and_upgrade(const Filesystem& fs, + const InstalledPaths& installed, + const StatusParagraphs& status_db) + { + return get_installed_files_impl(fs, installed, status_db); + } + std::string shorten_text(StringView desc, const size_t length) { Checks::check_exit(VCPKG_LINE_INFO, length >= 3);