diff --git a/include/vcpkg/abi.h b/include/vcpkg/abi.h new file mode 100644 index 0000000000..e2ff538fb8 --- /dev/null +++ b/include/vcpkg/abi.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include + +namespace vcpkg +{ + struct AbiEntry + { + std::string key; + std::string value; + + AbiEntry() = default; + AbiEntry(std::string key, std::string value) : key(std::move(key)), value(std::move(value)) { } + + bool operator<(const AbiEntry& other) const noexcept + { + return key < other.key || (key == other.key && value < other.value); + } + }; + + struct AbiInfo + { + // These should always be known if an AbiInfo exists + std::unique_ptr pre_build_info; + Optional toolset; + // These might not be known if compiler tracking is turned off or the port is --editable + Optional compiler_info; + Optional triplet_abi; + std::string package_abi; + + // Checks if full abi tag was created + bool abi_tag_complete() const noexcept { return package_abi.size() != 0; } + + void abi_file_contents(std::string&& abi_tag, std::string&& sbom_file) noexcept + { + abi_tag_file_contents.emplace(std::move(abi_tag)); + sbom_file_contents.emplace(std::move(sbom_file)); + } + + // PRE: abi_tag_complete() == true + // dir: Directory where the files should be saved, usually packages_dir/share + void save_abi_files(const Filesystem& fs, Path&& dir) const; + + private: + Optional abi_tag_file_contents; + Optional sbom_file_contents; + }; + + void compute_all_abis(const VcpkgPaths& paths, + ActionPlan& action_plan, + const CMakeVars::CMakeVarProvider& var_provider, + const StatusParagraphs& status_db); +} // namespace vcpkg diff --git a/include/vcpkg/base/path.h b/include/vcpkg/base/path.h index dce843d143..51f007c578 100644 --- a/include/vcpkg/base/path.h +++ b/include/vcpkg/base/path.h @@ -21,7 +21,8 @@ namespace vcpkg const char* c_str() const noexcept; - std::string generic_u8string() const; + std::string generic_u8string() const&; + std::string generic_u8string() &&; bool empty() const noexcept; diff --git a/include/vcpkg/commands.build.h b/include/vcpkg/commands.build.h index 747c130141..25a708f628 100644 --- a/include/vcpkg/commands.build.h +++ b/include/vcpkg/commands.build.h @@ -212,20 +212,6 @@ namespace vcpkg BuildInfo read_build_info(const ReadOnlyFilesystem& fs, const Path& filepath); - struct AbiEntry - { - std::string key; - std::string value; - - AbiEntry() = default; - AbiEntry(StringView key, StringView value) : key(key.to_string()), value(value.to_string()) { } - - bool operator<(const AbiEntry& other) const - { - return key < other.key || (key == other.key && value < other.value); - } - }; - struct CompilerInfo { std::string id; @@ -234,26 +220,6 @@ namespace vcpkg std::string path; }; - struct AbiInfo - { - // These should always be known if an AbiInfo exists - std::unique_ptr pre_build_info; - Optional toolset; - // These might not be known if compiler tracking is turned off or the port is --editable - Optional compiler_info; - Optional triplet_abi; - std::string package_abi; - Optional abi_tag_file; - std::vector relative_port_files; - std::vector relative_port_hashes; - std::vector heuristic_resources; - }; - - void compute_all_abis(const VcpkgPaths& paths, - ActionPlan& action_plan, - const CMakeVars::CMakeVarProvider& var_provider, - const StatusParagraphs& status_db); - struct EnvCache { explicit EnvCache(bool compiler_tracking) : m_compiler_tracking(compiler_tracking) { } diff --git a/include/vcpkg/dependencies.h b/include/vcpkg/dependencies.h index 994952581f..f3f80270a8 100644 --- a/include/vcpkg/dependencies.h +++ b/include/vcpkg/dependencies.h @@ -7,6 +7,7 @@ #include +#include #include #include #include diff --git a/include/vcpkg/fwd/abi.h b/include/vcpkg/fwd/abi.h new file mode 100644 index 0000000000..d5843cb0c5 --- /dev/null +++ b/include/vcpkg/fwd/abi.h @@ -0,0 +1,7 @@ +#pragma once + +namespace vcpkg +{ + struct AbiInfo; + struct AbiEntry; +} diff --git a/include/vcpkg/fwd/build.h b/include/vcpkg/fwd/build.h index ee0fe89f08..8bbc5274d1 100644 --- a/include/vcpkg/fwd/build.h +++ b/include/vcpkg/fwd/build.h @@ -134,9 +134,7 @@ namespace vcpkg struct PreBuildInfo; struct ExtendedBuildResult; struct BuildInfo; - struct AbiEntry; struct CompilerInfo; - struct AbiInfo; struct EnvCache; struct BuildCommand; } diff --git a/include/vcpkg/spdx.h b/include/vcpkg/spdx.h index 902b731872..d3c1998fd1 100644 --- a/include/vcpkg/spdx.h +++ b/include/vcpkg/spdx.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -23,11 +24,14 @@ namespace vcpkg /// @param resource_docs Additional documents to concatenate into the created document. These are intended to /// capture fetched resources, such as tools or source archives. std::string create_spdx_sbom(const InstallPlanAction& action, - View relative_paths, - View hashes, + View port_files_abi, std::string created_time, std::string document_namespace, std::vector&& resource_docs); Json::Value run_resource_heuristics(StringView contents, StringView portRawVersion); + std::string write_sbom(const InstallPlanAction& action, + std::vector&& heuristic_resources, + const std::vector& port_files_abi, + StringView uuid); } diff --git a/include/vcpkg/vcpkgpaths.h b/include/vcpkg/vcpkgpaths.h index 90fd4d093c..dc19622423 100644 --- a/include/vcpkg/vcpkgpaths.h +++ b/include/vcpkg/vcpkgpaths.h @@ -58,8 +58,6 @@ namespace vcpkg Path build_info_file_path(const PackageSpec& spec) const; const TripletDatabase& get_triplet_db() const; - const std::map& get_cmake_script_hashes() const; - StringView get_ports_cmake_hash() const; LockFile& get_installed_lockfile() const; void flush_lockfile() const; diff --git a/src/vcpkg-test/binarycaching.cpp b/src/vcpkg-test/binarycaching.cpp index 64800c976a..47e0de506e 100644 --- a/src/vcpkg-test/binarycaching.cpp +++ b/src/vcpkg-test/binarycaching.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include diff --git a/src/vcpkg-test/spdx.cpp b/src/vcpkg-test/spdx.cpp index 743e0207f2..4d6ad1bb19 100644 --- a/src/vcpkg-test/spdx.cpp +++ b/src/vcpkg-test/spdx.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -25,13 +26,11 @@ TEST_CASE ("spdx maximum serialization", "[spdx]") auto& abi = *(ipa.abi_info = AbiInfo{}).get(); abi.package_abi = "ABIHASH"; - const auto sbom = - create_spdx_sbom(ipa, - std::vector{"vcpkg.json", "portfile.cmake", "patches/patch1.diff"}, - std::vector{"vcpkg.json-hash", "portfile.cmake-hash", "patch1.diff-hash"}, - "now", - "https://test-document-namespace", - {}); + std::vector port_abi{{"vcpkg.json", "vcpkg.json-hash"}, + {"portfile.cmake", "portfile.cmake-hash"}, + {"patches/patch1.diff", "patch1.diff-hash"}}; + + const auto sbom = create_spdx_sbom(ipa, port_abi, "now", "https://test-document-namespace", {}); auto expected = Json::parse(R"json( { @@ -179,13 +178,9 @@ TEST_CASE ("spdx minimum serialization", "[spdx]") spec, scfl, "test_packages_root", RequestType::USER_REQUESTED, UseHeadVersion::No, Editable::No, {}, {}, {}); auto& abi = *(ipa.abi_info = AbiInfo{}).get(); abi.package_abi = "deadbeef"; + std::vector port_abi{{"vcpkg.json", "hash-vcpkg.json"}, {"portfile.cmake", "hash-portfile.cmake"}}; - const auto sbom = create_spdx_sbom(ipa, - std::vector{"vcpkg.json", "portfile.cmake"}, - std::vector{"hash-vcpkg.json", "hash-portfile.cmake"}, - "now+1", - "https://test-document-namespace-2", - {}); + const auto sbom = create_spdx_sbom(ipa, port_abi, "now+1", "https://test-document-namespace-2", {}); auto expected = Json::parse(R"json( { @@ -326,7 +321,7 @@ TEST_CASE ("spdx concat resources", "[spdx]") .value(VCPKG_LINE_INFO) .value; - const auto sbom = create_spdx_sbom(ipa, {}, {}, "now+1", "ns", {std::move(doc1), std::move(doc2)}); + const auto sbom = create_spdx_sbom(ipa, {}, "now+1", "ns", {std::move(doc1), std::move(doc2)}); auto expected = Json::parse(R"json( { diff --git a/src/vcpkg/abi.cpp b/src/vcpkg/abi.cpp new file mode 100644 index 0000000000..6943b76be9 --- /dev/null +++ b/src/vcpkg/abi.cpp @@ -0,0 +1,500 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace vcpkg +{ + using AbiEntries = std::vector; + + struct PrivateAbi + { + AbiEntries abi_entries; + std::vector heuristic_resources; + AbiEntries port_files_abi; + std::string sbom_uuid; + }; + + static std::string grdk_hash(const Filesystem& fs, const PreBuildInfo& pre_build_info) + { + static const Cache> grdk_cache; + if (auto game_dk_latest = pre_build_info.gamedk_latest_path.get()) + { + const auto grdk_header_path = *game_dk_latest / "GRDK/gameKit/Include/grdk.h"; + const auto& maybe_header_hash = grdk_cache.get_lazy(grdk_header_path, [&]() -> Optional { + auto maybe_hash = Hash::get_file_hash(fs, grdk_header_path, Hash::Algorithm::Sha256); + if (auto hash = maybe_hash.get()) + { + return std::move(*hash); + } + else + { + return nullopt; + } + }); + + if (auto header_hash = maybe_header_hash.get()) + { + return *header_hash; + } + } + + return "none"; + } + + static void hash_additional_files(const Filesystem& fs, const std::vector& files, AbiEntries& abi_tag_entries) + { + for (size_t i = 0; i < files.size(); ++i) + { + auto& file = files[i]; + if (file.is_relative() || !fs.is_regular_file(file)) + { + Checks::msg_exit_with_message(VCPKG_LINE_INFO, msgInvalidValueHashAdditionalFiles, msg::path = file); + } + abi_tag_entries.emplace_back( + fmt::format("additional_file_{}", i), + Hash::get_file_hash(fs, file, Hash::Algorithm::Sha256).value_or_exit(VCPKG_LINE_INFO)); + } + } + + static void abi_entries_from_pre_build_info(const Filesystem& fs, + const PreBuildInfo& pre_build_info, + AbiEntries& abi_tag_entries) + { + if (pre_build_info.public_abi_override) + { + abi_tag_entries.emplace_back( + "public_abi_override", + Hash::get_string_sha256(pre_build_info.public_abi_override.value_or_exit(VCPKG_LINE_INFO))); + } + + for (const auto& env_var : pre_build_info.passthrough_env_vars_tracked) + { + auto maybe_env_var_value = get_environment_variable(env_var); + if (auto env_var_value = maybe_env_var_value.get()) + { + abi_tag_entries.emplace_back("ENV:" + env_var, Hash::get_string_sha256(*env_var_value)); + } + } + + for (size_t i = 0; i < pre_build_info.post_portfile_includes.size(); ++i) + { + auto& file = pre_build_info.post_portfile_includes[i]; + if (file.is_relative() || !fs.is_regular_file(file) || file.extension() != ".cmake") + { + Checks::msg_exit_with_message(VCPKG_LINE_INFO, msgInvalidValuePostPortfileIncludes, msg::path = file); + } + + abi_tag_entries.emplace_back( + fmt::format("post_portfile_include_{}", i), + Hash::get_file_hash(fs, file, Hash::Algorithm::Sha256).value_or_exit(VCPKG_LINE_INFO)); + } + + if (pre_build_info.target_is_xbox) + { + abi_tag_entries.emplace_back("grdk.h", grdk_hash(fs, pre_build_info)); + } + hash_additional_files(fs, pre_build_info.hash_additional_files, abi_tag_entries); + } + + static AbiEntries get_common_abi(const VcpkgPaths& paths) + { + AbiEntries abi_entries; + auto& fs = paths.get_filesystem(); + + abi_entries.emplace_back("cmake", paths.get_tool_version(Tools::CMAKE, out_sink)); + abi_entries.emplace_back("ports.cmake", + Hash::get_file_hash(fs, paths.ports_cmake, Hash::Algorithm::Sha256).value_or("")); + abi_entries.emplace_back("post_build_checks", "2"); + + // This #ifdef is mirrored in tools.cpp's PowershellProvider +#if defined(_WIN32) + abi_entries.emplace_back("powershell", paths.get_tool_version("powershell-core", out_sink)); +#endif + return abi_entries; + } + + static AbiEntries get_cmake_script_hashes(const Filesystem& fs, const Path& scripts_dir) + { + auto files = fs.get_regular_files_non_recursive(scripts_dir / "cmake", VCPKG_LINE_INFO); + Util::erase_remove_if(files, [](const Path& file) { return file.filename() == FileDotDsStore; }); + + AbiEntries helpers(files.size()); + + parallel_transform(files, helpers.begin(), [&fs](auto&& file) { + return AbiEntry{file.stem().to_string(), + Hash::get_file_hash(fs, file, Hash::Algorithm::Sha256).value_or("")}; + }); + return helpers; + } + + static std::vector> get_ports_files_and_contents(const Filesystem& fs, + const Path& port_root_dir) + { + auto files = fs.get_regular_files_recursive_lexically_proximate(port_root_dir, VCPKG_LINE_INFO); + + std::vector> files_and_content; + files_and_content.reserve(files.size()); + + for (auto&& file : files) + { + if (file.filename() == FileDotDsStore) + { + continue; + } + files_and_content.emplace_back(std::move(file), fs.read_contents(port_root_dir / file, VCPKG_LINE_INFO)); + } + return files_and_content; + } + + // Get abi for per-port files + static std::pair get_port_files(const Filesystem& fs, + const SourceControlFileAndLocation& scfl) + { + auto files_and_content = get_ports_files_and_contents(fs, scfl.port_directory()); + + // If there is an unusually large number of files in the port then + // something suspicious is going on. + static constexpr size_t max_port_file_count = 100; + + if (files_and_content.size() > max_port_file_count) + { + msg::println_warning( + msgHashPortManyFiles, msg::package_name = scfl.to_name(), msg::count = files_and_content.size()); + } + + AbiEntries abi_entries; + abi_entries.reserve(files_and_content.size()); + + std::string portfile_cmake_contents; + for (auto&& [port_file, contents] : files_and_content) + { + if (port_file.extension() == ".cmake") + { + portfile_cmake_contents += contents; + } + abi_entries.emplace_back(std::move(port_file).generic_u8string(), Hash::get_string_sha256(contents)); + } + return {std::move(abi_entries), std::move(portfile_cmake_contents)}; + } + + static void get_feature_abi(InternalFeatureSet sorted_feature_list, AbiEntries& abi_entries) + { + // Check that no "default" feature is present. Default features must be resolved before attempting to calculate + // a package ABI, so the "default" should not have made it here. + const bool has_pseudo_features = + std::binary_search(sorted_feature_list.begin(), sorted_feature_list.end(), FeatureNameDefault); + Checks::check_exit(VCPKG_LINE_INFO, !has_pseudo_features); + Util::sort_unique_erase(sorted_feature_list); + + // Check that the "core" feature is present. After resolution into InternalFeatureSet "core" meaning "not + // default" should have already been handled so "core" should be here. + Checks::check_exit(VCPKG_LINE_INFO, + std::binary_search(sorted_feature_list.begin(), sorted_feature_list.end(), FeatureNameCore)); + + abi_entries.emplace_back("features", Strings::join(";", sorted_feature_list)); + } + + // PRE: Check if debugging is enabled + static void print_debug_info(const PackageSpec& spec, View abi_entries) + { + std::string message = Strings::concat("[DEBUG] \n"); + for (auto&& entry : abi_entries) + { + Strings::append(message, "[DEBUG] ", entry.key, "|", entry.value, "\n"); + } + Strings::append(message, "[DEBUG] \n"); + msg::write_unlocalized_text(Color::none, message); + } + + static void print_missing_abi_tags(View abi_entries) + { + bool printed_first_line = false; + for (const auto& abi_tag : abi_entries) + { + if (!abi_tag.value.empty()) + { + continue; + } + if (!printed_first_line) + { + Debug::print("Warning: abi keys are missing values:\n"); + printed_first_line = true; + } + Debug::println(abi_tag.key); + } + } + + // PRE: !action.abi_info.has_value() + static void initialize_abi_info(const VcpkgPaths& paths, + Optional& proto_abi_info, + const InstallPlanAction& action, + std::unique_ptr&& proto_pre_build_info) + { + const auto& pre_build_info = *proto_pre_build_info; + const auto& toolset = paths.get_toolset(pre_build_info); + auto& abi_info = proto_abi_info.emplace(); + abi_info.pre_build_info = std::move(proto_pre_build_info); + abi_info.toolset.emplace(toolset); + + // return when using editable or head flags + if (action.use_head_version == UseHeadVersion::Yes) + { + Debug::print("Binary caching for package ", action.spec, " is disabled due to --head\n"); + return; + } + if (action.editable == Editable::Yes) + { + Debug::print("Binary caching for package ", action.spec, " is disabled due to --editable\n"); + return; + } + + abi_info.compiler_info = paths.get_compiler_info(pre_build_info, toolset); + abi_info.triplet_abi.emplace(paths.get_triplet_info(pre_build_info, toolset)); + } + + // check if all dependencies have a hash + static bool check_dependency_hashes(View dependency_abis, const PackageSpec& spec) + { + if (!dependency_abis.empty()) + { + auto dep_no_hash_it = std::find_if(dependency_abis.begin(), dependency_abis.end(), [](const auto& dep_abi) { + return dep_abi.value.empty(); + }); + if (dep_no_hash_it != dependency_abis.end()) + { + Debug::print("Binary caching for package ", + spec, + " is disabled due to missing abi info for ", + dep_no_hash_it->key, + '\n'); + return false; + } + } + return true; + } + + static Optional get_private_abi(const Filesystem& fs, + const InstallPlanAction& action, + View common_abi, + View cmake_script_hashes) + { + if (action.use_head_version == UseHeadVersion::Yes || action.editable == Editable::Yes) + { + return nullopt; + } + + AbiEntries abi_entries; + abi_entries.emplace_back("triplet", action.spec.triplet().canonical_name()); + auto& scfl = action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO); + + // portfile hashes/contents + auto&& [port_files_abi, cmake_contents] = get_port_files(fs, scfl); + Util::Vectors::append(abi_entries, port_files_abi); + + // cmake helpers + for (auto&& helper : cmake_script_hashes) + { + if (Strings::case_insensitive_ascii_contains(cmake_contents, helper.key)) + { + abi_entries.push_back(helper); + } + } + + abi_entries.insert(abi_entries.end(), common_abi.begin(), common_abi.end()); + + // port features + get_feature_abi(action.feature_list, abi_entries); + + // make sbom file + std::vector heuristic_resources{ + run_resource_heuristics(cmake_contents, scfl.source_control_file->core_paragraph->version.text)}; + + return PrivateAbi{ + std::move(abi_entries), std::move(heuristic_resources), std::move(port_files_abi), generate_random_UUID()}; + } + + static std::string abi_entries_to_string(View abi_entries) + { + std::string out; + for (const auto& abi_entry : abi_entries) + { + out.append(abi_entry.key); + out.push_back(' '); + out.append(abi_entry.value); + out.push_back('\n'); + } + return out; + } + + // PRE: action.abi_info.has_value() + static void make_abi_tag(const VcpkgPaths& paths, + InstallPlanAction& action, + AbiEntries&& dependency_abis, + PrivateAbi&& private_abi) + { + auto& fs = paths.get_filesystem(); + auto& abi_info = *action.abi_info.get(); + auto abi_tag_entries = std::move(private_abi.abi_entries); + Util::Vectors::append(abi_tag_entries, std::move(dependency_abis)); + + abi_tag_entries.emplace_back("triplet_abi", *abi_info.triplet_abi.get()); + + abi_entries_from_pre_build_info(fs, *abi_info.pre_build_info, abi_tag_entries); + + Util::sort(abi_tag_entries); + + if (Debug::g_debugging) + { + // debug output + print_debug_info(action.spec, abi_tag_entries); + + // check for missing abi tags + print_missing_abi_tags(abi_tag_entries); + } + + // fill out port abi + std::string full_abi_info = abi_entries_to_string(abi_tag_entries); + abi_info.package_abi = Hash::get_string_sha256(full_abi_info); + + abi_info.abi_file_contents( + std::move(full_abi_info), + write_sbom( + action, std::move(private_abi.heuristic_resources), private_abi.port_files_abi, private_abi.sbom_uuid)); + } + + static AbiEntries get_dependency_abis(std::vector::const_iterator action_plan_begin, + std::vector::const_iterator current_action, + const StatusParagraphs& status_db) + { + AbiEntries dependency_abis; + dependency_abis.reserve(current_action->package_dependencies.size()); + + for (const auto& dependency_spec : current_action->package_dependencies) + { + // depends on itself? + if (dependency_spec == current_action->spec) + { + continue; + } + + // find dependency in downstream ports + auto dependency_it = + std::find_if(action_plan_begin, current_action, [&dependency_spec](const auto& action) { + return action.spec == dependency_spec; + }); + + if (dependency_it == current_action) + { + // dependency not found + // dependency already installed and therefore not in action plan? + auto status_it = status_db.find(dependency_spec); + if (status_it == status_db.end()) + { + // also not installed --> can't be true + Checks::unreachable(VCPKG_LINE_INFO, + fmt::format("Failed to find dependency abi for {} -> {}", + current_action->spec, + dependency_spec)); + } + + dependency_abis.emplace_back(dependency_spec.name(), status_it->get()->package.abi); + } + else + { + // dependency found in action plan + dependency_abis.emplace_back(dependency_spec.name(), dependency_it->public_abi()); + } + } + return dependency_abis; + } + + void compute_all_abis(const VcpkgPaths& paths, + ActionPlan& action_plan, + const CMakeVars::CMakeVarProvider& var_provider, + const StatusParagraphs& status_db) + { + auto& fs = paths.get_filesystem(); + static AbiEntries common_abi = get_common_abi(paths); + + auto private_abi_future = std::async(std::launch::async | std::launch::deferred, [&]() { + static AbiEntries cmake_script_hashes = get_cmake_script_hashes(fs, paths.scripts); + + std::vector> ret(action_plan.install_actions.size()); + + parallel_transform(action_plan.install_actions, ret.begin(), [&](const InstallPlanAction& action) { + return get_private_abi(fs, action, common_abi, cmake_script_hashes); + }); + return ret; + }); + + for (auto& action : action_plan.install_actions) + { + Checks::check_exit(VCPKG_LINE_INFO, !action.abi_info.has_value()); + + // get prebuildinfo + auto pre_build_info = std::make_unique( + paths, action.spec.triplet(), var_provider.get_tag_vars(action.spec).value_or_exit(VCPKG_LINE_INFO)); + + initialize_abi_info(paths, action.abi_info, action, std::move(pre_build_info)); + } + + auto private_abis = private_abi_future.get(); + + for (auto it = action_plan.install_actions.begin(); it != action_plan.install_actions.end(); ++it) + { + auto& action = *it; + size_t i = static_cast(std::distance(action_plan.install_actions.begin(), it)); + + if (!private_abis[i].has_value()) + { + continue; + } + + AbiEntries dependency_abis; + dependency_abis = get_dependency_abis(action_plan.install_actions.begin(), it, status_db); + if (!check_dependency_hashes(dependency_abis, action.spec)) + { + continue; + } + + make_abi_tag(paths, action, std::move(dependency_abis), std::move(*private_abis[i].get())); + } + } + + void AbiInfo::save_abi_files(const Filesystem& fs, Path&& dir) const + { + if (!abi_tag_file_contents.has_value() || !sbom_file_contents.has_value()) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + + // write abi tag file + fs.write_contents_and_dirs(dir / FileVcpkgAbiInfo, *abi_tag_file_contents.get(), VCPKG_LINE_INFO); + + // write sbom file + fs.write_contents(std::move(dir) / "vcpkg.spdx.json", *sbom_file_contents.get(), VCPKG_LINE_INFO); + } +} // namespace vcpkg diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index e466e97163..057ab65b70 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -967,7 +967,7 @@ namespace vcpkg const char* Path::c_str() const noexcept { return m_str.c_str(); } - std::string Path::generic_u8string() const + std::string Path::generic_u8string() const& { #if defined(_WIN32) auto result = m_str; @@ -978,6 +978,17 @@ namespace vcpkg #endif } + std::string Path::generic_u8string() && + { +#if defined(_WIN32) + auto result = std::move(m_str); + std::replace(result.begin(), result.end(), '\\', '/'); + return result; +#else + return std::move(m_str); +#endif + } + bool Path::empty() const noexcept { return m_str.empty(); } Path Path::operator/(StringView sv) const& diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 613b373d11..d21734e178 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -921,35 +922,6 @@ namespace vcpkg } } - static void write_sbom(const VcpkgPaths& paths, - const InstallPlanAction& action, - std::vector heuristic_resources) - { - auto& fs = paths.get_filesystem(); - const auto& scfl = action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO); - const auto& scf = *scfl.source_control_file; - - auto doc_ns = Strings::concat("https://spdx.org/spdxdocs/", - scf.to_name(), - '-', - action.spec.triplet(), - '-', - scf.to_version(), - '-', - generate_random_UUID()); - - const auto now = CTime::now_string(); - const auto& abi = action.abi_info.value_or_exit(VCPKG_LINE_INFO); - - const auto json_path = - action.package_dir.value_or_exit(VCPKG_LINE_INFO) / "share" / action.spec.name() / "vcpkg.spdx.json"; - fs.write_contents_and_dirs( - json_path, - create_spdx_sbom( - action, abi.relative_port_files, abi.relative_port_hashes, now, doc_ns, std::move(heuristic_resources)), - VCPKG_LINE_INFO); - } - static ExtendedBuildResult do_build_package(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet host_triplet, @@ -1075,7 +1047,6 @@ namespace vcpkg std::unique_ptr bcf = create_binary_control_file(action, build_info); - write_sbom(paths, action, abi_info.heuristic_resources); write_binary_control_file(paths.get_filesystem(), action.package_dir.value_or_exit(VCPKG_LINE_INFO), *bcf); return {BuildResult::Succeeded, std::move(bcf)}; } @@ -1103,298 +1074,6 @@ namespace vcpkg return result; } - static std::string grdk_hash(const Filesystem& fs, - Cache>& grdk_cache, - const PreBuildInfo& pre_build_info) - { - if (auto game_dk_latest = pre_build_info.gamedk_latest_path.get()) - { - const auto grdk_header_path = *game_dk_latest / "GRDK/gameKit/Include/grdk.h"; - const auto& maybe_header_hash = grdk_cache.get_lazy(grdk_header_path, [&]() -> Optional { - auto maybe_hash = Hash::get_file_hash(fs, grdk_header_path, Hash::Algorithm::Sha256); - if (auto hash = maybe_hash.get()) - { - return std::move(*hash); - } - else - { - return nullopt; - } - }); - - if (auto header_hash = maybe_header_hash.get()) - { - return *header_hash; - } - } - - return "none"; - } - - static void abi_entries_from_pre_build_info(const Filesystem& fs, - Cache>& grdk_cache, - const PreBuildInfo& pre_build_info, - std::vector& abi_tag_entries) - { - if (pre_build_info.public_abi_override) - { - abi_tag_entries.emplace_back( - AbiTagPublicAbiOverride, - Hash::get_string_hash(pre_build_info.public_abi_override.value_or_exit(VCPKG_LINE_INFO), - Hash::Algorithm::Sha256)); - } - - for (const auto& env_var : pre_build_info.passthrough_env_vars_tracked) - { - if (auto e = get_environment_variable(env_var)) - { - abi_tag_entries.emplace_back( - "ENV:" + env_var, Hash::get_string_hash(e.value_or_exit(VCPKG_LINE_INFO), Hash::Algorithm::Sha256)); - } - } - - if (pre_build_info.target_is_xbox) - { - abi_tag_entries.emplace_back(AbiTagGrdkH, grdk_hash(fs, grdk_cache, pre_build_info)); - } - } - - static void populate_abi_tag(const VcpkgPaths& paths, - InstallPlanAction& action, - std::unique_ptr&& proto_pre_build_info, - Span dependency_abis, - Cache>& grdk_cache) - { - Checks::check_exit(VCPKG_LINE_INFO, static_cast(proto_pre_build_info)); - const auto& pre_build_info = *proto_pre_build_info; - const auto& toolset = paths.get_toolset(pre_build_info); - auto& abi_info = action.abi_info.emplace(); - abi_info.pre_build_info = std::move(proto_pre_build_info); - abi_info.toolset.emplace(toolset); - - if (action.use_head_version == UseHeadVersion::Yes) - { - Debug::print("Binary caching for package ", action.spec, " is disabled due to --head\n"); - return; - } - if (action.editable == Editable::Yes) - { - Debug::print("Binary caching for package ", action.spec, " is disabled due to --editable\n"); - return; - } - - abi_info.compiler_info = paths.get_compiler_info(*abi_info.pre_build_info, toolset); - for (auto&& dep_abi : dependency_abis) - { - if (dep_abi.value.empty()) - { - Debug::print("Binary caching for package ", - action.spec, - " is disabled due to missing abi info for ", - dep_abi.key, - '\n'); - return; - } - } - - std::vector abi_tag_entries(dependency_abis.begin(), dependency_abis.end()); - - const auto& triplet_abi = paths.get_triplet_info(pre_build_info, toolset); - abi_info.triplet_abi.emplace(triplet_abi); - const auto& triplet_canonical_name = action.spec.triplet().canonical_name(); - abi_tag_entries.emplace_back(AbiTagTriplet, triplet_canonical_name); - abi_tag_entries.emplace_back(AbiTagTripletAbi, triplet_abi); - auto& fs = paths.get_filesystem(); - abi_entries_from_pre_build_info(fs, grdk_cache, pre_build_info, abi_tag_entries); - - // If there is an unusually large number of files in the port then - // something suspicious is going on. - constexpr int max_port_file_count = 100; - - std::string portfile_cmake_contents; - std::vector files; - std::vector hashes; - - for (size_t i = 0; i < abi_info.pre_build_info->hash_additional_files.size(); ++i) - { - auto& file = abi_info.pre_build_info->hash_additional_files[i]; - if (file.is_relative() || !fs.is_regular_file(file)) - { - Checks::msg_exit_with_message(VCPKG_LINE_INFO, msgInvalidValueHashAdditionalFiles, msg::path = file); - } - - abi_tag_entries.emplace_back( - fmt::format("additional_file_{}", i), - Hash::get_file_hash(fs, file, Hash::Algorithm::Sha256).value_or_exit(VCPKG_LINE_INFO)); - } - - for (size_t i = 0; i < abi_info.pre_build_info->post_portfile_includes.size(); ++i) - { - auto& file = abi_info.pre_build_info->post_portfile_includes[i]; - if (file.is_relative() || !fs.is_regular_file(file) || file.extension() != ".cmake") - { - Checks::msg_exit_with_message(VCPKG_LINE_INFO, msgInvalidValuePostPortfileIncludes, msg::path = file); - } - - abi_tag_entries.emplace_back( - fmt::format("post_portfile_include_{}", i), - Hash::get_file_hash(fs, file, Hash::Algorithm::Sha256).value_or_exit(VCPKG_LINE_INFO)); - } - - auto&& scfl = action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO); - auto port_dir = scfl.port_directory(); - auto raw_files = fs.get_regular_files_recursive_lexically_proximate(port_dir, VCPKG_LINE_INFO); - if (raw_files.size() > max_port_file_count) - { - msg::println_warning( - msgHashPortManyFiles, msg::package_name = action.spec.name(), msg::count = raw_files.size()); - } - - for (auto& port_file : raw_files) - { - if (port_file.filename() == FileDotDsStore) - { - continue; - } - - const auto& abs_port_file = files.emplace_back(port_dir / port_file); - if (port_file.extension() == ".cmake") - { - auto contents = fs.read_contents(abs_port_file, VCPKG_LINE_INFO); - portfile_cmake_contents += contents; - hashes.push_back(vcpkg::Hash::get_string_sha256(contents)); - } - else - { - hashes.push_back(vcpkg::Hash::get_file_hash(fs, abs_port_file, Hash::Algorithm::Sha256) - .value_or_exit(VCPKG_LINE_INFO)); - } - - abi_tag_entries.emplace_back(port_file, hashes.back()); - } - - abi_tag_entries.emplace_back(AbiTagCMake, paths.get_tool_version(Tools::CMAKE, out_sink)); - - // This #ifdef is mirrored in tools.cpp's PowershellProvider -#if defined(_WIN32) - abi_tag_entries.emplace_back(AbiTagPowershell, paths.get_tool_version("powershell-core", out_sink)); -#endif - - auto& helpers = paths.get_cmake_script_hashes(); - for (auto&& helper : helpers) - { - if (Strings::case_insensitive_ascii_contains(portfile_cmake_contents, helper.first)) - { - abi_tag_entries.emplace_back(helper.first, helper.second); - } - } - - abi_tag_entries.emplace_back(AbiTagPortsDotCMake, paths.get_ports_cmake_hash().to_string()); - abi_tag_entries.emplace_back(AbiTagPostBuildChecks, "2"); - InternalFeatureSet sorted_feature_list = action.feature_list; - // Check that no "default" feature is present. Default features must be resolved before attempting to calculate - // a package ABI, so the "default" should not have made it here. - const bool has_no_pseudo_features = std::none_of(sorted_feature_list.begin(), - sorted_feature_list.end(), - [](StringView s) { return s == FeatureNameDefault; }); - Checks::check_exit(VCPKG_LINE_INFO, has_no_pseudo_features); - Util::sort_unique_erase(sorted_feature_list); - - // Check that the "core" feature is present. After resolution into InternalFeatureSet "core" meaning "not - // default" should have already been handled so "core" should be here. - Checks::check_exit(VCPKG_LINE_INFO, - std::binary_search(sorted_feature_list.begin(), sorted_feature_list.end(), FeatureNameCore)); - - abi_tag_entries.emplace_back(AbiTagFeatures, Strings::join(";", sorted_feature_list)); - - Util::sort(abi_tag_entries); - - const std::string full_abi_info = - Strings::join("", abi_tag_entries, [](const AbiEntry& p) { return p.key + " " + p.value + "\n"; }); - - if (Debug::g_debugging) - { - std::string message = Strings::concat("[DEBUG] \n"); - for (auto&& entry : abi_tag_entries) - { - Strings::append(message, "[DEBUG] ", entry.key, "|", entry.value, "\n"); - } - Strings::append(message, "[DEBUG] \n"); - msg::write_unlocalized_text(Color::none, message); - } - - auto abi_tag_entries_missing = Util::filter(abi_tag_entries, [](const AbiEntry& p) { return p.value.empty(); }); - if (!abi_tag_entries_missing.empty()) - { - Debug::println("Warning: abi keys are missing values:\n", - Strings::join("\n", abi_tag_entries_missing, [](const AbiEntry& e) -> const std::string& { - return e.key; - })); - return; - } - - auto abi_file_path = paths.build_dir(action.spec); - fs.create_directory(abi_file_path, VCPKG_LINE_INFO); - abi_file_path /= triplet_canonical_name + ".vcpkg_abi_info.txt"; - fs.write_contents(abi_file_path, full_abi_info, VCPKG_LINE_INFO); - - auto& scf = scfl.source_control_file; - abi_info.package_abi = Hash::get_string_sha256(full_abi_info); - abi_info.abi_tag_file.emplace(std::move(abi_file_path)); - abi_info.relative_port_files = std::move(files); - abi_info.relative_port_hashes = std::move(hashes); - abi_info.heuristic_resources.push_back( - run_resource_heuristics(portfile_cmake_contents, scf->core_paragraph->version.text)); - } - - void compute_all_abis(const VcpkgPaths& paths, - ActionPlan& action_plan, - const CMakeVars::CMakeVarProvider& var_provider, - const StatusParagraphs& status_db) - { - Cache> grdk_cache; - for (auto it = action_plan.install_actions.begin(); it != action_plan.install_actions.end(); ++it) - { - auto& action = *it; - if (action.abi_info.has_value()) continue; - - std::vector dependency_abis; - for (auto&& pspec : action.package_dependencies) - { - if (pspec == action.spec) continue; - - auto pred = [&](const InstallPlanAction& ipa) { return ipa.spec == pspec; }; - auto it2 = std::find_if(action_plan.install_actions.begin(), it, pred); - if (it2 == it) - { - // Finally, look in current installed - auto status_it = status_db.find(pspec); - if (status_it == status_db.end()) - { - Checks::unreachable( - VCPKG_LINE_INFO, - fmt::format("Failed to find dependency abi for {} -> {}", action.spec, pspec)); - } - - dependency_abis.emplace_back(pspec.name(), status_it->get()->package.abi); - } - else - { - dependency_abis.emplace_back(pspec.name(), it2->public_abi()); - } - } - - populate_abi_tag( - paths, - action, - std::make_unique(paths, - action.spec.triplet(), - var_provider.get_tag_vars(action.spec).value_or_exit(VCPKG_LINE_INFO)), - dependency_abis, - grdk_cache); - } - } - ExtendedBuildResult build_package(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet host_triplet, @@ -1445,14 +1124,12 @@ namespace vcpkg auto& abi_info = action.abi_info.value_or_exit(VCPKG_LINE_INFO); ExtendedBuildResult result = do_build_package_and_clean_buildtrees( args, paths, host_triplet, build_options, action, all_dependencies_satisfied); - if (abi_info.abi_tag_file) + + if (abi_info.abi_tag_complete()) { - auto& abi_file = *abi_info.abi_tag_file.get(); - const auto abi_package_dir = action.package_dir.value_or_exit(VCPKG_LINE_INFO) / "share" / spec.name(); - const auto abi_file_in_package = abi_package_dir / FileVcpkgAbiInfo; build_logs_recorder.record_build_result(paths, spec, result.code); - filesystem.create_directories(abi_package_dir, VCPKG_LINE_INFO); - filesystem.copy_file(abi_file, abi_file_in_package, CopyOptions::none, VCPKG_LINE_INFO); + auto abi_package_dir = action.package_dir.value_or_exit(VCPKG_LINE_INFO) / "share" / spec.name(); + abi_info.save_abi_files(filesystem, std::move(abi_package_dir)); } return result; diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 115964d7ca..fc3ec8ff0b 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/vcpkg/dependencies.cpp b/src/vcpkg/dependencies.cpp index 7142a61067..48c6185802 100644 --- a/src/vcpkg/dependencies.cpp +++ b/src/vcpkg/dependencies.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/src/vcpkg/spdx.cpp b/src/vcpkg/spdx.cpp index bcaba964b6..21c335661b 100644 --- a/src/vcpkg/spdx.cpp +++ b/src/vcpkg/spdx.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -134,14 +135,11 @@ Json::Value vcpkg::run_resource_heuristics(StringView contents, StringView versi } std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, - View relative_paths, - View hashes, + View port_files_abi, std::string created_time, std::string document_namespace, std::vector&& resource_docs) { - Checks::check_exit(VCPKG_LINE_INFO, relative_paths.size() == hashes.size()); - const auto& scfl = action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO); const auto& cpgh = *scfl.source_control_file->core_paragraph; StringView abi{SpdxNone}; @@ -188,7 +186,7 @@ std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, rel.insert(SpdxRelationshipType, SpdxGenerates); rel.insert(SpdxRelatedSpdxElement, SpdxRefBinary); } - for (size_t i = 0; i < relative_paths.size(); ++i) + for (size_t i = 0; i < port_files_abi.size(); ++i) { auto& rel = rels.push_back(Json::Object()); rel.insert(SpdxElementId, SpdxRefPort); @@ -216,13 +214,13 @@ std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, auto& files = doc.insert(JsonIdFiles, Json::Array()); { - for (size_t i = 0; i < relative_paths.size(); ++i) + for (size_t i = 0; i < port_files_abi.size(); ++i) { - const auto& path = relative_paths[i]; - const auto& hash = hashes[i]; + const auto& path = port_files_abi[i].key; + const auto& hash = port_files_abi[i].value; auto& obj = files.push_back(Json::Object()); - obj.insert(SpdxFileName, "./" + path.generic_u8string()); + obj.insert(SpdxFileName, "./" + path); const auto ref = fmt::format("SPDXRef-file-{}", i); obj.insert(SpdxSpdxId, ref); auto& checksum = obj.insert(JsonIdChecksums, Json::Array()); @@ -258,3 +256,18 @@ std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, return Json::stringify(doc); } + +std::string vcpkg::write_sbom(const InstallPlanAction& action, + std::vector&& heuristic_resources, + const std::vector& port_files_abi, + StringView uuid) +{ + const auto& scfl = action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO); + const auto& scf = *scfl.source_control_file; + + auto doc_ns = Strings::concat( + "https://spdx.org/spdxdocs/", scf.to_name(), '-', action.spec.triplet(), '-', scf.to_version(), '-', uuid); + + const auto now = CTime::now_string(); + return create_spdx_sbom(action, port_files_abi, now, doc_ns, std::move(heuristic_resources)); +} diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index 77e107f5ce..d7a71eca8a 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -259,8 +259,6 @@ namespace { Lazy triplets_db; Lazy toolsets; - Lazy> cmake_script_hashes; - Lazy ports_cmake_hash; Optional m_installed_lock; }; @@ -735,33 +733,6 @@ namespace vcpkg }); } - const std::map& VcpkgPaths::get_cmake_script_hashes() const - { - return m_pimpl->cmake_script_hashes.get_lazy([this]() -> std::map { - auto& fs = this->get_filesystem(); - std::map helpers; - auto files = fs.get_regular_files_non_recursive(this->scripts / "cmake", VCPKG_LINE_INFO); - for (auto&& file : files) - { - if (file.filename() == FileDotDsStore) - { - continue; - } - helpers.emplace(file.stem().to_string(), - Hash::get_file_hash(fs, file, Hash::Algorithm::Sha256).value_or_exit(VCPKG_LINE_INFO)); - } - return helpers; - }); - } - - StringView VcpkgPaths::get_ports_cmake_hash() const - { - return m_pimpl->ports_cmake_hash.get_lazy([this]() -> std::string { - return Hash::get_file_hash(get_filesystem(), ports_cmake, Hash::Algorithm::Sha256) - .value_or_exit(VCPKG_LINE_INFO); - }); - } - LockFile& VcpkgPaths::get_installed_lockfile() const { if (!m_pimpl->m_installed_lock.has_value())