From 0cb0e9fcb02f268093ab651a8a63a44e01540b5c Mon Sep 17 00:00:00 2001 From: Tomasz Moniuszko Date: Wed, 29 May 2019 10:02:37 +0200 Subject: [PATCH] Add jumbo support This allows to generate Visual Studio projects that include source files that are merged together into jumbo files. --- .github/workflows/main.yml | 82 ++++++++++ build/gen.py | 9 +- src/gn/args.cc | 5 + src/gn/binary_target_generator.cc | 96 ++++++++++++ src/gn/binary_target_generator.h | 3 + src/gn/command_gen.cc | 56 +++++++ src/gn/gn_main.cc | 2 +- src/gn/jumbo_file_list_generator.cc | 140 +++++++++++++++++ src/gn/jumbo_file_list_generator.h | 59 +++++++ src/gn/jumbo_file_list_generator_unittest.cc | 155 +++++++++++++++++++ src/gn/jumbo_writer.cc | 66 ++++++++ src/gn/jumbo_writer.h | 34 ++++ src/gn/jumbo_writer_unittest.cc | 77 +++++++++ src/gn/ninja_binary_target_writer.cc | 38 ++++- src/gn/ninja_binary_target_writer.h | 10 ++ src/gn/ninja_c_binary_target_writer.cc | 139 +++++++++++------ src/gn/target.cc | 5 +- src/gn/target.h | 31 ++++ src/gn/variables.cc | 99 ++++++++++++ src/gn/variables.h | 16 ++ 20 files changed, 1073 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 src/gn/jumbo_file_list_generator.cc create mode 100644 src/gn/jumbo_file_list_generator.h create mode 100644 src/gn/jumbo_file_list_generator_unittest.cc create mode 100644 src/gn/jumbo_writer.cc create mode 100644 src/gn/jumbo_writer.h create mode 100644 src/gn/jumbo_writer_unittest.cc diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..28156755 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,82 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + build-ubuntu: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + with: { fetch-depth: 0 } + + - name: install dependencies + run: | + sudo apt-get update + sudo apt-get install -qq ninja-build + + # Runs a single command using the runners shell + - name: Gen project + run: python build/gen.py + + # Runs a set of commands using the runners shell + - name: Build + run: ninja -C out gn + + - name: Print gn version + run: ./out/gn --version + + - name: Rename gn to gn_lin + run: mv out/gn out/gn_lin + + - name: Upload gn_lin + uses: actions/upload-artifact@v2.1.4 + with: + path: out/gn_lin + # The desired behavior if no files are found using the provided path. + + build-mac: + # The type of runner that the job will run on + runs-on: macOS-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + with: { fetch-depth: 0 } + + - name: install dependencies + run: brew install ninja + + # Runs a single command using the runners shell + - name: Gen project + run: python build/gen.py + + # Runs a set of commands using the runners shell + - name: Build + run: ninja -C out gn + + - name: Print gn version + run: ./out/gn --version + + - name: Rename gn to gn_mac + run: mv out/gn out/gn_mac + + - name: Upload gn_mac + uses: actions/upload-artifact@v2.1.4 + with: + path: out/gn_mac + # The desired behavior if no files are found using the provided path. + diff --git a/build/gen.py b/build/gen.py index 265b1f56..5acfe03d 100755 --- a/build/gen.py +++ b/build/gen.py @@ -166,9 +166,10 @@ def main(argv): def GenerateLastCommitPosition(host, header): ROOT_TAG = 'initial-commit' + # Use HEAD@{1} so original Chromium and Opera (patched) versions are the same. describe_output = subprocess.check_output( - ['git', 'describe', 'HEAD', '--match', ROOT_TAG], shell=host.is_windows(), - cwd=REPO_ROOT) + ['git', 'describe', 'HEAD~', '--match', ROOT_TAG], + shell=host.is_windows(), cwd=REPO_ROOT) mo = re.match(ROOT_TAG + '-(\d+)-g([0-9a-f]+)', describe_output.decode()) if not mo: raise ValueError( @@ -591,6 +592,8 @@ def WriteGNNinja(path, platform, host, options): 'src/gn/input_file_manager.cc', 'src/gn/item.cc', 'src/gn/json_project_writer.cc', + 'src/gn/jumbo_file_list_generator.cc', + 'src/gn/jumbo_writer.cc', 'src/gn/label.cc', 'src/gn/label_pattern.cc', 'src/gn/lib_file.cc', @@ -714,6 +717,8 @@ def WriteGNNinja(path, platform, host, options): 'src/gn/inherited_libraries_unittest.cc', 'src/gn/input_conversion_unittest.cc', 'src/gn/json_project_writer_unittest.cc', + 'src/gn/jumbo_file_list_generator_unittest.cc', + 'src/gn/jumbo_writer_unittest.cc', 'src/gn/rust_project_writer_unittest.cc', 'src/gn/rust_project_writer_helpers_unittest.cc', 'src/gn/label_pattern_unittest.cc', diff --git a/src/gn/args.cc b/src/gn/args.cc index 2b673137..7f6de46b 100644 --- a/src/gn/args.cc +++ b/src/gn/args.cc @@ -377,6 +377,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const { // declared. This is so they can be overridden in a toolchain build args // override, and so that they will appear in the "gn args" output. Value empty_string(nullptr, std::string()); + Value false_bool(nullptr, false); Value os_val(nullptr, std::string(os)); dest->SetValue(variables::kHostOs, os_val, nullptr); @@ -388,6 +389,8 @@ void Args::SetSystemVarsLocked(Scope* dest) const { dest->SetValue(variables::kTargetCpu, empty_string, nullptr); dest->SetValue(variables::kCurrentCpu, empty_string, nullptr); + dest->SetValue(variables::kEnableNativeJumbo, false_bool, nullptr); + Scope::KeyValueMap& declared_arguments( DeclaredArgumentsForToolchainLocked(dest)); declared_arguments[variables::kHostOs] = os_val; @@ -396,6 +399,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const { declared_arguments[variables::kHostCpu] = arch_val; declared_arguments[variables::kCurrentCpu] = empty_string; declared_arguments[variables::kTargetCpu] = empty_string; + declared_arguments[variables::kEnableNativeJumbo] = false_bool; // Mark these variables used so the build config file can override them // without getting a warning about overwriting an unused variable. @@ -405,6 +409,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const { dest->MarkUsed(variables::kHostOs); dest->MarkUsed(variables::kCurrentOs); dest->MarkUsed(variables::kTargetOs); + dest->MarkUsed(variables::kEnableNativeJumbo); } void Args::ApplyOverridesLocked(const Scope::KeyValueMap& values, diff --git a/src/gn/binary_target_generator.cc b/src/gn/binary_target_generator.cc index 4cac96f9..5ec34210 100644 --- a/src/gn/binary_target_generator.cc +++ b/src/gn/binary_target_generator.cc @@ -4,11 +4,14 @@ #include "gn/binary_target_generator.h" +#include + #include "gn/config_values_generator.h" #include "gn/deps_iterator.h" #include "gn/err.h" #include "gn/filesystem_utils.h" #include "gn/functions.h" +#include "gn/jumbo_file_list_generator.h" #include "gn/parse_tree.h" #include "gn/rust_values_generator.h" #include "gn/rust_variables.h" @@ -87,6 +90,23 @@ void BinaryTargetGenerator::DoRun() { gen.Run(); if (err_->has_error()) return; + + if (!FillJumboAllowed()) + return; + + if (!FillJumboExcludedSources()) + return; + + if (!FillJumboFileMergeLimit()) + return; + + if (target_->is_jumbo_allowed()) { + JumboFileListGenerator jumbo_generator(target_, &target_->jumbo_files(), + err_); + jumbo_generator.Run(); + if (err_->has_error()) + return; + } } bool BinaryTargetGenerator::FillSources() { @@ -231,6 +251,82 @@ bool BinaryTargetGenerator::FillAllowCircularIncludesFrom() { return true; } +bool BinaryTargetGenerator::FillJumboAllowed() { + const Value* value = scope_->GetValue(variables::kJumboAllowed, true); + if (!value) + return true; + + if (!value->VerifyTypeIs(Value::BOOLEAN, err_)) + return false; + + target_->set_jumbo_allowed(value->boolean_value()); + return true; +} + +bool BinaryTargetGenerator::FillJumboExcludedSources() { + const Value* value = scope_->GetValue(variables::kJumboExcludedSources, true); + if (!value) + return true; + + if (!target_->is_jumbo_allowed()) { + // TODO(tmoniuszko): We currently don't report error here because we want + // to support non-native jumbo implementation from BUILD.gn scripts. + // For native-only jumbo we would display such error: + // *err_ = Err(*value, "Jumbo is not allowed for this target."); + return false; + } + + Target::FileList jumbo_excluded_sources; + if (!ExtractListOfRelativeFiles(scope_->settings()->build_settings(), *value, + scope_->GetSourceDir(), + &jumbo_excluded_sources, err_)) { + return false; + } + + // Excluded files should be in sources. jumbo_excluded_sources is intended to + // exclude only small amount of files that cause compilation issues so + // searching for every file in loop should be acceptable despite of time + // complexity. + const Target::FileList& sources = target_->sources(); + for (const SourceFile& file : jumbo_excluded_sources) { + if (std::find(sources.begin(), sources.end(), file) == sources.end()) { + *err_ = Err(*value, "Excluded file not in sources.", + "The file \"" + file.value() + "\" was not in \"sources\"." + + " " + sources.front().value()); + return false; + } + } + + target_->jumbo_excluded_sources().swap(jumbo_excluded_sources); + return true; +} + +bool BinaryTargetGenerator::FillJumboFileMergeLimit() { + const Value* value = scope_->GetValue(variables::kJumboFileMergeLimit, true); + if (!value) + return true; + + if (!target_->is_jumbo_allowed()) { + // TODO(tmoniuszko): We currently don't report error here because we want + // to support non-native jumbo implementation from BUILD.gn scripts. + // For native-only jumbo we would display such error: + // *err_ = Err(*value, "Jumbo is not allowed for this target."); + return false; + } + + if (!value->VerifyTypeIs(Value::INTEGER, err_)) + return false; + + int jumbo_file_merge_limit = value->int_value(); + if (jumbo_file_merge_limit < 2) { + *err_ = Err(*value, "Value must be greater than 1."); + return false; + } + + target_->set_jumbo_file_merge_limit(jumbo_file_merge_limit); + return true; +} + bool BinaryTargetGenerator::ValidateSources() { // For Rust targets, if the only source file is the root `sources` can be // omitted/empty. diff --git a/src/gn/binary_target_generator.h b/src/gn/binary_target_generator.h index 2c8b7695..e2f3c740 100644 --- a/src/gn/binary_target_generator.h +++ b/src/gn/binary_target_generator.h @@ -30,6 +30,9 @@ class BinaryTargetGenerator : public TargetGenerator { bool FillOutputPrefixOverride(); bool FillOutputDir(); bool FillAllowCircularIncludesFrom(); + bool FillJumboAllowed(); + bool FillJumboExcludedSources(); + bool FillJumboFileMergeLimit(); bool ValidateSources(); Target::OutputType output_type_; diff --git a/src/gn/command_gen.cc b/src/gn/command_gen.cc index e391d26d..1ec2cbc8 100644 --- a/src/gn/command_gen.cc +++ b/src/gn/command_gen.cc @@ -60,6 +60,7 @@ const char kSwitchJsonIdeScript[] = "json-ide-script"; const char kSwitchJsonIdeScriptArgs[] = "json-ide-script-args"; const char kSwitchExportCompileCommands[] = "export-compile-commands"; const char kSwitchExportRustProject[] = "export-rust-project"; +const char kSwitchJumboStats[] = "jumbo-stats"; // Collects Ninja rules for each toolchain. The lock protectes the rules. struct TargetWriteInfo { @@ -583,6 +584,16 @@ Compilation Database - "//foo:foo" and not match: - "//foo:bar" + of target_name is supplied, only targets that are reachable from the list + of target_name will be used for “command objects” generation, otherwise + all available targets will be used. This is used for various Clang-based + tooling, allowing for the replay of individual compilations independent + of the build system. + +Jumbo Build Mode + + --jumbo-stats + Shows statistics about Jumbo usage in targets. )"; int RunGen(const std::vector& args) { @@ -624,6 +635,10 @@ int RunGen(const std::vector& args) { if (!setup->Run()) return 1; + int jumbo_allowed_count = 0; + int jumbo_disallowed_count = 0; + std::set jumbo_not_configured_targets; + // Sort the targets in each toolchain according to their label. This makes // the ninja files have deterministic content. for (auto& cur_toolchain : write_info.rules) { @@ -632,6 +647,19 @@ int RunGen(const std::vector& args) { const NinjaWriter::TargetRulePair& b) { return a.first->label() < b.first->label(); }); + + if (command_line->HasSwitch(kSwitchJumboStats)) { + for (const NinjaWriter::TargetRulePair& rule : cur_toolchain.second) { + if (rule.first->is_jumbo_configured()) { + if (rule.first->is_jumbo_allowed()) + ++jumbo_allowed_count; + else + ++jumbo_disallowed_count; + } else if (rule.first->IsBinary()) { + jumbo_not_configured_targets.insert(rule.first); + } + } + } } Err err; @@ -683,6 +711,34 @@ int RunGen(const std::vector& args) { TickDelta elapsed_time = timer.Elapsed(); if (!command_line->HasSwitch(switches::kQuiet)) { + if (command_line->HasSwitch(kSwitchJumboStats)) { + std::vector not_configured( + jumbo_not_configured_targets.begin(), + jumbo_not_configured_targets.end()); + std::sort(not_configured.begin(), not_configured.end(), + [](const Target* a, const Target* b) { + if (a->sources().size() > b->sources().size()) + return true; + if (a->sources().size() < b->sources().size()) + return false; + return a->label().GetUserVisibleName(false) < + b->label().GetUserVisibleName(false); + }); + OutputString("Jumbo is not configured in following targets:\n"); + for (const Target* target : not_configured) { + OutputString(target->label().GetUserVisibleName(false) + + " (" + base::NumberToString(target->sources().size()) + + " sources)\n"); + } + OutputString("\nJumbo is not configured in " + + base::NumberToString(not_configured.size()) + " targets.\n"); + OutputString("Jumbo is allowed in " + + base::NumberToString(jumbo_allowed_count) + " targets.\n"); + OutputString("Jumbo is disallowed in " + + base::NumberToString(jumbo_disallowed_count) + + " targets.\n\n"); + } + OutputString("Done. ", DECORATION_GREEN); size_t targets_collected = 0; diff --git a/src/gn/gn_main.cc b/src/gn/gn_main.cc index e839b322..0c86311d 100644 --- a/src/gn/gn_main.cc +++ b/src/gn/gn_main.cc @@ -49,7 +49,7 @@ int main(int argc, char** argv) { command = commands::kHelp; } else if (cmdline.HasSwitch(switches::kVersion)) { // Make "--version" print the version and exit. - OutputString(std::string(LAST_COMMIT_POSITION) + "\n"); + OutputString(std::string(LAST_COMMIT_POSITION) + " - modified by Opera\n"); exit(0); } else if (args.empty()) { // No command, print error and exit. diff --git a/src/gn/jumbo_file_list_generator.cc b/src/gn/jumbo_file_list_generator.cc new file mode 100644 index 00000000..edadf9d5 --- /dev/null +++ b/src/gn/jumbo_file_list_generator.cc @@ -0,0 +1,140 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gn/jumbo_file_list_generator.h" + +#include +#include +#include +#include + +#include "base/logging.h" +#include "base/numerics/safe_conversions.h" +#include "base/strings/stringprintf.h" +#include "gn/filesystem_utils.h" + +namespace { + +// Constructs the file name for a jumbo file. +std::string GetJumboFileName(const std::string& target_name, + SourceFile::Type file_type, + int file_number) { + std::string_view extension; + switch (file_type) { + case SourceFile::SOURCE_C: + extension = "c"; + break; + case SourceFile::SOURCE_CPP: + extension = "cc"; + break; + case SourceFile::SOURCE_MM: + extension = "mm"; + break; + default: + return std::string(); + } + + return base::StringPrintf("%s_jumbo_%s_%d.%s", target_name.c_str(), + extension.data(), file_number, extension.data()); +} + +} // namespace + +JumboFileListGenerator::JumboFileListGenerator( + const Target* target, + Target::JumboFileList* jumbo_files, + Err* err) + : target_(target), + jumbo_files_dir_( + GetBuildDirForTargetAsSourceDir(target, BuildDirType::GEN)), + jumbo_files_(jumbo_files), + recent_jumbo_file_(nullptr), + recent_jumbo_file_type_(SourceFile::SOURCE_UNKNOWN), + err_(err) {} + +JumboFileListGenerator::~JumboFileListGenerator() = default; + +void JumboFileListGenerator::Run() { + const Target::FileList& excluded_sources = target_->jumbo_excluded_sources(); + + for (const SourceFile& input : target_->sources()) { + SourceFile::Type file_type = input.GetType(); + if (file_type != SourceFile::SOURCE_C && + file_type != SourceFile::SOURCE_CPP && + file_type != SourceFile::SOURCE_MM) { + continue; + } + + if (std::find(excluded_sources.begin(), excluded_sources.end(), input) != + excluded_sources.end()) { + continue; + } + + Target::JumboSourceFile* jumbo_file = FindJumboFile(file_type); + if (!jumbo_file) + jumbo_file = CreateJumboFile(file_type); + if (!jumbo_file) { + if (err_->has_error()) + return; + else + continue; // Source file type not supported by jumbo. Just skip it. + } + + jumbo_file->second.push_back(&input); + + recent_jumbo_file_ = jumbo_file; + recent_jumbo_file_type_ = file_type; + } +} + +Target::JumboSourceFile* JumboFileListGenerator::FindJumboFile( + SourceFile::Type file_type) const { + // Return recently used file if suitable. + if (recent_jumbo_file_type_ == file_type) { + return base::checked_cast(recent_jumbo_file_->second.size()) < + target_->jumbo_file_merge_limit() + ? recent_jumbo_file_ + : nullptr; + } + + // Return immediately if we don't have any files for |file_type|. + if (jumbo_file_numbers_.count(file_type) == 0) + return nullptr; + + // Search for file on |jumbo_files_| list. + for (auto it = jumbo_files_->rbegin(); it != jumbo_files_->rend(); ++it) { + if (it->first.GetType() == file_type) { + return base::checked_cast(it->second.size()) < + target_->jumbo_file_merge_limit() + ? &(*it) + : nullptr; + } + } + + NOTREACHED(); + return nullptr; +} + +Target::JumboSourceFile* JumboFileListGenerator::CreateJumboFile( + SourceFile::Type file_type) { + const auto it = jumbo_file_numbers_.find(file_type); + int file_number = it != jumbo_file_numbers_.end() ? it->second + 1 : 0; + jumbo_file_numbers_[file_type] = file_number; + + std::string file_name = + GetJumboFileName(target_->label().name(), file_type, file_number); + if (file_name.empty()) + return nullptr; + + SourceFile source_file = + jumbo_files_dir_.ResolveRelativeFile(Value(nullptr, file_name), err_); + if (source_file.is_null()) + return nullptr; + + jumbo_files_->push_back( + Target::JumboSourceFile(source_file, std::vector())); + Target::JumboSourceFile* jumbo_file = &jumbo_files_->back(); + jumbo_file->second.reserve(target_->jumbo_file_merge_limit()); + return jumbo_file; +} diff --git a/src/gn/jumbo_file_list_generator.h b/src/gn/jumbo_file_list_generator.h new file mode 100644 index 00000000..de42fb0c --- /dev/null +++ b/src/gn/jumbo_file_list_generator.h @@ -0,0 +1,59 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef GN_JUMBO_FILE_LIST_GENERATOR_H_ +#define GN_JUMBO_FILE_LIST_GENERATOR_H_ + +#include + +#include "gn/source_dir.h" +#include "gn/source_file.h" +#include "gn/target.h" + +/** + * Helper class responsible for generating jumbo file list for target sources. + */ +class JumboFileListGenerator { + public: + JumboFileListGenerator(const Target* target, + Target::JumboFileList* jumbo_files, + Err* err); + ~JumboFileListGenerator(); + + // Fills |jumbo_files| list passed to constructor. Sets the error passed to + // the constructor on failure. + void Run(); + + private: + // Returns JumboSourceFile object for given |file_type| if it exists and is + // suitable for adding more source files. + Target::JumboSourceFile* FindJumboFile(SourceFile::Type file_type) const; + + // Creates a new JumboSourceFile object for given |file_type| and adds it to + // |jumbo_files_|. + Target::JumboSourceFile* CreateJumboFile(SourceFile::Type file_type); + + const Target* target_; + + // Parent directory for jumbo files of |target_|. + SourceDir jumbo_files_dir_; + + // Generated list of jumbo files. + Target::JumboFileList* jumbo_files_; + + // Recently used numbers for jumbo files. Numbering is separate for each + // source file type. + std::map jumbo_file_numbers_; + + // Recently used jumbo file and its type. + Target::JumboSourceFile* recent_jumbo_file_; + SourceFile::Type recent_jumbo_file_type_; + + Err* err_; + + JumboFileListGenerator(const JumboFileListGenerator&) = delete; + JumboFileListGenerator& operator=(const JumboFileListGenerator&) = delete; +}; + +#endif // GN_JUMBO_FILE_LIST_GENERATOR_H_ diff --git a/src/gn/jumbo_file_list_generator_unittest.cc b/src/gn/jumbo_file_list_generator_unittest.cc new file mode 100644 index 00000000..b992e576 --- /dev/null +++ b/src/gn/jumbo_file_list_generator_unittest.cc @@ -0,0 +1,155 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gn/jumbo_file_list_generator.h" + +#include "base/strings/string_number_conversions.h" +#include "gn/filesystem_utils.h" +#include "gn/label.h" +#include "gn/source_dir.h" +#include "gn/source_file.h" +#include "gn/test_with_scope.h" +#include "util/test/test.h" + +namespace { + +class JumboFileListGeneratorTest : public testing::Test { + public: + JumboFileListGeneratorTest() + : target_(setup_.settings(), Label(SourceDir("foo"), "bar")) { + target_.set_jumbo_allowed(true); + } + + protected: + TestWithScope setup_; + Target target_; + Err err_; +}; + +} // namespace + +TEST_F(JumboFileListGeneratorTest, BasicList) { + target_.set_jumbo_file_merge_limit(3); + target_.sources().assign({ + SourceFile("a.cc"), SourceFile("a.h"), SourceFile("b.cc"), + SourceFile("A/c.cc"), SourceFile("B/d.cc"), SourceFile("A/B/e.cc"), + SourceFile("A/B/e.h"), + }); + + Target::JumboFileList jumbo_files; + JumboFileListGenerator generator(&target_, &jumbo_files, &err_); + generator.Run(); + + EXPECT_FALSE(err_.has_error()); + ASSERT_EQ(2u, jumbo_files.size()); + ASSERT_EQ(3u, jumbo_files[0].second.size()); + EXPECT_EQ("a.cc", jumbo_files[0].second[0]->value()); + EXPECT_EQ("b.cc", jumbo_files[0].second[1]->value()); + EXPECT_EQ("A/c.cc", jumbo_files[0].second[2]->value()); + ASSERT_EQ(2u, jumbo_files[1].second.size()); + EXPECT_EQ("B/d.cc", jumbo_files[1].second[0]->value()); + EXPECT_EQ("A/B/e.cc", jumbo_files[1].second[1]->value()); +} + +TEST_F(JumboFileListGeneratorTest, DefaultFileMergeLimit) { + // Default limit is 50. Add many source files. + for (int i = 0; i < 105; ++i) { + target_.sources().push_back(SourceFile(base::NumberToString(i) + ".cc")); + } + + Target::JumboFileList jumbo_files; + JumboFileListGenerator generator(&target_, &jumbo_files, &err_); + generator.Run(); + + EXPECT_FALSE(err_.has_error()); + ASSERT_EQ(3u, jumbo_files.size()); + EXPECT_EQ(50u, jumbo_files[0].second.size()); + EXPECT_EQ(50u, jumbo_files[1].second.size()); + EXPECT_EQ(5u, jumbo_files[2].second.size()); +} + +TEST_F(JumboFileListGeneratorTest, ExcludedSources) { + target_.set_jumbo_file_merge_limit(2); + target_.sources().assign({ + SourceFile("a.cc"), SourceFile("b.cc"), SourceFile("c.cc"), + SourceFile("d.cc"), SourceFile("e.cc"), + }); + target_.jumbo_excluded_sources().assign( + {SourceFile("b.cc"), SourceFile("d.cc")}); + + Target::JumboFileList jumbo_files; + JumboFileListGenerator generator(&target_, &jumbo_files, &err_); + generator.Run(); + + EXPECT_FALSE(err_.has_error()); + ASSERT_EQ(2u, jumbo_files.size()); + ASSERT_EQ(2u, jumbo_files[0].second.size()); + EXPECT_EQ("a.cc", jumbo_files[0].second[0]->value()); + EXPECT_EQ("c.cc", jumbo_files[0].second[1]->value()); + ASSERT_EQ(1u, jumbo_files[1].second.size()); + EXPECT_EQ("e.cc", jumbo_files[1].second[0]->value()); +} + +TEST_F(JumboFileListGeneratorTest, MixedSourceFileTypes) { + target_.set_jumbo_file_merge_limit(2); + target_.sources().assign({ + SourceFile("a.cc"), SourceFile("1.mm"), SourceFile("2.mm"), + SourceFile("3.mm"), SourceFile("b.cc"), SourceFile("c.cc"), + SourceFile("d.cc"), SourceFile("4.mm"), SourceFile("5.mm"), + SourceFile("e.cc"), + }); + + Target::JumboFileList jumbo_files; + JumboFileListGenerator generator(&target_, &jumbo_files, &err_); + generator.Run(); + + EXPECT_FALSE(err_.has_error()); + ASSERT_EQ(6u, jumbo_files.size()); + EXPECT_EQ("cc", FindExtension(&jumbo_files[0].first.value())); + ASSERT_EQ(2u, jumbo_files[0].second.size()); + EXPECT_EQ("a.cc", jumbo_files[0].second[0]->value()); + EXPECT_EQ("b.cc", jumbo_files[0].second[1]->value()); + EXPECT_EQ("mm", FindExtension(&jumbo_files[1].first.value())); + ASSERT_EQ(2u, jumbo_files[1].second.size()); + EXPECT_EQ("1.mm", jumbo_files[1].second[0]->value()); + EXPECT_EQ("2.mm", jumbo_files[1].second[1]->value()); + EXPECT_EQ("mm", FindExtension(&jumbo_files[2].first.value())); + ASSERT_EQ(2u, jumbo_files[2].second.size()); + EXPECT_EQ("3.mm", jumbo_files[2].second[0]->value()); + EXPECT_EQ("4.mm", jumbo_files[2].second[1]->value()); + EXPECT_EQ("cc", FindExtension(&jumbo_files[3].first.value())); + ASSERT_EQ(2u, jumbo_files[3].second.size()); + EXPECT_EQ("c.cc", jumbo_files[3].second[0]->value()); + EXPECT_EQ("d.cc", jumbo_files[3].second[1]->value()); + EXPECT_EQ("mm", FindExtension(&jumbo_files[4].first.value())); + ASSERT_EQ(1u, jumbo_files[4].second.size()); + EXPECT_EQ("5.mm", jumbo_files[4].second[0]->value()); + EXPECT_EQ("cc", FindExtension(&jumbo_files[5].first.value())); + ASSERT_EQ(1u, jumbo_files[5].second.size()); + EXPECT_EQ("e.cc", jumbo_files[5].second[0]->value()); +} + +TEST_F(JumboFileListGeneratorTest, SupportedSourceFileTypes) { + target_.sources().assign({ + SourceFile("x.cc"), SourceFile("x.cpp"), SourceFile("x.cxx"), + SourceFile("x.h"), SourceFile("x.hpp"), SourceFile("x.hxx"), + SourceFile("x.hh"), SourceFile("x.c"), SourceFile("x.m"), + SourceFile("x.mm"), SourceFile("x.rc"), SourceFile("x.S"), + SourceFile("x.s"), SourceFile("x.asm"), SourceFile("x.o"), + SourceFile("x.obj"), SourceFile("x.def"), + }); + + Target::JumboFileList jumbo_files; + JumboFileListGenerator generator(&target_, &jumbo_files, &err_); + generator.Run(); + + EXPECT_FALSE(err_.has_error()); + ASSERT_EQ(3u, jumbo_files.size()); + EXPECT_EQ("cc", FindExtension(&jumbo_files[0].first.value())); + ASSERT_EQ(3u, jumbo_files[0].second.size()); + EXPECT_EQ("c", FindExtension(&jumbo_files[1].first.value())); + ASSERT_EQ(1u, jumbo_files[1].second.size()); + EXPECT_EQ("mm", FindExtension(&jumbo_files[2].first.value())); + ASSERT_EQ(1u, jumbo_files[2].second.size()); +} diff --git a/src/gn/jumbo_writer.cc b/src/gn/jumbo_writer.cc new file mode 100644 index 00000000..38fa6b9d --- /dev/null +++ b/src/gn/jumbo_writer.cc @@ -0,0 +1,66 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gn/jumbo_writer.h" + +#include + +#include "base/files/file_util.h" +#include "gn/build_settings.h" +#include "gn/filesystem_utils.h" +#include "gn/scheduler.h" +#include "gn/settings.h" +#include "gn/string_output_buffer.h" + +JumboWriter::JumboWriter(const Target* target) + : target_(target), + path_output_(GetBuildDirForTargetAsSourceDir(target, BuildDirType::GEN), + target_->settings()->build_settings()->root_path_utf8(), + ESCAPE_NONE) {} + +JumboWriter::~JumboWriter() = default; + +void JumboWriter::RunAndWriteFiles(const Target* target) { + JumboWriter writer(target); + writer.Run(); +} + +void JumboWriter::Run() const { + if (target_->jumbo_files().empty()) + return; + + base::CreateDirectory(target_->settings() + ->build_settings() + ->GetFullPath(target_->jumbo_files()[0].first) + .DirName()); + + for (const Target::JumboSourceFile& jumbo_file : target_->jumbo_files()) { + if (!WriteJumboFile(jumbo_file)) + return; + } +} + +bool JumboWriter::WriteJumboFile( + const Target::JumboSourceFile& jumbo_file) const { + StringOutputBuffer storage; + std::ostream content(&storage); + content << "/* This is a Jumbo file. Don't edit. " + << "Generated with 'gn gen' command. */\n\n"; + + for (const SourceFile* source_file : jumbo_file.second) { + content << "#include \""; + path_output_.WriteFile(content, *source_file); + content << "\"\n"; + } + + Err err; + if (!storage.WriteToFileIfChanged( + target_->settings()->build_settings()->GetFullPath(jumbo_file.first), + &err)) { + g_scheduler->FailWithError(err); + return false; + } + + return true; +} diff --git a/src/gn/jumbo_writer.h b/src/gn/jumbo_writer.h new file mode 100644 index 00000000..eccd84ca --- /dev/null +++ b/src/gn/jumbo_writer.h @@ -0,0 +1,34 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef GN_JUMBO_WRITER_H_ +#define GN_JUMBO_WRITER_H_ + +#include +#include + +#include "gn/path_output.h" +#include "gn/target.h" + +// Helper class responsible for writing jumbo files. +class JumboWriter { + public: + explicit JumboWriter(const Target* target); + ~JumboWriter(); + + // Writes jumbo files for given |target|. + static void RunAndWriteFiles(const Target* target); + + private: + void Run() const; + bool WriteJumboFile(const Target::JumboSourceFile& jumbo_file) const; + + const Target* target_; + const PathOutput path_output_; + + JumboWriter(const JumboWriter&) = delete; + JumboWriter& operator=(const JumboWriter&) = delete; +}; + +#endif // GN_JUMBO_WRITER_H_ diff --git a/src/gn/jumbo_writer_unittest.cc b/src/gn/jumbo_writer_unittest.cc new file mode 100644 index 00000000..b06802eb --- /dev/null +++ b/src/gn/jumbo_writer_unittest.cc @@ -0,0 +1,77 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gn/jumbo_writer.h" + +#include +#include +#include + +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/files/scoped_temp_dir.h" +#include "base/strings/string_split.h" +#include "base/strings/string_util.h" +#include "gn/build_settings.h" +#include "gn/source_dir.h" +#include "gn/target.h" +#include "gn/test_with_scope.h" +#include "util/test/test.h" + +namespace { + +std::vector ReadFileLines(const base::FilePath& file_path) { + std::string contents; + base::ReadFileToString(file_path, &contents); + return base::SplitString(contents, "\n", base::TRIM_WHITESPACE, + base::SPLIT_WANT_NONEMPTY); +} + +// Returns #include line for |file_path|. As a side effect, prevents presubmit +// check from complaining about relative includes in this test. +std::string MakeIncludeLine(const char* file_path) { + return std::string("#include \"") + file_path + '\"'; +} + +} // namespace + +TEST(JumboWriter, WriteJumboFiles) { + TestWithScope setup; + + // Make a real directory for writing the files. + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + setup.build_settings()->SetRootPath(temp_dir.GetPath()); + + Target target(setup.settings(), Label(SourceDir("foo"), "bar")); + + SourceFile file_a("//foo/a.cc"); + SourceFile file_b("//foo/subdir/b.cc"); + SourceFile file_c("//foo/c.cc"); + + target.jumbo_files().assign( + {Target::JumboSourceFile(SourceFile("//out/Debug/gen/foo/jumbo_cc_1.cc"), + {&file_a, &file_b}), + Target::JumboSourceFile(SourceFile("//out/Debug/gen/foo/jumbo_cc_2.cc"), + {&file_c})}); + + JumboWriter::RunAndWriteFiles(&target); + + auto lines = ReadFileLines( + temp_dir.GetPath().AppendASCII("out/Debug/gen/foo/jumbo_cc_1.cc")); + ASSERT_EQ(3u, lines.size()); + // First line is a comment. + EXPECT_TRUE(base::StartsWith(lines[0], "/*", base::CompareCase::SENSITIVE)); + EXPECT_TRUE(base::EndsWith(lines[0], "*/", base::CompareCase::SENSITIVE)); + EXPECT_EQ(MakeIncludeLine("../../../../foo/a.cc"), lines[1]); + EXPECT_EQ(MakeIncludeLine("../../../../foo/subdir/b.cc"), lines[2]); + + lines = ReadFileLines( + temp_dir.GetPath().AppendASCII("out/Debug/gen/foo/jumbo_cc_2.cc")); + ASSERT_EQ(2u, lines.size()); + // First line is a comment. + EXPECT_TRUE(base::StartsWith(lines[0], "/*", base::CompareCase::SENSITIVE)); + EXPECT_TRUE(base::EndsWith(lines[0], "*/", base::CompareCase::SENSITIVE)); + EXPECT_EQ(MakeIncludeLine("../../../../foo/c.cc"), lines[1]); +} diff --git a/src/gn/ninja_binary_target_writer.cc b/src/gn/ninja_binary_target_writer.cc index 48f485ad..816b5ece 100644 --- a/src/gn/ninja_binary_target_writer.cc +++ b/src/gn/ninja_binary_target_writer.cc @@ -4,9 +4,12 @@ #include "gn/ninja_binary_target_writer.h" +#include #include #include "base/strings/string_util.h" +#include "gn/args.h" +#include "gn/build_settings.h" #include "gn/config_values_extractors.h" #include "gn/deps_iterator.h" #include "gn/filesystem_utils.h" @@ -50,6 +53,35 @@ void NinjaBinaryTargetWriter::Run() { writer.Run(); } +bool NinjaBinaryTargetWriter::IsJumboEnabledForTarget(const Target* target) { + if (!target->is_jumbo_allowed()) + return false; + + const Value* value = + target->settings()->build_settings()->build_args().GetArgOverride( + variables::kEnableNativeJumbo); + return value && value->type() == Value::BOOLEAN && value->boolean_value(); +} + +const Target::FileList& NinjaBinaryTargetWriter::GetSourcesForTarget( + const Target* target, + Target::FileList* sources) { + const bool use_jumbo = IsJumboEnabledForTarget(target); + + if (use_jumbo) { + std::transform(target->jumbo_files().begin(), target->jumbo_files().end(), + std::back_inserter(*sources), + [](const Target::JumboSourceFile& jumbo_file) -> SourceFile { + return jumbo_file.first; + }); + + sources->insert(sources->end(), target->jumbo_excluded_sources().begin(), + target->jumbo_excluded_sources().end()); + } + + return use_jumbo ? *sources : target->sources(); +} + std::vector NinjaBinaryTargetWriter::WriteInputsStampAndGetDep( size_t num_stamp_uses) const { CHECK(target_->toolchain()) << "Toolchain not set on target " @@ -207,9 +239,13 @@ void NinjaBinaryTargetWriter::AddSourceSetFiles( UniqueVector* obj_files) const { std::vector tool_outputs; // Prevent allocation in loop. + Target::FileList temp_sources; + const Target::FileList& sources = + GetSourcesForTarget(source_set, &temp_sources); + // Compute object files for all sources. Only link the first output from // the tool if there are more than one. - for (const auto& source : source_set->sources()) { + for (const auto& source : sources) { const char* tool_name = Tool::kToolNone; if (source_set->GetOutputFilesForSource(source, &tool_name, &tool_outputs)) obj_files->push_back(tool_outputs[0]); diff --git a/src/gn/ninja_binary_target_writer.h b/src/gn/ninja_binary_target_writer.h index e53e28eb..286bc0de 100644 --- a/src/gn/ninja_binary_target_writer.h +++ b/src/gn/ninja_binary_target_writer.h @@ -8,6 +8,7 @@ #include "gn/c_tool.h" #include "gn/config_values.h" #include "gn/ninja_target_writer.h" +#include "gn/target.h" #include "gn/toolchain.h" #include "gn/unique_vector.h" @@ -32,6 +33,15 @@ class NinjaBinaryTargetWriter : public NinjaTargetWriter { UniqueVector swiftmodule_deps; }; + // Returns true if jumbo mode is globally enabled and allowed for |target|. + static bool IsJumboEnabledForTarget(const Target* target); + + // Returns a list of files that should be compiled for |target| considering + // jumbo mode. Function may use |sources| as a handy storage and return a + // reference to it. + static const Target::FileList& GetSourcesForTarget(const Target* target, + Target::FileList* sources); + // Writes to the output stream a stamp rule for inputs, and // returns the file to be appended to source rules that encodes the // implicit dependencies for the current target. diff --git a/src/gn/ninja_c_binary_target_writer.cc b/src/gn/ninja_c_binary_target_writer.cc index d9c3ad2f..0b3ae39a 100644 --- a/src/gn/ninja_c_binary_target_writer.cc +++ b/src/gn/ninja_c_binary_target_writer.cc @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -19,6 +20,7 @@ #include "gn/escape.h" #include "gn/filesystem_utils.h" #include "gn/general_tool.h" +#include "gn/jumbo_writer.h" #include "gn/ninja_target_command_util.h" #include "gn/ninja_utils.h" #include "gn/scheduler.h" @@ -116,6 +118,66 @@ std::vector GetModuleDepsInformation(const Target* target) { return ret; } +// Prepares information needed to write build line for |source| file. Returns +// Tool::kToolNone if |source| doesn't need any build command. +const char* PrepareCompilerBuildLine( + const Target& target, + const SourceFile& source, + const std::vector& pch_deps, + const std::vector& input_deps, + const std::vector& module_dep_info, + std::vector* extra_deps, + std::vector* tool_outputs, + std::vector* other_files) { + const char* tool_name = Tool::kToolNone; + if (!target.GetOutputFilesForSource(source, &tool_name, tool_outputs)) { + if (source.IsDefType()) + other_files->push_back(source); + return nullptr; // No output for this source. + } + + std::copy(input_deps.begin(), input_deps.end(), + std::back_inserter(*extra_deps)); + + if (tool_name != Tool::kToolNone) { + // Only include PCH deps that correspond to the tool type, for instance, + // do not specify target_name.precompile.cc.obj (a CXX PCH file) as a dep + // for the output of a C tool type. + // + // This makes the assumption that pch_deps only contains pch output files + // with the naming scheme specified in GetWindowsPCHObjectExtension or + // GetGCCPCHOutputExtension. + const CTool* tool = target.toolchain()->GetToolAsC(tool_name); + if (tool->precompiled_header_type() != CTool::PCH_NONE) { + for (const auto& dep : pch_deps) { + const std::string& output_value = dep.value(); + size_t extension_offset = FindExtensionOffset(output_value); + if (extension_offset == std::string::npos) + continue; + std::string output_extension; + if (tool->precompiled_header_type() == CTool::PCH_MSVC) { + output_extension = GetWindowsPCHObjectExtension( + tool_name, output_value.substr(extension_offset - 1)); + } else if (tool->precompiled_header_type() == CTool::PCH_GCC) { + output_extension = GetGCCPCHOutputExtension(tool_name); + } + if (output_value.compare(output_value.size() - output_extension.size(), + output_extension.size(), + output_extension) == 0) { + extra_deps->push_back(dep); + } + } + } + } + + for (const auto& module_dep : module_dep_info) { + if ((*tool_outputs)[0] != module_dep.pcm) + extra_deps->push_back(module_dep.pcm); + } + + return tool_name; +} + } // namespace NinjaCBinaryTargetWriter::NinjaCBinaryTargetWriter(const Target* target, @@ -178,6 +240,9 @@ void NinjaCBinaryTargetWriter::Run() { std::vector* pch_files = !pch_obj_files.empty() ? &pch_obj_files : &pch_other_files; + if (IsJumboEnabledForTarget(target_)) + JumboWriter::RunAndWriteFiles(target_); + // Treat all pch output files as explicit dependencies of all // compiles that support them. Some notes: // @@ -546,67 +611,53 @@ void NinjaCBinaryTargetWriter::WriteSources( std::vector* object_files, std::vector* other_files) { DCHECK(!target_->source_types_used().SwiftSourceUsed()); - object_files->reserve(object_files->size() + target_->sources().size()); + Target::FileList temp_sources; + const Target::FileList& sources = GetSourcesForTarget(target_, &temp_sources); + + object_files->reserve(object_files->size() + sources.size()); std::vector tool_outputs; // Prevent reallocation in loop. std::vector deps; - for (const auto& source : target_->sources()) { + for (const auto& source : sources) { DCHECK_NE(source.GetType(), SourceFile::SOURCE_SWIFT); // Clear the vector but maintain the max capacity to prevent reallocations. deps.resize(0); - const char* tool_name = Tool::kToolNone; - if (!target_->GetOutputFilesForSource(source, &tool_name, &tool_outputs)) { - if (source.IsDefType()) - other_files->push_back(source); + const char* tool_name = + PrepareCompilerBuildLine(*target_, source, pch_deps, input_deps, + module_dep_info, &deps, &tool_outputs, + other_files); + if (!tool_name) continue; // No output for this source. - } - - std::copy(input_deps.begin(), input_deps.end(), std::back_inserter(deps)); if (tool_name != Tool::kToolNone) { - // Only include PCH deps that correspond to the tool type, for instance, - // do not specify target_name.precompile.cc.obj (a CXX PCH file) as a dep - // for the output of a C tool type. - // - // This makes the assumption that pch_deps only contains pch output files - // with the naming scheme specified in GetWindowsPCHObjectExtension or - // GetGCCPCHOutputExtension. - const CTool* tool = target_->toolchain()->GetToolAsC(tool_name); - if (tool->precompiled_header_type() != CTool::PCH_NONE) { - for (const auto& dep : pch_deps) { - const std::string& output_value = dep.value(); - size_t extension_offset = FindExtensionOffset(output_value); - if (extension_offset == std::string::npos) - continue; - std::string output_extension; - if (tool->precompiled_header_type() == CTool::PCH_MSVC) { - output_extension = GetWindowsPCHObjectExtension( - tool_name, output_value.substr(extension_offset - 1)); - } else if (tool->precompiled_header_type() == CTool::PCH_GCC) { - output_extension = GetGCCPCHOutputExtension(tool_name); - } - if (output_value.compare( - output_value.size() - output_extension.size(), - output_extension.size(), output_extension) == 0) { - deps.push_back(dep); - } - } - } - - for (const auto& module_dep : module_dep_info) { - if (tool_outputs[0] != module_dep.pcm) - deps.push_back(module_dep.pcm); - } - WriteCompilerBuildLine({source}, deps, order_only_deps, tool_name, tool_outputs); } // It's theoretically possible for a compiler to produce more than one // output, but we'll only link to the first output. - if (!source.IsModuleMapType()) { + if (!source.IsModuleMapType() && !tool_outputs.empty()) object_files->push_back(tool_outputs[0]); + } + + // Write build commands for source files that have been merged into jumbo + // files. Don't add outputs to |object_files| because we don't want to link + // them (outputs for jumbo files are linked instead). + if (IsJumboEnabledForTarget(target_)) { + for (const auto& jumbo_file : target_->jumbo_files()) { + for (const auto* source : jumbo_file.second) { + // Clear the vector but maintain the max capacity. + deps.resize(0); + const char* tool_name = + PrepareCompilerBuildLine(*target_, *source, pch_deps, input_deps, + module_dep_info, &deps, &tool_outputs, + other_files); + if (tool_name && tool_name != Tool::kToolNone) { + WriteCompilerBuildLine({*source}, deps, order_only_deps, tool_name, + tool_outputs); + } + } } } diff --git a/src/gn/target.cc b/src/gn/target.cc index 70713575..aa750f6d 100644 --- a/src/gn/target.cc +++ b/src/gn/target.cc @@ -24,6 +24,8 @@ namespace { using ConfigSet = std::set; +constexpr int kDefaultJumboFileMergeLimit = 50; + // Merges the public configs from the given target to the given config list. void MergePublicConfigsFrom(const Target* from_target, UniqueVector* dest) { @@ -281,7 +283,8 @@ Dependencies Target::Target(const Settings* settings, const Label& label, const SourceFileSet& build_dependency_files) - : Item(settings, label, build_dependency_files) {} + : Item(settings, label, build_dependency_files), + jumbo_file_merge_limit_(kDefaultJumboFileMergeLimit) {} Target::~Target() = default; diff --git a/src/gn/target.h b/src/gn/target.h index 293d53c6..0477f03d 100644 --- a/src/gn/target.h +++ b/src/gn/target.h @@ -5,8 +5,10 @@ #ifndef TOOLS_GN_TARGET_H_ #define TOOLS_GN_TARGET_H_ +#include #include #include +#include #include #include "base/gtest_prod_util.h" @@ -58,6 +60,8 @@ class Target : public Item { using FileList = std::vector; using StringVector = std::vector; + using JumboSourceFile = std::pair>; + using JumboFileList = std::vector; // We track the set of build files that may affect this target, please refer // to Scope for how this is determined. @@ -309,6 +313,27 @@ class Target : public Item { return assert_no_deps_; } + // Set to true if jumbo compilation is allowed for this target. + bool is_jumbo_allowed() const { return jumbo_allowed_.value_or(false); } + void set_jumbo_allowed(bool jumbo_allowed) { jumbo_allowed_ = jumbo_allowed; } + bool is_jumbo_configured() const { return jumbo_allowed_.has_value(); } + + // List of source files not merged in jumbo mode. + const FileList& jumbo_excluded_sources() const { + return jumbo_excluded_sources_; + } + FileList& jumbo_excluded_sources() { return jumbo_excluded_sources_; } + + // Maximum number of source files to group in jumbo mode. + int jumbo_file_merge_limit() const { return jumbo_file_merge_limit_; } + void set_jumbo_file_merge_limit(int limit) { + jumbo_file_merge_limit_ = limit; + } + + // List of jumbo source files with original merged source files. + const JumboFileList& jumbo_files() const { return jumbo_files_; } + JumboFileList& jumbo_files() { return jumbo_files_; } + // The toolchain is only known once this target is resolved (all if its // dependencies are known). They will be null until then. Generally, this can // only be used during target writing. @@ -491,6 +516,12 @@ class Target : public Item { // Toolchain used by this target. Null until target is resolved. const Toolchain* toolchain_ = nullptr; + // Jumbo mode configuration. + std::optional jumbo_allowed_; + FileList jumbo_excluded_sources_; + int jumbo_file_merge_limit_; + JumboFileList jumbo_files_; + // Output files. Empty until the target is resolved. std::vector computed_outputs_; OutputFile link_output_file_; diff --git a/src/gn/variables.cc b/src/gn/variables.cc index 285f3b94..fafcd940 100644 --- a/src/gn/variables.cc +++ b/src/gn/variables.cc @@ -11,6 +11,25 @@ namespace variables { // Built-in variables ---------------------------------------------------------- +const char kEnableNativeJumbo[] = "enable_native_jumbo"; +const char kEnableNativeJumbo_HelpShort[] = + "enable_native_jumbo: [boolean] Enable jumbo mode to speed up compilation."; +const char kEnableNativeJumbo_Help[] = + R"(enable_native_jumbo: [boolean] Enable jumbo mode to speed up compilation. + + Enable jumbo mode to speed up compilation of targets that allow it. + + In jumbo mode many translation units are merged and compiled together. This + reduces the amount of work needed to process shared header files. Linking is + faster because there is less redundant data (debug information, inline + functions) to merge. + + This is Opera feature. It's implemented natively in GN binary and replaces the + jumbo mechanism implemented in .gni files. + + See "gn help jumbo_allowed". +)"; + const char kGnVersion[] = "gn_version"; const char kGnVersion_HelpShort[] = "gn_version: [number] The version of gn."; @@ -1329,6 +1348,82 @@ Example } )"; +const char kJumboAllowed[] = "jumbo_allowed"; +const char kJumboAllowed_HelpShort[] = + "jumbo_allowed: [boolean] Allow jumbo mode for a target."; +const char kJumboAllowed_Help[] = + R"(jumbo_allowed: [boolean] Allow jumbo mode for a target. + + Allow merging files together for jumbo compilation in a target. Has no effect + if global variable "enable_native_jumbo" is not "true". + + See "gn help enable_native_jumbo" for more information. + + See also "gn help jumbo_excluded_sources" and + "gn help jumbo_file_merge_limit". + +Example + + source_set("doom_melon") { + jumbo_allowed = true + sources = [ "a.cc", "b.cc" ] # Both files will be compiled together. + } +)"; + +const char kJumboExcludedSources[] = "jumbo_excluded_sources"; +const char kJumboExcludedSources_HelpShort[] = + "jumbo_excluded_sources: [file list] Files not merged in jumbo mode."; +const char kJumboExcludedSources_Help[] = + R"(jumbo_excluded_sources: [file list] Files not merged in jumbo mode. + + List of source files that will not be merged with the rest in jumbo mode. This + can be necessary if merging the files causes compilation issues and fixing the + issues is impractical. + + See also "gn help jumbo_allowed". + +Example + + source_set("doom_melon") { + jumbo_allowed = true + # "a.cc", "c.cc" and "e.cc" will be compiled together. "b.cc" and "d.cc" + # will be compiled separately. + sources = [ "a.cc", "b.cc", "c.cc", "d.cc", "e.cc" ] + jumbo_excluded_sources = [ "b.cc", "d.cc" ] + } +)"; + +const char kJumboFileMergeLimit[] = "jumbo_file_merge_limit"; +const char kJumboFileMergeLimit_HelpShort[] = + "jumbo_file_merge_limit: [number] Maximum number of files to group."; +const char kJumboFileMergeLimit_Help[] = + R"(jumbo_file_merge_limit: [number] Maximum number of files to group. + + How many files to group at most. Smaller numbers give more parallellism, + higher numbers give less total CPU usage. Higher numbers also give longer + single-file recompilation times. The default is 50. + + Recommendations: + Higher numbers than 100 does not reduce wall clock compile times even for 4 + cores or less so no reason to go higher than 100. Going from 50 to 100 with a + 4 core CPU saves about 3% CPU time and 3% wall clock time. At the same time it + increases the compile time for the largest jumbo chunks by 10-20% and reduces + the chance to use all available CPU cores. So set the default to 50 to balance + between high and low-core build performance. + + See also "gn help jumbo_allowed". + +Example + + source_set("doom_melon") { + jumbo_allowed = true + jumbo_file_merge_limit = 3 + # "a.cc", "b.cc" and "c.cc" will be compiled together. "d.cc" and "e.cc" + # will be compiled together. + sources = [ "a.cc", "b.cc", "c.cc", "d.cc", "e.cc" ] + } +)"; + const char kLdflags[] = "ldflags"; const char kLdflags_HelpShort[] = "ldflags: [string list] Flags passed to the linker."; @@ -2243,6 +2338,7 @@ const VariableInfoMap& GetBuiltinVariables() { INSERT_VARIABLE(CurrentOs) INSERT_VARIABLE(CurrentToolchain) INSERT_VARIABLE(DefaultToolchain) + INSERT_VARIABLE(EnableNativeJumbo) INSERT_VARIABLE(GnVersion) INSERT_VARIABLE(HostCpu) INSERT_VARIABLE(HostOs) @@ -2299,6 +2395,9 @@ const VariableInfoMap& GetTargetVariables() { INSERT_VARIABLE(Frameworks) INSERT_VARIABLE(IncludeDirs) INSERT_VARIABLE(Inputs) + INSERT_VARIABLE(JumboAllowed) + INSERT_VARIABLE(JumboExcludedSources) + INSERT_VARIABLE(JumboFileMergeLimit) INSERT_VARIABLE(Ldflags) INSERT_VARIABLE(Libs) INSERT_VARIABLE(LibDirs) diff --git a/src/gn/variables.h b/src/gn/variables.h index f5794d1d..ade31099 100644 --- a/src/gn/variables.h +++ b/src/gn/variables.h @@ -36,6 +36,10 @@ extern const char kDefaultToolchain[]; extern const char kDefaultToolchain_HelpShort[]; extern const char kDefaultToolchain_Help[]; +extern const char kEnableNativeJumbo[]; +extern const char kEnableNativeJumbo_HelpShort[]; +extern const char kEnableNativeJumbo_Help[]; + extern const char kGnVersion[]; extern const char kGnVersion_HelpShort[]; extern const char kGnVersion_Help[]; @@ -226,6 +230,18 @@ extern const char kInputs[]; extern const char kInputs_HelpShort[]; extern const char kInputs_Help[]; +extern const char kJumboAllowed[]; +extern const char kJumboAllowed_HelpShort[]; +extern const char kJumboAllowed_Help[]; + +extern const char kJumboExcludedSources[]; +extern const char kJumboExcludedSources_HelpShort[]; +extern const char kJumboExcludedSources_Help[]; + +extern const char kJumboFileMergeLimit[]; +extern const char kJumboFileMergeLimit_HelpShort[]; +extern const char kJumboFileMergeLimit_Help[]; + extern const char kLdflags[]; extern const char kLdflags_HelpShort[]; extern const char kLdflags_Help[];