Add jumbo support

This allows to generate Visual Studio projects that include source files
that are merged together into jumbo files.

build/gen.py

@@ -125,9 +125,10 @@ def main(argv):
 
 def GenerateLastCommitPosition(host, header):
   ROOT_TAG = 'initial-commit'
+  # Use HEAD@{1} so original Chromium and Opera (patched) versions are the same.
   describe_output = subprocess.check_output(
-      ['git', 'describe', 'HEAD', '--match', ROOT_TAG], shell=host.is_windows(),
-      cwd=REPO_ROOT)
+      ['git', 'describe', 'HEAD@{1}', '--match', ROOT_TAG],
+      shell=host.is_windows(), cwd=REPO_ROOT)
   mo = re.match(ROOT_TAG + '-(\d+)-g([0-9a-f]+)', describe_output.decode())
   if not mo:
     raise ValueError(
@@ -497,6 +498,8 @@ def WriteGNNinja(path, platform, host, options):
         'src/gn/input_file_manager.cc',
         'src/gn/item.cc',
         'src/gn/json_project_writer.cc',
+        'src/gn/jumbo_file_list_generator.cc',
+        'src/gn/jumbo_writer.cc',
         'src/gn/label.cc',
         'src/gn/label_pattern.cc',
         'src/gn/lib_file.cc',
@@ -609,6 +612,8 @@ def WriteGNNinja(path, platform, host, options):
         'src/gn/inherited_libraries_unittest.cc',
         'src/gn/input_conversion_unittest.cc',
         'src/gn/json_project_writer_unittest.cc',
+        'src/gn/jumbo_file_list_generator_unittest.cc',
+        'src/gn/jumbo_writer_unittest.cc',
         'src/gn/label_pattern_unittest.cc',
         'src/gn/label_unittest.cc',
         'src/gn/loader_unittest.cc',

src/gn/args.cc

@@ -368,6 +368,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
   // declared. This is so they can be overridden in a toolchain build args
   // override, and so that they will appear in the "gn args" output.
   Value empty_string(nullptr, std::string());
+  Value false_bool(nullptr, false);
 
   Value os_val(nullptr, std::string(os));
   dest->SetValue(variables::kHostOs, os_val, nullptr);
@@ -379,6 +380,8 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
   dest->SetValue(variables::kTargetCpu, empty_string, nullptr);
   dest->SetValue(variables::kCurrentCpu, empty_string, nullptr);
 
+  dest->SetValue(variables::kEnableNativeJumbo, false_bool, nullptr);
+
   Scope::KeyValueMap& declared_arguments(
       DeclaredArgumentsForToolchainLocked(dest));
   declared_arguments[variables::kHostOs] = os_val;
@@ -387,6 +390,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
   declared_arguments[variables::kHostCpu] = arch_val;
   declared_arguments[variables::kCurrentCpu] = empty_string;
   declared_arguments[variables::kTargetCpu] = empty_string;
+  declared_arguments[variables::kEnableNativeJumbo] = false_bool;
 
   // Mark these variables used so the build config file can override them
   // without geting a warning about overwriting an unused variable.
@@ -396,6 +400,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
   dest->MarkUsed(variables::kHostOs);
   dest->MarkUsed(variables::kCurrentOs);
   dest->MarkUsed(variables::kTargetOs);
+  dest->MarkUsed(variables::kEnableNativeJumbo);
 }
 
 void Args::ApplyOverridesLocked(const Scope::KeyValueMap& values,

src/gn/binary_target_generator.cc

@@ -4,11 +4,14 @@
 
 #include "gn/binary_target_generator.h"
 
+#include <algorithm>
+
 #include "gn/config_values_generator.h"
 #include "gn/deps_iterator.h"
 #include "gn/err.h"
 #include "gn/filesystem_utils.h"
 #include "gn/functions.h"
+#include "gn/jumbo_file_list_generator.h"
 #include "gn/parse_tree.h"
 #include "gn/rust_values_generator.h"
 #include "gn/rust_variables.h"
@@ -79,6 +82,23 @@ void BinaryTargetGenerator::DoRun() {
   gen.Run();
   if (err_->has_error())
     return;
+
+  if (!FillJumboAllowed())
+    return;
+
+  if (!FillJumboExcludedSources())
+    return;
+
+  if (!FillJumboFileMergeLimit())
+    return;
+
+  if (target_->is_jumbo_allowed()) {
+    JumboFileListGenerator jumbo_generator(target_, &target_->jumbo_files(),
+                                           err_);
+    jumbo_generator.Run();
+    if (err_->has_error())
+      return;
+  }
 }
 
 bool BinaryTargetGenerator::FillSources() {
@@ -217,6 +237,82 @@ bool BinaryTargetGenerator::FillAllowCircularIncludesFrom() {
   return true;
 }
 
+bool BinaryTargetGenerator::FillJumboAllowed() {
+  const Value* value = scope_->GetValue(variables::kJumboAllowed, true);
+  if (!value)
+    return true;
+
+  if (!value->VerifyTypeIs(Value::BOOLEAN, err_))
+    return false;
+
+  target_->set_jumbo_allowed(value->boolean_value());
+  return true;
+}
+
+bool BinaryTargetGenerator::FillJumboExcludedSources() {
+  const Value* value = scope_->GetValue(variables::kJumboExcludedSources, true);
+  if (!value)
+    return true;
+
+  if (!target_->is_jumbo_allowed()) {
+    // TODO(tmoniuszko): We currently don't report error here because we want
+    // to support non-native jumbo implementation from BUILD.gn scripts.
+    // For native-only jumbo we would display such error:
+    // *err_ = Err(*value, "Jumbo is not allowed for this target.");
+    return false;
+  }
+
+  Target::FileList jumbo_excluded_sources;
+  if (!ExtractListOfRelativeFiles(scope_->settings()->build_settings(), *value,
+                                  scope_->GetSourceDir(),
+                                  &jumbo_excluded_sources, err_)) {
+    return false;
+  }
+
+  // Excluded files should be in sources. jumbo_excluded_sources is intended to
+  // exclude only small amount of files that cause compilation issues so
+  // searching for every file in loop should be acceptable despite of time
+  // complexity.
+  const Target::FileList& sources = target_->sources();
+  for (const SourceFile& file : jumbo_excluded_sources) {
+    if (std::find(sources.begin(), sources.end(), file) == sources.end()) {
+      *err_ = Err(*value, "Excluded file not in sources.",
+                  "The file \"" + file.value() + "\" was not in \"sources\"." +
+                      "  " + sources.front().value());
+      return false;
+    }
+  }
+
+  target_->jumbo_excluded_sources().swap(jumbo_excluded_sources);
+  return true;
+}
+
+bool BinaryTargetGenerator::FillJumboFileMergeLimit() {
+  const Value* value = scope_->GetValue(variables::kJumboFileMergeLimit, true);
+  if (!value)
+    return true;
+
+  if (!target_->is_jumbo_allowed()) {
+    // TODO(tmoniuszko): We currently don't report error here because we want
+    // to support non-native jumbo implementation from BUILD.gn scripts.
+    // For native-only jumbo we would display such error:
+    // *err_ = Err(*value, "Jumbo is not allowed for this target.");
+    return false;
+  }
+
+  if (!value->VerifyTypeIs(Value::INTEGER, err_))
+    return false;
+
+  int jumbo_file_merge_limit = value->int_value();
+  if (jumbo_file_merge_limit < 2) {
+    *err_ = Err(*value, "Value must be greater than 1.");
+    return false;
+  }
+
+  target_->set_jumbo_file_merge_limit(jumbo_file_merge_limit);
+  return true;
+}
+
 bool BinaryTargetGenerator::ValidateSources() {
   // For Rust targets, if the only source file is the root `sources` can be
   // omitted/empty.

src/gn/binary_target_generator.h

@@ -31,6 +31,9 @@ class BinaryTargetGenerator : public TargetGenerator {
   bool FillOutputPrefixOverride();
   bool FillOutputDir();
   bool FillAllowCircularIncludesFrom();
+  bool FillJumboAllowed();
+  bool FillJumboExcludedSources();
+  bool FillJumboFileMergeLimit();
   bool ValidateSources();
 
   Target::OutputType output_type_;

src/gn/command_gen.cc

@@ -51,6 +51,7 @@ const char kSwitchJsonFileName[] = "json-file-name";
 const char kSwitchJsonIdeScript[] = "json-ide-script";
 const char kSwitchJsonIdeScriptArgs[] = "json-ide-script-args";
 const char kSwitchExportCompileCommands[] = "export-compile-commands";
+const char kSwitchJumboStats[] = "jumbo-stats";
 
 // Collects Ninja rules for each toolchain. The lock protectes the rules.
 struct TargetWriteInfo {
@@ -431,6 +432,11 @@ Compilation Database
       all available targets will be used. This is used for various Clang-based
       tooling, allowing for the replay of individual compilations independent
       of the build system.
+
+Jumbo Build Mode
+
+  --jumbo-stats
+      Shows statistics about Jumbo usage in targets.
 )";
 
 int RunGen(const std::vector<std::string>& args) {
@@ -472,6 +478,10 @@ int RunGen(const std::vector<std::string>& args) {
   if (!setup->Run())
     return 1;
 
+  int jumbo_allowed_count = 0;
+  int jumbo_disallowed_count = 0;
+  std::set<const Target*> jumbo_not_configured_targets;
+
   // Sort the targets in each toolchain according to their label. This makes
   // the ninja files have deterministic content.
   for (auto& cur_toolchain : write_info.rules) {
@@ -480,6 +490,19 @@ int RunGen(const std::vector<std::string>& args) {
                  const NinjaWriter::TargetRulePair& b) {
                 return a.first->label() < b.first->label();
               });
+
+    if (command_line->HasSwitch(kSwitchJumboStats)) {
+      for (const NinjaWriter::TargetRulePair& rule : cur_toolchain.second) {
+        if (rule.first->is_jumbo_configured()) {
+          if (rule.first->is_jumbo_allowed())
+            ++jumbo_allowed_count;
+          else
+            ++jumbo_disallowed_count;
+        } else if (rule.first->IsBinary()) {
+          jumbo_not_configured_targets.insert(rule.first);
+        }
+      }
+    }
   }
 
   Err err;
@@ -515,6 +538,29 @@ int RunGen(const std::vector<std::string>& args) {
   TickDelta elapsed_time = timer.Elapsed();
 
   if (!command_line->HasSwitch(switches::kQuiet)) {
+    if (command_line->HasSwitch(kSwitchJumboStats)) {
+      std::vector<const Target*> not_configured(
+          jumbo_not_configured_targets.begin(),
+          jumbo_not_configured_targets.end());
+      std::sort(not_configured.begin(), not_configured.end(),
+                [](const Target* a, const Target* b) {
+                  return a->sources().size() < b->sources().size();
+                });
+      OutputString("Jumbo is not configured in following targets:\n");
+      for (const Target* target : not_configured) {
+        OutputString(target->label().GetUserVisibleName(false) +
+                         " (" + base::NumberToString(target->sources().size()) +
+                         " sources)\n");
+      }
+      OutputString("\nJumbo is not configured in " +
+                   base::NumberToString(not_configured.size()) + " targets.\n");
+      OutputString("Jumbo is allowed in " +
+                   base::NumberToString(jumbo_allowed_count) + " targets.\n");
+      OutputString("Jumbo is disallowed in " +
+                   base::NumberToString(jumbo_disallowed_count) +
+                   " targets.\n\n");
+    }
+
     OutputString("Done. ", DECORATION_GREEN);
 
     size_t targets_collected = 0;

src/gn/gn_main.cc

@@ -49,7 +49,8 @@ int main(int argc, char** argv) {
     command = commands::kHelp;
   } else if (cmdline.HasSwitch(switches::kVersion)) {
     // Make "--version" print the version and exit.
-    OutputString(std::string(LAST_COMMIT_POSITION) + "\n");
+    OutputString(std::string(LAST_COMMIT_POSITION) +
+                     " - modified by Opera Software AS\n");
     exit(0);
   } else if (args.empty()) {
     // No command, print error and exit.