Add jumbo support

This allows to generate Visual Studio projects that include source files
that are merged together into jumbo files.

.github/workflows/main.yml

@@ -0,0 +1,82 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  build-ubuntu:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+        with: { fetch-depth: 0 }
+
+      - name: install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -qq ninja-build
+
+      # Runs a single command using the runners shell
+      - name: Gen project
+        run:  python build/gen.py
+
+      # Runs a set of commands using the runners shell
+      - name: Build
+        run: ninja -C out gn
+
+      - name: Print gn version
+        run: ./out/gn --version
+
+      - name: Rename gn to gn_lin
+        run: mv out/gn out/gn_lin
+
+      - name: Upload gn_lin
+        uses: actions/[email protected]
+        with:
+          path: out/gn_lin
+          # The desired behavior if no files are found using the provided path.
+
+  build-mac:
+    # The type of runner that the job will run on
+    runs-on: macOS-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+        with: { fetch-depth: 0 }
+
+      - name: install dependencies
+        run: brew install ninja
+
+      # Runs a single command using the runners shell
+      - name: Gen project
+        run:  python build/gen.py
+
+      # Runs a set of commands using the runners shell
+      - name: Build
+        run: ninja -C out gn
+
+      - name: Print gn version
+        run: ./out/gn --version
+
+      - name: Rename gn to gn_mac
+        run: mv out/gn out/gn_mac
+
+      - name: Upload gn_mac
+        uses: actions/[email protected]
+        with:
+          path: out/gn_mac
+          # The desired behavior if no files are found using the provided path.
+

build/gen.py

@@ -166,9 +166,10 @@ def main(argv):
 
 def GenerateLastCommitPosition(host, header):
   ROOT_TAG = 'initial-commit'
+  # Use HEAD@{1} so original Chromium and Opera (patched) versions are the same.
   describe_output = subprocess.check_output(
-      ['git', 'describe', 'HEAD', '--match', ROOT_TAG], shell=host.is_windows(),
-      cwd=REPO_ROOT)
+      ['git', 'describe', 'HEAD~', '--match', ROOT_TAG],
+      shell=host.is_windows(), cwd=REPO_ROOT)
   mo = re.match(ROOT_TAG + '-(\d+)-g([0-9a-f]+)', describe_output.decode())
   if not mo:
     raise ValueError(
@@ -591,6 +592,8 @@ def WriteGNNinja(path, platform, host, options):
         'src/gn/input_file_manager.cc',
         'src/gn/item.cc',
         'src/gn/json_project_writer.cc',
+        'src/gn/jumbo_file_list_generator.cc',
+        'src/gn/jumbo_writer.cc',
         'src/gn/label.cc',
         'src/gn/label_pattern.cc',
         'src/gn/lib_file.cc',
@@ -714,6 +717,8 @@ def WriteGNNinja(path, platform, host, options):
         'src/gn/inherited_libraries_unittest.cc',
         'src/gn/input_conversion_unittest.cc',
         'src/gn/json_project_writer_unittest.cc',
+        'src/gn/jumbo_file_list_generator_unittest.cc',
+        'src/gn/jumbo_writer_unittest.cc',
         'src/gn/rust_project_writer_unittest.cc',
         'src/gn/rust_project_writer_helpers_unittest.cc',
         'src/gn/label_pattern_unittest.cc',

src/gn/args.cc

@@ -377,6 +377,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
   // declared. This is so they can be overridden in a toolchain build args
   // override, and so that they will appear in the "gn args" output.
   Value empty_string(nullptr, std::string());
+  Value false_bool(nullptr, false);
 
   Value os_val(nullptr, std::string(os));
   dest->SetValue(variables::kHostOs, os_val, nullptr);
@@ -388,6 +389,8 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
   dest->SetValue(variables::kTargetCpu, empty_string, nullptr);
   dest->SetValue(variables::kCurrentCpu, empty_string, nullptr);
 
+  dest->SetValue(variables::kEnableNativeJumbo, false_bool, nullptr);
+
   Scope::KeyValueMap& declared_arguments(
       DeclaredArgumentsForToolchainLocked(dest));
   declared_arguments[variables::kHostOs] = os_val;
@@ -396,6 +399,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
   declared_arguments[variables::kHostCpu] = arch_val;
   declared_arguments[variables::kCurrentCpu] = empty_string;
   declared_arguments[variables::kTargetCpu] = empty_string;
+  declared_arguments[variables::kEnableNativeJumbo] = false_bool;
 
   // Mark these variables used so the build config file can override them
   // without getting a warning about overwriting an unused variable.
@@ -405,6 +409,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
   dest->MarkUsed(variables::kHostOs);
   dest->MarkUsed(variables::kCurrentOs);
   dest->MarkUsed(variables::kTargetOs);
+  dest->MarkUsed(variables::kEnableNativeJumbo);
 }
 
 void Args::ApplyOverridesLocked(const Scope::KeyValueMap& values,

src/gn/binary_target_generator.cc

@@ -4,11 +4,14 @@
 
 #include "gn/binary_target_generator.h"
 
+#include <algorithm>
+
 #include "gn/config_values_generator.h"
 #include "gn/deps_iterator.h"
 #include "gn/err.h"
 #include "gn/filesystem_utils.h"
 #include "gn/functions.h"
+#include "gn/jumbo_file_list_generator.h"
 #include "gn/parse_tree.h"
 #include "gn/rust_values_generator.h"
 #include "gn/rust_variables.h"
@@ -87,6 +90,23 @@ void BinaryTargetGenerator::DoRun() {
   gen.Run();
   if (err_->has_error())
     return;
+
+  if (!FillJumboAllowed())
+    return;
+
+  if (!FillJumboExcludedSources())
+    return;
+
+  if (!FillJumboFileMergeLimit())
+    return;
+
+  if (target_->is_jumbo_allowed()) {
+    JumboFileListGenerator jumbo_generator(target_, &target_->jumbo_files(),
+                                           err_);
+    jumbo_generator.Run();
+    if (err_->has_error())
+      return;
+  }
 }
 
 bool BinaryTargetGenerator::FillSources() {
@@ -231,6 +251,82 @@ bool BinaryTargetGenerator::FillAllowCircularIncludesFrom() {
   return true;
 }
 
+bool BinaryTargetGenerator::FillJumboAllowed() {
+  const Value* value = scope_->GetValue(variables::kJumboAllowed, true);
+  if (!value)
+    return true;
+
+  if (!value->VerifyTypeIs(Value::BOOLEAN, err_))
+    return false;
+
+  target_->set_jumbo_allowed(value->boolean_value());
+  return true;
+}
+
+bool BinaryTargetGenerator::FillJumboExcludedSources() {
+  const Value* value = scope_->GetValue(variables::kJumboExcludedSources, true);
+  if (!value)
+    return true;
+
+  if (!target_->is_jumbo_allowed()) {
+    // TODO(tmoniuszko): We currently don't report error here because we want
+    // to support non-native jumbo implementation from BUILD.gn scripts.
+    // For native-only jumbo we would display such error:
+    // *err_ = Err(*value, "Jumbo is not allowed for this target.");
+    return false;
+  }
+
+  Target::FileList jumbo_excluded_sources;
+  if (!ExtractListOfRelativeFiles(scope_->settings()->build_settings(), *value,
+                                  scope_->GetSourceDir(),
+                                  &jumbo_excluded_sources, err_)) {
+    return false;
+  }
+
+  // Excluded files should be in sources. jumbo_excluded_sources is intended to
+  // exclude only small amount of files that cause compilation issues so
+  // searching for every file in loop should be acceptable despite of time
+  // complexity.
+  const Target::FileList& sources = target_->sources();
+  for (const SourceFile& file : jumbo_excluded_sources) {
+    if (std::find(sources.begin(), sources.end(), file) == sources.end()) {
+      *err_ = Err(*value, "Excluded file not in sources.",
+                  "The file \"" + file.value() + "\" was not in \"sources\"." +
+                      "  " + sources.front().value());
+      return false;
+    }
+  }
+
+  target_->jumbo_excluded_sources().swap(jumbo_excluded_sources);
+  return true;
+}
+
+bool BinaryTargetGenerator::FillJumboFileMergeLimit() {
+  const Value* value = scope_->GetValue(variables::kJumboFileMergeLimit, true);
+  if (!value)
+    return true;
+
+  if (!target_->is_jumbo_allowed()) {
+    // TODO(tmoniuszko): We currently don't report error here because we want
+    // to support non-native jumbo implementation from BUILD.gn scripts.
+    // For native-only jumbo we would display such error:
+    // *err_ = Err(*value, "Jumbo is not allowed for this target.");
+    return false;
+  }
+
+  if (!value->VerifyTypeIs(Value::INTEGER, err_))
+    return false;
+
+  int jumbo_file_merge_limit = value->int_value();
+  if (jumbo_file_merge_limit < 2) {
+    *err_ = Err(*value, "Value must be greater than 1.");
+    return false;
+  }
+
+  target_->set_jumbo_file_merge_limit(jumbo_file_merge_limit);
+  return true;
+}
+
 bool BinaryTargetGenerator::ValidateSources() {
   // For Rust targets, if the only source file is the root `sources` can be
   // omitted/empty.

src/gn/binary_target_generator.h

@@ -30,6 +30,9 @@ class BinaryTargetGenerator : public TargetGenerator {
   bool FillOutputPrefixOverride();
   bool FillOutputDir();
   bool FillAllowCircularIncludesFrom();
+  bool FillJumboAllowed();
+  bool FillJumboExcludedSources();
+  bool FillJumboFileMergeLimit();
   bool ValidateSources();
 
   Target::OutputType output_type_;

src/gn/command_gen.cc

@@ -60,6 +60,7 @@ const char kSwitchJsonIdeScript[] = "json-ide-script";
 const char kSwitchJsonIdeScriptArgs[] = "json-ide-script-args";
 const char kSwitchExportCompileCommands[] = "export-compile-commands";
 const char kSwitchExportRustProject[] = "export-rust-project";
+const char kSwitchJumboStats[] = "jumbo-stats";
 
 // Collects Ninja rules for each toolchain. The lock protectes the rules.
 struct TargetWriteInfo {
@@ -583,6 +584,16 @@ Compilation Database
       - "//foo:foo"
       and not match:
       - "//foo:bar"
+      of target_name is supplied, only targets that are reachable from the list
+      of target_name will be used for “command objects” generation, otherwise
+      all available targets will be used. This is used for various Clang-based
+      tooling, allowing for the replay of individual compilations independent
+      of the build system.
+
+Jumbo Build Mode
+
+  --jumbo-stats
+      Shows statistics about Jumbo usage in targets.
 )";
 
 int RunGen(const std::vector<std::string>& args) {
@@ -624,6 +635,10 @@ int RunGen(const std::vector<std::string>& args) {
   if (!setup->Run())
     return 1;
 
+  int jumbo_allowed_count = 0;
+  int jumbo_disallowed_count = 0;
+  std::set<const Target*> jumbo_not_configured_targets;
+
   // Sort the targets in each toolchain according to their label. This makes
   // the ninja files have deterministic content.
   for (auto& cur_toolchain : write_info.rules) {
@@ -632,6 +647,19 @@ int RunGen(const std::vector<std::string>& args) {
                  const NinjaWriter::TargetRulePair& b) {
                 return a.first->label() < b.first->label();
               });
+
+    if (command_line->HasSwitch(kSwitchJumboStats)) {
+      for (const NinjaWriter::TargetRulePair& rule : cur_toolchain.second) {
+        if (rule.first->is_jumbo_configured()) {
+          if (rule.first->is_jumbo_allowed())
+            ++jumbo_allowed_count;
+          else
+            ++jumbo_disallowed_count;
+        } else if (rule.first->IsBinary()) {
+          jumbo_not_configured_targets.insert(rule.first);
+        }
+      }
+    }
   }
 
   Err err;
@@ -683,6 +711,34 @@ int RunGen(const std::vector<std::string>& args) {
   TickDelta elapsed_time = timer.Elapsed();
 
   if (!command_line->HasSwitch(switches::kQuiet)) {
+    if (command_line->HasSwitch(kSwitchJumboStats)) {
+      std::vector<const Target*> not_configured(
+          jumbo_not_configured_targets.begin(),
+          jumbo_not_configured_targets.end());
+      std::sort(not_configured.begin(), not_configured.end(),
+                [](const Target* a, const Target* b) {
+                  if (a->sources().size() > b->sources().size())
+                    return true;
+                  if (a->sources().size() < b->sources().size())
+                    return false;
+                  return a->label().GetUserVisibleName(false) <
+                      b->label().GetUserVisibleName(false);
+                });
+      OutputString("Jumbo is not configured in following targets:\n");
+      for (const Target* target : not_configured) {
+        OutputString(target->label().GetUserVisibleName(false) +
+                         " (" + base::NumberToString(target->sources().size()) +
+                         " sources)\n");
+      }
+      OutputString("\nJumbo is not configured in " +
+                   base::NumberToString(not_configured.size()) + " targets.\n");
+      OutputString("Jumbo is allowed in " +
+                   base::NumberToString(jumbo_allowed_count) + " targets.\n");
+      OutputString("Jumbo is disallowed in " +
+                   base::NumberToString(jumbo_disallowed_count) +
+                   " targets.\n\n");
+    }
+
     OutputString("Done. ", DECORATION_GREEN);
 
     size_t targets_collected = 0;

src/gn/gn_main.cc

@@ -49,7 +49,7 @@ int main(int argc, char** argv) {
     command = commands::kHelp;
   } else if (cmdline.HasSwitch(switches::kVersion)) {
     // Make "--version" print the version and exit.
-    OutputString(std::string(LAST_COMMIT_POSITION) + "\n");
+    OutputString(std::string(LAST_COMMIT_POSITION) + " - modified by Opera\n");
     exit(0);
   } else if (args.empty()) {
     // No command, print error and exit.