Skip to content

Commit

Permalink
Add jumbo support
Browse files Browse the repository at this point in the history
This allows to generate Visual Studio projects that include source files
that are merged together into jumbo files.
  • Loading branch information
tmoniuszko-opera committed Apr 29, 2019
1 parent 64b846c commit cce6e37
Show file tree
Hide file tree
Showing 16 changed files with 921 additions and 47 deletions.
9 changes: 7 additions & 2 deletions build/gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,10 @@ def main(argv):

def GenerateLastCommitPosition(host, header):
ROOT_TAG = 'initial-commit'
# Use HEAD@{1} so original Chromium and Opera (patched) versions are the same.
describe_output = subprocess.check_output(
['git', 'describe', 'HEAD', '--match', ROOT_TAG], shell=host.is_windows(),
cwd=REPO_ROOT)
['git', 'describe', 'HEAD@{1}', '--match', ROOT_TAG],
shell=host.is_windows(), cwd=REPO_ROOT)
mo = re.match(ROOT_TAG + '-(\d+)-g([0-9a-f]+)', describe_output.decode())
if not mo:
raise ValueError(
Expand Down Expand Up @@ -475,6 +476,8 @@ def WriteGNNinja(path, platform, host, options):
'tools/gn/input_file_manager.cc',
'tools/gn/item.cc',
'tools/gn/json_project_writer.cc',
'tools/gn/jumbo_file_list_generator.cc',
'tools/gn/jumbo_writer.cc',
'tools/gn/label.cc',
'tools/gn/label_pattern.cc',
'tools/gn/lib_file.cc',
Expand Down Expand Up @@ -579,6 +582,8 @@ def WriteGNNinja(path, platform, host, options):
'tools/gn/header_checker_unittest.cc',
'tools/gn/inherited_libraries_unittest.cc',
'tools/gn/input_conversion_unittest.cc',
'tools/gn/jumbo_file_list_generator_unittest.cc',
'tools/gn/jumbo_writer_unittest.cc',
'tools/gn/label_pattern_unittest.cc',
'tools/gn/label_unittest.cc',
'tools/gn/loader_unittest.cc',
Expand Down
5 changes: 5 additions & 0 deletions tools/gn/args.cc
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
// declared. This is so they can be overridden in a toolchain build args
// override, and so that they will appear in the "gn args" output.
Value empty_string(nullptr, std::string());
Value false_bool(nullptr, false);

Value os_val(nullptr, std::string(os));
dest->SetValue(variables::kHostOs, os_val, nullptr);
Expand All @@ -355,6 +356,8 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
dest->SetValue(variables::kTargetCpu, empty_string, nullptr);
dest->SetValue(variables::kCurrentCpu, empty_string, nullptr);

dest->SetValue(variables::kEnableNativeJumbo, false_bool, nullptr);

Scope::KeyValueMap& declared_arguments(
DeclaredArgumentsForToolchainLocked(dest));
declared_arguments[variables::kHostOs] = os_val;
Expand All @@ -363,6 +366,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
declared_arguments[variables::kHostCpu] = arch_val;
declared_arguments[variables::kCurrentCpu] = empty_string;
declared_arguments[variables::kTargetCpu] = empty_string;
declared_arguments[variables::kEnableNativeJumbo] = false_bool;

// Mark these variables used so the build config file can override them
// without geting a warning about overwriting an unused variable.
Expand All @@ -372,6 +376,7 @@ void Args::SetSystemVarsLocked(Scope* dest) const {
dest->MarkUsed(variables::kHostOs);
dest->MarkUsed(variables::kCurrentOs);
dest->MarkUsed(variables::kTargetOs);
dest->MarkUsed(variables::kEnableNativeJumbo);
}

void Args::ApplyOverridesLocked(const Scope::KeyValueMap& values,
Expand Down
96 changes: 96 additions & 0 deletions tools/gn/binary_target_generator.cc
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@

#include "tools/gn/binary_target_generator.h"

#include <algorithm>

#include "tools/gn/config_values_generator.h"
#include "tools/gn/deps_iterator.h"
#include "tools/gn/err.h"
#include "tools/gn/filesystem_utils.h"
#include "tools/gn/functions.h"
#include "tools/gn/jumbo_file_list_generator.h"
#include "tools/gn/scope.h"
#include "tools/gn/settings.h"
#include "tools/gn/value_extractors.h"
Expand Down Expand Up @@ -66,6 +69,23 @@ void BinaryTargetGenerator::DoRun() {
gen.Run();
if (err_->has_error())
return;

if (!FillJumboAllowed())
return;

if (!FillJumboExcludedSources())
return;

if (!FillJumboFileMergeLimit())
return;

if (target_->is_jumbo_allowed()) {
JumboFileListGenerator jumbo_generator(target_, &target_->jumbo_files(),
err_);
jumbo_generator.Run();
if (err_->has_error())
return;
}
}

bool BinaryTargetGenerator::FillCompleteStaticLib() {
Expand Down Expand Up @@ -179,3 +199,79 @@ bool BinaryTargetGenerator::FillAllowCircularIncludesFrom() {
target_->allow_circular_includes_from().insert(cur);
return true;
}

bool BinaryTargetGenerator::FillJumboAllowed() {
const Value* value = scope_->GetValue(variables::kJumboAllowed, true);
if (!value)
return true;

if (!value->VerifyTypeIs(Value::BOOLEAN, err_))
return false;

target_->set_jumbo_allowed(value->boolean_value());
return true;
}

bool BinaryTargetGenerator::FillJumboExcludedSources() {
const Value* value = scope_->GetValue(variables::kJumboExcludedSources, true);
if (!value)
return true;

if (!target_->is_jumbo_allowed()) {
// TODO(tmoniuszko): We currently don't report error here because we want
// to support non-native jumbo implementation from BUILD.gn scripts.
// For native-only jumbo we would display such error:
// *err_ = Err(*value, "Jumbo is not allowed for this target.");
return false;
}

Target::FileList jumbo_excluded_sources;
if (!ExtractListOfRelativeFiles(scope_->settings()->build_settings(), *value,
scope_->GetSourceDir(),
&jumbo_excluded_sources, err_)) {
return false;
}

// Excluded files should be in sources. jumbo_excluded_sources is intended to
// exclude only small amount of files that cause compilation issues so
// searching for every file in loop should be acceptable despite of time
// complexity.
const Target::FileList& sources = target_->sources();
for (const SourceFile& file : jumbo_excluded_sources) {
if (std::find(sources.begin(), sources.end(), file) == sources.end()) {
*err_ = Err(*value, "Excluded file not in sources.",
"The file \"" + file.value() + "\" was not in \"sources\"." +
" " + sources.front().value());
return false;
}
}

target_->jumbo_excluded_sources().swap(jumbo_excluded_sources);
return true;
}

bool BinaryTargetGenerator::FillJumboFileMergeLimit() {
const Value* value = scope_->GetValue(variables::kJumboFileMergeLimit, true);
if (!value)
return true;

if (!target_->is_jumbo_allowed()) {
// TODO(tmoniuszko): We currently don't report error here because we want
// to support non-native jumbo implementation from BUILD.gn scripts.
// For native-only jumbo we would display such error:
// *err_ = Err(*value, "Jumbo is not allowed for this target.");
return false;
}

if (!value->VerifyTypeIs(Value::INTEGER, err_))
return false;

int jumbo_file_merge_limit = value->int_value();
if (jumbo_file_merge_limit < 2) {
*err_ = Err(*value, "Value must be greater than 1.");
return false;
}

target_->set_jumbo_file_merge_limit(jumbo_file_merge_limit);
return true;
}
3 changes: 3 additions & 0 deletions tools/gn/binary_target_generator.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ class BinaryTargetGenerator : public TargetGenerator {
bool FillOutputDir();
bool FillOutputExtension();
bool FillAllowCircularIncludesFrom();
bool FillJumboAllowed();
bool FillJumboExcludedSources();
bool FillJumboFileMergeLimit();

Target::OutputType output_type_;

Expand Down
2 changes: 1 addition & 1 deletion tools/gn/gn_main.cc
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ int main(int argc, char** argv) {
command = commands::kHelp;
} else if (cmdline.HasSwitch(switches::kVersion)) {
// Make "--version" print the version and exit.
OutputString(std::string(LAST_COMMIT_POSITION) + "\n");
OutputString(std::string(LAST_COMMIT_POSITION) + " - modified by Opera\n");
exit(0);
} else if (args.empty()) {
// No command, print error and exit.
Expand Down
139 changes: 139 additions & 0 deletions tools/gn/jumbo_file_list_generator.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "tools/gn/jumbo_file_list_generator.h"

#include <algorithm>
#include <string>
#include <vector>

#include "base/logging.h"
#include "base/numerics/safe_conversions.h"
#include "base/strings/string_piece.h"
#include "base/strings/stringprintf.h"
#include "tools/gn/filesystem_utils.h"

namespace {

// Constructs the file name for a jumbo file.
std::string GetJumboFileName(const std::string& target_name,
SourceFileType file_type,
int file_number) {
base::StringPiece extension;
switch (file_type) {
case SOURCE_C:
extension = "c";
break;
case SOURCE_CPP:
extension = "cc";
break;
case SOURCE_MM:
extension = "mm";
break;
default:
return std::string();
}

return base::StringPrintf("%s_jumbo_%s_%d.%s", target_name.c_str(),
extension.data(), file_number, extension.data());
}

} // namespace

JumboFileListGenerator::JumboFileListGenerator(
const Target* target,
Target::JumboFileList* jumbo_files,
Err* err)
: target_(target),
jumbo_files_dir_(
GetBuildDirForTargetAsSourceDir(target, BuildDirType::GEN)),
jumbo_files_(jumbo_files),
recent_jumbo_file_(nullptr),
recent_jumbo_file_type_(SOURCE_UNKNOWN),
err_(err) {}

JumboFileListGenerator::~JumboFileListGenerator() = default;

void JumboFileListGenerator::Run() {
const Target::FileList& excluded_sources = target_->jumbo_excluded_sources();

for (const SourceFile& input : target_->sources()) {
SourceFileType file_type = GetSourceFileType(input);
if (file_type != SOURCE_C && file_type != SOURCE_CPP &&
file_type != SOURCE_MM) {
continue;
}

if (std::find(excluded_sources.begin(), excluded_sources.end(), input) !=
excluded_sources.end()) {
continue;
}

Target::JumboSourceFile* jumbo_file = FindJumboFile(file_type);
if (!jumbo_file)
jumbo_file = CreateJumboFile(file_type);
if (!jumbo_file) {
if (err_->has_error())
return;
else
continue; // Source file type not supported by jumbo. Just skip it.
}

jumbo_file->second.push_back(&input);

recent_jumbo_file_ = jumbo_file;
recent_jumbo_file_type_ = file_type;
}
}

Target::JumboSourceFile* JumboFileListGenerator::FindJumboFile(
SourceFileType file_type) const {
// Return recently used file if suitable.
if (recent_jumbo_file_type_ == file_type) {
return base::checked_cast<int>(recent_jumbo_file_->second.size()) <
target_->jumbo_file_merge_limit()
? recent_jumbo_file_
: nullptr;
}

// Return immediately if we don't have any files for |file_type|.
if (jumbo_file_numbers_.count(file_type) == 0)
return nullptr;

// Search for file on |jumbo_files_| list.
for (auto it = jumbo_files_->rbegin(); it != jumbo_files_->rend(); ++it) {
if (GetSourceFileType(it->first) == file_type) {
return base::checked_cast<int>(it->second.size()) <
target_->jumbo_file_merge_limit()
? &(*it)
: nullptr;
}
}

NOTREACHED();
return nullptr;
}

Target::JumboSourceFile* JumboFileListGenerator::CreateJumboFile(
SourceFileType file_type) {
const auto it = jumbo_file_numbers_.find(file_type);
int file_number = it != jumbo_file_numbers_.end() ? it->second + 1 : 0;
jumbo_file_numbers_[file_type] = file_number;

std::string file_name =
GetJumboFileName(target_->label().name(), file_type, file_number);
if (file_name.empty())
return nullptr;

SourceFile source_file =
jumbo_files_dir_.ResolveRelativeFile(Value(nullptr, file_name), err_);
if (source_file.is_null())
return nullptr;

jumbo_files_->push_back(
Target::JumboSourceFile(source_file, std::vector<const SourceFile*>()));
Target::JumboSourceFile* jumbo_file = &jumbo_files_->back();
jumbo_file->second.reserve(target_->jumbo_file_merge_limit());
return jumbo_file;
}
Loading

0 comments on commit cce6e37

Please sign in to comment.