Skip to content

Commit

Permalink
Revert^2 "Refactor oat file writing."
Browse files Browse the repository at this point in the history 
This reverts commit 50c812a.

Fix the VerificationResults to be created before Runtime.
This is needed for recording verification data during early
Runtime initialization when compiling the boot image.
This also cleans up a TODO for bug 29790079.

Test: m test-art-host-gtest
Test: testrunner.py --host --optimizing
Test: Manual, prebuilt files do not change with this CL.
Bug: 142680736
Bug: 29790079
Change-Id: I4350e7a67a16ee1653a8f2fce6244e353a3597c4
  • Loading branch information
@vmarko
vmarko committed Oct 16, 2019
1 parent 765b87d commit 0866ea4
Show file tree
Hide file tree
Showing 5 changed files with 149 additions and 138 deletions.
180 changes: 90 additions & 90 deletions dex2oat/dex2oat.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1448,6 +1448,79 @@ class Dex2Oat final {
return dex2oat::ReturnCode::kOther;
}

{
TimingLogger::ScopedTiming t_dex("Writing and opening dex files", timings_);
for (size_t i = 0, size = oat_writers_.size(); i != size; ++i) {
// Unzip or copy dex files straight to the oat file.
std::vector<MemMap> opened_dex_files_map;
std::vector<std::unique_ptr<const DexFile>> opened_dex_files;
// No need to verify the dex file when we have a vdex file, which means it was already
// verified.
const bool verify = (input_vdex_file_ == nullptr);
if (!oat_writers_[i]->WriteAndOpenDexFiles(
vdex_files_[i].get(),
verify,
update_input_vdex_,
copy_dex_files_,
&opened_dex_files_map,
&opened_dex_files)) {
return dex2oat::ReturnCode::kOther;
}
dex_files_per_oat_file_.push_back(MakeNonOwningPointerVector(opened_dex_files));
if (opened_dex_files_map.empty()) {
DCHECK(opened_dex_files.empty());
} else {
for (MemMap& map : opened_dex_files_map) {
opened_dex_files_maps_.push_back(std::move(map));
}
for (std::unique_ptr<const DexFile>& dex_file : opened_dex_files) {
dex_file_oat_index_map_.emplace(dex_file.get(), i);
opened_dex_files_.push_back(std::move(dex_file));
}
}
}
}

compiler_options_->dex_files_for_oat_file_ = MakeNonOwningPointerVector(opened_dex_files_);
const std::vector<const DexFile*>& dex_files = compiler_options_->dex_files_for_oat_file_;

// Check if we need to downgrade the compiler-filter for size reasons.
// Note: This does not affect the compiler filter already stored in the key-value
// store which is used for determining whether the oat file is up to date,
// together with the boot class path locations and checksums stored below.
CompilerFilter::Filter original_compiler_filter = compiler_options_->GetCompilerFilter();
if (!IsBootImage() && IsVeryLarge(dex_files)) {
// Disable app image to make sure dex2oat unloading is enabled.
compiler_options_->image_type_ = CompilerOptions::ImageType::kNone;

// If we need to downgrade the compiler-filter for size reasons, do that early before we read
// it below for creating verification callbacks.
if (!CompilerFilter::IsAsGoodAs(kLargeAppFilter, compiler_options_->GetCompilerFilter())) {
LOG(INFO) << "Very large app, downgrading to verify.";
compiler_options_->SetCompilerFilter(kLargeAppFilter);
}
}

if (CompilerFilter::IsAnyCompilationEnabled(compiler_options_->GetCompilerFilter())) {
// Only modes with compilation require verification results, do this here instead of when we
// create the compilation callbacks since the compilation mode may have been changed by the
// very large app logic.
// Avoiding setting the verification results saves RAM by not adding the dex files later in
// the function.
// Note: When compiling boot image, this must be done before creating the Runtime.
verification_results_.reset(new VerificationResults(compiler_options_.get()));
callbacks_->SetVerificationResults(verification_results_.get());
}

if (IsBootImage()) {
// For boot image, pass opened dex files to the Runtime::Create().
// Note: Runtime acquires ownership of these dex files.
runtime_options.Set(RuntimeArgumentMap::BootClassPathDexList, &opened_dex_files_);
}
if (!CreateRuntime(std::move(runtime_options))) {
return dex2oat::ReturnCode::kCreateRuntime;
}

if (!compilation_reason_.empty()) {
key_value_store_->Put(OatHeader::kCompilationReasonKey, compilation_reason_);
}
Expand All @@ -1459,13 +1532,7 @@ class Dex2Oat final {
}

if (!IsBootImage()) {
// When compiling an app, create the runtime early to retrieve
// the boot image checksums needed for the oat header.
if (!CreateRuntime(std::move(runtime_options))) {
return dex2oat::ReturnCode::kCreateRuntime;
}

if (CompilerFilter::DependsOnImageChecksum(compiler_options_->GetCompilerFilter())) {
if (CompilerFilter::DependsOnImageChecksum(original_compiler_filter)) {
TimingLogger::ScopedTiming t3("Loading image checksum", timings_);
Runtime* runtime = Runtime::Current();
key_value_store_->Put(OatHeader::kBootClassPathKey,
Expand Down Expand Up @@ -1518,74 +1585,6 @@ class Dex2Oat final {
key_value_store_->Put(OatHeader::kClassPathKey, class_path_key);
}

// Now that we have finalized key_value_store_, start writing the oat file.
{
TimingLogger::ScopedTiming t_dex("Writing and opening dex files", timings_);
rodata_.reserve(oat_writers_.size());
for (size_t i = 0, size = oat_writers_.size(); i != size; ++i) {
rodata_.push_back(elf_writers_[i]->StartRoData());
// Unzip or copy dex files straight to the oat file.
std::vector<MemMap> opened_dex_files_map;
std::vector<std::unique_ptr<const DexFile>> opened_dex_files;
// No need to verify the dex file when we have a vdex file, which means it was already
// verified.
const bool verify = (input_vdex_file_ == nullptr);
if (!oat_writers_[i]->WriteAndOpenDexFiles(
vdex_files_[i].get(),
rodata_.back(),
(i == 0u) ? key_value_store_.get() : nullptr,
verify,
update_input_vdex_,
copy_dex_files_,
&opened_dex_files_map,
&opened_dex_files)) {
return dex2oat::ReturnCode::kOther;
}
dex_files_per_oat_file_.push_back(MakeNonOwningPointerVector(opened_dex_files));
if (opened_dex_files_map.empty()) {
DCHECK(opened_dex_files.empty());
} else {
for (MemMap& map : opened_dex_files_map) {
opened_dex_files_maps_.push_back(std::move(map));
}
for (std::unique_ptr<const DexFile>& dex_file : opened_dex_files) {
dex_file_oat_index_map_.emplace(dex_file.get(), i);
opened_dex_files_.push_back(std::move(dex_file));
}
}
}
}

compiler_options_->dex_files_for_oat_file_ = MakeNonOwningPointerVector(opened_dex_files_);
const std::vector<const DexFile*>& dex_files = compiler_options_->dex_files_for_oat_file_;

// If we need to downgrade the compiler-filter for size reasons.
if (!IsBootImage() && IsVeryLarge(dex_files)) {
// Disable app image to make sure dex2oat unloading is enabled.
compiler_options_->image_type_ = CompilerOptions::ImageType::kNone;

// If we need to downgrade the compiler-filter for size reasons, do that early before we read
// it below for creating verification callbacks.
if (!CompilerFilter::IsAsGoodAs(kLargeAppFilter, compiler_options_->GetCompilerFilter())) {
LOG(INFO) << "Very large app, downgrading to verify.";
// Note: this change won't be reflected in the key-value store, as that had to be
// finalized before loading the dex files. This setup is currently required
// to get the size from the DexFile objects.
// TODO: refactor. b/29790079
compiler_options_->SetCompilerFilter(kLargeAppFilter);
}
}

if (CompilerFilter::IsAnyCompilationEnabled(compiler_options_->GetCompilerFilter())) {
// Only modes with compilation require verification results, do this here instead of when we
// create the compilation callbacks since the compilation mode may have been changed by the
// very large app logic.
// Avoiding setting the verification results saves RAM by not adding the dex files later in
// the function.
verification_results_.reset(new VerificationResults(compiler_options_.get()));
callbacks_->SetVerificationResults(verification_results_.get());
}

// We had to postpone the swap decision till now, as this is the point when we actually
// know about the dex files we're going to use.

Expand All @@ -1602,14 +1601,6 @@ class Dex2Oat final {
}
}
// Note that dex2oat won't close the swap_fd_. The compiler driver's swap space will do that.
if (IsBootImage()) {
// For boot image, pass opened dex files to the Runtime::Create().
// Note: Runtime acquires ownership of these dex files.
runtime_options.Set(RuntimeArgumentMap::BootClassPathDexList, &opened_dex_files_);
if (!CreateRuntime(std::move(runtime_options))) {
return dex2oat::ReturnCode::kOther;
}
}

// If we're doing the image, override the compiler filter to force full compilation. Must be
// done ahead of WellKnownClasses::Init that causes verification. Note: doesn't force
Expand Down Expand Up @@ -1934,12 +1925,21 @@ class Dex2Oat final {
}
}

// Initialize the writers with the compiler driver, image writer, and their
// dex files. The writers were created without those being there yet.
for (size_t i = 0, size = oat_files_.size(); i != size; ++i) {
std::unique_ptr<linker::OatWriter>& oat_writer = oat_writers_[i];
std::vector<const DexFile*>& dex_files = dex_files_per_oat_file_[i];
oat_writer->Initialize(driver_.get(), image_writer_.get(), dex_files);
// Initialize the writers with the compiler driver, image writer and their dex files
// and start writing the .rodata sections.
{
TimingLogger::ScopedTiming t2("dex2oat Starting oat .rodata section(s)", timings_);
rodata_.reserve(oat_writers_.size());
for (size_t i = 0, size = oat_files_.size(); i != size; ++i) {
rodata_.push_back(elf_writers_[i]->StartRoData());
if (!oat_writers_[i]->StartRoData(driver_.get(),
image_writer_.get(),
dex_files_per_oat_file_[i],
rodata_.back(),
(i == 0u) ? key_value_store_.get() : nullptr)) {
return false;
}
}
}

{
Expand Down Expand Up @@ -2008,7 +2008,7 @@ class Dex2Oat final {
debug::DebugInfo debug_info = oat_writer->GetDebugInfo(); // Keep the variable alive.
elf_writer->PrepareDebugInfo(debug_info); // Processes the data on background thread.

OutputStream*& rodata = rodata_[i];
OutputStream* rodata = rodata_[i];
DCHECK(rodata != nullptr);
if (!oat_writer->WriteRodata(rodata)) {
LOG(ERROR) << "Failed to write .rodata section to the ELF file " << oat_file->GetPath();
Expand Down
9 changes: 6 additions & 3 deletions dex2oat/linker/image_test.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,8 +257,6 @@ inline void ImageTest::DoCompile(ImageHeader::StorageMode storage_mode,
std::vector<std::unique_ptr<const DexFile>> cur_opened_dex_files;
bool dex_files_ok = oat_writers[i]->WriteAndOpenDexFiles(
out_helper.vdex_files[i].GetFile(),
rodata.back(),
(i == 0u) ? &key_value_store : nullptr,
/* verify */ false, // Dex files may be dex-to-dex-ed, don't verify.
/* update_input_vdex */ false,
/* copy_dex_files */ CopyOption::kOnlyIfCompressed,
Expand Down Expand Up @@ -289,7 +287,12 @@ inline void ImageTest::DoCompile(ImageHeader::StorageMode storage_mode,
OatWriter* const oat_writer = oat_writers[i].get();
ElfWriter* const elf_writer = elf_writers[i].get();
std::vector<const DexFile*> cur_dex_files(1u, class_path[i]);
oat_writer->Initialize(driver, writer.get(), cur_dex_files);
bool initialize_ok = oat_writer->StartRoData(driver,
writer.get(),
cur_dex_files,
rodata[i],
(i == 0u) ? &key_value_store : nullptr);
ASSERT_TRUE(initialize_ok);

std::unique_ptr<BufferedOutputStream> vdex_out =
std::make_unique<BufferedOutputStream>(
Expand Down
67 changes: 35 additions & 32 deletions dex2oat/linker/oat_writer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -658,32 +658,16 @@ bool OatWriter::MayHaveCompiledMethods() const {

bool OatWriter::WriteAndOpenDexFiles(
File* vdex_file,
OutputStream* oat_rodata,
SafeMap<std::string, std::string>* key_value_store,
bool verify,
bool update_input_vdex,
CopyOption copy_dex_files,
/*out*/ std::vector<MemMap>* opened_dex_files_map,
/*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files) {
CHECK(write_state_ == WriteState::kAddingDexFileSources);

// Record the ELF rodata section offset, i.e. the beginning of the OAT data.
if (!RecordOatDataOffset(oat_rodata)) {
return false;
}

// Record whether this is the primary oat file.
primary_oat_file_ = (key_value_store != nullptr);

// Initialize VDEX and OAT headers.

// Reserve space for Vdex header and checksums.
vdex_size_ = sizeof(VdexFile::VerifierDepsHeader) +
oat_dex_files_.size() * sizeof(VdexFile::VdexChecksum);
oat_size_ = InitOatHeader(dchecked_integral_cast<uint32_t>(oat_dex_files_.size()),
key_value_store);

ChecksumUpdatingOutputStream checksum_updating_rodata(oat_rodata, this);

std::unique_ptr<BufferedOutputStream> vdex_out =
std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(vdex_file));
Expand All @@ -695,6 +679,37 @@ bool OatWriter::WriteAndOpenDexFiles(
return false;
}

*opened_dex_files_map = std::move(dex_files_map);
*opened_dex_files = std::move(dex_files);
write_state_ = WriteState::kStartRoData;
return true;
}

// Initialize the writer with the given parameters.
bool OatWriter::StartRoData(const CompilerDriver* compiler_driver,
ImageWriter* image_writer,
const std::vector<const DexFile*>& dex_files,
OutputStream* oat_rodata,
SafeMap<std::string, std::string>* key_value_store) {
CHECK(write_state_ == WriteState::kStartRoData);
compiler_driver_ = compiler_driver;
image_writer_ = image_writer;
dex_files_ = &dex_files;

// Record the ELF rodata section offset, i.e. the beginning of the OAT data.
if (!RecordOatDataOffset(oat_rodata)) {
return false;
}

// Record whether this is the primary oat file.
primary_oat_file_ = (key_value_store != nullptr);

// Initialize OAT header.
oat_size_ = InitOatHeader(dchecked_integral_cast<uint32_t>(oat_dex_files_.size()),
key_value_store);

ChecksumUpdatingOutputStream checksum_updating_rodata(oat_rodata, this);

// Write type lookup tables into the oat file.
if (!WriteTypeLookupTables(&checksum_updating_rodata, dex_files)) {
return false;
Expand All @@ -705,21 +720,10 @@ bool OatWriter::WriteAndOpenDexFiles(
return false;
}

*opened_dex_files_map = std::move(dex_files_map);
*opened_dex_files = std::move(dex_files);
write_state_ = WriteState::kPrepareLayout;
return true;
}

// Initialize the writer with the given parameters.
void OatWriter::Initialize(const CompilerDriver* compiler_driver,
ImageWriter* image_writer,
const std::vector<const DexFile*>& dex_files) {
compiler_driver_ = compiler_driver;
image_writer_ = image_writer;
dex_files_ = &dex_files;
}

void OatWriter::PrepareLayout(MultiOatRelativePatcher* relative_patcher) {
CHECK(write_state_ == WriteState::kPrepareLayout);

Expand Down Expand Up @@ -3768,9 +3772,8 @@ bool OatWriter::OpenDexFiles(
return true;
}

bool OatWriter::WriteTypeLookupTables(
OutputStream* oat_rodata,
const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files) {
bool OatWriter::WriteTypeLookupTables(OutputStream* oat_rodata,
const std::vector<const DexFile*>& opened_dex_files) {
TimingLogger::ScopedTiming split("WriteTypeLookupTables", timings_);

uint32_t expected_offset = oat_data_offset_ + oat_size_;
Expand Down Expand Up @@ -3852,11 +3855,11 @@ bool OatWriter::WriteTypeLookupTables(

bool OatWriter::WriteDexLayoutSections(
OutputStream* oat_rodata,
const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files) {
const std::vector<const DexFile*>& opened_dex_files) {
TimingLogger::ScopedTiming split(__FUNCTION__, timings_);

if (!kWriteDexLayoutInfo) {
return true;;
return true;
}

uint32_t expected_offset = oat_data_offset_ + oat_size_;
Expand Down
Loading

0 comments on commit 0866ea4

Please sign in to comment.