[Courgette] Using LabelManager to reduce Courgette-apply peak RAM by …

…25%.

AssemblyProgram previously allocates new Label instances as it parses
an executable and emits instructions. This CL replaces the flow by using
LabelManager to precompute Labels in one array. This allows us to reduce
Courgette-apply peak RAM by 25%, measured by "choke RAM until failure"
method. Details:
- We precompute Labels in AssemblyProgram::PrecomputeLabels(), which
  relies on RvaVisitor inherited classes for architecture-specific
  extraction of abs32 and rel32 targets.
- TrimLabel()'s complex post-processing flow is simplified using
  PrecomputeLabels(), which runs before main file parse.
  - This requires RemoveUnusedRel32Locations() to update rel32.
  - Deprecating C_TRIM_FAILED error message.
- Moving more common functionality to Disassembler, but duplicating
  some code for win32-x86 and win32-x64 to follow existing pattern.

BUG=613216

Review-Url: https://codereview.chromium.org/1935203002
Cr-Commit-Position: refs/heads/master@{#394815}

courgette/adjustment_method_unittest.cc

@@ -5,11 +5,13 @@
 #include <memory>
 #include <string>
 #include <utility>
+#include <vector>
 
 #include "base/strings/string_util.h"
 #include "courgette/assembly_program.h"
 #include "courgette/courgette.h"
 #include "courgette/encoded_program.h"
+#include "courgette/image_utils.h"
 #include "courgette/streams.h"
 
 #include "testing/gtest/include/gtest/gtest.h"
@@ -32,8 +34,20 @@ class AdjustmentMethodTest : public testing::Test {
         new courgette::AssemblyProgram(courgette::EXE_WIN_32_X86));
     prog->set_image_base(0x00400000);
 
-    courgette::Label* labelA = prog->FindOrMakeAbs32Label(0x00410000);
-    courgette::Label* labelB = prog->FindOrMakeAbs32Label(0x00410004);
+    courgette::RVA kRvaA = 0x00410000;
+    courgette::RVA kRvaB = 0x00410004;
+
+    std::vector<courgette::RVA> abs32_rvas;
+    abs32_rvas.push_back(kRvaA);
+    abs32_rvas.push_back(kRvaB);
+    std::vector<courgette::RVA> rel32_rvas;  // Stub.
+
+    courgette::TrivialRvaVisitor abs32_visitor(abs32_rvas);
+    courgette::TrivialRvaVisitor rel32_visitor(rel32_rvas);
+    prog->PrecomputeLabels(&abs32_visitor, &rel32_visitor);
+
+    courgette::Label* labelA = prog->FindAbs32Label(kRvaA);
+    courgette::Label* labelB = prog->FindAbs32Label(kRvaB);
 
     EXPECT_TRUE(prog->EmitAbs32(labelA));
     EXPECT_TRUE(prog->EmitAbs32(labelA));
@@ -88,7 +102,6 @@ class AdjustmentMethodTest : public testing::Test {
   }
 };
 
-
 void AdjustmentMethodTest::Test1() const {
   std::unique_ptr<courgette::AssemblyProgram> prog1 = MakeProgramA();
   std::unique_ptr<courgette::AssemblyProgram> prog2 = MakeProgramB();
@@ -109,7 +122,6 @@ void AdjustmentMethodTest::Test1() const {
   EXPECT_TRUE(s5 == s6);  // Adjustment did change B into A
 }
 
-
 TEST_F(AdjustmentMethodTest, All) {
   Test1();
 }

courgette/assembly_program.cc

@@ -111,11 +111,6 @@ AssemblyProgram::AssemblyProgram(ExecutableType kind)
   : kind_(kind), image_base_(0) {
 }
 
-static void DeleteContainedLabels(const RVAToLabel& labels) {
-  for (RVAToLabel::const_iterator p = labels.begin();  p != labels.end();  ++p)
-    UncheckedDelete(p->second);
-}
-
 AssemblyProgram::~AssemblyProgram() {
   for (size_t i = 0;  i < instructions_.size();  ++i) {
     Instruction* instruction = instructions_[i];
@@ -126,8 +121,6 @@ AssemblyProgram::~AssemblyProgram() {
     for (size_t i = 0;  i < 256;  ++i)
       UncheckedDelete(byte_instruction_cache_[i]);
   }
-  DeleteContainedLabels(rel32_labels_);
-  DeleteContainedLabels(abs32_labels_);
 }
 
 CheckBool AssemblyProgram::EmitPeRelocsInstruction() {
@@ -178,27 +171,50 @@ CheckBool AssemblyProgram::EmitAbs64(Label* label) {
       ScopedInstruction(UncheckedNew<InstructionWithLabel>(ABS64, label)));
 }
 
-Label* AssemblyProgram::FindOrMakeAbs32Label(RVA rva) {
-  return FindLabel(rva, &abs32_labels_);
+void AssemblyProgram::PrecomputeLabels(RvaVisitor* abs32_visitor,
+                                       RvaVisitor* rel32_visitor) {
+  abs32_label_manager_.Read(abs32_visitor);
+  rel32_label_manager_.Read(rel32_visitor);
+  TrimLabels();
 }
 
-Label* AssemblyProgram::FindOrMakeRel32Label(RVA rva) {
-  return FindLabel(rva, &rel32_labels_);
-}
+// Chosen empirically to give the best reduction in payload size for
+// an update from daisy_3701.98.0 to daisy_4206.0.0.
+const int AssemblyProgram::kLabelLowerLimit = 5;
 
-void AssemblyProgram::DefaultAssignIndexes() {
-  DefaultAssignIndexes(&abs32_labels_);
-  DefaultAssignIndexes(&rel32_labels_);
+void AssemblyProgram::TrimLabels() {
+  // For now only trim for ARM binaries.
+  if (kind() != EXE_ELF_32_ARM)
+    return;
+
+  int lower_limit = kLabelLowerLimit;
+
+  VLOG(1) << "TrimLabels: threshold " << lower_limit;
+
+  rel32_label_manager_.RemoveUnderusedLabels(lower_limit);
 }
 
 void AssemblyProgram::UnassignIndexes() {
-  UnassignIndexes(&abs32_labels_);
-  UnassignIndexes(&rel32_labels_);
+  abs32_label_manager_.UnassignIndexes();
+  rel32_label_manager_.UnassignIndexes();
+}
+
+void AssemblyProgram::DefaultAssignIndexes() {
+  abs32_label_manager_.DefaultAssignIndexes();
+  rel32_label_manager_.DefaultAssignIndexes();
 }
 
 void AssemblyProgram::AssignRemainingIndexes() {
-  AssignRemainingIndexes(&abs32_labels_);
-  AssignRemainingIndexes(&rel32_labels_);
+  abs32_label_manager_.AssignRemainingIndexes();
+  rel32_label_manager_.AssignRemainingIndexes();
+}
+
+Label* AssemblyProgram::FindAbs32Label(RVA rva) {
+  return abs32_label_manager_.Find(rva);
+}
+
+Label* AssemblyProgram::FindRel32Label(RVA rva) {
+  return rel32_label_manager_.Find(rva);
 }
 
 Label* AssemblyProgram::InstructionAbs32Label(
@@ -238,17 +254,6 @@ CheckBool AssemblyProgram::EmitShared(Instruction* instruction) {
   return instruction && instructions_.push_back(instruction);
 }
 
-Label* AssemblyProgram::FindLabel(RVA rva, RVAToLabel* labels) {
-  Label*& slot = (*labels)[rva];
-  if (slot == NULL) {
-    slot = UncheckedNew<Label>(rva);
-    if (slot == NULL)
-      return NULL;
-  }
-  slot->count_++;
-  return slot;
-}
-
 void AssemblyProgram::UnassignIndexes(RVAToLabel* labels) {
   for (RVAToLabel::iterator p = labels->begin();  p != labels->end();  ++p) {
     Label* current = p->second;
@@ -367,7 +372,7 @@ std::unique_ptr<EncodedProgram> AssemblyProgram::Encode() const {
 
   encoded->set_image_base(image_base_);
 
-  if (!encoded->DefineLabels(abs32_labels_, rel32_labels_))
+  if (!encoded->ImportLabels(abs32_label_manager_, rel32_label_manager_))
     return nullptr;
 
   for (size_t i = 0;  i < instructions_.size();  ++i) {
@@ -470,61 +475,6 @@ Instruction* AssemblyProgram::GetByteInstruction(uint8_t byte) {
   return byte_instruction_cache_[byte];
 }
 
-// Chosen empirically to give the best reduction in payload size for
-// an update from daisy_3701.98.0 to daisy_4206.0.0.
-const int AssemblyProgram::kLabelLowerLimit = 5;
-
-CheckBool AssemblyProgram::TrimLabels() {
-  // For now only trim for ARM binaries.
-  if (kind() != EXE_ELF_32_ARM)
-    return true;
-
-  int lower_limit = kLabelLowerLimit;
-
-  VLOG(1) << "TrimLabels: threshold " << lower_limit;
-
-  // Walk through the list of instructions, replacing trimmed labels
-  // with the original machine instruction.
-  for (size_t i = 0; i < instructions_.size(); ++i) {
-    Instruction* instruction = instructions_[i];
-    switch (instruction->op()) {
-      case REL32ARM: {
-        Label* label =
-            static_cast<InstructionWithLabelARM*>(instruction)->label();
-        if (label->count_ <= lower_limit) {
-          const uint8_t* arm_op =
-              static_cast<InstructionWithLabelARM*>(instruction)->arm_op();
-          uint16_t op_size =
-              static_cast<InstructionWithLabelARM*>(instruction)->op_size();
-
-          if (op_size < 1)
-            return false;
-          UncheckedDelete(instruction);
-          instructions_[i] = UncheckedNew<BytesInstruction>(arm_op, op_size);
-          if (!instructions_[i])
-            return false;
-        }
-        break;
-      }
-      default:
-        break;
-    }
-  }
-
-  // Remove and deallocate underused Labels.
-  RVAToLabel::iterator it = rel32_labels_.begin();
-  while (it != rel32_labels_.end()) {
-    if (it->second->count_ <= lower_limit) {
-      UncheckedDelete(it->second);
-      rel32_labels_.erase(it++);
-    } else {
-      ++it;
-    }
-  }
-
-  return true;
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 
 Status Encode(const AssemblyProgram& program,

courgette/assembly_program.h

@@ -123,16 +123,23 @@ class AssemblyProgram {
   // Generates 8-byte absolute reference to address of 'label'.
   CheckBool EmitAbs64(Label* label) WARN_UNUSED_RESULT;
 
-  // Looks up a label or creates a new one.  Might return NULL.
-  Label* FindOrMakeAbs32Label(RVA rva);
+  // Traverses RVAs in |abs32_visitor| and |rel32_visitor| to precompute Labels.
+  void PrecomputeLabels(RvaVisitor* abs32_visitor, RvaVisitor* rel32_visitor);
 
-  // Looks up a label or creates a new one.  Might return NULL.
-  Label* FindOrMakeRel32Label(RVA rva);
+  // Removes underused Labels. Thresholds used (0 = no trimming) is
+  // architecture-dependent.
+  void TrimLabels();
 
-  void DefaultAssignIndexes();
   void UnassignIndexes();
+  void DefaultAssignIndexes();
   void AssignRemainingIndexes();
 
+  // Looks up abs32 label. Returns null if none found.
+  Label* FindAbs32Label(RVA rva);
+
+  // Looks up rel32 label. Returns null if none found.
+  Label* FindRel32Label(RVA rva);
+
   std::unique_ptr<EncodedProgram> Encode() const;
 
   // Accessor for instruction list.
@@ -152,10 +159,6 @@ class AssemblyProgram {
   // otherwise returns NULL.
   Label* InstructionRel32Label(const Instruction* instruction) const;
 
-  // Removes underused Labels. Thresholds used (may be 0, i.e., no trimming) is
-  // dependent on architecture. Returns true on success, and false otherwise.
-  CheckBool TrimLabels();
-
  private:
   using ScopedInstruction =
       std::unique_ptr<Instruction, UncheckedDeleter<Instruction>>;
@@ -183,11 +186,10 @@ class AssemblyProgram {
 
   InstructionVector instructions_;  // All the instructions in program.
 
-  // These are lookup maps to find the label associated with a given address.
-  // We have separate label spaces for addresses referenced by rel32 labels and
-  // abs32 labels.  This is somewhat arbitrary.
-  RVAToLabel rel32_labels_;
-  RVAToLabel abs32_labels_;
+  // Storage and lookup of Labels associated with target addresses. We use
+  // separate abs32 and rel32 labels.
+  LabelManager abs32_label_manager_;
+  LabelManager rel32_label_manager_;
 
   DISALLOW_COPY_AND_ASSIGN(AssemblyProgram);
 };

courgette/courgette.h

@@ -48,7 +48,6 @@ enum Status {
   C_DISASSEMBLY_FAILED = 25,      //
   C_ASSEMBLY_FAILED = 26,         //
   C_ADJUSTMENT_FAILED = 27,       //
-  C_TRIM_FAILED = 28,             // TrimLabels failed
 };
 
 // What type of executable is something

courgette/disassembler.cc

@@ -4,10 +4,35 @@
 
 #include "courgette/disassembler.h"
 
+#include <memory>
+
 #include "base/logging.h"
+#include "courgette/assembly_program.h"
 
 namespace courgette {
 
+Disassembler::RvaVisitor_Abs32::RvaVisitor_Abs32(
+    const std::vector<RVA>& rva_locations,
+    const AddressTranslator& translator)
+    : VectorRvaVisitor<RVA>(rva_locations), translator_(translator) {
+}
+
+RVA Disassembler::RvaVisitor_Abs32::Get() const {
+  // For Abs32 targets, get target RVA from architecture-dependent functions.
+  return translator_.PointerToTargetRVA(translator_.RVAToPointer(*it_));
+}
+
+Disassembler::RvaVisitor_Rel32::RvaVisitor_Rel32(
+    const std::vector<RVA>& rva_locations,
+    const AddressTranslator& translator)
+    : VectorRvaVisitor<RVA>(rva_locations), translator_(translator) {
+}
+
+RVA Disassembler::RvaVisitor_Rel32::Get() const {
+  // For Rel32 targets, only handle 32-bit offsets.
+  return *it_ + 4 + Read32LittleEndian(translator_.RVAToPointer(*it_));
+}
+
 Disassembler::Disassembler(const void* start, size_t length)
     : failure_reason_("uninitialized") {
   start_ = reinterpret_cast<const uint8_t*>(start);
@@ -40,6 +65,12 @@ bool Disassembler::Bad(const char* reason) {
   return false;
 }
 
+void Disassembler::PrecomputeLabels(AssemblyProgram* program) {
+  std::unique_ptr<RvaVisitor> abs32_visitor(CreateAbs32TargetRvaVisitor());
+  std::unique_ptr<RvaVisitor> rel32_visitor(CreateRel32TargetRvaVisitor());
+  program->PrecomputeLabels(abs32_visitor.get(), rel32_visitor.get());
+}
+
 void Disassembler::ReduceLength(size_t reduced_length) {
   CHECK_LE(reduced_length, length_);
   length_ = reduced_length;