Finer branch profile data for inlined methods

src/hotspot/share/c1/c1_GraphBuilder.cpp

@@ -1217,6 +1217,19 @@ void GraphBuilder::_goto(int from_bci, int to_bci) {
   append(x);
 }
 
+void GraphBuilder::hash_inline_context_for_if_node(If* if_node) {
+  int h = INLINE_HASH_FIRST;
+  IRScope* s = if_node->scope();
+  int caller_bci = bci();
+  while (s != NULL) {
+    s->method()->compute_hash_for_method_and_bci(caller_bci, h);
+
+    log_info(compilation)("C1 insert profiling method: %s, hash: %x, caller bci: %d, caller level: %d", s->method()->get_Method()->external_name(), h, caller_bci, s->level());
+    caller_bci = s->caller_bci();
+    s = s->caller();
+  }
+  if_node->set_fine_profile_inline_hash(h);
+}
 
 void GraphBuilder::if_node(Value x, If::Condition cond, Value y, ValueStack* state_before) {
   BlockBegin* tsux = block_at(stream()->get_dest());
@@ -1241,6 +1254,7 @@ void GraphBuilder::if_node(Value x, If::Condition cond, Value y, ValueStack* sta
       if (profile_branches()) {
         // Successors can be rotated by the canonicalizer, check for this case.
         if_node->set_profiled_method(method());
+        hash_inline_context_for_if_node(if_node);
         if_node->set_should_profile(true);
         if (if_node->tsux() == fsux) {
           if_node->set_swapped(true);

src/hotspot/share/c1/c1_GraphBuilder.hpp

@@ -245,6 +245,9 @@ class GraphBuilder {
   void increment();
   void _goto(int from_bci, int to_bci);
   void if_node(Value x, If::Condition cond, Value y, ValueStack* stack_before);
+
+  void hash_inline_context_for_if_node(If* if_node);
+
   void if_zero(ValueType* type, If::Condition cond);
   void if_null(ValueType* type, If::Condition cond);
   void if_same(ValueType* type, If::Condition cond);

src/hotspot/share/c1/c1_IR.cpp

@@ -146,6 +146,7 @@ IRScope::IRScope(Compilation* compilation, IRScope* caller, int caller_bci, ciMe
   _wrote_fields       = false;
   _wrote_volatile     = false;
   _start              = NULL;
+  _caller_bci = caller_bci;
 
   if (osr_bci != -1) {
     // selective creation of phi functions is not possibel in osr-methods

src/hotspot/share/c1/c1_IR.hpp

@@ -138,6 +138,7 @@ class IRScope: public CompilationResourceObj {
   // hierarchy
   Compilation*  _compilation;                    // the current compilation
   IRScope*      _caller;                         // the caller scope, or NULL
+  int           _caller_bci;
   int           _level;                          // the inlining level
   ciMethod*     _method;                         // the corresponding method
   IRScopeList   _callees;                        // the inlined method scopes
@@ -163,6 +164,7 @@ class IRScope: public CompilationResourceObj {
   // accessors
   Compilation*  compilation() const              { return _compilation; }
   IRScope*      caller() const                   { return _caller; }
+  int      caller_bci() const                   { return _caller_bci; }
   int           level() const                    { return _level; }
   ciMethod*     method() const                   { return _method; }
   int           max_stack() const;               // NOTE: expensive

src/hotspot/share/c1/c1_Instruction.hpp

@@ -1988,6 +1988,7 @@ LEAF(If, BlockEnd)
   int         _profiled_bci; // Canonicalizer may alter bci of If node
   bool        _swapped;      // Is the order reversed with respect to the original If in the
                              // bytecode stream?
+  int _fine_profile_inline_hash;
  public:
   // creation
   // unordered_is_true is valid for float/double compares only
@@ -1999,6 +2000,7 @@ LEAF(If, BlockEnd)
   , _profiled_method(NULL)
   , _profiled_bci(0)
   , _swapped(false)
+  , _fine_profile_inline_hash(0)
   {
     ASSERT_VALUES
     set_flag(UnorderedIsTrueFlag, unordered_is_true);
@@ -2023,6 +2025,12 @@ LEAF(If, BlockEnd)
   int profiled_bci() const                       { return _profiled_bci; }    // set for profiled branches and tiered
   bool is_swapped() const                        { return _swapped; }
 
+  void set_fine_profile_inline_hash(int hash)   {
+    assert(_fine_profile_inline_hash == 0, "_fine_profile_inline_hash should not been set");
+    _fine_profile_inline_hash = hash;
+  }
+  int get_fine_profile_inline_hash()   { return _fine_profile_inline_hash; }
+
   // manipulation
   void swap_operands() {
     Value t = _x; _x = _y; _y = t;

src/hotspot/share/c1/c1_LIRGenerator.cpp

@@ -965,11 +965,27 @@ void LIRGenerator::profile_branch(If* if_instr, If::Condition cond) {
     assert(method != NULL, "method should be set if branch is profiled");
     ciMethodData* md = method->method_data_or_null();
     assert(md != NULL, "Sanity");
-    ciProfileData* data = md->bci_to_data(if_instr->profiled_bci());
-    assert(data != NULL, "must have profiling data");
-    assert(data->is_BranchData(), "need BranchData for two-way branches");
-    int taken_count_offset     = md->byte_offset_of_slot(data, BranchData::taken_offset());
-    int not_taken_count_offset = md->byte_offset_of_slot(data, BranchData::not_taken_offset());
+
+    int inline_hash = 0;
+
+    if (EnableLevel3FineProfiling) {
+      inline_hash = if_instr->get_fine_profile_inline_hash();
+    }
+
+    int taken_count_offset, not_taken_count_offset = 0;
+    if (inline_hash != 0) {
+      DataLayout* expanded_data = md->expand_profile_data(if_instr->profiled_bci(), inline_hash);
+      assert(expanded_data != NULL, "must have expanded data");
+      taken_count_offset     = md->byte_offset_of_raw_slot(md->constant_encoding(), expanded_data, BranchData::taken_offset());
+      not_taken_count_offset = md->byte_offset_of_raw_slot(md->constant_encoding(), expanded_data, BranchData::not_taken_offset());
+    } else {
+      ciProfileData* data = md->bci_to_data(if_instr->profiled_bci());
+      assert(data != NULL, "must have profiling data");
+      assert(data->is_BranchData(), "need BranchData for two-way branches");
+      taken_count_offset     = md->byte_offset_of_slot(data, BranchData::taken_offset());
+      not_taken_count_offset = md->byte_offset_of_slot(data, BranchData::not_taken_offset());
+    }
+    log_info(compilation)("fine profile inline hash: %x, method: %s, meta: %p, taken_offset: %d, not_taken_offset: %d", inline_hash, method->get_Method()->external_name(), md->constant_encoding(), taken_count_offset, not_taken_count_offset);
     if (if_instr->is_swapped()) {
       int t = taken_count_offset;
       taken_count_offset = not_taken_count_offset;

src/hotspot/share/ci/ciMethod.hpp

@@ -33,6 +33,10 @@
 #include "prims/methodHandles.hpp"
 #include "utilities/bitMap.hpp"
 
+#define INLINE_HASH_A 54059 /* a prime */
+#define INLINE_HASH_B 76963 /* another prime */
+#define INLINE_HASH_FIRST 37 /* also prime */
+
 class ciMethodBlocks;
 class MethodLiveness;
 class Arena;
@@ -172,6 +176,16 @@ class ciMethod : public ciMetadata {
     return m;
   }
 
+  void compute_hash_for_method_and_bci(int bci, int& h) {
+    const char* method_full_name = get_Method()->external_name();
+    size_t idx = 0;
+    while (method_full_name[idx]) {
+      h = (h * INLINE_HASH_A) ^ (method_full_name[idx] * INLINE_HASH_B);
+      idx++;
+    }
+    h = (h * INLINE_HASH_A) ^ (bci * INLINE_HASH_B);
+  }
+
   // Method code and related information.
   address code()                                 { if (_code == NULL) load_code(); return _code; }
   int code_size() const                          { check_is_loaded(); return _code_size; }

src/hotspot/share/ci/ciMethodData.cpp

@@ -262,6 +262,7 @@ bool ciMethodData::load_data() {
   return true;
 }
 
+
 void ciReceiverTypeData::translate_receiver_data_from(const ProfileData* data) {
   for (uint row = 0; row < row_limit(); row++) {
     Klass* k = data->as_ReceiverTypeData()->receiver(row);
@@ -337,6 +338,39 @@ ciProfileData* ciMethodData::data_at(int data_index) {
   };
 }
 
+ciProfileData* ciMethodData::data_at_layout(DataLayout* data_layout) {
+  switch (data_layout->tag()) {
+  case DataLayout::no_tag:
+  default:
+    ShouldNotReachHere();
+    return NULL;
+  case DataLayout::bit_data_tag:
+    return new ciBitData(data_layout);
+  case DataLayout::counter_data_tag:
+    return new ciCounterData(data_layout);
+  case DataLayout::jump_data_tag:
+    return new ciJumpData(data_layout);
+  case DataLayout::receiver_type_data_tag:
+    return new ciReceiverTypeData(data_layout);
+  case DataLayout::virtual_call_data_tag:
+    return new ciVirtualCallData(data_layout);
+  case DataLayout::ret_data_tag:
+    return new ciRetData(data_layout);
+  case DataLayout::branch_data_tag:
+    return new ciBranchData(data_layout);
+  case DataLayout::multi_branch_data_tag:
+    return new ciMultiBranchData(data_layout);
+  case DataLayout::arg_info_data_tag:
+    return new ciArgInfoData(data_layout);
+  case DataLayout::call_type_data_tag:
+    return new ciCallTypeData(data_layout);
+  case DataLayout::virtual_call_type_data_tag:
+    return new ciVirtualCallTypeData(data_layout);
+  case DataLayout::parameters_type_data_tag:
+    return new ciParametersTypeData(data_layout);
+  };
+}
+
 // Iteration over data.
 ciProfileData* ciMethodData::next_data(ciProfileData* current) {
   int current_index = dp_to_di(current->dp());
@@ -345,6 +379,11 @@ ciProfileData* ciMethodData::next_data(ciProfileData* current) {
   return next;
 }
 
+DataLayout* ciMethodData::get_expanded_data(int hash) {
+  assert(hash != 0, "Illegal expanded data");
+  return get_MethodData()->find_expanded_data_with_hash(hash);
+}
+
 ciProfileData* ciMethodData::bci_to_extra_data(int bci, ciMethod* m, bool& two_free_slots) {
   DataLayout* dp  = extra_data_base();
   DataLayout* end = args_data_limit();
@@ -592,7 +631,15 @@ ByteSize ciMethodData::offset_of_slot(ciProfileData* data, ByteSize slot_offset_
 
   // Add in counter_offset, the # of bytes into the ProfileData of counter or flag
   int offset = in_bytes(data_offset) + cell_offset + in_bytes(slot_offset_in_data);
+  return in_ByteSize(offset);
+}
 
+ByteSize ciMethodData::offset_of_raw_slot(Metadata* mdo, DataLayout* data, ByteSize slot_offset_in_data) {
+  // Get cell offset of the ProfileData within data array
+  int cell_offset = (address) data - (address) mdo;
+
+  // Add in counter_offset, the # of bytes into the ProfileData of counter or flag
+  int offset = cell_offset + in_bytes(slot_offset_in_data);
   return in_ByteSize(offset);
 }
 

src/hotspot/share/ci/ciMethodData.hpp

@@ -469,10 +469,16 @@ class ciMethodData : public ciMetadata {
 
   ciArgInfoData *arg_info() const;
 
+public:
   address data_base() const {
     return (address) _data;
   }
 
+  DataLayout* expand_profile_data(int bci, int inline_hash) {
+    return get_MethodData()->expand_data_for_branch(bci, inline_hash);
+  }
+
+private:
   void prepare_metadata();
   void load_remaining_extra_data();
   ciProfileData* bci_to_extra_data(int bci, ciMethod* m, bool& two_free_slots);
@@ -528,11 +534,15 @@ class ciMethodData : public ciMetadata {
   // Get the data at an arbitrary (sort of) data index.
   ciProfileData* data_at(int data_index);
 
+  ciProfileData* data_at_layout(DataLayout* data);
+
   // Walk through the data in order.
   ciProfileData* first_data() { return data_at(first_di()); }
   ciProfileData* next_data(ciProfileData* current);
   bool is_valid(ciProfileData* current) { return current != NULL; }
 
+  DataLayout* get_expanded_data(int hash);
+
   DataLayout* extra_data_base() const  { return data_layout_at(data_size()); }
   DataLayout* args_data_limit() const  { return data_layout_at(data_size() + extra_data_size() -
                                                                parameters_size()); }
@@ -591,7 +601,9 @@ class ciMethodData : public ciMetadata {
 
   // Code generation helper
   ByteSize offset_of_slot(ciProfileData* data, ByteSize slot_offset_in_data);
+  ByteSize offset_of_raw_slot(Metadata* mdo, DataLayout* data, ByteSize slot_offset_in_data);
   int      byte_offset_of_slot(ciProfileData* data, ByteSize slot_offset_in_data) { return in_bytes(offset_of_slot(data, slot_offset_in_data)); }
+  int      byte_offset_of_raw_slot(Metadata* mdo, DataLayout* data, ByteSize slot_offset_in_data) { return in_bytes(offset_of_raw_slot(mdo, data, slot_offset_in_data)); }
 
 #ifndef PRODUCT
   // printing support for method data

src/hotspot/share/oops/methodData.cpp

@@ -926,6 +926,12 @@ int MethodData::compute_allocation_size_in_bytes(const methodHandle& method) {
   if (args_cell > 0) {
     object_size += DataLayout::compute_size_in_bytes(args_cell);
   }
+
+  // for expanded
+  if (EnableLevel3FineProfiling) {
+    object_size += MethodData::expand_data_size_in_bytes();
+  }
+
   return object_size;
 }
 
@@ -1052,7 +1058,7 @@ int MethodData::initialize_data(BytecodeStream* stream,
            tag == DataLayout::counter_data_tag ||
            tag == DataLayout::virtual_call_type_data_tag ||
            tag == DataLayout::virtual_call_data_tag)) ||
-         cell_count == bytecode_cell_count(c), "cell counts must agree");
+         cell_count == bytecode_cell_count(c), "cell counts must agree %d, cell count: %d, verify count: %d", static_cast<int>(c), cell_count, bytecode_cell_count(c));
   if (cell_count >= 0) {
     assert(tag != DataLayout::no_tag, "bad tag");
     assert(bytecode_has_profile(c), "agree w/ BHP");
@@ -1140,6 +1146,50 @@ MethodData::MethodData(const methodHandle& method)
   initialize();
 }
 
+DataLayout* MethodData::expand_data_for_branch(int bci, int hash) {
+  // find avaiable space
+  size_t insert_index = 0;
+  for (; insert_index < FineProfilingExtraSize; insert_index++) {
+    if (_index_to_expand_profile_data[insert_index] == hash) {
+      int cell_count = BranchData::static_cell_count();
+      int total_size = DataLayout::compute_size_in_bytes(cell_count) * insert_index;
+      log_info(compilation)("expand data exist for hash: %x, data start: %p, expand start: %x, total size: %x", hash, data_base(), non_expand_data_size(), total_size);
+      return data_layout_at(non_expand_data_size() + total_size);
+    } else if (_index_to_expand_profile_data[insert_index] == 0) {
+      _index_to_expand_profile_data[insert_index] = hash;
+      break;
+    }
+  }
+  if (insert_index == FineProfilingExtraSize) {
+    log_info(compilation)("expanded data for %p is full", this);
+    return NULL;
+  }
+
+  // init expanded profile data
+  int cell_count = BranchData::static_cell_count();
+  int total_size = DataLayout::compute_size_in_bytes(cell_count) * insert_index;
+  int tag = DataLayout::branch_data_tag;
+  log_info(compilation)("expand data init for hash: %x, data start: %p, expand start: %x, total size: %x", hash, data_base(), non_expand_data_size(), total_size);
+  DataLayout* data_layout = data_layout_at(non_expand_data_size() + total_size);
+  data_layout->initialize(tag, bci, cell_count);
+
+  return data_layout;
+}
+
+DataLayout* MethodData::find_expanded_data_with_hash(int hash) {
+  size_t insert_index = 0;
+  for (; insert_index < FineProfilingExtraSize; insert_index++) {
+    if (_index_to_expand_profile_data[insert_index] == hash) {
+      int total_size = DataLayout::compute_size_in_bytes(BranchData::static_cell_count()) * insert_index;
+      DataLayout* data = data_layout_at(non_expand_data_size() + total_size);
+      log_info(compilation)("find data %p for hash: %x, data start %p, expand start: %x, total size: %x", data, hash, data_base(), non_expand_data_size(), total_size);
+      return data;
+    }
+  }
+  log_info(compilation)("not find data for hash: %x", hash);
+  return NULL;
+}
+
 void MethodData::initialize() {
   NoSafepointVerifier no_safepoint;  // init function atomic wrt GC
   ResourceMark rm;
@@ -1150,6 +1200,15 @@ void MethodData::initialize() {
   // corresponding data cells.
   int data_size = 0;
   int empty_bc_count = 0;  // number of bytecodes lacking data
+
+  if (EnableLevel3FineProfiling) {
+    _index_to_expand_profile_data = (int*) AllocateHeap(FineProfilingExtraSize * sizeof(int), mtClass);
+
+    for (size_t i = 0; i < FineProfilingExtraSize; i++) {
+      _index_to_expand_profile_data[i] = 0;
+    }
+  }
+
   _data[0] = 0;  // apparently not set below.
   BytecodeStream stream(method());
   Bytecodes::Code c;
@@ -1206,6 +1265,10 @@ void MethodData::initialize() {
 
   post_initialize(&stream);
 
+  _non_expand_data_size = data_size + extra_size + arg_data_size;
+  if (EnableLevel3FineProfiling) {
+    object_size += MethodData::expand_data_size_in_bytes();
+  }
   assert(object_size == compute_allocation_size_in_bytes(methodHandle(_method)), "MethodData: computed size != initialized size");
   set_size(object_size);
 }