poly

src/hotspot/share/ci/ciCallProfile.hpp

@@ -37,8 +37,9 @@ class ciCallProfile : StackObj {
   // Fields are initialized directly by ciMethod::call_profile_at_bci.
   friend class ciMethod;
   friend class ciMethodHandle;
+  friend class Compile;
 
-  enum { MorphismLimit = 2 }; // Max call site's morphism we care about
+  enum { MorphismLimit = 6 }; // Max call site's morphism we care about
   int  _limit;                // number of receivers have been determined
   int  _morphism;             // determined call site's morphism
   int  _count;                // # times has this call been executed
@@ -57,6 +58,9 @@ class ciCallProfile : StackObj {
 
   void add_receiver(ciKlass* receiver, int receiver_count);
 
+  bool _polymorphic_enabled = false;
+  const double _polymorphic_inline_ratio[2] = {PolymorphicRecv0InlineRatio, PolymorphicRecv1InlineRatio};
+
 public:
   // Note:  The following predicates return false for invalid profiles:
   bool      has_receiver(int i) const { return _limit > i; }
@@ -90,6 +94,22 @@ class ciCallProfile : StackObj {
     }
     return call;
   }
+
+  void enable_polymorphic() { _polymorphic_enabled = true; }
+
+  bool polymorphic_inline(int id) {
+    if (_polymorphic_enabled) {
+      return has_receiver(id) && receiver_prob(id) > _polymorphic_inline_ratio[id];
+    }
+    return false;
+  }
+
+  int polymorphic_devirtualize() {
+    if (_polymorphic_enabled) {
+      return _morphism;
+    }
+    return 0;
+  }
 };
 
 #endif // SHARE_VM_CI_CICALLPROFILE_HPP

src/hotspot/share/ci/ciMethod.cpp

@@ -468,7 +468,7 @@ const BitMap& ciMethod::bci_block_start() {
 ciCallProfile ciMethod::call_profile_at_bci(int bci) {
   ResourceMark rm;
   ciCallProfile result;
-  if (method_data() != NULL && method_data()->is_mature()) {
+  if (method_data() != NULL && (method_data()->is_mature() || PolymorphicInlining)) {
     ciProfileData* data = method_data()->bci_to_data(bci);
     if (data != NULL && data->is_CounterData()) {
       // Every profiled call site has a counter.
@@ -515,8 +515,8 @@ ciCallProfile ciMethod::call_profile_at_bci(int bci) {
         // The call site count is > 0 in the case of a polymorphic virtual call.
         if (morphism > 0 && morphism == result._limit) {
            // The morphism <= MorphismLimit.
-           if ((morphism <  ciCallProfile::MorphismLimit) ||
-               (morphism == ciCallProfile::MorphismLimit && count == 0)) {
+           if ((morphism ==  1) ||
+               (morphism <= ciCallProfile::MorphismLimit && count == 0)) {
 #ifdef ASSERT
              if (count > 0) {
                this->print_short_name(tty);

src/hotspot/share/code/codeCache.cpp

@@ -1678,9 +1678,9 @@ void CodeCache::print_summary(outputStream* st, bool detailed) {
   }
 
   if (detailed) {
-    st->print_cr(" total_blobs=" UINT32_FORMAT " nmethods=" UINT32_FORMAT
+    st->print_cr(" total_blobs=" UINT32_FORMAT " nmethods=" UINT32_FORMAT " nonprofiled nmethods=" UINT32_FORMAT
                        " adapters=" UINT32_FORMAT,
-                       blob_count(), nmethod_count(), adapter_count());
+                       blob_count(), nmethod_count(), nmethod_count(CodeBlobType::MethodNonProfiled), adapter_count());
     st->print_cr(" compilation: %s", CompileBroker::should_compile_new_jobs() ?
                  "enabled" : Arguments::mode() == Arguments::_int ?
                  "disabled (interpreter mode)" :

src/hotspot/share/opto/doCall.cpp

@@ -33,6 +33,7 @@
 #include "opto/callGenerator.hpp"
 #include "opto/castnode.hpp"
 #include "opto/cfgnode.hpp"
+#include "opto/library_call.hpp"
 #include "opto/mulnode.hpp"
 #include "opto/parse.hpp"
 #include "opto/rootnode.hpp"
@@ -231,23 +232,38 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool
           speculative_receiver_type = NULL;
         }
       }
+
+      if (PolymorphicInlining && morphism != 1) {
+        profile.enable_polymorphic();
+      }
+
       if (receiver_method == NULL &&
           (have_major_receiver || morphism == 1 ||
+           (morphism >= 2 && PolymorphicInlining) ||
            (morphism == 2 && UseBimorphicInlining))) {
         // receiver_method = profile.method();
         // Profiles do not suggest methods now.  Look it up in the major receiver.
         receiver_method = callee->resolve_invoke(jvms->method()->holder(),
                                                       profile.receiver(0));
       }
-      if (receiver_method != NULL) {
+      if (receiver_method != NULL && !(receiver_method->is_native() && cg_intrinsic)) {
         // The single majority receiver sufficiently outweighs the minority.
         CallGenerator* hit_cg = this->call_generator(receiver_method,
-              vtable_index, !call_does_dispatch, jvms, allow_inline, prof_factor);
+              vtable_index, !call_does_dispatch, jvms, allow_inline && profile.polymorphic_inline(0), prof_factor);
         if (hit_cg != NULL) {
           // Look up second receiver.
           CallGenerator* next_hit_cg = NULL;
           ciMethod* next_receiver_method = NULL;
-          if (morphism == 2 && UseBimorphicInlining) {
+
+          if (profile.polymorphic_inline(1)) {
+            next_receiver_method = callee->resolve_invoke(jvms->method()->holder(),
+                                                               profile.receiver(1));
+            if (next_receiver_method != NULL && !(next_receiver_method->is_native() && cg_intrinsic)) {
+              next_hit_cg = this->call_generator(next_receiver_method,
+                                  vtable_index, !call_does_dispatch, jvms,
+                                  allow_inline, prof_factor);
+            }
+          } else if (morphism == 2 && UseBimorphicInlining) {
             next_receiver_method = callee->resolve_invoke(jvms->method()->holder(),
                                                                profile.receiver(1));
             if (next_receiver_method != NULL) {
@@ -261,11 +277,27 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool
               }
             }
           }
+          bool polymorphic_recompile = PolymorphicInlining && next_hit_cg != NULL;
+          CallGenerator* hit_cg_devirtual[ciCallProfile::MorphismLimit] = {0};
+          for (int i = 2; i < profile.polymorphic_devirtualize(); i++) {
+            ciMethod*receiver_method_devirtual = callee->resolve_invoke(jvms->method()->holder(),
+                                                                profile.receiver(i));
+            if (receiver_method_devirtual != NULL && !(receiver_method_devirtual->is_native() && cg_intrinsic)) {
+              hit_cg_devirtual[i] = this->call_generator(
+                  receiver_method_devirtual,
+                                             vtable_index, !call_does_dispatch, jvms,
+                                             false, prof_factor);
+              if (hit_cg_devirtual[i] == NULL) {
+                polymorphic_recompile = false;
+              }
+            }
+          }
           CallGenerator* miss_cg;
           Deoptimization::DeoptReason reason = (morphism == 2
                                                ? Deoptimization::Reason_bimorphic
                                                : Deoptimization::reason_class_check(speculative_receiver_type != NULL));
-          if ((morphism == 1 || (morphism == 2 && next_hit_cg != NULL)) &&
+
+          if ((morphism == 1 || (morphism == 2 && next_hit_cg != NULL) || polymorphic_recompile) &&
               !too_many_traps_or_recompiles(caller, bci, reason)
              ) {
             // Generate uncommon trap for class check failure path
@@ -275,9 +307,22 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool
           } else {
             // Generate virtual call for class check failure path
             // in case of polymorphic virtual call site.
-            miss_cg = CallGenerator::for_virtual_call(callee, vtable_index);
+            if (PolymorphicInlining && cg_intrinsic != NULL) {
+              miss_cg = cg_intrinsic;
+            } else {
+              miss_cg = CallGenerator::for_virtual_call(callee, vtable_index);
+            }
           }
           if (miss_cg != NULL) {
+            for (int i = profile.polymorphic_devirtualize() - 1; i >= 2; i--) {
+              if (hit_cg_devirtual[i] != NULL) {
+                assert(speculative_receiver_type == NULL, "shouldn't end up here if we used speculation");
+                trace_type_profile(C, jvms->method(), jvms->depth() - 1, jvms->bci(), next_receiver_method, profile.receiver(i), site_count, profile.receiver_count(i));
+                // We don't need to record dependency on a receiver here and below.
+                // Whenever we inline, the dependency is added by Parse::Parse().
+                miss_cg = CallGenerator::for_predicted_call(profile.receiver(i), miss_cg, hit_cg_devirtual[i], PROB_MAX);
+              }
+            }
             if (next_hit_cg != NULL) {
               assert(speculative_receiver_type == NULL, "shouldn't end up here if we used speculation");
               trace_type_profile(C, jvms->method(), jvms->depth() - 1, jvms->bci(), next_receiver_method, profile.receiver(1), site_count, profile.receiver_count(1));
@@ -359,6 +404,9 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool
     const char* msg = "virtual call";
     if (C->print_inlining()) {
       print_inlining(callee, jvms->depth() - 1, jvms->bci(), msg);
+      if (!profile.has_receiver(0)) {
+        print_inlining(callee, jvms->depth() - 1, jvms->bci(), "no receiver");
+      }
     }
     C->log_inline_failure(msg);
     return CallGenerator::for_virtual_call(callee, vtable_index);

src/hotspot/share/runtime/arguments.cpp

@@ -4119,6 +4119,10 @@ jint Arguments::parse(const JavaVMInitArgs* initial_cmd_args) {
     PropertyList_add(&_system_properties, new SystemProperty("java.math.BigDecimal.optimization", "true", true));
   }
 
+  if (PolymorphicInlining) {
+    FLAG_SET_ERGO_IF_DEFAULT(intx, TypeProfileWidth, 8);
+  }
+
   // Set object alignment values.
   set_object_alignment();
 

src/hotspot/share/runtime/globals_ext.hpp

@@ -96,6 +96,14 @@
     product(bool, UseBigDecimalOpt, true,                                   \
           "use binary search in zero stripping of BigDecimal")              \
                                                                             \
+  product(bool, PolymorphicInlining, true,                                  \
+          "Inline caching multiple type of receivers")                      \
+                                                                            \
+  product(double, PolymorphicRecv0InlineRatio, 0.15,                        \
+          "Receiver calling ratio for polymorphic inline ")                 \
+                                                                            \
+  product(double, PolymorphicRecv1InlineRatio, 0.35,                        \
+          "Receiver calling ratio for polymorphic inline ")                 \
   //add new AJDK specific flags here