diff --git a/src/hotspot/share/ci/ciCallProfile.hpp b/src/hotspot/share/ci/ciCallProfile.hpp index 081014954db..96f5dc0ef95 100644 --- a/src/hotspot/share/ci/ciCallProfile.hpp +++ b/src/hotspot/share/ci/ciCallProfile.hpp @@ -37,8 +37,9 @@ class ciCallProfile : StackObj { // Fields are initialized directly by ciMethod::call_profile_at_bci. friend class ciMethod; friend class ciMethodHandle; + friend class Compile; - enum { MorphismLimit = 2 }; // Max call site's morphism we care about + enum { MorphismLimit = 6 }; // Max call site's morphism we care about int _limit; // number of receivers have been determined int _morphism; // determined call site's morphism int _count; // # times has this call been executed @@ -57,6 +58,9 @@ class ciCallProfile : StackObj { void add_receiver(ciKlass* receiver, int receiver_count); + bool _polymorphic_enabled = false; + const double _polymorphic_inline_ratio[2] = {PolymorphicRecv0InlineRatio, PolymorphicRecv1InlineRatio}; + public: // Note: The following predicates return false for invalid profiles: bool has_receiver(int i) const { return _limit > i; } @@ -90,6 +94,22 @@ class ciCallProfile : StackObj { } return call; } + + void enable_polymorphic() { _polymorphic_enabled = true; } + + bool polymorphic_inline(int id) { + if (_polymorphic_enabled) { + return has_receiver(id) && receiver_prob(id) > _polymorphic_inline_ratio[id]; + } + return false; + } + + int polymorphic_devirtualize() { + if (_polymorphic_enabled) { + return _morphism; + } + return 0; + } }; #endif // SHARE_VM_CI_CICALLPROFILE_HPP diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index b7f82ec09b5..6b3dfb9d35a 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -468,7 +468,7 @@ const BitMap& ciMethod::bci_block_start() { ciCallProfile ciMethod::call_profile_at_bci(int bci) { ResourceMark rm; ciCallProfile result; - if (method_data() != NULL && method_data()->is_mature()) { + if (method_data() != NULL && (method_data()->is_mature() || PolymorphicInlining)) { ciProfileData* data = method_data()->bci_to_data(bci); if (data != NULL && data->is_CounterData()) { // Every profiled call site has a counter. @@ -515,8 +515,8 @@ ciCallProfile ciMethod::call_profile_at_bci(int bci) { // The call site count is > 0 in the case of a polymorphic virtual call. if (morphism > 0 && morphism == result._limit) { // The morphism <= MorphismLimit. - if ((morphism < ciCallProfile::MorphismLimit) || - (morphism == ciCallProfile::MorphismLimit && count == 0)) { + if ((morphism == 1) || + (morphism <= ciCallProfile::MorphismLimit && count == 0)) { #ifdef ASSERT if (count > 0) { this->print_short_name(tty); diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp index 7aaa2337248..27bed36fee1 100644 --- a/src/hotspot/share/code/codeCache.cpp +++ b/src/hotspot/share/code/codeCache.cpp @@ -1678,9 +1678,9 @@ void CodeCache::print_summary(outputStream* st, bool detailed) { } if (detailed) { - st->print_cr(" total_blobs=" UINT32_FORMAT " nmethods=" UINT32_FORMAT + st->print_cr(" total_blobs=" UINT32_FORMAT " nmethods=" UINT32_FORMAT " nonprofiled nmethods=" UINT32_FORMAT " adapters=" UINT32_FORMAT, - blob_count(), nmethod_count(), adapter_count()); + blob_count(), nmethod_count(), nmethod_count(CodeBlobType::MethodNonProfiled), adapter_count()); st->print_cr(" compilation: %s", CompileBroker::should_compile_new_jobs() ? "enabled" : Arguments::mode() == Arguments::_int ? "disabled (interpreter mode)" : diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index 9031251107d..9f59dae354b 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -33,6 +33,7 @@ #include "opto/callGenerator.hpp" #include "opto/castnode.hpp" #include "opto/cfgnode.hpp" +#include "opto/library_call.hpp" #include "opto/mulnode.hpp" #include "opto/parse.hpp" #include "opto/rootnode.hpp" @@ -231,23 +232,38 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool speculative_receiver_type = NULL; } } + + if (PolymorphicInlining && morphism != 1) { + profile.enable_polymorphic(); + } + if (receiver_method == NULL && (have_major_receiver || morphism == 1 || + (morphism >= 2 && PolymorphicInlining) || (morphism == 2 && UseBimorphicInlining))) { // receiver_method = profile.method(); // Profiles do not suggest methods now. Look it up in the major receiver. receiver_method = callee->resolve_invoke(jvms->method()->holder(), profile.receiver(0)); } - if (receiver_method != NULL) { + if (receiver_method != NULL && !(receiver_method->is_native() && cg_intrinsic)) { // The single majority receiver sufficiently outweighs the minority. CallGenerator* hit_cg = this->call_generator(receiver_method, - vtable_index, !call_does_dispatch, jvms, allow_inline, prof_factor); + vtable_index, !call_does_dispatch, jvms, allow_inline && profile.polymorphic_inline(0), prof_factor); if (hit_cg != NULL) { // Look up second receiver. CallGenerator* next_hit_cg = NULL; ciMethod* next_receiver_method = NULL; - if (morphism == 2 && UseBimorphicInlining) { + + if (profile.polymorphic_inline(1)) { + next_receiver_method = callee->resolve_invoke(jvms->method()->holder(), + profile.receiver(1)); + if (next_receiver_method != NULL && !(next_receiver_method->is_native() && cg_intrinsic)) { + next_hit_cg = this->call_generator(next_receiver_method, + vtable_index, !call_does_dispatch, jvms, + allow_inline, prof_factor); + } + } else if (morphism == 2 && UseBimorphicInlining) { next_receiver_method = callee->resolve_invoke(jvms->method()->holder(), profile.receiver(1)); if (next_receiver_method != NULL) { @@ -261,11 +277,27 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool } } } + bool polymorphic_recompile = PolymorphicInlining && next_hit_cg != NULL; + CallGenerator* hit_cg_devirtual[ciCallProfile::MorphismLimit] = {0}; + for (int i = 2; i < profile.polymorphic_devirtualize(); i++) { + ciMethod*receiver_method_devirtual = callee->resolve_invoke(jvms->method()->holder(), + profile.receiver(i)); + if (receiver_method_devirtual != NULL && !(receiver_method_devirtual->is_native() && cg_intrinsic)) { + hit_cg_devirtual[i] = this->call_generator( + receiver_method_devirtual, + vtable_index, !call_does_dispatch, jvms, + false, prof_factor); + if (hit_cg_devirtual[i] == NULL) { + polymorphic_recompile = false; + } + } + } CallGenerator* miss_cg; Deoptimization::DeoptReason reason = (morphism == 2 ? Deoptimization::Reason_bimorphic : Deoptimization::reason_class_check(speculative_receiver_type != NULL)); - if ((morphism == 1 || (morphism == 2 && next_hit_cg != NULL)) && + + if ((morphism == 1 || (morphism == 2 && next_hit_cg != NULL) || polymorphic_recompile) && !too_many_traps_or_recompiles(caller, bci, reason) ) { // Generate uncommon trap for class check failure path @@ -275,9 +307,22 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool } else { // Generate virtual call for class check failure path // in case of polymorphic virtual call site. - miss_cg = CallGenerator::for_virtual_call(callee, vtable_index); + if (PolymorphicInlining && cg_intrinsic != NULL) { + miss_cg = cg_intrinsic; + } else { + miss_cg = CallGenerator::for_virtual_call(callee, vtable_index); + } } if (miss_cg != NULL) { + for (int i = profile.polymorphic_devirtualize() - 1; i >= 2; i--) { + if (hit_cg_devirtual[i] != NULL) { + assert(speculative_receiver_type == NULL, "shouldn't end up here if we used speculation"); + trace_type_profile(C, jvms->method(), jvms->depth() - 1, jvms->bci(), next_receiver_method, profile.receiver(i), site_count, profile.receiver_count(i)); + // We don't need to record dependency on a receiver here and below. + // Whenever we inline, the dependency is added by Parse::Parse(). + miss_cg = CallGenerator::for_predicted_call(profile.receiver(i), miss_cg, hit_cg_devirtual[i], PROB_MAX); + } + } if (next_hit_cg != NULL) { assert(speculative_receiver_type == NULL, "shouldn't end up here if we used speculation"); trace_type_profile(C, jvms->method(), jvms->depth() - 1, jvms->bci(), next_receiver_method, profile.receiver(1), site_count, profile.receiver_count(1)); @@ -359,6 +404,9 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool const char* msg = "virtual call"; if (C->print_inlining()) { print_inlining(callee, jvms->depth() - 1, jvms->bci(), msg); + if (!profile.has_receiver(0)) { + print_inlining(callee, jvms->depth() - 1, jvms->bci(), "no receiver"); + } } C->log_inline_failure(msg); return CallGenerator::for_virtual_call(callee, vtable_index); diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index 028d96520cb..2bb8c111206 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -4119,6 +4119,10 @@ jint Arguments::parse(const JavaVMInitArgs* initial_cmd_args) { PropertyList_add(&_system_properties, new SystemProperty("java.math.BigDecimal.optimization", "true", true)); } + if (PolymorphicInlining) { + FLAG_SET_ERGO_IF_DEFAULT(intx, TypeProfileWidth, 8); + } + // Set object alignment values. set_object_alignment(); diff --git a/src/hotspot/share/runtime/globals_ext.hpp b/src/hotspot/share/runtime/globals_ext.hpp index 1a54881c4f5..1c5e62f3c0f 100644 --- a/src/hotspot/share/runtime/globals_ext.hpp +++ b/src/hotspot/share/runtime/globals_ext.hpp @@ -96,6 +96,14 @@ product(bool, UseBigDecimalOpt, true, \ "use binary search in zero stripping of BigDecimal") \ \ + product(bool, PolymorphicInlining, true, \ + "Inline caching multiple type of receivers") \ + \ + product(double, PolymorphicRecv0InlineRatio, 0.15, \ + "Receiver calling ratio for polymorphic inline ") \ + \ + product(double, PolymorphicRecv1InlineRatio, 0.35, \ + "Receiver calling ratio for polymorphic inline ") \ //add new AJDK specific flags here