From 9ab7cd90a9ae466b4597e60a6edf513172b539a1 Mon Sep 17 00:00:00 2001 From: "Jin,Zhonghui" Date: Fri, 3 Nov 2023 01:09:24 +0800 Subject: [PATCH] [Backport] 8265126: [REDO] unified handling for VectorMask object re-materialization during de-optimization Summary: 8265126: [REDO] unified handling for VectorMask object re-materialization during de-optimization Test Plan: ci jtreg Reviewed-by: JoshuaZhuwj Issue: https://github.com/dragonwell-project/dragonwell11/issues/599 --- src/hotspot/share/opto/vector.cpp | 36 +++++++++----- src/hotspot/share/prims/vectorSupport.cpp | 57 +++++++++-------------- src/hotspot/share/prims/vectorSupport.hpp | 2 +- 3 files changed, 47 insertions(+), 48 deletions(-) diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 91f16b374c1..10bd14a1c8d 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -30,6 +30,15 @@ #include "opto/vector.hpp" #include "utilities/macros.hpp" +static bool is_vector_mask(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); +} + +static bool is_vector_shuffle(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); +} + + void PhaseVector::optimize_vector_boxes() { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); @@ -236,13 +245,24 @@ void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) { } } + ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass(); + int n_fields = iklass->nof_nonstatic_fields(); + assert(n_fields == 1, "sanity"); + + // If a mask is feeding into safepoint[s], then its value should be + // packed into a boolean/byte vector first, this will simplify the + // re-materialization logic for both predicated and non-predicated + // targets. + bool is_mask = is_vector_mask(iklass); + if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) { + const TypeVect* vt = vec_value->bottom_type()->is_vect(); + BasicType bt = vt->element_basic_type(); + vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length())); + } + while (safepoints.size() > 0) { SafePointNode* sfpt = safepoints.pop()->as_SafePoint(); - ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass(); - int n_fields = iklass->nof_nonstatic_fields(); - assert(n_fields == 1, "sanity"); - uint first_ind = (sfpt->req() - sfpt->jvms()->scloff()); Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), #ifdef ASSERT @@ -303,14 +323,6 @@ Node* PhaseVector::expand_vbox_node_helper(Node* vbox, } } -static bool is_vector_mask(ciKlass* klass) { - return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); -} - -static bool is_vector_shuffle(ciKlass* klass) { - return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); -} - Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc, Node* value, const TypeInstPtr* box_type, diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index b228f1e48ae..b0031afcdf8 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -62,6 +62,8 @@ BasicType VectorSupport::klass2bt(InstanceKlass* ik) { if (is_vector_shuffle(ik)) { return T_BYTE; + } else if (is_vector_mask(ik)) { + return T_BOOLEAN; } else { // vector and mask oop value = ik->java_mirror()->obj_field(fd.offset()); BasicType elem_bt = java_lang_Class::as_BasicType(value); @@ -83,48 +85,33 @@ jint VectorSupport::klass2length(InstanceKlass* ik) { return vlen; } -void VectorSupport::init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr) { - if (is_mask) { - // Masks require special handling: when boxed they are packed and stored in boolean - // arrays, but in scalarized form they have the same size as corresponding vectors. - // For example, Int512Mask is represented in memory as boolean[16], but - // occupies the whole 512-bit vector register when scalarized. - // (In generated code, the conversion is performed by VectorStoreMask.) - // - // TODO: revisit when predicate registers are fully supported. - switch (elem_bt) { - case T_BYTE: arr->bool_at_put(index, (*(jbyte*)addr) != 0); break; - case T_SHORT: arr->bool_at_put(index, (*(jshort*)addr) != 0); break; - case T_INT: // fall-through - case T_FLOAT: arr->bool_at_put(index, (*(jint*)addr) != 0); break; - case T_LONG: // fall-through - case T_DOUBLE: arr->bool_at_put(index, (*(jlong*)addr) != 0); break; - - default: fatal("unsupported: %s", type2name(elem_bt)); - } - } else { - switch (elem_bt) { - case T_BYTE: arr-> byte_at_put(index, *(jbyte*)addr); break; - case T_SHORT: arr-> short_at_put(index, *(jshort*)addr); break; - case T_INT: arr-> int_at_put(index, *(jint*)addr); break; - case T_FLOAT: arr-> float_at_put(index, *(jfloat*)addr); break; - case T_LONG: arr-> long_at_put(index, *(jlong*)addr); break; - case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break; - - default: fatal("unsupported: %s", type2name(elem_bt)); - } +// Masks require special handling: when boxed they are packed and stored in boolean +// arrays, but in scalarized form they have the same size as corresponding vectors. +// For example, Int512Mask is represented in memory as boolean[16], but +// occupies the whole 512-bit vector register when scalarized. +// During scalarization inserting a VectorStoreMask node between mask +// and safepoint node always ensures the existence of masks in a boolean array. + +void VectorSupport::init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr) { + switch (elem_bt) { + case T_BOOLEAN: arr->bool_at_put(index, *(jboolean*)addr); break; + case T_BYTE: arr->byte_at_put(index, *(jbyte*)addr); break; + case T_SHORT: arr->short_at_put(index, *(jshort*)addr); break; + case T_INT: arr->int_at_put(index, *(jint*)addr); break; + case T_FLOAT: arr->float_at_put(index, *(jfloat*)addr); break; + case T_LONG: arr->long_at_put(index, *(jlong*)addr); break; + case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break; + default: fatal("unsupported: %s", type2name(elem_bt)); } } Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS) { - bool is_mask = is_vector_mask(ik); - int num_elem = klass2length(ik); BasicType elem_bt = klass2bt(ik); int elem_size = type2aelembytes(elem_bt); // On-heap vector values are represented as primitive arrays. - TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(is_mask ? T_BOOLEAN : elem_bt)); + TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(elem_bt)); typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint @@ -137,13 +124,13 @@ Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* f int off = (i * elem_size) % VMRegImpl::stack_slot_size; address elem_addr = reg_map->location(vreg->next(vslot)) + off; - init_payload_element(arr, is_mask, elem_bt, i, elem_addr); + init_payload_element(arr, elem_bt, i, elem_addr); } } else { // Value was directly saved on the stack. address base_addr = ((address)fr->unextended_sp()) + location.stack_offset(); for (int i = 0; i < num_elem; i++) { - init_payload_element(arr, is_mask, elem_bt, i, base_addr + i * elem_size); + init_payload_element(arr, elem_bt, i, base_addr + i * elem_size); } } return Handle(THREAD, arr); diff --git a/src/hotspot/share/prims/vectorSupport.hpp b/src/hotspot/share/prims/vectorSupport.hpp index 19e885f9dcf..5debf65ac5b 100644 --- a/src/hotspot/share/prims/vectorSupport.hpp +++ b/src/hotspot/share/prims/vectorSupport.hpp @@ -42,7 +42,7 @@ class VectorSupport : AllStatic { static Handle allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS); static Handle allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS); - static void init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr); + static void init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr); static BasicType klass2bt(InstanceKlass* ik); static jint klass2length(InstanceKlass* ik);