[JIT] Backport 8318446: C2: optimize stores into primitive arrays by …

…combining values into larger store

Summary: include these patches for merge stores optimization
 8318446: C2: optimize stores into primitive arrays by combining values into larger store
 8319690: [AArch64] C2 compilation hits offset_ok_for_immed: assert "c2 compiler bug"
 8335390: C2 MergeStores: wrong result with Unsafe
 8331311: C2: Big Endian Port of 8318446: optimize stores into primitive arrays by combining values into larger store
 8331085: Crash in MergePrimitiveArrayStores::is_compatible_store()
 8331252: C2: MergeStores: handle negative shift values
 8331054: C2 MergeStores: assert failed: unexpected basic type after JDK-8318446 and JDK-8329555
 8335392: C2 MergeStores: enhanced pointer parsing
 8334342: Add MergeStore JMH benchmarks
 8226411: C2: Avoid memory barriers around off-heap unsafe accesses
 Fix is_ConI() query after port 8318446
 Fix for comments

Testing: CI/CD

Reviewers: zhuoren.wz, MaxXSoft

Issue: #920

Fix for comments

Add missing assertion in round_down_power_of_2()

src/hotspot/share/compiler/compilerDirectives.hpp

@@ -61,6 +61,7 @@
     cflags(PrintIntrinsics,         bool, PrintIntrinsics, PrintIntrinsics) \
 NOT_PRODUCT(cflags(TraceOptoPipelining, bool, TraceOptoPipelining, TraceOptoPipelining)) \
 NOT_PRODUCT(cflags(TraceOptoOutput,     bool, TraceOptoOutput, TraceOptoOutput)) \
+NOT_PRODUCT(cflags(TraceMergeStores, bool, TraceMergeStores, TraceMergeStores)) \
     cflags(TraceSpilling,           bool, TraceSpilling, TraceSpilling) \
     cflags(Vectorize,               bool, false, Vectorize) \
     cflags(VectorizeDebug,          uintx, 0, VectorizeDebug) \

src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp

@@ -607,12 +607,15 @@ Node* G1BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) c
   Node* adr = access.addr().node();
   Node* obj = access.base();
 
+  bool anonymous = (decorators & C2_UNSAFE_ACCESS) != 0;
   bool mismatched = (decorators & C2_MISMATCHED) != 0;
   bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
   bool in_heap = (decorators & IN_HEAP) != 0;
+  bool in_native = (decorators & IN_NATIVE) != 0;
   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
   bool is_unordered = (decorators & MO_UNORDERED) != 0;
-  bool need_cpu_mem_bar = !is_unordered || mismatched || !in_heap;
+  bool is_mixed = !in_heap && !in_native;
+  bool need_cpu_mem_bar = !is_unordered || mismatched || is_mixed;
 
   Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : kit->top();
   Node* load = CardTableBarrierSetC2::load_at_resolved(access, val_type);

src/hotspot/share/gc/shared/c2/barrierSetC2.cpp

@@ -40,13 +40,15 @@ void* C2Access::barrier_set_state() const {
 }
 
 bool C2Access::needs_cpu_membar() const {
-  bool mismatched = (_decorators & C2_MISMATCHED) != 0;
+  bool mismatched   = (_decorators & C2_MISMATCHED) != 0;
   bool is_unordered = (_decorators & MO_UNORDERED) != 0;
   bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
-  bool in_heap = (_decorators & IN_HEAP) != 0;
+  bool in_heap   = (_decorators & IN_HEAP) != 0;
+  bool in_native = (_decorators & IN_NATIVE) != 0;
+  bool is_mixed  = !in_heap && !in_native;
 
-  bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
-  bool is_read = (_decorators & C2_READ_ACCESS) != 0;
+  bool is_write  = (_decorators & C2_WRITE_ACCESS) != 0;
+  bool is_read   = (_decorators & C2_READ_ACCESS) != 0;
   bool is_atomic = is_read && is_write;
 
   if (is_atomic) {
@@ -60,9 +62,11 @@ bool C2Access::needs_cpu_membar() const {
     // the barriers get omitted and the unsafe reference begins to "pollute"
     // the alias analysis of the rest of the graph, either Compile::can_alias
     // or Compile::must_alias will throw a diagnostic assert.)
-    if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
+    if (is_mixed || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
       return true;
     }
+  } else {
+    assert(!is_mixed, "not unsafe");
   }
 
   return false;
@@ -78,7 +82,7 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons
   bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
 
   bool in_native = (decorators & IN_NATIVE) != 0;
-  assert(!in_native, "not supported yet");
+  assert(!in_native || (unsafe && !access.is_oop()), "not supported yet");
 
   if (access.type() == T_DOUBLE) {
     Node* new_val = kit->dstore_rounding(val.node());

src/hotspot/share/oops/accessDecorators.hpp

@@ -174,11 +174,11 @@ const DecoratorSet ON_DECORATOR_MASK  = ON_STRONG_OOP_REF | ON_WEAK_OOP_REF |
                                         ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF;
 
 // === Access Location ===
-// Accesses can take place in, e.g. the heap, old or young generation and different native roots.
+// Accesses can take place in, e.g. the heap, old or young generation, different native roots, or native memory off the heap.
 // The location is important to the GC as it may imply different actions. The following decorators are used:
 // * IN_HEAP: The access is performed in the heap. Many barriers such as card marking will
 //   be omitted if this decorator is not set.
-// * IN_NATIVE: The access is performed in an off-heap data structure pointing into the Java heap.
+// * IN_NATIVE: The access is performed in an off-heap data structure.
 const DecoratorSet IN_HEAP            = UCONST64(1) << 19;
 const DecoratorSet IN_NATIVE          = UCONST64(1) << 20;
 const DecoratorSet IN_DECORATOR_MASK  = IN_HEAP | IN_NATIVE;

src/hotspot/share/opto/addnode.cpp

@@ -704,9 +704,9 @@ Node* AddPNode::Ideal_base_and_offset(Node* ptr, PhaseTransform* phase,
 //------------------------------unpack_offsets----------------------------------
 // Collect the AddP offset values into the elements array, giving up
 // if there are more than length.
-int AddPNode::unpack_offsets(Node* elements[], int length) {
+int AddPNode::unpack_offsets(Node* elements[], int length) const {
   int count = 0;
-  Node* addr = this;
+  Node const* addr = this;
   Node* base = addr->in(AddPNode::Base);
   while (addr->is_AddP()) {
     if (addr->in(AddPNode::Base) != base) {

src/hotspot/share/opto/addnode.hpp

@@ -154,7 +154,7 @@ class AddPNode : public Node {
 
   // Collect the AddP offset values into the elements array, giving up
   // if there are more than length.
-  int unpack_offsets(Node* elements[], int length);
+  int unpack_offsets(Node* elements[], int length) const;
 
   // Do not match base-ptr edge
   virtual uint match_edge(uint idx) const;

src/hotspot/share/opto/c2_globals.hpp

@@ -354,6 +354,12 @@
   notproduct(bool, TraceNewVectors, false,                                  \
           "Trace creation of Vector nodes")                                 \
                                                                             \
+  diagnostic(bool, MergeStores, true,                                       \
+          "Optimize stores by combining values into larger store")          \
+                                                                            \
+  develop(bool, TraceMergeStores, false,                                    \
+          "Trace creation of merged stores")                                \
+                                                                            \
   product_pd(bool, OptoBundling,                                            \
           "Generate nops to fill i-cache lines")                            \
                                                                             \

src/hotspot/share/opto/connode.hpp

@@ -39,6 +39,7 @@ class ConNode : public TypeNode {
   ConNode( const Type *t ) : TypeNode(t->remove_speculative(),1) {
     init_req(0, (Node*)Compile::current()->root());
     init_flags(Flag_is_Con);
+    init_class_id(Class_Con);
   }
   virtual int  Opcode() const;
   virtual uint hash() const;
@@ -53,7 +54,9 @@ class ConNode : public TypeNode {
 // Simple integer constants
 class ConINode : public ConNode {
 public:
-  ConINode( const TypeInt *t ) : ConNode(t) {}
+  ConINode( const TypeInt *t ) : ConNode(t) {
+    init_class_id(Class_ConI);
+  }
   virtual int Opcode() const;
 
   // Factory method:

src/hotspot/share/opto/library_call.cpp

@@ -2201,10 +2201,14 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c
   offset = ConvL2X(offset);
   adr = make_unsafe_address(base, offset, type, kind == Relaxed);
 
-  if (_gvn.type(base)->isa_ptr() != TypePtr::NULL_PTR) {
-    heap_base_oop = base;
-  } else if (type == T_OBJECT) {
-    return false; // off-heap oop accesses are not supported
+  if (_gvn.type(base)->isa_ptr() == TypePtr::NULL_PTR) {
+    if (type != T_OBJECT) {
+      decorators |= IN_NATIVE; // off-heap primitive access
+    } else {
+      return false; // off-heap oop accesses are not supported
+    }
+  } else {
+    heap_base_oop = base; // on-heap or mixed access
   }
 
   // Can base be NULL? Otherwise, always on-heap access.