8346836: C2: Verify CastII/CastLL bounds at runtime

src/hotspot/cpu/aarch64/aarch64.ad

@@ -8144,6 +8144,7 @@ instruct castPP(iRegPNoSp dst)
 
 instruct castII(iRegI dst)
 %{
+  predicate(VerifyConstraintCasts == 0);
   match(Set dst (CastII dst));
 
   size(0);
@@ -8153,8 +8154,22 @@ instruct castII(iRegI dst)
   ins_pipe(pipe_class_empty);
 %}
 
+instruct castII_checked(iRegI dst, rFlagsReg cr)
+%{
+  predicate(VerifyConstraintCasts > 0);
+  match(Set dst (CastII dst));
+  effect(KILL cr);
+
+  format %{ "# castII_checked of $dst" %}
+  ins_encode %{
+    __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct castLL(iRegL dst)
 %{
+  predicate(VerifyConstraintCasts == 0);
   match(Set dst (CastLL dst));
 
   size(0);
@@ -8164,6 +8179,19 @@ instruct castLL(iRegL dst)
   ins_pipe(pipe_class_empty);
 %}
 
+instruct castLL_checked(iRegL dst, rFlagsReg cr)
+%{
+  predicate(VerifyConstraintCasts > 0);
+  match(Set dst (CastLL dst));
+  effect(KILL cr);
+
+  format %{ "# castLL_checked of $dst" %}
+  ins_encode %{
+    __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct castFF(vRegF dst)
 %{
   match(Set dst (CastFF dst));

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -2743,3 +2743,107 @@ bool C2_MacroAssembler::in_scratch_emit_size() {
   }
   return MacroAssembler::in_scratch_emit_size();
 }
+
+static void abort_verify_int_in_range(uint idx, jint val, jint lo, jint hi) {
+  fatal("Invalid CastII, idx: %u, val: %d, lo: %d, hi: %d", idx, val, lo, hi);
+}
+
+void C2_MacroAssembler::verify_int_in_range(uint idx, const TypeInt* t, Register rval, Register rtmp) {
+  assert(!t->empty() && !t->singleton(), "%s", Type::str(t));
+  if (t == TypeInt::INT) {
+    return;
+  }
+  BLOCK_COMMENT("verify_int_in_range {");
+  Label L_success, L_failure;
+
+  jint lo = t->_lo;
+  jint hi = t->_hi;
+
+  if (lo != min_jint && hi != max_jint) {
+    subsw(rtmp, rval, lo);
+    br(Assembler::LT, L_failure);
+    subsw(rtmp, rval, hi);
+    br(Assembler::LE, L_success);
+  } else if (lo != min_jint) {
+    subsw(rtmp, rval, lo);
+    br(Assembler::GE, L_success);
+  } else if (hi != max_jint) {
+    subsw(rtmp, rval, hi);
+    br(Assembler::LE, L_success);
+  } else {
+    ShouldNotReachHere();
+  }
+
+  bind(L_failure);
+  movw(c_rarg0, idx);
+  mov(c_rarg1, rval);
+  movw(c_rarg2, lo);
+  movw(c_rarg3, hi);
+  reconstruct_frame_pointer(rtmp);
+  rt_call(CAST_FROM_FN_PTR(address, abort_verify_int_in_range), rtmp);
+  hlt(0);
+
+  bind(L_success);
+  BLOCK_COMMENT("} verify_int_in_range");
+}
+
+static void abort_verify_long_in_range(uint idx, jlong val, jlong lo, jlong hi) {
+  fatal("Invalid CastLL, idx: %u, val: " JLONG_FORMAT ", lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, val, lo, hi);
+}
+
+void C2_MacroAssembler::verify_long_in_range(uint idx, const TypeLong* t, Register rval, Register rtmp) {
+  assert(!t->empty() && !t->singleton(), "%s", Type::str(t));
+  if (t == TypeLong::LONG) {
+    return;
+  }
+  BLOCK_COMMENT("verify_long_in_range {");
+  Label L_success, L_failure;
+
+  jlong lo = t->_lo;
+  jlong hi = t->_hi;
+
+  if (lo != min_jlong && hi != max_jlong) {
+    subs(rtmp, rval, lo);
+    br(Assembler::LT, L_failure);
+    subs(rtmp, rval, hi);
+    br(Assembler::LE, L_success);
+  } else if (lo != min_jlong) {
+    subs(rtmp, rval, lo);
+    br(Assembler::GE, L_success);
+  } else if (hi != max_jlong) {
+    subs(rtmp, rval, hi);
+    br(Assembler::LE, L_success);
+  } else {
+    ShouldNotReachHere();
+  }
+
+  bind(L_failure);
+  movw(c_rarg0, idx);
+  mov(c_rarg1, rval);
+  mov(c_rarg2, lo);
+  mov(c_rarg3, hi);
+  reconstruct_frame_pointer(rtmp);
+  rt_call(CAST_FROM_FN_PTR(address, abort_verify_long_in_range), rtmp);
+  hlt(0);
+
+  bind(L_success);
+  BLOCK_COMMENT("} verify_long_in_range");
+}
+
+void C2_MacroAssembler::reconstruct_frame_pointer(Register rtmp) {
+  const int framesize = Compile::current()->output()->frame_size_in_bytes();
+  if (PreserveFramePointer) {
+    // frame pointer is valid
+#ifdef ASSERT
+    // Verify frame pointer value in rfp.
+    add(rtmp, sp, framesize - 2 * wordSize);
+    Label L_success;
+    cmp(rfp, rtmp);
+    br(Assembler::EQ, L_success);
+    stop("frame pointer mismatch");
+    bind(L_success);
+#endif // ASSERT
+  } else {
+    add(rfp, sp, framesize - 2 * wordSize);
+  }
+}

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp

@@ -188,4 +188,9 @@
   void vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero,
                          FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T);
 
+  void verify_int_in_range(uint idx, const TypeInt* t, Register val, Register tmp);
+  void verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp);
+
+  void reconstruct_frame_pointer(Register rtmp);
+
 #endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -787,6 +787,119 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax,
   // C2 uses the value of ZF to determine the continuation.
 }
 
+static void abort_verify_int_in_range(uint idx, jint val, jint lo, jint hi) {
+  fatal("Invalid CastII, idx: %u, val: %d, lo: %d, hi: %d", idx, val, lo, hi);
+}
+
+static void reconstruct_frame_pointer_helper(MacroAssembler* masm, Register dst) {
+  const int framesize = Compile::current()->output()->frame_size_in_bytes();
+  masm->movptr(dst, rsp);
+  if (framesize > 2 * wordSize) {
+    masm->addptr(dst, framesize - 2 * wordSize);
+  }
+}
+
+void C2_MacroAssembler::reconstruct_frame_pointer(Register rtmp) {
+  if (PreserveFramePointer) {
+    // frame pointer is valid
+#ifdef ASSERT
+    // Verify frame pointer value in rbp.
+    reconstruct_frame_pointer_helper(this, rtmp);
+    Label L_success;
+    cmpq(rbp, rtmp);
+    jccb(Assembler::equal, L_success);
+    STOP("frame pointer mismatch");
+    bind(L_success);
+#endif // ASSERT
+  } else {
+    reconstruct_frame_pointer_helper(this, rbp);
+  }
+}
+
+void C2_MacroAssembler::verify_int_in_range(uint idx, const TypeInt* t, Register val) {
+  jint lo = t->_lo;
+  jint hi = t->_hi;
+  assert(lo < hi, "type should not be empty or constant, idx: %u, lo: %d, hi: %d", idx, lo, hi);
+  if (t == TypeInt::INT) {
+    return;
+  }
+
+  BLOCK_COMMENT("CastII {");
+  Label fail;
+  Label succeed;
+  if (hi == max_jint) {
+    cmpl(val, lo);
+    jccb(Assembler::greaterEqual, succeed);
+  } else {
+    if (lo != min_jint) {
+      cmpl(val, lo);
+      jccb(Assembler::less, fail);
+    }
+    cmpl(val, hi);
+    jccb(Assembler::lessEqual, succeed);
+  }
+
+  bind(fail);
+  movl(c_rarg0, idx);
+  movl(c_rarg1, val);
+  movl(c_rarg2, lo);
+  movl(c_rarg3, hi);
+  reconstruct_frame_pointer(rscratch1);
+  call(RuntimeAddress(CAST_FROM_FN_PTR(address, abort_verify_int_in_range)));
+  hlt();
+  bind(succeed);
+  BLOCK_COMMENT("} // CastII");
+}
+
+static void abort_verify_long_in_range(uint idx, jlong val, jlong lo, jlong hi) {
+  fatal("Invalid CastLL, idx: %u, val: " JLONG_FORMAT ", lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, val, lo, hi);
+}
+
+void C2_MacroAssembler::verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp) {
+  jlong lo = t->_lo;
+  jlong hi = t->_hi;
+  assert(lo < hi, "type should not be empty or constant, idx: %u, lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, lo, hi);
+  if (t == TypeLong::LONG) {
+    return;
+  }
+
+  BLOCK_COMMENT("CastLL {");
+  Label fail;
+  Label succeed;
+
+  auto cmp_val = [&](jlong bound) {
+    if (is_simm32(bound)) {
+      cmpq(val, checked_cast<int>(bound));
+    } else {
+      mov64(tmp, bound);
+      cmpq(val, tmp);
+    }
+  };
+
+  if (hi == max_jlong) {
+    cmp_val(lo);
+    jccb(Assembler::greaterEqual, succeed);
+  } else {
+    if (lo != min_jlong) {
+      cmp_val(lo);
+      jccb(Assembler::less, fail);
+    }
+    cmp_val(hi);
+    jccb(Assembler::lessEqual, succeed);
+  }
+
+  bind(fail);
+  movl(c_rarg0, idx);
+  movq(c_rarg1, val);
+  mov64(c_rarg2, lo);
+  mov64(c_rarg3, hi);
+  reconstruct_frame_pointer(rscratch1);
+  call(RuntimeAddress(CAST_FROM_FN_PTR(address, abort_verify_long_in_range)));
+  hlt();
+  bind(succeed);
+  BLOCK_COMMENT("} // CastLL");
+}
+
 //-------------------------------------------------------------------------------------------
 // Generic instructions support for use in .ad files C2 code generation
 

src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp

@@ -44,6 +44,9 @@
                              Register t, Register thread);
   void fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread);
 
+  void verify_int_in_range(uint idx, const TypeInt* t, Register val);
+  void verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp);
+
   // Generic instructions support for use in .ad files C2 code generation
   void vabsnegd(int opcode, XMMRegister dst, XMMRegister src);
   void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len);
@@ -574,4 +577,7 @@
 
   void scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
                           KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
+
+  void reconstruct_frame_pointer(Register rtmp);
+
 #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP

src/hotspot/cpu/x86/x86_64.ad

@@ -422,6 +422,18 @@ source_hpp %{
 
 #include "peephole_x86_64.hpp"
 
+bool castLL_is_imm32(const Node* n);
+
+%}
+
+source %{
+
+bool castLL_is_imm32(const Node* n) {
+  assert(n->is_CastLL(), "must be a CastLL");
+  const TypeLong* t = n->bottom_type()->is_long();
+  return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
+}
+
 %}
 
 // Register masks
@@ -7605,6 +7617,7 @@ instruct castPP(rRegP dst)
 
 instruct castII(rRegI dst)
 %{
+  predicate(VerifyConstraintCasts == 0);
   match(Set dst (CastII dst));
 
   size(0);
@@ -7614,8 +7627,22 @@ instruct castII(rRegI dst)
   ins_pipe(empty);
 %}
 
+instruct castII_checked(rRegI dst, rFlagsReg cr)
+%{
+  predicate(VerifyConstraintCasts > 0);
+  match(Set dst (CastII dst));
+
+  effect(KILL cr);
+  format %{ "# cast_checked_II $dst" %}
+  ins_encode %{
+    __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct castLL(rRegL dst)
 %{
+  predicate(VerifyConstraintCasts == 0);
   match(Set dst (CastLL dst));
 
   size(0);
@@ -7625,6 +7652,32 @@ instruct castLL(rRegL dst)
   ins_pipe(empty);
 %}
 
+instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
+%{
+  predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
+  match(Set dst (CastLL dst));
+
+  effect(KILL cr);
+  format %{ "# cast_checked_LL $dst" %}
+  ins_encode %{
+    __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
+%{
+  predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
+  match(Set dst (CastLL dst));
+
+  effect(KILL cr, TEMP tmp);
+  format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
+  ins_encode %{
+    __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct castFF(regF dst)
 %{
   match(Set dst (CastFF dst));