diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 76e3c92ddc261..3d25e97c0c577 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -8144,6 +8144,7 @@ instruct castPP(iRegPNoSp dst) instruct castII(iRegI dst) %{ + predicate(VerifyConstraintCasts == 0); match(Set dst (CastII dst)); size(0); @@ -8153,8 +8154,22 @@ instruct castII(iRegI dst) ins_pipe(pipe_class_empty); %} +instruct castII_checked(iRegI dst, rFlagsReg cr) +%{ + predicate(VerifyConstraintCasts > 0); + match(Set dst (CastII dst)); + effect(KILL cr); + + format %{ "# castII_checked of $dst" %} + ins_encode %{ + __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1); + %} + ins_pipe(pipe_slow); +%} + instruct castLL(iRegL dst) %{ + predicate(VerifyConstraintCasts == 0); match(Set dst (CastLL dst)); size(0); @@ -8164,6 +8179,19 @@ instruct castLL(iRegL dst) ins_pipe(pipe_class_empty); %} +instruct castLL_checked(iRegL dst, rFlagsReg cr) +%{ + predicate(VerifyConstraintCasts > 0); + match(Set dst (CastLL dst)); + effect(KILL cr); + + format %{ "# castLL_checked of $dst" %} + ins_encode %{ + __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1); + %} + ins_pipe(pipe_slow); +%} + instruct castFF(vRegF dst) %{ match(Set dst (CastFF dst)); diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp index 605a05a44a731..eeb992326c304 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp @@ -2743,3 +2743,107 @@ bool C2_MacroAssembler::in_scratch_emit_size() { } return MacroAssembler::in_scratch_emit_size(); } + +static void abort_verify_int_in_range(uint idx, jint val, jint lo, jint hi) { + fatal("Invalid CastII, idx: %u, val: %d, lo: %d, hi: %d", idx, val, lo, hi); +} + +void C2_MacroAssembler::verify_int_in_range(uint idx, const TypeInt* t, Register rval, Register rtmp) { + assert(!t->empty() && !t->singleton(), "%s", Type::str(t)); + if (t == TypeInt::INT) { + return; + } + BLOCK_COMMENT("verify_int_in_range {"); + Label L_success, L_failure; + + jint lo = t->_lo; + jint hi = t->_hi; + + if (lo != min_jint && hi != max_jint) { + subsw(rtmp, rval, lo); + br(Assembler::LT, L_failure); + subsw(rtmp, rval, hi); + br(Assembler::LE, L_success); + } else if (lo != min_jint) { + subsw(rtmp, rval, lo); + br(Assembler::GE, L_success); + } else if (hi != max_jint) { + subsw(rtmp, rval, hi); + br(Assembler::LE, L_success); + } else { + ShouldNotReachHere(); + } + + bind(L_failure); + movw(c_rarg0, idx); + mov(c_rarg1, rval); + movw(c_rarg2, lo); + movw(c_rarg3, hi); + reconstruct_frame_pointer(rtmp); + rt_call(CAST_FROM_FN_PTR(address, abort_verify_int_in_range), rtmp); + hlt(0); + + bind(L_success); + BLOCK_COMMENT("} verify_int_in_range"); +} + +static void abort_verify_long_in_range(uint idx, jlong val, jlong lo, jlong hi) { + fatal("Invalid CastLL, idx: %u, val: " JLONG_FORMAT ", lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, val, lo, hi); +} + +void C2_MacroAssembler::verify_long_in_range(uint idx, const TypeLong* t, Register rval, Register rtmp) { + assert(!t->empty() && !t->singleton(), "%s", Type::str(t)); + if (t == TypeLong::LONG) { + return; + } + BLOCK_COMMENT("verify_long_in_range {"); + Label L_success, L_failure; + + jlong lo = t->_lo; + jlong hi = t->_hi; + + if (lo != min_jlong && hi != max_jlong) { + subs(rtmp, rval, lo); + br(Assembler::LT, L_failure); + subs(rtmp, rval, hi); + br(Assembler::LE, L_success); + } else if (lo != min_jlong) { + subs(rtmp, rval, lo); + br(Assembler::GE, L_success); + } else if (hi != max_jlong) { + subs(rtmp, rval, hi); + br(Assembler::LE, L_success); + } else { + ShouldNotReachHere(); + } + + bind(L_failure); + movw(c_rarg0, idx); + mov(c_rarg1, rval); + mov(c_rarg2, lo); + mov(c_rarg3, hi); + reconstruct_frame_pointer(rtmp); + rt_call(CAST_FROM_FN_PTR(address, abort_verify_long_in_range), rtmp); + hlt(0); + + bind(L_success); + BLOCK_COMMENT("} verify_long_in_range"); +} + +void C2_MacroAssembler::reconstruct_frame_pointer(Register rtmp) { + const int framesize = Compile::current()->output()->frame_size_in_bytes(); + if (PreserveFramePointer) { + // frame pointer is valid +#ifdef ASSERT + // Verify frame pointer value in rfp. + add(rtmp, sp, framesize - 2 * wordSize); + Label L_success; + cmp(rfp, rtmp); + br(Assembler::EQ, L_success); + stop("frame pointer mismatch"); + bind(L_success); +#endif // ASSERT + } else { + add(rfp, sp, framesize - 2 * wordSize); + } +} diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp index e0eaa0b76e6e9..70e4265c7cc5e 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp @@ -188,4 +188,9 @@ void vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero, FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T); + void verify_int_in_range(uint idx, const TypeInt* t, Register val, Register tmp); + void verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp); + + void reconstruct_frame_pointer(Register rtmp); + #endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index a7967d83a4e7f..5e0d6749813f9 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -787,6 +787,119 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, // C2 uses the value of ZF to determine the continuation. } +static void abort_verify_int_in_range(uint idx, jint val, jint lo, jint hi) { + fatal("Invalid CastII, idx: %u, val: %d, lo: %d, hi: %d", idx, val, lo, hi); +} + +static void reconstruct_frame_pointer_helper(MacroAssembler* masm, Register dst) { + const int framesize = Compile::current()->output()->frame_size_in_bytes(); + masm->movptr(dst, rsp); + if (framesize > 2 * wordSize) { + masm->addptr(dst, framesize - 2 * wordSize); + } +} + +void C2_MacroAssembler::reconstruct_frame_pointer(Register rtmp) { + if (PreserveFramePointer) { + // frame pointer is valid +#ifdef ASSERT + // Verify frame pointer value in rbp. + reconstruct_frame_pointer_helper(this, rtmp); + Label L_success; + cmpq(rbp, rtmp); + jccb(Assembler::equal, L_success); + STOP("frame pointer mismatch"); + bind(L_success); +#endif // ASSERT + } else { + reconstruct_frame_pointer_helper(this, rbp); + } +} + +void C2_MacroAssembler::verify_int_in_range(uint idx, const TypeInt* t, Register val) { + jint lo = t->_lo; + jint hi = t->_hi; + assert(lo < hi, "type should not be empty or constant, idx: %u, lo: %d, hi: %d", idx, lo, hi); + if (t == TypeInt::INT) { + return; + } + + BLOCK_COMMENT("CastII {"); + Label fail; + Label succeed; + if (hi == max_jint) { + cmpl(val, lo); + jccb(Assembler::greaterEqual, succeed); + } else { + if (lo != min_jint) { + cmpl(val, lo); + jccb(Assembler::less, fail); + } + cmpl(val, hi); + jccb(Assembler::lessEqual, succeed); + } + + bind(fail); + movl(c_rarg0, idx); + movl(c_rarg1, val); + movl(c_rarg2, lo); + movl(c_rarg3, hi); + reconstruct_frame_pointer(rscratch1); + call(RuntimeAddress(CAST_FROM_FN_PTR(address, abort_verify_int_in_range))); + hlt(); + bind(succeed); + BLOCK_COMMENT("} // CastII"); +} + +static void abort_verify_long_in_range(uint idx, jlong val, jlong lo, jlong hi) { + fatal("Invalid CastLL, idx: %u, val: " JLONG_FORMAT ", lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, val, lo, hi); +} + +void C2_MacroAssembler::verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp) { + jlong lo = t->_lo; + jlong hi = t->_hi; + assert(lo < hi, "type should not be empty or constant, idx: %u, lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, lo, hi); + if (t == TypeLong::LONG) { + return; + } + + BLOCK_COMMENT("CastLL {"); + Label fail; + Label succeed; + + auto cmp_val = [&](jlong bound) { + if (is_simm32(bound)) { + cmpq(val, checked_cast(bound)); + } else { + mov64(tmp, bound); + cmpq(val, tmp); + } + }; + + if (hi == max_jlong) { + cmp_val(lo); + jccb(Assembler::greaterEqual, succeed); + } else { + if (lo != min_jlong) { + cmp_val(lo); + jccb(Assembler::less, fail); + } + cmp_val(hi); + jccb(Assembler::lessEqual, succeed); + } + + bind(fail); + movl(c_rarg0, idx); + movq(c_rarg1, val); + mov64(c_rarg2, lo); + mov64(c_rarg3, hi); + reconstruct_frame_pointer(rscratch1); + call(RuntimeAddress(CAST_FROM_FN_PTR(address, abort_verify_long_in_range))); + hlt(); + bind(succeed); + BLOCK_COMMENT("} // CastLL"); +} + //------------------------------------------------------------------------------------------- // Generic instructions support for use in .ad files C2 code generation diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp index dd2880d88c381..713eb73d68f38 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp @@ -44,6 +44,9 @@ Register t, Register thread); void fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread); + void verify_int_in_range(uint idx, const TypeInt* t, Register val); + void verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp); + // Generic instructions support for use in .ad files C2 code generation void vabsnegd(int opcode, XMMRegister dst, XMMRegister src); void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len); @@ -574,4 +577,7 @@ void scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2); + + void reconstruct_frame_pointer(Register rtmp); + #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index 078150c61fbc5..dc79108c6f807 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -422,6 +422,18 @@ source_hpp %{ #include "peephole_x86_64.hpp" +bool castLL_is_imm32(const Node* n); + +%} + +source %{ + +bool castLL_is_imm32(const Node* n) { + assert(n->is_CastLL(), "must be a CastLL"); + const TypeLong* t = n->bottom_type()->is_long(); + return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi)); +} + %} // Register masks @@ -7605,6 +7617,7 @@ instruct castPP(rRegP dst) instruct castII(rRegI dst) %{ + predicate(VerifyConstraintCasts == 0); match(Set dst (CastII dst)); size(0); @@ -7614,8 +7627,22 @@ instruct castII(rRegI dst) ins_pipe(empty); %} +instruct castII_checked(rRegI dst, rFlagsReg cr) +%{ + predicate(VerifyConstraintCasts > 0); + match(Set dst (CastII dst)); + + effect(KILL cr); + format %{ "# cast_checked_II $dst" %} + ins_encode %{ + __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register); + %} + ins_pipe(pipe_slow); +%} + instruct castLL(rRegL dst) %{ + predicate(VerifyConstraintCasts == 0); match(Set dst (CastLL dst)); size(0); @@ -7625,6 +7652,32 @@ instruct castLL(rRegL dst) ins_pipe(empty); %} +instruct castLL_checked_L32(rRegL dst, rFlagsReg cr) +%{ + predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n)); + match(Set dst (CastLL dst)); + + effect(KILL cr); + format %{ "# cast_checked_LL $dst" %} + ins_encode %{ + __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg); + %} + ins_pipe(pipe_slow); +%} + +instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr) +%{ + predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n)); + match(Set dst (CastLL dst)); + + effect(KILL cr, TEMP tmp); + format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %} + ins_encode %{ + __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register); + %} + ins_pipe(pipe_slow); +%} + instruct castFF(regF dst) %{ match(Set dst (CastFF dst)); diff --git a/src/hotspot/share/opto/c2_MacroAssembler.hpp b/src/hotspot/share/opto/c2_MacroAssembler.hpp index 41347313a8cf1..1fb7714153dd8 100644 --- a/src/hotspot/share/opto/c2_MacroAssembler.hpp +++ b/src/hotspot/share/opto/c2_MacroAssembler.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,8 @@ #include "utilities/macros.hpp" class C2EntryBarrierStub; +class TypeInt; +class TypeLong; class C2_MacroAssembler: public MacroAssembler { public: diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index 41b081610974a..662f2e0796591 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -669,6 +669,16 @@ develop(bool, VerifyAliases, false, \ "perform extra checks on the results of alias analysis") \ \ + product(uint, VerifyConstraintCasts, 0, DIAGNOSTIC, \ + "Perform runtime checks to verify the value of a " \ + "ConstraintCast lies inside its type" \ + "0 = does not perform any verification, " \ + "1 = perform verification on ConstraintCastNodes that are " \ + "present during code emission, " \ + "2 = Do not do widening of ConstraintCastNodes so that we can " \ + "have more verification coverage") \ + range(0, 2) \ + \ product(intx, MaxInlineLevel, 15, \ "maximum number of nested calls that are inlined by high tier " \ "compiler") \ diff --git a/src/hotspot/share/opto/castnode.cpp b/src/hotspot/share/opto/castnode.cpp index 198634af71bba..6b3058d94f6ff 100644 --- a/src/hotspot/share/opto/castnode.cpp +++ b/src/hotspot/share/opto/castnode.cpp @@ -531,6 +531,18 @@ const Type* ConstraintCastNode::widen_type(const PhaseGVN* phase, const Type* re if (!phase->C->post_loop_opts_phase()) { return res; } + + // At VerifyConstraintCasts == 1, we verify the ConstraintCastNodes that are present during code + // emission. This allows us detecting possible mis-scheduling due to these nodes being pinned at + // the wrong control nodes. + // At VerifyConstraintCasts == 2, we do not perform widening so that we can verify the + // correctness of more ConstraintCastNodes. This further helps us detect possible + // mis-transformations that may happen due to these nodes being pinned at the wrong control + // nodes. + if (VerifyConstraintCasts > 1) { + return res; + } + const TypeInteger* this_type = res->is_integer(bt); const TypeInteger* in_type = phase->type(in(1))->isa_integer(bt); if (in_type != nullptr && diff --git a/test/hotspot/jtreg/compiler/c2/TestVerifyConstraintCasts.java b/test/hotspot/jtreg/compiler/c2/TestVerifyConstraintCasts.java new file mode 100644 index 0000000000000..5c8a367c97cf3 --- /dev/null +++ b/test/hotspot/jtreg/compiler/c2/TestVerifyConstraintCasts.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8346836 + * @requires vm.debug == true & vm.flavor == "server" + * @summary Empty main program to run with flag VerifyConstraintCasts. + * + * @run main/othervm/timeout=300 -Xbatch -Xcomp -XX:+StressGCM -XX:VerifyConstraintCasts=1 compiler.c2.TestVerifyConstraintCasts + * @run main/othervm/timeout=300 -Xbatch -Xcomp -XX:+StressGCM -XX:VerifyConstraintCasts=2 compiler.c2.TestVerifyConstraintCasts + */ +package compiler.c2; + +public class TestVerifyConstraintCasts { + public static void main(String[] args) { + } +}