diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/aarch64/AArch64ReadBarrierSetLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/aarch64/AArch64ReadBarrierSetLIRGenerator.java index 7422134eb580..ac50d13d302f 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/aarch64/AArch64ReadBarrierSetLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/aarch64/AArch64ReadBarrierSetLIRGenerator.java @@ -28,6 +28,7 @@ import jdk.graal.compiler.core.common.memory.BarrierType; import jdk.graal.compiler.core.common.memory.MemoryOrderMode; import jdk.graal.compiler.lir.Variable; +import jdk.graal.compiler.lir.gen.BarrierSetLIRGeneratorTool; import jdk.graal.compiler.lir.gen.LIRGeneratorTool; import jdk.graal.compiler.lir.gen.ReadBarrierSetLIRGeneratorTool; import jdk.vm.ci.aarch64.AArch64Kind; @@ -37,7 +38,7 @@ /** * AArch64 specific LIR generation for GC barriers. */ -public interface AArch64ReadBarrierSetLIRGenerator extends ReadBarrierSetLIRGeneratorTool { +public interface AArch64ReadBarrierSetLIRGenerator extends BarrierSetLIRGeneratorTool { /** * Emit an atomic read-and-write instruction with any required GC barriers. diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyMidTier.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyMidTier.java index ecb657fad5d2..9481e1f00a81 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyMidTier.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyMidTier.java @@ -44,6 +44,5 @@ public EconomyMidTier() { appendPhase(new MidTierLoweringPhase(canonicalizer)); appendPhase(new FrameStateAssignmentPhase()); appendPhase(canonicalizer); - appendPhase(new WriteBarrierAdditionPhase()); } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java index dd551716ec40..b6b81f926c72 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java @@ -47,6 +47,7 @@ import jdk.graal.compiler.phases.common.PropagateDeoptimizeProbabilityPhase; import jdk.graal.compiler.phases.common.RemoveOpaqueValuePhase; import jdk.graal.compiler.phases.common.TransplantGraphsPhase; +import jdk.graal.compiler.phases.common.WriteBarrierAdditionPhase; import jdk.graal.compiler.phases.schedule.SchedulePhase; import jdk.graal.compiler.phases.schedule.SchedulePhase.SchedulingStrategy; import jdk.graal.compiler.phases.tiers.LowTierContext; @@ -87,6 +88,8 @@ public LowTier(OptionValues options) { appendPhase(new FixReadsPhase(true, new SchedulePhase(GraalOptions.StressTestEarlyReads.getValue(options) ? SchedulingStrategy.EARLIEST : SchedulingStrategy.LATEST_OUT_OF_LOOPS_IMPLICIT_NULL_CHECKS))); + appendPhase(new WriteBarrierAdditionPhase()); + appendPhase(canonicalizerWithoutGVN); /* diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/MidTier.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/MidTier.java index 17b80da4b114..b41f1774c165 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/MidTier.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/MidTier.java @@ -122,8 +122,6 @@ public MidTier(OptionValues options) { } appendPhase(canonicalizer); - - appendPhase(new WriteBarrierAdditionPhase()); } @Override diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java index d9c338c7c589..8b4407dd9f9b 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java @@ -1149,13 +1149,17 @@ private void replaceAtUsagePos(Node replacement, Node usage, Position pos) { } } + public void replaceAtUsages(Node replacement, InputType inputType) { + replaceAtUsages(replacement, inputType, (NodePredicate) n -> true); + } + /** * For each use of {@code this} in another node, {@code n}, replace it with {@code replacement} * if the type of the use is {@code inputType}. * * @see #replaceAtUsages(Node) */ - public void replaceAtUsages(Node replacement, InputType inputType) { + public void replaceAtUsages(Node replacement, InputType inputType, Predicate filter) { checkReplaceWith(replacement); int i = 0; int usageCount = this.getUsageCount(); @@ -1165,7 +1169,7 @@ public void replaceAtUsages(Node replacement, InputType inputType) { usages: while (i < usageCount) { Node usage = this.getUsageAt(i); for (Position pos : usage.inputPositions()) { - if (pos.getInputType() == inputType && pos.get(usage) == this) { + if (pos.getInputType() == inputType && pos.get(usage) == this && filter.test(usage)) { replaceAtUsagePos(replacement, usage, pos); this.movUsageFromEndTo(i); usageCount--; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java index 945e3a4d6b2c..f188463ec5d4 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java @@ -455,6 +455,13 @@ public int threadLastJavaFpOffset() { public final int g1CardQueueBufferOffset = getConstant("G1ThreadLocalData::dirty_card_queue_buffer_offset", Integer.class, -1, !g1LowLatencyPostWriteBarrierSupport); public final int g1CardTableBaseOffset = getConstant("G1ThreadLocalData::card_table_base_offset", Integer.class, -1, g1LowLatencyPostWriteBarrierSupport); + public final int shenandoahGCStateOffset = getConstant("ShenandoahThreadLocalData::gc_state_offset", Integer.class); + public final int shenandoahSATBIndexOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_index_offset", Integer.class); + public final int shenandoahSATBBufferOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_buffer_offset", Integer.class); + public final int shenandoahCardTableOffset = getConstant("ShenandoahThreadLocalData::card_table_offset", Integer.class); + public final int shenandoahGCRegionSizeBytesShift = getFieldValue("CompilerToVM::Data::shenandoah_region_size_bytes_shift", Integer.class, "int"); + public final long shenandoahGCCSetFastTestAddress = getFieldValue("CompilerToVM::Data::shenandoah_in_cset_fast_test_addr", Long.class, "address"); + public final int klassOffset = getFieldValue("java_lang_Class::_klass_offset", Integer.class, "int"); public final int arrayKlassOffset = getFieldValue("java_lang_Class::_array_klass_offset", Integer.class, "int"); @@ -632,6 +639,14 @@ private long getZGCAddressField(String name) { public final long zBarrierSetRuntimeLoadBarrierOnOopArray = getZGCAddressField("ZBarrierSetRuntime::load_barrier_on_oop_array"); public final int zPointerLoadShift = getConstant("ZPointerLoadShift", Integer.class, -1, osArch.equals("aarch64") && zgcSupport); + public final long shenandoahLoadBarrierStrong = getAddress("ShenandoahRuntime::load_reference_barrier_strong"); + public final long shenandoahLoadBarrierStrongNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_strong_narrow"); + public final long shenandoahLoadBarrierWeak = getAddress("ShenandoahRuntime::load_reference_barrier_weak"); + public final long shenandoahLoadBarrierWeakNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_weak_narrow"); + public final long shenandoahLoadBarrierPhantom = getAddress("ShenandoahRuntime::load_reference_barrier_phantom"); + public final long shenandoahLoadBarrierPhantomNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_phantom_narrow"); + public final long shenandoahPreBarrier = getAddress("ShenandoahRuntime::pre_barrier"); + // aarch64 specific nmethod entry barrier support // @formatter:off public final int BarrierSetAssembler_nmethod_patching_type = getFieldValue("CompilerToVM::Data::BarrierSetAssembler_nmethod_patching_type", Integer.class, "int", -1, osArch.equals("aarch64")); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfigAccess.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfigAccess.java index e7ee7156d090..3228f88b63e7 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfigAccess.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfigAccess.java @@ -190,7 +190,7 @@ protected void reportErrors() { List messages = new ArrayList<>(); if (!missing.isEmpty()) { messages.add(String.format("VM config values missing that should be present in %s:%n %s", runtime, - missing.stream().sorted().collect(Collectors.joining(System.lineSeparator() + " ")))); + missing.stream().sorted().collect(Collectors.joining( " / ")))); } if (!unexpected.isEmpty()) { messages.add(String.format("VM config values not expected to be present in %s:%n %s", runtime, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java index 47f3dd750bad..504db0194d05 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java @@ -259,6 +259,8 @@ private BarrierSet createBarrierSet(GraalHotSpotVMConfig config, MetaAccessProvi ResolvedJavaField referentField = HotSpotReplacementsUtil.referentField(metaAccess); if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { return new HotSpotZBarrierSet(objectArrayType, referentField); + } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + return new HotSpotShenandoahBarrierSet(objectArrayType, referentField, config); } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Epsilon) { return new NoBarrierSet(); } else if (config.useG1GC()) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java index fdfc96aa7eda..2e95a7566c6e 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java @@ -210,9 +210,7 @@ public enum HotSpotGC { G1("UseG1GC"), Z(JavaVersionUtil.JAVA_SPEC > 21, true, flagIsSet("UseZGC")), Epsilon(true, true, flagIsSet("UseEpsilonGC")), - - // Unsupported GCs - Shenandoah(false, true, flagIsSet("UseShenandoahGC")); + Shenandoah(JavaVersionUtil.JAVA_SPEC > 24, true, flagIsSet("UseShenandoahGC")); HotSpotGC(String flag) { this(true, true, flagIsSet(flag)); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java new file mode 100644 index 000000000000..5eaed1b45f9d --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java @@ -0,0 +1,101 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot; + +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.core.common.type.ObjectStamp; +import jdk.graal.compiler.hotspot.nodes.HotSpotCompressionNode; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.type.NarrowOopStamp; +import org.graalvm.word.LocationIdentity; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahBarrierSet; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * Specialization of {@link ShenandoahBarrierSet} that adds support for read barriers on handle locations. + */ +public class HotSpotShenandoahBarrierSet extends ShenandoahBarrierSet { + private CompressEncoding oopEncoding; + + public HotSpotShenandoahBarrierSet(ResolvedJavaType objectArrayType, ResolvedJavaField referentField, GraalHotSpotVMConfig config) { + super(objectArrayType, referentField); + this.oopEncoding = config.getOopEncoding(); + this.useLoadRefBarrier = config.getFlag("ShenandoahLoadRefBarrier", Boolean.class); + this.useSATBBarrier = config.getFlag("ShenandoahSATBBarrier", Boolean.class); + this.useCASBarrier = config.getFlag("ShenandoahCASBarrier", Boolean.class); + this.useCardBarrier = config.getFlag("ShenandoahCardBarrier", Boolean.class); + } + + @Override + protected BarrierType barrierForLocation(BarrierType currentBarrier, LocationIdentity location, JavaKind storageKind) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + return BarrierType.READ; + } + return super.barrierForLocation(currentBarrier, location, storageKind); + } + + @Override + public BarrierType readBarrierType(LocationIdentity location, ValueNode address, Stamp loadStamp) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + assert loadStamp instanceof AbstractObjectStamp : loadStamp; + return BarrierType.READ; + } + return super.readBarrierType(location, address, loadStamp); + } + + @Override + public BarrierType writeBarrierType(LocationIdentity location) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + return BarrierType.FIELD; + } + return BarrierType.NONE; + } + + @Override + protected ValueNode maybeUncompressReference(ValueNode value, boolean narrow) { + if (value != null && narrow) { + //System.out.println("Uncompressing " + value); + return HotSpotCompressionNode.uncompressWithoutUnique(value.graph(), value, oopEncoding); + } + return value; + } + + @Override + protected ValueNode maybeCompressReference(ValueNode value, boolean narrow) { + if (value != null && narrow) { + //System.out.println("Compressing " + value); + return HotSpotCompressionNode.compressWithoutUnique(value.graph(), value, oopEncoding); + } + return value; + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java index 528a12d9b47f..47be5cdbf1af 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java @@ -62,6 +62,7 @@ import jdk.graal.compiler.hotspot.HotSpotLIRGenerator; import jdk.graal.compiler.hotspot.HotSpotLockStack; import jdk.graal.compiler.hotspot.aarch64.g1.AArch64HotSpotG1BarrierSetLIRTool; +import jdk.graal.compiler.hotspot.aarch64.shenandoah.AArch64HotSpotShenandoahBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.aarch64.z.AArch64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.debug.BenchmarkCounters; import jdk.graal.compiler.hotspot.meta.HotSpotProviders; @@ -119,6 +120,9 @@ protected static BarrierSetLIRGeneratorTool getBarrierSet(GraalHotSpotVMConfig c if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { return new AArch64HotSpotZBarrierSetLIRGenerator(config, providers); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + return new AArch64HotSpotShenandoahBarrierSetLIRGenerator(config, providers); + } return null; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java new file mode 100644 index 000000000000..28dc375a8a5e --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java @@ -0,0 +1,144 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.core.aarch64.AArch64LIRGenerator; +import jdk.graal.compiler.core.aarch64.AArch64ReadBarrierSetLIRGenerator; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.core.common.LIRKind; +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.memory.MemoryExtendKind; +import jdk.graal.compiler.core.common.memory.MemoryOrderMode; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.LIRFrameState; +import jdk.graal.compiler.lir.Variable; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.aarch64.AArch64AtomicMove; +import jdk.graal.compiler.lir.aarch64.AArch64Move; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadBarrierNode; +import jdk.vm.ci.aarch64.AArch64Kind; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.PlatformKind; +import jdk.vm.ci.meta.Value; + +public class AArch64HotSpotShenandoahBarrierSetLIRGenerator implements ShenandoahBarrierSetLIRGeneratorTool { + public AArch64HotSpotShenandoahBarrierSetLIRGenerator(GraalHotSpotVMConfig config, HotSpotProviders providers) { + this.config = config; + this.providers = providers; + } + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + private ForeignCallLinkage getReadBarrierStub(LIRGeneratorTool tool, ShenandoahLoadBarrierNode.ReferenceStrength strength, boolean narrow) { + //System.out.println("barrier stub narrow: " + narrow + ", strength: " + strength); + return switch (strength) { + case STRONG -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_NARROW) : + tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER); + case WEAK -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK_NARROW) : + tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK); + case PHANTOM -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW) : + tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM); + }; + } + + @Override + public Value emitLoadReferenceBarrier(LIRGeneratorTool tool, Value obj, Value address, ShenandoahLoadBarrierNode.ReferenceStrength strength, boolean narrow, boolean notNull) { + PlatformKind platformKind = obj.getPlatformKind(); + LIRKind kind = LIRKind.reference(platformKind); + Value result = tool.newVariable(tool.toRegisterKind(kind)); + ForeignCallLinkage callTarget = getReadBarrierStub(tool, strength, narrow); + AllocatableValue object = tool.asAllocatable(obj); + AArch64AddressValue loadAddress = ((AArch64LIRGenerator) tool).asAddressValue(address, AArch64Address.ANY_SIZE); + tool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + tool.append(new AArch64HotSpotShenandoahReadBarrierOp(config, providers, tool.asAllocatable(result), object, loadAddress, callTarget, strength, notNull)); + return result; + } + + @Override + public void emitPreWriteBarrier(LIRGeneratorTool lirTool, Value address, AllocatableValue expectedObject, boolean nonNull) { + AllocatableValue temp = lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)); + // If the assembly must load the value then it's needs a temporary to store it + AllocatableValue temp2 = expectedObject.equals(Value.ILLEGAL) ? lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)) : Value.ILLEGAL; + + // Load the address into a register + AllocatableValue addressValue = lirTool.newVariable(address.getValueKind()); + lirTool.emitMove(addressValue, address); + + ForeignCallLinkage callTarget = lirTool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_PRE_BARRIER); + lirTool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + lirTool.append(new AArch64ShenandoahPreWriteBarrierOp(config, providers, addressValue, expectedObject, temp, temp2, callTarget, nonNull)); + } + + @Override + public void emitCardBarrier(LIRGeneratorTool lirTool, Value address) { + AArch64AddressValue addr = ((AArch64LIRGenerator) lirTool).asAddressValue(address, AArch64Address.ANY_SIZE); + AllocatableValue tmp = lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)); + lirTool.append(new AArch64HotSpotShenandoahCardBarrierOp(config, providers, addr, tmp)); + } + + private static ShenandoahLoadBarrierNode.ReferenceStrength getReferenceStrength(BarrierType barrierType) { + return switch (barrierType) { + case READ, NONE -> ShenandoahLoadBarrierNode.ReferenceStrength.STRONG; + case REFERENCE_GET, WEAK_REFERS_TO -> ShenandoahLoadBarrierNode.ReferenceStrength.WEAK; + case PHANTOM_REFERS_TO -> ShenandoahLoadBarrierNode.ReferenceStrength.PHANTOM; + case ARRAY, FIELD, UNKNOWN, POST_INIT_WRITE, AS_NO_KEEPALIVE_WRITE -> throw GraalError.shouldNotReachHere("Unexpected barrier type: " + barrierType); + }; + } + +// @Override +// public Value emitBarrieredLoad(LIRGeneratorTool tool, LIRKind kind, Value address, LIRFrameState state, MemoryOrderMode memoryOrder, BarrierType barrierType) { +// Value load = tool.getArithmetic().emitLoad(kind, address, state, memoryOrder, MemoryExtendKind.DEFAULT); +// return emitLoadReferenceBarrier(tool, load, address, getReferenceStrength(barrierType)); +// } + +// @Override +// public Value emitAtomicReadAndWrite(LIRGeneratorTool tool, LIRKind readKind, Value address, Value newValue, BarrierType barrierType) { +// Value xchg = tool.emitAtomicReadAndWrite(readKind, address, newValue, barrierType); +// return emitLoadReferenceBarrier(tool, xchg, address, getReferenceStrength(barrierType)); +// } + +// @Override +// public void emitCompareAndSwapOp(LIRGeneratorTool tool, boolean isLogic, Value address, MemoryOrderMode memoryOrder, AArch64Kind memKind, Variable result, AllocatableValue allocatableExpectedValue, AllocatableValue allocatableNewValue, BarrierType barrierType) { +// tool.append(new AArch64AtomicMove.CompareAndSwapOp(memKind, memoryOrder, isLogic, result, allocatableExpectedValue, allocatableNewValue, tool.asAllocatable(address))); +// if (!isLogic) { +// boolean compressed = result.getValueKind().getPlatformKind() == AArch64Kind.DWORD; +// Value lrb = emitLoadReferenceBarrier(tool, result, address, getReferenceStrength(barrierType)); +// if (compressed) { +// tool.append(new AArch64Move.Move(AArch64Kind.DWORD, result, tool.asAllocatable(lrb))); +// } else { +// tool.append(new AArch64Move.Move(AArch64Kind.QWORD, result, tool.asAllocatable(lrb))); +// } +// } +// } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java new file mode 100644 index 000000000000..881e7f9c6379 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.aarch64.AArch64.zr; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +public class AArch64HotSpotShenandoahCardBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahCardBarrierOp.class); + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + @Alive({COMPOSITE}) protected AArch64AddressValue address; + @Temp({REG}) protected AllocatableValue tmp; + protected AArch64HotSpotShenandoahCardBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AArch64AddressValue addr, AllocatableValue tmp) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = addr; + this.tmp = tmp; + } + + @Override + protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + try (AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister(); + AArch64MacroAssembler.ScratchRegister tmp3 = masm.getScratchRegister()) { + Register rtmp1 = asRegister(tmp); + Register rtmp2 = tmp2.getRegister(); + Register rtmp3 = tmp3.getRegister(); + AArch64Address storeAddr = address.toAddress(); + Register rthread = providers.getRegisters().getThreadRegister(); + + // Flatten address if necessary. + Register rAddr; + if (storeAddr.isBaseRegisterOnly()) { + rAddr = storeAddr.getBase(); + } else { + rAddr = rtmp1; + masm.loadAddress(rAddr, storeAddr); + } + + masm.lsr(64, rAddr, rAddr, HotSpotReplacementsUtil.cardTableShift(config)); + + AArch64Address currCTHolderAddr = AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED, rthread, HotSpotReplacementsUtil.shenandoahCardTableOffset(config)); + masm.ldr(64, rtmp2, currCTHolderAddr); + + AArch64Address cardAddr = AArch64Address.createRegisterOffsetAddress(8, rAddr, rtmp2, false); + if (HotSpotReplacementsUtil.useCondCardMark(config)) { + Label alreadyDirty = new Label(); + masm.ldr(8, rtmp3, cardAddr); + masm.cbz(8, rtmp3, alreadyDirty); + masm.str(8, zr, cardAddr); + masm.bind(alreadyDirty); + } else { + masm.str(8, zr, cardAddr); + } + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahReadBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahReadBarrierOp.java new file mode 100644 index 000000000000..45a6892b346c --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahReadBarrierOp.java @@ -0,0 +1,223 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64Assembler; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.aarch64.AArch64Call; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadBarrierNode; + +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +public class AArch64HotSpotShenandoahReadBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahReadBarrierOp.class); + + enum GCStateBitPos { + // Heap has forwarded objects: needs LRB barriers. + HAS_FORWARDED_BITPOS(0), + + // Heap is under marking: needs SATB barriers. + // For generational mode, it means either young or old marking, or both. + MARKING_BITPOS(1), + + // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED) + EVACUATION_BITPOS(2), + + // Heap is under updating: needs no additional barriers. + UPDATE_REFS_BITPOS(3), + + // Heap is under weak-reference/roots processing: needs weak-LRB barriers. + WEAK_ROOTS_BITPOS(4), + + // Young regions are under marking, need SATB barriers. + YOUNG_MARKING_BITPOS(5), + + // Old regions are under marking, need SATB barriers. + OLD_MARKING_BITPOS(6); + + private final int value; + + GCStateBitPos(int val) { + this.value = val; + } + + public int getValue() { + return this.value; + } + } + enum GCState { + HAS_FORWARDED(1 << GCStateBitPos.HAS_FORWARDED_BITPOS.value), + MARKING(1 << GCStateBitPos.MARKING_BITPOS.value), + EVACUATION(1 << GCStateBitPos.EVACUATION_BITPOS.value), + UPDATE_REFS(1 << GCStateBitPos.UPDATE_REFS_BITPOS.value), + WEAK_ROOTS(1 << GCStateBitPos.WEAK_ROOTS_BITPOS.value), + YOUNG_MARKING(1 << GCStateBitPos.YOUNG_MARKING_BITPOS.value), + OLD_MARKING(1 << GCStateBitPos.OLD_MARKING_BITPOS.value); + + private final int value; + GCState(int val) { this.value = val; } + public int getValue() { return this.value; } + } + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + @Def({REG}) protected AllocatableValue result; + @Use({REG}) protected AllocatableValue object; + @Alive({COMPOSITE}) protected AArch64AddressValue loadAddress; + + protected final ForeignCallLinkage callTarget; + + ShenandoahLoadBarrierNode.ReferenceStrength strength; + boolean notNull; + + public AArch64HotSpotShenandoahReadBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue result, AllocatableValue object, AArch64AddressValue loadAddress, + ForeignCallLinkage callTarget, + ShenandoahLoadBarrierNode.ReferenceStrength strength, + boolean notNull) { + super(TYPE); + this.providers = providers; + this.config = config; + this.result = result; + this.object = object; + this.loadAddress = loadAddress; + this.callTarget = callTarget; + this.strength = strength; + this.notNull = notNull; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + // System.out.println("Emitting Shenandoah load reference barrier"); + Register rscratch1 = sc1.getRegister(); + Register objectRegister = asRegister(object); + AArch64Address loadAddr = loadAddress.toAddress(); + Register resultRegister = asRegister(result); + + Register thread = providers.getRegisters().getThreadRegister(); + + Label done = new Label(); + Label cset_check = new Label(); + Label slow_path = new Label(); + + // Move object to result, in case the heap is stable and no barrier needs to be called. + masm.mov(64, resultRegister, objectRegister); + + if (!notNull) { + // Check for object being null. + masm.cbz(64, resultRegister, done); + } + + // Check for heap stability + int gcStateOffset = HotSpotReplacementsUtil.shenandoahGCStateOffset(config); + AArch64Address gcState = masm.makeAddress(8, thread, gcStateOffset); + masm.ldr(8, rscratch1, gcState); + if (strength != ShenandoahLoadBarrierNode.ReferenceStrength.STRONG) { + // This is needed because in a short-cut cycle we may get a trailing + // weak-roots phase but no evacuation/update-refs phase, and during that, + // we need to take the LRB to report null for unreachable weak-refs. + // This is true even for non-cset objects. + // Two tests because HAS_FORWARDED | WEAK_ROOTS currently is not representable + // as a single immediate. + masm.tst(64, rscratch1, GCState.HAS_FORWARDED.value); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, slow_path); + masm.tst(64, rscratch1, GCState.WEAK_ROOTS.value); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, slow_path); + } else { + masm.tst(64, rscratch1, GCState.HAS_FORWARDED.value); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, cset_check); + } + masm.bind(done); + + // Check for object in collection set in an out-of-line mid-path. + if (strength == ShenandoahLoadBarrierNode.ReferenceStrength.STRONG) { + crb.getLIR().addSlowPath(this, () -> { + try (AArch64MacroAssembler.ScratchRegister tmp1 = masm.getScratchRegister(); AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister()) { + Register rtmp1 = tmp1.getRegister(); + Register rtmp2 = tmp2.getRegister(); + masm.bind(cset_check); + masm.mov(rtmp1, HotSpotReplacementsUtil.shenandoahGCCSetFastTestAddr(config)); + masm.lsr(64, rtmp2, objectRegister, HotSpotReplacementsUtil.shenandoahGCRegionSizeBytesShift(config)); + masm.ldr(8, rtmp2, AArch64Address.createRegisterOffsetAddress(8, rtmp1, rtmp2, false)); + masm.cbnz(8, rtmp2, slow_path); + masm.jmp(done); + } + }); + } + // Call runtime slow-path LRB in out-of-line slow-path. + crb.getLIR().addSlowPath(this, () -> { + try (AArch64MacroAssembler.ScratchRegister tmp1 = masm.getScratchRegister(); AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister()) { + Register rtmp1 = tmp1.getRegister(); + Register rtmp2 = tmp2.getRegister(); + masm.bind(slow_path); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + assert cc.getArgumentCount() == 2 : "Expecting callTarget to have only 2 parameters. It has " + cc.getArgumentCount(); + + // Store first argument + AArch64Address cArg0 = (AArch64Address) crb.asAddress(cc.getArgument(0)); + masm.str(64, objectRegister, cArg0); + + // Store second argument + Register addressReg; + if (loadAddr.isBaseRegisterOnly()) { + // Can directly use the base register as the address + addressReg = loadAddr.getBase(); + } else { + addressReg = rtmp1; + masm.loadAddress(addressReg, loadAddr); + } + AArch64Address cArg1 = (AArch64Address) crb.asAddress(cc.getArgument(1)); + masm.str(64, addressReg, cArg1); + + // Make the call + AArch64Call.directCall(crb, masm, callTarget, AArch64Call.isNearCall(callTarget) ? null : rtmp2, null); + + // Retrieve result and move to the result register. + AArch64Address cRet = (AArch64Address) crb.asAddress(cc.getReturn()); + masm.ldr(64, resultRegister, cRet); + masm.jmp(done); + } + }); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64ShenandoahPreWriteBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64ShenandoahPreWriteBarrierOp.java new file mode 100644 index 000000000000..0a344b4e5043 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64ShenandoahPreWriteBarrierOp.java @@ -0,0 +1,189 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64Assembler; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotMacroAssembler; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.aarch64.AArch64Call; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.aarch64.AArch64Move; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.asm.aarch64.AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED; +import static jdk.graal.compiler.core.common.GraalOptions.AssemblyGCBarriersSlowPathOnly; +import static jdk.graal.compiler.core.common.GraalOptions.VerifyAssemblyGCBarriers; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +public class AArch64ShenandoahPreWriteBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64ShenandoahPreWriteBarrierOp.class); + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + @Alive + private Value address; + + @Alive({OperandFlag.REG, OperandFlag.ILLEGAL}) + private Value expectedObject; + + @Temp + private Value temp; + + @Temp({OperandFlag.REG, OperandFlag.ILLEGAL}) + private Value temp2; + + private final ForeignCallLinkage callTarget; + private final boolean nonNull; + + public AArch64ShenandoahPreWriteBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue address, AllocatableValue expectedObject, AllocatableValue temp, AllocatableValue temp2, ForeignCallLinkage callTarget, boolean nonNull) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = address; + assert expectedObject.equals(Value.ILLEGAL) ^ temp2.equals(Value.ILLEGAL) : "only one register is necessary"; + this.expectedObject = expectedObject; + this.temp = temp; + this.temp2 = temp2; + this.callTarget = callTarget; + this.nonNull = nonNull; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) || expectedObject.getPlatformKind().getSizeInBytes() == 8, "expected uncompressed pointer"); + } + + public void loadObject(AArch64MacroAssembler masm, Register preVal, Register immediateAddress) { + if (config.useCompressedOops) { + masm.ldr(32, preVal, AArch64Address.createImmediateAddress(32, IMMEDIATE_SIGNED_UNSCALED, immediateAddress, 0)); + CompressEncoding encoding = config.getOopEncoding(); + AArch64Move.UncompressPointerOp.emitUncompressCode(masm, preVal, preVal, encoding, false, providers.getRegisters().getHeapBaseRegister(), false); + } else { + masm.ldr(64, preVal, AArch64Address.createImmediateAddress(64, IMMEDIATE_SIGNED_UNSCALED, immediateAddress, 0)); + } + } + + @Override + protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + //System.out.println("Emitting Shenandoah SATB barrier"); + Register storeAddress = asRegister(address); + Register thread = providers.getRegisters().getThreadRegister(); + Register tmp = asRegister(temp); + Register previousValue = expectedObject.equals(Value.ILLEGAL) ? asRegister(temp2) : asRegister(expectedObject); + + guaranteeDifferentRegisters(storeAddress, thread, tmp, previousValue); + + Label done = new Label(); + Label midPath = new Label(); + Label runtime = new Label(); + + // Is marking active? + int gcStateOffset = HotSpotReplacementsUtil.shenandoahGCStateOffset(config); + AArch64Address gcState = masm.makeAddress(8, thread, gcStateOffset); + masm.ldr(8, tmp, gcState); + masm.tst(64, tmp, AArch64HotSpotShenandoahReadBarrierOp.GCState.MARKING.getValue()); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, midPath); + masm.bind(done); + + // Out of line mid-path. + crb.getLIR().addSlowPath(this, () -> { + masm.bind(midPath); + + // Do we need to load the previous value? + if (expectedObject.equals(Value.ILLEGAL)) { + loadObject(masm, previousValue, storeAddress); + } + + if (!nonNull) { + // Is the previous value null? + masm.cbz(64, previousValue, done); + } + + if (VerifyAssemblyGCBarriers.getValue(crb.getOptions())) { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register tmp2 = sc1.getRegister(); + verifyOop(masm, previousValue, tmp, tmp2, false, true); + } + } + + if (AssemblyGCBarriersSlowPathOnly.getValue(crb.getOptions())) { + masm.jmp(runtime); + } else { + int satbQueueIndexOffset = HotSpotReplacementsUtil.shenandoahSATBIndexOffset(config); + AArch64Address satbQueueIndex = masm.makeAddress(64, thread, satbQueueIndexOffset); + // tmp := *index_adr + // if tmp == 0 then goto runtime + masm.ldr(64, tmp, satbQueueIndex); + masm.cbz(64, tmp, runtime); + + // tmp := tmp - wordSize + // *index_adr := tmp + // tmp := tmp + *buffer_adr + masm.sub(64, tmp, tmp, 8); + masm.str(64, tmp, satbQueueIndex); + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register scratch1 = sc1.getRegister(); + int satbQueueBufferOffset = HotSpotReplacementsUtil.shenandoahSATBBufferOffset(config); + AArch64Address satbQueueBuffer = masm.makeAddress(64, thread, satbQueueBufferOffset); + masm.ldr(64, scratch1, satbQueueBuffer); + masm.add(64, tmp, tmp, scratch1); + } + + // Record the previous value + masm.str(64, previousValue, masm.makeAddress(64, tmp, 0)); + masm.jmp(done); + } + }); + + // Out of line slow path + crb.getLIR().addSlowPath(this, () -> { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register scratch1 = sc1.getRegister(); + masm.bind(runtime); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + AArch64Address cArg0 = (AArch64Address) crb.asAddress(cc.getArgument(0)); + masm.str(64, previousValue, cArg0); + AArch64Call.directCall(crb, masm, callTarget, AArch64Call.isNearCall(callTarget) ? null : scratch1, null); + masm.jmp(done); + } + }); + } + + private void verifyOop(AArch64MacroAssembler masm, Register previousValue, Register tmp, Register tmp2, boolean compressed, boolean nonNull) { + ((AArch64HotSpotMacroAssembler) masm).verifyOop(previousValue, tmp, tmp2, compressed, nonNull); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/z/AArch64HotSpotZBarrierSetLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/z/AArch64HotSpotZBarrierSetLIRGenerator.java index 73fe5e65d38d..3e9403764cde 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/z/AArch64HotSpotZBarrierSetLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/z/AArch64HotSpotZBarrierSetLIRGenerator.java @@ -113,8 +113,8 @@ static void zUncolor(AArch64MacroAssembler masm, GraalHotSpotVMConfig config, Re * isn't needed by this code otherwise and in some cases the destination register for the zColor * must be customized. */ - @SyncPort(from = "https://github.com/openjdk/jdk/blob/98a93e115137a305aed6b7dbf1d4a7d5906fe77c/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L166-L224", sha1 = "101b4c83516738a04bf6fb3f17bfc78f58ac5784") - @SyncPort(from = "https://github.com/openjdk/jdk/blob/98a93e115137a305aed6b7dbf1d4a7d5906fe77c/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L309-L370", sha1 = "755eb5d52e1ad8c30c9aa9c5f009d35f8c52bb78") + @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L167-L225", sha1 = "101b4c83516738a04bf6fb3f17bfc78f58ac5784") + @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L310-L371", sha1 = "755eb5d52e1ad8c30c9aa9c5f009d35f8c52bb78") static void emitStoreBarrier(CompilationResultBuilder crb, AArch64MacroAssembler masm, LIRInstruction op, @@ -209,7 +209,7 @@ static void emitStoreBarrier(CompilationResultBuilder crb, /** * Try to perform any local store barrier fixups or dispatch to the slow path. */ - @SyncPort(from = "https://github.com/openjdk/jdk/blob/98a93e115137a305aed6b7dbf1d4a7d5906fe77c/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L258-L307", sha1 = "061eaf13b97f69aee4f687ce51e500ac3b37071a") + @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L259-L308", sha1 = "061eaf13b97f69aee4f687ce51e500ac3b37071a") static void storeBarrierMedium(CompilationResultBuilder crb, AArch64MacroAssembler masm, GraalHotSpotVMConfig config, @@ -276,7 +276,7 @@ static void storeBarrierMedium(CompilationResultBuilder crb, /** * Add a value to the store buffer. */ - @SyncPort(from = "https://github.com/openjdk/jdk/blob/98a93e115137a305aed6b7dbf1d4a7d5906fe77c/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L226-L256", sha1 = "b52bb540cf136f455dfac53fece3cc029a240bf2") + @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L227-L257", sha1 = "b52bb540cf136f455dfac53fece3cc029a240bf2") static void storeBarrierBufferAdd(AArch64MacroAssembler masm, GraalHotSpotVMConfig config, AArch64Address refAddr, @@ -322,7 +322,7 @@ static void storeBarrierBufferAdd(AArch64MacroAssembler masm, * done with a special stack-only calling convention that saves and restores all registers * around the call. This simplifies the code generation as no extra registers are required. */ - @SyncPort(from = "https://github.com/openjdk/jdk/blob/98a93e115137a305aed6b7dbf1d4a7d5906fe77c/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L104-L164", sha1 = "2b500d0e7769c719aca0eb4d1707ac0cbf476727") + @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp#L105-L165", sha1 = "2b500d0e7769c719aca0eb4d1707ac0cbf476727") public static void emitLoadBarrier(CompilationResultBuilder crb, AArch64MacroAssembler masm, GraalHotSpotVMConfig config, @@ -391,27 +391,6 @@ public static void emitLoadBarrier(CompilationResultBuilder crb, } } - @Override - public Variable emitBarrieredLoad(LIRGeneratorTool tool, - LIRKind kind, - Value address, - LIRFrameState state, - MemoryOrderMode memoryOrder, - BarrierType barrierType) { - if (kind.getPlatformKind().getVectorLength() == 1) { - GraalError.guarantee(kind.getPlatformKind() == AArch64Kind.QWORD, "ZGC only uses uncompressed oops: %s", kind); - - ForeignCallLinkage callTarget = getReadBarrierStub(barrierType); - AArch64AddressValue loadAddress = ((AArch64LIRGenerator) tool).asAddressValue(address, 64); - Variable result = tool.newVariable(tool.toRegisterKind(kind)); - tool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); - boolean isNotStrong = barrierType == BarrierType.REFERENCE_GET || barrierType == BarrierType.WEAK_REFERS_TO || barrierType == BarrierType.PHANTOM_REFERS_TO; - tool.append(new AArch64HotSpotZReadBarrierOp(result, loadAddress, memoryOrder, state, config, callTarget, isNotStrong)); - return result; - } - throw GraalError.shouldNotReachHere("unhandled"); - } - @Override public void emitCompareAndSwapOp(LIRGeneratorTool tool, boolean isLogicVariant, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/z/AMD64HotSpotZBarrierSetLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/z/AMD64HotSpotZBarrierSetLIRGenerator.java index ec09991ad1e7..f4a95ae2914f 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/z/AMD64HotSpotZBarrierSetLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/z/AMD64HotSpotZBarrierSetLIRGenerator.java @@ -375,12 +375,12 @@ public static void emitLoadBarrier(CompilationResultBuilder crb, } @Override - public Variable emitBarrieredLoad(LIRGeneratorTool tool, - LIRKind kind, - Value address, - LIRFrameState state, - MemoryOrderMode memoryOrder, - BarrierType barrierType) { + public Value emitBarrieredLoad(LIRGeneratorTool tool, + LIRKind kind, + Value address, + LIRFrameState state, + MemoryOrderMode memoryOrder, + BarrierType barrierType) { if (kind.getPlatformKind().getVectorLength() == 1) { GraalError.guarantee(kind.getPlatformKind() == AMD64Kind.QWORD, "ZGC only uses uncompressed oops: %s", kind); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java index 9c12dc8e8775..955945c11e5e 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java @@ -70,6 +70,7 @@ import static jdk.graal.compiler.hotspot.HotSpotBackend.VM_ERROR; import static jdk.graal.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.COMPUTES_REGISTERS_KILLED; import static jdk.graal.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.DESTROYS_ALL_CALLER_SAVE_REGISTERS; +import static jdk.graal.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC.Shenandoah; import static jdk.graal.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC.Z; import static jdk.graal.compiler.hotspot.HotSpotHostBackend.DEOPT_BLOB_UNCOMMON_TRAP; import static jdk.graal.compiler.hotspot.HotSpotHostBackend.DEOPT_BLOB_UNPACK; @@ -106,6 +107,7 @@ import java.util.EnumMap; +import jdk.graal.compiler.debug.GraalError; import org.graalvm.collections.EconomicMap; import org.graalvm.word.LocationIdentity; @@ -228,6 +230,35 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall public static final HotSpotForeignCallDescriptor Z_ARRAY_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, NO_LOCATIONS, "load_barrier_on_oop_array", void.class, long.class, long.class); + /** + * Shenandoah runtime function entries. + */ + + // oopDesc* ShenandoahRuntime::load_reference_barrier_strong(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_strong", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_strong_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_strong_narrow", Object.class, Object.class, Word.class); + + // oopDesc* ShenandoahRuntime::load_reference_barrier_weak(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_WEAK = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_weak", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_weak_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_WEAK_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_weak_narrow", Object.class, Object.class, Word.class); + + // oopDesc* ShenandoahRuntime::load_reference_barrier_phantom(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_PHANTOM = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_phantom", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_phantom_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_phantom_narrow", Object.class, Object.class, Word.class); + + // void ShenandoahRuntime::pre_barrier(JavaThread*, oopDesc*) + public static final HotSpotForeignCallDescriptor SHENANDOAH_PRE_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, NO_SIDE_EFFECT, NO_LOCATIONS, "ShenandoahRuntime::pre_barrier_stack_only", + void.class, Object.class); + /** * Signature of an unsafe {@link System#arraycopy} stub. * @@ -538,6 +569,14 @@ public void initialize(HotSpotProviders providers, OptionValues options) { linkStackOnlyForeignCall(options, providers, G1WBPRECALL_STACK_ONLY, c.writeBarrierPreAddress, PREPEND_THREAD); linkStackOnlyForeignCall(options, providers, G1WBPOSTCALL_STACK_ONLY, c.writeBarrierPostAddress, PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER, c.shenandoahLoadBarrierStrong, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_NARROW, c.shenandoahLoadBarrierStrongNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_WEAK, c.shenandoahLoadBarrierWeak, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_WEAK_NARROW, c.shenandoahLoadBarrierWeakNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_PHANTOM, c.shenandoahLoadBarrierPhantom, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW, c.shenandoahLoadBarrierPhantomNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_PRE_BARRIER, c.shenandoahPreBarrier, PREPEND_THREAD); + linkForeignCall(options, providers, LOG_PRINTF, c.logPrintfAddress, PREPEND_THREAD); linkForeignCall(options, providers, LOG_OBJECT, c.logObjectAddress, PREPEND_THREAD); linkForeignCall(options, providers, LOG_PRIMITIVE, c.logPrimitiveAddress, PREPEND_THREAD); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java index 0da0867c6932..bff11c58f81e 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java @@ -64,6 +64,14 @@ public static CompressionNode uncompress(StructuredGraph graph, ValueNode input, return graph.unique(uncompress(input, encoding)); } + public static HotSpotCompressionNode compressWithoutUnique(StructuredGraph graph, ValueNode input, CompressEncoding encoding) { + return graph.addWithoutUnique(compress(input, encoding)); + } + + public static CompressionNode uncompressWithoutUnique(StructuredGraph graph, ValueNode input, CompressEncoding encoding) { + return graph.addWithoutUnique(uncompress(input, encoding)); + } + private static HotSpotCompressionNode compress(ValueNode input, CompressEncoding encoding) { return new HotSpotCompressionNode(CompressionOp.Compress, input, encoding); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java index 3557f992fae2..a5463e99cacd 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java @@ -717,6 +717,37 @@ public static boolean useCondCardMark(@InjectedParameter GraalHotSpotVMConfig co return config.useCondCardMark; } + @Fold + public static int shenandoahGCStateOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCStateOffset; + } + + @Fold + public static int shenandoahSATBIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahSATBIndexOffset; + } + + @Fold + public static int shenandoahSATBBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahSATBBufferOffset; + } + + @Fold + public static int shenandoahCardTableOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahCardTableOffset; + } + + @Fold + public static int shenandoahGCRegionSizeBytesShift(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCRegionSizeBytesShift; + } + + @Fold + public static long shenandoahGCCSetFastTestAddr(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCCSetFastTestAddress; + } + + public static final LocationIdentity KLASS_SUPER_CHECK_OFFSET_LOCATION = NamedLocationIdentity.immutable("Klass::_super_check_offset"); @Fold diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ReadBarrierSetLIRGeneratorTool.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ReadBarrierSetLIRGeneratorTool.java index 0b236533c1e1..86930fa40403 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ReadBarrierSetLIRGeneratorTool.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ReadBarrierSetLIRGeneratorTool.java @@ -28,7 +28,6 @@ import jdk.graal.compiler.core.common.memory.BarrierType; import jdk.graal.compiler.core.common.memory.MemoryOrderMode; import jdk.graal.compiler.lir.LIRFrameState; -import jdk.graal.compiler.lir.Variable; import jdk.vm.ci.meta.Value; /** @@ -41,5 +40,5 @@ public interface ReadBarrierSetLIRGeneratorTool extends BarrierSetLIRGeneratorTo * Emit a read of a memory location along with the required read barrier.. {@code barrierType} * will always be something besides {@link BarrierType#NONE}. */ - Variable emitBarrieredLoad(LIRGeneratorTool tool, LIRKind kind, Value address, LIRFrameState state, MemoryOrderMode memoryOrder, BarrierType barrierType); + Value emitBarrieredLoad(LIRGeneratorTool tool, LIRKind kind, Value address, LIRFrameState state, MemoryOrderMode memoryOrder, BarrierType barrierType); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java new file mode 100644 index 000000000000..463112a48264 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java @@ -0,0 +1,38 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.lir.gen; + +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadBarrierNode; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +public interface ShenandoahBarrierSetLIRGeneratorTool extends BarrierSetLIRGeneratorTool { + Value emitLoadReferenceBarrier(LIRGeneratorTool tool, Value obj, Value address, ShenandoahLoadBarrierNode.ReferenceStrength strength, boolean narrow, boolean notNull); + + void emitPreWriteBarrier(LIRGeneratorTool lirTool, Value address, AllocatableValue expectedObject, boolean nonNull); + + void emitCardBarrier(LIRGeneratorTool lirTool, Value address); +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java index a8e130b32b44..3698cecc1331 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java @@ -630,6 +630,7 @@ public enum StageFlag { VECTOR_LOWERING, EXPAND_LOGIC, FIXED_READS, + READ_BARRIER_ADDITION, PARTIAL_REDUNDANCY_SCHEDULE, ADDRESS_LOWERING, FINAL_CANONICALIZATION, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java index 86871a540585..eea8fea372ca 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java @@ -25,6 +25,8 @@ */ package jdk.graal.compiler.nodes.gc; +import jdk.graal.compiler.nodes.memory.ReadNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; import org.graalvm.word.LocationIdentity; import jdk.graal.compiler.core.common.memory.BarrierType; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java new file mode 100644 index 000000000000..6e9ec28e61db --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java @@ -0,0 +1,348 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import static jdk.graal.compiler.nodes.NamedLocationIdentity.OFF_HEAP_LOCATION; + +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.nodeinfo.InputType; +import jdk.graal.compiler.nodes.extended.ArrayRangeWrite; +import jdk.graal.compiler.nodes.gc.BarrierSet; +import jdk.graal.compiler.nodes.java.ValueCompareAndSwapNode; +import jdk.graal.compiler.nodes.spi.CoreProviders; +import jdk.graal.compiler.nodes.type.NarrowOopStamp; +import org.graalvm.word.LocationIdentity; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.Node; +import jdk.graal.compiler.nodes.FieldLocationIdentity; +import jdk.graal.compiler.nodes.NamedLocationIdentity; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.StructuredGraph; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.extended.RawStoreNode; +import jdk.graal.compiler.nodes.java.AbstractCompareAndSwapNode; +import jdk.graal.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import jdk.graal.compiler.nodes.memory.AddressableMemoryAccess; +import jdk.graal.compiler.nodes.memory.FixedAccessNode; +import jdk.graal.compiler.nodes.memory.LIRLowerableAccess; +import jdk.graal.compiler.nodes.memory.ReadNode; +import jdk.graal.compiler.nodes.memory.WriteNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.type.StampTool; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaType; + +public class ShenandoahBarrierSet implements BarrierSet { + + private final ResolvedJavaType objectArrayType; + private final ResolvedJavaField referentField; + protected boolean useLoadRefBarrier; + protected boolean useSATBBarrier; + protected boolean useCASBarrier; + protected boolean useCardBarrier; + + public ShenandoahBarrierSet(ResolvedJavaType objectArrayType, ResolvedJavaField referentField) { + this.referentField = referentField; + this.objectArrayType = objectArrayType; + this.useLoadRefBarrier = true; + this.useSATBBarrier = true; + this.useCASBarrier = true; + this.useCardBarrier = true; + } + + @Override + public BarrierType postAllocationInitBarrier(BarrierType original) { + assert original == BarrierType.FIELD || original == BarrierType.ARRAY : "only for write barriers: " + original; + return BarrierType.POST_INIT_WRITE; + } + + @Override + public BarrierType readBarrierType(LocationIdentity location, ValueNode address, Stamp loadStamp) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + assert loadStamp instanceof AbstractObjectStamp : loadStamp; + return BarrierType.READ; + } + if (location.equals(OFF_HEAP_LOCATION)) { + // Off heap locations are never expected to contain objects + assert !loadStamp.isObjectStamp() : location; + return BarrierType.NONE; + } + + if (loadStamp.isObjectStamp()) { + if (address.stamp(NodeView.DEFAULT).isObjectStamp()) { + // A read of an Object from an Object requires a barrier + return BarrierType.READ; + } + + if (address instanceof AddressNode addr) { + if (addr.getBase().stamp(NodeView.DEFAULT).isObjectStamp()) { + // A read of an Object from an Object requires a barrier + return BarrierType.READ; + } + } + // Objects aren't expected to be read from non-heap locations. + throw GraalError.shouldNotReachHere("Unexpected location type " + loadStamp); + } + + boolean mustBeObject = false; + if (location instanceof FieldLocationIdentity fieldLocationIdentity) { + mustBeObject = fieldLocationIdentity.getField().getJavaKind() == JavaKind.Object; + } + assert !mustBeObject : address; + return BarrierType.NONE; + } + + @Override + public BarrierType writeBarrierType(RawStoreNode store) { + if (store.object().isNullConstant()) { + return BarrierType.NONE; + } + return store.needsBarrier() ? readWriteBarrier(store.object(), store.value()) : BarrierType.NONE; + } + + @Override + public BarrierType fieldReadBarrierType(ResolvedJavaField field, JavaKind storageKind) { + if (field.getJavaKind() == JavaKind.Object && field.equals(referentField)) { + return BarrierType.REFERENCE_GET; + } + if (storageKind.isObject()) { + return BarrierType.READ; + } + return BarrierType.NONE; + } + + @Override + public BarrierType fieldWriteBarrierType(ResolvedJavaField field, JavaKind storageKind) { + return storageKind == JavaKind.Object ? BarrierType.FIELD : BarrierType.NONE; + } + + @Override + public BarrierType arrayWriteBarrierType(JavaKind storageKind) { + return storageKind == JavaKind.Object ? BarrierType.ARRAY : BarrierType.NONE; + } + + @Override + public BarrierType readWriteBarrier(ValueNode object, ValueNode value) { + if (value.getStackKind() == JavaKind.Object && object.getStackKind() == JavaKind.Object) { + ResolvedJavaType type = StampTool.typeOrNull(object); + if (type != null && type.isArray()) { + return BarrierType.ARRAY; + } else if (type == null || type.isAssignableFrom(objectArrayType)) { + return BarrierType.ARRAY; + } else { + return BarrierType.FIELD; + } + } + return BarrierType.NONE; + } + + @Override + public boolean hasWriteBarrier() { + return true; + } + + @Override + public boolean hasReadBarrier() { + return true; + } + + @Override + public void addBarriers(FixedAccessNode n, CoreProviders context) { + switch (n) { + case ReadNode readNode -> addReadNodeBarriers(readNode); + case WriteNode write -> addWriteBarriers(write, write.value(), null, true); + case LoweredAtomicReadAndWriteNode atomic -> { + if (useCASBarrier) { + addWriteBarriers(atomic, atomic.getNewValue(), atomic, false); + addReadNodeBarriers(atomic); + } + } + case AbstractCompareAndSwapNode cmpSwap -> { + if (useCASBarrier) { + addWriteBarriers(cmpSwap, cmpSwap.getNewValue(), cmpSwap.getExpectedValue(), false); + if (cmpSwap instanceof ValueCompareAndSwapNode) { + addReadNodeBarriers(cmpSwap); + } + } + } + case ArrayRangeWrite ignored -> GraalError.unimplemented("ArrayRangeWrite is not used"); + case null, default -> + GraalError.guarantee(n.getBarrierType() == BarrierType.NONE, "missed a node that requires a GC barrier: %s", n.getClass()); + } + } + + private void addWriteBarriers(FixedAccessNode node, ValueNode writtenValue, ValueNode expectedValue, boolean doLoad) { + BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case FIELD: + case ARRAY: + case UNKNOWN: + case POST_INIT_WRITE: + case AS_NO_KEEPALIVE_WRITE: + if (isObjectValue(writtenValue)) { + StructuredGraph graph = node.graph(); + boolean init = node.getLocationIdentity().isInit(); + if (!init && barrierType != BarrierType.AS_NO_KEEPALIVE_WRITE && useSATBBarrier) { + // The pre barrier does nothing if the value being read is null, so it can + // be explicitly skipped when this is an initializing store. + // No keep-alive means no need for the pre-barrier. + addShenandoahPreWriteBarrier(node, node.getAddress(), expectedValue, doLoad, graph); + } + if (!init && useCardBarrier && !StampTool.isPointerAlwaysNull(writtenValue)) { + graph.addAfterFixed(node, graph.add(new ShenandoahCardBarrierNode(node.getAddress()))); + } + } + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + private void addLoadReferenceBarrier(FixedAccessNode node, AddressNode address, BarrierType barrierType) { + GraalError.guarantee(node != null, "input value must not be null"); + StructuredGraph graph = node.graph(); + boolean narrow = node.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ValueNode uncompressed = maybeUncompressReference(node, narrow); + ShenandoahLoadBarrierNode lrb = graph.add(new ShenandoahLoadBarrierNode(uncompressed, address, barrierType, narrow)); + //graph.addAfterFixed(node, lrb); + ValueNode compValue = maybeCompressReference(lrb, narrow); + ValueNode newUsage = uncompressed != node ? uncompressed : lrb; + //System.out.println("Replacing node: " + node + " with new node: " + compValue + " when use is not: " + newUsage); + node.replaceAtUsages(compValue, InputType.Value, usage -> usage != newUsage); + } + + private void addReadNodeBarriers(FixedAccessNode node) { + + BarrierType barrierType = node.getBarrierType(); + StructuredGraph graph = node.graph(); + switch (barrierType) { + case NONE -> { + // No barriers required. + } + case REFERENCE_GET -> { + if (useLoadRefBarrier) { + addLoadReferenceBarrier(node, node.getAddress(), barrierType); + } + if (useSATBBarrier) { + boolean narrow = node.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ShenandoahReferentFieldReadBarrierNode barrier = graph.add(new ShenandoahReferentFieldReadBarrierNode(node.getAddress(), maybeUncompressReference(node, narrow))); + graph.addAfterFixed(node, barrier); + } + } + case WEAK_REFERS_TO, PHANTOM_REFERS_TO, READ, ARRAY, FIELD, UNKNOWN -> { + if (useLoadRefBarrier) { + addLoadReferenceBarrier(node, node.getAddress(), barrierType); + } + } + default -> throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + protected ValueNode maybeUncompressReference(ValueNode value, boolean narrow) { + return value; + } + protected ValueNode maybeCompressReference(ValueNode value, boolean narrow) { + return value; + } + + private void addShenandoahPreWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean doLoad, StructuredGraph graph) { + boolean narrow = value != null && value.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ShenandoahPreWriteBarrierNode preBarrier = graph.add(new ShenandoahPreWriteBarrierNode(address, maybeUncompressReference(value, narrow), doLoad)); + GraalError.guarantee(!node.getUsedAsNullCheck(), "trapping null checks are inserted after write barrier insertion: ", node); + node.setStateBefore(null); + graph.addBeforeFixed(node, preBarrier); + } + + private static boolean isObjectValue(ValueNode value) { + return value.stamp(NodeView.DEFAULT) instanceof AbstractObjectStamp; + } + + @Override + public boolean mayNeedPreWriteBarrier(JavaKind storageKind) { + return false; + } + + @Override + public void verifyBarriers(StructuredGraph graph) { + for (Node node : graph.getNodes()) { + if (node instanceof WriteNode write) { + Stamp stamp = write.getAccessStamp(NodeView.DEFAULT); + if (!stamp.isObjectStamp()) { + GraalError.guarantee(write.getBarrierType() == BarrierType.NONE, "no barriers for primitive writes: %s", write); + } + } else if (node instanceof ReadNode || + node instanceof AbstractCompareAndSwapNode || + node instanceof LoweredAtomicReadAndWriteNode) { + LIRLowerableAccess read = (LIRLowerableAccess) node; + Stamp stamp = read.getAccessStamp(NodeView.DEFAULT); + if (!stamp.isObjectStamp()) { + GraalError.guarantee(read.getBarrierType() == BarrierType.NONE, "no barriers for primitive reads: %s", read); + continue; + } + + BarrierType expectedBarrier = barrierForLocation(read.getBarrierType(), read.getLocationIdentity(), JavaKind.Object); + if (expectedBarrier != null) { + GraalError.guarantee(expectedBarrier == read.getBarrierType(), "expected %s but found %s in %s", expectedBarrier, read.getBarrierType(), read); + continue; + } + + ValueNode base = read.getAddress().getBase(); + if (!base.stamp(NodeView.DEFAULT).isObjectStamp()) { + GraalError.guarantee(read.getBarrierType() == BarrierType.NONE, "no barrier for non-heap read: %s", read); + } else { + GraalError.guarantee(read.getBarrierType() == BarrierType.READ, "missing barriers for heap read: %s", read); + } + } else if (node instanceof AddressableMemoryAccess access) { + if (access.getBarrierType() != BarrierType.NONE) { + throw new GraalError("Unexpected memory access with barrier : " + node); + } + } + } + } + + protected BarrierType barrierForLocation(BarrierType currentBarrier, LocationIdentity location, JavaKind storageKind) { + if (location instanceof FieldLocationIdentity fieldLocationIdentity) { + BarrierType barrierType = fieldReadBarrierType(fieldLocationIdentity.getField(), storageKind); + if (barrierType != currentBarrier && barrierType == BarrierType.REFERENCE_GET) { + if (currentBarrier == BarrierType.WEAK_REFERS_TO || currentBarrier == BarrierType.PHANTOM_REFERS_TO) { + return currentBarrier; + } + } + return barrierType; + } + if (location.equals(NamedLocationIdentity.getArrayLocation(JavaKind.Object))) { + return BarrierType.READ; + } + return null; + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java new file mode 100644 index 000000000000..a1e4a7208cab --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java @@ -0,0 +1,56 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_8; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_4; + +@NodeInfo(cycles = CYCLES_8, size = SIZE_4) +public class ShenandoahCardBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahCardBarrierNode.class); + + public ShenandoahCardBarrierNode(AddressNode address) { + super(TYPE, address, null, false); + } + + @Override + public Kind getKind() { + return Kind.POST_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool gen) { + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) gen.getLIRGeneratorTool().getBarrierSet(); + tool.emitCardBarrier(gen.getLIRGeneratorTool(), gen.operand(getAddress())); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadBarrierNode.java new file mode 100644 index 000000000000..aeb7f6ff5695 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadBarrierNode.java @@ -0,0 +1,81 @@ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.Node; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.InputType; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodeinfo.StructuralInput; +import jdk.graal.compiler.nodes.FixedNode; +import jdk.graal.compiler.nodes.FixedWithNextNode; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.calc.UnaryNode; +import jdk.graal.compiler.nodes.memory.LIRLowerableAccess; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.CanonicalizerTool; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_16; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class ShenandoahLoadBarrierNode extends ValueNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahLoadBarrierNode.class); + + @Input + private ValueNode value; + + public enum ReferenceStrength { + STRONG, WEAK, PHANTOM; + }; + + @Input(InputType.Association) + private AddressNode address; + + private final ReferenceStrength strength; + private final boolean narrow; + + private static ReferenceStrength getReferenceStrength(BarrierType barrierType) { + return switch (barrierType) { + case READ, FIELD, ARRAY, NONE -> ReferenceStrength.STRONG; + case REFERENCE_GET, WEAK_REFERS_TO -> ReferenceStrength.WEAK; + case PHANTOM_REFERS_TO -> ReferenceStrength.PHANTOM; + case UNKNOWN, POST_INIT_WRITE, AS_NO_KEEPALIVE_WRITE -> throw GraalError.shouldNotReachHere("Unexpected barrier type: " + barrierType); + }; + } + + public ShenandoahLoadBarrierNode(ValueNode value, AddressNode address, BarrierType barrierType, boolean narrow) { + super(TYPE, value.stamp(NodeView.DEFAULT)); + this.value = value; + this.address = address; + this.strength = getReferenceStrength(barrierType); + this.narrow = narrow; + //System.out.println("New ShenandoahLoadBarrierNode"); + } + + @Override + public void generate(NodeLIRBuilderTool gen) { +// System.out.println("generate LRB for input: " + value); +// System.out.println("usages:"); +// for (Node usage : usages()) { +// System.out.println("usage: " + usage); +// } + Stamp stamp; + if (value instanceof LIRLowerableAccess accessValue) { + stamp = accessValue.getAccessStamp(NodeView.DEFAULT); + } else { + stamp = value.stamp(NodeView.DEFAULT); + } + GraalError.guarantee(stamp.isObjectStamp(), "LRB value must be object"); + boolean notNull = ((AbstractObjectStamp)stamp).nonNull(); + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) gen.getLIRGeneratorTool().getBarrierSet(); + gen.setResult(this, tool.emitLoadReferenceBarrier(gen.getLIRGeneratorTool(), gen.operand(value), gen.operand(address), strength, narrow, notNull)); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahPreWriteBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahPreWriteBarrierNode.java new file mode 100644 index 000000000000..39fac4ba90c5 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahPreWriteBarrierNode.java @@ -0,0 +1,63 @@ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.core.common.type.ObjectStamp; +import jdk.graal.compiler.debug.Assertions; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class ShenandoahPreWriteBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahPreWriteBarrierNode.class); + + private final boolean doLoad; + + public ShenandoahPreWriteBarrierNode(AddressNode address, ValueNode expectedObject, boolean doLoad) { + super(TYPE, address, expectedObject, true); + assert doLoad == (expectedObject == null) : Assertions.errorMessageContext("adr", address, "expectedO", expectedObject, "doLoad", doLoad); + this.doLoad = doLoad; + } + + public ValueNode getExpectedObject() { + return getValue(); + } + + public boolean doLoad() { + return doLoad; + } + + @Override + public Kind getKind() { + return Kind.PRE_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + ValueNode expectedObject = getExpectedObject(); + if (expectedObject == null || !expectedObject.isJavaConstant() || !expectedObject.asJavaConstant().isNull()) { + AllocatableValue operand = Value.ILLEGAL; + boolean nonNull = false; + LIRGeneratorTool lirGen = generator.getLIRGeneratorTool(); + if (expectedObject != null) { + operand = lirGen.asAllocatable(generator.operand(expectedObject)); + nonNull = ((ObjectStamp) expectedObject.stamp(NodeView.DEFAULT)).nonNull(); + GraalError.guarantee(expectedObject.stamp(NodeView.DEFAULT) instanceof ObjectStamp, "expecting full size object"); + } + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) generator.getLIRGeneratorTool().getBarrierSet(); + tool.emitPreWriteBarrier(lirGen, generator.operand(getAddress()), operand, nonNull); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java new file mode 100644 index 000000000000..5481726fa853 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java @@ -0,0 +1,39 @@ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public class ShenandoahReferentFieldReadBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahReferentFieldReadBarrierNode.class); + + public ShenandoahReferentFieldReadBarrierNode(AddressNode address, ValueNode expectedObject) { + super(TYPE, address, expectedObject, true); + } + + public ValueNode getExpectedObject() { + return getValue(); + } + + @Override + public Kind getKind() { + return Kind.PRE_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + LIRGeneratorTool lirGen = generator.getLIRGeneratorTool(); + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) generator.getLIRGeneratorTool().getBarrierSet(); + tool.emitPreWriteBarrier(lirGen, generator.operand(getAddress()), lirGen.asAllocatable(generator.operand(getExpectedObject())), false); + } +}