diff --git a/src/interpreter/ByteCode.h b/src/interpreter/ByteCode.h index 5e040e390..d07a91a3d 100644 --- a/src/interpreter/ByteCode.h +++ b/src/interpreter/ByteCode.h @@ -586,7 +586,8 @@ class FunctionType; #define FOR_EACH_BYTECODE_ATOMIC_OTHER(F) \ F(MemoryAtomicNotify) \ F(MemoryAtomicWait32) \ - F(MemoryAtomicWait64) + F(MemoryAtomicWait64) \ + F(AtomicFence) #else // Extended Features #define FOR_EACH_BYTECODE_ATOMIC_LOAD_OP(F) #define FOR_EACH_BYTECODE_ATOMIC_STORE_OP(F) @@ -1864,6 +1865,22 @@ class MemoryAtomicNotify : public ByteCode { ByteCodeStackOffset m_src1Offset; ByteCodeStackOffset m_dstOffset; }; + +class AtomicFence : public ByteCode { +public: + AtomicFence() + : ByteCode(Opcode::AtomicFenceOpcode) + { + } + +#if !defined(NDEBUG) + void dump(size_t pos) + { + } +#endif +protected: + uint32_t m_offset; +}; #endif #if !defined(NDEBUG) diff --git a/src/interpreter/Interpreter.cpp b/src/interpreter/Interpreter.cpp index 3e617a46d..65c3fc215 100644 --- a/src/interpreter/Interpreter.cpp +++ b/src/interpreter/Interpreter.cpp @@ -1077,6 +1077,12 @@ ByteCodeStackOffset* Interpreter::interpret(ExecutionState& state, ADD_PROGRAM_COUNTER(MemoryAtomicNotify); NEXT_INSTRUCTION(); } + DEFINE_OPCODE(AtomicFence) + : + { + ADD_PROGRAM_COUNTER(AtomicFence); + NEXT_INSTRUCTION(); + } #endif // FOR_EACH_BYTECODE_SIMD_ETC_OP diff --git a/src/jit/Backend.cpp b/src/jit/Backend.cpp index 6c3207abc..2e0596bbb 100644 --- a/src/jit/Backend.cpp +++ b/src/jit/Backend.cpp @@ -1243,6 +1243,10 @@ void JITCompiler::compileFunction(JITFunction* jitFunc, bool isExternal) emitAtomic(m_compiler, item->asInstruction()); break; } + case Instruction::AtomicSync: { + emitAtomicSync(m_compiler, item->asInstruction()); + break; + } #endif /* ENABLE_EXTENDED_FEATURES */ default: { switch (item->asInstruction()->opcode()) { diff --git a/src/jit/ByteCodeParser.cpp b/src/jit/ByteCodeParser.cpp index 947bd25cd..331a53b9e 100644 --- a/src/jit/ByteCodeParser.cpp +++ b/src/jit/ByteCodeParser.cpp @@ -235,7 +235,10 @@ static bool isFloatGlobal(uint32_t globalIndex, Module* module) OL5(OTAtomicRmwI32, /* SSDTT */ I32, I32, I32 | TMP, PTR, I32 | S1) \ OL5(OTAtomicRmwI64, /* SSDTT */ I32, I64, I64 | TMP, PTR, I64 | S1) \ OL6(OTAtomicRmwCmpxchgI32, /* SSSDTT */ I32, I32, I32, I32 | TMP, PTR, I32 | S1) \ - OL6(OTAtomicRmwCmpxchgI64, /* SSSDTT */ I32, I64, I64, I64 | TMP, PTR, I64 | S1) + OL6(OTAtomicRmwCmpxchgI64, /* SSSDTT */ I32, I64, I64, I64 | TMP, PTR, I64 | S1) \ + OL6(OTAtomicWaitI32, /* SSSDTT */ I32, I32, I64, I32 | TMP, PTR, I32 | S0) \ + OL6(OTAtomicWaitI64, /* SSSDTT */ I32, I64, I64, I32 | TMP, PTR, I64 | S0) \ + OL5(OTAtomicNotify, /* SSDTT */ I32, I32, I32 | TMP, PTR, I32 | S0) #else /* !ENABLE_EXTENDED_FEATURES */ #define OPERAND_TYPE_LIST_EXTENDED #endif /* ENABLE_EXTENDED_FEATURES */ @@ -1958,6 +1961,53 @@ static void compileFunction(JITCompiler* compiler) operands[3] = STACK_OFFSET(atomicRmwCmpxchg->dstOffset()); break; } + case ByteCode::AtomicFenceOpcode: { + Instruction* instr = compiler->append(byteCode, Instruction::AtomicSync, opcode, 0, 0); + break; + } + case ByteCode::MemoryAtomicWait32Opcode: { + Instruction* instr = compiler->append(byteCode, Instruction::AtomicSync, opcode, 3, 1); + instr->addInfo(Instruction::kIsCallback); + + MemoryAtomicWait32* memoryAtomicWait = reinterpret_cast(byteCode); + Operand* operands = instr->operands(); + instr->setRequiredRegsDescriptor(OTAtomicWaitI32); + + operands[0] = STACK_OFFSET(memoryAtomicWait->src0Offset()); + operands[1] = STACK_OFFSET(memoryAtomicWait->src1Offset()); + operands[2] = STACK_OFFSET(memoryAtomicWait->src2Offset()); + operands[3] = STACK_OFFSET(memoryAtomicWait->dstOffset()); + + break; + } + case ByteCode::MemoryAtomicWait64Opcode: { + Instruction* instr = compiler->append(byteCode, Instruction::AtomicSync, opcode, 3, 1); + instr->addInfo(Instruction::kIsCallback); + + MemoryAtomicWait64* memoryAtomicWait = reinterpret_cast(byteCode); + Operand* operands = instr->operands(); + instr->setRequiredRegsDescriptor(OTAtomicWaitI64); + + operands[0] = STACK_OFFSET(memoryAtomicWait->src0Offset()); + operands[1] = STACK_OFFSET(memoryAtomicWait->src1Offset()); + operands[2] = STACK_OFFSET(memoryAtomicWait->src2Offset()); + operands[3] = STACK_OFFSET(memoryAtomicWait->dstOffset()); + + break; + } + case ByteCode::MemoryAtomicNotifyOpcode: { + Instruction* instr = compiler->append(byteCode, Instruction::AtomicSync, opcode, 2, 1); + instr->addInfo(Instruction::kIsCallback); + + MemoryAtomicNotify* memoryAtomicWait = reinterpret_cast(byteCode); + Operand* operands = instr->operands(); + instr->setRequiredRegsDescriptor(OTAtomicNotify); + + operands[0] = STACK_OFFSET(memoryAtomicWait->src0Offset()); + operands[1] = STACK_OFFSET(memoryAtomicWait->src1Offset()); + operands[2] = STACK_OFFSET(memoryAtomicWait->dstOffset()); + break; + } #endif /* ENABLE_EXTENDED_FEATURES */ default: { ASSERT_NOT_REACHED(); diff --git a/src/jit/Compiler.h b/src/jit/Compiler.h index 3bf40d012..decf9ddf9 100644 --- a/src/jit/Compiler.h +++ b/src/jit/Compiler.h @@ -109,6 +109,8 @@ class InstructionListItem { #if defined(ENABLE_EXTENDED_FEATURES) // Atomic memory operations (e.g. I32AtomicRmwAdd, I64AtomicRmw16OrU) Atomic, + // Thread synchronization operations (e.g. MemoryAtomicWait64, MemoryAtomicNotify) + AtomicSync, #endif /* ENABLE_EXTENDED_FEATURES */ }; diff --git a/src/jit/MemoryUtilInl.h b/src/jit/MemoryUtilInl.h index 9f912cb7f..1f200ef2b 100644 --- a/src/jit/MemoryUtilInl.h +++ b/src/jit/MemoryUtilInl.h @@ -147,3 +147,143 @@ static void emitDataDrop(sljit_compiler* compiler, Instruction* instr) sljit_sw addr = GET_FUNC_ADDR(sljit_sw, dropData); sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS2V(32, W), SLJIT_IMM, addr); } + +#if defined(ENABLE_EXTENDED_FEATURES) + +static void atomicNotifyCallback(ExecutionContext* context, sljit_s32 memOffset) +{ + Instance* instance = context->instance; + Memory** memories = reinterpret_cast(reinterpret_cast(instance) + Instance::alignedSize()); + uint32_t result = 0; + auto count = context->tmp1[0]; + memories[0]->atomicNotify(context->state, instance->module()->store(), memOffset, 0, count, &result); + context->tmp2[0] = result; +} + +static void atomicWaitCallback(ExecutionContext* context, sljit_s32 memOffset, sljit_s32 size) +{ + Instance* instance = context->instance; + Memory** memories = reinterpret_cast(reinterpret_cast(instance) + Instance::alignedSize()); + if (!memories[0]->isShared()) { + context->tmp1[0] = 1; + return; + } + uint32_t result = 0; + int64_t timeout = context->tmp2[0]; + int64_t expect = context->tmp1[0]; + if (size == 8) { + memories[0]->atomicWait(context->state, instance->module()->store(), memOffset, 0, expect, timeout, &result); + } else { + memories[0]->atomicWait(context->state, instance->module()->store(), memOffset, 0, (int32_t)expect, timeout, &result); + } + context->tmp2[0] = result; + context->tmp1[0] = 0; +} + +static void emitAtomicSync(sljit_compiler* compiler, Instruction* instr) +{ + CompileContext* context = CompileContext::get(compiler); + sljit_s32 size = 0; + + switch (instr->opcode()) { + case ByteCode::AtomicFenceOpcode: { + sljit_emit_op0(compiler, SLJIT_MEMORY_BARRIER); + break; + } + case ByteCode::MemoryAtomicWait64Opcode: { + size = 8; + FALLTHROUGH; + } + case ByteCode::MemoryAtomicWait32Opcode: { + if (size == 0) { + size = 4; + } + MemoryAtomicWait32* atomicWait32Operation = reinterpret_cast(instr->byteCode()); + sljit_s32 offset = atomicWait32Operation->offset(); + + Operand* operands = instr->operands(); + MemAddress addr(MemAddress::CheckNaturalAlignment, instr->requiredReg(0), instr->requiredReg(1), instr->requiredReg(2)); + addr.check(compiler, operands, offset, size); + +#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) + JITArgPair expectedPair; +#endif /* SLJIT_32BIT_ARCHITECTURE */ + JITArg expected; + + JITArg memOffset(operands + 0); +#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) + if (instr->opcode() == ByteCode::MemoryAtomicWait64Opcode) { + expectedPair = JITArgPair(operands + 1); + } else { + expected = JITArg(operands + 1); + } + JITArgPair timeout(operands + 2); +#else /* !SLJIT_32BIT_ARCHITECTURE */ + expected = JITArg(operands + 1); + JITArg timeout(operands + 2); +#endif /* SLJIT_32BIT_ARCHITECTURE */ + JITArg dst(operands + 3); + + struct sljit_jump* memoryShared; + +#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) + if (instr->opcode() == ByteCode::MemoryAtomicWait64Opcode) { + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET, expectedPair.arg1, expectedPair.arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET, expectedPair.arg2, expectedPair.arg2w); + } else { + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), expected.arg, expected.argw); + } +#else /* !SLJIT_32BIT_ARCHITECTURE */ + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), expected.arg, expected.argw); +#endif /* SLJIT_32BIT_ARCHITECTURE */ + +#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_LOW_OFFSET, timeout.arg1, timeout.arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_HIGH_OFFSET, timeout.arg2, timeout.arg2w); +#else /* !SLJIT_32BIT_ARCHITECTURE */ + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2), timeout.arg, timeout.argw); +#endif /* SLJIT_32BIT_ARCHITECTURE */ + + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, memOffset.arg, memOffset.argw); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, kContextReg, 0); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, size); + + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3V(P, W, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, atomicWaitCallback)); + + memoryShared = sljit_emit_cmp(compiler, SLJIT_EQUAL, SLJIT_IMM, 0, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1)); + context->appendTrapJump(ExecutionContext::ExpectedSharedMemError, sljit_emit_jump(compiler, SLJIT_JUMP)); + sljit_set_label(memoryShared, sljit_emit_label(compiler)); + + sljit_emit_op1(compiler, SLJIT_MOV, dst.arg, dst.argw, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2)); + break; + } + case ByteCode::MemoryAtomicNotifyOpcode: { + MemoryAtomicNotify* atomicNotifyOperation = reinterpret_cast(instr->byteCode()); + sljit_s32 offset = atomicNotifyOperation->offset(); + + Operand* operands = instr->operands(); + MemAddress addr(MemAddress::CheckNaturalAlignment, instr->requiredReg(0), instr->requiredReg(1), instr->requiredReg(2)); + addr.check(compiler, operands, offset, 4); + + JITArg memOffset(operands + 0); + JITArg count(operands + 1); + JITArg dst(operands + 2); + + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), count.arg, count.argw); + + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, memOffset.arg, memOffset.argw); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, kContextReg, 0); + + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS2V(P, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, atomicNotifyCallback)); + + sljit_emit_op1(compiler, SLJIT_MOV, dst.arg, dst.argw, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2)); + break; + } + default: { + RELEASE_ASSERT_NOT_REACHED(); + break; + } + } +} + +#endif /* ENABLE_EXTENDED_FEATURES */ diff --git a/src/parser/WASMParser.cpp b/src/parser/WASMParser.cpp index d7114559c..b4e56105e 100644 --- a/src/parser/WASMParser.cpp +++ b/src/parser/WASMParser.cpp @@ -2275,7 +2275,7 @@ class WASMBinaryReader : public wabt::WASMBinaryReaderDelegate { virtual void OnAtomicFenceExpr(uint32_t consistency_model) override { - // FIXME do nothing + pushByteCode(Walrus::AtomicFence(), WASMOpcode::AtomicFenceOpcode); } virtual void OnAtomicNotifyExpr(int opcode, Index memidx, Address alignmentLog2, Address offset) override diff --git a/src/runtime/JITExec.cpp b/src/runtime/JITExec.cpp index ca58add83..6d6ecc4c1 100644 --- a/src/runtime/JITExec.cpp +++ b/src/runtime/JITExec.cpp @@ -83,6 +83,9 @@ ByteCodeStackOffset* JITFunction::call(ExecutionState& state, Instance* instance case ExecutionContext::UnalignedAtomicError: Trap::throwException(state, "unaligned atomic"); return resultOffsets; + case ExecutionContext::ExpectedSharedMemError: + Trap::throwException(state, "expected shared memory"); + return resultOffsets; #endif /* ENABLE_EXTENDED_FEATURES */ default: Trap::throwException(state, "unknown exception"); diff --git a/src/runtime/JITExec.h b/src/runtime/JITExec.h index 9e6a745d2..06b112fbd 100644 --- a/src/runtime/JITExec.h +++ b/src/runtime/JITExec.h @@ -54,6 +54,7 @@ struct ExecutionContext { UnreachableError, #if defined(ENABLE_EXTENDED_FEATURES) UnalignedAtomicError, + ExpectedSharedMemError, #endif /* ENABLE_EXTENDED_FEATURES */ // These three in this order must be the last items of the list. diff --git a/third_party/sljit b/third_party/sljit index f9f512c08..2c105e246 160000 --- a/third_party/sljit +++ b/third_party/sljit @@ -1 +1 @@ -Subproject commit f9f512c0809dc9b29a580ca6b0204ea96620d237 +Subproject commit 2c105e2461b0d5b6c9c632753522457ca442f9dd