Jit: Threads: Add Fence, Wait, and Notify

Signed-off-by: Máté Tokodi [email protected]

src/interpreter/ByteCode.h

@@ -586,7 +586,8 @@ class FunctionType;
 #define FOR_EACH_BYTECODE_ATOMIC_OTHER(F) \
     F(MemoryAtomicNotify)                 \
     F(MemoryAtomicWait32)                 \
-    F(MemoryAtomicWait64)
+    F(MemoryAtomicWait64)                 \
+    F(AtomicFence)
 #else // Extended Features
 #define FOR_EACH_BYTECODE_ATOMIC_LOAD_OP(F)
 #define FOR_EACH_BYTECODE_ATOMIC_STORE_OP(F)
@@ -1864,6 +1865,22 @@ class MemoryAtomicNotify : public ByteCode {
     ByteCodeStackOffset m_src1Offset;
     ByteCodeStackOffset m_dstOffset;
 };
+
+class AtomicFence : public ByteCode {
+public:
+    AtomicFence()
+        : ByteCode(Opcode::AtomicFenceOpcode)
+    {
+    }
+
+#if !defined(NDEBUG)
+    void dump(size_t pos)
+    {
+    }
+#endif
+protected:
+    uint32_t m_offset;
+};
 #endif
 
 #if !defined(NDEBUG)

src/interpreter/Interpreter.cpp

@@ -1077,6 +1077,12 @@ ByteCodeStackOffset* Interpreter::interpret(ExecutionState& state,
         ADD_PROGRAM_COUNTER(MemoryAtomicNotify);
         NEXT_INSTRUCTION();
     }
+    DEFINE_OPCODE(AtomicFence)
+    :
+    {
+        ADD_PROGRAM_COUNTER(AtomicFence);
+        NEXT_INSTRUCTION();
+    }
 #endif
 
     // FOR_EACH_BYTECODE_SIMD_ETC_OP

src/jit/Backend.cpp

@@ -1243,6 +1243,10 @@ void JITCompiler::compileFunction(JITFunction* jitFunc, bool isExternal)
             emitAtomic(m_compiler, item->asInstruction());
             break;
         }
+        case Instruction::AtomicSync: {
+            emitAtomicSync(m_compiler, item->asInstruction());
+            break;
+        }
 #endif /* ENABLE_EXTENDED_FEATURES */
         default: {
             switch (item->asInstruction()->opcode()) {

src/jit/ByteCodeParser.cpp

@@ -235,7 +235,10 @@ static bool isFloatGlobal(uint32_t globalIndex, Module* module)
     OL5(OTAtomicRmwI32, /* SSDTT */ I32, I32, I32 | TMP, PTR, I32 | S1)              \
     OL5(OTAtomicRmwI64, /* SSDTT */ I32, I64, I64 | TMP, PTR, I64 | S1)              \
     OL6(OTAtomicRmwCmpxchgI32, /* SSSDTT */ I32, I32, I32, I32 | TMP, PTR, I32 | S1) \
-    OL6(OTAtomicRmwCmpxchgI64, /* SSSDTT */ I32, I64, I64, I64 | TMP, PTR, I64 | S1)
+    OL6(OTAtomicRmwCmpxchgI64, /* SSSDTT */ I32, I64, I64, I64 | TMP, PTR, I64 | S1) \
+    OL6(OTAtomicWaitI32, /* SSSDTT */ I32, I32, I64, I32 | TMP, PTR, I32 | S0)       \
+    OL6(OTAtomicWaitI64, /* SSSDTT */ I32, I64, I64, I32 | TMP, PTR, I64 | S0)       \
+    OL5(OTAtomicNotify, /* SSDTT */ I32, I32, I32 | TMP, PTR, I32 | S0)
 #else /* !ENABLE_EXTENDED_FEATURES */
 #define OPERAND_TYPE_LIST_EXTENDED
 #endif /* ENABLE_EXTENDED_FEATURES */
@@ -1958,6 +1961,53 @@ static void compileFunction(JITCompiler* compiler)
             operands[3] = STACK_OFFSET(atomicRmwCmpxchg->dstOffset());
             break;
         }
+        case ByteCode::AtomicFenceOpcode: {
+            Instruction* instr = compiler->append(byteCode, Instruction::AtomicSync, opcode, 0, 0);
+            break;
+        }
+        case ByteCode::MemoryAtomicWait32Opcode: {
+            Instruction* instr = compiler->append(byteCode, Instruction::AtomicSync, opcode, 3, 1);
+            instr->addInfo(Instruction::kIsCallback);
+
+            MemoryAtomicWait32* memoryAtomicWait = reinterpret_cast<MemoryAtomicWait32*>(byteCode);
+            Operand* operands = instr->operands();
+            instr->setRequiredRegsDescriptor(OTAtomicWaitI32);
+
+            operands[0] = STACK_OFFSET(memoryAtomicWait->src0Offset());
+            operands[1] = STACK_OFFSET(memoryAtomicWait->src1Offset());
+            operands[2] = STACK_OFFSET(memoryAtomicWait->src2Offset());
+            operands[3] = STACK_OFFSET(memoryAtomicWait->dstOffset());
+
+            break;
+        }
+        case ByteCode::MemoryAtomicWait64Opcode: {
+            Instruction* instr = compiler->append(byteCode, Instruction::AtomicSync, opcode, 3, 1);
+            instr->addInfo(Instruction::kIsCallback);
+
+            MemoryAtomicWait64* memoryAtomicWait = reinterpret_cast<MemoryAtomicWait64*>(byteCode);
+            Operand* operands = instr->operands();
+            instr->setRequiredRegsDescriptor(OTAtomicWaitI64);
+
+            operands[0] = STACK_OFFSET(memoryAtomicWait->src0Offset());
+            operands[1] = STACK_OFFSET(memoryAtomicWait->src1Offset());
+            operands[2] = STACK_OFFSET(memoryAtomicWait->src2Offset());
+            operands[3] = STACK_OFFSET(memoryAtomicWait->dstOffset());
+
+            break;
+        }
+        case ByteCode::MemoryAtomicNotifyOpcode: {
+            Instruction* instr = compiler->append(byteCode, Instruction::AtomicSync, opcode, 2, 1);
+            instr->addInfo(Instruction::kIsCallback);
+
+            MemoryAtomicNotify* memoryAtomicWait = reinterpret_cast<MemoryAtomicNotify*>(byteCode);
+            Operand* operands = instr->operands();
+            instr->setRequiredRegsDescriptor(OTAtomicNotify);
+
+            operands[0] = STACK_OFFSET(memoryAtomicWait->src0Offset());
+            operands[1] = STACK_OFFSET(memoryAtomicWait->src1Offset());
+            operands[2] = STACK_OFFSET(memoryAtomicWait->dstOffset());
+            break;
+        }
 #endif /* ENABLE_EXTENDED_FEATURES */
         default: {
             ASSERT_NOT_REACHED();

src/jit/Compiler.h

@@ -109,6 +109,8 @@ class InstructionListItem {
 #if defined(ENABLE_EXTENDED_FEATURES)
         // Atomic memory operations (e.g. I32AtomicRmwAdd, I64AtomicRmw16OrU)
         Atomic,
+        // Thread synchronization operations (e.g. MemoryAtomicWait64, MemoryAtomicNotify)
+        AtomicSync,
 #endif /* ENABLE_EXTENDED_FEATURES */
     };
 

src/jit/MemoryUtilInl.h

@@ -147,3 +147,143 @@ static void emitDataDrop(sljit_compiler* compiler, Instruction* instr)
     sljit_sw addr = GET_FUNC_ADDR(sljit_sw, dropData);
     sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS2V(32, W), SLJIT_IMM, addr);
 }
+
+#if defined(ENABLE_EXTENDED_FEATURES)
+
+static void atomicNotifyCallback(ExecutionContext* context, sljit_s32 memOffset)
+{
+    Instance* instance = context->instance;
+    Memory** memories = reinterpret_cast<Memory**>(reinterpret_cast<uintptr_t>(instance) + Instance::alignedSize());
+    uint32_t result = 0;
+    auto count = context->tmp1[0];
+    memories[0]->atomicNotify(context->state, instance->module()->store(), memOffset, 0, count, &result);
+    context->tmp2[0] = result;
+}
+
+static void atomicWaitCallback(ExecutionContext* context, sljit_s32 memOffset, sljit_s32 size)
+{
+    Instance* instance = context->instance;
+    Memory** memories = reinterpret_cast<Memory**>(reinterpret_cast<uintptr_t>(instance) + Instance::alignedSize());
+    if (!memories[0]->isShared()) {
+        context->tmp1[0] = 1;
+        return;
+    }
+    uint32_t result = 0;
+    int64_t timeout = context->tmp2[0];
+    int64_t expect = context->tmp1[0];
+    if (size == 8) {
+        memories[0]->atomicWait(context->state, instance->module()->store(), memOffset, 0, expect, timeout, &result);
+    } else {
+        memories[0]->atomicWait(context->state, instance->module()->store(), memOffset, 0, (int32_t)expect, timeout, &result);
+    }
+    context->tmp2[0] = result;
+    context->tmp1[0] = 0;
+}
+
+static void emitAtomicSync(sljit_compiler* compiler, Instruction* instr)
+{
+    CompileContext* context = CompileContext::get(compiler);
+    sljit_s32 size = 0;
+
+    switch (instr->opcode()) {
+    case ByteCode::AtomicFenceOpcode: {
+        sljit_emit_op0(compiler, SLJIT_MEMORY_BARRIER);
+        break;
+    }
+    case ByteCode::MemoryAtomicWait64Opcode: {
+        size = 8;
+        FALLTHROUGH;
+    }
+    case ByteCode::MemoryAtomicWait32Opcode: {
+        if (size == 0) {
+            size = 4;
+        }
+        MemoryAtomicWait32* atomicWait32Operation = reinterpret_cast<MemoryAtomicWait32*>(instr->byteCode());
+        sljit_s32 offset = atomicWait32Operation->offset();
+
+        Operand* operands = instr->operands();
+        MemAddress addr(MemAddress::CheckNaturalAlignment, instr->requiredReg(0), instr->requiredReg(1), instr->requiredReg(2));
+        addr.check(compiler, operands, offset, size);
+
+#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
+        JITArgPair expectedPair;
+#endif /* SLJIT_32BIT_ARCHITECTURE */
+        JITArg expected;
+
+        JITArg memOffset(operands + 0);
+#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
+        if (instr->opcode() == ByteCode::MemoryAtomicWait64Opcode) {
+            expectedPair = JITArgPair(operands + 1);
+        } else {
+            expected = JITArg(operands + 1);
+        }
+        JITArgPair timeout(operands + 2);
+#else /* !SLJIT_32BIT_ARCHITECTURE */
+        expected = JITArg(operands + 1);
+        JITArg timeout(operands + 2);
+#endif /* SLJIT_32BIT_ARCHITECTURE */
+        JITArg dst(operands + 3);
+
+        struct sljit_jump* memoryShared;
+
+#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
+        if (instr->opcode() == ByteCode::MemoryAtomicWait64Opcode) {
+            sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET, expectedPair.arg1, expectedPair.arg1w);
+            sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET, expectedPair.arg2, expectedPair.arg2w);
+        } else {
+            sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), expected.arg, expected.argw);
+        }
+#else /* !SLJIT_32BIT_ARCHITECTURE */
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), expected.arg, expected.argw);
+#endif /* SLJIT_32BIT_ARCHITECTURE */
+
+#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_LOW_OFFSET, timeout.arg1, timeout.arg1w);
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_HIGH_OFFSET, timeout.arg2, timeout.arg2w);
+#else /* !SLJIT_32BIT_ARCHITECTURE */
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2), timeout.arg, timeout.argw);
+#endif /* SLJIT_32BIT_ARCHITECTURE */
+
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, memOffset.arg, memOffset.argw);
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, kContextReg, 0);
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, size);
+
+        sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3V(P, W, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, atomicWaitCallback));
+
+        memoryShared = sljit_emit_cmp(compiler, SLJIT_EQUAL, SLJIT_IMM, 0, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1));
+        context->appendTrapJump(ExecutionContext::ExpectedSharedMemError, sljit_emit_jump(compiler, SLJIT_JUMP));
+        sljit_set_label(memoryShared, sljit_emit_label(compiler));
+
+        sljit_emit_op1(compiler, SLJIT_MOV, dst.arg, dst.argw, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2));
+        break;
+    }
+    case ByteCode::MemoryAtomicNotifyOpcode: {
+        MemoryAtomicNotify* atomicNotifyOperation = reinterpret_cast<MemoryAtomicNotify*>(instr->byteCode());
+        sljit_s32 offset = atomicNotifyOperation->offset();
+
+        Operand* operands = instr->operands();
+        MemAddress addr(MemAddress::CheckNaturalAlignment, instr->requiredReg(0), instr->requiredReg(1), instr->requiredReg(2));
+        addr.check(compiler, operands, offset, 4);
+
+        JITArg memOffset(operands + 0);
+        JITArg count(operands + 1);
+        JITArg dst(operands + 2);
+
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), count.arg, count.argw);
+
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, memOffset.arg, memOffset.argw);
+        sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, kContextReg, 0);
+
+        sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS2V(P, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, atomicNotifyCallback));
+
+        sljit_emit_op1(compiler, SLJIT_MOV, dst.arg, dst.argw, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2));
+        break;
+    }
+    default: {
+        RELEASE_ASSERT_NOT_REACHED();
+        break;
+    }
+    }
+}
+
+#endif /* ENABLE_EXTENDED_FEATURES */

src/parser/WASMParser.cpp

@@ -2275,7 +2275,7 @@ class WASMBinaryReader : public wabt::WASMBinaryReaderDelegate {
 
     virtual void OnAtomicFenceExpr(uint32_t consistency_model) override
     {
-        // FIXME do nothing
+        pushByteCode(Walrus::AtomicFence(), WASMOpcode::AtomicFenceOpcode);
     }
 
     virtual void OnAtomicNotifyExpr(int opcode, Index memidx, Address alignmentLog2, Address offset) override

src/runtime/JITExec.cpp

@@ -83,6 +83,9 @@ ByteCodeStackOffset* JITFunction::call(ExecutionState& state, Instance* instance
         case ExecutionContext::UnalignedAtomicError:
             Trap::throwException(state, "unaligned atomic");
             return resultOffsets;
+        case ExecutionContext::ExpectedSharedMemError:
+            Trap::throwException(state, "expected shared memory");
+            return resultOffsets;
 #endif /* ENABLE_EXTENDED_FEATURES */
         default:
             Trap::throwException(state, "unknown exception");

src/runtime/JITExec.h

@@ -54,6 +54,7 @@ struct ExecutionContext {
         UnreachableError,
 #if defined(ENABLE_EXTENDED_FEATURES)
         UnalignedAtomicError,
+        ExpectedSharedMemError,
 #endif /* ENABLE_EXTENDED_FEATURES */
 
         // These three in this order must be the last items of the list.