diff --git a/src/jit/Backend.cpp b/src/jit/Backend.cpp index 2ff5591c4..082d2e987 100644 --- a/src/jit/Backend.cpp +++ b/src/jit/Backend.cpp @@ -223,6 +223,7 @@ CompileContext::CompileContext(Module* module, JITCompiler* compiler) , shuffleOffset(0) #endif /* SLJIT_CONFIG_X86 */ , stackTmpStart(0) + , stackMemoryStart(0) , nextTryBlock(0) , currentTryBlock(InstanceConstData::globalTryBlock) , trapBlocksStart(0) @@ -1021,6 +1022,7 @@ JITCompiler::JITCompiler(Module* module, uint32_t JITFlags) , m_savedIntegerRegCount(0) , m_savedFloatRegCount(0) , m_stackTmpSize(0) + , m_useMemory0(false) { if (module->m_jitModule != nullptr) { ASSERT(module->m_jitModule->m_instanceConstData != nullptr); @@ -1038,6 +1040,10 @@ void JITCompiler::compileFunction(JITFunction* jitFunc, bool isExternal) m_functionList.push_back(FunctionList(jitFunc, isExternal, m_branchTableSize)); + sljit_uw stackTmpStart = m_useMemory0 ? sizeof(Memory::TargetBuffer) : 0; + // Align data. + m_context.stackTmpStart = static_cast((stackTmpStart + sizeof(sljit_sw) - 1) & ~(sizeof(sljit_sw) - 1)); + if (m_compiler == nullptr) { // First compiled function. m_compiler = sljit_create_compiler(nullptr); @@ -1465,6 +1471,7 @@ void JITCompiler::clear() m_last = nullptr; m_branchTableSize = 0; m_stackTmpSize = 0; + m_useMemory0 = false; #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) m_context.shuffleOffset = 0; #endif /* SLJIT_CONFIG_X86 */ @@ -1509,7 +1516,27 @@ void JITCompiler::emitProlog() sljit_emit_enter(m_compiler, options, SLJIT_ARGS0(P), SLJIT_NUMBER_OF_SCRATCH_REGISTERS | SLJIT_ENTER_FLOAT(SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS), - (m_savedIntegerRegCount + 2) | SLJIT_ENTER_FLOAT(m_savedFloatRegCount), m_stackTmpSize); + (m_savedIntegerRegCount + 2) | SLJIT_ENTER_FLOAT(m_savedFloatRegCount), m_context.stackTmpStart + m_stackTmpSize); + + if (hasMemory0()) { + sljit_sw stackMemoryStart = m_context.stackMemoryStart; + ASSERT(m_context.stackTmpStart >= stackMemoryStart + static_cast(sizeof(Memory::TargetBuffer))); + + sljit_emit_op1(m_compiler, SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(kContextReg), OffsetOfContextField(instance)); + sljit_emit_op1(m_compiler, SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_R0), Instance::alignedSize()); + + sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R0), offsetof(Memory, m_targetBuffers)); + sljit_emit_op1(m_compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), offsetof(Memory, m_sizeInByte) + WORD_LOW_OFFSET); + sljit_get_local_base(m_compiler, SLJIT_MEM1(SLJIT_R0), offsetof(Memory, m_targetBuffers), stackMemoryStart); + sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_R0), offsetof(Memory, m_buffer)); + +#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) + sljit_emit_op1(m_compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackMemoryStart + offsetof(Memory::TargetBuffer, sizeInByte) + WORD_HIGH_OFFSET, SLJIT_IMM, 0); +#endif /* SLJIT_32BIT_ARCHITECTURE */ + sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_SP), stackMemoryStart + offsetof(Memory::TargetBuffer, prev), SLJIT_R1, 0); + sljit_emit_op1(m_compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackMemoryStart + offsetof(Memory::TargetBuffer, sizeInByte) + WORD_LOW_OFFSET, SLJIT_R2, 0); + sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_SP), stackMemoryStart + offsetof(Memory::TargetBuffer, buffer), SLJIT_R0, 0); + } m_context.branchTableOffset = 0; size_t size = func.branchTableSize * sizeof(sljit_up); @@ -1530,6 +1557,20 @@ void JITCompiler::emitProlog() } } +void JITCompiler::emitRestoreMemories() +{ + if (!hasMemory0()) { + return; + } + + sljit_sw stackMemoryStart = m_context.stackMemoryStart; + + sljit_emit_op1(m_compiler, SLJIT_MOV, SLJIT_R1, 0, SLJIT_MEM1(kContextReg), OffsetOfContextField(instance)); + sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), stackMemoryStart + offsetof(Memory::TargetBuffer, prev)); + sljit_emit_op1(m_compiler, SLJIT_MOV, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R1), Instance::alignedSize()); + sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_R1), offsetof(Memory, m_targetBuffers), SLJIT_R2, 0); +} + void JITCompiler::emitEpilog() { FunctionList& func = m_functionList.back(); @@ -1547,6 +1588,7 @@ void JITCompiler::emitEpilog() m_context.earlyReturns.clear(); } + emitRestoreMemories(); sljit_emit_return(m_compiler, SLJIT_MOV_P, SLJIT_R0, 0); m_context.emitSlowCases(m_compiler); @@ -1607,6 +1649,8 @@ void JITCompiler::emitEpilog() sljit_emit_op_dst(m_compiler, SLJIT_GET_RETURN_ADDRESS, SLJIT_R1, 0); sljit_emit_op1(m_compiler, SLJIT_MOV, SLJIT_R0, 0, kContextReg, 0); sljit_emit_icall(m_compiler, SLJIT_CALL, SLJIT_ARGS2(W, W, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, getTrapHandler)); + + emitRestoreMemories(); sljit_emit_return_to(m_compiler, SLJIT_R0, 0); while (trapJumpIndex < trapJumps.size()) { diff --git a/src/jit/ByteCodeParser.cpp b/src/jit/ByteCodeParser.cpp index d9fec64b3..4e5a25ab4 100644 --- a/src/jit/ByteCodeParser.cpp +++ b/src/jit/ByteCodeParser.cpp @@ -1019,12 +1019,14 @@ static void compileFunction(JITCompiler* compiler) case ByteCode::Load32Opcode: { group = Instruction::Load; paramType = ParamTypes::ParamSrcDst; + compiler->useMemory0(); requiredInit = OTLoadI32; break; } case ByteCode::Load64Opcode: { group = Instruction::Load; paramType = ParamTypes::ParamSrcDst; + compiler->useMemory0(); requiredInit = OTLoadI64; break; } @@ -1044,6 +1046,7 @@ static void compileFunction(JITCompiler* compiler) case ByteCode::I64Load32UOpcode: { group = Instruction::Load; paramType = ParamTypes::ParamSrcDstValue; + compiler->useMemory0(); if (requiredInit == OTNone) { requiredInit = OTLoadI64; } @@ -1066,6 +1069,7 @@ static void compileFunction(JITCompiler* compiler) case ByteCode::V128Load64ZeroOpcode: { group = Instruction::Load; paramType = ParamTypes::ParamSrcDstValue; + compiler->useMemory0(); if (opcode == ByteCode::F32LoadOpcode) requiredInit = OTLoadF32; @@ -1082,6 +1086,7 @@ static void compileFunction(JITCompiler* compiler) SIMDMemoryLoad* loadOperation = reinterpret_cast(byteCode); Instruction* instr = compiler->append(byteCode, Instruction::LoadLaneSIMD, opcode, 2, 1); instr->setRequiredRegsDescriptor(OTLoadLaneV128); + compiler->useMemory0(); Operand* operands = instr->operands(); operands[0] = STACK_OFFSET(loadOperation->src0Offset()); @@ -1092,12 +1097,14 @@ static void compileFunction(JITCompiler* compiler) case ByteCode::Store32Opcode: { group = Instruction::Store; paramType = ParamTypes::ParamSrc2; + compiler->useMemory0(); requiredInit = OTStoreI32; break; } case ByteCode::Store64Opcode: { group = Instruction::Store; paramType = ParamTypes::ParamSrc2; + compiler->useMemory0(); requiredInit = OTStoreI64; break; } @@ -1117,6 +1124,7 @@ static void compileFunction(JITCompiler* compiler) case ByteCode::I64StoreOpcode: { group = Instruction::Store; paramType = ParamTypes::ParamSrc2Value; + compiler->useMemory0(); if (requiredInit == OTNone) { requiredInit = OTStoreI64; } @@ -1127,6 +1135,7 @@ static void compileFunction(JITCompiler* compiler) case ByteCode::V128StoreOpcode: { group = Instruction::Store; paramType = ParamTypes::ParamSrc2Value; + compiler->useMemory0(); if (opcode == ByteCode::F32StoreOpcode) requiredInit = OTStoreF32; @@ -1143,6 +1152,7 @@ static void compileFunction(JITCompiler* compiler) SIMDMemoryStore* storeOperation = reinterpret_cast(byteCode); Instruction* instr = compiler->append(byteCode, Instruction::Store, opcode, 2, 0); instr->setRequiredRegsDescriptor(OTStoreV128); + compiler->useMemory0(); Operand* operands = instr->operands(); operands[0] = STACK_OFFSET(storeOperation->src0Offset()); @@ -1317,6 +1327,7 @@ static void compileFunction(JITCompiler* compiler) Instruction* instr = compiler->append(byteCode, Instruction::Memory, opcode, 0, 1); instr->setRequiredRegsDescriptor(OTPutI32); + compiler->useMemory0(); *instr->operands() = STACK_OFFSET(memorySize->dstOffset()); break; diff --git a/src/jit/Compiler.h b/src/jit/Compiler.h index ae2874535..c830208ad 100644 --- a/src/jit/Compiler.h +++ b/src/jit/Compiler.h @@ -585,6 +585,7 @@ struct CompileContext { size_t tableStart; size_t functionsStart; sljit_sw stackTmpStart; + sljit_sw stackMemoryStart; size_t nextTryBlock; size_t currentTryBlock; size_t trapBlocksStart; @@ -756,6 +757,16 @@ class JITCompiler { } } + void useMemory0() + { + m_useMemory0 = true; + } + + bool hasMemory0() + { + return m_useMemory0; + } + void setModuleFunction(ModuleFunction* moduleFunction) { m_moduleFunction = moduleFunction; @@ -802,6 +813,7 @@ class JITCompiler { // Backend operations. void emitProlog(); void emitEpilog(); + void emitRestoreMemories(); #if !defined(NDEBUG) static const char* m_byteCodeNames[]; @@ -825,6 +837,7 @@ class JITCompiler { uint8_t m_savedIntegerRegCount; uint8_t m_savedFloatRegCount; uint8_t m_stackTmpSize; + bool m_useMemory0; std::vector m_tryBlocks; std::vector m_functionList; diff --git a/src/jit/IntMath64Inl.h b/src/jit/IntMath64Inl.h index 56c1f32c4..f10cae07d 100644 --- a/src/jit/IntMath64Inl.h +++ b/src/jit/IntMath64Inl.h @@ -16,6 +16,8 @@ /* Only included by jit-backend.cc */ +#define WORD_LOW_OFFSET 0 + enum DivRemOptions : sljit_s32 { DivRem32 = 1 << 1, DivRemSigned = 1 << 0, diff --git a/src/jit/MemoryInl.h b/src/jit/MemoryInl.h index 22ad85ae0..ee538d2ff 100644 --- a/src/jit/MemoryInl.h +++ b/src/jit/MemoryInl.h @@ -57,7 +57,9 @@ struct MemAddress { void MemAddress::check(sljit_compiler* compiler, Operand* offsetOperand, sljit_uw offset, sljit_u32 size) { CompileContext* context = CompileContext::get(compiler); + sljit_sw stackMemoryStart = context->stackMemoryStart; + ASSERT(context->compiler->hasMemory0()); ASSERT(!(options & LoadInteger) || baseReg != sourceReg); ASSERT(!(options & LoadInteger) || offsetReg != sourceReg); #if defined(ENABLE_EXTENDED_FEATURES) @@ -105,8 +107,8 @@ void MemAddress::check(sljit_compiler* compiler, Operand* offsetOperand, sljit_u if (offset + size <= context->initialMemorySize) { ASSERT(baseReg != 0); - sljit_emit_op1(compiler, SLJIT_MOV_P, baseReg, 0, SLJIT_MEM1(kContextReg), - OffsetOfContextField(memory0) + offsetof(Memory::TargetBuffer, buffer)); + sljit_emit_op1(compiler, SLJIT_MOV_P, baseReg, 0, SLJIT_MEM1(SLJIT_SP), + stackMemoryStart + offsetof(Memory::TargetBuffer, buffer)); memArg.arg = SLJIT_MEM1(baseReg); memArg.argw = offset; load(compiler); @@ -121,18 +123,13 @@ void MemAddress::check(sljit_compiler* compiler, Operand* offsetOperand, sljit_u } ASSERT(baseReg != 0 && offsetReg != 0); -#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE) - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TMP_DEST_REG, 0, SLJIT_MEM1(kContextReg), - OffsetOfContextField(memory0) + offsetof(Memory::TargetBuffer, sizeInByte)); -#else /* !SLJIT_64BIT_ARCHITECTURE */ /* The sizeInByte is always a 32 bit number on 32 bit systems. */ - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TMP_DEST_REG, 0, SLJIT_MEM1(kContextReg), - OffsetOfContextField(memory0) + offsetof(Memory::TargetBuffer, sizeInByte) + WORD_LOW_OFFSET); -#endif /* SLJIT_64BIT_ARCHITECTURE */ + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TMP_DEST_REG, 0, SLJIT_MEM1(SLJIT_SP), + stackMemoryStart + offsetof(Memory::TargetBuffer, sizeInByte) + WORD_LOW_OFFSET); sljit_emit_op1(compiler, SLJIT_MOV, offsetReg, 0, SLJIT_IMM, static_cast(offset + size)); - sljit_emit_op1(compiler, SLJIT_MOV_P, baseReg, 0, SLJIT_MEM1(kContextReg), - OffsetOfContextField(memory0) + offsetof(Memory::TargetBuffer, buffer)); + sljit_emit_op1(compiler, SLJIT_MOV_P, baseReg, 0, SLJIT_MEM1(SLJIT_SP), + stackMemoryStart + offsetof(Memory::TargetBuffer, buffer)); load(compiler); @@ -164,19 +161,14 @@ void MemAddress::check(sljit_compiler* compiler, Operand* offsetOperand, sljit_u sljit_emit_op1(compiler, SLJIT_MOV_U32, offsetReg, 0, offsetArg.arg, offsetArg.argw); if (context->initialMemorySize != context->maximumMemorySize) { -#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE) - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TMP_DEST_REG, 0, SLJIT_MEM1(kContextReg), - OffsetOfContextField(memory0) + offsetof(Memory::TargetBuffer, sizeInByte)); -#else /* !SLJIT_64BIT_ARCHITECTURE */ /* The sizeInByte is always a 32 bit number on 32 bit systems. */ - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TMP_DEST_REG, 0, SLJIT_MEM1(kContextReg), - OffsetOfContextField(memory0) + offsetof(Memory::TargetBuffer, sizeInByte) + WORD_LOW_OFFSET); -#endif /* SLJIT_64BIT_ARCHITECTURE */ + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TMP_DEST_REG, 0, SLJIT_MEM1(SLJIT_SP), + stackMemoryStart + offsetof(Memory::TargetBuffer, sizeInByte) + WORD_LOW_OFFSET); offset += size; } - sljit_emit_op1(compiler, SLJIT_MOV_P, baseReg, 0, SLJIT_MEM1(kContextReg), - OffsetOfContextField(memory0) + offsetof(Memory::TargetBuffer, buffer)); + sljit_emit_op1(compiler, SLJIT_MOV_P, baseReg, 0, SLJIT_MEM1(SLJIT_SP), + stackMemoryStart + offsetof(Memory::TargetBuffer, buffer)); load(compiler); diff --git a/src/jit/MemoryUtilInl.h b/src/jit/MemoryUtilInl.h index b455a7e2f..72cc41628 100644 --- a/src/jit/MemoryUtilInl.h +++ b/src/jit/MemoryUtilInl.h @@ -79,11 +79,13 @@ static void emitMemory(sljit_compiler* compiler, Instruction* instr) switch (opcode) { case ByteCode::MemorySizeOpcode: { ASSERT(!(instr->info() & Instruction::kIsCallback)); + ASSERT(context->compiler->hasMemory0()); JITArg dstArg(params); - sljit_emit_op2(compiler, SLJIT_LSHR32, dstArg.arg, dstArg.argw, - SLJIT_MEM1(kContextReg), OffsetOfContextField(memory0) + offsetof(Memory::TargetBuffer, sizeInByte), SLJIT_IMM, 16); + /* The sizeInByte is always a 32 bit number on 32 bit systems. */ + sljit_emit_op2(compiler, SLJIT_LSHR, dstArg.arg, dstArg.argw, SLJIT_MEM1(SLJIT_SP), + context->stackMemoryStart + offsetof(Memory::TargetBuffer, sizeInByte) + WORD_LOW_OFFSET, SLJIT_IMM, 16); return; } case ByteCode::MemoryInitOpcode: diff --git a/src/runtime/JITExec.cpp b/src/runtime/JITExec.cpp index ca58add83..36eec92dd 100644 --- a/src/runtime/JITExec.cpp +++ b/src/runtime/JITExec.cpp @@ -31,17 +31,8 @@ ByteCodeStackOffset* JITFunction::call(ExecutionState& state, Instance* instance ExecutionContext context(m_module->instanceConstData(), state, instance); Memory* memory0 = nullptr; - if (instance->module()->numberOfMemoryTypes() > 0) { - memory0 = instance->memory(0); - memory0->push(&context.memory0); - } - ByteCodeStackOffset* resultOffsets = m_module->exportCall()(&context, bp, m_exportEntry); - if (memory0 != nullptr) { - memory0->pop(&context.memory0); - } - if (context.error != ExecutionContext::NoError) { switch (context.error) { case ExecutionContext::CapturedException: diff --git a/src/runtime/JITExec.h b/src/runtime/JITExec.h index 780a6f111..a5309929f 100644 --- a/src/runtime/JITExec.h +++ b/src/runtime/JITExec.h @@ -65,7 +65,6 @@ struct ExecutionContext { ExecutionState& state; Instance* instance; Exception* capturedException; - Memory::TargetBuffer memory0; ErrorCodes error; }; diff --git a/src/runtime/Memory.h b/src/runtime/Memory.h index 726299ec3..77204dd2d 100644 --- a/src/runtime/Memory.h +++ b/src/runtime/Memory.h @@ -31,6 +31,8 @@ class Store; class DataSegment; class Memory : public Extern { + friend class JITCompiler; + public: static const uint32_t s_memoryPageSize = 1024 * 64; @@ -38,14 +40,14 @@ class Memory : public Extern { struct TargetBuffer { TargetBuffer() : prev(nullptr) - , sizeInByte(0) , buffer(nullptr) + , sizeInByte(0) { } TargetBuffer* prev; - uint64_t sizeInByte; uint8_t* buffer; + uint64_t sizeInByte; }; static Memory* createMemory(Store* store, uint64_t initialSizeInByte, uint64_t maximumSizeInByte, bool isShared); @@ -300,19 +302,6 @@ class Memory : public Extern { void copy(ExecutionState& state, uint32_t dstStart, uint32_t srcStart, uint32_t size); void fill(ExecutionState& state, uint32_t start, uint8_t value, uint32_t size); - inline void push(TargetBuffer* targetBuffer) - { - targetBuffer->prev = m_targetBuffers; - targetBuffer->sizeInByte = sizeInByte(); - targetBuffer->buffer = buffer(); - m_targetBuffers = targetBuffer; - } - - inline void pop(TargetBuffer* targetBuffer) - { - m_targetBuffers = targetBuffer->prev; - } - inline bool checkAccess(uint32_t offset, uint32_t size, uint32_t addend = 0) const { return !UNLIKELY(!((uint64_t)offset + (uint64_t)addend + (uint64_t)size <= m_sizeInByte)); diff --git a/test/jit/trycatch-mem.wast b/test/jit/trycatch-mem.wast new file mode 100644 index 000000000..e48ae0536 --- /dev/null +++ b/test/jit/trycatch-mem.wast @@ -0,0 +1,39 @@ +(module + (memory 1) + (tag $except0 (param i32 i64 i32)) + + (func $throw1 (param i64 i32) + local.get 1 + memory.grow + + local.get 0 + i64.const 0xffffffffffff + i64.add + + memory.size + throw $except0 + ) + + (func (export "try1") (result i32 i64 i32 i32 i32) + (try (result i32 i64 i32 i32 i32) + (do + i64.const 0x123456781234 + i32.const 1 + call $throw1 + + i32.const 0 + i64.const 0 + i32.const 0 + i32.const 0 + i32.const 0 + ) + (catch $except0 + i32.const 2 + memory.grow + memory.size + ) + ) + ) +) + +(assert_return (invoke "try1") (i32.const 1) (i64.const 0x1123456781233) (i32.const 2) (i32.const 2) (i32.const 4))