diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index 09a280f097..9916303a7d 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -68,6 +68,13 @@ namespace FEXCore::Context { MODE_SINGLESTEP = 1, }; + struct ExitFunctionLinkData { + uint64_t HostBranch; + uint64_t GuestRIP; + }; + + using BlockDelinkerFunc = void(*)(FEXCore::Core::CpuStateFrame *Frame, FEXCore::Context::ExitFunctionLinkData *Record); + class ContextImpl final : public FEXCore::Context::Context { public: // Context base class implementation. @@ -274,12 +281,7 @@ namespace FEXCore::Context { void SignalThread(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::SignalEvent Event); static void ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP); - static void ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, uintptr_t HostLink, const std::function &delinker); - - struct ExitFunctionLinkData { - uint64_t HostBranch; - uint64_t GuestRIP; - }; + static void ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData *HostLink, const BlockDelinkerFunc &delinker); template static uint64_t ThreadExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, ExitFunctionLinkData *Record) { diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 02e40f8d7a..5ba18b8cd2 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -1237,7 +1237,7 @@ namespace FEXCore::Context { } } - void ContextImpl::ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, uintptr_t HostLink, const std::function &delinker) { + void ContextImpl::ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData *HostLink, const FEXCore::Context::BlockDelinkerFunc &delinker) { auto lk = GuardSignalDeferringSection(static_cast(Thread->CTX)->CodeInvalidationMutex, Thread); Thread->LookupCache->AddBlockLink(GuestDestination, HostLink, delinker); @@ -1249,7 +1249,7 @@ namespace FEXCore::Context { std::lock_guard lk(Thread->LookupCache->WriteLock); Thread->DebugStore.erase(GuestRIP); - Thread->LookupCache->Erase(GuestRIP); + Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP); } CustomIRResult ContextImpl::AddCustomIREntrypoint(uintptr_t Entrypoint, CustomIREntrypointHandler Handler, void *Creator, void *Data) { diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index 390e9ab571..2fd3af18d1 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -481,7 +481,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { } -static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, FEXCore::Context::ContextImpl::ExitFunctionLinkData *Record) { +static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame *Frame, FEXCore::Context::ExitFunctionLinkData *Record) { + auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker; + uintptr_t branch = (uintptr_t)(Record) - 8; + FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 8); + FEXCore::ARMEmitter::ForwardLabel l_BranchHost; + emit.ldr(FEXCore::ARMEmitter::XReg::x0, &l_BranchHost); + emit.blr(FEXCore::ARMEmitter::Reg::r0); + emit.Bind(&l_BranchHost); + emit.dc64(LinkerAddress); + FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 8); +} + +static void IndirectBlockDelinker(FEXCore::Core::CpuStateFrame *Frame, FEXCore::Context::ExitFunctionLinkData *Record) { + auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker; + Record->HostBranch = LinkerAddress; +} + +static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, FEXCore::Context::ExitFunctionLinkData *Record) { auto Thread = Frame->Thread; auto GuestRip = Record->GuestRIP; @@ -493,34 +510,23 @@ static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Fram } uintptr_t branch = (uintptr_t)(Record) - 8; - auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker; auto offset = HostCode/4 - branch/4; if (vixl::IsInt26(offset)) { // optimal case - can branch directly // patch the code - FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 24); + FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 4); emit.b(offset); - FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 24); + FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 4); // Add de-linking handler - Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)Record, [branch, LinkerAddress]{ - FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 24); - FEXCore::ARMEmitter::ForwardLabel l_BranchHost; - emit.ldr(FEXCore::ARMEmitter::XReg::x0, &l_BranchHost); - emit.blr(FEXCore::ARMEmitter::Reg::r0); - emit.Bind(&l_BranchHost); - emit.dc64(LinkerAddress); - FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 24); - }); + Thread->LookupCache->AddBlockLink(GuestRip, Record, DirectBlockDelinker); } else { // fallback case - do a soft-er link by patching the pointer Record->HostBranch = HostCode; // Add de-linking handler - Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)Record, [Record, LinkerAddress]{ - Record->HostBranch = LinkerAddress; - }); + Thread->LookupCache->AddBlockLink(GuestRip, Record, IndirectBlockDelinker); } return HostCode; diff --git a/FEXCore/Source/Interface/Core/LookupCache.h b/FEXCore/Source/Interface/Core/LookupCache.h index 0f0e51bac5..d02d02a380 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.h +++ b/FEXCore/Source/Interface/Core/LookupCache.h @@ -100,15 +100,15 @@ class LookupCache { L1Entry.HostCode = (uintptr_t)HostCode; } - void Erase(uint64_t Address) { + void Erase(FEXCore::Core::CpuStateFrame *Frame, uint64_t Address) { std::lock_guard lk(WriteLock); // Sever any links to this block - auto lower = BlockLinks->lower_bound({Address, 0}); - auto upper = BlockLinks->upper_bound({Address, UINTPTR_MAX}); + auto lower = BlockLinks->lower_bound({Address, nullptr}); + auto upper = BlockLinks->upper_bound({Address, reinterpret_cast(UINTPTR_MAX)}); for (auto it = lower; it != upper; it = BlockLinks->erase(it)) { - it->second(); + it->second(Frame, it->first.HostLink); } // Remove from BlockList @@ -141,8 +141,7 @@ class LookupCache { BlockPointers[PageOffset].HostCode = 0; } - - void AddBlockLink(uint64_t GuestDestination, uintptr_t HostLink, const std::function &delinker) { + void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData * HostLink, const FEXCore::Context::BlockDelinkerFunc &delinker) { std::lock_guard lk(WriteLock); BlockLinks->insert({{GuestDestination, HostLink}, delinker}); @@ -224,7 +223,7 @@ class LookupCache { struct BlockLinkTag { uint64_t GuestDestination; - uintptr_t HostLink; + FEXCore::Context::ExitFunctionLinkData *HostLink; bool operator <(const BlockLinkTag& other) const { if (GuestDestination < other.GuestDestination) @@ -243,7 +242,7 @@ class LookupCache { // // This makes `BlockLinks` look like a raw pointer that could memory leak, but since it is backed by the MBR, it won't. std::pmr::monotonic_buffer_resource BlockLinks_mbr; - using BlockLinksMapType = std::pmr::map>; + using BlockLinksMapType = std::pmr::map; fextl::unique_ptr> BlockLinks_pma; BlockLinksMapType *BlockLinks;