diff --git a/src/coreclr/nativeaot/Runtime/CMakeLists.txt b/src/coreclr/nativeaot/Runtime/CMakeLists.txt index e64e8b0a67123a..8ce6ea0d296722 100644 --- a/src/coreclr/nativeaot/Runtime/CMakeLists.txt +++ b/src/coreclr/nativeaot/Runtime/CMakeLists.txt @@ -245,6 +245,10 @@ add_definitions(-D_LIB) if(WIN32) add_definitions(-DFEATURE_ETW) add_definitions(-DFEATURE_EVENT_TRACE) + # redirection on ARM64 should work, but needs to be tested + if (CLR_CMAKE_TARGET_ARCH_AMD64) + add_definitions(-DFEATURE_SUSPEND_REDIRECTION) + endif() else() add_definitions(-DNO_UI_ASSERT) include(unix/configure.cmake) diff --git a/src/coreclr/nativeaot/Runtime/ICodeManager.h b/src/coreclr/nativeaot/Runtime/ICodeManager.h index 587b10211c6f5b..fe2a4fe6a6b1bd 100644 --- a/src/coreclr/nativeaot/Runtime/ICodeManager.h +++ b/src/coreclr/nativeaot/Runtime/ICodeManager.h @@ -139,6 +139,8 @@ enum class AssociatedDataFlags : unsigned char class ICodeManager { public: + virtual bool IsSafePoint(PTR_VOID pvAddress) = 0; + virtual bool FindMethodInfo(PTR_VOID ControlPC, MethodInfo * pMethodInfoOut) = 0; @@ -150,7 +152,8 @@ class ICodeManager virtual void EnumGcRefs(MethodInfo * pMethodInfo, PTR_VOID safePointAddress, REGDISPLAY * pRegisterSet, - GCEnumContext * hCallback) = 0; + GCEnumContext * hCallback, + bool isActiveStackFrame) = 0; virtual bool UnwindStackFrame(MethodInfo * pMethodInfo, REGDISPLAY * pRegisterSet, // in/out diff --git a/src/coreclr/nativeaot/Runtime/PalRedhawk.h b/src/coreclr/nativeaot/Runtime/PalRedhawk.h index 951177d372fddc..82293920996ee1 100644 --- a/src/coreclr/nativeaot/Runtime/PalRedhawk.h +++ b/src/coreclr/nativeaot/Runtime/PalRedhawk.h @@ -405,6 +405,7 @@ typedef struct DECLSPEC_ALIGN(16) _CONTEXT { void SetArg1Reg(uintptr_t val) { X1 = val; } uintptr_t GetIp() { return Pc; } uintptr_t GetLr() { return Lr; } + uintptr_t GetSp() { return Sp; } } CONTEXT, *PCONTEXT; #elif defined(HOST_WASM) @@ -590,6 +591,11 @@ REDHAWK_PALIMPORT bool REDHAWK_PALAPI PalInit(); // Given the OS handle of a loaded module, compute the upper and lower virtual address bounds (inclusive). REDHAWK_PALIMPORT void REDHAWK_PALAPI PalGetModuleBounds(HANDLE hOsHandle, _Out_ uint8_t ** ppLowerBound, _Out_ uint8_t ** ppUpperBound); +REDHAWK_PALIMPORT CONTEXT* PalAllocateCompleteOSContext(_Out_ uint8_t** contextBuffer); +REDHAWK_PALIMPORT bool REDHAWK_PALAPI PalGetCompleteThreadContext(HANDLE hThread, _Out_ CONTEXT * pCtx); +REDHAWK_PALIMPORT bool REDHAWK_PALAPI PalSetThreadContext(HANDLE hThread, _Out_ CONTEXT * pCtx); +REDHAWK_PALIMPORT void REDHAWK_PALAPI PalRestoreContext(CONTEXT * pCtx); + REDHAWK_PALIMPORT bool REDHAWK_PALAPI PalGetThreadContext(HANDLE hThread, _Out_ PAL_LIMITED_CONTEXT * pCtx); REDHAWK_PALIMPORT int32_t REDHAWK_PALAPI PalGetProcessCpuCount(); diff --git a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp index 18d71fa41fb783..5a3197ff649c62 100644 --- a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp +++ b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp @@ -92,7 +92,18 @@ StackFrameIterator::StackFrameIterator(Thread * pThreadToWalk, PInvokeTransition { STRESS_LOG0(LF_STACKWALK, LL_INFO10000, "----Init---- [ GC ]\n"); ASSERT(!pThreadToWalk->DangerousCrossThreadIsHijacked()); - InternalInit(pThreadToWalk, pInitialTransitionFrame, GcStackWalkFlags); + +#ifdef FEATURE_SUSPEND_REDIRECTION + if (pInitialTransitionFrame == REDIRECTED_THREAD_MARKER) + { + InternalInit(pThreadToWalk, pThreadToWalk->GetRedirectionContext(), GcStackWalkFlags | ActiveStackFrame); + } + else +#endif + { + InternalInit(pThreadToWalk, pInitialTransitionFrame, GcStackWalkFlags); + } + PrepareToYieldFrame(); } @@ -497,6 +508,112 @@ void StackFrameIterator::InternalInit(Thread * pThreadToWalk, PTR_PAL_LIMITED_CO #endif // TARGET_ARM } +// Prepare to start a stack walk from the context listed in the supplied CONTEXT. +// The supplied context can describe a location in either managed or unmanaged code. In the +// latter case the iterator is left in an invalid state when this function returns. +void StackFrameIterator::InternalInit(Thread * pThreadToWalk, CONTEXT* pCtx, uint32_t dwFlags) +{ + ASSERT((dwFlags & MethodStateCalculated) == 0); + + EnterInitialInvalidState(pThreadToWalk); + + m_dwFlags = dwFlags; + + // We need to walk the ExInfo chain in parallel with the stackwalk so that we know when we cross over + // exception throw points. So we must find our initial point in the ExInfo chain here so that we can + // properly walk it in parallel. + ResetNextExInfoForSP(pCtx->GetSp()); + + // This codepath is used by the hijack stackwalk and we can get arbitrary ControlPCs from there. If this + // context has a non-managed control PC, then we're done. + if (!m_pInstance->IsManaged(dac_cast(pCtx->GetIp()))) + return; + + // + // control state + // + SetControlPC(dac_cast(pCtx->GetIp())); + m_RegDisplay.SP = pCtx->GetSp(); + m_RegDisplay.IP = pCtx->GetIp(); + +#ifdef TARGET_ARM64 + + m_RegDisplay.pIP = PTR_TO_MEMBER(CONTEXT, pCtx, Pc); + + // + // preserved regs + // + m_RegDisplay.pX19 = PTR_TO_MEMBER(CONTEXT, pCtx, X19); + m_RegDisplay.pX20 = PTR_TO_MEMBER(CONTEXT, pCtx, X20); + m_RegDisplay.pX21 = PTR_TO_MEMBER(CONTEXT, pCtx, X21); + m_RegDisplay.pX22 = PTR_TO_MEMBER(CONTEXT, pCtx, X22); + m_RegDisplay.pX23 = PTR_TO_MEMBER(CONTEXT, pCtx, X23); + m_RegDisplay.pX24 = PTR_TO_MEMBER(CONTEXT, pCtx, X24); + m_RegDisplay.pX25 = PTR_TO_MEMBER(CONTEXT, pCtx, X25); + m_RegDisplay.pX26 = PTR_TO_MEMBER(CONTEXT, pCtx, X26); + m_RegDisplay.pX27 = PTR_TO_MEMBER(CONTEXT, pCtx, X27); + m_RegDisplay.pX28 = PTR_TO_MEMBER(CONTEXT, pCtx, X28); + m_RegDisplay.pFP = PTR_TO_MEMBER(CONTEXT, pCtx, Fp); + m_RegDisplay.pLR = PTR_TO_MEMBER(CONTEXT, pCtx, Lr); + + // + // scratch regs + // + m_RegDisplay.pX0 = PTR_TO_MEMBER(CONTEXT, pCtx, X0); + m_RegDisplay.pX1 = PTR_TO_MEMBER(CONTEXT, pCtx, X1); + m_RegDisplay.pX2 = PTR_TO_MEMBER(CONTEXT, pCtx, X2); + m_RegDisplay.pX3 = PTR_TO_MEMBER(CONTEXT, pCtx, X3); + m_RegDisplay.pX4 = PTR_TO_MEMBER(CONTEXT, pCtx, X4); + m_RegDisplay.pX5 = PTR_TO_MEMBER(CONTEXT, pCtx, X5); + m_RegDisplay.pX6 = PTR_TO_MEMBER(CONTEXT, pCtx, X6); + m_RegDisplay.pX7 = PTR_TO_MEMBER(CONTEXT, pCtx, X7); + m_RegDisplay.pX8 = PTR_TO_MEMBER(CONTEXT, pCtx, X8); + m_RegDisplay.pX9 = PTR_TO_MEMBER(CONTEXT, pCtx, X9); + m_RegDisplay.pX10 = PTR_TO_MEMBER(CONTEXT, pCtx, X10); + m_RegDisplay.pX11 = PTR_TO_MEMBER(CONTEXT, pCtx, X11); + m_RegDisplay.pX12 = PTR_TO_MEMBER(CONTEXT, pCtx, X12); + m_RegDisplay.pX13 = PTR_TO_MEMBER(CONTEXT, pCtx, X13); + m_RegDisplay.pX14 = PTR_TO_MEMBER(CONTEXT, pCtx, X14); + m_RegDisplay.pX15 = PTR_TO_MEMBER(CONTEXT, pCtx, X15); + m_RegDisplay.pX16 = PTR_TO_MEMBER(CONTEXT, pCtx, X16); + m_RegDisplay.pX17 = PTR_TO_MEMBER(CONTEXT, pCtx, X17); + m_RegDisplay.pX18 = PTR_TO_MEMBER(CONTEXT, pCtx, X18); + +#elif defined(TARGET_X86) || defined(TARGET_AMD64) + + m_RegDisplay.pIP = (PTR_PCODE)PTR_TO_MEMBER(CONTEXT, pCtx, Rip); + + // + // preserved regs + // + m_RegDisplay.pRbp = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, Rbp); + m_RegDisplay.pRsi = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, Rsi); + m_RegDisplay.pRdi = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, Rdi); + m_RegDisplay.pRbx = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, Rbx); +#ifdef TARGET_AMD64 + m_RegDisplay.pR12 = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, R12); + m_RegDisplay.pR13 = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, R13); + m_RegDisplay.pR14 = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, R14); + m_RegDisplay.pR15 = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, R15); +#endif // TARGET_AMD64 + + // + // scratch regs + // + m_RegDisplay.pRax = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, Rax); + m_RegDisplay.pRcx = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, Rcx); + m_RegDisplay.pRdx = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, Rdx); +#ifdef TARGET_AMD64 + m_RegDisplay.pR8 = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, R8); + m_RegDisplay.pR9 = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, R9); + m_RegDisplay.pR10 = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, R10); + m_RegDisplay.pR11 = (PTR_UIntNative)PTR_TO_MEMBER(CONTEXT, pCtx, R11); +#endif // TARGET_AMD64 +#else + PORTABILITY_ASSERT("StackFrameIterator::InternalInit"); +#endif // TARGET_ARM +} + PTR_VOID StackFrameIterator::HandleExCollide(PTR_ExInfo pExInfo) { STRESS_LOG3(LF_STACKWALK, LL_INFO10000, " [ ex collide ] kind = %d, pass = %d, idxCurClause = %d\n", @@ -1290,7 +1407,7 @@ void StackFrameIterator::NextInternal() { UnwindOutOfCurrentManagedFrame: ASSERT(m_dwFlags & MethodStateCalculated); - m_dwFlags &= ~(ExCollide|MethodStateCalculated|UnwoundReversePInvoke); + m_dwFlags &= ~(ExCollide|MethodStateCalculated|UnwoundReversePInvoke|ActiveStackFrame); ASSERT(IsValid()); m_pHijackedReturnValue = NULL; @@ -1640,6 +1757,12 @@ MethodInfo * StackFrameIterator::GetMethodInfo() return &m_methodInfo; } +bool StackFrameIterator::IsActiveStackFrame() +{ + ASSERT(IsValid()); + return (m_dwFlags & ActiveStackFrame) != 0; +} + #ifdef DACCESS_COMPILE #define FAILFAST_OR_DAC_RETURN_FALSE(x) if(!(x)) return false; #else diff --git a/src/coreclr/nativeaot/Runtime/StackFrameIterator.h b/src/coreclr/nativeaot/Runtime/StackFrameIterator.h index 1f460fa8356ffc..ca0e48168db2f4 100644 --- a/src/coreclr/nativeaot/Runtime/StackFrameIterator.h +++ b/src/coreclr/nativeaot/Runtime/StackFrameIterator.h @@ -36,7 +36,6 @@ class StackFrameIterator StackFrameIterator(Thread * pThreadToWalk, PInvokeTransitionFrame* pInitialTransitionFrame); StackFrameIterator(Thread * pThreadToWalk, PTR_PAL_LIMITED_CONTEXT pCtx); - bool IsValid(); void CalculateCurrentMethodState(); void Next(); @@ -44,6 +43,7 @@ class StackFrameIterator REGDISPLAY * GetRegisterSet(); PTR_ICodeManager GetCodeManager(); MethodInfo * GetMethodInfo(); + bool IsActiveStackFrame(); bool GetHijackedReturnValueLocation(PTR_RtuObjectRef * pLocation, GCRefKind * pKind); void SetControlPC(PTR_VOID controlPC); @@ -82,6 +82,8 @@ class StackFrameIterator void InternalInit(Thread * pThreadToWalk, PTR_PInvokeTransitionFrame pFrame, uint32_t dwFlags); // GC stackwalk void InternalInit(Thread * pThreadToWalk, PTR_PAL_LIMITED_CONTEXT pCtx, uint32_t dwFlags); // EH and hijack stackwalk, and collided unwind + void InternalInit(Thread * pThreadToWalk, CONTEXT* pCtx, uint32_t dwFlags); // GC stackwalk of redirected thread + void InternalInitForEH(Thread * pThreadToWalk, PAL_LIMITED_CONTEXT * pCtx, bool instructionFault); // EH stackwalk void InternalInitForStackTrace(); // Environment.StackTrace @@ -137,6 +139,9 @@ class StackFrameIterator // This is a state returned by Next() which indicates that we just unwound a reverse pinvoke method UnwoundReversePInvoke = 0x20, + // The thread was interrupted in the current frame at the current IP by a signal, SuspendThread or similar. + ActiveStackFrame = 0x40, + GcStackWalkFlags = (CollapseFunclets | RemapHardwareFaultsToSafePoint), EHStackWalkFlags = ApplyReturnAddressAdjustment, StackTraceStackWalkFlags = GcStackWalkFlags diff --git a/src/coreclr/nativeaot/Runtime/arm64/GcProbe.asm b/src/coreclr/nativeaot/Runtime/arm64/GcProbe.asm index c794331eb3cb44..3921344e561711 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/GcProbe.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/GcProbe.asm @@ -8,7 +8,7 @@ EXTERN RhpGcPoll2 EXTERN g_fGcStressStarted -PROBE_SAVE_FLAGS_EVERYTHING equ DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_ALL_SCRATCH +PROBE_SAVE_FLAGS_EVERYTHING equ DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_ALL_SCRATCH + PTFF_SAVE_LR ;; Build a map of symbols representing offsets into the transition frame (see PInvokeTransitionFrame in ;; rhbinder.h) and keep these two in sync. diff --git a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp index f4f13221dfc111..8119fd42ffa5ee 100644 --- a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp +++ b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp @@ -515,7 +515,8 @@ void RedhawkGCInterface::EnumGcRefs(ICodeManager * pCodeManager, PTR_VOID safePointAddress, REGDISPLAY * pRegisterSet, void * pfnEnumCallback, - void * pvCallbackData) + void * pvCallbackData, + bool isActiveStackFrame) { EnumGcRefContext ctx; ctx.pCallback = EnumGcRefsCallback; @@ -526,7 +527,8 @@ void RedhawkGCInterface::EnumGcRefs(ICodeManager * pCodeManager, pCodeManager->EnumGcRefs(pMethodInfo, safePointAddress, pRegisterSet, - &ctx); + &ctx, + isActiveStackFrame); } // static diff --git a/src/coreclr/nativeaot/Runtime/gcrhinterface.h b/src/coreclr/nativeaot/Runtime/gcrhinterface.h index 976bd8e6bdf954..1379a208e16e06 100644 --- a/src/coreclr/nativeaot/Runtime/gcrhinterface.h +++ b/src/coreclr/nativeaot/Runtime/gcrhinterface.h @@ -120,7 +120,8 @@ class RedhawkGCInterface PTR_VOID safePointAddress, REGDISPLAY * pRegisterSet, void * pfnEnumCallback, - void * pvCallbackData); + void * pvCallbackData, + bool isActiveStackFrame); static void EnumGcRefsInRegionConservatively(PTR_RtuObjectRef pLowerBound, PTR_RtuObjectRef pUpperBound, diff --git a/src/coreclr/nativeaot/Runtime/thread.cpp b/src/coreclr/nativeaot/Runtime/thread.cpp index 0db1d1465a35f6..9baf409b1d5f0d 100644 --- a/src/coreclr/nativeaot/Runtime/thread.cpp +++ b/src/coreclr/nativeaot/Runtime/thread.cpp @@ -63,7 +63,6 @@ PInvokeTransitionFrame* Thread::GetTransitionFrame() PInvokeTransitionFrame* Thread::GetTransitionFrameForStackTrace() { - ASSERT_MSG(ThreadStore::GetSuspendingThread() == NULL, "Not allowed when suspended for GC."); ASSERT_MSG(this == ThreadStore::GetCurrentThread(), "Only supported for current thread."); ASSERT(Thread::IsCurrentThreadInCooperativeMode()); ASSERT(m_pDeferredTransitionFrame != NULL); @@ -80,6 +79,10 @@ void Thread::WaitForGC(PInvokeTransitionFrame* pTransitionFrame) { ASSERT(!IsDoNotTriggerGcSet()); + // The wait operation below may trash the last win32 error. We save the error here so that it can be + // restored after the wait operation; + int32_t lastErrorOnEntry = PalGetLastError(); + do { m_pTransitionFrame = pTransitionFrame; @@ -93,6 +96,9 @@ void Thread::WaitForGC(PInvokeTransitionFrame* pTransitionFrame) _ReadWriteBarrier(); } while (ThreadStore::IsTrapThreadsRequested()); + + // Restore the saved error + PalSetLastError(lastErrorOnEntry); } // @@ -305,6 +311,11 @@ void Thread::Construct() #endif // STRESS_LOG m_threadAbortException = NULL; + +#ifdef FEATURE_SUSPEND_REDIRECTION + m_redirectionContextBuffer = NULL; + m_redirectionContext = NULL; +#endif //FEATURE_SUSPEND_REDIRECTION } bool Thread::IsInitialized() @@ -391,6 +402,13 @@ void Thread::Destroy() ThreadStressLog* ptsl = reinterpret_cast(GetThreadStressLog()); StressLog::ThreadDetach(ptsl); #endif // STRESS_LOG + +#ifdef FEATURE_SUSPEND_REDIRECTION + if (m_redirectionContextBuffer != NULL) + { + delete[] m_redirectionContextBuffer; + } +#endif //FEATURE_SUSPEND_REDIRECTION } #ifdef HOST_WASM @@ -514,7 +532,8 @@ void Thread::GcScanRootsWorker(void * pfnEnumCallback, void * pvCallbackData, St frameIterator.GetEffectiveSafePointAddress(), frameIterator.GetRegisterSet(), pfnEnumCallback, - pvCallbackData); + pvCallbackData, + frameIterator.IsActiveStackFrame()); } // Each enumerated frame (including the first one) may have an associated stack range we need to @@ -562,6 +581,8 @@ void Thread::GcScanRootsWorker(void * pfnEnumCallback, void * pvCallbackData, St #ifndef DACCESS_COMPILE +EXTERN_C void FASTCALL RhpSuspendRedirected(); + #ifndef TARGET_ARM64 EXTERN_C void FASTCALL RhpGcProbeHijackScalar(); EXTERN_C void FASTCALL RhpGcProbeHijackObject(); @@ -655,14 +676,25 @@ UInt32_BOOL Thread::HijackCallback(HANDLE /*hThread*/, PAL_LIMITED_CONTEXT* pThr return true; } - if (!GetRuntimeInstance()->IsManaged((PTR_VOID)pThreadContext->IP)) + void* pvAddress = (void*)pThreadContext->IP; + RuntimeInstance* runtime = GetRuntimeInstance(); + if (!runtime->IsManaged(pvAddress)) { // Running in cooperative mode, but not managed. // We cannot continue. return false; } - // TODO: attempt to redirect + ICodeManager* codeManager = runtime->GetCodeManagerForAddress(pvAddress); + if (codeManager->IsSafePoint(pvAddress)) + { +#ifdef FEATURE_SUSPEND_REDIRECTION + if (pThread->Redirect()) + { + return true; + } +#endif //FEATURE_SUSPEND_REDIRECTION + } return pThread->InternalHijack(pThreadContext, NormalHijackTargets); } @@ -769,6 +801,43 @@ bool Thread::InternalHijack(PAL_LIMITED_CONTEXT * pSuspendCtx, void * pvHijackTa return fSuccess; } +#ifdef FEATURE_SUSPEND_REDIRECTION +CONTEXT* Thread::GetRedirectionContext() +{ + if (m_redirectionContext == NULL) + { + m_redirectionContext = PalAllocateCompleteOSContext(&m_redirectionContextBuffer); + } + + return m_redirectionContext; +} + +bool Thread::Redirect() +{ + if (IsDoNotTriggerGcSet()) + return false; + + CONTEXT* redirectionContext = GetRedirectionContext(); + if (redirectionContext == NULL) + return false; + + if (!PalGetCompleteThreadContext(m_hPalThread, redirectionContext)) + return false; + + uintptr_t origIP = redirectionContext->GetIp(); + redirectionContext->SetIp((uintptr_t)RhpSuspendRedirected); + if (!PalSetThreadContext(m_hPalThread, redirectionContext)) + return false; + + redirectionContext->SetIp(origIP); + + STRESS_LOG2(LF_STACKWALK, LL_INFO10000, "InternalRedirect: TgtThread = %llx, IP = %p\n", + GetPalThreadIdForLogging(), origIP); + + return true; +} +#endif //FEATURE_SUSPEND_REDIRECTION + // This is the standard Unhijack, which is only allowed to be called on your own thread. // Note that all the asm-implemented Unhijacks should also only be operating on their // own thread. @@ -949,20 +1018,11 @@ EXTERN_C NOINLINE void FASTCALL RhpWaitForSuspend2() // Standard calling convention variant and actual implementation for RhpWaitForGC EXTERN_C NOINLINE void FASTCALL RhpWaitForGC2(PInvokeTransitionFrame * pFrame) { - Thread * pThread = pFrame->m_pThread; - if (pThread->IsDoNotTriggerGcSet()) return; - // The wait operation below may trash the last win32 error. We save the error here so that it can be - // restored after the wait operation; - int32_t lastErrorOnEntry = PalGetLastError(); - pThread->WaitForGC(pFrame); - - // Restore the saved error - PalSetLastError(lastErrorOnEntry); } // Standard calling convention variant and actual implementation for RhpGcPoll @@ -974,6 +1034,20 @@ EXTERN_C NOINLINE void FASTCALL RhpGcPoll2(PInvokeTransitionFrame* pFrame) RhpWaitForGC2(pFrame); } +#ifdef FEATURE_SUSPEND_REDIRECTION + +EXTERN_C NOINLINE void FASTCALL RhpSuspendRedirected() +{ + Thread* pThread = ThreadStore::GetCurrentThread(); + pThread->WaitForGC(REDIRECTED_THREAD_MARKER); + + // restore execution at interrupted location + PalRestoreContext(pThread->GetRedirectionContext()); + UNREACHABLE(); +} + +#endif //FEATURE_SUSPEND_REDIRECTION + void Thread::PushExInfo(ExInfo * pExInfo) { ValidateExInfoStack(); diff --git a/src/coreclr/nativeaot/Runtime/thread.h b/src/coreclr/nativeaot/Runtime/thread.h index 2c50024f58c9da..f06dbf9ea5530b 100644 --- a/src/coreclr/nativeaot/Runtime/thread.h +++ b/src/coreclr/nativeaot/Runtime/thread.h @@ -28,6 +28,7 @@ class Thread; #endif // HOST_64BIT #define TOP_OF_STACK_MARKER ((PInvokeTransitionFrame*)(ptrdiff_t)-1) +#define REDIRECTED_THREAD_MARKER ((PInvokeTransitionFrame*)(ptrdiff_t)-2) #define DYNAMIC_TYPE_TLS_OFFSET_FLAG 0x80000000 @@ -90,6 +91,10 @@ struct ThreadBuffer uint64_t m_uPalThreadIdForLogging; // @TODO: likely debug-only EEThreadId m_threadId; PTR_VOID m_pThreadStressLog; // pointer to head of thread's StressLogChunks +#ifdef FEATURE_SUSPEND_REDIRECTION + uint8_t* m_redirectionContextBuffer; // storage for redirection context, allocated on demand + CONTEXT* m_redirectionContext; // legacy context somewhere inside the context buffer +#endif //FEATURE_SUSPEND_REDIRECTION #ifdef FEATURE_GC_STRESS uint32_t m_uRand; // current per-thread random number #endif // FEATURE_GC_STRESS @@ -139,6 +144,10 @@ class Thread : private ThreadBuffer static UInt32_BOOL HijackCallback(HANDLE hThread, PAL_LIMITED_CONTEXT* pThreadContext, void* pCallbackContext); bool InternalHijack(PAL_LIMITED_CONTEXT * pSuspendCtx, void * pvHijackTargets[]); +#ifdef FEATURE_SUSPEND_REDIRECTION + bool Redirect(); +#endif //FEATURE_SUSPEND_REDIRECTION + bool CacheTransitionFrameForSuspend(); void ResetCachedTransitionFrame(); void CrossThreadUnhijack(); @@ -264,6 +273,10 @@ class Thread : private ThreadBuffer Object* GetThreadStaticStorageForModule(uint32_t moduleIndex); bool SetThreadStaticStorageForModule(Object* pStorage, uint32_t moduleIndex); + +#ifdef FEATURE_SUSPEND_REDIRECTION + CONTEXT* GetRedirectionContext(); +#endif //FEATURE_SUSPEND_REDIRECTION }; #ifndef __GCENV_BASE_INCLUDED__ diff --git a/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.cpp b/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.cpp index 6ccc2491180213..4aa8e9ab7343ff 100644 --- a/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.cpp +++ b/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.cpp @@ -127,10 +127,17 @@ PTR_VOID UnixNativeCodeManager::GetFramePointer(MethodInfo * pMethodInfo, return NULL; } +bool UnixNativeCodeManager::IsSafePoint(PTR_VOID pvAddress) +{ + // @TODO: IsSafePoint + return false; +} + void UnixNativeCodeManager::EnumGcRefs(MethodInfo * pMethodInfo, PTR_VOID safePointAddress, REGDISPLAY * pRegisterSet, - GCEnumContext * hCallback) + GCEnumContext * hCallback, + bool isActiveStackFrame) { UnixNativeMethodInfo * pNativeMethodInfo = (UnixNativeMethodInfo *)pMethodInfo; @@ -149,7 +156,7 @@ void UnixNativeCodeManager::EnumGcRefs(MethodInfo * pMethodInfo, GcInfoDecoder decoder( GCInfoToken(p), GcInfoDecoderFlags(DECODE_GC_LIFETIMES | DECODE_SECURITY_OBJECT | DECODE_VARARG), - codeOffset - 1 // TODO: Is this adjustment correct? + codeOffset - 1 // TODO: isActiveStackFrame ); ICodeManagerFlags flags = (ICodeManagerFlags)0; diff --git a/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.h b/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.h index 6a4dc631f0aeef..709bd381e06f24 100644 --- a/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.h +++ b/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.h @@ -34,10 +34,13 @@ class UnixNativeCodeManager : public ICodeManager PTR_VOID GetFramePointer(MethodInfo * pMethodInfo, REGDISPLAY * pRegisterSet); + bool IsSafePoint(PTR_VOID pvAddress); + void EnumGcRefs(MethodInfo * pMethodInfo, PTR_VOID safePointAddress, REGDISPLAY * pRegisterSet, - GCEnumContext * hCallback); + GCEnumContext * hCallback, + bool isActiveStackFrame); bool UnwindStackFrame(MethodInfo * pMethodInfo, REGDISPLAY * pRegisterSet, // in/out diff --git a/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.cpp b/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.cpp index c2fdecb07e7221..4010dbea5241fe 100644 --- a/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.cpp +++ b/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.cpp @@ -331,13 +331,12 @@ PTR_VOID CoffNativeCodeManager::GetFramePointer(MethodInfo * pMethInfo, return NULL; } -void CoffNativeCodeManager::EnumGcRefs(MethodInfo * pMethodInfo, - PTR_VOID safePointAddress, - REGDISPLAY * pRegisterSet, - GCEnumContext * hCallback) +uint32_t CoffNativeCodeManager::GetCodeOffset(MethodInfo* pMethodInfo, PTR_VOID address, /*out*/ PTR_UInt8* gcInfo) { CoffNativeMethodInfo * pNativeMethodInfo = (CoffNativeMethodInfo *)pMethodInfo; + _ASSERTE(FindMethodInfo(address, pMethodInfo) && (MethodInfo*)pNativeMethodInfo == pMethodInfo); + size_t unwindDataBlobSize; PTR_VOID pUnwindDataBlob = GetUnwindDataBlob(m_moduleBase, pNativeMethodInfo->mainRuntimeFunction, &unwindDataBlobSize); @@ -351,24 +350,71 @@ void CoffNativeCodeManager::EnumGcRefs(MethodInfo * pMethodInfo, if ((unwindBlockFlags & UBF_FUNC_HAS_EHINFO) != 0) p += sizeof(int32_t); + *gcInfo = p; + TADDR methodStartAddress = m_moduleBase + pNativeMethodInfo->mainRuntimeFunction->BeginAddress; - uint32_t codeOffset = (uint32_t)(dac_cast(safePointAddress) - methodStartAddress); + return (uint32_t)(dac_cast(address) - methodStartAddress); +} + +bool CoffNativeCodeManager::IsSafePoint(PTR_VOID pvAddress) +{ + MethodInfo pMethodInfo; + if (!FindMethodInfo(pvAddress, &pMethodInfo)) + { + return false; + } + + PTR_UInt8 gcInfo; + uint32_t codeOffset = GetCodeOffset(&pMethodInfo, pvAddress, &gcInfo); + + GcInfoDecoder decoder( + GCInfoToken(gcInfo), + GcInfoDecoderFlags(DECODE_INTERRUPTIBILITY), + codeOffset + ); + + return decoder.IsInterruptible(); +} + +void CoffNativeCodeManager::EnumGcRefs(MethodInfo * pMethodInfo, + PTR_VOID safePointAddress, + REGDISPLAY * pRegisterSet, + GCEnumContext * hCallback, + bool isActiveStackFrame) +{ + PTR_UInt8 gcInfo; + uint32_t codeOffset = GetCodeOffset(pMethodInfo, safePointAddress, &gcInfo); + + if (!isActiveStackFrame) + { + // If we are not in the active method, we are currently pointing + // to the return address. That may not be reachable after a call (if call does not return) + // or reachable via a jump and thus have a different live set. + // Therefore we simply adjust the offset to inside of call instruction. + // NOTE: The GcInfoDecoder depends on this; if you change it, you must + // revisit the GcInfoEncoder/Decoder + codeOffset--; + } GcInfoDecoder decoder( - GCInfoToken(p), + GCInfoToken(gcInfo), GcInfoDecoderFlags(DECODE_GC_LIFETIMES | DECODE_SECURITY_OBJECT | DECODE_VARARG), - codeOffset - 1 // TODO: Is this adjustment correct? + codeOffset ); ICodeManagerFlags flags = (ICodeManagerFlags)0; - if (pNativeMethodInfo->executionAborted) + if (((CoffNativeMethodInfo *)pMethodInfo)->executionAborted) flags = ICodeManagerFlags::ExecutionAborted; + if (IsFilter(pMethodInfo)) flags = (ICodeManagerFlags)(flags | ICodeManagerFlags::NoReportUntracked); + if (isActiveStackFrame) + flags = (ICodeManagerFlags)(flags | ICodeManagerFlags::ActiveStackFrame); + if (!decoder.EnumerateLiveSlots( pRegisterSet, - false /* reportScratchSlots */, + isActiveStackFrame /* reportScratchSlots */, flags, hCallback->pCallback, hCallback @@ -673,11 +719,7 @@ bool CoffNativeCodeManager::GetReturnAddressHijackInfo(MethodInfo * pMethodIn p += sizeof(int32_t); // Decode the GC info for the current method to determine its return type - GcInfoDecoder decoder( - GCInfoToken(p), - GcInfoDecoderFlags(DECODE_RETURN_KIND), - 0 - ); + GcInfoDecoder decoder(GCInfoToken(p), DECODE_RETURN_KIND); GCRefKind gcRefKind = GetGcRefKind(decoder.GetReturnKind()); diff --git a/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.h b/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.h index eb2b03f6722544..d8606f62367327 100644 --- a/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.h +++ b/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.h @@ -70,10 +70,15 @@ class CoffNativeCodeManager : public ICodeManager PTR_VOID GetFramePointer(MethodInfo * pMethodInfo, REGDISPLAY * pRegisterSet); + uint32_t GetCodeOffset(MethodInfo * pMethodInfo, PTR_VOID address, /*out*/ PTR_UInt8* gcInfo); + + bool IsSafePoint(PTR_VOID pvAddress); + void EnumGcRefs(MethodInfo * pMethodInfo, PTR_VOID safePointAddress, REGDISPLAY * pRegisterSet, - GCEnumContext * hCallback); + GCEnumContext * hCallback, + bool isActiveStackFrame); bool UnwindStackFrame(MethodInfo * pMethodInfo, REGDISPLAY * pRegisterSet, // in/out diff --git a/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp b/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp index 9944742d23e3f5..631fbb61c4dd50 100644 --- a/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp +++ b/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp @@ -312,6 +312,124 @@ REDHAWK_PALEXPORT HANDLE REDHAWK_PALAPI PalCreateEventW(_In_opt_ LPSECURITY_ATTR return CreateEventW(pEventAttributes, manualReset, initialState, pName); } +typedef BOOL(WINAPI* PINITIALIZECONTEXT2)(PVOID Buffer, DWORD ContextFlags, PCONTEXT* Context, PDWORD ContextLength, ULONG64 XStateCompactionMask); +PINITIALIZECONTEXT2 pfnInitializeContext2 = NULL; + +#ifdef TARGET_X86 +typedef VOID(__cdecl* PRTLRESTORECONTEXT)(PCONTEXT ContextRecord, struct _EXCEPTION_RECORD* ExceptionRecord); +PRTLRESTORECONTEXT pfnRtlRestoreContext = NULL; + +#define CONTEXT_COMPLETE (CONTEXT_FULL | CONTEXT_FLOATING_POINT | \ + CONTEXT_DEBUG_REGISTERS | CONTEXT_EXTENDED_REGISTERS) +#else +#define CONTEXT_COMPLETE (CONTEXT_FULL | CONTEXT_DEBUG_REGISTERS) +#endif + +REDHAWK_PALEXPORT CONTEXT* PalAllocateCompleteOSContext(_Out_ uint8_t** contextBuffer) +{ + CONTEXT* pOSContext = NULL; + +#if (defined(TARGET_X86) || defined(TARGET_AMD64)) + DWORD context = CONTEXT_COMPLETE; + + if (pfnInitializeContext2 == NULL) + { + HMODULE hm = GetModuleHandleW(_T("kernel32.dll")); + if (hm != NULL) + { + pfnInitializeContext2 = (PINITIALIZECONTEXT2)GetProcAddress(hm, "InitializeContext2"); + } + } + +#ifdef TARGET_X86 + if (pfnRtlRestoreContext == NULL) + { + HMODULE hm = GetModuleHandleW(_T("ntdll.dll")); + pfnRtlRestoreContext = (PRTLRESTORECONTEXT)GetProcAddress(hm, "RtlRestoreContext"); + } +#endif //TARGET_X86 + + // Determine if the processor supports AVX so we could + // retrieve extended registers + DWORD64 FeatureMask = GetEnabledXStateFeatures(); + if ((FeatureMask & XSTATE_MASK_AVX) != 0) + { + context = context | CONTEXT_XSTATE; + } + + // Retrieve contextSize by passing NULL for Buffer + DWORD contextSize = 0; + ULONG64 xStateCompactionMask = XSTATE_MASK_LEGACY | XSTATE_MASK_AVX; + // The initialize call should fail but return contextSize + BOOL success = pfnInitializeContext2 ? + pfnInitializeContext2(NULL, context, NULL, &contextSize, xStateCompactionMask) : + InitializeContext(NULL, context, NULL, &contextSize); + + // Spec mentions that we may get a different error (it was observed on Windows7). + // In such case the contextSize is undefined. + if (success || GetLastError() != ERROR_INSUFFICIENT_BUFFER) + { + return NULL; + } + + // So now allocate a buffer of that size and call InitializeContext again + uint8_t* buffer = new (nothrow)uint8_t[contextSize]; + if (buffer != NULL) + { + success = pfnInitializeContext2 ? + pfnInitializeContext2(buffer, context, &pOSContext, &contextSize, xStateCompactionMask): + InitializeContext(buffer, context, &pOSContext, &contextSize); + + if (!success) + { + delete[] buffer; + buffer = NULL; + } + } + + if (!success) + { + pOSContext = NULL; + } + + *contextBuffer = buffer; + +#else + pOSContext = new (nothrow) CONTEXT; + pOSContext->ContextFlags = CONTEXT_COMPLETE; + *contextBuffer = NULL; +#endif + + return pOSContext; +} + +REDHAWK_PALEXPORT _Success_(return) bool REDHAWK_PALAPI PalGetCompleteThreadContext(HANDLE hThread, _Out_ CONTEXT * pCtx) +{ + _ASSERTE((pCtx->ContextFlags & CONTEXT_COMPLETE) == CONTEXT_COMPLETE); + +#if defined(TARGET_X86) || defined(TARGET_AMD64) + // Make sure that AVX feature mask is set, if supported. This should not normally fail. + // The system silently ignores any feature specified in the FeatureMask which is not enabled on the processor. + if (!SetXStateFeaturesMask(pCtx, XSTATE_MASK_AVX)) + { + _ASSERTE(!"Could not apply XSTATE_MASK_AVX"); + return FALSE; + } +#endif //defined(TARGET_X86) || defined(TARGET_AMD64) + + return GetThreadContext(hThread, pCtx); +} + +REDHAWK_PALEXPORT _Success_(return) bool REDHAWK_PALAPI PalSetThreadContext(HANDLE hThread, _Out_ CONTEXT * pCtx) +{ + return SetThreadContext(hThread, pCtx); +} + +REDHAWK_PALEXPORT void REDHAWK_PALAPI PalRestoreContext(CONTEXT * pCtx) +{ + RtlRestoreContext(pCtx, NULL); +} + REDHAWK_PALEXPORT _Success_(return) bool REDHAWK_PALAPI PalGetThreadContext(HANDLE hThread, _Out_ PAL_LIMITED_CONTEXT * pCtx) { CONTEXT win32ctx;