Separate refactoring changes in 43250

src/coreclr/jit/codegenarm.cpp

@@ -1849,28 +1849,22 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni
     {
         GetEmitter()->emitIns_R_I(INS_sub, EA_PTRSIZE, REG_SPBASE, frameSize);
     }
-    else if (frameSize < compiler->getVeryLargeFrameSize())
-    {
-        for (target_size_t probeOffset = pageSize; probeOffset <= frameSize; probeOffset += pageSize)
-        {
-            // Generate:
-            //    movw initReg, -probeOffset
-            //    ldr initReg, [SP + initReg]
-
-            instGen_Set_Reg_To_Imm(EA_PTRSIZE, initReg, -(ssize_t)probeOffset);
-            GetEmitter()->emitIns_R_R_R(INS_ldr, EA_PTRSIZE, initReg, REG_SPBASE, initReg);
-        }
-
-        regSet.verifyRegUsed(initReg);
-        *pInitRegZeroed = false; // The initReg does not contain zero
-
-        instGen_Set_Reg_To_Imm(EA_PTRSIZE, initReg, frameSize);
-        compiler->unwindPadding();
-        GetEmitter()->emitIns_R_R_R(INS_sub, EA_PTRSIZE, REG_SPBASE, REG_SPBASE, initReg);
-    }
     else
     {
-        assert(frameSize >= compiler->getVeryLargeFrameSize());
+        // Generate the following code:
+        //
+        //    movw  r4, #frameSize
+        //    sub   r4, sp, r4
+        //    bl    CORINFO_HELP_STACK_PROBE
+        //    mov   sp, r4
+        //
+        // If frameSize can not be encoded by movw immediate this becomes:
+        //
+        //    movw  r4, #frameSizeLo16
+        //    movt  r4, #frameSizeHi16
+        //    sub   r4, sp, r4
+        //    bl    CORINFO_HELP_STACK_PROBE
+        //    mov   sp, r4
 
         genInstrWithConstant(INS_sub, EA_PTRSIZE, REG_STACK_PROBE_HELPER_ARG, REG_SPBASE, frameSize,
                              INS_FLAGS_DONT_CARE, REG_STACK_PROBE_HELPER_ARG);

src/coreclr/jit/codegenarm64.cpp

@@ -9658,8 +9658,15 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni
     {
         lastTouchDelta = frameSize;
     }
-    else if (frameSize < compiler->getVeryLargeFrameSize())
+    else if (frameSize < 3 * pageSize)
     {
+        // The probing loop in "else"-case below would require at least 6 instructions (and more if
+        // 'frameSize' or 'pageSize' can not be encoded with mov-instruction immediate).
+        // Hence for frames that are smaller than 3 * PAGE_SIZE the JIT inlines the following probing code
+        // to decrease code size.
+        // TODO-ARM64: The probing mechanisms should be replaced by a call to stack probe helper
+        // as it is done on other platforms.
+
         lastTouchDelta = frameSize;
 
         for (target_size_t probeOffset = pageSize; probeOffset <= frameSize; probeOffset += pageSize)
@@ -9681,8 +9688,6 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni
     }
     else
     {
-        assert(frameSize >= compiler->getVeryLargeFrameSize());
-
         // Emit the following sequence to 'tickle' the pages. Note it is important that stack pointer not change
         // until this is complete since the tickles could cause a stack overflow, and we need to be able to crawl
         // the stack afterward (which means the stack pointer needs to be known).

src/coreclr/jit/codegenxarch.cpp

@@ -8950,4 +8950,61 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper)
 
 #endif // PROFILING_SUPPORTED
 
+//------------------------------------------------------------------------
+// genPushCalleeSavedRegisters: Push any callee-saved registers we have used.
+//
+void CodeGen::genPushCalleeSavedRegisters()
+{
+    assert(compiler->compGeneratingProlog);
+
+    // x86/x64 doesn't support push of xmm/ymm regs, therefore consider only integer registers for pushing onto stack
+    // here. Space for float registers to be preserved is stack allocated and saved as part of prolog sequence and not
+    // here.
+    regMaskTP rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_INT_CALLEE_SAVED;
+
+#if ETW_EBP_FRAMED
+    if (!isFramePointerUsed() && regSet.rsRegsModified(RBM_FPBASE))
+    {
+        noway_assert(!"Used register RBM_FPBASE as a scratch register!");
+    }
+#endif
+
+    // On X86/X64 we have already pushed the FP (frame-pointer) prior to calling this method
+    if (isFramePointerUsed())
+    {
+        rsPushRegs &= ~RBM_FPBASE;
+    }
+
+#ifdef DEBUG
+    if (compiler->compCalleeRegsPushed != genCountBits(rsPushRegs))
+    {
+        printf("Error: unexpected number of callee-saved registers to push. Expected: %d. Got: %d ",
+               compiler->compCalleeRegsPushed, genCountBits(rsPushRegs));
+        dspRegMask(rsPushRegs);
+        printf("\n");
+        assert(compiler->compCalleeRegsPushed == genCountBits(rsPushRegs));
+    }
+#endif // DEBUG
+
+    // Push backwards so we match the order we will pop them in the epilog
+    // and all the other code that expects it to be in this order.
+    for (regNumber reg = REG_INT_LAST; rsPushRegs != RBM_NONE; reg = REG_PREV(reg))
+    {
+        regMaskTP regBit = genRegMask(reg);
+
+        if ((regBit & rsPushRegs) != 0)
+        {
+            inst_RV(INS_push, reg, TYP_REF);
+            compiler->unwindPush(reg);
+#ifdef USING_SCOPE_INFO
+            if (!doubleAlignOrFramePointerUsed())
+            {
+                psiAdjustStackLevel(REGSIZE_BYTES);
+            }
+#endif // USING_SCOPE_INFO
+            rsPushRegs &= ~regBit;
+        }
+    }
+}
+
 #endif // TARGET_XARCH

src/coreclr/jit/compiler.h

@@ -7495,19 +7495,6 @@ class Compiler
         return (target_size_t)eeGetEEInfo()->osPageSize;
     }
 
-    // Returns the frame size at which we will generate a loop to probe the stack.
-    target_size_t getVeryLargeFrameSize()
-    {
-#ifdef TARGET_ARM
-        // The looping probe code is 40 bytes, whereas the straight-line probing for
-        // the (0x2000..0x3000) case is 44, so use looping for anything 0x2000 bytes
-        // or greater, to generate smaller code.
-        return 2 * eeGetPageSize();
-#else
-        return 3 * eeGetPageSize();
-#endif
-    }
-
     //------------------------------------------------------------------------
     // VirtualStubParam: virtual stub dispatch extra parameter (slot address).
     //

src/coreclr/vm/amd64/JitHelpers_Fast.asm

@@ -395,7 +395,7 @@ LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT
 ;
 ; NOTE: this helper will NOT modify a value of rsp and can be defined as a leaf function.
 
-PAGE_SIZE equ 1000h
+PROBE_PAGE_SIZE equ 1000h
 
 LEAF_ENTRY JIT_StackProbe, _TEXT
         ; On entry:
@@ -408,11 +408,11 @@ LEAF_ENTRY JIT_StackProbe, _TEXT
         ; NOTE: this helper will probe at least one page below the one pointed by rsp.
 
         mov     rax, rsp               ; rax points to some byte on the last probed page
-        and     rax, -PAGE_SIZE        ; rax points to the **lowest address** on the last probed page
+        and     rax, -PROBE_PAGE_SIZE  ; rax points to the **lowest address** on the last probed page
                                        ; This is done to make the following loop end condition simpler.
 
 ProbeLoop:
-        sub     rax, PAGE_SIZE         ; rax points to the lowest address of the **next page** to probe
+        sub     rax, PROBE_PAGE_SIZE   ; rax points to the lowest address of the **next page** to probe
         test    dword ptr [rax], eax   ; rax points to the lowest address on the **last probed** page
         cmp     rax, r11
         jg      ProbeLoop              ; If (rax > r11), then we need to probe at least one more page.

src/coreclr/vm/amd64/jithelpers_fast.S

@@ -425,7 +425,7 @@ LEAF_END JIT_WriteBarrier_Callable, _TEXT
 //
 // See also https://github.com/dotnet/runtime/issues/9899#issue-303331518 for more information.
 
-#define PAGE_SIZE 0x1000
+#define PROBE_PAGE_SIZE 0x1000
 
 LEAF_ENTRY JIT_StackProbe, _TEXT
         // On entry:
@@ -442,11 +442,11 @@ LEAF_ENTRY JIT_StackProbe, _TEXT
 
     END_PROLOGUE
 
-        and     rsp, -PAGE_SIZE        // rsp points to the **lowest address** on the last probed page
+        and     rsp, -PROBE_PAGE_SIZE  // rsp points to the **lowest address** on the last probed page
                                        // This is done to make the following loop end condition simpler.
 
 LOCAL_LABEL(ProbeLoop):
-        sub     rsp, PAGE_SIZE         // rsp points to the lowest address of the **next page** to probe
+        sub     rsp, PROBE_PAGE_SIZE   // rsp points to the lowest address of the **next page** to probe
         test    dword ptr [rsp], eax   // rsp points to the lowest address on the **last probed** page
         cmp     rsp, r11
         jg      LOCAL_LABEL(ProbeLoop) // if (rsp > r11), then we need to probe at least one more page.

src/coreclr/vm/arm/asmhelpers.S

@@ -1106,7 +1106,7 @@ DelayLoad_Helper\suffix:
 // The following helper will access ("probe") a word on each page of the stack
 // starting with the page right beneath sp down to the one pointed to by r4.
 // The procedure is needed to make sure that the "guard" page is pushed down below the allocated stack frame.
-// The call to the helper will be emitted by JIT in the function/funclet prolog when large (larger than 0x3000 bytes) stack frame is required.
+// The call to the helper will be emitted by JIT in the function/funclet prolog when stack frame is larger than an OS page.
 // On entry:
 //   r4 - points to the lowest address on the stack frame being allocated (i.e. [InitialSp - FrameSize])
 //   sp - points to some byte on the last probed page
@@ -1115,23 +1115,23 @@ DelayLoad_Helper\suffix:
 //   r5 - is not preserved
 //
 // NOTE: this helper will probe at least one page below the one pointed to by sp.
-#define PAGE_SIZE      0x1000
-#define PAGE_SIZE_LOG2 12
+#define PROBE_PAGE_SIZE      4096
+#define PROBE_PAGE_SIZE_LOG2 12
 
         LEAF_ENTRY JIT_StackProbe, _TEXT
         PROLOG_PUSH "{r7}"
         PROLOG_STACK_SAVE r7
 
-        mov r5, sp                       // r5 points to some byte on the last probed page
-        bfc r5, #0, #PAGE_SIZE_LOG2      // r5 points to the **lowest address** on the last probed page
+        mov r5, sp                         // r5 points to some byte on the last probed page
+        bfc r5, #0, #PROBE_PAGE_SIZE_LOG2  // r5 points to the **lowest address** on the last probed page
         mov sp, r5
 
 ProbeLoop:
-                                         // Immediate operand for the following instruction can not be greater than 4095.
-        sub sp, #(PAGE_SIZE - 4)         // sp points to the **fourth** byte on the **next page** to probe
-        ldr r5, [sp, #-4]!               // sp points to the lowest address on the **last probed** page
+                                           // Immediate operand for the following instruction can not be greater than 4095.
+        sub sp, #(PROBE_PAGE_SIZE - 4)     // sp points to the **fourth** byte on the **next page** to probe
+        ldr r5, [sp, #-4]!                 // sp points to the lowest address on the **last probed** page
         cmp sp, r4
-        bhi ProbeLoop                    // If (sp > r4), then we need to probe at least one more page.
+        bhi ProbeLoop                      // If (sp > r4), then we need to probe at least one more page.
 
         EPILOG_STACK_RESTORE r7
         EPILOG_POP "{r7}"

src/coreclr/vm/arm/asmhelpers.asm

@@ -1835,7 +1835,7 @@ $__RealName
 ;; The following helper will access ("probe") a word on each page of the stack
 ;; starting with the page right beneath sp down to the one pointed to by r4.
 ;; The procedure is needed to make sure that the "guard" page is pushed down below the allocated stack frame.
-;; The call to the helper will be emitted by JIT in the function/funclet prolog when large (larger than 0x3000 bytes) stack frame is required.
+;; The call to the helper will be emitted by JIT in the function/funclet prolog when stack frame is larger than an OS page.
 ;;-----------------------------------------------------------------------------
 ; On entry:
 ;   r4 - points to the lowest address on the stack frame being allocated (i.e. [InitialSp - FrameSize])
@@ -1845,21 +1845,23 @@ $__RealName
 ;   r5 - is not preserved
 ;
 ; NOTE: this helper will probe at least one page below the one pointed to by sp.
-#define PAGE_SIZE_LOG2 12
+#define PROBE_PAGE_SIZE      4096
+#define PROBE_PAGE_SIZE_LOG2 12
+
     LEAF_ENTRY JIT_StackProbe
     PROLOG_PUSH {r7}
     PROLOG_STACK_SAVE r7
 
-    mov r5, sp                       ; r5 points to some byte on the last probed page
-    bfc r5, #0, #PAGE_SIZE_LOG2      ; r5 points to the **lowest address** on the last probed page
+    mov r5, sp                         ; r5 points to some byte on the last probed page
+    bfc r5, #0, #PROBE_PAGE_SIZE_LOG2  ; r5 points to the **lowest address** on the last probed page
     mov sp, r5
 
 ProbeLoop
-                                     ; Immediate operand for the following instruction can not be greater than 4095.
-    sub sp, #(PAGE_SIZE - 4)         ; sp points to the **fourth** byte on the **next page** to probe
-    ldr r5, [sp, #-4]!               ; sp points to the lowest address on the **last probed** page
+                                       ; Immediate operand for the following instruction can not be greater than 4095.
+    sub sp, #(PROBE_PAGE_SIZE - 4)     ; sp points to the **fourth** byte on the **next page** to probe
+    ldr r5, [sp, #-4]!                 ; sp points to the lowest address on the **last probed** page
     cmp sp, r4
-    bhi ProbeLoop                    ; if (sp > r4), then we need to probe at least one more page.
+    bhi ProbeLoop                      ; if (sp > r4), then we need to probe at least one more page.
 
     EPILOG_STACK_RESTORE r7
     EPILOG_POP {r7}

src/coreclr/vm/i386/jithelp.S

@@ -618,7 +618,7 @@ LEAF_END JIT_Dbl2IntSSE2, _TEXT
 // NOTE: this helper will modify a value of esp and must establish the frame pointer.
 // NOTE: On Linux we must advance the stack pointer as we probe - it is not allowed to access 65535 bytes below esp.
 //
-#define PAGE_SIZE 0x1000
+#define PROBE_PAGE_SIZE 0x1000
 NESTED_ENTRY JIT_StackProbe, _TEXT, NoHandler
     // On entry:
     //   eax - the lowest address of the stack frame being allocated (i.e. [InitialSp - FrameSize])
@@ -627,11 +627,11 @@ NESTED_ENTRY JIT_StackProbe, _TEXT, NoHandler
     PROLOG_BEG
     PROLOG_END
 
-    and     esp, -PAGE_SIZE        // esp points to the **lowest address** on the last probed page
+    and     esp, -PROBE_PAGE_SIZE  // esp points to the **lowest address** on the last probed page
                                    // This is done to make the loop end condition simpler.
 
 LOCAL_LABEL(ProbeLoop):
-    sub     esp, PAGE_SIZE         // esp points to the lowest address of the **next page** to probe
+    sub     esp, PROBE_PAGE_SIZE   // esp points to the lowest address of the **next page** to probe
     test    [esp], eax             // esp points to the lowest address on the **last probed** page
     cmp     esp, eax
     jg      LOCAL_LABEL(ProbeLoop) // if esp > eax, then we need to probe at least one more page.

src/coreclr/vm/i386/jithelp.asm

@@ -1313,7 +1313,7 @@ JIT_EndCatch ENDP
 ; The call to the helper will be emitted by JIT in the function prolog when large (larger than 0x3000 bytes) stack frame is required.
 ;
 ; NOTE: this helper will modify a value of esp and must establish the frame pointer.
-PAGE_SIZE equ 1000h
+PROBE_PAGE_SIZE equ 1000h
 
 _JIT_StackProbe@0 PROC public
     ; On entry:
@@ -1323,13 +1323,13 @@ _JIT_StackProbe@0 PROC public
     push    ebp
     mov     ebp, esp
 
-    and     esp, -PAGE_SIZE      ; esp points to the **lowest address** on the last probed page
-                                 ; This is done to make the loop end condition simpler.
+    and     esp, -PROBE_PAGE_SIZE ; esp points to the **lowest address** on the last probed page
+                                  ; This is done to make the loop end condition simpler.
 ProbeLoop:
-    test    [esp - 4], eax       ; esp points to the lowest address on the **last probed** page
-    sub     esp, PAGE_SIZE       ; esp points to the lowest address of the **next page** to probe
+    test    [esp - 4], eax        ; esp points to the lowest address on the **last probed** page
+    sub     esp, PROBE_PAGE_SIZE  ; esp points to the lowest address of the **next page** to probe
     cmp     esp, eax
-    jg      ProbeLoop            ; if esp > eax, then we need to probe at least one more page.
+    jg      ProbeLoop             ; if esp > eax, then we need to probe at least one more page.
 
     mov     esp, ebp
     pop     ebp