Miscompilation of tail call on x86_64: RSP is erroneously increased

[cypheon]

In some cases, when calling a tailcc function in tail position, rsp is increased in a branch where it has never been decreased. This leads to corrupted stack pointer (and probably a segfault) in the following function.

Repro:

declare tailcc void @f2()
declare tailcc void @f1(ptr, i64, ptr, i8, ptr, ptr, ptr, ptr, ptr, ptr)

define tailcc void @repro(i64 %0) {
  %cond = icmp ugt i64 %0, 0
  br i1 %cond, label %a, label %b
a:
  musttail call tailcc void @f1(ptr null, i64 16, ptr null, i8 1, ptr null, ptr null, ptr null, ptr null, ptr null, ptr null)
  ret void
b:
  musttail call tailcc void @f2()
  ret void
}

Resulting assembly:

repro:                                  # @repro
# %bb.0:
        test    rdi, rdi
        je      .LBB0_2
# %bb.1:                                # %a
        sub     rsp, 32
        xorps   xmm0, xmm0
        movaps  xmmword ptr [rsp + 8], xmm0
        mov     rax, qword ptr [rsp + 32]
        movaps  xmmword ptr [rsp + 24], xmm0
        mov     qword ptr [rsp], rax
        mov     esi, 16
        xor     edi, edi
        xor     edx, edx
        mov     ecx, 1
        xor     r8d, r8d
        xor     r9d, r9d
        jmp     f1@PLT                          # TAILCALL
.LBB0_2:                                # %b
        add     rsp, 32  # <- rsp should not be increased here, as it is only decreased in branch "%a"
        jmp     f2@PLT                          # TAILCALL
                                        # -- End function

https://godbolt.org/z/ze7r8j67o

As far as I can see, the issue does not occur on aarch64. There the stack pointer is decremented right at the beginning (before the branch), so the increment before the tail call is correct.

[llvmbot]

@llvm/issue-subscribers-backend-x86

Author: Johann Rudloff (cypheon)

In some cases, when calling a tailcc function in tail position, `rsp` is increased in a branch where it has never been decreased. This leads to corrupted stack pointer (and probably a segfault) in the following function.

Repro:

declare tailcc void @<!-- -->f2()
declare tailcc void @<!-- -->f1(ptr, i64, ptr, i8, ptr, ptr, ptr, ptr, ptr, ptr)

define tailcc void @<!-- -->repro(i64 %0) {
  %cond = icmp ugt i64 %0, 0
  br i1 %cond, label %a, label %b
a:
  musttail call tailcc void @<!-- -->f1(ptr null, i64 16, ptr null, i8 1, ptr null, ptr null, ptr null, ptr null, ptr null, ptr null)
  ret void
b:
  musttail call tailcc void @<!-- -->f2()
  ret void
}

Resulting assembly:

repro:                                  # @<!-- -->repro
# %bb.0:
        test    rdi, rdi
        je      .LBB0_2
# %bb.1:                                # %a
        sub     rsp, 32
        xorps   xmm0, xmm0
        movaps  xmmword ptr [rsp + 8], xmm0
        mov     rax, qword ptr [rsp + 32]
        movaps  xmmword ptr [rsp + 24], xmm0
        mov     qword ptr [rsp], rax
        mov     esi, 16
        xor     edi, edi
        xor     edx, edx
        mov     ecx, 1
        xor     r8d, r8d
        xor     r9d, r9d
        jmp     f1@<!-- -->PLT                          # TAILCALL
.LBB0_2:                                # %b
        add     rsp, 32  # &lt;- rsp should not be increased here, as it is only decreased in branch "%a"
        jmp     f2@<!-- -->PLT                          # TAILCALL
                                        # -- End function

https://godbolt.org/z/ze7r8j67o

As far as I can see, the issue does not occur on aarch64. There the stack pointer is decremented right at the beginning (before the branch), so the increment before the tail call is correct.