From d67375558fde67f88559e7cc17d2245d95a95099 Mon Sep 17 00:00:00 2001 From: wangzhuowei Date: Mon, 18 Sep 2023 19:06:00 +0800 Subject: [PATCH] perf: use block syscall enter for epoll_wait --- poll_default_linux.go | 18 +++++++++++++++--- sys_epoll_linux.go | 27 ++++++++++++++++++++++----- sys_epoll_linux_arm64.go | 27 ++++++++++++++++++++++----- sys_epoll_linux_loong64.go | 27 ++++++++++++++++++++++----- sys_exec.go | 6 ++++++ 5 files changed, 87 insertions(+), 18 deletions(-) diff --git a/poll_default_linux.go b/poll_default_linux.go index a0087ee0..fa6a5683 100644 --- a/poll_default_linux.go +++ b/poll_default_linux.go @@ -97,13 +97,25 @@ func (p *defaultPoll) Wait() (err error) { if n == p.size && p.size < 128*1024 { p.Reset(p.size<<1, caps) } - n, err = EpollWait(p.fd, p.events, msec) + // msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall) + // poller's G will hold P at most 1ms + if msec > 0 { + n, err = EpollWaitRaw(p.fd, p.events, msec) + } else if msec == 0 { + n, err = EpollWaitRaw(p.fd, p.events, msec) + } else { // < 0 + n, err = EpollWaitBlock(p.fd, p.events, msec) + } if err != nil && err != syscall.EINTR { return err } if n <= 0 { - msec = -1 - runtime.Gosched() + if msec > 0 { + msec = -1 // no need to sched, because we will use block syscall + } else if msec == 0 { + msec = 1 + runtime.Gosched() + } continue } msec = 0 diff --git a/sys_epoll_linux.go b/sys_epoll_linux.go index 528c34ed..c91f750b 100644 --- a/sys_epoll_linux.go +++ b/sys_epoll_linux.go @@ -48,15 +48,32 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { return err } -// EpollWait implements epoll_wait. func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { var r0 uintptr var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil } + return int(r0), err +} + +func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) { + var r0 uintptr + var _p0 = unsafe.Pointer(&events[0]) + r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil + } + return int(r0), err +} + +func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) { + var r0 uintptr + var _p0 = unsafe.Pointer(&events[0]) + entersyscallblock() + r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + exitsyscall() if err == syscall.Errno(0) { err = nil } diff --git a/sys_epoll_linux_arm64.go b/sys_epoll_linux_arm64.go index e8d6094d..dab3c79e 100644 --- a/sys_epoll_linux_arm64.go +++ b/sys_epoll_linux_arm64.go @@ -46,15 +46,32 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { return err } -// EpollWait implements epoll_wait. func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { var r0 uintptr var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil } + return int(r0), err +} + +func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) { + var r0 uintptr + var _p0 = unsafe.Pointer(&events[0]) + r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil + } + return int(r0), err +} + +func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) { + var r0 uintptr + var _p0 = unsafe.Pointer(&events[0]) + entersyscallblock() + r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + exitsyscall() if err == syscall.Errno(0) { err = nil } diff --git a/sys_epoll_linux_loong64.go b/sys_epoll_linux_loong64.go index ecf36c13..1c966609 100644 --- a/sys_epoll_linux_loong64.go +++ b/sys_epoll_linux_loong64.go @@ -49,15 +49,32 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { return err } -// EpollWait implements epoll_wait. func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { var r0 uintptr var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil } + return int(r0), err +} + +func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) { + var r0 uintptr + var _p0 = unsafe.Pointer(&events[0]) + r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil + } + return int(r0), err +} + +func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) { + var r0 uintptr + var _p0 = unsafe.Pointer(&events[0]) + entersyscallblock() + r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + exitsyscall() if err == syscall.Errno(0) { err = nil } diff --git a/sys_exec.go b/sys_exec.go index 1c8e40e4..66a3e2e5 100644 --- a/sys_exec.go +++ b/sys_exec.go @@ -24,6 +24,12 @@ import ( "unsafe" ) +//go:linkname entersyscallblock runtime.entersyscallblock +func entersyscallblock() + +//go:linkname exitsyscall runtime.exitsyscall +func exitsyscall() + // GetSysFdPairs creates and returns the fds of a pair of sockets. func GetSysFdPairs() (r, w int) { fds, _ := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)