perf: use block syscall enter for epoll_wait

cloudwego · Sep 19, 2023 · d673755 · d673755
1 parent 5719b53
commit d673755
Show file tree

Hide file tree

Showing 5 changed files with 87 additions and 18 deletions.
diff --git a/poll_default_linux.go b/poll_default_linux.go
@@ -97,13 +97,25 @@ func (p *defaultPoll) Wait() (err error) {
 		if n == p.size && p.size < 128*1024 {
 			p.Reset(p.size<<1, caps)
 		}
-		n, err = EpollWait(p.fd, p.events, msec)
+		// msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall)
+		// poller's G will hold P at most 1ms
+		if msec > 0 {
+			n, err = EpollWaitRaw(p.fd, p.events, msec)
+		} else if msec == 0 {
+			n, err = EpollWaitRaw(p.fd, p.events, msec)
+		} else { // < 0
+			n, err = EpollWaitBlock(p.fd, p.events, msec)
+		}
 		if err != nil && err != syscall.EINTR {
 			return err
 		}
 		if n <= 0 {
-			msec = -1
-			runtime.Gosched()
+			if msec > 0 {
+				msec = -1 // no need to sched, because we will use block syscall
+			} else if msec == 0 {
+				msec = 1
+				runtime.Gosched()
+			}
 			continue
 		}
 		msec = 0

diff --git a/sys_epoll_linux.go b/sys_epoll_linux.go
@@ -48,15 +48,32 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
 	return err
 }
 
-// EpollWait implements epoll_wait.
 func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
 	var r0 uintptr
 	var _p0 = unsafe.Pointer(&events[0])
-	if msec == 0 {
-		r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
-	} else {
-		r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	if err == syscall.Errno(0) {
+		err = nil
 	}
+	return int(r0), err
+}
+
+func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) {
+	var r0 uintptr
+	var _p0 = unsafe.Pointer(&events[0])
+	r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	if err == syscall.Errno(0) {
+		err = nil
+	}
+	return int(r0), err
+}
+
+func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) {
+	var r0 uintptr
+	var _p0 = unsafe.Pointer(&events[0])
+	entersyscallblock()
+	r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	exitsyscall()
 	if err == syscall.Errno(0) {
 		err = nil
 	}

diff --git a/sys_epoll_linux_arm64.go b/sys_epoll_linux_arm64.go
@@ -46,15 +46,32 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
 	return err
 }
 
-// EpollWait implements epoll_wait.
 func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
 	var r0 uintptr
 	var _p0 = unsafe.Pointer(&events[0])
-	if msec == 0 {
-		r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
-	} else {
-		r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	if err == syscall.Errno(0) {
+		err = nil
 	}
+	return int(r0), err
+}
+
+func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) {
+	var r0 uintptr
+	var _p0 = unsafe.Pointer(&events[0])
+	r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	if err == syscall.Errno(0) {
+		err = nil
+	}
+	return int(r0), err
+}
+
+func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) {
+	var r0 uintptr
+	var _p0 = unsafe.Pointer(&events[0])
+	entersyscallblock()
+	r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	exitsyscall()
 	if err == syscall.Errno(0) {
 		err = nil
 	}

diff --git a/sys_epoll_linux_loong64.go b/sys_epoll_linux_loong64.go
@@ -49,15 +49,32 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
 	return err
 }
 
-// EpollWait implements epoll_wait.
 func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
 	var r0 uintptr
 	var _p0 = unsafe.Pointer(&events[0])
-	if msec == 0 {
-		r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
-	} else {
-		r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	if err == syscall.Errno(0) {
+		err = nil
 	}
+	return int(r0), err
+}
+
+func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) {
+	var r0 uintptr
+	var _p0 = unsafe.Pointer(&events[0])
+	r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	if err == syscall.Errno(0) {
+		err = nil
+	}
+	return int(r0), err
+}
+
+func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) {
+	var r0 uintptr
+	var _p0 = unsafe.Pointer(&events[0])
+	entersyscallblock()
+	r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	exitsyscall()
 	if err == syscall.Errno(0) {
 		err = nil
 	}

diff --git a/sys_exec.go b/sys_exec.go
@@ -24,6 +24,12 @@ import (
 	"unsafe"
 )
 
+//go:linkname entersyscallblock runtime.entersyscallblock
+func entersyscallblock()
+
+//go:linkname exitsyscall runtime.exitsyscall
+func exitsyscall()
+
 // GetSysFdPairs creates and returns the fds of a pair of sockets.
 func GetSysFdPairs() (r, w int) {
 	fds, _ := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)