diff --git a/poll_default_linux.go b/poll_default_linux.go index a0087ee0..759c36c5 100644 --- a/poll_default_linux.go +++ b/poll_default_linux.go @@ -97,13 +97,27 @@ func (p *defaultPoll) Wait() (err error) { if n == p.size && p.size < 128*1024 { p.Reset(p.size<<1, caps) } - n, err = EpollWait(p.fd, p.events, msec) + // msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall) + // poller's G will hold P at most 1ms + var eventptr = uintptr(unsafe.Pointer(&p.events[0])) + var eventlen = uintptr(len(p.events)) + if msec > 0 { + n, err = EpollWaitRaw(p.fd, eventptr, eventlen, msec) + } else if msec == 0 { + n, err = EpollWaitRaw(p.fd, eventptr, eventlen, msec) + } else { // < 0 + n, err = EpollWaitBlock(uintptr(p.fd), eventptr, eventlen, uintptr(msec)) + } if err != nil && err != syscall.EINTR { return err } if n <= 0 { - msec = -1 - runtime.Gosched() + if msec > 0 { + msec = -1 // no need to sched, because we will use block syscall + } else if msec == 0 { + msec = 1 + runtime.Gosched() + } continue } msec = 0 diff --git a/poll_default_linux_test.go b/poll_default_linux_test.go index 072963d7..e0991249 100644 --- a/poll_default_linux_test.go +++ b/poll_default_linux_test.go @@ -21,6 +21,7 @@ import ( "errors" "syscall" "testing" + "unsafe" "golang.org/x/sys/unix" ) @@ -331,7 +332,9 @@ func TestEpollConnectSameFD(t *testing.T) { func epollWaitUntil(epfd int, events []epollevent, msec int) (n int, err error) { WAIT: - n, err = EpollWait(epfd, events, msec) + var eventptr = uintptr(unsafe.Pointer(&events[0])) + var eventlen = uintptr(len(events)) + n, err = EpollWait(epfd, eventptr, eventlen, msec) if err == syscall.EINTR { goto WAIT } diff --git a/sys_epoll_linux.go b/sys_epoll_linux.go index 528c34ed..c896a677 100644 --- a/sys_epoll_linux.go +++ b/sys_epoll_linux.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !arm64 && !loong64 -// +build !arm64,!loong64 +//go:build linux +// +build linux package netpoll @@ -22,11 +22,20 @@ import ( "unsafe" ) -const EPOLLET = -syscall.EPOLLET +//go:linkname _entersyscallblock runtime.entersyscallblock +func _entersyscallblock() -type epollevent struct { - events uint32 - data [8]byte // unaligned uintptr +//go:linkname _exitsyscall runtime.exitsyscall +func _exitsyscall() + +//go:nosplit +func entersyscallblock() { + _entersyscallblock() +} + +//go:nosplit +func exitsyscall() { + _exitsyscall() } // EpollCreate implements epoll_create1. @@ -49,16 +58,31 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { } // EpollWait implements epoll_wait. -func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { +func EpollWait(epfd int, eventsptr uintptr, eventslen uintptr, msec int) (n int, err error) { var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), eventsptr, eventslen, uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil } + return int(r0), err +} + +func EpollWaitRaw(epfd int, eventsptr uintptr, eventslen uintptr, msec int) (n int, err error) { + var r0 uintptr + r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), eventsptr, eventslen, uintptr(msec), 0, 0) if err == syscall.Errno(0) { err = nil } return int(r0), err } + +//go:nosplit +func EpollWaitBlock(epfd uintptr, eventsptr uintptr, eventslen uintptr, msec uintptr) (n int, err error) { + entersyscallblock() + r0, _, errno := syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, epfd, eventsptr, eventslen, msec, 0, 0) + exitsyscall() + if errno == syscall.Errno(0) { + err = nil + } + return int(r0), err +} diff --git a/sys_epoll_linux_amd64.go b/sys_epoll_linux_amd64.go new file mode 100644 index 00000000..4a058aff --- /dev/null +++ b/sys_epoll_linux_amd64.go @@ -0,0 +1,29 @@ +// Copyright 2022 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !arm64 && !loong64 +// +build !arm64,!loong64 + +package netpoll + +import ( + "syscall" +) + +const EPOLLET = -syscall.EPOLLET + +type epollevent struct { + events uint32 + data [8]byte // unaligned uintptr +} diff --git a/sys_epoll_linux_arm64.go b/sys_epoll_linux_arm64.go index e8d6094d..a138fb12 100644 --- a/sys_epoll_linux_arm64.go +++ b/sys_epoll_linux_arm64.go @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build arm64 +// +build arm64 + package netpoll import ( "syscall" - "unsafe" ) const EPOLLET = syscall.EPOLLET @@ -26,37 +28,3 @@ type epollevent struct { _ int32 data [8]byte // unaligned uintptr } - -// EpollCreate implements epoll_create1. -func EpollCreate(flag int) (fd int, err error) { - var r0 uintptr - r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} - -// EpollCtl implements epoll_ctl. -func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { - _, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return err -} - -// EpollWait implements epoll_wait. -func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { - var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) - } - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} diff --git a/sys_epoll_linux_loong64.go b/sys_epoll_linux_loong64.go index ecf36c13..af9eabdf 100644 --- a/sys_epoll_linux_loong64.go +++ b/sys_epoll_linux_loong64.go @@ -12,14 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build linux && loong64 -// +build linux,loong64 +//go:build loong64 +// +build loong64 package netpoll import ( "syscall" - "unsafe" ) const EPOLLET = syscall.EPOLLET @@ -29,37 +28,3 @@ type epollevent struct { _ int32 data [8]byte // unaligned uintptr } - -// EpollCreate implements epoll_create1. -func EpollCreate(flag int) (fd int, err error) { - var r0 uintptr - r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} - -// EpollCtl implements epoll_ctl. -func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { - _, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return err -} - -// EpollWait implements epoll_wait. -func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { - var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) - } - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -}