diff --git a/go.mod b/go.mod index 0f672d8ec2d7..7ef1efa1db11 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/checkpoint-restore/checkpointctl v1.3.0 github.com/checkpoint-restore/go-criu/v7 v7.2.0 github.com/containerd/btrfs/v2 v2.0.0 - github.com/containerd/cgroups/v3 v3.0.3 + github.com/containerd/cgroups/v3 v3.0.4 github.com/containerd/console v1.0.4 github.com/containerd/containerd/api v1.8.0 github.com/containerd/continuity v0.4.5 @@ -92,7 +92,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cilium/ebpf v0.11.0 // indirect + github.com/cilium/ebpf v0.16.0 // indirect github.com/containers/ocicrypt v1.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -134,7 +134,7 @@ require ( go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.28.0 // indirect - golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect + golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/term v0.25.0 // indirect diff --git a/go.sum b/go.sum index 9c168c1d0af9..048f84ed3726 100644 --- a/go.sum +++ b/go.sum @@ -647,8 +647,8 @@ github.com/checkpoint-restore/go-criu/v7 v7.2.0/go.mod h1:u0LCWLg0w4yqqu14aXhiB4 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= -github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= +github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= +github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -665,8 +665,8 @@ github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/containerd/btrfs/v2 v2.0.0 h1:FN4wsx7KQrYoLXN7uLP0vBV4oVWHOIKDRQ1G2Z0oL5M= github.com/containerd/btrfs/v2 v2.0.0/go.mod h1:swkD/7j9HApWpzl8OHfrHNxppPd9l44DFZdF94BUj9k= -github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= -github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= +github.com/containerd/cgroups/v3 v3.0.4 h1:2fs7l3P0Qxb1nKWuJNFiwhp2CqiKzho71DQkDrHJIo4= +github.com/containerd/cgroups/v3 v3.0.4/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= @@ -748,8 +748,6 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -784,6 +782,8 @@ github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogB github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -929,6 +929,10 @@ github.com/intel/goresctrl v0.8.0 h1:N3shVbS3kA1Hk2AmcbHv8805Hjbv+zqsCIZCGktxx50 github.com/intel/goresctrl v0.8.0/go.mod h1:T3ZZnuHSNouwELB5wvOoUJaB7l/4Rm23rJy/wuWJlr0= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -969,6 +973,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= @@ -1205,8 +1211,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1548,8 +1554,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= +golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vendor/github.com/cilium/ebpf/.clang-format b/vendor/github.com/cilium/ebpf/.clang-format index 3f74dc023665..0ff4257606fb 100644 --- a/vendor/github.com/cilium/ebpf/.clang-format +++ b/vendor/github.com/cilium/ebpf/.clang-format @@ -4,6 +4,9 @@ BasedOnStyle: LLVM AlignAfterOpenBracket: DontAlign AlignConsecutiveAssignments: true AlignEscapedNewlines: DontAlign +# mkdocs annotations in source code are written as trailing comments +# and alignment pushes these really far away from the content. +AlignTrailingComments: false AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: false AllowAllParametersOfDeclarationOnNextLine: false @@ -16,4 +19,7 @@ UseTab: ForContinuationAndIndentation ColumnLimit: 1000 # Go compiler comments need to stay unindented. CommentPragmas: '^go:.*' +# linux/bpf.h needs to be included before bpf/bpf_helpers.h for types like __u64 +# and sorting makes this impossible. +SortIncludes: false ... diff --git a/vendor/github.com/cilium/ebpf/.gitattributes b/vendor/github.com/cilium/ebpf/.gitattributes new file mode 100644 index 000000000000..113f97b9804d --- /dev/null +++ b/vendor/github.com/cilium/ebpf/.gitattributes @@ -0,0 +1 @@ +internal/sys/types.go linguist-generated=false diff --git a/vendor/github.com/cilium/ebpf/.golangci.yaml b/vendor/github.com/cilium/ebpf/.golangci.yaml index 06743dfc91b4..65f91b910bf6 100644 --- a/vendor/github.com/cilium/ebpf/.golangci.yaml +++ b/vendor/github.com/cilium/ebpf/.golangci.yaml @@ -1,15 +1,7 @@ --- -issues: - exclude-rules: - # syscall param structs will have unused fields in Go code. - - path: syscall.*.go - linters: - - structcheck - linters: disable-all: true enable: - - errcheck - goimports - gosimple - govet @@ -19,8 +11,3 @@ linters: - typecheck - unused - gofmt - - # Could be enabled later: - # - gocyclo - # - maligned - # - gosec diff --git a/vendor/github.com/cilium/ebpf/.vimto.toml b/vendor/github.com/cilium/ebpf/.vimto.toml new file mode 100644 index 000000000000..49a12dbc090d --- /dev/null +++ b/vendor/github.com/cilium/ebpf/.vimto.toml @@ -0,0 +1,12 @@ +kernel="ghcr.io/cilium/ci-kernels:stable" +smp="cpus=2" +memory="1G" +user="root" +setup=[ + "mount -t cgroup2 -o nosuid,noexec,nodev cgroup2 /sys/fs/cgroup", + "/bin/sh -c 'modprobe bpf_testmod || true'", + "dmesg --clear", +] +teardown=[ + "dmesg --read-clear", +] diff --git a/vendor/github.com/cilium/ebpf/ARCHITECTURE.md b/vendor/github.com/cilium/ebpf/ARCHITECTURE.md deleted file mode 100644 index 26f555eb7a76..000000000000 --- a/vendor/github.com/cilium/ebpf/ARCHITECTURE.md +++ /dev/null @@ -1,92 +0,0 @@ -Architecture of the library -=== - -```mermaid -graph RL - Program --> ProgramSpec --> ELF - btf.Spec --> ELF - Map --> MapSpec --> ELF - Links --> Map & Program - ProgramSpec -.-> btf.Spec - MapSpec -.-> btf.Spec - subgraph Collection - Program & Map - end - subgraph CollectionSpec - ProgramSpec & MapSpec & btf.Spec - end -``` - -ELF ---- - -BPF is usually produced by using Clang to compile a subset of C. Clang outputs -an ELF file which contains program byte code (aka BPF), but also metadata for -maps used by the program. The metadata follows the conventions set by libbpf -shipped with the kernel. Certain ELF sections have special meaning -and contain structures defined by libbpf. Newer versions of clang emit -additional metadata in [BPF Type Format](#BTF). - -The library aims to be compatible with libbpf so that moving from a C toolchain -to a Go one creates little friction. To that end, the [ELF reader](elf_reader.go) -is tested against the Linux selftests and avoids introducing custom behaviour -if possible. - -The output of the ELF reader is a `CollectionSpec` which encodes -all of the information contained in the ELF in a form that is easy to work with -in Go. The returned `CollectionSpec` should be deterministic: reading the same ELF -file on different systems must produce the same output. -As a corollary, any changes that depend on the runtime environment like the -current kernel version must happen when creating [Objects](#Objects). - -Specifications ---- - -`CollectionSpec` is a very simple container for `ProgramSpec`, `MapSpec` and -`btf.Spec`. Avoid adding functionality to it if possible. - -`ProgramSpec` and `MapSpec` are blueprints for in-kernel -objects and contain everything necessary to execute the relevant `bpf(2)` -syscalls. They refer to `btf.Spec` for type information such as `Map` key and -value types. - -The [asm](asm/) package provides an assembler that can be used to generate -`ProgramSpec` on the fly. - -Objects ---- - -`Program` and `Map` are the result of loading specifications into the kernel. -Features that depend on knowledge of the current system (e.g kernel version) -are implemented at this point. - -Sometimes loading a spec will fail because the kernel is too old, or a feature is not -enabled. There are multiple ways the library deals with that: - -* Fallback: older kernels don't allow naming programs and maps. The library - automatically detects support for names, and omits them during load if - necessary. This works since name is primarily a debug aid. - -* Sentinel error: sometimes it's possible to detect that a feature isn't available. - In that case the library will return an error wrapping `ErrNotSupported`. - This is also useful to skip tests that can't run on the current kernel. - -Once program and map objects are loaded they expose the kernel's low-level API, -e.g. `NextKey`. Often this API is awkward to use in Go, so there are safer -wrappers on top of the low-level API, like `MapIterator`. The low-level API is -useful when our higher-level API doesn't support a particular use case. - -Links ---- - -Programs can be attached to many different points in the kernel and newer BPF hooks -tend to use bpf_link to do so. Older hooks unfortunately use a combination of -syscalls, netlink messages, etc. Adding support for a new link type should not -pull in large dependencies like netlink, so XDP programs or tracepoints are -out of scope. - -Each bpf_link_type has one corresponding Go type, e.g. `link.tracing` corresponds -to BPF_LINK_TRACING. In general, these types should be unexported as long as they -don't export methods outside of the Link interface. Each Go type may have multiple -exported constructors. For example `AttachTracing` and `AttachLSM` create a -tracing link, but are distinct functions since they may require different arguments. diff --git a/vendor/github.com/cilium/ebpf/CODEOWNERS b/vendor/github.com/cilium/ebpf/CODEOWNERS new file mode 100644 index 000000000000..ca65d23c09d2 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/CODEOWNERS @@ -0,0 +1,11 @@ +* @cilium/ebpf-lib-maintainers + +features/ @rgo3 +link/ @mmat11 + +perf/ @florianl +ringbuf/ @florianl + +btf/ @dylandreimerink + +cmd/bpf2go/ @mejedi diff --git a/vendor/github.com/cilium/ebpf/CONTRIBUTING.md b/vendor/github.com/cilium/ebpf/CONTRIBUTING.md index bf57da939539..673a9ac29074 100644 --- a/vendor/github.com/cilium/ebpf/CONTRIBUTING.md +++ b/vendor/github.com/cilium/ebpf/CONTRIBUTING.md @@ -1,48 +1,5 @@ -# How to contribute +# Contributing to ebpf-go -Development is on [GitHub](https://github.com/cilium/ebpf) and contributions in -the form of pull requests and issues reporting bugs or suggesting new features -are welcome. Please take a look at [the architecture](ARCHITECTURE.md) to get -a better understanding for the high-level goals. - -## Adding a new feature - -1. [Join](https://ebpf.io/slack) the -[#ebpf-go](https://cilium.slack.com/messages/ebpf-go) channel to discuss your requirements and how the feature can be implemented. The most important part is figuring out how much new exported API is necessary. **The less new API is required the easier it will be to land the feature.** -2. (*optional*) Create a draft PR if you want to discuss the implementation or have hit a problem. It's fine if this doesn't compile or contains debug statements. -3. Create a PR that is ready to merge. This must pass CI and have tests. - -### API stability - -The library doesn't guarantee the stability of its API at the moment. - -1. If possible avoid breakage by introducing new API and deprecating the old one - at the same time. If an API was deprecated in v0.x it can be removed in v0.x+1. -2. Breaking API in a way that causes compilation failures is acceptable but must - have good reasons. -3. Changing the semantics of the API without causing compilation failures is - heavily discouraged. - -## Running the tests - -Many of the tests require privileges to set resource limits and load eBPF code. -The easiest way to obtain these is to run the tests with `sudo`. - -To test the current package with your local kernel you can simply run: -``` -go test -exec sudo ./... -``` - -To test the current package with a different kernel version you can use the [run-tests.sh](run-tests.sh) script. -It requires [virtme](https://github.com/amluto/virtme) and qemu to be installed. - -Examples: - -```bash -# Run all tests on a 5.4 kernel -./run-tests.sh 5.4 - -# Run a subset of tests: -./run-tests.sh 5.4 ./link -``` +Want to contribute to ebpf-go? There are a few things you need to know. +We wrote a [contribution guide](https://ebpf-go.dev/contributing/) to help you get started. diff --git a/vendor/github.com/cilium/ebpf/Makefile b/vendor/github.com/cilium/ebpf/Makefile index abcd6c1a47c7..d355eea71cae 100644 --- a/vendor/github.com/cilium/ebpf/Makefile +++ b/vendor/github.com/cilium/ebpf/Makefile @@ -1,9 +1,9 @@ # The development version of clang is distributed as the 'clang' binary, # while stable/released versions have a version number attached. # Pin the default clang to a stable version. -CLANG ?= clang-14 -STRIP ?= llvm-strip-14 -OBJCOPY ?= llvm-objcopy-14 +CLANG ?= clang-17 +STRIP ?= llvm-strip-17 +OBJCOPY ?= llvm-objcopy-17 CFLAGS := -O2 -g -Wall -Werror $(CFLAGS) CI_KERNEL_URL ?= https://github.com/cilium/ci-kernels/raw/master/ @@ -21,12 +21,9 @@ CONTAINER_RUN_ARGS ?= $(if $(filter ${CONTAINER_ENGINE}, podman), --log-driver=n IMAGE := $(shell cat ${REPODIR}/testdata/docker/IMAGE) VERSION := $(shell cat ${REPODIR}/testdata/docker/VERSION) - -# clang <8 doesn't tag relocs properly (STT_NOTYPE) -# clang 9 is the first version emitting BTF TARGETS := \ - testdata/loader-clang-7 \ - testdata/loader-clang-9 \ + testdata/loader-clang-11 \ + testdata/loader-clang-14 \ testdata/loader-$(CLANG) \ testdata/manyprogs \ testdata/btf_map_init \ @@ -36,6 +33,7 @@ TARGETS := \ testdata/invalid_btf_map_init \ testdata/strings \ testdata/freplace \ + testdata/fentry_fexit \ testdata/iproute2_map_compat \ testdata/map_spin_lock \ testdata/subprog_reloc \ @@ -45,9 +43,12 @@ TARGETS := \ testdata/kfunc \ testdata/invalid-kfunc \ testdata/kfunc-kmod \ + testdata/constants \ + testdata/errors \ btf/testdata/relocs \ btf/testdata/relocs_read \ btf/testdata/relocs_read_tgt \ + btf/testdata/relocs_enum \ cmd/bpf2go/testdata/minimal .PHONY: all clean container-all container-shell generate @@ -56,22 +57,26 @@ TARGETS := \ # Build all ELF binaries using a containerized LLVM toolchain. container-all: - +${CONTAINER_ENGINE} run --rm -ti ${CONTAINER_RUN_ARGS} \ + +${CONTAINER_ENGINE} run --rm -t ${CONTAINER_RUN_ARGS} \ -v "${REPODIR}":/ebpf -w /ebpf --env MAKEFLAGS \ - --env CFLAGS="-fdebug-prefix-map=/ebpf=." \ --env HOME="/tmp" \ + --env BPF2GO_CC="$(CLANG)" \ + --env BPF2GO_FLAGS="-fdebug-prefix-map=/ebpf=. $(CFLAGS)" \ "${IMAGE}:${VERSION}" \ make all # (debug) Drop the user into a shell inside the container as root. +# Set BPF2GO_ envs to make 'make generate' just work. container-shell: ${CONTAINER_ENGINE} run --rm -ti \ -v "${REPODIR}":/ebpf -w /ebpf \ + --env BPF2GO_CC="$(CLANG)" \ + --env BPF2GO_FLAGS="-fdebug-prefix-map=/ebpf=. $(CFLAGS)" \ "${IMAGE}:${VERSION}" clean: - -$(RM) testdata/*.elf - -$(RM) btf/testdata/*.elf + find "$(CURDIR)" -name "*.elf" -delete + find "$(CURDIR)" -name "*.o" -delete format: find . -type f -name "*.c" | xargs clang-format -i @@ -80,11 +85,9 @@ all: format $(addsuffix -el.elf,$(TARGETS)) $(addsuffix -eb.elf,$(TARGETS)) gene ln -srf testdata/loader-$(CLANG)-el.elf testdata/loader-el.elf ln -srf testdata/loader-$(CLANG)-eb.elf testdata/loader-eb.elf -# $BPF_CLANG is used in go:generate invocations. -generate: export BPF_CLANG := $(CLANG) -generate: export BPF_CFLAGS := $(CFLAGS) generate: - go generate ./... + go generate -run "internal/cmd/gentypes" ./... + go generate -skip "internal/cmd/gentypes" ./... testdata/loader-%-el.elf: testdata/loader.c $* $(CFLAGS) -target bpfel -c $< -o $@ @@ -102,14 +105,8 @@ testdata/loader-%-eb.elf: testdata/loader.c $(CLANG) $(CFLAGS) -target bpfeb -c $< -o $@ $(STRIP) -g $@ -.PHONY: generate-btf -generate-btf: KERNEL_VERSION?=5.19 -generate-btf: - $(eval TMP := $(shell mktemp -d)) - curl -fL "$(CI_KERNEL_URL)/linux-$(KERNEL_VERSION).bz" -o "$(TMP)/bzImage" - /lib/modules/$(uname -r)/build/scripts/extract-vmlinux "$(TMP)/bzImage" > "$(TMP)/vmlinux" - $(OBJCOPY) --dump-section .BTF=/dev/stdout "$(TMP)/vmlinux" /dev/null | gzip > "btf/testdata/vmlinux.btf.gz" - curl -fL "$(CI_KERNEL_URL)/linux-$(KERNEL_VERSION)-selftests-bpf.tgz" -o "$(TMP)/selftests.tgz" - tar -xf "$(TMP)/selftests.tgz" --to-stdout tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.ko | \ - $(OBJCOPY) --dump-section .BTF="btf/testdata/btf_testmod.btf" - /dev/null - $(RM) -r "$(TMP)" +.PHONY: update-kernel-deps +update-kernel-deps: export KERNEL_VERSION?=6.8 +update-kernel-deps: + ./testdata/sh/update-kernel-deps.sh + $(MAKE) container-all diff --git a/vendor/github.com/cilium/ebpf/README.md b/vendor/github.com/cilium/ebpf/README.md index eff08d8df699..85871db1ae39 100644 --- a/vendor/github.com/cilium/ebpf/README.md +++ b/vendor/github.com/cilium/ebpf/README.md @@ -2,7 +2,7 @@ [![PkgGoDev](https://pkg.go.dev/badge/github.com/cilium/ebpf)](https://pkg.go.dev/github.com/cilium/ebpf) -![HoneyGopher](.github/images/cilium-ebpf.png) +![HoneyGopher](docs/ebpf/ebpf-go.png) ebpf-go is a pure Go library that provides utilities for loading, compiling, and debugging eBPF programs. It has minimal external dependencies and is intended to @@ -13,10 +13,9 @@ ecosystem. ## Getting Started -A small collection of Go and eBPF programs that serve as examples for building -your own tools can be found under [examples/](examples/). +Please take a look at our [Getting Started] guide. -[Contributions](CONTRIBUTING.md) are highly encouraged, as they highlight certain use cases of +[Contributions](https://ebpf-go.dev/contributing) are highly encouraged, as they highlight certain use cases of eBPF and the library, and help shape the future of the project. ## Getting Help @@ -59,19 +58,8 @@ This library includes the following packages: * A version of Go that is [supported by upstream](https://golang.org/doc/devel/release.html#policy) -* Linux >= 4.9. CI is run against kernel.org LTS releases. 4.4 should work but is - not tested against. - -## Regenerating Testdata - -Run `make` in the root of this repository to rebuild testdata in all -subpackages. This requires Docker, as it relies on a standardized build -environment to keep the build output stable. - -It is possible to regenerate data using Podman by overriding the `CONTAINER_*` -variables: `CONTAINER_ENGINE=podman CONTAINER_RUN_ARGS= make`. - -The toolchain image build files are kept in [testdata/docker/](testdata/docker/). +* CI is run against kernel.org LTS releases. >= 4.4 should work but EOL'ed versions + are not supported. ## License @@ -80,3 +68,5 @@ MIT ### eBPF Gopher The eBPF honeygopher is based on the Go gopher designed by Renee French. + +[Getting Started]: https://ebpf-go.dev/guides/getting-started/ diff --git a/vendor/github.com/cilium/ebpf/asm/alu.go b/vendor/github.com/cilium/ebpf/asm/alu.go index 3f60245f2b6d..282233d327fe 100644 --- a/vendor/github.com/cilium/ebpf/asm/alu.go +++ b/vendor/github.com/cilium/ebpf/asm/alu.go @@ -1,26 +1,26 @@ package asm -//go:generate stringer -output alu_string.go -type=Source,Endianness,ALUOp +//go:generate go run golang.org/x/tools/cmd/stringer@latest -output alu_string.go -type=Source,Endianness,ALUOp // Source of ALU / ALU64 / Branch operations // -// msb lsb -// +----+-+---+ -// |op |S|cls| -// +----+-+---+ -type Source uint8 +// msb lsb +// +------------+-+---+ +// | op |S|cls| +// +------------+-+---+ +type Source uint16 -const sourceMask OpCode = 0x08 +const sourceMask OpCode = 0x0008 // Source bitmask const ( // InvalidSource is returned by getters when invoked // on non ALU / branch OpCodes. - InvalidSource Source = 0xff + InvalidSource Source = 0xffff // ImmSource src is from constant - ImmSource Source = 0x00 + ImmSource Source = 0x0000 // RegSource src is from register - RegSource Source = 0x08 + RegSource Source = 0x0008 ) // The Endianness of a byte swap instruction. @@ -39,46 +39,56 @@ const ( // ALUOp are ALU / ALU64 operations // -// msb lsb -// +----+-+---+ -// |OP |s|cls| -// +----+-+---+ -type ALUOp uint8 +// msb lsb +// +-------+----+-+---+ +// | EXT | OP |s|cls| +// +-------+----+-+---+ +type ALUOp uint16 -const aluMask OpCode = 0xf0 +const aluMask OpCode = 0x3ff0 const ( // InvalidALUOp is returned by getters when invoked // on non ALU OpCodes - InvalidALUOp ALUOp = 0xff + InvalidALUOp ALUOp = 0xffff // Add - addition - Add ALUOp = 0x00 + Add ALUOp = 0x0000 // Sub - subtraction - Sub ALUOp = 0x10 + Sub ALUOp = 0x0010 // Mul - multiplication - Mul ALUOp = 0x20 + Mul ALUOp = 0x0020 // Div - division - Div ALUOp = 0x30 + Div ALUOp = 0x0030 + // SDiv - signed division + SDiv ALUOp = Div + 0x0100 // Or - bitwise or - Or ALUOp = 0x40 + Or ALUOp = 0x0040 // And - bitwise and - And ALUOp = 0x50 + And ALUOp = 0x0050 // LSh - bitwise shift left - LSh ALUOp = 0x60 + LSh ALUOp = 0x0060 // RSh - bitwise shift right - RSh ALUOp = 0x70 + RSh ALUOp = 0x0070 // Neg - sign/unsign signing bit - Neg ALUOp = 0x80 + Neg ALUOp = 0x0080 // Mod - modulo - Mod ALUOp = 0x90 + Mod ALUOp = 0x0090 + // SMod - signed modulo + SMod ALUOp = Mod + 0x0100 // Xor - bitwise xor - Xor ALUOp = 0xa0 + Xor ALUOp = 0x00a0 // Mov - move value from one place to another - Mov ALUOp = 0xb0 - // ArSh - arithmatic shift - ArSh ALUOp = 0xc0 + Mov ALUOp = 0x00b0 + // MovSX8 - move lower 8 bits, sign extended upper bits of target + MovSX8 ALUOp = Mov + 0x0100 + // MovSX16 - move lower 16 bits, sign extended upper bits of target + MovSX16 ALUOp = Mov + 0x0200 + // MovSX32 - move lower 32 bits, sign extended upper bits of target + MovSX32 ALUOp = Mov + 0x0300 + // ArSh - arithmetic shift + ArSh ALUOp = 0x00c0 // Swap - endian conversions - Swap ALUOp = 0xd0 + Swap ALUOp = 0x00d0 ) // HostTo converts from host to another endianness. @@ -102,6 +112,27 @@ func HostTo(endian Endianness, dst Register, size Size) Instruction { } } +// BSwap unconditionally reverses the order of bytes in a register. +func BSwap(dst Register, size Size) Instruction { + var imm int64 + switch size { + case Half: + imm = 16 + case Word: + imm = 32 + case DWord: + imm = 64 + default: + return Instruction{OpCode: InvalidOpCode} + } + + return Instruction{ + OpCode: OpCode(ALU64Class).SetALUOp(Swap), + Dst: dst, + Constant: imm, + } +} + // Op returns the OpCode for an ALU operation with a given source. func (op ALUOp) Op(source Source) OpCode { return OpCode(ALU64Class).SetALUOp(op).SetSource(source) diff --git a/vendor/github.com/cilium/ebpf/asm/alu_string.go b/vendor/github.com/cilium/ebpf/asm/alu_string.go index 72d3fe6292ea..35b406bf3f72 100644 --- a/vendor/github.com/cilium/ebpf/asm/alu_string.go +++ b/vendor/github.com/cilium/ebpf/asm/alu_string.go @@ -8,7 +8,7 @@ func _() { // An "invalid array index" compiler error signifies that the constant values have changed. // Re-run the stringer command to generate them again. var x [1]struct{} - _ = x[InvalidSource-255] + _ = x[InvalidSource-65535] _ = x[ImmSource-0] _ = x[RegSource-8] } @@ -25,7 +25,7 @@ func (i Source) String() string { return _Source_name_0 case i == 8: return _Source_name_1 - case i == 255: + case i == 65535: return _Source_name_2 default: return "Source(" + strconv.FormatInt(int64(i), 10) + ")" @@ -62,41 +62,51 @@ func _() { // An "invalid array index" compiler error signifies that the constant values have changed. // Re-run the stringer command to generate them again. var x [1]struct{} - _ = x[InvalidALUOp-255] + _ = x[InvalidALUOp-65535] _ = x[Add-0] _ = x[Sub-16] _ = x[Mul-32] _ = x[Div-48] + _ = x[SDiv-304] _ = x[Or-64] _ = x[And-80] _ = x[LSh-96] _ = x[RSh-112] _ = x[Neg-128] _ = x[Mod-144] + _ = x[SMod-400] _ = x[Xor-160] _ = x[Mov-176] + _ = x[MovSX8-432] + _ = x[MovSX16-688] + _ = x[MovSX32-944] _ = x[ArSh-192] _ = x[Swap-208] } -const _ALUOp_name = "AddSubMulDivOrAndLShRShNegModXorMovArShSwapInvalidALUOp" +const _ALUOp_name = "AddSubMulDivOrAndLShRShNegModXorMovArShSwapSDivSModMovSX8MovSX16MovSX32InvalidALUOp" var _ALUOp_map = map[ALUOp]string{ - 0: _ALUOp_name[0:3], - 16: _ALUOp_name[3:6], - 32: _ALUOp_name[6:9], - 48: _ALUOp_name[9:12], - 64: _ALUOp_name[12:14], - 80: _ALUOp_name[14:17], - 96: _ALUOp_name[17:20], - 112: _ALUOp_name[20:23], - 128: _ALUOp_name[23:26], - 144: _ALUOp_name[26:29], - 160: _ALUOp_name[29:32], - 176: _ALUOp_name[32:35], - 192: _ALUOp_name[35:39], - 208: _ALUOp_name[39:43], - 255: _ALUOp_name[43:55], + 0: _ALUOp_name[0:3], + 16: _ALUOp_name[3:6], + 32: _ALUOp_name[6:9], + 48: _ALUOp_name[9:12], + 64: _ALUOp_name[12:14], + 80: _ALUOp_name[14:17], + 96: _ALUOp_name[17:20], + 112: _ALUOp_name[20:23], + 128: _ALUOp_name[23:26], + 144: _ALUOp_name[26:29], + 160: _ALUOp_name[29:32], + 176: _ALUOp_name[32:35], + 192: _ALUOp_name[35:39], + 208: _ALUOp_name[39:43], + 304: _ALUOp_name[43:47], + 400: _ALUOp_name[47:51], + 432: _ALUOp_name[51:57], + 688: _ALUOp_name[57:64], + 944: _ALUOp_name[64:71], + 65535: _ALUOp_name[71:83], } func (i ALUOp) String() string { diff --git a/vendor/github.com/cilium/ebpf/asm/func.go b/vendor/github.com/cilium/ebpf/asm/func.go index 18f6a75db58a..84a40b2277f0 100644 --- a/vendor/github.com/cilium/ebpf/asm/func.go +++ b/vendor/github.com/cilium/ebpf/asm/func.go @@ -1,6 +1,6 @@ package asm -//go:generate stringer -output func_string.go -type=BuiltinFunc +//go:generate go run golang.org/x/tools/cmd/stringer@latest -output func_string.go -type=BuiltinFunc // BuiltinFunc is a built-in eBPF function. type BuiltinFunc int32 diff --git a/vendor/github.com/cilium/ebpf/asm/instruction.go b/vendor/github.com/cilium/ebpf/asm/instruction.go index ef01eaa35ae9..67cd39d6f672 100644 --- a/vendor/github.com/cilium/ebpf/asm/instruction.go +++ b/vendor/github.com/cilium/ebpf/asm/instruction.go @@ -60,6 +60,34 @@ func (ins *Instruction) Unmarshal(r io.Reader, bo binary.ByteOrder) (uint64, err } ins.Offset = int16(bo.Uint16(data[2:4])) + + if ins.OpCode.Class().IsALU() { + switch ins.OpCode.ALUOp() { + case Div: + if ins.Offset == 1 { + ins.OpCode = ins.OpCode.SetALUOp(SDiv) + ins.Offset = 0 + } + case Mod: + if ins.Offset == 1 { + ins.OpCode = ins.OpCode.SetALUOp(SMod) + ins.Offset = 0 + } + case Mov: + switch ins.Offset { + case 8: + ins.OpCode = ins.OpCode.SetALUOp(MovSX8) + ins.Offset = 0 + case 16: + ins.OpCode = ins.OpCode.SetALUOp(MovSX16) + ins.Offset = 0 + case 32: + ins.OpCode = ins.OpCode.SetALUOp(MovSX32) + ins.Offset = 0 + } + } + } + // Convert to int32 before widening to int64 // to ensure the signed bit is carried over. ins.Constant = int64(int32(bo.Uint32(data[4:8]))) @@ -106,8 +134,38 @@ func (ins Instruction) Marshal(w io.Writer, bo binary.ByteOrder) (uint64, error) return 0, fmt.Errorf("can't marshal registers: %s", err) } + if ins.OpCode.Class().IsALU() { + newOffset := int16(0) + switch ins.OpCode.ALUOp() { + case SDiv: + ins.OpCode = ins.OpCode.SetALUOp(Div) + newOffset = 1 + case SMod: + ins.OpCode = ins.OpCode.SetALUOp(Mod) + newOffset = 1 + case MovSX8: + ins.OpCode = ins.OpCode.SetALUOp(Mov) + newOffset = 8 + case MovSX16: + ins.OpCode = ins.OpCode.SetALUOp(Mov) + newOffset = 16 + case MovSX32: + ins.OpCode = ins.OpCode.SetALUOp(Mov) + newOffset = 32 + } + if newOffset != 0 && ins.Offset != 0 { + return 0, fmt.Errorf("extended ALU opcodes should have an .Offset of 0: %s", ins) + } + ins.Offset = newOffset + } + + op, err := ins.OpCode.bpfOpCode() + if err != nil { + return 0, err + } + data := make([]byte, InstructionSize) - data[0] = byte(ins.OpCode) + data[0] = op data[1] = byte(regs) bo.PutUint16(data[2:4], uint16(ins.Offset)) bo.PutUint32(data[4:8], uint32(cons)) @@ -298,9 +356,9 @@ func (ins Instruction) Format(f fmt.State, c rune) { goto ref } - fmt.Fprintf(f, "%v ", op) switch cls := op.Class(); { case cls.isLoadOrStore(): + fmt.Fprintf(f, "%v ", op) switch op.Mode() { case ImmMode: fmt.Fprintf(f, "dst: %s imm: %d", ins.Dst, ins.Constant) @@ -308,21 +366,30 @@ func (ins Instruction) Format(f fmt.State, c rune) { fmt.Fprintf(f, "imm: %d", ins.Constant) case IndMode: fmt.Fprintf(f, "dst: %s src: %s imm: %d", ins.Dst, ins.Src, ins.Constant) - case MemMode: + case MemMode, MemSXMode: fmt.Fprintf(f, "dst: %s src: %s off: %d imm: %d", ins.Dst, ins.Src, ins.Offset, ins.Constant) case XAddMode: fmt.Fprintf(f, "dst: %s src: %s", ins.Dst, ins.Src) } case cls.IsALU(): - fmt.Fprintf(f, "dst: %s ", ins.Dst) - if op.ALUOp() == Swap || op.Source() == ImmSource { + fmt.Fprintf(f, "%v", op) + if op == Swap.Op(ImmSource) { + fmt.Fprintf(f, "%d", ins.Constant) + } + + fmt.Fprintf(f, " dst: %s ", ins.Dst) + switch { + case op.ALUOp() == Swap: + break + case op.Source() == ImmSource: fmt.Fprintf(f, "imm: %d", ins.Constant) - } else { + default: fmt.Fprintf(f, "src: %s", ins.Src) } case cls.IsJump(): + fmt.Fprintf(f, "%v ", op) switch jop := op.JumpOp(); jop { case Call: switch ins.Src { @@ -336,6 +403,13 @@ func (ins Instruction) Format(f fmt.State, c rune) { fmt.Fprint(f, BuiltinFunc(ins.Constant)) } + case Ja: + if ins.OpCode.Class() == Jump32Class { + fmt.Fprintf(f, "imm: %d", ins.Constant) + } else { + fmt.Fprintf(f, "off: %d", ins.Offset) + } + default: fmt.Fprintf(f, "dst: %s off: %d ", ins.Dst, ins.Offset) if op.Source() == ImmSource { @@ -344,6 +418,8 @@ func (ins Instruction) Format(f fmt.State, c rune) { fmt.Fprintf(f, "src: %s", ins.Src) } } + default: + fmt.Fprintf(f, "%v ", op) } ref: @@ -772,7 +848,8 @@ func (insns Instructions) encodeFunctionReferences() error { } switch { - case ins.IsFunctionReference() && ins.Constant == -1: + case ins.IsFunctionReference() && ins.Constant == -1, + ins.OpCode == Ja.opCode(Jump32Class, ImmSource) && ins.Constant == -1: symOffset, ok := symbolOffsets[ins.Reference()] if !ok { return fmt.Errorf("%s at insn %d: symbol %q: %w", ins.OpCode, i, ins.Reference(), ErrUnsatisfiedProgramReference) diff --git a/vendor/github.com/cilium/ebpf/asm/jump.go b/vendor/github.com/cilium/ebpf/asm/jump.go index 2c8a3dbb7a36..2738d736b2d5 100644 --- a/vendor/github.com/cilium/ebpf/asm/jump.go +++ b/vendor/github.com/cilium/ebpf/asm/jump.go @@ -1,6 +1,6 @@ package asm -//go:generate stringer -output jump_string.go -type=JumpOp +//go:generate go run golang.org/x/tools/cmd/stringer@latest -output jump_string.go -type=JumpOp // JumpOp affect control flow. // @@ -10,7 +10,7 @@ package asm // +----+-+---+ type JumpOp uint8 -const jumpMask OpCode = aluMask +const jumpMask OpCode = 0xf0 const ( // InvalidJumpOp is returned by getters when invoked @@ -103,13 +103,21 @@ func (op JumpOp) Reg32(dst, src Register, label string) Instruction { } func (op JumpOp) opCode(class Class, source Source) OpCode { - if op == Exit || op == Call || op == Ja { + if op == Exit || op == Call { return InvalidOpCode } return OpCode(class).SetJumpOp(op).SetSource(source) } +// LongJump returns a jump always instruction with a range of [-2^31, 2^31 - 1]. +func LongJump(label string) Instruction { + return Instruction{ + OpCode: Ja.opCode(Jump32Class, ImmSource), + Constant: -1, + }.WithReference(label) +} + // Label adjusts PC to the address of the label. func (op JumpOp) Label(label string) Instruction { if op == Call { diff --git a/vendor/github.com/cilium/ebpf/asm/load_store.go b/vendor/github.com/cilium/ebpf/asm/load_store.go index f109497aebcb..cdb5c5cfa43b 100644 --- a/vendor/github.com/cilium/ebpf/asm/load_store.go +++ b/vendor/github.com/cilium/ebpf/asm/load_store.go @@ -1,6 +1,6 @@ package asm -//go:generate stringer -output load_store_string.go -type=Mode,Size +//go:generate go run golang.org/x/tools/cmd/stringer@latest -output load_store_string.go -type=Mode,Size // Mode for load and store operations // @@ -24,6 +24,8 @@ const ( IndMode Mode = 0x40 // MemMode - load from memory MemMode Mode = 0x60 + // MemSXMode - load from memory, sign extension + MemSXMode Mode = 0x80 // XAddMode - add atomically across processors. XAddMode Mode = 0xc0 ) @@ -73,6 +75,11 @@ func LoadMemOp(size Size) OpCode { return OpCode(LdXClass).SetMode(MemMode).SetSize(size) } +// LoadMemSXOp returns the OpCode to load a value of given size from memory sign extended. +func LoadMemSXOp(size Size) OpCode { + return OpCode(LdXClass).SetMode(MemSXMode).SetSize(size) +} + // LoadMem emits `dst = *(size *)(src + offset)`. func LoadMem(dst, src Register, offset int16, size Size) Instruction { return Instruction{ @@ -83,6 +90,20 @@ func LoadMem(dst, src Register, offset int16, size Size) Instruction { } } +// LoadMemSX emits `dst = *(size *)(src + offset)` but sign extends dst. +func LoadMemSX(dst, src Register, offset int16, size Size) Instruction { + if size == DWord { + return Instruction{OpCode: InvalidOpCode} + } + + return Instruction{ + OpCode: LoadMemSXOp(size), + Dst: dst, + Src: src, + Offset: offset, + } +} + // LoadImmOp returns the OpCode to load an immediate of given size. // // As of kernel 4.20, only DWord size is accepted. diff --git a/vendor/github.com/cilium/ebpf/asm/load_store_string.go b/vendor/github.com/cilium/ebpf/asm/load_store_string.go index 76d29a0756c6..c48080327c0b 100644 --- a/vendor/github.com/cilium/ebpf/asm/load_store_string.go +++ b/vendor/github.com/cilium/ebpf/asm/load_store_string.go @@ -13,6 +13,7 @@ func _() { _ = x[AbsMode-32] _ = x[IndMode-64] _ = x[MemMode-96] + _ = x[MemSXMode-128] _ = x[XAddMode-192] } @@ -21,8 +22,9 @@ const ( _Mode_name_1 = "AbsMode" _Mode_name_2 = "IndMode" _Mode_name_3 = "MemMode" - _Mode_name_4 = "XAddMode" - _Mode_name_5 = "InvalidMode" + _Mode_name_4 = "MemSXMode" + _Mode_name_5 = "XAddMode" + _Mode_name_6 = "InvalidMode" ) func (i Mode) String() string { @@ -35,10 +37,12 @@ func (i Mode) String() string { return _Mode_name_2 case i == 96: return _Mode_name_3 - case i == 192: + case i == 128: return _Mode_name_4 - case i == 255: + case i == 192: return _Mode_name_5 + case i == 255: + return _Mode_name_6 default: return "Mode(" + strconv.FormatInt(int64(i), 10) + ")" } diff --git a/vendor/github.com/cilium/ebpf/asm/opcode.go b/vendor/github.com/cilium/ebpf/asm/opcode.go index 9e3c30b0b3a9..1dfd0b171a4e 100644 --- a/vendor/github.com/cilium/ebpf/asm/opcode.go +++ b/vendor/github.com/cilium/ebpf/asm/opcode.go @@ -5,7 +5,7 @@ import ( "strings" ) -//go:generate stringer -output opcode_string.go -type=Class +//go:generate go run golang.org/x/tools/cmd/stringer@latest -output opcode_string.go -type=Class // Class of operations // @@ -66,18 +66,43 @@ func (cls Class) isJumpOrALU() bool { return cls.IsJump() || cls.IsALU() } -// OpCode is a packed eBPF opcode. +// OpCode represents a single operation. +// It is not a 1:1 mapping to real eBPF opcodes. // -// Its encoding is defined by a Class value: +// The encoding varies based on a 3-bit Class: // -// msb lsb -// +----+-+---+ -// | ???? |CLS| -// +----+-+---+ -type OpCode uint8 +// 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 +// ??? | CLS +// +// For ALUClass and ALUCLass32: +// +// 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 +// OPC |S| CLS +// +// For LdClass, LdXclass, StClass and StXClass: +// +// 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 +// 0 | MDE |SIZ| CLS +// +// For JumpClass, Jump32Class: +// +// 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 +// 0 | OPC |S| CLS +type OpCode uint16 // InvalidOpCode is returned by setters on OpCode -const InvalidOpCode OpCode = 0xff +const InvalidOpCode OpCode = 0xffff + +// bpfOpCode returns the actual BPF opcode. +func (op OpCode) bpfOpCode() (byte, error) { + const opCodeMask = 0xff + + if !valid(op, opCodeMask) { + return 0, fmt.Errorf("invalid opcode %x", op) + } + + return byte(op & opCodeMask), nil +} // rawInstructions returns the number of BPF instructions required // to encode this opcode. @@ -147,7 +172,7 @@ func (op OpCode) JumpOp() JumpOp { jumpOp := JumpOp(op & jumpMask) // Some JumpOps are only supported by JumpClass, not Jump32Class. - if op.Class() == Jump32Class && (jumpOp == Exit || jumpOp == Call || jumpOp == Ja) { + if op.Class() == Jump32Class && (jumpOp == Exit || jumpOp == Call) { return InvalidJumpOp } @@ -234,17 +259,24 @@ func (op OpCode) String() string { } case class.IsALU(): + if op.ALUOp() == Swap && op.Class() == ALU64Class { + // B to make BSwap, uncontitional byte swap + f.WriteString("B") + } + f.WriteString(op.ALUOp().String()) if op.ALUOp() == Swap { - // Width for Endian is controlled by Constant - f.WriteString(op.Endianness().String()) + if op.Class() == ALUClass { + // Width for Endian is controlled by Constant + f.WriteString(op.Endianness().String()) + } } else { + f.WriteString(strings.TrimSuffix(op.Source().String(), "Source")) + if class == ALUClass { f.WriteString("32") } - - f.WriteString(strings.TrimSuffix(op.Source().String(), "Source")) } case class.IsJump(): @@ -254,7 +286,7 @@ func (op OpCode) String() string { f.WriteString("32") } - if jop := op.JumpOp(); jop != Exit && jop != Call { + if jop := op.JumpOp(); jop != Exit && jop != Call && jop != Ja { f.WriteString(strings.TrimSuffix(op.Source().String(), "Source")) } diff --git a/vendor/github.com/cilium/ebpf/attachtype_string.go b/vendor/github.com/cilium/ebpf/attachtype_string.go index add2a3b5cc9b..bece896bb611 100644 --- a/vendor/github.com/cilium/ebpf/attachtype_string.go +++ b/vendor/github.com/cilium/ebpf/attachtype_string.go @@ -52,11 +52,24 @@ func _() { _ = x[AttachSkReuseportSelectOrMigrate-40] _ = x[AttachPerfEvent-41] _ = x[AttachTraceKprobeMulti-42] + _ = x[AttachLSMCgroup-43] + _ = x[AttachStructOps-44] + _ = x[AttachNetfilter-45] + _ = x[AttachTCXIngress-46] + _ = x[AttachTCXEgress-47] + _ = x[AttachTraceUprobeMulti-48] + _ = x[AttachCgroupUnixConnect-49] + _ = x[AttachCgroupUnixSendmsg-50] + _ = x[AttachCgroupUnixRecvmsg-51] + _ = x[AttachCgroupUnixGetpeername-52] + _ = x[AttachCgroupUnixGetsockname-53] + _ = x[AttachNetkitPrimary-54] + _ = x[AttachNetkitPeer-55] } -const _AttachType_name = "NoneCGroupInetEgressCGroupInetSockCreateCGroupSockOpsSkSKBStreamParserSkSKBStreamVerdictCGroupDeviceSkMsgVerdictCGroupInet4BindCGroupInet6BindCGroupInet4ConnectCGroupInet6ConnectCGroupInet4PostBindCGroupInet6PostBindCGroupUDP4SendmsgCGroupUDP6SendmsgLircMode2FlowDissectorCGroupSysctlCGroupUDP4RecvmsgCGroupUDP6RecvmsgCGroupGetsockoptCGroupSetsockoptTraceRawTpTraceFEntryTraceFExitModifyReturnLSMMacTraceIterCgroupInet4GetPeernameCgroupInet6GetPeernameCgroupInet4GetSocknameCgroupInet6GetSocknameXDPDevMapCgroupInetSockReleaseXDPCPUMapSkLookupXDPSkSKBVerdictSkReuseportSelectSkReuseportSelectOrMigratePerfEventTraceKprobeMulti" +const _AttachType_name = "NoneCGroupInetEgressCGroupInetSockCreateCGroupSockOpsSkSKBStreamParserSkSKBStreamVerdictCGroupDeviceSkMsgVerdictCGroupInet4BindCGroupInet6BindCGroupInet4ConnectCGroupInet6ConnectCGroupInet4PostBindCGroupInet6PostBindCGroupUDP4SendmsgCGroupUDP6SendmsgLircMode2FlowDissectorCGroupSysctlCGroupUDP4RecvmsgCGroupUDP6RecvmsgCGroupGetsockoptCGroupSetsockoptTraceRawTpTraceFEntryTraceFExitModifyReturnLSMMacTraceIterCgroupInet4GetPeernameCgroupInet6GetPeernameCgroupInet4GetSocknameCgroupInet6GetSocknameXDPDevMapCgroupInetSockReleaseXDPCPUMapSkLookupXDPSkSKBVerdictSkReuseportSelectSkReuseportSelectOrMigratePerfEventTraceKprobeMultiLSMCgroupStructOpsNetfilterTCXIngressTCXEgressTraceUprobeMultiCgroupUnixConnectCgroupUnixSendmsgCgroupUnixRecvmsgCgroupUnixGetpeernameCgroupUnixGetsocknameNetkitPrimaryNetkitPeer" -var _AttachType_index = [...]uint16{0, 4, 20, 40, 53, 70, 88, 100, 112, 127, 142, 160, 178, 197, 216, 233, 250, 259, 272, 284, 301, 318, 334, 350, 360, 371, 381, 393, 399, 408, 430, 452, 474, 496, 505, 526, 535, 543, 546, 558, 575, 601, 610, 626} +var _AttachType_index = [...]uint16{0, 4, 20, 40, 53, 70, 88, 100, 112, 127, 142, 160, 178, 197, 216, 233, 250, 259, 272, 284, 301, 318, 334, 350, 360, 371, 381, 393, 399, 408, 430, 452, 474, 496, 505, 526, 535, 543, 546, 558, 575, 601, 610, 626, 635, 644, 653, 663, 672, 688, 705, 722, 739, 760, 781, 794, 804} func (i AttachType) String() string { if i >= AttachType(len(_AttachType_index)-1) { diff --git a/vendor/github.com/cilium/ebpf/btf/btf.go b/vendor/github.com/cilium/ebpf/btf/btf.go index 86eb7d6819d5..671f680b2af2 100644 --- a/vendor/github.com/cilium/ebpf/btf/btf.go +++ b/vendor/github.com/cilium/ebpf/btf/btf.go @@ -14,7 +14,6 @@ import ( "github.com/cilium/ebpf/internal" "github.com/cilium/ebpf/internal/sys" - "github.com/cilium/ebpf/internal/unix" ) const btfMagic = 0xeB9F @@ -30,9 +29,8 @@ var ( // ID represents the unique ID of a BTF object. type ID = sys.BTFID -// Spec allows querying a set of Types and loading the set into the -// kernel. -type Spec struct { +// immutableTypes is a set of types which musn't be changed. +type immutableTypes struct { // All types contained by the spec, not including types from the base in // case the spec was parsed from split BTF. types []Type @@ -45,51 +43,140 @@ type Spec struct { // Types indexed by essential name. // Includes all struct flavors and types with the same name. - namedTypes map[essentialName][]Type + namedTypes map[essentialName][]TypeID - // String table from ELF, may be nil. - strings *stringTable - - // Byte order of the ELF we decoded the spec from, may be nil. + // Byte order of the types. This affects things like struct member order + // when using bitfields. byteOrder binary.ByteOrder } -var btfHeaderLen = binary.Size(&btfHeader{}) +func (s *immutableTypes) typeByID(id TypeID) (Type, bool) { + if id < s.firstTypeID { + return nil, false + } -type btfHeader struct { - Magic uint16 - Version uint8 - Flags uint8 - HdrLen uint32 + index := int(id - s.firstTypeID) + if index >= len(s.types) { + return nil, false + } - TypeOff uint32 - TypeLen uint32 - StringOff uint32 - StringLen uint32 + return s.types[index], true } -// typeStart returns the offset from the beginning of the .BTF section -// to the start of its type entries. -func (h *btfHeader) typeStart() int64 { - return int64(h.HdrLen + h.TypeOff) +// mutableTypes is a set of types which may be changed. +type mutableTypes struct { + imm immutableTypes + mu sync.RWMutex // protects copies below + copies map[Type]Type // map[orig]copy + copiedTypeIDs map[Type]TypeID // map[copy]origID } -// stringStart returns the offset from the beginning of the .BTF section -// to the start of its string table. -func (h *btfHeader) stringStart() int64 { - return int64(h.HdrLen + h.StringOff) +// add a type to the set of mutable types. +// +// Copies type and all of its children once. Repeated calls with the same type +// do not copy again. +func (mt *mutableTypes) add(typ Type, typeIDs map[Type]TypeID) Type { + mt.mu.RLock() + cpy, ok := mt.copies[typ] + mt.mu.RUnlock() + + if ok { + // Fast path: the type has been copied before. + return cpy + } + + // modifyGraphPreorder copies the type graph node by node, so we can't drop + // the lock in between. + mt.mu.Lock() + defer mt.mu.Unlock() + + return copyType(typ, typeIDs, mt.copies, mt.copiedTypeIDs) } -// newSpec creates a Spec containing only Void. -func newSpec() *Spec { - return &Spec{ - []Type{(*Void)(nil)}, - map[Type]TypeID{(*Void)(nil): 0}, - 0, - make(map[essentialName][]Type), - nil, - nil, +// copy a set of mutable types. +func (mt *mutableTypes) copy() *mutableTypes { + if mt == nil { + return nil + } + + mtCopy := &mutableTypes{ + mt.imm, + sync.RWMutex{}, + make(map[Type]Type, len(mt.copies)), + make(map[Type]TypeID, len(mt.copiedTypeIDs)), + } + + // Prevent concurrent modification of mt.copiedTypeIDs. + mt.mu.RLock() + defer mt.mu.RUnlock() + + copiesOfCopies := make(map[Type]Type, len(mt.copies)) + for orig, copy := range mt.copies { + // NB: We make a copy of copy, not orig, so that changes to mutable types + // are preserved. + copyOfCopy := copyType(copy, mt.copiedTypeIDs, copiesOfCopies, mtCopy.copiedTypeIDs) + mtCopy.copies[orig] = copyOfCopy + } + + return mtCopy +} + +func (mt *mutableTypes) typeID(typ Type) (TypeID, error) { + if _, ok := typ.(*Void); ok { + // Equality is weird for void, since it is a zero sized type. + return 0, nil + } + + mt.mu.RLock() + defer mt.mu.RUnlock() + + id, ok := mt.copiedTypeIDs[typ] + if !ok { + return 0, fmt.Errorf("no ID for type %s: %w", typ, ErrNotFound) + } + + return id, nil +} + +func (mt *mutableTypes) typeByID(id TypeID) (Type, bool) { + immT, ok := mt.imm.typeByID(id) + if !ok { + return nil, false + } + + return mt.add(immT, mt.imm.typeIDs), true +} + +func (mt *mutableTypes) anyTypesByName(name string) ([]Type, error) { + immTypes := mt.imm.namedTypes[newEssentialName(name)] + if len(immTypes) == 0 { + return nil, fmt.Errorf("type name %s: %w", name, ErrNotFound) + } + + // Return a copy to prevent changes to namedTypes. + result := make([]Type, 0, len(immTypes)) + for _, id := range immTypes { + immT, ok := mt.imm.typeByID(id) + if !ok { + return nil, fmt.Errorf("no type with ID %d", id) + } + + // Match against the full name, not just the essential one + // in case the type being looked up is a struct flavor. + if immT.TypeName() == name { + result = append(result, mt.add(immT, mt.imm.typeIDs)) + } } + return result, nil +} + +// Spec allows querying a set of Types and loading the set into the +// kernel. +type Spec struct { + *mutableTypes + + // String table from ELF. + strings *stringTable } // LoadSpec opens file and calls LoadSpecFromReader on it. @@ -220,7 +307,7 @@ func loadSpecFromELF(file *internal.SafeELFFile) (*Spec, error) { return nil, err } - err = fixupDatasec(spec.types, sectionSizes, offsets) + err = fixupDatasec(spec.imm.types, sectionSizes, offsets) if err != nil { return nil, err } @@ -236,14 +323,10 @@ func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec) (*Spec, error ) if base != nil { - if base.firstTypeID != 0 { + if base.imm.firstTypeID != 0 { return nil, fmt.Errorf("can't use split BTF as base") } - if base.strings == nil { - return nil, fmt.Errorf("parse split BTF: base must be loaded from an ELF") - } - baseStrings = base.strings firstTypeID, err = base.nextTypeID() @@ -252,12 +335,7 @@ func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec) (*Spec, error } } - rawTypes, rawStrings, err := parseBTF(btf, bo, baseStrings) - if err != nil { - return nil, err - } - - types, err := inflateRawTypes(rawTypes, rawStrings, base) + types, rawStrings, err := parseBTF(btf, bo, baseStrings, base) if err != nil { return nil, err } @@ -265,16 +343,23 @@ func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec) (*Spec, error typeIDs, typesByName := indexTypes(types, firstTypeID) return &Spec{ - namedTypes: typesByName, - typeIDs: typeIDs, - types: types, - firstTypeID: firstTypeID, - strings: rawStrings, - byteOrder: bo, + &mutableTypes{ + immutableTypes{ + types, + typeIDs, + firstTypeID, + typesByName, + bo, + }, + sync.RWMutex{}, + make(map[Type]Type), + make(map[Type]TypeID), + }, + rawStrings, }, nil } -func indexTypes(types []Type, firstTypeID TypeID) (map[Type]TypeID, map[essentialName][]Type) { +func indexTypes(types []Type, firstTypeID TypeID) (map[Type]TypeID, map[essentialName][]TypeID) { namedTypes := 0 for _, typ := range types { if typ.TypeName() != "" { @@ -286,150 +371,20 @@ func indexTypes(types []Type, firstTypeID TypeID) (map[Type]TypeID, map[essentia } typeIDs := make(map[Type]TypeID, len(types)) - typesByName := make(map[essentialName][]Type, namedTypes) + typesByName := make(map[essentialName][]TypeID, namedTypes) for i, typ := range types { + id := firstTypeID + TypeID(i) + typeIDs[typ] = id + if name := newEssentialName(typ.TypeName()); name != "" { - typesByName[name] = append(typesByName[name], typ) + typesByName[name] = append(typesByName[name], id) } - typeIDs[typ] = firstTypeID + TypeID(i) } return typeIDs, typesByName } -// LoadKernelSpec returns the current kernel's BTF information. -// -// Defaults to /sys/kernel/btf/vmlinux and falls back to scanning the file system -// for vmlinux ELFs. Returns an error wrapping ErrNotSupported if BTF is not enabled. -func LoadKernelSpec() (*Spec, error) { - spec, _, err := kernelSpec() - if err != nil { - return nil, err - } - return spec.Copy(), nil -} - -var kernelBTF struct { - sync.RWMutex - spec *Spec - // True if the spec was read from an ELF instead of raw BTF in /sys. - fallback bool -} - -// FlushKernelSpec removes any cached kernel type information. -func FlushKernelSpec() { - kernelBTF.Lock() - defer kernelBTF.Unlock() - - kernelBTF.spec, kernelBTF.fallback = nil, false -} - -func kernelSpec() (*Spec, bool, error) { - kernelBTF.RLock() - spec, fallback := kernelBTF.spec, kernelBTF.fallback - kernelBTF.RUnlock() - - if spec == nil { - kernelBTF.Lock() - defer kernelBTF.Unlock() - - spec, fallback = kernelBTF.spec, kernelBTF.fallback - } - - if spec != nil { - return spec, fallback, nil - } - - spec, fallback, err := loadKernelSpec() - if err != nil { - return nil, false, err - } - - kernelBTF.spec, kernelBTF.fallback = spec, fallback - return spec, fallback, nil -} - -func loadKernelSpec() (_ *Spec, fallback bool, _ error) { - fh, err := os.Open("/sys/kernel/btf/vmlinux") - if err == nil { - defer fh.Close() - - spec, err := loadRawSpec(fh, internal.NativeEndian, nil) - return spec, false, err - } - - file, err := findVMLinux() - if err != nil { - return nil, false, err - } - defer file.Close() - - spec, err := loadSpecFromELF(file) - return spec, true, err -} - -// findVMLinux scans multiple well-known paths for vmlinux kernel images. -func findVMLinux() (*internal.SafeELFFile, error) { - release, err := internal.KernelRelease() - if err != nil { - return nil, err - } - - // use same list of locations as libbpf - // https://github.com/libbpf/libbpf/blob/9a3a42608dbe3731256a5682a125ac1e23bced8f/src/btf.c#L3114-L3122 - locations := []string{ - "/boot/vmlinux-%s", - "/lib/modules/%s/vmlinux-%[1]s", - "/lib/modules/%s/build/vmlinux", - "/usr/lib/modules/%s/kernel/vmlinux", - "/usr/lib/debug/boot/vmlinux-%s", - "/usr/lib/debug/boot/vmlinux-%s.debug", - "/usr/lib/debug/lib/modules/%s/vmlinux", - } - - for _, loc := range locations { - file, err := internal.OpenSafeELFFile(fmt.Sprintf(loc, release)) - if errors.Is(err, os.ErrNotExist) { - continue - } - return file, err - } - - return nil, fmt.Errorf("no BTF found for kernel version %s: %w", release, internal.ErrNotSupported) -} - -// parseBTFHeader parses the header of the .BTF section. -func parseBTFHeader(r io.Reader, bo binary.ByteOrder) (*btfHeader, error) { - var header btfHeader - if err := binary.Read(r, bo, &header); err != nil { - return nil, fmt.Errorf("can't read header: %v", err) - } - - if header.Magic != btfMagic { - return nil, fmt.Errorf("incorrect magic value %v", header.Magic) - } - - if header.Version != 1 { - return nil, fmt.Errorf("unexpected version %v", header.Version) - } - - if header.Flags != 0 { - return nil, fmt.Errorf("unsupported flags %v", header.Flags) - } - - remainder := int64(header.HdrLen) - int64(binary.Size(&header)) - if remainder < 0 { - return nil, errors.New("header length shorter than btfHeader size") - } - - if _, err := io.CopyN(internal.DiscardZeroes{}, r, remainder); err != nil { - return nil, fmt.Errorf("header padding: %v", err) - } - - return &header, nil -} - func guessRawBTFByteOrder(r io.ReaderAt) binary.ByteOrder { buf := new(bufio.Reader) for _, bo := range []binary.ByteOrder{ @@ -447,7 +402,7 @@ func guessRawBTFByteOrder(r io.ReaderAt) binary.ByteOrder { // parseBTF reads a .BTF section into memory and parses it into a list of // raw types and a string table. -func parseBTF(btf io.ReaderAt, bo binary.ByteOrder, baseStrings *stringTable) ([]rawType, *stringTable, error) { +func parseBTF(btf io.ReaderAt, bo binary.ByteOrder, baseStrings *stringTable, base *Spec) ([]Type, *stringTable, error) { buf := internal.NewBufferedSectionReader(btf, 0, math.MaxInt64) header, err := parseBTFHeader(buf, bo) if err != nil { @@ -461,12 +416,12 @@ func parseBTF(btf io.ReaderAt, bo binary.ByteOrder, baseStrings *stringTable) ([ } buf.Reset(io.NewSectionReader(btf, header.typeStart(), int64(header.TypeLen))) - rawTypes, err := readTypes(buf, bo, header.TypeLen) + types, err := readAndInflateTypes(buf, bo, header.TypeLen, rawStrings, base) if err != nil { - return nil, nil, fmt.Errorf("can't read types: %w", err) + return nil, nil, err } - return rawTypes, rawStrings, nil + return types, rawStrings, nil } type symbol struct { @@ -571,17 +526,13 @@ func fixupDatasecLayout(ds *Datasec) error { // Copy creates a copy of Spec. func (s *Spec) Copy() *Spec { - types := copyTypes(s.types, nil) - typeIDs, typesByName := indexTypes(types, s.firstTypeID) + if s == nil { + return nil + } - // NB: Other parts of spec are not copied since they are immutable. return &Spec{ - types, - typeIDs, - s.firstTypeID, - typesByName, + s.mutableTypes.copy(), s.strings, - s.byteOrder, } } @@ -598,8 +549,8 @@ func (sw sliceWriter) Write(p []byte) (int, error) { // nextTypeID returns the next unallocated type ID or an error if there are no // more type IDs. func (s *Spec) nextTypeID() (TypeID, error) { - id := s.firstTypeID + TypeID(len(s.types)) - if id < s.firstTypeID { + id := s.imm.firstTypeID + TypeID(len(s.imm.types)) + if id < s.imm.firstTypeID { return 0, fmt.Errorf("no more type IDs") } return id, nil @@ -610,33 +561,19 @@ func (s *Spec) nextTypeID() (TypeID, error) { // Returns an error wrapping ErrNotFound if a Type with the given ID // does not exist in the Spec. func (s *Spec) TypeByID(id TypeID) (Type, error) { - if id < s.firstTypeID { - return nil, fmt.Errorf("look up type with ID %d (first ID is %d): %w", id, s.firstTypeID, ErrNotFound) - } - - index := int(id - s.firstTypeID) - if index >= len(s.types) { - return nil, fmt.Errorf("look up type with ID %d: %w", id, ErrNotFound) + typ, ok := s.typeByID(id) + if !ok { + return nil, fmt.Errorf("look up type with ID %d (first ID is %d): %w", id, s.imm.firstTypeID, ErrNotFound) } - return s.types[index], nil + return typ, nil } // TypeID returns the ID for a given Type. // -// Returns an error wrapping ErrNoFound if the type isn't part of the Spec. +// Returns an error wrapping [ErrNotFound] if the type isn't part of the Spec. func (s *Spec) TypeID(typ Type) (TypeID, error) { - if _, ok := typ.(*Void); ok { - // Equality is weird for void, since it is a zero sized type. - return 0, nil - } - - id, ok := s.typeIDs[typ] - if !ok { - return 0, fmt.Errorf("no ID for type %s: %w", typ, ErrNotFound) - } - - return id, nil + return s.mutableTypes.typeID(typ) } // AnyTypesByName returns a list of BTF Types with the given name. @@ -647,21 +584,7 @@ func (s *Spec) TypeID(typ Type) (TypeID, error) { // // Returns an error wrapping ErrNotFound if no matching Type exists in the Spec. func (s *Spec) AnyTypesByName(name string) ([]Type, error) { - types := s.namedTypes[newEssentialName(name)] - if len(types) == 0 { - return nil, fmt.Errorf("type name %s: %w", name, ErrNotFound) - } - - // Return a copy to prevent changes to namedTypes. - result := make([]Type, 0, len(types)) - for _, t := range types { - // Match against the full name, not just the essential one - // in case the type being looked up is a struct flavor. - if t.TypeName() == name { - result = append(result, t) - } - } - return result, nil + return s.mutableTypes.anyTypesByName(name) } // AnyTypeByName returns a Type with the given name. @@ -750,120 +673,27 @@ func LoadSplitSpecFromReader(r io.ReaderAt, base *Spec) (*Spec, error) { // TypesIterator iterates over types of a given spec. type TypesIterator struct { - types []Type - index int + spec *Spec + id TypeID + done bool // The last visited type in the spec. Type Type } // Iterate returns the types iterator. func (s *Spec) Iterate() *TypesIterator { - // We share the backing array of types with the Spec. This is safe since - // we don't allow deletion or shuffling of types. - return &TypesIterator{types: s.types, index: 0} + return &TypesIterator{spec: s, id: s.imm.firstTypeID} } // Next returns true as long as there are any remaining types. func (iter *TypesIterator) Next() bool { - if len(iter.types) <= iter.index { + if iter.done { return false } - iter.Type = iter.types[iter.index] - iter.index++ - return true -} - -// haveBTF attempts to load a BTF blob containing an Int. It should pass on any -// kernel that supports BPF_BTF_LOAD. -var haveBTF = internal.NewFeatureTest("BTF", "4.18", func() error { - // 0-length anonymous integer - err := probeBTF(&Int{}) - if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { - return internal.ErrNotSupported - } - return err -}) - -// haveMapBTF attempts to load a minimal BTF blob containing a Var. It is -// used as a proxy for .bss, .data and .rodata map support, which generally -// come with a Var and Datasec. These were introduced in Linux 5.2. -var haveMapBTF = internal.NewFeatureTest("Map BTF (Var/Datasec)", "5.2", func() error { - if err := haveBTF(); err != nil { - return err - } - - v := &Var{ - Name: "a", - Type: &Pointer{(*Void)(nil)}, - } - - err := probeBTF(v) - if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { - // Treat both EINVAL and EPERM as not supported: creating the map may still - // succeed without Btf* attrs. - return internal.ErrNotSupported - } - return err -}) - -// haveProgBTF attempts to load a BTF blob containing a Func and FuncProto. It -// is used as a proxy for ext_info (func_info) support, which depends on -// Func(Proto) by definition. -var haveProgBTF = internal.NewFeatureTest("Program BTF (func/line_info)", "5.0", func() error { - if err := haveBTF(); err != nil { - return err - } - - fn := &Func{ - Name: "a", - Type: &FuncProto{Return: (*Void)(nil)}, - } - - err := probeBTF(fn) - if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { - return internal.ErrNotSupported - } - return err -}) - -var haveFuncLinkage = internal.NewFeatureTest("BTF func linkage", "5.6", func() error { - if err := haveProgBTF(); err != nil { - return err - } - - fn := &Func{ - Name: "a", - Type: &FuncProto{Return: (*Void)(nil)}, - Linkage: GlobalFunc, - } - - err := probeBTF(fn) - if errors.Is(err, unix.EINVAL) { - return internal.ErrNotSupported - } - return err -}) - -func probeBTF(typ Type) error { - b, err := NewBuilder([]Type{typ}) - if err != nil { - return err - } - - buf, err := b.Marshal(nil, nil) - if err != nil { - return err - } - - fd, err := sys.BtfLoad(&sys.BtfLoadAttr{ - Btf: sys.NewSlicePointer(buf), - BtfSize: uint32(len(buf)), - }) - - if err == nil { - fd.Close() - } - - return err + var ok bool + iter.Type, ok = iter.spec.typeByID(iter.id) + iter.id++ + iter.done = !ok + return !iter.done } diff --git a/vendor/github.com/cilium/ebpf/btf/btf_types.go b/vendor/github.com/cilium/ebpf/btf/btf_types.go index a253b7c9b9e7..f0e327abc0e6 100644 --- a/vendor/github.com/cilium/ebpf/btf/btf_types.go +++ b/vendor/github.com/cilium/ebpf/btf/btf_types.go @@ -2,12 +2,15 @@ package btf import ( "encoding/binary" + "errors" "fmt" "io" "unsafe" + + "github.com/cilium/ebpf/internal" ) -//go:generate stringer -linecomment -output=btf_types_string.go -type=FuncLinkage,VarLinkage,btfKind +//go:generate go run golang.org/x/tools/cmd/stringer@latest -linecomment -output=btf_types_string.go -type=FuncLinkage,VarLinkage,btfKind // btfKind describes a Type. type btfKind uint8 @@ -69,6 +72,63 @@ const ( btfTypeKindFlagMask = 1 ) +var btfHeaderLen = binary.Size(&btfHeader{}) + +type btfHeader struct { + Magic uint16 + Version uint8 + Flags uint8 + HdrLen uint32 + + TypeOff uint32 + TypeLen uint32 + StringOff uint32 + StringLen uint32 +} + +// typeStart returns the offset from the beginning of the .BTF section +// to the start of its type entries. +func (h *btfHeader) typeStart() int64 { + return int64(h.HdrLen + h.TypeOff) +} + +// stringStart returns the offset from the beginning of the .BTF section +// to the start of its string table. +func (h *btfHeader) stringStart() int64 { + return int64(h.HdrLen + h.StringOff) +} + +// parseBTFHeader parses the header of the .BTF section. +func parseBTFHeader(r io.Reader, bo binary.ByteOrder) (*btfHeader, error) { + var header btfHeader + if err := binary.Read(r, bo, &header); err != nil { + return nil, fmt.Errorf("can't read header: %v", err) + } + + if header.Magic != btfMagic { + return nil, fmt.Errorf("incorrect magic value %v", header.Magic) + } + + if header.Version != 1 { + return nil, fmt.Errorf("unexpected version %v", header.Version) + } + + if header.Flags != 0 { + return nil, fmt.Errorf("unsupported flags %v", header.Flags) + } + + remainder := int64(header.HdrLen) - int64(binary.Size(&header)) + if remainder < 0 { + return nil, errors.New("header length shorter than btfHeader size") + } + + if _, err := io.CopyN(internal.DiscardZeroes{}, r, remainder); err != nil { + return nil, fmt.Errorf("header padding: %v", err) + } + + return &header, nil +} + var btfTypeLen = binary.Size(btfType{}) // btfType is equivalent to struct btf_type in Documentation/bpf/btf.rst. @@ -93,6 +153,19 @@ type btfType struct { SizeType uint32 } +var btfTypeSize = int(unsafe.Sizeof(btfType{})) + +func unmarshalBtfType(bt *btfType, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfTypeSize { + return 0, fmt.Errorf("not enough bytes to unmarshal btfType") + } + + bt.NameOff = bo.Uint32(b[0:]) + bt.Info = bo.Uint32(b[4:]) + bt.SizeType = bo.Uint32(b[8:]) + return btfTypeSize, nil +} + func mask(len uint32) uint32 { return (1 << len) - 1 } @@ -240,6 +313,17 @@ const ( btfIntBitsShift = 0 ) +var btfIntLen = int(unsafe.Sizeof(btfInt{})) + +func unmarshalBtfInt(bi *btfInt, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfIntLen { + return 0, fmt.Errorf("not enough bytes to unmarshal btfInt") + } + + bi.Raw = bo.Uint32(b[0:]) + return btfIntLen, nil +} + func (bi btfInt) Encoding() IntEncoding { return IntEncoding(readBits(bi.Raw, btfIntEncodingLen, btfIntEncodingShift)) } @@ -270,102 +354,166 @@ type btfArray struct { Nelems uint32 } +var btfArrayLen = int(unsafe.Sizeof(btfArray{})) + +func unmarshalBtfArray(ba *btfArray, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfArrayLen { + return 0, fmt.Errorf("not enough bytes to unmarshal btfArray") + } + + ba.Type = TypeID(bo.Uint32(b[0:])) + ba.IndexType = TypeID(bo.Uint32(b[4:])) + ba.Nelems = bo.Uint32(b[8:]) + return btfArrayLen, nil +} + type btfMember struct { NameOff uint32 Type TypeID Offset uint32 } +var btfMemberLen = int(unsafe.Sizeof(btfMember{})) + +func unmarshalBtfMembers(members []btfMember, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range members { + if off+btfMemberLen > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfMember %d", i) + } + + members[i].NameOff = bo.Uint32(b[off+0:]) + members[i].Type = TypeID(bo.Uint32(b[off+4:])) + members[i].Offset = bo.Uint32(b[off+8:]) + + off += btfMemberLen + } + + return off, nil +} + type btfVarSecinfo struct { Type TypeID Offset uint32 Size uint32 } +var btfVarSecinfoLen = int(unsafe.Sizeof(btfVarSecinfo{})) + +func unmarshalBtfVarSecInfos(secinfos []btfVarSecinfo, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range secinfos { + if off+btfVarSecinfoLen > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfVarSecinfo %d", i) + } + + secinfos[i].Type = TypeID(bo.Uint32(b[off+0:])) + secinfos[i].Offset = bo.Uint32(b[off+4:]) + secinfos[i].Size = bo.Uint32(b[off+8:]) + + off += btfVarSecinfoLen + } + + return off, nil +} + type btfVariable struct { Linkage uint32 } +var btfVariableLen = int(unsafe.Sizeof(btfVariable{})) + +func unmarshalBtfVariable(bv *btfVariable, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfVariableLen { + return 0, fmt.Errorf("not enough bytes to unmarshal btfVariable") + } + + bv.Linkage = bo.Uint32(b[0:]) + return btfVariableLen, nil +} + type btfEnum struct { NameOff uint32 Val uint32 } +var btfEnumLen = int(unsafe.Sizeof(btfEnum{})) + +func unmarshalBtfEnums(enums []btfEnum, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range enums { + if off+btfEnumLen > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfEnum %d", i) + } + + enums[i].NameOff = bo.Uint32(b[off+0:]) + enums[i].Val = bo.Uint32(b[off+4:]) + + off += btfEnumLen + } + + return off, nil +} + type btfEnum64 struct { NameOff uint32 ValLo32 uint32 ValHi32 uint32 } +var btfEnum64Len = int(unsafe.Sizeof(btfEnum64{})) + +func unmarshalBtfEnums64(enums []btfEnum64, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range enums { + if off+btfEnum64Len > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfEnum64 %d", i) + } + + enums[i].NameOff = bo.Uint32(b[off+0:]) + enums[i].ValLo32 = bo.Uint32(b[off+4:]) + enums[i].ValHi32 = bo.Uint32(b[off+8:]) + + off += btfEnum64Len + } + + return off, nil +} + type btfParam struct { NameOff uint32 Type TypeID } -type btfDeclTag struct { - ComponentIdx uint32 -} +var btfParamLen = int(unsafe.Sizeof(btfParam{})) -func readTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32) ([]rawType, error) { - var header btfType - // because of the interleaving between types and struct members it is difficult to - // precompute the numbers of raw types this will parse - // this "guess" is a good first estimation - sizeOfbtfType := uintptr(btfTypeLen) - tyMaxCount := uintptr(typeLen) / sizeOfbtfType / 2 - types := make([]rawType, 0, tyMaxCount) - - for id := TypeID(1); ; id++ { - if err := binary.Read(r, bo, &header); err == io.EOF { - return types, nil - } else if err != nil { - return nil, fmt.Errorf("can't read type info for id %v: %v", id, err) +func unmarshalBtfParams(params []btfParam, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range params { + if off+btfParamLen > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfParam %d", i) } - var data interface{} - switch header.Kind() { - case kindInt: - data = new(btfInt) - case kindPointer: - case kindArray: - data = new(btfArray) - case kindStruct: - fallthrough - case kindUnion: - data = make([]btfMember, header.Vlen()) - case kindEnum: - data = make([]btfEnum, header.Vlen()) - case kindForward: - case kindTypedef: - case kindVolatile: - case kindConst: - case kindRestrict: - case kindFunc: - case kindFuncProto: - data = make([]btfParam, header.Vlen()) - case kindVar: - data = new(btfVariable) - case kindDatasec: - data = make([]btfVarSecinfo, header.Vlen()) - case kindFloat: - case kindDeclTag: - data = new(btfDeclTag) - case kindTypeTag: - case kindEnum64: - data = make([]btfEnum64, header.Vlen()) - default: - return nil, fmt.Errorf("type id %v: unknown kind: %v", id, header.Kind()) - } + params[i].NameOff = bo.Uint32(b[off+0:]) + params[i].Type = TypeID(bo.Uint32(b[off+4:])) - if data == nil { - types = append(types, rawType{header, nil}) - continue - } + off += btfParamLen + } - if err := binary.Read(r, bo, data); err != nil { - return nil, fmt.Errorf("type id %d: kind %v: can't read %T: %v", id, header.Kind(), data, err) - } + return off, nil +} - types = append(types, rawType{header, data}) +type btfDeclTag struct { + ComponentIdx uint32 +} + +var btfDeclTagLen = int(unsafe.Sizeof(btfDeclTag{})) + +func unmarshalBtfDeclTag(bdt *btfDeclTag, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfDeclTagLen { + return 0, fmt.Errorf("not enough bytes to unmarshal btfDeclTag") } + + bdt.ComponentIdx = bo.Uint32(b[0:]) + return btfDeclTagLen, nil } diff --git a/vendor/github.com/cilium/ebpf/btf/core.go b/vendor/github.com/cilium/ebpf/btf/core.go index a5c40d36af46..ee89f98331a4 100644 --- a/vendor/github.com/cilium/ebpf/btf/core.go +++ b/vendor/github.com/cilium/ebpf/btf/core.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "reflect" + "slices" "strconv" "strings" @@ -15,11 +16,16 @@ import ( // Code in this file is derived from libbpf, which is available under a BSD // 2-Clause license. +// A constant used when CO-RE relocation has to remove instructions. +// +// Taken from libbpf. +const COREBadRelocationSentinel = 0xbad2310 + // COREFixup is the result of computing a CO-RE relocation for a target. type COREFixup struct { kind coreKind - local uint32 - target uint32 + local uint64 + target uint64 // True if there is no valid fixup. The instruction is replaced with an // invalid dummy. poison bool @@ -41,9 +47,22 @@ func (f *COREFixup) String() string { func (f *COREFixup) Apply(ins *asm.Instruction) error { if f.poison { - const badRelo = 0xbad2310 + // Relocation is poisoned, replace the instruction with an invalid one. + if ins.OpCode.IsDWordLoad() { + // Replace a dword load with a invalid dword load to preserve instruction size. + *ins = asm.LoadImm(asm.R10, COREBadRelocationSentinel, asm.DWord) + } else { + // Replace all single size instruction with a invalid call instruction. + *ins = asm.BuiltinFunc(COREBadRelocationSentinel).Call() + } + + // Add context to the kernel verifier output. + if source := ins.Source(); source != nil { + *ins = ins.WithSource(asm.Comment(fmt.Sprintf("instruction poisoned by CO-RE: %s", source))) + } else { + *ins = ins.WithSource(asm.Comment("instruction poisoned by CO-RE")) + } - *ins = asm.BuiltinFunc(badRelo).Call() return nil } @@ -119,10 +138,11 @@ const ( reloTypeSize /* type size in bytes */ reloEnumvalExists /* enum value existence in target kernel */ reloEnumvalValue /* enum value integer value */ + reloTypeMatches /* type matches kernel type */ ) func (k coreKind) checksForExistence() bool { - return k == reloEnumvalExists || k == reloTypeExists || k == reloFieldExists + return k == reloEnumvalExists || k == reloTypeExists || k == reloFieldExists || k == reloTypeMatches } func (k coreKind) String() string { @@ -151,30 +171,43 @@ func (k coreKind) String() string { return "enumval_exists" case reloEnumvalValue: return "enumval_value" + case reloTypeMatches: + return "type_matches" default: - return "unknown" + return fmt.Sprintf("unknown (%d)", k) } } // CORERelocate calculates changes needed to adjust eBPF instructions for differences // in types. // +// targets forms the set of types to relocate against. The first element has to be +// BTF for vmlinux, the following must be types for kernel modules. +// +// resolveLocalTypeID is called for each local type which requires a stable TypeID. +// Calling the function with the same type multiple times must produce the same +// result. It is the callers responsibility to ensure that the relocated instructions +// are loaded with matching BTF. +// // Returns a list of fixups which can be applied to instructions to make them // match the target type(s). // // Fixups are returned in the order of relos, e.g. fixup[i] is the solution // for relos[i]. -func CORERelocate(relos []*CORERelocation, target *Spec, bo binary.ByteOrder) ([]COREFixup, error) { - if target == nil { - var err error - target, _, err = kernelSpec() - if err != nil { - return nil, fmt.Errorf("load kernel spec: %w", err) - } +func CORERelocate(relos []*CORERelocation, targets []*Spec, bo binary.ByteOrder, resolveLocalTypeID func(Type) (TypeID, error)) ([]COREFixup, error) { + if len(targets) == 0 { + // Explicitly check for nil here since the argument used to be optional. + return nil, fmt.Errorf("targets must be provided") } - if bo != target.byteOrder { - return nil, fmt.Errorf("can't relocate %s against %s", bo, target.byteOrder) + // We can't encode type IDs that aren't for vmlinux into instructions at the + // moment. + resolveTargetTypeID := targets[0].TypeID + + for _, target := range targets { + if bo != target.imm.byteOrder { + return nil, fmt.Errorf("can't relocate %s against %s", bo, target.imm.byteOrder) + } } type reloGroup struct { @@ -194,14 +227,15 @@ func CORERelocate(relos []*CORERelocation, target *Spec, bo binary.ByteOrder) ([ return nil, fmt.Errorf("%s: unexpected accessor %v", relo.kind, relo.accessor) } + id, err := resolveLocalTypeID(relo.typ) + if err != nil { + return nil, fmt.Errorf("%s: get type id: %w", relo.kind, err) + } + result[i] = COREFixup{ - kind: relo.kind, - local: uint32(relo.id), - // NB: Using relo.id as the target here is incorrect, since - // it doesn't match the BTF we generate on the fly. This isn't - // too bad for now since there are no uses of the local type ID - // in the kernel, yet. - target: uint32(relo.id), + kind: relo.kind, + local: uint64(relo.id), + target: uint64(id), } continue } @@ -221,8 +255,23 @@ func CORERelocate(relos []*CORERelocation, target *Spec, bo binary.ByteOrder) ([ return nil, fmt.Errorf("relocate unnamed or anonymous type %s: %w", localType, ErrNotSupported) } - targets := target.namedTypes[newEssentialName(localTypeName)] - fixups, err := coreCalculateFixups(group.relos, target, targets, bo) + essentialName := newEssentialName(localTypeName) + + var targetTypes []Type + for _, target := range targets { + namedTypeIDs := target.imm.namedTypes[essentialName] + targetTypes = slices.Grow(targetTypes, len(namedTypeIDs)) + for _, id := range namedTypeIDs { + typ, err := target.TypeByID(id) + if err != nil { + return nil, err + } + + targetTypes = append(targetTypes, typ) + } + } + + fixups, err := coreCalculateFixups(group.relos, targetTypes, bo, resolveTargetTypeID) if err != nil { return nil, fmt.Errorf("relocate %s: %w", localType, err) } @@ -245,19 +294,14 @@ var errIncompatibleTypes = errors.New("incompatible types") // // The best target is determined by scoring: the less poisoning we have to do // the better the target is. -func coreCalculateFixups(relos []*CORERelocation, targetSpec *Spec, targets []Type, bo binary.ByteOrder) ([]COREFixup, error) { +func coreCalculateFixups(relos []*CORERelocation, targets []Type, bo binary.ByteOrder, resolveTargetTypeID func(Type) (TypeID, error)) ([]COREFixup, error) { bestScore := len(relos) var bestFixups []COREFixup for _, target := range targets { - targetID, err := targetSpec.TypeID(target) - if err != nil { - return nil, fmt.Errorf("target type ID: %w", err) - } - score := 0 // lower is better fixups := make([]COREFixup, 0, len(relos)) for _, relo := range relos { - fixup, err := coreCalculateFixup(relo, target, targetID, bo) + fixup, err := coreCalculateFixup(relo, target, bo, resolveTargetTypeID) if err != nil { return nil, fmt.Errorf("target %s: %s: %w", target, relo.kind, err) } @@ -308,13 +352,12 @@ func coreCalculateFixups(relos []*CORERelocation, targetSpec *Spec, targets []Ty var errNoSignedness = errors.New("no signedness") -// coreCalculateFixup calculates the fixup for a single local type, target type -// and relocation. -func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo binary.ByteOrder) (COREFixup, error) { - fixup := func(local, target uint32) (COREFixup, error) { +// coreCalculateFixup calculates the fixup given a relocation and a target type. +func coreCalculateFixup(relo *CORERelocation, target Type, bo binary.ByteOrder, resolveTargetTypeID func(Type) (TypeID, error)) (COREFixup, error) { + fixup := func(local, target uint64) (COREFixup, error) { return COREFixup{kind: relo.kind, local: local, target: target}, nil } - fixupWithoutValidation := func(local, target uint32) (COREFixup, error) { + fixupWithoutValidation := func(local, target uint64) (COREFixup, error) { return COREFixup{kind: relo.kind, local: local, target: target, skipLocalValidation: true}, nil } poison := func() (COREFixup, error) { @@ -328,12 +371,27 @@ func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo b local := relo.typ switch relo.kind { + case reloTypeMatches: + if len(relo.accessor) > 1 || relo.accessor[0] != 0 { + return zero, fmt.Errorf("unexpected accessor %v", relo.accessor) + } + + err := coreTypesMatch(local, target, nil) + if errors.Is(err, errIncompatibleTypes) { + return poison() + } + if err != nil { + return zero, err + } + + return fixup(1, 1) + case reloTypeIDTarget, reloTypeSize, reloTypeExists: if len(relo.accessor) > 1 || relo.accessor[0] != 0 { return zero, fmt.Errorf("unexpected accessor %v", relo.accessor) } - err := coreAreTypesCompatible(local, target) + err := CheckTypeCompatibility(local, target) if errors.Is(err, errIncompatibleTypes) { return poison() } @@ -346,7 +404,16 @@ func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo b return fixup(1, 1) case reloTypeIDTarget: - return fixup(uint32(relo.id), uint32(targetID)) + targetID, err := resolveTargetTypeID(target) + if errors.Is(err, ErrNotFound) { + // Probably a relocation trying to get the ID + // of a type from a kmod. + return poison() + } + if err != nil { + return zero, err + } + return fixup(uint64(relo.id), uint64(targetID)) case reloTypeSize: localSize, err := Sizeof(local) @@ -359,7 +426,7 @@ func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo b return zero, err } - return fixup(uint32(localSize), uint32(targetSize)) + return fixup(uint64(localSize), uint64(targetSize)) } case reloEnumvalValue, reloEnumvalExists: @@ -376,11 +443,11 @@ func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo b return fixup(1, 1) case reloEnumvalValue: - return fixup(uint32(localValue.Value), uint32(targetValue.Value)) + return fixup(localValue.Value, targetValue.Value) } case reloFieldByteOffset, reloFieldByteSize, reloFieldExists, reloFieldLShiftU64, reloFieldRShiftU64, reloFieldSigned: - if _, ok := as[*Fwd](target); ok { + if _, ok := As[*Fwd](target); ok { // We can't relocate fields using a forward declaration, so // skip it. If a non-forward declaration is present in the BTF // we'll find it in one of the other iterations. @@ -405,7 +472,7 @@ func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo b return fixup(1, 1) case reloFieldByteOffset: - return maybeSkipValidation(fixup(localField.offset, targetField.offset)) + return maybeSkipValidation(fixup(uint64(localField.offset), uint64(targetField.offset))) case reloFieldByteSize: localSize, err := Sizeof(localField.Type) @@ -417,24 +484,24 @@ func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo b if err != nil { return zero, err } - return maybeSkipValidation(fixup(uint32(localSize), uint32(targetSize))) + return maybeSkipValidation(fixup(uint64(localSize), uint64(targetSize))) case reloFieldLShiftU64: - var target uint32 + var target uint64 if bo == binary.LittleEndian { targetSize, err := targetField.sizeBits() if err != nil { return zero, err } - target = uint32(64 - targetField.bitfieldOffset - targetSize) + target = uint64(64 - targetField.bitfieldOffset - targetSize) } else { loadWidth, err := Sizeof(targetField.Type) if err != nil { return zero, err } - target = uint32(64 - Bits(loadWidth*8) + targetField.bitfieldOffset) + target = uint64(64 - Bits(loadWidth*8) + targetField.bitfieldOffset) } return fixupWithoutValidation(0, target) @@ -444,26 +511,26 @@ func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo b return zero, err } - return fixupWithoutValidation(0, uint32(64-targetSize)) + return fixupWithoutValidation(0, uint64(64-targetSize)) case reloFieldSigned: switch local := UnderlyingType(localField.Type).(type) { case *Enum: - target, ok := as[*Enum](targetField.Type) + target, ok := As[*Enum](targetField.Type) if !ok { return zero, fmt.Errorf("target isn't *Enum but %T", targetField.Type) } - return fixup(boolToUint32(local.Signed), boolToUint32(target.Signed)) + return fixup(boolToUint64(local.Signed), boolToUint64(target.Signed)) case *Int: - target, ok := as[*Int](targetField.Type) + target, ok := As[*Int](targetField.Type) if !ok { return zero, fmt.Errorf("target isn't *Int but %T", targetField.Type) } return fixup( - uint32(local.Encoding&Signed), - uint32(target.Encoding&Signed), + uint64(local.Encoding&Signed), + uint64(target.Encoding&Signed), ) default: return zero, fmt.Errorf("type %T: %w", local, errNoSignedness) @@ -474,7 +541,7 @@ func coreCalculateFixup(relo *CORERelocation, target Type, targetID TypeID, bo b return zero, ErrNotSupported } -func boolToUint32(val bool) uint32 { +func boolToUint64(val bool) uint64 { if val { return 1 } @@ -540,7 +607,7 @@ func (ca coreAccessor) String() string { } func (ca coreAccessor) enumValue(t Type) (*EnumValue, error) { - e, ok := as[*Enum](t) + e, ok := As[*Enum](t) if !ok { return nil, fmt.Errorf("not an enum: %s", t) } @@ -666,7 +733,7 @@ func coreFindField(localT Type, localAcc coreAccessor, targetT Type) (coreField, localMember := localMembers[acc] if localMember.Name == "" { - localMemberType, ok := as[composite](localMember.Type) + localMemberType, ok := As[composite](localMember.Type) if !ok { return coreField{}, coreField{}, fmt.Errorf("unnamed field with type %s: %s", localMember.Type, ErrNotSupported) } @@ -680,7 +747,7 @@ func coreFindField(localT Type, localAcc coreAccessor, targetT Type) (coreField, continue } - targetType, ok := as[composite](target.Type) + targetType, ok := As[composite](target.Type) if !ok { return coreField{}, coreField{}, fmt.Errorf("target not composite: %w", errImpossibleRelocation) } @@ -726,7 +793,7 @@ func coreFindField(localT Type, localAcc coreAccessor, targetT Type) (coreField, case *Array: // For arrays, acc is the index in the target. - targetType, ok := as[*Array](target.Type) + targetType, ok := As[*Array](target.Type) if !ok { return coreField{}, coreField{}, fmt.Errorf("target not array: %w", errImpossibleRelocation) } @@ -799,7 +866,7 @@ func coreFindMember(typ composite, name string) (Member, bool, error) { if visited[target] { continue } - if len(visited) >= maxTypeDepth { + if len(visited) >= maxResolveDepth { // This check is different than libbpf, which restricts the entire // path to BPF_CORE_SPEC_MAX_LEN items. return Member{}, false, fmt.Errorf("type is nested too deep") @@ -820,7 +887,7 @@ func coreFindMember(typ composite, name string) (Member, bool, error) { continue } - comp, ok := as[composite](member.Type) + comp, ok := As[composite](member.Type) if !ok { return Member{}, false, fmt.Errorf("anonymous non-composite type %T not allowed", member.Type) } @@ -839,7 +906,7 @@ func coreFindEnumValue(local Type, localAcc coreAccessor, target Type) (localVal return nil, nil, err } - targetEnum, ok := as[*Enum](target) + targetEnum, ok := As[*Enum](target) if !ok { return nil, nil, errImpossibleRelocation } @@ -860,7 +927,11 @@ func coreFindEnumValue(local Type, localAcc coreAccessor, target Type) (localVal // // Only layout compatibility is checked, ignoring names of the root type. func CheckTypeCompatibility(localType Type, targetType Type) error { - return coreAreTypesCompatible(localType, targetType) + return coreAreTypesCompatible(localType, targetType, nil) +} + +type pair struct { + A, B Type } /* The comment below is from bpf_core_types_are_compat in libbpf.c: @@ -886,59 +957,60 @@ func CheckTypeCompatibility(localType Type, targetType Type) error { * * Returns errIncompatibleTypes if types are not compatible. */ -func coreAreTypesCompatible(localType Type, targetType Type) error { +func coreAreTypesCompatible(localType Type, targetType Type, visited map[pair]struct{}) error { + localType = UnderlyingType(localType) + targetType = UnderlyingType(targetType) - var ( - localTs, targetTs typeDeque - l, t = &localType, &targetType - depth = 0 - ) + if reflect.TypeOf(localType) != reflect.TypeOf(targetType) { + return fmt.Errorf("type mismatch between %v and %v: %w", localType, targetType, errIncompatibleTypes) + } - for ; l != nil && t != nil; l, t = localTs.Shift(), targetTs.Shift() { - if depth >= maxTypeDepth { - return errors.New("types are nested too deep") - } + if _, ok := visited[pair{localType, targetType}]; ok { + return nil + } + if visited == nil { + visited = make(map[pair]struct{}) + } + visited[pair{localType, targetType}] = struct{}{} - localType = UnderlyingType(*l) - targetType = UnderlyingType(*t) + switch lv := localType.(type) { + case *Void, *Struct, *Union, *Enum, *Fwd, *Int: + return nil - if reflect.TypeOf(localType) != reflect.TypeOf(targetType) { - return fmt.Errorf("type mismatch: %w", errIncompatibleTypes) - } + case *Pointer: + tv := targetType.(*Pointer) + return coreAreTypesCompatible(lv.Target, tv.Target, visited) - switch lv := (localType).(type) { - case *Void, *Struct, *Union, *Enum, *Fwd, *Int: - // Nothing to do here + case *Array: + tv := targetType.(*Array) + if err := coreAreTypesCompatible(lv.Index, tv.Index, visited); err != nil { + return err + } - case *Pointer, *Array: - depth++ - walkType(localType, localTs.Push) - walkType(targetType, targetTs.Push) + return coreAreTypesCompatible(lv.Type, tv.Type, visited) - case *FuncProto: - tv := targetType.(*FuncProto) - if len(lv.Params) != len(tv.Params) { - return fmt.Errorf("function param mismatch: %w", errIncompatibleTypes) - } + case *FuncProto: + tv := targetType.(*FuncProto) + if err := coreAreTypesCompatible(lv.Return, tv.Return, visited); err != nil { + return err + } - depth++ - walkType(localType, localTs.Push) - walkType(targetType, targetTs.Push) + if len(lv.Params) != len(tv.Params) { + return fmt.Errorf("function param mismatch: %w", errIncompatibleTypes) + } - default: - return fmt.Errorf("unsupported type %T", localType) + for i, localParam := range lv.Params { + targetParam := tv.Params[i] + if err := coreAreTypesCompatible(localParam.Type, targetParam.Type, visited); err != nil { + return err + } } - } - if l != nil { - return fmt.Errorf("dangling local type %T", *l) - } + return nil - if t != nil { - return fmt.Errorf("dangling target type %T", *t) + default: + return fmt.Errorf("unsupported type %T", localType) } - - return nil } /* coreAreMembersCompatible checks two types for field-based relocation compatibility. @@ -970,19 +1042,6 @@ func coreAreMembersCompatible(localType Type, targetType Type) error { localType = UnderlyingType(localType) targetType = UnderlyingType(targetType) - doNamesMatch := func(a, b string) error { - if a == "" || b == "" { - // allow anonymous and named type to match - return nil - } - - if newEssentialName(a) == newEssentialName(b) { - return nil - } - - return fmt.Errorf("names don't match: %w", errImpossibleRelocation) - } - _, lok := localType.(composite) _, tok := targetType.(composite) if lok && tok { @@ -999,13 +1058,204 @@ func coreAreMembersCompatible(localType Type, targetType Type) error { case *Enum: tv := targetType.(*Enum) - return doNamesMatch(lv.Name, tv.Name) + if !coreEssentialNamesMatch(lv.Name, tv.Name) { + return fmt.Errorf("names %q and %q don't match: %w", lv.Name, tv.Name, errImpossibleRelocation) + } + + return nil case *Fwd: tv := targetType.(*Fwd) - return doNamesMatch(lv.Name, tv.Name) + if !coreEssentialNamesMatch(lv.Name, tv.Name) { + return fmt.Errorf("names %q and %q don't match: %w", lv.Name, tv.Name, errImpossibleRelocation) + } + + return nil default: return fmt.Errorf("type %s: %w", localType, ErrNotSupported) } } + +// coreEssentialNamesMatch compares two names while ignoring their flavour suffix. +// +// This should only be used on names which are in the global scope, like struct +// names, typedefs or enum values. +func coreEssentialNamesMatch(a, b string) bool { + if a == "" || b == "" { + // allow anonymous and named type to match + return true + } + + return newEssentialName(a) == newEssentialName(b) +} + +/* The comment below is from __bpf_core_types_match in relo_core.c: + * + * Check that two types "match". This function assumes that root types were + * already checked for name match. + * + * The matching relation is defined as follows: + * - modifiers and typedefs are stripped (and, hence, effectively ignored) + * - generally speaking types need to be of same kind (struct vs. struct, union + * vs. union, etc.) + * - exceptions are struct/union behind a pointer which could also match a + * forward declaration of a struct or union, respectively, and enum vs. + * enum64 (see below) + * Then, depending on type: + * - integers: + * - match if size and signedness match + * - arrays & pointers: + * - target types are recursively matched + * - structs & unions: + * - local members need to exist in target with the same name + * - for each member we recursively check match unless it is already behind a + * pointer, in which case we only check matching names and compatible kind + * - enums: + * - local variants have to have a match in target by symbolic name (but not + * numeric value) + * - size has to match (but enum may match enum64 and vice versa) + * - function pointers: + * - number and position of arguments in local type has to match target + * - for each argument and the return value we recursively check match + */ +func coreTypesMatch(localType Type, targetType Type, visited map[pair]struct{}) error { + localType = UnderlyingType(localType) + targetType = UnderlyingType(targetType) + + if !coreEssentialNamesMatch(localType.TypeName(), targetType.TypeName()) { + return fmt.Errorf("type name %q don't match %q: %w", localType.TypeName(), targetType.TypeName(), errIncompatibleTypes) + } + + if reflect.TypeOf(localType) != reflect.TypeOf(targetType) { + return fmt.Errorf("type mismatch between %v and %v: %w", localType, targetType, errIncompatibleTypes) + } + + if _, ok := visited[pair{localType, targetType}]; ok { + return nil + } + if visited == nil { + visited = make(map[pair]struct{}) + } + visited[pair{localType, targetType}] = struct{}{} + + switch lv := (localType).(type) { + case *Void: + + case *Fwd: + if targetType.(*Fwd).Kind != lv.Kind { + return fmt.Errorf("fwd kind mismatch between %v and %v: %w", localType, targetType, errIncompatibleTypes) + } + + case *Enum: + return coreEnumsMatch(lv, targetType.(*Enum)) + + case composite: + tv := targetType.(composite) + + if len(lv.members()) > len(tv.members()) { + return errIncompatibleTypes + } + + localMembers := lv.members() + targetMembers := map[string]Member{} + for _, member := range tv.members() { + targetMembers[member.Name] = member + } + + for _, localMember := range localMembers { + targetMember, found := targetMembers[localMember.Name] + if !found { + return fmt.Errorf("no field %q in %v: %w", localMember.Name, targetType, errIncompatibleTypes) + } + + err := coreTypesMatch(localMember.Type, targetMember.Type, visited) + if err != nil { + return err + } + } + + case *Int: + if !coreEncodingMatches(lv, targetType.(*Int)) { + return fmt.Errorf("int mismatch between %v and %v: %w", localType, targetType, errIncompatibleTypes) + } + + case *Pointer: + tv := targetType.(*Pointer) + + // Allow a pointer to a forward declaration to match a struct + // or union. + if fwd, ok := As[*Fwd](lv.Target); ok && fwd.matches(tv.Target) { + return nil + } + + if fwd, ok := As[*Fwd](tv.Target); ok && fwd.matches(lv.Target) { + return nil + } + + return coreTypesMatch(lv.Target, tv.Target, visited) + + case *Array: + tv := targetType.(*Array) + + if lv.Nelems != tv.Nelems { + return fmt.Errorf("array mismatch between %v and %v: %w", localType, targetType, errIncompatibleTypes) + } + + return coreTypesMatch(lv.Type, tv.Type, visited) + + case *FuncProto: + tv := targetType.(*FuncProto) + + if len(lv.Params) != len(tv.Params) { + return fmt.Errorf("function param mismatch: %w", errIncompatibleTypes) + } + + for i, lparam := range lv.Params { + if err := coreTypesMatch(lparam.Type, tv.Params[i].Type, visited); err != nil { + return err + } + } + + return coreTypesMatch(lv.Return, tv.Return, visited) + + default: + return fmt.Errorf("unsupported type %T", localType) + } + + return nil +} + +// coreEncodingMatches returns true if both ints have the same size and signedness. +// All encodings other than `Signed` are considered unsigned. +func coreEncodingMatches(local, target *Int) bool { + return local.Size == target.Size && (local.Encoding == Signed) == (target.Encoding == Signed) +} + +// coreEnumsMatch checks two enums match, which is considered to be the case if the following is true: +// - size has to match (but enum may match enum64 and vice versa) +// - local variants have to have a match in target by symbolic name (but not numeric value) +func coreEnumsMatch(local *Enum, target *Enum) error { + if local.Size != target.Size { + return fmt.Errorf("size mismatch between %v and %v: %w", local, target, errIncompatibleTypes) + } + + // If there are more values in the local than the target, there must be at least one value in the local + // that isn't in the target, and therefor the types are incompatible. + if len(local.Values) > len(target.Values) { + return fmt.Errorf("local has more values than target: %w", errIncompatibleTypes) + } + +outer: + for _, lv := range local.Values { + for _, rv := range target.Values { + if coreEssentialNamesMatch(lv.Name, rv.Name) { + continue outer + } + } + + return fmt.Errorf("no match for %v in %v: %w", lv, target, errIncompatibleTypes) + } + + return nil +} diff --git a/vendor/github.com/cilium/ebpf/btf/ext_info.go b/vendor/github.com/cilium/ebpf/btf/ext_info.go index b764fb7bcc1b..eb9044badf23 100644 --- a/vendor/github.com/cilium/ebpf/btf/ext_info.go +++ b/vendor/github.com/cilium/ebpf/btf/ext_info.go @@ -16,9 +16,9 @@ import ( // ExtInfos contains ELF section metadata. type ExtInfos struct { // The slices are sorted by offset in ascending order. - funcInfos map[string][]funcInfo - lineInfos map[string][]lineInfo - relocationInfos map[string][]coreRelocationInfo + funcInfos map[string]FuncInfos + lineInfos map[string]LineInfos + relocationInfos map[string]CORERelocationInfos } // loadExtInfosFromELF parses ext infos from the .BTF.ext section in an ELF. @@ -34,11 +34,11 @@ func loadExtInfosFromELF(file *internal.SafeELFFile, spec *Spec) (*ExtInfos, err return nil, fmt.Errorf("compressed ext_info is not supported") } - return loadExtInfos(section.ReaderAt, file.ByteOrder, spec, spec.strings) + return loadExtInfos(section.ReaderAt, file.ByteOrder, spec) } // loadExtInfos parses bare ext infos. -func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec, strings *stringTable) (*ExtInfos, error) { +func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec) (*ExtInfos, error) { // Open unbuffered section reader. binary.Read() calls io.ReadFull on // the header structs, resulting in one syscall per header. headerRd := io.NewSectionReader(r, 0, math.MaxInt64) @@ -53,12 +53,12 @@ func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec, strings *strin } buf := internal.NewBufferedSectionReader(r, extHeader.funcInfoStart(), int64(extHeader.FuncInfoLen)) - btfFuncInfos, err := parseFuncInfos(buf, bo, strings) + btfFuncInfos, err := parseFuncInfos(buf, bo, spec.strings) if err != nil { return nil, fmt.Errorf("parsing BTF function info: %w", err) } - funcInfos := make(map[string][]funcInfo, len(btfFuncInfos)) + funcInfos := make(map[string]FuncInfos, len(btfFuncInfos)) for section, bfis := range btfFuncInfos { funcInfos[section], err = newFuncInfos(bfis, spec) if err != nil { @@ -67,14 +67,14 @@ func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec, strings *strin } buf = internal.NewBufferedSectionReader(r, extHeader.lineInfoStart(), int64(extHeader.LineInfoLen)) - btfLineInfos, err := parseLineInfos(buf, bo, strings) + btfLineInfos, err := parseLineInfos(buf, bo, spec.strings) if err != nil { return nil, fmt.Errorf("parsing BTF line info: %w", err) } - lineInfos := make(map[string][]lineInfo, len(btfLineInfos)) + lineInfos := make(map[string]LineInfos, len(btfLineInfos)) for section, blis := range btfLineInfos { - lineInfos[section], err = newLineInfos(blis, strings) + lineInfos[section], err = newLineInfos(blis, spec.strings) if err != nil { return nil, fmt.Errorf("section %s: line infos: %w", section, err) } @@ -86,14 +86,14 @@ func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec, strings *strin var btfCORERelos map[string][]bpfCORERelo buf = internal.NewBufferedSectionReader(r, extHeader.coreReloStart(coreHeader), int64(coreHeader.COREReloLen)) - btfCORERelos, err = parseCORERelos(buf, bo, strings) + btfCORERelos, err = parseCORERelos(buf, bo, spec.strings) if err != nil { return nil, fmt.Errorf("parsing CO-RE relocation info: %w", err) } - coreRelos := make(map[string][]coreRelocationInfo, len(btfCORERelos)) + coreRelos := make(map[string]CORERelocationInfos, len(btfCORERelos)) for section, brs := range btfCORERelos { - coreRelos[section], err = newRelocationInfos(brs, spec, strings) + coreRelos[section], err = newRelocationInfos(brs, spec, spec.strings) if err != nil { return nil, fmt.Errorf("section %s: CO-RE relocations: %w", section, err) } @@ -111,21 +111,31 @@ func (ei *ExtInfos) Assign(insns asm.Instructions, section string) { lineInfos := ei.lineInfos[section] reloInfos := ei.relocationInfos[section] + AssignMetadataToInstructions(insns, funcInfos, lineInfos, reloInfos) +} + +// Assign per-instruction metadata to the instructions in insns. +func AssignMetadataToInstructions( + insns asm.Instructions, + funcInfos FuncInfos, + lineInfos LineInfos, + reloInfos CORERelocationInfos, +) { iter := insns.Iterate() for iter.Next() { - if len(funcInfos) > 0 && funcInfos[0].offset == iter.Offset { - *iter.Ins = WithFuncMetadata(*iter.Ins, funcInfos[0].fn) - funcInfos = funcInfos[1:] + if len(funcInfos.infos) > 0 && funcInfos.infos[0].offset == iter.Offset { + *iter.Ins = WithFuncMetadata(*iter.Ins, funcInfos.infos[0].fn) + funcInfos.infos = funcInfos.infos[1:] } - if len(lineInfos) > 0 && lineInfos[0].offset == iter.Offset { - *iter.Ins = iter.Ins.WithSource(lineInfos[0].line) - lineInfos = lineInfos[1:] + if len(lineInfos.infos) > 0 && lineInfos.infos[0].offset == iter.Offset { + *iter.Ins = iter.Ins.WithSource(lineInfos.infos[0].line) + lineInfos.infos = lineInfos.infos[1:] } - if len(reloInfos) > 0 && reloInfos[0].offset == iter.Offset { - iter.Ins.Metadata.Set(coreRelocationMeta{}, reloInfos[0].relo) - reloInfos = reloInfos[1:] + if len(reloInfos.infos) > 0 && reloInfos.infos[0].offset == iter.Offset { + iter.Ins.Metadata.Set(coreRelocationMeta{}, reloInfos.infos[0].relo) + reloInfos.infos = reloInfos.infos[1:] } } } @@ -133,27 +143,19 @@ func (ei *ExtInfos) Assign(insns asm.Instructions, section string) { // MarshalExtInfos encodes function and line info embedded in insns into kernel // wire format. // -// Returns ErrNotSupported if the kernel doesn't support BTF-associated programs. -func MarshalExtInfos(insns asm.Instructions) (_ *Handle, funcInfos, lineInfos []byte, _ error) { - // Bail out early if the kernel doesn't support Func(Proto). If this is the - // case, func_info will also be unsupported. - if err := haveProgBTF(); err != nil { - return nil, nil, nil, err - } - +// If an instruction has an [asm.Comment], it will be synthesized into a mostly +// empty line info. +func MarshalExtInfos(insns asm.Instructions, b *Builder) (funcInfos, lineInfos []byte, _ error) { iter := insns.Iterate() for iter.Next() { - _, ok := iter.Ins.Source().(*Line) - fn := FuncMetadata(iter.Ins) - if ok || fn != nil { + if iter.Ins.Source() != nil || FuncMetadata(iter.Ins) != nil { goto marshal } } - return nil, nil, nil, nil + return nil, nil, nil marshal: - var b Builder var fiBuf, liBuf bytes.Buffer for { if fn := FuncMetadata(iter.Ins); fn != nil { @@ -161,18 +163,27 @@ marshal: fn: fn, offset: iter.Offset, } - if err := fi.marshal(&fiBuf, &b); err != nil { - return nil, nil, nil, fmt.Errorf("write func info: %w", err) + if err := fi.marshal(&fiBuf, b); err != nil { + return nil, nil, fmt.Errorf("write func info: %w", err) } } - if line, ok := iter.Ins.Source().(*Line); ok { + if source := iter.Ins.Source(); source != nil { + var line *Line + if l, ok := source.(*Line); ok { + line = l + } else { + line = &Line{ + line: source.String(), + } + } + li := &lineInfo{ line: line, offset: iter.Offset, } - if err := li.marshal(&liBuf, &b); err != nil { - return nil, nil, nil, fmt.Errorf("write line info: %w", err) + if err := li.marshal(&liBuf, b); err != nil { + return nil, nil, fmt.Errorf("write line info: %w", err) } } @@ -181,8 +192,7 @@ marshal: } } - handle, err := NewHandle(&b) - return handle, fiBuf.Bytes(), liBuf.Bytes(), err + return fiBuf.Bytes(), liBuf.Bytes(), nil } // btfExtHeader is found at the start of the .BTF.ext section. @@ -323,6 +333,11 @@ func parseExtInfoRecordSize(r io.Reader, bo binary.ByteOrder) (uint32, error) { return recordSize, nil } +// FuncInfos contains a sorted list of func infos. +type FuncInfos struct { + infos []funcInfo +} + // The size of a FuncInfo in BTF wire format. var FuncInfoSize = uint32(binary.Size(bpfFuncInfo{})) @@ -359,21 +374,39 @@ func newFuncInfo(fi bpfFuncInfo, spec *Spec) (*funcInfo, error) { }, nil } -func newFuncInfos(bfis []bpfFuncInfo, spec *Spec) ([]funcInfo, error) { - fis := make([]funcInfo, 0, len(bfis)) +func newFuncInfos(bfis []bpfFuncInfo, spec *Spec) (FuncInfos, error) { + fis := FuncInfos{ + infos: make([]funcInfo, 0, len(bfis)), + } for _, bfi := range bfis { fi, err := newFuncInfo(bfi, spec) if err != nil { - return nil, fmt.Errorf("offset %d: %w", bfi.InsnOff, err) + return FuncInfos{}, fmt.Errorf("offset %d: %w", bfi.InsnOff, err) } - fis = append(fis, *fi) + fis.infos = append(fis.infos, *fi) } - sort.Slice(fis, func(i, j int) bool { - return fis[i].offset <= fis[j].offset + sort.Slice(fis.infos, func(i, j int) bool { + return fis.infos[i].offset <= fis.infos[j].offset }) return fis, nil } +// LoadFuncInfos parses BTF func info in kernel wire format. +func LoadFuncInfos(reader io.Reader, bo binary.ByteOrder, recordNum uint32, spec *Spec) (FuncInfos, error) { + fis, err := parseFuncInfoRecords( + reader, + bo, + FuncInfoSize, + recordNum, + false, + ) + if err != nil { + return FuncInfos{}, fmt.Errorf("parsing BTF func info: %w", err) + } + + return newFuncInfos(fis, spec) +} + // marshal into the BTF wire format. func (fi *funcInfo) marshal(w *bytes.Buffer, b *Builder) error { id, err := b.Add(fi.fn) @@ -409,7 +442,7 @@ func parseFuncInfos(r io.Reader, bo binary.ByteOrder, strings *stringTable) (map return nil, err } - records, err := parseFuncInfoRecords(r, bo, recordSize, infoHeader.NumInfo) + records, err := parseFuncInfoRecords(r, bo, recordSize, infoHeader.NumInfo, true) if err != nil { return nil, fmt.Errorf("section %v: %w", secName, err) } @@ -421,7 +454,7 @@ func parseFuncInfos(r io.Reader, bo binary.ByteOrder, strings *stringTable) (map // parseFuncInfoRecords parses a stream of func_infos into a funcInfos. // These records appear after a btf_ext_info_sec header in the func_info // sub-section of .BTF.ext. -func parseFuncInfoRecords(r io.Reader, bo binary.ByteOrder, recordSize uint32, recordNum uint32) ([]bpfFuncInfo, error) { +func parseFuncInfoRecords(r io.Reader, bo binary.ByteOrder, recordSize uint32, recordNum uint32, offsetInBytes bool) ([]bpfFuncInfo, error) { var out []bpfFuncInfo var fi bpfFuncInfo @@ -435,13 +468,15 @@ func parseFuncInfoRecords(r io.Reader, bo binary.ByteOrder, recordSize uint32, r return nil, fmt.Errorf("can't read function info: %v", err) } - if fi.InsnOff%asm.InstructionSize != 0 { - return nil, fmt.Errorf("offset %v is not aligned with instruction size", fi.InsnOff) - } + if offsetInBytes { + if fi.InsnOff%asm.InstructionSize != 0 { + return nil, fmt.Errorf("offset %v is not aligned with instruction size", fi.InsnOff) + } - // ELF tracks offset in bytes, the kernel expects raw BPF instructions. - // Convert as early as possible. - fi.InsnOff /= asm.InstructionSize + // ELF tracks offset in bytes, the kernel expects raw BPF instructions. + // Convert as early as possible. + fi.InsnOff /= asm.InstructionSize + } out = append(out, fi) } @@ -480,6 +515,11 @@ func (li *Line) String() string { return li.line } +// LineInfos contains a sorted list of line infos. +type LineInfos struct { + infos []lineInfo +} + type lineInfo struct { line *Line offset asm.RawInstructionOffset @@ -500,21 +540,37 @@ type bpfLineInfo struct { LineCol uint32 } -func newLineInfo(li bpfLineInfo, strings *stringTable) (*lineInfo, error) { +// LoadLineInfos parses BTF line info in kernel wire format. +func LoadLineInfos(reader io.Reader, bo binary.ByteOrder, recordNum uint32, spec *Spec) (LineInfos, error) { + lis, err := parseLineInfoRecords( + reader, + bo, + LineInfoSize, + recordNum, + false, + ) + if err != nil { + return LineInfos{}, fmt.Errorf("parsing BTF line info: %w", err) + } + + return newLineInfos(lis, spec.strings) +} + +func newLineInfo(li bpfLineInfo, strings *stringTable) (lineInfo, error) { line, err := strings.Lookup(li.LineOff) if err != nil { - return nil, fmt.Errorf("lookup of line: %w", err) + return lineInfo{}, fmt.Errorf("lookup of line: %w", err) } fileName, err := strings.Lookup(li.FileNameOff) if err != nil { - return nil, fmt.Errorf("lookup of filename: %w", err) + return lineInfo{}, fmt.Errorf("lookup of filename: %w", err) } lineNumber := li.LineCol >> bpfLineShift lineColumn := li.LineCol & bpfColumnMax - return &lineInfo{ + return lineInfo{ &Line{ fileName, line, @@ -525,17 +581,19 @@ func newLineInfo(li bpfLineInfo, strings *stringTable) (*lineInfo, error) { }, nil } -func newLineInfos(blis []bpfLineInfo, strings *stringTable) ([]lineInfo, error) { - lis := make([]lineInfo, 0, len(blis)) +func newLineInfos(blis []bpfLineInfo, strings *stringTable) (LineInfos, error) { + lis := LineInfos{ + infos: make([]lineInfo, 0, len(blis)), + } for _, bli := range blis { li, err := newLineInfo(bli, strings) if err != nil { - return nil, fmt.Errorf("offset %d: %w", bli.InsnOff, err) + return LineInfos{}, fmt.Errorf("offset %d: %w", bli.InsnOff, err) } - lis = append(lis, *li) + lis.infos = append(lis.infos, li) } - sort.Slice(lis, func(i, j int) bool { - return lis[i].offset <= lis[j].offset + sort.Slice(lis.infos, func(i, j int) bool { + return lis.infos[i].offset <= lis.infos[j].offset }) return lis, nil } @@ -595,7 +653,7 @@ func parseLineInfos(r io.Reader, bo binary.ByteOrder, strings *stringTable) (map return nil, err } - records, err := parseLineInfoRecords(r, bo, recordSize, infoHeader.NumInfo) + records, err := parseLineInfoRecords(r, bo, recordSize, infoHeader.NumInfo, true) if err != nil { return nil, fmt.Errorf("section %v: %w", secName, err) } @@ -607,8 +665,7 @@ func parseLineInfos(r io.Reader, bo binary.ByteOrder, strings *stringTable) (map // parseLineInfoRecords parses a stream of line_infos into a lineInfos. // These records appear after a btf_ext_info_sec header in the line_info // sub-section of .BTF.ext. -func parseLineInfoRecords(r io.Reader, bo binary.ByteOrder, recordSize uint32, recordNum uint32) ([]bpfLineInfo, error) { - var out []bpfLineInfo +func parseLineInfoRecords(r io.Reader, bo binary.ByteOrder, recordSize uint32, recordNum uint32, offsetInBytes bool) ([]bpfLineInfo, error) { var li bpfLineInfo if exp, got := uint32(binary.Size(li)), recordSize; exp != got { @@ -616,18 +673,21 @@ func parseLineInfoRecords(r io.Reader, bo binary.ByteOrder, recordSize uint32, r return nil, fmt.Errorf("expected LineInfo record size %d, but BTF blob contains %d", exp, got) } + out := make([]bpfLineInfo, 0, recordNum) for i := uint32(0); i < recordNum; i++ { if err := binary.Read(r, bo, &li); err != nil { return nil, fmt.Errorf("can't read line info: %v", err) } - if li.InsnOff%asm.InstructionSize != 0 { - return nil, fmt.Errorf("offset %v is not aligned with instruction size", li.InsnOff) - } + if offsetInBytes { + if li.InsnOff%asm.InstructionSize != 0 { + return nil, fmt.Errorf("offset %v is not aligned with instruction size", li.InsnOff) + } - // ELF tracks offset in bytes, the kernel expects raw BPF instructions. - // Convert as early as possible. - li.InsnOff /= asm.InstructionSize + // ELF tracks offset in bytes, the kernel expects raw BPF instructions. + // Convert as early as possible. + li.InsnOff /= asm.InstructionSize + } out = append(out, li) } @@ -661,6 +721,11 @@ func CORERelocationMetadata(ins *asm.Instruction) *CORERelocation { return relo } +// CORERelocationInfos contains a sorted list of co:re relocation infos. +type CORERelocationInfos struct { + infos []coreRelocationInfo +} + type coreRelocationInfo struct { relo *CORERelocation offset asm.RawInstructionOffset @@ -693,17 +758,19 @@ func newRelocationInfo(relo bpfCORERelo, spec *Spec, strings *stringTable) (*cor }, nil } -func newRelocationInfos(brs []bpfCORERelo, spec *Spec, strings *stringTable) ([]coreRelocationInfo, error) { - rs := make([]coreRelocationInfo, 0, len(brs)) +func newRelocationInfos(brs []bpfCORERelo, spec *Spec, strings *stringTable) (CORERelocationInfos, error) { + rs := CORERelocationInfos{ + infos: make([]coreRelocationInfo, 0, len(brs)), + } for _, br := range brs { relo, err := newRelocationInfo(br, spec, strings) if err != nil { - return nil, fmt.Errorf("offset %d: %w", br.InsnOff, err) + return CORERelocationInfos{}, fmt.Errorf("offset %d: %w", br.InsnOff, err) } - rs = append(rs, *relo) + rs.infos = append(rs.infos, *relo) } - sort.Slice(rs, func(i, j int) bool { - return rs[i].offset < rs[j].offset + sort.Slice(rs.infos, func(i, j int) bool { + return rs.infos[i].offset < rs.infos[j].offset }) return rs, nil } diff --git a/vendor/github.com/cilium/ebpf/btf/feature.go b/vendor/github.com/cilium/ebpf/btf/feature.go new file mode 100644 index 000000000000..6feb08dfbb0f --- /dev/null +++ b/vendor/github.com/cilium/ebpf/btf/feature.go @@ -0,0 +1,123 @@ +package btf + +import ( + "errors" + "math" + + "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/unix" +) + +// haveBTF attempts to load a BTF blob containing an Int. It should pass on any +// kernel that supports BPF_BTF_LOAD. +var haveBTF = internal.NewFeatureTest("BTF", "4.18", func() error { + // 0-length anonymous integer + err := probeBTF(&Int{}) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + return internal.ErrNotSupported + } + return err +}) + +// haveMapBTF attempts to load a minimal BTF blob containing a Var. It is +// used as a proxy for .bss, .data and .rodata map support, which generally +// come with a Var and Datasec. These were introduced in Linux 5.2. +var haveMapBTF = internal.NewFeatureTest("Map BTF (Var/Datasec)", "5.2", func() error { + if err := haveBTF(); err != nil { + return err + } + + v := &Var{ + Name: "a", + Type: &Pointer{(*Void)(nil)}, + } + + err := probeBTF(v) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + // Treat both EINVAL and EPERM as not supported: creating the map may still + // succeed without Btf* attrs. + return internal.ErrNotSupported + } + return err +}) + +// haveProgBTF attempts to load a BTF blob containing a Func and FuncProto. It +// is used as a proxy for ext_info (func_info) support, which depends on +// Func(Proto) by definition. +var haveProgBTF = internal.NewFeatureTest("Program BTF (func/line_info)", "5.0", func() error { + if err := haveBTF(); err != nil { + return err + } + + fn := &Func{ + Name: "a", + Type: &FuncProto{Return: (*Void)(nil)}, + } + + err := probeBTF(fn) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + return internal.ErrNotSupported + } + return err +}) + +var haveFuncLinkage = internal.NewFeatureTest("BTF func linkage", "5.6", func() error { + if err := haveProgBTF(); err != nil { + return err + } + + fn := &Func{ + Name: "a", + Type: &FuncProto{Return: (*Void)(nil)}, + Linkage: GlobalFunc, + } + + err := probeBTF(fn) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } + return err +}) + +var haveEnum64 = internal.NewFeatureTest("ENUM64", "6.0", func() error { + if err := haveBTF(); err != nil { + return err + } + + enum := &Enum{ + Size: 8, + Values: []EnumValue{ + {"TEST", math.MaxUint32 + 1}, + }, + } + + err := probeBTF(enum) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } + return err +}) + +func probeBTF(typ Type) error { + b, err := NewBuilder([]Type{typ}) + if err != nil { + return err + } + + buf, err := b.Marshal(nil, nil) + if err != nil { + return err + } + + fd, err := sys.BtfLoad(&sys.BtfLoadAttr{ + Btf: sys.NewSlicePointer(buf), + BtfSize: uint32(len(buf)), + }) + + if err == nil { + fd.Close() + } + + return err +} diff --git a/vendor/github.com/cilium/ebpf/btf/format.go b/vendor/github.com/cilium/ebpf/btf/format.go index e85220259e74..5e581b4a8518 100644 --- a/vendor/github.com/cilium/ebpf/btf/format.go +++ b/vendor/github.com/cilium/ebpf/btf/format.go @@ -77,7 +77,13 @@ func (gf *GoFormatter) writeTypeDecl(name string, typ Type) error { gf.w.WriteString("; const ( ") for _, ev := range e.Values { id := gf.enumIdentifier(name, ev.Name) - fmt.Fprintf(&gf.w, "%s %s = %d; ", id, name, ev.Value) + var value any + if e.Signed { + value = int64(ev.Value) + } else { + value = ev.Value + } + fmt.Fprintf(&gf.w, "%s %s = %d; ", id, name, value) } gf.w.WriteString(")") @@ -112,7 +118,7 @@ func (gf *GoFormatter) writeType(typ Type, depth int) error { // uint32 func (gf *GoFormatter) writeTypeLit(typ Type, depth int) error { depth++ - if depth > maxTypeDepth { + if depth > maxResolveDepth { return errNestedTooDeep } @@ -259,7 +265,7 @@ func (gf *GoFormatter) writeStructField(m Member, depth int) error { } depth++ - if depth > maxTypeDepth { + if depth > maxResolveDepth { return errNestedTooDeep } @@ -332,7 +338,7 @@ func (gf *GoFormatter) writePadding(bytes uint32) { func skipQualifiers(typ Type) Type { result := typ - for depth := 0; depth <= maxTypeDepth; depth++ { + for depth := 0; depth <= maxResolveDepth; depth++ { switch v := (result).(type) { case qualifier: result = v.qualify() diff --git a/vendor/github.com/cilium/ebpf/btf/handle.go b/vendor/github.com/cilium/ebpf/btf/handle.go index b6b3e87f504f..adfa6fed4bc3 100644 --- a/vendor/github.com/cilium/ebpf/btf/handle.go +++ b/vendor/github.com/cilium/ebpf/btf/handle.go @@ -41,6 +41,8 @@ func NewHandle(b *Builder) (*Handle, error) { // // Returns an error wrapping ErrNotSupported if the kernel doesn't support BTF. func NewHandleFromRawBTF(btf []byte) (*Handle, error) { + const minLogSize = 64 * 1024 + if uint64(len(btf)) > math.MaxUint32 { return nil, errors.New("BTF exceeds the maximum size") } @@ -50,26 +52,54 @@ func NewHandleFromRawBTF(btf []byte) (*Handle, error) { BtfSize: uint32(len(btf)), } - fd, err := sys.BtfLoad(attr) - if err == nil { - return &Handle{fd, attr.BtfSize, false}, nil + var ( + logBuf []byte + err error + ) + for { + var fd *sys.FD + fd, err = sys.BtfLoad(attr) + if err == nil { + return &Handle{fd, attr.BtfSize, false}, nil + } + + if attr.BtfLogTrueSize != 0 && attr.BtfLogSize >= attr.BtfLogTrueSize { + // The log buffer already has the correct size. + break + } + + if attr.BtfLogSize != 0 && !errors.Is(err, unix.ENOSPC) { + // Up until at least kernel 6.0, the BTF verifier does not return ENOSPC + // if there are other verification errors. ENOSPC is only returned when + // the BTF blob is correct, a log was requested, and the provided buffer + // is too small. We're therefore not sure whether we got the full + // log or not. + break + } + + // Make an educated guess how large the buffer should be. Start + // at a reasonable minimum and then double the size. + logSize := uint32(max(len(logBuf)*2, minLogSize)) + if int(logSize) < len(logBuf) { + return nil, errors.New("overflow while probing log buffer size") + } + + if attr.BtfLogTrueSize != 0 { + // The kernel has given us a hint how large the log buffer has to be. + logSize = attr.BtfLogTrueSize + } + + logBuf = make([]byte, logSize) + attr.BtfLogSize = logSize + attr.BtfLogBuf = sys.NewSlicePointer(logBuf) + attr.BtfLogLevel = 1 } if err := haveBTF(); err != nil { return nil, err } - logBuf := make([]byte, 64*1024) - attr.BtfLogBuf = sys.NewSlicePointer(logBuf) - attr.BtfLogSize = uint32(len(logBuf)) - attr.BtfLogLevel = 1 - - // Up until at least kernel 6.0, the BTF verifier does not return ENOSPC - // if there are other verification errors. ENOSPC is only returned when - // the BTF blob is correct, a log was requested, and the provided buffer - // is too small. - _, ve := sys.BtfLoad(attr) - return nil, internal.ErrorWithLog("load btf", err, logBuf, errors.Is(ve, unix.ENOSPC)) + return nil, internal.ErrorWithLog("load btf", err, logBuf) } // NewHandleFromID returns the BTF handle for a given id. diff --git a/vendor/github.com/cilium/ebpf/btf/kernel.go b/vendor/github.com/cilium/ebpf/btf/kernel.go new file mode 100644 index 000000000000..8584ebcb932a --- /dev/null +++ b/vendor/github.com/cilium/ebpf/btf/kernel.go @@ -0,0 +1,159 @@ +package btf + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/kallsyms" +) + +var kernelBTF = struct { + sync.RWMutex + kernel *Spec + modules map[string]*Spec +}{ + modules: make(map[string]*Spec), +} + +// FlushKernelSpec removes any cached kernel type information. +func FlushKernelSpec() { + kallsyms.FlushKernelModuleCache() + + kernelBTF.Lock() + defer kernelBTF.Unlock() + + kernelBTF.kernel = nil + kernelBTF.modules = make(map[string]*Spec) +} + +// LoadKernelSpec returns the current kernel's BTF information. +// +// Defaults to /sys/kernel/btf/vmlinux and falls back to scanning the file system +// for vmlinux ELFs. Returns an error wrapping ErrNotSupported if BTF is not enabled. +func LoadKernelSpec() (*Spec, error) { + kernelBTF.RLock() + spec := kernelBTF.kernel + kernelBTF.RUnlock() + + if spec == nil { + kernelBTF.Lock() + defer kernelBTF.Unlock() + + spec = kernelBTF.kernel + } + + if spec != nil { + return spec.Copy(), nil + } + + spec, _, err := loadKernelSpec() + if err != nil { + return nil, err + } + + kernelBTF.kernel = spec + return spec.Copy(), nil +} + +// LoadKernelModuleSpec returns the BTF information for the named kernel module. +// +// Defaults to /sys/kernel/btf/. +// Returns an error wrapping ErrNotSupported if BTF is not enabled. +// Returns an error wrapping fs.ErrNotExist if BTF for the specific module doesn't exist. +func LoadKernelModuleSpec(module string) (*Spec, error) { + kernelBTF.RLock() + spec := kernelBTF.modules[module] + kernelBTF.RUnlock() + + if spec != nil { + return spec.Copy(), nil + } + + base, err := LoadKernelSpec() + if err != nil { + return nil, fmt.Errorf("load kernel spec: %w", err) + } + + kernelBTF.Lock() + defer kernelBTF.Unlock() + + if spec = kernelBTF.modules[module]; spec != nil { + return spec.Copy(), nil + } + + spec, err = loadKernelModuleSpec(module, base) + if err != nil { + return nil, err + } + + kernelBTF.modules[module] = spec + return spec.Copy(), nil +} + +func loadKernelSpec() (_ *Spec, fallback bool, _ error) { + fh, err := os.Open("/sys/kernel/btf/vmlinux") + if err == nil { + defer fh.Close() + + spec, err := loadRawSpec(fh, internal.NativeEndian, nil) + return spec, false, err + } + + file, err := findVMLinux() + if err != nil { + return nil, false, err + } + defer file.Close() + + spec, err := LoadSpecFromReader(file) + return spec, true, err +} + +func loadKernelModuleSpec(module string, base *Spec) (*Spec, error) { + dir, file := filepath.Split(module) + if dir != "" || filepath.Ext(file) != "" { + return nil, fmt.Errorf("invalid module name %q", module) + } + + fh, err := os.Open(filepath.Join("/sys/kernel/btf", module)) + if err != nil { + return nil, err + } + defer fh.Close() + + return loadRawSpec(fh, internal.NativeEndian, base) +} + +// findVMLinux scans multiple well-known paths for vmlinux kernel images. +func findVMLinux() (*os.File, error) { + release, err := internal.KernelRelease() + if err != nil { + return nil, err + } + + // use same list of locations as libbpf + // https://github.com/libbpf/libbpf/blob/9a3a42608dbe3731256a5682a125ac1e23bced8f/src/btf.c#L3114-L3122 + locations := []string{ + "/boot/vmlinux-%s", + "/lib/modules/%s/vmlinux-%[1]s", + "/lib/modules/%s/build/vmlinux", + "/usr/lib/modules/%s/kernel/vmlinux", + "/usr/lib/debug/boot/vmlinux-%s", + "/usr/lib/debug/boot/vmlinux-%s.debug", + "/usr/lib/debug/lib/modules/%s/vmlinux", + } + + for _, loc := range locations { + file, err := os.Open(fmt.Sprintf(loc, release)) + if errors.Is(err, os.ErrNotExist) { + continue + } + return file, err + } + + return nil, fmt.Errorf("no BTF found for kernel version %s: %w", release, internal.ErrNotSupported) +} diff --git a/vendor/github.com/cilium/ebpf/btf/marshal.go b/vendor/github.com/cilium/ebpf/btf/marshal.go index bfe53b41072c..f14cfa6e9735 100644 --- a/vendor/github.com/cilium/ebpf/btf/marshal.go +++ b/vendor/github.com/cilium/ebpf/btf/marshal.go @@ -5,12 +5,12 @@ import ( "encoding/binary" "errors" "fmt" + "maps" "math" + "slices" "sync" "github.com/cilium/ebpf/internal" - - "golang.org/x/exp/slices" ) type MarshalOptions struct { @@ -18,13 +18,19 @@ type MarshalOptions struct { Order binary.ByteOrder // Remove function linkage information for compatibility with <5.6 kernels. StripFuncLinkage bool + // Replace Enum64 with a placeholder for compatibility with <6.0 kernels. + ReplaceEnum64 bool + // Prevent the "No type found" error when loading BTF without any types. + PreventNoTypeFound bool } // KernelMarshalOptions will generate BTF suitable for the current kernel. func KernelMarshalOptions() *MarshalOptions { return &MarshalOptions{ - Order: internal.NativeEndian, - StripFuncLinkage: haveFuncLinkage() != nil, + Order: internal.NativeEndian, + StripFuncLinkage: haveFuncLinkage() != nil, + ReplaceEnum64: haveEnum64() != nil, + PreventNoTypeFound: true, // All current kernels require this. } } @@ -36,6 +42,7 @@ type encoder struct { buf *bytes.Buffer strings *stringTableBuilder ids map[Type]TypeID + visited map[Type]struct{} lastID TypeID } @@ -90,6 +97,11 @@ func NewBuilder(types []Type) (*Builder, error) { return b, nil } +// Empty returns true if neither types nor strings have been added. +func (b *Builder) Empty() bool { + return len(b.types) == 0 && (b.strings == nil || b.strings.Length() == 0) +} + // Add a Type and allocate a stable ID for it. // // Adding the identical Type multiple times is valid and will return the same ID. @@ -156,15 +168,29 @@ func (b *Builder) Marshal(buf []byte, opts *MarshalOptions) ([]byte, error) { buf: w, strings: stb, lastID: TypeID(len(b.types)), - ids: make(map[Type]TypeID, len(b.types)), + visited: make(map[Type]struct{}, len(b.types)), + ids: maps.Clone(b.stableIDs), + } + + if e.ids == nil { + e.ids = make(map[Type]TypeID) + } + + types := b.types + if len(types) == 0 && stb.Length() > 0 && opts.PreventNoTypeFound { + // We have strings that need to be written out, + // but no types (besides the implicit Void). + // Kernels as recent as v6.7 refuse to load such BTF + // with a "No type found" error in the log. + // Fix this by adding a dummy type. + types = []Type{&Int{Size: 0}} } // Ensure that types are marshaled in the exact order they were Add()ed. // Otherwise the ID returned from Add() won't match. - e.pending.Grow(len(b.types)) - for _, typ := range b.types { + e.pending.Grow(len(types)) + for _, typ := range types { e.pending.Push(typ) - e.ids[typ] = b.stableIDs[typ] } if err := e.deflatePending(); err != nil { @@ -211,16 +237,28 @@ func (b *Builder) addString(str string) (uint32, error) { return b.strings.Add(str) } -func (e *encoder) allocateID(typ Type) error { - id := e.lastID + 1 - if id < e.lastID { - return errors.New("type ID overflow") - } +func (e *encoder) allocateIDs(root Type) (err error) { + visitInPostorder(root, e.visited, func(typ Type) bool { + if _, ok := typ.(*Void); ok { + return true + } - e.pending.Push(typ) - e.ids[typ] = id - e.lastID = id - return nil + if _, ok := e.ids[typ]; ok { + return true + } + + id := e.lastID + 1 + if id < e.lastID { + err = errors.New("type ID overflow") + return false + } + + e.pending.Push(typ) + e.ids[typ] = id + e.lastID = id + return true + }) + return } // id returns the ID for the given type or panics with an error. @@ -240,33 +278,13 @@ func (e *encoder) id(typ Type) TypeID { func (e *encoder) deflatePending() error { // Declare root outside of the loop to avoid repeated heap allocations. var root Type - skip := func(t Type) (skip bool) { - if t == root { - // Force descending into the current root type even if it already - // has an ID. Otherwise we miss children of types that have their - // ID pre-allocated via Add. - return false - } - - _, isVoid := t.(*Void) - _, alreadyEncoded := e.ids[t] - return isVoid || alreadyEncoded - } for !e.pending.Empty() { root = e.pending.Shift() // Allocate IDs for all children of typ, including transitive dependencies. - iter := postorderTraversal(root, skip) - for iter.Next() { - if iter.Type == root { - // The iterator yields root at the end, do not allocate another ID. - break - } - - if err := e.allocateID(iter.Type); err != nil { - return err - } + if err := e.allocateIDs(root); err != nil { + return err } if err := e.deflateType(root); err != nil { @@ -328,21 +346,13 @@ func (e *encoder) deflateType(typ Type) (err error) { raw.data, err = e.convertMembers(&raw.btfType, v.Members) case *Union: - raw.SetKind(kindUnion) - raw.SetSize(v.Size) - raw.data, err = e.convertMembers(&raw.btfType, v.Members) + err = e.deflateUnion(&raw, v) case *Enum: - raw.SetSize(v.size()) - raw.SetVlen(len(v.Values)) - raw.SetSigned(v.Signed) - - if v.has64BitValues() { - raw.SetKind(kindEnum64) - raw.data, err = e.deflateEnum64Values(v.Values) + if v.Size == 8 { + err = e.deflateEnum64(&raw, v) } else { - raw.SetKind(kindEnum) - raw.data, err = e.deflateEnumValues(v.Values) + err = e.deflateEnum(&raw, v) } case *Fwd: @@ -415,6 +425,13 @@ func (e *encoder) deflateType(typ Type) (err error) { return raw.Marshal(e.buf, e.Order) } +func (e *encoder) deflateUnion(raw *rawType, union *Union) (err error) { + raw.SetKind(kindUnion) + raw.SetSize(union.Size) + raw.data, err = e.convertMembers(&raw.btfType, union.Members) + return +} + func (e *encoder) convertMembers(header *btfType, members []Member) ([]btfMember, error) { bms := make([]btfMember, 0, len(members)) isBitfield := false @@ -443,16 +460,32 @@ func (e *encoder) convertMembers(header *btfType, members []Member) ([]btfMember return bms, nil } -func (e *encoder) deflateEnumValues(values []EnumValue) ([]btfEnum, error) { - bes := make([]btfEnum, 0, len(values)) - for _, value := range values { +func (e *encoder) deflateEnum(raw *rawType, enum *Enum) (err error) { + raw.SetKind(kindEnum) + raw.SetSize(enum.Size) + raw.SetVlen(len(enum.Values)) + // Signedness appeared together with ENUM64 support. + raw.SetSigned(enum.Signed && !e.ReplaceEnum64) + raw.data, err = e.deflateEnumValues(enum) + return +} + +func (e *encoder) deflateEnumValues(enum *Enum) ([]btfEnum, error) { + bes := make([]btfEnum, 0, len(enum.Values)) + for _, value := range enum.Values { nameOff, err := e.strings.Add(value.Name) if err != nil { return nil, err } - if value.Value > math.MaxUint32 { - return nil, fmt.Errorf("value of enum %q exceeds 32 bits", value.Name) + if enum.Signed { + if signedValue := int64(value.Value); signedValue < math.MinInt32 || signedValue > math.MaxInt32 { + return nil, fmt.Errorf("value %d of enum %q exceeds 32 bits", signedValue, value.Name) + } + } else { + if value.Value > math.MaxUint32 { + return nil, fmt.Errorf("value %d of enum %q exceeds 32 bits", value.Value, value.Name) + } } bes = append(bes, btfEnum{ @@ -464,6 +497,41 @@ func (e *encoder) deflateEnumValues(values []EnumValue) ([]btfEnum, error) { return bes, nil } +func (e *encoder) deflateEnum64(raw *rawType, enum *Enum) (err error) { + if e.ReplaceEnum64 { + // Replace the ENUM64 with a union of fields with the correct size. + // This matches libbpf behaviour on purpose. + placeholder := &Int{ + "enum64_placeholder", + enum.Size, + Unsigned, + } + if enum.Signed { + placeholder.Encoding = Signed + } + if err := e.allocateIDs(placeholder); err != nil { + return fmt.Errorf("add enum64 placeholder: %w", err) + } + + members := make([]Member, 0, len(enum.Values)) + for _, v := range enum.Values { + members = append(members, Member{ + Name: v.Name, + Type: placeholder, + }) + } + + return e.deflateUnion(raw, &Union{enum.Name, enum.Size, members}) + } + + raw.SetKind(kindEnum64) + raw.SetSize(enum.Size) + raw.SetVlen(len(enum.Values)) + raw.SetSigned(enum.Signed) + raw.data, err = e.deflateEnum64Values(enum.Values) + return +} + func (e *encoder) deflateEnum64Values(values []EnumValue) ([]btfEnum64, error) { bes := make([]btfEnum64, 0, len(values)) for _, value := range values { diff --git a/vendor/github.com/cilium/ebpf/btf/strings.go b/vendor/github.com/cilium/ebpf/btf/strings.go index bc6aff28142d..7c31461c3064 100644 --- a/vendor/github.com/cilium/ebpf/btf/strings.go +++ b/vendor/github.com/cilium/ebpf/btf/strings.go @@ -6,14 +6,15 @@ import ( "errors" "fmt" "io" + "maps" + "slices" "strings" - - "golang.org/x/exp/maps" ) type stringTable struct { base *stringTable offsets []uint32 + prevIdx int strings []string } @@ -60,7 +61,7 @@ func readStringTable(r sizedReader, base *stringTable) (*stringTable, error) { return nil, errors.New("first item in string table is non-empty") } - return &stringTable{base, offsets, strings}, nil + return &stringTable{base, offsets, 0, strings}, nil } func splitNull(data []byte, atEOF bool) (advance int, token []byte, err error) { @@ -83,26 +84,29 @@ func (st *stringTable) Lookup(offset uint32) (string, error) { } func (st *stringTable) lookup(offset uint32) (string, error) { - i := search(st.offsets, offset) - if i == len(st.offsets) || st.offsets[i] != offset { - return "", fmt.Errorf("offset %d isn't start of a string", offset) + // Fast path: zero offset is the empty string, looked up frequently. + if offset == 0 && st.base == nil { + return "", nil } - return st.strings[i], nil -} + // Accesses tend to be globally increasing, so check if the next string is + // the one we want. This skips the binary search in about 50% of cases. + if st.prevIdx+1 < len(st.offsets) && st.offsets[st.prevIdx+1] == offset { + st.prevIdx++ + return st.strings[st.prevIdx], nil + } -func (st *stringTable) Marshal(w io.Writer) error { - for _, str := range st.strings { - _, err := io.WriteString(w, str) - if err != nil { - return err - } - _, err = w.Write([]byte{0}) - if err != nil { - return err - } + i, found := slices.BinarySearch(st.offsets, offset) + if !found { + return "", fmt.Errorf("offset %d isn't start of a string", offset) } - return nil + + // Set the new increment index, but only if its greater than the current. + if i > st.prevIdx+1 { + st.prevIdx = i + } + + return st.strings[i], nil } // Num returns the number of strings in the table. @@ -110,26 +114,6 @@ func (st *stringTable) Num() int { return len(st.strings) } -// search is a copy of sort.Search specialised for uint32. -// -// Licensed under https://go.dev/LICENSE -func search(ints []uint32, needle uint32) int { - // Define f(-1) == false and f(n) == true. - // Invariant: f(i-1) == false, f(j) == true. - i, j := 0, len(ints) - for i < j { - h := int(uint(i+j) >> 1) // avoid overflow when computing h - // i ≤ h < j - if !(ints[h] >= needle) { - i = h + 1 // preserves f(i-1) == false - } else { - j = h // preserves f(j) == true - } - } - // i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i. - return i -} - // stringTableBuilder builds BTF string tables. type stringTableBuilder struct { length uint32 diff --git a/vendor/github.com/cilium/ebpf/btf/traversal.go b/vendor/github.com/cilium/ebpf/btf/traversal.go index a3a9dec940a9..c39dc66e46ce 100644 --- a/vendor/github.com/cilium/ebpf/btf/traversal.go +++ b/vendor/github.com/cilium/ebpf/btf/traversal.go @@ -2,93 +2,41 @@ package btf import ( "fmt" - - "github.com/cilium/ebpf/internal" ) // Functions to traverse a cyclic graph of types. The below was very useful: // https://eli.thegreenplace.net/2015/directed-graph-traversal-orderings-and-applications-to-data-flow-analysis/#post-order-and-reverse-post-order -type postorderIterator struct { - // Iteration skips types for which this function returns true. - skip func(Type) bool - // The root type. May be nil if skip(root) is true. - root Type - - // Contains types which need to be either walked or yielded. - types typeDeque - // Contains a boolean whether the type has been walked or not. - walked internal.Deque[bool] - // The set of types which has been pushed onto types. - pushed map[Type]struct{} - - // The current type. Only valid after a call to Next(). - Type Type -} - -// postorderTraversal iterates all types reachable from root by visiting the -// leaves of the graph first. +// Visit all types reachable from root in postorder. // -// Types for which skip returns true are ignored. skip may be nil. -func postorderTraversal(root Type, skip func(Type) (skip bool)) postorderIterator { - // Avoid allocations for the common case of a skipped root. - if skip != nil && skip(root) { - return postorderIterator{} - } - - po := postorderIterator{root: root, skip: skip} - walkType(root, po.push) - - return po -} - -func (po *postorderIterator) push(t *Type) { - if _, ok := po.pushed[*t]; ok || *t == po.root { - return - } - - if po.skip != nil && po.skip(*t) { - return +// Traversal stops if yield returns false. +// +// Returns false if traversal was aborted. +func visitInPostorder(root Type, visited map[Type]struct{}, yield func(typ Type) bool) bool { + if _, ok := visited[root]; ok { + return true } - - if po.pushed == nil { - // Lazily allocate pushed to avoid an allocation for Types without children. - po.pushed = make(map[Type]struct{}) + if visited == nil { + visited = make(map[Type]struct{}) } + visited[root] = struct{}{} - po.pushed[*t] = struct{}{} - po.types.Push(t) - po.walked.Push(false) -} - -// Next returns true if there is another Type to traverse. -func (po *postorderIterator) Next() bool { - for !po.types.Empty() { - t := po.types.Pop() - - if !po.walked.Pop() { - // Push the type again, so that we re-evaluate it in done state - // after all children have been handled. - po.types.Push(t) - po.walked.Push(true) - - // Add all direct children to todo. - walkType(*t, po.push) - } else { - // We've walked this type previously, so we now know that all - // children have been handled. - po.Type = *t - return true - } + cont := children(root, func(child *Type) bool { + return visitInPostorder(*child, visited, yield) + }) + if !cont { + return false } - // Only return root once. - po.Type, po.root = po.root, nil - return po.Type != nil + return yield(root) } -// walkType calls fn on each child of typ. -func walkType(typ Type, fn func(*Type)) { +// children calls yield on each child of typ. +// +// Traversal stops if yield returns false. +// +// Returns false if traversal was aborted. +func children(typ Type, yield func(child *Type) bool) bool { // Explicitly type switch on the most common types to allow the inliner to // do its work. This avoids allocating intermediate slices from walk() on // the heap. @@ -96,46 +44,80 @@ func walkType(typ Type, fn func(*Type)) { case *Void, *Int, *Enum, *Fwd, *Float: // No children to traverse. case *Pointer: - fn(&v.Target) + if !yield(&v.Target) { + return false + } case *Array: - fn(&v.Index) - fn(&v.Type) + if !yield(&v.Index) { + return false + } + if !yield(&v.Type) { + return false + } case *Struct: for i := range v.Members { - fn(&v.Members[i].Type) + if !yield(&v.Members[i].Type) { + return false + } } case *Union: for i := range v.Members { - fn(&v.Members[i].Type) + if !yield(&v.Members[i].Type) { + return false + } } case *Typedef: - fn(&v.Type) + if !yield(&v.Type) { + return false + } case *Volatile: - fn(&v.Type) + if !yield(&v.Type) { + return false + } case *Const: - fn(&v.Type) + if !yield(&v.Type) { + return false + } case *Restrict: - fn(&v.Type) + if !yield(&v.Type) { + return false + } case *Func: - fn(&v.Type) + if !yield(&v.Type) { + return false + } case *FuncProto: - fn(&v.Return) + if !yield(&v.Return) { + return false + } for i := range v.Params { - fn(&v.Params[i].Type) + if !yield(&v.Params[i].Type) { + return false + } } case *Var: - fn(&v.Type) + if !yield(&v.Type) { + return false + } case *Datasec: for i := range v.Vars { - fn(&v.Vars[i].Type) + if !yield(&v.Vars[i].Type) { + return false + } } case *declTag: - fn(&v.Type) + if !yield(&v.Type) { + return false + } case *typeTag: - fn(&v.Type) + if !yield(&v.Type) { + return false + } case *cycle: // cycle has children, but we ignore them deliberately. default: panic(fmt.Sprintf("don't know how to walk Type %T", v)) } + + return true } diff --git a/vendor/github.com/cilium/ebpf/btf/types.go b/vendor/github.com/cilium/ebpf/btf/types.go index 68d4a1757166..a3397460b9d5 100644 --- a/vendor/github.com/cilium/ebpf/btf/types.go +++ b/vendor/github.com/cilium/ebpf/btf/types.go @@ -1,11 +1,12 @@ package btf import ( + "encoding/binary" "errors" "fmt" "io" "math" - "reflect" + "slices" "strings" "github.com/cilium/ebpf/asm" @@ -13,7 +14,9 @@ import ( "github.com/cilium/ebpf/internal/sys" ) -const maxTypeDepth = 32 +// Mirrors MAX_RESOLVE_DEPTH in libbpf. +// https://github.com/libbpf/libbpf/blob/e26b84dc330c9644c07428c271ab491b0f01f4e1/src/btf.c#L761 +const maxResolveDepth = 32 // TypeID identifies a type in a BTF section. type TypeID = sys.TypeID @@ -116,7 +119,7 @@ type Int struct { } func (i *Int) Format(fs fmt.State, verb rune) { - formatType(fs, verb, i, i.Encoding, "size=", i.Size*8) + formatType(fs, verb, i, i.Encoding, "size=", i.Size) } func (i *Int) TypeName() string { return i.Name } @@ -278,21 +281,6 @@ func (e *Enum) copy() Type { return &cpy } -// has64BitValues returns true if the Enum contains a value larger than 32 bits. -// Kernels before 6.0 have enum values that overrun u32 replaced with zeroes. -// -// 64-bit enums have their Enum.Size attributes correctly set to 8, but if we -// use the size attribute as a heuristic during BTF marshaling, we'll emit -// ENUM64s to kernels that don't support them. -func (e *Enum) has64BitValues() bool { - for _, v := range e.Values { - if v.Value > math.MaxUint32 { - return true - } - } - return false -} - // FwdKind is the type of forward declaration. type FwdKind int @@ -330,6 +318,18 @@ func (f *Fwd) copy() Type { return &cpy } +func (f *Fwd) matches(typ Type) bool { + if _, ok := As[*Struct](typ); ok && f.Kind == FwdStruct { + return true + } + + if _, ok := As[*Union](typ); ok && f.Kind == FwdUnion { + return true + } + + return false +} + // Typedef is an alias of a Type. type Typedef struct { Name string @@ -605,7 +605,7 @@ func Sizeof(typ Type) (int, error) { elem int64 ) - for i := 0; i < maxTypeDepth; i++ { + for i := 0; i < maxResolveDepth; i++ { switch v := typ.(type) { case *Array: if n > 0 && int64(v.Nelems) > math.MaxInt64/n { @@ -667,90 +667,64 @@ func alignof(typ Type) (int, error) { return 0, fmt.Errorf("can't calculate alignment of %T", t) } - if !pow(n) { + if !internal.IsPow(n) { return 0, fmt.Errorf("alignment value %d is not a power of two", n) } return n, nil } -// pow returns true if n is a power of two. -func pow(n int) bool { - return n != 0 && (n&(n-1)) == 0 -} - -// Transformer modifies a given Type and returns the result. -// -// For example, UnderlyingType removes any qualifiers or typedefs from a type. -// See the example on Copy for how to use a transform. -type Transformer func(Type) Type - // Copy a Type recursively. // -// typ may form a cycle. If transform is not nil, it is called with the -// to be copied type, and the returned value is copied instead. -func Copy(typ Type, transform Transformer) Type { - copies := copier{copies: make(map[Type]Type)} - copies.copy(&typ, transform) - return typ +// typ may form a cycle. +func Copy(typ Type) Type { + return copyType(typ, nil, make(map[Type]Type), nil) } -// copy a slice of Types recursively. -// -// See Copy for the semantics. -func copyTypes(types []Type, transform Transformer) []Type { - result := make([]Type, len(types)) - copy(result, types) - - copies := copier{copies: make(map[Type]Type, len(types))} - for i := range result { - copies.copy(&result[i], transform) +func copyType(typ Type, ids map[Type]TypeID, copies map[Type]Type, copiedIDs map[Type]TypeID) Type { + if typ == nil { + return nil } - return result -} - -type copier struct { - copies map[Type]Type - work typeDeque -} + cpy, ok := copies[typ] + if ok { + // This has been copied previously, no need to continue. + return cpy + } -func (c *copier) copy(typ *Type, transform Transformer) { - for t := typ; t != nil; t = c.work.Pop() { - // *t is the identity of the type. - if cpy := c.copies[*t]; cpy != nil { - *t = cpy - continue - } + cpy = typ.copy() + copies[typ] = cpy - var cpy Type - if transform != nil { - cpy = transform(*t).copy() - } else { - cpy = (*t).copy() - } + if id, ok := ids[typ]; ok { + copiedIDs[cpy] = id + } - c.copies[*t] = cpy - *t = cpy + children(cpy, func(child *Type) bool { + *child = copyType(*child, ids, copies, copiedIDs) + return true + }) - // Mark any nested types for copying. - walkType(cpy, c.work.Push) - } + return cpy } type typeDeque = internal.Deque[*Type] -// inflateRawTypes takes a list of raw btf types linked via type IDs, and turns -// it into a graph of Types connected via pointers. +// readAndInflateTypes reads the raw btf type info and turns it into a graph +// of Types connected via pointers. // -// If base is provided, then the raw types are considered to be of a split BTF +// If base is provided, then the types are considered to be of a split BTF // (e.g., a kernel module). // // Returns a slice of types indexed by TypeID. Since BTF ignores compilation // units, multiple types may share the same name. A Type may form a cyclic graph // by pointing at itself. -func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([]Type, error) { - types := make([]Type, 0, len(rawTypes)+1) // +1 for Void added to base types +func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawStrings *stringTable, base *Spec) ([]Type, error) { + // because of the interleaving between types and struct members it is difficult to + // precompute the numbers of raw types this will parse + // this "guess" is a good first estimation + sizeOfbtfType := uintptr(btfTypeLen) + tyMaxCount := uintptr(typeLen) / sizeOfbtfType / 2 + types := make([]Type, 0, tyMaxCount) // Void is defined to always be type ID 0, and is thus omitted from BTF. types = append(types, (*Void)(nil)) @@ -773,11 +747,11 @@ func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([ } var fixups []fixupDef - fixup := func(id TypeID, typ *Type) bool { + fixup := func(id TypeID, typ *Type) { if id < firstTypeID { if baseType, err := base.TypeByID(id); err == nil { *typ = baseType - return true + return } } @@ -785,31 +759,10 @@ func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([ if idx < len(types) { // We've already inflated this type, fix it up immediately. *typ = types[idx] - return true + return } fixups = append(fixups, fixupDef{id, typ}) - return false - } - - type assertion struct { - id TypeID - typ *Type - want reflect.Type - } - - var assertions []assertion - fixupAndAssert := func(id TypeID, typ *Type, want reflect.Type) error { - if !fixup(id, typ) { - assertions = append(assertions, assertion{id, typ, want}) - return nil - } - - // The type has already been fixed up, check the type immediately. - if reflect.TypeOf(*typ) != want { - return fmt.Errorf("type ID %d: expected %s, got %T", id, want, *typ) - } - return nil } type bitfieldFixupDef struct { @@ -876,62 +829,128 @@ func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([ return members, nil } + var ( + buf = make([]byte, 1024) + header btfType + bInt btfInt + bArr btfArray + bMembers []btfMember + bEnums []btfEnum + bParams []btfParam + bVariable btfVariable + bSecInfos []btfVarSecinfo + bDeclTag btfDeclTag + bEnums64 []btfEnum64 + ) + var declTags []*declTag - for _, raw := range rawTypes { + for { var ( id = firstTypeID + TypeID(len(types)) typ Type ) + if _, err := io.ReadFull(r, buf[:btfTypeLen]); err == io.EOF { + break + } else if err != nil { + return nil, fmt.Errorf("can't read type info for id %v: %v", id, err) + } + + if _, err := unmarshalBtfType(&header, buf[:btfTypeLen], bo); err != nil { + return nil, fmt.Errorf("can't unmarshal type info for id %v: %v", id, err) + } + if id < firstTypeID { return nil, fmt.Errorf("no more type IDs") } - name, err := rawStrings.Lookup(raw.NameOff) + name, err := rawStrings.Lookup(header.NameOff) if err != nil { return nil, fmt.Errorf("get name for type id %d: %w", id, err) } - switch raw.Kind() { + switch header.Kind() { case kindInt: - size := raw.Size() - bi := raw.data.(*btfInt) - if bi.Offset() > 0 || bi.Bits().Bytes() != size { - legacyBitfields[id] = [2]Bits{bi.Offset(), bi.Bits()} + size := header.Size() + buf = buf[:btfIntLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfInt, id: %d: %w", id, err) + } + if _, err := unmarshalBtfInt(&bInt, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfInt, id: %d: %w", id, err) } - typ = &Int{name, raw.Size(), bi.Encoding()} + if bInt.Offset() > 0 || bInt.Bits().Bytes() != size { + legacyBitfields[id] = [2]Bits{bInt.Offset(), bInt.Bits()} + } + typ = &Int{name, header.Size(), bInt.Encoding()} case kindPointer: ptr := &Pointer{nil} - fixup(raw.Type(), &ptr.Target) + fixup(header.Type(), &ptr.Target) typ = ptr case kindArray: - btfArr := raw.data.(*btfArray) - arr := &Array{nil, nil, btfArr.Nelems} - fixup(btfArr.IndexType, &arr.Index) - fixup(btfArr.Type, &arr.Type) + buf = buf[:btfArrayLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfArray, id: %d: %w", id, err) + } + if _, err := unmarshalBtfArray(&bArr, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfArray, id: %d: %w", id, err) + } + + arr := &Array{nil, nil, bArr.Nelems} + fixup(bArr.IndexType, &arr.Index) + fixup(bArr.Type, &arr.Type) typ = arr case kindStruct: - members, err := convertMembers(raw.data.([]btfMember), raw.Bitfield()) + vlen := header.Vlen() + bMembers = slices.Grow(bMembers[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfMemberLen)[:vlen*btfMemberLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfMembers, id: %d: %w", id, err) + } + if _, err := unmarshalBtfMembers(bMembers, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfMembers, id: %d: %w", id, err) + } + + members, err := convertMembers(bMembers, header.Bitfield()) if err != nil { return nil, fmt.Errorf("struct %s (id %d): %w", name, id, err) } - typ = &Struct{name, raw.Size(), members} + typ = &Struct{name, header.Size(), members} case kindUnion: - members, err := convertMembers(raw.data.([]btfMember), raw.Bitfield()) + vlen := header.Vlen() + bMembers = slices.Grow(bMembers[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfMemberLen)[:vlen*btfMemberLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfMembers, id: %d: %w", id, err) + } + if _, err := unmarshalBtfMembers(bMembers, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfMembers, id: %d: %w", id, err) + } + + members, err := convertMembers(bMembers, header.Bitfield()) if err != nil { return nil, fmt.Errorf("union %s (id %d): %w", name, id, err) } - typ = &Union{name, raw.Size(), members} + typ = &Union{name, header.Size(), members} case kindEnum: - rawvals := raw.data.([]btfEnum) - vals := make([]EnumValue, 0, len(rawvals)) - signed := raw.Signed() - for i, btfVal := range rawvals { + vlen := header.Vlen() + bEnums = slices.Grow(bEnums[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfEnumLen)[:vlen*btfEnumLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfEnums, id: %d: %w", id, err) + } + if _, err := unmarshalBtfEnums(bEnums, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfEnums, id: %d: %w", id, err) + } + + vals := make([]EnumValue, 0, vlen) + signed := header.Signed() + for i, btfVal := range bEnums { name, err := rawStrings.Lookup(btfVal.NameOff) if err != nil { return nil, fmt.Errorf("get name for enum value %d: %s", i, err) @@ -943,42 +962,49 @@ func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([ } vals = append(vals, EnumValue{name, value}) } - typ = &Enum{name, raw.Size(), signed, vals} + typ = &Enum{name, header.Size(), signed, vals} case kindForward: - typ = &Fwd{name, raw.FwdKind()} + typ = &Fwd{name, header.FwdKind()} case kindTypedef: typedef := &Typedef{name, nil} - fixup(raw.Type(), &typedef.Type) + fixup(header.Type(), &typedef.Type) typ = typedef case kindVolatile: volatile := &Volatile{nil} - fixup(raw.Type(), &volatile.Type) + fixup(header.Type(), &volatile.Type) typ = volatile case kindConst: cnst := &Const{nil} - fixup(raw.Type(), &cnst.Type) + fixup(header.Type(), &cnst.Type) typ = cnst case kindRestrict: restrict := &Restrict{nil} - fixup(raw.Type(), &restrict.Type) + fixup(header.Type(), &restrict.Type) typ = restrict case kindFunc: - fn := &Func{name, nil, raw.Linkage()} - if err := fixupAndAssert(raw.Type(), &fn.Type, reflect.TypeOf((*FuncProto)(nil))); err != nil { - return nil, err - } + fn := &Func{name, nil, header.Linkage()} + fixup(header.Type(), &fn.Type) typ = fn case kindFuncProto: - rawparams := raw.data.([]btfParam) - params := make([]FuncParam, 0, len(rawparams)) - for i, param := range rawparams { + vlen := header.Vlen() + bParams = slices.Grow(bParams[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfParamLen)[:vlen*btfParamLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfParams, id: %d: %w", id, err) + } + if _, err := unmarshalBtfParams(bParams, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfParams, id: %d: %w", id, err) + } + + params := make([]FuncParam, 0, vlen) + for i, param := range bParams { name, err := rawStrings.Lookup(param.NameOff) if err != nil { return nil, fmt.Errorf("get name for func proto parameter %d: %s", i, err) @@ -988,57 +1014,90 @@ func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([ }) } for i := range params { - fixup(rawparams[i].Type, ¶ms[i].Type) + fixup(bParams[i].Type, ¶ms[i].Type) } fp := &FuncProto{nil, params} - fixup(raw.Type(), &fp.Return) + fixup(header.Type(), &fp.Return) typ = fp case kindVar: - variable := raw.data.(*btfVariable) - v := &Var{name, nil, VarLinkage(variable.Linkage)} - fixup(raw.Type(), &v.Type) + buf = buf[:btfVariableLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfVariable, id: %d: %w", id, err) + } + if _, err := unmarshalBtfVariable(&bVariable, buf, bo); err != nil { + return nil, fmt.Errorf("can't read btfVariable, id: %d: %w", id, err) + } + + v := &Var{name, nil, VarLinkage(bVariable.Linkage)} + fixup(header.Type(), &v.Type) typ = v case kindDatasec: - btfVars := raw.data.([]btfVarSecinfo) - vars := make([]VarSecinfo, 0, len(btfVars)) - for _, btfVar := range btfVars { + vlen := header.Vlen() + bSecInfos = slices.Grow(bSecInfos[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfVarSecinfoLen)[:vlen*btfVarSecinfoLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfVarSecInfos, id: %d: %w", id, err) + } + if _, err := unmarshalBtfVarSecInfos(bSecInfos, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfVarSecInfos, id: %d: %w", id, err) + } + + vars := make([]VarSecinfo, 0, vlen) + for _, btfVar := range bSecInfos { vars = append(vars, VarSecinfo{ Offset: btfVar.Offset, Size: btfVar.Size, }) } for i := range vars { - fixup(btfVars[i].Type, &vars[i].Type) + fixup(bSecInfos[i].Type, &vars[i].Type) } - typ = &Datasec{name, raw.Size(), vars} + typ = &Datasec{name, header.Size(), vars} case kindFloat: - typ = &Float{name, raw.Size()} + typ = &Float{name, header.Size()} case kindDeclTag: - btfIndex := raw.data.(*btfDeclTag).ComponentIdx + buf = buf[:btfDeclTagLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfDeclTag, id: %d: %w", id, err) + } + if _, err := unmarshalBtfDeclTag(&bDeclTag, buf, bo); err != nil { + return nil, fmt.Errorf("can't read btfDeclTag, id: %d: %w", id, err) + } + + btfIndex := bDeclTag.ComponentIdx if uint64(btfIndex) > math.MaxInt { return nil, fmt.Errorf("type id %d: index exceeds int", id) } dt := &declTag{nil, name, int(int32(btfIndex))} - fixup(raw.Type(), &dt.Type) + fixup(header.Type(), &dt.Type) typ = dt declTags = append(declTags, dt) case kindTypeTag: tt := &typeTag{nil, name} - fixup(raw.Type(), &tt.Type) + fixup(header.Type(), &tt.Type) typ = tt case kindEnum64: - rawvals := raw.data.([]btfEnum64) - vals := make([]EnumValue, 0, len(rawvals)) - for i, btfVal := range rawvals { + vlen := header.Vlen() + bEnums64 = slices.Grow(bEnums64[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfEnum64Len)[:vlen*btfEnum64Len] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfEnum64s, id: %d: %w", id, err) + } + if _, err := unmarshalBtfEnums64(bEnums64, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfEnum64s, id: %d: %w", id, err) + } + + vals := make([]EnumValue, 0, vlen) + for i, btfVal := range bEnums64 { name, err := rawStrings.Lookup(btfVal.NameOff) if err != nil { return nil, fmt.Errorf("get name for enum64 value %d: %s", i, err) @@ -1046,10 +1105,10 @@ func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([ value := (uint64(btfVal.ValHi32) << 32) | uint64(btfVal.ValLo32) vals = append(vals, EnumValue{name, value}) } - typ = &Enum{name, raw.Size(), raw.Signed(), vals} + typ = &Enum{name, header.Size(), header.Signed(), vals} default: - return nil, fmt.Errorf("type id %d: unknown kind: %v", id, raw.Kind()) + return nil, fmt.Errorf("type id %d: unknown kind: %v", id, header.Kind()) } types = append(types, typ) @@ -1081,12 +1140,6 @@ func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([ } } - for _, assertion := range assertions { - if reflect.TypeOf(*assertion.typ) != assertion.want { - return nil, fmt.Errorf("type ID %d: expected %s, got %T", assertion.id, assertion.want, *assertion.typ) - } - } - for _, dt := range declTags { switch t := dt.Type.(type) { case *Var, *Typedef: @@ -1100,7 +1153,12 @@ func inflateRawTypes(rawTypes []rawType, rawStrings *stringTable, base *Spec) ([ } case *Func: - if dt.Index >= len(t.Type.(*FuncProto).Params) { + fp, ok := t.Type.(*FuncProto) + if !ok { + return nil, fmt.Errorf("type %s: %s is not a FuncProto", dt, t.Type) + } + + if dt.Index >= len(fp.Params) { return nil, fmt.Errorf("type %s: index %d exceeds params of %s", dt, dt.Index, t) } @@ -1136,7 +1194,7 @@ func newEssentialName(name string) essentialName { // UnderlyingType skips qualifiers and Typedefs. func UnderlyingType(typ Type) Type { result := typ - for depth := 0; depth <= maxTypeDepth; depth++ { + for depth := 0; depth <= maxResolveDepth; depth++ { switch v := (result).(type) { case qualifier: result = v.qualify() @@ -1149,13 +1207,16 @@ func UnderlyingType(typ Type) Type { return &cycle{typ} } -// as returns typ if is of type T. Otherwise it peels qualifiers and Typedefs +// As returns typ if is of type T. Otherwise it peels qualifiers and Typedefs // until it finds a T. // // Returns the zero value and false if there is no T or if the type is nested // too deeply. -func as[T Type](typ Type) (T, bool) { - for depth := 0; depth <= maxTypeDepth; depth++ { +func As[T Type](typ Type) (T, bool) { + // NB: We can't make this function return (*T) since then + // we can't assert that a type matches an interface which + // embeds Type: as[composite](T). + for depth := 0; depth <= maxResolveDepth; depth++ { switch v := (typ).(type) { case T: return v, true diff --git a/vendor/github.com/cilium/ebpf/collection.go b/vendor/github.com/cilium/ebpf/collection.go index fb720bebdb7b..b2cb214adce0 100644 --- a/vendor/github.com/cilium/ebpf/collection.go +++ b/vendor/github.com/cilium/ebpf/collection.go @@ -11,6 +11,7 @@ import ( "github.com/cilium/ebpf/btf" "github.com/cilium/ebpf/internal" "github.com/cilium/ebpf/internal/kconfig" + "github.com/cilium/ebpf/internal/sysenc" ) // CollectionOptions control loading a collection into the kernel. @@ -56,7 +57,7 @@ func (cs *CollectionSpec) Copy() *CollectionSpec { Maps: make(map[string]*MapSpec, len(cs.Maps)), Programs: make(map[string]*ProgramSpec, len(cs.Programs)), ByteOrder: cs.ByteOrder, - Types: cs.Types, + Types: cs.Types.Copy(), } for name, spec := range cs.Maps { @@ -175,12 +176,12 @@ func (cs *CollectionSpec) RewriteConstants(consts map[string]interface{}) error return fmt.Errorf("section %s: offset %d(+%d) for variable %s is out of bounds", name, v.Offset, v.Size, vname) } - b, err := marshalBytes(replacement, int(v.Size)) + b, err := sysenc.Marshal(replacement, int(v.Size)) if err != nil { return fmt.Errorf("marshaling constant replacement %s: %w", vname, err) } - copy(cpy[v.Offset:v.Offset+v.Size], b) + b.CopyTo(cpy[v.Offset : v.Offset+v.Size]) replaced[vname] = true } @@ -308,7 +309,7 @@ func (cs *CollectionSpec) LoadAndAssign(to interface{}, opts *CollectionOptions) } // Populate the requested maps. Has a chance of lazy-loading other dependent maps. - if err := loader.populateMaps(); err != nil { + if err := loader.populateDeferredMaps(); err != nil { return err } @@ -388,7 +389,7 @@ func NewCollectionWithOptions(spec *CollectionSpec, opts CollectionOptions) (*Co // Maps can contain Program and Map stubs, so populate them after // all Maps and Programs have been successfully loaded. - if err := loader.populateMaps(); err != nil { + if err := loader.populateDeferredMaps(); err != nil { return nil, err } @@ -470,6 +471,15 @@ func (cl *collectionLoader) loadMap(mapName string) (*Map, error) { return nil, fmt.Errorf("map %s: %w", mapName, err) } + // Finalize 'scalar' maps that don't refer to any other eBPF resources + // potentially pending creation. This is needed for frozen maps like .rodata + // that need to be finalized before invoking the verifier. + if !mapSpec.Type.canStoreMapOrProgram() { + if err := m.finalize(mapSpec); err != nil { + return nil, fmt.Errorf("finalizing map %s: %w", mapName, err) + } + } + cl.maps[mapName] = m return m, nil } @@ -527,44 +537,50 @@ func (cl *collectionLoader) loadProgram(progName string) (*Program, error) { return prog, nil } -func (cl *collectionLoader) populateMaps() error { +// populateDeferredMaps iterates maps holding programs or other maps and loads +// any dependencies. Populates all maps in cl and freezes them if specified. +func (cl *collectionLoader) populateDeferredMaps() error { for mapName, m := range cl.maps { mapSpec, ok := cl.coll.Maps[mapName] if !ok { return fmt.Errorf("missing map spec %s", mapName) } + // Scalar maps without Map or Program references are finalized during + // creation. Don't finalize them again. + if !mapSpec.Type.canStoreMapOrProgram() { + continue + } + + mapSpec = mapSpec.Copy() + // MapSpecs that refer to inner maps or programs within the same // CollectionSpec do so using strings. These strings are used as the key // to look up the respective object in the Maps or Programs fields. // Resolve those references to actual Map or Program resources that // have been loaded into the kernel. - if mapSpec.Type.canStoreMap() || mapSpec.Type.canStoreProgram() { - mapSpec = mapSpec.Copy() + for i, kv := range mapSpec.Contents { + objName, ok := kv.Value.(string) + if !ok { + continue + } - for i, kv := range mapSpec.Contents { - objName, ok := kv.Value.(string) - if !ok { - continue + switch t := mapSpec.Type; { + case t.canStoreProgram(): + // loadProgram is idempotent and could return an existing Program. + prog, err := cl.loadProgram(objName) + if err != nil { + return fmt.Errorf("loading program %s, for map %s: %w", objName, mapName, err) } + mapSpec.Contents[i] = MapKV{kv.Key, prog} - switch t := mapSpec.Type; { - case t.canStoreProgram(): - // loadProgram is idempotent and could return an existing Program. - prog, err := cl.loadProgram(objName) - if err != nil { - return fmt.Errorf("loading program %s, for map %s: %w", objName, mapName, err) - } - mapSpec.Contents[i] = MapKV{kv.Key, prog} - - case t.canStoreMap(): - // loadMap is idempotent and could return an existing Map. - innerMap, err := cl.loadMap(objName) - if err != nil { - return fmt.Errorf("loading inner map %s, for map %s: %w", objName, mapName, err) - } - mapSpec.Contents[i] = MapKV{kv.Key, innerMap} + case t.canStoreMap(): + // loadMap is idempotent and could return an existing Map. + innerMap, err := cl.loadMap(objName) + if err != nil { + return fmt.Errorf("loading inner map %s, for map %s: %w", objName, mapName, err) } + mapSpec.Contents[i] = MapKV{kv.Key, innerMap} } } @@ -610,17 +626,20 @@ func resolveKconfig(m *MapSpec) error { internal.NativeEndian.PutUint32(data[vsi.Offset:], kv.Kernel()) case "LINUX_HAS_SYSCALL_WRAPPER": - if integer, ok := v.Type.(*btf.Int); !ok || integer.Size != 4 { - return fmt.Errorf("variable %s must be a 32 bits integer, got %s", n, v.Type) + integer, ok := v.Type.(*btf.Int) + if !ok { + return fmt.Errorf("variable %s must be an integer, got %s", n, v.Type) } - var value uint32 = 1 + var value uint64 = 1 if err := haveSyscallWrapper(); errors.Is(err, ErrNotSupported) { value = 0 } else if err != nil { return fmt.Errorf("unable to derive a value for LINUX_HAS_SYSCALL_WRAPPER: %w", err) } - internal.NativeEndian.PutUint32(data[vsi.Offset:], value) + if err := kconfig.PutInteger(data[vsi.Offset:], integer, value); err != nil { + return fmt.Errorf("set LINUX_HAS_SYSCALL_WRAPPER: %w", err) + } default: // Catch CONFIG_*. configs[n] = configInfo{ @@ -679,6 +698,71 @@ func LoadCollection(file string) (*Collection, error) { return NewCollection(spec) } +// Assign the contents of a Collection to a struct. +// +// This function bridges functionality between bpf2go generated +// code and any functionality better implemented in Collection. +// +// 'to' must be a pointer to a struct. A field of the +// struct is updated with values from Programs or Maps if it +// has an `ebpf` tag and its type is *Program or *Map. +// The tag's value specifies the name of the program or map as +// found in the CollectionSpec. +// +// struct { +// Foo *ebpf.Program `ebpf:"xdp_foo"` +// Bar *ebpf.Map `ebpf:"bar_map"` +// Ignored int +// } +// +// Returns an error if any of the eBPF objects can't be found, or +// if the same Map or Program is assigned multiple times. +// +// Ownership and Close()ing responsibility is transferred to `to` +// for any successful assigns. On error `to` is left in an undefined state. +func (coll *Collection) Assign(to interface{}) error { + assignedMaps := make(map[string]bool) + assignedProgs := make(map[string]bool) + + // Assign() only transfers already-loaded Maps and Programs. No extra + // loading is done. + getValue := func(typ reflect.Type, name string) (interface{}, error) { + switch typ { + + case reflect.TypeOf((*Program)(nil)): + if p := coll.Programs[name]; p != nil { + assignedProgs[name] = true + return p, nil + } + return nil, fmt.Errorf("missing program %q", name) + + case reflect.TypeOf((*Map)(nil)): + if m := coll.Maps[name]; m != nil { + assignedMaps[name] = true + return m, nil + } + return nil, fmt.Errorf("missing map %q", name) + + default: + return nil, fmt.Errorf("unsupported type %s", typ) + } + } + + if err := assignValues(to, getValue); err != nil { + return err + } + + // Finalize ownership transfer + for p := range assignedProgs { + delete(coll.Programs, p) + } + for m := range assignedMaps { + delete(coll.Maps, m) + } + + return nil +} + // Close frees all maps and programs associated with the collection. // // The collection mustn't be used afterwards. diff --git a/vendor/github.com/cilium/ebpf/internal/cpu.go b/vendor/github.com/cilium/ebpf/cpu.go similarity index 72% rename from vendor/github.com/cilium/ebpf/internal/cpu.go rename to vendor/github.com/cilium/ebpf/cpu.go index 9e908b610b5f..07e959efdcb2 100644 --- a/vendor/github.com/cilium/ebpf/internal/cpu.go +++ b/vendor/github.com/cilium/ebpf/cpu.go @@ -1,17 +1,32 @@ -package internal +package ebpf import ( "fmt" "os" "strings" + "sync" ) -// PossibleCPUs returns the max number of CPUs a system may possibly have -// Logical CPU numbers must be of the form 0-n -var PossibleCPUs = Memoize(func() (int, error) { +var possibleCPU = sync.OnceValues(func() (int, error) { return parseCPUsFromFile("/sys/devices/system/cpu/possible") }) +// PossibleCPU returns the max number of CPUs a system may possibly have +// Logical CPU numbers must be of the form 0-n +func PossibleCPU() (int, error) { + return possibleCPU() +} + +// MustPossibleCPU is a helper that wraps a call to PossibleCPU and panics if +// the error is non-nil. +func MustPossibleCPU() int { + cpus, err := PossibleCPU() + if err != nil { + panic(err) + } + return cpus +} + func parseCPUsFromFile(path string) (int, error) { spec, err := os.ReadFile(path) if err != nil { diff --git a/vendor/github.com/cilium/ebpf/elf_reader.go b/vendor/github.com/cilium/ebpf/elf_reader.go index 8d92672eb144..620037d80a8a 100644 --- a/vendor/github.com/cilium/ebpf/elf_reader.go +++ b/vendor/github.com/cilium/ebpf/elf_reader.go @@ -15,6 +15,7 @@ import ( "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/btf" "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/sys" "github.com/cilium/ebpf/internal/unix" ) @@ -25,7 +26,12 @@ type kconfigMeta struct { Offset uint32 } -type kfuncMeta struct{} +type kfuncMetaKey struct{} + +type kfuncMeta struct { + Binding elf.SymBind + Func *btf.Func +} // elfCode is a convenience to reduce the amount of arguments that have to // be passed around explicitly. You should treat its contents as immutable. @@ -81,6 +87,8 @@ func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) { // Collect all the sections we're interested in. This includes relocations // which we parse later. + // + // Keep the documentation at docs/ebpf/loading/elf-sections.md up-to-date. for i, sec := range f.Sections { idx := elf.SectionIndex(i) @@ -371,7 +379,7 @@ func (ec *elfCode) loadFunctions(section *elfSection) (map[string]asm.Instructio r := bufio.NewReader(section.Open()) // Decode the section's instruction stream. - var insns asm.Instructions + insns := make(asm.Instructions, 0, section.Size/asm.InstructionSize) if err := insns.Unmarshal(r, ec.ByteOrder); err != nil { return nil, fmt.Errorf("decoding instructions for section %s: %w", section.Name, err) } @@ -454,6 +462,8 @@ func jumpTarget(offset uint64, ins asm.Instruction) uint64 { return uint64(dest) } +var errUnsupportedBinding = errors.New("unsupported binding") + func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) error { var ( typ = elf.ST_TYPE(rel.Info) @@ -465,10 +475,14 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err switch target.kind { case mapSection, btfMapSection: - if bind != elf.STB_GLOBAL { + if bind == elf.STB_LOCAL { return fmt.Errorf("possible erroneous static qualifier on map definition: found reference to %q", name) } + if bind != elf.STB_GLOBAL { + return fmt.Errorf("map %q: %w: %s", name, errUnsupportedBinding, bind) + } + if typ != elf.STT_OBJECT && typ != elf.STT_NOTYPE { // STT_NOTYPE is generated on clang < 8 which doesn't tag // relocations appropriately. @@ -482,7 +496,7 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err switch typ { case elf.STT_SECTION: if bind != elf.STB_LOCAL { - return fmt.Errorf("direct load: %s: unsupported section relocation %s", name, bind) + return fmt.Errorf("direct load: %s: %w: %s", name, errUnsupportedBinding, bind) } // This is really a reference to a static symbol, which clang doesn't @@ -493,7 +507,7 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err case elf.STT_OBJECT: // LLVM 9 emits OBJECT-LOCAL symbols for anonymous constants. if bind != elf.STB_GLOBAL && bind != elf.STB_LOCAL { - return fmt.Errorf("direct load: %s: unsupported object relocation %s", name, bind) + return fmt.Errorf("direct load: %s: %w: %s", name, errUnsupportedBinding, bind) } offset = uint32(rel.Value) @@ -501,7 +515,7 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err case elf.STT_NOTYPE: // LLVM 7 emits NOTYPE-LOCAL symbols for anonymous constants. if bind != elf.STB_LOCAL { - return fmt.Errorf("direct load: %s: unsupported untyped relocation %s", name, bind) + return fmt.Errorf("direct load: %s: %w: %s", name, errUnsupportedBinding, bind) } offset = uint32(rel.Value) @@ -529,12 +543,12 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err switch typ { case elf.STT_NOTYPE, elf.STT_FUNC: if bind != elf.STB_GLOBAL { - return fmt.Errorf("call: %s: unsupported binding: %s", name, bind) + return fmt.Errorf("call: %s: %w: %s", name, errUnsupportedBinding, bind) } case elf.STT_SECTION: if bind != elf.STB_LOCAL { - return fmt.Errorf("call: %s: unsupported binding: %s", name, bind) + return fmt.Errorf("call: %s: %w: %s", name, errUnsupportedBinding, bind) } // The function we want to call is in the indicated section, @@ -557,12 +571,12 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err switch typ { case elf.STT_FUNC: if bind != elf.STB_GLOBAL { - return fmt.Errorf("load: %s: unsupported binding: %s", name, bind) + return fmt.Errorf("load: %s: %w: %s", name, errUnsupportedBinding, bind) } case elf.STT_SECTION: if bind != elf.STB_LOCAL { - return fmt.Errorf("load: %s: unsupported binding: %s", name, bind) + return fmt.Errorf("load: %s: %w: %s", name, errUnsupportedBinding, bind) } // ins.Constant already contains the offset in bytes from the @@ -591,8 +605,8 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err // function declarations, as well as extern kfunc declarations using __ksym // and extern kconfig variables declared using __kconfig. case undefSection: - if bind != elf.STB_GLOBAL { - return fmt.Errorf("asm relocation: %s: unsupported binding: %s", name, bind) + if bind != elf.STB_GLOBAL && bind != elf.STB_WEAK { + return fmt.Errorf("asm relocation: %s: %w: %s", name, errUnsupportedBinding, bind) } if typ != elf.STT_NOTYPE { @@ -601,13 +615,25 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err kf := ec.kfuncs[name] switch { - // If a Call instruction is found and the datasec has a btf.Func with a Name - // that matches the symbol name we mark the instruction as a call to a kfunc. + // If a Call / DWordLoad instruction is found and the datasec has a btf.Func with a Name + // that matches the symbol name we mark the instruction as a referencing a kfunc. case kf != nil && ins.OpCode.JumpOp() == asm.Call: - ins.Metadata.Set(kfuncMeta{}, kf) + ins.Metadata.Set(kfuncMetaKey{}, &kfuncMeta{ + Func: kf, + Binding: bind, + }) + ins.Src = asm.PseudoKfuncCall ins.Constant = -1 + case kf != nil && ins.OpCode.IsDWordLoad(): + ins.Metadata.Set(kfuncMetaKey{}, &kfuncMeta{ + Func: kf, + Binding: bind, + }) + + ins.Constant = 0 + // If no kconfig map is found, this must be a symbol reference from inline // asm (see testdata/loader.c:asm_relocation()) or a call to a forward // function declaration (see testdata/fwd_decl.c). Don't interfere, These @@ -617,6 +643,10 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err // require it to contain the symbol to disambiguate between inline asm // relos and kconfigs. case ec.kconfig != nil && ins.OpCode.IsDWordLoad(): + if bind != elf.STB_GLOBAL { + return fmt.Errorf("asm relocation: %s: %w: %s", name, errUnsupportedBinding, bind) + } + for _, vsi := range ec.kconfig.Value.(*btf.Datasec).Vars { if vsi.Type.(*btf.Var).Name != rel.Name { continue @@ -694,10 +724,6 @@ func (ec *elfCode) loadMaps() error { spec.Extra = bytes.NewReader(extra) } - if err := spec.clampPerfEventArraySize(); err != nil { - return fmt.Errorf("map %s: %w", mapName, err) - } - ec.maps[mapName] = &spec } } @@ -752,7 +778,7 @@ func (ec *elfCode) loadBTFMaps() error { } // Each Var representing a BTF map definition contains a Struct. - mapStruct, ok := v.Type.(*btf.Struct) + mapStruct, ok := btf.UnderlyingType(v.Type).(*btf.Struct) if !ok { return fmt.Errorf("expected struct, got %s", v.Type) } @@ -762,10 +788,6 @@ func (ec *elfCode) loadBTFMaps() error { return fmt.Errorf("map %v: %w", name, err) } - if err := mapSpec.clampPerfEventArraySize(); err != nil { - return fmt.Errorf("map %v: %w", name, err) - } - ec.maps[name] = mapSpec } @@ -785,7 +807,7 @@ func (ec *elfCode) loadBTFMaps() error { // mapSpecFromBTF produces a MapSpec based on a btf.Struct def representing // a BTF map definition. The name and spec arguments will be copied to the -// resulting MapSpec, and inner must be true on any resursive invocations. +// resulting MapSpec, and inner must be true on any recursive invocations. func mapSpecFromBTF(es *elfSection, vs *btf.VarSecinfo, def *btf.Struct, spec *btf.Spec, name string, inner bool) (*MapSpec, error) { var ( key, value btf.Type @@ -950,6 +972,9 @@ func mapSpecFromBTF(es *elfSection, vs *btf.VarSecinfo, def *btf.Struct, spec *b return nil, fmt.Errorf("resolving values contents: %w", err) } + case "map_extra": + return nil, fmt.Errorf("BTF map definition: field %s: %w", member.Name, ErrNotSupported) + default: return nil, fmt.Errorf("unrecognized field %s in BTF map definition", member.Name) } @@ -1150,7 +1175,7 @@ func (ec *elfCode) loadKconfigSection() error { KeySize: uint32(4), ValueSize: ds.Size, MaxEntries: 1, - Flags: unix.BPF_F_RDONLY_PROG | unix.BPF_F_MMAPABLE, + Flags: unix.BPF_F_RDONLY_PROG, Freeze: true, Key: &btf.Int{Size: 4}, Value: ds, @@ -1183,108 +1208,106 @@ func (ec *elfCode) loadKsymsSection() error { return nil } +type libbpfElfSectionDef struct { + pattern string + programType sys.ProgType + attachType sys.AttachType + flags libbpfElfSectionFlag +} + +type libbpfElfSectionFlag uint32 + +// The values correspond to enum sec_def_flags in libbpf. +const ( + _SEC_NONE libbpfElfSectionFlag = 0 + + _SEC_EXP_ATTACH_OPT libbpfElfSectionFlag = 1 << (iota - 1) + _SEC_ATTACHABLE + _SEC_ATTACH_BTF + _SEC_SLEEPABLE + _SEC_XDP_FRAGS + _SEC_USDT + + // Ignore any present extra in order to preserve backwards compatibility + // with earlier versions of the library. + ignoreExtra + + _SEC_ATTACHABLE_OPT = _SEC_ATTACHABLE | _SEC_EXP_ATTACH_OPT +) + +func init() { + // Compatibility with older versions of the library. + // We prepend libbpf definitions since they contain a prefix match + // for "xdp". + elfSectionDefs = append([]libbpfElfSectionDef{ + {"xdp.frags/", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP, _SEC_XDP_FRAGS | ignoreExtra}, + {"xdp.frags_devmap/", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP_DEVMAP, _SEC_XDP_FRAGS}, + {"xdp_devmap/", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP_DEVMAP, 0}, + {"xdp.frags_cpumap/", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP_CPUMAP, _SEC_XDP_FRAGS}, + {"xdp_cpumap/", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP_CPUMAP, 0}, + // This has been in the library since the beginning of time. Not sure + // where it came from. + {"seccomp", sys.BPF_PROG_TYPE_SOCKET_FILTER, 0, _SEC_NONE}, + }, elfSectionDefs...) +} + func getProgType(sectionName string) (ProgramType, AttachType, uint32, string) { - types := []struct { - prefix string - progType ProgramType - attachType AttachType - progFlags uint32 - }{ - // Please update the types from libbpf.c and follow the order of it. - // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/tools/lib/bpf/libbpf.c - {"socket", SocketFilter, AttachNone, 0}, - {"sk_reuseport/migrate", SkReuseport, AttachSkReuseportSelectOrMigrate, 0}, - {"sk_reuseport", SkReuseport, AttachSkReuseportSelect, 0}, - {"kprobe/", Kprobe, AttachNone, 0}, - {"uprobe/", Kprobe, AttachNone, 0}, - {"kretprobe/", Kprobe, AttachNone, 0}, - {"uretprobe/", Kprobe, AttachNone, 0}, - {"tc", SchedCLS, AttachNone, 0}, - {"classifier", SchedCLS, AttachNone, 0}, - {"action", SchedACT, AttachNone, 0}, - {"tracepoint/", TracePoint, AttachNone, 0}, - {"tp/", TracePoint, AttachNone, 0}, - {"raw_tracepoint/", RawTracepoint, AttachNone, 0}, - {"raw_tp/", RawTracepoint, AttachNone, 0}, - {"raw_tracepoint.w/", RawTracepointWritable, AttachNone, 0}, - {"raw_tp.w/", RawTracepointWritable, AttachNone, 0}, - {"tp_btf/", Tracing, AttachTraceRawTp, 0}, - {"fentry/", Tracing, AttachTraceFEntry, 0}, - {"fmod_ret/", Tracing, AttachModifyReturn, 0}, - {"fexit/", Tracing, AttachTraceFExit, 0}, - {"fentry.s/", Tracing, AttachTraceFEntry, unix.BPF_F_SLEEPABLE}, - {"fmod_ret.s/", Tracing, AttachModifyReturn, unix.BPF_F_SLEEPABLE}, - {"fexit.s/", Tracing, AttachTraceFExit, unix.BPF_F_SLEEPABLE}, - {"freplace/", Extension, AttachNone, 0}, - {"lsm/", LSM, AttachLSMMac, 0}, - {"lsm.s/", LSM, AttachLSMMac, unix.BPF_F_SLEEPABLE}, - {"iter/", Tracing, AttachTraceIter, 0}, - {"iter.s/", Tracing, AttachTraceIter, unix.BPF_F_SLEEPABLE}, - {"syscall", Syscall, AttachNone, 0}, - {"xdp.frags_devmap/", XDP, AttachXDPDevMap, unix.BPF_F_XDP_HAS_FRAGS}, - {"xdp_devmap/", XDP, AttachXDPDevMap, 0}, - {"xdp.frags_cpumap/", XDP, AttachXDPCPUMap, unix.BPF_F_XDP_HAS_FRAGS}, - {"xdp_cpumap/", XDP, AttachXDPCPUMap, 0}, - {"xdp.frags", XDP, AttachNone, unix.BPF_F_XDP_HAS_FRAGS}, - {"xdp", XDP, AttachNone, 0}, - {"perf_event", PerfEvent, AttachNone, 0}, - {"lwt_in", LWTIn, AttachNone, 0}, - {"lwt_out", LWTOut, AttachNone, 0}, - {"lwt_xmit", LWTXmit, AttachNone, 0}, - {"lwt_seg6local", LWTSeg6Local, AttachNone, 0}, - {"cgroup_skb/ingress", CGroupSKB, AttachCGroupInetIngress, 0}, - {"cgroup_skb/egress", CGroupSKB, AttachCGroupInetEgress, 0}, - {"cgroup/skb", CGroupSKB, AttachNone, 0}, - {"cgroup/sock_create", CGroupSock, AttachCGroupInetSockCreate, 0}, - {"cgroup/sock_release", CGroupSock, AttachCgroupInetSockRelease, 0}, - {"cgroup/sock", CGroupSock, AttachCGroupInetSockCreate, 0}, - {"cgroup/post_bind4", CGroupSock, AttachCGroupInet4PostBind, 0}, - {"cgroup/post_bind6", CGroupSock, AttachCGroupInet6PostBind, 0}, - {"cgroup/dev", CGroupDevice, AttachCGroupDevice, 0}, - {"sockops", SockOps, AttachCGroupSockOps, 0}, - {"sk_skb/stream_parser", SkSKB, AttachSkSKBStreamParser, 0}, - {"sk_skb/stream_verdict", SkSKB, AttachSkSKBStreamVerdict, 0}, - {"sk_skb", SkSKB, AttachNone, 0}, - {"sk_msg", SkMsg, AttachSkMsgVerdict, 0}, - {"lirc_mode2", LircMode2, AttachLircMode2, 0}, - {"flow_dissector", FlowDissector, AttachFlowDissector, 0}, - {"cgroup/bind4", CGroupSockAddr, AttachCGroupInet4Bind, 0}, - {"cgroup/bind6", CGroupSockAddr, AttachCGroupInet6Bind, 0}, - {"cgroup/connect4", CGroupSockAddr, AttachCGroupInet4Connect, 0}, - {"cgroup/connect6", CGroupSockAddr, AttachCGroupInet6Connect, 0}, - {"cgroup/sendmsg4", CGroupSockAddr, AttachCGroupUDP4Sendmsg, 0}, - {"cgroup/sendmsg6", CGroupSockAddr, AttachCGroupUDP6Sendmsg, 0}, - {"cgroup/recvmsg4", CGroupSockAddr, AttachCGroupUDP4Recvmsg, 0}, - {"cgroup/recvmsg6", CGroupSockAddr, AttachCGroupUDP6Recvmsg, 0}, - {"cgroup/getpeername4", CGroupSockAddr, AttachCgroupInet4GetPeername, 0}, - {"cgroup/getpeername6", CGroupSockAddr, AttachCgroupInet6GetPeername, 0}, - {"cgroup/getsockname4", CGroupSockAddr, AttachCgroupInet4GetSockname, 0}, - {"cgroup/getsockname6", CGroupSockAddr, AttachCgroupInet6GetSockname, 0}, - {"cgroup/sysctl", CGroupSysctl, AttachCGroupSysctl, 0}, - {"cgroup/getsockopt", CGroupSockopt, AttachCGroupGetsockopt, 0}, - {"cgroup/setsockopt", CGroupSockopt, AttachCGroupSetsockopt, 0}, - {"struct_ops+", StructOps, AttachNone, 0}, - {"sk_lookup/", SkLookup, AttachSkLookup, 0}, - {"seccomp", SocketFilter, AttachNone, 0}, - {"kprobe.multi", Kprobe, AttachTraceKprobeMulti, 0}, - {"kretprobe.multi", Kprobe, AttachTraceKprobeMulti, 0}, - } + // Skip optional program marking for now. + sectionName = strings.TrimPrefix(sectionName, "?") - for _, t := range types { - if !strings.HasPrefix(sectionName, t.prefix) { + for _, t := range elfSectionDefs { + extra, ok := matchSectionName(sectionName, t.pattern) + if !ok { continue } - if !strings.HasSuffix(t.prefix, "/") { - return t.progType, t.attachType, t.progFlags, "" + programType := ProgramType(t.programType) + attachType := AttachType(t.attachType) + + var flags uint32 + if t.flags&_SEC_SLEEPABLE > 0 { + flags |= unix.BPF_F_SLEEPABLE + } + if t.flags&_SEC_XDP_FRAGS > 0 { + flags |= unix.BPF_F_XDP_HAS_FRAGS + } + if t.flags&_SEC_EXP_ATTACH_OPT > 0 { + if programType == XDP { + // The library doesn't yet have code to fallback to not specifying + // attach type. Only do this for XDP since we've enforced correct + // attach type for all other program types. + attachType = AttachNone + } + } + if t.flags&ignoreExtra > 0 { + extra = "" } - return t.progType, t.attachType, t.progFlags, sectionName[len(t.prefix):] + return programType, attachType, flags, extra } return UnspecifiedProgram, AttachNone, 0, "" } +// matchSectionName checks a section name against a pattern. +// +// It's behaviour mirrors that of libbpf's sec_def_matches. +func matchSectionName(sectionName, pattern string) (extra string, found bool) { + have, extra, found := strings.Cut(sectionName, "/") + want := strings.TrimRight(pattern, "+/") + + if strings.HasSuffix(pattern, "/") { + // Section name must have a slash and extra may be empty. + return extra, have == want && found + } else if strings.HasSuffix(pattern, "+") { + // Section name may have a slash and extra may be empty. + return extra, have == want + } + + // Section name must have a prefix. extra is ignored. + return "", strings.HasPrefix(sectionName, pattern) +} + func (ec *elfCode) loadSectionRelocations(sec *elf.Section, symbols []elf.Symbol) (map[uint64]elf.Symbol, error) { rels := make(map[uint64]elf.Symbol) diff --git a/vendor/github.com/cilium/ebpf/elf_sections.go b/vendor/github.com/cilium/ebpf/elf_sections.go new file mode 100644 index 000000000000..4b58251d9ab4 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/elf_sections.go @@ -0,0 +1,109 @@ +// Code generated by internal/cmd/gensections.awk; DO NOT EDIT. + +package ebpf + +// Code in this file is derived from libbpf, available under BSD-2-Clause. + +import "github.com/cilium/ebpf/internal/sys" + +var elfSectionDefs = []libbpfElfSectionDef{ + {"socket", sys.BPF_PROG_TYPE_SOCKET_FILTER, 0, _SEC_NONE}, + {"sk_reuseport/migrate", sys.BPF_PROG_TYPE_SK_REUSEPORT, sys.BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, _SEC_ATTACHABLE}, + {"sk_reuseport", sys.BPF_PROG_TYPE_SK_REUSEPORT, sys.BPF_SK_REUSEPORT_SELECT, _SEC_ATTACHABLE}, + {"kprobe+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_NONE}, + {"uprobe+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_NONE}, + {"uprobe.s+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_SLEEPABLE}, + {"kretprobe+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_NONE}, + {"uretprobe+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_NONE}, + {"uretprobe.s+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_SLEEPABLE}, + {"kprobe.multi+", sys.BPF_PROG_TYPE_KPROBE, sys.BPF_TRACE_KPROBE_MULTI, _SEC_NONE}, + {"kretprobe.multi+", sys.BPF_PROG_TYPE_KPROBE, sys.BPF_TRACE_KPROBE_MULTI, _SEC_NONE}, + {"uprobe.multi+", sys.BPF_PROG_TYPE_KPROBE, sys.BPF_TRACE_UPROBE_MULTI, _SEC_NONE}, + {"uretprobe.multi+", sys.BPF_PROG_TYPE_KPROBE, sys.BPF_TRACE_UPROBE_MULTI, _SEC_NONE}, + {"uprobe.multi.s+", sys.BPF_PROG_TYPE_KPROBE, sys.BPF_TRACE_UPROBE_MULTI, _SEC_SLEEPABLE}, + {"uretprobe.multi.s+", sys.BPF_PROG_TYPE_KPROBE, sys.BPF_TRACE_UPROBE_MULTI, _SEC_SLEEPABLE}, + {"ksyscall+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_NONE}, + {"kretsyscall+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_NONE}, + {"usdt+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_USDT}, + {"usdt.s+", sys.BPF_PROG_TYPE_KPROBE, 0, _SEC_USDT | _SEC_SLEEPABLE}, + {"tc/ingress", sys.BPF_PROG_TYPE_SCHED_CLS, sys.BPF_TCX_INGRESS, _SEC_NONE}, + {"tc/egress", sys.BPF_PROG_TYPE_SCHED_CLS, sys.BPF_TCX_EGRESS, _SEC_NONE}, + {"tcx/ingress", sys.BPF_PROG_TYPE_SCHED_CLS, sys.BPF_TCX_INGRESS, _SEC_NONE}, + {"tcx/egress", sys.BPF_PROG_TYPE_SCHED_CLS, sys.BPF_TCX_EGRESS, _SEC_NONE}, + {"tc", sys.BPF_PROG_TYPE_SCHED_CLS, 0, _SEC_NONE}, + {"classifier", sys.BPF_PROG_TYPE_SCHED_CLS, 0, _SEC_NONE}, + {"action", sys.BPF_PROG_TYPE_SCHED_ACT, 0, _SEC_NONE}, + {"netkit/primary", sys.BPF_PROG_TYPE_SCHED_CLS, sys.BPF_NETKIT_PRIMARY, _SEC_NONE}, + {"netkit/peer", sys.BPF_PROG_TYPE_SCHED_CLS, sys.BPF_NETKIT_PEER, _SEC_NONE}, + {"tracepoint+", sys.BPF_PROG_TYPE_TRACEPOINT, 0, _SEC_NONE}, + {"tp+", sys.BPF_PROG_TYPE_TRACEPOINT, 0, _SEC_NONE}, + {"raw_tracepoint+", sys.BPF_PROG_TYPE_RAW_TRACEPOINT, 0, _SEC_NONE}, + {"raw_tp+", sys.BPF_PROG_TYPE_RAW_TRACEPOINT, 0, _SEC_NONE}, + {"raw_tracepoint.w+", sys.BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, 0, _SEC_NONE}, + {"raw_tp.w+", sys.BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, 0, _SEC_NONE}, + {"tp_btf+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_TRACE_RAW_TP, _SEC_ATTACH_BTF}, + {"fentry+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_TRACE_FENTRY, _SEC_ATTACH_BTF}, + {"fmod_ret+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_MODIFY_RETURN, _SEC_ATTACH_BTF}, + {"fexit+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_TRACE_FEXIT, _SEC_ATTACH_BTF}, + {"fentry.s+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_TRACE_FENTRY, _SEC_ATTACH_BTF | _SEC_SLEEPABLE}, + {"fmod_ret.s+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_MODIFY_RETURN, _SEC_ATTACH_BTF | _SEC_SLEEPABLE}, + {"fexit.s+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_TRACE_FEXIT, _SEC_ATTACH_BTF | _SEC_SLEEPABLE}, + {"freplace+", sys.BPF_PROG_TYPE_EXT, 0, _SEC_ATTACH_BTF}, + {"lsm+", sys.BPF_PROG_TYPE_LSM, sys.BPF_LSM_MAC, _SEC_ATTACH_BTF}, + {"lsm.s+", sys.BPF_PROG_TYPE_LSM, sys.BPF_LSM_MAC, _SEC_ATTACH_BTF | _SEC_SLEEPABLE}, + {"lsm_cgroup+", sys.BPF_PROG_TYPE_LSM, sys.BPF_LSM_CGROUP, _SEC_ATTACH_BTF}, + {"iter+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_TRACE_ITER, _SEC_ATTACH_BTF}, + {"iter.s+", sys.BPF_PROG_TYPE_TRACING, sys.BPF_TRACE_ITER, _SEC_ATTACH_BTF | _SEC_SLEEPABLE}, + {"syscall", sys.BPF_PROG_TYPE_SYSCALL, 0, _SEC_SLEEPABLE}, + {"xdp.frags/devmap", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP_DEVMAP, _SEC_XDP_FRAGS}, + {"xdp/devmap", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP_DEVMAP, _SEC_ATTACHABLE}, + {"xdp.frags/cpumap", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP_CPUMAP, _SEC_XDP_FRAGS}, + {"xdp/cpumap", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP_CPUMAP, _SEC_ATTACHABLE}, + {"xdp.frags", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP, _SEC_XDP_FRAGS}, + {"xdp", sys.BPF_PROG_TYPE_XDP, sys.BPF_XDP, _SEC_ATTACHABLE_OPT}, + {"perf_event", sys.BPF_PROG_TYPE_PERF_EVENT, 0, _SEC_NONE}, + {"lwt_in", sys.BPF_PROG_TYPE_LWT_IN, 0, _SEC_NONE}, + {"lwt_out", sys.BPF_PROG_TYPE_LWT_OUT, 0, _SEC_NONE}, + {"lwt_xmit", sys.BPF_PROG_TYPE_LWT_XMIT, 0, _SEC_NONE}, + {"lwt_seg6local", sys.BPF_PROG_TYPE_LWT_SEG6LOCAL, 0, _SEC_NONE}, + {"sockops", sys.BPF_PROG_TYPE_SOCK_OPS, sys.BPF_CGROUP_SOCK_OPS, _SEC_ATTACHABLE_OPT}, + {"sk_skb/stream_parser", sys.BPF_PROG_TYPE_SK_SKB, sys.BPF_SK_SKB_STREAM_PARSER, _SEC_ATTACHABLE_OPT}, + {"sk_skb/stream_verdict", sys.BPF_PROG_TYPE_SK_SKB, sys.BPF_SK_SKB_STREAM_VERDICT, _SEC_ATTACHABLE_OPT}, + {"sk_skb", sys.BPF_PROG_TYPE_SK_SKB, 0, _SEC_NONE}, + {"sk_msg", sys.BPF_PROG_TYPE_SK_MSG, sys.BPF_SK_MSG_VERDICT, _SEC_ATTACHABLE_OPT}, + {"lirc_mode2", sys.BPF_PROG_TYPE_LIRC_MODE2, sys.BPF_LIRC_MODE2, _SEC_ATTACHABLE_OPT}, + {"flow_dissector", sys.BPF_PROG_TYPE_FLOW_DISSECTOR, sys.BPF_FLOW_DISSECTOR, _SEC_ATTACHABLE_OPT}, + {"cgroup_skb/ingress", sys.BPF_PROG_TYPE_CGROUP_SKB, sys.BPF_CGROUP_INET_INGRESS, _SEC_ATTACHABLE_OPT}, + {"cgroup_skb/egress", sys.BPF_PROG_TYPE_CGROUP_SKB, sys.BPF_CGROUP_INET_EGRESS, _SEC_ATTACHABLE_OPT}, + {"cgroup/skb", sys.BPF_PROG_TYPE_CGROUP_SKB, 0, _SEC_NONE}, + {"cgroup/sock_create", sys.BPF_PROG_TYPE_CGROUP_SOCK, sys.BPF_CGROUP_INET_SOCK_CREATE, _SEC_ATTACHABLE}, + {"cgroup/sock_release", sys.BPF_PROG_TYPE_CGROUP_SOCK, sys.BPF_CGROUP_INET_SOCK_RELEASE, _SEC_ATTACHABLE}, + {"cgroup/sock", sys.BPF_PROG_TYPE_CGROUP_SOCK, sys.BPF_CGROUP_INET_SOCK_CREATE, _SEC_ATTACHABLE_OPT}, + {"cgroup/post_bind4", sys.BPF_PROG_TYPE_CGROUP_SOCK, sys.BPF_CGROUP_INET4_POST_BIND, _SEC_ATTACHABLE}, + {"cgroup/post_bind6", sys.BPF_PROG_TYPE_CGROUP_SOCK, sys.BPF_CGROUP_INET6_POST_BIND, _SEC_ATTACHABLE}, + {"cgroup/bind4", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_INET4_BIND, _SEC_ATTACHABLE}, + {"cgroup/bind6", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_INET6_BIND, _SEC_ATTACHABLE}, + {"cgroup/connect4", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_INET4_CONNECT, _SEC_ATTACHABLE}, + {"cgroup/connect6", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_INET6_CONNECT, _SEC_ATTACHABLE}, + {"cgroup/connect_unix", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UNIX_CONNECT, _SEC_ATTACHABLE}, + {"cgroup/sendmsg4", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UDP4_SENDMSG, _SEC_ATTACHABLE}, + {"cgroup/sendmsg6", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UDP6_SENDMSG, _SEC_ATTACHABLE}, + {"cgroup/sendmsg_unix", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UNIX_SENDMSG, _SEC_ATTACHABLE}, + {"cgroup/recvmsg4", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UDP4_RECVMSG, _SEC_ATTACHABLE}, + {"cgroup/recvmsg6", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UDP6_RECVMSG, _SEC_ATTACHABLE}, + {"cgroup/recvmsg_unix", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UNIX_RECVMSG, _SEC_ATTACHABLE}, + {"cgroup/getpeername4", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_INET4_GETPEERNAME, _SEC_ATTACHABLE}, + {"cgroup/getpeername6", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_INET6_GETPEERNAME, _SEC_ATTACHABLE}, + {"cgroup/getpeername_unix", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UNIX_GETPEERNAME, _SEC_ATTACHABLE}, + {"cgroup/getsockname4", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_INET4_GETSOCKNAME, _SEC_ATTACHABLE}, + {"cgroup/getsockname6", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_INET6_GETSOCKNAME, _SEC_ATTACHABLE}, + {"cgroup/getsockname_unix", sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR, sys.BPF_CGROUP_UNIX_GETSOCKNAME, _SEC_ATTACHABLE}, + {"cgroup/sysctl", sys.BPF_PROG_TYPE_CGROUP_SYSCTL, sys.BPF_CGROUP_SYSCTL, _SEC_ATTACHABLE}, + {"cgroup/getsockopt", sys.BPF_PROG_TYPE_CGROUP_SOCKOPT, sys.BPF_CGROUP_GETSOCKOPT, _SEC_ATTACHABLE}, + {"cgroup/setsockopt", sys.BPF_PROG_TYPE_CGROUP_SOCKOPT, sys.BPF_CGROUP_SETSOCKOPT, _SEC_ATTACHABLE}, + {"cgroup/dev", sys.BPF_PROG_TYPE_CGROUP_DEVICE, sys.BPF_CGROUP_DEVICE, _SEC_ATTACHABLE_OPT}, + {"struct_ops+", sys.BPF_PROG_TYPE_STRUCT_OPS, 0, _SEC_NONE}, + {"struct_ops.s+", sys.BPF_PROG_TYPE_STRUCT_OPS, 0, _SEC_SLEEPABLE}, + {"sk_lookup", sys.BPF_PROG_TYPE_SK_LOOKUP, sys.BPF_SK_LOOKUP, _SEC_ATTACHABLE}, + {"netfilter", sys.BPF_PROG_TYPE_NETFILTER, sys.BPF_NETFILTER, _SEC_NONE}, +} diff --git a/vendor/github.com/cilium/ebpf/info.go b/vendor/github.com/cilium/ebpf/info.go index a02e8a41618c..04c60c64b893 100644 --- a/vendor/github.com/cilium/ebpf/info.go +++ b/vendor/github.com/cilium/ebpf/info.go @@ -20,6 +20,23 @@ import ( "github.com/cilium/ebpf/internal/unix" ) +// The *Info structs expose metadata about a program or map. Most +// fields are exposed via a getter: +// +// func (*MapInfo) ID() (MapID, bool) +// +// This is because the metadata available changes based on kernel version. +// The second boolean return value indicates whether a particular field is +// available on the current kernel. +// +// Always add new metadata as such a getter, unless you can somehow get the +// value of the field on all supported kernels. Also document which version +// a particular field first appeared in. +// +// Some metadata is a buffer which needs additional parsing. In this case, +// store the undecoded data in the Info struct and provide a getter which +// decodes it when necessary. See ProgramInfo.Instructions for an example. + // MapInfo describes a map. type MapInfo struct { Type MapType @@ -30,6 +47,8 @@ type MapInfo struct { Flags uint32 // Name as supplied by user space at load time. Available from 4.15. Name string + + btf btf.ID } func newMapInfoFromFd(fd *sys.FD) (*MapInfo, error) { @@ -50,6 +69,7 @@ func newMapInfoFromFd(fd *sys.FD) (*MapInfo, error) { info.MaxEntries, uint32(info.MapFlags), unix.ByteSliceToString(info.Name[:]), + btf.ID(info.BtfId), }, nil } @@ -77,12 +97,27 @@ func (mi *MapInfo) ID() (MapID, bool) { return mi.id, mi.id > 0 } +// BTFID returns the BTF ID associated with the Map. +// +// The ID is only valid as long as the associated Map is kept alive. +// Available from 4.18. +// +// The bool return value indicates whether this optional field is available and +// populated. (The field may be available but not populated if the kernel +// supports the field but the Map was loaded without BTF information.) +func (mi *MapInfo) BTFID() (btf.ID, bool) { + return mi.btf, mi.btf > 0 +} + // programStats holds statistics of a program. type programStats struct { // Total accumulated runtime of the program ins ns. runtime time.Duration // Total number of times the program was called. runCount uint64 + // Total number of times the programm was NOT called. + // Added in commit 9ed9e9ba2337 ("bpf: Count the number of times recursion was prevented"). + recursionMisses uint64 } // ProgramInfo describes a program. @@ -101,6 +136,11 @@ type ProgramInfo struct { maps []MapID insns []byte + + lineInfos []byte + numLineInfos uint32 + funcInfos []byte + numFuncInfos uint32 } func newProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) { @@ -120,18 +160,22 @@ func newProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) { Name: unix.ByteSliceToString(info.Name[:]), btf: btf.ID(info.BtfId), stats: &programStats{ - runtime: time.Duration(info.RunTimeNs), - runCount: info.RunCnt, + runtime: time.Duration(info.RunTimeNs), + runCount: info.RunCnt, + recursionMisses: info.RecursionMisses, }, } // Start with a clean struct for the second call, otherwise we may get EFAULT. var info2 sys.ProgInfo + makeSecondCall := false + if info.NrMapIds > 0 { pi.maps = make([]MapID, info.NrMapIds) info2.NrMapIds = info.NrMapIds info2.MapIds = sys.NewPointer(unsafe.Pointer(&pi.maps[0])) + makeSecondCall = true } else if haveProgramInfoMapIDs() == nil { // This program really has no associated maps. pi.maps = make([]MapID, 0) @@ -150,9 +194,28 @@ func newProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) { pi.insns = make([]byte, info.XlatedProgLen) info2.XlatedProgLen = info.XlatedProgLen info2.XlatedProgInsns = sys.NewSlicePointer(pi.insns) + makeSecondCall = true + } + + if info.NrLineInfo > 0 { + pi.lineInfos = make([]byte, btf.LineInfoSize*info.NrLineInfo) + info2.LineInfo = sys.NewSlicePointer(pi.lineInfos) + info2.LineInfoRecSize = btf.LineInfoSize + info2.NrLineInfo = info.NrLineInfo + pi.numLineInfos = info.NrLineInfo + makeSecondCall = true + } + + if info.NrFuncInfo > 0 { + pi.funcInfos = make([]byte, btf.FuncInfoSize*info.NrFuncInfo) + info2.FuncInfo = sys.NewSlicePointer(pi.funcInfos) + info2.FuncInfoRecSize = btf.FuncInfoSize + info2.NrFuncInfo = info.NrFuncInfo + pi.numFuncInfos = info.NrFuncInfo + makeSecondCall = true } - if info.NrMapIds > 0 || info.XlatedProgLen > 0 { + if makeSecondCall { if err := sys.ObjInfo(fd, &info2); err != nil { return nil, err } @@ -232,6 +295,16 @@ func (pi *ProgramInfo) Runtime() (time.Duration, bool) { return time.Duration(0), false } +// RecursionMisses returns the total number of times the program was NOT called. +// This can happen when another bpf program is already running on the cpu, which +// is likely to happen for example when you interrupt bpf program execution. +func (pi *ProgramInfo) RecursionMisses() (uint64, bool) { + if pi.stats != nil { + return pi.stats.recursionMisses, true + } + return 0, false +} + // Instructions returns the 'xlated' instruction stream of the program // after it has been verified and rewritten by the kernel. These instructions // cannot be loaded back into the kernel as-is, this is mainly used for @@ -245,7 +318,13 @@ func (pi *ProgramInfo) Runtime() (time.Duration, bool) { // // The first instruction is marked as a symbol using the Program's name. // -// Available from 4.13. Requires CAP_BPF or equivalent. +// If available, the instructions will be annotated with metadata from the +// BTF. This includes line information and function information. Reading +// this metadata requires CAP_SYS_ADMIN or equivalent. If capability is +// unavailable, the instructions will be returned without metadata. +// +// Available from 4.13. Requires CAP_BPF or equivalent for plain instructions. +// Requires CAP_SYS_ADMIN for instructions with metadata. func (pi *ProgramInfo) Instructions() (asm.Instructions, error) { // If the calling process is not BPF-capable or if the kernel doesn't // support getting xlated instructions, the field will be zero. @@ -259,8 +338,55 @@ func (pi *ProgramInfo) Instructions() (asm.Instructions, error) { return nil, fmt.Errorf("unmarshaling instructions: %w", err) } - // Tag the first instruction with the name of the program, if available. - insns[0] = insns[0].WithSymbol(pi.Name) + if pi.btf != 0 { + btfh, err := btf.NewHandleFromID(pi.btf) + if err != nil { + // Getting a BTF handle requires CAP_SYS_ADMIN, if not available we get an -EPERM. + // Ignore it and fall back to instructions without metadata. + if !errors.Is(err, unix.EPERM) { + return nil, fmt.Errorf("unable to get BTF handle: %w", err) + } + } + + // If we have a BTF handle, we can use it to assign metadata to the instructions. + if btfh != nil { + defer btfh.Close() + + spec, err := btfh.Spec(nil) + if err != nil { + return nil, fmt.Errorf("unable to get BTF spec: %w", err) + } + + lineInfos, err := btf.LoadLineInfos( + bytes.NewReader(pi.lineInfos), + internal.NativeEndian, + pi.numLineInfos, + spec, + ) + if err != nil { + return nil, fmt.Errorf("parse line info: %w", err) + } + + funcInfos, err := btf.LoadFuncInfos( + bytes.NewReader(pi.funcInfos), + internal.NativeEndian, + pi.numFuncInfos, + spec, + ) + if err != nil { + return nil, fmt.Errorf("parse func info: %w", err) + } + + btf.AssignMetadataToInstructions(insns, funcInfos, lineInfos, btf.CORERelocationInfos{}) + } + } + + fn := btf.FuncMetadata(&insns[0]) + name := pi.Name + if fn != nil { + name = fn.Name + } + insns[0] = insns[0].WithSymbol(name) return insns, nil } diff --git a/vendor/github.com/cilium/ebpf/internal/auxv.go b/vendor/github.com/cilium/ebpf/internal/auxv.go new file mode 100644 index 000000000000..45fd0d37f132 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/auxv.go @@ -0,0 +1,60 @@ +package internal + +import ( + "errors" + "io" + _ "unsafe" +) + +type auxvPairReader interface { + Close() error + ReadAuxvPair() (uint64, uint64, error) +} + +// See https://elixir.bootlin.com/linux/v6.5.5/source/include/uapi/linux/auxvec.h +const ( + _AT_NULL = 0 // End of vector + _AT_SYSINFO_EHDR = 33 // Offset to vDSO blob in process image +) + +//go:linkname runtime_getAuxv runtime.getAuxv +func runtime_getAuxv() []uintptr + +type auxvRuntimeReader struct { + data []uintptr + index int +} + +func (r *auxvRuntimeReader) Close() error { + return nil +} + +func (r *auxvRuntimeReader) ReadAuxvPair() (uint64, uint64, error) { + if r.index >= len(r.data)+2 { + return 0, 0, io.EOF + } + + // we manually add the (_AT_NULL, _AT_NULL) pair at the end + // that is not provided by the go runtime + var tag, value uintptr + if r.index+1 < len(r.data) { + tag, value = r.data[r.index], r.data[r.index+1] + } else { + tag, value = _AT_NULL, _AT_NULL + } + r.index += 2 + return uint64(tag), uint64(value), nil +} + +func newAuxvRuntimeReader() (auxvPairReader, error) { + data := runtime_getAuxv() + + if len(data)%2 != 0 { + return nil, errors.New("malformed auxv passed from runtime") + } + + return &auxvRuntimeReader{ + data: data, + index: 0, + }, nil +} diff --git a/vendor/github.com/cilium/ebpf/internal/endian_be.go b/vendor/github.com/cilium/ebpf/internal/endian_be.go index 96a2ac0de22f..a37777f21f2a 100644 --- a/vendor/github.com/cilium/ebpf/internal/endian_be.go +++ b/vendor/github.com/cilium/ebpf/internal/endian_be.go @@ -6,7 +6,4 @@ import "encoding/binary" // NativeEndian is set to either binary.BigEndian or binary.LittleEndian, // depending on the host's endianness. -var NativeEndian binary.ByteOrder = binary.BigEndian - -// ClangEndian is set to either "el" or "eb" depending on the host's endianness. -const ClangEndian = "eb" +var NativeEndian = binary.BigEndian diff --git a/vendor/github.com/cilium/ebpf/internal/endian_le.go b/vendor/github.com/cilium/ebpf/internal/endian_le.go index fde4c55a6f5c..6dcd916d5df2 100644 --- a/vendor/github.com/cilium/ebpf/internal/endian_le.go +++ b/vendor/github.com/cilium/ebpf/internal/endian_le.go @@ -6,7 +6,4 @@ import "encoding/binary" // NativeEndian is set to either binary.BigEndian or binary.LittleEndian, // depending on the host's endianness. -var NativeEndian binary.ByteOrder = binary.LittleEndian - -// ClangEndian is set to either "el" or "eb" depending on the host's endianness. -const ClangEndian = "el" +var NativeEndian = binary.LittleEndian diff --git a/vendor/github.com/cilium/ebpf/internal/errors.go b/vendor/github.com/cilium/ebpf/internal/errors.go index bda01e2fde58..83a371ad35d6 100644 --- a/vendor/github.com/cilium/ebpf/internal/errors.go +++ b/vendor/github.com/cilium/ebpf/internal/errors.go @@ -12,7 +12,7 @@ import ( // // The default error output is a summary of the full log. The latter can be // accessed via VerifierError.Log or by formatting the error, see Format. -func ErrorWithLog(source string, err error, log []byte, truncated bool) *VerifierError { +func ErrorWithLog(source string, err error, log []byte) *VerifierError { const whitespace = "\t\r\v\n " // Convert verifier log C string by truncating it on the first 0 byte @@ -23,7 +23,7 @@ func ErrorWithLog(source string, err error, log []byte, truncated bool) *Verifie log = bytes.Trim(log, whitespace) if len(log) == 0 { - return &VerifierError{source, err, nil, truncated} + return &VerifierError{source, err, nil, false} } logLines := bytes.Split(log, []byte{'\n'}) @@ -34,7 +34,7 @@ func ErrorWithLog(source string, err error, log []byte, truncated bool) *Verifie lines = append(lines, string(bytes.TrimRight(line, whitespace))) } - return &VerifierError{source, err, lines, truncated} + return &VerifierError{source, err, lines, false} } // VerifierError includes information from the eBPF verifier. @@ -46,7 +46,7 @@ type VerifierError struct { Cause error // The verifier output split into lines. Log []string - // Whether the log output is truncated, based on several heuristics. + // Deprecated: the log is never truncated anymore. Truncated bool } @@ -70,7 +70,7 @@ func (le *VerifierError) Error() string { } lines := log[n-1:] - if n >= 2 && (includePreviousLine(log[n-1]) || le.Truncated) { + if n >= 2 && includePreviousLine(log[n-1]) { // Add one more line of context if it aids understanding the error. lines = log[n-2:] } @@ -81,22 +81,9 @@ func (le *VerifierError) Error() string { } omitted := len(le.Log) - len(lines) - if omitted == 0 && !le.Truncated { - return b.String() - } - - b.WriteString(" (") - if le.Truncated { - b.WriteString("truncated") - } - if omitted > 0 { - if le.Truncated { - b.WriteString(", ") - } - fmt.Fprintf(&b, "%d line(s) omitted", omitted) + fmt.Fprintf(&b, " (%d line(s) omitted)", omitted) } - b.WriteString(")") return b.String() } @@ -188,10 +175,6 @@ func (le *VerifierError) Format(f fmt.State, verb rune) { } } - if le.Truncated { - fmt.Fprintf(f, "\n\t(truncated)") - } - default: fmt.Fprintf(f, "%%!%c(BADVERB)", verb) } diff --git a/vendor/github.com/cilium/ebpf/internal/feature.go b/vendor/github.com/cilium/ebpf/internal/feature.go index b1f650751deb..2b856c735e7f 100644 --- a/vendor/github.com/cilium/ebpf/internal/feature.go +++ b/vendor/github.com/cilium/ebpf/internal/feature.go @@ -37,7 +37,7 @@ func (ufe *UnsupportedFeatureError) Is(target error) bool { type FeatureTest struct { // The name of the feature being detected. Name string - // Version in in the form Major.Minor[.Patch]. + // Version in the form Major.Minor[.Patch]. Version string // The feature test itself. Fn FeatureTestFn diff --git a/vendor/github.com/cilium/ebpf/internal/kallsyms/kallsyms.go b/vendor/github.com/cilium/ebpf/internal/kallsyms/kallsyms.go new file mode 100644 index 000000000000..776c7a10a28e --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/kallsyms/kallsyms.go @@ -0,0 +1,74 @@ +package kallsyms + +import ( + "bufio" + "bytes" + "io" + "os" + "sync" +) + +var kernelModules struct { + sync.RWMutex + // function to kernel module mapping + kmods map[string]string +} + +// KernelModule returns the kernel module, if any, a probe-able function is contained in. +func KernelModule(fn string) (string, error) { + kernelModules.RLock() + kmods := kernelModules.kmods + kernelModules.RUnlock() + + if kmods == nil { + kernelModules.Lock() + defer kernelModules.Unlock() + kmods = kernelModules.kmods + } + + if kmods != nil { + return kmods[fn], nil + } + + f, err := os.Open("/proc/kallsyms") + if err != nil { + return "", err + } + defer f.Close() + kmods, err = loadKernelModuleMapping(f) + if err != nil { + return "", err + } + + kernelModules.kmods = kmods + return kmods[fn], nil +} + +// FlushKernelModuleCache removes any cached information about function to kernel module mapping. +func FlushKernelModuleCache() { + kernelModules.Lock() + defer kernelModules.Unlock() + + kernelModules.kmods = nil +} + +func loadKernelModuleMapping(f io.Reader) (map[string]string, error) { + mods := make(map[string]string) + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := bytes.Fields(scanner.Bytes()) + if len(fields) < 4 { + continue + } + switch string(fields[1]) { + case "t", "T": + mods[string(fields[2])] = string(bytes.Trim(fields[3], "[]")) + default: + continue + } + } + if scanner.Err() != nil { + return nil, scanner.Err() + } + return mods, nil +} diff --git a/vendor/github.com/cilium/ebpf/internal/kconfig/kconfig.go b/vendor/github.com/cilium/ebpf/internal/kconfig/kconfig.go index d95e7eb0e5d2..1921e4f15ad7 100644 --- a/vendor/github.com/cilium/ebpf/internal/kconfig/kconfig.go +++ b/vendor/github.com/cilium/ebpf/internal/kconfig/kconfig.go @@ -250,17 +250,43 @@ func putValueNumber(data []byte, typ btf.Type, value string) error { return fmt.Errorf("cannot parse value: %w", err) } - switch size { + return PutInteger(data, integer, n) +} + +// PutInteger writes n into data. +// +// integer determines how much is written into data and what the valid values +// are. +func PutInteger(data []byte, integer *btf.Int, n uint64) error { + // This function should match set_kcfg_value_num in libbpf. + if integer.Encoding == btf.Bool && n > 1 { + return fmt.Errorf("invalid boolean value: %d", n) + } + + if len(data) < int(integer.Size) { + return fmt.Errorf("can't fit an integer of size %d into a byte slice of length %d", integer.Size, len(data)) + } + + switch integer.Size { case 1: + if integer.Encoding == btf.Signed && (int64(n) > math.MaxInt8 || int64(n) < math.MinInt8) { + return fmt.Errorf("can't represent %d as a signed integer of size %d", int64(n), integer.Size) + } data[0] = byte(n) case 2: + if integer.Encoding == btf.Signed && (int64(n) > math.MaxInt16 || int64(n) < math.MinInt16) { + return fmt.Errorf("can't represent %d as a signed integer of size %d", int64(n), integer.Size) + } internal.NativeEndian.PutUint16(data, uint16(n)) case 4: + if integer.Encoding == btf.Signed && (int64(n) > math.MaxInt32 || int64(n) < math.MinInt32) { + return fmt.Errorf("can't represent %d as a signed integer of size %d", int64(n), integer.Size) + } internal.NativeEndian.PutUint32(data, uint32(n)) case 8: internal.NativeEndian.PutUint64(data, uint64(n)) default: - return fmt.Errorf("size (%d) is not valid, expected: 1, 2, 4 or 8", size) + return fmt.Errorf("size (%d) is not valid, expected: 1, 2, 4 or 8", integer.Size) } return nil diff --git a/vendor/github.com/cilium/ebpf/internal/align.go b/vendor/github.com/cilium/ebpf/internal/math.go similarity index 63% rename from vendor/github.com/cilium/ebpf/internal/align.go rename to vendor/github.com/cilium/ebpf/internal/math.go index edc898fa968b..e95c8efde51a 100644 --- a/vendor/github.com/cilium/ebpf/internal/align.go +++ b/vendor/github.com/cilium/ebpf/internal/math.go @@ -6,3 +6,8 @@ import "golang.org/x/exp/constraints" func Align[I constraints.Integer](n, alignment I) I { return (n + alignment - 1) / alignment * alignment } + +// IsPow returns true if n is a power of two. +func IsPow[I constraints.Integer](n I) bool { + return n != 0 && (n&(n-1)) == 0 +} diff --git a/vendor/github.com/cilium/ebpf/internal/memoize.go b/vendor/github.com/cilium/ebpf/internal/memoize.go deleted file mode 100644 index 3de0a3fb95a5..000000000000 --- a/vendor/github.com/cilium/ebpf/internal/memoize.go +++ /dev/null @@ -1,26 +0,0 @@ -package internal - -import ( - "sync" -) - -type memoizedFunc[T any] struct { - once sync.Once - fn func() (T, error) - result T - err error -} - -func (mf *memoizedFunc[T]) do() (T, error) { - mf.once.Do(func() { - mf.result, mf.err = mf.fn() - }) - return mf.result, mf.err -} - -// Memoize the result of a function call. -// -// fn is only ever called once, even if it returns an error. -func Memoize[T any](fn func() (T, error)) func() (T, error) { - return (&memoizedFunc[T]{fn: fn}).do -} diff --git a/vendor/github.com/cilium/ebpf/internal/sys/mapflags_string.go b/vendor/github.com/cilium/ebpf/internal/sys/mapflags_string.go index c80744ae0e07..d9fe217222bf 100644 --- a/vendor/github.com/cilium/ebpf/internal/sys/mapflags_string.go +++ b/vendor/github.com/cilium/ebpf/internal/sys/mapflags_string.go @@ -21,24 +21,28 @@ func _() { _ = x[BPF_F_MMAPABLE-1024] _ = x[BPF_F_PRESERVE_ELEMS-2048] _ = x[BPF_F_INNER_MAP-4096] + _ = x[BPF_F_LINK-8192] + _ = x[BPF_F_PATH_FD-16384] } -const _MapFlags_name = "BPF_F_NO_PREALLOCBPF_F_NO_COMMON_LRUBPF_F_NUMA_NODEBPF_F_RDONLYBPF_F_WRONLYBPF_F_STACK_BUILD_IDBPF_F_ZERO_SEEDBPF_F_RDONLY_PROGBPF_F_WRONLY_PROGBPF_F_CLONEBPF_F_MMAPABLEBPF_F_PRESERVE_ELEMSBPF_F_INNER_MAP" +const _MapFlags_name = "BPF_F_NO_PREALLOCBPF_F_NO_COMMON_LRUBPF_F_NUMA_NODEBPF_F_RDONLYBPF_F_WRONLYBPF_F_STACK_BUILD_IDBPF_F_ZERO_SEEDBPF_F_RDONLY_PROGBPF_F_WRONLY_PROGBPF_F_CLONEBPF_F_MMAPABLEBPF_F_PRESERVE_ELEMSBPF_F_INNER_MAPBPF_F_LINKBPF_F_PATH_FD" var _MapFlags_map = map[MapFlags]string{ - 1: _MapFlags_name[0:17], - 2: _MapFlags_name[17:36], - 4: _MapFlags_name[36:51], - 8: _MapFlags_name[51:63], - 16: _MapFlags_name[63:75], - 32: _MapFlags_name[75:95], - 64: _MapFlags_name[95:110], - 128: _MapFlags_name[110:127], - 256: _MapFlags_name[127:144], - 512: _MapFlags_name[144:155], - 1024: _MapFlags_name[155:169], - 2048: _MapFlags_name[169:189], - 4096: _MapFlags_name[189:204], + 1: _MapFlags_name[0:17], + 2: _MapFlags_name[17:36], + 4: _MapFlags_name[36:51], + 8: _MapFlags_name[51:63], + 16: _MapFlags_name[63:75], + 32: _MapFlags_name[75:95], + 64: _MapFlags_name[95:110], + 128: _MapFlags_name[110:127], + 256: _MapFlags_name[127:144], + 512: _MapFlags_name[144:155], + 1024: _MapFlags_name[155:169], + 2048: _MapFlags_name[169:189], + 4096: _MapFlags_name[189:204], + 8192: _MapFlags_name[204:214], + 16384: _MapFlags_name[214:227], } func (i MapFlags) String() string { diff --git a/vendor/github.com/cilium/ebpf/internal/sys/signals.go b/vendor/github.com/cilium/ebpf/internal/sys/signals.go index 7494c030c01c..e5337191d69d 100644 --- a/vendor/github.com/cilium/ebpf/internal/sys/signals.go +++ b/vendor/github.com/cilium/ebpf/internal/sys/signals.go @@ -63,7 +63,7 @@ func sigsetAdd(set *unix.Sigset_t, signal unix.Signal) error { // For amd64, runtime.sigaddset() performs the following operation: // set[(signal-1)/32] |= 1 << ((uint32(signal) - 1) & 31) // - // This trick depends on sigset being two u32's, causing a signal in the the + // This trick depends on sigset being two u32's, causing a signal in the // bottom 31 bits to be written to the low word if bit 32 is low, or the high // word if bit 32 is high. diff --git a/vendor/github.com/cilium/ebpf/internal/sys/syscall.go b/vendor/github.com/cilium/ebpf/internal/sys/syscall.go index 4fae04db5d8c..f6b6e9345802 100644 --- a/vendor/github.com/cilium/ebpf/internal/sys/syscall.go +++ b/vendor/github.com/cilium/ebpf/internal/sys/syscall.go @@ -11,7 +11,7 @@ import ( // ENOTSUPP is a Linux internal error code that has leaked into UAPI. // // It is not the same as ENOTSUP or EOPNOTSUPP. -var ENOTSUPP = syscall.Errno(524) +const ENOTSUPP = syscall.Errno(524) // BPF wraps SYS_BPF. // @@ -71,12 +71,52 @@ func (i *LinkInfo) info() (unsafe.Pointer, uint32) { return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) } +func (i *TracingLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + +func (i *CgroupLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + +func (i *NetNsLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + +func (i *XDPLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + +func (i *TcxLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + +func (i *NetfilterLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + +func (i *NetkitLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + +func (i *KprobeMultiLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + +func (i *KprobeLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + var _ Info = (*BtfInfo)(nil) func (i *BtfInfo) info() (unsafe.Pointer, uint32) { return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) } +func (i *PerfEventLinkInfo) info() (unsafe.Pointer, uint32) { + return unsafe.Pointer(i), uint32(unsafe.Sizeof(*i)) +} + // ObjInfo retrieves information about a BPF Fd. // // info may be one of MapInfo, ProgInfo, LinkInfo and BtfInfo. @@ -123,7 +163,7 @@ type TypeID uint32 // MapFlags control map behaviour. type MapFlags uint32 -//go:generate stringer -type MapFlags +//go:generate go run golang.org/x/tools/cmd/stringer@latest -type MapFlags const ( BPF_F_NO_PREALLOC MapFlags = 1 << iota @@ -139,6 +179,17 @@ const ( BPF_F_MMAPABLE BPF_F_PRESERVE_ELEMS BPF_F_INNER_MAP + BPF_F_LINK + BPF_F_PATH_FD +) + +// Flags used by bpf_mprog. +const ( + BPF_F_REPLACE = 1 << (iota + 2) + BPF_F_BEFORE + BPF_F_AFTER + BPF_F_ID + BPF_F_LINK_MPROG = 1 << 13 // aka BPF_F_LINK ) // wrappedErrno wraps syscall.Errno to prevent direct comparisons with diff --git a/vendor/github.com/cilium/ebpf/internal/sys/types.go b/vendor/github.com/cilium/ebpf/internal/sys/types.go index 2af7759e5a30..70e754de71d4 100644 --- a/vendor/github.com/cilium/ebpf/internal/sys/types.go +++ b/vendor/github.com/cilium/ebpf/internal/sys/types.go @@ -59,7 +59,20 @@ const ( BPF_SK_REUSEPORT_SELECT_OR_MIGRATE AttachType = 40 BPF_PERF_EVENT AttachType = 41 BPF_TRACE_KPROBE_MULTI AttachType = 42 - __MAX_BPF_ATTACH_TYPE AttachType = 43 + BPF_LSM_CGROUP AttachType = 43 + BPF_STRUCT_OPS AttachType = 44 + BPF_NETFILTER AttachType = 45 + BPF_TCX_INGRESS AttachType = 46 + BPF_TCX_EGRESS AttachType = 47 + BPF_TRACE_UPROBE_MULTI AttachType = 48 + BPF_CGROUP_UNIX_CONNECT AttachType = 49 + BPF_CGROUP_UNIX_SENDMSG AttachType = 50 + BPF_CGROUP_UNIX_RECVMSG AttachType = 51 + BPF_CGROUP_UNIX_GETPEERNAME AttachType = 52 + BPF_CGROUP_UNIX_GETSOCKNAME AttachType = 53 + BPF_NETKIT_PRIMARY AttachType = 54 + BPF_NETKIT_PEER AttachType = 55 + __MAX_BPF_ATTACH_TYPE AttachType = 56 ) type Cmd uint32 @@ -311,7 +324,15 @@ const ( BPF_FUNC_dynptr_read FunctionId = 201 BPF_FUNC_dynptr_write FunctionId = 202 BPF_FUNC_dynptr_data FunctionId = 203 - __BPF_FUNC_MAX_ID FunctionId = 204 + BPF_FUNC_tcp_raw_gen_syncookie_ipv4 FunctionId = 204 + BPF_FUNC_tcp_raw_gen_syncookie_ipv6 FunctionId = 205 + BPF_FUNC_tcp_raw_check_syncookie_ipv4 FunctionId = 206 + BPF_FUNC_tcp_raw_check_syncookie_ipv6 FunctionId = 207 + BPF_FUNC_ktime_get_tai_ns FunctionId = 208 + BPF_FUNC_user_ringbuf_drain FunctionId = 209 + BPF_FUNC_cgrp_storage_get FunctionId = 210 + BPF_FUNC_cgrp_storage_delete FunctionId = 211 + __BPF_FUNC_MAX_ID FunctionId = 212 ) type HdrStartOff uint32 @@ -334,43 +355,63 @@ const ( BPF_LINK_TYPE_PERF_EVENT LinkType = 7 BPF_LINK_TYPE_KPROBE_MULTI LinkType = 8 BPF_LINK_TYPE_STRUCT_OPS LinkType = 9 - MAX_BPF_LINK_TYPE LinkType = 10 + BPF_LINK_TYPE_NETFILTER LinkType = 10 + BPF_LINK_TYPE_TCX LinkType = 11 + BPF_LINK_TYPE_UPROBE_MULTI LinkType = 12 + BPF_LINK_TYPE_NETKIT LinkType = 13 + __MAX_BPF_LINK_TYPE LinkType = 14 ) type MapType uint32 const ( - BPF_MAP_TYPE_UNSPEC MapType = 0 - BPF_MAP_TYPE_HASH MapType = 1 - BPF_MAP_TYPE_ARRAY MapType = 2 - BPF_MAP_TYPE_PROG_ARRAY MapType = 3 - BPF_MAP_TYPE_PERF_EVENT_ARRAY MapType = 4 - BPF_MAP_TYPE_PERCPU_HASH MapType = 5 - BPF_MAP_TYPE_PERCPU_ARRAY MapType = 6 - BPF_MAP_TYPE_STACK_TRACE MapType = 7 - BPF_MAP_TYPE_CGROUP_ARRAY MapType = 8 - BPF_MAP_TYPE_LRU_HASH MapType = 9 - BPF_MAP_TYPE_LRU_PERCPU_HASH MapType = 10 - BPF_MAP_TYPE_LPM_TRIE MapType = 11 - BPF_MAP_TYPE_ARRAY_OF_MAPS MapType = 12 - BPF_MAP_TYPE_HASH_OF_MAPS MapType = 13 - BPF_MAP_TYPE_DEVMAP MapType = 14 - BPF_MAP_TYPE_SOCKMAP MapType = 15 - BPF_MAP_TYPE_CPUMAP MapType = 16 - BPF_MAP_TYPE_XSKMAP MapType = 17 - BPF_MAP_TYPE_SOCKHASH MapType = 18 - BPF_MAP_TYPE_CGROUP_STORAGE MapType = 19 - BPF_MAP_TYPE_REUSEPORT_SOCKARRAY MapType = 20 - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE MapType = 21 - BPF_MAP_TYPE_QUEUE MapType = 22 - BPF_MAP_TYPE_STACK MapType = 23 - BPF_MAP_TYPE_SK_STORAGE MapType = 24 - BPF_MAP_TYPE_DEVMAP_HASH MapType = 25 - BPF_MAP_TYPE_STRUCT_OPS MapType = 26 - BPF_MAP_TYPE_RINGBUF MapType = 27 - BPF_MAP_TYPE_INODE_STORAGE MapType = 28 - BPF_MAP_TYPE_TASK_STORAGE MapType = 29 - BPF_MAP_TYPE_BLOOM_FILTER MapType = 30 + BPF_MAP_TYPE_UNSPEC MapType = 0 + BPF_MAP_TYPE_HASH MapType = 1 + BPF_MAP_TYPE_ARRAY MapType = 2 + BPF_MAP_TYPE_PROG_ARRAY MapType = 3 + BPF_MAP_TYPE_PERF_EVENT_ARRAY MapType = 4 + BPF_MAP_TYPE_PERCPU_HASH MapType = 5 + BPF_MAP_TYPE_PERCPU_ARRAY MapType = 6 + BPF_MAP_TYPE_STACK_TRACE MapType = 7 + BPF_MAP_TYPE_CGROUP_ARRAY MapType = 8 + BPF_MAP_TYPE_LRU_HASH MapType = 9 + BPF_MAP_TYPE_LRU_PERCPU_HASH MapType = 10 + BPF_MAP_TYPE_LPM_TRIE MapType = 11 + BPF_MAP_TYPE_ARRAY_OF_MAPS MapType = 12 + BPF_MAP_TYPE_HASH_OF_MAPS MapType = 13 + BPF_MAP_TYPE_DEVMAP MapType = 14 + BPF_MAP_TYPE_SOCKMAP MapType = 15 + BPF_MAP_TYPE_CPUMAP MapType = 16 + BPF_MAP_TYPE_XSKMAP MapType = 17 + BPF_MAP_TYPE_SOCKHASH MapType = 18 + BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED MapType = 19 + BPF_MAP_TYPE_CGROUP_STORAGE MapType = 19 + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY MapType = 20 + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED MapType = 21 + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE MapType = 21 + BPF_MAP_TYPE_QUEUE MapType = 22 + BPF_MAP_TYPE_STACK MapType = 23 + BPF_MAP_TYPE_SK_STORAGE MapType = 24 + BPF_MAP_TYPE_DEVMAP_HASH MapType = 25 + BPF_MAP_TYPE_STRUCT_OPS MapType = 26 + BPF_MAP_TYPE_RINGBUF MapType = 27 + BPF_MAP_TYPE_INODE_STORAGE MapType = 28 + BPF_MAP_TYPE_TASK_STORAGE MapType = 29 + BPF_MAP_TYPE_BLOOM_FILTER MapType = 30 + BPF_MAP_TYPE_USER_RINGBUF MapType = 31 + BPF_MAP_TYPE_CGRP_STORAGE MapType = 32 +) + +type PerfEventType uint32 + +const ( + BPF_PERF_EVENT_UNSPEC PerfEventType = 0 + BPF_PERF_EVENT_UPROBE PerfEventType = 1 + BPF_PERF_EVENT_URETPROBE PerfEventType = 2 + BPF_PERF_EVENT_KPROBE PerfEventType = 3 + BPF_PERF_EVENT_KRETPROBE PerfEventType = 4 + BPF_PERF_EVENT_TRACEPOINT PerfEventType = 5 + BPF_PERF_EVENT_EVENT PerfEventType = 6 ) type ProgType uint32 @@ -408,15 +449,17 @@ const ( BPF_PROG_TYPE_LSM ProgType = 29 BPF_PROG_TYPE_SK_LOOKUP ProgType = 30 BPF_PROG_TYPE_SYSCALL ProgType = 31 + BPF_PROG_TYPE_NETFILTER ProgType = 32 ) type RetCode uint32 const ( - BPF_OK RetCode = 0 - BPF_DROP RetCode = 2 - BPF_REDIRECT RetCode = 7 - BPF_LWT_REROUTE RetCode = 128 + BPF_OK RetCode = 0 + BPF_DROP RetCode = 2 + BPF_REDIRECT RetCode = 7 + BPF_LWT_REROUTE RetCode = 128 + BPF_FLOW_DISSECTOR_CONTINUE RetCode = 129 ) type SkAction uint32 @@ -440,6 +483,15 @@ const ( BPF_STATS_RUN_TIME StatsType = 0 ) +type TcxActionBase int32 + +const ( + TCX_NEXT TcxActionBase = -1 + TCX_PASS TcxActionBase = 0 + TCX_DROP TcxActionBase = 2 + TCX_REDIRECT TcxActionBase = 7 +) + type XdpAction uint32 const ( @@ -476,7 +528,7 @@ type LinkInfo struct { Id LinkID ProgId uint32 _ [4]byte - Extra [16]uint8 + Extra [48]uint8 } type MapInfo struct { @@ -521,10 +573,10 @@ type ProgInfo struct { JitedFuncLens uint64 BtfId BTFID FuncInfoRecSize uint32 - FuncInfo uint64 + FuncInfo Pointer NrFuncInfo uint32 NrLineInfo uint32 - LineInfo uint64 + LineInfo Pointer JitedLineInfo uint64 NrJitedLineInfo uint32 LineInfoRecSize uint32 @@ -535,6 +587,8 @@ type ProgInfo struct { RunCnt uint64 RecursionMisses uint64 VerifiedInsns uint32 + AttachBtfObjId BTFID + AttachBtfId TypeID _ [4]byte } @@ -583,12 +637,12 @@ func BtfGetNextId(attr *BtfGetNextIdAttr) error { } type BtfLoadAttr struct { - Btf Pointer - BtfLogBuf Pointer - BtfSize uint32 - BtfLogSize uint32 - BtfLogLevel uint32 - _ [4]byte + Btf Pointer + BtfLogBuf Pointer + BtfSize uint32 + BtfLogSize uint32 + BtfLogLevel uint32 + BtfLogTrueSize uint32 } func BtfLoad(attr *BtfLoadAttr) (*FD, error) { @@ -628,7 +682,7 @@ type LinkCreateAttr struct { AttachType AttachType Flags uint32 TargetBtfId TypeID - _ [28]byte + _ [44]byte } func LinkCreate(attr *LinkCreateAttr) (*FD, error) { @@ -646,7 +700,7 @@ type LinkCreateIterAttr struct { Flags uint32 IterInfo Pointer IterInfoLen uint32 - _ [20]byte + _ [36]byte } func LinkCreateIter(attr *LinkCreateIterAttr) (*FD, error) { @@ -667,6 +721,7 @@ type LinkCreateKprobeMultiAttr struct { Syms Pointer Addrs Pointer Cookies Pointer + _ [16]byte } func LinkCreateKprobeMulti(attr *LinkCreateKprobeMultiAttr) (*FD, error) { @@ -677,13 +732,52 @@ func LinkCreateKprobeMulti(attr *LinkCreateKprobeMultiAttr) (*FD, error) { return NewFD(int(fd)) } +type LinkCreateNetfilterAttr struct { + ProgFd uint32 + TargetFd uint32 + AttachType AttachType + Flags uint32 + Pf uint32 + Hooknum uint32 + Priority int32 + NetfilterFlags uint32 + _ [32]byte +} + +func LinkCreateNetfilter(attr *LinkCreateNetfilterAttr) (*FD, error) { + fd, err := BPF(BPF_LINK_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err != nil { + return nil, err + } + return NewFD(int(fd)) +} + +type LinkCreateNetkitAttr struct { + ProgFd uint32 + TargetIfindex uint32 + AttachType AttachType + Flags uint32 + RelativeFdOrId uint32 + _ [4]byte + ExpectedRevision uint64 + _ [32]byte +} + +func LinkCreateNetkit(attr *LinkCreateNetkitAttr) (*FD, error) { + fd, err := BPF(BPF_LINK_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err != nil { + return nil, err + } + return NewFD(int(fd)) +} + type LinkCreatePerfEventAttr struct { ProgFd uint32 TargetFd uint32 AttachType AttachType Flags uint32 BpfCookie uint64 - _ [24]byte + _ [40]byte } func LinkCreatePerfEvent(attr *LinkCreatePerfEventAttr) (*FD, error) { @@ -694,6 +788,25 @@ func LinkCreatePerfEvent(attr *LinkCreatePerfEventAttr) (*FD, error) { return NewFD(int(fd)) } +type LinkCreateTcxAttr struct { + ProgFd uint32 + TargetIfindex uint32 + AttachType AttachType + Flags uint32 + RelativeFdOrId uint32 + _ [4]byte + ExpectedRevision uint64 + _ [32]byte +} + +func LinkCreateTcx(attr *LinkCreateTcxAttr) (*FD, error) { + fd, err := BPF(BPF_LINK_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err != nil { + return nil, err + } + return NewFD(int(fd)) +} + type LinkCreateTracingAttr struct { ProgFd uint32 TargetFd uint32 @@ -702,7 +815,7 @@ type LinkCreateTracingAttr struct { TargetBtfId BTFID _ [4]byte Cookie uint64 - _ [16]byte + _ [32]byte } func LinkCreateTracing(attr *LinkCreateTracingAttr) (*FD, error) { @@ -713,6 +826,49 @@ func LinkCreateTracing(attr *LinkCreateTracingAttr) (*FD, error) { return NewFD(int(fd)) } +type LinkCreateUprobeMultiAttr struct { + ProgFd uint32 + TargetFd uint32 + AttachType AttachType + Flags uint32 + Path Pointer + Offsets Pointer + RefCtrOffsets Pointer + Cookies Pointer + Count uint32 + UprobeMultiFlags uint32 + Pid uint32 + _ [4]byte +} + +func LinkCreateUprobeMulti(attr *LinkCreateUprobeMultiAttr) (*FD, error) { + fd, err := BPF(BPF_LINK_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err != nil { + return nil, err + } + return NewFD(int(fd)) +} + +type LinkGetFdByIdAttr struct{ Id LinkID } + +func LinkGetFdById(attr *LinkGetFdByIdAttr) (*FD, error) { + fd, err := BPF(BPF_LINK_GET_FD_BY_ID, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err != nil { + return nil, err + } + return NewFD(int(fd)) +} + +type LinkGetNextIdAttr struct { + Id LinkID + NextId LinkID +} + +func LinkGetNextId(attr *LinkGetNextIdAttr) error { + _, err := BPF(BPF_LINK_GET_NEXT_ID, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + return err +} + type LinkUpdateAttr struct { LinkFd uint32 NewProgFd uint32 @@ -909,6 +1065,8 @@ type ObjGetAttr struct { Pathname Pointer BpfFd uint32 FileFlags uint32 + PathFd int32 + _ [4]byte } func ObjGet(attr *ObjGetAttr) (*FD, error) { @@ -934,6 +1092,8 @@ type ObjPinAttr struct { Pathname Pointer BpfFd uint32 FileFlags uint32 + PathFd int32 + _ [4]byte } func ObjPin(attr *ObjPinAttr) error { @@ -942,11 +1102,13 @@ func ObjPin(attr *ObjPinAttr) error { } type ProgAttachAttr struct { - TargetFd uint32 - AttachBpfFd uint32 - AttachType uint32 - AttachFlags uint32 - ReplaceBpfFd uint32 + TargetFdOrIfindex uint32 + AttachBpfFd uint32 + AttachType uint32 + AttachFlags uint32 + ReplaceBpfFd uint32 + RelativeFdOrId uint32 + ExpectedRevision uint64 } func ProgAttach(attr *ProgAttachAttr) error { @@ -966,9 +1128,13 @@ func ProgBindMap(attr *ProgBindMapAttr) error { } type ProgDetachAttr struct { - TargetFd uint32 - AttachBpfFd uint32 - AttachType uint32 + TargetFdOrIfindex uint32 + AttachBpfFd uint32 + AttachType uint32 + AttachFlags uint32 + _ [4]byte + RelativeFdOrId uint32 + ExpectedRevision uint64 } func ProgDetach(attr *ProgDetachAttr) error { @@ -1022,7 +1188,7 @@ type ProgLoadAttr struct { FdArray Pointer CoreRelos Pointer CoreReloRecSize uint32 - _ [4]byte + LogTrueSize uint32 } func ProgLoad(attr *ProgLoadAttr) (*FD, error) { @@ -1034,13 +1200,17 @@ func ProgLoad(attr *ProgLoadAttr) (*FD, error) { } type ProgQueryAttr struct { - TargetFd uint32 - AttachType AttachType - QueryFlags uint32 - AttachFlags uint32 - ProgIds Pointer - ProgCount uint32 - _ [4]byte + TargetFdOrIfindex uint32 + AttachType AttachType + QueryFlags uint32 + AttachFlags uint32 + ProgIds Pointer + Count uint32 + _ [4]byte + ProgAttachFlags Pointer + LinkIds Pointer + LinkAttachFlags Pointer + Revision uint64 } func ProgQuery(attr *ProgQueryAttr) error { @@ -1087,31 +1257,127 @@ func RawTracepointOpen(attr *RawTracepointOpenAttr) (*FD, error) { } type CgroupLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte CgroupId uint64 AttachType AttachType - _ [4]byte + _ [36]byte } type IterLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte TargetName Pointer TargetNameLen uint32 } +type KprobeLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte + PerfEventType PerfEventType + _ [4]byte + FuncName Pointer + NameLen uint32 + Offset uint32 + Addr uint64 + Missed uint64 + _ [8]byte +} + +type KprobeMultiLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte + Addrs Pointer + Count uint32 + Flags uint32 + Missed uint64 + _ [24]byte +} + type NetNsLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte NetnsIno uint32 AttachType AttachType + _ [40]byte +} + +type NetfilterLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte + Pf uint32 + Hooknum uint32 + Priority int32 + Flags uint32 + _ [32]byte +} + +type NetkitLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte + Ifindex uint32 + AttachType AttachType + _ [40]byte +} + +type PerfEventLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte + PerfEventType PerfEventType } type RawTracepointLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte TpName Pointer TpNameLen uint32 - _ [4]byte + _ [36]byte +} + +type TcxLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte + Ifindex uint32 + AttachType AttachType + _ [40]byte } type TracingLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte AttachType AttachType TargetObjId uint32 TargetBtfId TypeID + _ [36]byte } -type XDPLinkInfo struct{ Ifindex uint32 } +type XDPLinkInfo struct { + Type LinkType + Id LinkID + ProgId uint32 + _ [4]byte + Ifindex uint32 + _ [44]byte +} diff --git a/vendor/github.com/cilium/ebpf/internal/sysenc/buffer.go b/vendor/github.com/cilium/ebpf/internal/sysenc/buffer.go new file mode 100644 index 000000000000..d184ea196aeb --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/sysenc/buffer.go @@ -0,0 +1,83 @@ +package sysenc + +import ( + "unsafe" + + "github.com/cilium/ebpf/internal/sys" +) + +type Buffer struct { + ptr unsafe.Pointer + // Size of the buffer. syscallPointerOnly if created from UnsafeBuffer or when using + // zero-copy unmarshaling. + size int +} + +const syscallPointerOnly = -1 + +func newBuffer(buf []byte) Buffer { + if len(buf) == 0 { + return Buffer{} + } + return Buffer{unsafe.Pointer(&buf[0]), len(buf)} +} + +// UnsafeBuffer constructs a Buffer for zero-copy unmarshaling. +// +// [Pointer] is the only valid method to call on such a Buffer. +// Use [SyscallBuffer] instead if possible. +func UnsafeBuffer(ptr unsafe.Pointer) Buffer { + return Buffer{ptr, syscallPointerOnly} +} + +// SyscallOutput prepares a Buffer for a syscall to write into. +// +// size is the length of the desired buffer in bytes. +// The buffer may point at the underlying memory of dst, in which case [Unmarshal] +// becomes a no-op. +// +// The contents of the buffer are undefined and may be non-zero. +func SyscallOutput(dst any, size int) Buffer { + if dstBuf := unsafeBackingMemory(dst); len(dstBuf) == size { + buf := newBuffer(dstBuf) + buf.size = syscallPointerOnly + return buf + } + + return newBuffer(make([]byte, size)) +} + +// CopyTo copies the buffer into dst. +// +// Returns the number of copied bytes. +func (b Buffer) CopyTo(dst []byte) int { + return copy(dst, b.unsafeBytes()) +} + +// AppendTo appends the buffer onto dst. +func (b Buffer) AppendTo(dst []byte) []byte { + return append(dst, b.unsafeBytes()...) +} + +// Pointer returns the location where a syscall should write. +func (b Buffer) Pointer() sys.Pointer { + // NB: This deliberately ignores b.length to support zero-copy + // marshaling / unmarshaling using unsafe.Pointer. + return sys.NewPointer(b.ptr) +} + +// Unmarshal the buffer into the provided value. +func (b Buffer) Unmarshal(data any) error { + if b.size == syscallPointerOnly { + return nil + } + + return Unmarshal(data, b.unsafeBytes()) +} + +func (b Buffer) unsafeBytes() []byte { + if b.size == syscallPointerOnly { + return nil + } + return unsafe.Slice((*byte)(b.ptr), b.size) +} diff --git a/vendor/github.com/cilium/ebpf/internal/sysenc/doc.go b/vendor/github.com/cilium/ebpf/internal/sysenc/doc.go new file mode 100644 index 000000000000..676ad98ba1b8 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/sysenc/doc.go @@ -0,0 +1,3 @@ +// Package sysenc provides efficient conversion of Go values to system +// call interfaces. +package sysenc diff --git a/vendor/github.com/cilium/ebpf/internal/sysenc/layout.go b/vendor/github.com/cilium/ebpf/internal/sysenc/layout.go new file mode 100644 index 000000000000..52d111e7aff7 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/sysenc/layout.go @@ -0,0 +1,41 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.dev/LICENSE. + +package sysenc + +import ( + "reflect" + "sync" +) + +var hasUnexportedFieldsCache sync.Map // map[reflect.Type]bool + +func hasUnexportedFields(typ reflect.Type) bool { + switch typ.Kind() { + case reflect.Slice, reflect.Array, reflect.Pointer: + return hasUnexportedFields(typ.Elem()) + + case reflect.Struct: + if unexported, ok := hasUnexportedFieldsCache.Load(typ); ok { + return unexported.(bool) + } + + unexported := false + for i, n := 0, typ.NumField(); i < n; i++ { + field := typ.Field(i) + // Package binary allows _ fields but always writes zeroes into them. + if (!field.IsExported() && field.Name != "_") || hasUnexportedFields(field.Type) { + unexported = true + break + } + } + + hasUnexportedFieldsCache.Store(typ, unexported) + return unexported + + default: + // NB: It's not clear what this means for Chan and so on. + return false + } +} diff --git a/vendor/github.com/cilium/ebpf/internal/sysenc/marshal.go b/vendor/github.com/cilium/ebpf/internal/sysenc/marshal.go new file mode 100644 index 000000000000..0026af8f24fb --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/sysenc/marshal.go @@ -0,0 +1,177 @@ +package sysenc + +import ( + "bytes" + "encoding" + "encoding/binary" + "errors" + "fmt" + "reflect" + "slices" + "sync" + "unsafe" + + "github.com/cilium/ebpf/internal" +) + +// Marshal turns data into a byte slice using the system's native endianness. +// +// If possible, avoids allocations by directly using the backing memory +// of data. This means that the variable must not be modified for the lifetime +// of the returned [Buffer]. +// +// Returns an error if the data can't be turned into a byte slice according to +// the behaviour of [binary.Write]. +func Marshal(data any, size int) (Buffer, error) { + if data == nil { + return Buffer{}, errors.New("can't marshal a nil value") + } + + var buf []byte + var err error + switch value := data.(type) { + case encoding.BinaryMarshaler: + buf, err = value.MarshalBinary() + case string: + buf = unsafe.Slice(unsafe.StringData(value), len(value)) + case []byte: + buf = value + case int16: + buf = internal.NativeEndian.AppendUint16(make([]byte, 0, 2), uint16(value)) + case uint16: + buf = internal.NativeEndian.AppendUint16(make([]byte, 0, 2), value) + case int32: + buf = internal.NativeEndian.AppendUint32(make([]byte, 0, 4), uint32(value)) + case uint32: + buf = internal.NativeEndian.AppendUint32(make([]byte, 0, 4), value) + case int64: + buf = internal.NativeEndian.AppendUint64(make([]byte, 0, 8), uint64(value)) + case uint64: + buf = internal.NativeEndian.AppendUint64(make([]byte, 0, 8), value) + default: + if buf := unsafeBackingMemory(data); len(buf) == size { + return newBuffer(buf), nil + } + + wr := internal.NewBuffer(make([]byte, 0, size)) + defer internal.PutBuffer(wr) + + err = binary.Write(wr, internal.NativeEndian, value) + buf = wr.Bytes() + } + if err != nil { + return Buffer{}, err + } + + if len(buf) != size { + return Buffer{}, fmt.Errorf("%T doesn't marshal to %d bytes", data, size) + } + + return newBuffer(buf), nil +} + +var bytesReaderPool = sync.Pool{ + New: func() interface{} { + return new(bytes.Reader) + }, +} + +// Unmarshal a byte slice in the system's native endianness into data. +// +// Returns an error if buf can't be unmarshalled according to the behaviour +// of [binary.Read]. +func Unmarshal(data interface{}, buf []byte) error { + switch value := data.(type) { + case encoding.BinaryUnmarshaler: + return value.UnmarshalBinary(buf) + + case *string: + *value = string(buf) + return nil + + case *[]byte: + // Backwards compat: unmarshaling into a slice replaces the whole slice. + *value = slices.Clone(buf) + return nil + + default: + if dataBuf := unsafeBackingMemory(data); len(dataBuf) == len(buf) { + copy(dataBuf, buf) + return nil + } + + rd := bytesReaderPool.Get().(*bytes.Reader) + defer bytesReaderPool.Put(rd) + + rd.Reset(buf) + + if err := binary.Read(rd, internal.NativeEndian, value); err != nil { + return err + } + + if rd.Len() != 0 { + return fmt.Errorf("unmarshaling %T doesn't consume all data", data) + } + + return nil + } +} + +// unsafeBackingMemory returns the backing memory of data if it can be used +// instead of calling into package binary. +// +// Returns nil if the value is not a pointer or a slice, or if it contains +// padding or unexported fields. +func unsafeBackingMemory(data any) []byte { + if data == nil { + return nil + } + + value := reflect.ValueOf(data) + var valueSize int + switch value.Kind() { + case reflect.Pointer: + if value.IsNil() { + return nil + } + + if elemType := value.Type().Elem(); elemType.Kind() != reflect.Slice { + valueSize = int(elemType.Size()) + break + } + + // We're dealing with a pointer to a slice. Dereference and + // handle it like a regular slice. + value = value.Elem() + fallthrough + + case reflect.Slice: + valueSize = int(value.Type().Elem().Size()) * value.Len() + + default: + // Prevent Value.UnsafePointer from panicking. + return nil + } + + // Some nil pointer types currently crash binary.Size. Call it after our own + // code so that the panic isn't reachable. + // See https://github.com/golang/go/issues/60892 + if size := binary.Size(data); size == -1 || size != valueSize { + // The type contains padding or unsupported types. + return nil + } + + if hasUnexportedFields(reflect.TypeOf(data)) { + return nil + } + + // Reinterpret the pointer as a byte slice. This violates the unsafe.Pointer + // rules because it's very unlikely that the source data has "an equivalent + // memory layout". However, we can make it safe-ish because of the + // following reasons: + // - There is no alignment mismatch since we cast to a type with an + // alignment of 1. + // - There are no pointers in the source type so we don't upset the GC. + // - The length is verified at runtime. + return unsafe.Slice((*byte)(value.UnsafePointer()), valueSize) +} diff --git a/vendor/github.com/cilium/ebpf/internal/tracefs/kprobe.go b/vendor/github.com/cilium/ebpf/internal/tracefs/kprobe.go index 4059a099b087..897740fec0cb 100644 --- a/vendor/github.com/cilium/ebpf/internal/tracefs/kprobe.go +++ b/vendor/github.com/cilium/ebpf/internal/tracefs/kprobe.go @@ -8,6 +8,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "syscall" "github.com/cilium/ebpf/internal" @@ -20,7 +21,7 @@ var ( ErrInvalidMaxActive = errors.New("can only set maxactive on kretprobes") ) -//go:generate stringer -type=ProbeType -linecomment +//go:generate go run golang.org/x/tools/cmd/stringer@latest -type=ProbeType -linecomment type ProbeType uint8 @@ -110,7 +111,7 @@ func sanitizeTracefsPath(path ...string) (string, error) { // Since kernel 4.1 tracefs should be mounted by default at /sys/kernel/tracing, // but may be also be available at /sys/kernel/debug/tracing if debugfs is mounted. // The available tracefs paths will depends on distribution choices. -var getTracefsPath = internal.Memoize(func() (string, error) { +var getTracefsPath = sync.OnceValues(func() (string, error) { for _, p := range []struct { path string fsType int64 diff --git a/vendor/github.com/cilium/ebpf/internal/unix/types_linux.go b/vendor/github.com/cilium/ebpf/internal/unix/types_linux.go index 7c9705919a36..d725cfaa3941 100644 --- a/vendor/github.com/cilium/ebpf/internal/unix/types_linux.go +++ b/vendor/github.com/cilium/ebpf/internal/unix/types_linux.go @@ -25,6 +25,7 @@ const ( EACCES = linux.EACCES EILSEQ = linux.EILSEQ EOPNOTSUPP = linux.EOPNOTSUPP + ESTALE = linux.ESTALE ) const ( @@ -39,6 +40,8 @@ const ( BPF_F_MMAPABLE = linux.BPF_F_MMAPABLE BPF_F_INNER_MAP = linux.BPF_F_INNER_MAP BPF_F_KPROBE_MULTI_RETURN = linux.BPF_F_KPROBE_MULTI_RETURN + BPF_F_UPROBE_MULTI_RETURN = linux.BPF_F_UPROBE_MULTI_RETURN + BPF_F_LOCK = linux.BPF_F_LOCK BPF_OBJ_NAME_LEN = linux.BPF_OBJ_NAME_LEN BPF_TAG_SIZE = linux.BPF_TAG_SIZE BPF_RINGBUF_BUSY_BIT = linux.BPF_RINGBUF_BUSY_BIT @@ -85,6 +88,8 @@ const ( BPF_FS_MAGIC = linux.BPF_FS_MAGIC TRACEFS_MAGIC = linux.TRACEFS_MAGIC DEBUGFS_MAGIC = linux.DEBUGFS_MAGIC + BPF_RB_NO_WAKEUP = linux.BPF_RB_NO_WAKEUP + BPF_RB_FORCE_WAKEUP = linux.BPF_RB_FORCE_WAKEUP ) type Statfs_t = linux.Statfs_t @@ -96,6 +101,7 @@ type PerfEventMmapPage = linux.PerfEventMmapPage type EpollEvent = linux.EpollEvent type PerfEventAttr = linux.PerfEventAttr type Utsname = linux.Utsname +type CPUSet = linux.CPUSet func Syscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) { return linux.Syscall(trap, a1, a2, a3) @@ -200,3 +206,11 @@ func Fstat(fd int, stat *Stat_t) error { func SetsockoptInt(fd, level, opt, value int) error { return linux.SetsockoptInt(fd, level, opt, value) } + +func SchedSetaffinity(pid int, set *CPUSet) error { + return linux.SchedSetaffinity(pid, set) +} + +func SchedGetaffinity(pid int, set *CPUSet) error { + return linux.SchedGetaffinity(pid, set) +} diff --git a/vendor/github.com/cilium/ebpf/internal/unix/types_other.go b/vendor/github.com/cilium/ebpf/internal/unix/types_other.go index 5e86b5052a19..3ff8962716a2 100644 --- a/vendor/github.com/cilium/ebpf/internal/unix/types_other.go +++ b/vendor/github.com/cilium/ebpf/internal/unix/types_other.go @@ -27,6 +27,7 @@ const ( EACCES EILSEQ EOPNOTSUPP + ESTALE ) // Constants are distinct to avoid breaking switch statements. @@ -41,6 +42,7 @@ const ( BPF_F_MMAPABLE BPF_F_INNER_MAP BPF_F_KPROBE_MULTI_RETURN + BPF_F_UPROBE_MULTI_RETURN BPF_F_XDP_HAS_FRAGS BPF_OBJ_NAME_LEN BPF_TAG_SIZE @@ -89,6 +91,9 @@ const ( BPF_FS_MAGIC TRACEFS_MAGIC DEBUGFS_MAGIC + BPF_RB_NO_WAKEUP + BPF_RB_FORCE_WAKEUP + BPF_F_LOCK ) type Statfs_t struct { @@ -292,3 +297,15 @@ func Fstat(fd int, stat *Stat_t) error { func SetsockoptInt(fd, level, opt, value int) error { return errNonLinux } + +type CPUSet struct{} + +func (*CPUSet) Set(int) {} + +func SchedSetaffinity(pid int, set *CPUSet) error { + return errNonLinux +} + +func SchedGetaffinity(pid int, set *CPUSet) error { + return errNonLinux +} diff --git a/vendor/github.com/cilium/ebpf/internal/vdso.go b/vendor/github.com/cilium/ebpf/internal/vdso.go index 10e639bf06e9..1049278554e0 100644 --- a/vendor/github.com/cilium/ebpf/internal/vdso.go +++ b/vendor/github.com/cilium/ebpf/internal/vdso.go @@ -19,16 +19,11 @@ var ( // vdsoVersion returns the LINUX_VERSION_CODE embedded in the vDSO library // linked into the current process image. func vdsoVersion() (uint32, error) { - // Read data from the auxiliary vector, which is normally passed directly - // to the process. Go does not expose that data, so we must read it from procfs. - // https://man7.org/linux/man-pages/man3/getauxval.3.html - av, err := os.Open("/proc/self/auxv") - if errors.Is(err, unix.EACCES) { - return 0, fmt.Errorf("opening auxv: %w (process may not be dumpable due to file capabilities)", err) - } + av, err := newAuxvRuntimeReader() if err != nil { - return 0, fmt.Errorf("opening auxv: %w", err) + return 0, err } + defer av.Close() vdsoAddr, err := vdsoMemoryAddress(av) @@ -54,24 +49,19 @@ func vdsoVersion() (uint32, error) { // vdsoMemoryAddress returns the memory address of the vDSO library // linked into the current process image. r is an io.Reader into an auxv blob. -func vdsoMemoryAddress(r io.Reader) (uint64, error) { - const ( - _AT_NULL = 0 // End of vector - _AT_SYSINFO_EHDR = 33 // Offset to vDSO blob in process image - ) - +func vdsoMemoryAddress(r auxvPairReader) (uintptr, error) { // Loop through all tag/value pairs in auxv until we find `AT_SYSINFO_EHDR`, // the address of a page containing the virtual Dynamic Shared Object (vDSO). - aux := struct{ Tag, Val uint64 }{} for { - if err := binary.Read(r, NativeEndian, &aux); err != nil { - return 0, fmt.Errorf("reading auxv entry: %w", err) + tag, value, err := r.ReadAuxvPair() + if err != nil { + return 0, err } - switch aux.Tag { + switch tag { case _AT_SYSINFO_EHDR: - if aux.Val != 0 { - return aux.Val, nil + if value != 0 { + return uintptr(value), nil } return 0, fmt.Errorf("invalid vDSO address in auxv") // _AT_NULL is always the last tag/val pair in the aux vector diff --git a/vendor/github.com/cilium/ebpf/internal/version.go b/vendor/github.com/cilium/ebpf/internal/version.go index 9b17ffb44de5..acd4650af732 100644 --- a/vendor/github.com/cilium/ebpf/internal/version.go +++ b/vendor/github.com/cilium/ebpf/internal/version.go @@ -2,6 +2,7 @@ package internal import ( "fmt" + "sync" "github.com/cilium/ebpf/internal/unix" ) @@ -79,7 +80,7 @@ func (v Version) Kernel() uint32 { } // KernelVersion returns the version of the currently running kernel. -var KernelVersion = Memoize(func() (Version, error) { +var KernelVersion = sync.OnceValues(func() (Version, error) { return detectKernelVersion() }) diff --git a/vendor/github.com/cilium/ebpf/link/anchor.go b/vendor/github.com/cilium/ebpf/link/anchor.go new file mode 100644 index 000000000000..1a3b5f7681fc --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/anchor.go @@ -0,0 +1,137 @@ +package link + +import ( + "fmt" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/sys" +) + +const anchorFlags = sys.BPF_F_REPLACE | + sys.BPF_F_BEFORE | + sys.BPF_F_AFTER | + sys.BPF_F_ID | + sys.BPF_F_LINK_MPROG + +// Anchor is a reference to a link or program. +// +// It is used to describe where an attachment or detachment should take place +// for link types which support multiple attachment. +type Anchor interface { + // anchor returns an fd or ID and a set of flags. + // + // By default fdOrID is taken to reference a program, but BPF_F_LINK_MPROG + // changes this to refer to a link instead. + // + // BPF_F_BEFORE, BPF_F_AFTER, BPF_F_REPLACE modify where a link or program + // is attached. The default behaviour if none of these flags is specified + // matches BPF_F_AFTER. + anchor() (fdOrID, flags uint32, _ error) +} + +type firstAnchor struct{} + +func (firstAnchor) anchor() (fdOrID, flags uint32, _ error) { + return 0, sys.BPF_F_BEFORE, nil +} + +// Head is the position before all other programs or links. +func Head() Anchor { + return firstAnchor{} +} + +type lastAnchor struct{} + +func (lastAnchor) anchor() (fdOrID, flags uint32, _ error) { + return 0, sys.BPF_F_AFTER, nil +} + +// Tail is the position after all other programs or links. +func Tail() Anchor { + return lastAnchor{} +} + +// Before is the position just in front of target. +func BeforeLink(target Link) Anchor { + return anchor{target, sys.BPF_F_BEFORE} +} + +// After is the position just after target. +func AfterLink(target Link) Anchor { + return anchor{target, sys.BPF_F_AFTER} +} + +// Before is the position just in front of target. +func BeforeLinkByID(target ID) Anchor { + return anchor{target, sys.BPF_F_BEFORE} +} + +// After is the position just after target. +func AfterLinkByID(target ID) Anchor { + return anchor{target, sys.BPF_F_AFTER} +} + +// Before is the position just in front of target. +func BeforeProgram(target *ebpf.Program) Anchor { + return anchor{target, sys.BPF_F_BEFORE} +} + +// After is the position just after target. +func AfterProgram(target *ebpf.Program) Anchor { + return anchor{target, sys.BPF_F_AFTER} +} + +// Replace the target itself. +func ReplaceProgram(target *ebpf.Program) Anchor { + return anchor{target, sys.BPF_F_REPLACE} +} + +// Before is the position just in front of target. +func BeforeProgramByID(target ebpf.ProgramID) Anchor { + return anchor{target, sys.BPF_F_BEFORE} +} + +// After is the position just after target. +func AfterProgramByID(target ebpf.ProgramID) Anchor { + return anchor{target, sys.BPF_F_AFTER} +} + +// Replace the target itself. +func ReplaceProgramByID(target ebpf.ProgramID) Anchor { + return anchor{target, sys.BPF_F_REPLACE} +} + +type anchor struct { + target any + position uint32 +} + +func (ap anchor) anchor() (fdOrID, flags uint32, _ error) { + var typeFlag uint32 + switch target := ap.target.(type) { + case *ebpf.Program: + fd := target.FD() + if fd < 0 { + return 0, 0, sys.ErrClosedFd + } + fdOrID = uint32(fd) + typeFlag = 0 + case ebpf.ProgramID: + fdOrID = uint32(target) + typeFlag = sys.BPF_F_ID + case interface{ FD() int }: + fd := target.FD() + if fd < 0 { + return 0, 0, sys.ErrClosedFd + } + fdOrID = uint32(fd) + typeFlag = sys.BPF_F_LINK_MPROG + case ID: + fdOrID = uint32(target) + typeFlag = sys.BPF_F_LINK_MPROG | sys.BPF_F_ID + default: + return 0, 0, fmt.Errorf("invalid target %T", ap.target) + } + + return fdOrID, ap.position | typeFlag, nil +} diff --git a/vendor/github.com/cilium/ebpf/link/cgroup.go b/vendor/github.com/cilium/ebpf/link/cgroup.go index 58e85fe9d473..f17d34f03c03 100644 --- a/vendor/github.com/cilium/ebpf/link/cgroup.go +++ b/vendor/github.com/cilium/ebpf/link/cgroup.go @@ -6,6 +6,7 @@ import ( "os" "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/sys" ) type cgroupAttachFlags uint32 @@ -143,8 +144,7 @@ func (cg *progAttachCgroup) Update(prog *ebpf.Program) error { // Atomically replacing multiple programs requires at least // 5.5 (commit 7dd68b3279f17921 "bpf: Support replacing cgroup-bpf // program in MULTI mode") - args.Flags |= uint32(flagReplace) - args.Replace = cg.current + args.Anchor = ReplaceProgram(cg.current) } if err := RawAttachProgram(args); err != nil { @@ -188,3 +188,21 @@ func newLinkCgroup(cgroup *os.File, attach ebpf.AttachType, prog *ebpf.Program) return &linkCgroup{*link}, err } + +func (cg *linkCgroup) Info() (*Info, error) { + var info sys.CgroupLinkInfo + if err := sys.ObjInfo(cg.fd, &info); err != nil { + return nil, fmt.Errorf("cgroup link info: %s", err) + } + extra := &CgroupInfo{ + CgroupId: info.CgroupId, + AttachType: info.AttachType, + } + + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil +} diff --git a/vendor/github.com/cilium/ebpf/link/iter.go b/vendor/github.com/cilium/ebpf/link/iter.go index d2b32ef331cd..0a39faef8838 100644 --- a/vendor/github.com/cilium/ebpf/link/iter.go +++ b/vendor/github.com/cilium/ebpf/link/iter.go @@ -25,10 +25,6 @@ type IterOptions struct { // AttachIter attaches a BPF seq_file iterator. func AttachIter(opts IterOptions) (*Iter, error) { - if err := haveBPFLink(); err != nil { - return nil, err - } - progFd := opts.Program.FD() if progFd < 0 { return nil, fmt.Errorf("invalid program: %s", sys.ErrClosedFd) @@ -52,6 +48,9 @@ func AttachIter(opts IterOptions) (*Iter, error) { fd, err := sys.LinkCreateIter(&attr) if err != nil { + if haveFeatErr := haveBPFLink(); haveFeatErr != nil { + return nil, haveFeatErr + } return nil, fmt.Errorf("can't link iterator: %w", err) } diff --git a/vendor/github.com/cilium/ebpf/link/kprobe.go b/vendor/github.com/cilium/ebpf/link/kprobe.go index b54ca9085335..fe3f17c37174 100644 --- a/vendor/github.com/cilium/ebpf/link/kprobe.go +++ b/vendor/github.com/cilium/ebpf/link/kprobe.go @@ -59,6 +59,8 @@ func (ko *KprobeOptions) cookie() uint64 { // If attaching to symbol fails, automatically retries with the running // platform's syscall prefix (e.g. __x64_) to support attaching to syscalls // in a portable fashion. +// +// The returned Link may implement [PerfEvent]. func Kprobe(symbol string, prog *ebpf.Program, opts *KprobeOptions) (Link, error) { k, err := kprobe(symbol, prog, opts, false) if err != nil { @@ -90,6 +92,8 @@ func Kprobe(symbol string, prog *ebpf.Program, opts *KprobeOptions) (Link, error // // On kernels 5.10 and earlier, setting a kretprobe on a nonexistent symbol // incorrectly returns unix.EINVAL instead of os.ErrNotExist. +// +// The returned Link may implement [PerfEvent]. func Kretprobe(symbol string, prog *ebpf.Program, opts *KprobeOptions) (Link, error) { k, err := kprobe(symbol, prog, opts, true) if err != nil { @@ -274,7 +278,11 @@ func pmuProbe(args tracefs.ProbeArgs) (*perfEvent, error) { } } - rawFd, err := unix.PerfEventOpen(&attr, args.Pid, 0, -1, unix.PERF_FLAG_FD_CLOEXEC) + cpu := 0 + if args.Pid != perfAllThreads { + cpu = -1 + } + rawFd, err := unix.PerfEventOpen(&attr, args.Pid, cpu, -1, unix.PERF_FLAG_FD_CLOEXEC) // On some old kernels, kprobe PMU doesn't allow `.` in symbol names and // return -EINVAL. Return ErrNotSupported to allow falling back to tracefs. diff --git a/vendor/github.com/cilium/ebpf/link/kprobe_multi.go b/vendor/github.com/cilium/ebpf/link/kprobe_multi.go index 697c6d7362a2..f7a8291f9453 100644 --- a/vendor/github.com/cilium/ebpf/link/kprobe_multi.go +++ b/vendor/github.com/cilium/ebpf/link/kprobe_multi.go @@ -82,10 +82,6 @@ func kprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions, flags uint32) (Lin return nil, fmt.Errorf("Cookies must be exactly Symbols or Addresses in length: %w", errInvalidInput) } - if err := haveBPFLinkKprobeMulti(); err != nil { - return nil, err - } - attr := &sys.LinkCreateKprobeMultiAttr{ ProgFd: uint32(prog.FD()), AttachType: sys.BPF_TRACE_KPROBE_MULTI, @@ -113,7 +109,11 @@ func kprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions, flags uint32) (Lin if errors.Is(err, unix.EINVAL) { return nil, fmt.Errorf("%w (missing kernel symbol or prog's AttachType not AttachTraceKprobeMulti?)", err) } + if err != nil { + if haveFeatErr := haveBPFLinkKprobeMulti(); haveFeatErr != nil { + return nil, haveFeatErr + } return nil, err } @@ -130,12 +130,23 @@ func (kml *kprobeMultiLink) Update(prog *ebpf.Program) error { return fmt.Errorf("update kprobe_multi: %w", ErrNotSupported) } -func (kml *kprobeMultiLink) Pin(string) error { - return fmt.Errorf("pin kprobe_multi: %w", ErrNotSupported) -} +func (kml *kprobeMultiLink) Info() (*Info, error) { + var info sys.KprobeMultiLinkInfo + if err := sys.ObjInfo(kml.fd, &info); err != nil { + return nil, fmt.Errorf("kprobe multi link info: %s", err) + } + extra := &KprobeMultiInfo{ + count: info.Count, + flags: info.Flags, + missed: info.Missed, + } -func (kml *kprobeMultiLink) Unpin() error { - return fmt.Errorf("unpin kprobe_multi: %w", ErrNotSupported) + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil } var haveBPFLinkKprobeMulti = internal.NewFeatureTest("bpf_link_kprobe_multi", "5.18", func() error { diff --git a/vendor/github.com/cilium/ebpf/link/link.go b/vendor/github.com/cilium/ebpf/link/link.go index 36acd6ee4b95..9c34616c9a9a 100644 --- a/vendor/github.com/cilium/ebpf/link/link.go +++ b/vendor/github.com/cilium/ebpf/link/link.go @@ -1,9 +1,9 @@ package link import ( - "bytes" - "encoding/binary" + "errors" "fmt" + "os" "github.com/cilium/ebpf" "github.com/cilium/ebpf/btf" @@ -48,8 +48,15 @@ type Link interface { // NewLinkFromFD creates a link from a raw fd. // -// You should not use fd after calling this function. +// Deprecated: use [NewFromFD] instead. func NewLinkFromFD(fd int) (Link, error) { + return NewFromFD(fd) +} + +// NewFromFD creates a link from a raw fd. +// +// You should not use fd after calling this function. +func NewFromFD(fd int) (Link, error) { sysFD, err := sys.NewFD(fd) if err != nil { return nil, err @@ -58,6 +65,19 @@ func NewLinkFromFD(fd int) (Link, error) { return wrapRawLink(&RawLink{fd: sysFD}) } +// NewFromID returns the link associated with the given id. +// +// Returns ErrNotExist if there is no link with the given id. +func NewFromID(id ID) (Link, error) { + getFdAttr := &sys.LinkGetFdByIdAttr{Id: id} + fd, err := sys.LinkGetFdById(getFdAttr) + if err != nil { + return nil, fmt.Errorf("get link fd from ID %d: %w", id, err) + } + + return wrapRawLink(&RawLink{fd, ""}) +} + // LoadPinnedLink loads a link that was persisted into a bpffs. func LoadPinnedLink(fileName string, opts *ebpf.LoadPinOptions) (Link, error) { raw, err := loadPinnedRawLink(fileName, opts) @@ -96,8 +116,18 @@ func wrapRawLink(raw *RawLink) (_ Link, err error) { return &NetNsLink{*raw}, nil case KprobeMultiType: return &kprobeMultiLink{*raw}, nil + case UprobeMultiType: + return &uprobeMultiLink{*raw}, nil case PerfEventType: - return nil, fmt.Errorf("recovering perf event fd: %w", ErrNotSupported) + return &perfEventLink{*raw, nil}, nil + case TCXType: + return &tcxLink{*raw}, nil + case NetfilterType: + return &netfilterLink{*raw}, nil + case NetkitType: + return &netkitLink{*raw}, nil + case XDPType: + return &xdpLink{*raw}, nil default: return raw, nil } @@ -128,10 +158,85 @@ type Info struct { extra interface{} } -type TracingInfo sys.TracingLinkInfo -type CgroupInfo sys.CgroupLinkInfo -type NetNsInfo sys.NetNsLinkInfo -type XDPInfo sys.XDPLinkInfo +type TracingInfo struct { + AttachType sys.AttachType + TargetObjId uint32 + TargetBtfId sys.TypeID +} + +type CgroupInfo struct { + CgroupId uint64 + AttachType sys.AttachType + _ [4]byte +} + +type NetNsInfo struct { + NetnsIno uint32 + AttachType sys.AttachType +} + +type TCXInfo struct { + Ifindex uint32 + AttachType sys.AttachType +} + +type XDPInfo struct { + Ifindex uint32 +} + +type NetfilterInfo struct { + Pf uint32 + Hooknum uint32 + Priority int32 + Flags uint32 +} + +type NetkitInfo struct { + Ifindex uint32 + AttachType sys.AttachType +} + +type KprobeMultiInfo struct { + count uint32 + flags uint32 + missed uint64 +} + +// AddressCount is the number of addresses hooked by the kprobe. +func (kpm *KprobeMultiInfo) AddressCount() (uint32, bool) { + return kpm.count, kpm.count > 0 +} + +func (kpm *KprobeMultiInfo) Flags() (uint32, bool) { + return kpm.flags, kpm.count > 0 +} + +func (kpm *KprobeMultiInfo) Missed() (uint64, bool) { + return kpm.missed, kpm.count > 0 +} + +type PerfEventInfo struct { + Type sys.PerfEventType + extra interface{} +} + +func (r *PerfEventInfo) Kprobe() *KprobeInfo { + e, _ := r.extra.(*KprobeInfo) + return e +} + +type KprobeInfo struct { + address uint64 + missed uint64 +} + +func (kp *KprobeInfo) Address() (uint64, bool) { + return kp.address, kp.address > 0 +} + +func (kp *KprobeInfo) Missed() (uint64, bool) { + return kp.missed, kp.address > 0 +} // Tracing returns tracing type-specific link info. // @@ -157,7 +262,7 @@ func (r Info) NetNs() *NetNsInfo { return e } -// ExtraNetNs returns XDP type-specific link info. +// XDP returns XDP type-specific link info. // // Returns nil if the type-specific link info isn't available. func (r Info) XDP() *XDPInfo { @@ -165,6 +270,46 @@ func (r Info) XDP() *XDPInfo { return e } +// TCX returns TCX type-specific link info. +// +// Returns nil if the type-specific link info isn't available. +func (r Info) TCX() *TCXInfo { + e, _ := r.extra.(*TCXInfo) + return e +} + +// Netfilter returns netfilter type-specific link info. +// +// Returns nil if the type-specific link info isn't available. +func (r Info) Netfilter() *NetfilterInfo { + e, _ := r.extra.(*NetfilterInfo) + return e +} + +// Netkit returns netkit type-specific link info. +// +// Returns nil if the type-specific link info isn't available. +func (r Info) Netkit() *NetkitInfo { + e, _ := r.extra.(*NetkitInfo) + return e +} + +// KprobeMulti returns kprobe-multi type-specific link info. +// +// Returns nil if the type-specific link info isn't available. +func (r Info) KprobeMulti() *KprobeMultiInfo { + e, _ := r.extra.(*KprobeMultiInfo) + return e +} + +// PerfEvent returns perf-event type-specific link info. +// +// Returns nil if the type-specific link info isn't available. +func (r Info) PerfEvent() *PerfEventInfo { + e, _ := r.extra.(*PerfEventInfo) + return e +} + // RawLink is the low-level API to bpf_link. // // You should consider using the higher level interfaces in this @@ -295,6 +440,9 @@ func (l *RawLink) UpdateArgs(opts RawLinkUpdateOptions) error { } // Info returns metadata about the link. +// +// Linktype specific metadata is not included and can be retrieved +// via the linktype specific Info() method. func (l *RawLink) Info() (*Info, error) { var info sys.LinkInfo @@ -302,35 +450,81 @@ func (l *RawLink) Info() (*Info, error) { return nil, fmt.Errorf("link info: %s", err) } - var extra interface{} - switch info.Type { - case CgroupType: - extra = &CgroupInfo{} - case NetNsType: - extra = &NetNsInfo{} - case TracingType: - extra = &TracingInfo{} - case XDPType: - extra = &XDPInfo{} - case RawTracepointType, IterType, - PerfEventType, KprobeMultiType: - // Extra metadata not supported. - default: - return nil, fmt.Errorf("unknown link info type: %d", info.Type) - } - - if extra != nil { - buf := bytes.NewReader(info.Extra[:]) - err := binary.Read(buf, internal.NativeEndian, extra) - if err != nil { - return nil, fmt.Errorf("cannot read extra link info: %w", err) - } - } - return &Info{ info.Type, info.Id, ebpf.ProgramID(info.ProgId), - extra, + nil, }, nil } + +// Iterator allows iterating over links attached into the kernel. +type Iterator struct { + // The ID of the current link. Only valid after a call to Next + ID ID + // The current link. Only valid until a call to Next. + // See Take if you want to retain the link. + Link Link + err error +} + +// Next retrieves the next link. +// +// Returns true if another link was found. Call [Iterator.Err] after the function returns false. +func (it *Iterator) Next() bool { + id := it.ID + for { + getIdAttr := &sys.LinkGetNextIdAttr{Id: id} + err := sys.LinkGetNextId(getIdAttr) + if errors.Is(err, os.ErrNotExist) { + // There are no more links. + break + } else if err != nil { + it.err = fmt.Errorf("get next link ID: %w", err) + break + } + + id = getIdAttr.NextId + l, err := NewFromID(id) + if errors.Is(err, os.ErrNotExist) { + // Couldn't load the link fast enough. Try next ID. + continue + } else if err != nil { + it.err = fmt.Errorf("get link for ID %d: %w", id, err) + break + } + + if it.Link != nil { + it.Link.Close() + } + it.ID, it.Link = id, l + return true + } + + // No more links or we encountered an error. + if it.Link != nil { + it.Link.Close() + } + it.Link = nil + return false +} + +// Take the ownership of the current link. +// +// It's the callers responsibility to close the link. +func (it *Iterator) Take() Link { + l := it.Link + it.Link = nil + return l +} + +// Err returns an error if iteration failed for some reason. +func (it *Iterator) Err() error { + return it.err +} + +func (it *Iterator) Close() { + if it.Link != nil { + it.Link.Close() + } +} diff --git a/vendor/github.com/cilium/ebpf/link/netfilter.go b/vendor/github.com/cilium/ebpf/link/netfilter.go new file mode 100644 index 000000000000..34be39085976 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/netfilter.go @@ -0,0 +1,90 @@ +package link + +import ( + "fmt" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/sys" +) + +const NetfilterIPDefrag NetfilterAttachFlags = 0 // Enable IP packet defragmentation + +type NetfilterAttachFlags uint32 + +type NetfilterOptions struct { + // Program must be a netfilter BPF program. + Program *ebpf.Program + // The protocol family. + ProtocolFamily uint32 + // The number of the hook you are interested in. + HookNumber uint32 + // Priority within hook + Priority int32 + // Extra link flags + Flags uint32 + // Netfilter flags + NetfilterFlags NetfilterAttachFlags +} + +type netfilterLink struct { + RawLink +} + +// AttachNetfilter links a netfilter BPF program to a netfilter hook. +func AttachNetfilter(opts NetfilterOptions) (Link, error) { + if opts.Program == nil { + return nil, fmt.Errorf("netfilter program is nil") + } + + if t := opts.Program.Type(); t != ebpf.Netfilter { + return nil, fmt.Errorf("invalid program type %s, expected netfilter", t) + } + + progFd := opts.Program.FD() + if progFd < 0 { + return nil, fmt.Errorf("invalid program: %s", sys.ErrClosedFd) + } + + attr := sys.LinkCreateNetfilterAttr{ + ProgFd: uint32(opts.Program.FD()), + AttachType: sys.BPF_NETFILTER, + Flags: opts.Flags, + Pf: uint32(opts.ProtocolFamily), + Hooknum: uint32(opts.HookNumber), + Priority: opts.Priority, + NetfilterFlags: uint32(opts.NetfilterFlags), + } + + fd, err := sys.LinkCreateNetfilter(&attr) + if err != nil { + return nil, fmt.Errorf("attach netfilter link: %w", err) + } + + return &netfilterLink{RawLink{fd, ""}}, nil +} + +func (*netfilterLink) Update(new *ebpf.Program) error { + return fmt.Errorf("netfilter update: %w", ErrNotSupported) +} + +func (nf *netfilterLink) Info() (*Info, error) { + var info sys.NetfilterLinkInfo + if err := sys.ObjInfo(nf.fd, &info); err != nil { + return nil, fmt.Errorf("netfilter link info: %s", err) + } + extra := &NetfilterInfo{ + Pf: info.Pf, + Hooknum: info.Hooknum, + Priority: info.Priority, + Flags: info.Flags, + } + + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil +} + +var _ Link = (*netfilterLink)(nil) diff --git a/vendor/github.com/cilium/ebpf/link/netkit.go b/vendor/github.com/cilium/ebpf/link/netkit.go new file mode 100644 index 000000000000..5eee3b023ae6 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/netkit.go @@ -0,0 +1,89 @@ +package link + +import ( + "fmt" + "runtime" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/sys" +) + +type NetkitOptions struct { + // Index of the interface to attach to. + Interface int + // Program to attach. + Program *ebpf.Program + // One of the AttachNetkit* constants. + Attach ebpf.AttachType + // Attach relative to an anchor. Optional. + Anchor Anchor + // Only attach if the expected revision matches. + ExpectedRevision uint64 + // Flags control the attach behaviour. Specify an Anchor instead of + // F_LINK, F_ID, F_BEFORE, F_AFTER and R_REPLACE. Optional. + Flags uint32 +} + +func AttachNetkit(opts NetkitOptions) (Link, error) { + if opts.Interface < 0 { + return nil, fmt.Errorf("interface %d is out of bounds", opts.Interface) + } + + if opts.Flags&anchorFlags != 0 { + return nil, fmt.Errorf("disallowed flags: use Anchor to specify attach target") + } + + attr := sys.LinkCreateNetkitAttr{ + ProgFd: uint32(opts.Program.FD()), + AttachType: sys.AttachType(opts.Attach), + TargetIfindex: uint32(opts.Interface), + ExpectedRevision: opts.ExpectedRevision, + Flags: opts.Flags, + } + + if opts.Anchor != nil { + fdOrID, flags, err := opts.Anchor.anchor() + if err != nil { + return nil, fmt.Errorf("attach netkit link: %w", err) + } + + attr.RelativeFdOrId = fdOrID + attr.Flags |= flags + } + + fd, err := sys.LinkCreateNetkit(&attr) + runtime.KeepAlive(opts.Program) + runtime.KeepAlive(opts.Anchor) + if err != nil { + if haveFeatErr := haveNetkit(); haveFeatErr != nil { + return nil, haveFeatErr + } + return nil, fmt.Errorf("attach netkit link: %w", err) + } + + return &netkitLink{RawLink{fd, ""}}, nil +} + +type netkitLink struct { + RawLink +} + +var _ Link = (*netkitLink)(nil) + +func (netkit *netkitLink) Info() (*Info, error) { + var info sys.NetkitLinkInfo + if err := sys.ObjInfo(netkit.fd, &info); err != nil { + return nil, fmt.Errorf("netkit link info: %s", err) + } + extra := &NetkitInfo{ + Ifindex: info.Ifindex, + AttachType: info.AttachType, + } + + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil +} diff --git a/vendor/github.com/cilium/ebpf/link/netns.go b/vendor/github.com/cilium/ebpf/link/netns.go index 344ecced6bea..b1edd340a3fb 100644 --- a/vendor/github.com/cilium/ebpf/link/netns.go +++ b/vendor/github.com/cilium/ebpf/link/netns.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/sys" ) // NetNsLink is a program attached to a network namespace. @@ -34,3 +35,21 @@ func AttachNetNs(ns int, prog *ebpf.Program) (*NetNsLink, error) { return &NetNsLink{*link}, nil } + +func (ns *NetNsLink) Info() (*Info, error) { + var info sys.NetNsLinkInfo + if err := sys.ObjInfo(ns.fd, &info); err != nil { + return nil, fmt.Errorf("netns link info: %s", err) + } + extra := &NetNsInfo{ + NetnsIno: info.NetnsIno, + AttachType: info.AttachType, + } + + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil +} diff --git a/vendor/github.com/cilium/ebpf/link/perf_event.go b/vendor/github.com/cilium/ebpf/link/perf_event.go index 5f7a628b3d73..1d8feb58c1c0 100644 --- a/vendor/github.com/cilium/ebpf/link/perf_event.go +++ b/vendor/github.com/cilium/ebpf/link/perf_event.go @@ -3,6 +3,7 @@ package link import ( "errors" "fmt" + "os" "runtime" "unsafe" @@ -78,6 +79,18 @@ func (pe *perfEvent) Close() error { return nil } +// PerfEvent is implemented by some Link types which use a perf event under +// the hood. +type PerfEvent interface { + // PerfEvent returns a file for the underlying perf event. + // + // It is the callers responsibility to close the returned file. + // + // Making changes to the associated perf event lead to + // undefined behaviour. + PerfEvent() (*os.File, error) +} + // perfEventLink represents a bpf perf link. type perfEventLink struct { RawLink @@ -86,30 +99,16 @@ type perfEventLink struct { func (pl *perfEventLink) isLink() {} -// Pinning requires the underlying perf event FD to stay open. -// -// | PerfEvent FD | BpfLink FD | Works | -// |--------------|------------|-------| -// | Open | Open | Yes | -// | Closed | Open | No | -// | Open | Closed | No (Pin() -> EINVAL) | -// | Closed | Closed | No (Pin() -> EINVAL) | -// -// There is currently no pretty way to recover the perf event FD -// when loading a pinned link, so leave as not supported for now. -func (pl *perfEventLink) Pin(string) error { - return fmt.Errorf("perf event link pin: %w", ErrNotSupported) -} - -func (pl *perfEventLink) Unpin() error { - return fmt.Errorf("perf event link unpin: %w", ErrNotSupported) -} - func (pl *perfEventLink) Close() error { if err := pl.fd.Close(); err != nil { return fmt.Errorf("perf link close: %w", err) } + // when created from pinned link + if pl.pe == nil { + return nil + } + if err := pl.pe.Close(); err != nil { return fmt.Errorf("perf event close: %w", err) } @@ -120,6 +119,54 @@ func (pl *perfEventLink) Update(prog *ebpf.Program) error { return fmt.Errorf("perf event link update: %w", ErrNotSupported) } +var _ PerfEvent = (*perfEventLink)(nil) + +func (pl *perfEventLink) PerfEvent() (*os.File, error) { + // when created from pinned link + if pl.pe == nil { + return nil, ErrNotSupported + } + + fd, err := pl.pe.fd.Dup() + if err != nil { + return nil, err + } + + return fd.File("perf-event"), nil +} + +func (pl *perfEventLink) Info() (*Info, error) { + var info sys.PerfEventLinkInfo + if err := sys.ObjInfo(pl.fd, &info); err != nil { + return nil, fmt.Errorf("perf event link info: %s", err) + } + + var extra2 interface{} + switch info.PerfEventType { + case sys.BPF_PERF_EVENT_KPROBE, sys.BPF_PERF_EVENT_KRETPROBE: + var kprobeInfo sys.KprobeLinkInfo + if err := sys.ObjInfo(pl.fd, &kprobeInfo); err != nil { + return nil, fmt.Errorf("kprobe link info: %s", err) + } + extra2 = &KprobeInfo{ + address: kprobeInfo.Addr, + missed: kprobeInfo.Missed, + } + } + + extra := &PerfEventInfo{ + Type: info.PerfEventType, + extra: extra2, + } + + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil +} + // perfEventIoctl implements Link and handles the perf event lifecycle // via ioctl(). type perfEventIoctl struct { @@ -154,6 +201,17 @@ func (pi *perfEventIoctl) Info() (*Info, error) { return nil, fmt.Errorf("perf event ioctl info: %w", ErrNotSupported) } +var _ PerfEvent = (*perfEventIoctl)(nil) + +func (pi *perfEventIoctl) PerfEvent() (*os.File, error) { + fd, err := pi.fd.Dup() + if err != nil { + return nil, err + } + + return fd.File("perf-event"), nil +} + // attach the given eBPF prog to the perf event stored in pe. // pe must contain a valid perf event fd. // prog's type must match the program type stored in pe. @@ -229,7 +287,11 @@ func openTracepointPerfEvent(tid uint64, pid int) (*sys.FD, error) { Wakeup: 1, } - fd, err := unix.PerfEventOpen(&attr, pid, 0, -1, unix.PERF_FLAG_FD_CLOEXEC) + cpu := 0 + if pid != perfAllThreads { + cpu = -1 + } + fd, err := unix.PerfEventOpen(&attr, pid, cpu, -1, unix.PERF_FLAG_FD_CLOEXEC) if err != nil { return nil, fmt.Errorf("opening tracepoint perf event: %w", err) } diff --git a/vendor/github.com/cilium/ebpf/link/program.go b/vendor/github.com/cilium/ebpf/link/program.go index ea31817377fc..d8a2a15f9379 100644 --- a/vendor/github.com/cilium/ebpf/link/program.go +++ b/vendor/github.com/cilium/ebpf/link/program.go @@ -2,22 +2,27 @@ package link import ( "fmt" + "runtime" "github.com/cilium/ebpf" "github.com/cilium/ebpf/internal/sys" ) type RawAttachProgramOptions struct { - // File descriptor to attach to. This differs for each attach type. + // Target to query. This is usually a file descriptor but may refer to + // something else based on the attach type. Target int // Program to attach. Program *ebpf.Program - // Program to replace (cgroups). - Replace *ebpf.Program - // Attach must match the attach type of Program (and Replace). + // Attach must match the attach type of Program. Attach ebpf.AttachType - // Flags control the attach behaviour. This differs for each attach type. + // Attach relative to an anchor. Optional. + Anchor Anchor + // Flags control the attach behaviour. Specify an Anchor instead of + // F_LINK, F_ID, F_BEFORE, F_AFTER and F_REPLACE. Optional. Flags uint32 + // Only attach if the internal revision matches the given value. + ExpectedRevision uint64 } // RawAttachProgram is a low level wrapper around BPF_PROG_ATTACH. @@ -25,50 +30,76 @@ type RawAttachProgramOptions struct { // You should use one of the higher level abstractions available in this // package if possible. func RawAttachProgram(opts RawAttachProgramOptions) error { - if err := haveProgAttach(); err != nil { - return err + if opts.Flags&anchorFlags != 0 { + return fmt.Errorf("disallowed flags: use Anchor to specify attach target") } - var replaceFd uint32 - if opts.Replace != nil { - replaceFd = uint32(opts.Replace.FD()) + attr := sys.ProgAttachAttr{ + TargetFdOrIfindex: uint32(opts.Target), + AttachBpfFd: uint32(opts.Program.FD()), + AttachType: uint32(opts.Attach), + AttachFlags: uint32(opts.Flags), + ExpectedRevision: opts.ExpectedRevision, } - attr := sys.ProgAttachAttr{ - TargetFd: uint32(opts.Target), - AttachBpfFd: uint32(opts.Program.FD()), - ReplaceBpfFd: replaceFd, - AttachType: uint32(opts.Attach), - AttachFlags: uint32(opts.Flags), + if opts.Anchor != nil { + fdOrID, flags, err := opts.Anchor.anchor() + if err != nil { + return fmt.Errorf("attach program: %w", err) + } + + if flags == sys.BPF_F_REPLACE { + // Ensure that replacing a program works on old kernels. + attr.ReplaceBpfFd = fdOrID + } else { + attr.RelativeFdOrId = fdOrID + attr.AttachFlags |= flags + } } if err := sys.ProgAttach(&attr); err != nil { - return fmt.Errorf("can't attach program: %w", err) + if haveFeatErr := haveProgAttach(); haveFeatErr != nil { + return haveFeatErr + } + return fmt.Errorf("attach program: %w", err) } + runtime.KeepAlive(opts.Program) + return nil } -type RawDetachProgramOptions struct { - Target int - Program *ebpf.Program - Attach ebpf.AttachType -} +type RawDetachProgramOptions RawAttachProgramOptions // RawDetachProgram is a low level wrapper around BPF_PROG_DETACH. // // You should use one of the higher level abstractions available in this // package if possible. func RawDetachProgram(opts RawDetachProgramOptions) error { - if err := haveProgAttach(); err != nil { - return err + if opts.Flags&anchorFlags != 0 { + return fmt.Errorf("disallowed flags: use Anchor to specify attach target") } attr := sys.ProgDetachAttr{ - TargetFd: uint32(opts.Target), - AttachBpfFd: uint32(opts.Program.FD()), - AttachType: uint32(opts.Attach), + TargetFdOrIfindex: uint32(opts.Target), + AttachBpfFd: uint32(opts.Program.FD()), + AttachType: uint32(opts.Attach), + ExpectedRevision: opts.ExpectedRevision, } + + if opts.Anchor != nil { + fdOrID, flags, err := opts.Anchor.anchor() + if err != nil { + return fmt.Errorf("detach program: %w", err) + } + + attr.RelativeFdOrId = fdOrID + attr.AttachFlags |= flags + } + if err := sys.ProgDetach(&attr); err != nil { + if haveFeatErr := haveProgAttach(); haveFeatErr != nil { + return haveFeatErr + } return fmt.Errorf("can't detach program: %w", err) } diff --git a/vendor/github.com/cilium/ebpf/link/query.go b/vendor/github.com/cilium/ebpf/link/query.go index c05656512d55..fe534f8efadb 100644 --- a/vendor/github.com/cilium/ebpf/link/query.go +++ b/vendor/github.com/cilium/ebpf/link/query.go @@ -2,7 +2,6 @@ package link import ( "fmt" - "os" "unsafe" "github.com/cilium/ebpf" @@ -11,53 +10,102 @@ import ( // QueryOptions defines additional parameters when querying for programs. type QueryOptions struct { - // Path can be a path to a cgroup, netns or LIRC2 device - Path string + // Target to query. This is usually a file descriptor but may refer to + // something else based on the attach type. + Target int // Attach specifies the AttachType of the programs queried for Attach ebpf.AttachType // QueryFlags are flags for BPF_PROG_QUERY, e.g. BPF_F_QUERY_EFFECTIVE QueryFlags uint32 } -// QueryPrograms retrieves ProgramIDs associated with the AttachType. -// -// Returns (nil, nil) if there are no programs attached to the queried kernel -// resource. Calling QueryPrograms on a kernel missing PROG_QUERY will result in -// ErrNotSupported. -func QueryPrograms(opts QueryOptions) ([]ebpf.ProgramID, error) { - if haveProgQuery() != nil { - return nil, fmt.Errorf("can't query program IDs: %w", ErrNotSupported) - } +// QueryResult describes which programs and links are active. +type QueryResult struct { + // List of attached programs. + Programs []AttachedProgram - f, err := os.Open(opts.Path) - if err != nil { - return nil, fmt.Errorf("can't open file: %s", err) - } - defer f.Close() + // Incremented by one every time the set of attached programs changes. + // May be zero if not supported by the [ebpf.AttachType]. + Revision uint64 +} + +// HaveLinkInfo returns true if the kernel supports querying link information +// for a particular [ebpf.AttachType]. +func (qr *QueryResult) HaveLinkInfo() bool { + return qr.Revision > 0 +} + +type AttachedProgram struct { + ID ebpf.ProgramID + linkID ID +} + +// LinkID returns the ID associated with the program. +// +// Returns 0, false if the kernel doesn't support retrieving the ID or if the +// program wasn't attached via a link. See [QueryResult.HaveLinkInfo] if you +// need to tell the two apart. +func (ap *AttachedProgram) LinkID() (ID, bool) { + return ap.linkID, ap.linkID != 0 +} +// QueryPrograms retrieves a list of programs for the given AttachType. +// +// Returns a slice of attached programs, which may be empty. +// revision counts how many times the set of attached programs has changed and +// may be zero if not supported by the [ebpf.AttachType]. +// Returns ErrNotSupportd on a kernel without BPF_PROG_QUERY +func QueryPrograms(opts QueryOptions) (*QueryResult, error) { // query the number of programs to allocate correct slice size attr := sys.ProgQueryAttr{ - TargetFd: uint32(f.Fd()), - AttachType: sys.AttachType(opts.Attach), - QueryFlags: opts.QueryFlags, + TargetFdOrIfindex: uint32(opts.Target), + AttachType: sys.AttachType(opts.Attach), + QueryFlags: opts.QueryFlags, } - if err := sys.ProgQuery(&attr); err != nil { - return nil, fmt.Errorf("can't query program count: %w", err) + err := sys.ProgQuery(&attr) + if err != nil { + if haveFeatErr := haveProgQuery(); haveFeatErr != nil { + return nil, fmt.Errorf("query programs: %w", haveFeatErr) + } + return nil, fmt.Errorf("query programs: %w", err) } + if attr.Count == 0 { + return &QueryResult{Revision: attr.Revision}, nil + } + + // The minimum bpf_mprog revision is 1, so we can use the field to detect + // whether the attach type supports link ids. + haveLinkIDs := attr.Revision != 0 - // return nil if no progs are attached - if attr.ProgCount == 0 { - return nil, nil + count := attr.Count + progIds := make([]ebpf.ProgramID, count) + attr = sys.ProgQueryAttr{ + TargetFdOrIfindex: uint32(opts.Target), + AttachType: sys.AttachType(opts.Attach), + QueryFlags: opts.QueryFlags, + Count: count, + ProgIds: sys.NewPointer(unsafe.Pointer(&progIds[0])), + } + + var linkIds []ID + if haveLinkIDs { + linkIds = make([]ID, count) + attr.LinkIds = sys.NewPointer(unsafe.Pointer(&linkIds[0])) } - // we have at least one prog, so we query again - progIds := make([]ebpf.ProgramID, attr.ProgCount) - attr.ProgIds = sys.NewPointer(unsafe.Pointer(&progIds[0])) - attr.ProgCount = uint32(len(progIds)) if err := sys.ProgQuery(&attr); err != nil { - return nil, fmt.Errorf("can't query program IDs: %w", err) + return nil, fmt.Errorf("query programs: %w", err) } - return progIds, nil + // NB: attr.Count might have changed between the two syscalls. + var programs []AttachedProgram + for i, id := range progIds[:attr.Count] { + ap := AttachedProgram{ID: id} + if haveLinkIDs { + ap.linkID = linkIds[i] + } + programs = append(programs, ap) + } + return &QueryResult{programs, attr.Revision}, nil } diff --git a/vendor/github.com/cilium/ebpf/link/syscalls.go b/vendor/github.com/cilium/ebpf/link/syscalls.go index c9c998c2014c..d09b5acb0f35 100644 --- a/vendor/github.com/cilium/ebpf/link/syscalls.go +++ b/vendor/github.com/cilium/ebpf/link/syscalls.go @@ -24,6 +24,10 @@ const ( XDPType = sys.BPF_LINK_TYPE_XDP PerfEventType = sys.BPF_LINK_TYPE_PERF_EVENT KprobeMultiType = sys.BPF_LINK_TYPE_KPROBE_MULTI + TCXType = sys.BPF_LINK_TYPE_TCX + UprobeMultiType = sys.BPF_LINK_TYPE_UPROBE_MULTI + NetfilterType = sys.BPF_LINK_TYPE_NETFILTER + NetkitType = sys.BPF_LINK_TYPE_NETKIT ) var haveProgAttach = internal.NewFeatureTest("BPF_PROG_ATTACH", "4.10", func() error { @@ -60,9 +64,11 @@ var haveProgAttachReplace = internal.NewFeatureTest("BPF_PROG_ATTACH atomic repl asm.Return(), }, }) + if err != nil { return internal.ErrNotSupported } + defer prog.Close() // We know that we have BPF_PROG_ATTACH since we can load CGroupSKB programs. @@ -70,10 +76,10 @@ var haveProgAttachReplace = internal.NewFeatureTest("BPF_PROG_ATTACH atomic repl // present. attr := sys.ProgAttachAttr{ // We rely on this being checked after attachFlags. - TargetFd: ^uint32(0), - AttachBpfFd: uint32(prog.FD()), - AttachType: uint32(ebpf.AttachCGroupInetIngress), - AttachFlags: uint32(flagReplace), + TargetFdOrIfindex: ^uint32(0), + AttachBpfFd: uint32(prog.FD()), + AttachType: uint32(ebpf.AttachCGroupInetIngress), + AttachFlags: uint32(flagReplace), } err = sys.ProgAttach(&attr) @@ -108,16 +114,87 @@ var haveProgQuery = internal.NewFeatureTest("BPF_PROG_QUERY", "4.15", func() err // We rely on this being checked during the syscall. // With an otherwise correct payload we expect EBADF here // as an indication that the feature is present. - TargetFd: ^uint32(0), - AttachType: sys.AttachType(ebpf.AttachCGroupInetIngress), + TargetFdOrIfindex: ^uint32(0), + AttachType: sys.AttachType(ebpf.AttachCGroupInetIngress), } err := sys.ProgQuery(&attr) - if errors.Is(err, unix.EINVAL) { + + if errors.Is(err, unix.EBADF) { + return nil + } + if err != nil { + return ErrNotSupported + } + return errors.New("syscall succeeded unexpectedly") +}) + +var haveTCX = internal.NewFeatureTest("tcx", "6.6", func() error { + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Type: ebpf.SchedCLS, + License: "MIT", + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + }) + + if err != nil { return internal.ErrNotSupported } - if errors.Is(err, unix.EBADF) { + + defer prog.Close() + attr := sys.LinkCreateTcxAttr{ + // We rely on this being checked during the syscall. + // With an otherwise correct payload we expect ENODEV here + // as an indication that the feature is present. + TargetIfindex: ^uint32(0), + ProgFd: uint32(prog.FD()), + AttachType: sys.AttachType(ebpf.AttachTCXIngress), + } + + _, err = sys.LinkCreateTcx(&attr) + + if errors.Is(err, unix.ENODEV) { return nil } - return err + if err != nil { + return ErrNotSupported + } + return errors.New("syscall succeeded unexpectedly") +}) + +var haveNetkit = internal.NewFeatureTest("netkit", "6.7", func() error { + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Type: ebpf.SchedCLS, + License: "MIT", + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + }) + + if err != nil { + return internal.ErrNotSupported + } + + defer prog.Close() + attr := sys.LinkCreateNetkitAttr{ + // We rely on this being checked during the syscall. + // With an otherwise correct payload we expect ENODEV here + // as an indication that the feature is present. + TargetIfindex: ^uint32(0), + ProgFd: uint32(prog.FD()), + AttachType: sys.AttachType(ebpf.AttachNetkitPrimary), + } + + _, err = sys.LinkCreateNetkit(&attr) + + if errors.Is(err, unix.ENODEV) { + return nil + } + if err != nil { + return ErrNotSupported + } + return errors.New("syscall succeeded unexpectedly") }) diff --git a/vendor/github.com/cilium/ebpf/link/tcx.go b/vendor/github.com/cilium/ebpf/link/tcx.go new file mode 100644 index 000000000000..ac045b71da0e --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/tcx.go @@ -0,0 +1,89 @@ +package link + +import ( + "fmt" + "runtime" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/sys" +) + +type TCXOptions struct { + // Index of the interface to attach to. + Interface int + // Program to attach. + Program *ebpf.Program + // One of the AttachTCX* constants. + Attach ebpf.AttachType + // Attach relative to an anchor. Optional. + Anchor Anchor + // Only attach if the expected revision matches. + ExpectedRevision uint64 + // Flags control the attach behaviour. Specify an Anchor instead of + // F_LINK, F_ID, F_BEFORE, F_AFTER and R_REPLACE. Optional. + Flags uint32 +} + +func AttachTCX(opts TCXOptions) (Link, error) { + if opts.Interface < 0 { + return nil, fmt.Errorf("interface %d is out of bounds", opts.Interface) + } + + if opts.Flags&anchorFlags != 0 { + return nil, fmt.Errorf("disallowed flags: use Anchor to specify attach target") + } + + attr := sys.LinkCreateTcxAttr{ + ProgFd: uint32(opts.Program.FD()), + AttachType: sys.AttachType(opts.Attach), + TargetIfindex: uint32(opts.Interface), + ExpectedRevision: opts.ExpectedRevision, + Flags: opts.Flags, + } + + if opts.Anchor != nil { + fdOrID, flags, err := opts.Anchor.anchor() + if err != nil { + return nil, fmt.Errorf("attach tcx link: %w", err) + } + + attr.RelativeFdOrId = fdOrID + attr.Flags |= flags + } + + fd, err := sys.LinkCreateTcx(&attr) + runtime.KeepAlive(opts.Program) + runtime.KeepAlive(opts.Anchor) + if err != nil { + if haveFeatErr := haveTCX(); haveFeatErr != nil { + return nil, haveFeatErr + } + return nil, fmt.Errorf("attach tcx link: %w", err) + } + + return &tcxLink{RawLink{fd, ""}}, nil +} + +type tcxLink struct { + RawLink +} + +var _ Link = (*tcxLink)(nil) + +func (tcx *tcxLink) Info() (*Info, error) { + var info sys.TcxLinkInfo + if err := sys.ObjInfo(tcx.fd, &info); err != nil { + return nil, fmt.Errorf("tcx link info: %s", err) + } + extra := &TCXInfo{ + Ifindex: info.Ifindex, + AttachType: info.AttachType, + } + + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil +} diff --git a/vendor/github.com/cilium/ebpf/link/tracepoint.go b/vendor/github.com/cilium/ebpf/link/tracepoint.go index 95f5fae3b094..6fc78b982872 100644 --- a/vendor/github.com/cilium/ebpf/link/tracepoint.go +++ b/vendor/github.com/cilium/ebpf/link/tracepoint.go @@ -30,6 +30,8 @@ type TracepointOptions struct { // // Note that attaching eBPF programs to syscalls (sys_enter_*/sys_exit_*) is // only possible as of kernel 4.14 (commit cf5f5ce). +// +// The returned Link may implement [PerfEvent]. func Tracepoint(group, name string, prog *ebpf.Program, opts *TracepointOptions) (Link, error) { if group == "" || name == "" { return nil, fmt.Errorf("group and name cannot be empty: %w", errInvalidInput) diff --git a/vendor/github.com/cilium/ebpf/link/tracing.go b/vendor/github.com/cilium/ebpf/link/tracing.go index 1e1a7834d8eb..9e570afc96a8 100644 --- a/vendor/github.com/cilium/ebpf/link/tracing.go +++ b/vendor/github.com/cilium/ebpf/link/tracing.go @@ -18,6 +18,25 @@ func (f *tracing) Update(new *ebpf.Program) error { return fmt.Errorf("tracing update: %w", ErrNotSupported) } +func (f *tracing) Info() (*Info, error) { + var info sys.TracingLinkInfo + if err := sys.ObjInfo(f.fd, &info); err != nil { + return nil, fmt.Errorf("tracing link info: %s", err) + } + extra := &TracingInfo{ + TargetObjId: info.TargetObjId, + TargetBtfId: info.TargetBtfId, + AttachType: info.AttachType, + } + + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil +} + // AttachFreplace attaches the given eBPF program to the function it replaces. // // The program and name can either be provided at link time, or can be provided diff --git a/vendor/github.com/cilium/ebpf/link/uprobe.go b/vendor/github.com/cilium/ebpf/link/uprobe.go index 272bac4151d4..194d1d319a7b 100644 --- a/vendor/github.com/cilium/ebpf/link/uprobe.go +++ b/vendor/github.com/cilium/ebpf/link/uprobe.go @@ -18,9 +18,12 @@ var ( uprobeRefCtrOffsetShift = 32 haveRefCtrOffsetPMU = internal.NewFeatureTest("RefCtrOffsetPMU", "4.20", func() error { _, err := os.Stat(uprobeRefCtrOffsetPMUPath) - if err != nil { + if errors.Is(err, os.ErrNotExist) { return internal.ErrNotSupported } + if err != nil { + return err + } return nil }) @@ -33,10 +36,10 @@ var ( type Executable struct { // Path of the executable on the filesystem. path string - // Parsed ELF and dynamic symbols' addresses. - addresses map[string]uint64 + // Parsed ELF and dynamic symbols' cachedAddresses. + cachedAddresses map[string]uint64 // Keep track of symbol table lazy load. - addressesOnce sync.Once + cacheAddressesOnce sync.Once } // UprobeOptions defines additional parameters that will be used @@ -105,8 +108,8 @@ func OpenExecutable(path string) (*Executable, error) { } return &Executable{ - path: path, - addresses: make(map[string]uint64), + path: path, + cachedAddresses: make(map[string]uint64), }, nil } @@ -150,7 +153,7 @@ func (ex *Executable) load(f *internal.SafeELFFile) error { } } - ex.addresses[s.Name] = address + ex.cachedAddresses[s.Name] = address } return nil @@ -159,13 +162,13 @@ func (ex *Executable) load(f *internal.SafeELFFile) error { // address calculates the address of a symbol in the executable. // // opts must not be nil. -func (ex *Executable) address(symbol string, opts *UprobeOptions) (uint64, error) { - if opts.Address > 0 { - return opts.Address + opts.Offset, nil +func (ex *Executable) address(symbol string, address, offset uint64) (uint64, error) { + if address > 0 { + return address + offset, nil } var err error - ex.addressesOnce.Do(func() { + ex.cacheAddressesOnce.Do(func() { var f *internal.SafeELFFile f, err = internal.OpenSafeELFFile(ex.path) if err != nil { @@ -180,7 +183,7 @@ func (ex *Executable) address(symbol string, opts *UprobeOptions) (uint64, error return 0, fmt.Errorf("lazy load symbols: %w", err) } - address, ok := ex.addresses[symbol] + address, ok := ex.cachedAddresses[symbol] if !ok { return 0, fmt.Errorf("symbol %s: %w", symbol, ErrNoSymbol) } @@ -196,7 +199,7 @@ func (ex *Executable) address(symbol string, opts *UprobeOptions) (uint64, error "(consider providing UprobeOptions.Address)", ex.path, symbol, ErrNotSupported) } - return address + opts.Offset, nil + return address + offset, nil } // Uprobe attaches the given eBPF program to a perf event that fires when the @@ -219,6 +222,8 @@ func (ex *Executable) address(symbol string, opts *UprobeOptions) (uint64, error // // Functions provided by shared libraries can currently not be traced and // will result in an ErrNotSupported. +// +// The returned Link may implement [PerfEvent]. func (ex *Executable) Uprobe(symbol string, prog *ebpf.Program, opts *UprobeOptions) (Link, error) { u, err := ex.uprobe(symbol, prog, opts, false) if err != nil { @@ -253,6 +258,8 @@ func (ex *Executable) Uprobe(symbol string, prog *ebpf.Program, opts *UprobeOpti // // Functions provided by shared libraries can currently not be traced and // will result in an ErrNotSupported. +// +// The returned Link may implement [PerfEvent]. func (ex *Executable) Uretprobe(symbol string, prog *ebpf.Program, opts *UprobeOptions) (Link, error) { u, err := ex.uprobe(symbol, prog, opts, true) if err != nil { @@ -281,7 +288,7 @@ func (ex *Executable) uprobe(symbol string, prog *ebpf.Program, opts *UprobeOpti opts = &UprobeOptions{} } - offset, err := ex.address(symbol, opts) + offset, err := ex.address(symbol, opts.Address, opts.Offset) if err != nil { return nil, err } diff --git a/vendor/github.com/cilium/ebpf/link/uprobe_multi.go b/vendor/github.com/cilium/ebpf/link/uprobe_multi.go new file mode 100644 index 000000000000..aea807b329ae --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/uprobe_multi.go @@ -0,0 +1,216 @@ +package link + +import ( + "errors" + "fmt" + "os" + "unsafe" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/asm" + "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/unix" +) + +// UprobeMultiOptions defines additional parameters that will be used +// when opening a UprobeMulti Link. +type UprobeMultiOptions struct { + // Symbol addresses. If set, overrides the addresses eventually parsed from + // the executable. Mutually exclusive with UprobeMulti's symbols argument. + Addresses []uint64 + + // Offsets into functions provided by UprobeMulti's symbols argument. + // For example: to set uprobes to main+5 and _start+10, call UprobeMulti + // with: + // symbols: "main", "_start" + // opt.Offsets: 5, 10 + Offsets []uint64 + + // Optional list of associated ref counter offsets. + RefCtrOffsets []uint64 + + // Optional list of associated BPF cookies. + Cookies []uint64 + + // Only set the uprobe_multi link on the given process ID, zero PID means + // system-wide. + PID uint32 +} + +func (ex *Executable) UprobeMulti(symbols []string, prog *ebpf.Program, opts *UprobeMultiOptions) (Link, error) { + return ex.uprobeMulti(symbols, prog, opts, 0) +} + +func (ex *Executable) UretprobeMulti(symbols []string, prog *ebpf.Program, opts *UprobeMultiOptions) (Link, error) { + + // The return probe is not limited for symbols entry, so there's no special + // setup for return uprobes (other than the extra flag). The symbols, opts.Offsets + // and opts.Addresses arrays follow the same logic as for entry uprobes. + return ex.uprobeMulti(symbols, prog, opts, unix.BPF_F_UPROBE_MULTI_RETURN) +} + +func (ex *Executable) uprobeMulti(symbols []string, prog *ebpf.Program, opts *UprobeMultiOptions, flags uint32) (Link, error) { + if prog == nil { + return nil, errors.New("cannot attach a nil program") + } + + if opts == nil { + opts = &UprobeMultiOptions{} + } + + addresses, err := ex.addresses(symbols, opts.Addresses, opts.Offsets) + if err != nil { + return nil, err + } + + addrs := len(addresses) + cookies := len(opts.Cookies) + refCtrOffsets := len(opts.RefCtrOffsets) + + if addrs == 0 { + return nil, fmt.Errorf("Addresses are required: %w", errInvalidInput) + } + if refCtrOffsets > 0 && refCtrOffsets != addrs { + return nil, fmt.Errorf("RefCtrOffsets must be exactly Addresses in length: %w", errInvalidInput) + } + if cookies > 0 && cookies != addrs { + return nil, fmt.Errorf("Cookies must be exactly Addresses in length: %w", errInvalidInput) + } + + attr := &sys.LinkCreateUprobeMultiAttr{ + Path: sys.NewStringPointer(ex.path), + ProgFd: uint32(prog.FD()), + AttachType: sys.BPF_TRACE_UPROBE_MULTI, + UprobeMultiFlags: flags, + Count: uint32(addrs), + Offsets: sys.NewPointer(unsafe.Pointer(&addresses[0])), + Pid: opts.PID, + } + + if refCtrOffsets != 0 { + attr.RefCtrOffsets = sys.NewPointer(unsafe.Pointer(&opts.RefCtrOffsets[0])) + } + if cookies != 0 { + attr.Cookies = sys.NewPointer(unsafe.Pointer(&opts.Cookies[0])) + } + + fd, err := sys.LinkCreateUprobeMulti(attr) + if errors.Is(err, unix.ESRCH) { + return nil, fmt.Errorf("%w (specified pid not found?)", os.ErrNotExist) + } + if errors.Is(err, unix.EINVAL) { + return nil, fmt.Errorf("%w (missing symbol or prog's AttachType not AttachTraceUprobeMulti?)", err) + } + + if err != nil { + if haveFeatErr := haveBPFLinkUprobeMulti(); haveFeatErr != nil { + return nil, haveFeatErr + } + return nil, err + } + + return &uprobeMultiLink{RawLink{fd, ""}}, nil +} + +func (ex *Executable) addresses(symbols []string, addresses, offsets []uint64) ([]uint64, error) { + n := len(symbols) + if n == 0 { + n = len(addresses) + } + + if n == 0 { + return nil, fmt.Errorf("%w: neither symbols nor addresses given", errInvalidInput) + } + + if symbols != nil && len(symbols) != n { + return nil, fmt.Errorf("%w: have %d symbols but want %d", errInvalidInput, len(symbols), n) + } + + if addresses != nil && len(addresses) != n { + return nil, fmt.Errorf("%w: have %d addresses but want %d", errInvalidInput, len(addresses), n) + } + + if offsets != nil && len(offsets) != n { + return nil, fmt.Errorf("%w: have %d offsets but want %d", errInvalidInput, len(offsets), n) + } + + results := make([]uint64, 0, n) + for i := 0; i < n; i++ { + var sym string + if symbols != nil { + sym = symbols[i] + } + + var addr, off uint64 + if addresses != nil { + addr = addresses[i] + } + + if offsets != nil { + off = offsets[i] + } + + result, err := ex.address(sym, addr, off) + if err != nil { + return nil, err + } + + results = append(results, result) + } + + return results, nil +} + +type uprobeMultiLink struct { + RawLink +} + +var _ Link = (*uprobeMultiLink)(nil) + +func (kml *uprobeMultiLink) Update(prog *ebpf.Program) error { + return fmt.Errorf("update uprobe_multi: %w", ErrNotSupported) +} + +var haveBPFLinkUprobeMulti = internal.NewFeatureTest("bpf_link_uprobe_multi", "6.6", func() error { + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Name: "probe_upm_link", + Type: ebpf.Kprobe, + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + AttachType: ebpf.AttachTraceUprobeMulti, + License: "MIT", + }) + if errors.Is(err, unix.E2BIG) { + // Kernel doesn't support AttachType field. + return internal.ErrNotSupported + } + if err != nil { + return err + } + defer prog.Close() + + // We try to create uprobe multi link on '/' path which results in + // error with -EBADF in case uprobe multi link is supported. + fd, err := sys.LinkCreateUprobeMulti(&sys.LinkCreateUprobeMultiAttr{ + ProgFd: uint32(prog.FD()), + AttachType: sys.BPF_TRACE_UPROBE_MULTI, + Path: sys.NewStringPointer("/"), + Offsets: sys.NewPointer(unsafe.Pointer(&[]uint64{0})), + Count: 1, + }) + switch { + case errors.Is(err, unix.EBADF): + return nil + case errors.Is(err, unix.EINVAL): + return internal.ErrNotSupported + case err != nil: + return err + } + + // should not happen + fd.Close() + return errors.New("successfully attached uprobe_multi to /, kernel bug?") +}) diff --git a/vendor/github.com/cilium/ebpf/link/xdp.go b/vendor/github.com/cilium/ebpf/link/xdp.go index aa8dd3a4cb39..2ec441229a51 100644 --- a/vendor/github.com/cilium/ebpf/link/xdp.go +++ b/vendor/github.com/cilium/ebpf/link/xdp.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/sys" ) // XDPAttachFlags represents how XDP program will be attached to interface. @@ -50,5 +51,30 @@ func AttachXDP(opts XDPOptions) (Link, error) { Flags: uint32(opts.Flags), }) - return rawLink, err + if err != nil { + return nil, fmt.Errorf("failed to attach link: %w", err) + } + + return &xdpLink{*rawLink}, nil +} + +type xdpLink struct { + RawLink +} + +func (xdp *xdpLink) Info() (*Info, error) { + var info sys.XDPLinkInfo + if err := sys.ObjInfo(xdp.fd, &info); err != nil { + return nil, fmt.Errorf("xdp link info: %s", err) + } + extra := &XDPInfo{ + Ifindex: info.Ifindex, + } + + return &Info{ + info.Type, + info.Id, + ebpf.ProgramID(info.ProgId), + extra, + }, nil } diff --git a/vendor/github.com/cilium/ebpf/linker.go b/vendor/github.com/cilium/ebpf/linker.go index e0dbfcffd377..788f21b7b6fc 100644 --- a/vendor/github.com/cilium/ebpf/linker.go +++ b/vendor/github.com/cilium/ebpf/linker.go @@ -1,11 +1,14 @@ package ebpf import ( + "debug/elf" "encoding/binary" "errors" "fmt" "io" + "io/fs" "math" + "slices" "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/btf" @@ -40,10 +43,12 @@ func (hs handles) fdArray() []int32 { return fda } -func (hs handles) close() { - for _, h := range hs { - h.Close() +func (hs *handles) Close() error { + var errs []error + for _, h := range *hs { + errs = append(errs, h.Close()) } + return errors.Join(errs...) } // splitSymbols splits insns into subsections delimited by Symbol Instructions. @@ -55,21 +60,33 @@ func splitSymbols(insns asm.Instructions) (map[string]asm.Instructions, error) { return nil, errors.New("insns is empty") } - if insns[0].Symbol() == "" { + currentSym := insns[0].Symbol() + if currentSym == "" { return nil, errors.New("insns must start with a Symbol") } - var name string + start := 0 progs := make(map[string]asm.Instructions) - for _, ins := range insns { - if sym := ins.Symbol(); sym != "" { - if progs[sym] != nil { - return nil, fmt.Errorf("insns contains duplicate Symbol %s", sym) - } - name = sym + for i, ins := range insns[1:] { + i := i + 1 + + sym := ins.Symbol() + if sym == "" { + continue } - progs[name] = append(progs[name], ins) + // New symbol, flush the old one out. + progs[currentSym] = slices.Clone(insns[start:i]) + + if progs[sym] != nil { + return nil, fmt.Errorf("insns contains duplicate Symbol %s", sym) + } + currentSym = sym + start = i + } + + if tail := insns[start:]; len(tail) > 0 { + progs[currentSym] = slices.Clone(tail) } return progs, nil @@ -104,7 +121,7 @@ func hasFunctionReferences(insns asm.Instructions) bool { // // Passing a nil target will relocate against the running kernel. insns are // modified in place. -func applyRelocations(insns asm.Instructions, target *btf.Spec, bo binary.ByteOrder) error { +func applyRelocations(insns asm.Instructions, targets []*btf.Spec, kmodName string, bo binary.ByteOrder, b *btf.Builder) error { var relos []*btf.CORERelocation var reloInsns []*asm.Instruction iter := insns.Iterate() @@ -123,7 +140,26 @@ func applyRelocations(insns asm.Instructions, target *btf.Spec, bo binary.ByteOr bo = internal.NativeEndian } - fixups, err := btf.CORERelocate(relos, target, bo) + if len(targets) == 0 { + kernelTarget, err := btf.LoadKernelSpec() + if err != nil { + return fmt.Errorf("load kernel spec: %w", err) + } + targets = append(targets, kernelTarget) + + if kmodName != "" { + kmodTarget, err := btf.LoadKernelModuleSpec(kmodName) + // Ignore ErrNotExists to cater to kernels which have CONFIG_DEBUG_INFO_BTF_MODULES disabled. + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("load kernel module spec: %w", err) + } + if err == nil { + targets = append(targets, kmodTarget) + } + } + } + + fixups, err := btf.CORERelocate(relos, targets, bo, b.Add) if err != nil { return err } @@ -228,14 +264,24 @@ func fixupAndValidate(insns asm.Instructions) error { return nil } +// POISON_CALL_KFUNC_BASE in libbpf. +// https://github.com/libbpf/libbpf/blob/2778cbce609aa1e2747a69349f7f46a2f94f0522/src/libbpf.c#L5767 +const kfuncCallPoisonBase = 2002000000 + // fixupKfuncs loops over all instructions in search for kfunc calls. // If at least one is found, the current kernels BTF and module BTFis are searched to set Instruction.Constant // and Instruction.Offset to the correct values. -func fixupKfuncs(insns asm.Instructions) (handles, error) { +func fixupKfuncs(insns asm.Instructions) (_ handles, err error) { + closeOnError := func(c io.Closer) { + if err != nil { + c.Close() + } + } + iter := insns.Iterate() for iter.Next() { ins := iter.Ins - if ins.IsKfuncCall() { + if metadata := ins.Metadata.Get(kfuncMetaKey{}); metadata != nil { goto fixups } } @@ -250,10 +296,13 @@ fixups: } fdArray := make(handles, 0) + defer closeOnError(&fdArray) + for { ins := iter.Ins - if !ins.IsKfuncCall() { + metadata := ins.Metadata.Get(kfuncMetaKey{}) + if metadata == nil { if !iter.Next() { // break loop if this was the last instruction in the stream. break @@ -262,30 +311,49 @@ fixups: } // check meta, if no meta return err - kfm, _ := ins.Metadata.Get(kfuncMeta{}).(*btf.Func) + kfm, _ := metadata.(*kfuncMeta) if kfm == nil { - return nil, fmt.Errorf("kfunc call has no kfuncMeta") + return nil, fmt.Errorf("kfuncMetaKey doesn't contain kfuncMeta") } target := btf.Type((*btf.Func)(nil)) - spec, module, err := findTargetInKernel(kernelSpec, kfm.Name, &target) + spec, module, err := findTargetInKernel(kernelSpec, kfm.Func.Name, &target) + if kfm.Binding == elf.STB_WEAK && errors.Is(err, btf.ErrNotFound) { + if ins.IsKfuncCall() { + // If the kfunc call is weak and not found, poison the call. Use a recognizable constant + // to make it easier to debug. And set src to zero so the verifier doesn't complain + // about the invalid imm/offset values before dead-code elimination. + ins.Constant = kfuncCallPoisonBase + ins.Src = 0 + } else if ins.OpCode.IsDWordLoad() { + // If the kfunc DWordLoad is weak and not found, set its address to 0. + ins.Constant = 0 + ins.Src = 0 + } else { + return nil, fmt.Errorf("only kfunc calls and dword loads may have kfunc metadata") + } + + iter.Next() + continue + } + // Error on non-weak kfunc not found. if errors.Is(err, btf.ErrNotFound) { - return nil, fmt.Errorf("kfunc %q: %w", kfm.Name, ErrNotSupported) + return nil, fmt.Errorf("kfunc %q: %w", kfm.Func.Name, ErrNotSupported) } if err != nil { return nil, err } - if err := btf.CheckTypeCompatibility(kfm.Type, target.(*btf.Func).Type); err != nil { - return nil, &incompatibleKfuncError{kfm.Name, err} - } - - id, err := spec.TypeID(target) + idx, err := fdArray.add(module) if err != nil { return nil, err } - idx, err := fdArray.add(module) + if err := btf.CheckTypeCompatibility(kfm.Func.Type, target.(*btf.Func).Type); err != nil { + return nil, &incompatibleKfuncError{kfm.Func.Name, err} + } + + id, err := spec.TypeID(target) if err != nil { return nil, err } diff --git a/vendor/github.com/cilium/ebpf/map.go b/vendor/github.com/cilium/ebpf/map.go index a11664cc72da..0b62101c3cb0 100644 --- a/vendor/github.com/cilium/ebpf/map.go +++ b/vendor/github.com/cilium/ebpf/map.go @@ -9,12 +9,16 @@ import ( "os" "path/filepath" "reflect" + "slices" + "strings" + "sync" "time" "unsafe" "github.com/cilium/ebpf/btf" "github.com/cilium/ebpf/internal" "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/sysenc" "github.com/cilium/ebpf/internal/unix" ) @@ -25,6 +29,10 @@ var ( ErrIterationAborted = errors.New("iteration aborted") ErrMapIncompatible = errors.New("map spec is incompatible with existing map") errMapNoBTFValue = errors.New("map spec does not contain a BTF Value") + + // pre-allocating these errors here since they may get called in hot code paths + // and cause unnecessary memory allocations + errMapLookupKeyNotExist = fmt.Errorf("lookup: %w", sysErrKeyNotExist) ) // MapOptions control loading a map into the kernel. @@ -93,35 +101,73 @@ func (ms *MapSpec) Copy() *MapSpec { } cpy := *ms + cpy.Contents = slices.Clone(cpy.Contents) + cpy.Key = btf.Copy(cpy.Key) + cpy.Value = btf.Copy(cpy.Value) - cpy.Contents = make([]MapKV, len(ms.Contents)) - copy(cpy.Contents, ms.Contents) + if cpy.InnerMap == ms { + cpy.InnerMap = &cpy + } else { + cpy.InnerMap = ms.InnerMap.Copy() + } - cpy.InnerMap = ms.InnerMap.Copy() + if cpy.Extra != nil { + extra := *cpy.Extra + cpy.Extra = &extra + } return &cpy } -func (ms *MapSpec) clampPerfEventArraySize() error { - if ms.Type != PerfEventArray { - return nil - } +// fixupMagicFields fills fields of MapSpec which are usually +// left empty in ELF or which depend on runtime information. +// +// The method doesn't modify Spec, instead returning a copy. +// The copy is only performed if fixups are necessary, so callers mustn't mutate +// the returned spec. +func (spec *MapSpec) fixupMagicFields() (*MapSpec, error) { + switch spec.Type { + case ArrayOfMaps, HashOfMaps: + if spec.ValueSize != 0 && spec.ValueSize != 4 { + return nil, errors.New("ValueSize must be zero or four for map of map") + } - n, err := internal.PossibleCPUs() - if err != nil { - return fmt.Errorf("perf event array: %w", err) - } + spec = spec.Copy() + spec.ValueSize = 4 + + case PerfEventArray: + if spec.KeySize != 0 && spec.KeySize != 4 { + return nil, errors.New("KeySize must be zero or four for perf event array") + } + + if spec.ValueSize != 0 && spec.ValueSize != 4 { + return nil, errors.New("ValueSize must be zero or four for perf event array") + } + + spec = spec.Copy() + spec.KeySize = 4 + spec.ValueSize = 4 - if n := uint32(n); ms.MaxEntries > n { - ms.MaxEntries = n + n, err := PossibleCPU() + if err != nil { + return nil, fmt.Errorf("fixup perf event array: %w", err) + } + + if n := uint32(n); spec.MaxEntries == 0 || spec.MaxEntries > n { + // MaxEntries should be zero most of the time, but there is code + // out there which hardcodes large constants. Clamp the number + // of entries to the number of CPUs at most. Allow creating maps with + // less than n items since some kernel selftests relied on this + // behaviour in the past. + spec.MaxEntries = n + } } - return nil + return spec, nil } // dataSection returns the contents and BTF Datasec descriptor of the spec. func (ms *MapSpec) dataSection() ([]byte, *btf.Datasec, error) { - if ms.Value == nil { return nil, nil, errMapNoBTFValue } @@ -155,27 +201,37 @@ type MapKV struct { // // Returns an error wrapping [ErrMapIncompatible] otherwise. func (ms *MapSpec) Compatible(m *Map) error { - switch { - case m.typ != ms.Type: - return fmt.Errorf("expected type %v, got %v: %w", ms.Type, m.typ, ErrMapIncompatible) - - case m.keySize != ms.KeySize: - return fmt.Errorf("expected key size %v, got %v: %w", ms.KeySize, m.keySize, ErrMapIncompatible) + ms, err := ms.fixupMagicFields() + if err != nil { + return err + } - case m.valueSize != ms.ValueSize: - return fmt.Errorf("expected value size %v, got %v: %w", ms.ValueSize, m.valueSize, ErrMapIncompatible) + diffs := []string{} + if m.typ != ms.Type { + diffs = append(diffs, fmt.Sprintf("Type: %s changed to %s", m.typ, ms.Type)) + } + if m.keySize != ms.KeySize { + diffs = append(diffs, fmt.Sprintf("KeySize: %d changed to %d", m.keySize, ms.KeySize)) + } + if m.valueSize != ms.ValueSize { + diffs = append(diffs, fmt.Sprintf("ValueSize: %d changed to %d", m.valueSize, ms.ValueSize)) + } + if m.maxEntries != ms.MaxEntries { + diffs = append(diffs, fmt.Sprintf("MaxEntries: %d changed to %d", m.maxEntries, ms.MaxEntries)) + } - case !(ms.Type == PerfEventArray && ms.MaxEntries == 0) && - m.maxEntries != ms.MaxEntries: - return fmt.Errorf("expected max entries %v, got %v: %w", ms.MaxEntries, m.maxEntries, ErrMapIncompatible) + // BPF_F_RDONLY_PROG is set unconditionally for devmaps. Explicitly allow this + // mismatch. + if !((ms.Type == DevMap || ms.Type == DevMapHash) && m.flags^ms.Flags == unix.BPF_F_RDONLY_PROG) && + m.flags != ms.Flags { + diffs = append(diffs, fmt.Sprintf("Flags: %d changed to %d", m.flags, ms.Flags)) + } - // BPF_F_RDONLY_PROG is set unconditionally for devmaps. Explicitly allow - // this mismatch. - case !((ms.Type == DevMap || ms.Type == DevMapHash) && m.flags^ms.Flags == unix.BPF_F_RDONLY_PROG) && - m.flags != ms.Flags: - return fmt.Errorf("expected flags %v, got %v: %w", ms.Flags, m.flags, ErrMapIncompatible) + if len(diffs) == 0 { + return nil } - return nil + + return fmt.Errorf("%s: %w", strings.Join(diffs, ", "), ErrMapIncompatible) } // Map represents a Map file descriptor. @@ -350,60 +406,9 @@ func (spec *MapSpec) createMap(inner *sys.FD, opts MapOptions) (_ *Map, err erro } } - switch spec.Type { - case ArrayOfMaps, HashOfMaps: - if err := haveNestedMaps(); err != nil { - return nil, err - } - - if spec.ValueSize != 0 && spec.ValueSize != 4 { - return nil, errors.New("ValueSize must be zero or four for map of map") - } - - spec = spec.Copy() - spec.ValueSize = 4 - - case PerfEventArray: - if spec.KeySize != 0 && spec.KeySize != 4 { - return nil, errors.New("KeySize must be zero or four for perf event array") - } - - if spec.ValueSize != 0 && spec.ValueSize != 4 { - return nil, errors.New("ValueSize must be zero or four for perf event array") - } - - spec = spec.Copy() - spec.KeySize = 4 - spec.ValueSize = 4 - - if spec.MaxEntries == 0 { - n, err := internal.PossibleCPUs() - if err != nil { - return nil, fmt.Errorf("perf event array: %w", err) - } - spec.MaxEntries = uint32(n) - } - } - - if spec.Flags&(unix.BPF_F_RDONLY_PROG|unix.BPF_F_WRONLY_PROG) > 0 || spec.Freeze { - if err := haveMapMutabilityModifiers(); err != nil { - return nil, fmt.Errorf("map create: %w", err) - } - } - if spec.Flags&unix.BPF_F_MMAPABLE > 0 { - if err := haveMmapableMaps(); err != nil { - return nil, fmt.Errorf("map create: %w", err) - } - } - if spec.Flags&unix.BPF_F_INNER_MAP > 0 { - if err := haveInnerMaps(); err != nil { - return nil, fmt.Errorf("map create: %w", err) - } - } - if spec.Flags&unix.BPF_F_NO_PREALLOC > 0 { - if err := haveNoPreallocMaps(); err != nil { - return nil, fmt.Errorf("map create: %w", err) - } + spec, err = spec.fixupMagicFields() + if err != nil { + return nil, err } attr := sys.MapCreateAttr{ @@ -440,36 +445,76 @@ func (spec *MapSpec) createMap(inner *sys.FD, opts MapOptions) (_ *Map, err erro } fd, err := sys.MapCreate(&attr) + // Some map types don't support BTF k/v in earlier kernel versions. // Remove BTF metadata and retry map creation. if (errors.Is(err, sys.ENOTSUPP) || errors.Is(err, unix.EINVAL)) && attr.BtfFd != 0 { attr.BtfFd, attr.BtfKeyTypeId, attr.BtfValueTypeId = 0, 0, 0 fd, err = sys.MapCreate(&attr) } + if err != nil { + return nil, handleMapCreateError(attr, spec, err) + } + defer closeOnError(fd) + m, err := newMap(fd, spec.Name, spec.Type, spec.KeySize, spec.ValueSize, spec.MaxEntries, spec.Flags) if err != nil { - if errors.Is(err, unix.EPERM) { - return nil, fmt.Errorf("map create: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err) + return nil, fmt.Errorf("map create: %w", err) + } + return m, nil +} + +func handleMapCreateError(attr sys.MapCreateAttr, spec *MapSpec, err error) error { + if errors.Is(err, unix.EPERM) { + return fmt.Errorf("map create: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err) + } + if errors.Is(err, unix.EINVAL) && spec.MaxEntries == 0 { + return fmt.Errorf("map create: %w (MaxEntries may be incorrectly set to zero)", err) + } + if errors.Is(err, unix.EINVAL) && spec.Type == UnspecifiedMap { + return fmt.Errorf("map create: cannot use type %s", UnspecifiedMap) + } + if errors.Is(err, unix.EINVAL) && spec.Flags&unix.BPF_F_NO_PREALLOC > 0 { + return fmt.Errorf("map create: %w (noPrealloc flag may be incompatible with map type %s)", err, spec.Type) + } + + switch spec.Type { + case ArrayOfMaps, HashOfMaps: + if haveFeatErr := haveNestedMaps(); haveFeatErr != nil { + return fmt.Errorf("map create: %w", haveFeatErr) } - if errors.Is(err, unix.EINVAL) && attr.MaxEntries == 0 { - return nil, fmt.Errorf("map create: %w (MaxEntries may be incorrectly set to zero)", err) + } + if spec.Flags&(unix.BPF_F_RDONLY_PROG|unix.BPF_F_WRONLY_PROG) > 0 || spec.Freeze { + if haveFeatErr := haveMapMutabilityModifiers(); haveFeatErr != nil { + return fmt.Errorf("map create: %w", haveFeatErr) } - if errors.Is(err, unix.EINVAL) && spec.Type == UnspecifiedMap { - return nil, fmt.Errorf("map create: cannot use type %s", UnspecifiedMap) + } + if spec.Flags&unix.BPF_F_MMAPABLE > 0 { + if haveFeatErr := haveMmapableMaps(); haveFeatErr != nil { + return fmt.Errorf("map create: %w", haveFeatErr) } - if attr.BtfFd == 0 { - return nil, fmt.Errorf("map create: %w (without BTF k/v)", err) + } + if spec.Flags&unix.BPF_F_INNER_MAP > 0 { + if haveFeatErr := haveInnerMaps(); haveFeatErr != nil { + return fmt.Errorf("map create: %w", haveFeatErr) } - return nil, fmt.Errorf("map create: %w", err) } - defer closeOnError(fd) - - m, err := newMap(fd, spec.Name, spec.Type, spec.KeySize, spec.ValueSize, spec.MaxEntries, spec.Flags) - if err != nil { - return nil, fmt.Errorf("map create: %w", err) + if spec.Flags&unix.BPF_F_NO_PREALLOC > 0 { + if haveFeatErr := haveNoPreallocMaps(); haveFeatErr != nil { + return fmt.Errorf("map create: %w", haveFeatErr) + } + } + // BPF_MAP_TYPE_RINGBUF's max_entries must be a power-of-2 multiple of kernel's page size. + if errors.Is(err, unix.EINVAL) && + (attr.MapType == sys.BPF_MAP_TYPE_RINGBUF || attr.MapType == sys.BPF_MAP_TYPE_USER_RINGBUF) { + pageSize := uint32(os.Getpagesize()) + maxEntries := attr.MaxEntries + if maxEntries%pageSize != 0 || !internal.IsPow(maxEntries) { + return fmt.Errorf("map create: %w (ring map size %d not a multiple of page size %d)", err, maxEntries, pageSize) + } } - return m, nil + return fmt.Errorf("map create: %w", err) } // newMap allocates and returns a new Map structure. @@ -491,7 +536,7 @@ func newMap(fd *sys.FD, name string, typ MapType, keySize, valueSize, maxEntries return m, nil } - possibleCPUs, err := internal.PossibleCPUs() + possibleCPUs, err := PossibleCPU() if err != nil { return nil, err } @@ -537,11 +582,29 @@ func (m *Map) Info() (*MapInfo, error) { return newMapInfoFromFd(m.fd) } +// Handle returns a reference to the Map's type information in the kernel. +// +// Returns ErrNotSupported if the kernel has no BTF support, or if there is no +// BTF associated with the Map. +func (m *Map) Handle() (*btf.Handle, error) { + info, err := m.Info() + if err != nil { + return nil, err + } + + id, ok := info.BTFID() + if !ok { + return nil, fmt.Errorf("map %s: retrieve BTF ID: %w", m, ErrNotSupported) + } + + return btf.NewHandleFromID(id) +} + // MapLookupFlags controls the behaviour of the map lookup calls. type MapLookupFlags uint64 // LookupLock look up the value of a spin-locked map. -const LookupLock MapLookupFlags = 4 +const LookupLock MapLookupFlags = unix.BPF_F_LOCK // Lookup retrieves a value from a Map. // @@ -568,8 +631,8 @@ func (m *Map) LookupWithFlags(key, valueOut interface{}, flags MapLookupFlags) e return m.lookupPerCPU(key, valueOut, flags) } - valuePtr, valueBytes := makeBuffer(valueOut, m.fullValueSize) - if err := m.lookup(key, valuePtr, flags); err != nil { + valueBytes := makeMapSyscallOutput(valueOut, m.fullValueSize) + if err := m.lookup(key, valueBytes.Pointer(), flags); err != nil { return err } @@ -595,8 +658,8 @@ func (m *Map) LookupAndDeleteWithFlags(key, valueOut interface{}, flags MapLooku return m.lookupAndDeletePerCPU(key, valueOut, flags) } - valuePtr, valueBytes := makeBuffer(valueOut, m.fullValueSize) - if err := m.lookupAndDelete(key, valuePtr, flags); err != nil { + valueBytes := makeMapSyscallOutput(valueOut, m.fullValueSize) + if err := m.lookupAndDelete(key, valueBytes.Pointer(), flags); err != nil { return err } return m.unmarshalValue(valueOut, valueBytes) @@ -618,11 +681,15 @@ func (m *Map) LookupBytes(key interface{}) ([]byte, error) { } func (m *Map) lookupPerCPU(key, valueOut any, flags MapLookupFlags) error { + slice, err := ensurePerCPUSlice(valueOut) + if err != nil { + return err + } valueBytes := make([]byte, m.fullValueSize) if err := m.lookup(key, sys.NewSlicePointer(valueBytes), flags); err != nil { return err } - return unmarshalPerCPUValue(valueOut, int(m.valueSize), valueBytes) + return unmarshalPerCPUValue(slice, int(m.valueSize), valueBytes) } func (m *Map) lookup(key interface{}, valueOut sys.Pointer, flags MapLookupFlags) error { @@ -639,17 +706,62 @@ func (m *Map) lookup(key interface{}, valueOut sys.Pointer, flags MapLookupFlags } if err = sys.MapLookupElem(&attr); err != nil { + if errors.Is(err, unix.ENOENT) { + return errMapLookupKeyNotExist + } return fmt.Errorf("lookup: %w", wrapMapError(err)) } return nil } func (m *Map) lookupAndDeletePerCPU(key, valueOut any, flags MapLookupFlags) error { + slice, err := ensurePerCPUSlice(valueOut) + if err != nil { + return err + } valueBytes := make([]byte, m.fullValueSize) if err := m.lookupAndDelete(key, sys.NewSlicePointer(valueBytes), flags); err != nil { return err } - return unmarshalPerCPUValue(valueOut, int(m.valueSize), valueBytes) + return unmarshalPerCPUValue(slice, int(m.valueSize), valueBytes) +} + +// ensurePerCPUSlice allocates a slice for a per-CPU value if necessary. +func ensurePerCPUSlice(sliceOrPtr any) (any, error) { + sliceOrPtrType := reflect.TypeOf(sliceOrPtr) + if sliceOrPtrType.Kind() == reflect.Slice { + // The target is a slice, the caller is responsible for ensuring that + // size is correct. + return sliceOrPtr, nil + } + + slicePtrType := sliceOrPtrType + if slicePtrType.Kind() != reflect.Ptr || slicePtrType.Elem().Kind() != reflect.Slice { + return nil, fmt.Errorf("per-cpu value requires a slice or a pointer to slice") + } + + possibleCPUs, err := PossibleCPU() + if err != nil { + return nil, err + } + + sliceType := slicePtrType.Elem() + slice := reflect.MakeSlice(sliceType, possibleCPUs, possibleCPUs) + + sliceElemType := sliceType.Elem() + sliceElemIsPointer := sliceElemType.Kind() == reflect.Ptr + reflect.ValueOf(sliceOrPtr).Elem().Set(slice) + if !sliceElemIsPointer { + return slice.Interface(), nil + } + sliceElemType = sliceElemType.Elem() + + for i := 0; i < possibleCPUs; i++ { + newElem := reflect.New(sliceElemType) + slice.Index(i).Set(newElem) + } + + return slice.Interface(), nil } func (m *Map) lookupAndDelete(key any, valuePtr sys.Pointer, flags MapLookupFlags) error { @@ -764,13 +876,13 @@ func (m *Map) Delete(key interface{}) error { // // Returns ErrKeyNotExist if there is no next key. func (m *Map) NextKey(key, nextKeyOut interface{}) error { - nextKeyPtr, nextKeyBytes := makeBuffer(nextKeyOut, int(m.keySize)) + nextKeyBytes := makeMapSyscallOutput(nextKeyOut, int(m.keySize)) - if err := m.nextKey(key, nextKeyPtr); err != nil { + if err := m.nextKey(key, nextKeyBytes.Pointer()); err != nil { return err } - if err := m.unmarshalKey(nextKeyOut, nextKeyBytes); err != nil { + if err := nextKeyBytes.Unmarshal(nextKeyOut); err != nil { return fmt.Errorf("can't unmarshal next key: %w", err) } return nil @@ -837,7 +949,7 @@ func (m *Map) nextKey(key interface{}, nextKeyOut sys.Pointer) error { return nil } -var mmapProtectedPage = internal.Memoize(func() ([]byte, error) { +var mmapProtectedPage = sync.OnceValues(func() ([]byte, error) { return unix.Mmap(-1, 0, os.Getpagesize(), unix.PROT_NONE, unix.MAP_ANON|unix.MAP_SHARED) }) @@ -893,14 +1005,23 @@ func (m *Map) guessNonExistentKey() ([]byte, error) { // // "keysOut" and "valuesOut" must be of type slice, a pointer // to a slice or buffer will not work. -// "prevKey" is the key to start the batch lookup from, it will -// *not* be included in the results. Use nil to start at the first key. +// "cursor" is an pointer to an opaque handle. It must be non-nil. Pass +// "cursor" to subsequent calls of this function to continue the batching +// operation in the case of chunking. +// +// Warning: This API is not very safe to use as the kernel implementation for +// batching relies on the user to be aware of subtle details with regarding to +// different map type implementations. // // ErrKeyNotExist is returned when the batch lookup has reached // the end of all possible results, even when partial results // are returned. It should be used to evaluate when lookup is "done". -func (m *Map) BatchLookup(prevKey, nextKeyOut, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { - return m.batchLookup(sys.BPF_MAP_LOOKUP_BATCH, prevKey, nextKeyOut, keysOut, valuesOut, opts) +func (m *Map) BatchLookup(cursor *MapBatchCursor, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { + n, err := m.batchLookup(sys.BPF_MAP_LOOKUP_BATCH, cursor, keysOut, valuesOut, opts) + if err != nil { + return n, fmt.Errorf("map batch lookup: %w", err) + } + return n, nil } // BatchLookupAndDelete looks up many elements in a map at once, @@ -908,47 +1029,121 @@ func (m *Map) BatchLookup(prevKey, nextKeyOut, keysOut, valuesOut interface{}, o // It then deletes all those elements. // "keysOut" and "valuesOut" must be of type slice, a pointer // to a slice or buffer will not work. -// "prevKey" is the key to start the batch lookup from, it will -// *not* be included in the results. Use nil to start at the first key. +// "cursor" is an pointer to an opaque handle. It must be non-nil. Pass +// "cursor" to subsequent calls of this function to continue the batching +// operation in the case of chunking. +// +// Warning: This API is not very safe to use as the kernel implementation for +// batching relies on the user to be aware of subtle details with regarding to +// different map type implementations. // // ErrKeyNotExist is returned when the batch lookup has reached // the end of all possible results, even when partial results // are returned. It should be used to evaluate when lookup is "done". -func (m *Map) BatchLookupAndDelete(prevKey, nextKeyOut, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { - return m.batchLookup(sys.BPF_MAP_LOOKUP_AND_DELETE_BATCH, prevKey, nextKeyOut, keysOut, valuesOut, opts) +func (m *Map) BatchLookupAndDelete(cursor *MapBatchCursor, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { + n, err := m.batchLookup(sys.BPF_MAP_LOOKUP_AND_DELETE_BATCH, cursor, keysOut, valuesOut, opts) + if err != nil { + return n, fmt.Errorf("map batch lookup and delete: %w", err) + } + return n, nil } -func (m *Map) batchLookup(cmd sys.Cmd, startKey, nextKeyOut, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { - if err := haveBatchAPI(); err != nil { - return 0, err - } +// MapBatchCursor represents a starting point for a batch operation. +type MapBatchCursor struct { + m *Map + opaque []byte +} + +func (m *Map) batchLookup(cmd sys.Cmd, cursor *MapBatchCursor, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { if m.typ.hasPerCPUValue() { - return 0, ErrNotSupported + return m.batchLookupPerCPU(cmd, cursor, keysOut, valuesOut, opts) } - keysValue := reflect.ValueOf(keysOut) - if keysValue.Kind() != reflect.Slice { - return 0, fmt.Errorf("keys must be a slice") + + count, err := batchCount(keysOut, valuesOut) + if err != nil { + return 0, err } - valuesValue := reflect.ValueOf(valuesOut) - if valuesValue.Kind() != reflect.Slice { - return 0, fmt.Errorf("valuesOut must be a slice") + + valueBuf := sysenc.SyscallOutput(valuesOut, count*int(m.fullValueSize)) + + n, err := m.batchLookupCmd(cmd, cursor, count, keysOut, valueBuf.Pointer(), opts) + if errors.Is(err, unix.ENOSPC) { + // Hash tables return ENOSPC when the size of the batch is smaller than + // any bucket. + return n, fmt.Errorf("%w (batch size too small?)", err) + } else if err != nil { + return n, err } - count := keysValue.Len() - if count != valuesValue.Len() { - return 0, fmt.Errorf("keysOut and valuesOut must be the same length") + + err = valueBuf.Unmarshal(valuesOut) + if err != nil { + return 0, err } - keyBuf := make([]byte, count*int(m.keySize)) - keyPtr := sys.NewSlicePointer(keyBuf) + + return n, nil +} + +func (m *Map) batchLookupPerCPU(cmd sys.Cmd, cursor *MapBatchCursor, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { + count, err := sliceLen(keysOut) + if err != nil { + return 0, fmt.Errorf("keys: %w", err) + } + valueBuf := make([]byte, count*int(m.fullValueSize)) valuePtr := sys.NewSlicePointer(valueBuf) - nextPtr, nextBuf := makeBuffer(nextKeyOut, int(m.keySize)) + + n, sysErr := m.batchLookupCmd(cmd, cursor, count, keysOut, valuePtr, opts) + if sysErr != nil && !errors.Is(sysErr, unix.ENOENT) { + return 0, err + } + + err = unmarshalBatchPerCPUValue(valuesOut, count, int(m.valueSize), valueBuf) + if err != nil { + return 0, err + } + + return n, sysErr +} + +func (m *Map) batchLookupCmd(cmd sys.Cmd, cursor *MapBatchCursor, count int, keysOut any, valuePtr sys.Pointer, opts *BatchOptions) (int, error) { + cursorLen := int(m.keySize) + if cursorLen < 4 { + // * generic_map_lookup_batch requires that batch_out is key_size bytes. + // This is used by array and LPM maps. + // + // * __htab_map_lookup_and_delete_batch requires u32. This is used by the + // various hash maps. + // + // Use a minimum of 4 bytes to avoid having to distinguish between the two. + cursorLen = 4 + } + + inBatch := cursor.opaque + if inBatch == nil { + // This is the first lookup, allocate a buffer to hold the cursor. + cursor.opaque = make([]byte, cursorLen) + cursor.m = m + } else if cursor.m != m { + // Prevent reuse of a cursor across maps. First, it's unlikely to work. + // Second, the maps may require different cursorLen and cursor.opaque + // may therefore be too short. This could lead to the kernel clobbering + // user space memory. + return 0, errors.New("a cursor may not be reused across maps") + } + + if err := haveBatchAPI(); err != nil { + return 0, err + } + + keyBuf := sysenc.SyscallOutput(keysOut, count*int(m.keySize)) attr := sys.MapLookupBatchAttr{ MapFd: m.fd.Uint(), - Keys: keyPtr, + Keys: keyBuf.Pointer(), Values: valuePtr, Count: uint32(count), - OutBatch: nextPtr, + InBatch: sys.NewSlicePointer(inBatch), + OutBatch: sys.NewSlicePointer(cursor.opaque), } if opts != nil { @@ -956,30 +1151,13 @@ func (m *Map) batchLookup(cmd sys.Cmd, startKey, nextKeyOut, keysOut, valuesOut attr.Flags = opts.Flags } - var err error - if startKey != nil { - attr.InBatch, err = marshalPtr(startKey, int(m.keySize)) - if err != nil { - return 0, err - } - } - _, sysErr := sys.BPF(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) sysErr = wrapMapError(sysErr) if sysErr != nil && !errors.Is(sysErr, unix.ENOENT) { return 0, sysErr } - err = m.unmarshalKey(nextKeyOut, nextBuf) - if err != nil { - return 0, err - } - err = unmarshalBytes(keysOut, keyBuf) - if err != nil { - return 0, err - } - err = unmarshalBytes(valuesOut, valueBuf) - if err != nil { + if err := keyBuf.Unmarshal(keysOut); err != nil { return 0, err } @@ -991,33 +1169,25 @@ func (m *Map) batchLookup(cmd sys.Cmd, startKey, nextKeyOut, keysOut, valuesOut // "keys" and "values" must be of type slice, a pointer // to a slice or buffer will not work. func (m *Map) BatchUpdate(keys, values interface{}, opts *BatchOptions) (int, error) { - if err := haveBatchAPI(); err != nil { - return 0, err - } if m.typ.hasPerCPUValue() { - return 0, ErrNotSupported + return m.batchUpdatePerCPU(keys, values, opts) } - keysValue := reflect.ValueOf(keys) - if keysValue.Kind() != reflect.Slice { - return 0, fmt.Errorf("keys must be a slice") - } - valuesValue := reflect.ValueOf(values) - if valuesValue.Kind() != reflect.Slice { - return 0, fmt.Errorf("values must be a slice") - } - var ( - count = keysValue.Len() - valuePtr sys.Pointer - err error - ) - if count != valuesValue.Len() { - return 0, fmt.Errorf("keys and values must be the same length") + + count, err := batchCount(keys, values) + if err != nil { + return 0, err } - keyPtr, err := marshalPtr(keys, count*int(m.keySize)) + + valuePtr, err := marshalMapSyscallInput(values, count*int(m.valueSize)) if err != nil { return 0, err } - valuePtr, err = marshalPtr(values, count*int(m.valueSize)) + + return m.batchUpdate(count, keys, valuePtr, opts) +} + +func (m *Map) batchUpdate(count int, keys any, valuePtr sys.Pointer, opts *BatchOptions) (int, error) { + keyPtr, err := marshalMapSyscallInput(keys, count*int(m.keySize)) if err != nil { return 0, err } @@ -1035,27 +1205,38 @@ func (m *Map) BatchUpdate(keys, values interface{}, opts *BatchOptions) (int, er err = sys.MapUpdateBatch(&attr) if err != nil { + if haveFeatErr := haveBatchAPI(); haveFeatErr != nil { + return 0, haveFeatErr + } return int(attr.Count), fmt.Errorf("batch update: %w", wrapMapError(err)) } return int(attr.Count), nil } +func (m *Map) batchUpdatePerCPU(keys, values any, opts *BatchOptions) (int, error) { + count, err := sliceLen(keys) + if err != nil { + return 0, fmt.Errorf("keys: %w", err) + } + + valueBuf, err := marshalBatchPerCPUValue(values, count, int(m.valueSize)) + if err != nil { + return 0, err + } + + return m.batchUpdate(count, keys, sys.NewSlicePointer(valueBuf), opts) +} + // BatchDelete batch deletes entries in the map by keys. // "keys" must be of type slice, a pointer to a slice or buffer will not work. func (m *Map) BatchDelete(keys interface{}, opts *BatchOptions) (int, error) { - if err := haveBatchAPI(); err != nil { - return 0, err - } - if m.typ.hasPerCPUValue() { - return 0, ErrNotSupported - } - keysValue := reflect.ValueOf(keys) - if keysValue.Kind() != reflect.Slice { - return 0, fmt.Errorf("keys must be a slice") + count, err := sliceLen(keys) + if err != nil { + return 0, fmt.Errorf("keys: %w", err) } - count := keysValue.Len() - keyPtr, err := marshalPtr(keys, count*int(m.keySize)) + + keyPtr, err := marshalMapSyscallInput(keys, count*int(m.keySize)) if err != nil { return 0, fmt.Errorf("cannot marshal keys: %v", err) } @@ -1072,12 +1253,33 @@ func (m *Map) BatchDelete(keys interface{}, opts *BatchOptions) (int, error) { } if err = sys.MapDeleteBatch(&attr); err != nil { + if haveFeatErr := haveBatchAPI(); haveFeatErr != nil { + return 0, haveFeatErr + } return int(attr.Count), fmt.Errorf("batch delete: %w", wrapMapError(err)) } return int(attr.Count), nil } +func batchCount(keys, values any) (int, error) { + keysLen, err := sliceLen(keys) + if err != nil { + return 0, fmt.Errorf("keys: %w", err) + } + + valuesLen, err := sliceLen(values) + if err != nil { + return 0, fmt.Errorf("values: %w", err) + } + + if keysLen != valuesLen { + return 0, fmt.Errorf("keys and values must have the same length") + } + + return keysLen, nil +} + // Iterate traverses a map. // // It's safe to create multiple iterators at the same time. @@ -1176,15 +1378,14 @@ func (m *Map) IsPinned() bool { // // It makes no changes to kernel-side restrictions. func (m *Map) Freeze() error { - if err := haveMapMutabilityModifiers(); err != nil { - return fmt.Errorf("can't freeze map: %w", err) - } - attr := sys.MapFreezeAttr{ MapFd: m.fd.Uint(), } if err := sys.MapFreeze(&attr); err != nil { + if haveFeatErr := haveMapMutabilityModifiers(); haveFeatErr != nil { + return fmt.Errorf("can't freeze map: %w", haveFeatErr) + } return fmt.Errorf("can't freeze map: %w", err) } return nil @@ -1217,16 +1418,7 @@ func (m *Map) marshalKey(data interface{}) (sys.Pointer, error) { return sys.Pointer{}, errors.New("can't use nil as key of map") } - return marshalPtr(data, int(m.keySize)) -} - -func (m *Map) unmarshalKey(data interface{}, buf []byte) error { - if buf == nil { - // This is from a makeBuffer call, nothing do do here. - return nil - } - - return unmarshalBytes(data, buf) + return marshalMapSyscallInput(data, int(m.keySize)) } func (m *Map) marshalValue(data interface{}) (sys.Pointer, error) { @@ -1249,7 +1441,7 @@ func (m *Map) marshalValue(data interface{}) (sys.Pointer, error) { buf, err = marshalProgram(value, int(m.valueSize)) default: - return marshalPtr(data, int(m.valueSize)) + return marshalMapSyscallInput(data, int(m.valueSize)) } if err != nil { @@ -1259,16 +1451,7 @@ func (m *Map) marshalValue(data interface{}) (sys.Pointer, error) { return sys.NewSlicePointer(buf), nil } -func (m *Map) unmarshalValue(value interface{}, buf []byte) error { - if buf == nil { - // This is from a makeBuffer call, nothing do do here. - return nil - } - - if m.typ.hasPerCPUValue() { - return unmarshalPerCPUValue(value, int(m.valueSize), buf) - } - +func (m *Map) unmarshalValue(value any, buf sysenc.Buffer) error { switch value := value.(type) { case **Map: if !m.typ.canStoreMap() { @@ -1315,7 +1498,7 @@ func (m *Map) unmarshalValue(value interface{}, buf []byte) error { return errors.New("require pointer to *Program") } - return unmarshalBytes(value, buf) + return buf.Unmarshal(value) } // LoadPinnedMap loads a Map from a BPF file. @@ -1337,12 +1520,11 @@ func LoadPinnedMap(fileName string, opts *LoadPinOptions) (*Map, error) { } // unmarshalMap creates a map from a map ID encoded in host endianness. -func unmarshalMap(buf []byte) (*Map, error) { - if len(buf) != 4 { - return nil, errors.New("map id requires 4 byte value") +func unmarshalMap(buf sysenc.Buffer) (*Map, error) { + var id uint32 + if err := buf.Unmarshal(&id); err != nil { + return nil, err } - - id := internal.NativeEndian.Uint32(buf) return NewMapFromID(MapID(id)) } @@ -1361,8 +1543,10 @@ func marshalMap(m *Map, length int) ([]byte, error) { // // See Map.Iterate. type MapIterator struct { - target *Map - curKey []byte + target *Map + // Temporary storage to avoid allocations in Next(). This is any instead + // of []byte to avoid allocations. + cursor any count, maxEntries uint32 done bool err error @@ -1390,38 +1574,30 @@ func (mi *MapIterator) Next(keyOut, valueOut interface{}) bool { return false } - // For array-like maps NextKeyBytes returns nil only on after maxEntries + // For array-like maps NextKey returns nil only after maxEntries // iterations. for mi.count <= mi.maxEntries { - var nextKey []byte - if mi.curKey == nil { - // Pass nil interface to NextKeyBytes to make sure the Map's first key + if mi.cursor == nil { + // Pass nil interface to NextKey to make sure the Map's first key // is returned. If we pass an uninitialized []byte instead, it'll see a // non-nil interface and try to marshal it. - nextKey, mi.err = mi.target.NextKeyBytes(nil) - - mi.curKey = make([]byte, mi.target.keySize) + mi.cursor = make([]byte, mi.target.keySize) + mi.err = mi.target.NextKey(nil, mi.cursor) } else { - nextKey, mi.err = mi.target.NextKeyBytes(mi.curKey) - } - if mi.err != nil { - mi.err = fmt.Errorf("get next key: %w", mi.err) - return false + mi.err = mi.target.NextKey(mi.cursor, mi.cursor) } - if nextKey == nil { + if errors.Is(mi.err, ErrKeyNotExist) { mi.done = true + mi.err = nil + return false + } else if mi.err != nil { + mi.err = fmt.Errorf("get next key: %w", mi.err) return false } - // The user can get access to nextKey since unmarshalBytes - // does not copy when unmarshaling into a []byte. - // Make a copy to prevent accidental corruption of - // iterator state. - copy(mi.curKey, nextKey) - mi.count++ - mi.err = mi.target.Lookup(nextKey, valueOut) + mi.err = mi.target.Lookup(mi.cursor, valueOut) if errors.Is(mi.err, ErrKeyNotExist) { // Even though the key should be valid, we couldn't look up // its value. If we're iterating a hash map this is probably @@ -1438,7 +1614,13 @@ func (mi *MapIterator) Next(keyOut, valueOut interface{}) bool { return false } - mi.err = mi.target.unmarshalKey(keyOut, nextKey) + buf := mi.cursor.([]byte) + if ptr, ok := keyOut.(unsafe.Pointer); ok { + copy(unsafe.Slice((*byte)(ptr), len(buf)), buf) + } else { + mi.err = sysenc.Unmarshal(keyOut, buf) + } + return mi.err == nil } @@ -1476,3 +1658,12 @@ func NewMapFromID(id MapID) (*Map, error) { return newMapFromFD(fd) } + +// sliceLen returns the length if the value is a slice or an error otherwise. +func sliceLen(slice any) (int, error) { + sliceValue := reflect.ValueOf(slice) + if sliceValue.Kind() != reflect.Slice { + return 0, fmt.Errorf("%T is not a slice", slice) + } + return sliceValue.Len(), nil +} diff --git a/vendor/github.com/cilium/ebpf/marshalers.go b/vendor/github.com/cilium/ebpf/marshalers.go index a568bff92074..57a0a8e88af6 100644 --- a/vendor/github.com/cilium/ebpf/marshalers.go +++ b/vendor/github.com/cilium/ebpf/marshalers.go @@ -1,249 +1,210 @@ package ebpf import ( - "bytes" "encoding" - "encoding/binary" "errors" "fmt" "reflect" - "runtime" - "sync" + "slices" "unsafe" "github.com/cilium/ebpf/internal" "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/sysenc" ) -// marshalPtr converts an arbitrary value into a pointer suitable +// marshalMapSyscallInput converts an arbitrary value into a pointer suitable // to be passed to the kernel. // // As an optimization, it returns the original value if it is an // unsafe.Pointer. -func marshalPtr(data interface{}, length int) (sys.Pointer, error) { +func marshalMapSyscallInput(data any, length int) (sys.Pointer, error) { if ptr, ok := data.(unsafe.Pointer); ok { return sys.NewPointer(ptr), nil } - buf, err := marshalBytes(data, length) + buf, err := sysenc.Marshal(data, length) if err != nil { return sys.Pointer{}, err } - return sys.NewSlicePointer(buf), nil + return buf.Pointer(), nil } -// marshalBytes converts an arbitrary value into a byte buffer. -// -// Prefer using Map.marshalKey and Map.marshalValue if possible, since -// those have special cases that allow more types to be encoded. -// -// Returns an error if the given value isn't representable in exactly -// length bytes. -func marshalBytes(data interface{}, length int) (buf []byte, err error) { - if data == nil { - return nil, errors.New("can't marshal a nil value") - } - - switch value := data.(type) { - case encoding.BinaryMarshaler: - buf, err = value.MarshalBinary() - case string: - buf = []byte(value) - case []byte: - buf = value - case unsafe.Pointer: - err = errors.New("can't marshal from unsafe.Pointer") - case Map, *Map, Program, *Program: - err = fmt.Errorf("can't marshal %T", value) - default: - wr := internal.NewBuffer(make([]byte, 0, length)) - defer internal.PutBuffer(wr) - - err = binary.Write(wr, internal.NativeEndian, value) - if err != nil { - err = fmt.Errorf("encoding %T: %v", value, err) - } - buf = wr.Bytes() - } - if err != nil { - return nil, err +func makeMapSyscallOutput(dst any, length int) sysenc.Buffer { + if ptr, ok := dst.(unsafe.Pointer); ok { + return sysenc.UnsafeBuffer(ptr) } - if len(buf) != length { - return nil, fmt.Errorf("%T doesn't marshal to %d bytes", data, length) + _, ok := dst.(encoding.BinaryUnmarshaler) + if ok { + return sysenc.SyscallOutput(nil, length) } - return buf, nil + + return sysenc.SyscallOutput(dst, length) } -func makeBuffer(dst interface{}, length int) (sys.Pointer, []byte) { - if ptr, ok := dst.(unsafe.Pointer); ok { - return sys.NewPointer(ptr), nil +// appendPerCPUSlice encodes a slice containing one value per +// possible CPU into a buffer of bytes. +// +// Values are initialized to zero if the slice has less elements than CPUs. +func appendPerCPUSlice(buf []byte, slice any, possibleCPUs, elemLength, alignedElemLength int) ([]byte, error) { + sliceType := reflect.TypeOf(slice) + if sliceType.Kind() != reflect.Slice { + return nil, errors.New("per-CPU value requires slice") } - buf := make([]byte, length) - return sys.NewSlicePointer(buf), buf -} - -var bytesReaderPool = sync.Pool{ - New: func() interface{} { - return new(bytes.Reader) - }, -} + sliceValue := reflect.ValueOf(slice) + sliceLen := sliceValue.Len() + if sliceLen > possibleCPUs { + return nil, fmt.Errorf("per-CPU value greater than number of CPUs") + } -// unmarshalBytes converts a byte buffer into an arbitrary value. -// -// Prefer using Map.unmarshalKey and Map.unmarshalValue if possible, since -// those have special cases that allow more types to be encoded. -// -// The common int32 and int64 types are directly handled to avoid -// unnecessary heap allocations as happening in the default case. -func unmarshalBytes(data interface{}, buf []byte) error { - switch value := data.(type) { - case unsafe.Pointer: - dst := unsafe.Slice((*byte)(value), len(buf)) - copy(dst, buf) - runtime.KeepAlive(value) - return nil - case Map, *Map, Program, *Program: - return fmt.Errorf("can't unmarshal into %T", value) - case encoding.BinaryUnmarshaler: - return value.UnmarshalBinary(buf) - case *string: - *value = string(buf) - return nil - case *[]byte: - *value = buf - return nil - case *int32: - if len(buf) < 4 { - return errors.New("int32 requires 4 bytes") - } - *value = int32(internal.NativeEndian.Uint32(buf)) - return nil - case *uint32: - if len(buf) < 4 { - return errors.New("uint32 requires 4 bytes") - } - *value = internal.NativeEndian.Uint32(buf) - return nil - case *int64: - if len(buf) < 8 { - return errors.New("int64 requires 8 bytes") - } - *value = int64(internal.NativeEndian.Uint64(buf)) - return nil - case *uint64: - if len(buf) < 8 { - return errors.New("uint64 requires 8 bytes") - } - *value = internal.NativeEndian.Uint64(buf) - return nil - case string: - return errors.New("require pointer to string") - case []byte: - return errors.New("require pointer to []byte") - default: - rd := bytesReaderPool.Get().(*bytes.Reader) - rd.Reset(buf) - defer bytesReaderPool.Put(rd) - if err := binary.Read(rd, internal.NativeEndian, value); err != nil { - return fmt.Errorf("decoding %T: %v", value, err) + // Grow increases the slice's capacity, _if_necessary_ + buf = slices.Grow(buf, alignedElemLength*possibleCPUs) + for i := 0; i < sliceLen; i++ { + elem := sliceValue.Index(i).Interface() + elemBytes, err := sysenc.Marshal(elem, elemLength) + if err != nil { + return nil, err } - return nil + + buf = elemBytes.AppendTo(buf) + buf = append(buf, make([]byte, alignedElemLength-elemLength)...) } + + // Ensure buf is zero-padded full size. + buf = append(buf, make([]byte, (possibleCPUs-sliceLen)*alignedElemLength)...) + + return buf, nil } // marshalPerCPUValue encodes a slice containing one value per // possible CPU into a buffer of bytes. // // Values are initialized to zero if the slice has less elements than CPUs. -// -// slice must have a type like []elementType. -func marshalPerCPUValue(slice interface{}, elemLength int) (sys.Pointer, error) { - sliceType := reflect.TypeOf(slice) - if sliceType.Kind() != reflect.Slice { - return sys.Pointer{}, errors.New("per-CPU value requires slice") +func marshalPerCPUValue(slice any, elemLength int) (sys.Pointer, error) { + possibleCPUs, err := PossibleCPU() + if err != nil { + return sys.Pointer{}, err } - possibleCPUs, err := internal.PossibleCPUs() + alignedElemLength := internal.Align(elemLength, 8) + buf := make([]byte, 0, alignedElemLength*possibleCPUs) + buf, err = appendPerCPUSlice(buf, slice, possibleCPUs, elemLength, alignedElemLength) if err != nil { return sys.Pointer{}, err } - sliceValue := reflect.ValueOf(slice) - sliceLen := sliceValue.Len() - if sliceLen > possibleCPUs { - return sys.Pointer{}, fmt.Errorf("per-CPU value exceeds number of CPUs") + return sys.NewSlicePointer(buf), nil +} + +// marshalBatchPerCPUValue encodes a batch-sized slice of slices containing +// one value per possible CPU into a buffer of bytes. +func marshalBatchPerCPUValue(slice any, batchLen, elemLength int) ([]byte, error) { + sliceType := reflect.TypeOf(slice) + if sliceType.Kind() != reflect.Slice { + return nil, fmt.Errorf("batch value requires a slice") } + sliceValue := reflect.ValueOf(slice) + possibleCPUs, err := PossibleCPU() + if err != nil { + return nil, err + } + if sliceValue.Len() != batchLen*possibleCPUs { + return nil, fmt.Errorf("per-CPU slice has incorrect length, expected %d, got %d", + batchLen*possibleCPUs, sliceValue.Len()) + } alignedElemLength := internal.Align(elemLength, 8) - buf := make([]byte, alignedElemLength*possibleCPUs) - - for i := 0; i < sliceLen; i++ { - elem := sliceValue.Index(i).Interface() - elemBytes, err := marshalBytes(elem, elemLength) + buf := make([]byte, 0, batchLen*alignedElemLength*possibleCPUs) + for i := 0; i < batchLen; i++ { + batch := sliceValue.Slice(i*possibleCPUs, (i+1)*possibleCPUs).Interface() + buf, err = appendPerCPUSlice(buf, batch, possibleCPUs, elemLength, alignedElemLength) if err != nil { - return sys.Pointer{}, err + return nil, fmt.Errorf("batch %d: %w", i, err) } - - offset := i * alignedElemLength - copy(buf[offset:offset+elemLength], elemBytes) } - - return sys.NewSlicePointer(buf), nil + return buf, nil } // unmarshalPerCPUValue decodes a buffer into a slice containing one value per // possible CPU. // -// valueOut must have a type like *[]elementType -func unmarshalPerCPUValue(slicePtr interface{}, elemLength int, buf []byte) error { - slicePtrType := reflect.TypeOf(slicePtr) - if slicePtrType.Kind() != reflect.Ptr || slicePtrType.Elem().Kind() != reflect.Slice { - return fmt.Errorf("per-cpu value requires pointer to slice") +// slice must be a literal slice and not a pointer. +func unmarshalPerCPUValue(slice any, elemLength int, buf []byte) error { + sliceType := reflect.TypeOf(slice) + if sliceType.Kind() != reflect.Slice { + return fmt.Errorf("per-CPU value requires a slice") } - possibleCPUs, err := internal.PossibleCPUs() + possibleCPUs, err := PossibleCPU() if err != nil { return err } - sliceType := slicePtrType.Elem() - slice := reflect.MakeSlice(sliceType, possibleCPUs, possibleCPUs) + sliceValue := reflect.ValueOf(slice) + if sliceValue.Len() != possibleCPUs { + return fmt.Errorf("per-CPU slice has incorrect length, expected %d, got %d", + possibleCPUs, sliceValue.Len()) + } sliceElemType := sliceType.Elem() sliceElemIsPointer := sliceElemType.Kind() == reflect.Ptr - if sliceElemIsPointer { - sliceElemType = sliceElemType.Elem() - } - - step := len(buf) / possibleCPUs - if step < elemLength { - return fmt.Errorf("per-cpu element length is larger than available data") - } + stride := internal.Align(elemLength, 8) for i := 0; i < possibleCPUs; i++ { - var elem interface{} + var elem any + v := sliceValue.Index(i) if sliceElemIsPointer { - newElem := reflect.New(sliceElemType) - slice.Index(i).Set(newElem) - elem = newElem.Interface() + if !v.Elem().CanAddr() { + return fmt.Errorf("per-CPU slice elements cannot be nil") + } + elem = v.Elem().Addr().Interface() } else { - elem = slice.Index(i).Addr().Interface() + elem = v.Addr().Interface() } - - // Make a copy, since unmarshal can hold on to itemBytes - elemBytes := make([]byte, elemLength) - copy(elemBytes, buf[:elemLength]) - - err := unmarshalBytes(elem, elemBytes) + err := sysenc.Unmarshal(elem, buf[:elemLength]) if err != nil { return fmt.Errorf("cpu %d: %w", i, err) } - buf = buf[step:] + buf = buf[stride:] + } + return nil +} + +// unmarshalBatchPerCPUValue decodes a buffer into a batch-sized slice +// containing one value per possible CPU. +// +// slice must have length batchLen * PossibleCPUs(). +func unmarshalBatchPerCPUValue(slice any, batchLen, elemLength int, buf []byte) error { + sliceType := reflect.TypeOf(slice) + if sliceType.Kind() != reflect.Slice { + return fmt.Errorf("batch requires a slice") + } + + sliceValue := reflect.ValueOf(slice) + possibleCPUs, err := PossibleCPU() + if err != nil { + return err + } + if sliceValue.Len() != batchLen*possibleCPUs { + return fmt.Errorf("per-CPU slice has incorrect length, expected %d, got %d", + sliceValue.Len(), batchLen*possibleCPUs) + } + + fullValueSize := possibleCPUs * internal.Align(elemLength, 8) + if len(buf) != batchLen*fullValueSize { + return fmt.Errorf("input buffer has incorrect length, expected %d, got %d", + len(buf), batchLen*fullValueSize) } - reflect.ValueOf(slicePtr).Elem().Set(slice) + for i := 0; i < batchLen; i++ { + elem := sliceValue.Slice(i*possibleCPUs, (i+1)*possibleCPUs).Interface() + if err := unmarshalPerCPUValue(elem, elemLength, buf[:fullValueSize]); err != nil { + return fmt.Errorf("batch %d: %w", i, err) + } + buf = buf[fullValueSize:] + } return nil } diff --git a/vendor/github.com/cilium/ebpf/netlify.toml b/vendor/github.com/cilium/ebpf/netlify.toml new file mode 100644 index 000000000000..67c83f3b307f --- /dev/null +++ b/vendor/github.com/cilium/ebpf/netlify.toml @@ -0,0 +1,4 @@ +[build] + base = "docs/" + publish = "site/" + command = "mkdocs build" diff --git a/vendor/github.com/cilium/ebpf/prog.go b/vendor/github.com/cilium/ebpf/prog.go index 70aaef55327a..9bc6325f8879 100644 --- a/vendor/github.com/cilium/ebpf/prog.go +++ b/vendor/github.com/cilium/ebpf/prog.go @@ -15,13 +15,27 @@ import ( "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/btf" "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/kallsyms" "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/sysenc" "github.com/cilium/ebpf/internal/unix" ) // ErrNotSupported is returned whenever the kernel doesn't support a feature. var ErrNotSupported = internal.ErrNotSupported +// errBadRelocation is returned when the verifier rejects a program due to a +// bad CO-RE relocation. +// +// This error is detected based on heuristics and therefore may not be reliable. +var errBadRelocation = errors.New("bad CO-RE relocation") + +// errUnknownKfunc is returned when the verifier rejects a program due to an +// unknown kfunc. +// +// This error is detected based on heuristics and therefore may not be reliable. +var errUnknownKfunc = errors.New("unknown kfunc") + // ProgramID represents the unique ID of an eBPF program. type ProgramID uint32 @@ -32,13 +46,13 @@ const ( outputPad = 256 + 2 ) -// DefaultVerifierLogSize is the default number of bytes allocated for the -// verifier log. +// Deprecated: the correct log size is now detected automatically and this +// constant is unused. const DefaultVerifierLogSize = 64 * 1024 -// maxVerifierLogSize is the maximum size of verifier log buffer the kernel -// will accept before returning EINVAL. -const maxVerifierLogSize = math.MaxUint32 >> 2 +// minVerifierLogSize is the default number of bytes allocated for the +// verifier log. +const minVerifierLogSize = 64 * 1024 // ProgramOptions control loading a program into the kernel. type ProgramOptions struct { @@ -52,22 +66,15 @@ type ProgramOptions struct { // verifier output enabled. Upon error, the program load will be repeated // with LogLevelBranch and the given (or default) LogSize value. // - // Setting this to a non-zero value will unconditionally enable the verifier + // Unless LogDisabled is set, setting this to a non-zero value will enable the verifier // log, populating the [ebpf.Program.VerifierLog] field on successful loads // and including detailed verifier errors if the program is rejected. This // will always allocate an output buffer, but will result in only a single // attempt at loading the program. LogLevel LogLevel - // Controls the output buffer size for the verifier log, in bytes. See the - // documentation on ProgramOptions.LogLevel for details about how this value - // is used. - // - // If this value is set too low to fit the verifier log, the resulting - // [ebpf.VerifierError]'s Truncated flag will be true, and the error string - // will also contain a hint to that effect. - // - // Defaults to DefaultVerifierLogSize. + // Deprecated: the correct log buffer size is determined automatically + // and this field is ignored. LogSize int // Disables the verifier log completely, regardless of other options. @@ -79,6 +86,14 @@ type ProgramOptions struct { // (containers) or where it is in a non-standard location. Defaults to // use the kernel BTF from a well-known location if nil. KernelTypes *btf.Spec + + // Type information used for CO-RE relocations of kernel modules, + // indexed by module name. + // + // This is useful in environments where the kernel BTF is not available + // (containers) or where it is in a non-standard location. Defaults to + // use the kernel module BTF from a well-known location if nil. + KernelModuleTypes map[string]*btf.Spec } // ProgramSpec defines a Program. @@ -147,6 +162,28 @@ func (ps *ProgramSpec) Tag() (string, error) { return ps.Instructions.Tag(internal.NativeEndian) } +// KernelModule returns the kernel module, if any, the AttachTo function is contained in. +func (ps *ProgramSpec) KernelModule() (string, error) { + if ps.AttachTo == "" { + return "", nil + } + + switch ps.Type { + default: + return "", nil + case Tracing: + switch ps.AttachType { + default: + return "", nil + case AttachTraceFEntry: + case AttachTraceFExit: + } + fallthrough + case Kprobe: + return kallsyms.KernelModule(ps.AttachTo) + } +} + // VerifierError is returned by [NewProgram] and [NewProgramWithOptions] if a // program is rejected by the verifier. // @@ -196,6 +233,15 @@ func NewProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er return prog, err } +var ( + coreBadLoad = []byte(fmt.Sprintf("(18) r10 = 0x%x\n", btf.COREBadRelocationSentinel)) + // This log message was introduced by ebb676daa1a3 ("bpf: Print function name in + // addition to function id") which first appeared in v4.10 and has remained + // unchanged since. + coreBadCall = []byte(fmt.Sprintf("invalid func unknown#%d\n", btf.COREBadRelocationSentinel)) + kfuncBadCall = []byte(fmt.Sprintf("invalid func unknown#%d\n", kfuncCallPoisonBase)) +) + func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, error) { if len(spec.Instructions) == 0 { return nil, errors.New("instructions cannot be empty") @@ -209,10 +255,6 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er return nil, fmt.Errorf("can't load %s program on %s", spec.ByteOrder, internal.NativeEndian) } - if opts.LogSize < 0 { - return nil, errors.New("ProgramOptions.LogSize must be a positive value; disable verifier logs using ProgramOptions.LogDisabled") - } - // Kernels before 5.0 (6c4fc209fcf9 "bpf: remove useless version check for prog load") // require the version field to be set to the value of the KERNEL_VERSION // macro for kprobe-type programs. @@ -241,14 +283,41 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er insns := make(asm.Instructions, len(spec.Instructions)) copy(insns, spec.Instructions) - handle, fib, lib, err := btf.MarshalExtInfos(insns) - if err != nil && !errors.Is(err, btf.ErrNotSupported) { - return nil, fmt.Errorf("load ext_infos: %w", err) + kmodName, err := spec.KernelModule() + if err != nil { + return nil, fmt.Errorf("kernel module search: %w", err) } - if handle != nil { - defer handle.Close() - attr.ProgBtfFd = uint32(handle.FD()) + var targets []*btf.Spec + if opts.KernelTypes != nil { + targets = append(targets, opts.KernelTypes) + } + if kmodName != "" && opts.KernelModuleTypes != nil { + if modBTF, ok := opts.KernelModuleTypes[kmodName]; ok { + targets = append(targets, modBTF) + } + } + + var b btf.Builder + if err := applyRelocations(insns, targets, kmodName, spec.ByteOrder, &b); err != nil { + return nil, fmt.Errorf("apply CO-RE relocations: %w", err) + } + + errExtInfos := haveProgramExtInfos() + if !b.Empty() && errors.Is(errExtInfos, ErrNotSupported) { + // There is at least one CO-RE relocation which relies on a stable local + // type ID. + // Return ErrNotSupported instead of E2BIG if there is no BTF support. + return nil, errExtInfos + } + + if errExtInfos == nil { + // Only add func and line info if the kernel supports it. This allows + // BPF compiled with modern toolchains to work on old kernels. + fib, lib, err := btf.MarshalExtInfos(insns, &b) + if err != nil { + return nil, fmt.Errorf("marshal ext_infos: %w", err) + } attr.FuncInfoRecSize = btf.FuncInfoSize attr.FuncInfoCnt = uint32(len(fib)) / btf.FuncInfoSize @@ -259,8 +328,14 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er attr.LineInfo = sys.NewSlicePointer(lib) } - if err := applyRelocations(insns, opts.KernelTypes, spec.ByteOrder); err != nil { - return nil, fmt.Errorf("apply CO-RE relocations: %w", err) + if !b.Empty() { + handle, err := btf.NewHandle(&b) + if err != nil { + return nil, fmt.Errorf("load BTF: %w", err) + } + defer handle.Close() + + attr.ProgBtfFd = uint32(handle.FD()) } kconfig, err := resolveKconfigReferences(insns) @@ -277,7 +352,7 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er if err != nil { return nil, fmt.Errorf("fixing up kfuncs: %w", err) } - defer handles.close() + defer handles.Close() if len(handles) > 0 { fdArray := handles.fdArray() @@ -318,39 +393,67 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er } } - if opts.LogSize == 0 { - opts.LogSize = DefaultVerifierLogSize - } - - // The caller requested a specific verifier log level. Set up the log buffer. + // The caller requested a specific verifier log level. Set up the log buffer + // so that there is a chance of loading the program in a single shot. var logBuf []byte if !opts.LogDisabled && opts.LogLevel != 0 { - logBuf = make([]byte, opts.LogSize) + logBuf = make([]byte, minVerifierLogSize) attr.LogLevel = opts.LogLevel attr.LogSize = uint32(len(logBuf)) attr.LogBuf = sys.NewSlicePointer(logBuf) } - fd, err := sys.ProgLoad(attr) - if err == nil { - return &Program{unix.ByteSliceToString(logBuf), fd, spec.Name, "", spec.Type}, nil - } + for { + var fd *sys.FD + fd, err = sys.ProgLoad(attr) + if err == nil { + return &Program{unix.ByteSliceToString(logBuf), fd, spec.Name, "", spec.Type}, nil + } - // An error occurred loading the program, but the caller did not explicitly - // enable the verifier log. Re-run with branch-level verifier logs enabled to - // obtain more info. Preserve the original error to return it to the caller. - // An undersized log buffer will result in ENOSPC regardless of the underlying - // cause. - var err2 error - if !opts.LogDisabled && opts.LogLevel == 0 { - logBuf = make([]byte, opts.LogSize) - attr.LogLevel = LogLevelBranch - attr.LogSize = uint32(len(logBuf)) + if opts.LogDisabled { + break + } + + if attr.LogTrueSize != 0 && attr.LogSize >= attr.LogTrueSize { + // The log buffer already has the correct size. + break + } + + if attr.LogSize != 0 && !errors.Is(err, unix.ENOSPC) { + // Logging is enabled and the error is not ENOSPC, so we can infer + // that the log buffer is large enough. + break + } + + if attr.LogLevel == 0 { + // Logging is not enabled but loading the program failed. Enable + // basic logging. + attr.LogLevel = LogLevelBranch + } + + // Make an educated guess how large the buffer should be. Start + // at minVerifierLogSize and then double the size. + logSize := uint32(max(len(logBuf)*2, minVerifierLogSize)) + if int(logSize) < len(logBuf) { + return nil, errors.New("overflow while probing log buffer size") + } + + if attr.LogTrueSize != 0 { + // The kernel has given us a hint how large the log buffer has to be. + logSize = attr.LogTrueSize + } + + logBuf = make([]byte, logSize) + attr.LogSize = logSize attr.LogBuf = sys.NewSlicePointer(logBuf) + } - _, err2 = sys.ProgLoad(attr) + end := bytes.IndexByte(logBuf, 0) + if end < 0 { + end = len(logBuf) } + tail := logBuf[max(end-256, 0):end] switch { case errors.Is(err, unix.EPERM): if len(logBuf) > 0 && logBuf[0] == 0 { @@ -359,22 +462,31 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er return nil, fmt.Errorf("load program: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err) } - fallthrough - case errors.Is(err, unix.EINVAL): - if hasFunctionReferences(spec.Instructions) { - if err := haveBPFToBPFCalls(); err != nil { - return nil, fmt.Errorf("load program: %w", err) - } + if bytes.Contains(tail, coreBadCall) { + err = errBadRelocation + break + } else if bytes.Contains(tail, kfuncBadCall) { + err = errUnknownKfunc + break } - if opts.LogSize > maxVerifierLogSize { - return nil, fmt.Errorf("load program: %w (ProgramOptions.LogSize exceeds maximum value of %d)", err, maxVerifierLogSize) + case errors.Is(err, unix.EACCES): + if bytes.Contains(tail, coreBadLoad) { + err = errBadRelocation + break } } - truncated := errors.Is(err, unix.ENOSPC) || errors.Is(err2, unix.ENOSPC) - return nil, internal.ErrorWithLog("load program", err, logBuf, truncated) + // hasFunctionReferences may be expensive, so check it last. + if (errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM)) && + hasFunctionReferences(spec.Instructions) { + if err := haveBPFToBPFCalls(); err != nil { + return nil, fmt.Errorf("load program: %w", err) + } + } + + return nil, internal.ErrorWithLog("load program", err, logBuf) } // NewProgramFromFD creates a program from a raw fd. @@ -553,7 +665,7 @@ type RunOptions struct { } // Test runs the Program in the kernel with the given input and returns the -// value returned by the eBPF program. outLen may be zero. +// value returned by the eBPF program. // // Note: the kernel expects at least 14 bytes input for an ethernet header for // XDP and SKB programs. @@ -702,10 +814,6 @@ func (p *Program) run(opts *RunOptions) (uint32, time.Duration, error) { Cpu: opts.CPU, } - if attr.Repeat == 0 { - attr.Repeat = 1 - } - retry: for { err := sys.ProgRun(&attr) @@ -714,7 +822,7 @@ retry: } if errors.Is(err, unix.EINTR) { - if attr.Repeat == 1 { + if attr.Repeat <= 1 { // Older kernels check whether enough repetitions have been // executed only after checking for pending signals. // @@ -763,14 +871,14 @@ retry: return attr.Retval, total, nil } -func unmarshalProgram(buf []byte) (*Program, error) { - if len(buf) != 4 { - return nil, errors.New("program id requires 4 byte value") +func unmarshalProgram(buf sysenc.Buffer) (*Program, error) { + var id uint32 + if err := buf.Unmarshal(&id); err != nil { + return nil, err } // Looking up an entry in a nested map or prog array returns an id, // not an fd. - id := internal.NativeEndian.Uint32(buf) return NewProgramFromID(ProgramID(id)) } @@ -921,7 +1029,12 @@ func findProgramTargetInKernel(name string, progType ProgramType, attachType Att } id, err := spec.TypeID(target) - return module, id, err + if err != nil { + module.Close() + return nil, 0, err + } + + return module, id, nil } // findTargetInKernel attempts to find a named type in the current kernel. @@ -999,7 +1112,9 @@ func findTargetInProgram(prog *Program, name string, progType ProgramType, attac var typeName string switch (match{progType, attachType}) { - case match{Extension, AttachNone}: + case match{Extension, AttachNone}, + match{Tracing, AttachTraceFEntry}, + match{Tracing, AttachTraceFExit}: typeName = name default: return 0, errUnrecognizedAttachType diff --git a/vendor/github.com/cilium/ebpf/run-tests.sh b/vendor/github.com/cilium/ebpf/run-tests.sh deleted file mode 100644 index 1d1490ad1d97..000000000000 --- a/vendor/github.com/cilium/ebpf/run-tests.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -# Test the current package under a different kernel. -# Requires virtme and qemu to be installed. -# Examples: -# Run all tests on a 5.4 kernel -# $ ./run-tests.sh 5.4 -# Run a subset of tests: -# $ ./run-tests.sh 5.4 ./link -# Run using a local kernel image -# $ ./run-tests.sh /path/to/bzImage - -set -euo pipefail - -script="$(realpath "$0")" -readonly script - -# This script is a bit like a Matryoshka doll since it keeps re-executing itself -# in various different contexts: -# -# 1. invoked by the user like run-tests.sh 5.4 -# 2. invoked by go test like run-tests.sh --exec-vm -# 3. invoked by init in the vm like run-tests.sh --exec-test -# -# This allows us to use all available CPU on the host machine to compile our -# code, and then only use the VM to execute the test. This is because the VM -# is usually slower at compiling than the host. -if [[ "${1:-}" = "--exec-vm" ]]; then - shift - - input="$1" - shift - - # Use sudo if /dev/kvm isn't accessible by the current user. - sudo="" - if [[ ! -r /dev/kvm || ! -w /dev/kvm ]]; then - sudo="sudo" - fi - readonly sudo - - testdir="$(dirname "$1")" - output="$(mktemp -d)" - printf -v cmd "%q " "$@" - - if [[ "$(stat -c '%t:%T' -L /proc/$$/fd/0)" == "1:3" ]]; then - # stdin is /dev/null, which doesn't play well with qemu. Use a fifo as a - # blocking substitute. - mkfifo "${output}/fake-stdin" - # Open for reading and writing to avoid blocking. - exec 0<> "${output}/fake-stdin" - rm "${output}/fake-stdin" - fi - - for ((i = 0; i < 3; i++)); do - if ! $sudo virtme-run --kimg "${input}/bzImage" --memory 768M --pwd \ - --rwdir="${testdir}=${testdir}" \ - --rodir=/run/input="${input}" \ - --rwdir=/run/output="${output}" \ - --script-sh "PATH=\"$PATH\" CI_MAX_KERNEL_VERSION="${CI_MAX_KERNEL_VERSION:-}" \"$script\" --exec-test $cmd" \ - --kopt possible_cpus=2; then # need at least two CPUs for some tests - exit 23 - fi - - if [[ -e "${output}/status" ]]; then - break - fi - - if [[ -v CI ]]; then - echo "Retrying test run due to qemu crash" - continue - fi - - exit 42 - done - - rc=$(<"${output}/status") - $sudo rm -r "$output" - exit $rc -elif [[ "${1:-}" = "--exec-test" ]]; then - shift - - mount -t bpf bpf /sys/fs/bpf - mount -t tracefs tracefs /sys/kernel/debug/tracing - - if [[ -d "/run/input/bpf" ]]; then - export KERNEL_SELFTESTS="/run/input/bpf" - fi - - if [[ -f "/run/input/bpf/bpf_testmod/bpf_testmod.ko" ]]; then - insmod "/run/input/bpf/bpf_testmod/bpf_testmod.ko" - fi - - dmesg --clear - rc=0 - "$@" || rc=$? - dmesg - echo $rc > "/run/output/status" - exit $rc # this return code is "swallowed" by qemu -fi - -if [[ -z "${1:-}" ]]; then - echo "Expecting kernel version or path as first argument" - exit 1 -fi - -readonly input="$(mktemp -d)" -readonly tmp_dir="${TMPDIR:-/tmp}" - -fetch() { - echo Fetching "${1}" - pushd "${tmp_dir}" > /dev/null - curl --no-progress-meter -L -O --fail --etag-compare "${1}.etag" --etag-save "${1}.etag" "https://github.com/cilium/ci-kernels/raw/${BRANCH:-master}/${1}" - local ret=$? - popd > /dev/null - return $ret -} - -if [[ -f "${1}" ]]; then - readonly kernel="${1}" - cp "${1}" "${input}/bzImage" -else -# LINUX_VERSION_CODE test compares this to discovered value. - export KERNEL_VERSION="${1}" - - readonly kernel="linux-${1}.bz" - readonly selftests="linux-${1}-selftests-bpf.tgz" - - fetch "${kernel}" - cp "${tmp_dir}/${kernel}" "${input}/bzImage" - - if fetch "${selftests}"; then - echo "Decompressing selftests" - mkdir "${input}/bpf" - tar --strip-components=4 -xf "${tmp_dir}/${selftests}" -C "${input}/bpf" - else - echo "No selftests found, disabling" - fi -fi -shift - -args=(-short -coverpkg=./... -coverprofile=coverage.out -count 1 ./...) -if (( $# > 0 )); then - args=("$@") -fi - -export GOFLAGS=-mod=readonly -export CGO_ENABLED=0 - -echo Testing on "${kernel}" -go test -exec "$script --exec-vm $input" "${args[@]}" -echo "Test successful on ${kernel}" - -rm -r "${input}" diff --git a/vendor/github.com/cilium/ebpf/syscalls.go b/vendor/github.com/cilium/ebpf/syscalls.go index fd21dea24ffc..4aef7faebc8a 100644 --- a/vendor/github.com/cilium/ebpf/syscalls.go +++ b/vendor/github.com/cilium/ebpf/syscalls.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "math" "os" "runtime" @@ -119,6 +120,7 @@ var haveInnerMaps = internal.NewFeatureTest("inner maps", "5.10", func() error { MaxEntries: 1, MapFlags: unix.BPF_F_INNER_MAP, }) + if err != nil { return internal.ErrNotSupported } @@ -135,6 +137,7 @@ var haveNoPreallocMaps = internal.NewFeatureTest("prealloc maps", "4.6", func() MaxEntries: 1, MapFlags: unix.BPF_F_NO_PREALLOC, }) + if err != nil { return internal.ErrNotSupported } @@ -223,8 +226,8 @@ var haveBatchAPI = internal.NewFeatureTest("map batch api", "5.6", func() error keys := []uint32{1, 2} values := []uint32{3, 4} - kp, _ := marshalPtr(keys, 8) - vp, _ := marshalPtr(values, 8) + kp, _ := marshalMapSyscallInput(keys, 8) + vp, _ := marshalMapSyscallInput(values, 8) err = sys.MapUpdateBatch(&sys.MapUpdateBatchAttr{ MapFd: fd.Uint(), @@ -265,11 +268,8 @@ var haveBPFToBPFCalls = internal.NewFeatureTest("bpf2bpf calls", "4.16", func() } fd, err := progLoad(insns, SocketFilter, "MIT") - if errors.Is(err, unix.EINVAL) { - return internal.ErrNotSupported - } if err != nil { - return err + return internal.ErrNotSupported } _ = fd.Close() return nil @@ -303,3 +303,35 @@ var haveSyscallWrapper = internal.NewFeatureTest("syscall wrapper", "4.17", func return evt.Close() }) + +var haveProgramExtInfos = internal.NewFeatureTest("program ext_infos", "5.0", func() error { + insns := asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + } + + buf := bytes.NewBuffer(make([]byte, 0, insns.Size())) + if err := insns.Marshal(buf, internal.NativeEndian); err != nil { + return err + } + bytecode := buf.Bytes() + + _, err := sys.ProgLoad(&sys.ProgLoadAttr{ + ProgType: sys.ProgType(SocketFilter), + License: sys.NewStringPointer("MIT"), + Insns: sys.NewSlicePointer(bytecode), + InsnCnt: uint32(len(bytecode) / asm.InstructionSize), + FuncInfoCnt: 1, + ProgBtfFd: math.MaxUint32, + }) + + if errors.Is(err, unix.EBADF) { + return nil + } + + if errors.Is(err, unix.E2BIG) { + return ErrNotSupported + } + + return err +}) diff --git a/vendor/github.com/cilium/ebpf/types.go b/vendor/github.com/cilium/ebpf/types.go index 35927e2ab80b..542c2397cab4 100644 --- a/vendor/github.com/cilium/ebpf/types.go +++ b/vendor/github.com/cilium/ebpf/types.go @@ -5,7 +5,7 @@ import ( "github.com/cilium/ebpf/internal/unix" ) -//go:generate stringer -output types_string.go -type=MapType,ProgramType,PinType +//go:generate go run golang.org/x/tools/cmd/stringer@latest -output types_string.go -type=MapType,ProgramType,PinType // MapType indicates the type map structure // that will be initialized in the kernel. @@ -44,7 +44,7 @@ const ( // if an skb is from a socket belonging to a specific cgroup CGroupArray // LRUHash - This allows you to create a small hash structure that will purge the - // least recently used items rather than thow an error when you run out of memory + // least recently used items rather than throw an error when you run out of memory LRUHash // LRUCPUHash - This is NOT like PerCPUHash, this structure is shared among the CPUs, // it has more to do with including the CPU id with the LRU calculation so that if a @@ -102,6 +102,12 @@ func (mt MapType) hasPerCPUValue() bool { return mt == PerCPUHash || mt == PerCPUArray || mt == LRUCPUHash || mt == PerCPUCGroupStorage } +// canStoreMapOrProgram returns true if the Map stores references to another Map +// or Program. +func (mt MapType) canStoreMapOrProgram() bool { + return mt.canStoreMap() || mt.canStoreProgram() +} + // canStoreMap returns true if the map type accepts a map fd // for update and returns a map id for lookup. func (mt MapType) canStoreMap() bool { @@ -119,38 +125,39 @@ type ProgramType uint32 // eBPF program types const ( - UnspecifiedProgram ProgramType = iota - SocketFilter - Kprobe - SchedCLS - SchedACT - TracePoint - XDP - PerfEvent - CGroupSKB - CGroupSock - LWTIn - LWTOut - LWTXmit - SockOps - SkSKB - CGroupDevice - SkMsg - RawTracepoint - CGroupSockAddr - LWTSeg6Local - LircMode2 - SkReuseport - FlowDissector - CGroupSysctl - RawTracepointWritable - CGroupSockopt - Tracing - StructOps - Extension - LSM - SkLookup - Syscall + UnspecifiedProgram = ProgramType(sys.BPF_PROG_TYPE_UNSPEC) + SocketFilter = ProgramType(sys.BPF_PROG_TYPE_SOCKET_FILTER) + Kprobe = ProgramType(sys.BPF_PROG_TYPE_KPROBE) + SchedCLS = ProgramType(sys.BPF_PROG_TYPE_SCHED_CLS) + SchedACT = ProgramType(sys.BPF_PROG_TYPE_SCHED_ACT) + TracePoint = ProgramType(sys.BPF_PROG_TYPE_TRACEPOINT) + XDP = ProgramType(sys.BPF_PROG_TYPE_XDP) + PerfEvent = ProgramType(sys.BPF_PROG_TYPE_PERF_EVENT) + CGroupSKB = ProgramType(sys.BPF_PROG_TYPE_CGROUP_SKB) + CGroupSock = ProgramType(sys.BPF_PROG_TYPE_CGROUP_SOCK) + LWTIn = ProgramType(sys.BPF_PROG_TYPE_LWT_IN) + LWTOut = ProgramType(sys.BPF_PROG_TYPE_LWT_OUT) + LWTXmit = ProgramType(sys.BPF_PROG_TYPE_LWT_XMIT) + SockOps = ProgramType(sys.BPF_PROG_TYPE_SOCK_OPS) + SkSKB = ProgramType(sys.BPF_PROG_TYPE_SK_SKB) + CGroupDevice = ProgramType(sys.BPF_PROG_TYPE_CGROUP_DEVICE) + SkMsg = ProgramType(sys.BPF_PROG_TYPE_SK_MSG) + RawTracepoint = ProgramType(sys.BPF_PROG_TYPE_RAW_TRACEPOINT) + CGroupSockAddr = ProgramType(sys.BPF_PROG_TYPE_CGROUP_SOCK_ADDR) + LWTSeg6Local = ProgramType(sys.BPF_PROG_TYPE_LWT_SEG6LOCAL) + LircMode2 = ProgramType(sys.BPF_PROG_TYPE_LIRC_MODE2) + SkReuseport = ProgramType(sys.BPF_PROG_TYPE_SK_REUSEPORT) + FlowDissector = ProgramType(sys.BPF_PROG_TYPE_FLOW_DISSECTOR) + CGroupSysctl = ProgramType(sys.BPF_PROG_TYPE_CGROUP_SYSCTL) + RawTracepointWritable = ProgramType(sys.BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) + CGroupSockopt = ProgramType(sys.BPF_PROG_TYPE_CGROUP_SOCKOPT) + Tracing = ProgramType(sys.BPF_PROG_TYPE_TRACING) + StructOps = ProgramType(sys.BPF_PROG_TYPE_STRUCT_OPS) + Extension = ProgramType(sys.BPF_PROG_TYPE_EXT) + LSM = ProgramType(sys.BPF_PROG_TYPE_LSM) + SkLookup = ProgramType(sys.BPF_PROG_TYPE_SK_LOOKUP) + Syscall = ProgramType(sys.BPF_PROG_TYPE_SYSCALL) + Netfilter = ProgramType(sys.BPF_PROG_TYPE_NETFILTER) ) // AttachType of the eBPF program, needed to differentiate allowed context accesses in @@ -158,62 +165,75 @@ const ( // Will cause invalid argument (EINVAL) at program load time if set incorrectly. type AttachType uint32 -//go:generate stringer -type AttachType -trimprefix Attach +//go:generate go run golang.org/x/tools/cmd/stringer@latest -type AttachType -trimprefix Attach // AttachNone is an alias for AttachCGroupInetIngress for readability reasons. const AttachNone AttachType = 0 const ( - AttachCGroupInetIngress AttachType = iota - AttachCGroupInetEgress - AttachCGroupInetSockCreate - AttachCGroupSockOps - AttachSkSKBStreamParser - AttachSkSKBStreamVerdict - AttachCGroupDevice - AttachSkMsgVerdict - AttachCGroupInet4Bind - AttachCGroupInet6Bind - AttachCGroupInet4Connect - AttachCGroupInet6Connect - AttachCGroupInet4PostBind - AttachCGroupInet6PostBind - AttachCGroupUDP4Sendmsg - AttachCGroupUDP6Sendmsg - AttachLircMode2 - AttachFlowDissector - AttachCGroupSysctl - AttachCGroupUDP4Recvmsg - AttachCGroupUDP6Recvmsg - AttachCGroupGetsockopt - AttachCGroupSetsockopt - AttachTraceRawTp - AttachTraceFEntry - AttachTraceFExit - AttachModifyReturn - AttachLSMMac - AttachTraceIter - AttachCgroupInet4GetPeername - AttachCgroupInet6GetPeername - AttachCgroupInet4GetSockname - AttachCgroupInet6GetSockname - AttachXDPDevMap - AttachCgroupInetSockRelease - AttachXDPCPUMap - AttachSkLookup - AttachXDP - AttachSkSKBVerdict - AttachSkReuseportSelect - AttachSkReuseportSelectOrMigrate - AttachPerfEvent - AttachTraceKprobeMulti + AttachCGroupInetIngress = AttachType(sys.BPF_CGROUP_INET_INGRESS) + AttachCGroupInetEgress = AttachType(sys.BPF_CGROUP_INET_EGRESS) + AttachCGroupInetSockCreate = AttachType(sys.BPF_CGROUP_INET_SOCK_CREATE) + AttachCGroupSockOps = AttachType(sys.BPF_CGROUP_SOCK_OPS) + AttachSkSKBStreamParser = AttachType(sys.BPF_SK_SKB_STREAM_PARSER) + AttachSkSKBStreamVerdict = AttachType(sys.BPF_SK_SKB_STREAM_VERDICT) + AttachCGroupDevice = AttachType(sys.BPF_CGROUP_DEVICE) + AttachSkMsgVerdict = AttachType(sys.BPF_SK_MSG_VERDICT) + AttachCGroupInet4Bind = AttachType(sys.BPF_CGROUP_INET4_BIND) + AttachCGroupInet6Bind = AttachType(sys.BPF_CGROUP_INET6_BIND) + AttachCGroupInet4Connect = AttachType(sys.BPF_CGROUP_INET4_CONNECT) + AttachCGroupInet6Connect = AttachType(sys.BPF_CGROUP_INET6_CONNECT) + AttachCGroupInet4PostBind = AttachType(sys.BPF_CGROUP_INET4_POST_BIND) + AttachCGroupInet6PostBind = AttachType(sys.BPF_CGROUP_INET6_POST_BIND) + AttachCGroupUDP4Sendmsg = AttachType(sys.BPF_CGROUP_UDP4_SENDMSG) + AttachCGroupUDP6Sendmsg = AttachType(sys.BPF_CGROUP_UDP6_SENDMSG) + AttachLircMode2 = AttachType(sys.BPF_LIRC_MODE2) + AttachFlowDissector = AttachType(sys.BPF_FLOW_DISSECTOR) + AttachCGroupSysctl = AttachType(sys.BPF_CGROUP_SYSCTL) + AttachCGroupUDP4Recvmsg = AttachType(sys.BPF_CGROUP_UDP4_RECVMSG) + AttachCGroupUDP6Recvmsg = AttachType(sys.BPF_CGROUP_UDP6_RECVMSG) + AttachCGroupGetsockopt = AttachType(sys.BPF_CGROUP_GETSOCKOPT) + AttachCGroupSetsockopt = AttachType(sys.BPF_CGROUP_SETSOCKOPT) + AttachTraceRawTp = AttachType(sys.BPF_TRACE_RAW_TP) + AttachTraceFEntry = AttachType(sys.BPF_TRACE_FENTRY) + AttachTraceFExit = AttachType(sys.BPF_TRACE_FEXIT) + AttachModifyReturn = AttachType(sys.BPF_MODIFY_RETURN) + AttachLSMMac = AttachType(sys.BPF_LSM_MAC) + AttachTraceIter = AttachType(sys.BPF_TRACE_ITER) + AttachCgroupInet4GetPeername = AttachType(sys.BPF_CGROUP_INET4_GETPEERNAME) + AttachCgroupInet6GetPeername = AttachType(sys.BPF_CGROUP_INET6_GETPEERNAME) + AttachCgroupInet4GetSockname = AttachType(sys.BPF_CGROUP_INET4_GETSOCKNAME) + AttachCgroupInet6GetSockname = AttachType(sys.BPF_CGROUP_INET6_GETSOCKNAME) + AttachXDPDevMap = AttachType(sys.BPF_XDP_DEVMAP) + AttachCgroupInetSockRelease = AttachType(sys.BPF_CGROUP_INET_SOCK_RELEASE) + AttachXDPCPUMap = AttachType(sys.BPF_XDP_CPUMAP) + AttachSkLookup = AttachType(sys.BPF_SK_LOOKUP) + AttachXDP = AttachType(sys.BPF_XDP) + AttachSkSKBVerdict = AttachType(sys.BPF_SK_SKB_VERDICT) + AttachSkReuseportSelect = AttachType(sys.BPF_SK_REUSEPORT_SELECT) + AttachSkReuseportSelectOrMigrate = AttachType(sys.BPF_SK_REUSEPORT_SELECT_OR_MIGRATE) + AttachPerfEvent = AttachType(sys.BPF_PERF_EVENT) + AttachTraceKprobeMulti = AttachType(sys.BPF_TRACE_KPROBE_MULTI) + AttachLSMCgroup = AttachType(sys.BPF_LSM_CGROUP) + AttachStructOps = AttachType(sys.BPF_STRUCT_OPS) + AttachNetfilter = AttachType(sys.BPF_NETFILTER) + AttachTCXIngress = AttachType(sys.BPF_TCX_INGRESS) + AttachTCXEgress = AttachType(sys.BPF_TCX_EGRESS) + AttachTraceUprobeMulti = AttachType(sys.BPF_TRACE_UPROBE_MULTI) + AttachCgroupUnixConnect = AttachType(sys.BPF_CGROUP_UNIX_CONNECT) + AttachCgroupUnixSendmsg = AttachType(sys.BPF_CGROUP_UNIX_SENDMSG) + AttachCgroupUnixRecvmsg = AttachType(sys.BPF_CGROUP_UNIX_RECVMSG) + AttachCgroupUnixGetpeername = AttachType(sys.BPF_CGROUP_UNIX_GETPEERNAME) + AttachCgroupUnixGetsockname = AttachType(sys.BPF_CGROUP_UNIX_GETSOCKNAME) + AttachNetkitPrimary = AttachType(sys.BPF_NETKIT_PRIMARY) + AttachNetkitPeer = AttachType(sys.BPF_NETKIT_PEER) ) // AttachFlags of the eBPF program used in BPF_PROG_ATTACH command type AttachFlags uint32 // PinType determines whether a map is pinned into a BPFFS. -type PinType int +type PinType uint32 // Valid pin types. // diff --git a/vendor/github.com/cilium/ebpf/types_string.go b/vendor/github.com/cilium/ebpf/types_string.go index 5679f2254301..ee60b5be5b64 100644 --- a/vendor/github.com/cilium/ebpf/types_string.go +++ b/vendor/github.com/cilium/ebpf/types_string.go @@ -86,11 +86,12 @@ func _() { _ = x[LSM-29] _ = x[SkLookup-30] _ = x[Syscall-31] + _ = x[Netfilter-32] } -const _ProgramType_name = "UnspecifiedProgramSocketFilterKprobeSchedCLSSchedACTTracePointXDPPerfEventCGroupSKBCGroupSockLWTInLWTOutLWTXmitSockOpsSkSKBCGroupDeviceSkMsgRawTracepointCGroupSockAddrLWTSeg6LocalLircMode2SkReuseportFlowDissectorCGroupSysctlRawTracepointWritableCGroupSockoptTracingStructOpsExtensionLSMSkLookupSyscall" +const _ProgramType_name = "UnspecifiedProgramSocketFilterKprobeSchedCLSSchedACTTracePointXDPPerfEventCGroupSKBCGroupSockLWTInLWTOutLWTXmitSockOpsSkSKBCGroupDeviceSkMsgRawTracepointCGroupSockAddrLWTSeg6LocalLircMode2SkReuseportFlowDissectorCGroupSysctlRawTracepointWritableCGroupSockoptTracingStructOpsExtensionLSMSkLookupSyscallNetfilter" -var _ProgramType_index = [...]uint16{0, 18, 30, 36, 44, 52, 62, 65, 74, 83, 93, 98, 104, 111, 118, 123, 135, 140, 153, 167, 179, 188, 199, 212, 224, 245, 258, 265, 274, 283, 286, 294, 301} +var _ProgramType_index = [...]uint16{0, 18, 30, 36, 44, 52, 62, 65, 74, 83, 93, 98, 104, 111, 118, 123, 135, 140, 153, 167, 179, 188, 199, 212, 224, 245, 258, 265, 274, 283, 286, 294, 301, 310} func (i ProgramType) String() string { if i >= ProgramType(len(_ProgramType_index)-1) { @@ -111,7 +112,7 @@ const _PinType_name = "PinNonePinByName" var _PinType_index = [...]uint8{0, 7, 16} func (i PinType) String() string { - if i < 0 || i >= PinType(len(_PinType_index)-1) { + if i >= PinType(len(_PinType_index)-1) { return "PinType(" + strconv.FormatInt(int64(i), 10) + ")" } return _PinType_name[_PinType_index[i]:_PinType_index[i+1]] diff --git a/vendor/github.com/containerd/cgroups/v3/Protobuild.toml b/vendor/github.com/containerd/cgroups/v3/Protobuild.toml index cf94b1c1ab2a..d27d12b33c9c 100644 --- a/vendor/github.com/containerd/cgroups/v3/Protobuild.toml +++ b/vendor/github.com/containerd/cgroups/v3/Protobuild.toml @@ -13,7 +13,7 @@ generators = ["go"] # This is the default. after = ["/usr/local/include", "/usr/include"] -# Aggregrate the API descriptors to lock down API changes. +# Aggregate the API descriptors to lock down API changes. [[descriptors]] prefix = "github.com/containerd/cgroups/cgroup1/stats" target = "cgroup1/stats/metrics.pb.txt" diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/cgroup.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/cgroup.go index eae04f05bcbc..f7db0b536838 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/cgroup.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/cgroup.go @@ -196,7 +196,7 @@ func (c *cgroup) AddTask(process Process, subsystems ...Name) error { return c.add(process, cgroupTasks, subsystems...) } -// writeCgroupsProcs writes to the file, but retries on EINVAL. +// writeCgroupProcs writes to the file, but retries on EINVAL. func writeCgroupProcs(path string, content []byte, perm fs.FileMode) error { f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, perm) if err != nil { diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go index 52fe690755d7..dbf49b5dc801 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go @@ -433,7 +433,7 @@ func getMemorySettings(resources *specs.LinuxResources) []memorySettings { }, { name: "kmem.limit_in_bytes", - value: mem.Kernel, + value: mem.Kernel, //nolint:staticcheck // SA1019: mem.Kernel is deprecated }, { name: "kmem.tcp.limit_in_bytes", diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/opts.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/opts.go index 3aa7f4fbbbe8..03389452135c 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/opts.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/opts.go @@ -62,11 +62,19 @@ func RequireDevices(s Subsystem, _ Path, _ error) error { return ErrIgnoreSubsystem } -// WithHiearchy sets a list of cgroup subsystems. +// WithHierarchy sets a list of cgroup subsystems. // The default list is coming from /proc/self/mountinfo. -func WithHiearchy(h Hierarchy) InitOpts { +func WithHierarchy(h Hierarchy) InitOpts { return func(c *InitConfig) error { c.hierarchy = h return nil } } + +// WithHiearchy sets a list of cgroup subsystems. It is just kept for backward +// compatibility and will be removed in v4. +// +// Deprecated: use WithHierarchy instead. +func WithHiearchy(h Hierarchy) InitOpts { + return WithHierarchy(h) +} diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/pids.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/pids.go index 31e2dda164d3..89818332a95f 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/pids.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/pids.go @@ -66,13 +66,13 @@ func (p *pidsController) Stat(path string, stats *v1.Metrics) error { if err != nil { return err } - max, err := readUint(filepath.Join(p.Path(path), "pids.max")) + pidsMax, err := readUint(filepath.Join(p.Path(path), "pids.max")) if err != nil { return err } stats.Pids = &v1.PidsStat{ Current: current, - Limit: max, + Limit: pidsMax, } return nil } diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/subsystem.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/subsystem.go index d32ea2cae4eb..59ff02909530 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/subsystem.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/subsystem.go @@ -20,8 +20,8 @@ import ( "fmt" "os" - "github.com/containerd/cgroups/v3" v1 "github.com/containerd/cgroups/v3/cgroup1/stats" + "github.com/moby/sys/userns" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -60,7 +60,7 @@ func Subsystems() []Name { Blkio, Rdma, } - if !cgroups.RunningInUserNS() { + if !userns.RunningInUserNS() { n = append(n, Devices) } if _, err := os.Stat("/sys/kernel/mm/hugepages"); err == nil { diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/utils.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/utils.go index 2b7d55200165..264c3d501365 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/utils.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/utils.go @@ -28,6 +28,7 @@ import ( "github.com/containerd/cgroups/v3" units "github.com/docker/go-units" + "github.com/moby/sys/userns" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -53,7 +54,7 @@ func defaults(root string) ([]Subsystem, error) { } // only add the devices cgroup if we are not in a user namespace // because modifications are not allowed - if !cgroups.RunningInUserNS() { + if !userns.RunningInUserNS() { s = append(s, NewDevices(root)) } // add the hugetlb cgroup if error wasn't due to missing hugetlb @@ -196,7 +197,7 @@ func parseKV(raw string) (string, uint64, error) { // The resulting map does not have an element for cgroup v2 unified hierarchy. // Use [cgroups.ParseCgroupFileUnified] to get the unified path. func ParseCgroupFile(path string) (map[string]string, error) { - x, _, err := ParseCgroupFileUnified(path) + x, _, err := cgroups.ParseCgroupFileUnified(path) return x, err } @@ -236,9 +237,9 @@ func getCgroupDestination(subsystem string) (string, error) { return "", ErrNoCgroupMountDestination } -func pathers(subystems []Subsystem) []pather { +func pathers(subsystems []Subsystem) []pather { var out []pather - for _, s := range subystems { + for _, s := range subsystems { if p, ok := s.(pather); ok { out = append(out, p) } diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup2/devicefilter.go b/vendor/github.com/containerd/cgroups/v3/cgroup2/devicefilter.go index 0cd5f7f3ddff..94a2854c8da6 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup2/devicefilter.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup2/devicefilter.go @@ -14,7 +14,7 @@ limitations under the License. */ -// Devicefilter containes eBPF device filter program +// Devicefilter contains eBPF device filter program // // The implementation is based on https://github.com/containers/crun/blob/0.10.2/src/libcrun/ebpf.c // diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go b/vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go index e540322b7fdf..d579cda7edbc 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go @@ -31,10 +31,10 @@ import ( "github.com/containerd/cgroups/v3/cgroup2/stats" + "github.com/containerd/log" systemdDbus "github.com/coreos/go-systemd/v22/dbus" "github.com/godbus/dbus/v5" "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -242,8 +242,10 @@ func setResources(path string, resources *Resources) error { type CgroupType string const ( - Domain CgroupType = "domain" - Threaded CgroupType = "threaded" + Domain CgroupType = "domain" + DomainThreaded CgroupType = "domain threaded" + DomainInvalid CgroupType = "domain invalid" + Threaded CgroupType = "threaded" ) func (c *Manager) GetType() (CgroupType, error) { @@ -317,7 +319,7 @@ func (c *Manager) ToggleControllers(controllers []string, t ControllerToggle) er } filePath := filepath.Join(f, subtreeControl) if err := c.writeSubtreeControl(filePath, controllers, t); err != nil { - // When running as rootless, the user may face EPERM on parent groups, but it is neglible when the + // When running as rootless, the user may face EPERM on parent groups, but it is negligible when the // controller is already written. // So we only return the last error. lastErr = fmt.Errorf("failed to write subtree controllers %+v to %q: %w", controllers, filePath, err) @@ -401,7 +403,7 @@ func (c *Manager) Kill() error { if err == nil { return nil } - logrus.Warnf("falling back to slower kill implementation: %s", err) + log.L.Warnf("falling back to slower kill implementation: %s", err) // Fallback to slow method. return c.fallbackKill() } @@ -414,13 +416,15 @@ func (c *Manager) Kill() error { // // https://github.com/opencontainers/runc/blob/8da0a0b5675764feaaaaad466f6567a9983fcd08/libcontainer/init_linux.go#L523-L529 func (c *Manager) fallbackKill() error { + logger := log.G(context.TODO()).WithFields(log.Fields{"path": c.path}) + if err := c.Freeze(); err != nil { - logrus.Warn(err) + logger.WithError(err).Warn("freezing cgroup2.manager") } pids, err := c.Procs(true) if err != nil { if err := c.Thaw(); err != nil { - logrus.Warn(err) + logger.WithError(err).Warn("thawing cgroup2.manager") } return err } @@ -428,16 +432,16 @@ func (c *Manager) fallbackKill() error { for _, pid := range pids { p, err := os.FindProcess(int(pid)) if err != nil { - logrus.Warn(err) + logger.WithFields(log.Fields{"error": err, "pid": int(pid)}).Warnf("finding process") continue } procs = append(procs, p) if err := p.Signal(unix.SIGKILL); err != nil { - logrus.Warn(err) + logger.WithFields(log.Fields{"error": err, "pid": int(pid)}).Warnf("signaling process") } } if err := c.Thaw(); err != nil { - logrus.Warn(err) + logger.WithError(err).Warn("thawing cgroup2.manager") } subreaper, err := getSubreaper() @@ -459,7 +463,7 @@ func (c *Manager) fallbackKill() error { if subreaper == 0 { if _, err := p.Wait(); err != nil { if !errors.Is(err, unix.ECHILD) { - logrus.Warnf("wait on pid %d failed: %s", p.Pid, err) + logger.WithFields(log.Fields{"error": err, "pid": p.Pid}).Warn("waiting on process") } } } @@ -468,13 +472,33 @@ func (c *Manager) fallbackKill() error { } func (c *Manager) Delete() error { - // kernel prevents cgroups with running process from being removed, check the tree is empty - processes, err := c.Procs(true) + var ( + tasks []uint64 + threaded bool + ) + // Kernel prevents cgroups with running process from being removed, + // check the tree is empty. + // + // Pick the right file to read based on the cgs type. + cgType, err := c.GetType() + if err != nil { + if !os.IsNotExist(err) { + return err + } + } else { + threaded = cgType == Threaded + } + + if threaded { + tasks, err = c.Threads(true) + } else { + tasks, err = c.Procs(true) + } if err != nil { return err } - if len(processes) > 0 { - return fmt.Errorf("cgroups: unable to remove path %q: still contains running processes", c.path) + if len(tasks) > 0 { + return fmt.Errorf("cgroups: unable to remove path %q: still contains running tasks", c.path) } return remove(c.path) } @@ -928,24 +952,27 @@ func startUnit(conn *systemdDbus.Conn, group string, properties []systemdDbus.Pr } } + systemdStartUnitTimeout := 30 * time.Second select { case s := <-statusChan: if s != "done" { attemptFailedUnitReset(conn, group) return fmt.Errorf("error creating systemd unit `%s`: got `%s`", group, s) } - case <-time.After(30 * time.Second): - logrus.Warnf("Timed out while waiting for StartTransientUnit(%s) completion signal from dbus. Continuing...", group) + case <-time.After(systemdStartUnitTimeout): + attemptFailedUnitReset(conn, group) + return fmt.Errorf("timed out while waiting for StartTransientUnit(%s) completion signal from dbus after %v", group, systemdStartUnitTimeout) } return nil } func attemptFailedUnitReset(conn *systemdDbus.Conn, group string) { - err := conn.ResetFailedUnitContext(context.TODO(), group) + ctx := context.TODO() + err := conn.ResetFailedUnitContext(ctx, group) if err != nil { - logrus.Warnf("Unable to reset failed unit: %v", err) + log.G(ctx).Warnf("Unable to reset failed unit: %v", err) } } diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go b/vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go index 20be57ca47e6..0974ce6a435b 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go @@ -32,9 +32,9 @@ import ( "github.com/containerd/cgroups/v3/cgroup2/stats" + "github.com/containerd/log" "github.com/godbus/dbus/v5" "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -264,7 +264,7 @@ func getStatFileContentUint64(filePath string) uint64 { res, err := parseUint(trimmed, 10, 64) if err != nil { - logrus.Errorf("unable to parse %q as a uint from Cgroup file %q", trimmed, filePath) + log.L.Errorf("unable to parse %q as a uint from Cgroup file %q", trimmed, filePath) return res } @@ -432,7 +432,7 @@ func hugePageSizes() []string { hPageSizes, err = getHugePageSizeFromFilenames(files) if err != nil { - logrus.Warnf("hugePageSizes: %s", err) + log.L.Warnf("hugePageSizes: %s", err) } }) @@ -507,14 +507,16 @@ func getStatPSIFromFile(path string) *stats.PSIStats { if pv != nil { err = parsePSIData(parts[1:], pv) if err != nil { - logrus.Errorf("failed to read file %s: %v", path, err) + log.L.WithError(err).Errorf("failed to read file %s", path) return nil } } } if err := sc.Err(); err != nil { - logrus.Errorf("unable to parse PSI data: %v", err) + if !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EOPNOTSUPP) { + log.L.WithError(err).Error("unable to parse PSI data") + } return nil } return psistats diff --git a/vendor/github.com/containerd/cgroups/v3/utils.go b/vendor/github.com/containerd/cgroups/v3/utils.go index ebff755a7611..6b4d04fdff17 100644 --- a/vendor/github.com/containerd/cgroups/v3/utils.go +++ b/vendor/github.com/containerd/cgroups/v3/utils.go @@ -25,12 +25,11 @@ import ( "strings" "sync" + "github.com/moby/sys/userns" "golang.org/x/sys/unix" ) var ( - nsOnce sync.Once - inUserNS bool checkMode sync.Once cgMode CGMode ) @@ -77,35 +76,10 @@ func Mode() CGMode { // RunningInUserNS detects whether we are currently running in a user namespace. // Copied from github.com/lxc/lxd/shared/util.go +// +// Deprecated: use [userns.RunningInUserNS]. func RunningInUserNS() bool { - nsOnce.Do(func() { - file, err := os.Open("/proc/self/uid_map") - if err != nil { - // This kernel-provided file only exists if user namespaces are supported - return - } - defer file.Close() - - buf := bufio.NewReader(file) - l, _, err := buf.ReadLine() - if err != nil { - return - } - - line := string(l) - var a, b, c int64 - fmt.Sscanf(line, "%d %d %d", &a, &b, &c) - - /* - * We assume we are in the initial user namespace if we have a full - * range - 4294967295 uids starting at uid 0. - */ - if a == 0 && b == 0 && c == 4294967295 { - return - } - inUserNS = true - }) - return inUserNS + return userns.RunningInUserNS() } // ParseCgroupFileUnified returns legacy subsystem paths as the first value, diff --git a/vendor/golang.org/x/exp/LICENSE b/vendor/golang.org/x/exp/LICENSE index 6a66aea5eafe..2a7cf70da6e4 100644 --- a/vendor/golang.org/x/exp/LICENSE +++ b/vendor/golang.org/x/exp/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/vendor/golang.org/x/exp/maps/maps.go b/vendor/golang.org/x/exp/maps/maps.go deleted file mode 100644 index ecc0dabb74d9..000000000000 --- a/vendor/golang.org/x/exp/maps/maps.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package maps defines various functions useful with maps of any type. -package maps - -// Keys returns the keys of the map m. -// The keys will be in an indeterminate order. -func Keys[M ~map[K]V, K comparable, V any](m M) []K { - r := make([]K, 0, len(m)) - for k := range m { - r = append(r, k) - } - return r -} - -// Values returns the values of the map m. -// The values will be in an indeterminate order. -func Values[M ~map[K]V, K comparable, V any](m M) []V { - r := make([]V, 0, len(m)) - for _, v := range m { - r = append(r, v) - } - return r -} - -// Equal reports whether two maps contain the same key/value pairs. -// Values are compared using ==. -func Equal[M1, M2 ~map[K]V, K, V comparable](m1 M1, m2 M2) bool { - if len(m1) != len(m2) { - return false - } - for k, v1 := range m1 { - if v2, ok := m2[k]; !ok || v1 != v2 { - return false - } - } - return true -} - -// EqualFunc is like Equal, but compares values using eq. -// Keys are still compared with ==. -func EqualFunc[M1 ~map[K]V1, M2 ~map[K]V2, K comparable, V1, V2 any](m1 M1, m2 M2, eq func(V1, V2) bool) bool { - if len(m1) != len(m2) { - return false - } - for k, v1 := range m1 { - if v2, ok := m2[k]; !ok || !eq(v1, v2) { - return false - } - } - return true -} - -// Clear removes all entries from m, leaving it empty. -func Clear[M ~map[K]V, K comparable, V any](m M) { - for k := range m { - delete(m, k) - } -} - -// Clone returns a copy of m. This is a shallow clone: -// the new keys and values are set using ordinary assignment. -func Clone[M ~map[K]V, K comparable, V any](m M) M { - // Preserve nil in case it matters. - if m == nil { - return nil - } - r := make(M, len(m)) - for k, v := range m { - r[k] = v - } - return r -} - -// Copy copies all key/value pairs in src adding them to dst. -// When a key in src is already present in dst, -// the value in dst will be overwritten by the value associated -// with the key in src. -func Copy[M1 ~map[K]V, M2 ~map[K]V, K comparable, V any](dst M1, src M2) { - for k, v := range src { - dst[k] = v - } -} - -// DeleteFunc deletes any key/value pairs from m for which del returns true. -func DeleteFunc[M ~map[K]V, K comparable, V any](m M, del func(K, V) bool) { - for k, v := range m { - if del(k, v) { - delete(m, k) - } - } -} diff --git a/vendor/golang.org/x/exp/slices/cmp.go b/vendor/golang.org/x/exp/slices/cmp.go deleted file mode 100644 index fbf1934a0617..000000000000 --- a/vendor/golang.org/x/exp/slices/cmp.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package slices - -import "golang.org/x/exp/constraints" - -// min is a version of the predeclared function from the Go 1.21 release. -func min[T constraints.Ordered](a, b T) T { - if a < b || isNaN(a) { - return a - } - return b -} - -// max is a version of the predeclared function from the Go 1.21 release. -func max[T constraints.Ordered](a, b T) T { - if a > b || isNaN(a) { - return a - } - return b -} - -// cmpLess is a copy of cmp.Less from the Go 1.21 release. -func cmpLess[T constraints.Ordered](x, y T) bool { - return (isNaN(x) && !isNaN(y)) || x < y -} - -// cmpCompare is a copy of cmp.Compare from the Go 1.21 release. -func cmpCompare[T constraints.Ordered](x, y T) int { - xNaN := isNaN(x) - yNaN := isNaN(y) - if xNaN && yNaN { - return 0 - } - if xNaN || x < y { - return -1 - } - if yNaN || x > y { - return +1 - } - return 0 -} diff --git a/vendor/golang.org/x/exp/slices/slices.go b/vendor/golang.org/x/exp/slices/slices.go deleted file mode 100644 index 5e8158bba869..000000000000 --- a/vendor/golang.org/x/exp/slices/slices.go +++ /dev/null @@ -1,499 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package slices defines various functions useful with slices of any type. -package slices - -import ( - "unsafe" - - "golang.org/x/exp/constraints" -) - -// Equal reports whether two slices are equal: the same length and all -// elements equal. If the lengths are different, Equal returns false. -// Otherwise, the elements are compared in increasing index order, and the -// comparison stops at the first unequal pair. -// Floating point NaNs are not considered equal. -func Equal[S ~[]E, E comparable](s1, s2 S) bool { - if len(s1) != len(s2) { - return false - } - for i := range s1 { - if s1[i] != s2[i] { - return false - } - } - return true -} - -// EqualFunc reports whether two slices are equal using an equality -// function on each pair of elements. If the lengths are different, -// EqualFunc returns false. Otherwise, the elements are compared in -// increasing index order, and the comparison stops at the first index -// for which eq returns false. -func EqualFunc[S1 ~[]E1, S2 ~[]E2, E1, E2 any](s1 S1, s2 S2, eq func(E1, E2) bool) bool { - if len(s1) != len(s2) { - return false - } - for i, v1 := range s1 { - v2 := s2[i] - if !eq(v1, v2) { - return false - } - } - return true -} - -// Compare compares the elements of s1 and s2, using [cmp.Compare] on each pair -// of elements. The elements are compared sequentially, starting at index 0, -// until one element is not equal to the other. -// The result of comparing the first non-matching elements is returned. -// If both slices are equal until one of them ends, the shorter slice is -// considered less than the longer one. -// The result is 0 if s1 == s2, -1 if s1 < s2, and +1 if s1 > s2. -func Compare[S ~[]E, E constraints.Ordered](s1, s2 S) int { - for i, v1 := range s1 { - if i >= len(s2) { - return +1 - } - v2 := s2[i] - if c := cmpCompare(v1, v2); c != 0 { - return c - } - } - if len(s1) < len(s2) { - return -1 - } - return 0 -} - -// CompareFunc is like [Compare] but uses a custom comparison function on each -// pair of elements. -// The result is the first non-zero result of cmp; if cmp always -// returns 0 the result is 0 if len(s1) == len(s2), -1 if len(s1) < len(s2), -// and +1 if len(s1) > len(s2). -func CompareFunc[S1 ~[]E1, S2 ~[]E2, E1, E2 any](s1 S1, s2 S2, cmp func(E1, E2) int) int { - for i, v1 := range s1 { - if i >= len(s2) { - return +1 - } - v2 := s2[i] - if c := cmp(v1, v2); c != 0 { - return c - } - } - if len(s1) < len(s2) { - return -1 - } - return 0 -} - -// Index returns the index of the first occurrence of v in s, -// or -1 if not present. -func Index[S ~[]E, E comparable](s S, v E) int { - for i := range s { - if v == s[i] { - return i - } - } - return -1 -} - -// IndexFunc returns the first index i satisfying f(s[i]), -// or -1 if none do. -func IndexFunc[S ~[]E, E any](s S, f func(E) bool) int { - for i := range s { - if f(s[i]) { - return i - } - } - return -1 -} - -// Contains reports whether v is present in s. -func Contains[S ~[]E, E comparable](s S, v E) bool { - return Index(s, v) >= 0 -} - -// ContainsFunc reports whether at least one -// element e of s satisfies f(e). -func ContainsFunc[S ~[]E, E any](s S, f func(E) bool) bool { - return IndexFunc(s, f) >= 0 -} - -// Insert inserts the values v... into s at index i, -// returning the modified slice. -// The elements at s[i:] are shifted up to make room. -// In the returned slice r, r[i] == v[0], -// and r[i+len(v)] == value originally at r[i]. -// Insert panics if i is out of range. -// This function is O(len(s) + len(v)). -func Insert[S ~[]E, E any](s S, i int, v ...E) S { - m := len(v) - if m == 0 { - return s - } - n := len(s) - if i == n { - return append(s, v...) - } - if n+m > cap(s) { - // Use append rather than make so that we bump the size of - // the slice up to the next storage class. - // This is what Grow does but we don't call Grow because - // that might copy the values twice. - s2 := append(s[:i], make(S, n+m-i)...) - copy(s2[i:], v) - copy(s2[i+m:], s[i:]) - return s2 - } - s = s[:n+m] - - // before: - // s: aaaaaaaabbbbccccccccdddd - // ^ ^ ^ ^ - // i i+m n n+m - // after: - // s: aaaaaaaavvvvbbbbcccccccc - // ^ ^ ^ ^ - // i i+m n n+m - // - // a are the values that don't move in s. - // v are the values copied in from v. - // b and c are the values from s that are shifted up in index. - // d are the values that get overwritten, never to be seen again. - - if !overlaps(v, s[i+m:]) { - // Easy case - v does not overlap either the c or d regions. - // (It might be in some of a or b, or elsewhere entirely.) - // The data we copy up doesn't write to v at all, so just do it. - - copy(s[i+m:], s[i:]) - - // Now we have - // s: aaaaaaaabbbbbbbbcccccccc - // ^ ^ ^ ^ - // i i+m n n+m - // Note the b values are duplicated. - - copy(s[i:], v) - - // Now we have - // s: aaaaaaaavvvvbbbbcccccccc - // ^ ^ ^ ^ - // i i+m n n+m - // That's the result we want. - return s - } - - // The hard case - v overlaps c or d. We can't just shift up - // the data because we'd move or clobber the values we're trying - // to insert. - // So instead, write v on top of d, then rotate. - copy(s[n:], v) - - // Now we have - // s: aaaaaaaabbbbccccccccvvvv - // ^ ^ ^ ^ - // i i+m n n+m - - rotateRight(s[i:], m) - - // Now we have - // s: aaaaaaaavvvvbbbbcccccccc - // ^ ^ ^ ^ - // i i+m n n+m - // That's the result we want. - return s -} - -// Delete removes the elements s[i:j] from s, returning the modified slice. -// Delete panics if s[i:j] is not a valid slice of s. -// Delete is O(len(s)-j), so if many items must be deleted, it is better to -// make a single call deleting them all together than to delete one at a time. -// Delete might not modify the elements s[len(s)-(j-i):len(s)]. If those -// elements contain pointers you might consider zeroing those elements so that -// objects they reference can be garbage collected. -func Delete[S ~[]E, E any](s S, i, j int) S { - _ = s[i:j] // bounds check - - return append(s[:i], s[j:]...) -} - -// DeleteFunc removes any elements from s for which del returns true, -// returning the modified slice. -// When DeleteFunc removes m elements, it might not modify the elements -// s[len(s)-m:len(s)]. If those elements contain pointers you might consider -// zeroing those elements so that objects they reference can be garbage -// collected. -func DeleteFunc[S ~[]E, E any](s S, del func(E) bool) S { - i := IndexFunc(s, del) - if i == -1 { - return s - } - // Don't start copying elements until we find one to delete. - for j := i + 1; j < len(s); j++ { - if v := s[j]; !del(v) { - s[i] = v - i++ - } - } - return s[:i] -} - -// Replace replaces the elements s[i:j] by the given v, and returns the -// modified slice. Replace panics if s[i:j] is not a valid slice of s. -func Replace[S ~[]E, E any](s S, i, j int, v ...E) S { - _ = s[i:j] // verify that i:j is a valid subslice - - if i == j { - return Insert(s, i, v...) - } - if j == len(s) { - return append(s[:i], v...) - } - - tot := len(s[:i]) + len(v) + len(s[j:]) - if tot > cap(s) { - // Too big to fit, allocate and copy over. - s2 := append(s[:i], make(S, tot-i)...) // See Insert - copy(s2[i:], v) - copy(s2[i+len(v):], s[j:]) - return s2 - } - - r := s[:tot] - - if i+len(v) <= j { - // Easy, as v fits in the deleted portion. - copy(r[i:], v) - if i+len(v) != j { - copy(r[i+len(v):], s[j:]) - } - return r - } - - // We are expanding (v is bigger than j-i). - // The situation is something like this: - // (example has i=4,j=8,len(s)=16,len(v)=6) - // s: aaaaxxxxbbbbbbbbyy - // ^ ^ ^ ^ - // i j len(s) tot - // a: prefix of s - // x: deleted range - // b: more of s - // y: area to expand into - - if !overlaps(r[i+len(v):], v) { - // Easy, as v is not clobbered by the first copy. - copy(r[i+len(v):], s[j:]) - copy(r[i:], v) - return r - } - - // This is a situation where we don't have a single place to which - // we can copy v. Parts of it need to go to two different places. - // We want to copy the prefix of v into y and the suffix into x, then - // rotate |y| spots to the right. - // - // v[2:] v[:2] - // | | - // s: aaaavvvvbbbbbbbbvv - // ^ ^ ^ ^ - // i j len(s) tot - // - // If either of those two destinations don't alias v, then we're good. - y := len(v) - (j - i) // length of y portion - - if !overlaps(r[i:j], v) { - copy(r[i:j], v[y:]) - copy(r[len(s):], v[:y]) - rotateRight(r[i:], y) - return r - } - if !overlaps(r[len(s):], v) { - copy(r[len(s):], v[:y]) - copy(r[i:j], v[y:]) - rotateRight(r[i:], y) - return r - } - - // Now we know that v overlaps both x and y. - // That means that the entirety of b is *inside* v. - // So we don't need to preserve b at all; instead we - // can copy v first, then copy the b part of v out of - // v to the right destination. - k := startIdx(v, s[j:]) - copy(r[i:], v) - copy(r[i+len(v):], r[i+k:]) - return r -} - -// Clone returns a copy of the slice. -// The elements are copied using assignment, so this is a shallow clone. -func Clone[S ~[]E, E any](s S) S { - // Preserve nil in case it matters. - if s == nil { - return nil - } - return append(S([]E{}), s...) -} - -// Compact replaces consecutive runs of equal elements with a single copy. -// This is like the uniq command found on Unix. -// Compact modifies the contents of the slice s and returns the modified slice, -// which may have a smaller length. -// When Compact discards m elements in total, it might not modify the elements -// s[len(s)-m:len(s)]. If those elements contain pointers you might consider -// zeroing those elements so that objects they reference can be garbage collected. -func Compact[S ~[]E, E comparable](s S) S { - if len(s) < 2 { - return s - } - i := 1 - for k := 1; k < len(s); k++ { - if s[k] != s[k-1] { - if i != k { - s[i] = s[k] - } - i++ - } - } - return s[:i] -} - -// CompactFunc is like [Compact] but uses an equality function to compare elements. -// For runs of elements that compare equal, CompactFunc keeps the first one. -func CompactFunc[S ~[]E, E any](s S, eq func(E, E) bool) S { - if len(s) < 2 { - return s - } - i := 1 - for k := 1; k < len(s); k++ { - if !eq(s[k], s[k-1]) { - if i != k { - s[i] = s[k] - } - i++ - } - } - return s[:i] -} - -// Grow increases the slice's capacity, if necessary, to guarantee space for -// another n elements. After Grow(n), at least n elements can be appended -// to the slice without another allocation. If n is negative or too large to -// allocate the memory, Grow panics. -func Grow[S ~[]E, E any](s S, n int) S { - if n < 0 { - panic("cannot be negative") - } - if n -= cap(s) - len(s); n > 0 { - // TODO(https://go.dev/issue/53888): Make using []E instead of S - // to workaround a compiler bug where the runtime.growslice optimization - // does not take effect. Revert when the compiler is fixed. - s = append([]E(s)[:cap(s)], make([]E, n)...)[:len(s)] - } - return s -} - -// Clip removes unused capacity from the slice, returning s[:len(s):len(s)]. -func Clip[S ~[]E, E any](s S) S { - return s[:len(s):len(s)] -} - -// Rotation algorithm explanation: -// -// rotate left by 2 -// start with -// 0123456789 -// split up like this -// 01 234567 89 -// swap first 2 and last 2 -// 89 234567 01 -// join first parts -// 89234567 01 -// recursively rotate first left part by 2 -// 23456789 01 -// join at the end -// 2345678901 -// -// rotate left by 8 -// start with -// 0123456789 -// split up like this -// 01 234567 89 -// swap first 2 and last 2 -// 89 234567 01 -// join last parts -// 89 23456701 -// recursively rotate second part left by 6 -// 89 01234567 -// join at the end -// 8901234567 - -// TODO: There are other rotate algorithms. -// This algorithm has the desirable property that it moves each element exactly twice. -// The triple-reverse algorithm is simpler and more cache friendly, but takes more writes. -// The follow-cycles algorithm can be 1-write but it is not very cache friendly. - -// rotateLeft rotates b left by n spaces. -// s_final[i] = s_orig[i+r], wrapping around. -func rotateLeft[E any](s []E, r int) { - for r != 0 && r != len(s) { - if r*2 <= len(s) { - swap(s[:r], s[len(s)-r:]) - s = s[:len(s)-r] - } else { - swap(s[:len(s)-r], s[r:]) - s, r = s[len(s)-r:], r*2-len(s) - } - } -} -func rotateRight[E any](s []E, r int) { - rotateLeft(s, len(s)-r) -} - -// swap swaps the contents of x and y. x and y must be equal length and disjoint. -func swap[E any](x, y []E) { - for i := 0; i < len(x); i++ { - x[i], y[i] = y[i], x[i] - } -} - -// overlaps reports whether the memory ranges a[0:len(a)] and b[0:len(b)] overlap. -func overlaps[E any](a, b []E) bool { - if len(a) == 0 || len(b) == 0 { - return false - } - elemSize := unsafe.Sizeof(a[0]) - if elemSize == 0 { - return false - } - // TODO: use a runtime/unsafe facility once one becomes available. See issue 12445. - // Also see crypto/internal/alias/alias.go:AnyOverlap - return uintptr(unsafe.Pointer(&a[0])) <= uintptr(unsafe.Pointer(&b[len(b)-1]))+(elemSize-1) && - uintptr(unsafe.Pointer(&b[0])) <= uintptr(unsafe.Pointer(&a[len(a)-1]))+(elemSize-1) -} - -// startIdx returns the index in haystack where the needle starts. -// prerequisite: the needle must be aliased entirely inside the haystack. -func startIdx[E any](haystack, needle []E) int { - p := &needle[0] - for i := range haystack { - if p == &haystack[i] { - return i - } - } - // TODO: what if the overlap is by a non-integral number of Es? - panic("needle not found") -} - -// Reverse reverses the elements of the slice in place. -func Reverse[S ~[]E, E any](s S) { - for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { - s[i], s[j] = s[j], s[i] - } -} diff --git a/vendor/golang.org/x/exp/slices/sort.go b/vendor/golang.org/x/exp/slices/sort.go deleted file mode 100644 index b67897f76b5d..000000000000 --- a/vendor/golang.org/x/exp/slices/sort.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:generate go run $GOROOT/src/sort/gen_sort_variants.go -exp - -package slices - -import ( - "math/bits" - - "golang.org/x/exp/constraints" -) - -// Sort sorts a slice of any ordered type in ascending order. -// When sorting floating-point numbers, NaNs are ordered before other values. -func Sort[S ~[]E, E constraints.Ordered](x S) { - n := len(x) - pdqsortOrdered(x, 0, n, bits.Len(uint(n))) -} - -// SortFunc sorts the slice x in ascending order as determined by the cmp -// function. This sort is not guaranteed to be stable. -// cmp(a, b) should return a negative number when a < b, a positive number when -// a > b and zero when a == b. -// -// SortFunc requires that cmp is a strict weak ordering. -// See https://en.wikipedia.org/wiki/Weak_ordering#Strict_weak_orderings. -func SortFunc[S ~[]E, E any](x S, cmp func(a, b E) int) { - n := len(x) - pdqsortCmpFunc(x, 0, n, bits.Len(uint(n)), cmp) -} - -// SortStableFunc sorts the slice x while keeping the original order of equal -// elements, using cmp to compare elements in the same way as [SortFunc]. -func SortStableFunc[S ~[]E, E any](x S, cmp func(a, b E) int) { - stableCmpFunc(x, len(x), cmp) -} - -// IsSorted reports whether x is sorted in ascending order. -func IsSorted[S ~[]E, E constraints.Ordered](x S) bool { - for i := len(x) - 1; i > 0; i-- { - if cmpLess(x[i], x[i-1]) { - return false - } - } - return true -} - -// IsSortedFunc reports whether x is sorted in ascending order, with cmp as the -// comparison function as defined by [SortFunc]. -func IsSortedFunc[S ~[]E, E any](x S, cmp func(a, b E) int) bool { - for i := len(x) - 1; i > 0; i-- { - if cmp(x[i], x[i-1]) < 0 { - return false - } - } - return true -} - -// Min returns the minimal value in x. It panics if x is empty. -// For floating-point numbers, Min propagates NaNs (any NaN value in x -// forces the output to be NaN). -func Min[S ~[]E, E constraints.Ordered](x S) E { - if len(x) < 1 { - panic("slices.Min: empty list") - } - m := x[0] - for i := 1; i < len(x); i++ { - m = min(m, x[i]) - } - return m -} - -// MinFunc returns the minimal value in x, using cmp to compare elements. -// It panics if x is empty. If there is more than one minimal element -// according to the cmp function, MinFunc returns the first one. -func MinFunc[S ~[]E, E any](x S, cmp func(a, b E) int) E { - if len(x) < 1 { - panic("slices.MinFunc: empty list") - } - m := x[0] - for i := 1; i < len(x); i++ { - if cmp(x[i], m) < 0 { - m = x[i] - } - } - return m -} - -// Max returns the maximal value in x. It panics if x is empty. -// For floating-point E, Max propagates NaNs (any NaN value in x -// forces the output to be NaN). -func Max[S ~[]E, E constraints.Ordered](x S) E { - if len(x) < 1 { - panic("slices.Max: empty list") - } - m := x[0] - for i := 1; i < len(x); i++ { - m = max(m, x[i]) - } - return m -} - -// MaxFunc returns the maximal value in x, using cmp to compare elements. -// It panics if x is empty. If there is more than one maximal element -// according to the cmp function, MaxFunc returns the first one. -func MaxFunc[S ~[]E, E any](x S, cmp func(a, b E) int) E { - if len(x) < 1 { - panic("slices.MaxFunc: empty list") - } - m := x[0] - for i := 1; i < len(x); i++ { - if cmp(x[i], m) > 0 { - m = x[i] - } - } - return m -} - -// BinarySearch searches for target in a sorted slice and returns the position -// where target is found, or the position where target would appear in the -// sort order; it also returns a bool saying whether the target is really found -// in the slice. The slice must be sorted in increasing order. -func BinarySearch[S ~[]E, E constraints.Ordered](x S, target E) (int, bool) { - // Inlining is faster than calling BinarySearchFunc with a lambda. - n := len(x) - // Define x[-1] < target and x[n] >= target. - // Invariant: x[i-1] < target, x[j] >= target. - i, j := 0, n - for i < j { - h := int(uint(i+j) >> 1) // avoid overflow when computing h - // i ≤ h < j - if cmpLess(x[h], target) { - i = h + 1 // preserves x[i-1] < target - } else { - j = h // preserves x[j] >= target - } - } - // i == j, x[i-1] < target, and x[j] (= x[i]) >= target => answer is i. - return i, i < n && (x[i] == target || (isNaN(x[i]) && isNaN(target))) -} - -// BinarySearchFunc works like [BinarySearch], but uses a custom comparison -// function. The slice must be sorted in increasing order, where "increasing" -// is defined by cmp. cmp should return 0 if the slice element matches -// the target, a negative number if the slice element precedes the target, -// or a positive number if the slice element follows the target. -// cmp must implement the same ordering as the slice, such that if -// cmp(a, t) < 0 and cmp(b, t) >= 0, then a must precede b in the slice. -func BinarySearchFunc[S ~[]E, E, T any](x S, target T, cmp func(E, T) int) (int, bool) { - n := len(x) - // Define cmp(x[-1], target) < 0 and cmp(x[n], target) >= 0 . - // Invariant: cmp(x[i - 1], target) < 0, cmp(x[j], target) >= 0. - i, j := 0, n - for i < j { - h := int(uint(i+j) >> 1) // avoid overflow when computing h - // i ≤ h < j - if cmp(x[h], target) < 0 { - i = h + 1 // preserves cmp(x[i - 1], target) < 0 - } else { - j = h // preserves cmp(x[j], target) >= 0 - } - } - // i == j, cmp(x[i-1], target) < 0, and cmp(x[j], target) (= cmp(x[i], target)) >= 0 => answer is i. - return i, i < n && cmp(x[i], target) == 0 -} - -type sortedHint int // hint for pdqsort when choosing the pivot - -const ( - unknownHint sortedHint = iota - increasingHint - decreasingHint -) - -// xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf -type xorshift uint64 - -func (r *xorshift) Next() uint64 { - *r ^= *r << 13 - *r ^= *r >> 17 - *r ^= *r << 5 - return uint64(*r) -} - -func nextPowerOfTwo(length int) uint { - return 1 << bits.Len(uint(length)) -} - -// isNaN reports whether x is a NaN without requiring the math package. -// This will always return false if T is not floating-point. -func isNaN[T constraints.Ordered](x T) bool { - return x != x -} diff --git a/vendor/golang.org/x/exp/slices/zsortanyfunc.go b/vendor/golang.org/x/exp/slices/zsortanyfunc.go deleted file mode 100644 index 06f2c7a2481b..000000000000 --- a/vendor/golang.org/x/exp/slices/zsortanyfunc.go +++ /dev/null @@ -1,479 +0,0 @@ -// Code generated by gen_sort_variants.go; DO NOT EDIT. - -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package slices - -// insertionSortCmpFunc sorts data[a:b] using insertion sort. -func insertionSortCmpFunc[E any](data []E, a, b int, cmp func(a, b E) int) { - for i := a + 1; i < b; i++ { - for j := i; j > a && (cmp(data[j], data[j-1]) < 0); j-- { - data[j], data[j-1] = data[j-1], data[j] - } - } -} - -// siftDownCmpFunc implements the heap property on data[lo:hi]. -// first is an offset into the array where the root of the heap lies. -func siftDownCmpFunc[E any](data []E, lo, hi, first int, cmp func(a, b E) int) { - root := lo - for { - child := 2*root + 1 - if child >= hi { - break - } - if child+1 < hi && (cmp(data[first+child], data[first+child+1]) < 0) { - child++ - } - if !(cmp(data[first+root], data[first+child]) < 0) { - return - } - data[first+root], data[first+child] = data[first+child], data[first+root] - root = child - } -} - -func heapSortCmpFunc[E any](data []E, a, b int, cmp func(a, b E) int) { - first := a - lo := 0 - hi := b - a - - // Build heap with greatest element at top. - for i := (hi - 1) / 2; i >= 0; i-- { - siftDownCmpFunc(data, i, hi, first, cmp) - } - - // Pop elements, largest first, into end of data. - for i := hi - 1; i >= 0; i-- { - data[first], data[first+i] = data[first+i], data[first] - siftDownCmpFunc(data, lo, i, first, cmp) - } -} - -// pdqsortCmpFunc sorts data[a:b]. -// The algorithm based on pattern-defeating quicksort(pdqsort), but without the optimizations from BlockQuicksort. -// pdqsort paper: https://arxiv.org/pdf/2106.05123.pdf -// C++ implementation: https://github.com/orlp/pdqsort -// Rust implementation: https://docs.rs/pdqsort/latest/pdqsort/ -// limit is the number of allowed bad (very unbalanced) pivots before falling back to heapsort. -func pdqsortCmpFunc[E any](data []E, a, b, limit int, cmp func(a, b E) int) { - const maxInsertion = 12 - - var ( - wasBalanced = true // whether the last partitioning was reasonably balanced - wasPartitioned = true // whether the slice was already partitioned - ) - - for { - length := b - a - - if length <= maxInsertion { - insertionSortCmpFunc(data, a, b, cmp) - return - } - - // Fall back to heapsort if too many bad choices were made. - if limit == 0 { - heapSortCmpFunc(data, a, b, cmp) - return - } - - // If the last partitioning was imbalanced, we need to breaking patterns. - if !wasBalanced { - breakPatternsCmpFunc(data, a, b, cmp) - limit-- - } - - pivot, hint := choosePivotCmpFunc(data, a, b, cmp) - if hint == decreasingHint { - reverseRangeCmpFunc(data, a, b, cmp) - // The chosen pivot was pivot-a elements after the start of the array. - // After reversing it is pivot-a elements before the end of the array. - // The idea came from Rust's implementation. - pivot = (b - 1) - (pivot - a) - hint = increasingHint - } - - // The slice is likely already sorted. - if wasBalanced && wasPartitioned && hint == increasingHint { - if partialInsertionSortCmpFunc(data, a, b, cmp) { - return - } - } - - // Probably the slice contains many duplicate elements, partition the slice into - // elements equal to and elements greater than the pivot. - if a > 0 && !(cmp(data[a-1], data[pivot]) < 0) { - mid := partitionEqualCmpFunc(data, a, b, pivot, cmp) - a = mid - continue - } - - mid, alreadyPartitioned := partitionCmpFunc(data, a, b, pivot, cmp) - wasPartitioned = alreadyPartitioned - - leftLen, rightLen := mid-a, b-mid - balanceThreshold := length / 8 - if leftLen < rightLen { - wasBalanced = leftLen >= balanceThreshold - pdqsortCmpFunc(data, a, mid, limit, cmp) - a = mid + 1 - } else { - wasBalanced = rightLen >= balanceThreshold - pdqsortCmpFunc(data, mid+1, b, limit, cmp) - b = mid - } - } -} - -// partitionCmpFunc does one quicksort partition. -// Let p = data[pivot] -// Moves elements in data[a:b] around, so that data[i]

=p for inewpivot. -// On return, data[newpivot] = p -func partitionCmpFunc[E any](data []E, a, b, pivot int, cmp func(a, b E) int) (newpivot int, alreadyPartitioned bool) { - data[a], data[pivot] = data[pivot], data[a] - i, j := a+1, b-1 // i and j are inclusive of the elements remaining to be partitioned - - for i <= j && (cmp(data[i], data[a]) < 0) { - i++ - } - for i <= j && !(cmp(data[j], data[a]) < 0) { - j-- - } - if i > j { - data[j], data[a] = data[a], data[j] - return j, true - } - data[i], data[j] = data[j], data[i] - i++ - j-- - - for { - for i <= j && (cmp(data[i], data[a]) < 0) { - i++ - } - for i <= j && !(cmp(data[j], data[a]) < 0) { - j-- - } - if i > j { - break - } - data[i], data[j] = data[j], data[i] - i++ - j-- - } - data[j], data[a] = data[a], data[j] - return j, false -} - -// partitionEqualCmpFunc partitions data[a:b] into elements equal to data[pivot] followed by elements greater than data[pivot]. -// It assumed that data[a:b] does not contain elements smaller than the data[pivot]. -func partitionEqualCmpFunc[E any](data []E, a, b, pivot int, cmp func(a, b E) int) (newpivot int) { - data[a], data[pivot] = data[pivot], data[a] - i, j := a+1, b-1 // i and j are inclusive of the elements remaining to be partitioned - - for { - for i <= j && !(cmp(data[a], data[i]) < 0) { - i++ - } - for i <= j && (cmp(data[a], data[j]) < 0) { - j-- - } - if i > j { - break - } - data[i], data[j] = data[j], data[i] - i++ - j-- - } - return i -} - -// partialInsertionSortCmpFunc partially sorts a slice, returns true if the slice is sorted at the end. -func partialInsertionSortCmpFunc[E any](data []E, a, b int, cmp func(a, b E) int) bool { - const ( - maxSteps = 5 // maximum number of adjacent out-of-order pairs that will get shifted - shortestShifting = 50 // don't shift any elements on short arrays - ) - i := a + 1 - for j := 0; j < maxSteps; j++ { - for i < b && !(cmp(data[i], data[i-1]) < 0) { - i++ - } - - if i == b { - return true - } - - if b-a < shortestShifting { - return false - } - - data[i], data[i-1] = data[i-1], data[i] - - // Shift the smaller one to the left. - if i-a >= 2 { - for j := i - 1; j >= 1; j-- { - if !(cmp(data[j], data[j-1]) < 0) { - break - } - data[j], data[j-1] = data[j-1], data[j] - } - } - // Shift the greater one to the right. - if b-i >= 2 { - for j := i + 1; j < b; j++ { - if !(cmp(data[j], data[j-1]) < 0) { - break - } - data[j], data[j-1] = data[j-1], data[j] - } - } - } - return false -} - -// breakPatternsCmpFunc scatters some elements around in an attempt to break some patterns -// that might cause imbalanced partitions in quicksort. -func breakPatternsCmpFunc[E any](data []E, a, b int, cmp func(a, b E) int) { - length := b - a - if length >= 8 { - random := xorshift(length) - modulus := nextPowerOfTwo(length) - - for idx := a + (length/4)*2 - 1; idx <= a+(length/4)*2+1; idx++ { - other := int(uint(random.Next()) & (modulus - 1)) - if other >= length { - other -= length - } - data[idx], data[a+other] = data[a+other], data[idx] - } - } -} - -// choosePivotCmpFunc chooses a pivot in data[a:b]. -// -// [0,8): chooses a static pivot. -// [8,shortestNinther): uses the simple median-of-three method. -// [shortestNinther,∞): uses the Tukey ninther method. -func choosePivotCmpFunc[E any](data []E, a, b int, cmp func(a, b E) int) (pivot int, hint sortedHint) { - const ( - shortestNinther = 50 - maxSwaps = 4 * 3 - ) - - l := b - a - - var ( - swaps int - i = a + l/4*1 - j = a + l/4*2 - k = a + l/4*3 - ) - - if l >= 8 { - if l >= shortestNinther { - // Tukey ninther method, the idea came from Rust's implementation. - i = medianAdjacentCmpFunc(data, i, &swaps, cmp) - j = medianAdjacentCmpFunc(data, j, &swaps, cmp) - k = medianAdjacentCmpFunc(data, k, &swaps, cmp) - } - // Find the median among i, j, k and stores it into j. - j = medianCmpFunc(data, i, j, k, &swaps, cmp) - } - - switch swaps { - case 0: - return j, increasingHint - case maxSwaps: - return j, decreasingHint - default: - return j, unknownHint - } -} - -// order2CmpFunc returns x,y where data[x] <= data[y], where x,y=a,b or x,y=b,a. -func order2CmpFunc[E any](data []E, a, b int, swaps *int, cmp func(a, b E) int) (int, int) { - if cmp(data[b], data[a]) < 0 { - *swaps++ - return b, a - } - return a, b -} - -// medianCmpFunc returns x where data[x] is the median of data[a],data[b],data[c], where x is a, b, or c. -func medianCmpFunc[E any](data []E, a, b, c int, swaps *int, cmp func(a, b E) int) int { - a, b = order2CmpFunc(data, a, b, swaps, cmp) - b, c = order2CmpFunc(data, b, c, swaps, cmp) - a, b = order2CmpFunc(data, a, b, swaps, cmp) - return b -} - -// medianAdjacentCmpFunc finds the median of data[a - 1], data[a], data[a + 1] and stores the index into a. -func medianAdjacentCmpFunc[E any](data []E, a int, swaps *int, cmp func(a, b E) int) int { - return medianCmpFunc(data, a-1, a, a+1, swaps, cmp) -} - -func reverseRangeCmpFunc[E any](data []E, a, b int, cmp func(a, b E) int) { - i := a - j := b - 1 - for i < j { - data[i], data[j] = data[j], data[i] - i++ - j-- - } -} - -func swapRangeCmpFunc[E any](data []E, a, b, n int, cmp func(a, b E) int) { - for i := 0; i < n; i++ { - data[a+i], data[b+i] = data[b+i], data[a+i] - } -} - -func stableCmpFunc[E any](data []E, n int, cmp func(a, b E) int) { - blockSize := 20 // must be > 0 - a, b := 0, blockSize - for b <= n { - insertionSortCmpFunc(data, a, b, cmp) - a = b - b += blockSize - } - insertionSortCmpFunc(data, a, n, cmp) - - for blockSize < n { - a, b = 0, 2*blockSize - for b <= n { - symMergeCmpFunc(data, a, a+blockSize, b, cmp) - a = b - b += 2 * blockSize - } - if m := a + blockSize; m < n { - symMergeCmpFunc(data, a, m, n, cmp) - } - blockSize *= 2 - } -} - -// symMergeCmpFunc merges the two sorted subsequences data[a:m] and data[m:b] using -// the SymMerge algorithm from Pok-Son Kim and Arne Kutzner, "Stable Minimum -// Storage Merging by Symmetric Comparisons", in Susanne Albers and Tomasz -// Radzik, editors, Algorithms - ESA 2004, volume 3221 of Lecture Notes in -// Computer Science, pages 714-723. Springer, 2004. -// -// Let M = m-a and N = b-n. Wolog M < N. -// The recursion depth is bound by ceil(log(N+M)). -// The algorithm needs O(M*log(N/M + 1)) calls to data.Less. -// The algorithm needs O((M+N)*log(M)) calls to data.Swap. -// -// The paper gives O((M+N)*log(M)) as the number of assignments assuming a -// rotation algorithm which uses O(M+N+gcd(M+N)) assignments. The argumentation -// in the paper carries through for Swap operations, especially as the block -// swapping rotate uses only O(M+N) Swaps. -// -// symMerge assumes non-degenerate arguments: a < m && m < b. -// Having the caller check this condition eliminates many leaf recursion calls, -// which improves performance. -func symMergeCmpFunc[E any](data []E, a, m, b int, cmp func(a, b E) int) { - // Avoid unnecessary recursions of symMerge - // by direct insertion of data[a] into data[m:b] - // if data[a:m] only contains one element. - if m-a == 1 { - // Use binary search to find the lowest index i - // such that data[i] >= data[a] for m <= i < b. - // Exit the search loop with i == b in case no such index exists. - i := m - j := b - for i < j { - h := int(uint(i+j) >> 1) - if cmp(data[h], data[a]) < 0 { - i = h + 1 - } else { - j = h - } - } - // Swap values until data[a] reaches the position before i. - for k := a; k < i-1; k++ { - data[k], data[k+1] = data[k+1], data[k] - } - return - } - - // Avoid unnecessary recursions of symMerge - // by direct insertion of data[m] into data[a:m] - // if data[m:b] only contains one element. - if b-m == 1 { - // Use binary search to find the lowest index i - // such that data[i] > data[m] for a <= i < m. - // Exit the search loop with i == m in case no such index exists. - i := a - j := m - for i < j { - h := int(uint(i+j) >> 1) - if !(cmp(data[m], data[h]) < 0) { - i = h + 1 - } else { - j = h - } - } - // Swap values until data[m] reaches the position i. - for k := m; k > i; k-- { - data[k], data[k-1] = data[k-1], data[k] - } - return - } - - mid := int(uint(a+b) >> 1) - n := mid + m - var start, r int - if m > mid { - start = n - b - r = mid - } else { - start = a - r = m - } - p := n - 1 - - for start < r { - c := int(uint(start+r) >> 1) - if !(cmp(data[p-c], data[c]) < 0) { - start = c + 1 - } else { - r = c - } - } - - end := n - start - if start < m && m < end { - rotateCmpFunc(data, start, m, end, cmp) - } - if a < start && start < mid { - symMergeCmpFunc(data, a, start, mid, cmp) - } - if mid < end && end < b { - symMergeCmpFunc(data, mid, end, b, cmp) - } -} - -// rotateCmpFunc rotates two consecutive blocks u = data[a:m] and v = data[m:b] in data: -// Data of the form 'x u v y' is changed to 'x v u y'. -// rotate performs at most b-a many calls to data.Swap, -// and it assumes non-degenerate arguments: a < m && m < b. -func rotateCmpFunc[E any](data []E, a, m, b int, cmp func(a, b E) int) { - i := m - a - j := b - m - - for i != j { - if i > j { - swapRangeCmpFunc(data, m-i, m, j, cmp) - i -= j - } else { - swapRangeCmpFunc(data, m-i, m+j-i, i, cmp) - j -= i - } - } - // i == j - swapRangeCmpFunc(data, m-i, m, i, cmp) -} diff --git a/vendor/golang.org/x/exp/slices/zsortordered.go b/vendor/golang.org/x/exp/slices/zsortordered.go deleted file mode 100644 index 99b47c3986a4..000000000000 --- a/vendor/golang.org/x/exp/slices/zsortordered.go +++ /dev/null @@ -1,481 +0,0 @@ -// Code generated by gen_sort_variants.go; DO NOT EDIT. - -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package slices - -import "golang.org/x/exp/constraints" - -// insertionSortOrdered sorts data[a:b] using insertion sort. -func insertionSortOrdered[E constraints.Ordered](data []E, a, b int) { - for i := a + 1; i < b; i++ { - for j := i; j > a && cmpLess(data[j], data[j-1]); j-- { - data[j], data[j-1] = data[j-1], data[j] - } - } -} - -// siftDownOrdered implements the heap property on data[lo:hi]. -// first is an offset into the array where the root of the heap lies. -func siftDownOrdered[E constraints.Ordered](data []E, lo, hi, first int) { - root := lo - for { - child := 2*root + 1 - if child >= hi { - break - } - if child+1 < hi && cmpLess(data[first+child], data[first+child+1]) { - child++ - } - if !cmpLess(data[first+root], data[first+child]) { - return - } - data[first+root], data[first+child] = data[first+child], data[first+root] - root = child - } -} - -func heapSortOrdered[E constraints.Ordered](data []E, a, b int) { - first := a - lo := 0 - hi := b - a - - // Build heap with greatest element at top. - for i := (hi - 1) / 2; i >= 0; i-- { - siftDownOrdered(data, i, hi, first) - } - - // Pop elements, largest first, into end of data. - for i := hi - 1; i >= 0; i-- { - data[first], data[first+i] = data[first+i], data[first] - siftDownOrdered(data, lo, i, first) - } -} - -// pdqsortOrdered sorts data[a:b]. -// The algorithm based on pattern-defeating quicksort(pdqsort), but without the optimizations from BlockQuicksort. -// pdqsort paper: https://arxiv.org/pdf/2106.05123.pdf -// C++ implementation: https://github.com/orlp/pdqsort -// Rust implementation: https://docs.rs/pdqsort/latest/pdqsort/ -// limit is the number of allowed bad (very unbalanced) pivots before falling back to heapsort. -func pdqsortOrdered[E constraints.Ordered](data []E, a, b, limit int) { - const maxInsertion = 12 - - var ( - wasBalanced = true // whether the last partitioning was reasonably balanced - wasPartitioned = true // whether the slice was already partitioned - ) - - for { - length := b - a - - if length <= maxInsertion { - insertionSortOrdered(data, a, b) - return - } - - // Fall back to heapsort if too many bad choices were made. - if limit == 0 { - heapSortOrdered(data, a, b) - return - } - - // If the last partitioning was imbalanced, we need to breaking patterns. - if !wasBalanced { - breakPatternsOrdered(data, a, b) - limit-- - } - - pivot, hint := choosePivotOrdered(data, a, b) - if hint == decreasingHint { - reverseRangeOrdered(data, a, b) - // The chosen pivot was pivot-a elements after the start of the array. - // After reversing it is pivot-a elements before the end of the array. - // The idea came from Rust's implementation. - pivot = (b - 1) - (pivot - a) - hint = increasingHint - } - - // The slice is likely already sorted. - if wasBalanced && wasPartitioned && hint == increasingHint { - if partialInsertionSortOrdered(data, a, b) { - return - } - } - - // Probably the slice contains many duplicate elements, partition the slice into - // elements equal to and elements greater than the pivot. - if a > 0 && !cmpLess(data[a-1], data[pivot]) { - mid := partitionEqualOrdered(data, a, b, pivot) - a = mid - continue - } - - mid, alreadyPartitioned := partitionOrdered(data, a, b, pivot) - wasPartitioned = alreadyPartitioned - - leftLen, rightLen := mid-a, b-mid - balanceThreshold := length / 8 - if leftLen < rightLen { - wasBalanced = leftLen >= balanceThreshold - pdqsortOrdered(data, a, mid, limit) - a = mid + 1 - } else { - wasBalanced = rightLen >= balanceThreshold - pdqsortOrdered(data, mid+1, b, limit) - b = mid - } - } -} - -// partitionOrdered does one quicksort partition. -// Let p = data[pivot] -// Moves elements in data[a:b] around, so that data[i]

=p for inewpivot. -// On return, data[newpivot] = p -func partitionOrdered[E constraints.Ordered](data []E, a, b, pivot int) (newpivot int, alreadyPartitioned bool) { - data[a], data[pivot] = data[pivot], data[a] - i, j := a+1, b-1 // i and j are inclusive of the elements remaining to be partitioned - - for i <= j && cmpLess(data[i], data[a]) { - i++ - } - for i <= j && !cmpLess(data[j], data[a]) { - j-- - } - if i > j { - data[j], data[a] = data[a], data[j] - return j, true - } - data[i], data[j] = data[j], data[i] - i++ - j-- - - for { - for i <= j && cmpLess(data[i], data[a]) { - i++ - } - for i <= j && !cmpLess(data[j], data[a]) { - j-- - } - if i > j { - break - } - data[i], data[j] = data[j], data[i] - i++ - j-- - } - data[j], data[a] = data[a], data[j] - return j, false -} - -// partitionEqualOrdered partitions data[a:b] into elements equal to data[pivot] followed by elements greater than data[pivot]. -// It assumed that data[a:b] does not contain elements smaller than the data[pivot]. -func partitionEqualOrdered[E constraints.Ordered](data []E, a, b, pivot int) (newpivot int) { - data[a], data[pivot] = data[pivot], data[a] - i, j := a+1, b-1 // i and j are inclusive of the elements remaining to be partitioned - - for { - for i <= j && !cmpLess(data[a], data[i]) { - i++ - } - for i <= j && cmpLess(data[a], data[j]) { - j-- - } - if i > j { - break - } - data[i], data[j] = data[j], data[i] - i++ - j-- - } - return i -} - -// partialInsertionSortOrdered partially sorts a slice, returns true if the slice is sorted at the end. -func partialInsertionSortOrdered[E constraints.Ordered](data []E, a, b int) bool { - const ( - maxSteps = 5 // maximum number of adjacent out-of-order pairs that will get shifted - shortestShifting = 50 // don't shift any elements on short arrays - ) - i := a + 1 - for j := 0; j < maxSteps; j++ { - for i < b && !cmpLess(data[i], data[i-1]) { - i++ - } - - if i == b { - return true - } - - if b-a < shortestShifting { - return false - } - - data[i], data[i-1] = data[i-1], data[i] - - // Shift the smaller one to the left. - if i-a >= 2 { - for j := i - 1; j >= 1; j-- { - if !cmpLess(data[j], data[j-1]) { - break - } - data[j], data[j-1] = data[j-1], data[j] - } - } - // Shift the greater one to the right. - if b-i >= 2 { - for j := i + 1; j < b; j++ { - if !cmpLess(data[j], data[j-1]) { - break - } - data[j], data[j-1] = data[j-1], data[j] - } - } - } - return false -} - -// breakPatternsOrdered scatters some elements around in an attempt to break some patterns -// that might cause imbalanced partitions in quicksort. -func breakPatternsOrdered[E constraints.Ordered](data []E, a, b int) { - length := b - a - if length >= 8 { - random := xorshift(length) - modulus := nextPowerOfTwo(length) - - for idx := a + (length/4)*2 - 1; idx <= a+(length/4)*2+1; idx++ { - other := int(uint(random.Next()) & (modulus - 1)) - if other >= length { - other -= length - } - data[idx], data[a+other] = data[a+other], data[idx] - } - } -} - -// choosePivotOrdered chooses a pivot in data[a:b]. -// -// [0,8): chooses a static pivot. -// [8,shortestNinther): uses the simple median-of-three method. -// [shortestNinther,∞): uses the Tukey ninther method. -func choosePivotOrdered[E constraints.Ordered](data []E, a, b int) (pivot int, hint sortedHint) { - const ( - shortestNinther = 50 - maxSwaps = 4 * 3 - ) - - l := b - a - - var ( - swaps int - i = a + l/4*1 - j = a + l/4*2 - k = a + l/4*3 - ) - - if l >= 8 { - if l >= shortestNinther { - // Tukey ninther method, the idea came from Rust's implementation. - i = medianAdjacentOrdered(data, i, &swaps) - j = medianAdjacentOrdered(data, j, &swaps) - k = medianAdjacentOrdered(data, k, &swaps) - } - // Find the median among i, j, k and stores it into j. - j = medianOrdered(data, i, j, k, &swaps) - } - - switch swaps { - case 0: - return j, increasingHint - case maxSwaps: - return j, decreasingHint - default: - return j, unknownHint - } -} - -// order2Ordered returns x,y where data[x] <= data[y], where x,y=a,b or x,y=b,a. -func order2Ordered[E constraints.Ordered](data []E, a, b int, swaps *int) (int, int) { - if cmpLess(data[b], data[a]) { - *swaps++ - return b, a - } - return a, b -} - -// medianOrdered returns x where data[x] is the median of data[a],data[b],data[c], where x is a, b, or c. -func medianOrdered[E constraints.Ordered](data []E, a, b, c int, swaps *int) int { - a, b = order2Ordered(data, a, b, swaps) - b, c = order2Ordered(data, b, c, swaps) - a, b = order2Ordered(data, a, b, swaps) - return b -} - -// medianAdjacentOrdered finds the median of data[a - 1], data[a], data[a + 1] and stores the index into a. -func medianAdjacentOrdered[E constraints.Ordered](data []E, a int, swaps *int) int { - return medianOrdered(data, a-1, a, a+1, swaps) -} - -func reverseRangeOrdered[E constraints.Ordered](data []E, a, b int) { - i := a - j := b - 1 - for i < j { - data[i], data[j] = data[j], data[i] - i++ - j-- - } -} - -func swapRangeOrdered[E constraints.Ordered](data []E, a, b, n int) { - for i := 0; i < n; i++ { - data[a+i], data[b+i] = data[b+i], data[a+i] - } -} - -func stableOrdered[E constraints.Ordered](data []E, n int) { - blockSize := 20 // must be > 0 - a, b := 0, blockSize - for b <= n { - insertionSortOrdered(data, a, b) - a = b - b += blockSize - } - insertionSortOrdered(data, a, n) - - for blockSize < n { - a, b = 0, 2*blockSize - for b <= n { - symMergeOrdered(data, a, a+blockSize, b) - a = b - b += 2 * blockSize - } - if m := a + blockSize; m < n { - symMergeOrdered(data, a, m, n) - } - blockSize *= 2 - } -} - -// symMergeOrdered merges the two sorted subsequences data[a:m] and data[m:b] using -// the SymMerge algorithm from Pok-Son Kim and Arne Kutzner, "Stable Minimum -// Storage Merging by Symmetric Comparisons", in Susanne Albers and Tomasz -// Radzik, editors, Algorithms - ESA 2004, volume 3221 of Lecture Notes in -// Computer Science, pages 714-723. Springer, 2004. -// -// Let M = m-a and N = b-n. Wolog M < N. -// The recursion depth is bound by ceil(log(N+M)). -// The algorithm needs O(M*log(N/M + 1)) calls to data.Less. -// The algorithm needs O((M+N)*log(M)) calls to data.Swap. -// -// The paper gives O((M+N)*log(M)) as the number of assignments assuming a -// rotation algorithm which uses O(M+N+gcd(M+N)) assignments. The argumentation -// in the paper carries through for Swap operations, especially as the block -// swapping rotate uses only O(M+N) Swaps. -// -// symMerge assumes non-degenerate arguments: a < m && m < b. -// Having the caller check this condition eliminates many leaf recursion calls, -// which improves performance. -func symMergeOrdered[E constraints.Ordered](data []E, a, m, b int) { - // Avoid unnecessary recursions of symMerge - // by direct insertion of data[a] into data[m:b] - // if data[a:m] only contains one element. - if m-a == 1 { - // Use binary search to find the lowest index i - // such that data[i] >= data[a] for m <= i < b. - // Exit the search loop with i == b in case no such index exists. - i := m - j := b - for i < j { - h := int(uint(i+j) >> 1) - if cmpLess(data[h], data[a]) { - i = h + 1 - } else { - j = h - } - } - // Swap values until data[a] reaches the position before i. - for k := a; k < i-1; k++ { - data[k], data[k+1] = data[k+1], data[k] - } - return - } - - // Avoid unnecessary recursions of symMerge - // by direct insertion of data[m] into data[a:m] - // if data[m:b] only contains one element. - if b-m == 1 { - // Use binary search to find the lowest index i - // such that data[i] > data[m] for a <= i < m. - // Exit the search loop with i == m in case no such index exists. - i := a - j := m - for i < j { - h := int(uint(i+j) >> 1) - if !cmpLess(data[m], data[h]) { - i = h + 1 - } else { - j = h - } - } - // Swap values until data[m] reaches the position i. - for k := m; k > i; k-- { - data[k], data[k-1] = data[k-1], data[k] - } - return - } - - mid := int(uint(a+b) >> 1) - n := mid + m - var start, r int - if m > mid { - start = n - b - r = mid - } else { - start = a - r = m - } - p := n - 1 - - for start < r { - c := int(uint(start+r) >> 1) - if !cmpLess(data[p-c], data[c]) { - start = c + 1 - } else { - r = c - } - } - - end := n - start - if start < m && m < end { - rotateOrdered(data, start, m, end) - } - if a < start && start < mid { - symMergeOrdered(data, a, start, mid) - } - if mid < end && end < b { - symMergeOrdered(data, mid, end, b) - } -} - -// rotateOrdered rotates two consecutive blocks u = data[a:m] and v = data[m:b] in data: -// Data of the form 'x u v y' is changed to 'x v u y'. -// rotate performs at most b-a many calls to data.Swap, -// and it assumes non-degenerate arguments: a < m && m < b. -func rotateOrdered[E constraints.Ordered](data []E, a, m, b int) { - i := m - a - j := b - m - - for i != j { - if i > j { - swapRangeOrdered(data, m-i, m, j) - i -= j - } else { - swapRangeOrdered(data, m-i, m+j-i, i) - j -= i - } - } - // i == j - swapRangeOrdered(data, m-i, m, i) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1158b55d3a43..411b23d358e6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -81,22 +81,24 @@ github.com/checkpoint-restore/checkpointctl/lib ## explicit; go 1.20 github.com/checkpoint-restore/go-criu/v7 github.com/checkpoint-restore/go-criu/v7/rpc -# github.com/cilium/ebpf v0.11.0 -## explicit; go 1.19 +# github.com/cilium/ebpf v0.16.0 +## explicit; go 1.21 github.com/cilium/ebpf github.com/cilium/ebpf/asm github.com/cilium/ebpf/btf github.com/cilium/ebpf/internal +github.com/cilium/ebpf/internal/kallsyms github.com/cilium/ebpf/internal/kconfig github.com/cilium/ebpf/internal/sys +github.com/cilium/ebpf/internal/sysenc github.com/cilium/ebpf/internal/tracefs github.com/cilium/ebpf/internal/unix github.com/cilium/ebpf/link # github.com/containerd/btrfs/v2 v2.0.0 ## explicit; go 1.19 github.com/containerd/btrfs/v2 -# github.com/containerd/cgroups/v3 v3.0.3 -## explicit; go 1.18 +# github.com/containerd/cgroups/v3 v3.0.4 +## explicit; go 1.22.0 github.com/containerd/cgroups/v3 github.com/containerd/cgroups/v3/cgroup1 github.com/containerd/cgroups/v3/cgroup1/stats @@ -576,11 +578,9 @@ golang.org/x/crypto/openpgp/errors golang.org/x/crypto/openpgp/packet golang.org/x/crypto/openpgp/s2k golang.org/x/crypto/pbkdf2 -# golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 -## explicit; go 1.20 +# golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f +## explicit; go 1.22.0 golang.org/x/exp/constraints -golang.org/x/exp/maps -golang.org/x/exp/slices # golang.org/x/mod v0.22.0 ## explicit; go 1.22.0 golang.org/x/mod/semver