diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index d0b708e959c1..6d6230eb16fc 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -243,11 +243,69 @@ BPF_PROG(name, ##args) #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) +/* useful compiler attributes */ +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#define __maybe_unused __attribute__((__unused__)) + /* - * bpf_log2 - Compute the base 2 logarithm of a 32-bit exponential value. + * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They + * prevent compiler from caching, redoing or reordering reads or writes. + */ +typedef __u8 __attribute__((__may_alias__)) __u8_alias_t; +typedef __u16 __attribute__((__may_alias__)) __u16_alias_t; +typedef __u32 __attribute__((__may_alias__)) __u32_alias_t; +typedef __u64 __attribute__((__may_alias__)) __u64_alias_t; + +static __always_inline void __read_once_size(const volatile void *p, void *res, int size) +{ + switch (size) { + case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break; + case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break; + case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break; + case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break; + default: + barrier(); + __builtin_memcpy((void *)res, (const void *)p, size); + barrier(); + } +} + +static __always_inline void __write_once_size(volatile void *p, void *res, int size) +{ + switch (size) { + case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break; + case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break; + case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break; + case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break; + default: + barrier(); + __builtin_memcpy((void *)p, (const void *)res, size); + barrier(); + } +} + +#define READ_ONCE(x) \ +({ \ + union { typeof(x) __val; char __c[1]; } __u = \ + { .__c = { 0 } }; \ + __read_once_size(&(x), __u.__c, sizeof(x)); \ + __u.__val; \ +}) + +#define WRITE_ONCE(x, val) \ +({ \ + union { typeof(x) __val; char __c[1]; } __u = \ + { .__val = (val) }; \ + __write_once_size(&(x), __u.__c, sizeof(x)); \ + __u.__val; \ +}) + +/* + * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value. * @v: The value for which we're computing the base 2 logarithm. */ -static inline u32 bpf_log2(u32 v) +static inline u32 log2_u32(u32 v) { u32 r; u32 shift; @@ -261,24 +319,18 @@ static inline u32 bpf_log2(u32 v) } /* - * bpf_log2l - Compute the base 2 logarithm of a 64-bit exponential value. + * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value. * @v: The value for which we're computing the base 2 logarithm. */ -static inline u32 bpf_log2l(u64 v) +static inline u32 log2_u64(u64 v) { u32 hi = v >> 32; if (hi) - return bpf_log2(hi) + 32 + 1; + return log2_u32(hi) + 32 + 1; else - return bpf_log2(v) + 1; + return log2_u32(v) + 1; } -/* useful compiler attributes */ -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) -#define __maybe_unused __attribute__((__unused__)) - - void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; @@ -311,6 +363,16 @@ struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; void bpf_cgroup_release(struct cgroup *cgrp) __ksym; struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; +/* css iteration */ +struct bpf_iter_css; +struct cgroup_subsys_state; +extern int bpf_iter_css_new(struct bpf_iter_css *it, + struct cgroup_subsys_state *start, + unsigned int flags) __weak __ksym; +extern struct cgroup_subsys_state * +bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym; +extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym; + /* cpumask */ struct bpf_cpumask *bpf_cpumask_create(void) __ksym; struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; diff --git a/tools/sched_ext/include/scx/common.h b/tools/sched_ext/include/scx/common.h index 8d5a6775f64d..49234dad00a1 100644 --- a/tools/sched_ext/include/scx/common.h +++ b/tools/sched_ext/include/scx/common.h @@ -44,7 +44,8 @@ typedef int64_t s64; /** * RESIZE_ARRAY - Convenience macro for resizing a BPF array - * @elfsec: the data section of the BPF program in which to the array exists + * @__skel: the skeleton containing the array + * @elfsec: the data section of the BPF program in which the array exists * @arr: the name of the array * @n: the desired array element count * @@ -56,13 +57,13 @@ typedef int64_t s64; * for that custom data section so that it points to the newly memory mapped * region. */ -#define RESIZE_ARRAY(elfsec, arr, n) \ - do { \ - size_t __sz; \ - bpf_map__set_value_size(skel->maps.elfsec##_##arr, \ - sizeof(skel->elfsec##_##arr->arr[0]) * (n)); \ - skel->elfsec##_##arr = \ - bpf_map__initial_value(skel->maps.elfsec##_##arr, &__sz); \ +#define RESIZE_ARRAY(__skel, elfsec, arr, n) \ + do { \ + size_t __sz; \ + bpf_map__set_value_size((__skel)->maps.elfsec##_##arr, \ + sizeof((__skel)->elfsec##_##arr->arr[0]) * (n)); \ + (__skel)->elfsec##_##arr = \ + bpf_map__initial_value((__skel)->maps.elfsec##_##arr, &__sz); \ } while (0) #include "user_exit_info.h" diff --git a/tools/sched_ext/include/scx/user_exit_info.h b/tools/sched_ext/include/scx/user_exit_info.h index 2d86d01a9575..891693ee604e 100644 --- a/tools/sched_ext/include/scx/user_exit_info.h +++ b/tools/sched_ext/include/scx/user_exit_info.h @@ -53,10 +53,10 @@ struct user_exit_info { #include /* no need to call the following explicitly if SCX_OPS_LOAD() is used */ -#define UEI_SET_SIZE(__skel, __ops_name, __uei_name) ({ \ - u32 __len = (__skel)->struct_ops.__ops_name->exit_dump_len ?: UEI_DUMP_DFL_LEN; \ - (__skel)->rodata->__uei_name##_dump_len = __len; \ - RESIZE_ARRAY(data, __uei_name##_dump, __len); \ +#define UEI_SET_SIZE(__skel, __ops_name, __uei_name) ({ \ + u32 __len = (__skel)->struct_ops.__ops_name->exit_dump_len ?: UEI_DUMP_DFL_LEN; \ + (__skel)->rodata->__uei_name##_dump_len = __len; \ + RESIZE_ARRAY((__skel), data, __uei_name##_dump, __len); \ }) #define UEI_EXITED(__skel, __uei_name) ({ \ diff --git a/tools/sched_ext/scx_central.c b/tools/sched_ext/scx_central.c index 1e0568624ccc..21deea320bd7 100644 --- a/tools/sched_ext/scx_central.c +++ b/tools/sched_ext/scx_central.c @@ -77,8 +77,8 @@ int main(int argc, char **argv) } /* Resize arrays so their element count is equal to cpu count. */ - RESIZE_ARRAY(data, cpu_gimme_task, skel->rodata->nr_cpu_ids); - RESIZE_ARRAY(data, cpu_started_at, skel->rodata->nr_cpu_ids); + RESIZE_ARRAY(skel, data, cpu_gimme_task, skel->rodata->nr_cpu_ids); + RESIZE_ARRAY(skel, data, cpu_started_at, skel->rodata->nr_cpu_ids); SCX_OPS_LOAD(skel, central_ops, scx_central, uei);