diff --git a/README.md b/README.md index f0a9e5ec..3aa56896 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ This crate provides a way to soundly perform such operations. ## Platform Support -Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, and Xtensa are supported. +Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, Xtensa, and BPF are supported. | target_arch | primitives | load/store | swap/CAS | | ------------------------------- | --------------------------------------------------- |:----------:|:--------:| @@ -45,6 +45,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, | hexagon \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ | | m68k \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | | xtensa \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | +| bpf \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓\[9] | \[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.
\[2] Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets).
@@ -54,6 +55,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, \[6] Requires Rust 1.72+.
\[7] Requires Rust 1.84+.
\[8] Requires nightly due to `#![feature(asm_experimental_arch)]`.
+\[9] BPF's atomic RMW operations are only available for isize/usize/i64/u64.
See also [Atomic operation overview by architecture](https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md) for more information about atomic operations in these architectures. diff --git a/build.rs b/build.rs index 87daeff4..5f03e2d5 100644 --- a/build.rs +++ b/build.rs @@ -111,8 +111,8 @@ fn main() { } } } - "avr" | "m68k" | "mips" | "mips32r6" | "mips64" | "mips64r6" | "msp430" | "powerpc" - | "powerpc64" | "xtensa" => { + "avr" | "bpf" | "m68k" | "mips" | "mips32r6" | "mips64" | "mips64r6" | "msp430" + | "powerpc" | "powerpc64" | "xtensa" => { if version.nightly && is_allowed_feature("asm_experimental_arch") { println!("cargo:rustc-cfg=atomic_maybe_uninit_unstable_asm_experimental_arch"); } diff --git a/src/arch/README.md b/src/arch/README.md index 7dc2aa08..c055c024 100644 --- a/src/arch/README.md +++ b/src/arch/README.md @@ -8,6 +8,7 @@ This document describes the operations that are considered atomic by architectur - [AArch64](#aarch64) - [Arm](#arm) - [AVR](#avr) +- [BPF](#bpf) - [Hexagon](#hexagon) - [LoongArch](#loongarch) - [M68k](#m68k) @@ -55,6 +56,13 @@ This architecture is always single-core and the following operations are atomic: disabling and restoring implementation must imply compiler fences, e.g., asm without nomem/readonly) may be moved out of the critical section by compiler optimizations. +## BPF + +target_arch: bpf
+Implementation: [bpf.rs](bpf.rs)
+ +TODO + ## Hexagon target_arch: hexagon
diff --git a/src/arch/bpf.rs b/src/arch/bpf.rs new file mode 100644 index 00000000..8cd50e3a --- /dev/null +++ b/src/arch/bpf.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/* +BPF + +Refs: +- BPF Instruction Set Architecture (ISA) + https://github.com/torvalds/linux/blob/v6.12/Documentation/bpf/standardization/instruction-set.rst + +Generated asm: +- bpf https://godbolt.org/z/oEGqWY1f4 +*/ + +#[path = "cfgs/bpf.rs"] +mod cfgs; + +use core::{ + arch::asm, + mem::{self, MaybeUninit}, + sync::atomic::Ordering, +}; + +use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap}; + +macro_rules! atomic_load_store { + ($ty:ident, $size:tt) => { + impl AtomicLoad for $ty { + #[inline] + unsafe fn atomic_load( + src: *const MaybeUninit, + _order: Ordering, + ) -> MaybeUninit { + debug_assert!(src as usize % mem::size_of::<$ty>() == 0); + let out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + asm!( + concat!("{out} = *(", $size, " *) ({src} + 0)"), // atomic { out = *src } + src = in(reg) src, + out = lateout(reg) out, + options(nostack, preserves_flags), + ); + } + out + } + } + impl AtomicStore for $ty { + #[inline] + unsafe fn atomic_store( + dst: *mut MaybeUninit, + val: MaybeUninit, + _order: Ordering, + ) { + debug_assert!(dst as usize % mem::size_of::<$ty>() == 0); + + // SAFETY: the caller must uphold the safety contract. + unsafe { + asm!( + concat!("*(", $size, " *) ({dst} + 0) = {val}"), // atomic { *dst = val } + dst = in(reg) dst, + val = in(reg) val, + options(nostack, preserves_flags), + ); + } + } + } + }; +} + +macro_rules! atomic64 { + ($ty:ident) => { + atomic_load_store!($ty, "u64"); + impl AtomicSwap for $ty { + #[inline] + unsafe fn atomic_swap( + dst: *mut MaybeUninit, + val: MaybeUninit, + _order: Ordering, + ) -> MaybeUninit { + debug_assert!(dst as usize % mem::size_of::<$ty>() == 0); + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + asm!( + "{out} = xchg_64({dst} + 0, {val})", // atomic { _x = *dst; *dst = val; out = _x } + dst = in(reg) dst, + val = in(reg) val, + out = lateout(reg) out, + options(nostack, preserves_flags), + ); + } + out + } + } + impl AtomicCompareExchange for $ty { + #[inline] + unsafe fn atomic_compare_exchange( + dst: *mut MaybeUninit, + old: MaybeUninit, + new: MaybeUninit, + _success: Ordering, + _failure: Ordering, + ) -> (MaybeUninit, bool) { + debug_assert!(dst as usize % mem::size_of::<$ty>() == 0); + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + let mut r: u64 = 1; + asm!( + "r0 = cmpxchg_64({dst} + 0, r0, {new})", // atomic { _x = *dst; if _x == r0 { *dst = new }; r0 = _x } + "if r0 == {old} goto 2f", // if r0 == old { jump 'success } + "{r} = 0", // r = 0 + "2:", // 'success + dst = in(reg) dst, + old = in(reg) old, + new = in(reg) new, + r = inout(reg) r, + inout("r0") old => out, + options(nostack, preserves_flags), + ); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test + (out, r != 0) + } + } + } + }; +} + +atomic_load_store!(i8, "u8"); +atomic_load_store!(u8, "u8"); +atomic_load_store!(i16, "u16"); +atomic_load_store!(u16, "u16"); +atomic_load_store!(i32, "u32"); +atomic_load_store!(u32, "u32"); +atomic64!(i64); +atomic64!(u64); +atomic64!(isize); +atomic64!(usize); diff --git a/src/arch/cfgs/bpf.rs b/src/arch/cfgs/bpf.rs new file mode 100644 index 00000000..031569c4 --- /dev/null +++ b/src/arch/cfgs/bpf.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(missing_docs)] + +// TODO: RMW on {8,16,32}-bit +#[macro_export] +macro_rules! cfg_has_atomic_8 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_no_atomic_8 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_has_atomic_16 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_no_atomic_16 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_has_atomic_32 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_no_atomic_32 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_has_atomic_64 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_64 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_has_atomic_128 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_no_atomic_128 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_has_atomic_cas { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_cas { + ($($tt:tt)*) => {}; +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 7ad4b261..3cfce1f4 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -29,6 +29,7 @@ all( any( target_arch = "avr", + target_arch = "bpf", target_arch = "hexagon", target_arch = "m68k", target_arch = "mips", @@ -86,6 +87,9 @@ mod armv8; #[cfg(target_arch = "avr")] #[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] mod avr; +#[cfg(target_arch = "bpf")] +#[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] +mod bpf; #[cfg(target_arch = "hexagon")] #[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] mod hexagon; diff --git a/src/lib.rs b/src/lib.rs index e42773de..249186be 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,7 +12,7 @@ This crate provides a way to soundly perform such operations. ## Platform Support -Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, and Xtensa are supported. +Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, Xtensa, and BPF are supported. | target_arch | primitives | load/store | swap/CAS | | ------------------------------- | --------------------------------------------------- |:----------:|:--------:| @@ -39,6 +39,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, | hexagon \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ | | m68k \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | | xtensa \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | +| bpf \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓\[9] | \[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.
\[2] Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets).
@@ -48,6 +49,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, \[6] Requires Rust 1.72+.
\[7] Requires Rust 1.84+.
\[8] Requires nightly due to `#![feature(asm_experimental_arch)]`.
+\[9] BPF's atomic RMW operations are only available for isize/usize/i64/u64.
See also [Atomic operation overview by architecture](https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md) for more information about atomic operations in these architectures. diff --git a/src/utils.rs b/src/utils.rs index 65659acd..7b31e8cb 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -241,6 +241,9 @@ pub(crate) struct Pair { pub(crate) lo: MaybeUninit, } +#[cfg(target_arch = "bpf")] +type MinWord = u64; +#[cfg(not(target_arch = "bpf"))] type MinWord = u32; #[cfg(target_arch = "s390x")] type RetInt = u32; diff --git a/tools/build.sh b/tools/build.sh index 74ebc1f0..d262144f 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -135,6 +135,11 @@ default_targets=( # hexagon # rustc --print target-list | grep -E '^hexagon' hexagon-unknown-linux-musl + + # bpf + # rustc --print target-list | grep -E '^bpf' + bpfeb-unknown-none + bpfel-unknown-none ) x() {