Support BPF (experimental)

README.md

@@ -18,7 +18,7 @@ This crate provides a way to soundly perform such operations.
 
 ## Platform Support
 
-Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, and Xtensa are supported.
+Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, Xtensa, and BPF are supported.
 
 | target_arch                     | primitives                                          | load/store | swap/CAS |
 | ------------------------------- | --------------------------------------------------- |:----------:|:--------:|
@@ -45,6 +45,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS,
 | hexagon \[8] (experimental)     | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64           | ✓          | ✓        |
 | m68k \[8] (experimental)        | isize,usize,i8,u8,i16,u16,i32,u32                   | ✓          | ✓\[1]    |
 | xtensa \[8] (experimental)      | isize,usize,i8,u8,i16,u16,i32,u32                   | ✓          | ✓\[1]    |
+| bpf \[8] (experimental)         | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64           | ✓          | ✓\[9]    |
 
 \[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.<br>
 \[2] Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets).<br>
@@ -54,6 +55,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS,
 \[6] Requires Rust 1.72+.<br>
 \[7] Requires Rust 1.84+.<br>
 \[8] Requires nightly due to `#![feature(asm_experimental_arch)]`.<br>
+\[9] BPF's atomic RMW operations are only available for isize/usize/i64/u64.<br>
 
 See also [Atomic operation overview by architecture](https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md) for more information about atomic operations in these architectures.
 

build.rs

@@ -111,8 +111,8 @@ fn main() {
                 }
             }
         }
-        "avr" | "m68k" | "mips" | "mips32r6" | "mips64" | "mips64r6" | "msp430" | "powerpc"
-        | "powerpc64" | "xtensa" => {
+        "avr" | "bpf" | "m68k" | "mips" | "mips32r6" | "mips64" | "mips64r6" | "msp430"
+        | "powerpc" | "powerpc64" | "xtensa" => {
             if version.nightly && is_allowed_feature("asm_experimental_arch") {
                 println!("cargo:rustc-cfg=atomic_maybe_uninit_unstable_asm_experimental_arch");
             }

src/arch/README.md

@@ -8,6 +8,7 @@ This document describes the operations that are considered atomic by architectur
 - [AArch64](#aarch64)
 - [Arm](#arm)
 - [AVR](#avr)
+- [BPF](#bpf)
 - [Hexagon](#hexagon)
 - [LoongArch](#loongarch)
 - [M68k](#m68k)
@@ -55,6 +56,13 @@ This architecture is always single-core and the following operations are atomic:
   disabling and restoring implementation must imply compiler fences, e.g., asm without nomem/readonly)
   may be moved out of the critical section by compiler optimizations.
 
+## BPF
+
+target_arch: bpf<br>
+Implementation: [bpf.rs](bpf.rs)<br>
+
+TODO
+
 ## Hexagon
 
 target_arch: hexagon<br>

src/arch/bpf.rs

@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+/*
+BPF
+
+Refs:
+- BPF Instruction Set Architecture (ISA)
+  https://github.com/torvalds/linux/blob/v6.12/Documentation/bpf/standardization/instruction-set.rst
+
+Generated asm:
+- bpf https://godbolt.org/z/oEGqWY1f4
+*/
+
+#[path = "cfgs/bpf.rs"]
+mod cfgs;
+
+use core::{
+    arch::asm,
+    mem::{self, MaybeUninit},
+    sync::atomic::Ordering,
+};
+
+use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap};
+
+macro_rules! atomic_load_store {
+    ($ty:ident, $size:tt) => {
+        impl AtomicLoad for $ty {
+            #[inline]
+            unsafe fn atomic_load(
+                src: *const MaybeUninit<Self>,
+                _order: Ordering,
+            ) -> MaybeUninit<Self> {
+                debug_assert!(src as usize % mem::size_of::<$ty>() == 0);
+                let out: MaybeUninit<Self>;
+
+                // SAFETY: the caller must uphold the safety contract.
+                unsafe {
+                    asm!(
+                        concat!("{out} = *(", $size, " *) ({src} + 0)"), // atomic { out = *src }
+                        src = in(reg) src,
+                        out = lateout(reg) out,
+                        options(nostack, preserves_flags),
+                    );
+                }
+                out
+            }
+        }
+        impl AtomicStore for $ty {
+            #[inline]
+            unsafe fn atomic_store(
+                dst: *mut MaybeUninit<Self>,
+                val: MaybeUninit<Self>,
+                _order: Ordering,
+            ) {
+                debug_assert!(dst as usize % mem::size_of::<$ty>() == 0);
+
+                // SAFETY: the caller must uphold the safety contract.
+                unsafe {
+                    asm!(
+                        concat!("*(", $size, " *) ({dst} + 0) = {val}"), // atomic { *dst = val }
+                        dst = in(reg) dst,
+                        val = in(reg) val,
+                        options(nostack, preserves_flags),
+                    );
+                }
+            }
+        }
+    };
+}
+
+macro_rules! atomic64 {
+    ($ty:ident) => {
+        atomic_load_store!($ty, "u64");
+        impl AtomicSwap for $ty {
+            #[inline]
+            unsafe fn atomic_swap(
+                dst: *mut MaybeUninit<Self>,
+                val: MaybeUninit<Self>,
+                _order: Ordering,
+            ) -> MaybeUninit<Self> {
+                debug_assert!(dst as usize % mem::size_of::<$ty>() == 0);
+                let mut out: MaybeUninit<Self>;
+
+                // SAFETY: the caller must uphold the safety contract.
+                unsafe {
+                    asm!(
+                        "{out} = xchg_64({dst} + 0, {val})", // atomic { _x = *dst; *dst = val; out = _x }
+                        dst = in(reg) dst,
+                        val = in(reg) val,
+                        out = lateout(reg) out,
+                        options(nostack, preserves_flags),
+                    );
+                }
+                out
+            }
+        }
+        impl AtomicCompareExchange for $ty {
+            #[inline]
+            unsafe fn atomic_compare_exchange(
+                dst: *mut MaybeUninit<Self>,
+                old: MaybeUninit<Self>,
+                new: MaybeUninit<Self>,
+                _success: Ordering,
+                _failure: Ordering,
+            ) -> (MaybeUninit<Self>, bool) {
+                debug_assert!(dst as usize % mem::size_of::<$ty>() == 0);
+                let mut out: MaybeUninit<Self>;
+
+                // SAFETY: the caller must uphold the safety contract.
+                unsafe {
+                    let mut r: u64 = 1;
+                    asm!(
+                        "r0 = cmpxchg_64({dst} + 0, r0, {new})", // atomic { _x = *dst; if _x == r0 { *dst = new }; r0 = _x }
+                        "if r0 == {old} goto 2f",                // if r0 == old { jump 'success }
+                        "{r} = 0",                               // r = 0
+                        "2:", // 'success
+                        dst = in(reg) dst,
+                        old = in(reg) old,
+                        new = in(reg) new,
+                        r = inout(reg) r,
+                        inout("r0") old => out,
+                        options(nostack, preserves_flags),
+                    );
+                    crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
+                    (out, r != 0)
+                }
+            }
+        }
+    };
+}
+
+atomic_load_store!(i8, "u8");
+atomic_load_store!(u8, "u8");
+atomic_load_store!(i16, "u16");
+atomic_load_store!(u16, "u16");
+atomic_load_store!(i32, "u32");
+atomic_load_store!(u32, "u32");
+atomic64!(i64);
+atomic64!(u64);
+atomic64!(isize);
+atomic64!(usize);

src/arch/cfgs/bpf.rs

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+#![allow(missing_docs)]
+
+// TODO: RMW on {8,16,32}-bit
+#[macro_export]
+macro_rules! cfg_has_atomic_8 {
+    ($($tt:tt)*) => {};
+}
+#[macro_export]
+macro_rules! cfg_no_atomic_8 {
+    ($($tt:tt)*) => { $($tt)* };
+}
+#[macro_export]
+macro_rules! cfg_has_atomic_16 {
+    ($($tt:tt)*) => {};
+}
+#[macro_export]
+macro_rules! cfg_no_atomic_16 {
+    ($($tt:tt)*) => { $($tt)* };
+}
+#[macro_export]
+macro_rules! cfg_has_atomic_32 {
+    ($($tt:tt)*) => {};
+}
+#[macro_export]
+macro_rules! cfg_no_atomic_32 {
+    ($($tt:tt)*) => { $($tt)* };
+}
+#[macro_export]
+macro_rules! cfg_has_atomic_64 {
+    ($($tt:tt)*) => { $($tt)* };
+}
+#[macro_export]
+macro_rules! cfg_no_atomic_64 {
+    ($($tt:tt)*) => {};
+}
+#[macro_export]
+macro_rules! cfg_has_atomic_128 {
+    ($($tt:tt)*) => {};
+}
+#[macro_export]
+macro_rules! cfg_no_atomic_128 {
+    ($($tt:tt)*) => { $($tt)* };
+}
+#[macro_export]
+macro_rules! cfg_has_atomic_cas {
+    ($($tt:tt)*) => { $($tt)* };
+}
+#[macro_export]
+macro_rules! cfg_no_atomic_cas {
+    ($($tt:tt)*) => {};
+}

src/arch/mod.rs

@@ -29,6 +29,7 @@
     all(
         any(
             target_arch = "avr",
+            target_arch = "bpf",
             target_arch = "hexagon",
             target_arch = "m68k",
             target_arch = "mips",
@@ -86,6 +87,9 @@ mod armv8;
 #[cfg(target_arch = "avr")]
 #[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)]
 mod avr;
+#[cfg(target_arch = "bpf")]
+#[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)]
+mod bpf;
 #[cfg(target_arch = "hexagon")]
 #[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)]
 mod hexagon;

src/lib.rs

@@ -12,7 +12,7 @@ This crate provides a way to soundly perform such operations.
 
 ## Platform Support
 
-Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, and Xtensa are supported.
+Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, Xtensa, and BPF are supported.
 
 | target_arch                     | primitives                                          | load/store | swap/CAS |
 | ------------------------------- | --------------------------------------------------- |:----------:|:--------:|
@@ -39,6 +39,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS,
 | hexagon \[8] (experimental)     | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64           | ✓          | ✓        |
 | m68k \[8] (experimental)        | isize,usize,i8,u8,i16,u16,i32,u32                   | ✓          | ✓\[1]    |
 | xtensa \[8] (experimental)      | isize,usize,i8,u8,i16,u16,i32,u32                   | ✓          | ✓\[1]    |
+| bpf \[8] (experimental)         | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64           | ✓          | ✓\[9]    |
 
 \[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.<br>
 \[2] Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets).<br>
@@ -48,6 +49,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS,
 \[6] Requires Rust 1.72+.<br>
 \[7] Requires Rust 1.84+.<br>
 \[8] Requires nightly due to `#![feature(asm_experimental_arch)]`.<br>
+\[9] BPF's atomic RMW operations are only available for isize/usize/i64/u64.<br>
 
 See also [Atomic operation overview by architecture](https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md) for more information about atomic operations in these architectures.
 

src/utils.rs

@@ -241,6 +241,9 @@ pub(crate) struct Pair<T: Copy> {
     pub(crate) lo: MaybeUninit<T>,
 }
 
+#[cfg(target_arch = "bpf")]
+type MinWord = u64;
+#[cfg(not(target_arch = "bpf"))]
 type MinWord = u32;
 #[cfg(target_arch = "s390x")]
 type RetInt = u32;

tools/build.sh

@@ -135,6 +135,11 @@ default_targets=(
     # hexagon
     # rustc --print target-list | grep -E '^hexagon'
     hexagon-unknown-linux-musl
+
+    # bpf
+    # rustc --print target-list | grep -E '^bpf'
+    bpfeb-unknown-none
+    bpfel-unknown-none
 )
 
 x() {