From eaf5d3c8c67086a2cfa309df0c00254320496b38 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Sat, 23 Nov 2024 21:40:33 +0900 Subject: [PATCH] Support C-SKY (experimental) --- .github/.cspell/project-dictionary.txt | 3 + .github/workflows/ci.yml | 3 + README.md | 1 + build.rs | 4 +- src/arch/README.md | 2 +- src/arch/cfgs/csky.rs | 52 +++++ src/arch/csky.rs | 301 +++++++++++++++++++++++++ src/arch/mod.rs | 4 + src/lib.rs | 1 + tools/build.sh | 4 + 10 files changed, 372 insertions(+), 3 deletions(-) create mode 100644 src/arch/cfgs/csky.rs create mode 100644 src/arch/csky.rs diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index 6dcccabc..89d12272 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -16,6 +16,7 @@ cdsg CDSY cinc clrex +cmpne cmpw cmpxchg cset @@ -41,6 +42,7 @@ ldar ldarx ldaxp ldclrp +ldex ldiapp ldrd ldrex @@ -103,6 +105,7 @@ sreg srlv stbar stdcx +stex stilp stlxp stpq diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d154226..f9ecad48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,6 +138,9 @@ jobs: target: armeb-unknown-linux-gnueabi - rust: nightly target: armeb-unknown-linux-gnueabi + # TODO + # - rust: nightly + # target: csky-unknown-linux-gnuabiv2 - rust: '1.59' # LLVM 13 target: i586-unknown-linux-gnu - rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported) diff --git a/README.md b/README.md index a587f4ac..f8e4faa9 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, | hexagon \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ | | m68k \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | | xtensa \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | +| csky \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ | \[1] Arm's atomic RMW operations are not available on v6-m (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.
\[2] Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets).
diff --git a/build.rs b/build.rs index d286258c..2365ac42 100644 --- a/build.rs +++ b/build.rs @@ -106,8 +106,8 @@ fn main() { } } } - "avr" | "hexagon" | "m68k" | "mips" | "mips32r6" | "mips64" | "mips64r6" | "msp430" - | "powerpc" | "powerpc64" | "xtensa" => { + "avr" | "csky" | "hexagon" | "m68k" | "mips" | "mips32r6" | "mips64" | "mips64r6" + | "msp430" | "powerpc" | "powerpc64" | "xtensa" => { if version.nightly && is_allowed_feature("asm_experimental_arch") { println!("cargo:rustc-cfg=atomic_maybe_uninit_unstable_asm_experimental_arch"); } diff --git a/src/arch/README.md b/src/arch/README.md index b3a8f88b..72bed1ee 100644 --- a/src/arch/README.md +++ b/src/arch/README.md @@ -13,7 +13,7 @@ This document describes the operations that are considered atomic by architectur - [s390x](#s390x) - [SPARC](#sparc) -TODO: write sections for AArch64, Arm, Hexagon, LoongArch, MIPS, x86, Xtensa +TODO: write sections for AArch64, Arm, C-SKY, Hexagon, LoongArch, MIPS, x86, Xtensa ## AVR diff --git a/src/arch/cfgs/csky.rs b/src/arch/cfgs/csky.rs new file mode 100644 index 00000000..bc666998 --- /dev/null +++ b/src/arch/cfgs/csky.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(missing_docs)] + +#[macro_export] +macro_rules! cfg_has_atomic_8 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_8 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_has_atomic_16 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_16 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_has_atomic_32 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_32 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_has_atomic_64 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_no_atomic_64 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_has_atomic_128 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_no_atomic_128 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_has_atomic_cas { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_cas { + ($($tt:tt)*) => {}; +} diff --git a/src/arch/csky.rs b/src/arch/csky.rs new file mode 100644 index 00000000..2da89413 --- /dev/null +++ b/src/arch/csky.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/* +C-SKY + +Refs: +- CSKY Architecture user_guide + https://github.com/c-sky/csky-doc/blob/9f7121f7d40970ba5cc0f15716da033db2bb9d07/CSKY%20Architecture%20user_guide.pdf +- Linux kernel's C-SKY atomic implementation + https://github.com/torvalds/linux/blob/v6.11/arch/csky/include/asm/atomic.h + https://github.com/torvalds/linux/blob/v6.11/arch/csky/include/asm/barrier.h + https://github.com/torvalds/linux/blob/v6.11/arch/csky/include/asm/cmpxchg.h + +Generated asm: +- csky https://godbolt.org/z/jK4c68WeG +*/ + +#[path = "cfgs/csky.rs"] +mod cfgs; + +use core::{arch::asm, mem::MaybeUninit, sync::atomic::Ordering}; + +use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap}; + +macro_rules! atomic_rmw { + ($op:ident, $order:ident) => { + match $order { + Ordering::Relaxed => $op!("", ""), + Ordering::Acquire => $op!("sync32", ""), + Ordering::Release => $op!("", "sync32"), + Ordering::AcqRel | Ordering::SeqCst => $op!("sync32", "sync32"), + _ => unreachable!(), + } + }; +} + +#[rustfmt::skip] +macro_rules! atomic_load_store { + ($int_type:ident, $suffix:tt) => { + impl AtomicLoad for $int_type { + #[inline] + unsafe fn atomic_load( + src: *const MaybeUninit, + order: Ordering, + ) -> MaybeUninit { + let out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! atomic_load { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + concat!("ld32.", $suffix, " {out}, ({src}, 0x0)"), // atomic { out = *src } + $acquire, // fence + src = in(reg) ptr_reg!(src), + out = lateout(reg) out, + options(nostack, preserves_flags), + ) + }; + } + match order { + Ordering::Relaxed => atomic_load!("", ""), + Ordering::Acquire => atomic_load!("sync32", ""), + Ordering::SeqCst => atomic_load!("sync32", "sync32"), + _ => unreachable!(), + } + } + out + } + } + impl AtomicStore for $int_type { + #[inline] + unsafe fn atomic_store( + dst: *mut MaybeUninit, + val: MaybeUninit, + order: Ordering, + ) { + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! store { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + concat!("st32.", $suffix, " {val}, ({dst}, 0x0)"), // atomic { *dst = val } + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = in(reg) val, + options(nostack, preserves_flags), + ) + }; + } + atomic_rmw!(store, order); + } + } + } + }; +} + +#[rustfmt::skip] +macro_rules! atomic { + ($int_type:ident) => { + atomic_load_store!($int_type, "w"); + impl AtomicSwap for $int_type { + #[inline] + unsafe fn atomic_swap( + dst: *mut MaybeUninit, + val: MaybeUninit, + order: Ordering, + ) -> MaybeUninit { + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! swap { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + "2:", // 'retry: + "ldex32.w {out}, ({dst}, 0)", // atomic { out = *dst; EXCLUSIVE = dst } + "or32 {tmp}, {val}, {val}", // tmp = val + "stex32.w {tmp}, ({dst}, 0)", // atomic { if EXCLUSIVE == dst { *dst = tmp; tmp = 1 } else { tmp = 0 }; EXCLUSIVE = None } + "bez32 {tmp}, 2b", // if tmp == 0 { jump 'retry } + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = in(reg) val, + out = out(reg) out, + tmp = out(reg) _, + options(nostack, preserves_flags), + ) + }; + } + atomic_rmw!(swap, order); + } + out + } + } + impl AtomicCompareExchange for $int_type { + #[inline(never)] // TODO: there is no way to mark condition/carry bit (C) as clobbered + unsafe fn atomic_compare_exchange( + dst: *mut MaybeUninit, + old: MaybeUninit, + new: MaybeUninit, + success: Ordering, + failure: Ordering, + ) -> (MaybeUninit, bool) { + let order = crate::utils::upgrade_success_ordering(success, failure); + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + let mut r: u32; + macro_rules! cmpxchg { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + "2:", // 'retry: + "ldex32.w {out}, ({dst}, 0)", // atomic { out = *dst; EXCLUSIVE = dst } + "cmpne32 {out}, {old}", // if out != old { C = 1 } else { C = 0 } + "bt32 3f", // if C == 1 { jump 'cmp-fail } + "or32 {tmp}, {new}, {new}", // tmp = new + "stex32.w {tmp}, ({dst}, 0)", // atomic { if EXCLUSIVE == dst { *dst = tmp; tmp = 1 } else { tmp = 0 }; EXCLUSIVE = None } + "bez32 {tmp}, 2b", // if tmp == 0 { jump 'retry } + "br32 4f", // jump 'success + "3:", // 'cmp-fail: + "movi32 {tmp}, 0", // tmp = 0 + "4:", // 'success: + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + old = in(reg) old, + new = in(reg) new, + out = out(reg) out, + tmp = out(reg) r, + // Do not use `preserves_flags` because CMPNE modifies condition bit C. + options(nostack), + ) + }; + } + atomic_rmw!(cmpxchg, order); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test + (out, r != 0) + } + } + } + }; +} + +#[rustfmt::skip] +macro_rules! atomic_sub_word { + ($int_type:ident, $suffix:tt) => { + atomic_load_store!($int_type, $suffix); + impl AtomicSwap for $int_type { + #[inline] + unsafe fn atomic_swap( + dst: *mut MaybeUninit, + val: MaybeUninit, + order: Ordering, + ) -> MaybeUninit { + let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst); + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! swap { + ($acquire:tt, $release:tt) => { + // Implement sub-word atomic operations using word-sized LL/SC loop. + // See also create_sub_word_mask_values. + asm!( + "lsl32 {val}, {val}, {shift}", // val <<= shift + $release, // fence + "2:", // 'retry: + "ldex32.w {out}, ({dst}, 0)", // atomic { out = *dst; EXCLUSIVE = dst } + "andn32 {tmp}, {out}, {mask}", // tmp = out & !mask + "or32 {tmp}, {tmp}, {val}", // tmp |= val + "stex32.w {tmp}, ({dst}, 0)", // atomic { if EXCLUSIVE == dst { *dst = tmp; tmp = 1 } else { tmp = 0 }; EXCLUSIVE = None } + "bez32 {tmp}, 2b", // if tmp == 0 { jump 'retry } + "lsr32 {out}, {out}, {shift}", // out >>= shift + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = inout(reg) crate::utils::ZeroExtend::zero_extend(val) => _, + out = out(reg) out, + shift = in(reg) shift, + mask = in(reg) mask, + tmp = out(reg) _, + options(nostack, preserves_flags), + ) + }; + } + atomic_rmw!(swap, order); + } + out + } + } + impl AtomicCompareExchange for $int_type { + #[inline(never)] // TODO: there is no way to mark condition/carry bit (C) as clobbered + unsafe fn atomic_compare_exchange( + dst: *mut MaybeUninit, + old: MaybeUninit, + new: MaybeUninit, + success: Ordering, + failure: Ordering, + ) -> (MaybeUninit, bool) { + let order = crate::utils::upgrade_success_ordering(success, failure); + let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst); + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + let mut r: u32; + macro_rules! cmpxchg { + ($acquire:tt, $release:tt) => { + // Implement sub-word atomic operations using word-sized LL/SC loop. + // See also create_sub_word_mask_values. + asm!( + "lsl32 {old}, {old}, {shift}", // old <<= shift + "lsl32 {new}, {new}, {shift}", // new <<= shift + $release, // fence + "2:", // 'retry: + "ldex32.w {tmp}, ({dst}, 0)", // atomic { tmp = *dst; EXCLUSIVE = dst } + "and32 {out}, {tmp}, {mask}", // out = tmp & mask + "cmpne32 {out}, {old}", // if out != old { C = 1 } else { C = 0 } + "bt32 3f", // if C == 1 { jump 'cmp-fail } + "andn32 {tmp}, {tmp}, {mask}", // tmp &= !mask + "or32 {tmp}, {tmp}, {new}", // tmp |= new + "stex32.w {tmp}, ({dst}, 0)", // atomic { if EXCLUSIVE == dst { *dst = tmp; tmp = 1 } else { tmp = 0 }; EXCLUSIVE = None } + "bez32 {tmp}, 2b", // if tmp == 0 { jump 'retry } + "br32 4f", // jump 'success + "3:", // 'cmp-fail: + "movi32 {tmp}, 0", // tmp = 0 + "4:", // 'success: + "lsr32 {out}, {out}, {shift}", // out >>= shift + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + old = inout(reg) crate::utils::ZeroExtend::zero_extend(old) => _, + new = inout(reg) crate::utils::ZeroExtend::zero_extend(new) => _, + out = out(reg) out, + shift = in(reg) shift, + mask = in(reg) mask, + tmp = out(reg) r, + // Do not use `preserves_flags` because CMPNE modifies condition bit C. + options(nostack), + ) + }; + } + atomic_rmw!(cmpxchg, order); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test + (out, r != 0) + } + } + } + }; +} + +atomic_sub_word!(i8, "b"); +atomic_sub_word!(u8, "b"); +atomic_sub_word!(i16, "h"); +atomic_sub_word!(u16, "h"); +atomic!(i32); +atomic!(u32); +atomic!(isize); +atomic!(usize); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 7ad4b261..522acf09 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -29,6 +29,7 @@ all( any( target_arch = "avr", + target_arch = "csky", target_arch = "hexagon", target_arch = "m68k", target_arch = "mips", @@ -86,6 +87,9 @@ mod armv8; #[cfg(target_arch = "avr")] #[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] mod avr; +#[cfg(target_arch = "csky")] +#[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] +mod csky; #[cfg(target_arch = "hexagon")] #[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] mod hexagon; diff --git a/src/lib.rs b/src/lib.rs index 5c20ec48..0f8f3089 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, | hexagon \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ | | m68k \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | | xtensa \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | +| csky \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ | \[1] Arm's atomic RMW operations are not available on v6-m (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.
\[2] Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets).
diff --git a/tools/build.sh b/tools/build.sh index f61b9c5a..ef2c86bb 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -135,6 +135,10 @@ default_targets=( # hexagon # rustc --print target-list | grep -E '^hexagon' hexagon-unknown-linux-musl + + # csky + # rustc --print target-list | grep -E '^csky' + csky-unknown-linux-gnuabiv2 ) x() {