diff --git a/src/lib.rs b/src/lib.rs index 9e780ac..c9b8275 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ #![feature(inline_const)] use std::{ marker::PhantomData, - mem::offset_of, + mem::{offset_of, MaybeUninit}, ops::{Deref, DerefMut}, pin::Pin, ptr::{self, NonNull}, @@ -21,8 +21,8 @@ struct Inner { /// Intrusive link. struct ListHead { - prev: Option>>, - next: Option>>, + prev: MaybeUninit>>, + next: MaybeUninit>>, dtype: PhantomData, } @@ -32,12 +32,14 @@ impl LinkNode { let mut node = Self(Box::pin(Inner { data, list: ListHead { - prev: None, - next: None, + prev: MaybeUninit::uninit(), + next: MaybeUninit::uninit(), dtype: PhantomData, }, })); - node.list_mut().init_head(); + unsafe { + node.list_mut().init_head(); + } node } @@ -61,8 +63,10 @@ impl LinkNode { #[inline] pub fn take(&mut self) { let list = self.list_mut(); - unsafe { list.delist() }; - list.init_head(); + unsafe { + list.delist(); + list.init_head(); + } } pub fn for_each(&self, f: F) @@ -126,12 +130,17 @@ impl Drop for LinkNode { } impl ListHead { + #[inline(always)] + unsafe fn ptr(&mut self) -> NonNull> { + NonNull::new_unchecked(self as *mut ListHead) + } + /// Called when node is created. #[inline(always)] - fn init_head(&mut self) { - let list = NonNull::new(self as *mut ListHead); - self.prev = list; - self.next = list; + unsafe fn init_head(&mut self) { + let self_ptr = self.ptr(); + self.prev.write(self_ptr); + self.next.write(self_ptr); } /// Unlink the current node from its previous list. @@ -144,10 +153,10 @@ impl ListHead { /// or in `add()` to update to `prev` and `next` pointers. #[inline(always)] unsafe fn delist(&mut self) { - let mut prev = self.prev.unwrap(); - let mut next = self.next.unwrap(); - prev.as_mut().next = Some(next); - next.as_mut().prev = Some(prev); + let mut prev = self.prev.assume_init(); + let mut next = self.next.assume_init(); + prev.as_mut().next.write(next); + next.as_mut().prev.write(prev); } /// Add `other` between `self` and `self.next`. @@ -159,10 +168,12 @@ impl ListHead { /// is still complete. #[inline(always)] unsafe fn add(&mut self, other: &mut ListHead) { - other.prev = NonNull::new(self as *mut ListHead); + other.prev.write(self.ptr()); other.next = self.next; - self.next.unwrap().as_mut().prev = NonNull::new(other as *mut ListHead); - self.next = NonNull::new(other as *mut ListHead); + + let next = self.next.assume_init_mut().as_mut(); + next.prev.write(other.ptr()); + self.next.write(other.ptr()); } #[inline(always)] @@ -170,11 +181,12 @@ impl ListHead { where F: FnMut(&T), { + let self_ptr = self as *const Self; let mut this = self; loop { f(this.get()); - let next = this.next.unwrap(); - if ptr::eq(next.as_ptr(), self) { + let next = unsafe { this.next.assume_init_ref() }; + if ptr::eq(next.as_ptr(), self_ptr) { break; } this = unsafe { next.as_ref() }; @@ -186,11 +198,12 @@ impl ListHead { where F: FnMut(&mut T), { + let self_ptr = self as *const Self; let mut this = &mut *self; loop { f(this.get_mut()); - let mut next = this.next.unwrap(); - if ptr::eq(next.as_ptr(), self) { + let next = unsafe { this.next.assume_init_mut() }; + if ptr::eq(next.as_ptr(), self_ptr) { break; } this = unsafe { next.as_mut() }; @@ -202,11 +215,12 @@ impl ListHead { where F: FnMut(&T), { + let self_ptr = self as *const Self; let mut this = self; loop { f(this.get()); - let prev = this.prev.unwrap(); - if ptr::eq(prev.as_ptr(), self) { + let prev = unsafe { this.prev.assume_init_ref() }; + if ptr::eq(prev.as_ptr(), self_ptr) { break; } this = unsafe { prev.as_ref() }; @@ -218,11 +232,12 @@ impl ListHead { where F: FnMut(&mut T), { + let self_ptr = self as *const Self; let mut this = &mut *self; loop { f(this.get_mut()); - let mut prev = this.prev.unwrap(); - if ptr::eq(prev.as_ptr(), self) { + let prev = unsafe { this.prev.assume_init_mut() }; + if ptr::eq(prev.as_ptr(), self_ptr) { break; } this = unsafe { prev.as_mut() };