Pager integration (#249)

* Working on object store and pager.

* Working on pager integration.

* Fixed stall.

* Paging working.

* Paging working.

* Paging working.

* Fix merge.

src/bin/init/src/main.rs

@@ -30,14 +30,6 @@ fn initialize_pager() {
     .unwrap();
 
     pager::pager_start(queue.object().id(), queue2.object().id());
-
-    tracing::info!("sync call test");
-    twizzler_abi::syscall::sys_object_ctrl(
-        queue.object().id(),
-        twizzler_abi::syscall::ObjectControlCmd::Sync,
-    )
-    .unwrap();
-    tracing::info!("sync call done!");
 }
 
 fn main() {
@@ -223,21 +215,11 @@ use twizzler_abi::{
     object::{ObjID, Protections, MAX_SIZE, NULLPAGE_SIZE},
     pager::{CompletionToKernel, RequestFromKernel},
     syscall::{
-        sys_kaction,
-        sys_new_handle,
-        sys_thread_sync,
-        BackingType,
-        LifetimeType, //MapFlags,
-        NewHandleFlags,
-        ObjectCreate,
-        ObjectCreateFlags,
-        ThreadSync,
-        ThreadSyncFlags,
-        ThreadSyncOp,
-        ThreadSyncReference,
-        ThreadSyncSleep,
-        ThreadSyncWake,
+        sys_kaction, sys_new_handle, sys_object_create, sys_thread_sync, BackingType, LifetimeType,
+        NewHandleFlags, ObjectControlCmd, ObjectCreate, ObjectCreateFlags, ThreadSync,
+        ThreadSyncFlags, ThreadSyncOp, ThreadSyncReference, ThreadSyncSleep, ThreadSyncWake,
     },
     thread::{ExecutionState, ThreadRepr},
 };
 use twizzler_object::{CreateSpec, Object, ObjectInitFlags};
+use twizzler_rt_abi::object::{MapFlags, ObjectHandle};

src/bin/object-store-test/src/main.rs

@@ -78,7 +78,7 @@ fn test_khf_serde() {
     write_all(id, b"asdf", 0).unwrap();
     advance_epoch().unwrap();
     const ROOT_KEY: [u8; 32] = [0; 32];
-    let fs = FS.lock().unwrap();
+    let fs = FS.get().unwrap().lock().unwrap();
     let mut khf = KHF.lock().unwrap();
     *khf = MyKhf::load(ROOT_KEY, "lethe/khf", &fs).unwrap();
     drop(khf);
@@ -93,7 +93,8 @@ fn it_works() {
     // println!("{:?}", KHF.lock().unwrap());
     let out = (0..5).map(|_i| make_and_check_file(&mut working_bufs.0, &mut working_bufs.1));
     advance_epoch().unwrap();
-    *(KHF.lock().unwrap()) = MyKhf::load(ROOT_KEY, "lethe/khf", &FS.lock().unwrap()).unwrap();
+    *(KHF.lock().unwrap()) =
+        MyKhf::load(ROOT_KEY, "lethe/khf", &FS.get().unwrap().lock().unwrap()).unwrap();
     // println!("{:?}", KHF.lock().unwrap());
     for (value, id) in out {
         // make sure buf == read
@@ -103,7 +104,8 @@ fn it_works() {
         // unlink
         unlink_object(id).unwrap();
         advance_epoch().unwrap();
-        *(KHF.lock().unwrap()) = MyKhf::load(ROOT_KEY, "lethe/khf", &FS.lock().unwrap()).unwrap();
+        *(KHF.lock().unwrap()) =
+            MyKhf::load(ROOT_KEY, "lethe/khf", &FS.get().unwrap().lock().unwrap()).unwrap();
         // println!("{:?}", KHF.lock().unwrap());
         // make sure object is unlinked
         let v = read_exact(id, &mut buf, 0).expect_err("should be error");

src/kernel/src/memory/context/virtmem.rs

@@ -30,7 +30,7 @@ use crate::{
         PhysAddr,
     },
     mutex::Mutex,
-    obj::{self, pages::Page, ObjectRef, PageNumber},
+    obj::{self, pages::Page, range::PageStatus, ObjectRef, PageNumber},
     security::KERNEL_SCTX,
     spinlock::Spinlock,
     thread::{current_memory_context, current_thread_ref},
@@ -696,7 +696,9 @@ pub fn page_fault(addr: VirtAddr, cause: MemoryAccessKind, flags: PageFaultFlags
 
         let page_number = PageNumber::from_address(addr);
         let slot_mgr = ctx.slots.lock();
-        if let Some(info) = slot_mgr.get(&slot) {
+        let info = slot_mgr.get(&slot).cloned();
+        drop(slot_mgr);
+        if let Some(info) = info {
             let id = info.obj.id();
             let null_upcall = UpcallInfo::ObjectMemoryFault(ObjectMemoryFaultInfo::new(
                 id,
@@ -712,23 +714,25 @@ pub fn page_fault(addr: VirtAddr, cause: MemoryAccessKind, flags: PageFaultFlags
                 addr.into(),
             ));
 
+            if info.obj.use_pager() {
+                crate::pager::get_object_page(&info.obj, page_number);
+            }
+
             let mut obj_page_tree = info.obj.lock_page_tree();
             if page_number.is_zero() {
                 // drop these mutexes in case upcall sending generetes a page fault.
                 drop(obj_page_tree);
-                drop(slot_mgr);
                 current_thread_ref().unwrap().send_upcall(null_upcall);
                 return;
             }
             if page_number.as_byte_offset() >= MAX_SIZE {
                 // drop these mutexes in case upcall sending generetes a page fault.
                 drop(obj_page_tree);
-                drop(slot_mgr);
                 current_thread_ref().unwrap().send_upcall(oob_upcall);
                 return;
             }
 
-            if let Some((page, cow)) =
+            if let PageStatus::Ready(page, cow) =
                 obj_page_tree.get_page(page_number, cause == MemoryAccessKind::Write)
             {
                 ctx.with_arch(sctx_id, |arch| {
@@ -749,9 +753,11 @@ pub fn page_fault(addr: VirtAddr, cause: MemoryAccessKind, flags: PageFaultFlags
             } else {
                 let page = Page::new();
                 obj_page_tree.add_page(page_number, page);
-                let (page, cow) = obj_page_tree
-                    .get_page(page_number, cause == MemoryAccessKind::Write)
-                    .unwrap();
+                let PageStatus::Ready(page, cow) =
+                    obj_page_tree.get_page(page_number, cause == MemoryAccessKind::Write)
+                else {
+                    panic!("unreachable");
+                };
                 ctx.with_arch(sctx_id, |arch| {
                     // TODO: don't need all three every time.
                     arch.unmap(
@@ -769,7 +775,6 @@ pub fn page_fault(addr: VirtAddr, cause: MemoryAccessKind, flags: PageFaultFlags
                 });
             }
         } else {
-            drop(slot_mgr);
             current_thread_ref()
                 .unwrap()
                 .send_upcall(UpcallInfo::MemoryContextViolation(

src/kernel/src/obj/copy.rs

@@ -1,6 +1,6 @@
 use super::{
     pages::Page,
-    range::{PageRange, PageRangeTree},
+    range::{PageRange, PageRangeTree, PageStatus},
     InvalidateMode, ObjectRef, PageNumber,
 };
 use crate::mutex::LockGuard;
@@ -73,7 +73,7 @@ fn copy_single(
 ) {
     let src_page = src_tree.get_page(src_point, false);
     let (dest_page, _) = dest_tree.get_or_add_page(dest_point, true, |_, _| Page::new());
-    if let Some((src_page, _)) = src_page {
+    if let PageStatus::Ready(src_page, _) = src_page {
         dest_page.as_mut_slice()[offset..max].copy_from_slice(&src_page.as_slice()[offset..max]);
     } else {
         // TODO: could skip this on freshly created page, if we can detect that. That's just an
@@ -90,7 +90,7 @@ fn zero_single(
     max: usize,
 ) {
     // if there's no page here, our work is done
-    if let Some((dest_page, _)) = dest_tree.get_page(dest_point, true) {
+    if let PageStatus::Ready(dest_page, _) = dest_tree.get_page(dest_point, true) {
         dest_page.as_mut_slice()[offset..max].fill(0);
     }
 }
@@ -138,6 +138,7 @@ pub fn copy_ranges(
             (0, _) | (_, 0) => 1,
             (_, _) => 2,
         };
+    crate::pager::ensure_in_core(src, src_start, nr_pages);
     // Step 1: lock the page trees for the objects, in a canonical order.
     let (mut src_tree, mut dest_tree) = crate::utils::lock_two(&src.range_tree, &dest.range_tree);
 
@@ -267,7 +268,7 @@ fn copy_bytes(
         let this_src_offset = (src_off + count_sofar) % PageNumber::PAGE_SIZE;
         let this_dest_offset = (dest_off + count_sofar) % PageNumber::PAGE_SIZE;
 
-        let this_length = if let Some((src_page, _)) = src_page {
+        let this_length = if let PageStatus::Ready(src_page, _) = src_page {
             let this_length = core::cmp::min(
                 core::cmp::min(
                     PageNumber::PAGE_SIZE - this_src_offset,

src/kernel/src/obj/mod.rs

@@ -1,5 +1,5 @@
 use alloc::{
-    collections::{btree_map::Entry, BTreeMap},
+    collections::{btree_map::Entry, btree_set::BTreeSet, BTreeMap},
     sync::{Arc, Weak},
     vec::Vec,
 };
@@ -8,9 +8,11 @@ use core::{
     sync::atomic::{AtomicU32, Ordering},
 };
 
+use range::PageStatus;
 use twizzler_abi::{
     meta::MetaFlags,
     object::{ObjID, MAX_SIZE},
+    syscall::LifetimeType,
 };
 
 use self::{pages::Page, thread_sync::SleepInfo};
@@ -39,6 +41,7 @@ pub struct Object {
     sleep_info: Mutex<SleepInfo>,
     pin_info: Mutex<PinInfo>,
     contexts: Mutex<ContextInfo>,
+    lifetime_type: LifetimeType,
 }
 
 #[derive(Default)]
@@ -179,6 +182,10 @@ impl Object {
         self.flags.load(Ordering::SeqCst) & OBJ_DELETED != 0
     }
 
+    pub fn use_pager(&self) -> bool {
+        self.lifetime_type == LifetimeType::Persistent
+    }
+
     pub fn mark_for_delete(&self) {
         self.flags.fetch_or(OBJ_DELETED, Ordering::SeqCst);
     }
@@ -202,6 +209,7 @@ impl Object {
     }
 
     pub fn pin(&self, start: PageNumber, len: usize) -> Option<(Vec<PhysAddr>, u32)> {
+        assert!(!self.use_pager());
         let mut tree = self.lock_page_tree();
 
         let mut pin_info = self.pin_info.lock();
@@ -210,8 +218,8 @@ impl Object {
         for i in 0..len {
             // TODO: we'll need to handle failures here when we expand the paging system.
             let p = tree.get_page(start.offset(i), true);
-            if let Some(p) = p {
-                v.push(p.0.physical_address());
+            if let PageStatus::Ready(p, _) = p {
+                v.push(p.physical_address());
             } else {
                 let page = Page::new();
                 v.push(page.physical_address());
@@ -226,19 +234,23 @@ impl Object {
         Some((v, token))
     }
 
-    pub fn new(id: ObjID) -> Self {
+    pub fn new(id: ObjID, lifetime_type: LifetimeType) -> Self {
         Self {
             id,
             flags: AtomicU32::new(0),
             range_tree: Mutex::new(range::PageRangeTree::new()),
             sleep_info: Mutex::new(SleepInfo::new()),
             pin_info: Mutex::new(PinInfo::default()),
             contexts: Mutex::new(ContextInfo::default()),
+            lifetime_type,
         }
     }
 
     pub fn new_kernel() -> Self {
-        Self::new(calculate_new_id(0.into(), MetaFlags::default()))
+        Self::new(
+            calculate_new_id(0.into(), MetaFlags::default()),
+            LifetimeType::Volatile,
+        )
     }
 
     pub fn add_context(&self, ctx: &ContextRef) {
@@ -313,6 +325,7 @@ pub type ObjectRef = Arc<Object>;
 
 struct ObjectManager {
     map: Mutex<BTreeMap<ObjID, ObjectRef>>,
+    no_exist: Mutex<BTreeSet<ObjID>>,
 }
 
 bitflags::bitflags! {
@@ -351,10 +364,14 @@ impl ObjectManager {
     fn new() -> Self {
         Self {
             map: Mutex::new(BTreeMap::new()),
+            no_exist: Mutex::new(BTreeSet::new()),
         }
     }
 
     fn lookup_object(&self, id: ObjID, flags: LookupFlags) -> LookupResult {
+        if self.no_exist.lock().contains(&id) {
+            return LookupResult::WasDeleted;
+        }
         self.map
             .lock()
             .get(&id)
@@ -386,3 +403,7 @@ pub fn register_object(obj: Arc<Object>) {
     let om = &OBJ_MANAGER;
     om.register_object(obj);
 }
+
+pub fn no_exist(id: ObjID) {
+    OBJ_MANAGER.no_exist.lock().insert(id);
+}

src/kernel/src/obj/pages.rs

@@ -3,7 +3,7 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering};
 
 use twizzler_abi::device::{CacheType, MMIO_OFFSET};
 
-use super::{Object, PageNumber};
+use super::{range::PageStatus, Object, PageNumber};
 use crate::{
     arch::memory::{frame::FRAME_SIZE, phys_to_virt},
     memory::{
@@ -119,12 +119,13 @@ impl Page {
 
 impl Object {
     pub unsafe fn write_val_and_signal<T>(&self, offset: usize, val: T, wakeup_count: usize) {
+        assert!(!self.use_pager());
         {
             let mut obj_page_tree = self.lock_page_tree();
             let page_number = PageNumber::from_address(VirtAddr::new(offset as u64).unwrap());
             let page_offset = offset % PageNumber::PAGE_SIZE;
 
-            if let Some((page, _)) = obj_page_tree.get_page(page_number, true) {
+            if let PageStatus::Ready(page, _) = obj_page_tree.get_page(page_number, true) {
                 let t = page.get_mut_to_val::<T>(page_offset);
                 *t = val;
             } else {
@@ -141,11 +142,12 @@ impl Object {
     }
 
     pub unsafe fn read_atomic_u64(&self, offset: usize) -> u64 {
+        assert!(!self.use_pager());
         let mut obj_page_tree = self.lock_page_tree();
         let page_number = PageNumber::from_address(VirtAddr::new(offset as u64).unwrap());
         let page_offset = offset % PageNumber::PAGE_SIZE;
 
-        if let Some((page, _)) = obj_page_tree.get_page(page_number, true) {
+        if let PageStatus::Ready(page, _) = obj_page_tree.get_page(page_number, true) {
             let t = page.get_mut_to_val::<AtomicU64>(page_offset);
             (*t).load(Ordering::SeqCst)
         } else {
@@ -157,11 +159,12 @@ impl Object {
     }
 
     pub unsafe fn read_atomic_u32(&self, offset: usize) -> u32 {
+        assert!(!self.use_pager());
         let mut obj_page_tree = self.lock_page_tree();
         let page_number = PageNumber::from_address(VirtAddr::new(offset as u64).unwrap());
         let page_offset = offset % PageNumber::PAGE_SIZE;
 
-        if let Some((page, _)) = obj_page_tree.get_page(page_number, true) {
+        if let PageStatus::Ready(page, _) = obj_page_tree.get_page(page_number, true) {
             let t = page.get_mut_to_val::<AtomicU32>(page_offset);
             (*t).load(Ordering::SeqCst)
         } else {
@@ -186,7 +189,7 @@ impl Object {
 
                 let thislen = core::cmp::min(0x1000, len - count);
 
-                if let Some((page, _)) = obj_page_tree.get_page(page_number, true) {
+                if let PageStatus::Ready(page, _) = obj_page_tree.get_page(page_number, true) {
                     let dest = &mut page.as_mut_slice()[0..thislen];
                     dest.copy_from_slice(&bytes[count..(count + thislen)]);
                     //let t = page.get_mut_to_val::<T>(page_offset);
@@ -203,6 +206,9 @@ impl Object {
                 offset += thislen;
                 count += thislen;
             }
+            if self.use_pager() {
+                crate::pager::sync_object(self.id);
+            }
         }
     }
 

src/kernel/src/obj/pagevec.rs

@@ -81,6 +81,17 @@ impl PageVec {
         pv
     }
 
+    pub fn try_get_page(&self, offset: usize) -> Option<PageRef> {
+        if offset >= self.pages.len() {
+            return None;
+        }
+        if let Some(ref page) = self.pages[offset] {
+            Some(page.clone())
+        } else {
+            None
+        }
+    }
+
     pub fn get_page(&mut self, offset: usize) -> PageRef {
         if offset >= self.pages.len() {
             self.pages.resize(offset + 1, None)