Added Runtime Support for Naming (#252)

* Switch init to use new runtime.

* Cleanup tests.

* Reorg.

* Added secure gate naming library

* reverted changes

* refactored naming service

* removed changes in main

* did fmt

* removed unnecessary dependency

* updated naming service

* added naming support to etl system

* fixed bug with using twzid instead of path

* changed naming to twizzler vec

* Implement dynamic gate lookup.

* Implement dynamic gate calling.

* Move pager init and add pager dynamic gate lookup.

* Added runtime naming support

* Added std::fs support for etl_twizzler

* cargo fmt

---------

Co-authored-by: Daniel Bittman <[email protected]>

Cargo.toml

@@ -41,6 +41,7 @@ members = [
     "src/srv/pager-srv",
     "src/bin/logboi-test",
     "src/lib/virtio-net",
+    "src/lib/naming/naming-core",
     "src/lib/naming",
     "src/srv/naming-srv", 
     "src/bin/naming-test",

src/bin/etl_twizzler/Cargo.toml

@@ -10,7 +10,6 @@ twizzler-abi = { path = "../../lib/twizzler-abi" }
 twizzler-rt-abi = "0.99"
 twizzler-runtime = { path = "../../rt" }
 twizzler-object = { path = "../../lib/twizzler-object" }
-naming = { path = "../../lib/naming" }
 
 [dependencies]
 bincode = "1.3.3"

src/bin/etl_twizzler/src/etl.rs

@@ -2,14 +2,12 @@ use std::{
     fs::File,
     io::{self, BufRead, BufReader, Read, Seek, SeekFrom},
     path::PathBuf,
+    sync::Mutex,
 };
+
 use lazy_static::lazy_static;
-use std::sync::Mutex;
 use serde::{Deserialize, Serialize};
 use tar::Header;
-
-#[cfg(target_os = "twizzler")]
-use naming::NamingHandle;
 #[cfg(target_os = "twizzler")]
 use twizzler_abi::{
     object::{MAX_SIZE, NULLPAGE_SIZE},
@@ -21,14 +19,6 @@ use twizzler_abi::{
 #[cfg(target_os = "twizzler")]
 use twizzler_object::{ObjID, Object, ObjectInitFlags, Protections};
 
-// When the naming system gets integrated into the std::fs interface this will be removed
-#[cfg(target_os = "twizzler")]
-lazy_static! {
-    static ref NAMER: Mutex<NamingHandle> = {
-        Mutex::new(NamingHandle::new().unwrap())
-    };
-}
-
 // This type indicates what type of object you want to create, with the name inside
 #[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Copy)]
 pub enum PackType {
@@ -57,9 +47,7 @@ where
     pub fn new(storage: W) -> Pack<W> {
         let mut tarchive = tar::Builder::new(storage);
         tarchive.mode(tar::HeaderMode::Deterministic);
-        Pack { 
-            tarchive: tarchive, 
-        }
+        Pack { tarchive }
     }
 
     pub fn file_add(
@@ -68,9 +56,6 @@ where
         pack_type: PackType,
         offset: u64,
     ) -> std::io::Result<()> {
-        let old_path = &path;
-        #[cfg(target_os = "twizzler")]
-        let path = twizzler_name_create(&path.to_string_lossy()).to_string();
         let mut f = File::open(&path)?;
         let len = f.seek(SeekFrom::End(0))?;
         f.seek(SeekFrom::Start(0))?;
@@ -88,7 +73,7 @@ where
         header.set_size(len);
 
         self.tarchive
-            .append_data(&mut header, old_path, &mut buf_writer)?;
+            .append_data(&mut header, &path, &mut buf_writer)?;
 
         Ok(())
     }
@@ -125,7 +110,11 @@ where
 }
 
 #[cfg(target_os = "twizzler")]
-pub fn create_twizzler_object() -> twizzler_object::ObjID {
+pub fn form_twizzler_object<R: std::io::Read>(
+    mut stream: R,
+    name: String,
+    offset: u64,
+) -> std::io::Result<twizzler_object::ObjID> {
     let create = ObjectCreate::new(
         BackingType::Normal,
         LifetimeType::Persistent,
@@ -134,35 +123,6 @@ pub fn create_twizzler_object() -> twizzler_object::ObjID {
     );
     let twzid = twizzler_abi::syscall::sys_object_create(create, &[], &[]).unwrap();
 
-    twzid
-}
-
-#[cfg(target_os = "twizzler")]
-pub fn twizzler_name_create(name: &str) -> u128 {
-    let mut namer = NAMER.lock().unwrap();
-    match namer.get(&name) {
-        Some(id) => id,
-        None => {
-            let twzid = create_twizzler_object();
-            namer.put(&name, twzid.as_u128());
-            twzid.as_u128()
-        },
-    }
-}
-
-#[cfg(target_os = "twizzler")]
-pub fn twizzler_name_get(name: &str) -> u128 {
-    let mut namer = NAMER.lock().unwrap();
-    namer.get(&name).unwrap()
-}
-
-#[cfg(target_os = "twizzler")]
-pub fn form_twizzler_object<R: std::io::Read>(
-    mut stream: R,
-    name: String,
-    offset: u64,
-) -> std::io::Result<twizzler_object::ObjID> {
-    let twzid = create_twizzler_object();
     let handle =
         twizzler_rt_abi::object::twz_rt_map_object(twzid, Protections::WRITE.into()).unwrap();
     let mut stream = BufReader::new(stream);
@@ -230,10 +190,8 @@ where
                     .to_owned();
                 let bad_idea: SpecialData =
                     bincode::deserialize(&entry.header().as_old().pad).unwrap();
-                
+
                 println!("unpacked {}", path);
-                #[cfg(target_os = "twizzler")]
-                let path = twizzler_name_create(&path).to_string();
                 match bad_idea.kind {
                     PackType::StdFile => {
                         form_fs_file(entry, path, bad_idea.offset)?;

src/bin/etl_twizzler/src/main.rs

@@ -2,7 +2,7 @@ use clap::{Parser, Subcommand};
 use etl_twizzler::etl::{Pack, PackType, Unpack};
 
 #[cfg(target_os = "twizzler")]
-use naming::NamingHandle;
+use std::sync::atomic::{AtomicU64, Ordering};
 #[cfg(target_os = "twizzler")]
 use twizzler_abi::{
     object::{MAX_SIZE, NULLPAGE_SIZE},
@@ -13,10 +13,6 @@ use twizzler_abi::{
 };
 #[cfg(target_os = "twizzler")]
 use twizzler_object::{ObjID, Object, ObjectInitFlags, Protections};
-#[cfg(target_os = "twizzler")]
-use std::sync::atomic::{AtomicU64, Ordering};
-#[cfg(target_os = "twizzler")]
-use etl_twizzler::etl::{twizzler_name_get, twizzler_name_create};
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -65,9 +61,6 @@ fn main() {
             offset,
         } => {
             let archive_stream = if let Some(archive_name) = archive_name {
-                #[cfg(target_os = "twizzler")]
-                let archive_name = twizzler_name_create(&archive_name).to_string();
-
                 let archive = std::fs::File::create(archive_name).unwrap();
                 Box::new(archive) as Box<dyn std::io::Write>
             } else {
@@ -105,17 +98,11 @@ fn main() {
             pack.build();
         }
         Commands::Unpack { archive_path } => {
-            #[cfg(target_os = "twizzler")]
-            let archive_path = twizzler_name_get(&archive_path).to_string();
-
             let archive = std::fs::File::open(archive_path).unwrap();
             let unpack = Unpack::new(archive).unwrap();
             unpack.unpack().unwrap();
         }
         Commands::Inspect { archive_path } => {
-            #[cfg(target_os = "twizzler")]
-            let archive_path = twizzler_name_get(&archive_path).to_string();
-
             let archive = std::fs::File::open(archive_path).unwrap();
             let unpack = Unpack::new(archive).unwrap();
             let mut stdout = std::io::stdout().lock();
@@ -125,9 +112,6 @@ fn main() {
             archive_path,
             query,
         } => {
-            #[cfg(target_os = "twizzler")]
-            let archive_path = twizzler_name_get(&archive_path).to_string();
-
             let archive = std::fs::File::open(archive_path).unwrap();
             let unpack = Unpack::new(archive).unwrap();
             let mut stdout = std::io::stdout().lock();

src/bin/init/Cargo.toml

@@ -19,4 +19,3 @@ polling = "3.6.0"
 futures = "*"
 twizzler-futures = { path = "../../lib/twizzler-futures" }
 monitor-api = { path = "../../rt/monitor-api" }
-pager = { path = "../../lib/pager" }

src/bin/init/src/main.rs

@@ -29,7 +29,22 @@ fn initialize_pager() {
     )
     .unwrap();
 
-    pager::pager_start(queue.object().id(), queue2.object().id());
+    let pager_comp: CompartmentHandle = monitor_api::CompartmentLoader::new(
+        "pager-srv",
+        "libpager_srv.so",
+        monitor_api::NewCompartmentFlags::EXPORT_GATES,
+    )
+    .args(["pager-srv"])
+    .load()
+    .expect("failed to start pager");
+
+    let pager_start = unsafe {
+        pager_comp
+            .dynamic_gate::<(ObjID, ObjID), ()>("pager_start")
+            .unwrap()
+    };
+    pager_start(queue.object().id(), queue2.object().id());
+    std::mem::forget(pager_comp);
 }
 
 fn main() {

src/bin/naming-test/src/main.rs

@@ -1,11 +1,19 @@
 use std::fs::File;
 
-use naming::NamingHandle;
+use naming::{dynamic_naming_factory, static_naming_factory};
 
 fn main() {
-    let mut handle = NamingHandle::new().unwrap();
+    let mut handle = dynamic_naming_factory().unwrap();
 
-    println!("Behold the universe: {}", handle.enumerate_names().iter().map(|x| x.0.clone()).collect::<Vec<String>>().join(" "));
+    println!(
+        "Behold the universe: {}",
+        handle
+            .enumerate_names()
+            .iter()
+            .map(|x| x.0.clone())
+            .collect::<Vec<String>>()
+            .join(" ")
+    );
     let name = "hello world";
     match handle.get(name) {
         Some(x) => {

src/kernel/src/main.rs

@@ -174,6 +174,9 @@ pub fn idle_main() -> ! {
         current_processor().id
     );
     loop {
+        {
+            current_processor().cleanup_exited();
+        }
         sched::schedule(true);
         arch::processor::halt_and_wait();
     }

src/kernel/src/sched.rs

@@ -486,7 +486,7 @@ pub fn schedule_stattick(dt: Nanoseconds) {
     let cp = current_processor();
     let cur = current_thread_ref();
     if let Some(ref cur) = cur {
-        if !cur.is_critical() && (cur.is_in_user() || cur.is_idle_thread()) {
+        if !cur.is_critical() && cur.is_in_user() {
             cp.cleanup_exited();
         }
         if cur.is_idle_thread() {

src/lib/naming/Cargo.toml

@@ -8,6 +8,9 @@ crate-type = ["rlib"]
 
 [dependencies]
 arrayvec = "0.7.6"
-naming-srv = { path = "../../srv/naming-srv" }
 secgate = { path = "../../lib/secgate" }
+monitor-api = { path = "../../rt/monitor-api" }
 twizzler-rt-abi = "0.99"
+twizzler = { path = "../../lib/twizzler" }
+naming-core = { path = "naming-core" }
+naming-srv = { path = "../../srv/naming-srv" }

src/lib/naming/naming-core/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "naming-core"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+arrayvec = "0.7.6"
+secgate = { path = "../../../lib/secgate" }
+monitor-api = { path = "../../../rt/monitor-api" }
+twizzler-rt-abi = "0.99"
+twizzler = { path = "../../../lib/twizzler" }

src/lib/naming/naming-core/src/api.rs

@@ -0,0 +1,17 @@
+use monitor_api::CompartmentHandle;
+use secgate::{
+    util::{Descriptor, Handle, SimpleBuffer},
+    DynamicSecGate, SecGateReturn,
+};
+use twizzler_rt_abi::object::ObjID;
+
+use crate::NamingHandle;
+
+pub trait NamerAPI {
+    fn put(&self, desc: Descriptor) -> SecGateReturn<()>;
+    fn get(&self, desc: Descriptor) -> SecGateReturn<Option<u128>>;
+    fn open_handle(&self) -> SecGateReturn<Option<(Descriptor, ObjID)>>;
+    fn close_handle(&self, desc: Descriptor) -> SecGateReturn<()>;
+    fn enumerate_names(&self, desc: Descriptor) -> SecGateReturn<Option<usize>>;
+    fn remove(&self, desc: Descriptor) -> SecGateReturn<()>;
+}

src/lib/naming/naming-core/src/definitions.rs

@@ -0,0 +1,13 @@
+use arrayvec::ArrayString;
+use twizzler::marker::Invariant;
+
+pub const MAX_KEY_SIZE: usize = 255;
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy)]
+pub struct Schema {
+    pub key: ArrayString<MAX_KEY_SIZE>,
+    pub val: u128,
+}
+
+unsafe impl Invariant for Schema {}