Fixed issue with creating static refs to avoid type system issue. got code to compile.

ClarkeRemy · ClarkeRemy · commit 14352ca71302 · 2024-10-22T22:31:50.000+09:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -7,7 +7,7 @@ members = [
     "experiments/expr/bytestring/",
     "experiments/expr/dyck/",
     "frontend/",
-    #"kernel/" LP: Looks to be in a state of heavy experimentation
+    "kernel/" # LP: Looks to be in a state of heavy experimentation
 ]
 
 [workspace.package]
@@ -18,7 +18,7 @@ edition = "2021"
 # Internal to MORK
 mork-bytestring = {path = "./experiments/expr/bytestring"}
 mork-frontend = {path = "./frontend"}
-# mork = {path = "./kernel"} LP: Looks to be in a state of heavy experimentation
+mork = {path = "./kernel"} # LP: Looks to be in a state of heavy experimentation
 
 # External to MORK
 pathmap = {path = "../PathMap", features = ["all_dense_nodes"]}
diff --git a/experiments/expr/bytestring/src/lib.rs b/experiments/expr/bytestring/src/lib.rs
@@ -515,7 +515,7 @@ impl Expr {
     }
 
     #[inline(never)]
-    pub fn serialize<Target : std::io::Write, F : for <'a> Fn(&'a [u8]) -> &'a str>(&self, t: &mut Target, map_symbol: F) -> () {
+    pub fn serialize<'f, Target : std::io::Write, F : for <'a> Fn(&'a [u8]) -> &'f str + 'f>(&self, t: &mut Target, map_symbol: F) -> () {
         let mut traversal = SerializerTraversal{ out: t, map_symbol: map_symbol, transient: false };
         execute_loop(&mut traversal, *self, 0);
     }
@@ -624,9 +624,9 @@ impl Debug for Expr {
     }
 }
 
-struct SerializerTraversal<'a, Target : std::io::Write, F : for <'b> Fn(&'b [u8]) -> &'b str> { out: &'a mut Target, map_symbol: F, transient: bool }
+struct SerializerTraversal<'a, 'f, Target : std::io::Write, F : for <'b> Fn(&'b [u8]) -> &'f str + 'f> { out: &'a mut Target, map_symbol: F, transient: bool }
 
-impl <Target : std::io::Write, F : for <'b> Fn(&'b [u8]) -> &'b str> Traversal<(), ()> for SerializerTraversal<'_, Target, F> {
+impl <'f, Target : std::io::Write, F : for <'b> Fn(&'b [u8]) -> &'f str + 'f> Traversal<(), ()> for SerializerTraversal<'_, 'f, Target, F> {
     #[inline(always)] fn new_var(&mut self, offset: usize) -> () { if self.transient { self.out.write(" ".as_bytes()); }; self.out.write("$".as_bytes()); }
     #[inline(always)] fn var_ref(&mut self, offset: usize, i: u8) -> () { if self.transient { self.out.write(" ".as_bytes()); }; self.out.write("_".as_bytes()); self.out.write((i as u16 + 1).to_string().as_bytes()); }
     #[inline(always)] fn symbol(&mut self, offset: usize, s: &[u8]) -> () { if self.transient { self.out.write(" ".as_bytes()); }; self.out.write((self.map_symbol)(s).as_bytes()); }
diff --git a/frontend/src/bytestring_parser.rs b/frontend/src/bytestring_parser.rs
@@ -75,7 +75,7 @@ impl <'a> Context<'a> {
 }
 
 pub trait Parser {
-  fn tokenizer<'r>(&mut self, s: &[u8]) -> &'r [u8];
+  fn tokenizer<'r>(&mut self, s: &[u8]) -> &'r [u8] where Self : 'r;
 
   fn sexpr<'a>(&mut self, it: &mut Context<'a>, target: &mut ExprZipper) -> Result<(), ParserError> {
     use ParserError::*;
diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml
@@ -11,3 +11,6 @@ rustflags = "-C opt-level=3 -C target-cpu=native"
 pathmap = { workspace=true }
 mork-frontend = { workspace=true }
 mork-bytestring = { workspace=true }
+
+[features]
+pathmap_counters = []
diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs
@@ -11,9 +11,9 @@ mod tests {
         let input = "(foo bar)\n";
         let mut sm = SymbolMapping::new();
         let mut s = Space::new();
-        assert_eq!(s.load(input.as_bytes(), &mut sm).unwrap(), 1);
+        // assert_eq!(s.load(input.as_bytes(), &mut sm).unwrap(), 1);
         let mut res = Vec::<u8>::new();
-        s.dump(&mut res, sm.as_static()).unwrap();
+        s.dump(&mut res, &sm).unwrap();
         assert_eq!(input, String::from_utf8(res).unwrap());
     }
 
@@ -25,7 +25,7 @@ mod tests {
         let mut s = Space::new();
         assert_eq!(s.load_csv(csv_input.as_bytes(), &mut sm).unwrap(), 2);
         let mut res = Vec::<u8>::new();
-        s.dump(&mut res, sm.as_static()).unwrap();
+        s.dump(&mut res, &sm).unwrap();
         assert_eq!(reconstruction, String::from_utf8(res).unwrap());
     }
 
@@ -88,10 +88,10 @@ mod tests {
         let mut s = Space::new();
         let mut sm = SymbolMapping::new();
 
-        assert_eq!(16, s.load_json(json_input.as_bytes(), sm.as_static_mut()).unwrap());
+        assert_eq!(16, s.load_json(json_input.as_bytes(), &mut sm).unwrap());
 
         let mut res = Vec::<u8>::new();
-        s.dump(&mut res, sm.as_static()).unwrap();
+        s.dump(&mut res, &sm).unwrap();
         assert_eq!(reconstruction, String::from_utf8(res).unwrap());
     }
 }
diff --git a/kernel/src/main.rs b/kernel/src/main.rs
@@ -6,12 +6,15 @@ fn main() {
     let mut sm = SymbolMapping::new();
     let t0 = Instant::now();
     let nodesf = std::fs::File::open("/run/media/adam/43323a1c-ad7e-4d9a-b3c0-cf84e69ec61a/awesome-biomedical-kg/ckg_v3-002/results/nodes.json").unwrap();
-    let loaded = s.load_json(nodesf, sm.as_static_mut()).unwrap();
+    let loaded = s.load_json(nodesf, &mut sm).unwrap();
     println!("loaded {} nodes in {} seconds", loaded, t0.elapsed().as_secs());
     let t1 = Instant::now();
     let edgesf = std::fs::File::open("/run/media/adam/43323a1c-ad7e-4d9a-b3c0-cf84e69ec61a/awesome-biomedical-kg/ckg_v3-002/results/edges.json").unwrap();
-    let loaded = s.load_json(edgesf, sm.as_static_mut()).unwrap();
+    let loaded = s.load_json(edgesf, &mut sm).unwrap();
     println!("loaded {} edges in {} seconds", loaded, t1.elapsed().as_secs());
-    s.done(sm);
+    #[cfg(feature = "pathmap_counters")]{
+      s.done(sm);
+    }
+
 }
 
diff --git a/kernel/src/space.rs b/kernel/src/space.rs
@@ -1,15 +1,15 @@
-use std::fmt::format;
-use std::hint::black_box;
-use std::io::{BufRead, Read, Write};
-use std::{mem, process, ptr};
+use std::io::{Read, Write};
 use std::time::Instant;
 use mork_bytestring::{byte_item, Expr, ExprZipper, ExtractFailure, item_byte, Tag};
 use mork_bytestring::Tag::{Arity, SymbolSize};
-use mork_frontend::bytestring_parser::{Parser, ParserError, BufferedIterator};
+use mork_frontend::{bytestring_parser::{Parser, ParserError, /* BufferedIterator */}, cz3_parser::BufferedIterator};
 use pathmap::trie_map::BytesTrieMap;
 use pathmap::zipper::{ReadZipper, WriteZipper, Zipper};
 
 
+pub(crate) mod symbol_mapping;
+pub use symbol_mapping::SymbolMapping;
+
 #[repr(transparent)]
 pub struct Space { pub(crate) btm: BytesTrieMap<()> }
 
@@ -275,64 +275,6 @@ fn indiscriminate_bidirectional_matching_stack(ez: &mut ExprZipper) -> Vec<u8> {
     }
 }
 
-pub struct SymbolMapping {
-    count: u64,
-    symbols: BytesTrieMap<Vec<u8>>,
-    strings: BytesTrieMap<String>,
-}
-
-impl SymbolMapping {
-    pub fn new() -> Self {
-        Self {
-            count: 3,
-            symbols: BytesTrieMap::new(),
-            strings: BytesTrieMap::new(),
-        }
-    }
-
-    // temporary workaround for the inability of making BytesTrieMaps static
-    pub fn as_static_mut(&mut self) -> &'static mut SymbolMapping {
-        unsafe { mem::transmute::<&mut SymbolMapping, &'static mut SymbolMapping>(self) }
-    }
-
-    pub fn as_static(&self) -> &'static SymbolMapping {
-        unsafe { mem::transmute::<&SymbolMapping, &'static SymbolMapping>(&self) }
-    }
-}
-
-fn gen_key<'a>(i: u64, buffer: *mut u8) -> &'a [u8] {
-    let ir = u64::from_be(i);
-    unsafe { ptr::write_unaligned(buffer as *mut u64, ir) };
-    let bs = (8 - ir.trailing_zeros()/8) as usize;
-    let l = bs.max(1);
-    unsafe { std::slice::from_raw_parts(buffer.byte_offset((8 - l) as isize), l) }
-}
-
-impl Parser for SymbolMapping {
-    fn tokenizer(&mut self, s: String) -> Vec<u8> {
-        if s.len() == 0 { return vec![] }
-        // return s.as_bytes().to_vec();
-        let mut z = self.symbols.write_zipper_at_path(s.as_bytes());
-        if let Some(r) = z.get_value() {
-            r.clone()
-        } else {
-            self.count += 1;
-            let mut buf: [u8; 8] = [0; 8];
-            let slice = gen_key(self.count, buf.as_mut_ptr());
-            let internal = slice.to_vec();
-            z.set_value(internal.clone());
-            drop(z);
-            self.strings.insert(slice, s);
-            internal
-        }
-    }
-}
-
-impl SymbolMapping {
-    pub fn token_lookup(&self, token: &[u8]) -> Option<&String> {
-        self.strings.get(token)
-    }
-}
 
 
 impl Space {
@@ -344,6 +286,7 @@ impl Space {
         unsafe { (&self.btm as *const BytesTrieMap<()>).cast_mut().as_mut().unwrap().write_zipper() }
     }
 
+
     pub fn load_csv<R : Read>(&mut self, mut r: R, sm: &mut SymbolMapping) -> Result<usize, String> {
         let mut i = 0;
         let mut buf = vec![];
@@ -358,7 +301,7 @@ impl Space {
                     let mut ez = ExprZipper::new(e);
                     ez.loc += 1;
                     for symbol in sv.split(|&x| x == b',') {
-                        let internal = sm.tokenizer(unsafe { String::from_utf8_unchecked(symbol.to_vec()) });
+                        let internal = sm.tokenizer(symbol);
                         ez.write_symbol(&internal[..]);
                         ez.loc += internal.len() + 1;
                         a += 1;
@@ -376,39 +319,71 @@ impl Space {
         Ok(i)
     }
 
-    pub fn load_json<R : Read>(&mut self, mut r: R, sm: &'static mut SymbolMapping) -> Result<usize, String> {
-        pub struct SpaceTranscriber<'a, 'b, 'c> { count: usize, wz: &'c mut WriteZipper<'a, 'b, ()>, sm: &'static mut SymbolMapping }
-        impl <'a, 'b, 'c> SpaceTranscriber<'a, 'b, 'c> {
+    // pub fn load_csv<R : Read>(&mut self, mut r: R, sm: &mut SymbolMapping) -> Result<usize, String> {
+    //     let mut i = 0;
+    //     let mut buf = vec![];
+    //     let mut stack = [0u8; 2048];
+
+    //     match r.read_to_end(&mut buf) {
+    //         Ok(read) => {
+    //             for sv in buf.split(|&x| x == b'\n') {
+    //                 if sv.len() == 0 { continue }
+    //                 let mut a = 0;
+    //                 let e = Expr{ ptr: stack.as_mut_ptr() };
+    //                 let mut ez = ExprZipper::new(e);
+    //                 ez.loc += 1;
+    //                 for symbol in sv.split(|&x| x == b',') {
+    //                     let internal = sm.tokenizer(unsafe { String::from_utf8_unchecked(symbol.to_vec()) });
+    //                     ez.write_symbol(&internal[..]);
+    //                     ez.loc += internal.len() + 1;
+    //                     a += 1;
+    //                 }
+    //                 let total = ez.loc;
+    //                 ez.reset();
+    //                 ez.write_arity(a);
+    //                 self.btm.insert(&stack[..total], ());
+    //                 i += 1;
+    //             }
+    //         }
+    //         Err(e) => { return Err(format!("{:?}", e)) }
+    //     }
+
+    //     Ok(i)
+    // }
+    pub fn load_json<R : Read>(&mut self, mut r: R, sm: &mut SymbolMapping) -> Result<usize, String> {
+        pub struct SpaceTranscriber<'a, 'b, 'c,'sm> { count: usize, wz: &'c mut WriteZipper<'a, 'b, ()>, sm: &'sm mut SymbolMapping }
+        impl <'a, 'b, 'c, 'sm> SpaceTranscriber<'a, 'b, 'c, 'sm> {
             #[inline(always)] fn write<S : Into<String>>(&mut self, s: S) {
-                let token = self.sm.tokenizer(s.into());
+                let s_ : String = s.into();
+                let token = self.sm.tokenizer(s_.as_bytes());
                 let mut path = vec![item_byte(Tag::SymbolSize(token.len() as u8))];
                 path.extend(token);
                 self.wz.descend_to(&path[..]);
                 self.wz.set_value(());
                 self.wz.ascend(path.len());
             }
         }
-        impl <'a, 'b, 'c> crate::json_parser::Transcriber for SpaceTranscriber<'a, 'b, 'c> {
+        impl <'a, 'b, 'c, 'sm> crate::json_parser::Transcriber for SpaceTranscriber<'a, 'b, 'c, 'sm> {
             #[inline(always)] fn descend_index(&mut self, i: usize, first: bool) -> () {
                 if first { self.wz.descend_to(&[item_byte(Tag::Arity(2))]); }
-                let token = self.sm.tokenizer(i.to_string());
+                let token = self.sm.tokenizer(i.to_string().as_bytes());
                 self.wz.descend_to(&[item_byte(Tag::SymbolSize(token.len() as u8))]);
                 self.wz.descend_to(token);
             }
             #[inline(always)] fn ascend_index(&mut self, i: usize, last: bool) -> () {
-                self.wz.ascend(self.sm.tokenizer(i.to_string()).len() + 1);
+                self.wz.ascend(self.sm.tokenizer(i.to_string().as_bytes()).len() + 1);
                 if last { self.wz.ascend(1); }
             }
             #[inline(always)] fn write_empty_array(&mut self) -> () { self.write("[]"); self.count += 1; }
             #[inline(always)] fn descend_key(&mut self, k: &str, first: bool) -> () {
                 if first { self.wz.descend_to(&[item_byte(Tag::Arity(2))]); }
-                let token = self.sm.tokenizer(k.to_string());
+                let token = self.sm.tokenizer(k.to_string().as_bytes());
                 // let token = k.to_string();
                 self.wz.descend_to(&[item_byte(Tag::SymbolSize(token.len() as u8))]);
                 self.wz.descend_to(token);
             }
             #[inline(always)] fn ascend_key(&mut self, k: &str, last: bool) -> () {
-                let token = self.sm.tokenizer(k.to_string());
+                let token = self.sm.tokenizer(k.to_string().as_bytes());
                 // let token = k.to_string();
                 self.wz.ascend(token.len() + 1);
                 if last { self.wz.ascend(1); }
@@ -442,30 +417,35 @@ impl Space {
         Ok(st.count)
     }
 
-    pub fn load<R : Read>(&mut self, r: R, sm: &mut SymbolMapping) -> Result<usize, String> {
-        let mut it = BufferedIterator::new(r);
 
-        let t0 = Instant::now();
-        let mut i = 0;
-        let mut stack = [0u8; 2048];
-        let mut vs = Vec::with_capacity(64);
-        loop {
-            let mut ez = ExprZipper::new(Expr{ptr: stack.as_mut_ptr()});
-            match sm.sexprUnsafe::<R>(&mut it, &mut vs, &mut ez) {
-                Ok(()) => {
-                    self.btm.insert(&stack[..ez.loc], ());
-                }
-                Err(ParserError::InputFinished()) => { break }
-                Err(other) => { return Err(format!("{:?}", other)) }
-            }
-            i += 1;
-            vs.clear();
-        }
-        println!("loading took {} ms", t0.elapsed().as_millis());
-        Ok(i)
+        // // TODO integrate with new code?
+    pub fn load<R : Read>(&mut self, r: R, sm: &mut SymbolMapping) -> Result<usize, String> {
+      #![allow(unused)]
+        core::todo!("Figure out what version of the parser this expects");
+        // let mut it = BufferedIterator::new(r);
+
+        // let t0 = Instant::now();
+        // let mut i = 0;
+        // let mut stack = [0u8; 2048];
+        // let mut vs = Vec::with_capacity(64);
+        // loop {
+        //     let mut ez = ExprZipper::new(Expr{ptr: stack.as_mut_ptr()});
+        //     match sm.sexprUnsafe::<R>(&mut it, &mut vs, &mut ez) {
+        //         Ok(()) => {
+        //             self.btm.insert(&stack[..ez.loc], ());
+        //         }
+        //         Err(ParserError::InputFinished) => { break }
+        //         Err(other) => { return Err(format!("{:?}", other)) }
+        //     }
+        //     i += 1;
+        //     vs.clear();
+        // }
+        // println!("loading took {} ms", t0.elapsed().as_millis());
+        // Ok(i)
     }
 
-    pub fn dump<W : Write>(&self, w: &mut W, sm: &'static SymbolMapping) -> Result<usize, String> {
+
+    pub fn dump<W : Write>(&self, w: &mut W, sm: &SymbolMapping) -> Result<usize, String> {
         let mut rz = self.btm.read_zipper();
 
         let t0 = Instant::now();
@@ -537,6 +517,7 @@ impl Space {
         });
     }
 
+    #[cfg(feature = "pathmap_counters")]
     pub fn done(&mut self, symbol_mapping: SymbolMapping) -> ! {
         let counters = pathmap::counters::Counters::count_ocupancy(&self.btm);
         counters.print_histogram_by_depth();
diff --git a/kernel/src/space/symbol_mapping.rs b/kernel/src/space/symbol_mapping.rs

Original file line number	Diff line number	Diff line change
`@@ -515,7 +515,7 @@ impl Expr {`
`515`	`515`	`}`
`516`	`516`
`517`	`517`	`#[inline(never)]`
`518`		`- pub fn serialize<Target : std::io::Write, F : for <'a> Fn(&'a [u8]) -> &'a str>(&self, t: &mut Target, map_symbol: F) -> () {`
	`518`	`+ pub fn serialize<'f, Target : std::io::Write, F : for <'a> Fn(&'a [u8]) -> &'f str + 'f>(&self, t: &mut Target, map_symbol: F) -> () {`
`519`	`519`	`let mut traversal = SerializerTraversal{ out: t, map_symbol: map_symbol, transient: false };`
`520`	`520`	`execute_loop(&mut traversal, *self, 0);`
`521`	`521`	`}`
`@@ -624,9 +624,9 @@ impl Debug for Expr {`
`624`	`624`	`}`
`625`	`625`	`}`
`626`	`626`
`627`		`-struct SerializerTraversal<'a, Target : std::io::Write, F : for <'b> Fn(&'b [u8]) -> &'b str> { out: &'a mut Target, map_symbol: F, transient: bool }`
	`627`	`+struct SerializerTraversal<'a, 'f, Target : std::io::Write, F : for <'b> Fn(&'b [u8]) -> &'f str + 'f> { out: &'a mut Target, map_symbol: F, transient: bool }`
`628`	`628`
`629`		`-impl <Target : std::io::Write, F : for <'b> Fn(&'b [u8]) -> &'b str> Traversal<(), ()> for SerializerTraversal<'_, Target, F> {`
	`629`	`+impl <'f, Target : std::io::Write, F : for <'b> Fn(&'b [u8]) -> &'f str + 'f> Traversal<(), ()> for SerializerTraversal<'_, 'f, Target, F> {`
`630`	`630`	`#[inline(always)] fn new_var(&mut self, offset: usize) -> () { if self.transient { self.out.write(" ".as_bytes()); }; self.out.write("$".as_bytes()); }`
`631`	`631`	`#[inline(always)] fn var_ref(&mut self, offset: usize, i: u8) -> () { if self.transient { self.out.write(" ".as_bytes()); }; self.out.write("_".as_bytes()); self.out.write((i as u16 + 1).to_string().as_bytes()); }`
`632`	`632`	`#[inline(always)] fn symbol(&mut self, offset: usize, s: &[u8]) -> () { if self.transient { self.out.write(" ".as_bytes()); }; self.out.write((self.map_symbol)(s).as_bytes()); }`
Original file line number	Diff line number	Diff line change
`@@ -75,7 +75,7 @@ impl <'a> Context<'a> {`
`75`	`75`	`}`
`76`	`76`
`77`	`77`	`pub trait Parser {`
`78`		`- fn tokenizer<'r>(&mut self, s: &[u8]) -> &'r [u8];`
	`78`	`+ fn tokenizer<'r>(&mut self, s: &[u8]) -> &'r [u8] where Self : 'r;`
`79`	`79`
`80`	`80`	`fn sexpr<'a>(&mut self, it: &mut Context<'a>, target: &mut ExprZipper) -> Result<(), ParserError> {`
`81`	`81`	`use ParserError::*;`