Raw canonical ast separation

src/README.md

@@ -0,0 +1,9 @@
+# sexpr
+
+> S Expression parser
+
+## Overview
+
+Note when developing this S expression parser:
+
+- Tokenization, a.k.a lexing. The input is transformed into a stream of tokens. Tokenization never fails, although the output may contain error tokens.

src/lib.rs

@@ -1,69 +1,4 @@
 pub mod node;
-
-#[cfg(test)]
-mod tests {
-    use crate::node::*;
-    use crate::node::Node::*;
-
-    #[test]
-    fn atom_cmp() {
-        assert_eq!(Atom("ratatouille"), Atom("ratatouille"));
-        assert_ne!(Atom("ratatouille"), Atom("baguette"))
-    }
-
-    #[test]
-    fn list_cmp() {
-        assert_eq!(
-            List(vec![Atom("ratatouille")]),
-            List(vec![Atom("ratatouille")])
-        );
-
-        assert_ne!(
-            List(vec![Atom("ratatouille")]),
-            List(vec![Atom("baguette")])
-        );
-    }
-
-    #[test]
-    fn list_cons() {
-        use crate::node::cons;
-        assert_eq!(
-            cons(vec![Atom("ratatouille")], "baguette"),
-            List(vec![Atom("ratatouille"), Atom("baguette")])
-        )
-    }
-
-    #[test]
-    fn list_car() {
-        use crate::node::car;
-        assert_eq!(
-            car(vec![Atom("ratatouille"), Atom("baguette")]),
-            Atom("ratatouille")
-        );
-    }
-
-    #[test]
-    fn list_cdr() {
-        use crate::node::cdr;
-        assert_eq!(
-            cdr(vec![Atom("ratatouille"), Atom("baguette")]),
-            List(vec![Atom("baguette")])
-        );
-
-        assert_eq!(
-            cdr(vec![
-                Atom("ratatouille"),
-                Atom("baguette"),
-                Atom("aubergine")
-            ]),
-            List(vec![Atom("baguette"), Atom("aubergine")])
-        )
-    }
-
-    #[test]
-    fn type_of_term() {
-        assert_eq!(type_of(Atom("yo")), Type(0));
-        assert_eq!(type_of(List(vec![])), Type(0));
-        assert_eq!(type_of(Type(0)), Type(1))
-    }
-}
+pub mod sexpr_parser;
+pub mod sexpr_tokenizer;
+pub mod sexpr_tokenizer_2;

src/node.rs

@@ -1,37 +1,157 @@
 #[derive(Clone, PartialEq, Debug)]
 pub enum Node {
-    Atom(Atom),
-    List(List),
-    Type(i64)
+    Atom(&'static str),
+    PairNode(Pair<Node>),
+    TypeNode(Type),
+    Unit,
 }
 
-pub type Atom = &'static str;
+#[derive(Clone, PartialEq, Debug)]
+pub enum Type {
+    Universe(i64),
+    Atom,
+    Pair(Pair<Type>),
+    Unit,
+}
 
-// =====================================
-// List
-// =====================================
-pub type List = Vec<Node>;
+#[derive(Clone, PartialEq, Debug)]
+pub struct Pair<T>(pub Box<T>, pub Box<T>);
 
-pub fn cons(list: List, atom: Atom) -> Node {
-    let mut cloned_list = list.clone();
-    cloned_list.push(Node::Atom(atom));
-    Node::List(cloned_list)
-}
+impl<T> Pair<T> {
+    pub fn cons(node1: T, node2: T) -> Pair<T> {
+        Pair(Box::new(node1), Box::new(node2))
+    }
 
-pub fn car(list: List) -> Node {
-    list.first().unwrap().clone()
-}
+    pub fn car(self) -> T {
+        *self.0
+    }
 
-pub fn cdr(list: List) -> Node {
-    let mut cloned_list = list.clone();
-    cloned_list.remove(0);
-    Node::List(cloned_list)
+    pub fn cdr(self) -> T {
+        *self.1
+    }
 }
 
-pub fn type_of(node: Node) -> Node {
+pub fn type_of(node: Node) -> Type {
     match node {
-        Node::Atom(_) => Node::Type(0),
-        Node::List(_) => Node::Type(0),
-        Node::Type(n) => Node::Type(n + 1)
+        Node::Atom(_) => Type::Atom,
+        Node::PairNode(p) => Type::Pair(Pair(Box::new(type_of(*p.0)), Box::new(type_of(*p.1)))),
+        Node::TypeNode(Type::Universe(n)) => Type::Universe(n + 1),
+        Node::TypeNode(_) => Type::Universe(0),
+        Node::Unit => Type::Unit,
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::node::Node::*;
+    use crate::node::*;
+
+    #[test]
+    fn atom_cmp() {
+        assert_eq!(Atom("ratatouille"), Atom("ratatouille"));
+        assert_ne!(Atom("ratatouille"), Atom("baguette"))
+    }
+
+    #[test]
+    fn pair_cmp() {
+        assert_eq!(
+            PairNode(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Atom("ratatouille")),
+            )),
+            PairNode(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Atom("ratatouille")),
+            ))
+        );
+
+        assert_ne!(
+            PairNode(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Atom("ratatouille"))
+            )),
+            PairNode(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Atom("baguette")),
+            ))
+        );
+
+        assert_ne!(
+            PairNode(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Atom("ratatouille")),
+            )),
+            PairNode(Pair(
+                Box::new(Atom("baguette")),
+                Box::new(Atom("ratatouille")),
+            ))
+        );
+
+        assert_ne!(
+            PairNode(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Atom("baguette")),
+            )),
+            PairNode(Pair(Box::new(Atom("baguette")), Box::new(Atom("baguette")),))
+        );
+
+        assert_ne!(
+            PairNode(Pair(Box::new(Atom("baguette")), Box::new(Atom("baguette")),)),
+            PairNode(Pair(
+                Box::new(Atom("baguette")),
+                Box::new(Atom("ratatouille")),
+            ))
+        );
+    }
+
+    #[test]
+    fn pair_cons() {
+        assert_eq!(
+            Pair::cons(Atom("ratatouille"), Atom("baguette")),
+            Pair(Box::new(Atom("ratatouille")), Box::new(Atom("baguette")),)
+        )
+    }
+
+    #[test]
+    fn pair_car() {
+        assert_eq!(
+            Pair::car(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Atom("baguette"))
+            )),
+            Atom("ratatouille")
+        );
+    }
+
+    #[test]
+    fn pair_cdr() {
+        assert_eq!(
+            Pair::cdr(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Atom("baguette")),
+            )),
+            Atom("baguette")
+        );
+
+        assert_eq!(
+            Pair::cdr(Pair(
+                Box::new(Atom("ratatouille")),
+                Box::new(Node::PairNode(Pair(
+                    Box::new(Atom("baguette")),
+                    Box::new(Atom("aubergine")),
+                ))),
+            )),
+            Node::PairNode(Pair(
+                Box::new(Atom("baguette")),
+                Box::new(Atom("aubergine")),
+            ))
+        )
     }
-}
+
+    #[test]
+    fn type_of_term() {
+        use crate::node::type_of;
+        use crate::node::Type;
+        assert_eq!(type_of(Atom("yo")), Type::Atom);
+    }
+}