Raw canonical ast separation #10
Conversation
Renaming & test case added
| @@ -154,21 +154,50 @@ impl Parser { | |||
| } | |||
| } | |||
|
|
|||
| pub fn build_pair(expressions: Vec<Sexp>) -> Sexp { | |||
| pub fn build_pair(mut expressions: Vec<Sexp>) -> Sexp { | |||
| expressions.reverse(); | |||
There was a problem hiding this comment.
Hacky yeah, because rust does not have foldr
| /// An s-expression is either an atom or a list of s-expressions. This is | ||
| /// similar to the data format used by lisp. | ||
| /// | ||
| /// TODO: I don't know whether I need to add those seven Lisp primitives to |
There was a problem hiding this comment.
I think at the moment no need as we want to keep the scope small.
| @@ -1,3 +1,4 @@ | |||
| pub mod node; | |||
| pub mod sexpr_parser; | |||
| pub mod sexpr_tokenizer; | |||
| pub mod sexpr_tokenizer_2; | |||
There was a problem hiding this comment.
Why not put these modules into a separate directory?
| None => vec![token] | ||
| } | ||
| }) | ||
| }); |
| match tokens.last() { | ||
| Some(lastToken) => match (lastToken, token) { | ||
| (Token::Whitespace, Token::Whitespace) => | ||
| [&tokens[0..lineNumber]].concat(), |
There was a problem hiding this comment.
I think the concat is not needed here.
| /// All strings must be valid utf-8. | ||
| #[derive(PartialEq, Clone, PartialOrd, Debug)] | ||
| pub enum Atom { | ||
| /// N stands for node |
There was a problem hiding this comment.
I get that those are the short forms, but why short forms in the first place?
| } | ||
|
|
||
| pub struct Parser { | ||
| tokens: Vec<Token>, |
There was a problem hiding this comment.
Pure nit, can be ignored: Box<[Token]> because tokens will never grow under Parser
(tokens.into_boxed_slice() will do that)
| #[derive(Debug)] | ||
| pub struct TokenizerError { | ||
| /// The error message. | ||
| pub message: &'static str, |
There was a problem hiding this comment.
I feel like an enum TokenizerErrorKind is appropriate here, but I digress..
| /// This will consume text stream and produces tokens one by one | ||
| fn next_token(&self, chars: &mut Peekable<Chars<'_>>) -> Result<Option<Token>, TokenizerError> { | ||
| match chars.peek() { | ||
| Some(&ch) => match ch { |
There was a problem hiding this comment.
Missing \n case (only \r\n is handled)
| /// Read from `chars` until `predicate` returns `false` or EOF is hit. | ||
| /// Return the characters read as String, and keep the first non-matching | ||
| /// char available as `chars.next()`. | ||
| fn peeking_take_while( |
There was a problem hiding this comment.
I was thinking about take_while() but Peekable has disappointed me.
In this branch: