Skip to content

Commit

Permalink
Delim recovery (kcl-lang#216)
Browse files Browse the repository at this point in the history 
* paren_error_recovery

* Feat(error_recovey): paren_error_recovery

* Feat(error_recovey): paren_error_recovery

* Feat(error_recovey): paren_error_recovery

* Feat(error_recovey): paren_error_recovery

* handle brackets and braces

* handle brackets and braces

* Feat(error_recovery) using stack

* rewrite the code and delete unused code

* use bug!() to handle internal error

* add some comments
  • Loading branch information
@possible-fqz
possible-fqz authored Oct 8, 2022
1 parent d9a9975 commit 251d5e5
Show file tree
Hide file tree
Showing 5 changed files with 292 additions and 32 deletions.
158 changes: 127 additions & 31 deletions kclvm/parser/src/lexer/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ mod tests;
use kclvm_ast::ast::NumberBinarySuffix;
use kclvm_ast::token::{self, CommentKind, Token, TokenKind};
use kclvm_ast::token_stream::TokenStream;
use kclvm_error::bug;
use kclvm_lexer::Base;
use kclvm_span::symbol::Symbol;
use kclvm_span::{self, BytePos, Span};
Expand All @@ -47,7 +48,7 @@ pub fn parse_token_streams(sess: &ParseSession, src: &str, start_pos: BytePos) -
token: Token::dummy(),
},
indent_cxt: IndentContext {
nesting: 0,
delims: Vec::new(),
tabs: 0,
spaces: 0,
new_line_beginning: false,
Expand Down Expand Up @@ -129,12 +130,12 @@ struct Lexer<'a> {
}

struct IndentContext {
/// nested level counter
nesting: usize,

/// A new line flag
new_line_beginning: bool,

/// Delim stack
delims: Vec<TokenKind>,

/// tab counter
tabs: usize,

Expand Down Expand Up @@ -296,47 +297,118 @@ impl<'a> Lexer<'a> {
kclvm_lexer::TokenKind::Eq => token::Assign,
// Delim tokens
kclvm_lexer::TokenKind::OpenParen => {
self.indent_cxt.nesting += 1;
self.indent_cxt.delims.push(token::OpenDelim(token::Paren));
token::OpenDelim(token::Paren)
}
kclvm_lexer::TokenKind::CloseParen => {
if self.indent_cxt.nesting == 0 {
self.sess.struct_span_error(
kclvm_lexer::TokenKind::CloseParen => match self.indent_cxt.delims.pop() {
// check delim stack
Some(delim) => match delim {
// expected case
token::OpenDelim(token::Paren) => token::CloseDelim(token::Paren),
// error recovery
token::OpenDelim(token::Brace) => {
self.sess.struct_span_error_recovery(
"error nesting on close paren",
self.span(start, self.pos),
);
token::CloseDelim(token::Brace)
}
// error recovery
token::OpenDelim(token::Bracket) => {
self.sess.struct_span_error_recovery(
"error nesting on close paren",
self.span(start, self.pos),
);
token::CloseDelim(token::Bracket)
}
// impossible case
_ => bug!("Impossible!"),
},
// error recovery
None => {
self.sess.struct_span_error_recovery(
"error nesting on close paren",
self.span(start, self.pos),
)
);
token::CloseDelim(token::Paren)
}
self.indent_cxt.nesting -= 1;
token::CloseDelim(token::Paren)
}
},
kclvm_lexer::TokenKind::OpenBrace => {
self.indent_cxt.nesting += 1;
self.indent_cxt.delims.push(token::OpenDelim(token::Brace));
token::OpenDelim(token::Brace)
}
kclvm_lexer::TokenKind::CloseBrace => {
if self.indent_cxt.nesting == 0 {
self.sess.struct_span_error(
kclvm_lexer::TokenKind::CloseBrace => match self.indent_cxt.delims.pop() {
// check delim stack
Some(delim) => match delim {
// expected case
token::OpenDelim(token::Brace) => token::CloseDelim(token::Brace),
// error recovery
token::OpenDelim(token::Paren) => {
self.sess.struct_span_error_recovery(
"error nesting on close brace",
self.span(start, self.pos),
);
token::CloseDelim(token::Paren)
}
// error recovery
token::OpenDelim(token::Bracket) => {
self.sess.struct_span_error_recovery(
"error nesting on close brace",
self.span(start, self.pos),
);
token::CloseDelim(token::Bracket)
}
// impossible case
_ => bug!("Impossible!"),
},
// error recovery
None => {
self.sess.struct_span_error_recovery(
"error nesting on close brace",
self.span(start, self.pos),
)
);
token::CloseDelim(token::Brace)
}
self.indent_cxt.nesting -= 1;
token::CloseDelim(token::Brace)
}
},
kclvm_lexer::TokenKind::OpenBracket => {
self.indent_cxt.nesting += 1;
self.indent_cxt
.delims
.push(token::OpenDelim(token::Bracket));
token::OpenDelim(token::Bracket)
}
kclvm_lexer::TokenKind::CloseBracket => {
if self.indent_cxt.nesting == 0 {
self.sess.struct_span_error(
kclvm_lexer::TokenKind::CloseBracket => match self.indent_cxt.delims.pop() {
// check delim stack
Some(delim) => match delim {
// expected case
token::OpenDelim(token::Bracket) => token::CloseDelim(token::Bracket),
// error recovery
token::OpenDelim(token::Brace) => {
self.sess.struct_span_error_recovery(
"error nesting on close bracket",
self.span(start, self.pos),
);
token::CloseDelim(token::Brace)
}
// error recovery
token::OpenDelim(token::Paren) => {
self.sess.struct_span_error_recovery(
"error nesting on close bracket",
self.span(start, self.pos),
);
token::CloseDelim(token::Paren)
}
// impossible case
_ => bug!("Impossible!"),
},
// error recovery
None => {
self.sess.struct_span_error_recovery(
"error nesting on close bracket",
self.span(start, self.pos),
)
);
token::CloseDelim(token::Bracket)
}
self.indent_cxt.nesting -= 1;
token::CloseDelim(token::Bracket)
}
},
kclvm_lexer::TokenKind::LineContinue => return None,
kclvm_lexer::TokenKind::InvalidLineContinue => self.sess.struct_span_error(
"unexpected character after line continuation character",
Expand Down Expand Up @@ -560,11 +632,35 @@ impl<'a> Lexer<'a> {
fn eof(&mut self, buf: &mut TokenStreamBuilder) {
let start = self.pos;

if self.indent_cxt.nesting > 0 {
self.sess.struct_span_error(
if !self.indent_cxt.delims.is_empty() {
self.sess.struct_span_error_recovery(
"Unclosed nesting at the end of the file",
self.span(start, self.pos),
)
);

// Add CloseDelims
while !self.indent_cxt.delims.is_empty() {
match self.indent_cxt.delims.pop() {
Some(token::OpenDelim(token::Paren)) => buf.push(Token::new(
token::CloseDelim(token::Paren),
self.span(self.pos, self.pos),
)),
Some(token::OpenDelim(token::Brace)) => buf.push(Token::new(
token::CloseDelim(token::Brace),
self.span(self.pos, self.pos),
)),
Some(token::OpenDelim(token::Bracket)) => buf.push(Token::new(
token::CloseDelim(token::Bracket),
self.span(self.pos, self.pos),
)),
_ => {
self.sess.struct_span_error_recovery(
"Unknown delim at the end of the file",
self.span(start, self.pos),
);
}
}
}
}

if !self.indent_cxt.new_line_beginning {
Expand Down
3 changes: 2 additions & 1 deletion kclvm/parser/src/parser/expr.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1795,7 +1795,8 @@ impl<'a> Parser<'a> {
TokenKind::CloseDelim(DelimToken::Paren) => {
self.bump();
}
_ => self.sess.struct_token_error(

_ => self.sess.struct_token_error_recovery(
&[token::TokenKind::CloseDelim(token::DelimToken::Paren).into()],
self.token,
),
Expand Down
129 changes: 129 additions & 0 deletions kclvm/parser/src/parser/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1104,6 +1104,135 @@ fn test_parse_file() {
}
}

#[test]
fn expr_with_paren1() {
check_parsing_expr(
r####"(2+3)"####,
expect![[r#"
Node { node: Paren(ParenExpr { expr: Node { node: Binary(BinaryExpr { left: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 1, end_line: 1, end_column: 2 }, op: Bin(Add), right: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(3) }), filename: "", line: 1, column: 3, end_line: 1, end_column: 4 } }), filename: "", line: 1, column: 1, end_line: 1, end_column: 4 } }), filename: "", line: 1, column: 0, end_line: 1, end_column: 5 }
"#]],
);
}

#[test]
fn expr_with_paren2() {
check_parsing_expr(
r####"((2+3)"####,
expect![[r#"
Node { node: Paren(ParenExpr { expr: Node { node: Paren(ParenExpr { expr: Node { node: Binary(BinaryExpr { left: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 2, end_line: 1, end_column: 3 }, op: Bin(Add), right: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(3) }), filename: "", line: 1, column: 4, end_line: 1, end_column: 5 } }), filename: "", line: 1, column: 2, end_line: 1, end_column: 5 } }), filename: "", line: 1, column: 1, end_line: 1, end_column: 6 } }), filename: "", line: 1, column: 0, end_line: 1, end_column: 6 }
"#]],
);
}

#[test]
fn expr_with_paren3() {
check_parsing_expr(
r####"(2+3))"####,
expect![[r#"
Node { node: Paren(ParenExpr { expr: Node { node: Binary(BinaryExpr { left: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 1, end_line: 1, end_column: 2 }, op: Bin(Add), right: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(3) }), filename: "", line: 1, column: 3, end_line: 1, end_column: 4 } }), filename: "", line: 1, column: 1, end_line: 1, end_column: 4 } }), filename: "", line: 1, column: 0, end_line: 1, end_column: 5 }
"#]],
);
}

#[test]
fn expr_with_bracket1() {
check_parsing_expr(
r####"[2,3]"####,
expect![[r#"
Node { node: List(ListExpr { elts: [Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 1, end_line: 1, end_column: 2 }, Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(3) }), filename: "", line: 1, column: 3, end_line: 1, end_column: 4 }], ctx: Load }), filename: "", line: 1, column: 0, end_line: 1, end_column: 5 }
"#]],
);
}

#[test]
fn expr_with_bracket2() {
check_parsing_expr(
r####"[[2,3]"####,
expect![[r#"
Node { node: List(ListExpr { elts: [Node { node: List(ListExpr { elts: [Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 2, end_line: 1, end_column: 3 }, Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(3) }), filename: "", line: 1, column: 4, end_line: 1, end_column: 5 }], ctx: Load }), filename: "", line: 1, column: 1, end_line: 1, end_column: 6 }], ctx: Load }), filename: "", line: 1, column: 0, end_line: 1, end_column: 6 }
"#]],
);
}

#[test]
fn expr_with_bracket3() {
check_parsing_expr(
r####"[2,3]]"####,
expect![[r#"
Node { node: List(ListExpr { elts: [Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 1, end_line: 1, end_column: 2 }, Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(3) }), filename: "", line: 1, column: 3, end_line: 1, end_column: 4 }], ctx: Load }), filename: "", line: 1, column: 0, end_line: 1, end_column: 5 }
"#]],
);
}

#[test]
fn expr_with_brace1() {
check_parsing_expr(
r####"{a=2}"####,
expect![[r#"
Node { node: Config(ConfigExpr { items: [Node { node: ConfigEntry { key: Some(Node { node: Identifier(Identifier { names: ["a"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 1, end_line: 1, end_column: 2 }), value: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 3, end_line: 1, end_column: 4 }, operation: Override, insert_index: -1 }, filename: "", line: 1, column: 1, end_line: 1, end_column: 4 }] }), filename: "", line: 1, column: 0, end_line: 1, end_column: 5 }
"#]],
);
}

#[test]
fn expr_with_brace2() {
check_parsing_expr(
r####"{a=2}}"####,
expect![[r#"
Node { node: Config(ConfigExpr { items: [Node { node: ConfigEntry { key: Some(Node { node: Identifier(Identifier { names: ["a"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 1, end_line: 1, end_column: 2 }), value: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 3, end_line: 1, end_column: 4 }, operation: Override, insert_index: -1 }, filename: "", line: 1, column: 1, end_line: 1, end_column: 4 }] }), filename: "", line: 1, column: 0, end_line: 1, end_column: 5 }
"#]],
);
}

#[test]
fn expr_with_delim1() {
check_parsing_expr(
r####"({a=2}"####,
expect![[r#"
Node { node: Paren(ParenExpr { expr: Node { node: Config(ConfigExpr { items: [Node { node: ConfigEntry { key: Some(Node { node: Identifier(Identifier { names: ["a"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 2, end_line: 1, end_column: 3 }), value: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 4, end_line: 1, end_column: 5 }, operation: Override, insert_index: -1 }, filename: "", line: 1, column: 2, end_line: 1, end_column: 5 }] }), filename: "", line: 1, column: 1, end_line: 1, end_column: 6 } }), filename: "", line: 1, column: 0, end_line: 1, end_column: 6 }
"#]],
);
}

#[test]
fn expr_with_delim2() {
check_parsing_expr(
r####"({a=(2}"####,
expect![[r#"
Node { node: Paren(ParenExpr { expr: Node { node: Config(ConfigExpr { items: [Node { node: ConfigEntry { key: Some(Node { node: Identifier(Identifier { names: ["a"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 2, end_line: 1, end_column: 3 }), value: Node { node: Paren(ParenExpr { expr: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 5, end_line: 1, end_column: 6 } }), filename: "", line: 1, column: 4, end_line: 1, end_column: 7 }, operation: Override, insert_index: -1 }, filename: "", line: 1, column: 2, end_line: 1, end_column: 7 }] }), filename: "", line: 1, column: 1, end_line: 1, end_column: 7 } }), filename: "", line: 1, column: 0, end_line: 1, end_column: 7 }
"#]],
);
}

#[test]
fn expr_with_delim3() {
check_parsing_expr(
r####"{a=[2]"####,
expect![[r#"
Node { node: Config(ConfigExpr { items: [Node { node: ConfigEntry { key: Some(Node { node: Identifier(Identifier { names: ["a"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 1, end_line: 1, end_column: 2 }), value: Node { node: List(ListExpr { elts: [Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 4, end_line: 1, end_column: 5 }], ctx: Load }), filename: "", line: 1, column: 3, end_line: 1, end_column: 6 }, operation: Override, insert_index: -1 }, filename: "", line: 1, column: 1, end_line: 1, end_column: 6 }] }), filename: "", line: 1, column: 0, end_line: 1, end_column: 6 }
"#]],
);
}

#[test]
fn expr_with_delim4() {
check_parsing_expr(
r####"[{a=2}"####,
expect![[r#"
Node { node: List(ListExpr { elts: [Node { node: Config(ConfigExpr { items: [Node { node: ConfigEntry { key: Some(Node { node: Identifier(Identifier { names: ["a"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 2, end_line: 1, end_column: 3 }), value: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 4, end_line: 1, end_column: 5 }, operation: Override, insert_index: -1 }, filename: "", line: 1, column: 2, end_line: 1, end_column: 5 }] }), filename: "", line: 1, column: 1, end_line: 1, end_column: 6 }], ctx: Load }), filename: "", line: 1, column: 0, end_line: 1, end_column: 6 }
"#]],
);
}

#[test]
fn expr_with_delim5() {
check_parsing_expr(
r####"({a=[2}"####,
expect![[r#"
Node { node: Paren(ParenExpr { expr: Node { node: Config(ConfigExpr { items: [Node { node: ConfigEntry { key: Some(Node { node: Identifier(Identifier { names: ["a"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 2, end_line: 1, end_column: 3 }), value: Node { node: List(ListExpr { elts: [Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 1, column: 5, end_line: 1, end_column: 6 }], ctx: Load }), filename: "", line: 1, column: 4, end_line: 1, end_column: 7 }, operation: Override, insert_index: -1 }, filename: "", line: 1, column: 2, end_line: 1, end_column: 7 }] }), filename: "", line: 1, column: 1, end_line: 1, end_column: 7 } }), filename: "", line: 1, column: 0, end_line: 1, end_column: 7 }
"#]],
);
}
// TODO: enable file tests after pos & error added.
// #[test]
fn smoke_test_parsing_stmt() {
Expand Down
Loading

0 comments on commit 251d5e5

Please sign in to comment.