From d442279c356eb5587965e7b5219cae93c9b3c748 Mon Sep 17 00:00:00 2001 From: rzvxa Date: Mon, 4 Mar 2024 19:05:16 +0330 Subject: [PATCH] improvement(parser): better function parsing. --- crates/fuse-ast/src/ast.rs | 2 +- crates/fuse-parser/src/error.rs | 13 ++ crates/fuse-parser/src/lexer/operator.rs | 3 + crates/fuse-parser/src/parsers/expressions.rs | 73 ++++++---- crates/fuse-parser/src/parsers/statements.rs | 2 +- .../pass/function-expression-01/ast.snap | 6 +- .../pass/function-expression-02/ast.snap | 46 +++++++ .../pass/function-expression-02/case.fuse | 3 + .../pass/function-expression-02/tokens.snap | 86 ++++++++++++ .../pass/function-expression-03/ast.snap | 76 +++++++++++ .../pass/function-expression-03/case.fuse | 3 + .../pass/function-expression-03/tokens.snap | 127 ++++++++++++++++++ .../pass/string-interpolation-04/tokens.snap | 2 +- .../pass/string-interpolation-05/tokens.snap | 2 +- 14 files changed, 409 insertions(+), 35 deletions(-) create mode 100644 crates/fuse-parser/tests/cases/pass/function-expression-02/ast.snap create mode 100644 crates/fuse-parser/tests/cases/pass/function-expression-02/case.fuse create mode 100644 crates/fuse-parser/tests/cases/pass/function-expression-02/tokens.snap create mode 100644 crates/fuse-parser/tests/cases/pass/function-expression-03/ast.snap create mode 100644 crates/fuse-parser/tests/cases/pass/function-expression-03/case.fuse create mode 100644 crates/fuse-parser/tests/cases/pass/function-expression-03/tokens.snap diff --git a/crates/fuse-ast/src/ast.rs b/crates/fuse-ast/src/ast.rs index afbf82e..f3339f7 100644 --- a/crates/fuse-ast/src/ast.rs +++ b/crates/fuse-ast/src/ast.rs @@ -172,7 +172,7 @@ pub struct Function { pub span: Span, pub params: FunctionParameters, pub return_type: Option, - pub body: Option, + pub body: FunctionBody, } #[serializable] diff --git a/crates/fuse-parser/src/error.rs b/crates/fuse-parser/src/error.rs index 5a7536e..86c40dd 100644 --- a/crates/fuse-parser/src/error.rs +++ b/crates/fuse-parser/src/error.rs @@ -15,12 +15,21 @@ pub enum Error { token: TokenReference, expected: TokenKind, }, + #[error("{0}")] + DiagnosisError(DiagnosisError), #[error("Invalid number literal error at {0:?}")] InvalidNumberLiteralError(TokenReference), #[error("Unexpected error at {0:?}")] UnexpectedError(TokenReference), } +#[serializable] +#[derive(ThisError, Debug)] +pub enum DiagnosisError { + #[error("{0:?}\n{1}")] + GeneralError(TokenReference, String), +} + impl<'a> Parser<'a> { pub(crate) fn unexpected_error(token: &TokenReference) -> Error { Error::UnexpectedError(token.clone()) @@ -36,4 +45,8 @@ impl<'a> Parser<'a> { pub(crate) fn invalid_number_literal_error(token: &TokenReference) -> Error { Error::UnexpectedError(token.clone()) } + + pub(crate) fn diagnosis_general_error(token: &TokenReference, msg: &str) -> Error { + Error::DiagnosisError(DiagnosisError::GeneralError(token.clone(), msg.to_string())) + } } diff --git a/crates/fuse-parser/src/lexer/operator.rs b/crates/fuse-parser/src/lexer/operator.rs index 4537f2d..37940f6 100644 --- a/crates/fuse-parser/src/lexer/operator.rs +++ b/crates/fuse-parser/src/lexer/operator.rs @@ -5,6 +5,9 @@ impl<'a> Lexer<'a> { pub(super) fn operator(&mut self, start: u32, first: char) -> Option { let source = &mut self.source; let kind = flash_match! ((source, start, first) { + ',' => { + "" => TokenKind::Comma, + } '.' => { "" => TokenKind::Dot, "." => TokenKind::Dot2, diff --git a/crates/fuse-parser/src/parsers/expressions.rs b/crates/fuse-parser/src/parsers/expressions.rs index bc7823c..9f3c543 100644 --- a/crates/fuse-parser/src/parsers/expressions.rs +++ b/crates/fuse-parser/src/parsers/expressions.rs @@ -1,6 +1,9 @@ -use crate::{lexer::TokenKind, Parser, ParserResult}; +use crate::{ + lexer::{TokenKind, TokenReference}, + Parser, ParserResult, +}; use fuse_ast::{ - BindingPattern, BindingPatternKind, Block, BooleanLiteral, Expression, Function, + BindingPattern, BindingPatternKind, Block, BooleanLiteral, Expression, Function, FunctionBody, FunctionParameter, FunctionParameters, Identifier, TypeAnnotation, }; use fuse_common::Span; @@ -55,12 +58,12 @@ impl<'a> Parser<'a> { self.consume(); let params = self.parse_function_parameters()?; let return_type = self.parse_function_return_type()?; - let end = self.consume_expect(TokenKind::End); + let body = self.parse_function_body()?; Ok(Function { span: self.end_span(start), params, return_type, - body: None, + body, }) } @@ -76,31 +79,20 @@ impl<'a> Parser<'a> { } let mut params = Vec::new(); - let mut first_param = true; + let mut seen_comma = true; + while self.at(TokenKind::Identifier) { - if first_param { - first_param = false; - } else { - self.consume_expect(TokenKind::Comma)?; + if !seen_comma { + return Err(Self::unexpect_token_kind_error( + self.cur_token(), + TokenKind::Comma, + )); } - let binding = self.parse_binding()?; - match &binding { - BindingPattern { - kind: BindingPatternKind::Identifier(kind), - .. - } => params.push(FunctionParameter { - span: kind.span, - pattern: binding, - }), - _ => todo!( - "Diagnosis error here,\ - we don't allow variable deconstruction for function parameters." - ), - } + let (param, comma) = self.parse_function_parameter()?; + seen_comma = comma.is_some(); + params.push(param); } - // accept trailing commas. - self.consume_if(TokenKind::Comma); let close = self.consume_expect(TokenKind::RParen)?; Ok(FunctionParameters { @@ -110,6 +102,30 @@ impl<'a> Parser<'a> { }) } + fn parse_function_parameter( + &mut self, + ) -> ParserResult<(FunctionParameter, Option)> { + let binding = self.parse_binding()?; + let BindingPattern { + kind: BindingPatternKind::Identifier(kind), + .. + } = &binding + else { + todo!( + "Diagnosis error here,\ + we don't allow variable deconstruction for function parameters." + ); + }; + + Ok(( + FunctionParameter { + span: kind.span, + pattern: binding, + }, + self.consume_if(TokenKind::Comma), + )) + } + fn parse_function_return_type(&mut self) -> ParserResult> { if self.consume_if(TokenKind::ThinArrow) == None { return Ok(None); @@ -117,12 +133,11 @@ impl<'a> Parser<'a> { self.parse_type_annotation().map(|t| Some(t)) } - fn parse_function_body(&mut self) -> ParserResult { + fn parse_function_body(&mut self) -> ParserResult { if let Some(_) = self.consume_if(TokenKind::Arrow) { - self.parse_expression(); - todo!() + Ok(FunctionBody::Expression(self.parse_expression()?)) } else { - self.parse_block() + Ok(FunctionBody::Block(self.parse_block()?)) } } } diff --git a/crates/fuse-parser/src/parsers/statements.rs b/crates/fuse-parser/src/parsers/statements.rs index 82fbf12..0308338 100644 --- a/crates/fuse-parser/src/parsers/statements.rs +++ b/crates/fuse-parser/src/parsers/statements.rs @@ -11,7 +11,7 @@ impl<'a> Parser<'a> { pub(crate) fn parse_statements(&mut self) -> ParserResult> { let mut statements = Vec::new(); - while !self.at(TokenKind::Eof) { + while !self.at(TokenKind::Eof) && !self.at(TokenKind::End) { match self.parse_statement() { ParserResult::Ok(stmt) => { statements.push(stmt); diff --git a/crates/fuse-parser/tests/cases/pass/function-expression-01/ast.snap b/crates/fuse-parser/tests/cases/pass/function-expression-01/ast.snap index 579e023..238a2fd 100644 --- a/crates/fuse-parser/tests/cases/pass/function-expression-01/ast.snap +++ b/crates/fuse-parser/tests/cases/pass/function-expression-01/ast.snap @@ -13,7 +13,7 @@ Some(Chunk( Expression(Function(Function( span: Span( start: 0, - end: 15, + end: 11, ), params: FunctionParameters( span: Span( @@ -24,7 +24,9 @@ Some(Chunk( rest: None, ), return_type: None, - body: None, + body: Block(Block( + statements: [], + )), ))), ], ), diff --git a/crates/fuse-parser/tests/cases/pass/function-expression-02/ast.snap b/crates/fuse-parser/tests/cases/pass/function-expression-02/ast.snap new file mode 100644 index 0000000..4ea6acd --- /dev/null +++ b/crates/fuse-parser/tests/cases/pass/function-expression-02/ast.snap @@ -0,0 +1,46 @@ +--- +source: crates/fuse-parser/tests/cases/mod.rs +expression: parsed.chunk +input_file: crates/fuse-parser/tests/cases/pass/function-expression-02/case.fuse +--- +Some(Chunk( + span: Span( + start: 0, + end: 24, + ), + body: Block( + statements: [ + Expression(Function(Function( + span: Span( + start: 0, + end: 19, + ), + params: FunctionParameters( + span: Span( + start: 2, + end: 4, + ), + items: [], + rest: None, + ), + return_type: None, + body: Block(Block( + statements: [ + Expression(StringLiteral(StringLiteral( + span: Span( + start: 6, + end: 19, + ), + segments: [ + Literal(Unescaped(Span( + start: 7, + end: 19, + ))), + ], + ))), + ], + )), + ))), + ], + ), +)) diff --git a/crates/fuse-parser/tests/cases/pass/function-expression-02/case.fuse b/crates/fuse-parser/tests/cases/pass/function-expression-02/case.fuse new file mode 100644 index 0000000..4e4f53b --- /dev/null +++ b/crates/fuse-parser/tests/cases/pass/function-expression-02/case.fuse @@ -0,0 +1,3 @@ +fn() + "Hello world" +end diff --git a/crates/fuse-parser/tests/cases/pass/function-expression-02/tokens.snap b/crates/fuse-parser/tests/cases/pass/function-expression-02/tokens.snap new file mode 100644 index 0000000..fc95f76 --- /dev/null +++ b/crates/fuse-parser/tests/cases/pass/function-expression-02/tokens.snap @@ -0,0 +1,86 @@ +--- +source: crates/fuse-parser/tests/cases/mod.rs +expression: tokens +input_file: crates/fuse-parser/tests/cases/pass/function-expression-02/case.fuse +--- +[ + TokenReference( + token: Token( + span: Span( + start: 0, + end: 2, + ), + kind: Fn, + ), + leading_trivia: [], + trailing_trivia: [], + ), + TokenReference( + token: Token( + span: Span( + start: 2, + end: 3, + ), + kind: LParen, + ), + leading_trivia: [], + trailing_trivia: [], + ), + TokenReference( + token: Token( + span: Span( + start: 3, + end: 4, + ), + kind: RParen, + ), + leading_trivia: [], + trailing_trivia: [ + Token( + span: Span( + start: 4, + end: 6, + ), + kind: Whitespace, + ), + ], + ), + TokenReference( + token: Token( + span: Span( + start: 6, + end: 19, + ), + kind: StringLiteral, + ), + leading_trivia: [], + trailing_trivia: [ + Token( + span: Span( + start: 19, + end: 20, + ), + kind: Whitespace, + ), + ], + ), + TokenReference( + token: Token( + span: Span( + start: 20, + end: 23, + ), + kind: End, + ), + leading_trivia: [], + trailing_trivia: [ + Token( + span: Span( + start: 23, + end: 24, + ), + kind: Whitespace, + ), + ], + ), +] diff --git a/crates/fuse-parser/tests/cases/pass/function-expression-03/ast.snap b/crates/fuse-parser/tests/cases/pass/function-expression-03/ast.snap new file mode 100644 index 0000000..fb44bbe --- /dev/null +++ b/crates/fuse-parser/tests/cases/pass/function-expression-03/ast.snap @@ -0,0 +1,76 @@ +--- +source: crates/fuse-parser/tests/cases/mod.rs +expression: parsed.chunk +input_file: crates/fuse-parser/tests/cases/pass/function-expression-03/case.fuse +--- +Some(Chunk( + span: Span( + start: 0, + end: 16, + ), + body: Block( + statements: [ + Expression(Function(Function( + span: Span( + start: 0, + end: 11, + ), + params: FunctionParameters( + span: Span( + start: 2, + end: 8, + ), + items: [ + FunctionParameter( + span: Span( + start: 3, + end: 4, + ), + pattern: BindingPattern( + kind: Identifier(BindingIdentifier( + span: Span( + start: 3, + end: 4, + ), + atom: Atom("a"), + )), + type_annotation: None, + optional: false, + ), + ), + FunctionParameter( + span: Span( + start: 6, + end: 7, + ), + pattern: BindingPattern( + kind: Identifier(BindingIdentifier( + span: Span( + start: 6, + end: 7, + ), + atom: Atom("b"), + )), + type_annotation: None, + optional: false, + ), + ), + ], + rest: None, + ), + return_type: None, + body: Block(Block( + statements: [ + Expression(Identifier(Identifier( + span: Span( + start: 10, + end: 11, + ), + name: Atom("a"), + ))), + ], + )), + ))), + ], + ), +)) diff --git a/crates/fuse-parser/tests/cases/pass/function-expression-03/case.fuse b/crates/fuse-parser/tests/cases/pass/function-expression-03/case.fuse new file mode 100644 index 0000000..8a9381d --- /dev/null +++ b/crates/fuse-parser/tests/cases/pass/function-expression-03/case.fuse @@ -0,0 +1,3 @@ +fn(a, b) + a +end diff --git a/crates/fuse-parser/tests/cases/pass/function-expression-03/tokens.snap b/crates/fuse-parser/tests/cases/pass/function-expression-03/tokens.snap new file mode 100644 index 0000000..fcc85e9 --- /dev/null +++ b/crates/fuse-parser/tests/cases/pass/function-expression-03/tokens.snap @@ -0,0 +1,127 @@ +--- +source: crates/fuse-parser/tests/cases/mod.rs +expression: tokens +input_file: crates/fuse-parser/tests/cases/pass/function-expression-03/case.fuse +--- +[ + TokenReference( + token: Token( + span: Span( + start: 0, + end: 2, + ), + kind: Fn, + ), + leading_trivia: [], + trailing_trivia: [], + ), + TokenReference( + token: Token( + span: Span( + start: 2, + end: 3, + ), + kind: LParen, + ), + leading_trivia: [], + trailing_trivia: [], + ), + TokenReference( + token: Token( + span: Span( + start: 3, + end: 4, + ), + kind: Identifier, + ), + leading_trivia: [], + trailing_trivia: [], + ), + TokenReference( + token: Token( + span: Span( + start: 4, + end: 5, + ), + kind: Comma, + ), + leading_trivia: [], + trailing_trivia: [ + Token( + span: Span( + start: 5, + end: 6, + ), + kind: Whitespace, + ), + ], + ), + TokenReference( + token: Token( + span: Span( + start: 6, + end: 7, + ), + kind: Identifier, + ), + leading_trivia: [], + trailing_trivia: [], + ), + TokenReference( + token: Token( + span: Span( + start: 7, + end: 8, + ), + kind: RParen, + ), + leading_trivia: [], + trailing_trivia: [ + Token( + span: Span( + start: 8, + end: 10, + ), + kind: Whitespace, + ), + ], + ), + TokenReference( + token: Token( + span: Span( + start: 10, + end: 11, + ), + kind: Identifier, + ), + leading_trivia: [], + trailing_trivia: [ + Token( + span: Span( + start: 11, + end: 12, + ), + kind: Whitespace, + ), + ], + ), + TokenReference( + token: Token( + span: Span( + start: 12, + end: 15, + ), + kind: End, + ), + leading_trivia: [], + trailing_trivia: [ + Token( + span: Span( + start: 15, + end: 16, + ), + kind: Whitespace, + ), + ], + ), +] diff --git a/crates/fuse-parser/tests/cases/pass/string-interpolation-04/tokens.snap b/crates/fuse-parser/tests/cases/pass/string-interpolation-04/tokens.snap index e2bd096..d32ed05 100644 --- a/crates/fuse-parser/tests/cases/pass/string-interpolation-04/tokens.snap +++ b/crates/fuse-parser/tests/cases/pass/string-interpolation-04/tokens.snap @@ -43,7 +43,7 @@ input_file: crates/fuse-parser/tests/cases/pass/string-interpolation-04/case.fus start: 15, end: 16, ), - kind: Undetermined, + kind: Comma, ), leading_trivia: [], trailing_trivia: [ diff --git a/crates/fuse-parser/tests/cases/pass/string-interpolation-05/tokens.snap b/crates/fuse-parser/tests/cases/pass/string-interpolation-05/tokens.snap index 09dcfa3..57b9085 100644 --- a/crates/fuse-parser/tests/cases/pass/string-interpolation-05/tokens.snap +++ b/crates/fuse-parser/tests/cases/pass/string-interpolation-05/tokens.snap @@ -43,7 +43,7 @@ input_file: crates/fuse-parser/tests/cases/pass/string-interpolation-05/case.fus start: 16, end: 17, ), - kind: Undetermined, + kind: Comma, ), leading_trivia: [], trailing_trivia: [