From d266ba58b314040f569250a1b67a5ae3859ac375 Mon Sep 17 00:00:00 2001 From: JohnnyMorganz Date: Sat, 17 Apr 2021 19:28:22 +0100 Subject: [PATCH 1/5] Fix ordering of fields in structs so tokens() is better ordered --- full-moon/src/ast/mod.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/full-moon/src/ast/mod.rs b/full-moon/src/ast/mod.rs index 4c4c053e..c710e960 100644 --- a/full-moon/src/ast/mod.rs +++ b/full-moon/src/ast/mod.rs @@ -502,6 +502,9 @@ pub struct NumericFor<'a> { #[cfg_attr(feature = "serde", serde(borrow))] for_token: TokenReference<'a>, index_variable: TokenReference<'a>, + #[cfg(feature = "roblox")] + #[cfg_attr(feature = "serde", serde(borrow))] + type_specifier: Option>, equal_token: TokenReference<'a>, start: Expression<'a>, start_end_comma: TokenReference<'a>, @@ -511,9 +514,6 @@ pub struct NumericFor<'a> { do_token: TokenReference<'a>, block: Block<'a>, end_token: TokenReference<'a>, - #[cfg(feature = "roblox")] - #[cfg_attr(feature = "serde", serde(borrow))] - type_specifier: Option>, } impl<'a> NumericFor<'a> { @@ -734,14 +734,14 @@ pub struct GenericFor<'a> { #[cfg_attr(feature = "serde", serde(borrow))] for_token: TokenReference<'a>, names: Punctuated<'a, TokenReference<'a>>, + #[cfg(feature = "roblox")] + #[cfg_attr(feature = "serde", serde(borrow))] + type_specifiers: Vec>>, in_token: TokenReference<'a>, expr_list: Punctuated<'a, Expression<'a>>, do_token: TokenReference<'a>, block: Block<'a>, end_token: TokenReference<'a>, - #[cfg(feature = "roblox")] - #[cfg_attr(feature = "serde", serde(borrow))] - type_specifiers: Vec>>, } impl<'a> GenericFor<'a> { @@ -1676,10 +1676,10 @@ impl<'a> LocalFunction<'a> { pub struct LocalAssignment<'a> { #[cfg_attr(feature = "serde", serde(borrow))] local_token: TokenReference<'a>, + name_list: Punctuated<'a, TokenReference<'a>>, #[cfg(feature = "roblox")] #[cfg_attr(feature = "serde", serde(borrow))] type_specifiers: Vec>>, - name_list: Punctuated<'a, TokenReference<'a>>, equal_token: Option>, expr_list: Punctuated<'a, Expression<'a>>, } From b96e2a7d28f5a04b1100063b78fb3e91500922f7 Mon Sep 17 00:00:00 2001 From: JohnnyMorganz Date: Sat, 17 Apr 2021 21:15:03 +0100 Subject: [PATCH 2/5] Implement explicit node trait for TableConstructor --- full-moon/src/ast/mod.rs | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/full-moon/src/ast/mod.rs b/full-moon/src/ast/mod.rs index c710e960..7962514e 100644 --- a/full-moon/src/ast/mod.rs +++ b/full-moon/src/ast/mod.rs @@ -8,7 +8,9 @@ mod update_positions; mod visitors; use crate::{ - tokenizer::{Symbol, Token, TokenReference, TokenType}, + node::{Node as NodeTrait, TokenItem, Tokens}, + private::Sealed, + tokenizer::{Position, Symbol, Token, TokenReference, TokenType}, util::*, }; use derive_more::Display; @@ -209,12 +211,11 @@ pub enum Field<'a> { } /// A table being constructed, such as `{ 1, 2, 3 }` or `{ a = 1 }` -#[derive(Clone, Debug, Display, PartialEq, Owned, Node, Visit)] +#[derive(Clone, Debug, Display, PartialEq, Owned, Visit)] #[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] #[display(fmt = "{}{}{}", "braces.tokens().0", "fields", "braces.tokens().1")] pub struct TableConstructor<'a> { #[cfg_attr(feature = "serde", serde(borrow))] - #[node(full_range)] #[visit(contains = "fields")] braces: ContainedSpan<'a>, fields: Punctuated<'a, Field<'a>>, @@ -260,6 +261,31 @@ impl Default for TableConstructor<'_> { } } +impl Sealed for TableConstructor<'_> {} +impl<'a> NodeTrait<'a> for TableConstructor<'a> { + fn start_position(&self) -> Option { + self.braces.tokens().0.start_position() + } + + fn end_position(&self) -> Option { + self.braces.tokens().1.end_position() + } + + fn similar(&self, other: &Self) -> bool { + self.braces().similar(other.braces()) && self.fields().similar(other.fields()) + } + + fn tokens<'b>(&'b self) -> Tokens<'a, 'b> { + let mut items = Vec::with_capacity(3); + let (start_brace, end_brace) = self.braces().tokens(); + items.push(TokenItem::TokenReference(start_brace)); + items.push(TokenItem::MoreTokens(self.fields())); + items.push(TokenItem::TokenReference(end_brace)); + + Tokens { items } + } +} + /// An expression, mostly useful for getting values #[derive(Clone, Debug, Display, PartialEq, Owned, Node)] #[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] From 0e9ba096a2d8ccb17d58bafb8c3a6a18aa249de5 Mon Sep 17 00:00:00 2001 From: JohnnyMorganz Date: Sat, 17 Apr 2021 21:15:20 +0100 Subject: [PATCH 3/5] Add test for node constructor --- full-moon/src/ast/mod.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/full-moon/src/ast/mod.rs b/full-moon/src/ast/mod.rs index 7962514e..4038f112 100644 --- a/full-moon/src/ast/mod.rs +++ b/full-moon/src/ast/mod.rs @@ -2302,7 +2302,11 @@ pub(crate) fn extract_token_references(mut tokens: Vec) -> Vec Visitor<'ast> for NodesChecker { + fn visit_table_constructor(&mut self, table_constructor: &TableConstructor<'ast>) { + let mut tokens = table_constructor.tokens(); + assert!(tokens + .next() + .unwrap() + .similar(&TokenReference::symbol("{").unwrap())); + assert!(tokens + .next() + .unwrap() + .similar(&TokenReference::symbol("true").unwrap())); + assert!(tokens + .next() + .unwrap() + .similar(&TokenReference::symbol("}").unwrap())); + } + } + + NodesChecker.visit_ast(&ast); + } } From 96ca8451b0dd1722076ad521f7d1161dfcfff116 Mon Sep 17 00:00:00 2001 From: JohnnyMorganz Date: Sat, 17 Apr 2021 21:44:03 +0100 Subject: [PATCH 4/5] Update tests cases due to the fields changing order --- .../pass/compound_assignment/ast.snap | 17 +- .../tests/roblox_cases/pass/continue/ast.snap | 5 +- .../pass/decimal_seperators/ast.snap | 25 +- .../pass/no_roblox_syntax/ast.snap | 27 +- .../tests/roblox_cases/pass/types/ast.snap | 399 +++++++++--------- .../pass/types_indexable/ast.snap | 137 +++--- .../roblox_cases/pass/types_loops/ast.snap | 219 +++++----- 7 files changed, 411 insertions(+), 418 deletions(-) diff --git a/full-moon/tests/roblox_cases/pass/compound_assignment/ast.snap b/full-moon/tests/roblox_cases/pass/compound_assignment/ast.snap index 2c1bf6ad..6fae29ac 100644 --- a/full-moon/tests/roblox_cases/pass/compound_assignment/ast.snap +++ b/full-moon/tests/roblox_cases/pass/compound_assignment/ast.snap @@ -1,7 +1,6 @@ --- source: full-moon/tests/pass_cases.rs expression: ast.nodes() -input_file: full-moon/tests/roblox_cases/pass/compound_assignment --- stmts: @@ -32,8 +31,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -62,6 +59,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -146,8 +145,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -176,6 +173,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -1307,8 +1306,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -1337,6 +1334,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -1422,8 +1421,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -1452,6 +1449,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: diff --git a/full-moon/tests/roblox_cases/pass/continue/ast.snap b/full-moon/tests/roblox_cases/pass/continue/ast.snap index 5ba51059..38d18b88 100644 --- a/full-moon/tests/roblox_cases/pass/continue/ast.snap +++ b/full-moon/tests/roblox_cases/pass/continue/ast.snap @@ -1,7 +1,6 @@ --- source: full-moon/tests/pass_cases.rs expression: ast.nodes() -input_file: full-moon/tests/roblox_cases/pass/continue --- stmts: @@ -279,8 +278,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -309,6 +306,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: diff --git a/full-moon/tests/roblox_cases/pass/decimal_seperators/ast.snap b/full-moon/tests/roblox_cases/pass/decimal_seperators/ast.snap index 4c50a51f..268542b2 100644 --- a/full-moon/tests/roblox_cases/pass/decimal_seperators/ast.snap +++ b/full-moon/tests/roblox_cases/pass/decimal_seperators/ast.snap @@ -1,7 +1,6 @@ --- source: full-moon/tests/pass_cases.rs expression: ast.nodes() -input_file: full-moon/tests/roblox_cases/pass/decimal_seperators --- stmts: @@ -32,8 +31,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -62,6 +59,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -146,8 +145,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -176,6 +173,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -260,8 +259,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -290,6 +287,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -374,8 +373,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -404,6 +401,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -488,8 +487,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -518,6 +515,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -602,8 +601,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -632,6 +629,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: diff --git a/full-moon/tests/roblox_cases/pass/no_roblox_syntax/ast.snap b/full-moon/tests/roblox_cases/pass/no_roblox_syntax/ast.snap index 8a440c86..2d97421a 100644 --- a/full-moon/tests/roblox_cases/pass/no_roblox_syntax/ast.snap +++ b/full-moon/tests/roblox_cases/pass/no_roblox_syntax/ast.snap @@ -1,7 +1,6 @@ --- source: full-moon/tests/pass_cases.rs expression: ast.nodes() -input_file: full-moon/tests/roblox_cases/pass/no_roblox_syntax --- stmts: @@ -54,8 +53,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -84,6 +81,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -255,8 +254,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -285,6 +282,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -467,8 +466,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -497,6 +494,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -715,8 +714,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -745,6 +742,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -1053,8 +1052,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -1083,6 +1080,8 @@ stmts: token_type: type: Whitespace characters: "\n" + type_specifiers: + - ~ equal_token: ~ expr_list: pairs: [] @@ -3153,9 +3152,6 @@ last_stmt: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ - - ~ name_list: pairs: - Punctuated: @@ -3224,6 +3220,9 @@ last_stmt: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ + - ~ equal_token: ~ expr_list: pairs: [] diff --git a/full-moon/tests/roblox_cases/pass/types/ast.snap b/full-moon/tests/roblox_cases/pass/types/ast.snap index 1c8b5489..bad6fee4 100644 --- a/full-moon/tests/roblox_cases/pass/types/ast.snap +++ b/full-moon/tests/roblox_cases/pass/types/ast.snap @@ -1,7 +1,6 @@ --- source: full-moon/tests/pass_cases.rs expression: ast.nodes() -input_file: full-moon/tests/roblox_cases/pass/types --- stmts: @@ -2472,6 +2471,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 363 + line: 16 + character: 7 + end_position: + bytes: 366 + line: 16 + character: 10 + token_type: + type: Identifier + identifier: foo + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -2526,23 +2542,6 @@ stmts: token_type: type: Whitespace characters: " " - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 363 - line: 16 - character: 7 - end_position: - bytes: 366 - line: 16 - character: 10 - token_type: - type: Identifier - identifier: foo - trailing_trivia: [] equal_token: leading_trivia: [] token: @@ -2627,6 +2626,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 385 + line: 17 + character: 7 + end_position: + bytes: 388 + line: 17 + character: 10 + token_type: + type: Identifier + identifier: foo + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -2698,23 +2714,6 @@ stmts: token_type: type: Whitespace characters: "\n" - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 385 - line: 17 - character: 7 - end_position: - bytes: 388 - line: 17 - character: 10 - token_type: - type: Identifier - identifier: foo - trailing_trivia: [] equal_token: ~ expr_list: pairs: [] @@ -2746,6 +2745,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 404 + line: 18 + character: 7 + end_position: + bytes: 407 + line: 18 + character: 10 + token_type: + type: Identifier + identifier: foo + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -2849,23 +2865,6 @@ stmts: type: Identifier identifier: T trailing_trivia: [] - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 404 - line: 18 - character: 7 - end_position: - bytes: 407 - line: 18 - character: 10 - token_type: - type: Identifier - identifier: foo - trailing_trivia: [] equal_token: ~ expr_list: pairs: [] @@ -2897,6 +2896,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 424 + line: 19 + character: 7 + end_position: + bytes: 427 + line: 19 + character: 10 + token_type: + type: Identifier + identifier: foo + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -3041,23 +3057,6 @@ stmts: type: Identifier identifier: U trailing_trivia: [] - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 424 - line: 19 - character: 7 - end_position: - bytes: 427 - line: 19 - character: 10 - token_type: - type: Identifier - identifier: foo - trailing_trivia: [] equal_token: ~ expr_list: pairs: [] @@ -3089,8 +3088,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -3119,6 +3116,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -3258,6 +3257,63 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - Punctuated: + - leading_trivia: [] + token: + start_position: + bytes: 473 + line: 21 + character: 7 + end_position: + bytes: 476 + line: 21 + character: 10 + token_type: + type: Identifier + identifier: foo + trailing_trivia: [] + - leading_trivia: [] + token: + start_position: + bytes: 484 + line: 21 + character: 18 + end_position: + bytes: 485 + line: 21 + character: 19 + token_type: + type: Symbol + symbol: "," + trailing_trivia: + - start_position: + bytes: 485 + line: 21 + character: 19 + end_position: + bytes: 486 + line: 21 + character: 20 + token_type: + type: Whitespace + characters: " " + - End: + leading_trivia: [] + token: + start_position: + bytes: 486 + line: 21 + character: 20 + end_position: + bytes: 489 + line: 21 + character: 23 + token_type: + type: Identifier + identifier: bar + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -3354,63 +3410,6 @@ stmts: token_type: type: Whitespace characters: "\n" - name_list: - pairs: - - Punctuated: - - leading_trivia: [] - token: - start_position: - bytes: 473 - line: 21 - character: 7 - end_position: - bytes: 476 - line: 21 - character: 10 - token_type: - type: Identifier - identifier: foo - trailing_trivia: [] - - leading_trivia: [] - token: - start_position: - bytes: 484 - line: 21 - character: 18 - end_position: - bytes: 485 - line: 21 - character: 19 - token_type: - type: Symbol - symbol: "," - trailing_trivia: - - start_position: - bytes: 485 - line: 21 - character: 19 - end_position: - bytes: 486 - line: 21 - character: 20 - token_type: - type: Whitespace - characters: " " - - End: - leading_trivia: [] - token: - start_position: - bytes: 486 - line: 21 - character: 20 - end_position: - bytes: 489 - line: 21 - character: 23 - token_type: - type: Identifier - identifier: bar - trailing_trivia: [] equal_token: ~ expr_list: pairs: [] @@ -3453,6 +3452,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 505 + line: 23 + character: 7 + end_position: + bytes: 510 + line: 23 + character: 12 + token_type: + type: Identifier + identifier: union + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -3562,23 +3578,6 @@ stmts: token_type: type: Whitespace characters: "\n" - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 505 - line: 23 - character: 7 - end_position: - bytes: 510 - line: 23 - character: 12 - token_type: - type: Identifier - identifier: union - trailing_trivia: [] equal_token: ~ expr_list: pairs: [] @@ -3610,6 +3609,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 534 + line: 24 + character: 7 + end_position: + bytes: 544 + line: 24 + character: 17 + token_type: + type: Identifier + identifier: multiUnion + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -3774,23 +3790,6 @@ stmts: token_type: type: Whitespace characters: "\n" - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 534 - line: 24 - character: 7 - end_position: - bytes: 544 - line: 24 - character: 17 - token_type: - type: Identifier - identifier: multiUnion - trailing_trivia: [] equal_token: ~ expr_list: pairs: [] @@ -3833,6 +3832,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 575 + line: 26 + character: 7 + end_position: + bytes: 587 + line: 26 + character: 19 + token_type: + type: Identifier + identifier: intersection + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -3942,23 +3958,6 @@ stmts: token_type: type: Whitespace characters: "\n" - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 575 - line: 26 - character: 7 - end_position: - bytes: 587 - line: 26 - character: 19 - token_type: - type: Identifier - identifier: intersection - trailing_trivia: [] equal_token: ~ expr_list: pairs: [] @@ -3990,6 +3989,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 611 + line: 27 + character: 7 + end_position: + bytes: 628 + line: 27 + character: 24 + token_type: + type: Identifier + identifier: multiIntersection + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -4154,23 +4170,6 @@ stmts: token_type: type: Whitespace characters: "\n" - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 611 - line: 27 - character: 7 - end_position: - bytes: 628 - line: 27 - character: 24 - token_type: - type: Identifier - identifier: multiIntersection - trailing_trivia: [] equal_token: ~ expr_list: pairs: [] @@ -4840,8 +4839,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -4870,6 +4867,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: @@ -5205,8 +5204,6 @@ stmts: token_type: type: Whitespace characters: " " - type_specifiers: - - ~ name_list: pairs: - End: @@ -5235,6 +5232,8 @@ stmts: token_type: type: Whitespace characters: " " + type_specifiers: + - ~ equal_token: leading_trivia: [] token: diff --git a/full-moon/tests/roblox_cases/pass/types_indexable/ast.snap b/full-moon/tests/roblox_cases/pass/types_indexable/ast.snap index e0d95fe9..ab13018f 100644 --- a/full-moon/tests/roblox_cases/pass/types_indexable/ast.snap +++ b/full-moon/tests/roblox_cases/pass/types_indexable/ast.snap @@ -1,7 +1,6 @@ --- source: full-moon/tests/pass_cases.rs expression: ast.nodes() -input_file: full-moon/tests/roblox_cases/pass/types_indexable --- stmts: @@ -32,6 +31,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 6 + line: 1 + character: 7 + end_position: + bytes: 7 + line: 1 + character: 8 + token_type: + type: Identifier + identifier: x + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -118,23 +134,6 @@ stmts: token_type: type: Whitespace characters: " " - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 6 - line: 1 - character: 7 - end_position: - bytes: 7 - line: 1 - character: 8 - token_type: - type: Identifier - identifier: x - trailing_trivia: [] equal_token: leading_trivia: [] token: @@ -219,6 +218,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 32 + line: 2 + character: 7 + end_position: + bytes: 33 + line: 2 + character: 8 + token_type: + type: Identifier + identifier: x + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -354,23 +370,6 @@ stmts: type: Identifier identifier: string trailing_trivia: [] - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 32 - line: 2 - character: 7 - end_position: - bytes: 33 - line: 2 - character: 8 - token_type: - type: Identifier - identifier: x - trailing_trivia: [] equal_token: leading_trivia: [] token: @@ -514,6 +513,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 74 + line: 3 + character: 7 + end_position: + bytes: 75 + line: 3 + character: 8 + token_type: + type: Identifier + identifier: x + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -655,23 +671,6 @@ stmts: token_type: type: Whitespace characters: " " - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 74 - line: 3 - character: 7 - end_position: - bytes: 75 - line: 3 - character: 8 - token_type: - type: Identifier - identifier: x - trailing_trivia: [] equal_token: leading_trivia: [] token: @@ -757,6 +756,23 @@ stmts: token_type: type: Whitespace characters: " " + name_list: + pairs: + - End: + leading_trivia: [] + token: + start_position: + bytes: 111 + line: 4 + character: 7 + end_position: + bytes: 112 + line: 4 + character: 8 + token_type: + type: Identifier + identifier: x + trailing_trivia: [] type_specifiers: - punctuation: leading_trivia: [] @@ -860,23 +876,6 @@ stmts: token_type: type: Whitespace characters: " " - name_list: - pairs: - - End: - leading_trivia: [] - token: - start_position: - bytes: 111 - line: 4 - character: 7 - end_position: - bytes: 112 - line: 4 - character: 8 - token_type: - type: Identifier - identifier: x - trailing_trivia: [] equal_token: leading_trivia: [] token: diff --git a/full-moon/tests/roblox_cases/pass/types_loops/ast.snap b/full-moon/tests/roblox_cases/pass/types_loops/ast.snap index 9f44b814..03c5e04b 100644 --- a/full-moon/tests/roblox_cases/pass/types_loops/ast.snap +++ b/full-moon/tests/roblox_cases/pass/types_loops/ast.snap @@ -1,7 +1,6 @@ --- source: full-moon/tests/pass_cases.rs expression: ast.nodes() -input_file: full-moon/tests/roblox_cases/pass/types_loops --- stmts: @@ -89,6 +88,61 @@ stmts: type: Identifier identifier: v trailing_trivia: [] + type_specifiers: + - ~ + - punctuation: + leading_trivia: [] + token: + start_position: + bytes: 8 + line: 1 + character: 9 + end_position: + bytes: 9 + line: 1 + character: 10 + token_type: + type: Symbol + symbol: ":" + trailing_trivia: + - start_position: + bytes: 9 + line: 1 + character: 10 + end_position: + bytes: 10 + line: 1 + character: 11 + token_type: + type: Whitespace + characters: " " + type_info: + Basic: + leading_trivia: [] + token: + start_position: + bytes: 10 + line: 1 + character: 11 + end_position: + bytes: 16 + line: 1 + character: 17 + token_type: + type: Identifier + identifier: string + trailing_trivia: + - start_position: + bytes: 16 + line: 1 + character: 17 + end_position: + bytes: 17 + line: 1 + character: 18 + token_type: + type: Whitespace + characters: " " in_token: leading_trivia: [] token: @@ -248,61 +302,6 @@ stmts: token_type: type: Whitespace characters: "\n" - type_specifiers: - - ~ - - punctuation: - leading_trivia: [] - token: - start_position: - bytes: 8 - line: 1 - character: 9 - end_position: - bytes: 9 - line: 1 - character: 10 - token_type: - type: Symbol - symbol: ":" - trailing_trivia: - - start_position: - bytes: 9 - line: 1 - character: 10 - end_position: - bytes: 10 - line: 1 - character: 11 - token_type: - type: Whitespace - characters: " " - type_info: - Basic: - leading_trivia: [] - token: - start_position: - bytes: 10 - line: 1 - character: 11 - end_position: - bytes: 16 - line: 1 - character: 17 - token_type: - type: Identifier - identifier: string - trailing_trivia: - - start_position: - bytes: 16 - line: 1 - character: 17 - end_position: - bytes: 17 - line: 1 - character: 18 - token_type: - type: Whitespace - characters: " " - ~ - - NumericFor: for_token: @@ -357,6 +356,60 @@ stmts: type: Identifier identifier: i trailing_trivia: [] + type_specifier: + punctuation: + leading_trivia: [] + token: + start_position: + bytes: 42 + line: 5 + character: 6 + end_position: + bytes: 43 + line: 5 + character: 7 + token_type: + type: Symbol + symbol: ":" + trailing_trivia: + - start_position: + bytes: 43 + line: 5 + character: 7 + end_position: + bytes: 44 + line: 5 + character: 8 + token_type: + type: Whitespace + characters: " " + type_info: + Basic: + leading_trivia: [] + token: + start_position: + bytes: 44 + line: 5 + character: 8 + end_position: + bytes: 50 + line: 5 + character: 14 + token_type: + type: Identifier + identifier: number + trailing_trivia: + - start_position: + bytes: 50 + line: 5 + character: 14 + end_position: + bytes: 51 + line: 5 + character: 15 + token_type: + type: Whitespace + characters: " " equal_token: leading_trivia: [] token: @@ -551,59 +604,5 @@ stmts: type: Symbol symbol: end trailing_trivia: [] - type_specifier: - punctuation: - leading_trivia: [] - token: - start_position: - bytes: 42 - line: 5 - character: 6 - end_position: - bytes: 43 - line: 5 - character: 7 - token_type: - type: Symbol - symbol: ":" - trailing_trivia: - - start_position: - bytes: 43 - line: 5 - character: 7 - end_position: - bytes: 44 - line: 5 - character: 8 - token_type: - type: Whitespace - characters: " " - type_info: - Basic: - leading_trivia: [] - token: - start_position: - bytes: 44 - line: 5 - character: 8 - end_position: - bytes: 50 - line: 5 - character: 14 - token_type: - type: Identifier - identifier: number - trailing_trivia: - - start_position: - bytes: 50 - line: 5 - character: 14 - end_position: - bytes: 51 - line: 5 - character: 15 - token_type: - type: Whitespace - characters: " " - ~ From 250457c2d644bac584891d5f42ff2c603a9d918d Mon Sep 17 00:00:00 2001 From: JohnnyMorganz Date: Mon, 19 Apr 2021 20:03:19 +0100 Subject: [PATCH 5/5] Move around code --- full-moon/src/ast/mod.rs | 62 ++-------------------------------------- full-moon/src/node.rs | 60 +++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/full-moon/src/ast/mod.rs b/full-moon/src/ast/mod.rs index 4038f112..056a54c1 100644 --- a/full-moon/src/ast/mod.rs +++ b/full-moon/src/ast/mod.rs @@ -8,9 +8,7 @@ mod update_positions; mod visitors; use crate::{ - node::{Node as NodeTrait, TokenItem, Tokens}, - private::Sealed, - tokenizer::{Position, Symbol, Token, TokenReference, TokenType}, + tokenizer::{Symbol, Token, TokenReference, TokenType}, util::*, }; use derive_more::Display; @@ -261,31 +259,6 @@ impl Default for TableConstructor<'_> { } } -impl Sealed for TableConstructor<'_> {} -impl<'a> NodeTrait<'a> for TableConstructor<'a> { - fn start_position(&self) -> Option { - self.braces.tokens().0.start_position() - } - - fn end_position(&self) -> Option { - self.braces.tokens().1.end_position() - } - - fn similar(&self, other: &Self) -> bool { - self.braces().similar(other.braces()) && self.fields().similar(other.fields()) - } - - fn tokens<'b>(&'b self) -> Tokens<'a, 'b> { - let mut items = Vec::with_capacity(3); - let (start_brace, end_brace) = self.braces().tokens(); - items.push(TokenItem::TokenReference(start_brace)); - items.push(TokenItem::MoreTokens(self.fields())); - items.push(TokenItem::TokenReference(end_brace)); - - Tokens { items } - } -} - /// An expression, mostly useful for getting values #[derive(Clone, Debug, Display, PartialEq, Owned, Node)] #[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] @@ -2302,11 +2275,7 @@ pub(crate) fn extract_token_references(mut tokens: Vec) -> Vec Visitor<'ast> for NodesChecker { - fn visit_table_constructor(&mut self, table_constructor: &TableConstructor<'ast>) { - let mut tokens = table_constructor.tokens(); - assert!(tokens - .next() - .unwrap() - .similar(&TokenReference::symbol("{").unwrap())); - assert!(tokens - .next() - .unwrap() - .similar(&TokenReference::symbol("true").unwrap())); - assert!(tokens - .next() - .unwrap() - .similar(&TokenReference::symbol("}").unwrap())); - } - } - - NodesChecker.visit_ast(&ast); - } } diff --git a/full-moon/src/node.rs b/full-moon/src/node.rs index f0a85294..7828d223 100644 --- a/full-moon/src/node.rs +++ b/full-moon/src/node.rs @@ -1,5 +1,5 @@ use crate::{ - ast::Ast, + ast::{Ast, TableConstructor}, private, tokenizer::{Position, Token, TokenReference}, }; @@ -282,3 +282,61 @@ impl<'a, A: Node<'a>, B: Node<'a>> Node<'a> for (A, B) { Tokens { items } } } + +impl private::Sealed for TableConstructor<'_> {} +impl<'a> Node<'a> for TableConstructor<'a> { + fn start_position(&self) -> Option { + self.braces().tokens().0.start_position() + } + + fn end_position(&self) -> Option { + self.braces().tokens().1.end_position() + } + + fn similar(&self, other: &Self) -> bool { + self.braces().similar(other.braces()) && self.fields().similar(other.fields()) + } + + fn tokens<'b>(&'b self) -> Tokens<'a, 'b> { + let mut items = Vec::new(); + let (start_brace, end_brace) = self.braces().tokens(); + items.push(TokenItem::TokenReference(start_brace)); + items.push(TokenItem::MoreTokens(self.fields())); + items.push(TokenItem::TokenReference(end_brace)); + + Tokens { items } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{parse, visitors::Visitor}; + + // Test TableConstructor nodes are correctly ordered + #[test] + fn test_table_constructor_tokens() { + let ast = parse("local x = {true}").unwrap(); + + struct NodesChecker; + impl<'ast> Visitor<'ast> for NodesChecker { + fn visit_table_constructor(&mut self, table_constructor: &TableConstructor<'ast>) { + let mut tokens = table_constructor.tokens(); + assert!(tokens + .next() + .unwrap() + .similar(&TokenReference::symbol("{").unwrap())); + assert!(tokens + .next() + .unwrap() + .similar(&TokenReference::symbol("true").unwrap())); + assert!(tokens + .next() + .unwrap() + .similar(&TokenReference::symbol("}").unwrap())); + } + } + + NodesChecker.visit_ast(&ast); + } +}