diff --git a/fluent-bundle/src/resolver/errors.rs b/fluent-bundle/src/resolver/errors.rs index 7606faba..9b9f8e36 100644 --- a/fluent-bundle/src/resolver/errors.rs +++ b/fluent-bundle/src/resolver/errors.rs @@ -31,7 +31,7 @@ where InlineExpression::FunctionReference { id, .. } => Self::Function { id: id.name.to_string(), }, - InlineExpression::MessageReference { id, attribute } => Self::Message { + InlineExpression::MessageReference { id, attribute, .. } => Self::Message { id: id.name.to_string(), attribute: attribute.as_ref().map(|i| i.name.to_string()), }, diff --git a/fluent-bundle/src/resolver/expression.rs b/fluent-bundle/src/resolver/expression.rs index ce030e4c..a22505e6 100644 --- a/fluent-bundle/src/resolver/expression.rs +++ b/fluent-bundle/src/resolver/expression.rs @@ -23,15 +23,17 @@ impl<'bundle> WriteValue<'bundle> for ast::Expression<&'bundle str> { M: MemoizerKind, { match self { - Self::Inline(exp) => exp.write(w, scope), - Self::Select { selector, variants } => { + Self::Inline(exp, ..) => exp.write(w, scope), + Self::Select { + selector, variants, .. + } => { let selector = selector.resolve(scope); match selector { FluentValue::String(_) | FluentValue::Number(_) => { for variant in variants { let key = match variant.key { - ast::VariantKey::Identifier { name } => name.into(), - ast::VariantKey::NumberLiteral { value } => { + ast::VariantKey::Identifier { name, .. } => name.into(), + ast::VariantKey::NumberLiteral { value, .. } => { FluentValue::try_number(value) } }; @@ -59,7 +61,7 @@ impl<'bundle> WriteValue<'bundle> for ast::Expression<&'bundle str> { W: fmt::Write, { match self { - Self::Inline(exp) => exp.write_error(w), + Self::Inline(exp, ..) => exp.write_error(w), Self::Select { selector, .. } => selector.write_error(w), } } diff --git a/fluent-bundle/src/resolver/inline_expression.rs b/fluent-bundle/src/resolver/inline_expression.rs index 3f8c8d4f..cff88bd5 100644 --- a/fluent-bundle/src/resolver/inline_expression.rs +++ b/fluent-bundle/src/resolver/inline_expression.rs @@ -24,8 +24,8 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { M: MemoizerKind, { match self { - Self::StringLiteral { value } => unescape_unicode(w, value), - Self::MessageReference { id, attribute } => { + Self::StringLiteral { value, .. } => unescape_unicode(w, value), + Self::MessageReference { id, attribute, .. } => { if let Some(msg) = scope.bundle.get_entry_message(id.name) { if let Some(attr) = attribute { msg.attributes @@ -53,11 +53,12 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { scope.write_ref_error(w, self) } } - Self::NumberLiteral { value } => FluentValue::try_number(value).write(w, scope), + Self::NumberLiteral { value, .. } => FluentValue::try_number(value).write(w, scope), Self::TermReference { id, attribute, arguments, + .. } => { let (_, resolved_named_args) = scope.get_arguments(arguments.as_ref()); @@ -82,7 +83,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { scope.local_args = None; result } - Self::FunctionReference { id, arguments } => { + Self::FunctionReference { id, arguments, .. } => { let (resolved_positional_args, resolved_named_args) = scope.get_arguments(Some(arguments)); @@ -99,7 +100,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { scope.write_ref_error(w, self) } } - Self::VariableReference { id } => { + Self::VariableReference { id, .. } => { let args = scope.local_args.as_ref().or(scope.args); if let Some(arg) = args.and_then(|args| args.get(id.name)) { @@ -113,7 +114,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { w.write_char('}') } } - Self::Placeable { expression } => expression.write(w, scope), + Self::Placeable { expression, .. } => expression.write(w, scope), } } @@ -125,10 +126,12 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { Self::MessageReference { id, attribute: Some(attribute), + .. } => write!(w, "{}.{}", id.name, attribute.name), Self::MessageReference { id, attribute: None, + .. } => w.write_str(id.name), Self::TermReference { id, @@ -141,7 +144,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { .. } => write!(w, "-{}", id.name), Self::FunctionReference { id, .. } => write!(w, "{}()", id.name), - Self::VariableReference { id } => write!(w, "${}", id.name), + Self::VariableReference { id, .. } => write!(w, "${}", id.name), _ => unreachable!(), } } @@ -157,9 +160,9 @@ impl<'bundle> ResolveValue<'bundle> for ast::InlineExpression<&'bundle str> { M: MemoizerKind, { match self { - Self::StringLiteral { value } => unescape_unicode_to_string(value).into(), - Self::NumberLiteral { value } => FluentValue::try_number(value), - Self::VariableReference { id } => { + Self::StringLiteral { value, .. } => unescape_unicode_to_string(value).into(), + Self::NumberLiteral { value, .. } => FluentValue::try_number(value), + Self::VariableReference { id, .. } => { if let Some(local_args) = &scope.local_args { if let Some(arg) = local_args.get(id.name) { return arg.clone(); @@ -173,7 +176,7 @@ impl<'bundle> ResolveValue<'bundle> for ast::InlineExpression<&'bundle str> { } FluentValue::Error } - Self::FunctionReference { id, arguments } => { + Self::FunctionReference { id, arguments, .. } => { let (resolved_positional_args, resolved_named_args) = scope.get_arguments(Some(arguments)); diff --git a/fluent-bundle/src/resolver/pattern.rs b/fluent-bundle/src/resolver/pattern.rs index e20bfcde..d09547aa 100644 --- a/fluent-bundle/src/resolver/pattern.rs +++ b/fluent-bundle/src/resolver/pattern.rs @@ -32,14 +32,14 @@ impl<'bundle> WriteValue<'bundle> for ast::Pattern<&'bundle str> { } match elem { - ast::PatternElement::TextElement { value } => { + ast::PatternElement::TextElement { value, .. } => { if let Some(ref transform) = scope.bundle.transform { w.write_str(&transform(value))?; } else { w.write_str(value)?; } } - ast::PatternElement::Placeable { ref expression } => { + ast::PatternElement::Placeable { ref expression, .. } => { scope.placeables += 1; if scope.placeables > MAX_PLACEABLES { scope.dirty = true; @@ -51,13 +51,16 @@ impl<'bundle> WriteValue<'bundle> for ast::Pattern<&'bundle str> { && len > 1 && !matches!( expression, - ast::Expression::Inline(ast::InlineExpression::MessageReference { .. },) - | ast::Expression::Inline( - ast::InlineExpression::TermReference { .. }, - ) - | ast::Expression::Inline( - ast::InlineExpression::StringLiteral { .. }, - ) + ast::Expression::Inline( + ast::InlineExpression::MessageReference { .. }, + .. + ) | ast::Expression::Inline( + ast::InlineExpression::TermReference { .. }, + .. + ) | ast::Expression::Inline( + ast::InlineExpression::StringLiteral { .. }, + .. + ) ); if needs_isolation { w.write_char('\u{2068}')?; @@ -92,7 +95,7 @@ impl<'bundle> ResolveValue<'bundle> for ast::Pattern<&'bundle str> { let len = self.elements.len(); if len == 1 { - if let ast::PatternElement::TextElement { value } = self.elements[0] { + if let ast::PatternElement::TextElement { value, .. } = self.elements[0] { return scope .bundle .transform diff --git a/fluent-bundle/src/resolver/scope.rs b/fluent-bundle/src/resolver/scope.rs index 1ddff1a4..7f89c501 100644 --- a/fluent-bundle/src/resolver/scope.rs +++ b/fluent-bundle/src/resolver/scope.rs @@ -124,7 +124,7 @@ impl<'bundle, 'ast, 'args, 'errors, R, M> Scope<'bundle, 'ast, 'args, 'errors, R R: Borrow, M: MemoizerKind, { - if let Some(ast::CallArguments { positional, named }) = arguments { + if let Some(ast::CallArguments { positional, named, .. }) = arguments { let positional = positional.iter().map(|expr| expr.resolve(self)).collect(); let named = named diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index b8cf539a..e9efcd54 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -38,6 +38,7 @@ glob = "0.3" [features] default = [] +spans = [] serde = ["dep:serde"] json = ["serde", "dep:serde_json"] all-benchmarks = [] diff --git a/fluent-syntax/src/ast/helper.rs b/fluent-syntax/src/ast/helper.rs index 23851301..fb5b86d6 100644 --- a/fluent-syntax/src/ast/helper.rs +++ b/fluent-syntax/src/ast/helper.rs @@ -1,25 +1,49 @@ +use super::Comment; +#[cfg(feature = "spans")] +use super::Span; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use super::Comment; // This is a helper struct used to properly deserialize referential // JSON comments which are single continuous String, into a vec of // content slices. -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(untagged))] pub enum CommentDef { - Single { content: S }, - Multi { content: Vec }, + Single { + content: S, + #[cfg(feature = "spans")] + span: Span, + }, + Multi { + content: Vec, + #[cfg(feature = "spans")] + span: Span, + }, } impl From> for Comment { fn from(input: CommentDef) -> Self { match input { - CommentDef::Single { content } => Self { + CommentDef::Single { + content, + #[cfg(feature = "spans")] + span, + } => Self { content: vec![content], + #[cfg(feature = "spans")] + span, + }, + CommentDef::Multi { + content, + #[cfg(feature = "spans")] + span, + } => Self { + content, + #[cfg(feature = "spans")] + span, }, - CommentDef::Multi { content } => Self { content }, } } } diff --git a/fluent-syntax/src/ast/mod.rs b/fluent-syntax/src/ast/mod.rs index 273aceaf..21b1b879 100644 --- a/fluent-syntax/src/ast/mod.rs +++ b/fluent-syntax/src/ast/mod.rs @@ -88,6 +88,9 @@ mod helper; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +use std::ops::Deref; +#[cfg(feature = "spans")] +use std::ops::Range; /// Root node of a Fluent Translation List. /// @@ -111,10 +114,12 @@ use serde::{Deserialize, Serialize}; /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Resource { pub body: Vec>, + #[cfg(feature = "spans")] + pub span: Span, } /// A top-level node representing an entry of a [`Resource`]. @@ -193,7 +198,7 @@ pub struct Resource { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum Entry { @@ -202,7 +207,11 @@ pub enum Entry { Comment(Comment), GroupComment(Comment), ResourceComment(Comment), - Junk { content: S }, + Junk { + content: S, + #[cfg(feature = "spans")] + span: Span, + }, } /// Message node represents the most common [`Entry`] in an FTL [`Resource`]. @@ -253,13 +262,15 @@ pub enum Entry { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Message { pub id: Identifier, pub value: Option>, pub attributes: Vec>, pub comment: Option>, + #[cfg(feature = "spans")] + pub span: Span, } /// A Fluent [`Term`]. @@ -307,13 +318,15 @@ pub struct Message { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Term { pub id: Identifier, pub value: Pattern, pub attributes: Vec>, pub comment: Option>, + #[cfg(feature = "spans")] + pub span: Span, } /// Pattern contains a value of a [`Message`], [`Term`] or an [`Attribute`]. @@ -387,10 +400,12 @@ pub struct Term { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Pattern { pub elements: Vec>, + #[cfg(feature = "spans")] + pub span: Span, } /// `PatternElement` is an element of a [`Pattern`]. @@ -464,12 +479,20 @@ pub struct Pattern { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum PatternElement { - TextElement { value: S }, - Placeable { expression: Expression }, + TextElement { + value: S, + #[cfg(feature = "spans")] + span: Span, + }, + Placeable { + expression: Expression, + #[cfg(feature = "spans")] + span: Span, + }, } /// Attribute represents a part of a [`Message`] or [`Term`]. @@ -535,11 +558,13 @@ pub enum PatternElement { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Attribute { pub id: Identifier, pub value: Pattern, + #[cfg(feature = "spans")] + pub span: Span, } /// Identifier is part of nodes such as [`Message`], [`Term`] and [`Attribute`]. @@ -584,10 +609,12 @@ pub struct Attribute { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Identifier { pub name: S, + #[cfg(feature = "spans")] + pub span: Span, } /// Variant is a single branch of a value in a [`Select`](Expression::Select) expression. @@ -667,13 +694,15 @@ pub struct Identifier { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct Variant { pub key: VariantKey, pub value: Pattern, pub default: bool, + #[cfg(feature = "spans")] + pub span: Span, } /// A key of a [`Variant`]. @@ -752,12 +781,20 @@ pub struct Variant { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, PartialEq, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum VariantKey { - Identifier { name: S }, - NumberLiteral { value: S }, + Identifier { + name: S, + #[cfg(feature = "spans")] + span: Span, + }, + NumberLiteral { + value: S, + #[cfg(feature = "spans")] + span: Span, + }, } /// Fluent [`Comment`]. @@ -798,11 +835,13 @@ pub enum VariantKey { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(from = "helper::CommentDef"))] pub struct Comment { pub content: Vec, + #[cfg(feature = "spans")] + pub span: Span, } /// List of arguments for a [`FunctionReference`](InlineExpression::FunctionReference) or a @@ -879,12 +918,14 @@ pub struct Comment { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone, Default)] +#[derive(Debug, Clone, Default, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct CallArguments { pub positional: Vec>, pub named: Vec>, + #[cfg(feature = "spans")] + pub span: Span, } /// A key-value pair used in [`CallArguments`]. @@ -948,12 +989,14 @@ pub struct CallArguments { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct NamedArgument { pub name: Identifier, pub value: InlineExpression, + #[cfg(feature = "spans")] + pub span: Span, } /// A subset of expressions which can be used as [`Placeable`](PatternElement::Placeable), @@ -1004,7 +1047,8 @@ pub struct NamedArgument { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] + +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum InlineExpression { @@ -1053,7 +1097,11 @@ pub enum InlineExpression { /// } /// ); /// ``` - StringLiteral { value: S }, + StringLiteral { + value: S, + #[cfg(feature = "spans")] + span: Span, + }, /// A number literal. /// /// # Example @@ -1099,7 +1147,11 @@ pub enum InlineExpression { /// } /// ); /// ``` - NumberLiteral { value: S }, + NumberLiteral { + value: S, + #[cfg(feature = "spans")] + span: Span, + }, /// A function reference. /// /// # Example @@ -1151,6 +1203,8 @@ pub enum InlineExpression { FunctionReference { id: Identifier, arguments: CallArguments, + #[cfg(feature = "spans")] + span: Span, }, /// A reference to another message. /// @@ -1203,6 +1257,8 @@ pub enum InlineExpression { MessageReference { id: Identifier, attribute: Option>, + #[cfg(feature = "spans")] + span: Span, }, /// A reference to a term. /// @@ -1257,6 +1313,8 @@ pub enum InlineExpression { id: Identifier, attribute: Option>, arguments: Option>, + #[cfg(feature = "spans")] + span: Span, }, /// A reference to a variable. /// @@ -1305,7 +1363,11 @@ pub enum InlineExpression { /// } /// ); /// ``` - VariableReference { id: Identifier }, + VariableReference { + id: Identifier, + #[cfg(feature = "spans")] + span: Span, + }, /// A placeable which may contain another expression. /// /// # Example @@ -1357,7 +1419,26 @@ pub enum InlineExpression { /// } /// ); /// ``` - Placeable { expression: Box> }, + Placeable { + expression: Box>, + #[cfg(feature = "spans")] + span: Span, + }, +} + +#[cfg(feature = "spans")] +impl InlineExpression { + pub fn get_span(&self) -> &Span { + match self { + InlineExpression::StringLiteral { span, .. } + | InlineExpression::TermReference { span, .. } + | InlineExpression::VariableReference { span, .. } + | InlineExpression::Placeable { span, .. } + | InlineExpression::NumberLiteral { span, .. } + | InlineExpression::FunctionReference { span, .. } + | InlineExpression::MessageReference { span, .. } => span, + } + } } /// An expression that is either a select expression or an inline expression. @@ -1434,7 +1515,7 @@ pub enum InlineExpression { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(untagged))] pub enum Expression { @@ -1448,6 +1529,8 @@ pub enum Expression { Select { selector: InlineExpression, variants: Vec>, + #[cfg(feature = "spans")] + span: Span, }, /// An inline expression such as `${ username }`: @@ -1455,5 +1538,37 @@ pub enum Expression { /// ```ftl /// hello-user = Hello ${ username } /// ``` - Inline(InlineExpression), + Inline(InlineExpression, #[cfg(feature = "spans")] Span), +} + +/// A span of a node. Allows you to get the index of the start and end character of a node. +/// +/// # Example +/// +/// ``` +/// #![cfg(feature = "spans")] +/// +/// use fluent_syntax::parser; +/// use fluent_syntax::ast::*; +/// +/// let ftl = "hello-world = Hello, World!"; +/// +/// let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); +/// let Entry::Message(Message { ref id, .. }) = resource.body[0] else { unreachable!() }; +/// +/// assert_eq!(resource.span, Span { start: 0, end: 27 }); +/// assert_eq!(id.span, Span { start: 0, end: 11 }); // the span of hello-world identifier +/// ``` +#[cfg(feature = "spans")] +#[derive(Debug, Clone, Default, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Span(pub Range); + +#[cfg(feature = "spans")] +impl Deref for Span { + type Target = Range; + + fn deref(&self) -> &Self::Target { + &self.0 + } } diff --git a/fluent-syntax/src/parser/comment.rs b/fluent-syntax/src/parser/comment.rs index 1e30fc72..bfcff8ef 100644 --- a/fluent-syntax/src/parser/comment.rs +++ b/fluent-syntax/src/parser/comment.rs @@ -16,6 +16,9 @@ where pub(super) fn get_comment(&mut self) -> Result<(ast::Comment, Level)> { let mut level = Level::None; let mut content = vec![]; + + #[cfg(feature = "spans")] + let start_pos = self.ptr; while self.ptr < self.length { let line_level = self.get_comment_level(); @@ -47,7 +50,14 @@ where self.skip_eol(); } - Ok((ast::Comment { content }, level)) + Ok(( + ast::Comment { + content, + #[cfg(feature = "spans")] + span: ast::Span(start_pos..self.ptr), + }, + level, + )) } pub(super) fn skip_comment(&mut self) { diff --git a/fluent-syntax/src/parser/core.rs b/fluent-syntax/src/parser/core.rs index cb7149a8..ed69c4c3 100644 --- a/fluent-syntax/src/parser/core.rs +++ b/fluent-syntax/src/parser/core.rs @@ -3,7 +3,7 @@ use super::{ errors::{ErrorKind, ParserError}, slice::Slice, }; -use crate::ast; +use crate::ast::{self, Span}; pub type Result = std::result::Result; @@ -67,7 +67,11 @@ where err.slice = Some(entry_start..self.ptr); errors.push(err); let content = self.source.slice(entry_start..self.ptr); - body.push(ast::Entry::Junk { content }); + body.push(ast::Entry::Junk { + content, + #[cfg(feature = "spans")] + span: ast::Span(entry_start..self.ptr), + }); } } last_blank_count = self.skip_blank_block(); @@ -77,9 +81,9 @@ where body.push(ast::Entry::Comment(last_comment)); } if errors.is_empty() { - Ok(ast::Resource { body }) + Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span(0..self.length) }) } else { - Err((ast::Resource { body }, errors)) + Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span(0..self.length) }, errors)) } } @@ -123,6 +127,8 @@ where value: pattern, attributes, comment: None, + #[cfg(feature = "spans")] + span: ast::Span(entry_start..self.ptr), }) } @@ -145,6 +151,8 @@ where value, attributes, comment: None, + #[cfg(feature = "spans")] + span: ast::Span(entry_start..self.ptr), }) } else { error!( @@ -184,12 +192,19 @@ where let pattern = self.get_pattern()?; match pattern { - Some(pattern) => Ok(ast::Attribute { id, value: pattern }), + Some(pattern) => Ok(ast::Attribute { + id, + value: pattern, + #[cfg(feature = "spans")] + span: ast::Span(self.ptr - 1..self.ptr), + }), None => error!(ErrorKind::MissingValue, self.ptr), } } pub(super) fn get_identifier_unchecked(&mut self) -> ast::Identifier { + #[cfg(feature = "spans")] + let start = self.ptr - 1; let mut ptr = self.ptr; while matches!(get_byte!(self, ptr), Some(b) if b.is_ascii_alphanumeric() || *b == b'-' || *b == b'_') @@ -200,7 +215,11 @@ where let name = self.source.slice(self.ptr - 1..ptr); self.ptr = ptr; - ast::Identifier { name } + ast::Identifier { + name, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), + } } pub(super) fn get_identifier(&mut self) -> Result> { @@ -229,12 +248,22 @@ where self.skip_blank(); let key = if self.is_number_start() { + #[cfg(feature = "spans")] + let start = self.ptr; + ast::VariantKey::NumberLiteral { value: self.get_number_literal()?, + #[cfg(feature = "spans")] + span: Span(start..self.ptr), } } else { + #[cfg(feature = "spans")] + let start = self.ptr; + ast::VariantKey::Identifier { name: self.get_identifier()?.name, + #[cfg(feature = "spans")] + span: Span(start..self.ptr), } }; @@ -259,6 +288,9 @@ where } } + #[cfg(feature = "spans")] + let start = self.ptr; + if !self.take_byte_if(b'[') { break; } @@ -272,6 +304,8 @@ where key, value, default, + #[cfg(feature = "spans")] + span: ast::Span((if default { start - 1 } else { start })..self.ptr), }); self.skip_blank(); } else { @@ -293,9 +327,10 @@ where self.expect_byte(b'}')?; let invalid_expression_found = match &exp { - ast::Expression::Inline(ast::InlineExpression::TermReference { - ref attribute, .. - }) => attribute.is_some(), + ast::Expression::Inline( + ast::InlineExpression::TermReference { ref attribute, .. }, + .., + ) => attribute.is_some(), _ => false, }; if invalid_expression_found { diff --git a/fluent-syntax/src/parser/expression.rs b/fluent-syntax/src/parser/expression.rs index c5ccb32b..db72351d 100644 --- a/fluent-syntax/src/parser/expression.rs +++ b/fluent-syntax/src/parser/expression.rs @@ -9,6 +9,9 @@ where pub(super) fn get_expression(&mut self) -> Result> { let exp = self.get_inline_expression(false)?; + #[cfg(feature = "spans")] + let start_span = exp.get_span().start; + self.skip_blank(); if !self.is_current_byte(b'-') || !self.is_byte_at(b'>', self.ptr + 1) { @@ -17,7 +20,11 @@ where return error!(ErrorKind::TermAttributeAsPlaceable, self.ptr); } } - return Ok(ast::Expression::Inline(exp)); + return Ok(ast::Expression::Inline( + exp, + #[cfg(feature = "spans")] + ast::Span(start_span..self.ptr), + )); } match exp { @@ -60,6 +67,8 @@ where Ok(ast::Expression::Select { selector: exp, variants, + #[cfg(feature = "spans")] + span: ast::Span(start_span..self.ptr), }) } @@ -67,10 +76,10 @@ where &mut self, only_literal: bool, ) -> Result> { + let start = self.ptr; match get_current_byte!(self) { Some(b'"') => { self.ptr += 1; // " - let start = self.ptr; while let Some(b) = get_current_byte!(self) { match b { b'\\' => match get_byte!(self, self.ptr + 1) { @@ -99,12 +108,20 @@ where } self.expect_byte(b'"')?; - let slice = self.source.slice(start..self.ptr - 1); - Ok(ast::InlineExpression::StringLiteral { value: slice }) + let slice = self.source.slice(start + 1..self.ptr - 1); + Ok(ast::InlineExpression::StringLiteral { + value: slice, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), + }) } Some(b) if b.is_ascii_digit() => { let num = self.get_number_literal()?; - Ok(ast::InlineExpression::NumberLiteral { value: num }) + Ok(ast::InlineExpression::NumberLiteral { + value: num, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), + }) } Some(b'-') if !only_literal => { self.ptr += 1; // - @@ -117,17 +134,27 @@ where id, attribute, arguments, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), }) } else { self.ptr -= 1; let num = self.get_number_literal()?; - Ok(ast::InlineExpression::NumberLiteral { value: num }) + Ok(ast::InlineExpression::NumberLiteral { + value: num, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), + }) } } Some(b'$') if !only_literal => { self.ptr += 1; // $ let id = self.get_identifier()?; - Ok(ast::InlineExpression::VariableReference { id }) + Ok(ast::InlineExpression::VariableReference { + id, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), + }) } Some(b) if b.is_ascii_alphabetic() => { self.ptr += 1; @@ -138,10 +165,20 @@ where return error!(ErrorKind::ForbiddenCallee, self.ptr); } - Ok(ast::InlineExpression::FunctionReference { id, arguments }) + Ok(ast::InlineExpression::FunctionReference { + id, + arguments, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), + }) } else { let attribute = self.get_attribute_accessor()?; - Ok(ast::InlineExpression::MessageReference { id, attribute }) + Ok(ast::InlineExpression::MessageReference { + id, + attribute, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), + }) } } Some(b'{') if !only_literal => { @@ -149,6 +186,8 @@ where let exp = self.get_placeable()?; Ok(ast::InlineExpression::Placeable { expression: Box::new(exp), + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), }) } _ if only_literal => error!(ErrorKind::ExpectedLiteral, self.ptr), @@ -161,6 +200,8 @@ where if !self.take_byte_if(b'(') { return Ok(None); } + #[cfg(feature = "spans")] + let start = self.ptr - 1; let mut positional = vec![]; let mut named = vec![]; @@ -178,6 +219,7 @@ where if let ast::InlineExpression::MessageReference { ref id, attribute: None, + .. } = expr { self.skip_blank(); @@ -196,8 +238,12 @@ where named.push(ast::NamedArgument { name: ast::Identifier { name: id.name.clone(), + #[cfg(feature = "spans")] + span: id.span.clone(), }, value: val, + #[cfg(feature = "spans")] + span: ast::Span(id.span.start..self.ptr), }); } else { if !argument_names.is_empty() { @@ -219,6 +265,11 @@ where self.expect_byte(b')')?; - Ok(Some(ast::CallArguments { positional, named })) + Ok(Some(ast::CallArguments { + positional, + named, + #[cfg(feature = "spans")] + span: ast::Span(start..self.ptr), + })) } } diff --git a/fluent-syntax/src/parser/pattern.rs b/fluent-syntax/src/parser/pattern.rs index 9ca1229a..6c822646 100644 --- a/fluent-syntax/src/parser/pattern.rs +++ b/fluent-syntax/src/parser/pattern.rs @@ -1,6 +1,8 @@ use super::errors::{ErrorKind, ParserError}; use super::{core::Parser, core::Result, slice::Slice}; use crate::ast; +#[cfg(feature = "spans")] +use std::ops::Range; #[derive(Debug, PartialEq)] enum TextElementTermination { @@ -24,7 +26,7 @@ enum TextElementPosition { // cheaper since they'll happen on the pointers, rather than extracted slices. #[derive(Debug)] enum PatternElementPlaceholders { - Placeable(ast::Expression), + Placeable(ast::Expression, #[cfg(feature = "spans")] Range), // (start, end, indent, position) TextElement(usize, usize, usize, TextElementPosition), } @@ -49,6 +51,9 @@ where self.skip_blank_inline(); + #[cfg(feature = "spans")] + let start_pos = self.ptr; + let mut text_element_role = if self.skip_eol() { self.skip_blank_block(); TextElementPosition::LineStart @@ -58,12 +63,23 @@ where while self.ptr < self.length { if self.take_byte_if(b'{') { + #[cfg(feature = "spans")] + let slice_start = self.ptr - 1; if text_element_role == TextElementPosition::LineStart { common_indent = Some(0); } let exp = self.get_placeable()?; last_non_blank = Some(elements.len()); + + #[cfg(feature = "spans")] + elements.push(PatternElementPlaceholders::Placeable( + exp, + slice_start..self.ptr - 1, + )); + + #[cfg(not(feature = "spans"))] elements.push(PatternElementPlaceholders::Placeable(exp)); + text_element_role = TextElementPosition::Continuation; } else { let slice_start = self.ptr; @@ -127,6 +143,14 @@ where .take(last_non_blank + 1) .enumerate() .map(|(i, elem)| match elem { + #[cfg(feature = "spans")] + PatternElementPlaceholders::Placeable(expression, range) => { + ast::PatternElement::Placeable { + expression, + span: ast::Span(range), + } + } + #[cfg(not(feature = "spans"))] PatternElementPlaceholders::Placeable(expression) => { ast::PatternElement::Placeable { expression } } @@ -143,11 +167,19 @@ where if last_non_blank == i { value.trim(); } - ast::PatternElement::TextElement { value } + ast::PatternElement::TextElement { + value, + #[cfg(feature = "spans")] + span: ast::Span(start..end), + } } }) .collect(); - return Ok(Some(ast::Pattern { elements })); + return Ok(Some(ast::Pattern { + elements, + #[cfg(feature = "spans")] + span: ast::Span(start_pos..self.ptr), + })); } Ok(None) diff --git a/fluent-syntax/src/parser/runtime.rs b/fluent-syntax/src/parser/runtime.rs index e116ceae..a95383dc 100644 --- a/fluent-syntax/src/parser/runtime.rs +++ b/fluent-syntax/src/parser/runtime.rs @@ -34,16 +34,31 @@ where err.slice = Some(entry_start..self.ptr); errors.push(err); let content = self.source.slice(entry_start..self.ptr); - body.push(ast::Entry::Junk { content }); + body.push(ast::Entry::Junk { + content, + #[cfg(feature = "spans")] + span: ast::Span(entry_start..self.ptr), + }); } } self.skip_blank_block(); } if errors.is_empty() { - Ok(ast::Resource { body }) + Ok(ast::Resource { + body, + #[cfg(feature = "spans")] + span: ast::Span(0..self.length), + }) } else { - Err((ast::Resource { body }, errors)) + Err(( + ast::Resource { + body, + #[cfg(feature = "spans")] + span: ast::Span(0..self.length), + }, + errors, + )) } } diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index a3442429..01e72bf6 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -87,7 +87,7 @@ impl Serializer { Entry::Comment(comment) => self.serialize_free_comment(comment, "#"), Entry::GroupComment(comment) => self.serialize_free_comment(comment, "##"), Entry::ResourceComment(comment) => self.serialize_free_comment(comment, "###"), - Entry::Junk { content } => { + Entry::Junk { content, .. } => { if self.options.with_junk { self.serialize_junk(content.as_ref()); } @@ -203,9 +203,9 @@ impl Serializer { fn serialize_element<'s, S: Slice<'s>>(&mut self, elem: &PatternElement) { match elem { - PatternElement::TextElement { value } => self.writer.write_literal(value.as_ref()), - PatternElement::Placeable { expression } => match expression { - Expression::Inline(InlineExpression::Placeable { expression }) => { + PatternElement::TextElement { value, .. } => self.writer.write_literal(value.as_ref()), + PatternElement::Placeable { expression, .. } => match expression { + Expression::Inline(InlineExpression::Placeable { expression, .. }, ..) => { // A placeable inside a placeable is a special case because we // don't want the braces to look silly (e.g. "{ { Foo() } }"). self.writer.write_literal("{{ "); @@ -219,7 +219,7 @@ impl Serializer { self.serialize_expression(expression); self.writer.write_literal("}"); } - Expression::Inline(_) => { + Expression::Inline( .. ) => { self.writer.write_literal("{ "); self.serialize_expression(expression); self.writer.write_literal(" }"); @@ -230,8 +230,10 @@ impl Serializer { fn serialize_expression<'s, S: Slice<'s>>(&mut self, expr: &Expression) { match expr { - Expression::Inline(inline) => self.serialize_inline_expression(inline), - Expression::Select { selector, variants } => { + Expression::Inline(inline, ..) => self.serialize_inline_expression(inline), + Expression::Select { + selector, variants, .. + } => { self.serialize_select_expression(selector, variants); } } @@ -239,23 +241,26 @@ impl Serializer { fn serialize_inline_expression<'s, S: Slice<'s>>(&mut self, expr: &InlineExpression) { match expr { - InlineExpression::StringLiteral { value } => { + InlineExpression::StringLiteral { value, .. } => { self.writer.write_literal("\""); self.writer.write_literal(value.as_ref()); self.writer.write_literal("\""); } - InlineExpression::NumberLiteral { value } => self.writer.write_literal(value.as_ref()), + InlineExpression::NumberLiteral { value, .. } => { + self.writer.write_literal(value.as_ref()) + } InlineExpression::VariableReference { - id: Identifier { name: value }, + id: Identifier { name: value, .. }, + .. } => { self.writer.write_literal("$"); self.writer.write_literal(value.as_ref()); } - InlineExpression::FunctionReference { id, arguments } => { + InlineExpression::FunctionReference { id, arguments, .. } => { self.writer.write_literal(id.name.as_ref()); self.serialize_call_arguments(arguments); } - InlineExpression::MessageReference { id, attribute } => { + InlineExpression::MessageReference { id, attribute, .. } => { self.writer.write_literal(id.name.as_ref()); if let Some(attr) = attribute.as_ref() { @@ -267,6 +272,7 @@ impl Serializer { id, attribute, arguments, + .. } => { self.writer.write_literal("-"); self.writer.write_literal(id.name.as_ref()); @@ -279,7 +285,7 @@ impl Serializer { self.serialize_call_arguments(args); } } - InlineExpression::Placeable { expression } => { + InlineExpression::Placeable { expression, .. } => { self.writer.write_literal("{"); self.serialize_expression(expression); self.writer.write_literal("}"); @@ -319,7 +325,7 @@ impl Serializer { fn serialize_variant_key<'s, S: Slice<'s>>(&mut self, key: &VariantKey) { match key { - VariantKey::NumberLiteral { value } | VariantKey::Identifier { name: value } => { + VariantKey::NumberLiteral { value, .. } | VariantKey::Identifier { name: value, .. } => { self.writer.write_literal(value.as_ref()); } } @@ -361,13 +367,13 @@ impl<'s, S: Slice<'s>> Pattern { fn is_multiline(&self) -> bool { self.elements.iter().any(|elem| match elem { - PatternElement::TextElement { value } => value.as_ref().contains('\n'), - PatternElement::Placeable { expression } => is_select_expr(expression), + PatternElement::TextElement { value, .. } => value.as_ref().contains('\n'), + PatternElement::Placeable { expression, .. } => is_select_expr(expression), }) } fn has_leading_text_dot(&self) -> bool { - if let Some(PatternElement::TextElement { value }) = self.elements.first() { + if let Some(PatternElement::TextElement { value, .. }) = self.elements.first() { value.as_ref().starts_with('.') } else { false @@ -378,10 +384,10 @@ impl<'s, S: Slice<'s>> Pattern { fn is_select_expr<'s, S: Slice<'s>>(expr: &Expression) -> bool { match expr { Expression::Select { .. } => true, - Expression::Inline(InlineExpression::Placeable { expression }) => { + Expression::Inline(InlineExpression::Placeable { expression, .. }, ..) => { is_select_expr(expression) } - Expression::Inline(_) => false, + Expression::Inline(..) => false, } } @@ -478,12 +484,24 @@ mod test { macro_rules! text_message { ($name:expr, $value:expr) => { Entry::Message(Message { - id: Identifier { name: $name }, + id: Identifier { + name: $name, + #[cfg(feature = "spans")] + span: Span::default(), + }, value: Some(Pattern { - elements: vec![PatternElement::TextElement { value: $value }], + elements: vec![PatternElement::TextElement { + value: $value, + #[cfg(feature = "spans")] + span: Span::default(), + }], + #[cfg(feature = "spans")] + span: Span::default(), }), attributes: vec![], comment: None, + #[cfg(feature = "spans")] + span: Span::default(), }) }; } @@ -506,14 +524,14 @@ mod test { impl<'a> PatternElement<&'a str> { fn as_text(&mut self) -> &mut &'a str { match self { - Self::TextElement { value } => value, + Self::TextElement { value, .. } => value, _ => panic!("Expected TextElement"), } } fn as_expression(&mut self) -> &mut Expression<&'a str> { match self { - Self::Placeable { expression } => expression, + Self::Placeable { expression, .. } => expression, _ => panic!("Expected Placeable"), } } @@ -528,7 +546,7 @@ mod test { } fn as_inline_variable_id(&mut self) -> &mut Identifier<&'a str> { match self { - Self::Inline(InlineExpression::VariableReference { id }) => id, + Self::Inline(InlineExpression::VariableReference { id, .. }, ..) => id, _ => panic!("Expected Inline"), } } @@ -559,18 +577,38 @@ mod test { let mut ast = parse(message).expect("failed to parse ftl resource"); let one_variant = Variant { - key: VariantKey::Identifier { name: "one" }, + key: VariantKey::Identifier { name: "one", span: Span::default() }, value: Pattern { elements: vec![ PatternElement::Placeable { - expression: Expression::Inline(InlineExpression::VariableReference { - id: Identifier { name: "num" }, - }), + expression: Expression::Inline( + InlineExpression::VariableReference { + id: Identifier { + name: "num", + #[cfg(feature = "spans")] + span: Span::default(), + }, + #[cfg(feature = "spans")] + span: Span::default(), + }, + #[cfg(feature = "spans")] + Span::default(), + ), + #[cfg(feature = "spans")] + span: Span::default(), + }, + PatternElement::TextElement { + value: " bar", + #[cfg(feature = "spans")] + span: Span::default(), }, - PatternElement::TextElement { value: " bar" }, ], + #[cfg(feature = "spans")] + span: Span::default(), }, default: false, + #[cfg(feature = "spans")] + span: Span::default(), }; ast.body[0].as_message().as_pattern().elements[0] .as_expression() @@ -630,6 +668,8 @@ mod test { let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); ast.body[0].as_message().comment.replace(Comment { content: vec!["great message!"], + #[cfg(feature = "spans")] + span: Span::default(), }); assert_eq!("# great message!\nfoo = bar\n", serialize(&ast)); } diff --git a/fluent-syntax/tests/fixtures/formatted/any_char.ftl b/fluent-syntax/tests/fixtures/formatted/any_char.ftl new file mode 100644 index 00000000..76a1839b --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/any_char.ftl @@ -0,0 +1,6 @@ +# ↓ BEL, U+0007 +control0 = abcdef +# ↓ DEL, U+007F +delete = abcdef +# ↓ BPM, U+0082 +control1 = abcΒ‚def diff --git a/fluent-syntax/tests/fixtures/formatted/astral.ftl b/fluent-syntax/tests/fixtures/formatted/astral.ftl new file mode 100644 index 00000000..f650be0f --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/astral.ftl @@ -0,0 +1,14 @@ +face-with-tears-of-joy = πŸ˜‚ +tetragram-for-centre = πŒ† +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = { "\uD83D\uDE02" } +surrogates-in-adjacent-strings = { "\uD83D" }{ "\uDE02" } +emoji-in-text = A face πŸ˜‚ with tears of joy. +emoji-in-string = { "A face πŸ˜‚ with tears of joy." } + +# ERROR Invalid identifier + +# ERROR Invalid expression + +# ERROR Invalid variant key + diff --git a/fluent-syntax/tests/fixtures/formatted/call_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/call_expressions.ftl new file mode 100644 index 00000000..ffef8dca --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/call_expressions.ftl @@ -0,0 +1,56 @@ +## Function names + +valid-func-name-01 = { FUN1() } +valid-func-name-02 = { FUN_FUN() } +valid-func-name-03 = { FUN-FUN() } + +# JUNK 0 is not a valid Identifier start + +# JUNK Function names may not be lowercase + +# JUNK Function names may not contain lowercase character + +# JUNK ? is not a valid Identifier character + +## Arguments + +positional-args = { FUN(1, "a", msg) } +named-args = { FUN(x: 1, y: "Y") } +dense-named-args = { FUN(x: 1, y: "Y") } +mixed-args = { FUN(1, "a", msg, x: 1, y: "Y") } + +# ERROR Positional arg must not follow keyword args + +# ERROR Named arguments must be unique + +## Whitespace around arguments + +sparse-inline-call = { FUN("a", msg, x: 1) } +empty-inline-call = { FUN() } +multiline-call = { FUN("a", msg, x: 1) } +sparse-multiline-call = { FUN("a", msg, x: 1) } +empty-multiline-call = { FUN() } +unindented-arg-number = { FUN(1) } +unindented-arg-string = { FUN("a") } +unindented-arg-msg-ref = { FUN(msg) } +unindented-arg-term-ref = { FUN(-msg) } +unindented-arg-var-ref = { FUN($var) } +unindented-arg-call = { FUN(OTHER()) } +unindented-named-arg = { FUN(x: 1) } +unindented-closing-paren = { FUN(x) } + +## Optional trailing comma + +one-argument = { FUN(1) } +many-arguments = { FUN(1, 2, 3) } +inline-sparse-args = { FUN(1, 2, 3) } +mulitline-args = { FUN(1, 2) } +mulitline-sparse-args = { FUN(1, 2) } + +## Syntax errors for trailing comma + +## Whitespace in named arguments + +sparse-named-arg = { FUN(x: 1, y: 2, z: 3) } +unindented-colon = { FUN(x: 1) } +unindented-value = { FUN(x: 1) } diff --git a/fluent-syntax/tests/fixtures/formatted/callee_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/callee_expressions.ftl new file mode 100644 index 00000000..edc68ab5 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/callee_expressions.ftl @@ -0,0 +1,36 @@ +## Callees in placeables. + +function-callee-placeable = { FUNCTION() } +term-callee-placeable = { -term() } + +# ERROR Messages cannot be parameterized. + +# ERROR Equivalent to a MessageReference callee. + +# ERROR Message attributes cannot be parameterized. + +# ERROR Term attributes may not be used in Placeables. + +# ERROR Variables cannot be parameterized. + +## Callees in selectors. + +function-callee-selector = + { FUNCTION() -> + *[key] Value + } +term-attr-callee-selector = + { -term.attr() -> + *[key] Value + } + +# ERROR Messages cannot be parameterized. + +# ERROR Equivalent to a MessageReference callee. + +# ERROR Message attributes cannot be parameterized. + +# ERROR Term values may not be used as selectors. + +# ERROR Variables cannot be parameterized. + diff --git a/fluent-syntax/tests/fixtures/formatted/comments.ftl b/fluent-syntax/tests/fixtures/formatted/comments.ftl new file mode 100644 index 00000000..45754cd9 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/comments.ftl @@ -0,0 +1,22 @@ +# Standalone Comment + +# Message Comment +foo = Foo +# Term Comment +# with a blank last line. +# +-term = Term + +# Another standalone +# +# with indent + + +## Group Comment + + +### Resource Comment + + +# Errors + diff --git a/fluent-syntax/tests/fixtures/formatted/cr.ftl b/fluent-syntax/tests/fixtures/formatted/cr.ftl new file mode 100644 index 00000000..ace98efb --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/cr.ftl @@ -0,0 +1,2 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_comment.ftl b/fluent-syntax/tests/fixtures/formatted/eof_comment.ftl new file mode 100644 index 00000000..d262c3b3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_comment.ftl @@ -0,0 +1,5 @@ +### NOTE: Disable final newline insertion when editing this file. + + +# No EOL + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_empty.ftl b/fluent-syntax/tests/fixtures/formatted/eof_empty.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/fixtures/formatted/eof_id.ftl b/fluent-syntax/tests/fixtures/formatted/eof_id.ftl new file mode 100644 index 00000000..c938b0bc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_id.ftl @@ -0,0 +1,2 @@ +### NOTE: Disable final newline insertion when editing this file. + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_id_equals.ftl b/fluent-syntax/tests/fixtures/formatted/eof_id_equals.ftl new file mode 100644 index 00000000..c938b0bc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_id_equals.ftl @@ -0,0 +1,2 @@ +### NOTE: Disable final newline insertion when editing this file. + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_junk.ftl b/fluent-syntax/tests/fixtures/formatted/eof_junk.ftl new file mode 100644 index 00000000..c938b0bc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_junk.ftl @@ -0,0 +1,2 @@ +### NOTE: Disable final newline insertion when editing this file. + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_value.ftl b/fluent-syntax/tests/fixtures/formatted/eof_value.ftl new file mode 100644 index 00000000..23ee1b60 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_value.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +no-eol = No EOL diff --git a/fluent-syntax/tests/fixtures/formatted/escaped_characters.ftl b/fluent-syntax/tests/fixtures/formatted/escaped_characters.ftl new file mode 100644 index 00000000..29c56f50 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/escaped_characters.ftl @@ -0,0 +1,38 @@ +## Literal text + +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{ placeable } +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 + +## String literals + +quote-in-string = { "\"" } +backslash-in-string = { "\\" } + +# ERROR Mismatched quote + +# ERROR Unknown escape + +# ERROR Multiline literal + +## Unicode escapes + +string-unicode-4digits = { "\u0041" } +escape-unicode-4digits = { "\\u0041" } +string-unicode-6digits = { "\U01F602" } +escape-unicode-6digits = { "\\U01F602" } +# OK The trailing "00" is part of the literal value. +string-too-many-4digits = { "\u004100" } +# OK The trailing "00" is part of the literal value. +string-too-many-6digits = { "\U01F60200" } + +# ERROR Too few hex digits after \u. + +# ERROR Too few hex digits after \U. + +## Literal braces + +brace-open = An opening { "{" } brace. +brace-close = A closing { "}" } brace. diff --git a/fluent-syntax/tests/fixtures/formatted/junk.ftl b/fluent-syntax/tests/fixtures/formatted/junk.ftl new file mode 100644 index 00000000..d0ff0158 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junk.ftl @@ -0,0 +1,16 @@ +## Two adjacent Junks. + +# A single Junk. + +# A single Junk. + +# The COMMENT ends this junk. + +# COMMENT + + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. + +# COMMENT + diff --git a/fluent-syntax/tests/fixtures/formatted/junked/any_char.ftl b/fluent-syntax/tests/fixtures/formatted/junked/any_char.ftl new file mode 100644 index 00000000..76a1839b --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/any_char.ftl @@ -0,0 +1,6 @@ +# ↓ BEL, U+0007 +control0 = abcdef +# ↓ DEL, U+007F +delete = abcdef +# ↓ BPM, U+0082 +control1 = abcΒ‚def diff --git a/fluent-syntax/tests/fixtures/formatted/junked/astral.ftl b/fluent-syntax/tests/fixtures/formatted/junked/astral.ftl new file mode 100644 index 00000000..f29fef0a --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/astral.ftl @@ -0,0 +1,21 @@ +face-with-tears-of-joy = πŸ˜‚ +tetragram-for-centre = πŒ† +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = { "\uD83D\uDE02" } +surrogates-in-adjacent-strings = { "\uD83D" }{ "\uDE02" } +emoji-in-text = A face πŸ˜‚ with tears of joy. +emoji-in-string = { "A face πŸ˜‚ with tears of joy." } + +# ERROR Invalid identifier + +err-πŸ˜‚ = Value + +# ERROR Invalid expression + +err-invalid-expression = { πŸ˜‚ } + +# ERROR Invalid variant key + +err-invalid-variant-key = { $sel -> + *[πŸ˜‚] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/call_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/call_expressions.ftl new file mode 100644 index 00000000..46a531fe --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/call_expressions.ftl @@ -0,0 +1,71 @@ +## Function names + +valid-func-name-01 = { FUN1() } +valid-func-name-02 = { FUN_FUN() } +valid-func-name-03 = { FUN-FUN() } + +# JUNK 0 is not a valid Identifier start + +invalid-func-name-01 = {0FUN()} +# JUNK Function names may not be lowercase + +invalid-func-name-02 = {fun()} +# JUNK Function names may not contain lowercase character + +invalid-func-name-03 = {Fun()} +# JUNK ? is not a valid Identifier character + +invalid-func-name-04 = {FUN?()} + +## Arguments + +positional-args = { FUN(1, "a", msg) } +named-args = { FUN(x: 1, y: "Y") } +dense-named-args = { FUN(x: 1, y: "Y") } +mixed-args = { FUN(1, "a", msg, x: 1, y: "Y") } + +# ERROR Positional arg must not follow keyword args + +shuffled-args = {FUN(1, x: 1, "a", y: "Y", msg)} + +# ERROR Named arguments must be unique + +duplicate-named-args = {FUN(x: 1, x: "X")} + + +## Whitespace around arguments + +sparse-inline-call = { FUN("a", msg, x: 1) } +empty-inline-call = { FUN() } +multiline-call = { FUN("a", msg, x: 1) } +sparse-multiline-call = { FUN("a", msg, x: 1) } +empty-multiline-call = { FUN() } +unindented-arg-number = { FUN(1) } +unindented-arg-string = { FUN("a") } +unindented-arg-msg-ref = { FUN(msg) } +unindented-arg-term-ref = { FUN(-msg) } +unindented-arg-var-ref = { FUN($var) } +unindented-arg-call = { FUN(OTHER()) } +unindented-named-arg = { FUN(x: 1) } +unindented-closing-paren = { FUN(x) } + +## Optional trailing comma + +one-argument = { FUN(1) } +many-arguments = { FUN(1, 2, 3) } +inline-sparse-args = { FUN(1, 2, 3) } +mulitline-args = { FUN(1, 2) } +mulitline-sparse-args = { FUN(1, 2) } + +## Syntax errors for trailing comma + +one-argument = {FUN(1,,)} +missing-arg = {FUN(,)} +missing-sparse-arg = {FUN( , )} + + +## Whitespace in named arguments + +sparse-named-arg = { FUN(x: 1, y: 2, z: 3) } +unindented-colon = { FUN(x: 1) } +unindented-value = { FUN(x: 1) } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/callee_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/callee_expressions.ftl new file mode 100644 index 00000000..ca98eae7 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/callee_expressions.ftl @@ -0,0 +1,58 @@ +## Callees in placeables. + +function-callee-placeable = { FUNCTION() } +term-callee-placeable = { -term() } + +# ERROR Messages cannot be parameterized. + +message-callee-placeable = {message()} +# ERROR Equivalent to a MessageReference callee. + +mixed-case-callee-placeable = {Function()} +# ERROR Message attributes cannot be parameterized. + +message-attr-callee-placeable = {message.attr()} +# ERROR Term attributes may not be used in Placeables. + +term-attr-callee-placeable = {-term.attr()} +# ERROR Variables cannot be parameterized. + +variable-callee-placeable = {$variable()} + + +## Callees in selectors. + +function-callee-selector = + { FUNCTION() -> + *[key] Value + } +term-attr-callee-selector = + { -term.attr() -> + *[key] Value + } + +# ERROR Messages cannot be parameterized. + +message-callee-selector = {message() -> + *[key] Value +} +# ERROR Equivalent to a MessageReference callee. + +mixed-case-callee-selector = {Function() -> + *[key] Value +} +# ERROR Message attributes cannot be parameterized. + +message-attr-callee-selector = {message.attr() -> + *[key] Value +} +# ERROR Term values may not be used as selectors. + +term-callee-selector = {-term() -> + *[key] Value +} +# ERROR Variables cannot be parameterized. + +variable-callee-selector = {$variable() -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/comments.ftl b/fluent-syntax/tests/fixtures/formatted/junked/comments.ftl new file mode 100644 index 00000000..be10bad2 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/comments.ftl @@ -0,0 +1,25 @@ +# Standalone Comment + +# Message Comment +foo = Foo +# Term Comment +# with a blank last line. +# +-term = Term + +# Another standalone +# +# with indent + + +## Group Comment + + +### Resource Comment + + +# Errors + +#error +##error +###error diff --git a/fluent-syntax/tests/fixtures/formatted/junked/cr.ftl b/fluent-syntax/tests/fixtures/formatted/junked/cr.ftl new file mode 100644 index 00000000..ace98efb --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/cr.ftl @@ -0,0 +1,2 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } + diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_comment.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_comment.ftl new file mode 100644 index 00000000..d262c3b3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_comment.ftl @@ -0,0 +1,5 @@ +### NOTE: Disable final newline insertion when editing this file. + + +# No EOL + diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_empty.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_empty.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_id.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_id.ftl new file mode 100644 index 00000000..63fa86d6 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_id.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_id_equals.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_id_equals.ftl new file mode 100644 index 00000000..7d0d953a --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_id_equals.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id = \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_junk.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_junk.ftl new file mode 100644 index 00000000..dbafd3a3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_junk.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +000 \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_value.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_value.ftl new file mode 100644 index 00000000..23ee1b60 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_value.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +no-eol = No EOL diff --git a/fluent-syntax/tests/fixtures/formatted/junked/escaped_characters.ftl b/fluent-syntax/tests/fixtures/formatted/junked/escaped_characters.ftl new file mode 100644 index 00000000..b6025b76 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/escaped_characters.ftl @@ -0,0 +1,46 @@ +## Literal text + +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{ placeable } +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 + +## String literals + +quote-in-string = { "\"" } +backslash-in-string = { "\\" } + +# ERROR Mismatched quote + +mismatched-quote = {"\\""} +# ERROR Unknown escape + +unknown-escape = {"\x"} +# ERROR Multiline literal + +invalid-multiline-literal = {" + "} + +## Unicode escapes + +string-unicode-4digits = { "\u0041" } +escape-unicode-4digits = { "\\u0041" } +string-unicode-6digits = { "\U01F602" } +escape-unicode-6digits = { "\\U01F602" } +# OK The trailing "00" is part of the literal value. +string-too-many-4digits = { "\u004100" } +# OK The trailing "00" is part of the literal value. +string-too-many-6digits = { "\U01F60200" } + +# ERROR Too few hex digits after \u. + +string-too-few-4digits = {"\u41"} +# ERROR Too few hex digits after \U. + +string-too-few-6digits = {"\U1F602"} + +## Literal braces + +brace-open = An opening { "{" } brace. +brace-close = A closing { "}" } brace. diff --git a/fluent-syntax/tests/fixtures/formatted/junked/junk.ftl b/fluent-syntax/tests/fixtures/formatted/junked/junk.ftl new file mode 100644 index 00000000..6085bb92 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/junk.ftl @@ -0,0 +1,28 @@ +## Two adjacent Junks. + +err01 = {1x} +err02 = {2x} + +# A single Junk. + +err03 = {1x +2 + +# A single Junk. + +Δ…=Invalid identifier +Δ‡=Another one + +# The COMMENT ends this junk. + +err04 = { +# COMMENT + + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. + +err04 = { +# COMMENT + +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/leading_dots.ftl b/fluent-syntax/tests/fixtures/formatted/junked/leading_dots.ftl new file mode 100644 index 00000000..e87e4097 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/leading_dots.ftl @@ -0,0 +1,70 @@ +key01 = .Value +key02 = …Value +key03 = { "." }Value +key04 = { "." }Value +key05 = + Value + { "." }Continued +key06 = .Value +{ "." }Continued +# MESSAGE (value = "Value", attributes = []) +# JUNK (attr .Continued" must have a value) +key07 = Value + .Continued + +# JUNK (attr .Value must have a value) + +key08 = + .Value + +# JUNK (attr .Value must have a value) + +key09 = + .Value + Continued + +key10 = + .Value = + which is an attribute + Continued +key11 = + { "." }Value = which looks like an attribute + Continued +key12 = + .accesskey = A +key13 = + .attribute = .Value +key14 = + .attribute = { "." }Value +key15 = + { 1 -> + [one] .Value + *[other] { "." }Value + } + +# JUNK (variant must have a value) + +key16 = + { 1 -> + *[one] + .Value + } + +# JUNK (unclosed placeable) + +key17 = + { 1 -> + *[one] Value + .Continued + } + +# JUNK (attr .Value must have a value) + +key18 = +.Value + +key19 = + .attribute = + Value + Continued +key20 = { "." }Value diff --git a/fluent-syntax/tests/fixtures/formatted/junked/literal_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/literal_expressions.ftl new file mode 100644 index 00000000..8b45c427 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/literal_expressions.ftl @@ -0,0 +1,3 @@ +string-expression = { "abc" } +number-expression = { 123 } +number-expression = { -3.14 } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/member_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/member_expressions.ftl new file mode 100644 index 00000000..3a1089c8 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/member_expressions.ftl @@ -0,0 +1,23 @@ +## Member expressions in placeables. + +# OK Message attributes may be interpolated in values. +message-attribute-expression-placeable = { msg.attr } + +# ERROR Term attributes may not be used for interpolation. + +term-attribute-expression-placeable = {-term.attr} + + +## Member expressions in selectors. + +# OK Term attributes may be used as selectors. +term-attribute-expression-selector = + { -term.attr -> + *[key] Value + } + +# ERROR Message attributes may not be used as selectors. + +message-attribute-expression-selector = {msg.attr -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/messages.ftl b/fluent-syntax/tests/fixtures/formatted/junked/messages.ftl new file mode 100644 index 00000000..a5bcd1f1 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/messages.ftl @@ -0,0 +1,45 @@ +key01 = Value +key02 = Value + .attr = Attribute +key02 = Value + .attr1 = Attribute 1 + .attr2 = Attribute 2 +key03 = + .attr = Attribute +key04 = + .attr1 = Attribute 1 + .attr2 = Attribute 2 +# < whitespace > +key05 = + .attr1 = Attribute 1 +no-whitespace = Value + .attr1 = Attribute 1 +extra-whitespace = Value + .attr1 = Attribute 1 +key06 = { "" } + +# JUNK Missing value + +key07 = + +# JUNK Missing = + +key08 + +KEY09 = Value 09 +key-10 = Value 10 +key_11 = Value 11 +key-12- = Value 12 +key_13_ = Value 13 + +# JUNK Invalid id + +0err-14 = Value 14 + +# JUNK Invalid id + +err-15? = Value 15 + +# JUNK Invalid id + +err-Δ…Δ™-16 = Value 16 diff --git a/fluent-syntax/tests/fixtures/formatted/junked/mixed_entries.ftl b/fluent-syntax/tests/fixtures/formatted/junked/mixed_entries.ftl new file mode 100644 index 00000000..57480b45 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/mixed_entries.ftl @@ -0,0 +1,24 @@ +# License Comment + + +### Resource Comment + +-brand-name = Aurora + +## Group Comment + +key01 = + .attr = Attribute +Δ…=Invalid identifier +Δ‡=Another one + +# Message Comment +key02 = Value + +# Standalone Comment + + .attr = Dangling attribute + +# There are 5 spaces on the line between key03 and key04. +key03 = Value 03 +key04 = Value 04 diff --git a/fluent-syntax/tests/fixtures/formatted/junked/numbers.ftl b/fluent-syntax/tests/fixtures/formatted/junked/numbers.ftl new file mode 100644 index 00000000..08d6e6b1 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/numbers.ftl @@ -0,0 +1,32 @@ +int-zero = { 0 } +int-positive = { 1 } +int-negative = { -1 } +int-negative-zero = { -0 } +int-positive-padded = { 01 } +int-negative-padded = { -01 } +int-zero-padded = { 00 } +int-negative-zero-padded = { -00 } +float-zero = { 0.0 } +float-positive = { 0.01 } +float-positive-one = { 1.03 } +float-positive-without-fraction = { 1.000 } +float-negative = { -0.01 } +float-negative-one = { -1.03 } +float-negative-zero = { -0.0 } +float-negative-without-fraction = { -1.000 } +float-positive-padded-left = { 01.03 } +float-positive-padded-right = { 1.0300 } +float-positive-padded-both = { 01.0300 } +float-negative-padded-left = { -01.03 } +float-negative-padded-right = { -1.0300 } +float-negative-padded-both = { -01.0300 } + +## ERRORS + +err01 = {1.} +err02 = {.02} +err03 = {1.02.03} +err04 = {1. 02} +err05 = {1 .02} +err06 = {- 1} +err07 = {1,02} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/obsolete.ftl b/fluent-syntax/tests/fixtures/formatted/junked/obsolete.ftl new file mode 100644 index 00000000..e0869e3a --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/obsolete.ftl @@ -0,0 +1,29 @@ +### The syntax in this file has been discontinued. It is no longer part of the +### Fluent specification and should not be implemented nor used. We're keeping +### these fixtures around to protect against accidental syntax reuse. + + +## Variant lists. + +message-variant-list = + { + *[key] Value + } + +-term-variant-list = + { + *[key] Value + } + + +## Variant expressions. + +message-variant-expression-placeable = {msg[case]} +message-variant-expression-selector = {msg[case] -> + *[key] Value +} + +term-variant-expression-placeable = {-term[case]} +term-variant-expression-selector = {-term[case] -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/placeables.ftl b/fluent-syntax/tests/fixtures/formatted/junked/placeables.ftl new file mode 100644 index 00000000..1a2013b9 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/placeables.ftl @@ -0,0 +1,19 @@ +nested-placeable = {{ {1} }} +padded-placeable = { 1 } +sparse-placeable = {{ 1 }} + +# ERROR Unmatched opening brace + +unmatched-open1 = { 1 + +# ERROR Unmatched opening brace + +unmatched-open2 = {{ 1 } + +# ERROR Unmatched closing brace + +unmatched-close1 = 1 } + +# ERROR Unmatched closing brace + +unmatched-close2 = { 1 }} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/reference_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/reference_expressions.ftl new file mode 100644 index 00000000..1f4b8674 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/reference_expressions.ftl @@ -0,0 +1,32 @@ +## Reference expressions in placeables. + +message-reference-placeable = { msg } +term-reference-placeable = { -term } +variable-reference-placeable = { $var } +# Function references are invalid outside of call expressions. +# This parses as a valid MessageReference. +function-reference-placeable = { FUN } + +## Reference expressions in selectors. + +variable-reference-selector = + { $var -> + *[key] Value + } + +# ERROR Message values may not be used as selectors. + +message-reference-selector = {msg -> + *[key] Value +} +# ERROR Term values may not be used as selectors. + +term-reference-selector = {-term -> + *[key] Value +} +# ERROR Function references are invalid outside of call expressions, and this +# parses as a MessageReference which isn't a valid selector. + +function-expression-selector = {FUN -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/select_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/select_expressions.ftl new file mode 100644 index 00000000..4598442e --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/select_expressions.ftl @@ -0,0 +1,68 @@ +new-messages = + { BUILTIN() -> + [0] Zero + *[other] { "" }Other + } +valid-selector-term-attribute = + { -term.case -> + *[key] value + } + +# ERROR Term values are not valid selectors + +invalid-selector-term-value = + { -term -> + *[key] value + } + +# ERROR CallExpressions on Terms are similar to TermReferences + +invalid-selector-term-variant = + { -term(case: "nominative") -> + *[key] value + } + +# ERROR Nested expressions are not valid selectors + +invalid-selector-nested-expression = + { { 3 } -> + *[key] default + } + +# ERROR Select expressions are not valid selectors + +invalid-selector-select-expression = + { { $sel -> + *[key] value + } -> + *[key] default + } + +empty-variant = + { $sel -> + *[key] { "" } + } +reduced-whitespace = + { FOO() -> + *[key] { "" } + } +nested-select = + { $sel -> + *[one] + { $sel -> + *[two] Value + } + } + +# ERROR Missing selector + +missing-selector = + { + *[key] Value + } + +# ERROR Missing line end after variant list + +missing-line-end = + { $sel -> + *[key] Value} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/select_indent.ftl b/fluent-syntax/tests/fixtures/formatted/junked/select_indent.ftl new file mode 100644 index 00000000..3c21d71f --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/select_indent.ftl @@ -0,0 +1,70 @@ +select-1tbs-inline = + { $selector -> + *[key] Value + } +select-1tbs-newline = + { $selector -> + *[key] Value + } +select-1tbs-indent = + { $selector -> + *[key] Value + } +select-allman-inline = + { $selector -> + *[key] Value + [other] Other + } +select-allman-newline = + { $selector -> + *[key] Value + } +select-allman-indent = + { $selector -> + *[key] Value + } +select-gnu-inline = + { $selector -> + *[key] Value + } +select-gnu-newline = + { $selector -> + *[key] Value + } +select-gnu-indent = + { $selector -> + *[key] Value + } +select-no-indent = + { $selector -> + *[key] Value + [other] Other + } +select-no-indent-multiline = + { $selector -> + *[key] + Value + Continued + [other] + Other + Multiline + } + +# ERROR (Multiline text must be indented) + +select-no-indent-multiline = { $selector -> + *[key] Value +Continued without indent. +} + +select-flat = + { $selector -> + *[key] Value + [other] Other + } +# Each line ends with 5 spaces. +select-flat-with-trailing-spaces = + { $selector -> + *[key] Value + [other] Other + } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/sparse_entries.ftl b/fluent-syntax/tests/fixtures/formatted/junked/sparse_entries.ftl new file mode 100644 index 00000000..dfa85945 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/sparse_entries.ftl @@ -0,0 +1,17 @@ +key01 = Value +key02 = + .attr = Attribute +key03 = + Value + Continued + + + Over multiple + Lines + .attr = Attribute +key05 = Value +key06 = + { 1 -> + [one] One + *[two] Two + } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/special_chars.ftl b/fluent-syntax/tests/fixtures/formatted/junked/special_chars.ftl new file mode 100644 index 00000000..5224bad7 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/special_chars.ftl @@ -0,0 +1,14 @@ +## OK + +bracket-inline = [Value] +dot-inline = .Value +star-inline = *Value + +## ERRORS + +bracket-newline = + [Value] +dot-newline = + .Value +star-newline = + *Value diff --git a/fluent-syntax/tests/fixtures/formatted/junked/tab.ftl b/fluent-syntax/tests/fixtures/formatted/junked/tab.ftl new file mode 100644 index 00000000..46b5aa9f --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/tab.ftl @@ -0,0 +1,21 @@ +# OK (tab after = is part of the value) +key01 = Value 01 + +# Error (tab before =) + +key02 = Value 02 + +# Error (tab is not a valid indent) + +key03 = + This line isn't properly indented. + +# Partial Error (tab is not a valid indent) +key04 = This line is indented by 4 spaces, + whereas this line by 1 tab. + +# OK (value is a single tab) +key05 = +# OK (attribute value is two tabs) +key06 = + .attr = diff --git a/fluent-syntax/tests/fixtures/formatted/junked/term_parameters.ftl b/fluent-syntax/tests/fixtures/formatted/junked/term_parameters.ftl new file mode 100644 index 00000000..8ee2ed74 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/term_parameters.ftl @@ -0,0 +1,8 @@ +-term = + { $arg -> + *[key] Value + } +key01 = { -term } +key02 = { -term() } +key03 = { -term(arg: 1) } +key04 = { -term("positional", narg1: 1, narg2: 2) } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/terms.ftl b/fluent-syntax/tests/fixtures/formatted/junked/terms.ftl new file mode 100644 index 00000000..4bd9e6cc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/terms.ftl @@ -0,0 +1,32 @@ +-term01 = Value + .attr = Attribute +-term02 = { "" } + +# JUNK Missing value + +-term03 = + .attr = Attribute + +# JUNK Missing value +# < whitespace > + +-term04 = + .attr1 = Attribute 1 + +# JUNK Missing value + +-term05 = + +# JUNK Missing value +# < whitespace > + +-term06 = + +# JUNK Missing = + +-term07 + +-term08 = Value + .attr = Attribute +-term09 = Value + .attr = Attribute diff --git a/fluent-syntax/tests/fixtures/formatted/junked/variables.ftl b/fluent-syntax/tests/fixtures/formatted/junked/variables.ftl new file mode 100644 index 00000000..1a12cc59 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/variables.ftl @@ -0,0 +1,17 @@ +key01 = { $var } +key02 = { $var } +key03 = { $var } +key04 = { $var } + +## Errors + + +# ERROR Missing variable identifier + +err01 = {$} +# ERROR Double $$ + +err02 = {$$var} +# ERROR Invalid first char of the identifier + +err03 = {$-var} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/variant_keys.ftl b/fluent-syntax/tests/fixtures/formatted/junked/variant_keys.ftl new file mode 100644 index 00000000..e3ee17f2 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/variant_keys.ftl @@ -0,0 +1,37 @@ +simple-identifier = + { $sel -> + *[key] value + } +identifier-surrounded-by-whitespace = + { $sel -> + *[key] value + } +int-number = + { $sel -> + *[1] value + } +float-number = + { $sel -> + *[3.14] value + } + +# ERROR + +invalid-identifier = + { $sel -> + *[two words] value + } + +# ERROR + +invalid-int = + { $sel -> + *[1 apple] value + } + +# ERROR + +invalid-int = + { $sel -> + *[3.14 apples] value + } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/whitespace_in_value.ftl b/fluent-syntax/tests/fixtures/formatted/junked/whitespace_in_value.ftl new file mode 100644 index 00000000..0cbcd371 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/whitespace_in_value.ftl @@ -0,0 +1,10 @@ +# Caution, lines 6 and 7 contain white-space-only lines +key = + first line + + + + + + + last line diff --git a/fluent-syntax/tests/fixtures/formatted/junked/zero_length.ftl b/fluent-syntax/tests/fixtures/formatted/junked/zero_length.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/fixtures/formatted/leading_dots.ftl b/fluent-syntax/tests/fixtures/formatted/leading_dots.ftl new file mode 100644 index 00000000..d63f60e3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/leading_dots.ftl @@ -0,0 +1,46 @@ +key01 = .Value +key02 = …Value +key03 = { "." }Value +key04 = { "." }Value +key05 = + Value + { "." }Continued +key06 = .Value +{ "." }Continued +# MESSAGE (value = "Value", attributes = []) +# JUNK (attr .Continued" must have a value) +key07 = Value +# JUNK (attr .Value must have a value) + +# JUNK (attr .Value must have a value) + +key10 = + .Value = + which is an attribute + Continued +key11 = + { "." }Value = which looks like an attribute + Continued +key12 = + .accesskey = A +key13 = + .attribute = .Value +key14 = + .attribute = { "." }Value +key15 = + { 1 -> + [one] .Value + *[other] { "." }Value + } + +# JUNK (variant must have a value) + +# JUNK (unclosed placeable) + +# JUNK (attr .Value must have a value) + +key19 = + .attribute = + Value + Continued +key20 = { "." }Value diff --git a/fluent-syntax/tests/fixtures/formatted/literal_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/literal_expressions.ftl new file mode 100644 index 00000000..8b45c427 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/literal_expressions.ftl @@ -0,0 +1,3 @@ +string-expression = { "abc" } +number-expression = { 123 } +number-expression = { -3.14 } diff --git a/fluent-syntax/tests/fixtures/formatted/member_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/member_expressions.ftl new file mode 100644 index 00000000..809c2608 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/member_expressions.ftl @@ -0,0 +1,17 @@ +## Member expressions in placeables. + +# OK Message attributes may be interpolated in values. +message-attribute-expression-placeable = { msg.attr } + +# ERROR Term attributes may not be used for interpolation. + +## Member expressions in selectors. + +# OK Term attributes may be used as selectors. +term-attribute-expression-selector = + { -term.attr -> + *[key] Value + } + +# ERROR Message attributes may not be used as selectors. + diff --git a/fluent-syntax/tests/fixtures/formatted/messages.ftl b/fluent-syntax/tests/fixtures/formatted/messages.ftl new file mode 100644 index 00000000..72403bd3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/messages.ftl @@ -0,0 +1,36 @@ +key01 = Value +key02 = Value + .attr = Attribute +key02 = Value + .attr1 = Attribute 1 + .attr2 = Attribute 2 +key03 = + .attr = Attribute +key04 = + .attr1 = Attribute 1 + .attr2 = Attribute 2 +# < whitespace > +key05 = + .attr1 = Attribute 1 +no-whitespace = Value + .attr1 = Attribute 1 +extra-whitespace = Value + .attr1 = Attribute 1 +key06 = { "" } + +# JUNK Missing value + +# JUNK Missing = + +KEY09 = Value 09 +key-10 = Value 10 +key_11 = Value 11 +key-12- = Value 12 +key_13_ = Value 13 + +# JUNK Invalid id + +# JUNK Invalid id + +# JUNK Invalid id + diff --git a/fluent-syntax/tests/fixtures/formatted/mixed_entries.ftl b/fluent-syntax/tests/fixtures/formatted/mixed_entries.ftl new file mode 100644 index 00000000..f8e54c29 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/mixed_entries.ftl @@ -0,0 +1,19 @@ +# License Comment + + +### Resource Comment + +-brand-name = Aurora + +## Group Comment + +key01 = + .attr = Attribute +# Message Comment +key02 = Value + +# Standalone Comment + +# There are 5 spaces on the line between key03 and key04. +key03 = Value 03 +key04 = Value 04 diff --git a/fluent-syntax/tests/fixtures/formatted/numbers.ftl b/fluent-syntax/tests/fixtures/formatted/numbers.ftl new file mode 100644 index 00000000..521f5a23 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/numbers.ftl @@ -0,0 +1,25 @@ +int-zero = { 0 } +int-positive = { 1 } +int-negative = { -1 } +int-negative-zero = { -0 } +int-positive-padded = { 01 } +int-negative-padded = { -01 } +int-zero-padded = { 00 } +int-negative-zero-padded = { -00 } +float-zero = { 0.0 } +float-positive = { 0.01 } +float-positive-one = { 1.03 } +float-positive-without-fraction = { 1.000 } +float-negative = { -0.01 } +float-negative-one = { -1.03 } +float-negative-zero = { -0.0 } +float-negative-without-fraction = { -1.000 } +float-positive-padded-left = { 01.03 } +float-positive-padded-right = { 1.0300 } +float-positive-padded-both = { 01.0300 } +float-negative-padded-left = { -01.03 } +float-negative-padded-right = { -1.0300 } +float-negative-padded-both = { -01.0300 } + +## ERRORS + diff --git a/fluent-syntax/tests/fixtures/formatted/obsolete.ftl b/fluent-syntax/tests/fixtures/formatted/obsolete.ftl new file mode 100644 index 00000000..314a35ef --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/obsolete.ftl @@ -0,0 +1,9 @@ +### The syntax in this file has been discontinued. It is no longer part of the +### Fluent specification and should not be implemented nor used. We're keeping +### these fixtures around to protect against accidental syntax reuse. + + +## Variant lists. + +## Variant expressions. + diff --git a/fluent-syntax/tests/fixtures/formatted/placeables.ftl b/fluent-syntax/tests/fixtures/formatted/placeables.ftl new file mode 100644 index 00000000..6a5167be --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/placeables.ftl @@ -0,0 +1,12 @@ +nested-placeable = {{ {1} }} +padded-placeable = { 1 } +sparse-placeable = {{ 1 }} + +# ERROR Unmatched opening brace + +# ERROR Unmatched opening brace + +# ERROR Unmatched closing brace + +# ERROR Unmatched closing brace + diff --git a/fluent-syntax/tests/fixtures/formatted/reference_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/reference_expressions.ftl new file mode 100644 index 00000000..5132517f --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/reference_expressions.ftl @@ -0,0 +1,23 @@ +## Reference expressions in placeables. + +message-reference-placeable = { msg } +term-reference-placeable = { -term } +variable-reference-placeable = { $var } +# Function references are invalid outside of call expressions. +# This parses as a valid MessageReference. +function-reference-placeable = { FUN } + +## Reference expressions in selectors. + +variable-reference-selector = + { $var -> + *[key] Value + } + +# ERROR Message values may not be used as selectors. + +# ERROR Term values may not be used as selectors. + +# ERROR Function references are invalid outside of call expressions, and this +# parses as a MessageReference which isn't a valid selector. + diff --git a/fluent-syntax/tests/fixtures/formatted/select_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/select_expressions.ftl new file mode 100644 index 00000000..8e1021b4 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/select_expressions.ftl @@ -0,0 +1,38 @@ +new-messages = + { BUILTIN() -> + [0] Zero + *[other] { "" }Other + } +valid-selector-term-attribute = + { -term.case -> + *[key] value + } + +# ERROR Term values are not valid selectors + +# ERROR CallExpressions on Terms are similar to TermReferences + +# ERROR Nested expressions are not valid selectors + +# ERROR Select expressions are not valid selectors + +empty-variant = + { $sel -> + *[key] { "" } + } +reduced-whitespace = + { FOO() -> + *[key] { "" } + } +nested-select = + { $sel -> + *[one] + { $sel -> + *[two] Value + } + } + +# ERROR Missing selector + +# ERROR Missing line end after variant list + diff --git a/fluent-syntax/tests/fixtures/formatted/select_indent.ftl b/fluent-syntax/tests/fixtures/formatted/select_indent.ftl new file mode 100644 index 00000000..9aa2a995 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/select_indent.ftl @@ -0,0 +1,65 @@ +select-1tbs-inline = + { $selector -> + *[key] Value + } +select-1tbs-newline = + { $selector -> + *[key] Value + } +select-1tbs-indent = + { $selector -> + *[key] Value + } +select-allman-inline = + { $selector -> + *[key] Value + [other] Other + } +select-allman-newline = + { $selector -> + *[key] Value + } +select-allman-indent = + { $selector -> + *[key] Value + } +select-gnu-inline = + { $selector -> + *[key] Value + } +select-gnu-newline = + { $selector -> + *[key] Value + } +select-gnu-indent = + { $selector -> + *[key] Value + } +select-no-indent = + { $selector -> + *[key] Value + [other] Other + } +select-no-indent-multiline = + { $selector -> + *[key] + Value + Continued + [other] + Other + Multiline + } + +# ERROR (Multiline text must be indented) + +select-flat = + { $selector -> + *[key] Value + [other] Other + } +# Each line ends with 5 spaces. +select-flat-with-trailing-spaces = + { $selector -> + *[key] Value + [other] Other + } diff --git a/fluent-syntax/tests/fixtures/formatted/sparse_entries.ftl b/fluent-syntax/tests/fixtures/formatted/sparse_entries.ftl new file mode 100644 index 00000000..dfa85945 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/sparse_entries.ftl @@ -0,0 +1,17 @@ +key01 = Value +key02 = + .attr = Attribute +key03 = + Value + Continued + + + Over multiple + Lines + .attr = Attribute +key05 = Value +key06 = + { 1 -> + [one] One + *[two] Two + } diff --git a/fluent-syntax/tests/fixtures/formatted/special_chars.ftl b/fluent-syntax/tests/fixtures/formatted/special_chars.ftl new file mode 100644 index 00000000..ab32df9d --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/special_chars.ftl @@ -0,0 +1,8 @@ +## OK + +bracket-inline = [Value] +dot-inline = .Value +star-inline = *Value + +## ERRORS + diff --git a/fluent-syntax/tests/fixtures/formatted/tab.ftl b/fluent-syntax/tests/fixtures/formatted/tab.ftl new file mode 100644 index 00000000..bbbd7332 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/tab.ftl @@ -0,0 +1,14 @@ +# OK (tab after = is part of the value) +key01 = Value 01 + +# Error (tab before =) + +# Error (tab is not a valid indent) + +# Partial Error (tab is not a valid indent) +key04 = This line is indented by 4 spaces, +# OK (value is a single tab) +key05 = +# OK (attribute value is two tabs) +key06 = + .attr = diff --git a/fluent-syntax/tests/fixtures/formatted/term_parameters.ftl b/fluent-syntax/tests/fixtures/formatted/term_parameters.ftl new file mode 100644 index 00000000..8ee2ed74 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/term_parameters.ftl @@ -0,0 +1,8 @@ +-term = + { $arg -> + *[key] Value + } +key01 = { -term } +key02 = { -term() } +key03 = { -term(arg: 1) } +key04 = { -term("positional", narg1: 1, narg2: 2) } diff --git a/fluent-syntax/tests/fixtures/formatted/terms.ftl b/fluent-syntax/tests/fixtures/formatted/terms.ftl new file mode 100644 index 00000000..37d68113 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/terms.ftl @@ -0,0 +1,20 @@ +-term01 = Value + .attr = Attribute +-term02 = { "" } + +# JUNK Missing value + +# JUNK Missing value +# < whitespace > + +# JUNK Missing value + +# JUNK Missing value +# < whitespace > + +# JUNK Missing = + +-term08 = Value + .attr = Attribute +-term09 = Value + .attr = Attribute diff --git a/fluent-syntax/tests/fixtures/formatted/variables.ftl b/fluent-syntax/tests/fixtures/formatted/variables.ftl new file mode 100644 index 00000000..25d58359 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/variables.ftl @@ -0,0 +1,14 @@ +key01 = { $var } +key02 = { $var } +key03 = { $var } +key04 = { $var } + +## Errors + + +# ERROR Missing variable identifier + +# ERROR Double $$ + +# ERROR Invalid first char of the identifier + diff --git a/fluent-syntax/tests/fixtures/formatted/variant_keys.ftl b/fluent-syntax/tests/fixtures/formatted/variant_keys.ftl new file mode 100644 index 00000000..d0313bcc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/variant_keys.ftl @@ -0,0 +1,23 @@ +simple-identifier = + { $sel -> + *[key] value + } +identifier-surrounded-by-whitespace = + { $sel -> + *[key] value + } +int-number = + { $sel -> + *[1] value + } +float-number = + { $sel -> + *[3.14] value + } + +# ERROR + +# ERROR + +# ERROR + diff --git a/fluent-syntax/tests/fixtures/formatted/whitespace_in_value.ftl b/fluent-syntax/tests/fixtures/formatted/whitespace_in_value.ftl new file mode 100644 index 00000000..0cbcd371 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/whitespace_in_value.ftl @@ -0,0 +1,10 @@ +# Caution, lines 6 and 7 contain white-space-only lines +key = + first line + + + + + + + last line diff --git a/fluent-syntax/tests/fixtures/formatted/zero_length.ftl b/fluent-syntax/tests/fixtures/formatted/zero_length.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/serializer_fixtures.rs b/fluent-syntax/tests/serializer_fixtures.rs index 4af214a7..dc1e7075 100644 --- a/fluent-syntax/tests/serializer_fixtures.rs +++ b/fluent-syntax/tests/serializer_fixtures.rs @@ -1,4 +1,3 @@ -use fluent_syntax::ast::{Entry, Resource}; use glob::glob; use std::ffi::OsStr; use std::fs; @@ -22,17 +21,6 @@ fn is_ignored(path: &Path) -> bool { .unwrap_or_default() } -fn clone_without_junk<'a>(original: &Resource<&'a str>) -> Resource<&'a str> { - Resource { - body: original - .body - .iter() - .filter(|entry| !matches!(entry, Entry::Junk { .. })) - .cloned() - .collect(), - } -} - #[test] fn roundtrip_normalized_fixtures() { for entry in glob("./tests/fixtures/normalized/*.ftl").expect("Failed to read glob pattern") { @@ -57,15 +45,26 @@ fn roundtrip_unnormalized_fixtures() { let content = fs::read_to_string(&path).expect("Failed to read file"); let parsed = parse(content.as_str()).unwrap_or_else(|(res, _)| res); - let parsed_without_junk = clone_without_junk(&parsed); + let reserialized = serialize_with_options(&parsed, Options { with_junk: true }); let reserialized_without_junk = serialize_with_options(&parsed, Options { with_junk: false }); + + let formatted_path = path.parent().unwrap().join("formatted"); + + let formatted_content = fs::read_to_string(formatted_path.join("junked").join(path.file_name().unwrap())).unwrap(); + assert_eq!(formatted_content, reserialized); + let formatted = parse(formatted_content.as_str()).unwrap_or_else(|(res, _)| res); + + let formatted_content_without_junk = fs::read_to_string(formatted_path.join(path.file_name().unwrap())).unwrap(); + assert_eq!(formatted_content_without_junk, reserialized_without_junk); + let formatted_without_junk = + parse(formatted_content_without_junk.as_str()).unwrap_or_else(|(res, _)| res); + let reparsed = parse(reserialized.as_str()).unwrap_or_else(|(res, _)| res); - let reparsed_without_junk = - parse(reserialized_without_junk.as_str()).unwrap_or_else(|(res, _)| res); + let reparsed_without_junk = parse(reserialized_without_junk.as_str()).unwrap_or_else(|(res, _)| res); - assert_eq!(reparsed_without_junk, parsed_without_junk); - assert_eq!(reparsed, parsed); + assert_eq!(formatted_without_junk, reparsed_without_junk); + assert_eq!(formatted, reparsed); } }