From 4d64dbfda371324b7d74aaf7a5dce54a9e3b1073 Mon Sep 17 00:00:00 2001 From: Ertanic Date: Sat, 16 Nov 2024 19:27:43 +0300 Subject: [PATCH 01/10] Add span to nodes --- fluent-bundle/src/resolver/errors.rs | 2 +- fluent-bundle/src/resolver/expression.rs | 8 +- .../src/resolver/inline_expression.rs | 25 +- fluent-bundle/src/resolver/pattern.rs | 23 +- fluent-bundle/src/resolver/scope.rs | 2 +- fluent-syntax/Cargo.toml | 1 + fluent-syntax/src/ast/helper.rs | 51 ++- fluent-syntax/src/ast/mod.rs | 369 ++++++++++++++++-- fluent-syntax/src/parser/comment.rs | 12 +- fluent-syntax/src/parser/core.rs | 41 +- fluent-syntax/src/parser/expression.rs | 71 +++- fluent-syntax/src/parser/pattern.rs | 38 +- fluent-syntax/src/parser/runtime.rs | 10 +- fluent-syntax/src/serializer.rs | 94 +++-- fluent-syntax/tests/serializer_fixtures.rs | 2 + 15 files changed, 643 insertions(+), 106 deletions(-) diff --git a/fluent-bundle/src/resolver/errors.rs b/fluent-bundle/src/resolver/errors.rs index 7606faba..9b9f8e36 100644 --- a/fluent-bundle/src/resolver/errors.rs +++ b/fluent-bundle/src/resolver/errors.rs @@ -31,7 +31,7 @@ where InlineExpression::FunctionReference { id, .. } => Self::Function { id: id.name.to_string(), }, - InlineExpression::MessageReference { id, attribute } => Self::Message { + InlineExpression::MessageReference { id, attribute, .. } => Self::Message { id: id.name.to_string(), attribute: attribute.as_ref().map(|i| i.name.to_string()), }, diff --git a/fluent-bundle/src/resolver/expression.rs b/fluent-bundle/src/resolver/expression.rs index ce030e4c..19cc3d24 100644 --- a/fluent-bundle/src/resolver/expression.rs +++ b/fluent-bundle/src/resolver/expression.rs @@ -23,8 +23,10 @@ impl<'bundle> WriteValue<'bundle> for ast::Expression<&'bundle str> { M: MemoizerKind, { match self { - Self::Inline(exp) => exp.write(w, scope), - Self::Select { selector, variants } => { + Self::Inline(exp, ..) => exp.write(w, scope), + Self::Select { + selector, variants, .. + } => { let selector = selector.resolve(scope); match selector { FluentValue::String(_) | FluentValue::Number(_) => { @@ -59,7 +61,7 @@ impl<'bundle> WriteValue<'bundle> for ast::Expression<&'bundle str> { W: fmt::Write, { match self { - Self::Inline(exp) => exp.write_error(w), + Self::Inline(exp, ..) => exp.write_error(w), Self::Select { selector, .. } => selector.write_error(w), } } diff --git a/fluent-bundle/src/resolver/inline_expression.rs b/fluent-bundle/src/resolver/inline_expression.rs index 3f8c8d4f..cff88bd5 100644 --- a/fluent-bundle/src/resolver/inline_expression.rs +++ b/fluent-bundle/src/resolver/inline_expression.rs @@ -24,8 +24,8 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { M: MemoizerKind, { match self { - Self::StringLiteral { value } => unescape_unicode(w, value), - Self::MessageReference { id, attribute } => { + Self::StringLiteral { value, .. } => unescape_unicode(w, value), + Self::MessageReference { id, attribute, .. } => { if let Some(msg) = scope.bundle.get_entry_message(id.name) { if let Some(attr) = attribute { msg.attributes @@ -53,11 +53,12 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { scope.write_ref_error(w, self) } } - Self::NumberLiteral { value } => FluentValue::try_number(value).write(w, scope), + Self::NumberLiteral { value, .. } => FluentValue::try_number(value).write(w, scope), Self::TermReference { id, attribute, arguments, + .. } => { let (_, resolved_named_args) = scope.get_arguments(arguments.as_ref()); @@ -82,7 +83,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { scope.local_args = None; result } - Self::FunctionReference { id, arguments } => { + Self::FunctionReference { id, arguments, .. } => { let (resolved_positional_args, resolved_named_args) = scope.get_arguments(Some(arguments)); @@ -99,7 +100,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { scope.write_ref_error(w, self) } } - Self::VariableReference { id } => { + Self::VariableReference { id, .. } => { let args = scope.local_args.as_ref().or(scope.args); if let Some(arg) = args.and_then(|args| args.get(id.name)) { @@ -113,7 +114,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { w.write_char('}') } } - Self::Placeable { expression } => expression.write(w, scope), + Self::Placeable { expression, .. } => expression.write(w, scope), } } @@ -125,10 +126,12 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { Self::MessageReference { id, attribute: Some(attribute), + .. } => write!(w, "{}.{}", id.name, attribute.name), Self::MessageReference { id, attribute: None, + .. } => w.write_str(id.name), Self::TermReference { id, @@ -141,7 +144,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> { .. } => write!(w, "-{}", id.name), Self::FunctionReference { id, .. } => write!(w, "{}()", id.name), - Self::VariableReference { id } => write!(w, "${}", id.name), + Self::VariableReference { id, .. } => write!(w, "${}", id.name), _ => unreachable!(), } } @@ -157,9 +160,9 @@ impl<'bundle> ResolveValue<'bundle> for ast::InlineExpression<&'bundle str> { M: MemoizerKind, { match self { - Self::StringLiteral { value } => unescape_unicode_to_string(value).into(), - Self::NumberLiteral { value } => FluentValue::try_number(value), - Self::VariableReference { id } => { + Self::StringLiteral { value, .. } => unescape_unicode_to_string(value).into(), + Self::NumberLiteral { value, .. } => FluentValue::try_number(value), + Self::VariableReference { id, .. } => { if let Some(local_args) = &scope.local_args { if let Some(arg) = local_args.get(id.name) { return arg.clone(); @@ -173,7 +176,7 @@ impl<'bundle> ResolveValue<'bundle> for ast::InlineExpression<&'bundle str> { } FluentValue::Error } - Self::FunctionReference { id, arguments } => { + Self::FunctionReference { id, arguments, .. } => { let (resolved_positional_args, resolved_named_args) = scope.get_arguments(Some(arguments)); diff --git a/fluent-bundle/src/resolver/pattern.rs b/fluent-bundle/src/resolver/pattern.rs index e20bfcde..d09547aa 100644 --- a/fluent-bundle/src/resolver/pattern.rs +++ b/fluent-bundle/src/resolver/pattern.rs @@ -32,14 +32,14 @@ impl<'bundle> WriteValue<'bundle> for ast::Pattern<&'bundle str> { } match elem { - ast::PatternElement::TextElement { value } => { + ast::PatternElement::TextElement { value, .. } => { if let Some(ref transform) = scope.bundle.transform { w.write_str(&transform(value))?; } else { w.write_str(value)?; } } - ast::PatternElement::Placeable { ref expression } => { + ast::PatternElement::Placeable { ref expression, .. } => { scope.placeables += 1; if scope.placeables > MAX_PLACEABLES { scope.dirty = true; @@ -51,13 +51,16 @@ impl<'bundle> WriteValue<'bundle> for ast::Pattern<&'bundle str> { && len > 1 && !matches!( expression, - ast::Expression::Inline(ast::InlineExpression::MessageReference { .. },) - | ast::Expression::Inline( - ast::InlineExpression::TermReference { .. }, - ) - | ast::Expression::Inline( - ast::InlineExpression::StringLiteral { .. }, - ) + ast::Expression::Inline( + ast::InlineExpression::MessageReference { .. }, + .. + ) | ast::Expression::Inline( + ast::InlineExpression::TermReference { .. }, + .. + ) | ast::Expression::Inline( + ast::InlineExpression::StringLiteral { .. }, + .. + ) ); if needs_isolation { w.write_char('\u{2068}')?; @@ -92,7 +95,7 @@ impl<'bundle> ResolveValue<'bundle> for ast::Pattern<&'bundle str> { let len = self.elements.len(); if len == 1 { - if let ast::PatternElement::TextElement { value } = self.elements[0] { + if let ast::PatternElement::TextElement { value, .. } = self.elements[0] { return scope .bundle .transform diff --git a/fluent-bundle/src/resolver/scope.rs b/fluent-bundle/src/resolver/scope.rs index 1ddff1a4..7f89c501 100644 --- a/fluent-bundle/src/resolver/scope.rs +++ b/fluent-bundle/src/resolver/scope.rs @@ -124,7 +124,7 @@ impl<'bundle, 'ast, 'args, 'errors, R, M> Scope<'bundle, 'ast, 'args, 'errors, R R: Borrow, M: MemoizerKind, { - if let Some(ast::CallArguments { positional, named }) = arguments { + if let Some(ast::CallArguments { positional, named, .. }) = arguments { let positional = positional.iter().map(|expr| expr.resolve(self)).collect(); let named = named diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index b8cf539a..e9efcd54 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -38,6 +38,7 @@ glob = "0.3" [features] default = [] +spans = [] serde = ["dep:serde"] json = ["serde", "dep:serde_json"] all-benchmarks = [] diff --git a/fluent-syntax/src/ast/helper.rs b/fluent-syntax/src/ast/helper.rs index 23851301..c803e1ac 100644 --- a/fluent-syntax/src/ast/helper.rs +++ b/fluent-syntax/src/ast/helper.rs @@ -1,25 +1,64 @@ +use super::Comment; +#[cfg(feature = "spans")] +use super::Span; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use super::Comment; // This is a helper struct used to properly deserialize referential // JSON comments which are single continuous String, into a vec of // content slices. -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "spans"), derive(PartialEq, Eq))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(untagged))] pub enum CommentDef { - Single { content: S }, - Multi { content: Vec }, + Single { + content: S, + #[cfg(feature = "spans")] + span: Span, + }, + Multi { + content: Vec, + #[cfg(feature = "spans")] + span: Span, + }, +} + +#[cfg(feature = "spans")] +impl Eq for CommentDef {} + +#[cfg(feature = "spans")] +impl PartialEq for CommentDef { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Single { content: l_content, .. }, Self::Single { content: r_content, .. }) => l_content == r_content, + (Self::Multi { content: l_content, .. }, Self::Multi { content: r_content, .. }) => l_content == r_content, + _ => false, + } + } } impl From> for Comment { fn from(input: CommentDef) -> Self { match input { - CommentDef::Single { content } => Self { + CommentDef::Single { + content, + #[cfg(feature = "spans")] + span, + } => Self { content: vec![content], + #[cfg(feature = "spans")] + span, + }, + CommentDef::Multi { + content, + #[cfg(feature = "spans")] + span, + } => Self { + content, + #[cfg(feature = "spans")] + span, }, - CommentDef::Multi { content } => Self { content }, } } } diff --git a/fluent-syntax/src/ast/mod.rs b/fluent-syntax/src/ast/mod.rs index 273aceaf..bf321259 100644 --- a/fluent-syntax/src/ast/mod.rs +++ b/fluent-syntax/src/ast/mod.rs @@ -88,6 +88,8 @@ mod helper; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +#[cfg(feature = "spans")] +use std::ops::Range; /// Root node of a Fluent Translation List. /// @@ -111,10 +113,20 @@ use serde::{Deserialize, Serialize}; /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "spans"), derive(PartialEq))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Resource { pub body: Vec>, + #[cfg(feature = "spans")] + pub span: Span, +} + +#[cfg(feature = "spans")] +impl PartialEq for Resource { + fn eq(&self, other: &Self) -> bool { + self.body == other.body + } } /// A top-level node representing an entry of a [`Resource`]. @@ -193,7 +205,8 @@ pub struct Resource { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "spans"), derive(PartialEq))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum Entry { @@ -202,7 +215,33 @@ pub enum Entry { Comment(Comment), GroupComment(Comment), ResourceComment(Comment), - Junk { content: S }, + Junk { + content: S, + #[cfg(feature = "spans")] + span: Span, + }, +} + +#[cfg(feature = "spans")] +impl PartialEq for Entry { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Message(l0), Self::Message(r0)) => l0 == r0, + (Self::Term(l0), Self::Term(r0)) => l0 == r0, + (Self::Comment(l0), Self::Comment(r0)) => l0 == r0, + (Self::GroupComment(l0), Self::GroupComment(r0)) => l0 == r0, + (Self::ResourceComment(l0), Self::ResourceComment(r0)) => l0 == r0, + ( + Self::Junk { + content: l_content, .. + }, + Self::Junk { + content: r_content, .. + }, + ) => l_content == r_content, + _ => false, + } + } } /// Message node represents the most common [`Entry`] in an FTL [`Resource`]. @@ -253,13 +292,24 @@ pub enum Entry { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Message { pub id: Identifier, pub value: Option>, pub attributes: Vec>, pub comment: Option>, + #[cfg(feature = "spans")] + pub span: Span, +} + +impl PartialEq for Message { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + && self.value == other.value + && self.attributes == other.attributes + && self.comment == other.comment + } } /// A Fluent [`Term`]. @@ -307,13 +357,26 @@ pub struct Message { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "spans"), derive(PartialEq))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Term { pub id: Identifier, pub value: Pattern, pub attributes: Vec>, pub comment: Option>, + #[cfg(feature = "spans")] + pub span: Span, +} + +#[cfg(feature = "spans")] +impl PartialEq for Term { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + && self.value == other.value + && self.attributes == other.attributes + && self.comment == other.comment + } } /// Pattern contains a value of a [`Message`], [`Term`] or an [`Attribute`]. @@ -387,10 +450,18 @@ pub struct Term { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Pattern { pub elements: Vec>, + #[cfg(feature = "spans")] + pub span: Span, +} + +impl PartialEq for Pattern { + fn eq(&self, other: &Self) -> bool { + self.elements == other.elements + } } /// `PatternElement` is an element of a [`Pattern`]. @@ -464,12 +535,42 @@ pub struct Pattern { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum PatternElement { - TextElement { value: S }, - Placeable { expression: Expression }, + TextElement { + value: S, + #[cfg(feature = "spans")] + span: Span, + }, + Placeable { + expression: Expression, + #[cfg(feature = "spans")] + span: Span, + }, +} + +impl PartialEq for PatternElement { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + Self::TextElement { value: l_value, .. }, + Self::TextElement { value: r_value, .. }, + ) => l_value == r_value, + ( + Self::Placeable { + expression: l_expression, + .. + }, + Self::Placeable { + expression: r_expression, + .. + }, + ) => l_expression == r_expression, + _ => false, + } + } } /// Attribute represents a part of a [`Message`] or [`Term`]. @@ -535,11 +636,21 @@ pub enum PatternElement { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "spans"), derive(PartialEq))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Attribute { pub id: Identifier, pub value: Pattern, + #[cfg(feature = "spans")] + pub span: Span, +} + +#[cfg(feature = "spans")] +impl PartialEq for Attribute { + fn eq(&self, other: &Self) -> bool { + self.id == other.id && self.value == other.value + } } /// Identifier is part of nodes such as [`Message`], [`Term`] and [`Attribute`]. @@ -584,10 +695,20 @@ pub struct Attribute { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Identifier { pub name: S, + #[cfg(feature = "spans")] + pub span: Span, +} + +impl Eq for Identifier {} + +impl PartialEq for Identifier { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } } /// Variant is a single branch of a value in a [`Select`](Expression::Select) expression. @@ -667,13 +788,23 @@ pub struct Identifier { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "spans"), derive(PartialEq))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct Variant { pub key: VariantKey, pub value: Pattern, pub default: bool, + #[cfg(feature = "spans")] + pub span: Span, +} + +#[cfg(feature = "spans")] +impl PartialEq for Variant { + fn eq(&self, other: &Self) -> bool { + self.key == other.key && self.value == other.value && self.default == other.default + } } /// A key of a [`Variant`]. @@ -798,11 +929,21 @@ pub enum VariantKey { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(from = "helper::CommentDef"))] pub struct Comment { pub content: Vec, + #[cfg(feature = "spans")] + pub span: Span, +} + +impl Eq for Comment {} + +impl PartialEq for Comment { + fn eq(&self, other: &Self) -> bool { + self.content == other.content + } } /// List of arguments for a [`FunctionReference`](InlineExpression::FunctionReference) or a @@ -879,12 +1020,20 @@ pub struct Comment { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone, Default)] +#[derive(Debug, Clone, Default)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct CallArguments { pub positional: Vec>, pub named: Vec>, + #[cfg(feature = "spans")] + pub span: Span, +} + +impl PartialEq for CallArguments { + fn eq(&self, other: &Self) -> bool { + self.positional == other.positional && self.named == other.named + } } /// A key-value pair used in [`CallArguments`]. @@ -948,12 +1097,22 @@ pub struct CallArguments { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "spans"), derive(PartialEq))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct NamedArgument { pub name: Identifier, pub value: InlineExpression, + #[cfg(feature = "spans")] + pub span: Span, +} + +#[cfg(feature = "spans")] +impl PartialEq for NamedArgument { + fn eq(&self, other: &Self) -> bool { + self.name == other.name && self.value == other.value + } } /// A subset of expressions which can be used as [`Placeable`](PatternElement::Placeable), @@ -1004,7 +1163,7 @@ pub struct NamedArgument { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum InlineExpression { @@ -1053,7 +1212,11 @@ pub enum InlineExpression { /// } /// ); /// ``` - StringLiteral { value: S }, + StringLiteral { + value: S, + #[cfg(feature = "spans")] + span: Span, + }, /// A number literal. /// /// # Example @@ -1099,7 +1262,11 @@ pub enum InlineExpression { /// } /// ); /// ``` - NumberLiteral { value: S }, + NumberLiteral { + value: S, + #[cfg(feature = "spans")] + span: Span, + }, /// A function reference. /// /// # Example @@ -1151,6 +1318,8 @@ pub enum InlineExpression { FunctionReference { id: Identifier, arguments: CallArguments, + #[cfg(feature = "spans")] + span: Span, }, /// A reference to another message. /// @@ -1203,6 +1372,8 @@ pub enum InlineExpression { MessageReference { id: Identifier, attribute: Option>, + #[cfg(feature = "spans")] + span: Span, }, /// A reference to a term. /// @@ -1257,6 +1428,8 @@ pub enum InlineExpression { id: Identifier, attribute: Option>, arguments: Option>, + #[cfg(feature = "spans")] + span: Span, }, /// A reference to a variable. /// @@ -1305,7 +1478,11 @@ pub enum InlineExpression { /// } /// ); /// ``` - VariableReference { id: Identifier }, + VariableReference { + id: Identifier, + #[cfg(feature = "spans")] + span: Span, + }, /// A placeable which may contain another expression. /// /// # Example @@ -1357,7 +1534,94 @@ pub enum InlineExpression { /// } /// ); /// ``` - Placeable { expression: Box> }, + Placeable { + expression: Box>, + #[cfg(feature = "spans")] + span: Span, + }, +} + +impl PartialEq for InlineExpression { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + Self::StringLiteral { value: l_value, .. }, + Self::StringLiteral { value: r_value, .. }, + ) => l_value == r_value, + ( + Self::NumberLiteral { value: l_value, .. }, + Self::NumberLiteral { value: r_value, .. }, + ) => l_value == r_value, + ( + Self::FunctionReference { + id: l_id, + arguments: l_arguments, + .. + }, + Self::FunctionReference { + id: r_id, + arguments: r_arguments, + .. + }, + ) => l_id == r_id && l_arguments == r_arguments, + ( + Self::MessageReference { + id: l_id, + attribute: l_attribute, + .. + }, + Self::MessageReference { + id: r_id, + attribute: r_attribute, + .. + }, + ) => l_id == r_id && l_attribute == r_attribute, + ( + Self::TermReference { + id: l_id, + attribute: l_attribute, + arguments: l_arguments, + .. + }, + Self::TermReference { + id: r_id, + attribute: r_attribute, + arguments: r_arguments, + .. + }, + ) => l_id == r_id && l_attribute == r_attribute && l_arguments == r_arguments, + ( + Self::VariableReference { id: l_id, .. }, + Self::VariableReference { id: r_id, .. }, + ) => l_id == r_id, + ( + Self::Placeable { + expression: l_expression, + .. + }, + Self::Placeable { + expression: r_expression, + .. + }, + ) => l_expression == r_expression, + _ => false, + } + } +} + +#[cfg(feature = "spans")] +impl InlineExpression { + pub fn get_span(&self) -> Span { + match self { + InlineExpression::StringLiteral { span, .. } + | InlineExpression::TermReference { span, .. } + | InlineExpression::VariableReference { span, .. } + | InlineExpression::Placeable { span, .. } + | InlineExpression::NumberLiteral { span, .. } + | InlineExpression::FunctionReference { span, .. } + | InlineExpression::MessageReference { span, .. } => *span, + } + } } /// An expression that is either a select expression or an inline expression. @@ -1434,7 +1698,8 @@ pub enum InlineExpression { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, Clone)] +#[cfg_attr(not(feature = "spans"), derive(PartialEq))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(untagged))] pub enum Expression { @@ -1448,6 +1713,8 @@ pub enum Expression { Select { selector: InlineExpression, variants: Vec>, + #[cfg(feature = "spans")] + span: Span, }, /// An inline expression such as `${ username }`: @@ -1455,5 +1722,63 @@ pub enum Expression { /// ```ftl /// hello-user = Hello ${ username } /// ``` - Inline(InlineExpression), + Inline(InlineExpression, #[cfg(feature = "spans")] Span), +} + +#[cfg(feature = "spans")] +impl PartialEq for Expression { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + Self::Select { + selector: l_selector, + variants: l_variants, + .. + }, + Self::Select { + selector: r_selector, + variants: r_variants, + .. + }, + ) => l_selector == r_selector && l_variants == r_variants, + (Self::Inline(l0, ..), Self::Inline(r0, ..)) => l0 == r0, + _ => false, + } + } +} + +/// A span of a node. Allows you to get the index of the start and end character of a node. +/// +/// # Example +/// +/// ``` +/// #![cfg(feature = "spans")] +/// +/// use fluent_syntax::parser; +/// use fluent_syntax::ast::*; +/// +/// let ftl = "hello-world = Hello, World!"; +/// +/// let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); +/// let Entry::Message(Message { ref id, .. }) = resource.body[0] else { unreachable!() }; +/// +/// assert_eq!(resource.span, Span { start: 0, end: 27 }); +/// assert_eq!(id.span, Span { start: 0, end: 11 }); // the span of hello-world identifier +/// ``` +#[cfg(feature = "spans")] +#[derive(Debug, Clone, Copy, Default, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Span { + pub start: usize, + pub end: usize, +} + +#[cfg(feature = "spans")] +impl Span { + pub fn new(range: Range) -> Self { + Self { + start: range.start, + end: range.end, + } + } } diff --git a/fluent-syntax/src/parser/comment.rs b/fluent-syntax/src/parser/comment.rs index 1e30fc72..97c3ff6e 100644 --- a/fluent-syntax/src/parser/comment.rs +++ b/fluent-syntax/src/parser/comment.rs @@ -16,6 +16,9 @@ where pub(super) fn get_comment(&mut self) -> Result<(ast::Comment, Level)> { let mut level = Level::None; let mut content = vec![]; + + #[cfg(feature = "spans")] + let start_pos = self.ptr; while self.ptr < self.length { let line_level = self.get_comment_level(); @@ -47,7 +50,14 @@ where self.skip_eol(); } - Ok((ast::Comment { content }, level)) + Ok(( + ast::Comment { + content, + #[cfg(feature = "spans")] + span: ast::Span::new(start_pos..self.ptr), + }, + level, + )) } pub(super) fn skip_comment(&mut self) { diff --git a/fluent-syntax/src/parser/core.rs b/fluent-syntax/src/parser/core.rs index 68ad8dc0..8200a9ab 100644 --- a/fluent-syntax/src/parser/core.rs +++ b/fluent-syntax/src/parser/core.rs @@ -67,7 +67,11 @@ where err.slice = Some(entry_start..self.ptr); errors.push(err); let content = self.source.slice(entry_start..self.ptr); - body.push(ast::Entry::Junk { content }); + body.push(ast::Entry::Junk { + content, + #[cfg(feature = "spans")] + span: ast::Span::new(entry_start..self.ptr), + }); } } last_blank_count = self.skip_blank_block(); @@ -77,9 +81,9 @@ where body.push(ast::Entry::Comment(last_comment)); } if errors.is_empty() { - Ok(ast::Resource { body }) + Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }) } else { - Err((ast::Resource { body }, errors)) + Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }, errors)) } } @@ -123,6 +127,8 @@ where value: pattern, attributes, comment: None, + #[cfg(feature = "spans")] + span: ast::Span::new(entry_start..self.ptr), }) } @@ -145,6 +151,8 @@ where value, attributes, comment: None, + #[cfg(feature = "spans")] + span: ast::Span::new(entry_start..self.ptr), }) } else { error!( @@ -184,12 +192,19 @@ where let pattern = self.get_pattern()?; match pattern { - Some(pattern) => Ok(ast::Attribute { id, value: pattern }), + Some(pattern) => Ok(ast::Attribute { + id, + value: pattern, + #[cfg(feature = "spans")] + span: ast::Span::new(self.ptr - 1..self.ptr), + }), None => error!(ErrorKind::MissingValue, self.ptr), } } pub(super) fn get_identifier_unchecked(&mut self) -> ast::Identifier { + #[cfg(feature = "spans")] + let start = self.ptr - 1; let mut ptr = self.ptr; while matches!(get_byte!(self, ptr), Some(b) if b.is_ascii_alphanumeric() || *b == b'-' || *b == b'_') @@ -200,7 +215,11 @@ where let name = self.source.slice(self.ptr - 1..ptr); self.ptr = ptr; - ast::Identifier { name } + ast::Identifier { + name, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), + } } pub(super) fn get_identifier(&mut self) -> Result> { @@ -259,6 +278,9 @@ where } } + #[cfg(feature = "spans")] + let start = self.ptr; + if !self.take_byte_if(b'[') { break; } @@ -272,6 +294,8 @@ where key, value, default, + #[cfg(feature = "spans")] + span: ast::Span::new((if default { start - 1 } else { start })..self.ptr), }); self.skip_blank(); } else { @@ -293,9 +317,10 @@ where self.expect_byte(b'}')?; let invalid_expression_found = match &exp { - ast::Expression::Inline(ast::InlineExpression::TermReference { - ref attribute, .. - }) => attribute.is_some(), + ast::Expression::Inline( + ast::InlineExpression::TermReference { ref attribute, .. }, + .., + ) => attribute.is_some(), _ => false, }; if invalid_expression_found { diff --git a/fluent-syntax/src/parser/expression.rs b/fluent-syntax/src/parser/expression.rs index c5ccb32b..014228e0 100644 --- a/fluent-syntax/src/parser/expression.rs +++ b/fluent-syntax/src/parser/expression.rs @@ -9,6 +9,9 @@ where pub(super) fn get_expression(&mut self) -> Result> { let exp = self.get_inline_expression(false)?; + #[cfg(feature = "spans")] + let start_span = exp.get_span().start; + self.skip_blank(); if !self.is_current_byte(b'-') || !self.is_byte_at(b'>', self.ptr + 1) { @@ -17,7 +20,11 @@ where return error!(ErrorKind::TermAttributeAsPlaceable, self.ptr); } } - return Ok(ast::Expression::Inline(exp)); + return Ok(ast::Expression::Inline( + exp, + #[cfg(feature = "spans")] + ast::Span::new(start_span..self.ptr), + )); } match exp { @@ -60,6 +67,8 @@ where Ok(ast::Expression::Select { selector: exp, variants, + #[cfg(feature = "spans")] + span: ast::Span::new(start_span..self.ptr), }) } @@ -67,10 +76,10 @@ where &mut self, only_literal: bool, ) -> Result> { + let start = self.ptr; match get_current_byte!(self) { Some(b'"') => { self.ptr += 1; // " - let start = self.ptr; while let Some(b) = get_current_byte!(self) { match b { b'\\' => match get_byte!(self, self.ptr + 1) { @@ -99,12 +108,20 @@ where } self.expect_byte(b'"')?; - let slice = self.source.slice(start..self.ptr - 1); - Ok(ast::InlineExpression::StringLiteral { value: slice }) + let slice = self.source.slice(start + 1..self.ptr - 1); + Ok(ast::InlineExpression::StringLiteral { + value: slice, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), + }) } Some(b) if b.is_ascii_digit() => { let num = self.get_number_literal()?; - Ok(ast::InlineExpression::NumberLiteral { value: num }) + Ok(ast::InlineExpression::NumberLiteral { + value: num, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), + }) } Some(b'-') if !only_literal => { self.ptr += 1; // - @@ -117,17 +134,27 @@ where id, attribute, arguments, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), }) } else { self.ptr -= 1; let num = self.get_number_literal()?; - Ok(ast::InlineExpression::NumberLiteral { value: num }) + Ok(ast::InlineExpression::NumberLiteral { + value: num, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), + }) } } Some(b'$') if !only_literal => { self.ptr += 1; // $ let id = self.get_identifier()?; - Ok(ast::InlineExpression::VariableReference { id }) + Ok(ast::InlineExpression::VariableReference { + id, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), + }) } Some(b) if b.is_ascii_alphabetic() => { self.ptr += 1; @@ -138,10 +165,20 @@ where return error!(ErrorKind::ForbiddenCallee, self.ptr); } - Ok(ast::InlineExpression::FunctionReference { id, arguments }) + Ok(ast::InlineExpression::FunctionReference { + id, + arguments, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), + }) } else { let attribute = self.get_attribute_accessor()?; - Ok(ast::InlineExpression::MessageReference { id, attribute }) + Ok(ast::InlineExpression::MessageReference { + id, + attribute, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), + }) } } Some(b'{') if !only_literal => { @@ -149,6 +186,8 @@ where let exp = self.get_placeable()?; Ok(ast::InlineExpression::Placeable { expression: Box::new(exp), + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), }) } _ if only_literal => error!(ErrorKind::ExpectedLiteral, self.ptr), @@ -161,6 +200,8 @@ where if !self.take_byte_if(b'(') { return Ok(None); } + #[cfg(feature = "spans")] + let start = self.ptr - 1; let mut positional = vec![]; let mut named = vec![]; @@ -178,6 +219,7 @@ where if let ast::InlineExpression::MessageReference { ref id, attribute: None, + .. } = expr { self.skip_blank(); @@ -196,8 +238,12 @@ where named.push(ast::NamedArgument { name: ast::Identifier { name: id.name.clone(), + #[cfg(feature = "spans")] + span: id.span, }, value: val, + #[cfg(feature = "spans")] + span: ast::Span::new(id.span.start..self.ptr), }); } else { if !argument_names.is_empty() { @@ -219,6 +265,11 @@ where self.expect_byte(b')')?; - Ok(Some(ast::CallArguments { positional, named })) + Ok(Some(ast::CallArguments { + positional, + named, + #[cfg(feature = "spans")] + span: ast::Span::new(start..self.ptr), + })) } } diff --git a/fluent-syntax/src/parser/pattern.rs b/fluent-syntax/src/parser/pattern.rs index 9ca1229a..d0a6330d 100644 --- a/fluent-syntax/src/parser/pattern.rs +++ b/fluent-syntax/src/parser/pattern.rs @@ -1,6 +1,8 @@ use super::errors::{ErrorKind, ParserError}; use super::{core::Parser, core::Result, slice::Slice}; use crate::ast; +#[cfg(feature = "spans")] +use std::ops::Range; #[derive(Debug, PartialEq)] enum TextElementTermination { @@ -24,7 +26,7 @@ enum TextElementPosition { // cheaper since they'll happen on the pointers, rather than extracted slices. #[derive(Debug)] enum PatternElementPlaceholders { - Placeable(ast::Expression), + Placeable(ast::Expression, #[cfg(feature = "spans")] Range), // (start, end, indent, position) TextElement(usize, usize, usize, TextElementPosition), } @@ -49,6 +51,9 @@ where self.skip_blank_inline(); + #[cfg(feature = "spans")] + let start_pos = self.ptr; + let mut text_element_role = if self.skip_eol() { self.skip_blank_block(); TextElementPosition::LineStart @@ -58,12 +63,23 @@ where while self.ptr < self.length { if self.take_byte_if(b'{') { + #[cfg(feature = "spans")] + let slice_start = self.ptr - 1; if text_element_role == TextElementPosition::LineStart { common_indent = Some(0); } let exp = self.get_placeable()?; last_non_blank = Some(elements.len()); + + #[cfg(feature = "spans")] + elements.push(PatternElementPlaceholders::Placeable( + exp, + slice_start..self.ptr - 1, + )); + + #[cfg(not(feature = "spans"))] elements.push(PatternElementPlaceholders::Placeable(exp)); + text_element_role = TextElementPosition::Continuation; } else { let slice_start = self.ptr; @@ -127,6 +143,14 @@ where .take(last_non_blank + 1) .enumerate() .map(|(i, elem)| match elem { + #[cfg(feature = "spans")] + PatternElementPlaceholders::Placeable(expression, range) => { + ast::PatternElement::Placeable { + expression, + span: ast::Span::new(range), + } + } + #[cfg(not(feature = "spans"))] PatternElementPlaceholders::Placeable(expression) => { ast::PatternElement::Placeable { expression } } @@ -143,11 +167,19 @@ where if last_non_blank == i { value.trim(); } - ast::PatternElement::TextElement { value } + ast::PatternElement::TextElement { + value, + #[cfg(feature = "spans")] + span: ast::Span::new(start..end), + } } }) .collect(); - return Ok(Some(ast::Pattern { elements })); + return Ok(Some(ast::Pattern { + elements, + #[cfg(feature = "spans")] + span: ast::Span::new(start_pos..self.ptr), + })); } Ok(None) diff --git a/fluent-syntax/src/parser/runtime.rs b/fluent-syntax/src/parser/runtime.rs index e116ceae..d350e046 100644 --- a/fluent-syntax/src/parser/runtime.rs +++ b/fluent-syntax/src/parser/runtime.rs @@ -34,16 +34,20 @@ where err.slice = Some(entry_start..self.ptr); errors.push(err); let content = self.source.slice(entry_start..self.ptr); - body.push(ast::Entry::Junk { content }); + body.push(ast::Entry::Junk { + content, + #[cfg(feature = "spans")] + span: ast::Span::new(entry_start..self.ptr), + }); } } self.skip_blank_block(); } if errors.is_empty() { - Ok(ast::Resource { body }) + Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }) } else { - Err((ast::Resource { body }, errors)) + Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }, errors)) } } diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index a3442429..5610de35 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -87,7 +87,7 @@ impl Serializer { Entry::Comment(comment) => self.serialize_free_comment(comment, "#"), Entry::GroupComment(comment) => self.serialize_free_comment(comment, "##"), Entry::ResourceComment(comment) => self.serialize_free_comment(comment, "###"), - Entry::Junk { content } => { + Entry::Junk { content, .. } => { if self.options.with_junk { self.serialize_junk(content.as_ref()); } @@ -203,9 +203,9 @@ impl Serializer { fn serialize_element<'s, S: Slice<'s>>(&mut self, elem: &PatternElement) { match elem { - PatternElement::TextElement { value } => self.writer.write_literal(value.as_ref()), - PatternElement::Placeable { expression } => match expression { - Expression::Inline(InlineExpression::Placeable { expression }) => { + PatternElement::TextElement { value, .. } => self.writer.write_literal(value.as_ref()), + PatternElement::Placeable { expression, .. } => match expression { + Expression::Inline(InlineExpression::Placeable { expression, .. }, ..) => { // A placeable inside a placeable is a special case because we // don't want the braces to look silly (e.g. "{ { Foo() } }"). self.writer.write_literal("{{ "); @@ -219,7 +219,7 @@ impl Serializer { self.serialize_expression(expression); self.writer.write_literal("}"); } - Expression::Inline(_) => { + Expression::Inline( .. ) => { self.writer.write_literal("{ "); self.serialize_expression(expression); self.writer.write_literal(" }"); @@ -230,8 +230,10 @@ impl Serializer { fn serialize_expression<'s, S: Slice<'s>>(&mut self, expr: &Expression) { match expr { - Expression::Inline(inline) => self.serialize_inline_expression(inline), - Expression::Select { selector, variants } => { + Expression::Inline(inline, ..) => self.serialize_inline_expression(inline), + Expression::Select { + selector, variants, .. + } => { self.serialize_select_expression(selector, variants); } } @@ -239,23 +241,26 @@ impl Serializer { fn serialize_inline_expression<'s, S: Slice<'s>>(&mut self, expr: &InlineExpression) { match expr { - InlineExpression::StringLiteral { value } => { + InlineExpression::StringLiteral { value, .. } => { self.writer.write_literal("\""); self.writer.write_literal(value.as_ref()); self.writer.write_literal("\""); } - InlineExpression::NumberLiteral { value } => self.writer.write_literal(value.as_ref()), + InlineExpression::NumberLiteral { value, .. } => { + self.writer.write_literal(value.as_ref()) + } InlineExpression::VariableReference { - id: Identifier { name: value }, + id: Identifier { name: value, .. }, + .. } => { self.writer.write_literal("$"); self.writer.write_literal(value.as_ref()); } - InlineExpression::FunctionReference { id, arguments } => { + InlineExpression::FunctionReference { id, arguments, .. } => { self.writer.write_literal(id.name.as_ref()); self.serialize_call_arguments(arguments); } - InlineExpression::MessageReference { id, attribute } => { + InlineExpression::MessageReference { id, attribute, .. } => { self.writer.write_literal(id.name.as_ref()); if let Some(attr) = attribute.as_ref() { @@ -267,6 +272,7 @@ impl Serializer { id, attribute, arguments, + .. } => { self.writer.write_literal("-"); self.writer.write_literal(id.name.as_ref()); @@ -279,7 +285,7 @@ impl Serializer { self.serialize_call_arguments(args); } } - InlineExpression::Placeable { expression } => { + InlineExpression::Placeable { expression, .. } => { self.writer.write_literal("{"); self.serialize_expression(expression); self.writer.write_literal("}"); @@ -361,13 +367,13 @@ impl<'s, S: Slice<'s>> Pattern { fn is_multiline(&self) -> bool { self.elements.iter().any(|elem| match elem { - PatternElement::TextElement { value } => value.as_ref().contains('\n'), - PatternElement::Placeable { expression } => is_select_expr(expression), + PatternElement::TextElement { value, .. } => value.as_ref().contains('\n'), + PatternElement::Placeable { expression, .. } => is_select_expr(expression), }) } fn has_leading_text_dot(&self) -> bool { - if let Some(PatternElement::TextElement { value }) = self.elements.first() { + if let Some(PatternElement::TextElement { value, .. }) = self.elements.first() { value.as_ref().starts_with('.') } else { false @@ -378,10 +384,10 @@ impl<'s, S: Slice<'s>> Pattern { fn is_select_expr<'s, S: Slice<'s>>(expr: &Expression) -> bool { match expr { Expression::Select { .. } => true, - Expression::Inline(InlineExpression::Placeable { expression }) => { + Expression::Inline(InlineExpression::Placeable { expression, .. }, ..) => { is_select_expr(expression) } - Expression::Inline(_) => false, + Expression::Inline(..) => false, } } @@ -478,12 +484,24 @@ mod test { macro_rules! text_message { ($name:expr, $value:expr) => { Entry::Message(Message { - id: Identifier { name: $name }, + id: Identifier { + name: $name, + #[cfg(feature = "spans")] + span: Span::new(0..0), + }, value: Some(Pattern { - elements: vec![PatternElement::TextElement { value: $value }], + elements: vec![PatternElement::TextElement { + value: $value, + #[cfg(feature = "spans")] + span: Span::new(0..0), + }], + #[cfg(feature = "spans")] + span: Span::new(0..0), }), attributes: vec![], comment: None, + #[cfg(feature = "spans")] + span: Span::new(0..0), }) }; } @@ -506,14 +524,14 @@ mod test { impl<'a> PatternElement<&'a str> { fn as_text(&mut self) -> &mut &'a str { match self { - Self::TextElement { value } => value, + Self::TextElement { value, .. } => value, _ => panic!("Expected TextElement"), } } fn as_expression(&mut self) -> &mut Expression<&'a str> { match self { - Self::Placeable { expression } => expression, + Self::Placeable { expression, .. } => expression, _ => panic!("Expected Placeable"), } } @@ -528,7 +546,7 @@ mod test { } fn as_inline_variable_id(&mut self) -> &mut Identifier<&'a str> { match self { - Self::Inline(InlineExpression::VariableReference { id }) => id, + Self::Inline(InlineExpression::VariableReference { id, .. }, ..) => id, _ => panic!("Expected Inline"), } } @@ -563,14 +581,34 @@ mod test { value: Pattern { elements: vec![ PatternElement::Placeable { - expression: Expression::Inline(InlineExpression::VariableReference { - id: Identifier { name: "num" }, - }), + expression: Expression::Inline( + InlineExpression::VariableReference { + id: Identifier { + name: "num", + #[cfg(feature = "spans")] + span: Span::new(0..0), + }, + #[cfg(feature = "spans")] + span: Span::new(0..0), + }, + #[cfg(feature = "spans")] + Span::new(0..0), + ), + #[cfg(feature = "spans")] + span: Span::new(0..0), + }, + PatternElement::TextElement { + value: " bar", + #[cfg(feature = "spans")] + span: Span::new(0..0), }, - PatternElement::TextElement { value: " bar" }, ], + #[cfg(feature = "spans")] + span: Span::new(0..0), }, default: false, + #[cfg(feature = "spans")] + span: Span::new(0..0), }; ast.body[0].as_message().as_pattern().elements[0] .as_expression() @@ -630,6 +668,8 @@ mod test { let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); ast.body[0].as_message().comment.replace(Comment { content: vec!["great message!"], + #[cfg(feature = "spans")] + span: Span::new(0..0), }); assert_eq!("# great message!\nfoo = bar\n", serialize(&ast)); } diff --git a/fluent-syntax/tests/serializer_fixtures.rs b/fluent-syntax/tests/serializer_fixtures.rs index 4af214a7..ab5b7225 100644 --- a/fluent-syntax/tests/serializer_fixtures.rs +++ b/fluent-syntax/tests/serializer_fixtures.rs @@ -30,6 +30,8 @@ fn clone_without_junk<'a>(original: &Resource<&'a str>) -> Resource<&'a str> { .filter(|entry| !matches!(entry, Entry::Junk { .. })) .cloned() .collect(), + #[cfg(feature = "spans")] + span: original.span, } } From e25b189f586b87d45fa798ec647c87be77a8eab5 Mon Sep 17 00:00:00 2001 From: Ertanic Date: Sun, 17 Nov 2024 20:52:01 +0300 Subject: [PATCH 02/10] Replace `new` to `default` --- fluent-syntax/src/serializer.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 5610de35..0231622d 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -487,21 +487,21 @@ mod test { id: Identifier { name: $name, #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }, value: Some(Pattern { elements: vec![PatternElement::TextElement { value: $value, #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }], #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }), attributes: vec![], comment: None, #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }) }; } @@ -586,29 +586,29 @@ mod test { id: Identifier { name: "num", #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }, #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }, #[cfg(feature = "spans")] - Span::new(0..0), + Span::default(), ), #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }, PatternElement::TextElement { value: " bar", #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }, ], #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }, default: false, #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }; ast.body[0].as_message().as_pattern().elements[0] .as_expression() @@ -669,7 +669,7 @@ mod test { ast.body[0].as_message().comment.replace(Comment { content: vec!["great message!"], #[cfg(feature = "spans")] - span: Span::new(0..0), + span: Span::default(), }); assert_eq!("# great message!\nfoo = bar\n", serialize(&ast)); } From fbd193e8b88cb938507052a1c1f091bf374037b4 Mon Sep 17 00:00:00 2001 From: Ertanic Date: Sat, 25 Jan 2025 16:37:54 +0300 Subject: [PATCH 03/10] Removed manual implementation of PartialEq, and oundtrip_unnormalized_fixtures test now uses pre-formatted messages that are expected with the correct spans --- fluent-syntax/src/ast/helper.rs | 17 +- fluent-syntax/src/ast/mod.rs | 249 ++---------------- .../tests/fixtures/formatted/any_char.ftl | 6 + .../tests/fixtures/formatted/astral.ftl | 14 + .../fixtures/formatted/call_expressions.ftl | 56 ++++ .../fixtures/formatted/callee_expressions.ftl | 36 +++ .../tests/fixtures/formatted/comments.ftl | 22 ++ fluent-syntax/tests/fixtures/formatted/cr.ftl | 2 + .../tests/fixtures/formatted/eof_comment.ftl | 5 + .../tests/fixtures/formatted/eof_empty.ftl | 0 .../tests/fixtures/formatted/eof_id.ftl | 2 + .../fixtures/formatted/eof_id_equals.ftl | 2 + .../tests/fixtures/formatted/eof_junk.ftl | 2 + .../tests/fixtures/formatted/eof_value.ftl | 3 + .../fixtures/formatted/escaped_characters.ftl | 38 +++ .../tests/fixtures/formatted/junk.ftl | 16 ++ .../fixtures/formatted/junked/any_char.ftl | 6 + .../fixtures/formatted/junked/astral.ftl | 21 ++ .../formatted/junked/call_expressions.ftl | 71 +++++ .../formatted/junked/callee_expressions.ftl | 58 ++++ .../fixtures/formatted/junked/comments.ftl | 25 ++ .../tests/fixtures/formatted/junked/cr.ftl | 2 + .../fixtures/formatted/junked/eof_comment.ftl | 5 + .../fixtures/formatted/junked/eof_empty.ftl | 0 .../fixtures/formatted/junked/eof_id.ftl | 3 + .../formatted/junked/eof_id_equals.ftl | 3 + .../fixtures/formatted/junked/eof_junk.ftl | 3 + .../fixtures/formatted/junked/eof_value.ftl | 3 + .../formatted/junked/escaped_characters.ftl | 46 ++++ .../tests/fixtures/formatted/junked/junk.ftl | 28 ++ .../formatted/junked/leading_dots.ftl | 70 +++++ .../formatted/junked/literal_expressions.ftl | 3 + .../formatted/junked/member_expressions.ftl | 23 ++ .../fixtures/formatted/junked/messages.ftl | 45 ++++ .../formatted/junked/mixed_entries.ftl | 24 ++ .../fixtures/formatted/junked/numbers.ftl | 32 +++ .../fixtures/formatted/junked/obsolete.ftl | 29 ++ .../fixtures/formatted/junked/placeables.ftl | 19 ++ .../junked/reference_expressions.ftl | 32 +++ .../formatted/junked/select_expressions.ftl | 68 +++++ .../formatted/junked/select_indent.ftl | 70 +++++ .../formatted/junked/sparse_entries.ftl | 17 ++ .../formatted/junked/special_chars.ftl | 14 + .../tests/fixtures/formatted/junked/tab.ftl | 21 ++ .../formatted/junked/term_parameters.ftl | 8 + .../tests/fixtures/formatted/junked/terms.ftl | 32 +++ .../fixtures/formatted/junked/variables.ftl | 17 ++ .../formatted/junked/variant_keys.ftl | 37 +++ .../formatted/junked/whitespace_in_value.ftl | 10 + .../fixtures/formatted/junked/zero_length.ftl | 0 .../tests/fixtures/formatted/leading_dots.ftl | 46 ++++ .../formatted/literal_expressions.ftl | 3 + .../fixtures/formatted/member_expressions.ftl | 17 ++ .../tests/fixtures/formatted/messages.ftl | 36 +++ .../fixtures/formatted/mixed_entries.ftl | 19 ++ .../tests/fixtures/formatted/numbers.ftl | 25 ++ .../tests/fixtures/formatted/obsolete.ftl | 9 + .../tests/fixtures/formatted/placeables.ftl | 12 + .../formatted/reference_expressions.ftl | 23 ++ .../fixtures/formatted/select_expressions.ftl | 38 +++ .../fixtures/formatted/select_indent.ftl | 65 +++++ .../fixtures/formatted/sparse_entries.ftl | 17 ++ .../fixtures/formatted/special_chars.ftl | 8 + .../tests/fixtures/formatted/tab.ftl | 14 + .../fixtures/formatted/term_parameters.ftl | 8 + .../tests/fixtures/formatted/terms.ftl | 20 ++ .../tests/fixtures/formatted/variables.ftl | 14 + .../tests/fixtures/formatted/variant_keys.ftl | 23 ++ .../formatted/whitespace_in_value.ftl | 10 + .../tests/fixtures/formatted/zero_length.ftl | 0 fluent-syntax/tests/serializer_fixtures.rs | 35 ++- 71 files changed, 1490 insertions(+), 267 deletions(-) create mode 100644 fluent-syntax/tests/fixtures/formatted/any_char.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/astral.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/call_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/callee_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/comments.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/cr.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/eof_comment.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/eof_empty.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/eof_id.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/eof_id_equals.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/eof_junk.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/eof_value.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/escaped_characters.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junk.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/any_char.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/astral.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/call_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/callee_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/comments.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/cr.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/eof_comment.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/eof_empty.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/eof_id.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/eof_id_equals.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/eof_junk.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/eof_value.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/escaped_characters.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/junk.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/leading_dots.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/literal_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/member_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/messages.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/mixed_entries.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/numbers.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/obsolete.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/placeables.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/reference_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/select_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/select_indent.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/sparse_entries.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/special_chars.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/tab.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/term_parameters.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/terms.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/variables.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/variant_keys.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/whitespace_in_value.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/junked/zero_length.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/leading_dots.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/literal_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/member_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/messages.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/mixed_entries.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/numbers.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/obsolete.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/placeables.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/reference_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/select_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/select_indent.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/sparse_entries.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/special_chars.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/tab.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/term_parameters.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/terms.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/variables.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/variant_keys.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/whitespace_in_value.ftl create mode 100644 fluent-syntax/tests/fixtures/formatted/zero_length.ftl diff --git a/fluent-syntax/src/ast/helper.rs b/fluent-syntax/src/ast/helper.rs index c803e1ac..fb5b86d6 100644 --- a/fluent-syntax/src/ast/helper.rs +++ b/fluent-syntax/src/ast/helper.rs @@ -7,8 +7,7 @@ use serde::{Deserialize, Serialize}; // This is a helper struct used to properly deserialize referential // JSON comments which are single continuous String, into a vec of // content slices. -#[derive(Debug, Clone)] -#[cfg_attr(not(feature = "spans"), derive(PartialEq, Eq))] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(untagged))] pub enum CommentDef { @@ -24,20 +23,6 @@ pub enum CommentDef { }, } -#[cfg(feature = "spans")] -impl Eq for CommentDef {} - -#[cfg(feature = "spans")] -impl PartialEq for CommentDef { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Single { content: l_content, .. }, Self::Single { content: r_content, .. }) => l_content == r_content, - (Self::Multi { content: l_content, .. }, Self::Multi { content: r_content, .. }) => l_content == r_content, - _ => false, - } - } -} - impl From> for Comment { fn from(input: CommentDef) -> Self { match input { diff --git a/fluent-syntax/src/ast/mod.rs b/fluent-syntax/src/ast/mod.rs index bf321259..ecf7c967 100644 --- a/fluent-syntax/src/ast/mod.rs +++ b/fluent-syntax/src/ast/mod.rs @@ -113,8 +113,7 @@ use std::ops::Range; /// } /// ); /// ``` -#[derive(Debug, Clone)] -#[cfg_attr(not(feature = "spans"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Resource { pub body: Vec>, @@ -122,13 +121,6 @@ pub struct Resource { pub span: Span, } -#[cfg(feature = "spans")] -impl PartialEq for Resource { - fn eq(&self, other: &Self) -> bool { - self.body == other.body - } -} - /// A top-level node representing an entry of a [`Resource`]. /// /// Every [`Entry`] is a standalone element and the parser is capable @@ -205,8 +197,7 @@ impl PartialEq for Resource { /// } /// ); /// ``` -#[derive(Debug, Clone)] -#[cfg_attr(not(feature = "spans"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum Entry { @@ -222,28 +213,6 @@ pub enum Entry { }, } -#[cfg(feature = "spans")] -impl PartialEq for Entry { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Message(l0), Self::Message(r0)) => l0 == r0, - (Self::Term(l0), Self::Term(r0)) => l0 == r0, - (Self::Comment(l0), Self::Comment(r0)) => l0 == r0, - (Self::GroupComment(l0), Self::GroupComment(r0)) => l0 == r0, - (Self::ResourceComment(l0), Self::ResourceComment(r0)) => l0 == r0, - ( - Self::Junk { - content: l_content, .. - }, - Self::Junk { - content: r_content, .. - }, - ) => l_content == r_content, - _ => false, - } - } -} - /// Message node represents the most common [`Entry`] in an FTL [`Resource`]. /// /// A message is a localization unit with a [`Identifier`] unique within a given @@ -292,7 +261,7 @@ impl PartialEq for Entry { /// } /// ); /// ``` -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Message { pub id: Identifier, @@ -303,15 +272,6 @@ pub struct Message { pub span: Span, } -impl PartialEq for Message { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - && self.value == other.value - && self.attributes == other.attributes - && self.comment == other.comment - } -} - /// A Fluent [`Term`]. /// /// Terms are semantically similar to [`Message`] nodes, but @@ -357,8 +317,7 @@ impl PartialEq for Message { /// } /// ); /// ``` -#[derive(Debug, Clone)] -#[cfg_attr(not(feature = "spans"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Term { pub id: Identifier, @@ -369,16 +328,6 @@ pub struct Term { pub span: Span, } -#[cfg(feature = "spans")] -impl PartialEq for Term { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - && self.value == other.value - && self.attributes == other.attributes - && self.comment == other.comment - } -} - /// Pattern contains a value of a [`Message`], [`Term`] or an [`Attribute`]. /// /// Each pattern is a list of [`PatternElement`] nodes representing @@ -450,7 +399,7 @@ impl PartialEq for Term { /// } /// ); /// ``` -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Pattern { pub elements: Vec>, @@ -458,12 +407,6 @@ pub struct Pattern { pub span: Span, } -impl PartialEq for Pattern { - fn eq(&self, other: &Self) -> bool { - self.elements == other.elements - } -} - /// `PatternElement` is an element of a [`Pattern`]. /// /// Each [`PatternElement`] node represents @@ -535,7 +478,7 @@ impl PartialEq for Pattern { /// } /// ); /// ``` -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum PatternElement { @@ -551,28 +494,6 @@ pub enum PatternElement { }, } -impl PartialEq for PatternElement { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - ( - Self::TextElement { value: l_value, .. }, - Self::TextElement { value: r_value, .. }, - ) => l_value == r_value, - ( - Self::Placeable { - expression: l_expression, - .. - }, - Self::Placeable { - expression: r_expression, - .. - }, - ) => l_expression == r_expression, - _ => false, - } - } -} - /// Attribute represents a part of a [`Message`] or [`Term`]. /// /// Attributes are used to express a compound list of keyed @@ -636,8 +557,7 @@ impl PartialEq for PatternElement { /// } /// ); /// ``` -#[derive(Debug, Clone)] -#[cfg_attr(not(feature = "spans"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Attribute { pub id: Identifier, @@ -646,13 +566,6 @@ pub struct Attribute { pub span: Span, } -#[cfg(feature = "spans")] -impl PartialEq for Attribute { - fn eq(&self, other: &Self) -> bool { - self.id == other.id && self.value == other.value - } -} - /// Identifier is part of nodes such as [`Message`], [`Term`] and [`Attribute`]. /// /// It is used to associate a unique key with an [`Entry`] or an [`Attribute`] @@ -695,7 +608,7 @@ impl PartialEq for Attribute { /// } /// ); /// ``` -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Identifier { pub name: S, @@ -703,14 +616,6 @@ pub struct Identifier { pub span: Span, } -impl Eq for Identifier {} - -impl PartialEq for Identifier { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - } -} - /// Variant is a single branch of a value in a [`Select`](Expression::Select) expression. /// /// It's a pair of [`VariantKey`] and [`Pattern`]. If the selector match the @@ -788,8 +693,7 @@ impl PartialEq for Identifier { /// } /// ); /// ``` -#[derive(Debug, Clone)] -#[cfg_attr(not(feature = "spans"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct Variant { @@ -800,13 +704,6 @@ pub struct Variant { pub span: Span, } -#[cfg(feature = "spans")] -impl PartialEq for Variant { - fn eq(&self, other: &Self) -> bool { - self.key == other.key && self.value == other.value && self.default == other.default - } -} - /// A key of a [`Variant`]. /// /// Variant key can be either an identifier or a number. @@ -883,7 +780,7 @@ impl PartialEq for Variant { /// } /// ); /// ``` -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, PartialEq, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum VariantKey { @@ -929,7 +826,7 @@ pub enum VariantKey { /// } /// ); /// ``` -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(from = "helper::CommentDef"))] pub struct Comment { @@ -938,14 +835,6 @@ pub struct Comment { pub span: Span, } -impl Eq for Comment {} - -impl PartialEq for Comment { - fn eq(&self, other: &Self) -> bool { - self.content == other.content - } -} - /// List of arguments for a [`FunctionReference`](InlineExpression::FunctionReference) or a /// [`TermReference`](InlineExpression::TermReference). /// @@ -1020,7 +909,7 @@ impl PartialEq for Comment { /// } /// ); /// ``` -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct CallArguments { @@ -1030,12 +919,6 @@ pub struct CallArguments { pub span: Span, } -impl PartialEq for CallArguments { - fn eq(&self, other: &Self) -> bool { - self.positional == other.positional && self.named == other.named - } -} - /// A key-value pair used in [`CallArguments`]. /// /// # Example @@ -1097,8 +980,7 @@ impl PartialEq for CallArguments { /// } /// ); /// ``` -#[derive(Debug, Clone)] -#[cfg_attr(not(feature = "spans"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub struct NamedArgument { @@ -1108,13 +990,6 @@ pub struct NamedArgument { pub span: Span, } -#[cfg(feature = "spans")] -impl PartialEq for NamedArgument { - fn eq(&self, other: &Self) -> bool { - self.name == other.name && self.value == other.value - } -} - /// A subset of expressions which can be used as [`Placeable`](PatternElement::Placeable), /// [`selector`](Expression::Select), or in [`CallArguments`]. /// @@ -1163,7 +1038,8 @@ impl PartialEq for NamedArgument { /// } /// ); /// ``` -#[derive(Debug, Clone)] + +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum InlineExpression { @@ -1541,74 +1417,6 @@ pub enum InlineExpression { }, } -impl PartialEq for InlineExpression { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - ( - Self::StringLiteral { value: l_value, .. }, - Self::StringLiteral { value: r_value, .. }, - ) => l_value == r_value, - ( - Self::NumberLiteral { value: l_value, .. }, - Self::NumberLiteral { value: r_value, .. }, - ) => l_value == r_value, - ( - Self::FunctionReference { - id: l_id, - arguments: l_arguments, - .. - }, - Self::FunctionReference { - id: r_id, - arguments: r_arguments, - .. - }, - ) => l_id == r_id && l_arguments == r_arguments, - ( - Self::MessageReference { - id: l_id, - attribute: l_attribute, - .. - }, - Self::MessageReference { - id: r_id, - attribute: r_attribute, - .. - }, - ) => l_id == r_id && l_attribute == r_attribute, - ( - Self::TermReference { - id: l_id, - attribute: l_attribute, - arguments: l_arguments, - .. - }, - Self::TermReference { - id: r_id, - attribute: r_attribute, - arguments: r_arguments, - .. - }, - ) => l_id == r_id && l_attribute == r_attribute && l_arguments == r_arguments, - ( - Self::VariableReference { id: l_id, .. }, - Self::VariableReference { id: r_id, .. }, - ) => l_id == r_id, - ( - Self::Placeable { - expression: l_expression, - .. - }, - Self::Placeable { - expression: r_expression, - .. - }, - ) => l_expression == r_expression, - _ => false, - } - } -} - #[cfg(feature = "spans")] impl InlineExpression { pub fn get_span(&self) -> Span { @@ -1698,8 +1506,7 @@ impl InlineExpression { /// } /// ); /// ``` -#[derive(Debug, Clone)] -#[cfg_attr(not(feature = "spans"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(untagged))] pub enum Expression { @@ -1725,28 +1532,6 @@ pub enum Expression { Inline(InlineExpression, #[cfg(feature = "spans")] Span), } -#[cfg(feature = "spans")] -impl PartialEq for Expression { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - ( - Self::Select { - selector: l_selector, - variants: l_variants, - .. - }, - Self::Select { - selector: r_selector, - variants: r_variants, - .. - }, - ) => l_selector == r_selector && l_variants == r_variants, - (Self::Inline(l0, ..), Self::Inline(r0, ..)) => l0 == r0, - _ => false, - } - } -} - /// A span of a node. Allows you to get the index of the start and end character of a node. /// /// # Example @@ -1781,4 +1566,4 @@ impl Span { end: range.end, } } -} +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/formatted/any_char.ftl b/fluent-syntax/tests/fixtures/formatted/any_char.ftl new file mode 100644 index 00000000..76a1839b --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/any_char.ftl @@ -0,0 +1,6 @@ +# ↓ BEL, U+0007 +control0 = abcdef +# ↓ DEL, U+007F +delete = abcdef +# ↓ BPM, U+0082 +control1 = abc‚def diff --git a/fluent-syntax/tests/fixtures/formatted/astral.ftl b/fluent-syntax/tests/fixtures/formatted/astral.ftl new file mode 100644 index 00000000..f650be0f --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/astral.ftl @@ -0,0 +1,14 @@ +face-with-tears-of-joy = 😂 +tetragram-for-centre = 𝌆 +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = { "\uD83D\uDE02" } +surrogates-in-adjacent-strings = { "\uD83D" }{ "\uDE02" } +emoji-in-text = A face 😂 with tears of joy. +emoji-in-string = { "A face 😂 with tears of joy." } + +# ERROR Invalid identifier + +# ERROR Invalid expression + +# ERROR Invalid variant key + diff --git a/fluent-syntax/tests/fixtures/formatted/call_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/call_expressions.ftl new file mode 100644 index 00000000..ffef8dca --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/call_expressions.ftl @@ -0,0 +1,56 @@ +## Function names + +valid-func-name-01 = { FUN1() } +valid-func-name-02 = { FUN_FUN() } +valid-func-name-03 = { FUN-FUN() } + +# JUNK 0 is not a valid Identifier start + +# JUNK Function names may not be lowercase + +# JUNK Function names may not contain lowercase character + +# JUNK ? is not a valid Identifier character + +## Arguments + +positional-args = { FUN(1, "a", msg) } +named-args = { FUN(x: 1, y: "Y") } +dense-named-args = { FUN(x: 1, y: "Y") } +mixed-args = { FUN(1, "a", msg, x: 1, y: "Y") } + +# ERROR Positional arg must not follow keyword args + +# ERROR Named arguments must be unique + +## Whitespace around arguments + +sparse-inline-call = { FUN("a", msg, x: 1) } +empty-inline-call = { FUN() } +multiline-call = { FUN("a", msg, x: 1) } +sparse-multiline-call = { FUN("a", msg, x: 1) } +empty-multiline-call = { FUN() } +unindented-arg-number = { FUN(1) } +unindented-arg-string = { FUN("a") } +unindented-arg-msg-ref = { FUN(msg) } +unindented-arg-term-ref = { FUN(-msg) } +unindented-arg-var-ref = { FUN($var) } +unindented-arg-call = { FUN(OTHER()) } +unindented-named-arg = { FUN(x: 1) } +unindented-closing-paren = { FUN(x) } + +## Optional trailing comma + +one-argument = { FUN(1) } +many-arguments = { FUN(1, 2, 3) } +inline-sparse-args = { FUN(1, 2, 3) } +mulitline-args = { FUN(1, 2) } +mulitline-sparse-args = { FUN(1, 2) } + +## Syntax errors for trailing comma + +## Whitespace in named arguments + +sparse-named-arg = { FUN(x: 1, y: 2, z: 3) } +unindented-colon = { FUN(x: 1) } +unindented-value = { FUN(x: 1) } diff --git a/fluent-syntax/tests/fixtures/formatted/callee_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/callee_expressions.ftl new file mode 100644 index 00000000..edc68ab5 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/callee_expressions.ftl @@ -0,0 +1,36 @@ +## Callees in placeables. + +function-callee-placeable = { FUNCTION() } +term-callee-placeable = { -term() } + +# ERROR Messages cannot be parameterized. + +# ERROR Equivalent to a MessageReference callee. + +# ERROR Message attributes cannot be parameterized. + +# ERROR Term attributes may not be used in Placeables. + +# ERROR Variables cannot be parameterized. + +## Callees in selectors. + +function-callee-selector = + { FUNCTION() -> + *[key] Value + } +term-attr-callee-selector = + { -term.attr() -> + *[key] Value + } + +# ERROR Messages cannot be parameterized. + +# ERROR Equivalent to a MessageReference callee. + +# ERROR Message attributes cannot be parameterized. + +# ERROR Term values may not be used as selectors. + +# ERROR Variables cannot be parameterized. + diff --git a/fluent-syntax/tests/fixtures/formatted/comments.ftl b/fluent-syntax/tests/fixtures/formatted/comments.ftl new file mode 100644 index 00000000..45754cd9 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/comments.ftl @@ -0,0 +1,22 @@ +# Standalone Comment + +# Message Comment +foo = Foo +# Term Comment +# with a blank last line. +# +-term = Term + +# Another standalone +# +# with indent + + +## Group Comment + + +### Resource Comment + + +# Errors + diff --git a/fluent-syntax/tests/fixtures/formatted/cr.ftl b/fluent-syntax/tests/fixtures/formatted/cr.ftl new file mode 100644 index 00000000..ace98efb --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/cr.ftl @@ -0,0 +1,2 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_comment.ftl b/fluent-syntax/tests/fixtures/formatted/eof_comment.ftl new file mode 100644 index 00000000..d262c3b3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_comment.ftl @@ -0,0 +1,5 @@ +### NOTE: Disable final newline insertion when editing this file. + + +# No EOL + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_empty.ftl b/fluent-syntax/tests/fixtures/formatted/eof_empty.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/fixtures/formatted/eof_id.ftl b/fluent-syntax/tests/fixtures/formatted/eof_id.ftl new file mode 100644 index 00000000..c938b0bc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_id.ftl @@ -0,0 +1,2 @@ +### NOTE: Disable final newline insertion when editing this file. + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_id_equals.ftl b/fluent-syntax/tests/fixtures/formatted/eof_id_equals.ftl new file mode 100644 index 00000000..c938b0bc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_id_equals.ftl @@ -0,0 +1,2 @@ +### NOTE: Disable final newline insertion when editing this file. + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_junk.ftl b/fluent-syntax/tests/fixtures/formatted/eof_junk.ftl new file mode 100644 index 00000000..c938b0bc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_junk.ftl @@ -0,0 +1,2 @@ +### NOTE: Disable final newline insertion when editing this file. + diff --git a/fluent-syntax/tests/fixtures/formatted/eof_value.ftl b/fluent-syntax/tests/fixtures/formatted/eof_value.ftl new file mode 100644 index 00000000..23ee1b60 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/eof_value.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +no-eol = No EOL diff --git a/fluent-syntax/tests/fixtures/formatted/escaped_characters.ftl b/fluent-syntax/tests/fixtures/formatted/escaped_characters.ftl new file mode 100644 index 00000000..29c56f50 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/escaped_characters.ftl @@ -0,0 +1,38 @@ +## Literal text + +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{ placeable } +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 + +## String literals + +quote-in-string = { "\"" } +backslash-in-string = { "\\" } + +# ERROR Mismatched quote + +# ERROR Unknown escape + +# ERROR Multiline literal + +## Unicode escapes + +string-unicode-4digits = { "\u0041" } +escape-unicode-4digits = { "\\u0041" } +string-unicode-6digits = { "\U01F602" } +escape-unicode-6digits = { "\\U01F602" } +# OK The trailing "00" is part of the literal value. +string-too-many-4digits = { "\u004100" } +# OK The trailing "00" is part of the literal value. +string-too-many-6digits = { "\U01F60200" } + +# ERROR Too few hex digits after \u. + +# ERROR Too few hex digits after \U. + +## Literal braces + +brace-open = An opening { "{" } brace. +brace-close = A closing { "}" } brace. diff --git a/fluent-syntax/tests/fixtures/formatted/junk.ftl b/fluent-syntax/tests/fixtures/formatted/junk.ftl new file mode 100644 index 00000000..d0ff0158 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junk.ftl @@ -0,0 +1,16 @@ +## Two adjacent Junks. + +# A single Junk. + +# A single Junk. + +# The COMMENT ends this junk. + +# COMMENT + + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. + +# COMMENT + diff --git a/fluent-syntax/tests/fixtures/formatted/junked/any_char.ftl b/fluent-syntax/tests/fixtures/formatted/junked/any_char.ftl new file mode 100644 index 00000000..76a1839b --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/any_char.ftl @@ -0,0 +1,6 @@ +# ↓ BEL, U+0007 +control0 = abcdef +# ↓ DEL, U+007F +delete = abcdef +# ↓ BPM, U+0082 +control1 = abc‚def diff --git a/fluent-syntax/tests/fixtures/formatted/junked/astral.ftl b/fluent-syntax/tests/fixtures/formatted/junked/astral.ftl new file mode 100644 index 00000000..f29fef0a --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/astral.ftl @@ -0,0 +1,21 @@ +face-with-tears-of-joy = 😂 +tetragram-for-centre = 𝌆 +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = { "\uD83D\uDE02" } +surrogates-in-adjacent-strings = { "\uD83D" }{ "\uDE02" } +emoji-in-text = A face 😂 with tears of joy. +emoji-in-string = { "A face 😂 with tears of joy." } + +# ERROR Invalid identifier + +err-😂 = Value + +# ERROR Invalid expression + +err-invalid-expression = { 😂 } + +# ERROR Invalid variant key + +err-invalid-variant-key = { $sel -> + *[😂] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/call_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/call_expressions.ftl new file mode 100644 index 00000000..46a531fe --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/call_expressions.ftl @@ -0,0 +1,71 @@ +## Function names + +valid-func-name-01 = { FUN1() } +valid-func-name-02 = { FUN_FUN() } +valid-func-name-03 = { FUN-FUN() } + +# JUNK 0 is not a valid Identifier start + +invalid-func-name-01 = {0FUN()} +# JUNK Function names may not be lowercase + +invalid-func-name-02 = {fun()} +# JUNK Function names may not contain lowercase character + +invalid-func-name-03 = {Fun()} +# JUNK ? is not a valid Identifier character + +invalid-func-name-04 = {FUN?()} + +## Arguments + +positional-args = { FUN(1, "a", msg) } +named-args = { FUN(x: 1, y: "Y") } +dense-named-args = { FUN(x: 1, y: "Y") } +mixed-args = { FUN(1, "a", msg, x: 1, y: "Y") } + +# ERROR Positional arg must not follow keyword args + +shuffled-args = {FUN(1, x: 1, "a", y: "Y", msg)} + +# ERROR Named arguments must be unique + +duplicate-named-args = {FUN(x: 1, x: "X")} + + +## Whitespace around arguments + +sparse-inline-call = { FUN("a", msg, x: 1) } +empty-inline-call = { FUN() } +multiline-call = { FUN("a", msg, x: 1) } +sparse-multiline-call = { FUN("a", msg, x: 1) } +empty-multiline-call = { FUN() } +unindented-arg-number = { FUN(1) } +unindented-arg-string = { FUN("a") } +unindented-arg-msg-ref = { FUN(msg) } +unindented-arg-term-ref = { FUN(-msg) } +unindented-arg-var-ref = { FUN($var) } +unindented-arg-call = { FUN(OTHER()) } +unindented-named-arg = { FUN(x: 1) } +unindented-closing-paren = { FUN(x) } + +## Optional trailing comma + +one-argument = { FUN(1) } +many-arguments = { FUN(1, 2, 3) } +inline-sparse-args = { FUN(1, 2, 3) } +mulitline-args = { FUN(1, 2) } +mulitline-sparse-args = { FUN(1, 2) } + +## Syntax errors for trailing comma + +one-argument = {FUN(1,,)} +missing-arg = {FUN(,)} +missing-sparse-arg = {FUN( , )} + + +## Whitespace in named arguments + +sparse-named-arg = { FUN(x: 1, y: 2, z: 3) } +unindented-colon = { FUN(x: 1) } +unindented-value = { FUN(x: 1) } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/callee_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/callee_expressions.ftl new file mode 100644 index 00000000..ca98eae7 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/callee_expressions.ftl @@ -0,0 +1,58 @@ +## Callees in placeables. + +function-callee-placeable = { FUNCTION() } +term-callee-placeable = { -term() } + +# ERROR Messages cannot be parameterized. + +message-callee-placeable = {message()} +# ERROR Equivalent to a MessageReference callee. + +mixed-case-callee-placeable = {Function()} +# ERROR Message attributes cannot be parameterized. + +message-attr-callee-placeable = {message.attr()} +# ERROR Term attributes may not be used in Placeables. + +term-attr-callee-placeable = {-term.attr()} +# ERROR Variables cannot be parameterized. + +variable-callee-placeable = {$variable()} + + +## Callees in selectors. + +function-callee-selector = + { FUNCTION() -> + *[key] Value + } +term-attr-callee-selector = + { -term.attr() -> + *[key] Value + } + +# ERROR Messages cannot be parameterized. + +message-callee-selector = {message() -> + *[key] Value +} +# ERROR Equivalent to a MessageReference callee. + +mixed-case-callee-selector = {Function() -> + *[key] Value +} +# ERROR Message attributes cannot be parameterized. + +message-attr-callee-selector = {message.attr() -> + *[key] Value +} +# ERROR Term values may not be used as selectors. + +term-callee-selector = {-term() -> + *[key] Value +} +# ERROR Variables cannot be parameterized. + +variable-callee-selector = {$variable() -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/comments.ftl b/fluent-syntax/tests/fixtures/formatted/junked/comments.ftl new file mode 100644 index 00000000..be10bad2 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/comments.ftl @@ -0,0 +1,25 @@ +# Standalone Comment + +# Message Comment +foo = Foo +# Term Comment +# with a blank last line. +# +-term = Term + +# Another standalone +# +# with indent + + +## Group Comment + + +### Resource Comment + + +# Errors + +#error +##error +###error diff --git a/fluent-syntax/tests/fixtures/formatted/junked/cr.ftl b/fluent-syntax/tests/fixtures/formatted/junked/cr.ftl new file mode 100644 index 00000000..ace98efb --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/cr.ftl @@ -0,0 +1,2 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } + diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_comment.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_comment.ftl new file mode 100644 index 00000000..d262c3b3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_comment.ftl @@ -0,0 +1,5 @@ +### NOTE: Disable final newline insertion when editing this file. + + +# No EOL + diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_empty.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_empty.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_id.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_id.ftl new file mode 100644 index 00000000..63fa86d6 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_id.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_id_equals.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_id_equals.ftl new file mode 100644 index 00000000..7d0d953a --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_id_equals.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id = \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_junk.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_junk.ftl new file mode 100644 index 00000000..dbafd3a3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_junk.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +000 \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/formatted/junked/eof_value.ftl b/fluent-syntax/tests/fixtures/formatted/junked/eof_value.ftl new file mode 100644 index 00000000..23ee1b60 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/eof_value.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +no-eol = No EOL diff --git a/fluent-syntax/tests/fixtures/formatted/junked/escaped_characters.ftl b/fluent-syntax/tests/fixtures/formatted/junked/escaped_characters.ftl new file mode 100644 index 00000000..b6025b76 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/escaped_characters.ftl @@ -0,0 +1,46 @@ +## Literal text + +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{ placeable } +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 + +## String literals + +quote-in-string = { "\"" } +backslash-in-string = { "\\" } + +# ERROR Mismatched quote + +mismatched-quote = {"\\""} +# ERROR Unknown escape + +unknown-escape = {"\x"} +# ERROR Multiline literal + +invalid-multiline-literal = {" + "} + +## Unicode escapes + +string-unicode-4digits = { "\u0041" } +escape-unicode-4digits = { "\\u0041" } +string-unicode-6digits = { "\U01F602" } +escape-unicode-6digits = { "\\U01F602" } +# OK The trailing "00" is part of the literal value. +string-too-many-4digits = { "\u004100" } +# OK The trailing "00" is part of the literal value. +string-too-many-6digits = { "\U01F60200" } + +# ERROR Too few hex digits after \u. + +string-too-few-4digits = {"\u41"} +# ERROR Too few hex digits after \U. + +string-too-few-6digits = {"\U1F602"} + +## Literal braces + +brace-open = An opening { "{" } brace. +brace-close = A closing { "}" } brace. diff --git a/fluent-syntax/tests/fixtures/formatted/junked/junk.ftl b/fluent-syntax/tests/fixtures/formatted/junked/junk.ftl new file mode 100644 index 00000000..6085bb92 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/junk.ftl @@ -0,0 +1,28 @@ +## Two adjacent Junks. + +err01 = {1x} +err02 = {2x} + +# A single Junk. + +err03 = {1x +2 + +# A single Junk. + +ą=Invalid identifier +ć=Another one + +# The COMMENT ends this junk. + +err04 = { +# COMMENT + + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. + +err04 = { +# COMMENT + +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/leading_dots.ftl b/fluent-syntax/tests/fixtures/formatted/junked/leading_dots.ftl new file mode 100644 index 00000000..e87e4097 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/leading_dots.ftl @@ -0,0 +1,70 @@ +key01 = .Value +key02 = …Value +key03 = { "." }Value +key04 = { "." }Value +key05 = + Value + { "." }Continued +key06 = .Value +{ "." }Continued +# MESSAGE (value = "Value", attributes = []) +# JUNK (attr .Continued" must have a value) +key07 = Value + .Continued + +# JUNK (attr .Value must have a value) + +key08 = + .Value + +# JUNK (attr .Value must have a value) + +key09 = + .Value + Continued + +key10 = + .Value = + which is an attribute + Continued +key11 = + { "." }Value = which looks like an attribute + Continued +key12 = + .accesskey = A +key13 = + .attribute = .Value +key14 = + .attribute = { "." }Value +key15 = + { 1 -> + [one] .Value + *[other] { "." }Value + } + +# JUNK (variant must have a value) + +key16 = + { 1 -> + *[one] + .Value + } + +# JUNK (unclosed placeable) + +key17 = + { 1 -> + *[one] Value + .Continued + } + +# JUNK (attr .Value must have a value) + +key18 = +.Value + +key19 = + .attribute = + Value + Continued +key20 = { "." }Value diff --git a/fluent-syntax/tests/fixtures/formatted/junked/literal_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/literal_expressions.ftl new file mode 100644 index 00000000..8b45c427 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/literal_expressions.ftl @@ -0,0 +1,3 @@ +string-expression = { "abc" } +number-expression = { 123 } +number-expression = { -3.14 } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/member_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/member_expressions.ftl new file mode 100644 index 00000000..3a1089c8 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/member_expressions.ftl @@ -0,0 +1,23 @@ +## Member expressions in placeables. + +# OK Message attributes may be interpolated in values. +message-attribute-expression-placeable = { msg.attr } + +# ERROR Term attributes may not be used for interpolation. + +term-attribute-expression-placeable = {-term.attr} + + +## Member expressions in selectors. + +# OK Term attributes may be used as selectors. +term-attribute-expression-selector = + { -term.attr -> + *[key] Value + } + +# ERROR Message attributes may not be used as selectors. + +message-attribute-expression-selector = {msg.attr -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/messages.ftl b/fluent-syntax/tests/fixtures/formatted/junked/messages.ftl new file mode 100644 index 00000000..a5bcd1f1 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/messages.ftl @@ -0,0 +1,45 @@ +key01 = Value +key02 = Value + .attr = Attribute +key02 = Value + .attr1 = Attribute 1 + .attr2 = Attribute 2 +key03 = + .attr = Attribute +key04 = + .attr1 = Attribute 1 + .attr2 = Attribute 2 +# < whitespace > +key05 = + .attr1 = Attribute 1 +no-whitespace = Value + .attr1 = Attribute 1 +extra-whitespace = Value + .attr1 = Attribute 1 +key06 = { "" } + +# JUNK Missing value + +key07 = + +# JUNK Missing = + +key08 + +KEY09 = Value 09 +key-10 = Value 10 +key_11 = Value 11 +key-12- = Value 12 +key_13_ = Value 13 + +# JUNK Invalid id + +0err-14 = Value 14 + +# JUNK Invalid id + +err-15? = Value 15 + +# JUNK Invalid id + +err-ąę-16 = Value 16 diff --git a/fluent-syntax/tests/fixtures/formatted/junked/mixed_entries.ftl b/fluent-syntax/tests/fixtures/formatted/junked/mixed_entries.ftl new file mode 100644 index 00000000..57480b45 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/mixed_entries.ftl @@ -0,0 +1,24 @@ +# License Comment + + +### Resource Comment + +-brand-name = Aurora + +## Group Comment + +key01 = + .attr = Attribute +ą=Invalid identifier +ć=Another one + +# Message Comment +key02 = Value + +# Standalone Comment + + .attr = Dangling attribute + +# There are 5 spaces on the line between key03 and key04. +key03 = Value 03 +key04 = Value 04 diff --git a/fluent-syntax/tests/fixtures/formatted/junked/numbers.ftl b/fluent-syntax/tests/fixtures/formatted/junked/numbers.ftl new file mode 100644 index 00000000..08d6e6b1 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/numbers.ftl @@ -0,0 +1,32 @@ +int-zero = { 0 } +int-positive = { 1 } +int-negative = { -1 } +int-negative-zero = { -0 } +int-positive-padded = { 01 } +int-negative-padded = { -01 } +int-zero-padded = { 00 } +int-negative-zero-padded = { -00 } +float-zero = { 0.0 } +float-positive = { 0.01 } +float-positive-one = { 1.03 } +float-positive-without-fraction = { 1.000 } +float-negative = { -0.01 } +float-negative-one = { -1.03 } +float-negative-zero = { -0.0 } +float-negative-without-fraction = { -1.000 } +float-positive-padded-left = { 01.03 } +float-positive-padded-right = { 1.0300 } +float-positive-padded-both = { 01.0300 } +float-negative-padded-left = { -01.03 } +float-negative-padded-right = { -1.0300 } +float-negative-padded-both = { -01.0300 } + +## ERRORS + +err01 = {1.} +err02 = {.02} +err03 = {1.02.03} +err04 = {1. 02} +err05 = {1 .02} +err06 = {- 1} +err07 = {1,02} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/obsolete.ftl b/fluent-syntax/tests/fixtures/formatted/junked/obsolete.ftl new file mode 100644 index 00000000..e0869e3a --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/obsolete.ftl @@ -0,0 +1,29 @@ +### The syntax in this file has been discontinued. It is no longer part of the +### Fluent specification and should not be implemented nor used. We're keeping +### these fixtures around to protect against accidental syntax reuse. + + +## Variant lists. + +message-variant-list = + { + *[key] Value + } + +-term-variant-list = + { + *[key] Value + } + + +## Variant expressions. + +message-variant-expression-placeable = {msg[case]} +message-variant-expression-selector = {msg[case] -> + *[key] Value +} + +term-variant-expression-placeable = {-term[case]} +term-variant-expression-selector = {-term[case] -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/placeables.ftl b/fluent-syntax/tests/fixtures/formatted/junked/placeables.ftl new file mode 100644 index 00000000..1a2013b9 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/placeables.ftl @@ -0,0 +1,19 @@ +nested-placeable = {{ {1} }} +padded-placeable = { 1 } +sparse-placeable = {{ 1 }} + +# ERROR Unmatched opening brace + +unmatched-open1 = { 1 + +# ERROR Unmatched opening brace + +unmatched-open2 = {{ 1 } + +# ERROR Unmatched closing brace + +unmatched-close1 = 1 } + +# ERROR Unmatched closing brace + +unmatched-close2 = { 1 }} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/reference_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/reference_expressions.ftl new file mode 100644 index 00000000..1f4b8674 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/reference_expressions.ftl @@ -0,0 +1,32 @@ +## Reference expressions in placeables. + +message-reference-placeable = { msg } +term-reference-placeable = { -term } +variable-reference-placeable = { $var } +# Function references are invalid outside of call expressions. +# This parses as a valid MessageReference. +function-reference-placeable = { FUN } + +## Reference expressions in selectors. + +variable-reference-selector = + { $var -> + *[key] Value + } + +# ERROR Message values may not be used as selectors. + +message-reference-selector = {msg -> + *[key] Value +} +# ERROR Term values may not be used as selectors. + +term-reference-selector = {-term -> + *[key] Value +} +# ERROR Function references are invalid outside of call expressions, and this +# parses as a MessageReference which isn't a valid selector. + +function-expression-selector = {FUN -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/select_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/junked/select_expressions.ftl new file mode 100644 index 00000000..4598442e --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/select_expressions.ftl @@ -0,0 +1,68 @@ +new-messages = + { BUILTIN() -> + [0] Zero + *[other] { "" }Other + } +valid-selector-term-attribute = + { -term.case -> + *[key] value + } + +# ERROR Term values are not valid selectors + +invalid-selector-term-value = + { -term -> + *[key] value + } + +# ERROR CallExpressions on Terms are similar to TermReferences + +invalid-selector-term-variant = + { -term(case: "nominative") -> + *[key] value + } + +# ERROR Nested expressions are not valid selectors + +invalid-selector-nested-expression = + { { 3 } -> + *[key] default + } + +# ERROR Select expressions are not valid selectors + +invalid-selector-select-expression = + { { $sel -> + *[key] value + } -> + *[key] default + } + +empty-variant = + { $sel -> + *[key] { "" } + } +reduced-whitespace = + { FOO() -> + *[key] { "" } + } +nested-select = + { $sel -> + *[one] + { $sel -> + *[two] Value + } + } + +# ERROR Missing selector + +missing-selector = + { + *[key] Value + } + +# ERROR Missing line end after variant list + +missing-line-end = + { $sel -> + *[key] Value} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/select_indent.ftl b/fluent-syntax/tests/fixtures/formatted/junked/select_indent.ftl new file mode 100644 index 00000000..3c21d71f --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/select_indent.ftl @@ -0,0 +1,70 @@ +select-1tbs-inline = + { $selector -> + *[key] Value + } +select-1tbs-newline = + { $selector -> + *[key] Value + } +select-1tbs-indent = + { $selector -> + *[key] Value + } +select-allman-inline = + { $selector -> + *[key] Value + [other] Other + } +select-allman-newline = + { $selector -> + *[key] Value + } +select-allman-indent = + { $selector -> + *[key] Value + } +select-gnu-inline = + { $selector -> + *[key] Value + } +select-gnu-newline = + { $selector -> + *[key] Value + } +select-gnu-indent = + { $selector -> + *[key] Value + } +select-no-indent = + { $selector -> + *[key] Value + [other] Other + } +select-no-indent-multiline = + { $selector -> + *[key] + Value + Continued + [other] + Other + Multiline + } + +# ERROR (Multiline text must be indented) + +select-no-indent-multiline = { $selector -> + *[key] Value +Continued without indent. +} + +select-flat = + { $selector -> + *[key] Value + [other] Other + } +# Each line ends with 5 spaces. +select-flat-with-trailing-spaces = + { $selector -> + *[key] Value + [other] Other + } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/sparse_entries.ftl b/fluent-syntax/tests/fixtures/formatted/junked/sparse_entries.ftl new file mode 100644 index 00000000..dfa85945 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/sparse_entries.ftl @@ -0,0 +1,17 @@ +key01 = Value +key02 = + .attr = Attribute +key03 = + Value + Continued + + + Over multiple + Lines + .attr = Attribute +key05 = Value +key06 = + { 1 -> + [one] One + *[two] Two + } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/special_chars.ftl b/fluent-syntax/tests/fixtures/formatted/junked/special_chars.ftl new file mode 100644 index 00000000..5224bad7 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/special_chars.ftl @@ -0,0 +1,14 @@ +## OK + +bracket-inline = [Value] +dot-inline = .Value +star-inline = *Value + +## ERRORS + +bracket-newline = + [Value] +dot-newline = + .Value +star-newline = + *Value diff --git a/fluent-syntax/tests/fixtures/formatted/junked/tab.ftl b/fluent-syntax/tests/fixtures/formatted/junked/tab.ftl new file mode 100644 index 00000000..46b5aa9f --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/tab.ftl @@ -0,0 +1,21 @@ +# OK (tab after = is part of the value) +key01 = Value 01 + +# Error (tab before =) + +key02 = Value 02 + +# Error (tab is not a valid indent) + +key03 = + This line isn't properly indented. + +# Partial Error (tab is not a valid indent) +key04 = This line is indented by 4 spaces, + whereas this line by 1 tab. + +# OK (value is a single tab) +key05 = +# OK (attribute value is two tabs) +key06 = + .attr = diff --git a/fluent-syntax/tests/fixtures/formatted/junked/term_parameters.ftl b/fluent-syntax/tests/fixtures/formatted/junked/term_parameters.ftl new file mode 100644 index 00000000..8ee2ed74 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/term_parameters.ftl @@ -0,0 +1,8 @@ +-term = + { $arg -> + *[key] Value + } +key01 = { -term } +key02 = { -term() } +key03 = { -term(arg: 1) } +key04 = { -term("positional", narg1: 1, narg2: 2) } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/terms.ftl b/fluent-syntax/tests/fixtures/formatted/junked/terms.ftl new file mode 100644 index 00000000..4bd9e6cc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/terms.ftl @@ -0,0 +1,32 @@ +-term01 = Value + .attr = Attribute +-term02 = { "" } + +# JUNK Missing value + +-term03 = + .attr = Attribute + +# JUNK Missing value +# < whitespace > + +-term04 = + .attr1 = Attribute 1 + +# JUNK Missing value + +-term05 = + +# JUNK Missing value +# < whitespace > + +-term06 = + +# JUNK Missing = + +-term07 + +-term08 = Value + .attr = Attribute +-term09 = Value + .attr = Attribute diff --git a/fluent-syntax/tests/fixtures/formatted/junked/variables.ftl b/fluent-syntax/tests/fixtures/formatted/junked/variables.ftl new file mode 100644 index 00000000..1a12cc59 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/variables.ftl @@ -0,0 +1,17 @@ +key01 = { $var } +key02 = { $var } +key03 = { $var } +key04 = { $var } + +## Errors + + +# ERROR Missing variable identifier + +err01 = {$} +# ERROR Double $$ + +err02 = {$$var} +# ERROR Invalid first char of the identifier + +err03 = {$-var} diff --git a/fluent-syntax/tests/fixtures/formatted/junked/variant_keys.ftl b/fluent-syntax/tests/fixtures/formatted/junked/variant_keys.ftl new file mode 100644 index 00000000..e3ee17f2 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/variant_keys.ftl @@ -0,0 +1,37 @@ +simple-identifier = + { $sel -> + *[key] value + } +identifier-surrounded-by-whitespace = + { $sel -> + *[key] value + } +int-number = + { $sel -> + *[1] value + } +float-number = + { $sel -> + *[3.14] value + } + +# ERROR + +invalid-identifier = + { $sel -> + *[two words] value + } + +# ERROR + +invalid-int = + { $sel -> + *[1 apple] value + } + +# ERROR + +invalid-int = + { $sel -> + *[3.14 apples] value + } diff --git a/fluent-syntax/tests/fixtures/formatted/junked/whitespace_in_value.ftl b/fluent-syntax/tests/fixtures/formatted/junked/whitespace_in_value.ftl new file mode 100644 index 00000000..0cbcd371 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/junked/whitespace_in_value.ftl @@ -0,0 +1,10 @@ +# Caution, lines 6 and 7 contain white-space-only lines +key = + first line + + + + + + + last line diff --git a/fluent-syntax/tests/fixtures/formatted/junked/zero_length.ftl b/fluent-syntax/tests/fixtures/formatted/junked/zero_length.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/fixtures/formatted/leading_dots.ftl b/fluent-syntax/tests/fixtures/formatted/leading_dots.ftl new file mode 100644 index 00000000..d63f60e3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/leading_dots.ftl @@ -0,0 +1,46 @@ +key01 = .Value +key02 = …Value +key03 = { "." }Value +key04 = { "." }Value +key05 = + Value + { "." }Continued +key06 = .Value +{ "." }Continued +# MESSAGE (value = "Value", attributes = []) +# JUNK (attr .Continued" must have a value) +key07 = Value +# JUNK (attr .Value must have a value) + +# JUNK (attr .Value must have a value) + +key10 = + .Value = + which is an attribute + Continued +key11 = + { "." }Value = which looks like an attribute + Continued +key12 = + .accesskey = A +key13 = + .attribute = .Value +key14 = + .attribute = { "." }Value +key15 = + { 1 -> + [one] .Value + *[other] { "." }Value + } + +# JUNK (variant must have a value) + +# JUNK (unclosed placeable) + +# JUNK (attr .Value must have a value) + +key19 = + .attribute = + Value + Continued +key20 = { "." }Value diff --git a/fluent-syntax/tests/fixtures/formatted/literal_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/literal_expressions.ftl new file mode 100644 index 00000000..8b45c427 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/literal_expressions.ftl @@ -0,0 +1,3 @@ +string-expression = { "abc" } +number-expression = { 123 } +number-expression = { -3.14 } diff --git a/fluent-syntax/tests/fixtures/formatted/member_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/member_expressions.ftl new file mode 100644 index 00000000..809c2608 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/member_expressions.ftl @@ -0,0 +1,17 @@ +## Member expressions in placeables. + +# OK Message attributes may be interpolated in values. +message-attribute-expression-placeable = { msg.attr } + +# ERROR Term attributes may not be used for interpolation. + +## Member expressions in selectors. + +# OK Term attributes may be used as selectors. +term-attribute-expression-selector = + { -term.attr -> + *[key] Value + } + +# ERROR Message attributes may not be used as selectors. + diff --git a/fluent-syntax/tests/fixtures/formatted/messages.ftl b/fluent-syntax/tests/fixtures/formatted/messages.ftl new file mode 100644 index 00000000..72403bd3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/messages.ftl @@ -0,0 +1,36 @@ +key01 = Value +key02 = Value + .attr = Attribute +key02 = Value + .attr1 = Attribute 1 + .attr2 = Attribute 2 +key03 = + .attr = Attribute +key04 = + .attr1 = Attribute 1 + .attr2 = Attribute 2 +# < whitespace > +key05 = + .attr1 = Attribute 1 +no-whitespace = Value + .attr1 = Attribute 1 +extra-whitespace = Value + .attr1 = Attribute 1 +key06 = { "" } + +# JUNK Missing value + +# JUNK Missing = + +KEY09 = Value 09 +key-10 = Value 10 +key_11 = Value 11 +key-12- = Value 12 +key_13_ = Value 13 + +# JUNK Invalid id + +# JUNK Invalid id + +# JUNK Invalid id + diff --git a/fluent-syntax/tests/fixtures/formatted/mixed_entries.ftl b/fluent-syntax/tests/fixtures/formatted/mixed_entries.ftl new file mode 100644 index 00000000..f8e54c29 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/mixed_entries.ftl @@ -0,0 +1,19 @@ +# License Comment + + +### Resource Comment + +-brand-name = Aurora + +## Group Comment + +key01 = + .attr = Attribute +# Message Comment +key02 = Value + +# Standalone Comment + +# There are 5 spaces on the line between key03 and key04. +key03 = Value 03 +key04 = Value 04 diff --git a/fluent-syntax/tests/fixtures/formatted/numbers.ftl b/fluent-syntax/tests/fixtures/formatted/numbers.ftl new file mode 100644 index 00000000..521f5a23 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/numbers.ftl @@ -0,0 +1,25 @@ +int-zero = { 0 } +int-positive = { 1 } +int-negative = { -1 } +int-negative-zero = { -0 } +int-positive-padded = { 01 } +int-negative-padded = { -01 } +int-zero-padded = { 00 } +int-negative-zero-padded = { -00 } +float-zero = { 0.0 } +float-positive = { 0.01 } +float-positive-one = { 1.03 } +float-positive-without-fraction = { 1.000 } +float-negative = { -0.01 } +float-negative-one = { -1.03 } +float-negative-zero = { -0.0 } +float-negative-without-fraction = { -1.000 } +float-positive-padded-left = { 01.03 } +float-positive-padded-right = { 1.0300 } +float-positive-padded-both = { 01.0300 } +float-negative-padded-left = { -01.03 } +float-negative-padded-right = { -1.0300 } +float-negative-padded-both = { -01.0300 } + +## ERRORS + diff --git a/fluent-syntax/tests/fixtures/formatted/obsolete.ftl b/fluent-syntax/tests/fixtures/formatted/obsolete.ftl new file mode 100644 index 00000000..314a35ef --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/obsolete.ftl @@ -0,0 +1,9 @@ +### The syntax in this file has been discontinued. It is no longer part of the +### Fluent specification and should not be implemented nor used. We're keeping +### these fixtures around to protect against accidental syntax reuse. + + +## Variant lists. + +## Variant expressions. + diff --git a/fluent-syntax/tests/fixtures/formatted/placeables.ftl b/fluent-syntax/tests/fixtures/formatted/placeables.ftl new file mode 100644 index 00000000..6a5167be --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/placeables.ftl @@ -0,0 +1,12 @@ +nested-placeable = {{ {1} }} +padded-placeable = { 1 } +sparse-placeable = {{ 1 }} + +# ERROR Unmatched opening brace + +# ERROR Unmatched opening brace + +# ERROR Unmatched closing brace + +# ERROR Unmatched closing brace + diff --git a/fluent-syntax/tests/fixtures/formatted/reference_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/reference_expressions.ftl new file mode 100644 index 00000000..5132517f --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/reference_expressions.ftl @@ -0,0 +1,23 @@ +## Reference expressions in placeables. + +message-reference-placeable = { msg } +term-reference-placeable = { -term } +variable-reference-placeable = { $var } +# Function references are invalid outside of call expressions. +# This parses as a valid MessageReference. +function-reference-placeable = { FUN } + +## Reference expressions in selectors. + +variable-reference-selector = + { $var -> + *[key] Value + } + +# ERROR Message values may not be used as selectors. + +# ERROR Term values may not be used as selectors. + +# ERROR Function references are invalid outside of call expressions, and this +# parses as a MessageReference which isn't a valid selector. + diff --git a/fluent-syntax/tests/fixtures/formatted/select_expressions.ftl b/fluent-syntax/tests/fixtures/formatted/select_expressions.ftl new file mode 100644 index 00000000..8e1021b4 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/select_expressions.ftl @@ -0,0 +1,38 @@ +new-messages = + { BUILTIN() -> + [0] Zero + *[other] { "" }Other + } +valid-selector-term-attribute = + { -term.case -> + *[key] value + } + +# ERROR Term values are not valid selectors + +# ERROR CallExpressions on Terms are similar to TermReferences + +# ERROR Nested expressions are not valid selectors + +# ERROR Select expressions are not valid selectors + +empty-variant = + { $sel -> + *[key] { "" } + } +reduced-whitespace = + { FOO() -> + *[key] { "" } + } +nested-select = + { $sel -> + *[one] + { $sel -> + *[two] Value + } + } + +# ERROR Missing selector + +# ERROR Missing line end after variant list + diff --git a/fluent-syntax/tests/fixtures/formatted/select_indent.ftl b/fluent-syntax/tests/fixtures/formatted/select_indent.ftl new file mode 100644 index 00000000..9aa2a995 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/select_indent.ftl @@ -0,0 +1,65 @@ +select-1tbs-inline = + { $selector -> + *[key] Value + } +select-1tbs-newline = + { $selector -> + *[key] Value + } +select-1tbs-indent = + { $selector -> + *[key] Value + } +select-allman-inline = + { $selector -> + *[key] Value + [other] Other + } +select-allman-newline = + { $selector -> + *[key] Value + } +select-allman-indent = + { $selector -> + *[key] Value + } +select-gnu-inline = + { $selector -> + *[key] Value + } +select-gnu-newline = + { $selector -> + *[key] Value + } +select-gnu-indent = + { $selector -> + *[key] Value + } +select-no-indent = + { $selector -> + *[key] Value + [other] Other + } +select-no-indent-multiline = + { $selector -> + *[key] + Value + Continued + [other] + Other + Multiline + } + +# ERROR (Multiline text must be indented) + +select-flat = + { $selector -> + *[key] Value + [other] Other + } +# Each line ends with 5 spaces. +select-flat-with-trailing-spaces = + { $selector -> + *[key] Value + [other] Other + } diff --git a/fluent-syntax/tests/fixtures/formatted/sparse_entries.ftl b/fluent-syntax/tests/fixtures/formatted/sparse_entries.ftl new file mode 100644 index 00000000..dfa85945 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/sparse_entries.ftl @@ -0,0 +1,17 @@ +key01 = Value +key02 = + .attr = Attribute +key03 = + Value + Continued + + + Over multiple + Lines + .attr = Attribute +key05 = Value +key06 = + { 1 -> + [one] One + *[two] Two + } diff --git a/fluent-syntax/tests/fixtures/formatted/special_chars.ftl b/fluent-syntax/tests/fixtures/formatted/special_chars.ftl new file mode 100644 index 00000000..ab32df9d --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/special_chars.ftl @@ -0,0 +1,8 @@ +## OK + +bracket-inline = [Value] +dot-inline = .Value +star-inline = *Value + +## ERRORS + diff --git a/fluent-syntax/tests/fixtures/formatted/tab.ftl b/fluent-syntax/tests/fixtures/formatted/tab.ftl new file mode 100644 index 00000000..bbbd7332 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/tab.ftl @@ -0,0 +1,14 @@ +# OK (tab after = is part of the value) +key01 = Value 01 + +# Error (tab before =) + +# Error (tab is not a valid indent) + +# Partial Error (tab is not a valid indent) +key04 = This line is indented by 4 spaces, +# OK (value is a single tab) +key05 = +# OK (attribute value is two tabs) +key06 = + .attr = diff --git a/fluent-syntax/tests/fixtures/formatted/term_parameters.ftl b/fluent-syntax/tests/fixtures/formatted/term_parameters.ftl new file mode 100644 index 00000000..8ee2ed74 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/term_parameters.ftl @@ -0,0 +1,8 @@ +-term = + { $arg -> + *[key] Value + } +key01 = { -term } +key02 = { -term() } +key03 = { -term(arg: 1) } +key04 = { -term("positional", narg1: 1, narg2: 2) } diff --git a/fluent-syntax/tests/fixtures/formatted/terms.ftl b/fluent-syntax/tests/fixtures/formatted/terms.ftl new file mode 100644 index 00000000..37d68113 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/terms.ftl @@ -0,0 +1,20 @@ +-term01 = Value + .attr = Attribute +-term02 = { "" } + +# JUNK Missing value + +# JUNK Missing value +# < whitespace > + +# JUNK Missing value + +# JUNK Missing value +# < whitespace > + +# JUNK Missing = + +-term08 = Value + .attr = Attribute +-term09 = Value + .attr = Attribute diff --git a/fluent-syntax/tests/fixtures/formatted/variables.ftl b/fluent-syntax/tests/fixtures/formatted/variables.ftl new file mode 100644 index 00000000..25d58359 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/variables.ftl @@ -0,0 +1,14 @@ +key01 = { $var } +key02 = { $var } +key03 = { $var } +key04 = { $var } + +## Errors + + +# ERROR Missing variable identifier + +# ERROR Double $$ + +# ERROR Invalid first char of the identifier + diff --git a/fluent-syntax/tests/fixtures/formatted/variant_keys.ftl b/fluent-syntax/tests/fixtures/formatted/variant_keys.ftl new file mode 100644 index 00000000..d0313bcc --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/variant_keys.ftl @@ -0,0 +1,23 @@ +simple-identifier = + { $sel -> + *[key] value + } +identifier-surrounded-by-whitespace = + { $sel -> + *[key] value + } +int-number = + { $sel -> + *[1] value + } +float-number = + { $sel -> + *[3.14] value + } + +# ERROR + +# ERROR + +# ERROR + diff --git a/fluent-syntax/tests/fixtures/formatted/whitespace_in_value.ftl b/fluent-syntax/tests/fixtures/formatted/whitespace_in_value.ftl new file mode 100644 index 00000000..0cbcd371 --- /dev/null +++ b/fluent-syntax/tests/fixtures/formatted/whitespace_in_value.ftl @@ -0,0 +1,10 @@ +# Caution, lines 6 and 7 contain white-space-only lines +key = + first line + + + + + + + last line diff --git a/fluent-syntax/tests/fixtures/formatted/zero_length.ftl b/fluent-syntax/tests/fixtures/formatted/zero_length.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/serializer_fixtures.rs b/fluent-syntax/tests/serializer_fixtures.rs index ab5b7225..dc1e7075 100644 --- a/fluent-syntax/tests/serializer_fixtures.rs +++ b/fluent-syntax/tests/serializer_fixtures.rs @@ -1,4 +1,3 @@ -use fluent_syntax::ast::{Entry, Resource}; use glob::glob; use std::ffi::OsStr; use std::fs; @@ -22,19 +21,6 @@ fn is_ignored(path: &Path) -> bool { .unwrap_or_default() } -fn clone_without_junk<'a>(original: &Resource<&'a str>) -> Resource<&'a str> { - Resource { - body: original - .body - .iter() - .filter(|entry| !matches!(entry, Entry::Junk { .. })) - .cloned() - .collect(), - #[cfg(feature = "spans")] - span: original.span, - } -} - #[test] fn roundtrip_normalized_fixtures() { for entry in glob("./tests/fixtures/normalized/*.ftl").expect("Failed to read glob pattern") { @@ -59,15 +45,26 @@ fn roundtrip_unnormalized_fixtures() { let content = fs::read_to_string(&path).expect("Failed to read file"); let parsed = parse(content.as_str()).unwrap_or_else(|(res, _)| res); - let parsed_without_junk = clone_without_junk(&parsed); + let reserialized = serialize_with_options(&parsed, Options { with_junk: true }); let reserialized_without_junk = serialize_with_options(&parsed, Options { with_junk: false }); + + let formatted_path = path.parent().unwrap().join("formatted"); + + let formatted_content = fs::read_to_string(formatted_path.join("junked").join(path.file_name().unwrap())).unwrap(); + assert_eq!(formatted_content, reserialized); + let formatted = parse(formatted_content.as_str()).unwrap_or_else(|(res, _)| res); + + let formatted_content_without_junk = fs::read_to_string(formatted_path.join(path.file_name().unwrap())).unwrap(); + assert_eq!(formatted_content_without_junk, reserialized_without_junk); + let formatted_without_junk = + parse(formatted_content_without_junk.as_str()).unwrap_or_else(|(res, _)| res); + let reparsed = parse(reserialized.as_str()).unwrap_or_else(|(res, _)| res); - let reparsed_without_junk = - parse(reserialized_without_junk.as_str()).unwrap_or_else(|(res, _)| res); + let reparsed_without_junk = parse(reserialized_without_junk.as_str()).unwrap_or_else(|(res, _)| res); - assert_eq!(reparsed_without_junk, parsed_without_junk); - assert_eq!(reparsed, parsed); + assert_eq!(formatted_without_junk, reparsed_without_junk); + assert_eq!(formatted, reparsed); } } From a31f76a4d851c8d4b03a5065b84fa655a62e149a Mon Sep 17 00:00:00 2001 From: Ertanic Date: Sun, 9 Mar 2025 07:38:48 +0300 Subject: [PATCH 04/10] Span struct has been replaced by std::ops::Range --- fluent-syntax/src/ast/helper.rs | 6 +- fluent-syntax/src/ast/mod.rs | 84 ++++++++------------------ fluent-syntax/src/parser/comment.rs | 2 +- fluent-syntax/src/parser/core.rs | 16 ++--- fluent-syntax/src/parser/expression.rs | 29 +++++---- fluent-syntax/src/parser/pattern.rs | 6 +- fluent-syntax/src/parser/runtime.rs | 6 +- fluent-syntax/src/serializer.rs | 27 +++++---- 8 files changed, 73 insertions(+), 103 deletions(-) diff --git a/fluent-syntax/src/ast/helper.rs b/fluent-syntax/src/ast/helper.rs index fb5b86d6..5a51e13d 100644 --- a/fluent-syntax/src/ast/helper.rs +++ b/fluent-syntax/src/ast/helper.rs @@ -1,6 +1,6 @@ use super::Comment; #[cfg(feature = "spans")] -use super::Span; +use super::Range; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -14,12 +14,12 @@ pub enum CommentDef { Single { content: S, #[cfg(feature = "spans")] - span: Span, + span: Range, }, Multi { content: Vec, #[cfg(feature = "spans")] - span: Span, + span: Range, }, } diff --git a/fluent-syntax/src/ast/mod.rs b/fluent-syntax/src/ast/mod.rs index ecf7c967..eee56efc 100644 --- a/fluent-syntax/src/ast/mod.rs +++ b/fluent-syntax/src/ast/mod.rs @@ -118,7 +118,7 @@ use std::ops::Range; pub struct Resource { pub body: Vec>, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// A top-level node representing an entry of a [`Resource`]. @@ -209,7 +209,7 @@ pub enum Entry { Junk { content: S, #[cfg(feature = "spans")] - span: Span, + span: Range, }, } @@ -269,7 +269,7 @@ pub struct Message { pub attributes: Vec>, pub comment: Option>, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// A Fluent [`Term`]. @@ -325,7 +325,7 @@ pub struct Term { pub attributes: Vec>, pub comment: Option>, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// Pattern contains a value of a [`Message`], [`Term`] or an [`Attribute`]. @@ -404,7 +404,7 @@ pub struct Term { pub struct Pattern { pub elements: Vec>, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// `PatternElement` is an element of a [`Pattern`]. @@ -485,12 +485,12 @@ pub enum PatternElement { TextElement { value: S, #[cfg(feature = "spans")] - span: Span, + span: Range, }, Placeable { expression: Expression, #[cfg(feature = "spans")] - span: Span, + span: Range, }, } @@ -563,7 +563,7 @@ pub struct Attribute { pub id: Identifier, pub value: Pattern, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// Identifier is part of nodes such as [`Message`], [`Term`] and [`Attribute`]. @@ -613,7 +613,7 @@ pub struct Attribute { pub struct Identifier { pub name: S, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// Variant is a single branch of a value in a [`Select`](Expression::Select) expression. @@ -701,7 +701,7 @@ pub struct Variant { pub value: Pattern, pub default: bool, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// A key of a [`Variant`]. @@ -832,7 +832,7 @@ pub enum VariantKey { pub struct Comment { pub content: Vec, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// List of arguments for a [`FunctionReference`](InlineExpression::FunctionReference) or a @@ -916,7 +916,7 @@ pub struct CallArguments { pub positional: Vec>, pub named: Vec>, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// A key-value pair used in [`CallArguments`]. @@ -987,7 +987,7 @@ pub struct NamedArgument { pub name: Identifier, pub value: InlineExpression, #[cfg(feature = "spans")] - pub span: Span, + pub span: Range, } /// A subset of expressions which can be used as [`Placeable`](PatternElement::Placeable), @@ -1091,7 +1091,7 @@ pub enum InlineExpression { StringLiteral { value: S, #[cfg(feature = "spans")] - span: Span, + span: Range, }, /// A number literal. /// @@ -1141,7 +1141,7 @@ pub enum InlineExpression { NumberLiteral { value: S, #[cfg(feature = "spans")] - span: Span, + span: Range, }, /// A function reference. /// @@ -1195,7 +1195,7 @@ pub enum InlineExpression { id: Identifier, arguments: CallArguments, #[cfg(feature = "spans")] - span: Span, + span: Range, }, /// A reference to another message. /// @@ -1249,7 +1249,7 @@ pub enum InlineExpression { id: Identifier, attribute: Option>, #[cfg(feature = "spans")] - span: Span, + span: Range, }, /// A reference to a term. /// @@ -1305,7 +1305,7 @@ pub enum InlineExpression { attribute: Option>, arguments: Option>, #[cfg(feature = "spans")] - span: Span, + span: Range, }, /// A reference to a variable. /// @@ -1357,7 +1357,7 @@ pub enum InlineExpression { VariableReference { id: Identifier, #[cfg(feature = "spans")] - span: Span, + span: Range, }, /// A placeable which may contain another expression. /// @@ -1413,13 +1413,13 @@ pub enum InlineExpression { Placeable { expression: Box>, #[cfg(feature = "spans")] - span: Span, + span: Range, }, } #[cfg(feature = "spans")] impl InlineExpression { - pub fn get_span(&self) -> Span { + pub fn get_span(&self) -> Range { match self { InlineExpression::StringLiteral { span, .. } | InlineExpression::TermReference { span, .. } @@ -1427,7 +1427,7 @@ impl InlineExpression { | InlineExpression::Placeable { span, .. } | InlineExpression::NumberLiteral { span, .. } | InlineExpression::FunctionReference { span, .. } - | InlineExpression::MessageReference { span, .. } => *span, + | InlineExpression::MessageReference { span, .. } => span.clone(), // Why doesn't std::ops::Range implement the Copy trait...? } } } @@ -1521,7 +1521,7 @@ pub enum Expression { selector: InlineExpression, variants: Vec>, #[cfg(feature = "spans")] - span: Span, + span: Range, }, /// An inline expression such as `${ username }`: @@ -1529,41 +1529,5 @@ pub enum Expression { /// ```ftl /// hello-user = Hello ${ username } /// ``` - Inline(InlineExpression, #[cfg(feature = "spans")] Span), -} - -/// A span of a node. Allows you to get the index of the start and end character of a node. -/// -/// # Example -/// -/// ``` -/// #![cfg(feature = "spans")] -/// -/// use fluent_syntax::parser; -/// use fluent_syntax::ast::*; -/// -/// let ftl = "hello-world = Hello, World!"; -/// -/// let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); -/// let Entry::Message(Message { ref id, .. }) = resource.body[0] else { unreachable!() }; -/// -/// assert_eq!(resource.span, Span { start: 0, end: 27 }); -/// assert_eq!(id.span, Span { start: 0, end: 11 }); // the span of hello-world identifier -/// ``` -#[cfg(feature = "spans")] -#[derive(Debug, Clone, Copy, Default, PartialEq)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct Span { - pub start: usize, - pub end: usize, -} - -#[cfg(feature = "spans")] -impl Span { - pub fn new(range: Range) -> Self { - Self { - start: range.start, - end: range.end, - } - } + Inline(InlineExpression, #[cfg(feature = "spans")] Range), } \ No newline at end of file diff --git a/fluent-syntax/src/parser/comment.rs b/fluent-syntax/src/parser/comment.rs index 97c3ff6e..a4d22ac9 100644 --- a/fluent-syntax/src/parser/comment.rs +++ b/fluent-syntax/src/parser/comment.rs @@ -54,7 +54,7 @@ where ast::Comment { content, #[cfg(feature = "spans")] - span: ast::Span::new(start_pos..self.ptr), + span: start_pos..self.ptr, }, level, )) diff --git a/fluent-syntax/src/parser/core.rs b/fluent-syntax/src/parser/core.rs index 70ebe9dc..ceb0bc0b 100644 --- a/fluent-syntax/src/parser/core.rs +++ b/fluent-syntax/src/parser/core.rs @@ -70,7 +70,7 @@ where body.push(ast::Entry::Junk { content, #[cfg(feature = "spans")] - span: ast::Span::new(entry_start..self.ptr), + span: entry_start..self.ptr, }); } } @@ -81,9 +81,9 @@ where body.push(ast::Entry::Comment(last_comment)); } if errors.is_empty() { - Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }) + Ok(ast::Resource { body, #[cfg(feature = "spans")] span: 0..self.length }) } else { - Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }, errors)) + Err((ast::Resource { body, #[cfg(feature = "spans")] span: 0..self.length }, errors)) } } @@ -128,7 +128,7 @@ where attributes, comment: None, #[cfg(feature = "spans")] - span: ast::Span::new(entry_start..self.ptr), + span: entry_start..self.ptr, }) } @@ -152,7 +152,7 @@ where attributes, comment: None, #[cfg(feature = "spans")] - span: ast::Span::new(entry_start..self.ptr), + span: entry_start..self.ptr, }) } else { error!( @@ -196,7 +196,7 @@ where id, value: pattern, #[cfg(feature = "spans")] - span: ast::Span::new(self.ptr - 1..self.ptr), + span: self.ptr - 1..self.ptr, }), None => error!(ErrorKind::MissingValue, self.ptr), } @@ -218,7 +218,7 @@ where ast::Identifier { name, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, } } @@ -295,7 +295,7 @@ where value, default, #[cfg(feature = "spans")] - span: ast::Span::new((if default { start - 1 } else { start })..self.ptr), + span: (if default { start - 1 } else { start })..self.ptr, }); self.skip_blank(); } else { diff --git a/fluent-syntax/src/parser/expression.rs b/fluent-syntax/src/parser/expression.rs index 014228e0..245a845c 100644 --- a/fluent-syntax/src/parser/expression.rs +++ b/fluent-syntax/src/parser/expression.rs @@ -2,6 +2,9 @@ use super::errors::{ErrorKind, ParserError}; use super::{core::Parser, core::Result, slice::Slice}; use crate::ast; +#[cfg(feature = "spans")] +use std::ops::Range; + impl<'s, S> Parser where S: Slice<'s>, @@ -23,7 +26,7 @@ where return Ok(ast::Expression::Inline( exp, #[cfg(feature = "spans")] - ast::Span::new(start_span..self.ptr), + Range { start: start_span, end: self.ptr }, )); } @@ -68,7 +71,7 @@ where selector: exp, variants, #[cfg(feature = "spans")] - span: ast::Span::new(start_span..self.ptr), + span: start_span..self.ptr, }) } @@ -112,7 +115,7 @@ where Ok(ast::InlineExpression::StringLiteral { value: slice, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, }) } Some(b) if b.is_ascii_digit() => { @@ -120,7 +123,7 @@ where Ok(ast::InlineExpression::NumberLiteral { value: num, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, }) } Some(b'-') if !only_literal => { @@ -135,7 +138,7 @@ where attribute, arguments, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, }) } else { self.ptr -= 1; @@ -143,7 +146,7 @@ where Ok(ast::InlineExpression::NumberLiteral { value: num, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, }) } } @@ -153,7 +156,7 @@ where Ok(ast::InlineExpression::VariableReference { id, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, }) } Some(b) if b.is_ascii_alphabetic() => { @@ -169,7 +172,7 @@ where id, arguments, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, }) } else { let attribute = self.get_attribute_accessor()?; @@ -177,7 +180,7 @@ where id, attribute, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, }) } } @@ -187,7 +190,7 @@ where Ok(ast::InlineExpression::Placeable { expression: Box::new(exp), #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, }) } _ if only_literal => error!(ErrorKind::ExpectedLiteral, self.ptr), @@ -239,11 +242,11 @@ where name: ast::Identifier { name: id.name.clone(), #[cfg(feature = "spans")] - span: id.span, + span: id.span.clone(), }, value: val, #[cfg(feature = "spans")] - span: ast::Span::new(id.span.start..self.ptr), + span: id.span.start..self.ptr, }); } else { if !argument_names.is_empty() { @@ -269,7 +272,7 @@ where positional, named, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: start..self.ptr, })) } } diff --git a/fluent-syntax/src/parser/pattern.rs b/fluent-syntax/src/parser/pattern.rs index d0a6330d..cb0be596 100644 --- a/fluent-syntax/src/parser/pattern.rs +++ b/fluent-syntax/src/parser/pattern.rs @@ -147,7 +147,7 @@ where PatternElementPlaceholders::Placeable(expression, range) => { ast::PatternElement::Placeable { expression, - span: ast::Span::new(range), + span: range, } } #[cfg(not(feature = "spans"))] @@ -170,7 +170,7 @@ where ast::PatternElement::TextElement { value, #[cfg(feature = "spans")] - span: ast::Span::new(start..end), + span: start..end, } } }) @@ -178,7 +178,7 @@ where return Ok(Some(ast::Pattern { elements, #[cfg(feature = "spans")] - span: ast::Span::new(start_pos..self.ptr), + span: start_pos..self.ptr, })); } diff --git a/fluent-syntax/src/parser/runtime.rs b/fluent-syntax/src/parser/runtime.rs index d350e046..c2b4d738 100644 --- a/fluent-syntax/src/parser/runtime.rs +++ b/fluent-syntax/src/parser/runtime.rs @@ -37,7 +37,7 @@ where body.push(ast::Entry::Junk { content, #[cfg(feature = "spans")] - span: ast::Span::new(entry_start..self.ptr), + span: entry_start..self.ptr, }); } } @@ -45,9 +45,9 @@ where } if errors.is_empty() { - Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }) + Ok(ast::Resource { body, #[cfg(feature = "spans")] span: 0..self.length }) } else { - Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }, errors)) + Err((ast::Resource { body, #[cfg(feature = "spans")] span: 0..self.length }, errors)) } } diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 0231622d..d10f25ee 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -24,6 +24,9 @@ use crate::{ast::*, parser::matches_fluent_ws, parser::Slice}; use std::fmt::Write; +#[cfg(feature = "spans")] +use std::ops::Range; + /// Serializes an abstract syntax tree representing a Fluent Translation List into a /// String. /// @@ -487,21 +490,21 @@ mod test { id: Identifier { name: $name, #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }, value: Some(Pattern { elements: vec![PatternElement::TextElement { value: $value, #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }], #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }), attributes: vec![], comment: None, #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }) }; } @@ -586,29 +589,29 @@ mod test { id: Identifier { name: "num", #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }, #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }, #[cfg(feature = "spans")] - Span::default(), + Range::default(), ), #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }, PatternElement::TextElement { value: " bar", #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }, ], #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }, default: false, #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }; ast.body[0].as_message().as_pattern().elements[0] .as_expression() @@ -669,7 +672,7 @@ mod test { ast.body[0].as_message().comment.replace(Comment { content: vec!["great message!"], #[cfg(feature = "spans")] - span: Span::default(), + span: Range::default(), }); assert_eq!("# great message!\nfoo = bar\n", serialize(&ast)); } From e4612e88c48b5599a348da763c1007aefdc77942 Mon Sep 17 00:00:00 2001 From: Ertanic Date: Tue, 11 Mar 2025 18:53:04 +0300 Subject: [PATCH 05/10] Revert "Span struct has been replaced by std::ops::Range" This reverts commit a31f76a4d851c8d4b03a5065b84fa655a62e149a. --- fluent-syntax/src/ast/helper.rs | 6 +- fluent-syntax/src/ast/mod.rs | 84 ++++++++++++++++++-------- fluent-syntax/src/parser/comment.rs | 2 +- fluent-syntax/src/parser/core.rs | 16 ++--- fluent-syntax/src/parser/expression.rs | 29 ++++----- fluent-syntax/src/parser/pattern.rs | 6 +- fluent-syntax/src/parser/runtime.rs | 6 +- fluent-syntax/src/serializer.rs | 27 ++++----- 8 files changed, 103 insertions(+), 73 deletions(-) diff --git a/fluent-syntax/src/ast/helper.rs b/fluent-syntax/src/ast/helper.rs index 5a51e13d..fb5b86d6 100644 --- a/fluent-syntax/src/ast/helper.rs +++ b/fluent-syntax/src/ast/helper.rs @@ -1,6 +1,6 @@ use super::Comment; #[cfg(feature = "spans")] -use super::Range; +use super::Span; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -14,12 +14,12 @@ pub enum CommentDef { Single { content: S, #[cfg(feature = "spans")] - span: Range, + span: Span, }, Multi { content: Vec, #[cfg(feature = "spans")] - span: Range, + span: Span, }, } diff --git a/fluent-syntax/src/ast/mod.rs b/fluent-syntax/src/ast/mod.rs index eee56efc..ecf7c967 100644 --- a/fluent-syntax/src/ast/mod.rs +++ b/fluent-syntax/src/ast/mod.rs @@ -118,7 +118,7 @@ use std::ops::Range; pub struct Resource { pub body: Vec>, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// A top-level node representing an entry of a [`Resource`]. @@ -209,7 +209,7 @@ pub enum Entry { Junk { content: S, #[cfg(feature = "spans")] - span: Range, + span: Span, }, } @@ -269,7 +269,7 @@ pub struct Message { pub attributes: Vec>, pub comment: Option>, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// A Fluent [`Term`]. @@ -325,7 +325,7 @@ pub struct Term { pub attributes: Vec>, pub comment: Option>, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// Pattern contains a value of a [`Message`], [`Term`] or an [`Attribute`]. @@ -404,7 +404,7 @@ pub struct Term { pub struct Pattern { pub elements: Vec>, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// `PatternElement` is an element of a [`Pattern`]. @@ -485,12 +485,12 @@ pub enum PatternElement { TextElement { value: S, #[cfg(feature = "spans")] - span: Range, + span: Span, }, Placeable { expression: Expression, #[cfg(feature = "spans")] - span: Range, + span: Span, }, } @@ -563,7 +563,7 @@ pub struct Attribute { pub id: Identifier, pub value: Pattern, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// Identifier is part of nodes such as [`Message`], [`Term`] and [`Attribute`]. @@ -613,7 +613,7 @@ pub struct Attribute { pub struct Identifier { pub name: S, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// Variant is a single branch of a value in a [`Select`](Expression::Select) expression. @@ -701,7 +701,7 @@ pub struct Variant { pub value: Pattern, pub default: bool, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// A key of a [`Variant`]. @@ -832,7 +832,7 @@ pub enum VariantKey { pub struct Comment { pub content: Vec, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// List of arguments for a [`FunctionReference`](InlineExpression::FunctionReference) or a @@ -916,7 +916,7 @@ pub struct CallArguments { pub positional: Vec>, pub named: Vec>, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// A key-value pair used in [`CallArguments`]. @@ -987,7 +987,7 @@ pub struct NamedArgument { pub name: Identifier, pub value: InlineExpression, #[cfg(feature = "spans")] - pub span: Range, + pub span: Span, } /// A subset of expressions which can be used as [`Placeable`](PatternElement::Placeable), @@ -1091,7 +1091,7 @@ pub enum InlineExpression { StringLiteral { value: S, #[cfg(feature = "spans")] - span: Range, + span: Span, }, /// A number literal. /// @@ -1141,7 +1141,7 @@ pub enum InlineExpression { NumberLiteral { value: S, #[cfg(feature = "spans")] - span: Range, + span: Span, }, /// A function reference. /// @@ -1195,7 +1195,7 @@ pub enum InlineExpression { id: Identifier, arguments: CallArguments, #[cfg(feature = "spans")] - span: Range, + span: Span, }, /// A reference to another message. /// @@ -1249,7 +1249,7 @@ pub enum InlineExpression { id: Identifier, attribute: Option>, #[cfg(feature = "spans")] - span: Range, + span: Span, }, /// A reference to a term. /// @@ -1305,7 +1305,7 @@ pub enum InlineExpression { attribute: Option>, arguments: Option>, #[cfg(feature = "spans")] - span: Range, + span: Span, }, /// A reference to a variable. /// @@ -1357,7 +1357,7 @@ pub enum InlineExpression { VariableReference { id: Identifier, #[cfg(feature = "spans")] - span: Range, + span: Span, }, /// A placeable which may contain another expression. /// @@ -1413,13 +1413,13 @@ pub enum InlineExpression { Placeable { expression: Box>, #[cfg(feature = "spans")] - span: Range, + span: Span, }, } #[cfg(feature = "spans")] impl InlineExpression { - pub fn get_span(&self) -> Range { + pub fn get_span(&self) -> Span { match self { InlineExpression::StringLiteral { span, .. } | InlineExpression::TermReference { span, .. } @@ -1427,7 +1427,7 @@ impl InlineExpression { | InlineExpression::Placeable { span, .. } | InlineExpression::NumberLiteral { span, .. } | InlineExpression::FunctionReference { span, .. } - | InlineExpression::MessageReference { span, .. } => span.clone(), // Why doesn't std::ops::Range implement the Copy trait...? + | InlineExpression::MessageReference { span, .. } => *span, } } } @@ -1521,7 +1521,7 @@ pub enum Expression { selector: InlineExpression, variants: Vec>, #[cfg(feature = "spans")] - span: Range, + span: Span, }, /// An inline expression such as `${ username }`: @@ -1529,5 +1529,41 @@ pub enum Expression { /// ```ftl /// hello-user = Hello ${ username } /// ``` - Inline(InlineExpression, #[cfg(feature = "spans")] Range), + Inline(InlineExpression, #[cfg(feature = "spans")] Span), +} + +/// A span of a node. Allows you to get the index of the start and end character of a node. +/// +/// # Example +/// +/// ``` +/// #![cfg(feature = "spans")] +/// +/// use fluent_syntax::parser; +/// use fluent_syntax::ast::*; +/// +/// let ftl = "hello-world = Hello, World!"; +/// +/// let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); +/// let Entry::Message(Message { ref id, .. }) = resource.body[0] else { unreachable!() }; +/// +/// assert_eq!(resource.span, Span { start: 0, end: 27 }); +/// assert_eq!(id.span, Span { start: 0, end: 11 }); // the span of hello-world identifier +/// ``` +#[cfg(feature = "spans")] +#[derive(Debug, Clone, Copy, Default, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Span { + pub start: usize, + pub end: usize, +} + +#[cfg(feature = "spans")] +impl Span { + pub fn new(range: Range) -> Self { + Self { + start: range.start, + end: range.end, + } + } } \ No newline at end of file diff --git a/fluent-syntax/src/parser/comment.rs b/fluent-syntax/src/parser/comment.rs index a4d22ac9..97c3ff6e 100644 --- a/fluent-syntax/src/parser/comment.rs +++ b/fluent-syntax/src/parser/comment.rs @@ -54,7 +54,7 @@ where ast::Comment { content, #[cfg(feature = "spans")] - span: start_pos..self.ptr, + span: ast::Span::new(start_pos..self.ptr), }, level, )) diff --git a/fluent-syntax/src/parser/core.rs b/fluent-syntax/src/parser/core.rs index ceb0bc0b..70ebe9dc 100644 --- a/fluent-syntax/src/parser/core.rs +++ b/fluent-syntax/src/parser/core.rs @@ -70,7 +70,7 @@ where body.push(ast::Entry::Junk { content, #[cfg(feature = "spans")] - span: entry_start..self.ptr, + span: ast::Span::new(entry_start..self.ptr), }); } } @@ -81,9 +81,9 @@ where body.push(ast::Entry::Comment(last_comment)); } if errors.is_empty() { - Ok(ast::Resource { body, #[cfg(feature = "spans")] span: 0..self.length }) + Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }) } else { - Err((ast::Resource { body, #[cfg(feature = "spans")] span: 0..self.length }, errors)) + Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }, errors)) } } @@ -128,7 +128,7 @@ where attributes, comment: None, #[cfg(feature = "spans")] - span: entry_start..self.ptr, + span: ast::Span::new(entry_start..self.ptr), }) } @@ -152,7 +152,7 @@ where attributes, comment: None, #[cfg(feature = "spans")] - span: entry_start..self.ptr, + span: ast::Span::new(entry_start..self.ptr), }) } else { error!( @@ -196,7 +196,7 @@ where id, value: pattern, #[cfg(feature = "spans")] - span: self.ptr - 1..self.ptr, + span: ast::Span::new(self.ptr - 1..self.ptr), }), None => error!(ErrorKind::MissingValue, self.ptr), } @@ -218,7 +218,7 @@ where ast::Identifier { name, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), } } @@ -295,7 +295,7 @@ where value, default, #[cfg(feature = "spans")] - span: (if default { start - 1 } else { start })..self.ptr, + span: ast::Span::new((if default { start - 1 } else { start })..self.ptr), }); self.skip_blank(); } else { diff --git a/fluent-syntax/src/parser/expression.rs b/fluent-syntax/src/parser/expression.rs index 245a845c..014228e0 100644 --- a/fluent-syntax/src/parser/expression.rs +++ b/fluent-syntax/src/parser/expression.rs @@ -2,9 +2,6 @@ use super::errors::{ErrorKind, ParserError}; use super::{core::Parser, core::Result, slice::Slice}; use crate::ast; -#[cfg(feature = "spans")] -use std::ops::Range; - impl<'s, S> Parser where S: Slice<'s>, @@ -26,7 +23,7 @@ where return Ok(ast::Expression::Inline( exp, #[cfg(feature = "spans")] - Range { start: start_span, end: self.ptr }, + ast::Span::new(start_span..self.ptr), )); } @@ -71,7 +68,7 @@ where selector: exp, variants, #[cfg(feature = "spans")] - span: start_span..self.ptr, + span: ast::Span::new(start_span..self.ptr), }) } @@ -115,7 +112,7 @@ where Ok(ast::InlineExpression::StringLiteral { value: slice, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), }) } Some(b) if b.is_ascii_digit() => { @@ -123,7 +120,7 @@ where Ok(ast::InlineExpression::NumberLiteral { value: num, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), }) } Some(b'-') if !only_literal => { @@ -138,7 +135,7 @@ where attribute, arguments, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), }) } else { self.ptr -= 1; @@ -146,7 +143,7 @@ where Ok(ast::InlineExpression::NumberLiteral { value: num, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), }) } } @@ -156,7 +153,7 @@ where Ok(ast::InlineExpression::VariableReference { id, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), }) } Some(b) if b.is_ascii_alphabetic() => { @@ -172,7 +169,7 @@ where id, arguments, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), }) } else { let attribute = self.get_attribute_accessor()?; @@ -180,7 +177,7 @@ where id, attribute, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), }) } } @@ -190,7 +187,7 @@ where Ok(ast::InlineExpression::Placeable { expression: Box::new(exp), #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), }) } _ if only_literal => error!(ErrorKind::ExpectedLiteral, self.ptr), @@ -242,11 +239,11 @@ where name: ast::Identifier { name: id.name.clone(), #[cfg(feature = "spans")] - span: id.span.clone(), + span: id.span, }, value: val, #[cfg(feature = "spans")] - span: id.span.start..self.ptr, + span: ast::Span::new(id.span.start..self.ptr), }); } else { if !argument_names.is_empty() { @@ -272,7 +269,7 @@ where positional, named, #[cfg(feature = "spans")] - span: start..self.ptr, + span: ast::Span::new(start..self.ptr), })) } } diff --git a/fluent-syntax/src/parser/pattern.rs b/fluent-syntax/src/parser/pattern.rs index cb0be596..d0a6330d 100644 --- a/fluent-syntax/src/parser/pattern.rs +++ b/fluent-syntax/src/parser/pattern.rs @@ -147,7 +147,7 @@ where PatternElementPlaceholders::Placeable(expression, range) => { ast::PatternElement::Placeable { expression, - span: range, + span: ast::Span::new(range), } } #[cfg(not(feature = "spans"))] @@ -170,7 +170,7 @@ where ast::PatternElement::TextElement { value, #[cfg(feature = "spans")] - span: start..end, + span: ast::Span::new(start..end), } } }) @@ -178,7 +178,7 @@ where return Ok(Some(ast::Pattern { elements, #[cfg(feature = "spans")] - span: start_pos..self.ptr, + span: ast::Span::new(start_pos..self.ptr), })); } diff --git a/fluent-syntax/src/parser/runtime.rs b/fluent-syntax/src/parser/runtime.rs index c2b4d738..d350e046 100644 --- a/fluent-syntax/src/parser/runtime.rs +++ b/fluent-syntax/src/parser/runtime.rs @@ -37,7 +37,7 @@ where body.push(ast::Entry::Junk { content, #[cfg(feature = "spans")] - span: entry_start..self.ptr, + span: ast::Span::new(entry_start..self.ptr), }); } } @@ -45,9 +45,9 @@ where } if errors.is_empty() { - Ok(ast::Resource { body, #[cfg(feature = "spans")] span: 0..self.length }) + Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }) } else { - Err((ast::Resource { body, #[cfg(feature = "spans")] span: 0..self.length }, errors)) + Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }, errors)) } } diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index d10f25ee..0231622d 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -24,9 +24,6 @@ use crate::{ast::*, parser::matches_fluent_ws, parser::Slice}; use std::fmt::Write; -#[cfg(feature = "spans")] -use std::ops::Range; - /// Serializes an abstract syntax tree representing a Fluent Translation List into a /// String. /// @@ -490,21 +487,21 @@ mod test { id: Identifier { name: $name, #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }, value: Some(Pattern { elements: vec![PatternElement::TextElement { value: $value, #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }], #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }), attributes: vec![], comment: None, #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }) }; } @@ -589,29 +586,29 @@ mod test { id: Identifier { name: "num", #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }, #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }, #[cfg(feature = "spans")] - Range::default(), + Span::default(), ), #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }, PatternElement::TextElement { value: " bar", #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }, ], #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }, default: false, #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }; ast.body[0].as_message().as_pattern().elements[0] .as_expression() @@ -672,7 +669,7 @@ mod test { ast.body[0].as_message().comment.replace(Comment { content: vec!["great message!"], #[cfg(feature = "spans")] - span: Range::default(), + span: Span::default(), }); assert_eq!("# great message!\nfoo = bar\n", serialize(&ast)); } From b3b75bd137fb29335613eaf4b4dfa05b3ac71d9a Mon Sep 17 00:00:00 2001 From: Ertanic Date: Tue, 11 Mar 2025 19:09:10 +0300 Subject: [PATCH 06/10] Now `Span` wraps over `std::ops::Range` --- fluent-syntax/Cargo.toml | 2 +- fluent-syntax/src/ast/mod.rs | 23 ++++++++++------------- fluent-syntax/src/parser/comment.rs | 2 +- fluent-syntax/src/parser/core.rs | 16 ++++++++-------- fluent-syntax/src/parser/expression.rs | 26 +++++++++++++------------- fluent-syntax/src/parser/pattern.rs | 6 +++--- fluent-syntax/src/parser/runtime.rs | 17 ++++++++++++++--- 7 files changed, 50 insertions(+), 42 deletions(-) diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index e9efcd54..f84e226c 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -37,7 +37,7 @@ serde_json.workspace = true glob = "0.3" [features] -default = [] +default = ["spans"] spans = [] serde = ["dep:serde"] json = ["serde", "dep:serde_json"] diff --git a/fluent-syntax/src/ast/mod.rs b/fluent-syntax/src/ast/mod.rs index ecf7c967..2ace9df1 100644 --- a/fluent-syntax/src/ast/mod.rs +++ b/fluent-syntax/src/ast/mod.rs @@ -88,6 +88,7 @@ mod helper; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +use std::ops::Deref; #[cfg(feature = "spans")] use std::ops::Range; @@ -1419,7 +1420,7 @@ pub enum InlineExpression { #[cfg(feature = "spans")] impl InlineExpression { - pub fn get_span(&self) -> Span { + pub fn get_span(&self) -> &Span { match self { InlineExpression::StringLiteral { span, .. } | InlineExpression::TermReference { span, .. } @@ -1427,7 +1428,7 @@ impl InlineExpression { | InlineExpression::Placeable { span, .. } | InlineExpression::NumberLiteral { span, .. } | InlineExpression::FunctionReference { span, .. } - | InlineExpression::MessageReference { span, .. } => *span, + | InlineExpression::MessageReference { span, .. } => span, } } } @@ -1551,19 +1552,15 @@ pub enum Expression { /// assert_eq!(id.span, Span { start: 0, end: 11 }); // the span of hello-world identifier /// ``` #[cfg(feature = "spans")] -#[derive(Debug, Clone, Copy, Default, PartialEq)] +#[derive(Debug, Clone, Default, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct Span { - pub start: usize, - pub end: usize, -} +pub struct Span(pub Range); #[cfg(feature = "spans")] -impl Span { - pub fn new(range: Range) -> Self { - Self { - start: range.start, - end: range.end, - } +impl Deref for Span { + type Target = Range; + + fn deref(&self) -> &Self::Target { + &self.0 } } \ No newline at end of file diff --git a/fluent-syntax/src/parser/comment.rs b/fluent-syntax/src/parser/comment.rs index 97c3ff6e..bfcff8ef 100644 --- a/fluent-syntax/src/parser/comment.rs +++ b/fluent-syntax/src/parser/comment.rs @@ -54,7 +54,7 @@ where ast::Comment { content, #[cfg(feature = "spans")] - span: ast::Span::new(start_pos..self.ptr), + span: ast::Span(start_pos..self.ptr), }, level, )) diff --git a/fluent-syntax/src/parser/core.rs b/fluent-syntax/src/parser/core.rs index 70ebe9dc..b8141d71 100644 --- a/fluent-syntax/src/parser/core.rs +++ b/fluent-syntax/src/parser/core.rs @@ -70,7 +70,7 @@ where body.push(ast::Entry::Junk { content, #[cfg(feature = "spans")] - span: ast::Span::new(entry_start..self.ptr), + span: ast::Span(entry_start..self.ptr), }); } } @@ -81,9 +81,9 @@ where body.push(ast::Entry::Comment(last_comment)); } if errors.is_empty() { - Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }) + Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span(0..self.length) }) } else { - Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }, errors)) + Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span(0..self.length) }, errors)) } } @@ -128,7 +128,7 @@ where attributes, comment: None, #[cfg(feature = "spans")] - span: ast::Span::new(entry_start..self.ptr), + span: ast::Span(entry_start..self.ptr), }) } @@ -152,7 +152,7 @@ where attributes, comment: None, #[cfg(feature = "spans")] - span: ast::Span::new(entry_start..self.ptr), + span: ast::Span(entry_start..self.ptr), }) } else { error!( @@ -196,7 +196,7 @@ where id, value: pattern, #[cfg(feature = "spans")] - span: ast::Span::new(self.ptr - 1..self.ptr), + span: ast::Span(self.ptr - 1..self.ptr), }), None => error!(ErrorKind::MissingValue, self.ptr), } @@ -218,7 +218,7 @@ where ast::Identifier { name, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), } } @@ -295,7 +295,7 @@ where value, default, #[cfg(feature = "spans")] - span: ast::Span::new((if default { start - 1 } else { start })..self.ptr), + span: ast::Span((if default { start - 1 } else { start })..self.ptr), }); self.skip_blank(); } else { diff --git a/fluent-syntax/src/parser/expression.rs b/fluent-syntax/src/parser/expression.rs index 014228e0..db72351d 100644 --- a/fluent-syntax/src/parser/expression.rs +++ b/fluent-syntax/src/parser/expression.rs @@ -23,7 +23,7 @@ where return Ok(ast::Expression::Inline( exp, #[cfg(feature = "spans")] - ast::Span::new(start_span..self.ptr), + ast::Span(start_span..self.ptr), )); } @@ -68,7 +68,7 @@ where selector: exp, variants, #[cfg(feature = "spans")] - span: ast::Span::new(start_span..self.ptr), + span: ast::Span(start_span..self.ptr), }) } @@ -112,7 +112,7 @@ where Ok(ast::InlineExpression::StringLiteral { value: slice, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), }) } Some(b) if b.is_ascii_digit() => { @@ -120,7 +120,7 @@ where Ok(ast::InlineExpression::NumberLiteral { value: num, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), }) } Some(b'-') if !only_literal => { @@ -135,7 +135,7 @@ where attribute, arguments, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), }) } else { self.ptr -= 1; @@ -143,7 +143,7 @@ where Ok(ast::InlineExpression::NumberLiteral { value: num, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), }) } } @@ -153,7 +153,7 @@ where Ok(ast::InlineExpression::VariableReference { id, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), }) } Some(b) if b.is_ascii_alphabetic() => { @@ -169,7 +169,7 @@ where id, arguments, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), }) } else { let attribute = self.get_attribute_accessor()?; @@ -177,7 +177,7 @@ where id, attribute, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), }) } } @@ -187,7 +187,7 @@ where Ok(ast::InlineExpression::Placeable { expression: Box::new(exp), #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), }) } _ if only_literal => error!(ErrorKind::ExpectedLiteral, self.ptr), @@ -239,11 +239,11 @@ where name: ast::Identifier { name: id.name.clone(), #[cfg(feature = "spans")] - span: id.span, + span: id.span.clone(), }, value: val, #[cfg(feature = "spans")] - span: ast::Span::new(id.span.start..self.ptr), + span: ast::Span(id.span.start..self.ptr), }); } else { if !argument_names.is_empty() { @@ -269,7 +269,7 @@ where positional, named, #[cfg(feature = "spans")] - span: ast::Span::new(start..self.ptr), + span: ast::Span(start..self.ptr), })) } } diff --git a/fluent-syntax/src/parser/pattern.rs b/fluent-syntax/src/parser/pattern.rs index d0a6330d..6c822646 100644 --- a/fluent-syntax/src/parser/pattern.rs +++ b/fluent-syntax/src/parser/pattern.rs @@ -147,7 +147,7 @@ where PatternElementPlaceholders::Placeable(expression, range) => { ast::PatternElement::Placeable { expression, - span: ast::Span::new(range), + span: ast::Span(range), } } #[cfg(not(feature = "spans"))] @@ -170,7 +170,7 @@ where ast::PatternElement::TextElement { value, #[cfg(feature = "spans")] - span: ast::Span::new(start..end), + span: ast::Span(start..end), } } }) @@ -178,7 +178,7 @@ where return Ok(Some(ast::Pattern { elements, #[cfg(feature = "spans")] - span: ast::Span::new(start_pos..self.ptr), + span: ast::Span(start_pos..self.ptr), })); } diff --git a/fluent-syntax/src/parser/runtime.rs b/fluent-syntax/src/parser/runtime.rs index d350e046..a95383dc 100644 --- a/fluent-syntax/src/parser/runtime.rs +++ b/fluent-syntax/src/parser/runtime.rs @@ -37,7 +37,7 @@ where body.push(ast::Entry::Junk { content, #[cfg(feature = "spans")] - span: ast::Span::new(entry_start..self.ptr), + span: ast::Span(entry_start..self.ptr), }); } } @@ -45,9 +45,20 @@ where } if errors.is_empty() { - Ok(ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }) + Ok(ast::Resource { + body, + #[cfg(feature = "spans")] + span: ast::Span(0..self.length), + }) } else { - Err((ast::Resource { body, #[cfg(feature = "spans")] span: ast::Span::new(0..self.length) }, errors)) + Err(( + ast::Resource { + body, + #[cfg(feature = "spans")] + span: ast::Span(0..self.length), + }, + errors, + )) } } From 460c36514d5948172766f396f8a0214be2c6dd1e Mon Sep 17 00:00:00 2001 From: Ertanic Date: Tue, 11 Mar 2025 19:13:21 +0300 Subject: [PATCH 07/10] Removed spans feature from default --- fluent-syntax/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index f84e226c..e9efcd54 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -37,7 +37,7 @@ serde_json.workspace = true glob = "0.3" [features] -default = ["spans"] +default = [] spans = [] serde = ["dep:serde"] json = ["serde", "dep:serde_json"] From 512131803db332207735399e9db268d75adba4ff Mon Sep 17 00:00:00 2001 From: Ertanic Date: Mon, 17 Mar 2025 13:30:01 +0300 Subject: [PATCH 08/10] Revert "Removed spans feature from default" This reverts commit 460c36514d5948172766f396f8a0214be2c6dd1e. --- fluent-syntax/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index e9efcd54..f84e226c 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -37,7 +37,7 @@ serde_json.workspace = true glob = "0.3" [features] -default = [] +default = ["spans"] spans = [] serde = ["dep:serde"] json = ["serde", "dep:serde_json"] From 27615271d3f8d990e935d3c99053329863d6129e Mon Sep 17 00:00:00 2001 From: Ertanic Date: Mon, 17 Mar 2025 13:30:06 +0300 Subject: [PATCH 09/10] Reapply "Removed spans feature from default" This reverts commit 512131803db332207735399e9db268d75adba4ff. --- fluent-syntax/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index f84e226c..e9efcd54 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -37,7 +37,7 @@ serde_json.workspace = true glob = "0.3" [features] -default = ["spans"] +default = [] spans = [] serde = ["dep:serde"] json = ["serde", "dep:serde_json"] From be46c67f276bf201dd2bd2846507da29a523a529 Mon Sep 17 00:00:00 2001 From: Ertanic Date: Thu, 1 May 2025 16:13:15 +0300 Subject: [PATCH 10/10] Add span to VariantKey --- fluent-bundle/src/resolver/expression.rs | 4 ++-- fluent-syntax/src/ast/mod.rs | 20 ++++++++++++++------ fluent-syntax/src/parser/core.rs | 12 +++++++++++- fluent-syntax/src/serializer.rs | 4 ++-- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/fluent-bundle/src/resolver/expression.rs b/fluent-bundle/src/resolver/expression.rs index 19cc3d24..a22505e6 100644 --- a/fluent-bundle/src/resolver/expression.rs +++ b/fluent-bundle/src/resolver/expression.rs @@ -32,8 +32,8 @@ impl<'bundle> WriteValue<'bundle> for ast::Expression<&'bundle str> { FluentValue::String(_) | FluentValue::Number(_) => { for variant in variants { let key = match variant.key { - ast::VariantKey::Identifier { name } => name.into(), - ast::VariantKey::NumberLiteral { value } => { + ast::VariantKey::Identifier { name, .. } => name.into(), + ast::VariantKey::NumberLiteral { value, .. } => { FluentValue::try_number(value) } }; diff --git a/fluent-syntax/src/ast/mod.rs b/fluent-syntax/src/ast/mod.rs index 2ace9df1..21b1b879 100644 --- a/fluent-syntax/src/ast/mod.rs +++ b/fluent-syntax/src/ast/mod.rs @@ -785,8 +785,16 @@ pub struct Variant { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] pub enum VariantKey { - Identifier { name: S }, - NumberLiteral { value: S }, + Identifier { + name: S, + #[cfg(feature = "spans")] + span: Span, + }, + NumberLiteral { + value: S, + #[cfg(feature = "spans")] + span: Span, + }, } /// Fluent [`Comment`]. @@ -1539,12 +1547,12 @@ pub enum Expression { /// /// ``` /// #![cfg(feature = "spans")] -/// +/// /// use fluent_syntax::parser; /// use fluent_syntax::ast::*; -/// +/// /// let ftl = "hello-world = Hello, World!"; -/// +/// /// let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); /// let Entry::Message(Message { ref id, .. }) = resource.body[0] else { unreachable!() }; /// @@ -1563,4 +1571,4 @@ impl Deref for Span { fn deref(&self) -> &Self::Target { &self.0 } -} \ No newline at end of file +} diff --git a/fluent-syntax/src/parser/core.rs b/fluent-syntax/src/parser/core.rs index b8141d71..ed69c4c3 100644 --- a/fluent-syntax/src/parser/core.rs +++ b/fluent-syntax/src/parser/core.rs @@ -3,7 +3,7 @@ use super::{ errors::{ErrorKind, ParserError}, slice::Slice, }; -use crate::ast; +use crate::ast::{self, Span}; pub type Result = std::result::Result; @@ -248,12 +248,22 @@ where self.skip_blank(); let key = if self.is_number_start() { + #[cfg(feature = "spans")] + let start = self.ptr; + ast::VariantKey::NumberLiteral { value: self.get_number_literal()?, + #[cfg(feature = "spans")] + span: Span(start..self.ptr), } } else { + #[cfg(feature = "spans")] + let start = self.ptr; + ast::VariantKey::Identifier { name: self.get_identifier()?.name, + #[cfg(feature = "spans")] + span: Span(start..self.ptr), } }; diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 0231622d..01e72bf6 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -325,7 +325,7 @@ impl Serializer { fn serialize_variant_key<'s, S: Slice<'s>>(&mut self, key: &VariantKey) { match key { - VariantKey::NumberLiteral { value } | VariantKey::Identifier { name: value } => { + VariantKey::NumberLiteral { value, .. } | VariantKey::Identifier { name: value, .. } => { self.writer.write_literal(value.as_ref()); } } @@ -577,7 +577,7 @@ mod test { let mut ast = parse(message).expect("failed to parse ftl resource"); let one_variant = Variant { - key: VariantKey::Identifier { name: "one" }, + key: VariantKey::Identifier { name: "one", span: Span::default() }, value: Pattern { elements: vec![ PatternElement::Placeable {