Refactor swayfmt to support arbitrary lexed trees (#6806)

## Description

This PR refactors `swayfmt` to be able generate code from arbitrary
lexed trees. Arbitrary means a lexed tree that can be fully or partially
created in-memory by manipulating the tree structure in code. It does
not need to necessarily be backed by source code. This is needed to be
able to reuse `swayfmt` for generating source code from tools that
analyze and modify Sway code, as explained in #6779.

This PR makes the `swayfmt` independent of spans backed by the source
code, by doing the following changes:
- Keywords are rendered by using theirs `AS_STR` associated constants.
- The `Token` trait is extended with the `AS_STR` associated constant,
and tokens are rendered by using that constant.
- `Ident`s are rendered by using theirs `as_str()` methods. This method
takes the textual implementation from `Ident::name_override_opt` if it
is provided, and ignores the span in that case.
- `Literal`s are rendered based on the literal value and not their
spans. The exception are numeric literals that do not have empty spans.
Those are considered to be backed by source code and are rendered as
written in code, e.g., `1_000_000u64`.

The PR also fixes some existing bugs in `swayfmt`:
- `use path::{single_import,};` will be formatted as `use
path::single_import;`
- `use path::{a,b,};` will be formatted as `use path::{a, b};`
- partial addresses #6802 by rendering annotations for final values.
- partial addresses #6805 by properly rendering final values, but still
not using the expected field alignment.

The PR also removes the `path` parameter from the `parse_file`. It was
used only to provide an unnecessary dummy `source_id` which was always
pointing to the package root file.

Closes #6779.

## Checklist

- [x] I have linked to any relevant issues.
- [x] I have commented my code, particularly in hard-to-understand
areas.
- [ ] I have updated the documentation where relevant (API docs, the
reference, and the Sway book).
- [ ] If my change requires substantial documentation changes, I have
[requested support from the DevRel
team](https://github.com/FuelLabs/devrel-requests/issues/new/choose)
- [x] I have added tests that prove my fix is effective or that my
feature works.
- [ ] I have added (or requested a maintainer to add) the necessary
`Breaking*` or `New Feature` labels where relevant.
- [x] I have done my best to ensure that my PR adheres to [the Fuel Labs
Code Review
Standards](https://github.com/FuelLabs/rfcs/blob/master/text/code-standards/external-contributors.md).
- [x] I have requested a review from the relevant team or maintainers.

examples/advanced_storage_variables/src/main.sw

@@ -1,6 +1,6 @@
 contract;
 
-use std::{bytes::Bytes, string::String,};
+use std::{bytes::Bytes, string::String};
 
 // ANCHOR: temp_hash_import
 use std::hash::Hash;

examples/converting_types/src/byte_arrays.sw

@@ -1,7 +1,7 @@
 library;
 
 // ANCHOR: to_byte_array_import
-use std::array_conversions::{b256::*, u16::*, u256::*, u32::*, u64::*,};
+use std::array_conversions::{b256::*, u16::*, u256::*, u32::*, u64::*};
 // ANCHOR_END: to_byte_array_import
 
 pub fn to_byte_array() {

examples/converting_types/src/bytes.sw

@@ -1,7 +1,7 @@
 library;
 
 // ANCHOR: to_bytes_import
-use std::{bytes::Bytes, bytes_conversions::{b256::*, u16::*, u256::*, u32::*, u64::*,}};
+use std::{bytes::Bytes, bytes_conversions::{b256::*, u16::*, u256::*, u32::*, u64::*}};
 // ANCHOR_END: to_bytes_import
 
 pub fn convert_to_bytes() {

examples/wallet_smart_contract/src/main.sw

@@ -1,7 +1,7 @@
 // ANCHOR: full_wallet
 contract;
 
-use std::{asset::transfer, call_frames::msg_asset_id, context::msg_amount,};
+use std::{asset::transfer, call_frames::msg_asset_id, context::msg_amount};
 
 // ANCHOR: abi_import
 use wallet_abi::Wallet;

forc-plugins/forc-fmt/src/main.rs

@@ -15,7 +15,6 @@ use std::{
     path::{Path, PathBuf},
     sync::Arc,
 };
-use sway_core::{BuildConfig, BuildTarget};
 use sway_utils::{constants, find_parent_manifest_dir, get_sway_files, is_sway_file};
 use swayfmt::Formatter;
 use taplo::formatter as taplo_fmt;
@@ -77,13 +76,8 @@ fn run() -> Result<()> {
     if let Some(f) = app.file.as_ref() {
         let file_path = &PathBuf::from(f);
 
-        // If we're formatting a single file, find the nearest manifest if within a project.
-        // Otherwise, we simply provide 'None' to format_file().
-        let manifest_file = find_parent_manifest_dir(file_path)
-            .map(|path| path.join(constants::MANIFEST_FILE_NAME));
-
         if is_sway_file(file_path) {
-            format_file(&app, file_path.to_path_buf(), manifest_file, &mut formatter)?;
+            format_file(&app, file_path.to_path_buf(), &mut formatter)?;
             return Ok(());
         }
 
@@ -142,12 +136,7 @@ fn get_sway_dirs(workspace_dir: PathBuf) -> Vec<PathBuf> {
 /// - Ok(true) if executed successfully and formatted,
 /// - Ok(false) if executed successfully and not formatted,
 /// - Err if it fails to execute at all.
-fn format_file(
-    app: &App,
-    file: PathBuf,
-    manifest_file: Option<PathBuf>,
-    formatter: &mut Formatter,
-) -> Result<bool> {
+fn format_file(app: &App, file: PathBuf, formatter: &mut Formatter) -> Result<bool> {
     let file = file.canonicalize()?;
     if is_file_dirty(&file) {
         bail!(
@@ -160,14 +149,7 @@ fn format_file(
     if let Ok(file_content) = fs::read_to_string(&file) {
         let mut edited = false;
         let file_content: Arc<str> = Arc::from(file_content);
-        let build_config = manifest_file.map(|f| {
-            BuildConfig::root_from_file_name_and_manifest_path(
-                file.clone(),
-                f,
-                BuildTarget::default(),
-            )
-        });
-        match Formatter::format(formatter, file_content.clone(), build_config.as_ref()) {
+        match Formatter::format(formatter, file_content.clone()) {
             Ok(formatted_content) => {
                 if app.check {
                     if *file_content != formatted_content {
@@ -213,12 +195,7 @@ fn format_workspace_at_dir(app: &App, workspace: &WorkspaceManifestFile, dir: &P
         for entry in read_dir.filter_map(|res| res.ok()) {
             let path = entry.path();
             if is_sway_file(&path) {
-                format_file(
-                    app,
-                    path,
-                    Some(workspace.dir().to_path_buf()),
-                    &mut formatter,
-                )?;
+                format_file(app, path, &mut formatter)?;
             }
         }
     }
@@ -295,7 +272,7 @@ fn format_pkg_at_dir(app: &App, dir: &Path, formatter: &mut Formatter) -> Result
             let mut contains_edits = false;
 
             for file in files {
-                contains_edits |= format_file(app, file, Some(manifest_file.clone()), formatter)?;
+                contains_edits |= format_file(app, file, formatter)?;
             }
             // format manifest using taplo formatter
             contains_edits |= format_manifest(app, manifest_file)?;

sway-ast/src/expr/mod.rs

@@ -306,6 +306,18 @@ impl ReassignmentOpVariant {
             ReassignmentOpVariant::ShrEquals => "rsh",
         }
     }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ReassignmentOpVariant::Equals => EqToken::AS_STR,
+            ReassignmentOpVariant::AddEquals => AddEqToken::AS_STR,
+            ReassignmentOpVariant::SubEquals => SubEqToken::AS_STR,
+            ReassignmentOpVariant::MulEquals => StarEqToken::AS_STR,
+            ReassignmentOpVariant::DivEquals => DivEqToken::AS_STR,
+            ReassignmentOpVariant::ShlEquals => ShlEqToken::AS_STR,
+            ReassignmentOpVariant::ShrEquals => ShrEqToken::AS_STR,
+        }
+    }
 }
 
 #[derive(Clone, Debug, Serialize)]

sway-ast/src/expr/op_code.rs

@@ -123,6 +123,14 @@ macro_rules! define_op_codes (
                 }
             }
 
+            pub fn op_code_as_str(&self) -> &'static str {
+                match self {
+                    $(Instruction::$op_name { .. } => {
+                        $s
+                    },)*
+                }
+            }
+
             #[allow(clippy::vec_init_then_push)]
             pub fn register_arg_idents(&self) -> Vec<Ident> {
                 match self {

sway-ast/src/keywords.rs

@@ -98,10 +98,13 @@ pub trait Token: Spanned + Sized {
 
     /// Punctuations that will not follow the token.
     const NOT_FOLLOWED_BY: &'static [PunctKind];
+
+    /// What the string representation of the token is when lexing.
+    const AS_STR: &'static str;
 }
 
 macro_rules! define_token (
-    ($ty_name:ident, $description:literal, [$($punct_kinds:ident),*], [$($not_followed_by:ident),*]) => {
+    ($ty_name:ident, $description:literal, $as_str:literal, [$($punct_kinds:ident),*], [$($not_followed_by:ident),*]) => {
         #[derive(Clone, Debug, Serialize)]
         pub struct $ty_name {
             span: Span,
@@ -132,6 +135,7 @@ macro_rules! define_token (
 
             const PUNCT_KINDS: &'static [PunctKind] = &[$(PunctKind::$punct_kinds,)*];
             const NOT_FOLLOWED_BY: &'static [PunctKind] = &[$(PunctKind::$not_followed_by,)*];
+            const AS_STR: &'static str = $as_str;
         }
 
         impl From<$ty_name> for Ident {
@@ -142,93 +146,121 @@ macro_rules! define_token (
     };
 );
 
-define_token!(SemicolonToken, "a semicolon", [Semicolon], []);
+define_token!(SemicolonToken, "a semicolon", ";", [Semicolon], []);
 define_token!(
     ForwardSlashToken,
     "a forward slash",
+    "/",
     [ForwardSlash],
     [Equals]
 );
 define_token!(
     DoubleColonToken,
     "a double colon (::)",
+    "::",
     [Colon, Colon],
     [Colon]
 );
-define_token!(StarToken, "an asterisk (*)", [Star], [Equals]);
-define_token!(DoubleStarToken, "`**`", [Star, Star], []);
-define_token!(CommaToken, "a comma", [Comma], []);
-define_token!(ColonToken, "a colon", [Colon], [Colon]);
+define_token!(StarToken, "an asterisk (*)", "*", [Star], [Equals]);
+define_token!(DoubleStarToken, "`**`", "**", [Star, Star], []);
+define_token!(CommaToken, "a comma", ",", [Comma], []);
+define_token!(ColonToken, "a colon", ":", [Colon], [Colon]);
 define_token!(
     RightArrowToken,
     "`->`",
+    "->",
     [Sub, GreaterThan],
     [GreaterThan, Equals]
 );
-define_token!(LessThanToken, "`<`", [LessThan], [LessThan, Equals]);
+define_token!(LessThanToken, "`<`", "<", [LessThan], [LessThan, Equals]);
 define_token!(
     GreaterThanToken,
     "`>`",
+    ">",
     [GreaterThan],
     [GreaterThan, Equals]
 );
-define_token!(OpenAngleBracketToken, "`<`", [LessThan], []);
-define_token!(CloseAngleBracketToken, "`>`", [GreaterThan], []);
-define_token!(EqToken, "`=`", [Equals], [GreaterThan, Equals]);
-define_token!(AddEqToken, "`+=`", [Add, Equals], []);
-define_token!(SubEqToken, "`-=`", [Sub, Equals], []);
-define_token!(StarEqToken, "`*=`", [Star, Equals], []);
-define_token!(DivEqToken, "`/=`", [ForwardSlash, Equals], []);
-define_token!(ShlEqToken, "`<<=`", [LessThan, LessThan, Equals], []);
-define_token!(ShrEqToken, "`>>=`", [GreaterThan, GreaterThan, Equals], []);
+define_token!(OpenAngleBracketToken, "`<`", "<", [LessThan], []);
+define_token!(CloseAngleBracketToken, "`>`", ">", [GreaterThan], []);
+define_token!(EqToken, "`=`", "=", [Equals], [GreaterThan, Equals]);
+define_token!(AddEqToken, "`+=`", "+=", [Add, Equals], []);
+define_token!(SubEqToken, "`-=`", "-=", [Sub, Equals], []);
+define_token!(StarEqToken, "`*=`", "*=", [Star, Equals], []);
+define_token!(DivEqToken, "`/=`", "/=", [ForwardSlash, Equals], []);
+define_token!(ShlEqToken, "`<<=`", "<<=", [LessThan, LessThan, Equals], []);
+define_token!(
+    ShrEqToken,
+    "`>>=`",
+    ">>=",
+    [GreaterThan, GreaterThan, Equals],
+    []
+);
 define_token!(
     FatRightArrowToken,
     "`=>`",
+    "=>",
     [Equals, GreaterThan],
     [GreaterThan, Equals]
 );
-define_token!(DotToken, "`.`", [Dot], []);
-define_token!(DoubleDotToken, "`..`", [Dot, Dot], [Dot]);
-define_token!(BangToken, "`!`", [Bang], [Equals]);
-define_token!(PercentToken, "`%`", [Percent], []);
-define_token!(AddToken, "`+`", [Add], [Equals]);
-define_token!(SubToken, "`-`", [Sub], [Equals]);
+define_token!(DotToken, "`.`", ".", [Dot], []);
+define_token!(DoubleDotToken, "`..`", "..", [Dot, Dot], [Dot]);
+define_token!(BangToken, "`!`", "!", [Bang], [Equals]);
+define_token!(PercentToken, "`%`", "%", [Percent], []);
+define_token!(AddToken, "`+`", "+", [Add], [Equals]);
+define_token!(SubToken, "`-`", "-", [Sub], [Equals]);
 define_token!(
     ShrToken,
     "`>>`",
+    ">>",
     [GreaterThan, GreaterThan],
     [GreaterThan, Equals]
 );
-define_token!(ShlToken, "`<<`", [LessThan, LessThan], [LessThan, Equals]);
-define_token!(AmpersandToken, "`&`", [Ampersand], [Ampersand]);
-define_token!(CaretToken, "`^`", [Caret], []);
-define_token!(PipeToken, "`|`", [Pipe], [Pipe]);
+define_token!(
+    ShlToken,
+    "`<<`",
+    "<<",
+    [LessThan, LessThan],
+    [LessThan, Equals]
+);
+define_token!(AmpersandToken, "`&`", "&", [Ampersand], [Ampersand]);
+define_token!(CaretToken, "`^`", "^", [Caret], []);
+define_token!(PipeToken, "`|`", "|", [Pipe], [Pipe]);
 define_token!(
     DoubleEqToken,
     "`==`",
+    "==",
     [Equals, Equals],
     [Equals, GreaterThan]
 );
-define_token!(BangEqToken, "`!=`", [Bang, Equals], [Equals, GreaterThan]);
+define_token!(
+    BangEqToken,
+    "`!=`",
+    "!=",
+    [Bang, Equals],
+    [Equals, GreaterThan]
+);
 define_token!(
     GreaterThanEqToken,
     "`>=`",
+    ">=",
     [GreaterThan, Equals],
     [Equals, GreaterThan]
 );
 define_token!(
     LessThanEqToken,
     "`<=`",
+    "<=",
     [LessThan, Equals],
     [Equals, GreaterThan]
 );
 define_token!(
     DoubleAmpersandToken,
     "`&&`",
+    "&&",
     [Ampersand, Ampersand],
     [Ampersand]
 );
-define_token!(DoublePipeToken, "`||`", [Pipe, Pipe], [Pipe]);
-define_token!(UnderscoreToken, "`_`", [Underscore], [Underscore]);
-define_token!(HashToken, "`#`", [Sharp], []);
-define_token!(HashBangToken, "`#!`", [Sharp, Bang], []);
+define_token!(DoublePipeToken, "`||`", "||", [Pipe, Pipe], [Pipe]);
+define_token!(UnderscoreToken, "`_`", "_", [Underscore], [Underscore]);
+define_token!(HashToken, "`#`", "#", [Sharp], []);
+define_token!(HashBangToken, "`#!`", "#!", [Sharp, Bang], []);

sway-ast/src/punctuated.rs

@@ -20,6 +20,19 @@ impl<T, P> Punctuated<T, P> {
             final_value_opt: Some(Box::new(value)),
         }
     }
+
+    /// Returns true if the [Punctuated] ends with the punctuation token.
+    /// E.g., `fn fun(x: u64, y: u64,)`.
+    pub fn has_trailing_punctuation(&self) -> bool {
+        !self.value_separator_pairs.is_empty() && self.final_value_opt.is_none()
+    }
+
+    /// Returns true if the [Punctuated] has neither value separator pairs,
+    /// nor the final value.
+    /// E.g., `fn fun()`.
+    pub fn is_empty(&self) -> bool {
+        self.value_separator_pairs.is_empty() && self.final_value_opt.is_none()
+    }
 }
 
 impl<T, P> IntoIterator for Punctuated<T, P> {

sway-lib-std/src/prelude.sw

@@ -27,7 +27,7 @@ pub use ::revert::{require, revert, revert_with_log};
 pub use ::convert::From;
 
 // Primitive conversions
-pub use ::primitive_conversions::{b256::*, str::*, u16::*, u256::*, u32::*, u64::*, u8::*,};
+pub use ::primitive_conversions::{b256::*, str::*, u16::*, u256::*, u32::*, u64::*, u8::*};
 
 // Logging
 pub use ::logging::log;

sway-lsp/src/capabilities/formatting.rs

@@ -24,7 +24,7 @@ pub fn get_page_text_edit(
 ) -> Result<TextEdit, LanguageServerError> {
     // we only format if code is correct
     let formatted_code = formatter
-        .format(text.clone(), None)
+        .format(text.clone())
         .map_err(LanguageServerError::FormatError)?;
 
     let text_lines_count = text.split('\n').count();

sway-types/src/ast.rs

@@ -6,14 +6,14 @@ pub enum Delimiter {
 }
 
 impl Delimiter {
-    pub fn as_open_char(self) -> char {
+    pub const fn as_open_char(self) -> char {
         match self {
             Delimiter::Parenthesis => '(',
             Delimiter::Brace => '{',
             Delimiter::Bracket => '[',
         }
     }
-    pub fn as_close_char(self) -> char {
+    pub const fn as_close_char(self) -> char {
         match self {
             Delimiter::Parenthesis => ')',
             Delimiter::Brace => '}',

sway-types/src/span.rs

@@ -203,6 +203,10 @@ impl Span {
     pub fn is_dummy(&self) -> bool {
         self.eq(&DUMMY_SPAN)
     }
+
+    pub fn is_empty(&self) -> bool {
+        self.start == self.end
+    }
 }
 
 impl fmt::Debug for Span {