From eba0d0172b41203f7b4ac8dd0c958bc9a86f10da Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 17:31:00 -0700 Subject: [PATCH 01/71] save --- .../latex/latex-annotation/src/test_helpers.rs | 2 +- .../src/test_helpers/annotations.rs | 1 + crates/latex/latex-ast/src/parser.rs | 11 +++++++---- crates/latex/latex-ast/src/region.rs | 18 ++++++++++++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 crates/latex/latex-annotation/src/test_helpers/annotations.rs diff --git a/crates/latex/latex-annotation/src/test_helpers.rs b/crates/latex/latex-annotation/src/test_helpers.rs index 8b13789179..441c53fe5d 100644 --- a/crates/latex/latex-annotation/src/test_helpers.rs +++ b/crates/latex/latex-annotation/src/test_helpers.rs @@ -1 +1 @@ - +pub mod annotations; diff --git a/crates/latex/latex-annotation/src/test_helpers/annotations.rs b/crates/latex/latex-annotation/src/test_helpers/annotations.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/latex/latex-annotation/src/test_helpers/annotations.rs @@ -0,0 +1 @@ + diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index ba8e24e615..9b070a21e8 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -1,4 +1,7 @@ -use crate::ast::{LxAstArena, LxAstData, LxAstIdx, LxAstIdxRange}; +use crate::{ + ast::{LxAstArena, LxAstData, LxAstIdx, LxAstIdxRange}, + region::LxAstRegionData, +}; use latex_annotation::{ annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}, annotations::{walker::LxAnnotationsWalker, LxAnnotations}, @@ -30,7 +33,10 @@ impl<'a> LxAstParser<'a> { arena, } } +} +/// # actions +impl<'a> LxAstParser<'a> { pub(crate) fn alloc_asts(&mut self, asts: Vec) -> LxAstIdxRange { self.arena.alloc_batch(asts) } @@ -38,10 +44,7 @@ impl<'a> LxAstParser<'a> { pub(crate) fn alloc_ast(&mut self, ast: LxAstData) -> LxAstIdx { self.arena.alloc_one(ast) } -} -/// # actions -impl<'a> LxAstParser<'a> { pub(crate) fn peek_token(&mut self) -> Option { self.lexer.peek().map(|&(_, _, _, data)| data) } diff --git a/crates/latex/latex-ast/src/region.rs b/crates/latex/latex-ast/src/region.rs index b26643f977..f374c4c19e 100644 --- a/crates/latex/latex-ast/src/region.rs +++ b/crates/latex/latex-ast/src/region.rs @@ -1,2 +1,20 @@ +use crate::ast::LxAstArena; + #[salsa::tracked] pub struct LxAstRegion {} + +pub struct LxAstRegionData { + arena: LxAstArena, +} + +impl LxAstRegionData { + pub(crate) fn new(arena: LxAstArena) -> Self { + Self { arena } + } +} + +impl LxAstRegionData { + pub fn arena(&self) -> &LxAstArena { + &self.arena + } +} From 1f21a997d2f23cdb536c1cf419e10d593e1da552 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 17:31:51 -0700 Subject: [PATCH 02/71] save --- crates/latex/latex-ast/src/district.rs | 2 -- crates/latex/latex-ast/src/jar.rs | 2 +- crates/latex/latex-ast/src/lib.rs | 2 -- crates/latex/latex-ast/src/token.rs | 1 - 4 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 crates/latex/latex-ast/src/district.rs delete mode 100644 crates/latex/latex-ast/src/token.rs diff --git a/crates/latex/latex-ast/src/district.rs b/crates/latex/latex-ast/src/district.rs deleted file mode 100644 index 7451cd830b..0000000000 --- a/crates/latex/latex-ast/src/district.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[salsa::input] -pub struct LxAstDistrict {} diff --git a/crates/latex/latex-ast/src/jar.rs b/crates/latex/latex-ast/src/jar.rs index 680716a0f4..c2b39a1eee 100644 --- a/crates/latex/latex-ast/src/jar.rs +++ b/crates/latex/latex-ast/src/jar.rs @@ -1,2 +1,2 @@ #[salsa::jar] -pub struct LxAstJar(crate::district::LxAstDistrict, crate::region::LxAstRegion); +pub struct LxAstJar(crate::region::LxAstRegion); diff --git a/crates/latex/latex-ast/src/lib.rs b/crates/latex/latex-ast/src/lib.rs index 82c5f10c0d..8fa925b525 100644 --- a/crates/latex/latex-ast/src/lib.rs +++ b/crates/latex/latex-ast/src/lib.rs @@ -1,12 +1,10 @@ pub mod ast; -pub mod district; pub mod jar; mod parser; pub mod region; pub mod sheet; #[cfg(test)] mod tests; -pub mod token; use self::jar::LxAstJar as Jar; #[cfg(test)] diff --git a/crates/latex/latex-ast/src/token.rs b/crates/latex/latex-ast/src/token.rs deleted file mode 100644 index 8b13789179..0000000000 --- a/crates/latex/latex-ast/src/token.rs +++ /dev/null @@ -1 +0,0 @@ - From 79d4f162c0a06894c21edc7a8b4ff17d83498e91 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 17:36:56 -0700 Subject: [PATCH 03/71] save --- crates/latex/latex-ast/src/ast/math.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 4baf481b1e..2e3bf446d5 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -48,6 +48,7 @@ impl<'a> LxAstParser<'a> { } } + // here we differ from the latex syntax, we see all possible delimiters as latex delimiters fn parse_delimited( &mut self, left_delimiter_token_idx: LxTokenIdx, From 0936df6966311d31a30aaadcb4bec56db815a485 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 18:01:25 -0700 Subject: [PATCH 04/71] save --- Cargo.lock | 2 + crates/latex/latex-annotation/Cargo.toml | 3 ++ .../latex-annotation/src/test_helpers.rs | 2 +- .../src/test_helpers/annotations.rs | 1 - .../src/test_helpers/example.rs | 46 +++++++++++++++++++ 5 files changed, 52 insertions(+), 2 deletions(-) delete mode 100644 crates/latex/latex-annotation/src/test_helpers/annotations.rs create mode 100644 crates/latex/latex-annotation/src/test_helpers/example.rs diff --git a/Cargo.lock b/Cargo.lock index 688426ebf3..23778db978 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8034,6 +8034,8 @@ dependencies = [ "expect-test", "husky-print-utils", "latex-annotation", + "latex-prelude", + "lazy_static", ] [[package]] diff --git a/crates/latex/latex-annotation/Cargo.toml b/crates/latex/latex-annotation/Cargo.toml index bec30f5ff7..f30bbaf8b2 100644 --- a/crates/latex/latex-annotation/Cargo.toml +++ b/crates/latex/latex-annotation/Cargo.toml @@ -12,8 +12,11 @@ categories.workspace = true keywords.workspace = true [dependencies] +lazy_static.workspace = true # abstractions enum-class.workspace = true +# latex +latex-prelude.workspace = true # utils husky-print-utils.workspace = true diff --git a/crates/latex/latex-annotation/src/test_helpers.rs b/crates/latex/latex-annotation/src/test_helpers.rs index 441c53fe5d..d4d8a94c88 100644 --- a/crates/latex/latex-annotation/src/test_helpers.rs +++ b/crates/latex/latex-annotation/src/test_helpers.rs @@ -1 +1 @@ -pub mod annotations; +pub mod example; diff --git a/crates/latex/latex-annotation/src/test_helpers/annotations.rs b/crates/latex/latex-annotation/src/test_helpers/annotations.rs deleted file mode 100644 index 8b13789179..0000000000 --- a/crates/latex/latex-annotation/src/test_helpers/annotations.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/latex/latex-annotation/src/test_helpers/example.rs b/crates/latex/latex-annotation/src/test_helpers/example.rs new file mode 100644 index 0000000000..1753806ee0 --- /dev/null +++ b/crates/latex/latex-annotation/src/test_helpers/example.rs @@ -0,0 +1,46 @@ +use crate::{ + annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}, + annotations::LxAnnotations, +}; + +pub struct LxAnnotationsExample { + pub root_mode: LxMode, + pub input: String, + pub annotations: LxAnnotations, +} + +pub type LxAnnotationsExamples = Vec; + +impl LxAnnotationsExample { + fn collect_from_sparse( + examples: &[( + LxMode, + &str, + &[((&str, &str), LxTokenAnnotation)], + &[((&str, &str), LxSpaceAnnotation)], + )], + ) -> LxAnnotationsExamples { + examples + .iter() + .map( + |&(root_mode, input, token_annotations, space_annotations)| Self { + root_mode, + input: input.to_string(), + annotations: LxAnnotations::from_sparse( + input, + token_annotations.iter().copied(), + space_annotations.iter().copied(), + ), + }, + ) + .collect() + } +} + +use latex_prelude::mode::LxMode; +use lazy_static::lazy_static; + +lazy_static! { + pub static ref EXAMPLES: LxAnnotationsExamples = + LxAnnotationsExample::collect_from_sparse(&[(LxMode::Math, "", &[], &[])]); +} From b8e1a985362f3e2861c9185184a8c6f827f590c8 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 18:02:20 -0700 Subject: [PATCH 05/71] save --- .../src/test_helpers/example.rs | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/latex/latex-annotation/src/test_helpers/example.rs b/crates/latex/latex-annotation/src/test_helpers/example.rs index 1753806ee0..bf61ffd1b9 100644 --- a/crates/latex/latex-annotation/src/test_helpers/example.rs +++ b/crates/latex/latex-annotation/src/test_helpers/example.rs @@ -1,5 +1,8 @@ use crate::{ - annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}, + annotation::{ + space::{LxApplyAnnotation, LxSpaceAnnotation}, + token::{LxTokenAnnotation, LxVariableAnnotation}, + }, annotations::LxAnnotations, }; @@ -41,6 +44,42 @@ use latex_prelude::mode::LxMode; use lazy_static::lazy_static; lazy_static! { - pub static ref EXAMPLES: LxAnnotationsExamples = - LxAnnotationsExample::collect_from_sparse(&[(LxMode::Math, "", &[], &[])]); + pub static ref EXAMPLES: LxAnnotationsExamples = LxAnnotationsExample::collect_from_sparse(&[ + (LxMode::Math, "", &[], &[]), + ( + LxMode::Math, + "x y", + &[ + ( + ("x", "x"), + LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) + ), + ( + ("y", "y"), + LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) + ), + ], + &[( + ("x", "y"), + LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarMul) + ),] + ), + ( + LxMode::Math, + "dx", + &[ + (("d", "d"), LxTokenAnnotation::Differential), + ( + ("x", "x"), + LxTokenAnnotation::Variable( + LxVariableAnnotation::SingleVariableIntegralVariableDecl + ) + ), + ], + &[( + ("d", "x"), + LxSpaceAnnotation::Apply(LxApplyAnnotation::Differentiation) + ),] + ), + ]); } From b2b7c5f5e1305148c7faa1fbf7b8394ffa32937c Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 18:07:40 -0700 Subject: [PATCH 06/71] save --- .../latex-annotation/src/annotation/space.rs | 2 - .../src/annotations/builder/sparce.rs | 18 ++------- .../src/annotations/walker.rs | 38 ++----------------- .../src/test_helpers/example.rs | 20 +++++----- 4 files changed, 19 insertions(+), 59 deletions(-) diff --git a/crates/latex/latex-annotation/src/annotation/space.rs b/crates/latex/latex-annotation/src/annotation/space.rs index ffe1ec1aa0..e20101ca08 100644 --- a/crates/latex/latex-annotation/src/annotation/space.rs +++ b/crates/latex/latex-annotation/src/annotation/space.rs @@ -11,8 +11,6 @@ pub enum LxSpaceAnnotation { pub enum LxApplyAnnotation { ScalarMul, ScalarDifferentialFormMul, - Differentiation, - Integration, } #[enum_class::from_variants] diff --git a/crates/latex/latex-annotation/src/annotations/builder/sparce.rs b/crates/latex/latex-annotation/src/annotations/builder/sparce.rs index 3fc6f4f3cd..05883a5b69 100644 --- a/crates/latex/latex-annotation/src/annotations/builder/sparce.rs +++ b/crates/latex/latex-annotation/src/annotations/builder/sparce.rs @@ -83,20 +83,10 @@ mod tests { ), ]; - let space_annotations = vec![ - ( - ("\\int ", "x"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::Integration), - ), - ( - ("\\int x", "d"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), - ), - ( - ("\\int xd", "x"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::Differentiation), - ), - ]; + let space_annotations = vec![( + ("\\int x", "d"), + LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), + )]; let result = collect_from_sparse_annotations( input, diff --git a/crates/latex/latex-annotation/src/annotations/walker.rs b/crates/latex/latex-annotation/src/annotations/walker.rs index 8d837070ff..ba8991f6b3 100644 --- a/crates/latex/latex-annotation/src/annotations/walker.rs +++ b/crates/latex/latex-annotation/src/annotations/walker.rs @@ -105,20 +105,10 @@ mod tests { ), ]; - let space_annotations = vec![ - ( - ("\\int ", "x"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::Integration), - ), - ( - ("\\int x", "d"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), - ), - ( - ("\\int xd", "x"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::Differentiation), - ), - ]; + let space_annotations = vec![( + ("\\int x", "d"), + LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), + )]; LxAnnotations::from_sparse( &input, @@ -143,15 +133,6 @@ mod tests { ) ); - // Test at offset 5 ("x" after space) - assert_eq!( - walker.next(5, 6), - ( - LxTokenAnnotation::Variable(LxVariableAnnotation::Usage), - LxSpaceAnnotation::Apply(LxApplyAnnotation::Integration) - ) - ); - // Test at offset 6 ("d" after "x") assert_eq!( walker.next(6, 7), @@ -161,17 +142,6 @@ mod tests { ) ); - // Test at offset 7 ("x" after "d") - assert_eq!( - walker.next(7, 8), - ( - LxTokenAnnotation::Variable( - LxVariableAnnotation::SingleVariableIntegralVariableDecl - ), - LxSpaceAnnotation::Apply(LxApplyAnnotation::Differentiation) - ) - ); - // Test at offset 8 (end of input) assert_eq!( walker.next(8, 9), diff --git a/crates/latex/latex-annotation/src/test_helpers/example.rs b/crates/latex/latex-annotation/src/test_helpers/example.rs index bf61ffd1b9..ad5a968012 100644 --- a/crates/latex/latex-annotation/src/test_helpers/example.rs +++ b/crates/latex/latex-annotation/src/test_helpers/example.rs @@ -48,14 +48,14 @@ lazy_static! { (LxMode::Math, "", &[], &[]), ( LxMode::Math, - "x y", + "xy", &[ ( - ("x", "x"), + ("", "x"), LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) ), ( - ("y", "y"), + ("x", "y"), LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) ), ], @@ -68,18 +68,20 @@ lazy_static! { LxMode::Math, "dx", &[ - (("d", "d"), LxTokenAnnotation::Differential), + (("", "d"), LxTokenAnnotation::Differential), ( - ("x", "x"), + ("d", "x"), LxTokenAnnotation::Variable( LxVariableAnnotation::SingleVariableIntegralVariableDecl ) ), ], - &[( - ("d", "x"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::Differentiation) - ),] + &[] ), ]); } + +#[test] +fn latex_annotations_examples_works() { + let examples = &*EXAMPLES; // Dereference the lazy_static +} From 266fb5d5e44b74e48e28f7fdb8aa156be729b48f Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 21:17:16 -0700 Subject: [PATCH 07/71] save --- .../expect-files/annotations/examples.md | 62 +++++++++++++++++++ .../src/annotations/builder/sparce.rs | 14 ----- .../src/annotations/walker.rs | 17 ++--- .../src/test_helpers/example.rs | 3 + .../expect-files/husky_lang_packages.txt | 16 ++--- 5 files changed, 84 insertions(+), 28 deletions(-) create mode 100644 crates/latex/latex-annotation/expect-files/annotations/examples.md diff --git a/crates/latex/latex-annotation/expect-files/annotations/examples.md b/crates/latex/latex-annotation/expect-files/annotations/examples.md new file mode 100644 index 0000000000..3a53e75861 --- /dev/null +++ b/crates/latex/latex-annotation/expect-files/annotations/examples.md @@ -0,0 +1,62 @@ +[ + LxAnnotationsExample { + root_mode: Math, + input: "", + annotations: LxAnnotations { + token_annotations: [], + space_annotations: [], + }, + }, + LxAnnotationsExample { + root_mode: Math, + input: "xy", + annotations: LxAnnotations { + token_annotations: [ + LxAnnotationEntry { + start: 0, + end: 1, + annotation: Variable( + Usage, + ), + }, + LxAnnotationEntry { + start: 1, + end: 2, + annotation: Variable( + Usage, + ), + }, + ], + space_annotations: [ + LxAnnotationEntry { + start: 1, + end: 2, + annotation: Apply( + ScalarMul, + ), + }, + ], + }, + }, + LxAnnotationsExample { + root_mode: Math, + input: "dx", + annotations: LxAnnotations { + token_annotations: [ + LxAnnotationEntry { + start: 0, + end: 1, + annotation: Differential, + }, + LxAnnotationEntry { + start: 1, + end: 2, + annotation: Variable( + SingleVariableIntegralVariableDecl, + ), + }, + ], + space_annotations: [], + }, + }, +] \ No newline at end of file diff --git a/crates/latex/latex-annotation/src/annotations/builder/sparce.rs b/crates/latex/latex-annotation/src/annotations/builder/sparce.rs index 05883a5b69..4f5f1a662a 100644 --- a/crates/latex/latex-annotation/src/annotations/builder/sparce.rs +++ b/crates/latex/latex-annotation/src/annotations/builder/sparce.rs @@ -125,13 +125,6 @@ mod tests { }, ], space_annotations: [ - LxAnnotationEntry { - start: 5, - end: 6, - annotation: Apply( - Integration, - ), - }, LxAnnotationEntry { start: 6, end: 7, @@ -139,13 +132,6 @@ mod tests { ScalarDifferentialFormMul, ), }, - LxAnnotationEntry { - start: 7, - end: 8, - annotation: Apply( - Differentiation, - ), - }, ], } "#]] diff --git a/crates/latex/latex-annotation/src/annotations/walker.rs b/crates/latex/latex-annotation/src/annotations/walker.rs index ba8991f6b3..96254fc511 100644 --- a/crates/latex/latex-annotation/src/annotations/walker.rs +++ b/crates/latex/latex-annotation/src/annotations/walker.rs @@ -69,7 +69,7 @@ fn next_annotation_aux( *next_token_annotation_index += 1; result } else { - panic!() + panic!("Invalid annotation sequence: annotation start ({start1}) is before token start ({start})"); } } @@ -133,6 +133,15 @@ mod tests { ) ); + // Test at offset 5 ("x" after "\\int ") + assert_eq!( + walker.next(5, 6), + ( + LxTokenAnnotation::Variable(LxVariableAnnotation::Usage), + LxSpaceAnnotation::Void + ) + ); + // Test at offset 6 ("d" after "x") assert_eq!( walker.next(6, 7), @@ -141,12 +150,6 @@ mod tests { LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul) ) ); - - // Test at offset 8 (end of input) - assert_eq!( - walker.next(8, 9), - (LxTokenAnnotation::default(), LxSpaceAnnotation::default()) - ); } // Additional tests for next_token_annotation and next_space_annotation can be added here diff --git a/crates/latex/latex-annotation/src/test_helpers/example.rs b/crates/latex/latex-annotation/src/test_helpers/example.rs index ad5a968012..6bc1109311 100644 --- a/crates/latex/latex-annotation/src/test_helpers/example.rs +++ b/crates/latex/latex-annotation/src/test_helpers/example.rs @@ -6,6 +6,7 @@ use crate::{ annotations::LxAnnotations, }; +#[derive(Debug)] pub struct LxAnnotationsExample { pub root_mode: LxMode, pub input: String, @@ -84,4 +85,6 @@ lazy_static! { #[test] fn latex_annotations_examples_works() { let examples = &*EXAMPLES; // Dereference the lazy_static + expect_test::expect_file!["../../expect-files/annotations/examples.md"] + .assert_eq(&format!("{:#?}", examples)); } diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 7842cea1c1..86e6539514 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -3554,9 +3554,18 @@ dependencies: [ "enum-class", "husky-print-utils", + "latex-prelude", + "lazy_static", "expect-test", ], }, + PackageSummary { + name: "latex-prelude", + relative_path: "crates/latex/latex-prelude", + dependencies: [ + "salsa", + ], + }, PackageSummary { name: "latex-math-letter", relative_path: "crates/latex/latex-math-letter", @@ -3567,13 +3576,6 @@ relative_path: "crates/latex/latex-math-opr", dependencies: [], }, - PackageSummary { - name: "latex-prelude", - relative_path: "crates/latex/latex-prelude", - dependencies: [ - "salsa", - ], - }, PackageSummary { name: "latex-token", relative_path: "crates/latex/latex-token", From a85ffed267b4d615b51869f16e0896fb42b14354 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 21:56:58 -0700 Subject: [PATCH 08/71] save --- Cargo.lock | 1 + .../latex/latex-annotation/src/annotations.rs | 4 +- .../src/test_helpers/example.rs | 77 +++++++-------- crates/latex/latex-ast/Cargo.toml | 3 + .../latex/latex-ast/expect-files/examples.txt | 98 +++++++++++++++++++ crates/latex/latex-ast/src/lib.rs | 2 + crates/latex/latex-ast/src/test_helpers.rs | 1 + .../latex-ast/src/test_helpers/example.rs | 44 +++++++++ 8 files changed, 189 insertions(+), 41 deletions(-) create mode 100644 crates/latex/latex-ast/expect-files/examples.txt create mode 100644 crates/latex/latex-ast/src/test_helpers.rs create mode 100644 crates/latex/latex-ast/src/test_helpers/example.rs diff --git a/Cargo.lock b/Cargo.lock index 23778db978..fde0301613 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8048,6 +8048,7 @@ dependencies = [ "husky-coword", "idx-arena", "latex-annotation", + "latex-ast", "latex-math-letter", "latex-math-opr", "latex-prelude", diff --git a/crates/latex/latex-annotation/src/annotations.rs b/crates/latex/latex-annotation/src/annotations.rs index 56fc6d9414..5a0b82e206 100644 --- a/crates/latex/latex-annotation/src/annotations.rs +++ b/crates/latex/latex-annotation/src/annotations.rs @@ -5,7 +5,7 @@ use crate::annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}; use builder::sparce::collect_from_sparse_annotations; use walker::LxAnnotationsWalker; -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct LxAnnotationEntry { pub start: usize, pub end: usize, @@ -15,7 +15,7 @@ pub struct LxAnnotationEntry { pub type LxTokenAnnotationEntry = LxAnnotationEntry; pub type LxSpaceAnnotationEntry = LxAnnotationEntry; -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct LxAnnotations { token_annotations: Vec, space_annotations: Vec, diff --git a/crates/latex/latex-annotation/src/test_helpers/example.rs b/crates/latex/latex-annotation/src/test_helpers/example.rs index 6bc1109311..181aa820c5 100644 --- a/crates/latex/latex-annotation/src/test_helpers/example.rs +++ b/crates/latex/latex-annotation/src/test_helpers/example.rs @@ -6,15 +6,13 @@ use crate::{ annotations::LxAnnotations, }; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct LxAnnotationsExample { pub root_mode: LxMode, pub input: String, pub annotations: LxAnnotations, } -pub type LxAnnotationsExamples = Vec; - impl LxAnnotationsExample { fn collect_from_sparse( examples: &[( @@ -23,7 +21,7 @@ impl LxAnnotationsExample { &[((&str, &str), LxTokenAnnotation)], &[((&str, &str), LxSpaceAnnotation)], )], - ) -> LxAnnotationsExamples { + ) -> Vec { examples .iter() .map( @@ -45,46 +43,47 @@ use latex_prelude::mode::LxMode; use lazy_static::lazy_static; lazy_static! { - pub static ref EXAMPLES: LxAnnotationsExamples = LxAnnotationsExample::collect_from_sparse(&[ - (LxMode::Math, "", &[], &[]), - ( - LxMode::Math, - "xy", - &[ - ( - ("", "x"), - LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) - ), - ( + pub static ref LX_ANNOTATIONS_EXAMPLES: Vec = + LxAnnotationsExample::collect_from_sparse(&[ + (LxMode::Math, "", &[], &[]), + ( + LxMode::Math, + "xy", + &[ + ( + ("", "x"), + LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) + ), + ( + ("x", "y"), + LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) + ), + ], + &[( ("x", "y"), - LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) - ), - ], - &[( - ("x", "y"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarMul) - ),] - ), - ( - LxMode::Math, - "dx", - &[ - (("", "d"), LxTokenAnnotation::Differential), - ( - ("d", "x"), - LxTokenAnnotation::Variable( - LxVariableAnnotation::SingleVariableIntegralVariableDecl - ) - ), - ], - &[] - ), - ]); + LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarMul) + ),] + ), + ( + LxMode::Math, + "dx", + &[ + (("", "d"), LxTokenAnnotation::Differential), + ( + ("d", "x"), + LxTokenAnnotation::Variable( + LxVariableAnnotation::SingleVariableIntegralVariableDecl + ) + ), + ], + &[] + ), + ]); } #[test] fn latex_annotations_examples_works() { - let examples = &*EXAMPLES; // Dereference the lazy_static + let examples = &*LX_ANNOTATIONS_EXAMPLES; // Dereference the lazy_static expect_test::expect_file!["../../expect-files/annotations/examples.md"] .assert_eq(&format!("{:#?}", examples)); } diff --git a/crates/latex/latex-ast/Cargo.toml b/crates/latex/latex-ast/Cargo.toml index 53ad5620ce..76fe1e3228 100644 --- a/crates/latex/latex-ast/Cargo.toml +++ b/crates/latex/latex-ast/Cargo.toml @@ -29,9 +29,12 @@ latex-annotation.workspace = true [dev-dependencies] expect-test.workspace = true +# latex +latex-ast = { workspace = true, features = ["test_helpers"] } [lints] workspace = true [features] egui = ["dep:egui"] +test_helpers = ["latex-annotation/test_helpers"] diff --git a/crates/latex/latex-ast/expect-files/examples.txt b/crates/latex/latex-ast/expect-files/examples.txt new file mode 100644 index 0000000000..9ba9de6ce1 --- /dev/null +++ b/crates/latex/latex-ast/expect-files/examples.txt @@ -0,0 +1,98 @@ +[ + LatexAstsExample { + annotations: LxAnnotationsExample { + root_mode: Math, + input: "", + annotations: LxAnnotations { + token_annotations: [], + space_annotations: [], + }, + }, + ast_arena: Arena { + data: [], + }, + asts: ArenaIdxRange( + 0..0, + ), + }, + LatexAstsExample { + annotations: LxAnnotationsExample { + root_mode: Math, + input: "xy", + annotations: LxAnnotations { + token_annotations: [ + LxAnnotationEntry { + start: 0, + end: 1, + annotation: Variable( + Usage, + ), + }, + LxAnnotationEntry { + start: 1, + end: 2, + annotation: Variable( + Usage, + ), + }, + ], + space_annotations: [ + LxAnnotationEntry { + start: 1, + end: 2, + annotation: Apply( + ScalarMul, + ), + }, + ], + }, + }, + ast_arena: Arena { + data: [ + Math( + Letter( + LowerX, + ), + ), + ], + }, + asts: ArenaIdxRange( + 0..1, + ), + }, + LatexAstsExample { + annotations: LxAnnotationsExample { + root_mode: Math, + input: "dx", + annotations: LxAnnotations { + token_annotations: [ + LxAnnotationEntry { + start: 0, + end: 1, + annotation: Differential, + }, + LxAnnotationEntry { + start: 1, + end: 2, + annotation: Variable( + SingleVariableIntegralVariableDecl, + ), + }, + ], + space_annotations: [], + }, + }, + ast_arena: Arena { + data: [ + Math( + Letter( + LowerD, + ), + ), + ], + }, + asts: ArenaIdxRange( + 0..1, + ), + }, +] diff --git a/crates/latex/latex-ast/src/lib.rs b/crates/latex/latex-ast/src/lib.rs index 8fa925b525..bfa84ffc24 100644 --- a/crates/latex/latex-ast/src/lib.rs +++ b/crates/latex/latex-ast/src/lib.rs @@ -3,6 +3,8 @@ pub mod jar; mod parser; pub mod region; pub mod sheet; +#[cfg(feature = "test_helpers")] +pub mod test_helpers; #[cfg(test)] mod tests; diff --git a/crates/latex/latex-ast/src/test_helpers.rs b/crates/latex/latex-ast/src/test_helpers.rs new file mode 100644 index 0000000000..d4d8a94c88 --- /dev/null +++ b/crates/latex/latex-ast/src/test_helpers.rs @@ -0,0 +1 @@ +pub mod example; diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs new file mode 100644 index 0000000000..1d3b18bd77 --- /dev/null +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -0,0 +1,44 @@ +use crate::ast::{parse_latex_input_into_asts, LxAstArena, LxAstIdxRange}; +use latex_annotation::test_helpers::example::{LxAnnotationsExample, LX_ANNOTATIONS_EXAMPLES}; + +#[derive(Debug)] +pub struct LatexAstsExample { + pub annotations: LxAnnotationsExample, + pub ast_arena: LxAstArena, + pub asts: LxAstIdxRange, +} + +impl LatexAstsExample { + pub fn from_annotations(annotations: LxAnnotationsExample, db: &salsa::Db) -> Self { + let mut ast_arena = LxAstArena::default(); + let asts = parse_latex_input_into_asts( + db, + &annotations.input, + &annotations.annotations, + annotations.root_mode, + &mut ast_arena, + ); + Self { + annotations, + ast_arena, + asts, + } + } +} + +pub fn lx_asts_examples(db: &salsa::Db) -> Vec { + LX_ANNOTATIONS_EXAMPLES + .iter() + .cloned() + .map(|eg| LatexAstsExample::from_annotations(eg, db)) + .collect() +} + +#[test] +fn test_lx_asts_examples() { + use expect_test::expect_file; + + let db = crate::DB::default(); + let examples = lx_asts_examples(&db); + expect_file!["../../expect-files/examples.txt"].assert_debug_eq(&examples); +} From 0bd9fe10e707d256ac81fd343896195e6c2498f7 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 21:57:37 -0700 Subject: [PATCH 09/71] save --- .../expect-files/annotations/{examples.md => examples.txt} | 0 crates/latex/latex-annotation/src/test_helpers/example.rs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename crates/latex/latex-annotation/expect-files/annotations/{examples.md => examples.txt} (100%) diff --git a/crates/latex/latex-annotation/expect-files/annotations/examples.md b/crates/latex/latex-annotation/expect-files/annotations/examples.txt similarity index 100% rename from crates/latex/latex-annotation/expect-files/annotations/examples.md rename to crates/latex/latex-annotation/expect-files/annotations/examples.txt diff --git a/crates/latex/latex-annotation/src/test_helpers/example.rs b/crates/latex/latex-annotation/src/test_helpers/example.rs index 181aa820c5..70954c89e5 100644 --- a/crates/latex/latex-annotation/src/test_helpers/example.rs +++ b/crates/latex/latex-annotation/src/test_helpers/example.rs @@ -84,6 +84,6 @@ lazy_static! { #[test] fn latex_annotations_examples_works() { let examples = &*LX_ANNOTATIONS_EXAMPLES; // Dereference the lazy_static - expect_test::expect_file!["../../expect-files/annotations/examples.md"] + expect_test::expect_file!["../../expect-files/annotations/examples.txt"] .assert_eq(&format!("{:#?}", examples)); } From cd27e1ae30943fbf8595213e78e3ef8906e560a5 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 22:18:36 -0700 Subject: [PATCH 10/71] cursor is great --- crates/latex/latex-ast/src/ast.rs | 8 +++++-- crates/latex/latex-ast/src/parser.rs | 16 +++++++------ crates/latex/latex-token/src/lexer.rs | 17 +++++++++++--- crates/latex/latex-token/src/lib.rs | 3 +++ crates/latex/latex-token/src/storage.rs | 22 +++++++++++++----- crates/latex/latex-token/src/stream.rs | 30 +++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 crates/latex/latex-token/src/stream.rs diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 7b61d2160e..6aa9c97a46 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -13,7 +13,10 @@ use latex_annotation::{ use latex_math_letter::LxMathLetter; use latex_math_opr::LxMathOpr; use latex_prelude::{mode::LxMode, script::LxScriptKind}; -use latex_token::data::{math::LxMathTokenData, rose::LxRoseTokenData, LxTokenData}; +use latex_token::{ + data::{math::LxMathTokenData, rose::LxRoseTokenData, LxTokenData}, + lexer::lex_latex_input, +}; #[enum_class::from_variants] #[salsa::derive_debug_with_db] @@ -34,7 +37,8 @@ pub fn parse_latex_input_into_asts<'a>( mode: LxMode, arena: &'a mut LxAstArena, ) -> LxAstIdxRange { - let mut parser = LxAstParser::new(db, input, annotations, mode, arena); + let tokens = lex_latex_input(input, mode, db); + let mut parser = LxAstParser::new(db, &tokens, annotations, mode, arena); let asts = parser.parse_asts(); asts } diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index 9b070a21e8..f555279ff0 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -7,12 +7,14 @@ use latex_annotation::{ annotations::{walker::LxAnnotationsWalker, LxAnnotations}, }; use latex_prelude::mode::LxMode; -use latex_token::{data::LxTokenData, idx::LxTokenIdx, lexer::LxLexer}; -use std::iter::Peekable; +use latex_token::{ + data::LxTokenData, idx::LxTokenIdx, storage::LxTokenStorage, stream::LxTokenStream, +}; +use std::{borrow::BorrowMut, iter::Peekable}; pub(crate) struct LxAstParser<'a> { db: &'a ::salsa::Db, - lexer: Peekable>, + token_stream: Peekable>, annotations_walker: LxAnnotationsWalker<'a>, arena: &'a mut LxAstArena, } @@ -21,14 +23,14 @@ pub(crate) struct LxAstParser<'a> { impl<'a> LxAstParser<'a> { pub(crate) fn new( db: &'a ::salsa::Db, - input: &'a str, + tokens: &'a LxTokenStorage, annotations: &'a LxAnnotations, mode: LxMode, arena: &'a mut LxAstArena, ) -> Self { Self { db, - lexer: LxLexer::new(db, input, mode).peekable(), + token_stream: tokens.stream().peekable(), annotations_walker: annotations.walker(), arena, } @@ -46,7 +48,7 @@ impl<'a> LxAstParser<'a> { } pub(crate) fn peek_token(&mut self) -> Option { - self.lexer.peek().map(|&(_, _, _, data)| data) + self.token_stream.peek().map(|&(_, _, _, data)| data) } pub(crate) fn next_token( @@ -57,7 +59,7 @@ impl<'a> LxAstParser<'a> { LxTokenAnnotation, LxSpaceAnnotation, )> { - let (token_idx, (start, end), _, token_data) = self.lexer.next()?; + let (token_idx, (start, end), _, token_data) = self.token_stream.next()?; let (token_annotation, space_annotation) = self.annotations_walker.next(start, end); Some((token_idx, token_data, token_annotation, space_annotation)) } diff --git a/crates/latex/latex-token/src/lexer.rs b/crates/latex/latex-token/src/lexer.rs index 801f7c4113..eb7a3c2746 100644 --- a/crates/latex/latex-token/src/lexer.rs +++ b/crates/latex/latex-token/src/lexer.rs @@ -11,7 +11,13 @@ use husky_coword::Coword; use husky_text_protocol::{char::TextCharIter, range::TextRange}; use latex_prelude::mode::LxMode; -pub struct LxLexer<'a> { +pub fn lex_latex_input(input: &str, mode: LxMode, db: &::salsa::Db) -> LxTokenStorage { + let mut lexer = LxLexer::new(db, input, mode); + for _ in &mut lexer {} + lexer.finish() +} + +pub(crate) struct LxLexer<'a> { pub(crate) db: &'a ::salsa::Db, pub(crate) chars: TextCharIter<'a>, pub(crate) mode: LxMode, @@ -20,7 +26,7 @@ pub struct LxLexer<'a> { /// # constructor impl<'a> LxLexer<'a> { - pub fn new(db: &'a ::salsa::Db, input: &'a str, mode: LxMode) -> Self { + pub(crate) fn new(db: &'a ::salsa::Db, input: &'a str, mode: LxMode) -> Self { Self { db, chars: TextCharIter::new(input), @@ -28,6 +34,10 @@ impl<'a> LxLexer<'a> { storage: Default::default(), } } + + pub(crate) fn finish(self) -> LxTokenStorage { + self.storage + } } /// # actions @@ -81,7 +91,8 @@ impl<'a> Iterator for LxLexer<'a> { end: self.chars.current_position(), }; Some(( - self.storage.alloc(range, token_data), + self.storage + .alloc(start_offset, end_offset, range, token_data), (start_offset, end_offset), range, token_data, diff --git a/crates/latex/latex-token/src/lib.rs b/crates/latex/latex-token/src/lib.rs index dad89a7dc3..1d6157b9aa 100644 --- a/crates/latex/latex-token/src/lib.rs +++ b/crates/latex/latex-token/src/lib.rs @@ -4,8 +4,11 @@ pub mod idx; pub mod lexer; pub mod reserved_char; pub mod storage; +pub mod stream; #[cfg(test)] mod tests; +use self::storage::LxTokenStorage; #[cfg(test)] use self::tests::*; +use latex_prelude::mode::LxMode; diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index ec01eafb91..e07648764e 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -1,12 +1,13 @@ use crate::{ data::LxTokenData, idx::{LxTokenIdx, LxTokenIdxRange}, + stream::LxTokenStream, }; use husky_text_protocol::range::TextRange; #[derive(Default)] pub struct LxTokenStorage { - ranged_tokens: Vec<(TextRange, LxTokenData)>, + ranged_tokens: Vec<((usize, usize), TextRange, LxTokenData)>, } /// # getters @@ -15,7 +16,7 @@ impl std::ops::Index for LxTokenStorage { type Output = LxTokenData; fn index(&self, idx: LxTokenIdx) -> &Self::Output { - &self.ranged_tokens[idx.index()].1 + &self.ranged_tokens[idx.index()].2 } } @@ -29,16 +30,27 @@ impl LxTokenStorage { } pub fn token_text_range(&self, token_idx: LxTokenIdx) -> TextRange { - self.ranged_tokens[token_idx.index()].0 + self.ranged_tokens[token_idx.index()].1 + } + + pub fn stream(&self) -> LxTokenStream { + LxTokenStream::new(&self.ranged_tokens) } } /// # actions impl LxTokenStorage { - pub(crate) fn alloc(&mut self, range: TextRange, token_data: LxTokenData) -> LxTokenIdx { + pub(crate) fn alloc( + &mut self, + start_offset: usize, + end_offset: usize, + range: TextRange, + token_data: LxTokenData, + ) -> LxTokenIdx { let idx = LxTokenIdx::from_index(self.ranged_tokens.len()); - self.ranged_tokens.push((range, token_data)); + self.ranged_tokens + .push(((start_offset, end_offset), range, token_data)); idx } } diff --git a/crates/latex/latex-token/src/stream.rs b/crates/latex/latex-token/src/stream.rs new file mode 100644 index 0000000000..2b3b909f37 --- /dev/null +++ b/crates/latex/latex-token/src/stream.rs @@ -0,0 +1,30 @@ +use crate::{data::LxTokenData, idx::LxTokenIdx}; +use husky_text_protocol::range::TextRange; + +pub struct LxTokenStream<'a> { + ranged_tokens: &'a [((usize, usize), TextRange, LxTokenData)], + current_idx: usize, +} + +impl<'a> LxTokenStream<'a> { + pub(crate) fn new(ranged_tokens: &'a [((usize, usize), TextRange, LxTokenData)]) -> Self { + Self { + ranged_tokens, + current_idx: 0, + } + } +} + +impl<'a> Iterator for LxTokenStream<'a> { + type Item = (LxTokenIdx, (usize, usize), TextRange, LxTokenData); + + fn next(&mut self) -> Option { + if self.current_idx >= self.ranged_tokens.len() { + return None; + } + let idx = self.current_idx; + let (offset_range, text_range, data) = self.ranged_tokens[idx]; + self.current_idx += 1; + Some((LxTokenIdx::from_index(idx), offset_range, text_range, data)) + } +} From 25819813212dcc36fbaf8380a0d9519ed0bc2ed9 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 22:26:00 -0700 Subject: [PATCH 11/71] init lx ast range --- crates/latex/latex-ast/src/ast.rs | 4 ++- crates/latex/latex-ast/src/lib.rs | 1 + crates/latex/latex-ast/src/range.rs | 52 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 crates/latex/latex-ast/src/range.rs diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 6aa9c97a46..6d4ea3915f 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -5,7 +5,7 @@ use self::{math::LxMathAstData, rose::LxRoseAstData}; use crate::parser::LxAstParser; #[cfg(test)] use crate::*; -use idx_arena::{Arena, ArenaIdx, ArenaIdxRange}; +use idx_arena::{map::ArenaMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; use latex_annotation::{ annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}, annotations::LxAnnotations, @@ -27,6 +27,8 @@ pub enum LxAstData { } pub type LxAstArena = Arena; +pub type LxAstArenaRef<'a> = ArenaRef<'a, LxAstData>; +pub type LxAstArenaMap = ArenaMap; pub type LxAstIdx = ArenaIdx; pub type LxAstIdxRange = ArenaIdxRange; diff --git a/crates/latex/latex-ast/src/lib.rs b/crates/latex/latex-ast/src/lib.rs index bfa84ffc24..79c5a0d959 100644 --- a/crates/latex/latex-ast/src/lib.rs +++ b/crates/latex/latex-ast/src/lib.rs @@ -1,6 +1,7 @@ pub mod ast; pub mod jar; mod parser; +pub mod range; pub mod region; pub mod sheet; #[cfg(feature = "test_helpers")] diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs new file mode 100644 index 0000000000..9476a9c309 --- /dev/null +++ b/crates/latex/latex-ast/src/range.rs @@ -0,0 +1,52 @@ +use crate::ast::{ + math::LxMathAstData, LxAstArena, LxAstArenaMap, LxAstArenaRef, LxAstData, LxAstIdx, +}; +use latex_token::idx::LxTokenIdxRange; + +pub struct LxAstRangeMap { + data: LxAstArenaMap, +} + +struct LxAstOffsetRangeCalculator<'a> { + db: &'a ::salsa::Db, + ast_arena: LxAstArenaRef<'a>, + data: LxAstArenaMap, +} + +impl<'a> LxAstOffsetRangeCalculator<'a> { + fn new(db: &'a ::salsa::Db, arena: &'a LxAstArena) -> Self { + Self { + db, + ast_arena: arena.as_arena_ref(), + data: LxAstArenaMap::new(arena), + } + } +} + +impl<'a> LxAstOffsetRangeCalculator<'a> { + fn infer_all(&mut self) { + self.ast_arena.indexed_iter().for_each(|(idx, ast)| { + self.data[idx] = self.calc_ast(ast); + }); + } + + fn calc_ast(&self, data: &LxAstData) -> LxTokenIdxRange { + match *data { + LxAstData::Math(ref data) => match data { + LxMathAstData::Letter(lx_math_letter) => todo!(), + LxMathAstData::Opr(lx_math_opr) => todo!(), + LxMathAstData::Digit(lx_math_digit) => todo!(), + LxMathAstData::TextEdit { buffer } => todo!(), + LxMathAstData::Attach { base, scripts } => todo!(), + LxMathAstData::Delimited { + left_delimiter_token_idx, + left_delimiter, + asts, + right_delimiter_token_idx, + right_delimiter, + } => todo!(), + }, + LxAstData::Rose(ref data) => todo!(), + } + } +} From e97e8fff819fa6b39c292c994b7f39274e029e16 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 22:32:50 -0700 Subject: [PATCH 12/71] include ast token idx range map in asts example --- .../latex/latex-ast/expect-files/examples.txt | 29 ++++++++++++++++++ crates/latex/latex-ast/src/ast.rs | 30 +++++++++++++++++++ crates/latex/latex-ast/src/ast/math.rs | 12 ++++---- crates/latex/latex-ast/src/range.rs | 28 +++++++++-------- .../latex-ast/src/test_helpers/example.rs | 8 ++++- crates/latex/latex-token/src/idx.rs | 8 +++++ 6 files changed, 96 insertions(+), 19 deletions(-) diff --git a/crates/latex/latex-ast/expect-files/examples.txt b/crates/latex/latex-ast/expect-files/examples.txt index 9ba9de6ce1..e250507dde 100644 --- a/crates/latex/latex-ast/expect-files/examples.txt +++ b/crates/latex/latex-ast/expect-files/examples.txt @@ -14,6 +14,9 @@ asts: ArenaIdxRange( 0..0, ), + ast_token_idx_range_map: ArenaMap { + data: [], + }, }, LatexAstsExample { annotations: LxAnnotationsExample { @@ -51,6 +54,9 @@ data: [ Math( Letter( + LxTokenIdx( + 0, + ), LowerX, ), ), @@ -59,6 +65,16 @@ asts: ArenaIdxRange( 0..1, ), + ast_token_idx_range_map: ArenaMap { + data: [ + Some( + LxTokenIdxRange { + start: 0, + end: 1, + }, + ), + ], + }, }, LatexAstsExample { annotations: LxAnnotationsExample { @@ -86,6 +102,9 @@ data: [ Math( Letter( + LxTokenIdx( + 0, + ), LowerD, ), ), @@ -94,5 +113,15 @@ asts: ArenaIdxRange( 0..1, ), + ast_token_idx_range_map: ArenaMap { + data: [ + Some( + LxTokenIdxRange { + start: 0, + end: 1, + }, + ), + ], + }, }, ] diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 6d4ea3915f..a7f91489eb 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -189,11 +189,17 @@ fn parse_tex_input_into_asts_works() { data: [ LxAstData::Math( LxMathAstData::Letter( + LxTokenIdx( + 0, + ), LowerX, ), ), LxAstData::Math( LxMathAstData::Opr( + LxTokenIdx( + 1, + ), Add, ), ), @@ -216,11 +222,17 @@ fn parse_tex_input_into_asts_works() { data: [ LxAstData::Math( LxMathAstData::Letter( + LxTokenIdx( + 0, + ), LowerX, ), ), LxAstData::Math( LxMathAstData::Digit( + LxTokenIdx( + 2, + ), Two, ), ), @@ -254,11 +266,17 @@ fn parse_tex_input_into_asts_works() { data: [ LxAstData::Math( LxMathAstData::Letter( + LxTokenIdx( + 0, + ), LowerX, ), ), LxAstData::Math( LxMathAstData::Digit( + LxTokenIdx( + 2, + ), Two, ), ), @@ -292,21 +310,33 @@ fn parse_tex_input_into_asts_works() { data: [ LxAstData::Math( LxMathAstData::Letter( + LxTokenIdx( + 0, + ), LowerX, ), ), LxAstData::Math( LxMathAstData::Letter( + LxTokenIdx( + 3, + ), LowerI, ), ), LxAstData::Math( LxMathAstData::Opr( + LxTokenIdx( + 4, + ), Add, ), ), LxAstData::Math( LxMathAstData::Digit( + LxTokenIdx( + 5, + ), Two, ), ), diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 2e3bf446d5..a97b43d2b1 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -8,9 +8,9 @@ use super::*; #[salsa::derive_debug_with_db] #[derive(Debug, Clone, PartialEq, Eq)] pub enum LxMathAstData { - Letter(LxMathLetter), - Opr(LxMathOpr), - Digit(LxMathDigit), + Letter(LxTokenIdx, LxMathLetter), + Opr(LxTokenIdx, LxMathOpr), + Digit(LxTokenIdx, LxMathDigit), /// not obtained through parsing, but through ui TextEdit { buffer: String, @@ -38,9 +38,9 @@ impl<'a> LxAstParser<'a> { LxMathTokenData::Command(_) => todo!(), LxMathTokenData::LeftDelimiter(delimiter) => self.parse_delimited(idx, delimiter), LxMathTokenData::RightDelimiter(_) => todo!(), - LxMathTokenData::Letter(letter) => LxMathAstData::Letter(letter), - LxMathTokenData::Opr(opr) => LxMathAstData::Opr(opr), // it's not constructed into a tree yet in the ast stage - LxMathTokenData::Digit(digit) => LxMathAstData::Digit(digit), + LxMathTokenData::Letter(letter) => LxMathAstData::Letter(idx, letter), + LxMathTokenData::Opr(opr) => LxMathAstData::Opr(idx, opr), // it's not constructed into a tree yet in the ast stage + LxMathTokenData::Digit(digit) => LxMathAstData::Digit(idx, digit), LxMathTokenData::Other(_) => todo!(), LxMathTokenData::Subscript => todo!(), LxMathTokenData::Superscript => todo!(), diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index 9476a9c309..b03fc001c3 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -3,17 +3,21 @@ use crate::ast::{ }; use latex_token::idx::LxTokenIdxRange; -pub struct LxAstRangeMap { - data: LxAstArenaMap, +pub type LxAstTokenIdxRangeMap = LxAstArenaMap; + +pub fn calc_ast_token_idx_range_map(db: &salsa::Db, arena: &LxAstArena) -> LxAstTokenIdxRangeMap { + let mut calculator = LxAstTokenIdxRangeCalculator::new(db, arena); + calculator.infer_all(); + calculator.data } -struct LxAstOffsetRangeCalculator<'a> { +struct LxAstTokenIdxRangeCalculator<'a> { db: &'a ::salsa::Db, ast_arena: LxAstArenaRef<'a>, data: LxAstArenaMap, } -impl<'a> LxAstOffsetRangeCalculator<'a> { +impl<'a> LxAstTokenIdxRangeCalculator<'a> { fn new(db: &'a ::salsa::Db, arena: &'a LxAstArena) -> Self { Self { db, @@ -23,21 +27,21 @@ impl<'a> LxAstOffsetRangeCalculator<'a> { } } -impl<'a> LxAstOffsetRangeCalculator<'a> { +impl<'a> LxAstTokenIdxRangeCalculator<'a> { fn infer_all(&mut self) { self.ast_arena.indexed_iter().for_each(|(idx, ast)| { - self.data[idx] = self.calc_ast(ast); + self.data.insert_new(idx, self.calc_ast(ast)); }); } fn calc_ast(&self, data: &LxAstData) -> LxTokenIdxRange { match *data { - LxAstData::Math(ref data) => match data { - LxMathAstData::Letter(lx_math_letter) => todo!(), - LxMathAstData::Opr(lx_math_opr) => todo!(), - LxMathAstData::Digit(lx_math_digit) => todo!(), - LxMathAstData::TextEdit { buffer } => todo!(), - LxMathAstData::Attach { base, scripts } => todo!(), + LxAstData::Math(ref data) => match *data { + LxMathAstData::Letter(idx, _) => LxTokenIdxRange::new_single(idx), + LxMathAstData::Opr(idx, _) => LxTokenIdxRange::new_single(idx), + LxMathAstData::Digit(idx, _) => LxTokenIdxRange::new_single(idx), + LxMathAstData::TextEdit { ref buffer } => todo!(), + LxMathAstData::Attach { base, ref scripts } => todo!(), LxMathAstData::Delimited { left_delimiter_token_idx, left_delimiter, diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs index 1d3b18bd77..1bd72b88a6 100644 --- a/crates/latex/latex-ast/src/test_helpers/example.rs +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -1,4 +1,7 @@ -use crate::ast::{parse_latex_input_into_asts, LxAstArena, LxAstIdxRange}; +use crate::{ + ast::{parse_latex_input_into_asts, LxAstArena, LxAstIdxRange}, + range::{calc_ast_token_idx_range_map, LxAstTokenIdxRangeMap}, +}; use latex_annotation::test_helpers::example::{LxAnnotationsExample, LX_ANNOTATIONS_EXAMPLES}; #[derive(Debug)] @@ -6,6 +9,7 @@ pub struct LatexAstsExample { pub annotations: LxAnnotationsExample, pub ast_arena: LxAstArena, pub asts: LxAstIdxRange, + pub ast_token_idx_range_map: LxAstTokenIdxRangeMap, } impl LatexAstsExample { @@ -18,10 +22,12 @@ impl LatexAstsExample { annotations.root_mode, &mut ast_arena, ); + let ast_token_idx_range_map = calc_ast_token_idx_range_map(db, &ast_arena); Self { annotations, ast_arena, asts, + ast_token_idx_range_map, } } } diff --git a/crates/latex/latex-token/src/idx.rs b/crates/latex/latex-token/src/idx.rs index 93ab486248..f38af9a4bf 100644 --- a/crates/latex/latex-token/src/idx.rs +++ b/crates/latex/latex-token/src/idx.rs @@ -41,6 +41,7 @@ impl Step for LxTokenIdx { } } +#[derive(Debug)] pub struct LxTokenIdxRange { start: ShiftedU32, end: ShiftedU32, @@ -71,4 +72,11 @@ impl LxTokenIdxRange { end: range.end.into(), } } + + pub fn new_single(idx: LxTokenIdx) -> Self { + Self { + start: idx.0, + end: (idx + 1).0, + } + } } From 065f71a9c2ec28e8040a8a70daf45d7cda0d3b95 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Tue, 29 Oct 2024 23:34:14 -0700 Subject: [PATCH 13/71] towards show latex asts --- Cargo.lock | 314 ++++++++++++++++-- Cargo.toml | 1 + crates/latex/latex-ast/Cargo.toml | 1 + crates/latex/latex-ast/src/helpers.rs | 1 + crates/latex/latex-ast/src/helpers/show.rs | 1 + .../src/helpers/show/display_tree.rs | 44 +++ crates/latex/latex-ast/src/lib.rs | 1 + .../expect-files/husky_lang_packages.txt | 1 + 8 files changed, 339 insertions(+), 25 deletions(-) create mode 100644 crates/latex/latex-ast/src/helpers.rs create mode 100644 crates/latex/latex-ast/src/helpers/show.rs create mode 100644 crates/latex/latex-ast/src/helpers/show/display_tree.rs diff --git a/Cargo.lock b/Cargo.lock index fde0301613..b4baf9777a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,6 +245,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anstream" version = "0.6.15" @@ -746,7 +755,7 @@ checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", - "base64", + "base64 0.22.1", "bytes", "futures-util", "http", @@ -817,6 +826,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -844,7 +859,7 @@ dependencies = [ "clang-sys", "clap 3.2.25", "env_logger 0.9.3", - "lazy_static", + "lazy_static 1.5.0", "lazycell", "log", "peeking_take_while", @@ -888,6 +903,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "bitmaps" @@ -1062,7 +1080,7 @@ dependencies = [ "anstream", "anstyle", "anyhow", - "base64", + "base64 0.22.1", "bytesize", "cargo-credential", "cargo-credential-libsecret", @@ -1546,7 +1564,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" dependencies = [ - "lazy_static", + "lazy_static 1.5.0", "windows-sys 0.48.0", ] @@ -1604,6 +1622,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "config" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" +dependencies = [ + "async-trait", + "convert_case", + "json5", + "nom", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml", + "yaml-rust2 0.8.1", +] + [[package]] name = "console" version = "0.15.8" @@ -1611,7 +1648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", - "lazy_static", + "lazy_static 1.5.0", "libc", "unicode-width", "windows-sys 0.52.0", @@ -1623,6 +1660,26 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + [[package]] name = "const_format" version = "0.2.32" @@ -1917,7 +1974,7 @@ dependencies = [ "husky-fmt-utils", "husky-print-utils", "husky-wild-utils", - "lazy_static", + "lazy_static 1.5.0", "paste", "shifted-unsigned-int", ] @@ -1944,7 +2001,7 @@ dependencies = [ "husky-text-protocol", "husky-wild-utils", "indexmap 2.4.0", - "lazy_static", + "lazy_static 1.5.0", "ordered-float 4.2.2", "rmp-serde", "serde", @@ -1978,7 +2035,7 @@ dependencies = [ "husky-rng-utils", "husky-standard-linket-impl", "husky-standard-value", - "lazy_static", + "lazy_static 1.5.0", "ml-task", "serde", "shifted-unsigned-int", @@ -2130,13 +2187,22 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys 0.4.1", +] + [[package]] name = "dirs" version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" dependencies = [ - "dirs-sys", + "dirs-sys 0.3.7", ] [[package]] @@ -2145,7 +2211,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ - "dirs-sys", + "dirs-sys 0.3.7", ] [[package]] @@ -2159,6 +2225,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dispatch" version = "0.2.0" @@ -2191,6 +2269,15 @@ dependencies = [ "libloading", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "document-features" version = "0.2.10" @@ -2757,6 +2844,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.0.33" @@ -3783,7 +3876,7 @@ version = "0.42.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "421dcccab01b41a15d97b226ad97a8f9262295044e34fbd37b10e493b0a6481f" dependencies = [ - "base64", + "base64 0.22.1", "bstr", "curl", "gix-command", @@ -4075,6 +4168,15 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -4526,7 +4628,7 @@ version = "0.1.0" dependencies = [ "serde", "toml", - "yaml-rust2", + "yaml-rust2 0.9.0", ] [[package]] @@ -7491,10 +7593,10 @@ version = "0.1.0" name = "husky-yaml-utils" version = "0.1.0" dependencies = [ - "hashlink", + "hashlink 0.9.1", "serde", "vec-like", - "yaml-rust2", + "yaml-rust2 0.9.0", ] [[package]] @@ -7956,6 +8058,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "khronos-egl" version = "6.0.0" @@ -8035,7 +8148,7 @@ dependencies = [ "husky-print-utils", "latex-annotation", "latex-prelude", - "lazy_static", + "lazy_static 1.5.0", ] [[package]] @@ -8053,6 +8166,7 @@ dependencies = [ "latex-math-opr", "latex-prelude", "latex-token", + "ptree", "salsa", "time-capsule", ] @@ -8124,6 +8238,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "lazy_static" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" + [[package]] name = "lazy_static" version = "1.5.0" @@ -8523,7 +8643,7 @@ dependencies = [ "toml", "vec-like", "walkdir", - "yaml-rust2", + "yaml-rust2 0.9.0", ] [[package]] @@ -8715,7 +8835,7 @@ dependencies = [ "husky-rng-utils", "husky-standard-linket-impl", "husky-standard-value", - "lazy_static", + "lazy_static 1.5.0", "ml-task", "serde", "shifted-unsigned-int", @@ -8851,7 +8971,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668b3abeae3e0637740340e0e32a9bf9308380e146ea6797950f9ff16e88d88a" dependencies = [ - "lazy_static", + "lazy_static 1.5.0", "ndarray", "num-traits", "regex", @@ -9468,6 +9588,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "optional" version = "0.5.0" @@ -9503,6 +9629,16 @@ dependencies = [ "serde", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -9693,6 +9829,61 @@ dependencies = [ "libc", ] +[[package]] +name = "pest" +version = "2.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.76", +] + +[[package]] +name = "pest_meta" +version = "2.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.4.0", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -9942,6 +10133,22 @@ dependencies = [ "cc", ] +[[package]] +name = "ptree" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709c3b241d6a6ccc1933b1c6d7d997fae2b3dff8981f6780eac67df03c32f3ef" +dependencies = [ + "ansi_term", + "atty", + "config", + "directories", + "petgraph", + "serde", + "serde-value", + "tint", +] + [[package]] name = "pyo3" version = "0.22.3" @@ -10334,6 +10541,18 @@ dependencies = [ name = "rocq-hir-expr" version = "0.1.0" +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags 2.6.0", + "serde", + "serde_derive", +] + [[package]] name = "rowan" version = "0.15.16" @@ -10355,11 +10574,21 @@ dependencies = [ "bitflags 2.6.0", "fallible-iterator", "fallible-streaming-iterator", - "hashlink", + "hashlink 0.9.1", "libsqlite3-sys", "smallvec", ] +[[package]] +name = "rust-ini" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rust-optimal-transport" version = "0.2.0" @@ -10626,7 +10855,7 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "base64", + "base64 0.22.1", "rustls-pki-types", ] @@ -10668,7 +10897,7 @@ dependencies = [ "either", "enum-index", "expect-test", - "hashlink", + "hashlink 0.9.1", "husky-salsa-log-utils", "indexmap 2.4.0", "log", @@ -10947,7 +11176,7 @@ version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", @@ -11005,7 +11234,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ - "lazy_static", + "lazy_static 1.5.0", ] [[package]] @@ -11563,6 +11792,24 @@ dependencies = [ "time-core", ] +[[package]] +name = "tint" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af24570664a3074673dbbf69a65bdae0ae0b72f2949b1adfbacb736ee4d6896" +dependencies = [ + "lazy_static 0.2.11", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tiny-skia" version = "0.11.4" @@ -11993,6 +12240,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "uds_windows" version = "1.1.0" @@ -12144,7 +12397,7 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" dependencies = [ - "base64", + "base64 0.22.1", "flate2", "log", "native-tls", @@ -13226,6 +13479,17 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "539a77ee7c0de333dcc6da69b177380a0b81e0dacfa4f7344c465a36871ee601" +[[package]] +name = "yaml-rust2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink 0.8.4", +] + [[package]] name = "yaml-rust2" version = "0.9.0" @@ -13234,7 +13498,7 @@ checksum = "2a1a1c0bc9823338a3bdf8c61f994f23ac004c6fa32c08cd152984499b445e8d" dependencies = [ "arraydeque", "encoding_rs", - "hashlink", + "hashlink 0.9.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e3d161ef15..1bc61be820 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,6 +152,7 @@ parking_lot = "0.12.1" paste = "1.0.6" pathdiff = "0.2.1" proc-macro2 = "1.0.86" +ptree = "0.5.0" pyo3 = "0.22.3" quote = "1.0.37" rand = "0.8.4" diff --git a/crates/latex/latex-ast/Cargo.toml b/crates/latex/latex-ast/Cargo.toml index 76fe1e3228..e076ccf831 100644 --- a/crates/latex/latex-ast/Cargo.toml +++ b/crates/latex/latex-ast/Cargo.toml @@ -16,6 +16,7 @@ egui = { workspace = true, optional = true } # abstractions enum-class.workspace = true idx-arena.workspace = true +ptree.workspace = true salsa.workspace = true time-capsule.workspace = true # kernel diff --git a/crates/latex/latex-ast/src/helpers.rs b/crates/latex/latex-ast/src/helpers.rs new file mode 100644 index 0000000000..78775b6ff2 --- /dev/null +++ b/crates/latex/latex-ast/src/helpers.rs @@ -0,0 +1 @@ +pub mod show; diff --git a/crates/latex/latex-ast/src/helpers/show.rs b/crates/latex/latex-ast/src/helpers/show.rs new file mode 100644 index 0000000000..e04a65fb33 --- /dev/null +++ b/crates/latex/latex-ast/src/helpers/show.rs @@ -0,0 +1 @@ +pub mod display_tree; diff --git a/crates/latex/latex-ast/src/helpers/show/display_tree.rs b/crates/latex/latex-ast/src/helpers/show/display_tree.rs new file mode 100644 index 0000000000..d7698bc316 --- /dev/null +++ b/crates/latex/latex-ast/src/helpers/show/display_tree.rs @@ -0,0 +1,44 @@ +use crate::{ + ast::{LxAstArenaRef, LxAstIdx, LxAstIdxRange}, + range::LxAstTokenIdxRangeMap, +}; +use ptree::TreeBuilder; + +struct LxAstDisplayTreeBuilder<'a> { + db: &'a salsa::Db, + input: &'a str, + ast_arena: LxAstArenaRef<'a>, + ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, + tree_builder: TreeBuilder, +} + +/// # construction +impl<'a> LxAstDisplayTreeBuilder<'a> { + fn new( + db: &'a salsa::Db, + input: &'a str, + ast_arena: LxAstArenaRef<'a>, + ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, + ) -> Self { + Self { + db, + input, + ast_arena, + ast_token_idx_range_map, + tree_builder: TreeBuilder::new(input.to_string()), + } + } +} + +/// # actions +impl<'a> LxAstDisplayTreeBuilder<'a> { + fn render_asts(&mut self, asts: LxAstIdxRange) { + for ast in asts { + self.render_ast(ast); + } + } + + fn render_ast(&mut self, ast: LxAstIdx) { + todo!() + } +} diff --git a/crates/latex/latex-ast/src/lib.rs b/crates/latex/latex-ast/src/lib.rs index 79c5a0d959..12cc82d638 100644 --- a/crates/latex/latex-ast/src/lib.rs +++ b/crates/latex/latex-ast/src/lib.rs @@ -1,4 +1,5 @@ pub mod ast; +pub mod helpers; pub mod jar; mod parser; pub mod range; diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 86e6539514..08d3f046f8 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -3543,6 +3543,7 @@ "latex-math-opr", "latex-prelude", "latex-token", + "ptree", "salsa", "time-capsule", "expect-test", From 871e584abd878dbc85cfc276a976a9a8eb5444f8 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 00:25:30 -0700 Subject: [PATCH 14/71] refactor latex ast --- .../latex/latex-ast/expect-files/examples.txt | 110 +++++--- crates/latex/latex-ast/src/ast.rs | 244 +++++++++++++----- crates/latex/latex-ast/src/ast/math.rs | 14 +- crates/latex/latex-ast/src/ast/rose.rs | 6 + .../src/helpers/show/display_tree.rs | 21 +- crates/latex/latex-ast/src/parser.rs | 22 +- crates/latex/latex-ast/src/range.rs | 33 ++- crates/latex/latex-ast/src/sheet.rs | 16 +- .../latex-ast/src/sheet/action/alloc_ast.rs | 8 +- .../latex-ast/src/sheet/action/alloc_asts.rs | 8 +- .../latex-ast/src/sheet/action/text_edit.rs | 15 +- crates/latex/latex-ast/src/sheet/egui.rs | 25 +- .../latex-ast/src/test_helpers/example.rs | 2 +- 13 files changed, 348 insertions(+), 176 deletions(-) diff --git a/crates/latex/latex-ast/expect-files/examples.txt b/crates/latex/latex-ast/expect-files/examples.txt index e250507dde..d765777452 100644 --- a/crates/latex/latex-ast/expect-files/examples.txt +++ b/crates/latex/latex-ast/expect-files/examples.txt @@ -8,14 +8,22 @@ space_annotations: [], }, }, - ast_arena: Arena { - data: [], + ast_arena: LxAstArena { + math: Arena { + data: [], + }, + rose: Arena { + data: [], + }, }, - asts: ArenaIdxRange( - 0..0, - ), - ast_token_idx_range_map: ArenaMap { - data: [], + asts: None, + ast_token_idx_range_map: LxAstArenaMap { + math: ArenaMap { + data: [], + }, + rose: ArenaMap { + data: [], + }, }, }, LatexAstsExample { @@ -50,30 +58,42 @@ ], }, }, - ast_arena: Arena { - data: [ - Math( + ast_arena: LxAstArena { + math: Arena { + data: [ Letter( LxTokenIdx( 0, ), LowerX, ), - ), - ], + ], + }, + rose: Arena { + data: [], + }, }, - asts: ArenaIdxRange( - 0..1, - ), - ast_token_idx_range_map: ArenaMap { - data: [ - Some( - LxTokenIdxRange { - start: 0, - end: 1, - }, + asts: Some( + Math( + ArenaIdxRange( + 0..1, ), - ], + ), + ), + ast_token_idx_range_map: LxAstArenaMap { + math: ArenaMap { + data: [ + Some( + LxTokenIdxRange { + start: 0, + end: 1, + }, + ), + ], + }, + rose: ArenaMap { + data: [], + }, }, }, LatexAstsExample { @@ -98,30 +118,42 @@ space_annotations: [], }, }, - ast_arena: Arena { - data: [ - Math( + ast_arena: LxAstArena { + math: Arena { + data: [ Letter( LxTokenIdx( 0, ), LowerD, ), - ), - ], + ], + }, + rose: Arena { + data: [], + }, }, - asts: ArenaIdxRange( - 0..1, - ), - ast_token_idx_range_map: ArenaMap { - data: [ - Some( - LxTokenIdxRange { - start: 0, - end: 1, - }, + asts: Some( + Math( + ArenaIdxRange( + 0..1, ), - ], + ), + ), + ast_token_idx_range_map: LxAstArenaMap { + math: ArenaMap { + data: [ + Some( + LxTokenIdxRange { + start: 0, + end: 1, + }, + ), + ], + }, + rose: ArenaMap { + data: [], + }, }, }, ] diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index a7f91489eb..22a5873987 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -1,7 +1,10 @@ pub mod math; pub mod rose; -use self::{math::LxMathAstData, rose::LxRoseAstData}; +use self::{ + math::{LxMathAstArena, LxMathAstArenaMap, LxMathAstArenaRef, LxMathAstData}, + rose::{LxRoseAstArena, LxRoseAstArenaMap, LxRoseAstArenaRef, LxRoseAstData}, +}; use crate::parser::LxAstParser; #[cfg(test)] use crate::*; @@ -14,9 +17,15 @@ use latex_math_letter::LxMathLetter; use latex_math_opr::LxMathOpr; use latex_prelude::{mode::LxMode, script::LxScriptKind}; use latex_token::{ - data::{math::LxMathTokenData, rose::LxRoseTokenData, LxTokenData}, + data::{ + math::{LxMathDelimiter, LxMathTokenData}, + rose::LxRoseTokenData, + LxTokenData, + }, lexer::lex_latex_input, }; +use math::{LxMathAstIdx, LxMathAstIdxRange}; +use rose::{LxRoseAstIdx, LxRoseAstIdxRange}; #[enum_class::from_variants] #[salsa::derive_debug_with_db] @@ -26,11 +35,69 @@ pub enum LxAstData { Rose(LxRoseAstData), } -pub type LxAstArena = Arena; -pub type LxAstArenaRef<'a> = ArenaRef<'a, LxAstData>; -pub type LxAstArenaMap = ArenaMap; -pub type LxAstIdx = ArenaIdx; -pub type LxAstIdxRange = ArenaIdxRange; +#[salsa::derive_debug_with_db] +#[derive(Default, Debug)] +pub struct LxAstArena { + pub(crate) math: LxMathAstArena, + pub(crate) rose: LxRoseAstArena, +} +impl LxAstArena { + pub(crate) fn as_arena_ref(&self) -> LxAstArenaRef { + LxAstArenaRef { + math: self.math.as_arena_ref(), + rose: self.rose.as_arena_ref(), + } + } +} + +#[salsa::derive_debug_with_db] +#[derive(Debug, PartialEq, Eq)] +pub struct LxAstArenaRef<'a> { + math: LxMathAstArenaRef<'a>, + rose: LxRoseAstArenaRef<'a>, +} + +impl<'a> LxAstArenaRef<'a> { + pub fn math(&self) -> LxMathAstArenaRef<'a> { + self.math + } + + pub fn rose(&self) -> LxRoseAstArenaRef<'a> { + self.rose + } +} + +#[salsa::derive_debug_with_db] +#[derive(Debug, PartialEq, Eq)] +pub struct LxAstArenaMap { + pub(crate) math: LxMathAstArenaMap, + pub(crate) rose: LxRoseAstArenaMap, +} + +impl LxAstArenaMap { + pub(crate) fn new(arena: &LxAstArena) -> Self { + Self { + math: LxMathAstArenaMap::new(&arena.math), + rose: LxRoseAstArenaMap::new(&arena.rose), + } + } +} + +#[salsa::derive_debug_with_db] +#[enum_class::from_variants] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LxAstIdx { + Math(LxMathAstIdx), + Rose(LxRoseAstIdx), +} + +#[salsa::derive_debug_with_db] +#[enum_class::from_variants] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LxAstIdxRange { + Math(LxMathAstIdxRange), + Rose(LxRoseAstIdxRange), +} pub fn parse_latex_input_into_asts<'a>( db: &'a ::salsa::Db, @@ -38,7 +105,7 @@ pub fn parse_latex_input_into_asts<'a>( annotations: &'a LxAnnotations, mode: LxMode, arena: &'a mut LxAstArena, -) -> LxAstIdxRange { +) -> Option { let tokens = lex_latex_input(input, mode, db); let mut parser = LxAstParser::new(db, &tokens, annotations, mode, arena); let asts = parser.parse_asts(); @@ -46,12 +113,27 @@ pub fn parse_latex_input_into_asts<'a>( } impl<'a> LxAstParser<'a> { - pub(crate) fn parse_asts(&mut self) -> LxAstIdxRange { + pub(crate) fn parse_asts(&mut self) -> Option { + match self.peek_token()? { + LxTokenData::Math(_) => { + let mut asts = vec![]; + while let Some(ast) = self.parse_ast() { + let LxAstData::Math(ast) = ast else { todo!() }; + asts.push(ast) + } + Some(self.alloc_math_asts(asts).into()) + } + LxTokenData::Rose(_) => todo!(), + } + } + + pub(crate) fn parse_math_asts(&mut self) -> LxMathAstIdxRange { let mut asts = vec![]; while let Some(ast) = self.parse_ast() { + let LxAstData::Math(ast) = ast else { todo!() }; asts.push(ast) } - self.alloc_asts(asts) + self.alloc_math_asts(asts) } fn parse_ast(&mut self) -> Option { @@ -64,14 +146,15 @@ impl<'a> LxAstParser<'a> { }; ast = match ast { LxAstData::Math(LxMathAstData::Attach { .. }) => ast, - base => { - let base = self.alloc_ast(base.into()); + LxAstData::Math(base) => { + let base = self.alloc_math_ast(base); LxMathAstData::Attach { base, scripts: Default::default(), } .into() } + _ => todo!(), }; let LxAstData::Math(LxMathAstData::Attach { ref mut scripts, .. @@ -85,7 +168,8 @@ impl<'a> LxAstParser<'a> { _ => todo!(), }; let ast = match self.parse_atomic_ast() { - Some(new_subscript) => self.alloc_ast(new_subscript), + Some(LxAstData::Math(new_subscript)) => self.alloc_math_ast(new_subscript), + Some(LxAstData::Rose(_)) => todo!("err: expected math ast"), None => todo!("err: expected subscript"), }; scripts.push((script_kind, ast)); @@ -153,12 +237,15 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( - Arena { - data: [], + LxAstArena { + math: Arena { + data: [], + }, + rose: Arena { + data: [], + }, }, - ArenaIdxRange( - 0..0, - ), + None, ) "#]], ); @@ -169,11 +256,20 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( - Arena { - data: [], + LxAstArena { + math: Arena { + data: [], + }, + rose: Arena { + data: [], + }, }, - ArenaIdxRange( - 0..0, + Some( + LxAstIdxRange::Math( + ArenaIdxRange( + 0..0, + ), + ), ), ) "#]], @@ -185,28 +281,33 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( - Arena { - data: [ - LxAstData::Math( + LxAstArena { + math: Arena { + data: [ LxMathAstData::Letter( LxTokenIdx( 0, ), LowerX, ), - ), - LxAstData::Math( LxMathAstData::Opr( LxTokenIdx( 1, ), Add, ), - ), - ], + ], + }, + rose: Arena { + data: [], + }, }, - ArenaIdxRange( - 0..2, + Some( + LxAstIdxRange::Math( + ArenaIdxRange( + 0..2, + ), + ), ), ) "#]], @@ -218,25 +319,21 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( - Arena { - data: [ - LxAstData::Math( + LxAstArena { + math: Arena { + data: [ LxMathAstData::Letter( LxTokenIdx( 0, ), LowerX, ), - ), - LxAstData::Math( LxMathAstData::Digit( LxTokenIdx( 2, ), Two, ), - ), - LxAstData::Math( LxMathAstData::Attach { base: 0, scripts: [ @@ -246,11 +343,18 @@ fn parse_tex_input_into_asts_works() { ), ], }, - ), - ], + ], + }, + rose: Arena { + data: [], + }, }, - ArenaIdxRange( - 2..3, + Some( + LxAstIdxRange::Math( + ArenaIdxRange( + 2..3, + ), + ), ), ) "#]], @@ -262,25 +366,21 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( - Arena { - data: [ - LxAstData::Math( + LxAstArena { + math: Arena { + data: [ LxMathAstData::Letter( LxTokenIdx( 0, ), LowerX, ), - ), - LxAstData::Math( LxMathAstData::Digit( LxTokenIdx( 2, ), Two, ), - ), - LxAstData::Math( LxMathAstData::Attach { base: 0, scripts: [ @@ -290,11 +390,18 @@ fn parse_tex_input_into_asts_works() { ), ], }, - ), - ], + ], + }, + rose: Arena { + data: [], + }, }, - ArenaIdxRange( - 2..3, + Some( + LxAstIdxRange::Math( + ArenaIdxRange( + 2..3, + ), + ), ), ) "#]], @@ -306,41 +413,33 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( - Arena { - data: [ - LxAstData::Math( + LxAstArena { + math: Arena { + data: [ LxMathAstData::Letter( LxTokenIdx( 0, ), LowerX, ), - ), - LxAstData::Math( LxMathAstData::Letter( LxTokenIdx( 3, ), LowerI, ), - ), - LxAstData::Math( LxMathAstData::Opr( LxTokenIdx( 4, ), Add, ), - ), - LxAstData::Math( LxMathAstData::Digit( LxTokenIdx( 5, ), Two, ), - ), - LxAstData::Math( LxMathAstData::Delimited { left_delimiter_token_idx: LxTokenIdx( 2, @@ -354,8 +453,6 @@ fn parse_tex_input_into_asts_works() { ), right_delimiter: Curl, }, - ), - LxAstData::Math( LxMathAstData::Attach { base: 0, scripts: [ @@ -365,11 +462,18 @@ fn parse_tex_input_into_asts_works() { ), ], }, - ), - ], + ], + }, + rose: Arena { + data: [], + }, }, - ArenaIdxRange( - 5..6, + Some( + LxAstIdxRange::Math( + ArenaIdxRange( + 5..6, + ), + ), ), ) "#]], diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index a97b43d2b1..8ef46d4f90 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -16,18 +16,24 @@ pub enum LxMathAstData { buffer: String, }, Attach { - base: LxAstIdx, - scripts: Vec<(LxScriptKind, LxAstIdx)>, + base: LxMathAstIdx, + scripts: Vec<(LxScriptKind, LxMathAstIdx)>, }, Delimited { left_delimiter_token_idx: LxTokenIdx, left_delimiter: LxMathDelimiter, - asts: LxAstIdxRange, + asts: LxMathAstIdxRange, right_delimiter_token_idx: LxTokenIdx, right_delimiter: LxMathDelimiter, }, } +pub type LxMathAstArena = Arena; +pub type LxMathAstArenaRef<'a> = ArenaRef<'a, LxMathAstData>; +pub type LxMathAstArenaMap = ArenaMap; +pub type LxMathAstIdx = ArenaIdx; +pub type LxMathAstIdxRange = ArenaIdxRange; + impl<'a> LxAstParser<'a> { pub(super) fn parse_atomic_math_ast( &mut self, @@ -54,7 +60,7 @@ impl<'a> LxAstParser<'a> { left_delimiter_token_idx: LxTokenIdx, left_delimiter: LxMathDelimiter, ) -> LxMathAstData { - let asts = self.parse_asts(); + let asts = self.parse_math_asts(); let Some((idx, token, _, _)) = self.next_token() else { todo!() }; diff --git a/crates/latex/latex-ast/src/ast/rose.rs b/crates/latex/latex-ast/src/ast/rose.rs index fb866e248d..5eeef00f4d 100644 --- a/crates/latex/latex-ast/src/ast/rose.rs +++ b/crates/latex/latex-ast/src/ast/rose.rs @@ -8,6 +8,12 @@ pub enum LxRoseAstData { TextEdit { buffer: String }, } +pub type LxRoseAstArena = Arena; +pub type LxRoseAstArenaRef<'a> = ArenaRef<'a, LxRoseAstData>; +pub type LxRoseAstArenaMap = ArenaMap; +pub type LxRoseAstIdx = ArenaIdx; +pub type LxRoseAstIdxRange = ArenaIdxRange; + impl<'a> LxAstParser<'a> { pub(super) fn parse_atomic_text_ast( &mut self, diff --git a/crates/latex/latex-ast/src/helpers/show/display_tree.rs b/crates/latex/latex-ast/src/helpers/show/display_tree.rs index d7698bc316..d513d1b92e 100644 --- a/crates/latex/latex-ast/src/helpers/show/display_tree.rs +++ b/crates/latex/latex-ast/src/helpers/show/display_tree.rs @@ -1,5 +1,5 @@ use crate::{ - ast::{LxAstArenaRef, LxAstIdx, LxAstIdxRange}, + ast::{math::LxMathAstIdx, rose::LxRoseAstIdx, LxAstArenaRef, LxAstIdx, LxAstIdxRange}, range::LxAstTokenIdxRangeMap, }; use ptree::TreeBuilder; @@ -33,12 +33,25 @@ impl<'a> LxAstDisplayTreeBuilder<'a> { /// # actions impl<'a> LxAstDisplayTreeBuilder<'a> { fn render_asts(&mut self, asts: LxAstIdxRange) { - for ast in asts { - self.render_ast(ast); + match asts { + LxAstIdxRange::Math(range) => { + for ast in range { + self.render_math_ast(ast); + } + } + LxAstIdxRange::Rose(range) => { + for ast in range { + self.render_rose_ast(ast); + } + } } } - fn render_ast(&mut self, ast: LxAstIdx) { + fn render_math_ast(&mut self, ast: LxMathAstIdx) { + todo!() + } + + fn render_rose_ast(&mut self, ast: LxRoseAstIdx) { todo!() } } diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index f555279ff0..812cf5ad8e 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -1,5 +1,9 @@ use crate::{ - ast::{LxAstArena, LxAstData, LxAstIdx, LxAstIdxRange}, + ast::{ + math::{LxMathAstData, LxMathAstIdx, LxMathAstIdxRange}, + rose::{LxRoseAstData, LxRoseAstIdx, LxRoseAstIdxRange}, + LxAstArena, LxAstData, LxAstIdx, LxAstIdxRange, + }, region::LxAstRegionData, }; use latex_annotation::{ @@ -39,12 +43,20 @@ impl<'a> LxAstParser<'a> { /// # actions impl<'a> LxAstParser<'a> { - pub(crate) fn alloc_asts(&mut self, asts: Vec) -> LxAstIdxRange { - self.arena.alloc_batch(asts) + pub(crate) fn alloc_math_asts(&mut self, asts: Vec) -> LxMathAstIdxRange { + self.arena.math.alloc_batch(asts) } - pub(crate) fn alloc_ast(&mut self, ast: LxAstData) -> LxAstIdx { - self.arena.alloc_one(ast) + pub(crate) fn alloc_math_ast(&mut self, ast: LxMathAstData) -> LxMathAstIdx { + self.arena.math.alloc_one(ast) + } + + pub(crate) fn alloc_rose_ast(&mut self, ast: LxRoseAstData) -> LxRoseAstIdx { + self.arena.rose.alloc_one(ast) + } + + pub(crate) fn alloc_rose_asts(&mut self, asts: Vec) -> LxRoseAstIdxRange { + self.arena.rose.alloc_batch(asts) } pub(crate) fn peek_token(&mut self) -> Option { diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index b03fc001c3..e533eb8509 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -29,28 +29,25 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { impl<'a> LxAstTokenIdxRangeCalculator<'a> { fn infer_all(&mut self) { - self.ast_arena.indexed_iter().for_each(|(idx, ast)| { - self.data.insert_new(idx, self.calc_ast(ast)); + self.ast_arena.math().indexed_iter().for_each(|(idx, ast)| { + self.data.math.insert_new(idx, self.calc_math_ast(ast)); }); } - fn calc_ast(&self, data: &LxAstData) -> LxTokenIdxRange { + fn calc_math_ast(&self, data: &LxMathAstData) -> LxTokenIdxRange { match *data { - LxAstData::Math(ref data) => match *data { - LxMathAstData::Letter(idx, _) => LxTokenIdxRange::new_single(idx), - LxMathAstData::Opr(idx, _) => LxTokenIdxRange::new_single(idx), - LxMathAstData::Digit(idx, _) => LxTokenIdxRange::new_single(idx), - LxMathAstData::TextEdit { ref buffer } => todo!(), - LxMathAstData::Attach { base, ref scripts } => todo!(), - LxMathAstData::Delimited { - left_delimiter_token_idx, - left_delimiter, - asts, - right_delimiter_token_idx, - right_delimiter, - } => todo!(), - }, - LxAstData::Rose(ref data) => todo!(), + LxMathAstData::Letter(idx, _) => LxTokenIdxRange::new_single(idx), + LxMathAstData::Opr(idx, _) => LxTokenIdxRange::new_single(idx), + LxMathAstData::Digit(idx, _) => LxTokenIdxRange::new_single(idx), + LxMathAstData::TextEdit { ref buffer } => todo!(), + LxMathAstData::Attach { base, ref scripts } => todo!(), + LxMathAstData::Delimited { + left_delimiter_token_idx, + left_delimiter, + asts, + right_delimiter_token_idx, + right_delimiter, + } => todo!(), } } } diff --git a/crates/latex/latex-ast/src/sheet.rs b/crates/latex/latex-ast/src/sheet.rs index 3dbe94d46e..45cfd84d36 100644 --- a/crates/latex/latex-ast/src/sheet.rs +++ b/crates/latex/latex-ast/src/sheet.rs @@ -4,7 +4,10 @@ pub mod egui; pub mod event; use self::event::*; -use crate::ast::{LxAstArena, LxAstData, LxAstIdx, LxAstIdxRange}; +use crate::ast::{ + math::{LxMathAstData, LxMathAstIdx, LxMathAstIdxRange}, + LxAstArena, LxAstData, LxAstIdx, LxAstIdxRange, +}; use time_capsule::{capsule::TimeCapsule, state::IsTimeCapsuleState}; #[derive(Default)] @@ -15,12 +18,15 @@ pub struct LxAstSheet { pub type LxAstSheetTimeCapsule = TimeCapsule; impl LxAstSheet { - fn alloc_ast(&mut self, ast_data: LxAstData) -> LxAstIdx { - self.arena.alloc_one(ast_data) + fn alloc_math_ast(&mut self, ast_data: LxMathAstData) -> LxMathAstIdx { + self.arena.math.alloc_one(ast_data) } - fn alloc_asts(&mut self, asts: impl IntoIterator) -> LxAstIdxRange { - self.arena.alloc_batch(asts) + fn alloc_math_asts( + &mut self, + asts: impl IntoIterator, + ) -> LxMathAstIdxRange { + self.arena.math.alloc_batch(asts) } pub fn arena(&self) -> &LxAstArena { diff --git a/crates/latex/latex-ast/src/sheet/action/alloc_ast.rs b/crates/latex/latex-ast/src/sheet/action/alloc_ast.rs index 76a1d2a9f8..4bd785613b 100644 --- a/crates/latex/latex-ast/src/sheet/action/alloc_ast.rs +++ b/crates/latex/latex-ast/src/sheet/action/alloc_ast.rs @@ -1,11 +1,11 @@ use super::*; pub(in crate::sheet) struct MathAstAllocAstAction { - ast_data: LxAstData, + ast_data: LxMathAstData, } impl MathAstAllocAstAction { - pub(in crate::sheet) fn new(ast_data: LxAstData) -> Self { + pub(in crate::sheet) fn new(ast_data: LxMathAstData) -> Self { Self { ast_data } } } @@ -13,7 +13,7 @@ impl MathAstAllocAstAction { impl IsTimeCapsuleAction for MathAstAllocAstAction { type Event = MathAstEvent; - type Outcome = LxAstIdx; + type Outcome = LxMathAstIdx; fn add_to_event_buffer(&self, _event: &mut MathAstEventBuffer) { // ignored because sheet is append only, no need to undo @@ -21,6 +21,6 @@ impl IsTimeCapsuleAction for MathAstAllocAstAction { } fn exec(self, sheet: &mut LxAstSheet) -> Self::Outcome { - sheet.alloc_ast(self.ast_data) + sheet.alloc_math_ast(self.ast_data) } } diff --git a/crates/latex/latex-ast/src/sheet/action/alloc_asts.rs b/crates/latex/latex-ast/src/sheet/action/alloc_asts.rs index 0ca7f68ffa..3522d10c53 100644 --- a/crates/latex/latex-ast/src/sheet/action/alloc_asts.rs +++ b/crates/latex/latex-ast/src/sheet/action/alloc_asts.rs @@ -1,11 +1,11 @@ use super::*; pub(in crate::sheet) struct MathAstAllocAstAction { - asts_data: Vec, + asts_data: Vec, } impl MathAstAllocAstAction { - pub(in crate::sheet) fn new(asts_data: Vec) -> Self { + pub(in crate::sheet) fn new(asts_data: Vec) -> Self { Self { asts_data } } } @@ -13,7 +13,7 @@ impl MathAstAllocAstAction { impl IsTimeCapsuleAction for MathAstAllocAstAction { type Event = MathAstEvent; - type Outcome = LxAstIdxRange; + type Outcome = LxMathAstIdxRange; fn add_to_event_buffer(&self, _event: &mut MathAstEventBuffer) { // ignored because sheet is append only, no need to undo @@ -21,6 +21,6 @@ impl IsTimeCapsuleAction for MathAstAllocAstAction { } fn exec(self, sheet: &mut LxAstSheet) -> Self::Outcome { - sheet.alloc_asts(self.asts_data) + sheet.alloc_math_asts(self.asts_data) } } diff --git a/crates/latex/latex-ast/src/sheet/action/text_edit.rs b/crates/latex/latex-ast/src/sheet/action/text_edit.rs index 8969a8322b..43f60cb7fe 100644 --- a/crates/latex/latex-ast/src/sheet/action/text_edit.rs +++ b/crates/latex/latex-ast/src/sheet/action/text_edit.rs @@ -5,7 +5,7 @@ pub(in crate::sheet) struct MathAstTextEditAction where F: FnOnce(&mut String), { - ast_idx: LxAstIdx, + ast_idx: LxMathAstIdx, f: F, } @@ -13,7 +13,7 @@ impl MathAstTextEditAction where F: FnOnce(&mut String), { - pub(in crate::sheet) fn new(ast_idx: LxAstIdx, f: F) -> Self { + pub(in crate::sheet) fn new(ast_idx: LxMathAstIdx, f: F) -> Self { Self { ast_idx, f } } } @@ -32,10 +32,9 @@ where } fn exec(self, sheet: &mut LxAstSheet) -> Self::Outcome { - sheet.arena.update(self.ast_idx, |ast| match ast { - LxAstData::Math(LxMathAstData::TextEdit { ref mut buffer, .. }) => (self.f)(buffer), - LxAstData::Rose(LxRoseAstData::TextEdit { ref mut buffer, .. }) => (self.f)(buffer), - _ => unreachable!("shouldn't use this"), + sheet.arena.math.update(self.ast_idx, |ast| match ast { + LxMathAstData::TextEdit { ref mut buffer, .. } => (self.f)(buffer), + _ => unreachable!(), }) } } @@ -43,7 +42,7 @@ where #[test] fn math_ast_text_edit_action_works() { let mut sheet: LxAstSheet = Default::default(); - let ast_idx = sheet.alloc_ast( + let ast_idx = sheet.alloc_math_ast( LxMathAstData::TextEdit { buffer: "hello,".to_string(), } @@ -51,7 +50,7 @@ fn math_ast_text_edit_action_works() { ); let action = MathAstTextEditAction::new(ast_idx, |s| *s += " world"); action.exec(&mut sheet); - let LxAstData::Math(LxMathAstData::TextEdit { ref buffer, .. }) = sheet.arena[ast_idx] else { + let LxMathAstData::TextEdit { ref buffer, .. } = sheet.arena.math[ast_idx] else { unreachable!() }; assert_eq!(buffer, "hello, world") diff --git a/crates/latex/latex-ast/src/sheet/egui.rs b/crates/latex/latex-ast/src/sheet/egui.rs index aa0390797a..c4c787f51a 100644 --- a/crates/latex/latex-ast/src/sheet/egui.rs +++ b/crates/latex/latex-ast/src/sheet/egui.rs @@ -12,22 +12,19 @@ pub struct LxAstView<'a> { impl<'a> LxAstView<'a> { pub fn ui(self, ui: &mut egui::Ui) { - match self.time_capsule.arena()[self.ast_idx] { - LxAstData::Math(LxMathAstData::TextEdit { .. }) => { - self.time_capsule.add_event(|event_builder| { - event_builder.add_action(MathAstTextEditAction::new(self.ast_idx, |text| { + match self.ast_idx { + LxAstIdx::Math(idx) => match self.time_capsule.arena().math[idx] { + LxMathAstData::TextEdit { .. } => self.time_capsule.add_event(|event_builder| { + event_builder.add_action(MathAstTextEditAction::new(idx, |text| { ui.text_edit_singleline(text); })) - }) - } - LxAstData::Rose(LxRoseAstData::TextEdit { .. }) => { - self.time_capsule.add_event(|event_builder| { - event_builder.add_action(MathAstTextEditAction::new(self.ast_idx, |text| { - ui.text_edit_singleline(text); - })) - }) - } - _ => todo!(), + }), + _ => todo!(), + }, + LxAstIdx::Rose(idx) => match self.time_capsule.arena().rose[idx] { + LxRoseAstData::TextEdit { .. } => todo!(), + _ => todo!(), + }, } } } diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs index 1bd72b88a6..1f0b8267e1 100644 --- a/crates/latex/latex-ast/src/test_helpers/example.rs +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -8,7 +8,7 @@ use latex_annotation::test_helpers::example::{LxAnnotationsExample, LX_ANNOTATIO pub struct LatexAstsExample { pub annotations: LxAnnotationsExample, pub ast_arena: LxAstArena, - pub asts: LxAstIdxRange, + pub asts: Option, pub ast_token_idx_range_map: LxAstTokenIdxRangeMap, } From 1fc37d6d46db703a2153c313875d9503be0690eb Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 00:32:32 -0700 Subject: [PATCH 15/71] save --- crates/latex/latex-ast/src/test_helpers/example.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs index 1f0b8267e1..76b19d03a3 100644 --- a/crates/latex/latex-ast/src/test_helpers/example.rs +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -5,14 +5,14 @@ use crate::{ use latex_annotation::test_helpers::example::{LxAnnotationsExample, LX_ANNOTATIONS_EXAMPLES}; #[derive(Debug)] -pub struct LatexAstsExample { +pub struct LxAstsExample { pub annotations: LxAnnotationsExample, pub ast_arena: LxAstArena, pub asts: Option, pub ast_token_idx_range_map: LxAstTokenIdxRangeMap, } -impl LatexAstsExample { +impl LxAstsExample { pub fn from_annotations(annotations: LxAnnotationsExample, db: &salsa::Db) -> Self { let mut ast_arena = LxAstArena::default(); let asts = parse_latex_input_into_asts( @@ -32,11 +32,11 @@ impl LatexAstsExample { } } -pub fn lx_asts_examples(db: &salsa::Db) -> Vec { +pub fn lx_asts_examples(db: &salsa::Db) -> Vec { LX_ANNOTATIONS_EXAMPLES .iter() .cloned() - .map(|eg| LatexAstsExample::from_annotations(eg, db)) + .map(|eg| LxAstsExample::from_annotations(eg, db)) .collect() } From fe71ae92ae307437e5f71ec86232035c1ebcad48 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 02:13:50 -0700 Subject: [PATCH 16/71] move annotations to visored --- Cargo.lock | 27 ++++--- Cargo.toml | 2 +- crates/latex/latex-ast/Cargo.toml | 3 +- crates/latex/latex-ast/src/ast.rs | 34 ++------ crates/latex/latex-ast/src/ast/math.rs | 2 +- crates/latex/latex-ast/src/parser.rs | 21 +---- .../latex-ast/src/test_helpers/example.rs | 34 ++------ crates/latex/latex-token/src/storage.rs | 2 +- .../expect-files/husky_lang_jar_packages.txt | 16 ++-- .../expect-files/husky_lang_packages.txt | 78 +++++++++---------- .../visored-annotation}/Cargo.toml | 6 +- .../expect-files/annotations/examples.txt | 0 .../visored-annotation}/src/annotation.rs | 0 .../src/annotation/space.rs | 0 .../src/annotation/token.rs | 0 .../visored-annotation}/src/annotations.rs | 0 .../src/annotations/builder.rs | 0 .../src/annotations/builder/sparce.rs | 0 .../src/annotations/walker.rs | 0 .../visored-annotation}/src/lib.rs | 0 .../visored-annotation}/src/test_helpers.rs | 0 .../src/test_helpers/example.rs | 0 crates/visored/visored-sem-expr/Cargo.toml | 2 +- 23 files changed, 82 insertions(+), 145 deletions(-) rename crates/{latex/latex-annotation => visored/visored-annotation}/Cargo.toml (84%) rename crates/{latex/latex-annotation => visored/visored-annotation}/expect-files/annotations/examples.txt (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/annotation.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/annotation/space.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/annotation/token.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/annotations.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/annotations/builder.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/annotations/builder/sparce.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/annotations/walker.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/lib.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/test_helpers.rs (100%) rename crates/{latex/latex-annotation => visored/visored-annotation}/src/test_helpers/example.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index b4baf9777a..47603eefbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8139,18 +8139,6 @@ dependencies = [ "libc", ] -[[package]] -name = "latex-annotation" -version = "0.1.0" -dependencies = [ - "enum-class", - "expect-test", - "husky-print-utils", - "latex-annotation", - "latex-prelude", - "lazy_static 1.5.0", -] - [[package]] name = "latex-ast" version = "0.1.0" @@ -8160,7 +8148,6 @@ dependencies = [ "expect-test", "husky-coword", "idx-arena", - "latex-annotation", "latex-ast", "latex-math-letter", "latex-math-opr", @@ -12506,6 +12493,18 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visored-annotation" +version = "0.1.0" +dependencies = [ + "enum-class", + "expect-test", + "husky-print-utils", + "latex-prelude", + "lazy_static 1.5.0", + "visored-sem-expr", +] + [[package]] name = "visored-coword" version = "0.1.0" @@ -12568,10 +12567,10 @@ version = "0.1.0" dependencies = [ "husky-coword", "idx-arena", - "latex-annotation", "latex-prelude", "latex-syn-expr", "salsa", + "visored-annotation", "visored-opr", "visored-sem-expr", "visored-zfs-ty", diff --git a/Cargo.toml b/Cargo.toml index 1bc61be820..07a92b7a16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -316,7 +316,6 @@ husky-ki = { path = "crates/ki/husky-ki" } husky-ki-repr = { path = "crates/ki/husky-ki-repr" } husky-ki-repr-interface = { path = "crates/ki/husky-ki-repr-interface" } # latex -latex-annotation = { path = "crates/latex/latex-annotation" } latex-ast = { path = "crates/latex/latex-ast" } latex-command = { path = "crates/latex/latex-command" } latex-math-letter = { path = "crates/latex/latex-math-letter" } @@ -428,6 +427,7 @@ husky-standard-value = { path = "crates/value/husky-standard-value" } husky-standard-value-macros = { path = "crates/value/husky-standard-value-macros" } husky-virtual-value = { path = "crates/value/husky-virtual-value" } # visored +visored-annotation = { path = "crates/visored/visored-annotation" } visored-coword = { path = "crates/visored/visored-coword" } visored-hir-expr = { path = "crates/visored/visored-hir-expr" } visored-isabelle-hol-transpilation = { path = "crates/visored/visored-isabelle-hol-transpilation" } diff --git a/crates/latex/latex-ast/Cargo.toml b/crates/latex/latex-ast/Cargo.toml index e076ccf831..28295d081d 100644 --- a/crates/latex/latex-ast/Cargo.toml +++ b/crates/latex/latex-ast/Cargo.toml @@ -26,7 +26,6 @@ latex-math-letter.workspace = true latex-math-opr.workspace = true latex-prelude.workspace = true latex-token.workspace = true -latex-annotation.workspace = true [dev-dependencies] expect-test.workspace = true @@ -38,4 +37,4 @@ workspace = true [features] egui = ["dep:egui"] -test_helpers = ["latex-annotation/test_helpers"] +test_helpers = [] diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 22a5873987..ac5b0cd409 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -9,10 +9,6 @@ use crate::parser::LxAstParser; #[cfg(test)] use crate::*; use idx_arena::{map::ArenaMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; -use latex_annotation::{ - annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}, - annotations::LxAnnotations, -}; use latex_math_letter::LxMathLetter; use latex_math_opr::LxMathOpr; use latex_prelude::{mode::LxMode, script::LxScriptKind}; @@ -102,12 +98,11 @@ pub enum LxAstIdxRange { pub fn parse_latex_input_into_asts<'a>( db: &'a ::salsa::Db, input: &'a str, - annotations: &'a LxAnnotations, mode: LxMode, arena: &'a mut LxAstArena, ) -> Option { let tokens = lex_latex_input(input, mode, db); - let mut parser = LxAstParser::new(db, &tokens, annotations, mode, arena); + let mut parser = LxAstParser::new(db, &tokens, mode, arena); let asts = parser.parse_asts(); asts } @@ -141,7 +136,7 @@ impl<'a> LxAstParser<'a> { match self.peek_token()? { // TODO include more cases, like \limits LxTokenData::Math(LxMathTokenData::Subscript | LxMathTokenData::Superscript) => { - let (idx, LxTokenData::Math(token), _, _) = self.next_token().unwrap() else { + let (idx, LxTokenData::Math(token)) = self.next_token().unwrap() else { unreachable!() }; ast = match ast { @@ -205,7 +200,7 @@ impl<'a> LxAstParser<'a> { LxRoseTokenData::NewParagraph => todo!(), }, } - let (idx, token, token_annotation, space_annotation) = self.next_token().unwrap(); + let (idx, token) = self.next_token().unwrap(); Some(match token { LxTokenData::Math(token) => self.parse_atomic_math_ast(idx, token).into(), LxTokenData::Rose(token) => self.parse_atomic_text_ast(idx, token).into(), @@ -217,23 +212,14 @@ impl<'a> LxAstParser<'a> { fn parse_tex_input_into_asts_works() { use expect_test::Expect; - fn t( - input: &str, - token_annotations: Vec<((&str, &str), LxTokenAnnotation)>, - space_annotations: Vec<((&str, &str), LxSpaceAnnotation)>, - mode: LxMode, - expected: Expect, - ) { + fn t(input: &str, mode: LxMode, expected: Expect) { let db = &DB::default(); let mut arena = LxAstArena::default(); - let annotations = &LxAnnotations::from_sparse(input, token_annotations, space_annotations); - let asts = parse_latex_input_into_asts(db, input, annotations, mode, &mut arena); + let asts = parse_latex_input_into_asts(db, input, mode, &mut arena); expected.assert_debug_eq(&((arena, asts).debug(db))); } t( "", - vec![], - vec![], LxMode::Math, expect![[r#" ( @@ -251,8 +237,6 @@ fn parse_tex_input_into_asts_works() { ); t( "x", - vec![], - vec![], LxMode::Math, expect![[r#" ( @@ -276,8 +260,6 @@ fn parse_tex_input_into_asts_works() { ); t( "x+1", - vec![], - vec![], LxMode::Math, expect![[r#" ( @@ -314,8 +296,6 @@ fn parse_tex_input_into_asts_works() { ); t( "x^2", - vec![], - vec![], LxMode::Math, expect![[r#" ( @@ -361,8 +341,6 @@ fn parse_tex_input_into_asts_works() { ); t( "x_2", - vec![], - vec![], LxMode::Math, expect![[r#" ( @@ -408,8 +386,6 @@ fn parse_tex_input_into_asts_works() { ); t( "x^{i+2}", - vec![], - vec![], LxMode::Math, expect![[r#" ( diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 8ef46d4f90..289bf4ad25 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -61,7 +61,7 @@ impl<'a> LxAstParser<'a> { left_delimiter: LxMathDelimiter, ) -> LxMathAstData { let asts = self.parse_math_asts(); - let Some((idx, token, _, _)) = self.next_token() else { + let Some((idx, token)) = self.next_token() else { todo!() }; match token { diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index 812cf5ad8e..3e544bfdd6 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -6,10 +6,6 @@ use crate::{ }, region::LxAstRegionData, }; -use latex_annotation::{ - annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}, - annotations::{walker::LxAnnotationsWalker, LxAnnotations}, -}; use latex_prelude::mode::LxMode; use latex_token::{ data::LxTokenData, idx::LxTokenIdx, storage::LxTokenStorage, stream::LxTokenStream, @@ -19,7 +15,6 @@ use std::{borrow::BorrowMut, iter::Peekable}; pub(crate) struct LxAstParser<'a> { db: &'a ::salsa::Db, token_stream: Peekable>, - annotations_walker: LxAnnotationsWalker<'a>, arena: &'a mut LxAstArena, } @@ -28,14 +23,12 @@ impl<'a> LxAstParser<'a> { pub(crate) fn new( db: &'a ::salsa::Db, tokens: &'a LxTokenStorage, - annotations: &'a LxAnnotations, mode: LxMode, arena: &'a mut LxAstArena, ) -> Self { Self { db, token_stream: tokens.stream().peekable(), - annotations_walker: annotations.walker(), arena, } } @@ -63,16 +56,8 @@ impl<'a> LxAstParser<'a> { self.token_stream.peek().map(|&(_, _, _, data)| data) } - pub(crate) fn next_token( - &mut self, - ) -> Option<( - LxTokenIdx, - LxTokenData, - LxTokenAnnotation, - LxSpaceAnnotation, - )> { - let (token_idx, (start, end), _, token_data) = self.token_stream.next()?; - let (token_annotation, space_annotation) = self.annotations_walker.next(start, end); - Some((token_idx, token_data, token_annotation, space_annotation)) + pub(crate) fn next_token(&mut self) -> Option<(LxTokenIdx, LxTokenData)> { + let (token_idx, _, _, token_data) = self.token_stream.next()?; + Some((token_idx, token_data)) } } diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs index 76b19d03a3..65286e6913 100644 --- a/crates/latex/latex-ast/src/test_helpers/example.rs +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -2,49 +2,27 @@ use crate::{ ast::{parse_latex_input_into_asts, LxAstArena, LxAstIdxRange}, range::{calc_ast_token_idx_range_map, LxAstTokenIdxRangeMap}, }; -use latex_annotation::test_helpers::example::{LxAnnotationsExample, LX_ANNOTATIONS_EXAMPLES}; +use latex_prelude::mode::LxMode; +use latex_token::storage::LxTokenStorage; #[derive(Debug)] pub struct LxAstsExample { - pub annotations: LxAnnotationsExample, + pub token_storage: LxTokenStorage, pub ast_arena: LxAstArena, pub asts: Option, pub ast_token_idx_range_map: LxAstTokenIdxRangeMap, } impl LxAstsExample { - pub fn from_annotations(annotations: LxAnnotationsExample, db: &salsa::Db) -> Self { + pub fn new(input: &str, root_mode: LxMode, db: &salsa::Db) -> Self { let mut ast_arena = LxAstArena::default(); - let asts = parse_latex_input_into_asts( - db, - &annotations.input, - &annotations.annotations, - annotations.root_mode, - &mut ast_arena, - ); + let asts = parse_latex_input_into_asts(db, input, root_mode, &mut ast_arena); let ast_token_idx_range_map = calc_ast_token_idx_range_map(db, &ast_arena); Self { - annotations, + token_storage: todo!(), ast_arena, asts, ast_token_idx_range_map, } } } - -pub fn lx_asts_examples(db: &salsa::Db) -> Vec { - LX_ANNOTATIONS_EXAMPLES - .iter() - .cloned() - .map(|eg| LxAstsExample::from_annotations(eg, db)) - .collect() -} - -#[test] -fn test_lx_asts_examples() { - use expect_test::expect_file; - - let db = crate::DB::default(); - let examples = lx_asts_examples(&db); - expect_file!["../../expect-files/examples.txt"].assert_debug_eq(&examples); -} diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index e07648764e..5d8253788f 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -5,7 +5,7 @@ use crate::{ }; use husky_text_protocol::range::TextRange; -#[derive(Default)] +#[derive(Default, Debug)] pub struct LxTokenStorage { ranged_tokens: Vec<((usize, usize), TextRange, LxTokenData)>, } diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt index e37728a4b6..57997cce53 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt @@ -1305,13 +1305,11 @@ dependencies: [], }, JarPackageSummary { - name: "visored-hir-expr", + name: "visored-sem-expr", dependencies: [ "visored-opr", - "visored-sem-expr", "visored-zfs-ty", "husky-coword", - "visored-sem-expr", ], }, JarPackageSummary { @@ -1319,17 +1317,19 @@ dependencies: [], }, JarPackageSummary { - name: "visored-sem-expr", + name: "visored-zfs-ty", + dependencies: [], + }, + JarPackageSummary { + name: "visored-hir-expr", dependencies: [ "visored-opr", + "visored-sem-expr", "visored-zfs-ty", "husky-coword", + "visored-sem-expr", ], }, - JarPackageSummary { - name: "visored-zfs-ty", - dependencies: [], - }, JarPackageSummary { name: "visored-lean-transpilation", dependencies: [ diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 08d3f046f8..248ed854e8 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -3538,7 +3538,6 @@ "enum-class", "husky-coword", "idx-arena", - "latex-annotation", "latex-math-letter", "latex-math-opr", "latex-prelude", @@ -3549,24 +3548,6 @@ "expect-test", ], }, - PackageSummary { - name: "latex-annotation", - relative_path: "crates/latex/latex-annotation", - dependencies: [ - "enum-class", - "husky-print-utils", - "latex-prelude", - "lazy_static", - "expect-test", - ], - }, - PackageSummary { - name: "latex-prelude", - relative_path: "crates/latex/latex-prelude", - dependencies: [ - "salsa", - ], - }, PackageSummary { name: "latex-math-letter", relative_path: "crates/latex/latex-math-letter", @@ -3577,6 +3558,13 @@ relative_path: "crates/latex/latex-math-opr", dependencies: [], }, + PackageSummary { + name: "latex-prelude", + relative_path: "crates/latex/latex-prelude", + dependencies: [ + "salsa", + ], + }, PackageSummary { name: "latex-token", relative_path: "crates/latex/latex-token", @@ -4221,23 +4209,29 @@ dependencies: [], }, PackageSummary { - name: "visored-coword", - relative_path: "crates/visored/visored-coword", + name: "visored-annotation", + relative_path: "crates/visored/visored-annotation", dependencies: [ - "husky-coword", + "enum-class", + "husky-print-utils", + "latex-prelude", + "lazy_static", + "expect-test", + "visored-sem-expr", ], }, PackageSummary { - name: "visored-hir-expr", - relative_path: "crates/visored/visored-hir-expr", + name: "visored-sem-expr", + relative_path: "crates/visored/visored-sem-expr", dependencies: [ "idx-arena", + "latex-prelude", + "latex-syn-expr", "salsa", + "visored-annotation", "visored-opr", - "visored-sem-expr", "visored-zfs-ty", "husky-coword", - "visored-sem-expr", ], }, PackageSummary { @@ -4248,27 +4242,33 @@ ], }, PackageSummary { - name: "visored-sem-expr", - relative_path: "crates/visored/visored-sem-expr", + name: "visored-zfs-ty", + relative_path: "crates/visored/visored-zfs-ty", dependencies: [ - "idx-arena", - "latex-annotation", - "latex-prelude", - "latex-syn-expr", + "enum-class", "salsa", - "visored-opr", - "visored-zfs-ty", + "smallvec", + "visored-coword", + ], + }, + PackageSummary { + name: "visored-coword", + relative_path: "crates/visored/visored-coword", + dependencies: [ "husky-coword", ], }, PackageSummary { - name: "visored-zfs-ty", - relative_path: "crates/visored/visored-zfs-ty", + name: "visored-hir-expr", + relative_path: "crates/visored/visored-hir-expr", dependencies: [ - "enum-class", + "idx-arena", "salsa", - "smallvec", - "visored-coword", + "visored-opr", + "visored-sem-expr", + "visored-zfs-ty", + "husky-coword", + "visored-sem-expr", ], }, PackageSummary { diff --git a/crates/latex/latex-annotation/Cargo.toml b/crates/visored/visored-annotation/Cargo.toml similarity index 84% rename from crates/latex/latex-annotation/Cargo.toml rename to crates/visored/visored-annotation/Cargo.toml index f30bbaf8b2..20321a1f84 100644 --- a/crates/latex/latex-annotation/Cargo.toml +++ b/crates/visored/visored-annotation/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "latex-annotation" +name = "visored-annotation" version.workspace = true rust-version.workspace = true authors.workspace = true @@ -22,8 +22,8 @@ husky-print-utils.workspace = true [dev-dependencies] expect-test.workspace = true -# latex -latex-annotation = { workspace = true, features = ["test_helpers"] } +# visored +visored-sem-expr = { workspace = true, features = ["test_helpers"] } [lints] workspace = true diff --git a/crates/latex/latex-annotation/expect-files/annotations/examples.txt b/crates/visored/visored-annotation/expect-files/annotations/examples.txt similarity index 100% rename from crates/latex/latex-annotation/expect-files/annotations/examples.txt rename to crates/visored/visored-annotation/expect-files/annotations/examples.txt diff --git a/crates/latex/latex-annotation/src/annotation.rs b/crates/visored/visored-annotation/src/annotation.rs similarity index 100% rename from crates/latex/latex-annotation/src/annotation.rs rename to crates/visored/visored-annotation/src/annotation.rs diff --git a/crates/latex/latex-annotation/src/annotation/space.rs b/crates/visored/visored-annotation/src/annotation/space.rs similarity index 100% rename from crates/latex/latex-annotation/src/annotation/space.rs rename to crates/visored/visored-annotation/src/annotation/space.rs diff --git a/crates/latex/latex-annotation/src/annotation/token.rs b/crates/visored/visored-annotation/src/annotation/token.rs similarity index 100% rename from crates/latex/latex-annotation/src/annotation/token.rs rename to crates/visored/visored-annotation/src/annotation/token.rs diff --git a/crates/latex/latex-annotation/src/annotations.rs b/crates/visored/visored-annotation/src/annotations.rs similarity index 100% rename from crates/latex/latex-annotation/src/annotations.rs rename to crates/visored/visored-annotation/src/annotations.rs diff --git a/crates/latex/latex-annotation/src/annotations/builder.rs b/crates/visored/visored-annotation/src/annotations/builder.rs similarity index 100% rename from crates/latex/latex-annotation/src/annotations/builder.rs rename to crates/visored/visored-annotation/src/annotations/builder.rs diff --git a/crates/latex/latex-annotation/src/annotations/builder/sparce.rs b/crates/visored/visored-annotation/src/annotations/builder/sparce.rs similarity index 100% rename from crates/latex/latex-annotation/src/annotations/builder/sparce.rs rename to crates/visored/visored-annotation/src/annotations/builder/sparce.rs diff --git a/crates/latex/latex-annotation/src/annotations/walker.rs b/crates/visored/visored-annotation/src/annotations/walker.rs similarity index 100% rename from crates/latex/latex-annotation/src/annotations/walker.rs rename to crates/visored/visored-annotation/src/annotations/walker.rs diff --git a/crates/latex/latex-annotation/src/lib.rs b/crates/visored/visored-annotation/src/lib.rs similarity index 100% rename from crates/latex/latex-annotation/src/lib.rs rename to crates/visored/visored-annotation/src/lib.rs diff --git a/crates/latex/latex-annotation/src/test_helpers.rs b/crates/visored/visored-annotation/src/test_helpers.rs similarity index 100% rename from crates/latex/latex-annotation/src/test_helpers.rs rename to crates/visored/visored-annotation/src/test_helpers.rs diff --git a/crates/latex/latex-annotation/src/test_helpers/example.rs b/crates/visored/visored-annotation/src/test_helpers/example.rs similarity index 100% rename from crates/latex/latex-annotation/src/test_helpers/example.rs rename to crates/visored/visored-annotation/src/test_helpers/example.rs diff --git a/crates/visored/visored-sem-expr/Cargo.toml b/crates/visored/visored-sem-expr/Cargo.toml index 142f3b9941..e0cbb0ceb8 100644 --- a/crates/visored/visored-sem-expr/Cargo.toml +++ b/crates/visored/visored-sem-expr/Cargo.toml @@ -16,10 +16,10 @@ keywords.workspace = true salsa.workspace = true idx-arena.workspace = true # latex -latex-annotation.workspace = true latex-prelude.workspace = true latex-syn-expr.workspace = true # visored +visored-annotation.workspace = true visored-opr.workspace = true visored-zfs-ty.workspace = true From 7f09d6e3ffbec9d5e3b2625bf06dcd0f71232648 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 02:20:20 -0700 Subject: [PATCH 17/71] save --- crates/abstractions/salsa-macros/src/lib.rs | 2 +- crates/latex/latex-ast/src/ast.rs | 233 +++++++++++++++++- crates/latex/latex-ast/src/parser.rs | 14 +- .../latex-ast/src/test_helpers/example.rs | 4 +- crates/latex/latex-token/src/data/math.rs | 3 +- crates/latex/latex-token/src/data/rose.rs | 3 +- crates/latex/latex-token/src/lexer.rs | 23 +- crates/latex/latex-token/src/storage.rs | 1 + 8 files changed, 254 insertions(+), 29 deletions(-) diff --git a/crates/abstractions/salsa-macros/src/lib.rs b/crates/abstractions/salsa-macros/src/lib.rs index 247ba60be1..8d79672bb4 100644 --- a/crates/abstractions/salsa-macros/src/lib.rs +++ b/crates/abstractions/salsa-macros/src/lib.rs @@ -53,7 +53,7 @@ pub fn jar(args: TokenStream, input: TokenStream) -> TokenStream { jar::jar(args, input) } -#[deprecated(note = "try using derive(DebugWithDb) instead")] +// TODO: try using derive(DebugWithDb) instead #[proc_macro_attribute] pub fn derive_debug_with_db(args: TokenStream, input: TokenStream) -> TokenStream { debug_with_db::debug_with_db(args, input) diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index ac5b0cd409..027959ec57 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -18,7 +18,7 @@ use latex_token::{ rose::LxRoseTokenData, LxTokenData, }, - lexer::lex_latex_input, + storage::LxTokenStorage, }; use math::{LxMathAstIdx, LxMathAstIdxRange}; use rose::{LxRoseAstIdx, LxRoseAstIdxRange}; @@ -99,10 +99,10 @@ pub fn parse_latex_input_into_asts<'a>( db: &'a ::salsa::Db, input: &'a str, mode: LxMode, + token_storage: &'a mut LxTokenStorage, arena: &'a mut LxAstArena, ) -> Option { - let tokens = lex_latex_input(input, mode, db); - let mut parser = LxAstParser::new(db, &tokens, mode, arena); + let mut parser = LxAstParser::new(db, input, mode, token_storage, arena); let asts = parser.parse_asts(); asts } @@ -208,6 +208,7 @@ impl<'a> LxAstParser<'a> { } } +// TODO replace it with example #[test] fn parse_tex_input_into_asts_works() { use expect_test::Expect; @@ -215,14 +216,18 @@ fn parse_tex_input_into_asts_works() { fn t(input: &str, mode: LxMode, expected: Expect) { let db = &DB::default(); let mut arena = LxAstArena::default(); - let asts = parse_latex_input_into_asts(db, input, mode, &mut arena); - expected.assert_debug_eq(&((arena, asts).debug(db))); + let mut token_storage = LxTokenStorage::default(); + let asts = parse_latex_input_into_asts(db, input, mode, &mut token_storage, &mut arena); + expected.assert_debug_eq(&((token_storage, arena, asts).debug(db))); } t( "", LxMode::Math, expect![[r#" ( + LxTokenStorage { + ranged_tokens: [], + }, LxAstArena { math: Arena { data: [], @@ -240,6 +245,22 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( + LxTokenStorage { + ranged_tokens: [ + ( + ( + 0, + 1, + ), + [1:1, 1:2), + Math( + Letter( + LowerX, + ), + ), + ), + ], + }, LxAstArena { math: Arena { data: [], @@ -263,6 +284,46 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( + LxTokenStorage { + ranged_tokens: [ + ( + ( + 0, + 1, + ), + [1:1, 1:2), + Math( + Letter( + LowerX, + ), + ), + ), + ( + ( + 1, + 2, + ), + [1:2, 1:3), + Math( + Opr( + Add, + ), + ), + ), + ( + ( + 2, + 3, + ), + [1:3, 1:4), + Math( + Digit( + One, + ), + ), + ), + ], + }, LxAstArena { math: Arena { data: [ @@ -299,6 +360,44 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( + LxTokenStorage { + ranged_tokens: [ + ( + ( + 0, + 1, + ), + [1:1, 1:2), + Math( + Letter( + LowerX, + ), + ), + ), + ( + ( + 1, + 2, + ), + [1:2, 1:3), + Math( + Superscript, + ), + ), + ( + ( + 2, + 3, + ), + [1:3, 1:4), + Math( + Digit( + Two, + ), + ), + ), + ], + }, LxAstArena { math: Arena { data: [ @@ -344,6 +443,44 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( + LxTokenStorage { + ranged_tokens: [ + ( + ( + 0, + 1, + ), + [1:1, 1:2), + Math( + Letter( + LowerX, + ), + ), + ), + ( + ( + 1, + 2, + ), + [1:2, 1:3), + Math( + Subscript, + ), + ), + ( + ( + 2, + 3, + ), + [1:3, 1:4), + Math( + Digit( + Two, + ), + ), + ), + ], + }, LxAstArena { math: Arena { data: [ @@ -389,6 +526,92 @@ fn parse_tex_input_into_asts_works() { LxMode::Math, expect![[r#" ( + LxTokenStorage { + ranged_tokens: [ + ( + ( + 0, + 1, + ), + [1:1, 1:2), + Math( + Letter( + LowerX, + ), + ), + ), + ( + ( + 1, + 2, + ), + [1:2, 1:3), + Math( + Superscript, + ), + ), + ( + ( + 2, + 3, + ), + [1:3, 1:4), + Math( + LeftDelimiter( + Curl, + ), + ), + ), + ( + ( + 3, + 4, + ), + [1:4, 1:5), + Math( + Letter( + LowerI, + ), + ), + ), + ( + ( + 4, + 5, + ), + [1:5, 1:6), + Math( + Opr( + Add, + ), + ), + ), + ( + ( + 5, + 6, + ), + [1:6, 1:7), + Math( + Digit( + Two, + ), + ), + ), + ( + ( + 6, + 7, + ), + [1:7, 1:8), + Math( + RightDelimiter( + Curl, + ), + ), + ), + ], + }, LxAstArena { math: Arena { data: [ diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index 3e544bfdd6..6bb6074478 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -8,13 +8,14 @@ use crate::{ }; use latex_prelude::mode::LxMode; use latex_token::{ - data::LxTokenData, idx::LxTokenIdx, storage::LxTokenStorage, stream::LxTokenStream, + data::LxTokenData, idx::LxTokenIdx, lexer::LxLexer, storage::LxTokenStorage, + stream::LxTokenStream, }; use std::{borrow::BorrowMut, iter::Peekable}; pub(crate) struct LxAstParser<'a> { db: &'a ::salsa::Db, - token_stream: Peekable>, + lexer: Peekable>, arena: &'a mut LxAstArena, } @@ -22,13 +23,14 @@ pub(crate) struct LxAstParser<'a> { impl<'a> LxAstParser<'a> { pub(crate) fn new( db: &'a ::salsa::Db, - tokens: &'a LxTokenStorage, + input: &'a str, mode: LxMode, + token_storage: &'a mut LxTokenStorage, arena: &'a mut LxAstArena, ) -> Self { Self { db, - token_stream: tokens.stream().peekable(), + lexer: LxLexer::new(db, input, mode, token_storage).peekable(), arena, } } @@ -53,11 +55,11 @@ impl<'a> LxAstParser<'a> { } pub(crate) fn peek_token(&mut self) -> Option { - self.token_stream.peek().map(|&(_, _, _, data)| data) + self.lexer.peek().map(|&(_, _, _, data)| data) } pub(crate) fn next_token(&mut self) -> Option<(LxTokenIdx, LxTokenData)> { - let (token_idx, _, _, token_data) = self.token_stream.next()?; + let (token_idx, _, _, token_data) = self.lexer.next()?; Some((token_idx, token_data)) } } diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs index 65286e6913..deb674890f 100644 --- a/crates/latex/latex-ast/src/test_helpers/example.rs +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -16,7 +16,9 @@ pub struct LxAstsExample { impl LxAstsExample { pub fn new(input: &str, root_mode: LxMode, db: &salsa::Db) -> Self { let mut ast_arena = LxAstArena::default(); - let asts = parse_latex_input_into_asts(db, input, root_mode, &mut ast_arena); + let mut token_storage = LxTokenStorage::default(); + let asts = + parse_latex_input_into_asts(db, input, root_mode, &mut token_storage, &mut ast_arena); let ast_token_idx_range_map = calc_ast_token_idx_range_map(db, &ast_arena); Self { token_storage: todo!(), diff --git a/crates/latex/latex-token/src/data/math.rs b/crates/latex/latex-token/src/data/math.rs index a299ec4bbb..551899aee8 100644 --- a/crates/latex/latex-token/src/data/math.rs +++ b/crates/latex/latex-token/src/data/math.rs @@ -101,7 +101,8 @@ impl<'a> LxLexer<'a> { fn next_text_token_data_works() { fn t(input: &str, expected: &Expect) { let db = &DB::default(); - let tokenizer = LxLexer::new(db, input, LxMode::Math); + let mut storage = LxTokenStorage::default(); + let tokenizer = LxLexer::new(db, input, LxMode::Math, &mut storage); let tokens: Vec<_> = tokenizer.map(|(_, _, _, token_data)| token_data).collect(); expected.assert_debug_eq(&(tokens.debug(db))); } diff --git a/crates/latex/latex-token/src/data/rose.rs b/crates/latex/latex-token/src/data/rose.rs index 05eac3f559..e510a4167c 100644 --- a/crates/latex/latex-token/src/data/rose.rs +++ b/crates/latex/latex-token/src/data/rose.rs @@ -64,7 +64,8 @@ impl<'a> LxLexer<'a> { fn next_text_token_data_works() { fn t(input: &str, expected: &Expect) { let db = &DB::default(); - let lexer = LxLexer::new(db, input, LxMode::Rose); + let mut storage = LxTokenStorage::default(); + let lexer = LxLexer::new(db, input, LxMode::Rose, &mut storage); let tokens: Vec<_> = lexer.map(|(_, _, _, token_data)| token_data).collect(); expected.assert_debug_eq(&(tokens.debug(db))); } diff --git a/crates/latex/latex-token/src/lexer.rs b/crates/latex/latex-token/src/lexer.rs index eb7a3c2746..aba00f16f6 100644 --- a/crates/latex/latex-token/src/lexer.rs +++ b/crates/latex/latex-token/src/lexer.rs @@ -11,33 +11,28 @@ use husky_coword::Coword; use husky_text_protocol::{char::TextCharIter, range::TextRange}; use latex_prelude::mode::LxMode; -pub fn lex_latex_input(input: &str, mode: LxMode, db: &::salsa::Db) -> LxTokenStorage { - let mut lexer = LxLexer::new(db, input, mode); - for _ in &mut lexer {} - lexer.finish() -} - -pub(crate) struct LxLexer<'a> { +pub struct LxLexer<'a> { pub(crate) db: &'a ::salsa::Db, pub(crate) chars: TextCharIter<'a>, pub(crate) mode: LxMode, - pub(crate) storage: LxTokenStorage, + pub(crate) storage: &'a mut LxTokenStorage, } /// # constructor impl<'a> LxLexer<'a> { - pub(crate) fn new(db: &'a ::salsa::Db, input: &'a str, mode: LxMode) -> Self { + pub fn new( + db: &'a ::salsa::Db, + input: &'a str, + mode: LxMode, + storage: &'a mut LxTokenStorage, + ) -> Self { Self { db, chars: TextCharIter::new(input), mode, - storage: Default::default(), + storage, } } - - pub(crate) fn finish(self) -> LxTokenStorage { - self.storage - } } /// # actions diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index 5d8253788f..88aa6d873c 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -5,6 +5,7 @@ use crate::{ }; use husky_text_protocol::range::TextRange; +#[salsa::derive_debug_with_db] #[derive(Default, Debug)] pub struct LxTokenStorage { ranged_tokens: Vec<((usize, usize), TextRange, LxTokenData)>, From 7cd871e76213209579f626ea17eb5c533a1e9acc Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 02:41:19 -0700 Subject: [PATCH 18/71] refactor latex ast --- crates/latex/latex-ast/src/ast.rs | 127 ++++++------------ crates/latex/latex-ast/src/ast/math.rs | 25 ++-- crates/latex/latex-ast/src/parser.rs | 12 +- .../latex-ast/src/test_helpers/example.rs | 2 +- crates/latex/latex-math-opr/src/lib.rs | 116 +++++++++------- crates/latex/latex-token/src/data/math.rs | 26 ++-- crates/latex/latex-token/src/lexer.rs | 9 ++ 7 files changed, 151 insertions(+), 166 deletions(-) diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 027959ec57..533fb24f0c 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -10,7 +10,7 @@ use crate::parser::LxAstParser; use crate::*; use idx_arena::{map::ArenaMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; use latex_math_letter::LxMathLetter; -use latex_math_opr::LxMathOpr; +use latex_math_opr::LxMathPunctuation; use latex_prelude::{mode::LxMode, script::LxScriptKind}; use latex_token::{ data::{ @@ -101,47 +101,44 @@ pub fn parse_latex_input_into_asts<'a>( mode: LxMode, token_storage: &'a mut LxTokenStorage, arena: &'a mut LxAstArena, -) -> Option { +) -> LxAstIdxRange { let mut parser = LxAstParser::new(db, input, mode, token_storage, arena); - let asts = parser.parse_asts(); - asts + parser.parse_asts() } impl<'a> LxAstParser<'a> { - pub(crate) fn parse_asts(&mut self) -> Option { - match self.peek_token()? { - LxTokenData::Math(_) => { + pub(crate) fn parse_asts(&mut self) -> LxAstIdxRange { + match self.mode() { + LxMode::Rose => todo!(), + LxMode::Math => { let mut asts = vec![]; - while let Some(ast) = self.parse_ast() { - let LxAstData::Math(ast) = ast else { todo!() }; + while let Some(ast) = self.parse_math_ast() { asts.push(ast) } - Some(self.alloc_math_asts(asts).into()) + self.alloc_math_asts(asts).into() } - LxTokenData::Rose(_) => todo!(), } } pub(crate) fn parse_math_asts(&mut self) -> LxMathAstIdxRange { let mut asts = vec![]; - while let Some(ast) = self.parse_ast() { - let LxAstData::Math(ast) = ast else { todo!() }; + while let Some(ast) = self.parse_math_ast() { asts.push(ast) } self.alloc_math_asts(asts) } - fn parse_ast(&mut self) -> Option { - let mut ast = self.parse_atomic_ast()?; - match self.peek_token()? { + fn parse_math_ast(&mut self) -> Option { + let mut ast = self.parse_atomic_math_ast()?; + match self.peek_char()? { // TODO include more cases, like \limits - LxTokenData::Math(LxMathTokenData::Subscript | LxMathTokenData::Superscript) => { + '_' | '^' => { let (idx, LxTokenData::Math(token)) = self.next_token().unwrap() else { unreachable!() }; ast = match ast { - LxAstData::Math(LxMathAstData::Attach { .. }) => ast, - LxAstData::Math(base) => { + LxMathAstData::Attach { .. } => ast, + base => { let base = self.alloc_math_ast(base); LxMathAstData::Attach { base, @@ -151,9 +148,9 @@ impl<'a> LxAstParser<'a> { } _ => todo!(), }; - let LxAstData::Math(LxMathAstData::Attach { + let LxMathAstData::Attach { ref mut scripts, .. - }) = ast + } = ast else { unreachable!() }; @@ -162,9 +159,8 @@ impl<'a> LxAstParser<'a> { LxMathTokenData::Superscript => LxScriptKind::Superscript, _ => todo!(), }; - let ast = match self.parse_atomic_ast() { - Some(LxAstData::Math(new_subscript)) => self.alloc_math_ast(new_subscript), - Some(LxAstData::Rose(_)) => todo!("err: expected math ast"), + let ast = match self.parse_atomic_math_ast() { + Some(new_subscript) => self.alloc_math_ast(new_subscript), None => todo!("err: expected subscript"), }; scripts.push((script_kind, ast)); @@ -173,39 +169,6 @@ impl<'a> LxAstParser<'a> { }; Some(ast) } - - fn parse_atomic_ast(&mut self) -> Option { - match self.peek_token()? { - LxTokenData::Math(token) => { - match token { - LxMathTokenData::Command(_) => todo!(), - LxMathTokenData::LeftDelimiter(_) => (), - LxMathTokenData::RightDelimiter(_) => return None, - LxMathTokenData::Letter(_) => (), - LxMathTokenData::Opr(_) => (), - LxMathTokenData::Digit(_) => (), - LxMathTokenData::Other(_) => todo!(), - LxMathTokenData::Subscript => todo!(), - LxMathTokenData::Superscript => todo!(), - LxMathTokenData::Error(_) => todo!(), - }; - } - LxTokenData::Rose(token) => match token { - LxRoseTokenData::Word(_) => todo!(), - LxRoseTokenData::Command(_) => todo!(), - LxRoseTokenData::Dollar => todo!(), - LxRoseTokenData::EscapedLpar => todo!(), - LxRoseTokenData::EscapedLbox => todo!(), - LxRoseTokenData::Nat32(_) => todo!(), - LxRoseTokenData::NewParagraph => todo!(), - }, - } - let (idx, token) = self.next_token().unwrap(); - Some(match token { - LxTokenData::Math(token) => self.parse_atomic_math_ast(idx, token).into(), - LxTokenData::Rose(token) => self.parse_atomic_text_ast(idx, token).into(), - }) - } } // TODO replace it with example @@ -236,7 +199,11 @@ fn parse_tex_input_into_asts_works() { data: [], }, }, - None, + LxAstIdxRange::Math( + ArenaIdxRange( + 0..0, + ), + ), ) "#]], ); @@ -269,11 +236,9 @@ fn parse_tex_input_into_asts_works() { data: [], }, }, - Some( - LxAstIdxRange::Math( - ArenaIdxRange( - 0..0, - ), + LxAstIdxRange::Math( + ArenaIdxRange( + 0..0, ), ), ) @@ -305,7 +270,7 @@ fn parse_tex_input_into_asts_works() { ), [1:2, 1:3), Math( - Opr( + Punctuation( Add, ), ), @@ -345,11 +310,9 @@ fn parse_tex_input_into_asts_works() { data: [], }, }, - Some( - LxAstIdxRange::Math( - ArenaIdxRange( - 0..2, - ), + LxAstIdxRange::Math( + ArenaIdxRange( + 0..2, ), ), ) @@ -428,11 +391,9 @@ fn parse_tex_input_into_asts_works() { data: [], }, }, - Some( - LxAstIdxRange::Math( - ArenaIdxRange( - 2..3, - ), + LxAstIdxRange::Math( + ArenaIdxRange( + 2..3, ), ), ) @@ -511,11 +472,9 @@ fn parse_tex_input_into_asts_works() { data: [], }, }, - Some( - LxAstIdxRange::Math( - ArenaIdxRange( - 2..3, - ), + LxAstIdxRange::Math( + ArenaIdxRange( + 2..3, ), ), ) @@ -581,7 +540,7 @@ fn parse_tex_input_into_asts_works() { ), [1:5, 1:6), Math( - Opr( + Punctuation( Add, ), ), @@ -667,11 +626,9 @@ fn parse_tex_input_into_asts_works() { data: [], }, }, - Some( - LxAstIdxRange::Math( - ArenaIdxRange( - 5..6, - ), + LxAstIdxRange::Math( + ArenaIdxRange( + 5..6, ), ), ) diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 289bf4ad25..234dea1e2e 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -9,7 +9,7 @@ use super::*; #[derive(Debug, Clone, PartialEq, Eq)] pub enum LxMathAstData { Letter(LxTokenIdx, LxMathLetter), - Opr(LxTokenIdx, LxMathOpr), + Opr(LxTokenIdx, LxMathPunctuation), Digit(LxTokenIdx, LxMathDigit), /// not obtained through parsing, but through ui TextEdit { @@ -35,23 +35,26 @@ pub type LxMathAstIdx = ArenaIdx; pub type LxMathAstIdxRange = ArenaIdxRange; impl<'a> LxAstParser<'a> { - pub(super) fn parse_atomic_math_ast( - &mut self, - idx: LxTokenIdx, - token: LxMathTokenData, - ) -> LxMathAstData { - match token { + pub(super) fn parse_atomic_math_ast(&mut self) -> Option { + match self.peek_char()? { + '}' => return None, + _ => (), + }; + let (idx, LxTokenData::Math(token)) = self.next_token()? else { + unreachable!() + }; + Some(match token { LxMathTokenData::Command(_) => todo!(), LxMathTokenData::LeftDelimiter(delimiter) => self.parse_delimited(idx, delimiter), - LxMathTokenData::RightDelimiter(_) => todo!(), + LxMathTokenData::RightDelimiter(_) => unreachable!(), LxMathTokenData::Letter(letter) => LxMathAstData::Letter(idx, letter), - LxMathTokenData::Opr(opr) => LxMathAstData::Opr(idx, opr), // it's not constructed into a tree yet in the ast stage + LxMathTokenData::Punctuation(opr) => LxMathAstData::Opr(idx, opr), // it's not constructed into a tree yet in the ast stage LxMathTokenData::Digit(digit) => LxMathAstData::Digit(idx, digit), LxMathTokenData::Other(_) => todo!(), LxMathTokenData::Subscript => todo!(), LxMathTokenData::Superscript => todo!(), LxMathTokenData::Error(_) => todo!(), - } + }) } // here we differ from the latex syntax, we see all possible delimiters as latex delimiters @@ -76,7 +79,7 @@ impl<'a> LxAstParser<'a> { right_delimiter, }, LxMathTokenData::Letter(_) => todo!(), - LxMathTokenData::Opr(_) => todo!(), + LxMathTokenData::Punctuation(_) => todo!(), LxMathTokenData::Digit(_) => todo!(), LxMathTokenData::Other(_) => todo!(), LxMathTokenData::Subscript => todo!(), diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index 6bb6074478..2a080f2fdc 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -15,7 +15,7 @@ use std::{borrow::BorrowMut, iter::Peekable}; pub(crate) struct LxAstParser<'a> { db: &'a ::salsa::Db, - lexer: Peekable>, + lexer: LxLexer<'a>, arena: &'a mut LxAstArena, } @@ -30,10 +30,14 @@ impl<'a> LxAstParser<'a> { ) -> Self { Self { db, - lexer: LxLexer::new(db, input, mode, token_storage).peekable(), + lexer: LxLexer::new(db, input, mode, token_storage), arena, } } + + pub(crate) fn mode(&self) -> LxMode { + self.lexer.mode() + } } /// # actions @@ -54,8 +58,8 @@ impl<'a> LxAstParser<'a> { self.arena.rose.alloc_batch(asts) } - pub(crate) fn peek_token(&mut self) -> Option { - self.lexer.peek().map(|&(_, _, _, data)| data) + pub(crate) fn peek_char(&mut self) -> Option { + self.lexer.peek_char() } pub(crate) fn next_token(&mut self) -> Option<(LxTokenIdx, LxTokenData)> { diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs index deb674890f..8f08525a6d 100644 --- a/crates/latex/latex-ast/src/test_helpers/example.rs +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -9,7 +9,7 @@ use latex_token::storage::LxTokenStorage; pub struct LxAstsExample { pub token_storage: LxTokenStorage, pub ast_arena: LxAstArena, - pub asts: Option, + pub asts: LxAstIdxRange, pub ast_token_idx_range_map: LxAstTokenIdxRangeMap, } diff --git a/crates/latex/latex-math-opr/src/lib.rs b/crates/latex/latex-math-opr/src/lib.rs index bf4ad885c1..e17644ab72 100644 --- a/crates/latex/latex-math-opr/src/lib.rs +++ b/crates/latex/latex-math-opr/src/lib.rs @@ -1,6 +1,6 @@ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u8)] -pub enum LxMathOpr { +pub enum LxMathPunctuation { /// Addition `+` Add, /// Subtraction `-` @@ -45,60 +45,82 @@ pub enum LxMathOpr { PlusMinus, /// Times `\times` or `×` Times, + /// Left parenthesis `(` + Lpar, + /// Right parenthesis `)` + Rpar, + /// Left box `[` + Lbox, + /// Right box `]` + Rbox, + /// Escaped left curly `{` + EscapedLcurl, + /// Escaped right curly `}` + EscapedRcurl, } -impl LxMathOpr { - pub fn to_char(self) -> char { +impl LxMathPunctuation { + pub fn to_str(self) -> &'static str { match self { - LxMathOpr::Add => '+', - LxMathOpr::Sub => '-', - LxMathOpr::Mul => '·', - LxMathOpr::Div => '/', - LxMathOpr::In => '∈', - LxMathOpr::NotIn => '∉', - LxMathOpr::Subset => '⊂', - LxMathOpr::Superset => '⊃', - LxMathOpr::SubsetEq => '⊆', - LxMathOpr::SupersetEq => '⊇', - LxMathOpr::ForAll => '∀', - LxMathOpr::Exists => '∃', - LxMathOpr::NotExists => '∄', - LxMathOpr::Infinity => '∞', - LxMathOpr::Equals => '=', - LxMathOpr::NotEquals => '≠', - LxMathOpr::LessThan => '<', - LxMathOpr::GreaterThan => '>', - LxMathOpr::LessEq => '≤', - LxMathOpr::GreaterEq => '≥', - LxMathOpr::PlusMinus => '±', - LxMathOpr::Times => '×', + LxMathPunctuation::Add => "+", + LxMathPunctuation::Sub => "-", + LxMathPunctuation::Mul => "·", + LxMathPunctuation::Div => "/", + LxMathPunctuation::In => "∈", + LxMathPunctuation::NotIn => "∉", + LxMathPunctuation::Subset => "⊂", + LxMathPunctuation::Superset => "⊃", + LxMathPunctuation::SubsetEq => "⊆", + LxMathPunctuation::SupersetEq => "⊇", + LxMathPunctuation::ForAll => "∀", + LxMathPunctuation::Exists => "∃", + LxMathPunctuation::NotExists => "∄", + LxMathPunctuation::Infinity => "∞", + LxMathPunctuation::Equals => "=", + LxMathPunctuation::NotEquals => "≠", + LxMathPunctuation::LessThan => "<", + LxMathPunctuation::GreaterThan => ">", + LxMathPunctuation::LessEq => "≤", + LxMathPunctuation::GreaterEq => "≥", + LxMathPunctuation::PlusMinus => "±", + LxMathPunctuation::Times => "×", + LxMathPunctuation::Lpar => "(", + LxMathPunctuation::Rpar => ")", + LxMathPunctuation::Lbox => "[", + LxMathPunctuation::Rbox => "]", + LxMathPunctuation::EscapedLcurl => "\\{", + LxMathPunctuation::EscapedRcurl => "\\}", } } pub fn try_from_char(c: char) -> Option { match c { - '+' => Some(LxMathOpr::Add), - '-' => Some(LxMathOpr::Sub), - '·' => Some(LxMathOpr::Mul), - '/' => Some(LxMathOpr::Div), - '∈' => Some(LxMathOpr::In), - '∉' => Some(LxMathOpr::NotIn), - '⊂' => Some(LxMathOpr::Subset), - '⊃' => Some(LxMathOpr::Superset), - '⊆' => Some(LxMathOpr::SubsetEq), - '⊇' => Some(LxMathOpr::SupersetEq), - '∀' => Some(LxMathOpr::ForAll), - '∃' => Some(LxMathOpr::Exists), - '∄' => Some(LxMathOpr::NotExists), - '∞' => Some(LxMathOpr::Infinity), - '=' => Some(LxMathOpr::Equals), - '≠' => Some(LxMathOpr::NotEquals), - '<' => Some(LxMathOpr::LessThan), - '>' => Some(LxMathOpr::GreaterThan), - '≤' => Some(LxMathOpr::LessEq), - '≥' => Some(LxMathOpr::GreaterEq), - '±' => Some(LxMathOpr::PlusMinus), - '×' => Some(LxMathOpr::Times), + '+' => Some(LxMathPunctuation::Add), + '-' => Some(LxMathPunctuation::Sub), + '·' => Some(LxMathPunctuation::Mul), + '/' => Some(LxMathPunctuation::Div), + '∈' => Some(LxMathPunctuation::In), + '∉' => Some(LxMathPunctuation::NotIn), + '⊂' => Some(LxMathPunctuation::Subset), + '⊃' => Some(LxMathPunctuation::Superset), + '⊆' => Some(LxMathPunctuation::SubsetEq), + '⊇' => Some(LxMathPunctuation::SupersetEq), + '∀' => Some(LxMathPunctuation::ForAll), + '∃' => Some(LxMathPunctuation::Exists), + '∄' => Some(LxMathPunctuation::NotExists), + '∞' => Some(LxMathPunctuation::Infinity), + '=' => Some(LxMathPunctuation::Equals), + '≠' => Some(LxMathPunctuation::NotEquals), + '<' => Some(LxMathPunctuation::LessThan), + '>' => Some(LxMathPunctuation::GreaterThan), + '≤' => Some(LxMathPunctuation::LessEq), + '≥' => Some(LxMathPunctuation::GreaterEq), + '±' => Some(LxMathPunctuation::PlusMinus), + '×' => Some(LxMathPunctuation::Times), + '(' => Some(LxMathPunctuation::Lpar), + ')' => Some(LxMathPunctuation::Rpar), + '[' => Some(LxMathPunctuation::Lbox), + ']' => Some(LxMathPunctuation::Rbox), _ => None, } } diff --git a/crates/latex/latex-token/src/data/math.rs b/crates/latex/latex-token/src/data/math.rs index 551899aee8..e82510d7ec 100644 --- a/crates/latex/latex-token/src/data/math.rs +++ b/crates/latex/latex-token/src/data/math.rs @@ -4,7 +4,7 @@ use self::digit::LxMathDigit; use super::*; use latex_command::path::LxCommandPath; use latex_math_letter::LxMathLetter; -use latex_math_opr::LxMathOpr; +use latex_math_opr::LxMathPunctuation; #[salsa::derive_debug_with_db] #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -13,7 +13,7 @@ pub enum LxMathTokenData { LeftDelimiter(LxMathDelimiter), RightDelimiter(LxMathDelimiter), Letter(LxMathLetter), - Opr(LxMathOpr), + Punctuation(LxMathPunctuation), Digit(LxMathDigit), Other(char), Subscript, @@ -25,12 +25,6 @@ pub enum LxMathTokenData { pub enum LxMathDelimiter { /// `{`, `}` Curl, - /// `(`, `)` - Par, - /// `[`, `]` - Box, - /// `\{`, `\}` - EscapedCurl, } #[salsa::derive_debug_with_db] @@ -54,11 +48,11 @@ impl<'a> LxLexer<'a> { c => { self.chars.eat_char(); match c { - '{' => Some(LxMathTokenData::LeftDelimiter( - LxMathDelimiter::EscapedCurl, + '{' => Some(LxMathTokenData::Punctuation( + LxMathPunctuation::EscapedLcurl, )), - '}' => Some(LxMathTokenData::RightDelimiter( - LxMathDelimiter::EscapedCurl, + '}' => Some(LxMathTokenData::Punctuation( + LxMathPunctuation::EscapedRcurl, )), _ => todo!(), } @@ -78,15 +72,11 @@ impl<'a> LxLexer<'a> { '^' => Some(LxMathTokenData::Superscript), '{' => Some(LxMathTokenData::LeftDelimiter(LxMathDelimiter::Curl)), '}' => Some(LxMathTokenData::RightDelimiter(LxMathDelimiter::Curl)), - '(' => Some(LxMathTokenData::LeftDelimiter(LxMathDelimiter::Par)), - ')' => Some(LxMathTokenData::RightDelimiter(LxMathDelimiter::Par)), - '[' => Some(LxMathTokenData::LeftDelimiter(LxMathDelimiter::Box)), - ']' => Some(LxMathTokenData::RightDelimiter(LxMathDelimiter::Box)), c => { if let Some(letter) = LxMathLetter::try_from_char(c) { Some(LxMathTokenData::Letter(letter)) - } else if let Some(opr) = LxMathOpr::try_from_char(c) { - Some(LxMathTokenData::Opr(opr)) + } else if let Some(opr) = LxMathPunctuation::try_from_char(c) { + Some(LxMathTokenData::Punctuation(opr)) } else { Some(LxMathTokenData::Other(c)) } diff --git a/crates/latex/latex-token/src/lexer.rs b/crates/latex/latex-token/src/lexer.rs index aba00f16f6..31d9055010 100644 --- a/crates/latex/latex-token/src/lexer.rs +++ b/crates/latex/latex-token/src/lexer.rs @@ -33,6 +33,15 @@ impl<'a> LxLexer<'a> { storage, } } + + pub fn peek_char(&mut self) -> Option { + self.chars.eat_chars_while(|c| c == ' '); + self.chars.peek() + } + + pub fn mode(&self) -> LxMode { + self.mode + } } /// # actions From d6a4fc7de6951282fd6aa6dccb6c014e8f6f1c87 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 02:48:16 -0700 Subject: [PATCH 19/71] rename visored annotations --- .../latex/latex-ast/expect-files/examples.txt | 22 ++--- crates/latex/latex-ast/src/ast/math.rs | 2 +- crates/latex/latex-token/src/data/math.rs | 82 +++++++++---------- .../expect-files/annotations/examples.txt | 22 ++--- .../src/annotation/space.rs | 2 +- .../src/annotation/token.rs | 2 +- .../visored-annotation/src/annotations.rs | 34 ++++---- .../src/annotations/builder/sparce.rs | 46 +++++------ .../src/annotations/walker.rs | 58 ++++++------- .../src/test_helpers/example.rs | 32 ++++---- 10 files changed, 151 insertions(+), 151 deletions(-) diff --git a/crates/latex/latex-ast/expect-files/examples.txt b/crates/latex/latex-ast/expect-files/examples.txt index d765777452..21fb0098e5 100644 --- a/crates/latex/latex-ast/expect-files/examples.txt +++ b/crates/latex/latex-ast/expect-files/examples.txt @@ -1,9 +1,9 @@ [ LatexAstsExample { - annotations: LxAnnotationsExample { + annotations: VdAnnotationsExample { root_mode: Math, input: "", - annotations: LxAnnotations { + annotations: VdAnnotations { token_annotations: [], space_annotations: [], }, @@ -27,19 +27,19 @@ }, }, LatexAstsExample { - annotations: LxAnnotationsExample { + annotations: VdAnnotationsExample { root_mode: Math, input: "xy", - annotations: LxAnnotations { + annotations: VdAnnotations { token_annotations: [ - LxAnnotationEntry { + VdAnnotationEntry { start: 0, end: 1, annotation: Variable( Usage, ), }, - LxAnnotationEntry { + VdAnnotationEntry { start: 1, end: 2, annotation: Variable( @@ -48,7 +48,7 @@ }, ], space_annotations: [ - LxAnnotationEntry { + VdAnnotationEntry { start: 1, end: 2, annotation: Apply( @@ -97,17 +97,17 @@ }, }, LatexAstsExample { - annotations: LxAnnotationsExample { + annotations: VdAnnotationsExample { root_mode: Math, input: "dx", - annotations: LxAnnotations { + annotations: VdAnnotations { token_annotations: [ - LxAnnotationEntry { + VdAnnotationEntry { start: 0, end: 1, annotation: Differential, }, - LxAnnotationEntry { + VdAnnotationEntry { start: 1, end: 2, annotation: Variable( diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 234dea1e2e..56072d07c5 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -37,7 +37,7 @@ pub type LxMathAstIdxRange = ArenaIdxRange; impl<'a> LxAstParser<'a> { pub(super) fn parse_atomic_math_ast(&mut self) -> Option { match self.peek_char()? { - '}' => return None, + '}' | '$' => return None, _ => (), }; let (idx, LxTokenData::Math(token)) = self.next_token()? else { diff --git a/crates/latex/latex-token/src/data/math.rs b/crates/latex/latex-token/src/data/math.rs index e82510d7ec..7c1692c4f2 100644 --- a/crates/latex/latex-token/src/data/math.rs +++ b/crates/latex/latex-token/src/data/math.rs @@ -235,58 +235,58 @@ fn next_text_token_data_works() { t( "(", &expect![[r#" - [ - LxTokenData::Math( - LxMathTokenData::LeftDelimiter( - Par, + [ + LxTokenData::Math( + LxMathTokenData::Punctuation( + Lpar, + ), ), - ), - ] - "#]], + ] + "#]], ); t( ")", &expect![[r#" - [ - LxTokenData::Math( - LxMathTokenData::RightDelimiter( - Par, + [ + LxTokenData::Math( + LxMathTokenData::Punctuation( + Rpar, + ), ), - ), - ] - "#]], + ] + "#]], ); t( "[", &expect![[r#" - [ - LxTokenData::Math( - LxMathTokenData::LeftDelimiter( - Box, + [ + LxTokenData::Math( + LxMathTokenData::Punctuation( + Lbox, + ), ), - ), - ] - "#]], + ] + "#]], ); t( "]", &expect![[r#" - [ - LxTokenData::Math( - LxMathTokenData::RightDelimiter( - Box, + [ + LxTokenData::Math( + LxMathTokenData::Punctuation( + Rbox, + ), ), - ), - ] - "#]], + ] + "#]], ); t( "\\{", &expect![[r#" [ LxTokenData::Math( - LxMathTokenData::LeftDelimiter( - EscapedCurl, + LxMathTokenData::Punctuation( + EscapedLcurl, ), ), ] @@ -297,8 +297,8 @@ fn next_text_token_data_works() { &expect![[r#" [ LxTokenData::Math( - LxMathTokenData::RightDelimiter( - EscapedCurl, + LxMathTokenData::Punctuation( + EscapedRcurl, ), ), ] @@ -307,14 +307,14 @@ fn next_text_token_data_works() { t( "+", &expect![[r#" - [ - LxTokenData::Math( - LxMathTokenData::Opr( - Add, + [ + LxTokenData::Math( + LxMathTokenData::Punctuation( + Add, + ), ), - ), - ] - "#]], + ] + "#]], ); t( "x+1", @@ -326,7 +326,7 @@ fn next_text_token_data_works() { ), ), LxTokenData::Math( - LxMathTokenData::Opr( + LxMathTokenData::Punctuation( Add, ), ), @@ -364,7 +364,7 @@ fn next_text_token_data_works() { ), ), LxTokenData::Math( - LxMathTokenData::Opr( + LxMathTokenData::Punctuation( Add, ), ), diff --git a/crates/visored/visored-annotation/expect-files/annotations/examples.txt b/crates/visored/visored-annotation/expect-files/annotations/examples.txt index 3a53e75861..9eda51ee67 100644 --- a/crates/visored/visored-annotation/expect-files/annotations/examples.txt +++ b/crates/visored/visored-annotation/expect-files/annotations/examples.txt @@ -1,25 +1,25 @@ [ - LxAnnotationsExample { + VdAnnotationsExample { root_mode: Math, input: "", - annotations: LxAnnotations { + annotations: VdAnnotations { token_annotations: [], space_annotations: [], }, }, - LxAnnotationsExample { + VdAnnotationsExample { root_mode: Math, input: "xy", - annotations: LxAnnotations { + annotations: VdAnnotations { token_annotations: [ - LxAnnotationEntry { + VdAnnotationEntry { start: 0, end: 1, annotation: Variable( Usage, ), }, - LxAnnotationEntry { + VdAnnotationEntry { start: 1, end: 2, annotation: Variable( @@ -28,7 +28,7 @@ }, ], space_annotations: [ - LxAnnotationEntry { + VdAnnotationEntry { start: 1, end: 2, annotation: Apply( @@ -38,17 +38,17 @@ ], }, }, - LxAnnotationsExample { + VdAnnotationsExample { root_mode: Math, input: "dx", - annotations: LxAnnotations { + annotations: VdAnnotations { token_annotations: [ - LxAnnotationEntry { + VdAnnotationEntry { start: 0, end: 1, annotation: Differential, }, - LxAnnotationEntry { + VdAnnotationEntry { start: 1, end: 2, annotation: Variable( diff --git a/crates/visored/visored-annotation/src/annotation/space.rs b/crates/visored/visored-annotation/src/annotation/space.rs index e20101ca08..47221c71de 100644 --- a/crates/visored/visored-annotation/src/annotation/space.rs +++ b/crates/visored/visored-annotation/src/annotation/space.rs @@ -1,6 +1,6 @@ #[enum_class::from_variants] #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] -pub enum LxSpaceAnnotation { +pub enum VdSpaceAnnotation { #[default] Void, Apply(LxApplyAnnotation), diff --git a/crates/visored/visored-annotation/src/annotation/token.rs b/crates/visored/visored-annotation/src/annotation/token.rs index 403df29437..9efb163e0f 100644 --- a/crates/visored/visored-annotation/src/annotation/token.rs +++ b/crates/visored/visored-annotation/src/annotation/token.rs @@ -1,6 +1,6 @@ #[enum_class::from_variants] #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] -pub enum LxTokenAnnotation { +pub enum VdTokenAnnotation { #[default] Void, Integral(LxIntegralAnnotation), diff --git a/crates/visored/visored-annotation/src/annotations.rs b/crates/visored/visored-annotation/src/annotations.rs index 5a0b82e206..6119c74baa 100644 --- a/crates/visored/visored-annotation/src/annotations.rs +++ b/crates/visored/visored-annotation/src/annotations.rs @@ -1,30 +1,30 @@ mod builder; pub mod walker; -use crate::annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}; +use crate::annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}; use builder::sparce::collect_from_sparse_annotations; -use walker::LxAnnotationsWalker; +use walker::VdAnnotationsWalker; #[derive(Debug, PartialEq, Eq, Clone)] -pub struct LxAnnotationEntry { +pub struct VdAnnotationEntry { pub start: usize, pub end: usize, pub annotation: A, } -pub type LxTokenAnnotationEntry = LxAnnotationEntry; -pub type LxSpaceAnnotationEntry = LxAnnotationEntry; +pub type VdTokenAnnotationEntry = VdAnnotationEntry; +pub type VdSpaceAnnotationEntry = VdAnnotationEntry; #[derive(Debug, PartialEq, Eq, Clone)] -pub struct LxAnnotations { - token_annotations: Vec, - space_annotations: Vec, +pub struct VdAnnotations { + token_annotations: Vec, + space_annotations: Vec, } -impl LxAnnotations { +impl VdAnnotations { pub fn new( - token_annotations: Vec, - space_annotations: Vec, + token_annotations: Vec, + space_annotations: Vec, ) -> Self { Self { token_annotations, @@ -34,8 +34,8 @@ impl LxAnnotations { pub fn from_sparse<'a>( input: &'a str, - token_annotations: impl IntoIterator, - space_annotations: impl IntoIterator, + token_annotations: impl IntoIterator, + space_annotations: impl IntoIterator, ) -> Self { collect_from_sparse_annotations( input, @@ -44,15 +44,15 @@ impl LxAnnotations { ) } - pub fn token_annotations(&self) -> &[LxTokenAnnotationEntry] { + pub fn token_annotations(&self) -> &[VdTokenAnnotationEntry] { &self.token_annotations } - pub fn space_annotations(&self) -> &[LxSpaceAnnotationEntry] { + pub fn space_annotations(&self) -> &[VdSpaceAnnotationEntry] { &self.space_annotations } - pub fn walker(&self) -> LxAnnotationsWalker { - LxAnnotationsWalker::new(&self.token_annotations, &self.space_annotations) + pub fn walker(&self) -> VdAnnotationsWalker { + VdAnnotationsWalker::new(&self.token_annotations, &self.space_annotations) } } diff --git a/crates/visored/visored-annotation/src/annotations/builder/sparce.rs b/crates/visored/visored-annotation/src/annotations/builder/sparce.rs index 4f5f1a662a..2bbd475a54 100644 --- a/crates/visored/visored-annotation/src/annotations/builder/sparce.rs +++ b/crates/visored/visored-annotation/src/annotations/builder/sparce.rs @@ -1,46 +1,46 @@ use crate::{ - annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}, + annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}, annotations::{ - LxAnnotationEntry, LxAnnotations, LxSpaceAnnotationEntry, LxTokenAnnotationEntry, + VdAnnotationEntry, VdAnnotations, VdSpaceAnnotationEntry, VdTokenAnnotationEntry, }, }; pub(crate) fn collect_from_sparse_annotations<'a>( raw_text: &'a str, - token_annotation_iter: impl Iterator, - space_annotation_iter: impl Iterator, -) -> LxAnnotations { + token_annotation_iter: impl Iterator, + space_annotation_iter: impl Iterator, +) -> VdAnnotations { let token_annotations = collect_from_sparse_token_annotations(raw_text, token_annotation_iter); let space_annotations = collect_from_sparse_space_annotations(raw_text, space_annotation_iter); - LxAnnotations::new(token_annotations, space_annotations) + VdAnnotations::new(token_annotations, space_annotations) } fn collect_from_sparse_token_annotations<'a>( raw_text: &'a str, - token_annotation_iter: impl Iterator, -) -> Vec { + token_annotation_iter: impl Iterator, +) -> Vec { collect_from_sparse_annotations_aux(raw_text, token_annotation_iter) } fn collect_from_sparse_space_annotations<'a>( raw_text: &'a str, - space_annotation_iter: impl Iterator, -) -> Vec { + space_annotation_iter: impl Iterator, +) -> Vec { collect_from_sparse_annotations_aux(raw_text, space_annotation_iter) } fn collect_from_sparse_annotations_aux<'a, A>( raw_text: &'a str, annotation_iter: impl Iterator, -) -> Vec> { +) -> Vec> { let mut annotations = Vec::new(); for ((prev_s, token_s), annotation) in annotation_iter { let start = prev_s.len(); let end = prev_s.len() + token_s.len(); assert_eq!(&raw_text[..start], prev_s); assert_eq!(&raw_text[start..end], token_s); - annotations.push(LxAnnotationEntry { + annotations.push(VdAnnotationEntry { start, end, annotation, @@ -66,18 +66,18 @@ mod tests { let token_annotations = vec![ ( ("", "\\int"), - LxTokenAnnotation::Integral( + VdTokenAnnotation::Integral( LxIntegralAnnotation::SingleVariableIndefiniteIntegralOverReal, ), ), ( ("\\int ", "x"), - LxTokenAnnotation::Variable(LxVariableAnnotation::Usage), + VdTokenAnnotation::Variable(LxVariableAnnotation::Usage), ), - (("\\int x", "d"), LxTokenAnnotation::Differential), + (("\\int x", "d"), VdTokenAnnotation::Differential), ( ("\\int xd", "x"), - LxTokenAnnotation::Variable( + VdTokenAnnotation::Variable( LxVariableAnnotation::SingleVariableIntegralVariableDecl, ), ), @@ -85,7 +85,7 @@ mod tests { let space_annotations = vec![( ("\\int x", "d"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), + VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), )]; let result = collect_from_sparse_annotations( @@ -95,28 +95,28 @@ mod tests { ); expect![[r#" - LxAnnotations { + VdAnnotations { token_annotations: [ - LxAnnotationEntry { + VdAnnotationEntry { start: 0, end: 4, annotation: Integral( SingleVariableIndefiniteIntegralOverReal, ), }, - LxAnnotationEntry { + VdAnnotationEntry { start: 5, end: 6, annotation: Variable( Usage, ), }, - LxAnnotationEntry { + VdAnnotationEntry { start: 6, end: 7, annotation: Differential, }, - LxAnnotationEntry { + VdAnnotationEntry { start: 7, end: 8, annotation: Variable( @@ -125,7 +125,7 @@ mod tests { }, ], space_annotations: [ - LxAnnotationEntry { + VdAnnotationEntry { start: 6, end: 7, annotation: Apply( diff --git a/crates/visored/visored-annotation/src/annotations/walker.rs b/crates/visored/visored-annotation/src/annotations/walker.rs index 96254fc511..5797bf75ab 100644 --- a/crates/visored/visored-annotation/src/annotations/walker.rs +++ b/crates/visored/visored-annotation/src/annotations/walker.rs @@ -1,17 +1,17 @@ -use super::{LxAnnotationEntry, LxSpaceAnnotationEntry, LxTokenAnnotationEntry}; -use crate::annotation::{space::LxSpaceAnnotation, token::LxTokenAnnotation}; +use super::{VdAnnotationEntry, VdSpaceAnnotationEntry, VdTokenAnnotationEntry}; +use crate::annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}; -pub struct LxAnnotationsWalker<'a> { - token_annotations: &'a [LxTokenAnnotationEntry], - space_annotations: &'a [LxSpaceAnnotationEntry], +pub struct VdAnnotationsWalker<'a> { + token_annotations: &'a [VdTokenAnnotationEntry], + space_annotations: &'a [VdSpaceAnnotationEntry], next_token_annotation_index: usize, next_space_annotation_index: usize, } -impl<'a> LxAnnotationsWalker<'a> { +impl<'a> VdAnnotationsWalker<'a> { pub fn new( - token_annotations: &'a [LxTokenAnnotationEntry], - space_annotations: &'a [LxSpaceAnnotationEntry], + token_annotations: &'a [VdTokenAnnotationEntry], + space_annotations: &'a [VdSpaceAnnotationEntry], ) -> Self { Self { token_annotations, @@ -22,15 +22,15 @@ impl<'a> LxAnnotationsWalker<'a> { } } -impl<'a> LxAnnotationsWalker<'a> { - pub fn next(&mut self, start: usize, end: usize) -> (LxTokenAnnotation, LxSpaceAnnotation) { +impl<'a> VdAnnotationsWalker<'a> { + pub fn next(&mut self, start: usize, end: usize) -> (VdTokenAnnotation, VdSpaceAnnotation) { ( self.next_token_annotation(start, end), self.next_space_annotation(start, end), ) } - fn next_token_annotation(&mut self, start: usize, end: usize) -> LxTokenAnnotation { + fn next_token_annotation(&mut self, start: usize, end: usize) -> VdTokenAnnotation { next_annotation_aux( start, end, @@ -39,7 +39,7 @@ impl<'a> LxAnnotationsWalker<'a> { ) } - fn next_space_annotation(&mut self, start: usize, end: usize) -> LxSpaceAnnotation { + fn next_space_annotation(&mut self, start: usize, end: usize) -> VdSpaceAnnotation { next_annotation_aux( start, end, @@ -52,7 +52,7 @@ impl<'a> LxAnnotationsWalker<'a> { fn next_annotation_aux( start: usize, end: usize, - token_annotations: &[LxAnnotationEntry], + token_annotations: &[VdAnnotationEntry], next_token_annotation_index: &mut usize, ) -> A { if *next_token_annotation_index >= token_annotations.len() { @@ -76,30 +76,30 @@ fn next_annotation_aux( #[cfg(test)] mod tests { use super::*; - use crate::annotation::token::{LxIntegralAnnotation, LxTokenAnnotation, LxVariableAnnotation}; + use crate::annotation::token::{LxIntegralAnnotation, LxVariableAnnotation, VdTokenAnnotation}; use crate::{ - annotation::space::{LxApplyAnnotation, LxSpaceAnnotation}, - annotations::LxAnnotations, + annotation::space::{LxApplyAnnotation, VdSpaceAnnotation}, + annotations::VdAnnotations, }; - fn setup_test_data() -> LxAnnotations { + fn setup_test_data() -> VdAnnotations { let input = "\\int xdx".to_string(); let token_annotations = vec![ ( ("", "\\int"), - LxTokenAnnotation::Integral( + VdTokenAnnotation::Integral( LxIntegralAnnotation::SingleVariableIndefiniteIntegralOverReal, ), ), ( ("\\int ", "x"), - LxTokenAnnotation::Variable(LxVariableAnnotation::Usage), + VdTokenAnnotation::Variable(LxVariableAnnotation::Usage), ), - (("\\int x", "d"), LxTokenAnnotation::Differential), + (("\\int x", "d"), VdTokenAnnotation::Differential), ( ("\\int xd", "x"), - LxTokenAnnotation::Variable( + VdTokenAnnotation::Variable( LxVariableAnnotation::SingleVariableIntegralVariableDecl, ), ), @@ -107,10 +107,10 @@ mod tests { let space_annotations = vec![( ("\\int x", "d"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), + VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), )]; - LxAnnotations::from_sparse( + VdAnnotations::from_sparse( &input, token_annotations.into_iter(), space_annotations.into_iter(), @@ -126,10 +126,10 @@ mod tests { assert_eq!( walker.next(0, 4), ( - LxTokenAnnotation::Integral( + VdTokenAnnotation::Integral( LxIntegralAnnotation::SingleVariableIndefiniteIntegralOverReal ), - LxSpaceAnnotation::default() + VdSpaceAnnotation::default() ) ); @@ -137,8 +137,8 @@ mod tests { assert_eq!( walker.next(5, 6), ( - LxTokenAnnotation::Variable(LxVariableAnnotation::Usage), - LxSpaceAnnotation::Void + VdTokenAnnotation::Variable(LxVariableAnnotation::Usage), + VdSpaceAnnotation::Void ) ); @@ -146,8 +146,8 @@ mod tests { assert_eq!( walker.next(6, 7), ( - LxTokenAnnotation::Differential, - LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul) + VdTokenAnnotation::Differential, + VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul) ) ); } diff --git a/crates/visored/visored-annotation/src/test_helpers/example.rs b/crates/visored/visored-annotation/src/test_helpers/example.rs index 70954c89e5..72f2486f72 100644 --- a/crates/visored/visored-annotation/src/test_helpers/example.rs +++ b/crates/visored/visored-annotation/src/test_helpers/example.rs @@ -1,25 +1,25 @@ use crate::{ annotation::{ - space::{LxApplyAnnotation, LxSpaceAnnotation}, - token::{LxTokenAnnotation, LxVariableAnnotation}, + space::{LxApplyAnnotation, VdSpaceAnnotation}, + token::{LxVariableAnnotation, VdTokenAnnotation}, }, - annotations::LxAnnotations, + annotations::VdAnnotations, }; #[derive(Debug, Clone)] -pub struct LxAnnotationsExample { +pub struct VdAnnotationsExample { pub root_mode: LxMode, pub input: String, - pub annotations: LxAnnotations, + pub annotations: VdAnnotations, } -impl LxAnnotationsExample { +impl VdAnnotationsExample { fn collect_from_sparse( examples: &[( LxMode, &str, - &[((&str, &str), LxTokenAnnotation)], - &[((&str, &str), LxSpaceAnnotation)], + &[((&str, &str), VdTokenAnnotation)], + &[((&str, &str), VdSpaceAnnotation)], )], ) -> Vec { examples @@ -28,7 +28,7 @@ impl LxAnnotationsExample { |&(root_mode, input, token_annotations, space_annotations)| Self { root_mode, input: input.to_string(), - annotations: LxAnnotations::from_sparse( + annotations: VdAnnotations::from_sparse( input, token_annotations.iter().copied(), space_annotations.iter().copied(), @@ -43,8 +43,8 @@ use latex_prelude::mode::LxMode; use lazy_static::lazy_static; lazy_static! { - pub static ref LX_ANNOTATIONS_EXAMPLES: Vec = - LxAnnotationsExample::collect_from_sparse(&[ + pub static ref LX_ANNOTATIONS_EXAMPLES: Vec = + VdAnnotationsExample::collect_from_sparse(&[ (LxMode::Math, "", &[], &[]), ( LxMode::Math, @@ -52,26 +52,26 @@ lazy_static! { &[ ( ("", "x"), - LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) + VdTokenAnnotation::Variable(LxVariableAnnotation::Usage) ), ( ("x", "y"), - LxTokenAnnotation::Variable(LxVariableAnnotation::Usage) + VdTokenAnnotation::Variable(LxVariableAnnotation::Usage) ), ], &[( ("x", "y"), - LxSpaceAnnotation::Apply(LxApplyAnnotation::ScalarMul) + VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarMul) ),] ), ( LxMode::Math, "dx", &[ - (("", "d"), LxTokenAnnotation::Differential), + (("", "d"), VdTokenAnnotation::Differential), ( ("d", "x"), - LxTokenAnnotation::Variable( + VdTokenAnnotation::Variable( LxVariableAnnotation::SingleVariableIntegralVariableDecl ) ), From f98715fa511bd9952a624c45f5d2b68f1d556cbe Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 10:11:19 -0700 Subject: [PATCH 20/71] save --- crates/latex/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/latex/README.md b/crates/latex/README.md index 5c07e1d35d..37dd8aa8e7 100644 --- a/crates/latex/README.md +++ b/crates/latex/README.md @@ -1,4 +1,4 @@ -# HuskyTex +# LaTeX In the normal language, text -> token -> ast -> syn expr -> sema expr. @@ -8,15 +8,15 @@ Further text <--> token <--> ast is totally handled automatically. Basically, the user modify directly the asts. -HuskyTex asts is centered around displayability, what looks the same is processed the same way. +LaTex asts is centered around displayability, what looks the same is processed the same way. -HuskyTex differs from LaTex in that +We differs from standard implementation of LaTex in that - greek letters like `α`, `β`, ... and special symbols like `∀`, `∃` are allowed for better readability; - `{...}` in math mode can be interpreted directly as set notation if the content inside doesn't begin with `_` or `^`; - all `/` is interpreted as fraction, and thus no fraction command is needed for faster typing; - ... -However, HuskyTex is designed to be as backward-compatible with LaTex as possible for easier data acquirement. +However, LaTex is designed to be as a limited yet more structured subset of LaTex as possible for easier data acquirement. It should yield meaningful error messages for AI agents to fix. Modes: - root. From ec751ea0b33c8dc6d62e519d4e087627c7e48687 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 10:12:10 -0700 Subject: [PATCH 21/71] save --- crates/latex/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/latex/README.md b/crates/latex/README.md index 37dd8aa8e7..7e5d69989b 100644 --- a/crates/latex/README.md +++ b/crates/latex/README.md @@ -1,4 +1,4 @@ -# LaTeX +# LaTeX (the Husky version) In the normal language, text -> token -> ast -> syn expr -> sema expr. @@ -16,7 +16,7 @@ We differs from standard implementation of LaTex in that - all `/` is interpreted as fraction, and thus no fraction command is needed for faster typing; - ... -However, LaTex is designed to be as a limited yet more structured subset of LaTex as possible for easier data acquirement. It should yield meaningful error messages for AI agents to fix. +Our LaTex is designed to be as a limited yet more structured subset of LaTex as possible for easier data acquirement. It should yield meaningful error messages for AI agents to fix. We'll gradually add more features to it as we see fit. Modes: - root. From 1c6630ce68a95692bc6b0acd138a0513d3a43a97 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 10:12:22 -0700 Subject: [PATCH 22/71] save --- crates/latex/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/latex/README.md b/crates/latex/README.md index 7e5d69989b..bb95ecf110 100644 --- a/crates/latex/README.md +++ b/crates/latex/README.md @@ -10,7 +10,7 @@ Basically, the user modify directly the asts. LaTex asts is centered around displayability, what looks the same is processed the same way. -We differs from standard implementation of LaTex in that +We differs from the standard implementation of LaTex in that - greek letters like `α`, `β`, ... and special symbols like `∀`, `∃` are allowed for better readability; - `{...}` in math mode can be interpreted directly as set notation if the content inside doesn't begin with `_` or `^`; - all `/` is interpreted as fraction, and thus no fraction command is needed for faster typing; From d579ca0fe27e94342933321d78c6a749aabb8441 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 10:46:00 -0700 Subject: [PATCH 23/71] refactor lexer peek --- crates/latex/latex-ast/src/ast.rs | 8 +- crates/latex/latex-ast/src/ast/math.rs | 45 +++---- crates/latex/latex-ast/src/parser.rs | 29 +++-- crates/latex/latex-token/src/data.rs | 9 -- crates/latex/latex-token/src/data/math.rs | 7 +- crates/latex/latex-token/src/data/rose.rs | 8 +- crates/latex/latex-token/src/lexer.rs | 134 ++++++++++++-------- crates/latex/latex-token/src/storage.rs | 5 - crates/latex/latex-token/src/stream.rs | 32 +---- crates/latex/latex-token/src/stream/math.rs | 19 +++ crates/latex/latex-token/src/stream/rose.rs | 19 +++ 11 files changed, 173 insertions(+), 142 deletions(-) create mode 100644 crates/latex/latex-token/src/stream/math.rs create mode 100644 crates/latex/latex-token/src/stream/rose.rs diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 533fb24f0c..fbcaddc2e8 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -130,12 +130,10 @@ impl<'a> LxAstParser<'a> { fn parse_math_ast(&mut self) -> Option { let mut ast = self.parse_atomic_math_ast()?; - match self.peek_char()? { + match self.peek_math_token_data()? { // TODO include more cases, like \limits - '_' | '^' => { - let (idx, LxTokenData::Math(token)) = self.next_token().unwrap() else { - unreachable!() - }; + LxMathTokenData::Subscript | LxMathTokenData::Superscript => { + let (idx, token) = self.next_math_token().unwrap(); ast = match ast { LxMathAstData::Attach { .. } => ast, base => { diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 56072d07c5..dec9183c6f 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -36,13 +36,11 @@ pub type LxMathAstIdxRange = ArenaIdxRange; impl<'a> LxAstParser<'a> { pub(super) fn parse_atomic_math_ast(&mut self) -> Option { - match self.peek_char()? { - '}' | '$' => return None, + match self.peek_math_token_data()? { + LxMathTokenData::RightDelimiter(_) | LxMathTokenData::MathModeEnd => return None, _ => (), }; - let (idx, LxTokenData::Math(token)) = self.next_token()? else { - unreachable!() - }; + let (idx, token) = self.next_math_token()?; Some(match token { LxMathTokenData::Command(_) => todo!(), LxMathTokenData::LeftDelimiter(delimiter) => self.parse_delimited(idx, delimiter), @@ -54,6 +52,7 @@ impl<'a> LxAstParser<'a> { LxMathTokenData::Subscript => todo!(), LxMathTokenData::Superscript => todo!(), LxMathTokenData::Error(_) => todo!(), + LxMathTokenData::MathModeEnd => unreachable!(), }) } @@ -64,29 +63,27 @@ impl<'a> LxAstParser<'a> { left_delimiter: LxMathDelimiter, ) -> LxMathAstData { let asts = self.parse_math_asts(); - let Some((idx, token)) = self.next_token() else { + let Some((idx, token)) = self.next_math_token() else { todo!() }; match token { - LxTokenData::Math(token) => match token { - LxMathTokenData::Command(_) => todo!(), - LxMathTokenData::LeftDelimiter(_) => todo!(), - LxMathTokenData::RightDelimiter(right_delimiter) => LxMathAstData::Delimited { - left_delimiter_token_idx, - left_delimiter, - asts, - right_delimiter_token_idx: idx, - right_delimiter, - }, - LxMathTokenData::Letter(_) => todo!(), - LxMathTokenData::Punctuation(_) => todo!(), - LxMathTokenData::Digit(_) => todo!(), - LxMathTokenData::Other(_) => todo!(), - LxMathTokenData::Subscript => todo!(), - LxMathTokenData::Superscript => todo!(), - LxMathTokenData::Error(_) => todo!(), + LxMathTokenData::Command(_) => todo!(), + LxMathTokenData::LeftDelimiter(_) => todo!(), + LxMathTokenData::RightDelimiter(right_delimiter) => LxMathAstData::Delimited { + left_delimiter_token_idx, + left_delimiter, + asts, + right_delimiter_token_idx: idx, + right_delimiter, }, - LxTokenData::Rose(_) => todo!(), + LxMathTokenData::Letter(_) => todo!(), + LxMathTokenData::Punctuation(_) => todo!(), + LxMathTokenData::Digit(_) => todo!(), + LxMathTokenData::Other(_) => todo!(), + LxMathTokenData::Subscript => todo!(), + LxMathTokenData::Superscript => todo!(), + LxMathTokenData::Error(_) => todo!(), + LxMathTokenData::MathModeEnd => todo!(), } } } diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index 2a080f2fdc..e77766e6df 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -8,14 +8,17 @@ use crate::{ }; use latex_prelude::mode::LxMode; use latex_token::{ - data::LxTokenData, idx::LxTokenIdx, lexer::LxLexer, storage::LxTokenStorage, - stream::LxTokenStream, + data::{math::LxMathTokenData, rose::LxRoseTokenData, LxTokenData}, + idx::LxTokenIdx, + lexer::LxLexer, + storage::LxTokenStorage, }; use std::{borrow::BorrowMut, iter::Peekable}; pub(crate) struct LxAstParser<'a> { db: &'a ::salsa::Db, lexer: LxLexer<'a>, + mode: LxMode, arena: &'a mut LxAstArena, } @@ -30,13 +33,14 @@ impl<'a> LxAstParser<'a> { ) -> Self { Self { db, - lexer: LxLexer::new(db, input, mode, token_storage), + lexer: LxLexer::new(db, input, token_storage), + mode, arena, } } pub(crate) fn mode(&self) -> LxMode { - self.lexer.mode() + self.mode } } @@ -58,12 +62,19 @@ impl<'a> LxAstParser<'a> { self.arena.rose.alloc_batch(asts) } - pub(crate) fn peek_char(&mut self) -> Option { - self.lexer.peek_char() + pub(crate) fn peek_math_token_data(&mut self) -> Option { + self.lexer.peek_math_token_data() } - pub(crate) fn next_token(&mut self) -> Option<(LxTokenIdx, LxTokenData)> { - let (token_idx, _, _, token_data) = self.lexer.next()?; - Some((token_idx, token_data)) + pub(crate) fn peek_rose_token_data(&mut self) -> Option { + self.lexer.peek_rose_token_data() + } + + pub(crate) fn next_math_token(&mut self) -> Option<(LxTokenIdx, LxMathTokenData)> { + self.lexer.next_math_token() + } + + pub(crate) fn next_rose_token(&mut self) -> Option<(LxTokenIdx, LxRoseTokenData)> { + self.lexer.next_rose_token() } } diff --git a/crates/latex/latex-token/src/data.rs b/crates/latex/latex-token/src/data.rs index 5f14a1ba43..9ffa03572b 100644 --- a/crates/latex/latex-token/src/data.rs +++ b/crates/latex/latex-token/src/data.rs @@ -14,12 +14,3 @@ pub enum LxTokenData { Math(LxMathTokenData), Rose(LxRoseTokenData), } - -impl<'a> LxLexer<'a> { - pub(crate) fn next_token_data(&mut self) -> Option { - match self.mode { - LxMode::Rose => self.next_text_token_data().map(Into::into), - LxMode::Math => self.next_math_token_data().map(Into::into), - } - } -} diff --git a/crates/latex/latex-token/src/data/math.rs b/crates/latex/latex-token/src/data/math.rs index 7c1692c4f2..301aaab8ae 100644 --- a/crates/latex/latex-token/src/data/math.rs +++ b/crates/latex/latex-token/src/data/math.rs @@ -19,6 +19,7 @@ pub enum LxMathTokenData { Subscript, Superscript, Error(LxMathTokenError), + MathModeEnd, } #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -34,7 +35,7 @@ pub enum LxMathTokenError { } impl<'a> LxLexer<'a> { - pub(super) fn next_math_token_data(&mut self) -> Option { + pub(crate) fn next_math_token_data(&mut self) -> Option { match self.chars.peek()? { '\\' => { self.chars.eat_char(); @@ -92,8 +93,8 @@ fn next_text_token_data_works() { fn t(input: &str, expected: &Expect) { let db = &DB::default(); let mut storage = LxTokenStorage::default(); - let tokenizer = LxLexer::new(db, input, LxMode::Math, &mut storage); - let tokens: Vec<_> = tokenizer.map(|(_, _, _, token_data)| token_data).collect(); + let stream = LxLexer::new(db, input, &mut storage).into_math_stream(); + let tokens: Vec<_> = stream.map(|(_, token_data)| token_data).collect(); expected.assert_debug_eq(&(tokens.debug(db))); } t( diff --git a/crates/latex/latex-token/src/data/rose.rs b/crates/latex/latex-token/src/data/rose.rs index e510a4167c..0aae7969d8 100644 --- a/crates/latex/latex-token/src/data/rose.rs +++ b/crates/latex/latex-token/src/data/rose.rs @@ -19,7 +19,7 @@ pub enum LxRoseTokenData { } impl<'a> LxLexer<'a> { - pub(super) fn next_text_token_data(&mut self) -> Option { + pub(crate) fn next_rose_token_data(&mut self) -> Option { match self.chars.peek()? { '\\' => { self.chars.eat_char(); @@ -65,8 +65,10 @@ fn next_text_token_data_works() { fn t(input: &str, expected: &Expect) { let db = &DB::default(); let mut storage = LxTokenStorage::default(); - let lexer = LxLexer::new(db, input, LxMode::Rose, &mut storage); - let tokens: Vec<_> = lexer.map(|(_, _, _, token_data)| token_data).collect(); + let mut stream = LxLexer::new(db, input, &mut storage) + .into_rose_stream() + .map(|(_, token_data)| token_data); + let mut tokens: Vec<_> = stream.collect(); expected.assert_debug_eq(&(tokens.debug(db))); } t( diff --git a/crates/latex/latex-token/src/lexer.rs b/crates/latex/latex-token/src/lexer.rs index 31d9055010..cecc93cf2b 100644 --- a/crates/latex/latex-token/src/lexer.rs +++ b/crates/latex/latex-token/src/lexer.rs @@ -6,6 +6,7 @@ use crate::{ }, idx::LxTokenIdx, storage::LxTokenStorage, + stream::{math::LxMathTokenStream, rose::LxRoseTokenStream}, }; use husky_coword::Coword; use husky_text_protocol::{char::TextCharIter, range::TextRange}; @@ -14,92 +15,117 @@ use latex_prelude::mode::LxMode; pub struct LxLexer<'a> { pub(crate) db: &'a ::salsa::Db, pub(crate) chars: TextCharIter<'a>, - pub(crate) mode: LxMode, pub(crate) storage: &'a mut LxTokenStorage, } /// # constructor impl<'a> LxLexer<'a> { - pub fn new( - db: &'a ::salsa::Db, - input: &'a str, - mode: LxMode, - storage: &'a mut LxTokenStorage, - ) -> Self { + pub fn new(db: &'a ::salsa::Db, input: &'a str, storage: &'a mut LxTokenStorage) -> Self { Self { db, chars: TextCharIter::new(input), - mode, storage, } } - - pub fn peek_char(&mut self) -> Option { - self.chars.eat_chars_while(|c| c == ' '); - self.chars.peek() - } - - pub fn mode(&self) -> LxMode { - self.mode - } } /// # actions impl<'a> LxLexer<'a> { - pub(crate) fn next_coword_with(&mut self, predicate: impl Fn(char) -> bool) -> Option { - let coword_str_slice = self.chars.next_str_slice_while(|c| c.is_alphanumeric()); - if coword_str_slice.is_empty() { - return None; - } - Some(Coword::from_ref(self.db, coword_str_slice)) + pub fn next_math_token(&mut self) -> Option<(LxTokenIdx, LxMathTokenData)> { + let ((start_offset, end_offset), range, token_data) = self.next_math_token_aux()?; + Some(( + self.storage + .alloc(start_offset, end_offset, range, token_data.into()), + token_data, + )) } -} -impl<'a> Iterator for LxLexer<'a> { - type Item = (LxTokenIdx, (usize, usize), TextRange, LxTokenData); - - fn next(&mut self) -> Option { + fn next_math_token_aux(&mut self) -> Option<((usize, usize), TextRange, LxMathTokenData)> { self.chars.eat_chars_while(|c| c == ' '); let mut start_offset = self.chars.current_offset(); let mut start_position = self.chars.current_position(); - let token_data = if self.chars.eat_char_if(|c| c == '\n') { self.chars.eat_chars_while(|c| c == ' '); - match self.mode { - LxMode::Rose => { - if self.chars.eat_char_if(|c| c == '\n') { - self.chars.eat_chars_while(|c| c == '\n' || c == ' '); - LxRoseTokenData::NewParagraph.into() - } else { - start_offset = self.chars.current_offset(); - start_position = self.chars.current_position(); - self.next_token_data()? - } - } - LxMode::Math => { - if self.chars.eat_char_if(|c| c == '\n') { - LxMathTokenData::Error(LxMathTokenError::UnexpectedNewParagraph).into() - } else { - start_offset = self.chars.current_offset(); - start_position = self.chars.current_position(); - self.next_token_data()? - } - } + if self.chars.eat_char_if(|c| c == '\n') { + Some(LxMathTokenData::Error( + LxMathTokenError::UnexpectedNewParagraph, + )) + } else { + self.next_math_token_data() } } else { - self.next_token_data()? - }; + self.next_math_token_data() + }?; let end_offset = self.chars.current_offset(); let range = TextRange { start: start_position, end: self.chars.current_position(), }; + Some(((start_offset, end_offset), range, token_data)) + } + + pub fn peek_math_token_data(&mut self) -> Option { + let chars = self.chars.clone(); + let (_, _, token_data) = self.next_math_token_aux()?; + self.chars = chars; + Some(token_data) + } + + pub fn next_rose_token(&mut self) -> Option<(LxTokenIdx, LxRoseTokenData)> { + let ((start_offset, end_offset), range, token_data) = self.next_rose_token_aux()?; Some(( self.storage - .alloc(start_offset, end_offset, range, token_data), - (start_offset, end_offset), - range, + .alloc(start_offset, end_offset, range, token_data.into()), token_data, )) } + + fn next_rose_token_aux(&mut self) -> Option<((usize, usize), TextRange, LxRoseTokenData)> { + self.chars.eat_chars_while(|c| c == ' '); + let mut start_offset = self.chars.current_offset(); + let mut start_position = self.chars.current_position(); + + let token_data = if self.chars.eat_char_if(|c| c == '\n') { + self.chars.eat_chars_while(|c| c == ' '); + if self.chars.eat_char_if(|c| c == '\n') { + self.chars.eat_chars_while(|c| c == '\n' || c == ' '); + LxRoseTokenData::NewParagraph + } else { + start_offset = self.chars.current_offset(); + start_position = self.chars.current_position(); + self.next_rose_token_data()? + } + } else { + self.next_rose_token_data()? + }; + let end_offset = self.chars.current_offset(); + let range = TextRange { + start: start_position, + end: self.chars.current_position(), + }; + Some(((start_offset, end_offset), range, token_data)) + } + + pub fn peek_rose_token_data(&mut self) -> Option { + let chars = self.chars.clone(); + let (_, _, token_data) = self.next_rose_token_aux()?; + self.chars = chars; + Some(token_data) + } + + pub(crate) fn next_coword_with(&mut self, predicate: impl Fn(char) -> bool) -> Option { + let coword_str_slice = self.chars.next_str_slice_while(|c| c.is_alphanumeric()); + if coword_str_slice.is_empty() { + return None; + } + Some(Coword::from_ref(self.db, coword_str_slice)) + } + + pub fn into_math_stream(self) -> LxMathTokenStream<'a> { + LxMathTokenStream::new(self) + } + + pub(crate) fn into_rose_stream(self) -> LxRoseTokenStream<'a> { + LxRoseTokenStream::new(self) + } } diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index 88aa6d873c..56105639ed 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -1,7 +1,6 @@ use crate::{ data::LxTokenData, idx::{LxTokenIdx, LxTokenIdxRange}, - stream::LxTokenStream, }; use husky_text_protocol::range::TextRange; @@ -33,10 +32,6 @@ impl LxTokenStorage { pub fn token_text_range(&self, token_idx: LxTokenIdx) -> TextRange { self.ranged_tokens[token_idx.index()].1 } - - pub fn stream(&self) -> LxTokenStream { - LxTokenStream::new(&self.ranged_tokens) - } } /// # actions diff --git a/crates/latex/latex-token/src/stream.rs b/crates/latex/latex-token/src/stream.rs index 2b3b909f37..7fa0fff207 100644 --- a/crates/latex/latex-token/src/stream.rs +++ b/crates/latex/latex-token/src/stream.rs @@ -1,30 +1,2 @@ -use crate::{data::LxTokenData, idx::LxTokenIdx}; -use husky_text_protocol::range::TextRange; - -pub struct LxTokenStream<'a> { - ranged_tokens: &'a [((usize, usize), TextRange, LxTokenData)], - current_idx: usize, -} - -impl<'a> LxTokenStream<'a> { - pub(crate) fn new(ranged_tokens: &'a [((usize, usize), TextRange, LxTokenData)]) -> Self { - Self { - ranged_tokens, - current_idx: 0, - } - } -} - -impl<'a> Iterator for LxTokenStream<'a> { - type Item = (LxTokenIdx, (usize, usize), TextRange, LxTokenData); - - fn next(&mut self) -> Option { - if self.current_idx >= self.ranged_tokens.len() { - return None; - } - let idx = self.current_idx; - let (offset_range, text_range, data) = self.ranged_tokens[idx]; - self.current_idx += 1; - Some((LxTokenIdx::from_index(idx), offset_range, text_range, data)) - } -} +pub mod math; +pub mod rose; diff --git a/crates/latex/latex-token/src/stream/math.rs b/crates/latex/latex-token/src/stream/math.rs new file mode 100644 index 0000000000..787b685a64 --- /dev/null +++ b/crates/latex/latex-token/src/stream/math.rs @@ -0,0 +1,19 @@ +use crate::{data::math::LxMathTokenData, idx::LxTokenIdx, lexer::LxLexer}; + +pub struct LxMathTokenStream<'a> { + lexer: LxLexer<'a>, +} + +impl<'a> LxMathTokenStream<'a> { + pub fn new(lexer: LxLexer<'a>) -> Self { + Self { lexer } + } +} + +impl<'a> Iterator for LxMathTokenStream<'a> { + type Item = (LxTokenIdx, LxMathTokenData); + + fn next(&mut self) -> Option { + self.lexer.next_math_token() + } +} diff --git a/crates/latex/latex-token/src/stream/rose.rs b/crates/latex/latex-token/src/stream/rose.rs new file mode 100644 index 0000000000..00f7bc447f --- /dev/null +++ b/crates/latex/latex-token/src/stream/rose.rs @@ -0,0 +1,19 @@ +use crate::{data::rose::LxRoseTokenData, idx::LxTokenIdx, lexer::LxLexer}; + +pub struct LxRoseTokenStream<'a> { + lexer: LxLexer<'a>, +} + +impl<'a> LxRoseTokenStream<'a> { + pub fn new(lexer: LxLexer<'a>) -> Self { + Self { lexer } + } +} + +impl<'a> Iterator for LxRoseTokenStream<'a> { + type Item = (LxTokenIdx, LxRoseTokenData); + + fn next(&mut self) -> Option { + self.lexer.next_rose_token() + } +} From e3ec0c08df19031f42b2aa34ac30ae62156853bf Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 11:00:30 -0700 Subject: [PATCH 24/71] save --- crates/latex/latex-ast/src/ast.rs | 139 ++++------ crates/latex/latex-ast/src/ast/math.rs | 14 +- crates/latex/latex-ast/src/ast/rose.rs | 4 +- crates/latex/latex-ast/src/parser.rs | 8 +- crates/latex/latex-ast/src/range.rs | 41 ++- crates/latex/latex-token/src/data.rs | 8 - crates/latex/latex-token/src/data/math.rs | 272 +++++++------------- crates/latex/latex-token/src/data/rose.rs | 80 ++---- crates/latex/latex-token/src/idx.rs | 84 +----- crates/latex/latex-token/src/idx/math.rs | 82 ++++++ crates/latex/latex-token/src/idx/rose.rs | 82 ++++++ crates/latex/latex-token/src/lexer.rs | 11 +- crates/latex/latex-token/src/storage.rs | 57 ++-- crates/latex/latex-token/src/stream/math.rs | 4 +- crates/latex/latex-token/src/stream/rose.rs | 4 +- 15 files changed, 432 insertions(+), 458 deletions(-) create mode 100644 crates/latex/latex-token/src/idx/math.rs create mode 100644 crates/latex/latex-token/src/idx/rose.rs diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index fbcaddc2e8..5764f34a0f 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -16,7 +16,6 @@ use latex_token::{ data::{ math::{LxMathDelimiter, LxMathTokenData}, rose::LxRoseTokenData, - LxTokenData, }, storage::LxTokenStorage, }; @@ -187,7 +186,8 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_tokens: [], + ranged_math_tokens: [], + ranged_rose_tokens: [], }, LxAstArena { math: Arena { @@ -211,20 +211,19 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_tokens: [ + ranged_math_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Math( - Letter( - LowerX, - ), + Letter( + LowerX, ), ), ], + ranged_rose_tokens: [], }, LxAstArena { math: Arena { @@ -248,17 +247,15 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_tokens: [ + ranged_math_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Math( - Letter( - LowerX, - ), + Letter( + LowerX, ), ), ( @@ -267,10 +264,8 @@ fn parse_tex_input_into_asts_works() { 2, ), [1:2, 1:3), - Math( - Punctuation( - Add, - ), + Punctuation( + Add, ), ), ( @@ -279,25 +274,24 @@ fn parse_tex_input_into_asts_works() { 3, ), [1:3, 1:4), - Math( - Digit( - One, - ), + Digit( + One, ), ), ], + ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( - LxTokenIdx( + LxMathTokenIdx( 0, ), LowerX, ), LxMathAstData::Opr( - LxTokenIdx( + LxMathTokenIdx( 1, ), Add, @@ -322,17 +316,15 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_tokens: [ + ranged_math_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Math( - Letter( - LowerX, - ), + Letter( + LowerX, ), ), ( @@ -341,9 +333,7 @@ fn parse_tex_input_into_asts_works() { 2, ), [1:2, 1:3), - Math( - Superscript, - ), + Superscript, ), ( ( @@ -351,25 +341,24 @@ fn parse_tex_input_into_asts_works() { 3, ), [1:3, 1:4), - Math( - Digit( - Two, - ), + Digit( + Two, ), ), ], + ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( - LxTokenIdx( + LxMathTokenIdx( 0, ), LowerX, ), LxMathAstData::Digit( - LxTokenIdx( + LxMathTokenIdx( 2, ), Two, @@ -403,17 +392,15 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_tokens: [ + ranged_math_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Math( - Letter( - LowerX, - ), + Letter( + LowerX, ), ), ( @@ -422,9 +409,7 @@ fn parse_tex_input_into_asts_works() { 2, ), [1:2, 1:3), - Math( - Subscript, - ), + Subscript, ), ( ( @@ -432,25 +417,24 @@ fn parse_tex_input_into_asts_works() { 3, ), [1:3, 1:4), - Math( - Digit( - Two, - ), + Digit( + Two, ), ), ], + ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( - LxTokenIdx( + LxMathTokenIdx( 0, ), LowerX, ), LxMathAstData::Digit( - LxTokenIdx( + LxMathTokenIdx( 2, ), Two, @@ -484,17 +468,15 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_tokens: [ + ranged_math_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Math( - Letter( - LowerX, - ), + Letter( + LowerX, ), ), ( @@ -503,9 +485,7 @@ fn parse_tex_input_into_asts_works() { 2, ), [1:2, 1:3), - Math( - Superscript, - ), + Superscript, ), ( ( @@ -513,10 +493,8 @@ fn parse_tex_input_into_asts_works() { 3, ), [1:3, 1:4), - Math( - LeftDelimiter( - Curl, - ), + LeftDelimiter( + Curl, ), ), ( @@ -525,10 +503,8 @@ fn parse_tex_input_into_asts_works() { 4, ), [1:4, 1:5), - Math( - Letter( - LowerI, - ), + Letter( + LowerI, ), ), ( @@ -537,10 +513,8 @@ fn parse_tex_input_into_asts_works() { 5, ), [1:5, 1:6), - Math( - Punctuation( - Add, - ), + Punctuation( + Add, ), ), ( @@ -549,10 +523,8 @@ fn parse_tex_input_into_asts_works() { 6, ), [1:6, 1:7), - Math( - Digit( - Two, - ), + Digit( + Two, ), ), ( @@ -561,50 +533,49 @@ fn parse_tex_input_into_asts_works() { 7, ), [1:7, 1:8), - Math( - RightDelimiter( - Curl, - ), + RightDelimiter( + Curl, ), ), ], + ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( - LxTokenIdx( + LxMathTokenIdx( 0, ), LowerX, ), LxMathAstData::Letter( - LxTokenIdx( + LxMathTokenIdx( 3, ), LowerI, ), LxMathAstData::Opr( - LxTokenIdx( + LxMathTokenIdx( 4, ), Add, ), LxMathAstData::Digit( - LxTokenIdx( + LxMathTokenIdx( 5, ), Two, ), LxMathAstData::Delimited { - left_delimiter_token_idx: LxTokenIdx( + left_delimiter_token_idx: LxMathTokenIdx( 2, ), left_delimiter: Curl, asts: ArenaIdxRange( 1..4, ), - right_delimiter_token_idx: LxTokenIdx( + right_delimiter_token_idx: LxMathTokenIdx( 6, ), right_delimiter: Curl, diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index dec9183c6f..0d05197a76 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -1,6 +1,6 @@ use latex_token::{ data::math::{digit::LxMathDigit, LxMathDelimiter}, - idx::LxTokenIdx, + idx::math::LxMathTokenIdx, }; use super::*; @@ -8,9 +8,9 @@ use super::*; #[salsa::derive_debug_with_db] #[derive(Debug, Clone, PartialEq, Eq)] pub enum LxMathAstData { - Letter(LxTokenIdx, LxMathLetter), - Opr(LxTokenIdx, LxMathPunctuation), - Digit(LxTokenIdx, LxMathDigit), + Letter(LxMathTokenIdx, LxMathLetter), + Opr(LxMathTokenIdx, LxMathPunctuation), + Digit(LxMathTokenIdx, LxMathDigit), /// not obtained through parsing, but through ui TextEdit { buffer: String, @@ -20,10 +20,10 @@ pub enum LxMathAstData { scripts: Vec<(LxScriptKind, LxMathAstIdx)>, }, Delimited { - left_delimiter_token_idx: LxTokenIdx, + left_delimiter_token_idx: LxMathTokenIdx, left_delimiter: LxMathDelimiter, asts: LxMathAstIdxRange, - right_delimiter_token_idx: LxTokenIdx, + right_delimiter_token_idx: LxMathTokenIdx, right_delimiter: LxMathDelimiter, }, } @@ -59,7 +59,7 @@ impl<'a> LxAstParser<'a> { // here we differ from the latex syntax, we see all possible delimiters as latex delimiters fn parse_delimited( &mut self, - left_delimiter_token_idx: LxTokenIdx, + left_delimiter_token_idx: LxMathTokenIdx, left_delimiter: LxMathDelimiter, ) -> LxMathAstData { let asts = self.parse_math_asts(); diff --git a/crates/latex/latex-ast/src/ast/rose.rs b/crates/latex/latex-ast/src/ast/rose.rs index 5eeef00f4d..9da7e5189c 100644 --- a/crates/latex/latex-ast/src/ast/rose.rs +++ b/crates/latex/latex-ast/src/ast/rose.rs @@ -1,6 +1,6 @@ //! means the prose mode use super::*; -use latex_token::{data::rose::LxRoseTokenData, idx::LxTokenIdx}; +use latex_token::{data::rose::LxRoseTokenData, idx::rose::LxRoseTokenIdx}; #[salsa::derive_debug_with_db] #[derive(Debug, Clone, PartialEq, Eq)] @@ -17,7 +17,7 @@ pub type LxRoseAstIdxRange = ArenaIdxRange; impl<'a> LxAstParser<'a> { pub(super) fn parse_atomic_text_ast( &mut self, - idx: LxTokenIdx, + idx: LxRoseTokenIdx, token: LxRoseTokenData, ) -> LxRoseAstData { match token { diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index e77766e6df..51af263772 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -8,8 +8,8 @@ use crate::{ }; use latex_prelude::mode::LxMode; use latex_token::{ - data::{math::LxMathTokenData, rose::LxRoseTokenData, LxTokenData}, - idx::LxTokenIdx, + data::{math::LxMathTokenData, rose::LxRoseTokenData}, + idx::{math::LxMathTokenIdx, rose::LxRoseTokenIdx}, lexer::LxLexer, storage::LxTokenStorage, }; @@ -70,11 +70,11 @@ impl<'a> LxAstParser<'a> { self.lexer.peek_rose_token_data() } - pub(crate) fn next_math_token(&mut self) -> Option<(LxTokenIdx, LxMathTokenData)> { + pub(crate) fn next_math_token(&mut self) -> Option<(LxMathTokenIdx, LxMathTokenData)> { self.lexer.next_math_token() } - pub(crate) fn next_rose_token(&mut self) -> Option<(LxTokenIdx, LxRoseTokenData)> { + pub(crate) fn next_rose_token(&mut self) -> Option<(LxRoseTokenIdx, LxRoseTokenData)> { self.lexer.next_rose_token() } } diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index e533eb8509..16286e7714 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -1,20 +1,31 @@ +use latex_token::idx::{math::LxMathTokenIdxRange, rose::LxRoseTokenIdxRange}; + use crate::ast::{ - math::LxMathAstData, LxAstArena, LxAstArenaMap, LxAstArenaRef, LxAstData, LxAstIdx, + math::{LxMathAstArenaMap, LxMathAstData}, + rose::{LxRoseAstArenaMap, LxRoseAstData}, + LxAstArena, LxAstArenaMap, LxAstArenaRef, LxAstData, LxAstIdx, }; -use latex_token::idx::LxTokenIdxRange; -pub type LxAstTokenIdxRangeMap = LxAstArenaMap; +#[derive(Debug)] +pub struct LxAstTokenIdxRangeMap { + pub(crate) math: LxMathAstArenaMap, + pub(crate) rose: LxRoseAstArenaMap, +} pub fn calc_ast_token_idx_range_map(db: &salsa::Db, arena: &LxAstArena) -> LxAstTokenIdxRangeMap { let mut calculator = LxAstTokenIdxRangeCalculator::new(db, arena); calculator.infer_all(); - calculator.data + LxAstTokenIdxRangeMap { + math: calculator.math_data, + rose: calculator.rose_data, + } } struct LxAstTokenIdxRangeCalculator<'a> { db: &'a ::salsa::Db, ast_arena: LxAstArenaRef<'a>, - data: LxAstArenaMap, + math_data: LxMathAstArenaMap, + rose_data: LxRoseAstArenaMap, } impl<'a> LxAstTokenIdxRangeCalculator<'a> { @@ -22,7 +33,8 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { Self { db, ast_arena: arena.as_arena_ref(), - data: LxAstArenaMap::new(arena), + math_data: LxMathAstArenaMap::new(&arena.math), + rose_data: LxRoseAstArenaMap::new(&arena.rose), } } } @@ -30,15 +42,18 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { impl<'a> LxAstTokenIdxRangeCalculator<'a> { fn infer_all(&mut self) { self.ast_arena.math().indexed_iter().for_each(|(idx, ast)| { - self.data.math.insert_new(idx, self.calc_math_ast(ast)); + self.math_data.insert_new(idx, self.calc_math_ast(ast)); + }); + self.ast_arena.rose().indexed_iter().for_each(|(idx, ast)| { + self.rose_data.insert_new(idx, self.calc_rose_ast(ast)); }); } - fn calc_math_ast(&self, data: &LxMathAstData) -> LxTokenIdxRange { + fn calc_math_ast(&self, data: &LxMathAstData) -> LxMathTokenIdxRange { match *data { - LxMathAstData::Letter(idx, _) => LxTokenIdxRange::new_single(idx), - LxMathAstData::Opr(idx, _) => LxTokenIdxRange::new_single(idx), - LxMathAstData::Digit(idx, _) => LxTokenIdxRange::new_single(idx), + LxMathAstData::Letter(idx, _) => LxMathTokenIdxRange::new_single(idx), + LxMathAstData::Opr(idx, _) => LxMathTokenIdxRange::new_single(idx), + LxMathAstData::Digit(idx, _) => LxMathTokenIdxRange::new_single(idx), LxMathAstData::TextEdit { ref buffer } => todo!(), LxMathAstData::Attach { base, ref scripts } => todo!(), LxMathAstData::Delimited { @@ -50,4 +65,8 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { } => todo!(), } } + + fn calc_rose_ast(&self, data: &LxRoseAstData) -> LxRoseTokenIdxRange { + todo!() + } } diff --git a/crates/latex/latex-token/src/data.rs b/crates/latex/latex-token/src/data.rs index 9ffa03572b..7e3aebcfb0 100644 --- a/crates/latex/latex-token/src/data.rs +++ b/crates/latex/latex-token/src/data.rs @@ -6,11 +6,3 @@ use crate::lexer::LxLexer; #[cfg(test)] use crate::*; use latex_prelude::mode::LxMode; - -#[salsa::derive_debug_with_db] -#[enum_class::from_variants] -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum LxTokenData { - Math(LxMathTokenData), - Rose(LxRoseTokenData), -} diff --git a/crates/latex/latex-token/src/data/math.rs b/crates/latex/latex-token/src/data/math.rs index 301aaab8ae..a0303332f5 100644 --- a/crates/latex/latex-token/src/data/math.rs +++ b/crates/latex/latex-token/src/data/math.rs @@ -101,30 +101,20 @@ fn next_text_token_data_works() { "hello", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Letter( - LowerH, - ), + LxMathTokenData::Letter( + LowerH, ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerE, - ), + LxMathTokenData::Letter( + LowerE, ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerL, - ), + LxMathTokenData::Letter( + LowerL, ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerL, - ), + LxMathTokenData::Letter( + LowerL, ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerO, - ), + LxMathTokenData::Letter( + LowerO, ), ] "#]], @@ -133,10 +123,8 @@ fn next_text_token_data_works() { "0", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Digit( - Zero, - ), + LxMathTokenData::Digit( + Zero, ), ] "#]], @@ -145,10 +133,8 @@ fn next_text_token_data_works() { "0", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Digit( - Zero, - ), + LxMathTokenData::Digit( + Zero, ), ] "#]], @@ -157,15 +143,11 @@ fn next_text_token_data_works() { "0 0", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Digit( - Zero, - ), + LxMathTokenData::Digit( + Zero, ), - LxTokenData::Math( - LxMathTokenData::Digit( - Zero, - ), + LxMathTokenData::Digit( + Zero, ), ] "#]], @@ -174,15 +156,11 @@ fn next_text_token_data_works() { "0\n0", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Digit( - Zero, - ), + LxMathTokenData::Digit( + Zero, ), - LxTokenData::Math( - LxMathTokenData::Digit( - Zero, - ), + LxMathTokenData::Digit( + Zero, ), ] "#]], @@ -191,20 +169,14 @@ fn next_text_token_data_works() { "0\n\n0", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Digit( - Zero, - ), + LxMathTokenData::Digit( + Zero, ), - LxTokenData::Math( - LxMathTokenData::Error( - LxMathTokenError::UnexpectedNewParagraph, - ), + LxMathTokenData::Error( + LxMathTokenError::UnexpectedNewParagraph, ), - LxTokenData::Math( - LxMathTokenData::Digit( - Zero, - ), + LxMathTokenData::Digit( + Zero, ), ] "#]], @@ -212,35 +184,29 @@ fn next_text_token_data_works() { t( "{", &expect![[r#" - [ - LxTokenData::Math( + [ LxMathTokenData::LeftDelimiter( Curl, ), - ), - ] - "#]], + ] + "#]], ); t( "}", &expect![[r#" - [ - LxTokenData::Math( + [ LxMathTokenData::RightDelimiter( Curl, ), - ), - ] - "#]], + ] + "#]], ); t( "(", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Punctuation( - Lpar, - ), + LxMathTokenData::Punctuation( + Lpar, ), ] "#]], @@ -249,10 +215,8 @@ fn next_text_token_data_works() { ")", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Punctuation( - Rpar, - ), + LxMathTokenData::Punctuation( + Rpar, ), ] "#]], @@ -261,10 +225,8 @@ fn next_text_token_data_works() { "[", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Punctuation( - Lbox, - ), + LxMathTokenData::Punctuation( + Lbox, ), ] "#]], @@ -273,10 +235,8 @@ fn next_text_token_data_works() { "]", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Punctuation( - Rbox, - ), + LxMathTokenData::Punctuation( + Rbox, ), ] "#]], @@ -285,10 +245,8 @@ fn next_text_token_data_works() { "\\{", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Punctuation( - EscapedLcurl, - ), + LxMathTokenData::Punctuation( + EscapedLcurl, ), ] "#]], @@ -297,10 +255,8 @@ fn next_text_token_data_works() { "\\}", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Punctuation( - EscapedRcurl, - ), + LxMathTokenData::Punctuation( + EscapedRcurl, ), ] "#]], @@ -309,10 +265,8 @@ fn next_text_token_data_works() { "+", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Punctuation( - Add, - ), + LxMathTokenData::Punctuation( + Add, ), ] "#]], @@ -321,20 +275,14 @@ fn next_text_token_data_works() { "x+1", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Letter( - LowerX, - ), + LxMathTokenData::Letter( + LowerX, ), - LxTokenData::Math( - LxMathTokenData::Punctuation( - Add, - ), + LxMathTokenData::Punctuation( + Add, ), - LxTokenData::Math( - LxMathTokenData::Digit( - One, - ), + LxMathTokenData::Digit( + One, ), ] "#]], @@ -343,36 +291,22 @@ fn next_text_token_data_works() { "x_1^a+1", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Letter( - LowerX, - ), - ), - LxTokenData::Math( - LxMathTokenData::Subscript, - ), - LxTokenData::Math( - LxMathTokenData::Digit( - One, - ), + LxMathTokenData::Letter( + LowerX, ), - LxTokenData::Math( - LxMathTokenData::Superscript, + LxMathTokenData::Subscript, + LxMathTokenData::Digit( + One, ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerA, - ), + LxMathTokenData::Superscript, + LxMathTokenData::Letter( + LowerA, ), - LxTokenData::Math( - LxMathTokenData::Punctuation( - Add, - ), + LxMathTokenData::Punctuation( + Add, ), - LxTokenData::Math( - LxMathTokenData::Digit( - One, - ), + LxMathTokenData::Digit( + One, ), ] "#]], @@ -381,12 +315,10 @@ fn next_text_token_data_works() { "\\int", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Command( - LxCommandPath::Coword( - Coword( - "int", - ), + LxMathTokenData::Command( + LxCommandPath::Coword( + Coword( + "int", ), ), ), @@ -397,59 +329,39 @@ fn next_text_token_data_works() { "\\int x^3\\sin^3xdx", &expect![[r#" [ - LxTokenData::Math( - LxMathTokenData::Command( - LxCommandPath::Coword( - Coword( - "int", - ), + LxMathTokenData::Command( + LxCommandPath::Coword( + Coword( + "int", ), ), ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerX, - ), - ), - LxTokenData::Math( - LxMathTokenData::Superscript, + LxMathTokenData::Letter( + LowerX, ), - LxTokenData::Math( - LxMathTokenData::Digit( - Three, - ), + LxMathTokenData::Superscript, + LxMathTokenData::Digit( + Three, ), - LxTokenData::Math( - LxMathTokenData::Command( - LxCommandPath::Coword( - Coword( - "sin", - ), + LxMathTokenData::Command( + LxCommandPath::Coword( + Coword( + "sin", ), ), ), - LxTokenData::Math( - LxMathTokenData::Superscript, + LxMathTokenData::Superscript, + LxMathTokenData::Digit( + Three, ), - LxTokenData::Math( - LxMathTokenData::Digit( - Three, - ), + LxMathTokenData::Letter( + LowerX, ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerX, - ), + LxMathTokenData::Letter( + LowerD, ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerD, - ), - ), - LxTokenData::Math( - LxMathTokenData::Letter( - LowerX, - ), + LxMathTokenData::Letter( + LowerX, ), ] "#]], diff --git a/crates/latex/latex-token/src/data/rose.rs b/crates/latex/latex-token/src/data/rose.rs index 0aae7969d8..31c4a9bc53 100644 --- a/crates/latex/latex-token/src/data/rose.rs +++ b/crates/latex/latex-token/src/data/rose.rs @@ -99,9 +99,7 @@ fn next_text_token_data_works() { "\n\n", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::NewParagraph, - ), + LxRoseTokenData::NewParagraph, ] "#]], ); @@ -109,11 +107,9 @@ fn next_text_token_data_works() { "hello", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::Word( - Coword( - "hello", - ), + LxRoseTokenData::Word( + Coword( + "hello", ), ), ] @@ -123,10 +119,8 @@ fn next_text_token_data_works() { "0", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::Nat32( - 0, - ), + LxRoseTokenData::Nat32( + 0, ), ] "#]], @@ -135,10 +129,8 @@ fn next_text_token_data_works() { " 0", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::Nat32( - 0, - ), + LxRoseTokenData::Nat32( + 0, ), ] "#]], @@ -147,15 +139,11 @@ fn next_text_token_data_works() { "0 0", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::Nat32( - 0, - ), + LxRoseTokenData::Nat32( + 0, ), - LxTokenData::Rose( - LxRoseTokenData::Nat32( - 0, - ), + LxRoseTokenData::Nat32( + 0, ), ] "#]], @@ -164,15 +152,11 @@ fn next_text_token_data_works() { "0\n0", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::Nat32( - 0, - ), + LxRoseTokenData::Nat32( + 0, ), - LxTokenData::Rose( - LxRoseTokenData::Nat32( - 0, - ), + LxRoseTokenData::Nat32( + 0, ), ] "#]], @@ -181,15 +165,11 @@ fn next_text_token_data_works() { "0 0", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::Nat32( - 0, - ), + LxRoseTokenData::Nat32( + 0, ), - LxTokenData::Rose( - LxRoseTokenData::Nat32( - 0, - ), + LxRoseTokenData::Nat32( + 0, ), ] "#]], @@ -198,12 +178,10 @@ fn next_text_token_data_works() { "\\emph", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::Command( - LxCommandPath::Coword( - Coword( - "emph", - ), + LxRoseTokenData::Command( + LxCommandPath::Coword( + Coword( + "emph", ), ), ), @@ -214,12 +192,10 @@ fn next_text_token_data_works() { "\\emph", &expect![[r#" [ - LxTokenData::Rose( - LxRoseTokenData::Command( - LxCommandPath::Coword( - Coword( - "emph", - ), + LxRoseTokenData::Command( + LxCommandPath::Coword( + Coword( + "emph", ), ), ), diff --git a/crates/latex/latex-token/src/idx.rs b/crates/latex/latex-token/src/idx.rs index f38af9a4bf..7fa0fff207 100644 --- a/crates/latex/latex-token/src/idx.rs +++ b/crates/latex/latex-token/src/idx.rs @@ -1,82 +1,2 @@ -use shifted_unsigned_int::ShiftedU32; -use std::iter::Step; -use std::ops::{Add, Sub}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct LxTokenIdx(ShiftedU32); - -impl LxTokenIdx { - pub(crate) fn from_index(index: usize) -> LxTokenIdx { - Self(index.into()) - } - - pub(crate) fn index(self) -> usize { - self.0.index() - } -} - -impl Add for LxTokenIdx { - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - Self(self.0 + rhs) - } -} - -impl Sub for LxTokenIdx { - type Output = Self; - fn sub(self, rhs: usize) -> Self::Output { - Self(self.0 - rhs) - } -} - -impl Step for LxTokenIdx { - fn steps_between(start: &Self, end: &Self) -> Option { - end.0.index().checked_sub(start.0.index()).map(|diff| diff) - } - fn forward_checked(start: Self, count: usize) -> Option { - start.0.checked_add(count.try_into().ok()?).map(Self) - } - fn backward_checked(start: Self, count: usize) -> Option { - start.0.checked_sub(count.try_into().ok()?).map(Self) - } -} - -#[derive(Debug)] -pub struct LxTokenIdxRange { - start: ShiftedU32, - end: ShiftedU32, -} - -impl IntoIterator for LxTokenIdxRange { - type Item = LxTokenIdx; - type IntoIter = std::ops::Range; - - fn into_iter(self) -> Self::IntoIter { - LxTokenIdx::from_index(self.start.index())..LxTokenIdx::from_index(self.end.index()) - } -} - -impl From> for LxTokenIdxRange { - fn from(range: std::ops::Range) -> Self { - Self { - start: range.start.into(), - end: range.end.into(), - } - } -} - -impl LxTokenIdxRange { - pub(crate) fn new(range: std::ops::Range) -> Self { - Self { - start: range.start.into(), - end: range.end.into(), - } - } - - pub fn new_single(idx: LxTokenIdx) -> Self { - Self { - start: idx.0, - end: (idx + 1).0, - } - } -} +pub mod math; +pub mod rose; diff --git a/crates/latex/latex-token/src/idx/math.rs b/crates/latex/latex-token/src/idx/math.rs new file mode 100644 index 0000000000..954d7f24d6 --- /dev/null +++ b/crates/latex/latex-token/src/idx/math.rs @@ -0,0 +1,82 @@ +use shifted_unsigned_int::ShiftedU32; +use std::iter::Step; +use std::ops::{Add, Sub}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct LxMathTokenIdx(ShiftedU32); + +impl LxMathTokenIdx { + pub(crate) fn from_index(index: usize) -> LxMathTokenIdx { + Self(index.into()) + } + + pub(crate) fn index(self) -> usize { + self.0.index() + } +} + +impl Add for LxMathTokenIdx { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} + +impl Sub for LxMathTokenIdx { + type Output = Self; + fn sub(self, rhs: usize) -> Self::Output { + Self(self.0 - rhs) + } +} + +impl Step for LxMathTokenIdx { + fn steps_between(start: &Self, end: &Self) -> Option { + end.0.index().checked_sub(start.0.index()).map(|diff| diff) + } + fn forward_checked(start: Self, count: usize) -> Option { + start.0.checked_add(count.try_into().ok()?).map(Self) + } + fn backward_checked(start: Self, count: usize) -> Option { + start.0.checked_sub(count.try_into().ok()?).map(Self) + } +} + +#[derive(Debug)] +pub struct LxMathTokenIdxRange { + start: ShiftedU32, + end: ShiftedU32, +} + +impl IntoIterator for LxMathTokenIdxRange { + type Item = LxMathTokenIdx; + type IntoIter = std::ops::Range; + + fn into_iter(self) -> Self::IntoIter { + LxMathTokenIdx::from_index(self.start.index())..LxMathTokenIdx::from_index(self.end.index()) + } +} + +impl From> for LxMathTokenIdxRange { + fn from(range: std::ops::Range) -> Self { + Self { + start: range.start.into(), + end: range.end.into(), + } + } +} + +impl LxMathTokenIdxRange { + pub(crate) fn new(range: std::ops::Range) -> Self { + Self { + start: range.start.into(), + end: range.end.into(), + } + } + + pub fn new_single(idx: LxMathTokenIdx) -> Self { + Self { + start: idx.0, + end: (idx + 1).0, + } + } +} diff --git a/crates/latex/latex-token/src/idx/rose.rs b/crates/latex/latex-token/src/idx/rose.rs new file mode 100644 index 0000000000..a595ac1420 --- /dev/null +++ b/crates/latex/latex-token/src/idx/rose.rs @@ -0,0 +1,82 @@ +use shifted_unsigned_int::ShiftedU32; +use std::iter::Step; +use std::ops::{Add, Sub}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct LxRoseTokenIdx(ShiftedU32); + +impl LxRoseTokenIdx { + pub(crate) fn from_index(index: usize) -> LxRoseTokenIdx { + Self(index.into()) + } + + pub(crate) fn index(self) -> usize { + self.0.index() + } +} + +impl Add for LxRoseTokenIdx { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} + +impl Sub for LxRoseTokenIdx { + type Output = Self; + fn sub(self, rhs: usize) -> Self::Output { + Self(self.0 - rhs) + } +} + +impl Step for LxRoseTokenIdx { + fn steps_between(start: &Self, end: &Self) -> Option { + end.0.index().checked_sub(start.0.index()).map(|diff| diff) + } + fn forward_checked(start: Self, count: usize) -> Option { + start.0.checked_add(count.try_into().ok()?).map(Self) + } + fn backward_checked(start: Self, count: usize) -> Option { + start.0.checked_sub(count.try_into().ok()?).map(Self) + } +} + +#[derive(Debug)] +pub struct LxRoseTokenIdxRange { + start: ShiftedU32, + end: ShiftedU32, +} + +impl IntoIterator for LxRoseTokenIdxRange { + type Item = LxRoseTokenIdx; + type IntoIter = std::ops::Range; + + fn into_iter(self) -> Self::IntoIter { + LxRoseTokenIdx::from_index(self.start.index())..LxRoseTokenIdx::from_index(self.end.index()) + } +} + +impl From> for LxRoseTokenIdxRange { + fn from(range: std::ops::Range) -> Self { + Self { + start: range.start.into(), + end: range.end.into(), + } + } +} + +impl LxRoseTokenIdxRange { + pub(crate) fn new(range: std::ops::Range) -> Self { + Self { + start: range.start.into(), + end: range.end.into(), + } + } + + pub fn new_single(idx: LxRoseTokenIdx) -> Self { + Self { + start: idx.0, + end: (idx + 1).0, + } + } +} diff --git a/crates/latex/latex-token/src/lexer.rs b/crates/latex/latex-token/src/lexer.rs index cecc93cf2b..6de94c1477 100644 --- a/crates/latex/latex-token/src/lexer.rs +++ b/crates/latex/latex-token/src/lexer.rs @@ -2,9 +2,8 @@ use crate::{ data::{ math::{LxMathTokenData, LxMathTokenError}, rose::LxRoseTokenData, - LxTokenData, }, - idx::LxTokenIdx, + idx::{math::LxMathTokenIdx, rose::LxRoseTokenIdx}, storage::LxTokenStorage, stream::{math::LxMathTokenStream, rose::LxRoseTokenStream}, }; @@ -31,11 +30,11 @@ impl<'a> LxLexer<'a> { /// # actions impl<'a> LxLexer<'a> { - pub fn next_math_token(&mut self) -> Option<(LxTokenIdx, LxMathTokenData)> { + pub fn next_math_token(&mut self) -> Option<(LxMathTokenIdx, LxMathTokenData)> { let ((start_offset, end_offset), range, token_data) = self.next_math_token_aux()?; Some(( self.storage - .alloc(start_offset, end_offset, range, token_data.into()), + .alloc_math_token(start_offset, end_offset, range, token_data), token_data, )) } @@ -71,11 +70,11 @@ impl<'a> LxLexer<'a> { Some(token_data) } - pub fn next_rose_token(&mut self) -> Option<(LxTokenIdx, LxRoseTokenData)> { + pub fn next_rose_token(&mut self) -> Option<(LxRoseTokenIdx, LxRoseTokenData)> { let ((start_offset, end_offset), range, token_data) = self.next_rose_token_aux()?; Some(( self.storage - .alloc(start_offset, end_offset, range, token_data.into()), + .alloc_rose_token(start_offset, end_offset, range, token_data), token_data, )) } diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index 56105639ed..b0d021c1f2 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -1,51 +1,72 @@ use crate::{ - data::LxTokenData, - idx::{LxTokenIdx, LxTokenIdxRange}, + data::{math::LxMathTokenData, rose::LxRoseTokenData}, + idx::{ + math::{LxMathTokenIdx, LxMathTokenIdxRange}, + rose::{LxRoseTokenIdx, LxRoseTokenIdxRange}, + }, }; use husky_text_protocol::range::TextRange; #[salsa::derive_debug_with_db] #[derive(Default, Debug)] pub struct LxTokenStorage { - ranged_tokens: Vec<((usize, usize), TextRange, LxTokenData)>, + ranged_math_tokens: Vec<((usize, usize), TextRange, LxMathTokenData)>, + ranged_rose_tokens: Vec<((usize, usize), TextRange, LxRoseTokenData)>, } /// # getters -impl std::ops::Index for LxTokenStorage { - type Output = LxTokenData; +impl std::ops::Index for LxTokenStorage { + type Output = LxMathTokenData; - fn index(&self, idx: LxTokenIdx) -> &Self::Output { - &self.ranged_tokens[idx.index()].2 + fn index(&self, idx: LxMathTokenIdx) -> &Self::Output { + &self.ranged_math_tokens[idx.index()].2 } } impl LxTokenStorage { - pub fn whole_token_idx_range(&self) -> LxTokenIdxRange { - LxTokenIdxRange::new(0..self.ranged_tokens.len()) + pub fn whole_math_token_idx_range(&self) -> LxMathTokenIdxRange { + LxMathTokenIdxRange::new(0..self.ranged_math_tokens.len()) } - pub fn whole_token_idx_range_without_the_first(&self) -> LxTokenIdxRange { - LxTokenIdxRange::new(1..self.ranged_tokens.len()) + pub fn whole_rose_token_idx_range(&self) -> LxRoseTokenIdxRange { + LxRoseTokenIdxRange::new(0..self.ranged_rose_tokens.len()) } - pub fn token_text_range(&self, token_idx: LxTokenIdx) -> TextRange { - self.ranged_tokens[token_idx.index()].1 + pub fn math_token_text_range(&self, token_idx: LxMathTokenIdx) -> TextRange { + self.ranged_math_tokens[token_idx.index()].1 + } + + pub fn rose_token_text_range(&self, token_idx: LxRoseTokenIdx) -> TextRange { + self.ranged_rose_tokens[token_idx.index()].1 } } /// # actions impl LxTokenStorage { - pub(crate) fn alloc( + pub(crate) fn alloc_math_token( + &mut self, + start_offset: usize, + end_offset: usize, + range: TextRange, + token_data: LxMathTokenData, + ) -> LxMathTokenIdx { + let idx = LxMathTokenIdx::from_index(self.ranged_math_tokens.len()); + self.ranged_math_tokens + .push(((start_offset, end_offset), range, token_data)); + idx + } + + pub(crate) fn alloc_rose_token( &mut self, start_offset: usize, end_offset: usize, range: TextRange, - token_data: LxTokenData, - ) -> LxTokenIdx { - let idx = LxTokenIdx::from_index(self.ranged_tokens.len()); - self.ranged_tokens + token_data: LxRoseTokenData, + ) -> LxRoseTokenIdx { + let idx = LxRoseTokenIdx::from_index(self.ranged_rose_tokens.len()); + self.ranged_rose_tokens .push(((start_offset, end_offset), range, token_data)); idx } diff --git a/crates/latex/latex-token/src/stream/math.rs b/crates/latex/latex-token/src/stream/math.rs index 787b685a64..e6fdbd50f4 100644 --- a/crates/latex/latex-token/src/stream/math.rs +++ b/crates/latex/latex-token/src/stream/math.rs @@ -1,4 +1,4 @@ -use crate::{data::math::LxMathTokenData, idx::LxTokenIdx, lexer::LxLexer}; +use crate::{data::math::LxMathTokenData, idx::math::LxMathTokenIdx, lexer::LxLexer}; pub struct LxMathTokenStream<'a> { lexer: LxLexer<'a>, @@ -11,7 +11,7 @@ impl<'a> LxMathTokenStream<'a> { } impl<'a> Iterator for LxMathTokenStream<'a> { - type Item = (LxTokenIdx, LxMathTokenData); + type Item = (LxMathTokenIdx, LxMathTokenData); fn next(&mut self) -> Option { self.lexer.next_math_token() diff --git a/crates/latex/latex-token/src/stream/rose.rs b/crates/latex/latex-token/src/stream/rose.rs index 00f7bc447f..8f428192d3 100644 --- a/crates/latex/latex-token/src/stream/rose.rs +++ b/crates/latex/latex-token/src/stream/rose.rs @@ -1,4 +1,4 @@ -use crate::{data::rose::LxRoseTokenData, idx::LxTokenIdx, lexer::LxLexer}; +use crate::{data::rose::LxRoseTokenData, idx::rose::LxRoseTokenIdx, lexer::LxLexer}; pub struct LxRoseTokenStream<'a> { lexer: LxLexer<'a>, @@ -11,7 +11,7 @@ impl<'a> LxRoseTokenStream<'a> { } impl<'a> Iterator for LxRoseTokenStream<'a> { - type Item = (LxTokenIdx, LxRoseTokenData); + type Item = (LxRoseTokenIdx, LxRoseTokenData); fn next(&mut self) -> Option { self.lexer.next_rose_token() From 3fcf4b088ccc59520ae046c0c5403aeacf797414 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 11:39:00 -0700 Subject: [PATCH 25/71] save --- crates/latex/latex-ast/src/range.rs | 30 +++++++++++++++++++++--- crates/latex/latex-token/src/idx/math.rs | 2 +- crates/latex/latex-token/src/idx/rose.rs | 2 +- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index 16286e7714..39d6e9363b 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -1,8 +1,8 @@ use latex_token::idx::{math::LxMathTokenIdxRange, rose::LxRoseTokenIdxRange}; use crate::ast::{ - math::{LxMathAstArenaMap, LxMathAstData}, - rose::{LxRoseAstArenaMap, LxRoseAstData}, + math::{LxMathAstArenaMap, LxMathAstData, LxMathAstIdx}, + rose::{LxRoseAstArenaMap, LxRoseAstData, LxRoseAstIdx}, LxAstArena, LxAstArenaMap, LxAstArenaRef, LxAstData, LxAstIdx, }; @@ -42,13 +42,20 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { impl<'a> LxAstTokenIdxRangeCalculator<'a> { fn infer_all(&mut self) { self.ast_arena.math().indexed_iter().for_each(|(idx, ast)| { - self.math_data.insert_new(idx, self.calc_math_ast(ast)); + self.infer_math_ast(idx, ast); }); self.ast_arena.rose().indexed_iter().for_each(|(idx, ast)| { self.rose_data.insert_new(idx, self.calc_rose_ast(ast)); }); } + fn infer_math_ast(&mut self, idx: LxMathAstIdx, ast: &LxMathAstData) { + if !self.math_data.has(idx) { + let range = self.calc_math_ast(ast); + self.math_data.insert_new(idx, range); + } + } + fn calc_math_ast(&self, data: &LxMathAstData) -> LxMathTokenIdxRange { match *data { LxMathAstData::Letter(idx, _) => LxMathTokenIdxRange::new_single(idx), @@ -66,7 +73,24 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { } } + fn get_math_ast_range(&mut self, idx: LxMathAstIdx) -> LxMathTokenIdxRange { + self.infer_math_ast(idx, &self.ast_arena.math()[idx]); + self.math_data[idx] + } + + fn infer_rose_ast(&mut self, idx: LxRoseAstIdx, ast: &LxRoseAstData) { + if !self.rose_data.has(idx) { + let range = self.calc_rose_ast(ast); + self.rose_data.insert_new(idx, range); + } + } + fn calc_rose_ast(&self, data: &LxRoseAstData) -> LxRoseTokenIdxRange { todo!() } + + fn get_rose_ast_range(&mut self, idx: LxRoseAstIdx) -> LxRoseTokenIdxRange { + self.infer_rose_ast(idx, &self.ast_arena.rose()[idx]); + self.rose_data[idx] + } } diff --git a/crates/latex/latex-token/src/idx/math.rs b/crates/latex/latex-token/src/idx/math.rs index 954d7f24d6..ec093f4b17 100644 --- a/crates/latex/latex-token/src/idx/math.rs +++ b/crates/latex/latex-token/src/idx/math.rs @@ -41,7 +41,7 @@ impl Step for LxMathTokenIdx { } } -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LxMathTokenIdxRange { start: ShiftedU32, end: ShiftedU32, diff --git a/crates/latex/latex-token/src/idx/rose.rs b/crates/latex/latex-token/src/idx/rose.rs index a595ac1420..f62110ff92 100644 --- a/crates/latex/latex-token/src/idx/rose.rs +++ b/crates/latex/latex-token/src/idx/rose.rs @@ -41,7 +41,7 @@ impl Step for LxRoseTokenIdx { } } -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LxRoseTokenIdxRange { start: ShiftedU32, end: ShiftedU32, From 9ec4bd9dddbe0e4a7aef54816d7392d66cb6c9a8 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 14:26:29 -0700 Subject: [PATCH 26/71] tree utils --- Cargo.lock | 7 + .../src/helpers/show/display_tree.rs | 14 +- crates/latex/latex-ast/src/range.rs | 16 ++ crates/latex/latex-token/src/idx/math.rs | 16 ++ crates/latex/latex-token/src/idx/rose.rs | 16 ++ crates/latex/latex-token/src/storage.rs | 30 ++++ .../expect-files/husky_lang_packages.txt | 7 + crates/utils/husky-tree-utils/Cargo.toml | 20 +++ crates/utils/husky-tree-utils/src/display.rs | 155 ++++++++++++++++++ .../husky-tree-utils/src/display/char_set.rs | 33 ++++ crates/utils/husky-tree-utils/src/lib.rs | 1 + 11 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 crates/utils/husky-tree-utils/Cargo.toml create mode 100644 crates/utils/husky-tree-utils/src/display.rs create mode 100644 crates/utils/husky-tree-utils/src/display/char_set.rs create mode 100644 crates/utils/husky-tree-utils/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 47603eefbb..7bf4da8215 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7297,6 +7297,13 @@ version = "0.1.0" name = "husky-trai-analysis" version = "0.1.0" +[[package]] +name = "husky-tree-utils" +version = "0.1.0" +dependencies = [ + "expect-test", +] + [[package]] name = "husky-unicode-symbols" version = "0.1.0" diff --git a/crates/latex/latex-ast/src/helpers/show/display_tree.rs b/crates/latex/latex-ast/src/helpers/show/display_tree.rs index d513d1b92e..4ccd4669f4 100644 --- a/crates/latex/latex-ast/src/helpers/show/display_tree.rs +++ b/crates/latex/latex-ast/src/helpers/show/display_tree.rs @@ -2,14 +2,14 @@ use crate::{ ast::{math::LxMathAstIdx, rose::LxRoseAstIdx, LxAstArenaRef, LxAstIdx, LxAstIdxRange}, range::LxAstTokenIdxRangeMap, }; -use ptree::TreeBuilder; +use latex_token::storage::LxTokenStorage; struct LxAstDisplayTreeBuilder<'a> { db: &'a salsa::Db, input: &'a str, ast_arena: LxAstArenaRef<'a>, ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, - tree_builder: TreeBuilder, + token_storage: &'a LxTokenStorage, } /// # construction @@ -17,6 +17,7 @@ impl<'a> LxAstDisplayTreeBuilder<'a> { fn new( db: &'a salsa::Db, input: &'a str, + token_storage: &'a LxTokenStorage, ast_arena: LxAstArenaRef<'a>, ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, ) -> Self { @@ -25,7 +26,7 @@ impl<'a> LxAstDisplayTreeBuilder<'a> { input, ast_arena, ast_token_idx_range_map, - tree_builder: TreeBuilder::new(input.to_string()), + token_storage, } } } @@ -48,6 +49,13 @@ impl<'a> LxAstDisplayTreeBuilder<'a> { } fn render_math_ast(&mut self, ast: LxMathAstIdx) { + let ast_token_idx_range = self.ast_token_idx_range_map[ast]; + let (start, end) = self + .token_storage + .math_token_idx_range_offset_range(ast_token_idx_range); + // self.tree_builder + // .add_empty_child(text) + // .add_node(self.input[start..end].to_string()); todo!() } diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index 39d6e9363b..bc9512a740 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -12,6 +12,22 @@ pub struct LxAstTokenIdxRangeMap { pub(crate) rose: LxRoseAstArenaMap, } +impl std::ops::Index for LxAstTokenIdxRangeMap { + type Output = LxMathTokenIdxRange; + + fn index(&self, index: LxMathAstIdx) -> &Self::Output { + &self.math[index] + } +} + +impl std::ops::Index for LxAstTokenIdxRangeMap { + type Output = LxRoseTokenIdxRange; + + fn index(&self, index: LxRoseAstIdx) -> &Self::Output { + &self.rose[index] + } +} + pub fn calc_ast_token_idx_range_map(db: &salsa::Db, arena: &LxAstArena) -> LxAstTokenIdxRangeMap { let mut calculator = LxAstTokenIdxRangeCalculator::new(db, arena); calculator.infer_all(); diff --git a/crates/latex/latex-token/src/idx/math.rs b/crates/latex/latex-token/src/idx/math.rs index ec093f4b17..e49d51a8e8 100644 --- a/crates/latex/latex-token/src/idx/math.rs +++ b/crates/latex/latex-token/src/idx/math.rs @@ -79,4 +79,20 @@ impl LxMathTokenIdxRange { end: (idx + 1).0, } } + + pub(crate) fn is_empty(self) -> bool { + self.start == self.end + } + + pub fn start(&self) -> LxMathTokenIdx { + LxMathTokenIdx(self.start) + } + + pub fn last(&self) -> Option { + if self.is_empty() { + None + } else { + Some(LxMathTokenIdx(self.end - 1)) + } + } } diff --git a/crates/latex/latex-token/src/idx/rose.rs b/crates/latex/latex-token/src/idx/rose.rs index f62110ff92..c7d0ffc8c4 100644 --- a/crates/latex/latex-token/src/idx/rose.rs +++ b/crates/latex/latex-token/src/idx/rose.rs @@ -79,4 +79,20 @@ impl LxRoseTokenIdxRange { end: (idx + 1).0, } } + + pub(crate) fn is_empty(self) -> bool { + self.start == self.end + } + + pub fn start(self) -> LxRoseTokenIdx { + LxRoseTokenIdx(self.start) + } + + pub fn last(self) -> Option { + if self.is_empty() { + None + } else { + Some(LxRoseTokenIdx(self.end - 1)) + } + } } diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index b0d021c1f2..b6ba4087eb 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -33,6 +33,14 @@ impl LxTokenStorage { LxRoseTokenIdxRange::new(0..self.ranged_rose_tokens.len()) } + pub fn math_token_offset_range(&self, token_idx: LxMathTokenIdx) -> (usize, usize) { + self.ranged_math_tokens[token_idx.index()].0 + } + + pub fn rose_token_offset_range(&self, token_idx: LxRoseTokenIdx) -> (usize, usize) { + self.ranged_rose_tokens[token_idx.index()].0 + } + pub fn math_token_text_range(&self, token_idx: LxMathTokenIdx) -> TextRange { self.ranged_math_tokens[token_idx.index()].1 } @@ -40,6 +48,28 @@ impl LxTokenStorage { pub fn rose_token_text_range(&self, token_idx: LxRoseTokenIdx) -> TextRange { self.ranged_rose_tokens[token_idx.index()].1 } + + pub fn math_token_idx_range_offset_range(&self, range: LxMathTokenIdxRange) -> (usize, usize) { + let first = self.math_token_offset_range(range.start()); + match range.last() { + Some(last) => { + let last = self.math_token_offset_range(last); + (first.0, last.1) + } + None => todo!(), + } + } + + pub fn rose_token_idx_range_offset_range(&self, range: LxRoseTokenIdxRange) -> (usize, usize) { + let first = self.rose_token_offset_range(range.start()); + match range.last() { + Some(last) => { + let last = self.rose_token_offset_range(last); + (first.0, last.1) + } + None => todo!(), + } + } } /// # actions diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 248ed854e8..70bb1ed4eb 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4203,6 +4203,13 @@ relative_path: "crates/utils/husky-tracing-utils", dependencies: [], }, + PackageSummary { + name: "husky-tree-utils", + relative_path: "crates/utils/husky-tree-utils", + dependencies: [ + "expect-test", + ], + }, PackageSummary { name: "husky-virtual-value-macros", relative_path: "crates/value/husky-virtual-value-macros", diff --git a/crates/utils/husky-tree-utils/Cargo.toml b/crates/utils/husky-tree-utils/Cargo.toml new file mode 100644 index 0000000000..3deb8a8712 --- /dev/null +++ b/crates/utils/husky-tree-utils/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "husky-tree-utils" +version.workspace = true +rust-version.workspace = true +authors.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true +readme.workspace = true +license.workspace = true +categories.workspace = true +keywords.workspace = true + +[dependencies] + +[dev-dependencies] +expect-test.workspace = true + +[lints] +workspace = true diff --git a/crates/utils/husky-tree-utils/src/display.rs b/crates/utils/husky-tree-utils/src/display.rs new file mode 100644 index 0000000000..783c7a62ca --- /dev/null +++ b/crates/utils/husky-tree-utils/src/display.rs @@ -0,0 +1,155 @@ +use char_set::CharSet; + +pub mod char_set; + +pub struct DisplayTree { + value: String, + children: Vec, +} + +impl DisplayTree { + pub fn new(value: String, children: Vec) -> Self { + Self { value, children } + } + + pub fn show(&self, chars: &CharSet) -> String { + let mut builder = String::new(); + self.build_string("", "", &mut builder, &chars); + builder + } + + fn build_string( + &self, + prefix: &str, + child_prefix: &str, + builder: &mut String, + chars: &CharSet, + ) { + // Add current node + builder.push_str(&format!("{}{}\n", prefix, self.value)); + + // Process children + if !self.children.is_empty() { + for (i, child) in self.children.iter().enumerate() { + let is_last = i == self.children.len() - 1; + let (next_prefix, next_child_prefix) = if is_last { + ( + format!("{}{}─ ", child_prefix, chars.end_connector), + format!("{} ", child_prefix), + ) + } else { + ( + format!("{}{}─ ", child_prefix, chars.connector), + format!("{}{} ", child_prefix, chars.vertical), + ) + }; + child.build_string(&next_prefix, &next_child_prefix, builder, chars); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use expect_test::expect; + + #[test] + fn test_single_node() { + let tree = DisplayTree::new("root".to_string(), vec![]); + expect![[r#" + root + "#]] + .assert_eq(&tree.show(&CharSet::default())); + } + + #[test] + fn test_single_child() { + let tree = DisplayTree::new( + "root".to_string(), + vec![DisplayTree::new("child".to_string(), vec![])], + ); + let expected = "\ +root +└─ child +"; + expect![[r#" + root + └─ child + "#]] + .assert_eq(&tree.show(&CharSet::default())); + } + + #[test] + fn test_multiple_children() { + let tree = DisplayTree::new( + "root".to_string(), + vec![ + DisplayTree::new("child1".to_string(), vec![]), + DisplayTree::new("child2".to_string(), vec![]), + ], + ); + expect![[r#" + root + ├─ child1 + └─ child2 + "#]] + .assert_eq(&tree.show(&CharSet::default())); + } + + #[test] + fn test_nested_children() { + let tree = DisplayTree::new( + "root".to_string(), + vec![ + DisplayTree::new( + "child1".to_string(), + vec![DisplayTree::new("grandchild".to_string(), vec![])], + ), + DisplayTree::new("child2".to_string(), vec![]), + ], + ); + let expected = "\ +root +├─ child1 +│ └─ grandchild +└─ child2 +"; + expect![[r#" + root + ├─ child1 + │ └─ grandchild + └─ child2 + "#]] + .assert_eq(&tree.show(&CharSet::default())); + } + + #[test] + fn test_complex_tree() { + let tree = DisplayTree::new( + "+".to_string(), + vec![ + DisplayTree::new( + "-".to_string(), + vec![DisplayTree::new( + "Int".to_string(), + vec![DisplayTree::new("2".to_string(), vec![])], + )], + ), + DisplayTree::new( + "Int".to_string(), + vec![DisplayTree::new("7".to_string(), vec![])], + ), + ], + ); + expect![[r#" + + + ├─ - + │ └─ Int + │ └─ 2 + └─ Int + └─ 7 + "#]] + .assert_eq(&tree.show(&CharSet::default())); + } +} diff --git a/crates/utils/husky-tree-utils/src/display/char_set.rs b/crates/utils/husky-tree-utils/src/display/char_set.rs new file mode 100644 index 0000000000..fc8fd85105 --- /dev/null +++ b/crates/utils/husky-tree-utils/src/display/char_set.rs @@ -0,0 +1,33 @@ +#[derive(Clone, Copy)] +pub struct CharSet { + /// The characters used in the horizontal portion of a branch. + /// + /// Should resemble a plain horizontal line, eg. '─'. + pub horizontal: char, + /// The character used in the space between branches in place of + /// [`connector`](CharSet::connector). + /// + /// Should resemble a plain vertical line, eg. '│'. + pub vertical: char, + /// The character connecting the vertical and horizontal portions of a + /// branch. + /// + /// Should resemble a vertical line with an offshoot on the right, eg. '├'. + pub connector: char, + /// The character connecting the vertical and horizontal portions of the + /// last branch under a node. + /// + /// Should resemble an "L" shape, eg. '└'. + pub end_connector: char, +} + +impl Default for CharSet { + fn default() -> Self { + Self { + horizontal: '─', + vertical: '│', + connector: '├', + end_connector: '└', + } + } +} diff --git a/crates/utils/husky-tree-utils/src/lib.rs b/crates/utils/husky-tree-utils/src/lib.rs new file mode 100644 index 0000000000..5e63c4dc07 --- /dev/null +++ b/crates/utils/husky-tree-utils/src/lib.rs @@ -0,0 +1 @@ +mod display; From 675cfcde52af7a074d85d571765ddceaf48eb08f Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 15:47:45 -0700 Subject: [PATCH 27/71] display tree for latex math asts --- Cargo.lock | 1 + Cargo.toml | 1 + crates/latex/latex-ast/Cargo.toml | 2 + crates/latex/latex-ast/src/ast/math.rs | 12 +++++ .../src/helpers/show/display_tree.rs | 46 +++++++++++-------- .../expect-files/husky_lang_packages.txt | 15 +++--- crates/utils/husky-tree-utils/src/lib.rs | 2 +- 7 files changed, 53 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bf4da8215..1239425d7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8154,6 +8154,7 @@ dependencies = [ "enum-class", "expect-test", "husky-coword", + "husky-tree-utils", "idx-arena", "latex-ast", "latex-math-letter", diff --git a/Cargo.toml b/Cargo.toml index 07a92b7a16..6f5da873e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -415,6 +415,7 @@ husky-screen-utils = { path = "crates/utils/husky-screen-utils" } husky-sha-utils = { path = "crates/utils/husky-sha-utils" } husky-sync-utils = { path = "crates/utils/husky-sync-utils" } husky-toml-utils = { path = "crates/utils/husky-toml-utils" } +husky-tree-utils = { path = "crates/utils/husky-tree-utils" } husky-rng-utils = { path = "crates/utils/husky-rng-utils" } husky-unicode-symbols = { path = "crates/utils/husky-unicode-symbols" } husky-wild-utils = { path = "crates/utils/husky-wild-utils" } diff --git a/crates/latex/latex-ast/Cargo.toml b/crates/latex/latex-ast/Cargo.toml index 28295d081d..8413aa2b03 100644 --- a/crates/latex/latex-ast/Cargo.toml +++ b/crates/latex/latex-ast/Cargo.toml @@ -26,6 +26,8 @@ latex-math-letter.workspace = true latex-math-opr.workspace = true latex-prelude.workspace = true latex-token.workspace = true +# utils +husky-tree-utils.workspace = true [dev-dependencies] expect-test.workspace = true diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 0d05197a76..af635ac49c 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -28,6 +28,18 @@ pub enum LxMathAstData { }, } +impl LxMathAstData { + pub fn children(&self) -> Vec { + match *self { + LxMathAstData::Delimited { asts, .. } => asts.into_iter().collect(), + LxMathAstData::Attach { ref scripts, .. } => { + scripts.iter().map(|&(_, ast)| ast).collect() + } + _ => vec![], + } + } +} + pub type LxMathAstArena = Arena; pub type LxMathAstArenaRef<'a> = ArenaRef<'a, LxMathAstData>; pub type LxMathAstArenaMap = ArenaMap; diff --git a/crates/latex/latex-ast/src/helpers/show/display_tree.rs b/crates/latex/latex-ast/src/helpers/show/display_tree.rs index 4ccd4669f4..acb81e2fe9 100644 --- a/crates/latex/latex-ast/src/helpers/show/display_tree.rs +++ b/crates/latex/latex-ast/src/helpers/show/display_tree.rs @@ -1,7 +1,12 @@ use crate::{ - ast::{math::LxMathAstIdx, rose::LxRoseAstIdx, LxAstArenaRef, LxAstIdx, LxAstIdxRange}, + ast::{ + math::{LxMathAstIdx, LxMathAstIdxRange}, + rose::LxRoseAstIdx, + LxAstArenaRef, LxAstIdx, LxAstIdxRange, + }, range::LxAstTokenIdxRangeMap, }; +use husky_tree_utils::display::DisplayTree; use latex_token::storage::LxTokenStorage; struct LxAstDisplayTreeBuilder<'a> { @@ -33,33 +38,38 @@ impl<'a> LxAstDisplayTreeBuilder<'a> { /// # actions impl<'a> LxAstDisplayTreeBuilder<'a> { - fn render_asts(&mut self, asts: LxAstIdxRange) { + fn render_asts(&self, asts: LxAstIdxRange) -> Vec { match asts { - LxAstIdxRange::Math(range) => { - for ast in range { - self.render_math_ast(ast); - } - } - LxAstIdxRange::Rose(range) => { - for ast in range { - self.render_rose_ast(ast); - } - } + LxAstIdxRange::Math(range) => self.render_math_asts(range), + LxAstIdxRange::Rose(range) => self.render_rose_asts(range), } } - fn render_math_ast(&mut self, ast: LxMathAstIdx) { + fn render_math_asts(&self, asts: impl IntoIterator) -> Vec { + asts.into_iter() + .map(|ast| self.render_math_ast(ast)) + .collect() + } + + fn render_rose_asts(&self, asts: impl IntoIterator) -> Vec { + asts.into_iter() + .map(|ast| self.render_rose_ast(ast)) + .collect() + } + + fn render_math_ast(&self, ast: LxMathAstIdx) -> DisplayTree { let ast_token_idx_range = self.ast_token_idx_range_map[ast]; let (start, end) = self .token_storage .math_token_idx_range_offset_range(ast_token_idx_range); - // self.tree_builder - // .add_empty_child(text) - // .add_node(self.input[start..end].to_string()); - todo!() + let value = self.input[start..end].to_string(); + DisplayTree::new( + value, + self.render_math_asts(self.ast_arena.math()[ast].children()), + ) } - fn render_rose_ast(&mut self, ast: LxRoseAstIdx) { + fn render_rose_ast(&self, ast: LxRoseAstIdx) -> DisplayTree { todo!() } } diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 70bb1ed4eb..1176d81fa0 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -3537,6 +3537,7 @@ "egui", "enum-class", "husky-coword", + "husky-tree-utils", "idx-arena", "latex-math-letter", "latex-math-opr", @@ -3548,6 +3549,13 @@ "expect-test", ], }, + PackageSummary { + name: "husky-tree-utils", + relative_path: "crates/utils/husky-tree-utils", + dependencies: [ + "expect-test", + ], + }, PackageSummary { name: "latex-math-letter", relative_path: "crates/latex/latex-math-letter", @@ -4203,13 +4211,6 @@ relative_path: "crates/utils/husky-tracing-utils", dependencies: [], }, - PackageSummary { - name: "husky-tree-utils", - relative_path: "crates/utils/husky-tree-utils", - dependencies: [ - "expect-test", - ], - }, PackageSummary { name: "husky-virtual-value-macros", relative_path: "crates/value/husky-virtual-value-macros", diff --git a/crates/utils/husky-tree-utils/src/lib.rs b/crates/utils/husky-tree-utils/src/lib.rs index 5e63c4dc07..8754563b71 100644 --- a/crates/utils/husky-tree-utils/src/lib.rs +++ b/crates/utils/husky-tree-utils/src/lib.rs @@ -1 +1 @@ -mod display; +pub mod display; From 00effd5abdd8f2a2cccf3cba48f33b978244ecd1 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 18:42:40 -0700 Subject: [PATCH 28/71] save --- crates/latex/latex-ast/src/ast.rs | 59 +++++++++++++++++++ .../src/helpers/show/display_tree.rs | 10 +++- crates/latex/latex-ast/src/range.rs | 15 ++++- .../latex-ast/src/test_helpers/example.rs | 23 +++++++- crates/latex/latex-token/src/idx/math.rs | 16 +++++ crates/utils/husky-tree-utils/src/display.rs | 6 -- 6 files changed, 116 insertions(+), 13 deletions(-) diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 5764f34a0f..eb950d6723 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -604,3 +604,62 @@ fn parse_tex_input_into_asts_works() { "#]], ); } + +#[test] +fn parse_tex_input_into_asts_then_show_works() { + use crate::test_helpers::example::LxAstsExample; + use expect_test::Expect; + + fn t(input: &str, mode: LxMode, expected: Expect) { + let db = &DB::default(); + let example = LxAstsExample::new(input, mode, db); + let show = example.show(db); + expected.assert_eq(&show); + } + t( + "x", + LxMode::Math, + expect![[r#" + x + "#]], + ); + t( + "x+1", + LxMode::Math, + expect![[r#" + x+1 + ├─ x + └─ + + "#]], + ); + t( + "x^2", + LxMode::Math, + expect![[r#" + x^2 + └─ x^2 + └─ 2 + "#]], + ); + t( + "x_2", + LxMode::Math, + expect![[r#" + x_2 + └─ x_2 + └─ 2 + "#]], + ); + t( + "x^{i+2}", + LxMode::Math, + expect![[r#" + x^{i+2} + └─ x^{i+2} + └─ {i+2} + ├─ i + ├─ + + └─ 2 + "#]], + ); +} diff --git a/crates/latex/latex-ast/src/helpers/show/display_tree.rs b/crates/latex/latex-ast/src/helpers/show/display_tree.rs index acb81e2fe9..fb661870d3 100644 --- a/crates/latex/latex-ast/src/helpers/show/display_tree.rs +++ b/crates/latex/latex-ast/src/helpers/show/display_tree.rs @@ -9,7 +9,7 @@ use crate::{ use husky_tree_utils::display::DisplayTree; use latex_token::storage::LxTokenStorage; -struct LxAstDisplayTreeBuilder<'a> { +pub struct LxAstDisplayTreeBuilder<'a> { db: &'a salsa::Db, input: &'a str, ast_arena: LxAstArenaRef<'a>, @@ -19,7 +19,7 @@ struct LxAstDisplayTreeBuilder<'a> { /// # construction impl<'a> LxAstDisplayTreeBuilder<'a> { - fn new( + pub fn new( db: &'a salsa::Db, input: &'a str, token_storage: &'a LxTokenStorage, @@ -38,7 +38,11 @@ impl<'a> LxAstDisplayTreeBuilder<'a> { /// # actions impl<'a> LxAstDisplayTreeBuilder<'a> { - fn render_asts(&self, asts: LxAstIdxRange) -> Vec { + pub fn render_all(&self, asts: LxAstIdxRange) -> DisplayTree { + DisplayTree::new(self.input.to_string(), self.render_asts(asts)) + } + + pub fn render_asts(&self, asts: LxAstIdxRange) -> Vec { match asts { LxAstIdxRange::Math(range) => self.render_math_asts(range), LxAstIdxRange::Rose(range) => self.render_rose_asts(range), diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index bc9512a740..2d14a5e8c1 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -72,20 +72,29 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { } } - fn calc_math_ast(&self, data: &LxMathAstData) -> LxMathTokenIdxRange { + fn calc_math_ast(&mut self, data: &LxMathAstData) -> LxMathTokenIdxRange { match *data { LxMathAstData::Letter(idx, _) => LxMathTokenIdxRange::new_single(idx), LxMathAstData::Opr(idx, _) => LxMathTokenIdxRange::new_single(idx), LxMathAstData::Digit(idx, _) => LxMathTokenIdxRange::new_single(idx), LxMathAstData::TextEdit { ref buffer } => todo!(), - LxMathAstData::Attach { base, ref scripts } => todo!(), + LxMathAstData::Attach { base, ref scripts } => { + let mut range = self.get_math_ast_range(base); + for &(_, script) in scripts { + let script_range = self.get_math_ast_range(script); + range = range.join(script_range); + } + range + } LxMathAstData::Delimited { left_delimiter_token_idx, left_delimiter, asts, right_delimiter_token_idx, right_delimiter, - } => todo!(), + } => { + LxMathTokenIdxRange::new_closed(left_delimiter_token_idx, right_delimiter_token_idx) + } } } diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs index 8f08525a6d..30b569932a 100644 --- a/crates/latex/latex-ast/src/test_helpers/example.rs +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -1,5 +1,6 @@ use crate::{ ast::{parse_latex_input_into_asts, LxAstArena, LxAstIdxRange}, + helpers::show::display_tree::LxAstDisplayTreeBuilder, range::{calc_ast_token_idx_range_map, LxAstTokenIdxRangeMap}, }; use latex_prelude::mode::LxMode; @@ -7,6 +8,7 @@ use latex_token::storage::LxTokenStorage; #[derive(Debug)] pub struct LxAstsExample { + pub input: String, pub token_storage: LxTokenStorage, pub ast_arena: LxAstArena, pub asts: LxAstIdxRange, @@ -21,10 +23,29 @@ impl LxAstsExample { parse_latex_input_into_asts(db, input, root_mode, &mut token_storage, &mut ast_arena); let ast_token_idx_range_map = calc_ast_token_idx_range_map(db, &ast_arena); Self { - token_storage: todo!(), + input: input.to_string(), + token_storage, ast_arena, asts, ast_token_idx_range_map, } } } + +impl LxAstsExample { + pub fn show(&self, db: &salsa::Db) -> String { + let display_tree_builder = LxAstDisplayTreeBuilder::new( + db, + &self.input, + &self.token_storage, + self.ast_arena.as_arena_ref(), + &self.ast_token_idx_range_map, + ); + format!( + "{}", + display_tree_builder + .render_all(self.asts) + .show(&Default::default()) + ) + } +} diff --git a/crates/latex/latex-token/src/idx/math.rs b/crates/latex/latex-token/src/idx/math.rs index e49d51a8e8..2dfe4b51bc 100644 --- a/crates/latex/latex-token/src/idx/math.rs +++ b/crates/latex/latex-token/src/idx/math.rs @@ -80,6 +80,15 @@ impl LxMathTokenIdxRange { } } + pub fn new_closed(first: LxMathTokenIdx, last: LxMathTokenIdx) -> Self { + Self { + start: first.0, + end: last.0 + 1usize, + } + } +} + +impl LxMathTokenIdxRange { pub(crate) fn is_empty(self) -> bool { self.start == self.end } @@ -95,4 +104,11 @@ impl LxMathTokenIdxRange { Some(LxMathTokenIdx(self.end - 1)) } } + + pub fn join(self, other: Self) -> Self { + Self { + start: self.start.min(other.start), + end: self.end.max(other.end), + } + } } diff --git a/crates/utils/husky-tree-utils/src/display.rs b/crates/utils/husky-tree-utils/src/display.rs index 783c7a62ca..829866df38 100644 --- a/crates/utils/husky-tree-utils/src/display.rs +++ b/crates/utils/husky-tree-utils/src/display.rs @@ -109,12 +109,6 @@ root DisplayTree::new("child2".to_string(), vec![]), ], ); - let expected = "\ -root -├─ child1 -│ └─ grandchild -└─ child2 -"; expect![[r#" root ├─ child1 From 547e277e3bccbc88fc78cd86a63091094b2c9955 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 18:45:57 -0700 Subject: [PATCH 29/71] save --- crates/latex/latex-ast/src/ast.rs | 19 +++++++++++-------- crates/latex/latex-ast/src/ast/math.rs | 9 ++++++--- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index eb950d6723..2af0a44638 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -636,19 +636,21 @@ fn parse_tex_input_into_asts_then_show_works() { "x^2", LxMode::Math, expect![[r#" - x^2 - └─ x^2 - └─ 2 - "#]], + x^2 + └─ x^2 + ├─ x + └─ 2 + "#]], ); t( "x_2", LxMode::Math, expect![[r#" - x_2 - └─ x_2 - └─ 2 - "#]], + x_2 + └─ x_2 + ├─ x + └─ 2 + "#]], ); t( "x^{i+2}", @@ -656,6 +658,7 @@ fn parse_tex_input_into_asts_then_show_works() { expect![[r#" x^{i+2} └─ x^{i+2} + ├─ x └─ {i+2} ├─ i ├─ + diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index af635ac49c..3b9136dd8d 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -32,9 +32,12 @@ impl LxMathAstData { pub fn children(&self) -> Vec { match *self { LxMathAstData::Delimited { asts, .. } => asts.into_iter().collect(), - LxMathAstData::Attach { ref scripts, .. } => { - scripts.iter().map(|&(_, ast)| ast).collect() - } + LxMathAstData::Attach { + base, ref scripts, .. + } => [base] + .into_iter() + .chain(scripts.iter().map(|&(_, ast)| ast)) + .collect(), _ => vec![], } } From 3c6d0caac881b30be40dcb22909bc0ef6557de96 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 19:28:36 -0700 Subject: [PATCH 30/71] bug fixed --- crates/latex/latex-ast/src/ast.rs | 152 ++++++++++++++++++++---------- 1 file changed, 103 insertions(+), 49 deletions(-) diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 2af0a44638..b322cdb811 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -109,13 +109,7 @@ impl<'a> LxAstParser<'a> { pub(crate) fn parse_asts(&mut self) -> LxAstIdxRange { match self.mode() { LxMode::Rose => todo!(), - LxMode::Math => { - let mut asts = vec![]; - while let Some(ast) = self.parse_math_ast() { - asts.push(ast) - } - self.alloc_math_asts(asts).into() - } + LxMode::Math => self.parse_math_asts().into(), } } @@ -129,41 +123,43 @@ impl<'a> LxAstParser<'a> { fn parse_math_ast(&mut self) -> Option { let mut ast = self.parse_atomic_math_ast()?; - match self.peek_math_token_data()? { - // TODO include more cases, like \limits - LxMathTokenData::Subscript | LxMathTokenData::Superscript => { - let (idx, token) = self.next_math_token().unwrap(); - ast = match ast { - LxMathAstData::Attach { .. } => ast, - base => { - let base = self.alloc_math_ast(base); - LxMathAstData::Attach { - base, - scripts: Default::default(), + if let Some(token_data) = self.peek_math_token_data() { + match token_data { + // TODO include more cases, like \limits + LxMathTokenData::Subscript | LxMathTokenData::Superscript => { + let (idx, token) = self.next_math_token().unwrap(); + ast = match ast { + LxMathAstData::Attach { .. } => ast, + base => { + let base = self.alloc_math_ast(base); + LxMathAstData::Attach { + base, + scripts: Default::default(), + } + .into() } - .into() - } - _ => todo!(), - }; - let LxMathAstData::Attach { - ref mut scripts, .. - } = ast - else { - unreachable!() - }; - let script_kind = match token { - LxMathTokenData::Subscript => LxScriptKind::Subscript, - LxMathTokenData::Superscript => LxScriptKind::Superscript, - _ => todo!(), - }; - let ast = match self.parse_atomic_math_ast() { - Some(new_subscript) => self.alloc_math_ast(new_subscript), - None => todo!("err: expected subscript"), - }; - scripts.push((script_kind, ast)); + _ => todo!(), + }; + let LxMathAstData::Attach { + ref mut scripts, .. + } = ast + else { + unreachable!() + }; + let script_kind = match token { + LxMathTokenData::Subscript => LxScriptKind::Subscript, + LxMathTokenData::Superscript => LxScriptKind::Superscript, + _ => todo!(), + }; + let ast = match self.parse_atomic_math_ast() { + Some(new_subscript) => self.alloc_math_ast(new_subscript), + None => todo!("err: expected subscript"), + }; + scripts.push((script_kind, ast)); + } + _ => (), } - _ => (), - }; + } Some(ast) } } @@ -205,6 +201,49 @@ fn parse_tex_input_into_asts_works() { ) "#]], ); + t( + "1", + LxMode::Math, + expect![[r#" + ( + LxTokenStorage { + ranged_math_tokens: [ + ( + ( + 0, + 1, + ), + [1:1, 1:2), + Digit( + One, + ), + ), + ], + ranged_rose_tokens: [], + }, + LxAstArena { + math: Arena { + data: [ + LxMathAstData::Digit( + LxMathTokenIdx( + 0, + ), + One, + ), + ], + }, + rose: Arena { + data: [], + }, + }, + LxAstIdxRange::Math( + ArenaIdxRange( + 0..1, + ), + ), + ) + "#]], + ); t( "x", LxMode::Math, @@ -227,7 +266,14 @@ fn parse_tex_input_into_asts_works() { }, LxAstArena { math: Arena { - data: [], + data: [ + LxMathAstData::Letter( + LxMathTokenIdx( + 0, + ), + LowerX, + ), + ], }, rose: Arena { data: [], @@ -235,7 +281,7 @@ fn parse_tex_input_into_asts_works() { }, LxAstIdxRange::Math( ArenaIdxRange( - 0..0, + 0..1, ), ), ) @@ -296,6 +342,12 @@ fn parse_tex_input_into_asts_works() { ), Add, ), + LxMathAstData::Digit( + LxMathTokenIdx( + 2, + ), + One, + ), ], }, rose: Arena { @@ -304,7 +356,7 @@ fn parse_tex_input_into_asts_works() { }, LxAstIdxRange::Math( ArenaIdxRange( - 0..2, + 0..3, ), ), ) @@ -620,17 +672,19 @@ fn parse_tex_input_into_asts_then_show_works() { "x", LxMode::Math, expect![[r#" - x - "#]], + x + └─ x + "#]], ); t( "x+1", LxMode::Math, expect![[r#" - x+1 - ├─ x - └─ + - "#]], + x+1 + ├─ x + ├─ + + └─ 1 + "#]], ); t( "x^2", From 4f7b326c4a610aaba2704475175ca77fdda4ff18 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 21:10:35 -0700 Subject: [PATCH 31/71] remove latex-syn-expr --- Cargo.lock | 5 ----- Cargo.toml | 1 - crates/latex/latex-syn-expr/Cargo.toml | 17 ----------------- crates/latex/latex-syn-expr/src/lib.rs | 14 -------------- .../expect-files/husky_lang_packages.txt | 6 ------ crates/visored/visored-sem-expr/Cargo.toml | 1 - 6 files changed, 44 deletions(-) delete mode 100644 crates/latex/latex-syn-expr/Cargo.toml delete mode 100644 crates/latex/latex-syn-expr/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1239425d7c..d0378a6ace 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8199,10 +8199,6 @@ dependencies = [ "salsa", ] -[[package]] -name = "latex-syn-expr" -version = "0.1.0" - [[package]] name = "latex-token" version = "0.1.0" @@ -12576,7 +12572,6 @@ dependencies = [ "husky-coword", "idx-arena", "latex-prelude", - "latex-syn-expr", "salsa", "visored-annotation", "visored-opr", diff --git a/Cargo.toml b/Cargo.toml index 6f5da873e9..f4e07f17e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -322,7 +322,6 @@ latex-math-letter = { path = "crates/latex/latex-math-letter" } latex-math-opr = { path = "crates/latex/latex-math-opr" } latex-prelude = { path = "crates/latex/latex-prelude" } latex-sem-expr = { path = "crates/latex/latex-sem-expr" } -latex-syn-expr = { path = "crates/latex/latex-syn-expr" } latex-text = { path = "crates/latex/latex-text" } latex-token = { path = "crates/latex/latex-token" } # lean diff --git a/crates/latex/latex-syn-expr/Cargo.toml b/crates/latex/latex-syn-expr/Cargo.toml deleted file mode 100644 index d212a5f735..0000000000 --- a/crates/latex/latex-syn-expr/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "latex-syn-expr" -version.workspace = true -rust-version.workspace = true -authors.workspace = true -edition.workspace = true -homepage.workspace = true -repository.workspace = true -readme.workspace = true -license.workspace = true -categories.workspace = true -keywords.workspace = true - -[dependencies] - -[lints] -workspace = true diff --git a/crates/latex/latex-syn-expr/src/lib.rs b/crates/latex/latex-syn-expr/src/lib.rs deleted file mode 100644 index 7d12d9af81..0000000000 --- a/crates/latex/latex-syn-expr/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 1176d81fa0..be8b003f69 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4128,11 +4128,6 @@ relative_path: "crates/syntax/husky-syn-fmt", dependencies: [], }, - PackageSummary { - name: "latex-syn-expr", - relative_path: "crates/latex/latex-syn-expr", - dependencies: [], - }, PackageSummary { name: "husky-bindgen-utils", relative_path: "crates/utils/husky-bindgen-utils", @@ -4234,7 +4229,6 @@ dependencies: [ "idx-arena", "latex-prelude", - "latex-syn-expr", "salsa", "visored-annotation", "visored-opr", diff --git a/crates/visored/visored-sem-expr/Cargo.toml b/crates/visored/visored-sem-expr/Cargo.toml index e0cbb0ceb8..833ba21a74 100644 --- a/crates/visored/visored-sem-expr/Cargo.toml +++ b/crates/visored/visored-sem-expr/Cargo.toml @@ -17,7 +17,6 @@ salsa.workspace = true idx-arena.workspace = true # latex latex-prelude.workspace = true -latex-syn-expr.workspace = true # visored visored-annotation.workspace = true visored-opr.workspace = true From b5e561ded7e7e5cd2a0899b650bd9e35b307134a Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 21:32:44 -0700 Subject: [PATCH 32/71] init visored-syn-expr --- Cargo.lock | 11 +++ Cargo.toml | 1 + crates/abstractions/salsa/src/jar.rs | 1 + .../expect-files/husky_lang_jar_packages.txt | 7 ++ .../expect-files/husky_lang_jar_tree.json | 4 + .../expect-files/husky_lang_jar_tree.txt | 4 + .../husky_lang_jar_tree_trimmed.json | 4 + .../husky_lang_jar_tree_trimmed.txt | 4 + .../expect-files/husky_lang_packages.txt | 11 +++ crates/visored/visored-syn-expr/Cargo.toml | 24 ++++++ .../visored/visored-syn-expr/src/builder.rs | 1 + crates/visored/visored-syn-expr/src/clause.rs | 11 +++ crates/visored/visored-syn-expr/src/expr.rs | 85 +++++++++++++++++++ .../visored-syn-expr/src/expr/attach.rs | 1 + .../visored-syn-expr/src/expr/binary.rs | 1 + .../visored-syn-expr/src/expr/literal.rs | 1 + .../visored-syn-expr/src/expr/notation.rs | 1 + .../visored-syn-expr/src/expr/prefix.rs | 1 + .../visored-syn-expr/src/expr/suffix.rs | 1 + .../src/expr/uniadic_array.rs | 1 + .../src/expr/uniadic_chain.rs | 1 + .../src/expr/variadic_array.rs | 1 + .../src/expr/variadic_chain.rs | 1 + .../visored/visored-syn-expr/src/helpers.rs | 1 + .../visored-syn-expr/src/helpers/show.rs | 1 + crates/visored/visored-syn-expr/src/jar.rs | 2 + crates/visored/visored-syn-expr/src/lib.rs | 16 ++++ crates/visored/visored-syn-expr/src/phrase.rs | 19 +++++ .../visored-syn-expr/src/phrase/noun.rs | 5 ++ crates/visored/visored-syn-expr/src/region.rs | 1 + .../visored/visored-syn-expr/src/sentence.rs | 12 +++ .../visored-syn-expr/src/test_helpers.rs | 1 + crates/visored/visored-syn-expr/src/tests.rs | 1 + 33 files changed, 237 insertions(+) create mode 100644 crates/visored/visored-syn-expr/Cargo.toml create mode 100644 crates/visored/visored-syn-expr/src/builder.rs create mode 100644 crates/visored/visored-syn-expr/src/clause.rs create mode 100644 crates/visored/visored-syn-expr/src/expr.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/attach.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/binary.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/literal.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/notation.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/prefix.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/suffix.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/uniadic_array.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/uniadic_chain.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/variadic_array.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/variadic_chain.rs create mode 100644 crates/visored/visored-syn-expr/src/helpers.rs create mode 100644 crates/visored/visored-syn-expr/src/helpers/show.rs create mode 100644 crates/visored/visored-syn-expr/src/jar.rs create mode 100644 crates/visored/visored-syn-expr/src/lib.rs create mode 100644 crates/visored/visored-syn-expr/src/phrase.rs create mode 100644 crates/visored/visored-syn-expr/src/phrase/noun.rs create mode 100644 crates/visored/visored-syn-expr/src/region.rs create mode 100644 crates/visored/visored-syn-expr/src/sentence.rs create mode 100644 crates/visored/visored-syn-expr/src/test_helpers.rs create mode 100644 crates/visored/visored-syn-expr/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index d0378a6ace..6803528386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12579,6 +12579,17 @@ dependencies = [ "visored-zfs-ty", ] +[[package]] +name = "visored-syn-expr" +version = "0.1.0" +dependencies = [ + "idx-arena", + "latex-prelude", + "salsa", + "visored-opr", + "visored-zfs-ty", +] + [[package]] name = "visored-zfs-ty" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f4e07f17e7..5b3df4a9cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -433,6 +433,7 @@ visored-hir-expr = { path = "crates/visored/visored-hir-expr" } visored-isabelle-hol-transpilation = { path = "crates/visored/visored-isabelle-hol-transpilation" } visored-lean-transpilation = { path = "crates/visored/visored-lean-transpilation" } visored-sem-expr = { path = "crates/visored/visored-sem-expr" } +visored-syn-expr = { path = "crates/visored/visored-syn-expr" } visored-opr = { path = "crates/visored/visored-opr" } visored-zfs-ty = { path = "crates/visored/visored-zfs-ty" } # vm diff --git a/crates/abstractions/salsa/src/jar.rs b/crates/abstractions/salsa/src/jar.rs index aeeaf816d6..918bb04597 100644 --- a/crates/abstractions/salsa/src/jar.rs +++ b/crates/abstractions/salsa/src/jar.rs @@ -195,6 +195,7 @@ pub enum JarIndex { // visored VdZfsTypeJar, VdOprJar, + VdSynExprJar, VdSemExprJar, VdHirExprJar, VdLeanTranspilationJar, diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt index 57997cce53..2b8b9f8181 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt @@ -1344,4 +1344,11 @@ "visored-sem-expr", ], }, + JarPackageSummary { + name: "visored-syn-expr", + dependencies: [ + "visored-opr", + "visored-zfs-ty", + ], + }, ] \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json index 44900a3a9b..076569497e 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json @@ -1195,5 +1195,9 @@ "visored-opr", "visored-zfs-ty" ], + "visored-syn-expr": [ + "visored-opr", + "visored-zfs-ty" + ], "visored-zfs-ty": [] } \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt index e9e18fddac..b315cfe159 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt @@ -1195,5 +1195,9 @@ "visored-opr", "visored-zfs-ty", }, + "visored-syn-expr": { + "visored-opr", + "visored-zfs-ty", + }, "visored-zfs-ty": {}, } \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json index d7e576b1dd..fc5f01adae 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json @@ -195,5 +195,9 @@ "visored-opr", "visored-zfs-ty" ], + "visored-syn-expr": [ + "visored-opr", + "visored-zfs-ty" + ], "visored-zfs-ty": [] } \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt index 2cf5c7d15b..232b7644e8 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt @@ -195,5 +195,9 @@ "visored-opr", "visored-zfs-ty", }, + "visored-syn-expr": { + "visored-opr", + "visored-zfs-ty", + }, "visored-zfs-ty": {}, } \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index be8b003f69..4aa1dd467c 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4304,6 +4304,17 @@ "idx-arena", ], }, + PackageSummary { + name: "visored-syn-expr", + relative_path: "crates/visored/visored-syn-expr", + dependencies: [ + "idx-arena", + "latex-prelude", + "salsa", + "visored-opr", + "visored-zfs-ty", + ], + }, PackageSummary { name: "cybertron-mini-lean-tokens", relative_path: "registry/cybertron-mini-lean-tokens-0.1.0", diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml new file mode 100644 index 0000000000..db1e6e608e --- /dev/null +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "visored-syn-expr" +version.workspace = true +rust-version.workspace = true +authors.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true +readme.workspace = true +license.workspace = true +categories.workspace = true +keywords.workspace = true + +[dependencies] +# abstractions +salsa.workspace = true +idx-arena.workspace = true +# latex +latex-prelude.workspace = true +visored-opr.workspace = true +visored-zfs-ty.workspace = true + +[lints] +workspace = true diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/clause.rs b/crates/visored/visored-syn-expr/src/clause.rs new file mode 100644 index 0000000000..d780e1142b --- /dev/null +++ b/crates/visored/visored-syn-expr/src/clause.rs @@ -0,0 +1,11 @@ +use super::*; +use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; + +pub enum VdSynClauseData { + Verb, +} + +pub type VdSynClauseArena = Arena; +pub type VdSynClauseArenaRef<'a> = ArenaRef<'a, VdSynClauseData>; +pub type VdSynClauseIdx = ArenaIdx; +pub type VdSynClauseIdxRange = ArenaIdxRange; diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs new file mode 100644 index 0000000000..e05d787700 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -0,0 +1,85 @@ +pub mod attach; +pub mod binary; +pub mod literal; +pub mod notation; +pub mod prefix; +pub mod suffix; +pub mod uniadic_array; +pub mod uniadic_chain; +pub mod variadic_array; +pub mod variadic_chain; + +use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; +use latex_prelude::script::LxScriptKind; +use visored_opr::opr::binary::VdBinaryOpr; +use visored_zfs_ty::term::literal::VdZfsLiteral; + +/// It's a tree of both form and meaning +#[derive(Debug, PartialEq, Eq)] +pub enum VdSynExprData { + Literal { + literal: VdZfsLiteral, + }, + Notation, + Binary { + lopd: VdSynExprIdx, + opr: VdBinaryOpr, + ropd: VdSynExprIdx, + }, + Prefix { + opr: VdSynExprIdx, + opd: VdSynExprIdx, + }, + Suffix { + opd: VdSynExprIdx, + opr: VdSynExprIdx, + }, + Attach { + base: VdSynExprIdx, + // INVARIANCE: at least one of these are some + scripts: Vec<(LxScriptKind, VdSynExprIdx)>, + }, + UniadicChain, + VariadicChain, + UniadicArray, + VariadicArray, +} + +pub type VdSynExprIdx = ArenaIdx; +pub type VdSynExprIdxRange = ArenaIdxRange; +pub type VdSynExprArena = Arena; +pub type VdSynExprArenaRef<'a> = ArenaRef<'a, VdSynExprData>; + +impl VdSynExprData { + pub fn children(&self) -> Vec { + match *self { + VdSynExprData::Literal { literal } => vec![], + VdSynExprData::Notation => vec![], + VdSynExprData::Binary { lopd, opr, ropd } => vec![lopd, ropd], + VdSynExprData::Prefix { opr, opd } => vec![opr, opd], + VdSynExprData::Suffix { opd, opr } => vec![opd, opr], + VdSynExprData::Attach { base, ref scripts } => [base] + .into_iter() + .chain(scripts.iter().map(|&(_, script)| script)) + .collect(), + VdSynExprData::UniadicChain => vec![], + VdSynExprData::VariadicChain => vec![], + VdSynExprData::UniadicArray => vec![], + VdSynExprData::VariadicArray => vec![], + } + } + + pub fn class(&self) -> VdSynExprClass { + match *self { + // ad hoc + _ => VdSynExprClass::Separator, + _ => unreachable!(), + } + } +} + +pub enum VdSynExprClass { + Prefix, + Suffix, + Separator, +} diff --git a/crates/visored/visored-syn-expr/src/expr/attach.rs b/crates/visored/visored-syn-expr/src/expr/attach.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/attach.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/binary.rs b/crates/visored/visored-syn-expr/src/expr/binary.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/binary.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/literal.rs b/crates/visored/visored-syn-expr/src/expr/literal.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/literal.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/notation.rs b/crates/visored/visored-syn-expr/src/expr/notation.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/notation.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/prefix.rs b/crates/visored/visored-syn-expr/src/expr/prefix.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/prefix.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/suffix.rs b/crates/visored/visored-syn-expr/src/expr/suffix.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/suffix.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/uniadic_array.rs b/crates/visored/visored-syn-expr/src/expr/uniadic_array.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/uniadic_array.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/uniadic_chain.rs b/crates/visored/visored-syn-expr/src/expr/uniadic_chain.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/uniadic_chain.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/variadic_array.rs b/crates/visored/visored-syn-expr/src/expr/variadic_array.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/variadic_array.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/expr/variadic_chain.rs b/crates/visored/visored-syn-expr/src/expr/variadic_chain.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/variadic_chain.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/helpers.rs b/crates/visored/visored-syn-expr/src/helpers.rs new file mode 100644 index 0000000000..78775b6ff2 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/helpers.rs @@ -0,0 +1 @@ +pub mod show; diff --git a/crates/visored/visored-syn-expr/src/helpers/show.rs b/crates/visored/visored-syn-expr/src/helpers/show.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/helpers/show.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/jar.rs b/crates/visored/visored-syn-expr/src/jar.rs new file mode 100644 index 0000000000..34e6e52c60 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/jar.rs @@ -0,0 +1,2 @@ +#[salsa::jar] +pub struct VdSynExprJar(); diff --git a/crates/visored/visored-syn-expr/src/lib.rs b/crates/visored/visored-syn-expr/src/lib.rs new file mode 100644 index 0000000000..974a06658c --- /dev/null +++ b/crates/visored/visored-syn-expr/src/lib.rs @@ -0,0 +1,16 @@ +mod builder; +pub mod clause; +pub mod expr; +pub mod helpers; +pub mod jar; +pub mod phrase; +pub mod region; +pub mod sentence; +#[cfg(feature = "test_helpers")] +pub mod test_helpers; +#[cfg(test)] +mod tests; + +use self::jar::VdSynExprJar as Jar; +#[cfg(test)] +use self::tests::*; diff --git a/crates/visored/visored-syn-expr/src/phrase.rs b/crates/visored/visored-syn-expr/src/phrase.rs new file mode 100644 index 0000000000..973b58cc59 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/phrase.rs @@ -0,0 +1,19 @@ +pub mod noun; + +use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; +use noun::VdSynNounPhraseData; + +use crate::expr::VdSynExprIdx; + +pub enum VdSynPhrase { + Expr(VdSynExprIdx), +} + +pub enum VdSynPhraseData { + Noun(VdSynNounPhraseData), +} + +pub type VdSynPhraseArena = Arena; +pub type VdSynPhraseArenaRef<'a> = ArenaRef<'a, VdSynPhraseData>; +pub type VdSynPhraseIdx = ArenaIdx; +pub type VdSynPhraseIdxRange = ArenaIdxRange; diff --git a/crates/visored/visored-syn-expr/src/phrase/noun.rs b/crates/visored/visored-syn-expr/src/phrase/noun.rs new file mode 100644 index 0000000000..c6d03bae0b --- /dev/null +++ b/crates/visored/visored-syn-expr/src/phrase/noun.rs @@ -0,0 +1,5 @@ +use super::*; + +pub struct VdSynNounPhraseData {} + +pub struct NounPhrase(VdSynPhraseIdx); diff --git a/crates/visored/visored-syn-expr/src/region.rs b/crates/visored/visored-syn-expr/src/region.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/region.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/sentence.rs b/crates/visored/visored-syn-expr/src/sentence.rs new file mode 100644 index 0000000000..12ab3dfd59 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/sentence.rs @@ -0,0 +1,12 @@ +use crate::clause::VdSynClauseIdxRange; +use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum VdSynSentenceData { + Clauses(VdSynClauseIdxRange), +} + +pub type VdSynSentenceArena = Arena; +pub type VdSynSentenceArenaRef<'a> = ArenaRef<'a, VdSynSentenceData>; +pub type VdSynSentenceIdx = ArenaIdx; +pub type VdSynSentenceIdxRange = ArenaIdxRange; diff --git a/crates/visored/visored-syn-expr/src/test_helpers.rs b/crates/visored/visored-syn-expr/src/test_helpers.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/test_helpers.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/tests.rs b/crates/visored/visored-syn-expr/src/tests.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/tests.rs @@ -0,0 +1 @@ + From 6522168aa26b642c2fdd443fbca5dd03f96483d7 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 21:42:28 -0700 Subject: [PATCH 33/71] more --- Cargo.lock | 3 + .../expect-files/husky_lang_jar_packages.txt | 16 +- .../expect-files/husky_lang_jar_tree.json | 4 + .../expect-files/husky_lang_jar_tree.txt | 4 + .../husky_lang_jar_tree_trimmed.json | 5 +- .../husky_lang_jar_tree_trimmed.txt | 5 +- .../expect-files/husky_lang_packages.txt | 24 +-- crates/visored/visored-sem-expr/Cargo.toml | 1 + crates/visored/visored-sem-expr/src/tests.rs | 7 +- crates/visored/visored-syn-expr/Cargo.toml | 9 ++ .../visored/visored-syn-expr/src/builder.rs | 25 +++ .../visored/visored-syn-expr/src/helpers.rs | 3 + .../visored-syn-expr/src/helpers/latex_fmt.rs | 149 ++++++++++++++++++ crates/visored/visored-syn-expr/src/region.rs | 44 ++++++ .../visored-syn-expr/src/test_helpers.rs | 2 +- .../src/test_helpers/builder.rs | 91 +++++++++++ crates/visored/visored-syn-expr/src/tests.rs | 3 +- 17 files changed, 368 insertions(+), 27 deletions(-) create mode 100644 crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs create mode 100644 crates/visored/visored-syn-expr/src/test_helpers/builder.rs diff --git a/Cargo.lock b/Cargo.lock index 6803528386..4adcd87e41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12576,6 +12576,7 @@ dependencies = [ "visored-annotation", "visored-opr", "visored-sem-expr", + "visored-syn-expr", "visored-zfs-ty", ] @@ -12583,10 +12584,12 @@ dependencies = [ name = "visored-syn-expr" version = "0.1.0" dependencies = [ + "husky-coword", "idx-arena", "latex-prelude", "salsa", "visored-opr", + "visored-syn-expr", "visored-zfs-ty", ] diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt index 2b8b9f8181..cec4ad4a19 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt @@ -1308,6 +1308,7 @@ name: "visored-sem-expr", dependencies: [ "visored-opr", + "visored-syn-expr", "visored-zfs-ty", "husky-coword", ], @@ -1316,6 +1317,14 @@ name: "visored-opr", dependencies: [], }, + JarPackageSummary { + name: "visored-syn-expr", + dependencies: [ + "visored-opr", + "visored-zfs-ty", + "husky-coword", + ], + }, JarPackageSummary { name: "visored-zfs-ty", dependencies: [], @@ -1344,11 +1353,4 @@ "visored-sem-expr", ], }, - JarPackageSummary { - name: "visored-syn-expr", - dependencies: [ - "visored-opr", - "visored-zfs-ty", - ], - }, ] \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json index 076569497e..fdc1a52f64 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json @@ -1177,6 +1177,7 @@ "husky-coword", "visored-opr", "visored-sem-expr", + "visored-syn-expr", "visored-zfs-ty" ], "visored-lean-transpilation": [ @@ -1187,15 +1188,18 @@ "visored-hir-expr", "visored-opr", "visored-sem-expr", + "visored-syn-expr", "visored-zfs-ty" ], "visored-opr": [], "visored-sem-expr": [ "husky-coword", "visored-opr", + "visored-syn-expr", "visored-zfs-ty" ], "visored-syn-expr": [ + "husky-coword", "visored-opr", "visored-zfs-ty" ], diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt index b315cfe159..6395ed6fa6 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt @@ -1177,6 +1177,7 @@ "husky-coword", "visored-opr", "visored-sem-expr", + "visored-syn-expr", "visored-zfs-ty", }, "visored-lean-transpilation": { @@ -1187,15 +1188,18 @@ "visored-hir-expr", "visored-opr", "visored-sem-expr", + "visored-syn-expr", "visored-zfs-ty", }, "visored-opr": {}, "visored-sem-expr": { "husky-coword", "visored-opr", + "visored-syn-expr", "visored-zfs-ty", }, "visored-syn-expr": { + "husky-coword", "visored-opr", "visored-zfs-ty", }, diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json index fc5f01adae..45aa39e030 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json @@ -191,11 +191,10 @@ ], "visored-opr": [], "visored-sem-expr": [ - "husky-coword", - "visored-opr", - "visored-zfs-ty" + "visored-syn-expr" ], "visored-syn-expr": [ + "husky-coword", "visored-opr", "visored-zfs-ty" ], diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt index 232b7644e8..506c5a3080 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt @@ -191,11 +191,10 @@ }, "visored-opr": {}, "visored-sem-expr": { - "husky-coword", - "visored-opr", - "visored-zfs-ty", + "visored-syn-expr", }, "visored-syn-expr": { + "husky-coword", "visored-opr", "visored-zfs-ty", }, diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 4aa1dd467c..6044e3cb02 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4232,6 +4232,7 @@ "salsa", "visored-annotation", "visored-opr", + "visored-syn-expr", "visored-zfs-ty", "husky-coword", ], @@ -4243,6 +4244,18 @@ "salsa", ], }, + PackageSummary { + name: "visored-syn-expr", + relative_path: "crates/visored/visored-syn-expr", + dependencies: [ + "idx-arena", + "latex-prelude", + "salsa", + "visored-opr", + "visored-zfs-ty", + "husky-coword", + ], + }, PackageSummary { name: "visored-zfs-ty", relative_path: "crates/visored/visored-zfs-ty", @@ -4304,17 +4317,6 @@ "idx-arena", ], }, - PackageSummary { - name: "visored-syn-expr", - relative_path: "crates/visored/visored-syn-expr", - dependencies: [ - "idx-arena", - "latex-prelude", - "salsa", - "visored-opr", - "visored-zfs-ty", - ], - }, PackageSummary { name: "cybertron-mini-lean-tokens", relative_path: "registry/cybertron-mini-lean-tokens-0.1.0", diff --git a/crates/visored/visored-sem-expr/Cargo.toml b/crates/visored/visored-sem-expr/Cargo.toml index 833ba21a74..99383c58c6 100644 --- a/crates/visored/visored-sem-expr/Cargo.toml +++ b/crates/visored/visored-sem-expr/Cargo.toml @@ -21,6 +21,7 @@ latex-prelude.workspace = true visored-annotation.workspace = true visored-opr.workspace = true visored-zfs-ty.workspace = true +visored-syn-expr.workspace = true [dev-dependencies] # kernel diff --git a/crates/visored/visored-sem-expr/src/tests.rs b/crates/visored/visored-sem-expr/src/tests.rs index 79d342de26..5567ea80aa 100644 --- a/crates/visored/visored-sem-expr/src/tests.rs +++ b/crates/visored/visored-sem-expr/src/tests.rs @@ -1,2 +1,7 @@ -#[salsa::db(husky_coword::jar::CowordJar, visored_zfs_ty::jar::VdZfsTypeJar)] +#[salsa::db( + husky_coword::jar::CowordJar, + visored_zfs_ty::jar::VdZfsTypeJar, + visored_opr::jar::VdOprJar, + visored_syn_expr::jar::VdSynExprJar +)] pub(crate) struct DB {} diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index db1e6e608e..c13e294fcf 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -20,5 +20,14 @@ latex-prelude.workspace = true visored-opr.workspace = true visored-zfs-ty.workspace = true +[dev-dependencies] +# kernel +husky-coword.workspace = true +# visored +visored-syn-expr = { workspace = true, features = ["test_helpers"] } + [lints] workspace = true + +[features] +test_helpers = [] diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index 8b13789179..f9a0f7faa4 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -1 +1,26 @@ +use crate::{ + clause::VdSynClauseArena, + expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx}, + phrase::{VdSynPhraseArena, VdSynPhraseData, VdSynPhraseIdx}, + region::VdSynExprRegionData, + sentence::{VdSynSentenceArena, VdSynSentenceData, VdSynSentenceIdx}, +}; +pub(crate) struct VdSemExprBuilder<'db> { + db: &'db ::salsa::Db, + expr_arena: VdSynExprArena, + phrase_arena: VdSynPhraseArena, + clause_arena: VdSynClauseArena, + sentence_arena: VdSynSentenceArena, +} + +impl<'db> VdSemExprBuilder<'db> { + pub fn finish(self) -> VdSynExprRegionData { + VdSynExprRegionData::new( + self.expr_arena, + self.phrase_arena, + self.clause_arena, + self.sentence_arena, + ) + } +} diff --git a/crates/visored/visored-syn-expr/src/helpers.rs b/crates/visored/visored-syn-expr/src/helpers.rs index 78775b6ff2..3f0ca21d63 100644 --- a/crates/visored/visored-syn-expr/src/helpers.rs +++ b/crates/visored/visored-syn-expr/src/helpers.rs @@ -1 +1,4 @@ +pub mod latex_fmt; pub mod show; + +use crate::*; diff --git a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs new file mode 100644 index 0000000000..7c5eea7521 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs @@ -0,0 +1,149 @@ +use super::*; +#[cfg(test)] +use crate::test_helpers::builder::VdSynExprTestBuilder; +use crate::{ + clause::{VdSynClauseArenaRef, VdSynClauseData, VdSynClauseIdx}, + expr::{VdSynExprArenaRef, VdSynExprData, VdSynExprIdx}, + phrase::{ + noun::VdSynNounPhraseData, VdSynPhrase, VdSynPhraseArenaRef, VdSynPhraseData, + VdSynPhraseIdx, + }, + sentence::{VdSynSentenceArenaRef, VdSynSentenceData, VdSynSentenceIdx, VdSynSentenceIdxRange}, +}; +use visored_opr::opr::binary::VdBinaryOpr; +use visored_zfs_ty::{menu::vd_zfs_ty_menu, term::literal::VdZfsLiteralData}; + +pub struct VdSynExprLaTeXFormatter<'a> { + db: &'a ::salsa::Db, + expr_arena: VdSynExprArenaRef<'a>, + phrase_arena: VdSynPhraseArenaRef<'a>, + clause_arena: VdSynClauseArenaRef<'a>, + sentence_arena: VdSynSentenceArenaRef<'a>, + result: String, +} + +impl<'a> VdSynExprLaTeXFormatter<'a> { + pub fn new( + db: &'a ::salsa::Db, + expr_arena: VdSynExprArenaRef<'a>, + phrase_arena: VdSynPhraseArenaRef<'a>, + clause_arena: VdSynClauseArenaRef<'a>, + sentence_arena: VdSynSentenceArenaRef<'a>, + ) -> Self { + Self { + db, + expr_arena, + phrase_arena, + clause_arena, + sentence_arena, + result: Default::default(), + } + } + + pub fn fmt_sentences(&mut self, sentences: VdSynSentenceIdxRange) { + for sentence_idx in sentences { + self.fmt_sentence(sentence_idx); + self.result.push_str("\n\n"); + } + } + + pub fn fmt_sentence(&mut self, sentence_idx: VdSynSentenceIdx) { + match self.sentence_arena[sentence_idx] { + VdSynSentenceData::Clauses(clauses) => { + for (index, clause_idx) in clauses.into_iter().enumerate() { + self.fmt_clause(clause_idx); + if index < clauses.len() - 1 { + self.result.push_str(", "); + } else { + self.result.push_str(". "); + } + } + } + } + } + + pub fn fmt_clause(&mut self, clause_idx: VdSynClauseIdx) { + match self.clause_arena[clause_idx] { + VdSynClauseData::Verb => todo!(), + } + } + + pub fn fmt_phrase(&mut self, phrase_idx: VdSynPhraseIdx) { + match self.phrase_arena[phrase_idx] { + VdSynPhraseData::Noun(ref vd_syn_noun_phrase_data) => todo!(), + } + } + + fn fmt_noun_phrase(&mut self, noun_phrase: &VdSynNounPhraseData) { + // Implement noun phrase formatting + // This is a placeholder implementation + // self.result.push_str("\\textbf{"); + // self.fmt_expr(noun_phrase.head); + // self.result.push('}'); + todo!() + } + + pub fn fmt_expr(&mut self, expr_idx: VdSynExprIdx) { + let db = self.db; + match self.expr_arena[expr_idx] { + VdSynExprData::Literal { literal } => match literal.data(db) { + VdZfsLiteralData::NaturalNumber(s) => { + if self + .result + .chars() + .last() + .map_or(false, |c| c.is_alphanumeric()) + { + self.result.push(' '); + } + self.result.push_str(s); + } + VdZfsLiteralData::NegativeInteger(_) => todo!(), + VdZfsLiteralData::FiniteDecimalRepresentation(_) => { + todo!() + } + VdZfsLiteralData::SpecialConstant(vd_zfs_special_constant) => todo!(), + }, + VdSynExprData::Notation => todo!(), + VdSynExprData::Binary { + lopd, opr, ropd, .. + } => { + self.fmt_expr(lopd); + self.result += opr.latex_code(); + self.fmt_expr(ropd); + } + VdSynExprData::Prefix { opr, opd } => todo!(), + VdSynExprData::Suffix { opd, opr } => todo!(), + VdSynExprData::Attach { .. } => todo!(), + VdSynExprData::UniadicChain => todo!(), + VdSynExprData::VariadicChain => todo!(), + VdSynExprData::UniadicArray => todo!(), + VdSynExprData::VariadicArray => todo!(), + } + } + + pub fn finish(self) -> String { + self.result + } +} + +#[test] +fn latex_fmt_works() { + let db = &DB::default(); + let menu = vd_zfs_ty_menu(db); + let mut builder = VdSynExprTestBuilder::new(db); + let one = builder.new_expr_checked( + VdSynExprData::Literal { + literal: menu.one_literal(), + }, + "1", + ); + let one_add_one = builder.new_expr_checked( + VdSynExprData::Binary { + lopd: one, + opr: VdBinaryOpr::Add, + ropd: one, + }, + "1+1", + ); +} diff --git a/crates/visored/visored-syn-expr/src/region.rs b/crates/visored/visored-syn-expr/src/region.rs index 8b13789179..ea3b161b1b 100644 --- a/crates/visored/visored-syn-expr/src/region.rs +++ b/crates/visored/visored-syn-expr/src/region.rs @@ -1 +1,45 @@ +use crate::{ + clause::{VdSynClauseArena, VdSynClauseArenaRef}, + expr::{VdSynExprArena, VdSynExprArenaRef, VdSynExprData, VdSynExprIdx}, + phrase::{VdSynPhraseArena, VdSynPhraseArenaRef, VdSynPhraseData, VdSynPhraseIdx}, + sentence::{VdSynSentenceArena, VdSynSentenceArenaRef, VdSynSentenceData, VdSynSentenceIdx}, +}; +pub struct VdSynExprRegionData { + expr_arena: VdSynExprArena, + phrase_arena: VdSynPhraseArena, + clause_arena: VdSynClauseArena, + sentence_arena: VdSynSentenceArena, +} + +impl VdSynExprRegionData { + pub(crate) fn new( + expr_arena: VdSynExprArena, + phrase_arena: VdSynPhraseArena, + clause_arena: VdSynClauseArena, + sentence_arena: VdSynSentenceArena, + ) -> Self { + Self { + expr_arena, + phrase_arena, + clause_arena, + sentence_arena, + } + } + + pub fn expr_arena(&self) -> VdSynExprArenaRef { + self.expr_arena.as_arena_ref() + } + + pub fn phrase_arena(&self) -> VdSynPhraseArenaRef { + self.phrase_arena.as_arena_ref() + } + + pub fn clause_arena(&self) -> VdSynClauseArenaRef { + self.clause_arena.as_arena_ref() + } + + pub fn sentence_arena(&self) -> VdSynSentenceArenaRef { + self.sentence_arena.as_arena_ref() + } +} diff --git a/crates/visored/visored-syn-expr/src/test_helpers.rs b/crates/visored/visored-syn-expr/src/test_helpers.rs index 8b13789179..5575a85eb3 100644 --- a/crates/visored/visored-syn-expr/src/test_helpers.rs +++ b/crates/visored/visored-syn-expr/src/test_helpers.rs @@ -1 +1 @@ - +pub mod builder; diff --git a/crates/visored/visored-syn-expr/src/test_helpers/builder.rs b/crates/visored/visored-syn-expr/src/test_helpers/builder.rs new file mode 100644 index 0000000000..4e9b4202dc --- /dev/null +++ b/crates/visored/visored-syn-expr/src/test_helpers/builder.rs @@ -0,0 +1,91 @@ +use crate::{ + clause::{VdSynClauseArena, VdSynClauseData, VdSynClauseIdx}, + expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx}, + helpers::latex_fmt::VdSynExprLaTeXFormatter, + phrase::{VdSynPhraseArena, VdSynPhraseData, VdSynPhraseIdx}, + region::VdSynExprRegionData, + sentence::{VdSynSentenceArena, VdSynSentenceData, VdSynSentenceIdx}, +}; + +pub struct VdSynExprTestBuilder<'db> { + db: &'db ::salsa::Db, + expr_arena: VdSynExprArena, + phrase_arena: VdSynPhraseArena, + clause_arena: VdSynClauseArena, + sentence_arena: VdSynSentenceArena, +} + +impl<'db> VdSynExprTestBuilder<'db> { + pub fn new(db: &'db ::salsa::Db) -> Self { + Self { + db, + expr_arena: VdSynExprArena::default(), + phrase_arena: VdSynPhraseArena::default(), + clause_arena: VdSynClauseArena::default(), + sentence_arena: VdSynSentenceArena::default(), + } + } +} + +impl<'db> VdSynExprTestBuilder<'db> { + pub fn latex_formatter<'a>(&'a self) -> VdSynExprLaTeXFormatter<'a> { + VdSynExprLaTeXFormatter::new( + self.db, + self.expr_arena.as_arena_ref(), + self.phrase_arena.as_arena_ref(), + self.clause_arena.as_arena_ref(), + self.sentence_arena.as_arena_ref(), + ) + } +} + +impl<'db> VdSynExprTestBuilder<'db> { + #[track_caller] + pub fn new_expr_checked(&mut self, data: VdSynExprData, expected: &str) -> VdSynExprIdx { + let expr = self.expr_arena.alloc_one(data); + let mut formatter = self.latex_formatter(); + formatter.fmt_expr(expr); + assert_eq!(&formatter.finish(), expected); + expr + } + + #[track_caller] + pub fn new_phrase_checked(&mut self, data: VdSynPhraseData, expected: &str) -> VdSynPhraseIdx { + let phrase = self.phrase_arena.alloc_one(data); + let mut formatter = self.latex_formatter(); + formatter.fmt_phrase(phrase); + assert_eq!(&formatter.finish(), expected); + phrase + } + + #[track_caller] + pub fn new_clause_checked(&mut self, data: VdSynClauseData, expected: &str) -> VdSynClauseIdx { + let clause = self.clause_arena.alloc_one(data); + let mut formatter = self.latex_formatter(); + formatter.fmt_clause(clause); + assert_eq!(&formatter.finish(), expected); + clause + } + + #[track_caller] + pub fn new_sentence_checked( + &mut self, + data: VdSynSentenceData, + expected: &str, + ) -> VdSynSentenceIdx { + let sentence = self.sentence_arena.alloc_one(data); + let mut formatter = self.latex_formatter(); + formatter.fmt_sentence(sentence); + assert_eq!(&formatter.finish(), expected); + sentence + } + + pub fn finish(self) -> VdSynExprRegionData { + VdSynExprRegionData::new( + self.expr_arena, + self.phrase_arena, + self.clause_arena, + self.sentence_arena, + ) + } +} diff --git a/crates/visored/visored-syn-expr/src/tests.rs b/crates/visored/visored-syn-expr/src/tests.rs index 8b13789179..79d342de26 100644 --- a/crates/visored/visored-syn-expr/src/tests.rs +++ b/crates/visored/visored-syn-expr/src/tests.rs @@ -1 +1,2 @@ - +#[salsa::db(husky_coword::jar::CowordJar, visored_zfs_ty::jar::VdZfsTypeJar)] +pub(crate) struct DB {} From f2c42bdd8eab9ffdd5ab1b25148bef50fce09b2d Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 22:48:58 -0700 Subject: [PATCH 34/71] save --- Cargo.lock | 2 ++ .../expect-files/husky_lang_jar_packages.txt | 2 ++ .../husky-jar-utils/expect-files/husky_lang_packages.txt | 2 ++ crates/visored/visored-hir-expr/Cargo.toml | 1 + crates/visored/visored-hir-expr/src/tests.rs | 1 + crates/visored/visored-lean-transpilation/Cargo.toml | 1 + crates/visored/visored-lean-transpilation/src/tests.rs | 1 + crates/visored/visored-syn-expr/src/builder.rs | 7 +++++-- 8 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4adcd87e41..e6dd246b76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12525,6 +12525,7 @@ dependencies = [ "salsa", "visored-opr", "visored-sem-expr", + "visored-syn-expr", "visored-zfs-ty", ] @@ -12548,6 +12549,7 @@ dependencies = [ "visored-hir-expr", "visored-opr", "visored-sem-expr", + "visored-syn-expr", "visored-zfs-ty", ] diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt index cec4ad4a19..2b80f5f64e 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt @@ -1337,6 +1337,7 @@ "visored-zfs-ty", "husky-coword", "visored-sem-expr", + "visored-syn-expr", ], }, JarPackageSummary { @@ -1351,6 +1352,7 @@ "lean-sem-expr", "lean-term", "visored-sem-expr", + "visored-syn-expr", ], }, ] \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 6044e3cb02..38ea7d4c93 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4284,6 +4284,7 @@ "visored-zfs-ty", "husky-coword", "visored-sem-expr", + "visored-syn-expr", ], }, PackageSummary { @@ -4308,6 +4309,7 @@ "lean-sem-expr", "lean-term", "visored-sem-expr", + "visored-syn-expr", ], }, PackageSummary { diff --git a/crates/visored/visored-hir-expr/Cargo.toml b/crates/visored/visored-hir-expr/Cargo.toml index b815570c80..7750ab9bc7 100644 --- a/crates/visored/visored-hir-expr/Cargo.toml +++ b/crates/visored/visored-hir-expr/Cargo.toml @@ -25,6 +25,7 @@ visored-zfs-ty.workspace = true husky-coword.workspace = true # visored visored-sem-expr = { workspace = true, features = ["test_helpers"] } +visored-syn-expr = { workspace = true, features = ["test_helpers"] } [lints] workspace = true diff --git a/crates/visored/visored-hir-expr/src/tests.rs b/crates/visored/visored-hir-expr/src/tests.rs index 54efa3f0cf..665d41d90f 100644 --- a/crates/visored/visored-hir-expr/src/tests.rs +++ b/crates/visored/visored-hir-expr/src/tests.rs @@ -12,6 +12,7 @@ use crate::{builder::VdHirExprBuilder, ToVdHir}; visored_zfs_ty::jar::VdZfsTypeJar, visored_opr::jar::VdOprJar, visored_sem_expr::jar::VdSemExprJar, + visored_syn_expr::jar::VdSynExprJar, crate::jar::VdHirExprJar )] pub(crate) struct DB {} diff --git a/crates/visored/visored-lean-transpilation/Cargo.toml b/crates/visored/visored-lean-transpilation/Cargo.toml index 9479f67b26..07e3beadaa 100644 --- a/crates/visored/visored-lean-transpilation/Cargo.toml +++ b/crates/visored/visored-lean-transpilation/Cargo.toml @@ -31,6 +31,7 @@ lean-term.workspace = true lean-sem-expr.workspace = true # visored visored-sem-expr = { workspace = true, features = ["test_helpers"] } +visored-syn-expr = { workspace = true, features = ["test_helpers"] } [lints] workspace = true diff --git a/crates/visored/visored-lean-transpilation/src/tests.rs b/crates/visored/visored-lean-transpilation/src/tests.rs index 870b470f42..4da2423a7e 100644 --- a/crates/visored/visored-lean-transpilation/src/tests.rs +++ b/crates/visored/visored-lean-transpilation/src/tests.rs @@ -17,6 +17,7 @@ use crate::{builder::VdLeanTranspilationBuilder, ToLean}; visored_opr::jar::VdOprJar, visored_sem_expr::jar::VdSemExprJar, visored_hir_expr::jar::VdHirExprJar, + visored_syn_expr::jar::VdSynExprJar, crate::Jar )] pub(crate) struct DB {} diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index f9a0f7faa4..27c2a2f0f1 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -6,7 +6,7 @@ use crate::{ sentence::{VdSynSentenceArena, VdSynSentenceData, VdSynSentenceIdx}, }; -pub(crate) struct VdSemExprBuilder<'db> { +pub(crate) struct VdSynExprBuilder<'db> { db: &'db ::salsa::Db, expr_arena: VdSynExprArena, phrase_arena: VdSynPhraseArena, @@ -14,7 +14,7 @@ pub(crate) struct VdSemExprBuilder<'db> { sentence_arena: VdSynSentenceArena, } -impl<'db> VdSemExprBuilder<'db> { +impl<'db> VdSynExprBuilder<'db> { pub fn finish(self) -> VdSynExprRegionData { VdSynExprRegionData::new( self.expr_arena, @@ -24,3 +24,6 @@ impl<'db> VdSemExprBuilder<'db> { ) } } +pub trait ToVdSyn { + fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> T; +} From 08899fdce6c71bd5758e030ce92c395ba9f7e25a Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 23:29:36 -0700 Subject: [PATCH 35/71] more --- Cargo.lock | 6 +++ crates/syntax/husky-syn-expr/src/parser.rs | 2 +- .../husky-syn-expr/src/parser/expr_stack.rs | 4 +- .../expect-files/husky_lang_jar_packages.txt | 4 ++ .../expect-files/husky_lang_jar_tree.json | 4 ++ .../expect-files/husky_lang_jar_tree.txt | 4 ++ .../husky_lang_jar_tree_trimmed.json | 2 +- .../husky_lang_jar_tree_trimmed.txt | 2 +- .../expect-files/husky_lang_packages.txt | 6 +++ crates/visored/visored-hir-expr/Cargo.toml | 2 + crates/visored/visored-hir-expr/src/tests.rs | 1 + .../visored-lean-transpilation/Cargo.toml | 2 + .../visored-lean-transpilation/src/tests.rs | 1 + crates/visored/visored-opr/src/delimiter.rs | 5 ++ crates/visored/visored-opr/src/lib.rs | 2 + crates/visored/visored-opr/src/precedence.rs | 8 ++-- crates/visored/visored-opr/src/separator.rs | 7 +++ crates/visored/visored-sem-expr/Cargo.toml | 2 + crates/visored/visored-sem-expr/src/tests.rs | 2 + crates/visored/visored-syn-expr/Cargo.toml | 4 ++ crates/visored/visored-syn-expr/src/expr.rs | 1 + .../visored-syn-expr/src/expr/list_item.rs | 29 ++++++++++++ crates/visored/visored-syn-expr/src/lib.rs | 2 + crates/visored/visored-syn-expr/src/parser.rs | 15 ++++++ .../visored-syn-expr/src/parser/accept.rs | 1 + .../visored-syn-expr/src/parser/debug.rs | 1 + .../src/parser/disambiguate_token.rs | 1 + .../visored-syn-expr/src/parser/env.rs | 1 + .../visored-syn-expr/src/parser/expr_stack.rs | 7 +++ .../src/parser/incomplete_expr.rs | 46 +++++++++++++++++++ .../src/parser/incomplete_expr/call_list.rs | 2 + .../parser/incomplete_expr/separated_list.rs | 2 + 32 files changed, 170 insertions(+), 8 deletions(-) create mode 100644 crates/visored/visored-opr/src/delimiter.rs create mode 100644 crates/visored/visored-opr/src/separator.rs create mode 100644 crates/visored/visored-syn-expr/src/expr/list_item.rs create mode 100644 crates/visored/visored-syn-expr/src/parser.rs create mode 100644 crates/visored/visored-syn-expr/src/parser/accept.rs create mode 100644 crates/visored/visored-syn-expr/src/parser/debug.rs create mode 100644 crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs create mode 100644 crates/visored/visored-syn-expr/src/parser/env.rs create mode 100644 crates/visored/visored-syn-expr/src/parser/expr_stack.rs create mode 100644 crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs create mode 100644 crates/visored/visored-syn-expr/src/parser/incomplete_expr/call_list.rs create mode 100644 crates/visored/visored-syn-expr/src/parser/incomplete_expr/separated_list.rs diff --git a/Cargo.lock b/Cargo.lock index e6dd246b76..a0934f5d65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12522,6 +12522,7 @@ version = "0.1.0" dependencies = [ "husky-coword", "idx-arena", + "latex-ast", "salsa", "visored-opr", "visored-sem-expr", @@ -12541,6 +12542,7 @@ name = "visored-lean-transpilation" version = "0.1.0" dependencies = [ "husky-coword", + "latex-ast", "lean-hir-expr", "lean-opr", "lean-sem-expr", @@ -12573,6 +12575,7 @@ version = "0.1.0" dependencies = [ "husky-coword", "idx-arena", + "latex-ast", "latex-prelude", "salsa", "visored-annotation", @@ -12586,10 +12589,13 @@ dependencies = [ name = "visored-syn-expr" version = "0.1.0" dependencies = [ + "enum-class", "husky-coword", "idx-arena", + "latex-ast", "latex-prelude", "salsa", + "smallvec", "visored-opr", "visored-syn-expr", "visored-zfs-ty", diff --git a/crates/syntax/husky-syn-expr/src/parser.rs b/crates/syntax/husky-syn-expr/src/parser.rs index f3794ea067..cd1eb1b971 100644 --- a/crates/syntax/husky-syn-expr/src/parser.rs +++ b/crates/syntax/husky-syn-expr/src/parser.rs @@ -26,7 +26,7 @@ where pub(crate) context: C, env_stack: ExprEnvironmentStack, token_stream: RegionalTokenStream<'a>, - stack: ExprStack, + stack: SynExprStack, } pub type StandaloneSynExprParser<'a> = SynExprParser<'a, SynExprContext<'a>>; diff --git a/crates/syntax/husky-syn-expr/src/parser/expr_stack.rs b/crates/syntax/husky-syn-expr/src/parser/expr_stack.rs index 10de474003..8b6656f88a 100644 --- a/crates/syntax/husky-syn-expr/src/parser/expr_stack.rs +++ b/crates/syntax/husky-syn-expr/src/parser/expr_stack.rs @@ -11,7 +11,7 @@ use husky_token_data::delimiter::Delimiter; /// ``` /// in the above `c + d` would be the finished expression, `a +`, `b *` and `(c + d` would be unfinished expressions. #[derive(Default, Debug)] -pub(crate) struct ExprStack { +pub(crate) struct SynExprStack { incomplete_exprs: Vec<(IncompleteSynExprData, Precedence)>, complete_expr: Option, } @@ -48,7 +48,7 @@ impl From for TopSynExpr { } } -impl ExprStack { +impl SynExprStack { pub(super) fn prev_unfinished_expr_precedence(&self) -> Option { self.incomplete_exprs .last() diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt index 2b80f5f64e..c3965dede3 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt @@ -1311,6 +1311,7 @@ "visored-syn-expr", "visored-zfs-ty", "husky-coword", + "latex-ast", ], }, JarPackageSummary { @@ -1320,6 +1321,7 @@ JarPackageSummary { name: "visored-syn-expr", dependencies: [ + "latex-ast", "visored-opr", "visored-zfs-ty", "husky-coword", @@ -1336,6 +1338,7 @@ "visored-sem-expr", "visored-zfs-ty", "husky-coword", + "latex-ast", "visored-sem-expr", "visored-syn-expr", ], @@ -1349,6 +1352,7 @@ "visored-opr", "visored-zfs-ty", "husky-coword", + "latex-ast", "lean-sem-expr", "lean-term", "visored-sem-expr", diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json index fdc1a52f64..003981e99c 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json @@ -1175,6 +1175,7 @@ "namekian-ast": [], "visored-hir-expr": [ "husky-coword", + "latex-ast", "visored-opr", "visored-sem-expr", "visored-syn-expr", @@ -1182,6 +1183,7 @@ ], "visored-lean-transpilation": [ "husky-coword", + "latex-ast", "lean-hir-expr", "lean-sem-expr", "lean-term", @@ -1194,12 +1196,14 @@ "visored-opr": [], "visored-sem-expr": [ "husky-coword", + "latex-ast", "visored-opr", "visored-syn-expr", "visored-zfs-ty" ], "visored-syn-expr": [ "husky-coword", + "latex-ast", "visored-opr", "visored-zfs-ty" ], diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt index 6395ed6fa6..640e861eb9 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt @@ -1175,6 +1175,7 @@ "namekian-ast": {}, "visored-hir-expr": { "husky-coword", + "latex-ast", "visored-opr", "visored-sem-expr", "visored-syn-expr", @@ -1182,6 +1183,7 @@ }, "visored-lean-transpilation": { "husky-coword", + "latex-ast", "lean-hir-expr", "lean-sem-expr", "lean-term", @@ -1194,12 +1196,14 @@ "visored-opr": {}, "visored-sem-expr": { "husky-coword", + "latex-ast", "visored-opr", "visored-syn-expr", "visored-zfs-ty", }, "visored-syn-expr": { "husky-coword", + "latex-ast", "visored-opr", "visored-zfs-ty", }, diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json index 45aa39e030..cb1d2a9a71 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json @@ -194,7 +194,7 @@ "visored-syn-expr" ], "visored-syn-expr": [ - "husky-coword", + "latex-ast", "visored-opr", "visored-zfs-ty" ], diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt index 506c5a3080..e651da9470 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt @@ -194,7 +194,7 @@ "visored-syn-expr", }, "visored-syn-expr": { - "husky-coword", + "latex-ast", "visored-opr", "visored-zfs-ty", }, diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 38ea7d4c93..83429fa370 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4235,6 +4235,7 @@ "visored-syn-expr", "visored-zfs-ty", "husky-coword", + "latex-ast", ], }, PackageSummary { @@ -4248,9 +4249,12 @@ name: "visored-syn-expr", relative_path: "crates/visored/visored-syn-expr", dependencies: [ + "enum-class", "idx-arena", + "latex-ast", "latex-prelude", "salsa", + "smallvec", "visored-opr", "visored-zfs-ty", "husky-coword", @@ -4283,6 +4287,7 @@ "visored-sem-expr", "visored-zfs-ty", "husky-coword", + "latex-ast", "visored-sem-expr", "visored-syn-expr", ], @@ -4306,6 +4311,7 @@ "visored-opr", "visored-zfs-ty", "husky-coword", + "latex-ast", "lean-sem-expr", "lean-term", "visored-sem-expr", diff --git a/crates/visored/visored-hir-expr/Cargo.toml b/crates/visored/visored-hir-expr/Cargo.toml index 7750ab9bc7..00744a2691 100644 --- a/crates/visored/visored-hir-expr/Cargo.toml +++ b/crates/visored/visored-hir-expr/Cargo.toml @@ -21,6 +21,8 @@ visored-opr.workspace = true visored-zfs-ty.workspace = true [dev-dependencies] +# latex +latex-ast.workspace = true # kernel husky-coword.workspace = true # visored diff --git a/crates/visored/visored-hir-expr/src/tests.rs b/crates/visored/visored-hir-expr/src/tests.rs index 665d41d90f..d33564f2cc 100644 --- a/crates/visored/visored-hir-expr/src/tests.rs +++ b/crates/visored/visored-hir-expr/src/tests.rs @@ -9,6 +9,7 @@ use crate::{builder::VdHirExprBuilder, ToVdHir}; #[salsa::db( husky_coword::jar::CowordJar, + latex_ast::jar::LxAstJar, visored_zfs_ty::jar::VdZfsTypeJar, visored_opr::jar::VdOprJar, visored_sem_expr::jar::VdSemExprJar, diff --git a/crates/visored/visored-lean-transpilation/Cargo.toml b/crates/visored/visored-lean-transpilation/Cargo.toml index 07e3beadaa..f31365a53e 100644 --- a/crates/visored/visored-lean-transpilation/Cargo.toml +++ b/crates/visored/visored-lean-transpilation/Cargo.toml @@ -26,6 +26,8 @@ visored-zfs-ty.workspace = true [dev-dependencies] # kernel husky-coword.workspace = true +# latex +latex-ast.workspace = true # lean lean-term.workspace = true lean-sem-expr.workspace = true diff --git a/crates/visored/visored-lean-transpilation/src/tests.rs b/crates/visored/visored-lean-transpilation/src/tests.rs index 4da2423a7e..a8e8b906f7 100644 --- a/crates/visored/visored-lean-transpilation/src/tests.rs +++ b/crates/visored/visored-lean-transpilation/src/tests.rs @@ -10,6 +10,7 @@ use crate::{builder::VdLeanTranspilationBuilder, ToLean}; #[salsa::db( husky_coword::jar::CowordJar, + latex_ast::jar::LxAstJar, lean_term::jar::LnTermJar, lean_sem_expr::jar::LnSemExprJar, lean_hir_expr::jar::LnHirExprJar, diff --git a/crates/visored/visored-opr/src/delimiter.rs b/crates/visored/visored-opr/src/delimiter.rs new file mode 100644 index 0000000000..56e3545cc6 --- /dev/null +++ b/crates/visored/visored-opr/src/delimiter.rs @@ -0,0 +1,5 @@ +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdDelimiter { + Comma, + Semicolon, +} diff --git a/crates/visored/visored-opr/src/lib.rs b/crates/visored/visored-opr/src/lib.rs index 444f8bbed5..da57226322 100644 --- a/crates/visored/visored-opr/src/lib.rs +++ b/crates/visored/visored-opr/src/lib.rs @@ -1,3 +1,5 @@ +pub mod delimiter; pub mod jar; pub mod opr; pub mod precedence; +pub mod separator; diff --git a/crates/visored/visored-opr/src/precedence.rs b/crates/visored/visored-opr/src/precedence.rs index 95a01fa58c..1037fcf0e5 100644 --- a/crates/visored/visored-opr/src/precedence.rs +++ b/crates/visored/visored-opr/src/precedence.rs @@ -1,6 +1,8 @@ #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] -pub enum VdPrecedence { - Application, +pub struct VdPrecedence(u64); + +impl VdPrecedence { + pub const APPLICATION: Self = VdPrecedence(0); } #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -12,7 +14,7 @@ pub enum VdPrecedenceRange { /// # constants impl VdPrecedenceRange { - pub const APPLICATION_SUBEXPR: Self = VdPrecedenceRange::Greater(VdPrecedence::Application); + pub const APPLICATION_SUBEXPR: Self = VdPrecedenceRange::Greater(VdPrecedence::APPLICATION); } /// # methods diff --git a/crates/visored/visored-opr/src/separator.rs b/crates/visored/visored-opr/src/separator.rs new file mode 100644 index 0000000000..bc3e8da777 --- /dev/null +++ b/crates/visored/visored-opr/src/separator.rs @@ -0,0 +1,7 @@ +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdSeparator { + Comma, + Semicolon, + Add, + Mul, +} diff --git a/crates/visored/visored-sem-expr/Cargo.toml b/crates/visored/visored-sem-expr/Cargo.toml index 99383c58c6..9e03aec490 100644 --- a/crates/visored/visored-sem-expr/Cargo.toml +++ b/crates/visored/visored-sem-expr/Cargo.toml @@ -26,6 +26,8 @@ visored-syn-expr.workspace = true [dev-dependencies] # kernel husky-coword.workspace = true +# latex +latex-ast.workspace = true # visored visored-sem-expr = { workspace = true, features = ["test_helpers"] } diff --git a/crates/visored/visored-sem-expr/src/tests.rs b/crates/visored/visored-sem-expr/src/tests.rs index 5567ea80aa..36a138519c 100644 --- a/crates/visored/visored-sem-expr/src/tests.rs +++ b/crates/visored/visored-sem-expr/src/tests.rs @@ -2,6 +2,8 @@ husky_coword::jar::CowordJar, visored_zfs_ty::jar::VdZfsTypeJar, visored_opr::jar::VdOprJar, + latex_ast::jar::LxAstJar, + crate::jar::VdSemExprJar, visored_syn_expr::jar::VdSynExprJar )] pub(crate) struct DB {} diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index c13e294fcf..db56a9bbae 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -12,11 +12,15 @@ categories.workspace = true keywords.workspace = true [dependencies] +enum-class.workspace = true +smallvec.workspace = true # abstractions salsa.workspace = true idx-arena.workspace = true # latex latex-prelude.workspace = true +latex-ast.workspace = true +# visored visored-opr.workspace = true visored-zfs-ty.workspace = true diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index e05d787700..0f36a6282c 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -1,5 +1,6 @@ pub mod attach; pub mod binary; +pub mod list_item; pub mod literal; pub mod notation; pub mod prefix; diff --git a/crates/visored/visored-syn-expr/src/expr/list_item.rs b/crates/visored/visored-syn-expr/src/expr/list_item.rs new file mode 100644 index 0000000000..a99bbd4797 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/expr/list_item.rs @@ -0,0 +1,29 @@ +use super::*; +use latex_ast::ast::LxAstIdx; + +#[salsa::derive_debug_with_db] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct VdSynSeparatedListItem { + syn_expr_idx: VdSynExprIdx, + separator_latex_ast_idx: Option, +} + +impl VdSynSeparatedListItem { + pub(crate) fn new( + syn_expr_idx: VdSynExprIdx, + separator_latex_ast_idx: Option, + ) -> Self { + Self { + syn_expr_idx, + separator_latex_ast_idx, + } + } + + pub fn syn_expr_idx(self) -> VdSynExprIdx { + self.syn_expr_idx + } + + pub fn separator_latex_ast_idx(self) -> Option { + self.separator_latex_ast_idx + } +} diff --git a/crates/visored/visored-syn-expr/src/lib.rs b/crates/visored/visored-syn-expr/src/lib.rs index 974a06658c..890e77fe59 100644 --- a/crates/visored/visored-syn-expr/src/lib.rs +++ b/crates/visored/visored-syn-expr/src/lib.rs @@ -3,6 +3,7 @@ pub mod clause; pub mod expr; pub mod helpers; pub mod jar; +pub mod parser; pub mod phrase; pub mod region; pub mod sentence; @@ -14,3 +15,4 @@ mod tests; use self::jar::VdSynExprJar as Jar; #[cfg(test)] use self::tests::*; +use smallvec::SmallVec; diff --git a/crates/visored/visored-syn-expr/src/parser.rs b/crates/visored/visored-syn-expr/src/parser.rs new file mode 100644 index 0000000000..5dcb297966 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser.rs @@ -0,0 +1,15 @@ +mod accept; +mod debug; +mod disambiguate_token; +mod env; +mod expr_stack; +pub(crate) mod incomplete_expr; + +use self::expr_stack::VdSynExprStack; +use crate::builder::VdSynExprBuilder; +use crate::*; + +pub struct VdSynExprParser<'a, 'db> { + builder: &'a mut VdSynExprBuilder<'db>, + stack: VdSynExprStack, +} diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/parser/debug.rs b/crates/visored/visored-syn-expr/src/parser/debug.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser/debug.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/parser/env.rs b/crates/visored/visored-syn-expr/src/parser/env.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser/env.rs @@ -0,0 +1 @@ + diff --git a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs new file mode 100644 index 0000000000..c666decbfc --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs @@ -0,0 +1,7 @@ +use super::{expr::VdSynExprData, incomplete_expr::IncompleteVdSynExprData}; +use visored_opr::precedence::VdPrecedence; + +pub(crate) struct VdSynExprStack { + incomplete_exprs: Vec<(IncompleteVdSynExprData, VdPrecedence)>, + complete_expr: Option, +} diff --git a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs new file mode 100644 index 0000000000..e3ce7db60e --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs @@ -0,0 +1,46 @@ +mod call_list; +mod separated_list; + +pub(super) use self::call_list::*; +pub(super) use self::separated_list::*; + +use super::*; +use crate::expr::VdSynExprData; +use expr::list_item::VdSynSeparatedListItem; +use visored_opr::{delimiter::VdDelimiter, precedence::VdPrecedence}; + +#[derive(Debug, PartialEq, Eq)] +pub(super) enum IncompleteVdSynExprData { + Binary { + lopd: VdSynExprData, + opr: VdSynExprData, + }, + Prefix { + opr: VdSynExprData, + }, + /// list separated by commas + /// ```husky + /// A(a, b, c) + /// ``` + SeparatedList { + opr: IncompleteSeparatedListOpr, + bra: VdDelimiter, + items: SmallVec<[VdSynSeparatedListItem; 4]>, + }, + /// call list includes more separators like `;` + CallList { + opr: IncompleteCallListOpr, + items: SmallVec<[VdSynSeparatedListItem; 4]>, + }, +} + +impl IncompleteVdSynExprData { + pub(super) fn precedence(&self) -> VdPrecedence { + match self { + IncompleteVdSynExprData::Binary { opr, .. } => todo!(), + IncompleteVdSynExprData::Prefix { opr, .. } => todo!(), + IncompleteVdSynExprData::SeparatedList { .. } + | IncompleteVdSynExprData::CallList { .. } => todo!(), + } + } +} diff --git a/crates/visored/visored-syn-expr/src/parser/incomplete_expr/call_list.rs b/crates/visored/visored-syn-expr/src/parser/incomplete_expr/call_list.rs new file mode 100644 index 0000000000..fe86394c60 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser/incomplete_expr/call_list.rs @@ -0,0 +1,2 @@ +#[derive(Debug, PartialEq, Eq)] +pub enum IncompleteCallListOpr {} diff --git a/crates/visored/visored-syn-expr/src/parser/incomplete_expr/separated_list.rs b/crates/visored/visored-syn-expr/src/parser/incomplete_expr/separated_list.rs new file mode 100644 index 0000000000..4adcd40227 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/parser/incomplete_expr/separated_list.rs @@ -0,0 +1,2 @@ +#[derive(Debug, PartialEq, Eq)] +pub enum IncompleteSeparatedListOpr {} From ce94191751b98b3e1e6855ff0078f57bdfeb0f1f Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Wed, 30 Oct 2024 23:38:27 -0700 Subject: [PATCH 36/71] save --- Cargo.lock | 1 + .../expect-files/husky_lang_packages.txt | 1 + crates/visored/visored-opr/src/opr.rs | 9 ++++++ crates/visored/visored-syn-expr/Cargo.toml | 1 + crates/visored/visored-syn-expr/src/expr.rs | 32 +++++++++++++++---- .../visored-syn-expr/src/helpers/latex_fmt.rs | 9 ++++-- 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0934f5d65..a1b1ca8f10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12589,6 +12589,7 @@ dependencies = [ name = "visored-syn-expr" version = "0.1.0" dependencies = [ + "either", "enum-class", "husky-coword", "idx-arena", diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 83429fa370..4db9299330 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4249,6 +4249,7 @@ name: "visored-syn-expr", relative_path: "crates/visored/visored-syn-expr", dependencies: [ + "either", "enum-class", "idx-arena", "latex-ast", diff --git a/crates/visored/visored-opr/src/opr.rs b/crates/visored/visored-opr/src/opr.rs index 3baf018a0d..fee5cc325e 100644 --- a/crates/visored/visored-opr/src/opr.rs +++ b/crates/visored/visored-opr/src/opr.rs @@ -1,3 +1,12 @@ pub mod binary; pub mod prefix; pub mod suffix; + +use self::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr}; + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub enum VdOpr { + Binary(VdBinaryOpr), + Prefix(VdPrefixOpr), + Suffix(VdSuffixOpr), +} diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index db56a9bbae..410c530eb4 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -12,6 +12,7 @@ categories.workspace = true keywords.workspace = true [dependencies] +either.workspace = true enum-class.workspace = true smallvec.workspace = true # abstractions diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 0f36a6282c..b2a8c2b3dc 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -10,9 +10,10 @@ pub mod uniadic_chain; pub mod variadic_array; pub mod variadic_chain; +use either::*; use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; use latex_prelude::script::LxScriptKind; -use visored_opr::opr::binary::VdBinaryOpr; +use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}; use visored_zfs_ty::term::literal::VdZfsLiteral; /// It's a tree of both form and meaning @@ -22,18 +23,21 @@ pub enum VdSynExprData { literal: VdZfsLiteral, }, Notation, + Opr { + opr: VdOpr, + }, Binary { lopd: VdSynExprIdx, - opr: VdBinaryOpr, + opr: Either, ropd: VdSynExprIdx, }, Prefix { - opr: VdSynExprIdx, + opr: Either, opd: VdSynExprIdx, }, Suffix { opd: VdSynExprIdx, - opr: VdSynExprIdx, + opr: Either, }, Attach { base: VdSynExprIdx, @@ -56,16 +60,30 @@ impl VdSynExprData { match *self { VdSynExprData::Literal { literal } => vec![], VdSynExprData::Notation => vec![], - VdSynExprData::Binary { lopd, opr, ropd } => vec![lopd, ropd], - VdSynExprData::Prefix { opr, opd } => vec![opr, opd], - VdSynExprData::Suffix { opd, opr } => vec![opd, opr], + VdSynExprData::Opr { opr } => vec![], + VdSynExprData::Binary { lopd, opr, ropd } => match opr { + Left(_) => vec![lopd, ropd], + Right(opr) => vec![lopd, opr, ropd], + }, + VdSynExprData::Prefix { opr, opd } => match opr { + Left(_) => vec![opd], + Right(opr) => vec![opr, opd], + }, + VdSynExprData::Suffix { opd, opr } => match opr { + Left(_) => vec![opd], + Right(opr) => vec![opd, opr], + }, VdSynExprData::Attach { base, ref scripts } => [base] .into_iter() .chain(scripts.iter().map(|&(_, script)| script)) .collect(), + // ad hoc VdSynExprData::UniadicChain => vec![], + // ad hoc VdSynExprData::VariadicChain => vec![], + // ad hoc VdSynExprData::UniadicArray => vec![], + // ad hoc VdSynExprData::VariadicArray => vec![], } } diff --git a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs index 7c5eea7521..5f1d75402c 100644 --- a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs +++ b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs @@ -10,6 +10,7 @@ use crate::{ }, sentence::{VdSynSentenceArenaRef, VdSynSentenceData, VdSynSentenceIdx, VdSynSentenceIdxRange}, }; +use either::*; use visored_opr::opr::binary::VdBinaryOpr; use visored_zfs_ty::{menu::vd_zfs_ty_menu, term::literal::VdZfsLiteralData}; @@ -109,7 +110,10 @@ impl<'a> VdSynExprLaTeXFormatter<'a> { lopd, opr, ropd, .. } => { self.fmt_expr(lopd); - self.result += opr.latex_code(); + match opr { + Left(opr) => self.result.push_str(opr.latex_code()), + Either::Right(opr) => self.fmt_expr(opr), + } self.fmt_expr(ropd); } VdSynExprData::Prefix { opr, opd } => todo!(), @@ -119,6 +123,7 @@ impl<'a> VdSynExprLaTeXFormatter<'a> { VdSynExprData::VariadicChain => todo!(), VdSynExprData::UniadicArray => todo!(), VdSynExprData::VariadicArray => todo!(), + VdSynExprData::Opr { opr } => todo!(), } } @@ -141,7 +146,7 @@ fn latex_fmt_works() { let one_add_one = builder.new_expr_checked( VdSynExprData::Binary { lopd: one, - opr: VdBinaryOpr::Add, + opr: Either::Left(VdBinaryOpr::Add), ropd: one, }, "1+1", From 241ad673a6bd6ae32872ac7acc7b86736f7730e7 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 00:12:02 -0700 Subject: [PATCH 37/71] save --- Cargo.lock | 1 + .../expect-files/husky_lang_packages.txt | 1 + crates/visored/visored-opr/src/delimiter.rs | 8 +++---- crates/visored/visored-syn-expr/Cargo.toml | 1 + crates/visored/visored-syn-expr/src/expr.rs | 9 ++++++++ .../src/parser/disambiguate_token.rs | 21 +++++++++++++++++++ .../src/parser/incomplete_expr.rs | 4 ++-- 7 files changed, 39 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1b1ca8f10..55dd594e06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12597,6 +12597,7 @@ dependencies = [ "latex-prelude", "salsa", "smallvec", + "visored-annotation", "visored-opr", "visored-syn-expr", "visored-zfs-ty", diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 4db9299330..952601e355 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4256,6 +4256,7 @@ "latex-prelude", "salsa", "smallvec", + "visored-annotation", "visored-opr", "visored-zfs-ty", "husky-coword", diff --git a/crates/visored/visored-opr/src/delimiter.rs b/crates/visored/visored-opr/src/delimiter.rs index 56e3545cc6..37b95c9a15 100644 --- a/crates/visored/visored-opr/src/delimiter.rs +++ b/crates/visored/visored-opr/src/delimiter.rs @@ -1,5 +1,5 @@ #[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum VdDelimiter { - Comma, - Semicolon, -} +pub enum VdLeftDelimiter {} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdRightDelimiter {} diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index 410c530eb4..2368d8edca 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -22,6 +22,7 @@ idx-arena.workspace = true latex-prelude.workspace = true latex-ast.workspace = true # visored +visored-annotation.workspace = true visored-opr.workspace = true visored-zfs-ty.workspace = true diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index b2a8c2b3dc..33efbdf0ea 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -12,10 +12,13 @@ pub mod variadic_chain; use either::*; use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; +use latex_ast::ast::math::LxMathAstIdxRange; use latex_prelude::script::LxScriptKind; use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}; use visored_zfs_ty::term::literal::VdZfsLiteral; +use crate::builder::{ToVdSyn, VdSynExprBuilder}; + /// It's a tree of both form and meaning #[derive(Debug, PartialEq, Eq)] pub enum VdSynExprData { @@ -102,3 +105,9 @@ pub enum VdSynExprClass { Suffix, Separator, } + +impl ToVdSyn for LxMathAstIdxRange { + fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> VdSynExprIdx { + todo!() + } +} diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs index 8b13789179..7af11c4954 100644 --- a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -1 +1,22 @@ +use latex_ast::ast::math::LxMathAstIdx; +use visored_annotation::annotation::space::VdSpaceAnnotation; +use visored_opr::{ + delimiter::{VdLeftDelimiter, VdRightDelimiter}, + opr::VdOpr, + separator::VdSeparator, +}; +use super::expr::{VdSynExprClass, VdSynExprIdx}; + +pub struct DisambiguatedMathAst { + ast: LxMathAstIdx, + preceding_space_annotation: Option, +} + +pub enum DisambiguatedToken { + Expr(VdSynExprIdx, VdSynExprClass), + Opr(VdOpr), + Separator(VdSeparator), + LeftDelimiter(VdLeftDelimiter), + RightDelimiter(VdRightDelimiter), +} diff --git a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs index e3ce7db60e..289efbfe4b 100644 --- a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs +++ b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs @@ -7,7 +7,7 @@ pub(super) use self::separated_list::*; use super::*; use crate::expr::VdSynExprData; use expr::list_item::VdSynSeparatedListItem; -use visored_opr::{delimiter::VdDelimiter, precedence::VdPrecedence}; +use visored_opr::{delimiter::VdLeftDelimiter, precedence::VdPrecedence}; #[derive(Debug, PartialEq, Eq)] pub(super) enum IncompleteVdSynExprData { @@ -24,7 +24,7 @@ pub(super) enum IncompleteVdSynExprData { /// ``` SeparatedList { opr: IncompleteSeparatedListOpr, - bra: VdDelimiter, + bra: VdLeftDelimiter, items: SmallVec<[VdSynSeparatedListItem; 4]>, }, /// call list includes more separators like `;` From 251f68f32dc8889a38827373d0de99945a93e614 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 10:41:39 -0700 Subject: [PATCH 38/71] add error --- Cargo.lock | 1 + crates/visored/visored-syn-expr/Cargo.toml | 1 + crates/visored/visored-syn-expr/src/error.rs | 24 ++++++++++++++++++++ crates/visored/visored-syn-expr/src/lib.rs | 1 + 4 files changed, 27 insertions(+) create mode 100644 crates/visored/visored-syn-expr/src/error.rs diff --git a/Cargo.lock b/Cargo.lock index 55dd594e06..c179765fa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12597,6 +12597,7 @@ dependencies = [ "latex-prelude", "salsa", "smallvec", + "thiserror", "visored-annotation", "visored-opr", "visored-syn-expr", diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index 2368d8edca..c93128cc2c 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -15,6 +15,7 @@ keywords.workspace = true either.workspace = true enum-class.workspace = true smallvec.workspace = true +thiserror.workspace = true # abstractions salsa.workspace = true idx-arena.workspace = true diff --git a/crates/visored/visored-syn-expr/src/error.rs b/crates/visored/visored-syn-expr/src/error.rs new file mode 100644 index 0000000000..6a0dd9ce09 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/error.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum VdSynExprError { + #[error("original error({0})")] + Original(OriginalVdSynExprError), + #[error("derived error({0})")] + Derived(DerivedVdSynExprError), +} + +#[derive(Debug, Error)] +pub enum OriginalVdSynExprError { + #[error("todo")] + Todo, +} + +#[derive(Debug, Error)] +pub enum DerivedVdSynExprError { + #[error("todo")] + Todo, +} + +pub type VdSynExprResult = Result; +pub type VdSynExprResultRef<'a, T> = Result; diff --git a/crates/visored/visored-syn-expr/src/lib.rs b/crates/visored/visored-syn-expr/src/lib.rs index 890e77fe59..e7619f87ed 100644 --- a/crates/visored/visored-syn-expr/src/lib.rs +++ b/crates/visored/visored-syn-expr/src/lib.rs @@ -1,5 +1,6 @@ mod builder; pub mod clause; +pub mod error; pub mod expr; pub mod helpers; pub mod jar; From 0f15bf1e8b467b37ceeb36922aa6a857c0a29aeb Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 10:48:27 -0700 Subject: [PATCH 39/71] add vd sem expr error --- Cargo.lock | 1 + crates/visored/visored-sem-expr/Cargo.toml | 1 + crates/visored/visored-sem-expr/src/error.rs | 24 ++++++++++++++++++++ crates/visored/visored-sem-expr/src/lib.rs | 1 + 4 files changed, 27 insertions(+) create mode 100644 crates/visored/visored-sem-expr/src/error.rs diff --git a/Cargo.lock b/Cargo.lock index c179765fa3..5ac1c074a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12578,6 +12578,7 @@ dependencies = [ "latex-ast", "latex-prelude", "salsa", + "thiserror", "visored-annotation", "visored-opr", "visored-sem-expr", diff --git a/crates/visored/visored-sem-expr/Cargo.toml b/crates/visored/visored-sem-expr/Cargo.toml index 9e03aec490..989553689b 100644 --- a/crates/visored/visored-sem-expr/Cargo.toml +++ b/crates/visored/visored-sem-expr/Cargo.toml @@ -12,6 +12,7 @@ categories.workspace = true keywords.workspace = true [dependencies] +thiserror.workspace = true # abstractions salsa.workspace = true idx-arena.workspace = true diff --git a/crates/visored/visored-sem-expr/src/error.rs b/crates/visored/visored-sem-expr/src/error.rs new file mode 100644 index 0000000000..07b85c4177 --- /dev/null +++ b/crates/visored/visored-sem-expr/src/error.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum VdSemExprError { + #[error("original error({0})")] + Original(OriginalVdSemExprError), + #[error("derived error({0})")] + Derived(DerivedVdSemExprError), +} + +#[derive(Debug, Error)] +pub enum OriginalVdSemExprError { + #[error("todo")] + Todo, +} + +#[derive(Debug, Error)] +pub enum DerivedVdSemExprError { + #[error("todo")] + Todo, +} + +pub type VdSemExprResult = Result; +pub type VdSemExprResultRef<'a, T> = Result; diff --git a/crates/visored/visored-sem-expr/src/lib.rs b/crates/visored/visored-sem-expr/src/lib.rs index da7cbe4ffc..8f9b084eec 100644 --- a/crates/visored/visored-sem-expr/src/lib.rs +++ b/crates/visored/visored-sem-expr/src/lib.rs @@ -1,5 +1,6 @@ mod builder; pub mod clause; +pub mod error; pub mod expr; pub mod helpers; pub mod jar; From 5b108777343b28c73ac1c7b7f298fd48118517bd Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 11:22:08 -0700 Subject: [PATCH 40/71] save --- Cargo.lock | 3 + crates/abstractions/idx-arena/src/lib.rs | 4 + crates/latex/latex-ast/src/ast.rs | 2 +- .../expect-files/husky_lang_packages.txt | 5 + crates/visored/visored-syn-expr/Cargo.toml | 4 + .../visored/visored-syn-expr/src/builder.rs | 27 ++++ crates/visored/visored-syn-expr/src/expr.rs | 35 ++++- .../visored-syn-expr/src/helpers/show.rs | 2 +- .../src/helpers/show/display_tree.rs | 128 ++++++++++++++++++ crates/visored/visored-syn-expr/src/lib.rs | 1 + crates/visored/visored-syn-expr/src/range.rs | 10 ++ crates/visored/visored-syn-expr/src/tests.rs | 8 +- 12 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs create mode 100644 crates/visored/visored-syn-expr/src/range.rs diff --git a/Cargo.lock b/Cargo.lock index 5ac1c074a2..6dfa9a67bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12592,10 +12592,13 @@ version = "0.1.0" dependencies = [ "either", "enum-class", + "expect-test", "husky-coword", + "husky-tree-utils", "idx-arena", "latex-ast", "latex-prelude", + "latex-token", "salsa", "smallvec", "thiserror", diff --git a/crates/abstractions/idx-arena/src/lib.rs b/crates/abstractions/idx-arena/src/lib.rs index 429f156aa8..7ef8d55b5f 100644 --- a/crates/abstractions/idx-arena/src/lib.rs +++ b/crates/abstractions/idx-arena/src/lib.rs @@ -129,6 +129,10 @@ impl ArenaIdxRange { self.end.index() - self.start.index() } + pub fn is_empty(&self) -> bool { + self.start == self.end + } + pub fn start(&self) -> ArenaIdx { self.start } diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index b322cdb811..4436cf60be 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -37,7 +37,7 @@ pub struct LxAstArena { pub(crate) rose: LxRoseAstArena, } impl LxAstArena { - pub(crate) fn as_arena_ref(&self) -> LxAstArenaRef { + pub fn as_arena_ref(&self) -> LxAstArenaRef { LxAstArenaRef { math: self.math.as_arena_ref(), rose: self.rose.as_arena_ref(), diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 952601e355..903d393bdf 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4230,6 +4230,7 @@ "idx-arena", "latex-prelude", "salsa", + "thiserror", "visored-annotation", "visored-opr", "visored-syn-expr", @@ -4251,14 +4252,18 @@ dependencies: [ "either", "enum-class", + "husky-tree-utils", "idx-arena", "latex-ast", "latex-prelude", + "latex-token", "salsa", "smallvec", + "thiserror", "visored-annotation", "visored-opr", "visored-zfs-ty", + "expect-test", "husky-coword", ], }, diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index c93128cc2c..6b11fc1740 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -21,13 +21,17 @@ salsa.workspace = true idx-arena.workspace = true # latex latex-prelude.workspace = true +latex-token.workspace = true latex-ast.workspace = true +# utils +husky-tree-utils.workspace = true # visored visored-annotation.workspace = true visored-opr.workspace = true visored-zfs-ty.workspace = true [dev-dependencies] +expect-test.workspace = true # kernel husky-coword.workspace = true # visored diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index 27c2a2f0f1..fea4f3919e 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -1,3 +1,7 @@ +use latex_ast::ast::{LxAstArena, LxAstArenaRef}; +use latex_token::storage::LxTokenStorage; +use visored_annotation::annotations::VdAnnotations; + use crate::{ clause::VdSynClauseArena, expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx}, @@ -8,12 +12,35 @@ use crate::{ pub(crate) struct VdSynExprBuilder<'db> { db: &'db ::salsa::Db, + token_storage: &'db LxTokenStorage, + ast_arena: LxAstArenaRef<'db>, + annotations: &'db VdAnnotations, expr_arena: VdSynExprArena, phrase_arena: VdSynPhraseArena, clause_arena: VdSynClauseArena, sentence_arena: VdSynSentenceArena, } +impl<'db> VdSynExprBuilder<'db> { + pub fn new( + db: &'db ::salsa::Db, + token_storage: &'db LxTokenStorage, + ast_arena: &'db LxAstArena, + annotations: &'db VdAnnotations, + ) -> Self { + Self { + db, + token_storage, + ast_arena: ast_arena.as_arena_ref(), + annotations, + expr_arena: Default::default(), + phrase_arena: Default::default(), + clause_arena: Default::default(), + sentence_arena: Default::default(), + } + } +} + impl<'db> VdSynExprBuilder<'db> { pub fn finish(self) -> VdSynExprRegionData { VdSynExprRegionData::new( diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 33efbdf0ea..95147527fd 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -10,15 +10,15 @@ pub mod uniadic_chain; pub mod variadic_array; pub mod variadic_chain; +use crate::builder::{ToVdSyn, VdSynExprBuilder}; +use crate::*; use either::*; -use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; -use latex_ast::ast::math::LxMathAstIdxRange; +use idx_arena::{map::ArenaMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; +use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; use latex_prelude::script::LxScriptKind; use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}; use visored_zfs_ty::term::literal::VdZfsLiteral; -use crate::builder::{ToVdSyn, VdSynExprBuilder}; - /// It's a tree of both form and meaning #[derive(Debug, PartialEq, Eq)] pub enum VdSynExprData { @@ -56,6 +56,7 @@ pub enum VdSynExprData { pub type VdSynExprIdx = ArenaIdx; pub type VdSynExprIdxRange = ArenaIdxRange; pub type VdSynExprArena = Arena; +pub type VdSynExprMap = ArenaMap; pub type VdSynExprArenaRef<'a> = ArenaRef<'a, VdSynExprData>; impl VdSynExprData { @@ -111,3 +112,29 @@ impl ToVdSyn for LxMathAstIdxRange { todo!() } } + +impl ToVdSyn for LxMathAstIdx { + fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> VdSynExprIdx { + todo!() + } +} + +#[test] +fn math_ast_idx_to_vd_syn_expr_idx_works() { + use crate::helpers::show::display_tree::VdSynExprDisplayTreeBuilder; + + let db = &DB::default(); + let builder = VdSynExprBuilder::new(&db, todo!(), todo!(), todo!()); + let math_ast_idx: LxMathAstIdx = todo!(); + let vd_syn_expr_idx = math_ast_idx.to_vd_syn(&mut builder); + let display_tree_builder = + VdSynExprDisplayTreeBuilder::new(db, todo!(), todo!(), todo!(), todo!(), todo!(), todo!()); + expect_test::expect![[r#" + + "#]] + .assert_eq( + &display_tree_builder + .render_expr(vd_syn_expr_idx) + .show(&Default::default()), + ); +} diff --git a/crates/visored/visored-syn-expr/src/helpers/show.rs b/crates/visored/visored-syn-expr/src/helpers/show.rs index 8b13789179..e04a65fb33 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show.rs @@ -1 +1 @@ - +pub mod display_tree; diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs new file mode 100644 index 0000000000..ac46beeb70 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -0,0 +1,128 @@ +use crate::{ + builder::VdSynExprBuilder, + expr::{VdSynExprArenaRef, VdSynExprIdx, VdSynExprIdxRange}, + range::{VdSynExprRange, VdSynExprRangeMap}, +}; +use husky_tree_utils::display::DisplayTree; +use latex_ast::{ + ast::{ + math::{LxMathAstIdx, LxMathAstIdxRange}, + rose::{LxRoseAstIdx, LxRoseAstIdxRange}, + LxAstArenaRef, LxAstIdx, LxAstIdxRange, + }, + range::LxAstTokenIdxRangeMap, +}; +use latex_token::storage::LxTokenStorage; + +pub struct VdSynExprDisplayTreeBuilder<'a> { + db: &'a salsa::Db, + input: &'a str, + token_storage: &'a LxTokenStorage, + ast_arena: LxAstArenaRef<'a>, + ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, + expr_arena: VdSynExprArenaRef<'a>, + expr_range_map: &'a VdSynExprRangeMap<'a>, +} + +/// # construction +impl<'a> VdSynExprDisplayTreeBuilder<'a> { + pub fn new( + db: &'a salsa::Db, + input: &'a str, + token_storage: &'a LxTokenStorage, + ast_arena: LxAstArenaRef<'a>, + ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, + expr_arena: VdSynExprArenaRef<'a>, + expr_range_map: &'a VdSynExprRangeMap<'a>, + ) -> Self { + Self { + db, + input, + token_storage, + ast_arena, + ast_token_idx_range_map, + expr_arena, + expr_range_map, + } + } +} + +/// # actions +impl<'a> VdSynExprDisplayTreeBuilder<'a> { + pub fn render_all(&self, exprs: VdSynExprIdxRange) -> DisplayTree { + // TODO: maybe just use the range covered by these exprs + DisplayTree::new(self.input.to_string(), self.render_exprs(exprs)) + } + + pub fn render_exprs(&self, exprs: impl IntoIterator) -> Vec { + exprs + .into_iter() + .map(|expr| self.render_expr(expr)) + .collect() + } + + pub fn render_expr(&self, expr: VdSynExprIdx) -> DisplayTree { + let expr_range = self.expr_range_map[expr]; + let (start, end) = match expr_range { + VdSynExprRange::Ast(ast) => self.ast_offset_range(ast), + VdSynExprRange::Asts(asts) => todo!(), + }; + let value = self.input[(start..end)].to_string(); + DisplayTree::new(value, self.render_exprs(self.expr_arena[expr].children())) + } + + fn ast_offset_range(&self, ast: LxAstIdx) -> (usize, usize) { + match ast { + LxAstIdx::Math(ast) => self.math_ast_offset_range(ast), + LxAstIdx::Rose(ast) => self.rose_ast_offset_range(ast), + } + } + + fn math_ast_offset_range(&self, ast: LxMathAstIdx) -> (usize, usize) { + let range = self.ast_token_idx_range_map[ast]; + self.token_storage.math_token_idx_range_offset_range(range) + } + + fn rose_ast_offset_range(&self, ast: LxRoseAstIdx) -> (usize, usize) { + let range = self.ast_token_idx_range_map[ast]; + self.token_storage.rose_token_idx_range_offset_range(range) + } + + fn asts_offset_range(&self, asts: LxAstIdxRange) -> (usize, usize) { + match asts { + LxAstIdxRange::Math(asts) => self.math_asts_offset_range(asts), + LxAstIdxRange::Rose(asts) => self.rose_asts_offset_range(asts), + } + } + + fn math_asts_offset_range(&self, asts: LxMathAstIdxRange) -> (usize, usize) { + let first = asts.start(); + let Some(last) = asts.last() else { todo!() }; + ( + self.math_ast_offset_range(first).0, + self.math_ast_offset_range(last).1, + ) + } + + fn rose_asts_offset_range(&self, asts: LxRoseAstIdxRange) -> (usize, usize) { + let first = asts.start(); + let Some(last) = asts.last() else { todo!() }; + ( + self.rose_ast_offset_range(first).0, + self.rose_ast_offset_range(last).1, + ) + } +} + +// impl<'db> VdSynExprBuilder<'db> { +// pub fn display_tree(&self) -> DisplayTree { +// let display_tree_builder = LxAstDisplayTreeBuilder::new( +// self.db, +// self.input, +// self.token_storage, +// self.ast_arena, +// self.ast_token_idx_range_map, +// ); +// display_tree_builder.render_all(self.expr_range_map.keys()) +// } +// } diff --git a/crates/visored/visored-syn-expr/src/lib.rs b/crates/visored/visored-syn-expr/src/lib.rs index e7619f87ed..8b76d14afe 100644 --- a/crates/visored/visored-syn-expr/src/lib.rs +++ b/crates/visored/visored-syn-expr/src/lib.rs @@ -6,6 +6,7 @@ pub mod helpers; pub mod jar; pub mod parser; pub mod phrase; +pub mod range; pub mod region; pub mod sentence; #[cfg(feature = "test_helpers")] diff --git a/crates/visored/visored-syn-expr/src/range.rs b/crates/visored/visored-syn-expr/src/range.rs new file mode 100644 index 0000000000..da94b3863a --- /dev/null +++ b/crates/visored/visored-syn-expr/src/range.rs @@ -0,0 +1,10 @@ +use crate::expr::VdSynExprMap; +use latex_ast::ast::{LxAstIdx, LxAstIdxRange}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VdSynExprRange { + Ast(LxAstIdx), + Asts(LxAstIdxRange), +} + +pub type VdSynExprRangeMap<'a> = VdSynExprMap; diff --git a/crates/visored/visored-syn-expr/src/tests.rs b/crates/visored/visored-syn-expr/src/tests.rs index 79d342de26..8aebefc2f2 100644 --- a/crates/visored/visored-syn-expr/src/tests.rs +++ b/crates/visored/visored-syn-expr/src/tests.rs @@ -1,2 +1,8 @@ -#[salsa::db(husky_coword::jar::CowordJar, visored_zfs_ty::jar::VdZfsTypeJar)] +#[salsa::db( + husky_coword::jar::CowordJar, + visored_zfs_ty::jar::VdZfsTypeJar, + latex_ast::jar::LxAstJar, + visored_opr::jar::VdOprJar, + crate::jar::VdSynExprJar +)] pub(crate) struct DB {} From cb9417a84f897d174a54d94a14fca61931320a87 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 12:49:51 -0700 Subject: [PATCH 41/71] save --- crates/visored/visored-syn-expr/Cargo.toml | 2 +- .../visored/visored-syn-expr/src/builder.rs | 61 +++++++++++++++++++ crates/visored/visored-syn-expr/src/clause.rs | 6 +- crates/visored/visored-syn-expr/src/expr.rs | 47 +++++++++----- .../src/helpers/show/display_tree.rs | 55 ++++++++++++++++- crates/visored/visored-syn-expr/src/phrase.rs | 6 +- crates/visored/visored-syn-expr/src/range.rs | 15 ++++- .../visored/visored-syn-expr/src/sentence.rs | 6 +- 8 files changed, 175 insertions(+), 23 deletions(-) diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index 6b11fc1740..a90607d44c 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -41,4 +41,4 @@ visored-syn-expr = { workspace = true, features = ["test_helpers"] } workspace = true [features] -test_helpers = [] +test_helpers = ["latex-ast/test_helpers"] diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index fea4f3919e..ae297587e2 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -6,6 +6,7 @@ use crate::{ clause::VdSynClauseArena, expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx}, phrase::{VdSynPhraseArena, VdSynPhraseData, VdSynPhraseIdx}, + range::{VdSynClauseRangeMap, VdSynExprRangeMap, VdSynPhraseRangeMap, VdSynSentenceRangeMap}, region::VdSynExprRegionData, sentence::{VdSynSentenceArena, VdSynSentenceData, VdSynSentenceIdx}, }; @@ -16,11 +17,16 @@ pub(crate) struct VdSynExprBuilder<'db> { ast_arena: LxAstArenaRef<'db>, annotations: &'db VdAnnotations, expr_arena: VdSynExprArena, + expr_range_map: VdSynExprRangeMap, phrase_arena: VdSynPhraseArena, + phrase_range_map: VdSynPhraseRangeMap, clause_arena: VdSynClauseArena, + clause_range_map: VdSynClauseRangeMap, sentence_arena: VdSynSentenceArena, + sentence_range_map: VdSynSentenceRangeMap, } +/// # constructor impl<'db> VdSynExprBuilder<'db> { pub fn new( db: &'db ::salsa::Db, @@ -37,10 +43,65 @@ impl<'db> VdSynExprBuilder<'db> { phrase_arena: Default::default(), clause_arena: Default::default(), sentence_arena: Default::default(), + expr_range_map: Default::default(), + phrase_range_map: Default::default(), + clause_range_map: Default::default(), + sentence_range_map: Default::default(), } } } +/// # getters +impl<'db> VdSynExprBuilder<'db> { + pub(crate) fn db(&self) -> &'db ::salsa::Db { + self.db + } + + pub(crate) fn token_storage(&self) -> &LxTokenStorage { + self.token_storage + } + + pub(crate) fn ast_arena(&self) -> &LxAstArenaRef<'db> { + &self.ast_arena + } + + pub(crate) fn annotations(&self) -> &VdAnnotations { + self.annotations + } + + pub(crate) fn expr_arena(&self) -> &VdSynExprArena { + &self.expr_arena + } + + pub(crate) fn expr_range_map(&self) -> &VdSynExprRangeMap { + &self.expr_range_map + } + + pub(crate) fn phrase_arena(&self) -> &VdSynPhraseArena { + &self.phrase_arena + } + + pub(crate) fn phrase_range_map(&self) -> &VdSynPhraseRangeMap { + &self.phrase_range_map + } + + pub(crate) fn clause_arena(&self) -> &VdSynClauseArena { + &self.clause_arena + } + + pub(crate) fn clause_range_map(&self) -> &VdSynClauseRangeMap { + &self.clause_range_map + } + + pub(crate) fn sentence_arena(&self) -> &VdSynSentenceArena { + &self.sentence_arena + } + + pub(crate) fn sentence_range_map(&self) -> &VdSynSentenceRangeMap { + &self.sentence_range_map + } +} + impl<'db> VdSynExprBuilder<'db> { pub fn finish(self) -> VdSynExprRegionData { VdSynExprRegionData::new( diff --git a/crates/visored/visored-syn-expr/src/clause.rs b/crates/visored/visored-syn-expr/src/clause.rs index d780e1142b..da2dce98ab 100644 --- a/crates/visored/visored-syn-expr/src/clause.rs +++ b/crates/visored/visored-syn-expr/src/clause.rs @@ -1,5 +1,7 @@ use super::*; -use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; +use idx_arena::{ + map::ArenaMap, ordered_map::ArenaOrderedMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef, +}; pub enum VdSynClauseData { Verb, @@ -9,3 +11,5 @@ pub type VdSynClauseArena = Arena; pub type VdSynClauseArenaRef<'a> = ArenaRef<'a, VdSynClauseData>; pub type VdSynClauseIdx = ArenaIdx; pub type VdSynClauseIdxRange = ArenaIdxRange; +pub type VdSynClauseMap = ArenaMap; +pub type VdSynClauseOrderedMap = ArenaOrderedMap; diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 95147527fd..441410985a 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -13,7 +13,9 @@ pub mod variadic_chain; use crate::builder::{ToVdSyn, VdSynExprBuilder}; use crate::*; use either::*; -use idx_arena::{map::ArenaMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; +use idx_arena::{ + map::ArenaMap, ordered_map::ArenaOrderedMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef, +}; use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; use latex_prelude::script::LxScriptKind; use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}; @@ -57,6 +59,7 @@ pub type VdSynExprIdx = ArenaIdx; pub type VdSynExprIdxRange = ArenaIdxRange; pub type VdSynExprArena = Arena; pub type VdSynExprMap = ArenaMap; +pub type VdSynExprOrderedMap = ArenaOrderedMap; pub type VdSynExprArenaRef<'a> = ArenaRef<'a, VdSynExprData>; impl VdSynExprData { @@ -121,20 +124,36 @@ impl ToVdSyn for LxMathAstIdx { #[test] fn math_ast_idx_to_vd_syn_expr_idx_works() { - use crate::helpers::show::display_tree::VdSynExprDisplayTreeBuilder; + use expect_test::{expect, Expect}; + use latex_ast::test_helpers::example::LxAstsExample; + use latex_prelude::mode::LxMode; - let db = &DB::default(); - let builder = VdSynExprBuilder::new(&db, todo!(), todo!(), todo!()); - let math_ast_idx: LxMathAstIdx = todo!(); - let vd_syn_expr_idx = math_ast_idx.to_vd_syn(&mut builder); - let display_tree_builder = - VdSynExprDisplayTreeBuilder::new(db, todo!(), todo!(), todo!(), todo!(), todo!(), todo!()); - expect_test::expect![[r#" + fn t(input: &str, expected: &Expect) { + use crate::helpers::show::display_tree::VdSynExprDisplayTreeBuilder; + + let db = &DB::default(); + let lx_asts_example = LxAstsExample::new(input, LxMode::Math, db); + let builder = VdSynExprBuilder::new( + &db, + &lx_asts_example.token_storage, + &lx_asts_example.ast_arena, + todo!(), + ); + let math_ast_idx: LxMathAstIdx = todo!(); + let vd_syn_expr_idx = math_ast_idx.to_vd_syn(&mut builder); + let display_tree_builder = + VdSynExprDisplayTreeBuilder::new2(&lx_asts_example, &builder, db); + expected.assert_eq( + &display_tree_builder + .render_expr(vd_syn_expr_idx) + .show(&Default::default()), + ); + } + + t( + "", + &expect![[r#" - "#]] - .assert_eq( - &display_tree_builder - .render_expr(vd_syn_expr_idx) - .show(&Default::default()), + "#]], ); } diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index ac46beeb70..dce229697f 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -1,9 +1,17 @@ use crate::{ builder::VdSynExprBuilder, + clause::VdSynClauseArenaRef, expr::{VdSynExprArenaRef, VdSynExprIdx, VdSynExprIdxRange}, - range::{VdSynExprRange, VdSynExprRangeMap}, + phrase::VdSynPhraseArenaRef, + range::{ + VdSynClauseRangeMap, VdSynExprRange, VdSynExprRangeMap, VdSynPhraseRangeMap, + VdSynSentenceRangeMap, + }, + sentence::VdSynSentenceArenaRef, }; use husky_tree_utils::display::DisplayTree; +#[cfg(feature = "test_helpers")] +use latex_ast::test_helpers::example::LxAstsExample; use latex_ast::{ ast::{ math::{LxMathAstIdx, LxMathAstIdxRange}, @@ -21,7 +29,13 @@ pub struct VdSynExprDisplayTreeBuilder<'a> { ast_arena: LxAstArenaRef<'a>, ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, expr_arena: VdSynExprArenaRef<'a>, - expr_range_map: &'a VdSynExprRangeMap<'a>, + expr_range_map: &'a VdSynExprRangeMap, + phrase_arena: VdSynPhraseArenaRef<'a>, + phrase_range_map: &'a VdSynPhraseRangeMap, + clause_arena: VdSynClauseArenaRef<'a>, + clause_range_map: &'a VdSynClauseRangeMap, + sentence_arena: VdSynSentenceArenaRef<'a>, + sentence_range_map: &'a VdSynSentenceRangeMap, } /// # construction @@ -33,7 +47,13 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { ast_arena: LxAstArenaRef<'a>, ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, expr_arena: VdSynExprArenaRef<'a>, - expr_range_map: &'a VdSynExprRangeMap<'a>, + expr_range_map: &'a VdSynExprRangeMap, + phrase_arena: VdSynPhraseArenaRef<'a>, + phrase_range_map: &'a VdSynPhraseRangeMap, + clause_arena: VdSynClauseArenaRef<'a>, + clause_range_map: &'a VdSynClauseRangeMap, + sentence_arena: VdSynSentenceArenaRef<'a>, + sentence_range_map: &'a VdSynSentenceRangeMap, ) -> Self { Self { db, @@ -43,8 +63,37 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { ast_token_idx_range_map, expr_arena, expr_range_map, + phrase_arena, + phrase_range_map, + clause_arena, + clause_range_map, + sentence_arena, + sentence_range_map, } } + + #[cfg(feature = "test_helpers")] + pub fn new2( + lx_asts_example: &'a LxAstsExample, + builder: &'a VdSynExprBuilder<'a>, + db: &'a salsa::Db, + ) -> Self { + VdSynExprDisplayTreeBuilder::new( + db, + &lx_asts_example.input, + &lx_asts_example.token_storage, + lx_asts_example.ast_arena.as_arena_ref(), + &lx_asts_example.ast_token_idx_range_map, + builder.expr_arena().as_arena_ref(), + &builder.expr_range_map(), + builder.phrase_arena().as_arena_ref(), + &builder.phrase_range_map(), + builder.clause_arena().as_arena_ref(), + &builder.clause_range_map(), + builder.sentence_arena().as_arena_ref(), + &builder.sentence_range_map(), + ) + } } /// # actions diff --git a/crates/visored/visored-syn-expr/src/phrase.rs b/crates/visored/visored-syn-expr/src/phrase.rs index 973b58cc59..c4bb42d132 100644 --- a/crates/visored/visored-syn-expr/src/phrase.rs +++ b/crates/visored/visored-syn-expr/src/phrase.rs @@ -1,6 +1,8 @@ pub mod noun; -use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; +use idx_arena::{ + map::ArenaMap, ordered_map::ArenaOrderedMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef, +}; use noun::VdSynNounPhraseData; use crate::expr::VdSynExprIdx; @@ -17,3 +19,5 @@ pub type VdSynPhraseArena = Arena; pub type VdSynPhraseArenaRef<'a> = ArenaRef<'a, VdSynPhraseData>; pub type VdSynPhraseIdx = ArenaIdx; pub type VdSynPhraseIdxRange = ArenaIdxRange; +pub type VdSynPhraseMap = ArenaMap; +pub type VdSynPhraseOrderedMap = ArenaOrderedMap; diff --git a/crates/visored/visored-syn-expr/src/range.rs b/crates/visored/visored-syn-expr/src/range.rs index da94b3863a..6fe658d0ac 100644 --- a/crates/visored/visored-syn-expr/src/range.rs +++ b/crates/visored/visored-syn-expr/src/range.rs @@ -1,10 +1,21 @@ -use crate::expr::VdSynExprMap; use latex_ast::ast::{LxAstIdx, LxAstIdxRange}; +use crate::{ + clause::VdSynClauseOrderedMap, expr::VdSynExprOrderedMap, phrase::VdSynPhraseOrderedMap, + sentence::VdSynSentenceOrderedMap, +}; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VdSynExprRange { Ast(LxAstIdx), Asts(LxAstIdxRange), } -pub type VdSynExprRangeMap<'a> = VdSynExprMap; +pub type VdSynPhraseRange = LxAstIdxRange; +pub type VdSynClauseRange = LxAstIdxRange; +pub type VdSynSentenceRange = LxAstIdxRange; + +pub type VdSynExprRangeMap = VdSynExprOrderedMap; +pub type VdSynPhraseRangeMap = VdSynPhraseOrderedMap; +pub type VdSynClauseRangeMap = VdSynClauseOrderedMap; +pub type VdSynSentenceRangeMap = VdSynSentenceOrderedMap; diff --git a/crates/visored/visored-syn-expr/src/sentence.rs b/crates/visored/visored-syn-expr/src/sentence.rs index 12ab3dfd59..5de7bfffb3 100644 --- a/crates/visored/visored-syn-expr/src/sentence.rs +++ b/crates/visored/visored-syn-expr/src/sentence.rs @@ -1,5 +1,7 @@ use crate::clause::VdSynClauseIdxRange; -use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; +use idx_arena::{ + map::ArenaMap, ordered_map::ArenaOrderedMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef, +}; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum VdSynSentenceData { @@ -10,3 +12,5 @@ pub type VdSynSentenceArena = Arena; pub type VdSynSentenceArenaRef<'a> = ArenaRef<'a, VdSynSentenceData>; pub type VdSynSentenceIdx = ArenaIdx; pub type VdSynSentenceIdxRange = ArenaIdxRange; +pub type VdSynSentenceMap = ArenaMap; +pub type VdSynSentenceOrderedMap = ArenaOrderedMap; From 8c37242e65e3bfce9bcb7fda8af9012eae750d2e Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 12:51:06 -0700 Subject: [PATCH 42/71] save --- crates/visored/visored-syn-expr/src/expr.rs | 63 +++++++++++---------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 441410985a..a2a99f434b 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -122,38 +122,43 @@ impl ToVdSyn for LxMathAstIdx { } } -#[test] -fn math_ast_idx_to_vd_syn_expr_idx_works() { - use expect_test::{expect, Expect}; - use latex_ast::test_helpers::example::LxAstsExample; - use latex_prelude::mode::LxMode; +#[cfg(test)] +mod tests { + use super::*; - fn t(input: &str, expected: &Expect) { - use crate::helpers::show::display_tree::VdSynExprDisplayTreeBuilder; + #[test] + fn math_ast_idx_to_vd_syn_expr_idx_works() { + use expect_test::{expect, Expect}; + use latex_ast::test_helpers::example::LxAstsExample; + use latex_prelude::mode::LxMode; - let db = &DB::default(); - let lx_asts_example = LxAstsExample::new(input, LxMode::Math, db); - let builder = VdSynExprBuilder::new( - &db, - &lx_asts_example.token_storage, - &lx_asts_example.ast_arena, - todo!(), - ); - let math_ast_idx: LxMathAstIdx = todo!(); - let vd_syn_expr_idx = math_ast_idx.to_vd_syn(&mut builder); - let display_tree_builder = - VdSynExprDisplayTreeBuilder::new2(&lx_asts_example, &builder, db); - expected.assert_eq( - &display_tree_builder - .render_expr(vd_syn_expr_idx) - .show(&Default::default()), - ); - } + fn t(input: &str, expected: &Expect) { + use crate::helpers::show::display_tree::VdSynExprDisplayTreeBuilder; - t( - "", - &expect![[r#" + let db = &DB::default(); + let lx_asts_example = LxAstsExample::new(input, LxMode::Math, db); + let builder = VdSynExprBuilder::new( + &db, + &lx_asts_example.token_storage, + &lx_asts_example.ast_arena, + todo!(), + ); + let math_ast_idx: LxMathAstIdx = todo!(); + let vd_syn_expr_idx = math_ast_idx.to_vd_syn(&mut builder); + let display_tree_builder = + VdSynExprDisplayTreeBuilder::new2(&lx_asts_example, &builder, db); + expected.assert_eq( + &display_tree_builder + .render_expr(vd_syn_expr_idx) + .show(&Default::default()), + ); + } + + t( + "", + &expect![[r#" "#]], - ); + ); + } } From ea275f9e39b91c4e9f8043341f85756ea94de605 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 13:42:53 -0700 Subject: [PATCH 43/71] set up test framework for visored syn expr --- crates/latex/latex-ast/src/ast.rs | 4 +- .../latex-ast/src/test_helpers/example.rs | 6 +- .../src/test_helpers/example.rs | 2 +- .../visored/visored-syn-expr/src/builder.rs | 43 ++++++- crates/visored/visored-syn-expr/src/expr.rs | 35 +++--- .../src/helpers/show/display_tree.rs | 4 +- crates/visored/visored-syn-expr/src/lib.rs | 2 + .../visored-syn-expr/src/test_helpers.rs | 3 + .../src/test_helpers/example.rs | 114 ++++++++++++++++++ 9 files changed, 183 insertions(+), 30 deletions(-) create mode 100644 crates/visored/visored-syn-expr/src/test_helpers/example.rs diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 4436cf60be..dd5d3362da 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -659,12 +659,12 @@ fn parse_tex_input_into_asts_works() { #[test] fn parse_tex_input_into_asts_then_show_works() { - use crate::test_helpers::example::LxAstsExample; + use crate::test_helpers::example::LxAstExample; use expect_test::Expect; fn t(input: &str, mode: LxMode, expected: Expect) { let db = &DB::default(); - let example = LxAstsExample::new(input, mode, db); + let example = LxAstExample::new(input, mode, db); let show = example.show(db); expected.assert_eq(&show); } diff --git a/crates/latex/latex-ast/src/test_helpers/example.rs b/crates/latex/latex-ast/src/test_helpers/example.rs index 30b569932a..1a7c4ff6e9 100644 --- a/crates/latex/latex-ast/src/test_helpers/example.rs +++ b/crates/latex/latex-ast/src/test_helpers/example.rs @@ -7,7 +7,7 @@ use latex_prelude::mode::LxMode; use latex_token::storage::LxTokenStorage; #[derive(Debug)] -pub struct LxAstsExample { +pub struct LxAstExample { pub input: String, pub token_storage: LxTokenStorage, pub ast_arena: LxAstArena, @@ -15,7 +15,7 @@ pub struct LxAstsExample { pub ast_token_idx_range_map: LxAstTokenIdxRangeMap, } -impl LxAstsExample { +impl LxAstExample { pub fn new(input: &str, root_mode: LxMode, db: &salsa::Db) -> Self { let mut ast_arena = LxAstArena::default(); let mut token_storage = LxTokenStorage::default(); @@ -32,7 +32,7 @@ impl LxAstsExample { } } -impl LxAstsExample { +impl LxAstExample { pub fn show(&self, db: &salsa::Db) -> String { let display_tree_builder = LxAstDisplayTreeBuilder::new( db, diff --git a/crates/visored/visored-annotation/src/test_helpers/example.rs b/crates/visored/visored-annotation/src/test_helpers/example.rs index 72f2486f72..6ea365a85b 100644 --- a/crates/visored/visored-annotation/src/test_helpers/example.rs +++ b/crates/visored/visored-annotation/src/test_helpers/example.rs @@ -8,8 +8,8 @@ use crate::{ #[derive(Debug, Clone)] pub struct VdAnnotationsExample { - pub root_mode: LxMode, pub input: String, + pub root_mode: LxMode, pub annotations: VdAnnotations, } diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index ae297587e2..53a741bb31 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -1,7 +1,3 @@ -use latex_ast::ast::{LxAstArena, LxAstArenaRef}; -use latex_token::storage::LxTokenStorage; -use visored_annotation::annotations::VdAnnotations; - use crate::{ clause::VdSynClauseArena, expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx}, @@ -10,6 +6,10 @@ use crate::{ region::VdSynExprRegionData, sentence::{VdSynSentenceArena, VdSynSentenceData, VdSynSentenceIdx}, }; +use either::*; +use latex_ast::ast::{LxAstArena, LxAstArenaRef, LxAstIdxRange}; +use latex_token::storage::LxTokenStorage; +use visored_annotation::annotations::VdAnnotations; pub(crate) struct VdSynExprBuilder<'db> { db: &'db ::salsa::Db, @@ -103,7 +103,7 @@ impl<'db> VdSynExprBuilder<'db> { } impl<'db> VdSynExprBuilder<'db> { - pub fn finish(self) -> VdSynExprRegionData { + pub fn finish_to_region_data(self) -> VdSynExprRegionData { VdSynExprRegionData::new( self.expr_arena, self.phrase_arena, @@ -111,7 +111,40 @@ impl<'db> VdSynExprBuilder<'db> { self.sentence_arena, ) } + + pub fn finish( + self, + ) -> ( + VdSynExprArena, + VdSynExprRangeMap, + VdSynPhraseArena, + VdSynPhraseRangeMap, + VdSynClauseArena, + VdSynClauseRangeMap, + VdSynSentenceArena, + VdSynSentenceRangeMap, + ) { + ( + self.expr_arena, + self.expr_range_map, + self.phrase_arena, + self.phrase_range_map, + self.clause_arena, + self.clause_range_map, + self.sentence_arena, + self.sentence_range_map, + ) + } } pub trait ToVdSyn { fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> T; } + +impl ToVdSyn> for LxAstIdxRange { + fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> Either { + match self { + LxAstIdxRange::Math(slf) => Either::Left(slf.to_vd_syn(builder)), + LxAstIdxRange::Rose(slf) => Either::Right(todo!()), + } + } +} diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index a2a99f434b..c88805f3a6 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -124,38 +124,39 @@ impl ToVdSyn for LxMathAstIdx { #[cfg(test)] mod tests { + use visored_annotation::annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}; + use super::*; #[test] fn math_ast_idx_to_vd_syn_expr_idx_works() { + use crate::test_helpers::example::VdSynExprExample; use expect_test::{expect, Expect}; - use latex_ast::test_helpers::example::LxAstsExample; use latex_prelude::mode::LxMode; - fn t(input: &str, expected: &Expect) { + fn t( + input: &str, + token_annotations: &[((&str, &str), VdTokenAnnotation)], + space_annotations: &[((&str, &str), VdSpaceAnnotation)], + expected: &Expect, + ) { use crate::helpers::show::display_tree::VdSynExprDisplayTreeBuilder; let db = &DB::default(); - let lx_asts_example = LxAstsExample::new(input, LxMode::Math, db); - let builder = VdSynExprBuilder::new( - &db, - &lx_asts_example.token_storage, - &lx_asts_example.ast_arena, - todo!(), - ); - let math_ast_idx: LxMathAstIdx = todo!(); - let vd_syn_expr_idx = math_ast_idx.to_vd_syn(&mut builder); - let display_tree_builder = - VdSynExprDisplayTreeBuilder::new2(&lx_asts_example, &builder, db); - expected.assert_eq( - &display_tree_builder - .render_expr(vd_syn_expr_idx) - .show(&Default::default()), + let example = VdSynExprExample::new( + input, + LxMode::Math, + token_annotations, + space_annotations, + db, ); + expected.assert_eq(&example.show_display_tree(db)); } t( "", + &[], + &[], &expect![[r#" "#]], diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index dce229697f..5a180665f3 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -11,7 +11,7 @@ use crate::{ }; use husky_tree_utils::display::DisplayTree; #[cfg(feature = "test_helpers")] -use latex_ast::test_helpers::example::LxAstsExample; +use latex_ast::test_helpers::example::LxAstExample; use latex_ast::{ ast::{ math::{LxMathAstIdx, LxMathAstIdxRange}, @@ -74,7 +74,7 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { #[cfg(feature = "test_helpers")] pub fn new2( - lx_asts_example: &'a LxAstsExample, + lx_asts_example: &'a LxAstExample, builder: &'a VdSynExprBuilder<'a>, db: &'a salsa::Db, ) -> Self { diff --git a/crates/visored/visored-syn-expr/src/lib.rs b/crates/visored/visored-syn-expr/src/lib.rs index 8b76d14afe..1125a8506f 100644 --- a/crates/visored/visored-syn-expr/src/lib.rs +++ b/crates/visored/visored-syn-expr/src/lib.rs @@ -17,4 +17,6 @@ mod tests; use self::jar::VdSynExprJar as Jar; #[cfg(test)] use self::tests::*; +use crate::builder::ToVdSyn; +use either::*; use smallvec::SmallVec; diff --git a/crates/visored/visored-syn-expr/src/test_helpers.rs b/crates/visored/visored-syn-expr/src/test_helpers.rs index 5575a85eb3..b350042e4b 100644 --- a/crates/visored/visored-syn-expr/src/test_helpers.rs +++ b/crates/visored/visored-syn-expr/src/test_helpers.rs @@ -1 +1,4 @@ pub mod builder; +pub mod example; + +use crate::*; diff --git a/crates/visored/visored-syn-expr/src/test_helpers/example.rs b/crates/visored/visored-syn-expr/src/test_helpers/example.rs new file mode 100644 index 0000000000..f3e539a049 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/test_helpers/example.rs @@ -0,0 +1,114 @@ +use super::*; +use crate::{ + builder::VdSynExprBuilder, + clause::VdSynClauseArena, + expr::VdSynExprArena, + phrase::VdSynPhraseArena, + range::{VdSynClauseRangeMap, VdSynExprRangeMap, VdSynPhraseRangeMap, VdSynSentenceRangeMap}, + sentence::VdSynSentenceArena, +}; +use expr::VdSynExprIdx; +use helpers::show::display_tree::VdSynExprDisplayTreeBuilder; +use latex_ast::{ + ast::{parse_latex_input_into_asts, LxAstArena, LxAstIdxRange}, + range::{calc_ast_token_idx_range_map, LxAstTokenIdxRangeMap}, + test_helpers::example::LxAstExample, +}; +use latex_prelude::mode::LxMode; +use latex_token::storage::LxTokenStorage; +use visored_annotation::{ + annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}, + annotations::VdAnnotations, +}; + +pub struct VdSynExprExample { + pub input: String, + pub root_mode: LxMode, + pub annotations: VdAnnotations, + pub token_storage: LxTokenStorage, + pub ast_arena: LxAstArena, + pub asts: LxAstIdxRange, + pub ast_token_idx_range_map: LxAstTokenIdxRangeMap, + pub result: Either, + pub expr_arena: VdSynExprArena, + pub expr_range_map: VdSynExprRangeMap, + pub phrase_arena: VdSynPhraseArena, + pub phrase_range_map: VdSynPhraseRangeMap, + pub clause_arena: VdSynClauseArena, + pub clause_range_map: VdSynClauseRangeMap, + pub sentence_arena: VdSynSentenceArena, + pub sentence_range_map: VdSynSentenceRangeMap, +} + +impl VdSynExprExample { + pub fn new( + input: &str, + root_mode: LxMode, + token_annotations: &[((&str, &str), VdTokenAnnotation)], + space_annotations: &[((&str, &str), VdSpaceAnnotation)], + db: &salsa::Db, + ) -> Self { + let mut ast_arena = LxAstArena::default(); + let mut token_storage = LxTokenStorage::default(); + let asts = + parse_latex_input_into_asts(db, input, root_mode, &mut token_storage, &mut ast_arena); + let ast_token_idx_range_map = calc_ast_token_idx_range_map(db, &ast_arena); + let annotations = VdAnnotations::from_sparse( + input, + token_annotations.iter().copied(), + space_annotations.iter().copied(), + ); + let mut builder = VdSynExprBuilder::new(db, &token_storage, &ast_arena, &annotations); + let result = asts.to_vd_syn(&mut builder); + let ( + expr_arena, + expr_range_map, + phrase_arena, + phrase_range_map, + clause_arena, + clause_range_map, + sentence_arena, + sentence_range_map, + ) = builder.finish(); + Self { + input: input.to_string(), + root_mode, + annotations, + token_storage, + ast_arena, + asts, + ast_token_idx_range_map, + result, + expr_arena, + expr_range_map, + phrase_arena, + phrase_range_map, + clause_arena, + clause_range_map, + sentence_arena, + sentence_range_map, + } + } + + pub(crate) fn show_display_tree(&self, db: &salsa::Db) -> String { + let builder = VdSynExprDisplayTreeBuilder::new( + db, + &self.input, + &self.token_storage, + self.ast_arena.as_arena_ref(), + &self.ast_token_idx_range_map, + self.expr_arena.as_arena_ref(), + &self.expr_range_map, + self.phrase_arena.as_arena_ref(), + &self.phrase_range_map, + self.clause_arena.as_arena_ref(), + &self.clause_range_map, + self.sentence_arena.as_arena_ref(), + &self.sentence_range_map, + ); + match self.result { + Left(expr) => builder.render_expr(expr).show(&Default::default()), + Right(_) => todo!(), + } + } +} From c20846e4159ec3a14bddf1c967f1384fe04e03e3 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 13:43:47 -0700 Subject: [PATCH 44/71] save --- crates/visored/visored-syn-expr/src/expr.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index c88805f3a6..55c3c53b94 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -124,9 +124,8 @@ impl ToVdSyn for LxMathAstIdx { #[cfg(test)] mod tests { - use visored_annotation::annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}; - use super::*; + use visored_annotation::annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}; #[test] fn math_ast_idx_to_vd_syn_expr_idx_works() { From 04b75767e086ad209955ed868411108f3a165779 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 13:52:32 -0700 Subject: [PATCH 45/71] bug encountered --- crates/abstractions/idx-arena/src/arena_idx.rs | 1 + crates/abstractions/idx-arena/src/lib.rs | 6 +++++- crates/abstractions/idx-arena/src/map.rs | 1 + crates/visored/visored-syn-expr/src/builder.rs | 17 +++++++++++++++-- crates/visored/visored-syn-expr/src/error.rs | 12 +++++++----- crates/visored/visored-syn-expr/src/expr.rs | 13 ++++++++++++- .../visored-syn-expr/src/helpers/latex_fmt.rs | 1 + .../src/helpers/show/display_tree.rs | 10 ++++++---- crates/visored/visored-syn-expr/src/range.rs | 4 ++-- 9 files changed, 50 insertions(+), 15 deletions(-) diff --git a/crates/abstractions/idx-arena/src/arena_idx.rs b/crates/abstractions/idx-arena/src/arena_idx.rs index 57a9ec2d3c..b0a26b72df 100644 --- a/crates/abstractions/idx-arena/src/arena_idx.rs +++ b/crates/abstractions/idx-arena/src/arena_idx.rs @@ -94,6 +94,7 @@ impl Sub for ArenaIdx { impl std::ops::Sub for ArenaIdx { type Output = Self; + #[track_caller] fn sub(self, rhs: usize) -> Self::Output { Self::new(self.index() - rhs) } diff --git a/crates/abstractions/idx-arena/src/lib.rs b/crates/abstractions/idx-arena/src/lib.rs index 7ef8d55b5f..0bd9a77914 100644 --- a/crates/abstractions/idx-arena/src/lib.rs +++ b/crates/abstractions/idx-arena/src/lib.rs @@ -146,7 +146,11 @@ impl ArenaIdxRange { } pub fn last(&self) -> Option> { - (self.start < self.end).then_some(self.end - 1) + if self.start < self.end { + Some(self.end - 1) + } else { + None + } } pub fn new_single(idx: ArenaIdx) -> Self { diff --git a/crates/abstractions/idx-arena/src/map.rs b/crates/abstractions/idx-arena/src/map.rs index e0c9e7fe08..656645d8e0 100644 --- a/crates/abstractions/idx-arena/src/map.rs +++ b/crates/abstractions/idx-arena/src/map.rs @@ -92,6 +92,7 @@ impl ArenaMap { self.data[idx.index()].is_some() } + #[track_caller] pub fn get(&self, idx: ArenaIdx) -> Option<&V> { match self.data[idx.index()] { Some(ref v) => Some(v), diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index 53a741bb31..5c6b50ed6c 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -1,8 +1,11 @@ use crate::{ clause::VdSynClauseArena, - expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx}, + expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx, VdSynExprIdxRange}, phrase::{VdSynPhraseArena, VdSynPhraseData, VdSynPhraseIdx}, - range::{VdSynClauseRangeMap, VdSynExprRangeMap, VdSynPhraseRangeMap, VdSynSentenceRangeMap}, + range::{ + VdSynClauseRangeMap, VdSynExprAstRange, VdSynExprRangeMap, VdSynPhraseRangeMap, + VdSynSentenceRangeMap, + }, region::VdSynExprRegionData, sentence::{VdSynSentenceArena, VdSynSentenceData, VdSynSentenceIdx}, }; @@ -49,6 +52,16 @@ impl<'db> VdSynExprBuilder<'db> { sentence_range_map: Default::default(), } } + + pub(crate) fn alloc_expr( + &mut self, + data: VdSynExprData, + range: VdSynExprAstRange, + ) -> VdSynExprIdx { + let idx = self.expr_arena.alloc_one(data); + self.expr_range_map.insert_next(idx, range); + idx + } } /// # getters diff --git a/crates/visored/visored-syn-expr/src/error.rs b/crates/visored/visored-syn-expr/src/error.rs index 6a0dd9ce09..f20296bc5b 100644 --- a/crates/visored/visored-syn-expr/src/error.rs +++ b/crates/visored/visored-syn-expr/src/error.rs @@ -1,20 +1,22 @@ use thiserror::Error; -#[derive(Debug, Error)] +#[derive(Debug, Error, PartialEq, Eq)] pub enum VdSynExprError { #[error("original error({0})")] - Original(OriginalVdSynExprError), + Original(#[from] OriginalVdSynExprError), #[error("derived error({0})")] - Derived(DerivedVdSynExprError), + Derived(#[from] DerivedVdSynExprError), } -#[derive(Debug, Error)] +#[derive(Debug, Error, PartialEq, Eq)] pub enum OriginalVdSynExprError { + #[error("empty")] + Empty, #[error("todo")] Todo, } -#[derive(Debug, Error)] +#[derive(Debug, Error, PartialEq, Eq)] pub enum DerivedVdSynExprError { #[error("todo")] Todo, diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 55c3c53b94..4ad9e48dbf 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -13,11 +13,13 @@ pub mod variadic_chain; use crate::builder::{ToVdSyn, VdSynExprBuilder}; use crate::*; use either::*; +use error::{OriginalVdSynExprError, VdSynExprError}; use idx_arena::{ map::ArenaMap, ordered_map::ArenaOrderedMap, Arena, ArenaIdx, ArenaIdxRange, ArenaRef, }; use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; use latex_prelude::script::LxScriptKind; +use range::VdSynExprAstRange; use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}; use visored_zfs_ty::term::literal::VdZfsLiteral; @@ -53,6 +55,7 @@ pub enum VdSynExprData { VariadicChain, UniadicArray, VariadicArray, + Err(VdSynExprError), } pub type VdSynExprIdx = ArenaIdx; @@ -92,6 +95,7 @@ impl VdSynExprData { VdSynExprData::UniadicArray => vec![], // ad hoc VdSynExprData::VariadicArray => vec![], + VdSynExprData::Err(ref error) => vec![], } } @@ -112,7 +116,14 @@ pub enum VdSynExprClass { impl ToVdSyn for LxMathAstIdxRange { fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> VdSynExprIdx { - todo!() + if self.is_empty() { + builder.alloc_expr( + VdSynExprData::Err(OriginalVdSynExprError::Empty.into()), + VdSynExprAstRange::Asts(self.into()), + ) + } else { + todo!() + } } } diff --git a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs index 5f1d75402c..fc347e97f8 100644 --- a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs +++ b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs @@ -124,6 +124,7 @@ impl<'a> VdSynExprLaTeXFormatter<'a> { VdSynExprData::UniadicArray => todo!(), VdSynExprData::VariadicArray => todo!(), VdSynExprData::Opr { opr } => todo!(), + VdSynExprData::Err(ref error) => unreachable!("{error}"), } } diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index 5a180665f3..de7ee52cbf 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -4,7 +4,7 @@ use crate::{ expr::{VdSynExprArenaRef, VdSynExprIdx, VdSynExprIdxRange}, phrase::VdSynPhraseArenaRef, range::{ - VdSynClauseRangeMap, VdSynExprRange, VdSynExprRangeMap, VdSynPhraseRangeMap, + VdSynClauseRangeMap, VdSynExprAstRange, VdSynExprRangeMap, VdSynPhraseRangeMap, VdSynSentenceRangeMap, }, sentence::VdSynSentenceArenaRef, @@ -113,8 +113,8 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { pub fn render_expr(&self, expr: VdSynExprIdx) -> DisplayTree { let expr_range = self.expr_range_map[expr]; let (start, end) = match expr_range { - VdSynExprRange::Ast(ast) => self.ast_offset_range(ast), - VdSynExprRange::Asts(asts) => todo!(), + VdSynExprAstRange::Ast(ast) => self.ast_offset_range(ast), + VdSynExprAstRange::Asts(asts) => self.asts_offset_range(asts), }; let value = self.input[(start..end)].to_string(); DisplayTree::new(value, self.render_exprs(self.expr_arena[expr].children())) @@ -146,7 +146,9 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { fn math_asts_offset_range(&self, asts: LxMathAstIdxRange) -> (usize, usize) { let first = asts.start(); - let Some(last) = asts.last() else { todo!() }; + let Some(last) = asts.last() else { + return self.math_ast_offset_range(first); + }; ( self.math_ast_offset_range(first).0, self.math_ast_offset_range(last).1, diff --git a/crates/visored/visored-syn-expr/src/range.rs b/crates/visored/visored-syn-expr/src/range.rs index 6fe658d0ac..13840f4ef1 100644 --- a/crates/visored/visored-syn-expr/src/range.rs +++ b/crates/visored/visored-syn-expr/src/range.rs @@ -6,7 +6,7 @@ use crate::{ }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VdSynExprRange { +pub enum VdSynExprAstRange { Ast(LxAstIdx), Asts(LxAstIdxRange), } @@ -15,7 +15,7 @@ pub type VdSynPhraseRange = LxAstIdxRange; pub type VdSynClauseRange = LxAstIdxRange; pub type VdSynSentenceRange = LxAstIdxRange; -pub type VdSynExprRangeMap = VdSynExprOrderedMap; +pub type VdSynExprRangeMap = VdSynExprOrderedMap; pub type VdSynPhraseRangeMap = VdSynPhraseOrderedMap; pub type VdSynClauseRangeMap = VdSynClauseOrderedMap; pub type VdSynSentenceRangeMap = VdSynSentenceOrderedMap; From d502686d53566ac4b3704d11a40b2e953968fb8b Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 14:24:54 -0700 Subject: [PATCH 46/71] fix bug --- crates/visored/visored-syn-expr/src/expr.rs | 4 ++-- .../visored/visored-syn-expr/src/helpers/show/display_tree.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 4ad9e48dbf..5420acae76 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -168,8 +168,8 @@ mod tests { &[], &[], &expect![[r#" - - "#]], + + "#]], ); } } diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index de7ee52cbf..31fa4a3a31 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -147,7 +147,7 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { fn math_asts_offset_range(&self, asts: LxMathAstIdxRange) -> (usize, usize) { let first = asts.start(); let Some(last) = asts.last() else { - return self.math_ast_offset_range(first); + return (0, 0); }; ( self.math_ast_offset_range(first).0, From a17a8e87c546ed4541ec7c067926e4dcbbb69bdc Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 14:26:52 -0700 Subject: [PATCH 47/71] save --- crates/visored/visored-syn-expr/src/expr.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 5420acae76..2542f03389 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -169,6 +169,14 @@ mod tests { &[], &expect![[r#" + "#]], + ); + t( + "1", + &[], + &[], + &expect![[r#" + "#]], ); } From 3a82115ac14ca90f76ce55101b28510c07af4a23 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 16:27:09 -0700 Subject: [PATCH 48/71] save --- crates/visored/visored-syn-expr/src/lib.rs | 1 + crates/visored/visored-syn-expr/src/variable.rs | 5 +++++ crates/visored/visored-syn-expr/src/variable/scope.rs | 1 + 3 files changed, 7 insertions(+) create mode 100644 crates/visored/visored-syn-expr/src/variable.rs create mode 100644 crates/visored/visored-syn-expr/src/variable/scope.rs diff --git a/crates/visored/visored-syn-expr/src/lib.rs b/crates/visored/visored-syn-expr/src/lib.rs index 1125a8506f..965bea4446 100644 --- a/crates/visored/visored-syn-expr/src/lib.rs +++ b/crates/visored/visored-syn-expr/src/lib.rs @@ -13,6 +13,7 @@ pub mod sentence; pub mod test_helpers; #[cfg(test)] mod tests; +pub mod variable; use self::jar::VdSynExprJar as Jar; #[cfg(test)] diff --git a/crates/visored/visored-syn-expr/src/variable.rs b/crates/visored/visored-syn-expr/src/variable.rs new file mode 100644 index 0000000000..18ddc26540 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/variable.rs @@ -0,0 +1,5 @@ +pub mod scope; + +pub struct VdSynExprVariableIdx {} + +pub struct VdSynExprVariableData {} diff --git a/crates/visored/visored-syn-expr/src/variable/scope.rs b/crates/visored/visored-syn-expr/src/variable/scope.rs new file mode 100644 index 0000000000..40056866c5 --- /dev/null +++ b/crates/visored/visored-syn-expr/src/variable/scope.rs @@ -0,0 +1 @@ +pub struct VdSynExprVariableScope {} From 4a37e3d63abad9baed613c9030b72a7c07e16649 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 17:04:01 -0700 Subject: [PATCH 49/71] refactor latex token storage --- crates/cybertron/cybertron/src/seq/idx.rs | 6 +- crates/latex/latex-ast/src/ast.rs | 183 ++++++++++++------ crates/latex/latex-ast/src/ast/math.rs | 2 +- crates/latex/latex-ast/src/ast/rose.rs | 2 +- .../src/helpers/show/display_tree.rs | 2 +- crates/latex/latex-ast/src/parser.rs | 2 +- crates/latex/latex-ast/src/range.rs | 32 ++- crates/latex/latex-token/src/data.rs | 8 + crates/latex/latex-token/src/idx.rs | 160 ++++++++++++++- crates/latex/latex-token/src/idx/math.rs | 114 ----------- crates/latex/latex-token/src/idx/rose.rs | 98 ---------- crates/latex/latex-token/src/lexer.rs | 2 +- crates/latex/latex-token/src/storage.rs | 75 +++---- crates/latex/latex-token/src/stream/math.rs | 2 +- crates/latex/latex-token/src/stream/rose.rs | 2 +- crates/visored/visored-syn-expr/src/expr.rs | 18 +- .../src/helpers/show/display_tree.rs | 6 +- .../visored-syn-expr/src/variable/scope.rs | 6 +- 18 files changed, 356 insertions(+), 364 deletions(-) delete mode 100644 crates/latex/latex-token/src/idx/math.rs delete mode 100644 crates/latex/latex-token/src/idx/rose.rs diff --git a/crates/cybertron/cybertron/src/seq/idx.rs b/crates/cybertron/cybertron/src/seq/idx.rs index 97872db71f..6881e0a096 100644 --- a/crates/cybertron/cybertron/src/seq/idx.rs +++ b/crates/cybertron/cybertron/src/seq/idx.rs @@ -94,7 +94,7 @@ where ) } - pub fn nearest_left2(self) -> Seq<(Option2<(Idx, T)>)> { + pub fn nearest_left2(self) -> Seq> { let ts = self.data(); Seq::new( (0..ts.len()) @@ -128,7 +128,7 @@ where { (1..=i) .into_iter() - .filter_map(|j| ts[(i - j)].map(|t| (idx!(i - j), t))) + .filter_map(|j| ts[i - j].map(|t| (idx!(i - j), t))) .next() } @@ -138,7 +138,7 @@ where { (1..=i) .into_iter() - .filter_map(|j| ts[(i - j)].map(|t| (idx!(i - j), t))) + .filter_map(|j| ts[i - j].map(|t| (idx!(i - j), t))) .collect() } diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index dd5d3362da..652c7f1df5 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -182,8 +182,7 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_math_tokens: [], - ranged_rose_tokens: [], + ranged_tokens: [], }, LxAstArena { math: Arena { @@ -207,26 +206,29 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_math_tokens: [ + ranged_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Digit( - One, + Math( + Digit( + One, + ), ), ), ], - ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Digit( LxMathTokenIdx( - 0, + LxTokenIdx( + 0, + ), ), One, ), @@ -250,26 +252,29 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_math_tokens: [ + ranged_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Letter( - LowerX, + Math( + Letter( + LowerX, + ), ), ), ], - ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( LxMathTokenIdx( - 0, + LxTokenIdx( + 0, + ), ), LowerX, ), @@ -293,15 +298,17 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_math_tokens: [ + ranged_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Letter( - LowerX, + Math( + Letter( + LowerX, + ), ), ), ( @@ -310,8 +317,10 @@ fn parse_tex_input_into_asts_works() { 2, ), [1:2, 1:3), - Punctuation( - Add, + Math( + Punctuation( + Add, + ), ), ), ( @@ -320,31 +329,38 @@ fn parse_tex_input_into_asts_works() { 3, ), [1:3, 1:4), - Digit( - One, + Math( + Digit( + One, + ), ), ), ], - ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( LxMathTokenIdx( - 0, + LxTokenIdx( + 0, + ), ), LowerX, ), LxMathAstData::Opr( LxMathTokenIdx( - 1, + LxTokenIdx( + 1, + ), ), Add, ), LxMathAstData::Digit( LxMathTokenIdx( - 2, + LxTokenIdx( + 2, + ), ), One, ), @@ -368,15 +384,17 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_math_tokens: [ + ranged_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Letter( - LowerX, + Math( + Letter( + LowerX, + ), ), ), ( @@ -385,7 +403,9 @@ fn parse_tex_input_into_asts_works() { 2, ), [1:2, 1:3), - Superscript, + Math( + Superscript, + ), ), ( ( @@ -393,25 +413,30 @@ fn parse_tex_input_into_asts_works() { 3, ), [1:3, 1:4), - Digit( - Two, + Math( + Digit( + Two, + ), ), ), ], - ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( LxMathTokenIdx( - 0, + LxTokenIdx( + 0, + ), ), LowerX, ), LxMathAstData::Digit( LxMathTokenIdx( - 2, + LxTokenIdx( + 2, + ), ), Two, ), @@ -444,15 +469,17 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_math_tokens: [ + ranged_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Letter( - LowerX, + Math( + Letter( + LowerX, + ), ), ), ( @@ -461,7 +488,9 @@ fn parse_tex_input_into_asts_works() { 2, ), [1:2, 1:3), - Subscript, + Math( + Subscript, + ), ), ( ( @@ -469,25 +498,30 @@ fn parse_tex_input_into_asts_works() { 3, ), [1:3, 1:4), - Digit( - Two, + Math( + Digit( + Two, + ), ), ), ], - ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( LxMathTokenIdx( - 0, + LxTokenIdx( + 0, + ), ), LowerX, ), LxMathAstData::Digit( LxMathTokenIdx( - 2, + LxTokenIdx( + 2, + ), ), Two, ), @@ -520,15 +554,17 @@ fn parse_tex_input_into_asts_works() { expect![[r#" ( LxTokenStorage { - ranged_math_tokens: [ + ranged_tokens: [ ( ( 0, 1, ), [1:1, 1:2), - Letter( - LowerX, + Math( + Letter( + LowerX, + ), ), ), ( @@ -537,7 +573,9 @@ fn parse_tex_input_into_asts_works() { 2, ), [1:2, 1:3), - Superscript, + Math( + Superscript, + ), ), ( ( @@ -545,8 +583,10 @@ fn parse_tex_input_into_asts_works() { 3, ), [1:3, 1:4), - LeftDelimiter( - Curl, + Math( + LeftDelimiter( + Curl, + ), ), ), ( @@ -555,8 +595,10 @@ fn parse_tex_input_into_asts_works() { 4, ), [1:4, 1:5), - Letter( - LowerI, + Math( + Letter( + LowerI, + ), ), ), ( @@ -565,8 +607,10 @@ fn parse_tex_input_into_asts_works() { 5, ), [1:5, 1:6), - Punctuation( - Add, + Math( + Punctuation( + Add, + ), ), ), ( @@ -575,8 +619,10 @@ fn parse_tex_input_into_asts_works() { 6, ), [1:6, 1:7), - Digit( - Two, + Math( + Digit( + Two, + ), ), ), ( @@ -585,50 +631,63 @@ fn parse_tex_input_into_asts_works() { 7, ), [1:7, 1:8), - RightDelimiter( - Curl, + Math( + RightDelimiter( + Curl, + ), ), ), ], - ranged_rose_tokens: [], }, LxAstArena { math: Arena { data: [ LxMathAstData::Letter( LxMathTokenIdx( - 0, + LxTokenIdx( + 0, + ), ), LowerX, ), LxMathAstData::Letter( LxMathTokenIdx( - 3, + LxTokenIdx( + 3, + ), ), LowerI, ), LxMathAstData::Opr( LxMathTokenIdx( - 4, + LxTokenIdx( + 4, + ), ), Add, ), LxMathAstData::Digit( LxMathTokenIdx( - 5, + LxTokenIdx( + 5, + ), ), Two, ), LxMathAstData::Delimited { left_delimiter_token_idx: LxMathTokenIdx( - 2, + LxTokenIdx( + 2, + ), ), left_delimiter: Curl, asts: ArenaIdxRange( 1..4, ), right_delimiter_token_idx: LxMathTokenIdx( - 6, + LxTokenIdx( + 6, + ), ), right_delimiter: Curl, }, diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 3b9136dd8d..8e05d0435a 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -1,6 +1,6 @@ use latex_token::{ data::math::{digit::LxMathDigit, LxMathDelimiter}, - idx::math::LxMathTokenIdx, + idx::LxMathTokenIdx, }; use super::*; diff --git a/crates/latex/latex-ast/src/ast/rose.rs b/crates/latex/latex-ast/src/ast/rose.rs index 9da7e5189c..080c2c5396 100644 --- a/crates/latex/latex-ast/src/ast/rose.rs +++ b/crates/latex/latex-ast/src/ast/rose.rs @@ -1,6 +1,6 @@ //! means the prose mode use super::*; -use latex_token::{data::rose::LxRoseTokenData, idx::rose::LxRoseTokenIdx}; +use latex_token::{data::rose::LxRoseTokenData, idx::LxRoseTokenIdx}; #[salsa::derive_debug_with_db] #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/latex/latex-ast/src/helpers/show/display_tree.rs b/crates/latex/latex-ast/src/helpers/show/display_tree.rs index fb661870d3..7659f728de 100644 --- a/crates/latex/latex-ast/src/helpers/show/display_tree.rs +++ b/crates/latex/latex-ast/src/helpers/show/display_tree.rs @@ -65,7 +65,7 @@ impl<'a> LxAstDisplayTreeBuilder<'a> { let ast_token_idx_range = self.ast_token_idx_range_map[ast]; let (start, end) = self .token_storage - .math_token_idx_range_offset_range(ast_token_idx_range); + .token_idx_range_offset_range(ast_token_idx_range); let value = self.input[start..end].to_string(); DisplayTree::new( value, diff --git a/crates/latex/latex-ast/src/parser.rs b/crates/latex/latex-ast/src/parser.rs index 51af263772..20a293559f 100644 --- a/crates/latex/latex-ast/src/parser.rs +++ b/crates/latex/latex-ast/src/parser.rs @@ -9,7 +9,7 @@ use crate::{ use latex_prelude::mode::LxMode; use latex_token::{ data::{math::LxMathTokenData, rose::LxRoseTokenData}, - idx::{math::LxMathTokenIdx, rose::LxRoseTokenIdx}, + idx::{LxMathTokenIdx, LxRoseTokenIdx}, lexer::LxLexer, storage::LxTokenStorage, }; diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index 2d14a5e8c1..c4e09f3754 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -1,4 +1,4 @@ -use latex_token::idx::{math::LxMathTokenIdxRange, rose::LxRoseTokenIdxRange}; +use latex_token::idx::LxTokenIdxRange; use crate::ast::{ math::{LxMathAstArenaMap, LxMathAstData, LxMathAstIdx}, @@ -8,12 +8,12 @@ use crate::ast::{ #[derive(Debug)] pub struct LxAstTokenIdxRangeMap { - pub(crate) math: LxMathAstArenaMap, - pub(crate) rose: LxRoseAstArenaMap, + pub(crate) math: LxMathAstArenaMap, + pub(crate) rose: LxRoseAstArenaMap, } impl std::ops::Index for LxAstTokenIdxRangeMap { - type Output = LxMathTokenIdxRange; + type Output = LxTokenIdxRange; fn index(&self, index: LxMathAstIdx) -> &Self::Output { &self.math[index] @@ -21,7 +21,7 @@ impl std::ops::Index for LxAstTokenIdxRangeMap { } impl std::ops::Index for LxAstTokenIdxRangeMap { - type Output = LxRoseTokenIdxRange; + type Output = LxTokenIdxRange; fn index(&self, index: LxRoseAstIdx) -> &Self::Output { &self.rose[index] @@ -40,8 +40,8 @@ pub fn calc_ast_token_idx_range_map(db: &salsa::Db, arena: &LxAstArena) -> LxAst struct LxAstTokenIdxRangeCalculator<'a> { db: &'a ::salsa::Db, ast_arena: LxAstArenaRef<'a>, - math_data: LxMathAstArenaMap, - rose_data: LxRoseAstArenaMap, + math_data: LxMathAstArenaMap, + rose_data: LxRoseAstArenaMap, } impl<'a> LxAstTokenIdxRangeCalculator<'a> { @@ -72,11 +72,11 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { } } - fn calc_math_ast(&mut self, data: &LxMathAstData) -> LxMathTokenIdxRange { + fn calc_math_ast(&mut self, data: &LxMathAstData) -> LxTokenIdxRange { match *data { - LxMathAstData::Letter(idx, _) => LxMathTokenIdxRange::new_single(idx), - LxMathAstData::Opr(idx, _) => LxMathTokenIdxRange::new_single(idx), - LxMathAstData::Digit(idx, _) => LxMathTokenIdxRange::new_single(idx), + LxMathAstData::Letter(idx, _) => LxTokenIdxRange::new_single(*idx), + LxMathAstData::Opr(idx, _) => LxTokenIdxRange::new_single(*idx), + LxMathAstData::Digit(idx, _) => LxTokenIdxRange::new_single(*idx), LxMathAstData::TextEdit { ref buffer } => todo!(), LxMathAstData::Attach { base, ref scripts } => { let mut range = self.get_math_ast_range(base); @@ -92,13 +92,11 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { asts, right_delimiter_token_idx, right_delimiter, - } => { - LxMathTokenIdxRange::new_closed(left_delimiter_token_idx, right_delimiter_token_idx) - } + } => LxTokenIdxRange::new_closed(*left_delimiter_token_idx, *right_delimiter_token_idx), } } - fn get_math_ast_range(&mut self, idx: LxMathAstIdx) -> LxMathTokenIdxRange { + fn get_math_ast_range(&mut self, idx: LxMathAstIdx) -> LxTokenIdxRange { self.infer_math_ast(idx, &self.ast_arena.math()[idx]); self.math_data[idx] } @@ -110,11 +108,11 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { } } - fn calc_rose_ast(&self, data: &LxRoseAstData) -> LxRoseTokenIdxRange { + fn calc_rose_ast(&self, data: &LxRoseAstData) -> LxTokenIdxRange { todo!() } - fn get_rose_ast_range(&mut self, idx: LxRoseAstIdx) -> LxRoseTokenIdxRange { + fn get_rose_ast_range(&mut self, idx: LxRoseAstIdx) -> LxTokenIdxRange { self.infer_rose_ast(idx, &self.ast_arena.rose()[idx]); self.rose_data[idx] } diff --git a/crates/latex/latex-token/src/data.rs b/crates/latex/latex-token/src/data.rs index 7e3aebcfb0..439247a8b6 100644 --- a/crates/latex/latex-token/src/data.rs +++ b/crates/latex/latex-token/src/data.rs @@ -6,3 +6,11 @@ use crate::lexer::LxLexer; #[cfg(test)] use crate::*; use latex_prelude::mode::LxMode; + +#[enum_class::from_variants] +#[salsa::derive_debug_with_db] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum LxTokenData { + Math(LxMathTokenData), + Rose(LxRoseTokenData), +} diff --git a/crates/latex/latex-token/src/idx.rs b/crates/latex/latex-token/src/idx.rs index 7fa0fff207..a011ad00ff 100644 --- a/crates/latex/latex-token/src/idx.rs +++ b/crates/latex/latex-token/src/idx.rs @@ -1,2 +1,158 @@ -pub mod math; -pub mod rose; +use shifted_unsigned_int::ShiftedU32; +use std::iter::Step; +use std::ops::{Add, Sub}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct LxTokenIdx(ShiftedU32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct LxMathTokenIdx(pub(crate) LxTokenIdx); + +impl std::ops::Deref for LxMathTokenIdx { + type Target = LxTokenIdx; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::convert::AsRef for LxMathTokenIdx { + fn as_ref(&self) -> &LxTokenIdx { + &self.0 + } +} + +impl std::borrow::Borrow for LxMathTokenIdx { + fn borrow(&self) -> &LxTokenIdx { + &self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct LxRoseTokenIdx(pub(crate) LxTokenIdx); + +impl std::ops::Deref for LxRoseTokenIdx { + type Target = LxTokenIdx; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::convert::AsRef for LxRoseTokenIdx { + fn as_ref(&self) -> &LxTokenIdx { + &self.0 + } +} + +impl std::borrow::Borrow for LxRoseTokenIdx { + fn borrow(&self) -> &LxTokenIdx { + &self.0 + } +} + +impl LxTokenIdx { + pub(crate) fn from_index(index: usize) -> Self { + Self(index.into()) + } + + pub(crate) fn index(self) -> usize { + self.0.index() + } +} + +impl Add for LxTokenIdx { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} + +impl Sub for LxTokenIdx { + type Output = Self; + fn sub(self, rhs: usize) -> Self::Output { + Self(self.0 - rhs) + } +} + +impl Step for LxTokenIdx { + fn steps_between(start: &Self, end: &Self) -> Option { + end.0.index().checked_sub(start.0.index()).map(|diff| diff) + } + fn forward_checked(start: Self, count: usize) -> Option { + start.0.checked_add(count.try_into().ok()?).map(Self) + } + fn backward_checked(start: Self, count: usize) -> Option { + start.0.checked_sub(count.try_into().ok()?).map(Self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LxTokenIdxRange { + start: ShiftedU32, + end: ShiftedU32, +} + +impl IntoIterator for LxTokenIdxRange { + type Item = LxTokenIdx; + type IntoIter = std::ops::Range; + + fn into_iter(self) -> Self::IntoIter { + LxTokenIdx::from_index(self.start.index())..LxTokenIdx::from_index(self.end.index()) + } +} + +impl From> for LxTokenIdxRange { + fn from(range: std::ops::Range) -> Self { + Self { + start: range.start.into(), + end: range.end.into(), + } + } +} + +impl LxTokenIdxRange { + pub(crate) fn new(range: std::ops::Range) -> Self { + Self { + start: range.start.into(), + end: range.end.into(), + } + } + + pub fn new_single(idx: LxTokenIdx) -> Self { + Self { + start: idx.0, + end: (idx + 1).0, + } + } + + pub fn new_closed(first: LxTokenIdx, last: LxTokenIdx) -> Self { + Self { + start: first.0, + end: last.0 + 1usize, + } + } +} + +impl LxTokenIdxRange { + pub(crate) fn is_empty(self) -> bool { + self.start == self.end + } + + pub fn start(&self) -> LxTokenIdx { + LxTokenIdx(self.start) + } + + pub fn last(&self) -> Option { + if self.is_empty() { + None + } else { + Some(LxTokenIdx(self.end - 1)) + } + } + + pub fn join(self, other: Self) -> Self { + Self { + start: self.start.min(other.start), + end: self.end.max(other.end), + } + } +} diff --git a/crates/latex/latex-token/src/idx/math.rs b/crates/latex/latex-token/src/idx/math.rs deleted file mode 100644 index 2dfe4b51bc..0000000000 --- a/crates/latex/latex-token/src/idx/math.rs +++ /dev/null @@ -1,114 +0,0 @@ -use shifted_unsigned_int::ShiftedU32; -use std::iter::Step; -use std::ops::{Add, Sub}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct LxMathTokenIdx(ShiftedU32); - -impl LxMathTokenIdx { - pub(crate) fn from_index(index: usize) -> LxMathTokenIdx { - Self(index.into()) - } - - pub(crate) fn index(self) -> usize { - self.0.index() - } -} - -impl Add for LxMathTokenIdx { - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - Self(self.0 + rhs) - } -} - -impl Sub for LxMathTokenIdx { - type Output = Self; - fn sub(self, rhs: usize) -> Self::Output { - Self(self.0 - rhs) - } -} - -impl Step for LxMathTokenIdx { - fn steps_between(start: &Self, end: &Self) -> Option { - end.0.index().checked_sub(start.0.index()).map(|diff| diff) - } - fn forward_checked(start: Self, count: usize) -> Option { - start.0.checked_add(count.try_into().ok()?).map(Self) - } - fn backward_checked(start: Self, count: usize) -> Option { - start.0.checked_sub(count.try_into().ok()?).map(Self) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct LxMathTokenIdxRange { - start: ShiftedU32, - end: ShiftedU32, -} - -impl IntoIterator for LxMathTokenIdxRange { - type Item = LxMathTokenIdx; - type IntoIter = std::ops::Range; - - fn into_iter(self) -> Self::IntoIter { - LxMathTokenIdx::from_index(self.start.index())..LxMathTokenIdx::from_index(self.end.index()) - } -} - -impl From> for LxMathTokenIdxRange { - fn from(range: std::ops::Range) -> Self { - Self { - start: range.start.into(), - end: range.end.into(), - } - } -} - -impl LxMathTokenIdxRange { - pub(crate) fn new(range: std::ops::Range) -> Self { - Self { - start: range.start.into(), - end: range.end.into(), - } - } - - pub fn new_single(idx: LxMathTokenIdx) -> Self { - Self { - start: idx.0, - end: (idx + 1).0, - } - } - - pub fn new_closed(first: LxMathTokenIdx, last: LxMathTokenIdx) -> Self { - Self { - start: first.0, - end: last.0 + 1usize, - } - } -} - -impl LxMathTokenIdxRange { - pub(crate) fn is_empty(self) -> bool { - self.start == self.end - } - - pub fn start(&self) -> LxMathTokenIdx { - LxMathTokenIdx(self.start) - } - - pub fn last(&self) -> Option { - if self.is_empty() { - None - } else { - Some(LxMathTokenIdx(self.end - 1)) - } - } - - pub fn join(self, other: Self) -> Self { - Self { - start: self.start.min(other.start), - end: self.end.max(other.end), - } - } -} diff --git a/crates/latex/latex-token/src/idx/rose.rs b/crates/latex/latex-token/src/idx/rose.rs deleted file mode 100644 index c7d0ffc8c4..0000000000 --- a/crates/latex/latex-token/src/idx/rose.rs +++ /dev/null @@ -1,98 +0,0 @@ -use shifted_unsigned_int::ShiftedU32; -use std::iter::Step; -use std::ops::{Add, Sub}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct LxRoseTokenIdx(ShiftedU32); - -impl LxRoseTokenIdx { - pub(crate) fn from_index(index: usize) -> LxRoseTokenIdx { - Self(index.into()) - } - - pub(crate) fn index(self) -> usize { - self.0.index() - } -} - -impl Add for LxRoseTokenIdx { - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - Self(self.0 + rhs) - } -} - -impl Sub for LxRoseTokenIdx { - type Output = Self; - fn sub(self, rhs: usize) -> Self::Output { - Self(self.0 - rhs) - } -} - -impl Step for LxRoseTokenIdx { - fn steps_between(start: &Self, end: &Self) -> Option { - end.0.index().checked_sub(start.0.index()).map(|diff| diff) - } - fn forward_checked(start: Self, count: usize) -> Option { - start.0.checked_add(count.try_into().ok()?).map(Self) - } - fn backward_checked(start: Self, count: usize) -> Option { - start.0.checked_sub(count.try_into().ok()?).map(Self) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct LxRoseTokenIdxRange { - start: ShiftedU32, - end: ShiftedU32, -} - -impl IntoIterator for LxRoseTokenIdxRange { - type Item = LxRoseTokenIdx; - type IntoIter = std::ops::Range; - - fn into_iter(self) -> Self::IntoIter { - LxRoseTokenIdx::from_index(self.start.index())..LxRoseTokenIdx::from_index(self.end.index()) - } -} - -impl From> for LxRoseTokenIdxRange { - fn from(range: std::ops::Range) -> Self { - Self { - start: range.start.into(), - end: range.end.into(), - } - } -} - -impl LxRoseTokenIdxRange { - pub(crate) fn new(range: std::ops::Range) -> Self { - Self { - start: range.start.into(), - end: range.end.into(), - } - } - - pub fn new_single(idx: LxRoseTokenIdx) -> Self { - Self { - start: idx.0, - end: (idx + 1).0, - } - } - - pub(crate) fn is_empty(self) -> bool { - self.start == self.end - } - - pub fn start(self) -> LxRoseTokenIdx { - LxRoseTokenIdx(self.start) - } - - pub fn last(self) -> Option { - if self.is_empty() { - None - } else { - Some(LxRoseTokenIdx(self.end - 1)) - } - } -} diff --git a/crates/latex/latex-token/src/lexer.rs b/crates/latex/latex-token/src/lexer.rs index 6de94c1477..dcc681edb2 100644 --- a/crates/latex/latex-token/src/lexer.rs +++ b/crates/latex/latex-token/src/lexer.rs @@ -3,7 +3,7 @@ use crate::{ math::{LxMathTokenData, LxMathTokenError}, rose::LxRoseTokenData, }, - idx::{math::LxMathTokenIdx, rose::LxRoseTokenIdx}, + idx::{LxMathTokenIdx, LxRoseTokenIdx}, storage::LxTokenStorage, stream::{math::LxMathTokenStream, rose::LxRoseTokenStream}, }; diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index b6ba4087eb..a44cc6032d 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -1,17 +1,17 @@ +use crate::{ + data::LxTokenData, + idx::{LxRoseTokenIdx, LxTokenIdxRange}, +}; use crate::{ data::{math::LxMathTokenData, rose::LxRoseTokenData}, - idx::{ - math::{LxMathTokenIdx, LxMathTokenIdxRange}, - rose::{LxRoseTokenIdx, LxRoseTokenIdxRange}, - }, + idx::{LxMathTokenIdx, LxTokenIdx}, }; use husky_text_protocol::range::TextRange; #[salsa::derive_debug_with_db] #[derive(Default, Debug)] pub struct LxTokenStorage { - ranged_math_tokens: Vec<((usize, usize), TextRange, LxMathTokenData)>, - ranged_rose_tokens: Vec<((usize, usize), TextRange, LxRoseTokenData)>, + ranged_tokens: Vec<((usize, usize), TextRange, LxTokenData)>, } /// # getters @@ -20,54 +20,33 @@ impl std::ops::Index for LxTokenStorage { type Output = LxMathTokenData; fn index(&self, idx: LxMathTokenIdx) -> &Self::Output { - &self.ranged_math_tokens[idx.index()].2 + match &self.ranged_tokens[idx.0.index()].2 { + LxTokenData::Math(data) => data, + _ => unreachable!(), + } } } impl LxTokenStorage { - pub fn whole_math_token_idx_range(&self) -> LxMathTokenIdxRange { - LxMathTokenIdxRange::new(0..self.ranged_math_tokens.len()) - } - - pub fn whole_rose_token_idx_range(&self) -> LxRoseTokenIdxRange { - LxRoseTokenIdxRange::new(0..self.ranged_rose_tokens.len()) - } - - pub fn math_token_offset_range(&self, token_idx: LxMathTokenIdx) -> (usize, usize) { - self.ranged_math_tokens[token_idx.index()].0 - } - - pub fn rose_token_offset_range(&self, token_idx: LxRoseTokenIdx) -> (usize, usize) { - self.ranged_rose_tokens[token_idx.index()].0 + pub fn whole_token_idx_range(&self) -> LxTokenIdxRange { + LxTokenIdxRange::new(0..self.ranged_tokens.len()) } - pub fn math_token_text_range(&self, token_idx: LxMathTokenIdx) -> TextRange { - self.ranged_math_tokens[token_idx.index()].1 - } - - pub fn rose_token_text_range(&self, token_idx: LxRoseTokenIdx) -> TextRange { - self.ranged_rose_tokens[token_idx.index()].1 - } - - pub fn math_token_idx_range_offset_range(&self, range: LxMathTokenIdxRange) -> (usize, usize) { - let first = self.math_token_offset_range(range.start()); - match range.last() { - Some(last) => { - let last = self.math_token_offset_range(last); - (first.0, last.1) - } - None => todo!(), - } + pub fn token_offset_range( + &self, + token_idx: impl std::borrow::Borrow, + ) -> (usize, usize) { + self.ranged_tokens[token_idx.borrow().index()].0 } - pub fn rose_token_idx_range_offset_range(&self, range: LxRoseTokenIdxRange) -> (usize, usize) { - let first = self.rose_token_offset_range(range.start()); + pub fn token_idx_range_offset_range(&self, range: LxTokenIdxRange) -> (usize, usize) { + let first = self.token_offset_range(range.start()); match range.last() { Some(last) => { - let last = self.rose_token_offset_range(last); + let last = self.token_offset_range(last); (first.0, last.1) } - None => todo!(), + None => (0, 0), } } } @@ -82,9 +61,9 @@ impl LxTokenStorage { range: TextRange, token_data: LxMathTokenData, ) -> LxMathTokenIdx { - let idx = LxMathTokenIdx::from_index(self.ranged_math_tokens.len()); - self.ranged_math_tokens - .push(((start_offset, end_offset), range, token_data)); + let idx = LxMathTokenIdx(LxTokenIdx::from_index(self.ranged_tokens.len())); + self.ranged_tokens + .push(((start_offset, end_offset), range, token_data.into())); idx } @@ -95,9 +74,9 @@ impl LxTokenStorage { range: TextRange, token_data: LxRoseTokenData, ) -> LxRoseTokenIdx { - let idx = LxRoseTokenIdx::from_index(self.ranged_rose_tokens.len()); - self.ranged_rose_tokens - .push(((start_offset, end_offset), range, token_data)); + let idx = LxRoseTokenIdx(LxTokenIdx::from_index(self.ranged_tokens.len())); + self.ranged_tokens + .push(((start_offset, end_offset), range, token_data.into())); idx } } diff --git a/crates/latex/latex-token/src/stream/math.rs b/crates/latex/latex-token/src/stream/math.rs index e6fdbd50f4..c74f3e52fc 100644 --- a/crates/latex/latex-token/src/stream/math.rs +++ b/crates/latex/latex-token/src/stream/math.rs @@ -1,4 +1,4 @@ -use crate::{data::math::LxMathTokenData, idx::math::LxMathTokenIdx, lexer::LxLexer}; +use crate::{data::math::LxMathTokenData, idx::LxMathTokenIdx, lexer::LxLexer}; pub struct LxMathTokenStream<'a> { lexer: LxLexer<'a>, diff --git a/crates/latex/latex-token/src/stream/rose.rs b/crates/latex/latex-token/src/stream/rose.rs index 8f428192d3..9526b33dd6 100644 --- a/crates/latex/latex-token/src/stream/rose.rs +++ b/crates/latex/latex-token/src/stream/rose.rs @@ -1,4 +1,4 @@ -use crate::{data::rose::LxRoseTokenData, idx::rose::LxRoseTokenIdx, lexer::LxLexer}; +use crate::{data::rose::LxRoseTokenData, idx::LxRoseTokenIdx, lexer::LxLexer}; pub struct LxRoseTokenStream<'a> { lexer: LxLexer<'a>, diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 2542f03389..a3d4b2b163 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -160,7 +160,7 @@ mod tests { space_annotations, db, ); - expected.assert_eq(&example.show_display_tree(db)); + expected.assert_debug_eq(&example.show_display_tree(db)); } t( @@ -168,16 +168,16 @@ mod tests { &[], &[], &expect![[r#" - + "\n" "#]], ); - t( - "1", - &[], - &[], - &expect![[r#" + // t( + // "1", + // &[], + // &[], + // &expect![[r#" - "#]], - ); + // "#]], + // ); } } diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index 31fa4a3a31..72afa6091c 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -116,7 +116,7 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { VdSynExprAstRange::Ast(ast) => self.ast_offset_range(ast), VdSynExprAstRange::Asts(asts) => self.asts_offset_range(asts), }; - let value = self.input[(start..end)].to_string(); + let value = self.input[start..end].to_string(); DisplayTree::new(value, self.render_exprs(self.expr_arena[expr].children())) } @@ -129,12 +129,12 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { fn math_ast_offset_range(&self, ast: LxMathAstIdx) -> (usize, usize) { let range = self.ast_token_idx_range_map[ast]; - self.token_storage.math_token_idx_range_offset_range(range) + self.token_storage.token_idx_range_offset_range(range) } fn rose_ast_offset_range(&self, ast: LxRoseAstIdx) -> (usize, usize) { let range = self.ast_token_idx_range_map[ast]; - self.token_storage.rose_token_idx_range_offset_range(range) + self.token_storage.token_idx_range_offset_range(range) } fn asts_offset_range(&self, asts: LxAstIdxRange) -> (usize, usize) { diff --git a/crates/visored/visored-syn-expr/src/variable/scope.rs b/crates/visored/visored-syn-expr/src/variable/scope.rs index 40056866c5..3ecde61674 100644 --- a/crates/visored/visored-syn-expr/src/variable/scope.rs +++ b/crates/visored/visored-syn-expr/src/variable/scope.rs @@ -1 +1,5 @@ -pub struct VdSynExprVariableScope {} +use latex_token::idx::LxTokenIdxRange; + +pub struct VdSynExprVariableScope { + range: LxTokenIdxRange, +} From 2cc0921f8229d7694de7247fb71dab7a812a113c Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 17:18:50 -0700 Subject: [PATCH 50/71] add token storage as an argument to VdAnnotations --- Cargo.lock | 9 ++++ crates/latex/latex-ast/Cargo.toml | 2 + crates/latex/latex-ast/src/ast/math.rs | 11 ++++- crates/latex/latex-ast/src/range.rs | 4 ++ crates/latex/latex-ast/src/tests.rs | 2 +- .../expect-files/husky_lang_jar_packages.txt | 5 +++ .../expect-files/husky_lang_jar_tree.json | 7 +++- .../expect-files/husky_lang_jar_tree.txt | 5 +++ .../husky_lang_jar_tree_trimmed.json | 2 +- .../husky_lang_jar_tree_trimmed.txt | 2 +- .../expect-files/husky_lang_packages.txt | 42 ++++++++++++------- crates/visored/visored-annotation/Cargo.toml | 8 ++++ .../visored-annotation/src/annotations.rs | 9 +++- .../src/annotations/builder.rs | 2 + .../src/annotations/builder/sparce.rs | 23 ++++++++-- .../src/annotations/walker.rs | 17 +++++++- crates/visored/visored-annotation/src/lib.rs | 5 +++ .../visored/visored-annotation/src/tests.rs | 6 +++ crates/visored/visored-hir-expr/Cargo.toml | 1 + crates/visored/visored-hir-expr/src/tests.rs | 1 + crates/visored/visored-sem-expr/Cargo.toml | 1 + crates/visored/visored-sem-expr/src/tests.rs | 1 + crates/visored/visored-syn-expr/Cargo.toml | 2 + .../src/test_helpers/example.rs | 1 + crates/visored/visored-syn-expr/src/tests.rs | 1 + 25 files changed, 143 insertions(+), 26 deletions(-) create mode 100644 crates/visored/visored-annotation/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 6dfa9a67bc..1a263b7f7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8157,6 +8157,7 @@ dependencies = [ "husky-tree-utils", "idx-arena", "latex-ast", + "latex-command", "latex-math-letter", "latex-math-opr", "latex-prelude", @@ -12503,9 +12504,14 @@ version = "0.1.0" dependencies = [ "enum-class", "expect-test", + "husky-coword", "husky-print-utils", + "latex-ast", + "latex-command", "latex-prelude", + "latex-token", "lazy_static 1.5.0", + "salsa", "visored-sem-expr", ] @@ -12523,6 +12529,7 @@ dependencies = [ "husky-coword", "idx-arena", "latex-ast", + "latex-command", "salsa", "visored-opr", "visored-sem-expr", @@ -12576,6 +12583,7 @@ dependencies = [ "husky-coword", "idx-arena", "latex-ast", + "latex-command", "latex-prelude", "salsa", "thiserror", @@ -12597,6 +12605,7 @@ dependencies = [ "husky-tree-utils", "idx-arena", "latex-ast", + "latex-command", "latex-prelude", "latex-token", "salsa", diff --git a/crates/latex/latex-ast/Cargo.toml b/crates/latex/latex-ast/Cargo.toml index 8413aa2b03..e97436ac30 100644 --- a/crates/latex/latex-ast/Cargo.toml +++ b/crates/latex/latex-ast/Cargo.toml @@ -22,6 +22,7 @@ time-capsule.workspace = true # kernel husky-coword.workspace = true # latex +latex-command.workspace = true latex-math-letter.workspace = true latex-math-opr.workspace = true latex-prelude.workspace = true @@ -32,6 +33,7 @@ husky-tree-utils.workspace = true [dev-dependencies] expect-test.workspace = true # latex +latex-command.workspace = true latex-ast = { workspace = true, features = ["test_helpers"] } [lints] diff --git a/crates/latex/latex-ast/src/ast/math.rs b/crates/latex/latex-ast/src/ast/math.rs index 8e05d0435a..b55e1457dd 100644 --- a/crates/latex/latex-ast/src/ast/math.rs +++ b/crates/latex/latex-ast/src/ast/math.rs @@ -1,3 +1,4 @@ +use latex_command::path::LxCommandPath; use latex_token::{ data::math::{digit::LxMathDigit, LxMathDelimiter}, idx::LxMathTokenIdx, @@ -26,6 +27,11 @@ pub enum LxMathAstData { right_delimiter_token_idx: LxMathTokenIdx, right_delimiter: LxMathDelimiter, }, + Command { + command_token_idx: LxMathTokenIdx, + command_path: LxCommandPath, + // TODO: command arguments + }, } impl LxMathAstData { @@ -57,7 +63,10 @@ impl<'a> LxAstParser<'a> { }; let (idx, token) = self.next_math_token()?; Some(match token { - LxMathTokenData::Command(_) => todo!(), + LxMathTokenData::Command(command_path) => LxMathAstData::Command { + command_token_idx: idx, + command_path, + }, LxMathTokenData::LeftDelimiter(delimiter) => self.parse_delimited(idx, delimiter), LxMathTokenData::RightDelimiter(_) => unreachable!(), LxMathTokenData::Letter(letter) => LxMathAstData::Letter(idx, letter), diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index c4e09f3754..14c6e079f4 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -93,6 +93,10 @@ impl<'a> LxAstTokenIdxRangeCalculator<'a> { right_delimiter_token_idx, right_delimiter, } => LxTokenIdxRange::new_closed(*left_delimiter_token_idx, *right_delimiter_token_idx), + LxMathAstData::Command { + command_token_idx, + command_path, + } => LxTokenIdxRange::new_single(*command_token_idx), // TODO: consider command arguments } } diff --git a/crates/latex/latex-ast/src/tests.rs b/crates/latex/latex-ast/src/tests.rs index 2edeb99c1d..866bd40107 100644 --- a/crates/latex/latex-ast/src/tests.rs +++ b/crates/latex/latex-ast/src/tests.rs @@ -3,5 +3,5 @@ pub(crate) use salsa::DebugWithDb; use husky_coword::jar::CowordJar; -#[salsa::db(CowordJar)] +#[salsa::db(CowordJar, crate::jar::LxAstJar, latex_command::jar::LxCommandJar)] pub struct DB {} diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt index c3965dede3..a96b942c33 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt @@ -1265,6 +1265,8 @@ name: "latex-ast", dependencies: [ "husky-coword", + "latex-command", + "latex-command", ], }, JarPackageSummary { @@ -1312,6 +1314,7 @@ "visored-zfs-ty", "husky-coword", "latex-ast", + "latex-command", ], }, JarPackageSummary { @@ -1325,6 +1328,7 @@ "visored-opr", "visored-zfs-ty", "husky-coword", + "latex-command", ], }, JarPackageSummary { @@ -1339,6 +1343,7 @@ "visored-zfs-ty", "husky-coword", "latex-ast", + "latex-command", "visored-sem-expr", "visored-syn-expr", ], diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json index 003981e99c..6e9ffe2881 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json @@ -1162,7 +1162,8 @@ "husky-vfs" ], "latex-ast": [ - "husky-coword" + "husky-coword", + "latex-command" ], "latex-command": [ "husky-coword" @@ -1176,6 +1177,7 @@ "visored-hir-expr": [ "husky-coword", "latex-ast", + "latex-command", "visored-opr", "visored-sem-expr", "visored-syn-expr", @@ -1184,6 +1186,7 @@ "visored-lean-transpilation": [ "husky-coword", "latex-ast", + "latex-command", "lean-hir-expr", "lean-sem-expr", "lean-term", @@ -1197,6 +1200,7 @@ "visored-sem-expr": [ "husky-coword", "latex-ast", + "latex-command", "visored-opr", "visored-syn-expr", "visored-zfs-ty" @@ -1204,6 +1208,7 @@ "visored-syn-expr": [ "husky-coword", "latex-ast", + "latex-command", "visored-opr", "visored-zfs-ty" ], diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt index 640e861eb9..733a8ac984 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt @@ -1163,6 +1163,7 @@ }, "latex-ast": { "husky-coword", + "latex-command", }, "latex-command": { "husky-coword", @@ -1176,6 +1177,7 @@ "visored-hir-expr": { "husky-coword", "latex-ast", + "latex-command", "visored-opr", "visored-sem-expr", "visored-syn-expr", @@ -1184,6 +1186,7 @@ "visored-lean-transpilation": { "husky-coword", "latex-ast", + "latex-command", "lean-hir-expr", "lean-sem-expr", "lean-term", @@ -1197,6 +1200,7 @@ "visored-sem-expr": { "husky-coword", "latex-ast", + "latex-command", "visored-opr", "visored-syn-expr", "visored-zfs-ty", @@ -1204,6 +1208,7 @@ "visored-syn-expr": { "husky-coword", "latex-ast", + "latex-command", "visored-opr", "visored-zfs-ty", }, diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json index cb1d2a9a71..03270dd322 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json @@ -170,7 +170,7 @@ "husky-ki-repr" ], "latex-ast": [ - "husky-coword" + "latex-command" ], "latex-command": [ "husky-coword" diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt index e651da9470..b9a5fcc9cd 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt @@ -170,7 +170,7 @@ "husky-ki-repr", }, "latex-ast": { - "husky-coword", + "latex-command", }, "latex-command": { "husky-coword", diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 903d393bdf..239f57eda8 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -3539,6 +3539,7 @@ "husky-coword", "husky-tree-utils", "idx-arena", + "latex-command", "latex-math-letter", "latex-math-opr", "latex-prelude", @@ -3547,6 +3548,7 @@ "salsa", "time-capsule", "expect-test", + "latex-command", ], }, PackageSummary { @@ -3556,6 +3558,22 @@ "expect-test", ], }, + PackageSummary { + name: "latex-command", + relative_path: "crates/latex/latex-command", + dependencies: [ + "husky-coword", + "latex-prelude", + "salsa", + ], + }, + PackageSummary { + name: "latex-prelude", + relative_path: "crates/latex/latex-prelude", + dependencies: [ + "salsa", + ], + }, PackageSummary { name: "latex-math-letter", relative_path: "crates/latex/latex-math-letter", @@ -3566,13 +3584,6 @@ relative_path: "crates/latex/latex-math-opr", dependencies: [], }, - PackageSummary { - name: "latex-prelude", - relative_path: "crates/latex/latex-prelude", - dependencies: [ - "salsa", - ], - }, PackageSummary { name: "latex-token", relative_path: "crates/latex/latex-token", @@ -3590,15 +3601,6 @@ "expect-test", ], }, - PackageSummary { - name: "latex-command", - relative_path: "crates/latex/latex-command", - dependencies: [ - "husky-coword", - "latex-prelude", - "salsa", - ], - }, PackageSummary { name: "husky-aarch64", relative_path: "crates/archs/husky-aarch64", @@ -4217,9 +4219,14 @@ dependencies: [ "enum-class", "husky-print-utils", + "latex-ast", "latex-prelude", + "latex-token", "lazy_static", "expect-test", + "husky-coword", + "latex-command", + "salsa", "visored-sem-expr", ], }, @@ -4237,6 +4244,7 @@ "visored-zfs-ty", "husky-coword", "latex-ast", + "latex-command", ], }, PackageSummary { @@ -4265,6 +4273,7 @@ "visored-zfs-ty", "expect-test", "husky-coword", + "latex-command", ], }, PackageSummary { @@ -4295,6 +4304,7 @@ "visored-zfs-ty", "husky-coword", "latex-ast", + "latex-command", "visored-sem-expr", "visored-syn-expr", ], diff --git a/crates/visored/visored-annotation/Cargo.toml b/crates/visored/visored-annotation/Cargo.toml index 20321a1f84..8e5e948d51 100644 --- a/crates/visored/visored-annotation/Cargo.toml +++ b/crates/visored/visored-annotation/Cargo.toml @@ -17,11 +17,19 @@ lazy_static.workspace = true enum-class.workspace = true # latex latex-prelude.workspace = true +latex-token.workspace = true +latex-ast.workspace = true # utils husky-print-utils.workspace = true [dev-dependencies] expect-test.workspace = true +# abstractions +salsa.workspace = true +# kernel +husky-coword.workspace = true +# latex +latex-command.workspace = true # visored visored-sem-expr = { workspace = true, features = ["test_helpers"] } diff --git a/crates/visored/visored-annotation/src/annotations.rs b/crates/visored/visored-annotation/src/annotations.rs index 6119c74baa..b284d785ea 100644 --- a/crates/visored/visored-annotation/src/annotations.rs +++ b/crates/visored/visored-annotation/src/annotations.rs @@ -1,8 +1,12 @@ mod builder; pub mod walker; -use crate::annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}; +use crate::{ + annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}, + *, +}; use builder::sparce::collect_from_sparse_annotations; +use latex_token::storage::LxTokenStorage; use walker::VdAnnotationsWalker; #[derive(Debug, PartialEq, Eq, Clone)] @@ -25,6 +29,7 @@ impl VdAnnotations { pub fn new( token_annotations: Vec, space_annotations: Vec, + token_storage: &LxTokenStorage, ) -> Self { Self { token_annotations, @@ -36,11 +41,13 @@ impl VdAnnotations { input: &'a str, token_annotations: impl IntoIterator, space_annotations: impl IntoIterator, + token_storage: &LxTokenStorage, ) -> Self { collect_from_sparse_annotations( input, token_annotations.into_iter(), space_annotations.into_iter(), + token_storage, ) } diff --git a/crates/visored/visored-annotation/src/annotations/builder.rs b/crates/visored/visored-annotation/src/annotations/builder.rs index c00d338927..4e682e8d90 100644 --- a/crates/visored/visored-annotation/src/annotations/builder.rs +++ b/crates/visored/visored-annotation/src/annotations/builder.rs @@ -1 +1,3 @@ pub mod sparce; + +use super::*; diff --git a/crates/visored/visored-annotation/src/annotations/builder/sparce.rs b/crates/visored/visored-annotation/src/annotations/builder/sparce.rs index 2bbd475a54..7611dc444c 100644 --- a/crates/visored/visored-annotation/src/annotations/builder/sparce.rs +++ b/crates/visored/visored-annotation/src/annotations/builder/sparce.rs @@ -1,19 +1,23 @@ +use super::*; use crate::{ annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}, annotations::{ VdAnnotationEntry, VdAnnotations, VdSpaceAnnotationEntry, VdTokenAnnotationEntry, }, }; +use latex_ast::ast::LxAstArena; +use latex_token::storage::LxTokenStorage; pub(crate) fn collect_from_sparse_annotations<'a>( raw_text: &'a str, token_annotation_iter: impl Iterator, space_annotation_iter: impl Iterator, + token_storage: &LxTokenStorage, ) -> VdAnnotations { let token_annotations = collect_from_sparse_token_annotations(raw_text, token_annotation_iter); let space_annotations = collect_from_sparse_space_annotations(raw_text, space_annotation_iter); - VdAnnotations::new(token_annotations, space_annotations) + VdAnnotations::new(token_annotations, space_annotations, token_storage) } fn collect_from_sparse_token_annotations<'a>( @@ -51,16 +55,18 @@ fn collect_from_sparse_annotations_aux<'a, A>( #[cfg(test)] mod tests { + use super::*; use crate::annotation::{ space::LxApplyAnnotation, token::{LxIntegralAnnotation, LxVariableAnnotation}, }; use expect_test::expect; - - use super::*; + use latex_ast::ast::parse_latex_input_into_asts; + use latex_prelude::mode::LxMode; #[test] fn test_collect_from_sparse_annotations_integral() { + let db = &DB::default(); let input = "\\int xdx"; let token_annotations = vec![ @@ -88,10 +94,21 @@ mod tests { VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), )]; + let mut token_storage = LxTokenStorage::default(); + let mut ast_arena = LxAstArena::default(); + let asts = parse_latex_input_into_asts( + db, + input, + LxMode::Math, + &mut token_storage, + &mut ast_arena, + ); + let result = collect_from_sparse_annotations( input, token_annotations.iter().copied(), space_annotations.iter().copied(), + &token_storage, ); expect![[r#" diff --git a/crates/visored/visored-annotation/src/annotations/walker.rs b/crates/visored/visored-annotation/src/annotations/walker.rs index 5797bf75ab..dda7658f81 100644 --- a/crates/visored/visored-annotation/src/annotations/walker.rs +++ b/crates/visored/visored-annotation/src/annotations/walker.rs @@ -1,4 +1,4 @@ -use super::{VdAnnotationEntry, VdSpaceAnnotationEntry, VdTokenAnnotationEntry}; +use super::*; use crate::annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}; pub struct VdAnnotationsWalker<'a> { @@ -75,6 +75,9 @@ fn next_annotation_aux( #[cfg(test)] mod tests { + use latex_ast::ast::{parse_latex_input_into_asts, LxAstArena}; + use latex_prelude::mode::LxMode; + use super::*; use crate::annotation::token::{LxIntegralAnnotation, LxVariableAnnotation, VdTokenAnnotation}; use crate::{ @@ -83,6 +86,8 @@ mod tests { }; fn setup_test_data() -> VdAnnotations { + let db = DB::default(); + let input = "\\int xdx".to_string(); let token_annotations = vec![ @@ -110,10 +115,20 @@ mod tests { VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), )]; + let mut token_storage = LxTokenStorage::default(); + let mut ast_arena = LxAstArena::default(); + let asts = parse_latex_input_into_asts( + &db, + &input, + LxMode::Math, + &mut token_storage, + &mut ast_arena, + ); VdAnnotations::from_sparse( &input, token_annotations.into_iter(), space_annotations.into_iter(), + &token_storage, ) } diff --git a/crates/visored/visored-annotation/src/lib.rs b/crates/visored/visored-annotation/src/lib.rs index 4153d0cf24..ee85f21d4d 100644 --- a/crates/visored/visored-annotation/src/lib.rs +++ b/crates/visored/visored-annotation/src/lib.rs @@ -3,3 +3,8 @@ pub mod annotation; pub mod annotations; #[cfg(feature = "test_helpers")] pub mod test_helpers; +#[cfg(test)] +mod tests; + +#[cfg(test)] +use crate::tests::*; diff --git a/crates/visored/visored-annotation/src/tests.rs b/crates/visored/visored-annotation/src/tests.rs new file mode 100644 index 0000000000..026d70ad2b --- /dev/null +++ b/crates/visored/visored-annotation/src/tests.rs @@ -0,0 +1,6 @@ +#[salsa::db( + husky_coword::jar::CowordJar, + latex_ast::jar::LxAstJar, + latex_command::jar::LxCommandJar +)] +pub(crate) struct DB; diff --git a/crates/visored/visored-hir-expr/Cargo.toml b/crates/visored/visored-hir-expr/Cargo.toml index 00744a2691..da0fcc2e12 100644 --- a/crates/visored/visored-hir-expr/Cargo.toml +++ b/crates/visored/visored-hir-expr/Cargo.toml @@ -23,6 +23,7 @@ visored-zfs-ty.workspace = true [dev-dependencies] # latex latex-ast.workspace = true +latex-command.workspace = true # kernel husky-coword.workspace = true # visored diff --git a/crates/visored/visored-hir-expr/src/tests.rs b/crates/visored/visored-hir-expr/src/tests.rs index d33564f2cc..510e7ca321 100644 --- a/crates/visored/visored-hir-expr/src/tests.rs +++ b/crates/visored/visored-hir-expr/src/tests.rs @@ -10,6 +10,7 @@ use crate::{builder::VdHirExprBuilder, ToVdHir}; #[salsa::db( husky_coword::jar::CowordJar, latex_ast::jar::LxAstJar, + latex_command::jar::LxCommandJar, visored_zfs_ty::jar::VdZfsTypeJar, visored_opr::jar::VdOprJar, visored_sem_expr::jar::VdSemExprJar, diff --git a/crates/visored/visored-sem-expr/Cargo.toml b/crates/visored/visored-sem-expr/Cargo.toml index 989553689b..a0417719cd 100644 --- a/crates/visored/visored-sem-expr/Cargo.toml +++ b/crates/visored/visored-sem-expr/Cargo.toml @@ -29,6 +29,7 @@ visored-syn-expr.workspace = true husky-coword.workspace = true # latex latex-ast.workspace = true +latex-command.workspace = true # visored visored-sem-expr = { workspace = true, features = ["test_helpers"] } diff --git a/crates/visored/visored-sem-expr/src/tests.rs b/crates/visored/visored-sem-expr/src/tests.rs index 36a138519c..21d330135f 100644 --- a/crates/visored/visored-sem-expr/src/tests.rs +++ b/crates/visored/visored-sem-expr/src/tests.rs @@ -3,6 +3,7 @@ visored_zfs_ty::jar::VdZfsTypeJar, visored_opr::jar::VdOprJar, latex_ast::jar::LxAstJar, + latex_command::jar::LxCommandJar, crate::jar::VdSemExprJar, visored_syn_expr::jar::VdSynExprJar )] diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index a90607d44c..2df93ba11f 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -34,6 +34,8 @@ visored-zfs-ty.workspace = true expect-test.workspace = true # kernel husky-coword.workspace = true +# latex +latex-command.workspace = true # visored visored-syn-expr = { workspace = true, features = ["test_helpers"] } diff --git a/crates/visored/visored-syn-expr/src/test_helpers/example.rs b/crates/visored/visored-syn-expr/src/test_helpers/example.rs index f3e539a049..a2a1f8d0dd 100644 --- a/crates/visored/visored-syn-expr/src/test_helpers/example.rs +++ b/crates/visored/visored-syn-expr/src/test_helpers/example.rs @@ -57,6 +57,7 @@ impl VdSynExprExample { input, token_annotations.iter().copied(), space_annotations.iter().copied(), + &token_storage, ); let mut builder = VdSynExprBuilder::new(db, &token_storage, &ast_arena, &annotations); let result = asts.to_vd_syn(&mut builder); diff --git a/crates/visored/visored-syn-expr/src/tests.rs b/crates/visored/visored-syn-expr/src/tests.rs index 8aebefc2f2..add60bbc61 100644 --- a/crates/visored/visored-syn-expr/src/tests.rs +++ b/crates/visored/visored-syn-expr/src/tests.rs @@ -2,6 +2,7 @@ husky_coword::jar::CowordJar, visored_zfs_ty::jar::VdZfsTypeJar, latex_ast::jar::LxAstJar, + latex_command::jar::LxCommandJar, visored_opr::jar::VdOprJar, crate::jar::VdSynExprJar )] From 23a59595ac04c7fe35b1a6d2275cc7bcc6f67550 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 17:25:59 -0700 Subject: [PATCH 51/71] save --- Cargo.lock | 2 + .../expect-files/husky_lang_jar_packages.txt | 1 + .../expect-files/husky_lang_packages.txt | 3 + crates/visored/visored-annotation/Cargo.toml | 1 + .../visored-annotation/src/annotations.rs | 33 +++++++--- .../visored-annotation/src/test_helpers.rs | 2 + .../src/test_helpers/example.rs | 62 ++++++++++++++----- .../visored-lean-transpilation/Cargo.toml | 2 + .../visored-lean-transpilation/src/tests.rs | 1 + 9 files changed, 80 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a263b7f7b..8d8bbf3dab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12550,11 +12550,13 @@ version = "0.1.0" dependencies = [ "husky-coword", "latex-ast", + "latex-command", "lean-hir-expr", "lean-opr", "lean-sem-expr", "lean-term", "salsa", + "visored-annotation", "visored-hir-expr", "visored-opr", "visored-sem-expr", diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt index a96b942c33..645545c652 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt @@ -1358,6 +1358,7 @@ "visored-zfs-ty", "husky-coword", "latex-ast", + "latex-command", "lean-sem-expr", "lean-term", "visored-sem-expr", diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 239f57eda8..7d2464c873 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4223,6 +4223,7 @@ "latex-prelude", "latex-token", "lazy_static", + "salsa", "expect-test", "husky-coword", "latex-command", @@ -4329,8 +4330,10 @@ "visored-zfs-ty", "husky-coword", "latex-ast", + "latex-command", "lean-sem-expr", "lean-term", + "visored-annotation", "visored-sem-expr", "visored-syn-expr", ], diff --git a/crates/visored/visored-annotation/Cargo.toml b/crates/visored/visored-annotation/Cargo.toml index 8e5e948d51..34310a281d 100644 --- a/crates/visored/visored-annotation/Cargo.toml +++ b/crates/visored/visored-annotation/Cargo.toml @@ -15,6 +15,7 @@ keywords.workspace = true lazy_static.workspace = true # abstractions enum-class.workspace = true +salsa.workspace = true # latex latex-prelude.workspace = true latex-token.workspace = true diff --git a/crates/visored/visored-annotation/src/annotations.rs b/crates/visored/visored-annotation/src/annotations.rs index b284d785ea..c814acc327 100644 --- a/crates/visored/visored-annotation/src/annotations.rs +++ b/crates/visored/visored-annotation/src/annotations.rs @@ -21,19 +21,22 @@ pub type VdSpaceAnnotationEntry = VdAnnotationEntry; #[derive(Debug, PartialEq, Eq, Clone)] pub struct VdAnnotations { - token_annotations: Vec, - space_annotations: Vec, + token_annotation_entries: Vec, + space_annotation_entries: Vec, + token_annotations: Vec, } impl VdAnnotations { pub fn new( - token_annotations: Vec, - space_annotations: Vec, + token_annotation_entries: Vec, + space_annotation_entries: Vec, token_storage: &LxTokenStorage, ) -> Self { + let token_annotations = collect_token_annotations(&token_annotation_entries, token_storage); Self { + token_annotation_entries, + space_annotation_entries, token_annotations, - space_annotations, } } @@ -51,15 +54,25 @@ impl VdAnnotations { ) } - pub fn token_annotations(&self) -> &[VdTokenAnnotationEntry] { - &self.token_annotations + pub fn token_annotation_entries(&self) -> &[VdTokenAnnotationEntry] { + &self.token_annotation_entries } - pub fn space_annotations(&self) -> &[VdSpaceAnnotationEntry] { - &self.space_annotations + pub fn space_annotation_entries(&self) -> &[VdSpaceAnnotationEntry] { + &self.space_annotation_entries } pub fn walker(&self) -> VdAnnotationsWalker { - VdAnnotationsWalker::new(&self.token_annotations, &self.space_annotations) + VdAnnotationsWalker::new( + &self.token_annotation_entries, + &self.space_annotation_entries, + ) } } + +fn collect_token_annotations( + token_annotation_entries: &[VdTokenAnnotationEntry], + token_storage: &LxTokenStorage, +) -> Vec { + todo!() +} diff --git a/crates/visored/visored-annotation/src/test_helpers.rs b/crates/visored/visored-annotation/src/test_helpers.rs index d4d8a94c88..19fceccd70 100644 --- a/crates/visored/visored-annotation/src/test_helpers.rs +++ b/crates/visored/visored-annotation/src/test_helpers.rs @@ -1 +1,3 @@ pub mod example; + +use crate::*; diff --git a/crates/visored/visored-annotation/src/test_helpers/example.rs b/crates/visored/visored-annotation/src/test_helpers/example.rs index 6ea365a85b..c54ae59e25 100644 --- a/crates/visored/visored-annotation/src/test_helpers/example.rs +++ b/crates/visored/visored-annotation/src/test_helpers/example.rs @@ -1,3 +1,4 @@ +use super::*; use crate::{ annotation::{ space::{LxApplyAnnotation, VdSpaceAnnotation}, @@ -21,30 +22,54 @@ impl VdAnnotationsExample { &[((&str, &str), VdTokenAnnotation)], &[((&str, &str), VdSpaceAnnotation)], )], + db: &::salsa::Db, ) -> Vec { examples .iter() .map( - |&(root_mode, input, token_annotations, space_annotations)| Self { - root_mode, - input: input.to_string(), - annotations: VdAnnotations::from_sparse( + |&(root_mode, input, token_annotations, space_annotations)| { + let mut token_storage = LxTokenStorage::default(); + let mut ast_arena = LxAstArena::default(); + let asts = parse_latex_input_into_asts( + &db, + &input, + LxMode::Math, + &mut token_storage, + &mut ast_arena, + ); + let mut ast_arena = LxAstArena::default(); + let asts = parse_latex_input_into_asts( + &db, + &input, + LxMode::Math, + &mut token_storage, + &mut ast_arena, + ); + let annotations = VdAnnotations::from_sparse( input, token_annotations.iter().copied(), space_annotations.iter().copied(), - ), + &token_storage, + ); + Self { + root_mode, + input: input.to_string(), + annotations, + } }, ) .collect() } } +use latex_ast::ast::{parse_latex_input_into_asts, LxAstArena}; use latex_prelude::mode::LxMode; +use latex_token::storage::LxTokenStorage; use lazy_static::lazy_static; -lazy_static! { - pub static ref LX_ANNOTATIONS_EXAMPLES: Vec = - VdAnnotationsExample::collect_from_sparse(&[ +pub fn lx_annotations_examples(db: &::salsa::Db) -> Vec { + VdAnnotationsExample::collect_from_sparse( + &[ (LxMode::Math, "", &[], &[]), ( LxMode::Math, @@ -52,17 +77,17 @@ lazy_static! { &[ ( ("", "x"), - VdTokenAnnotation::Variable(LxVariableAnnotation::Usage) + VdTokenAnnotation::Variable(LxVariableAnnotation::Usage), ), ( ("x", "y"), - VdTokenAnnotation::Variable(LxVariableAnnotation::Usage) + VdTokenAnnotation::Variable(LxVariableAnnotation::Usage), ), ], &[( ("x", "y"), - VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarMul) - ),] + VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarMul), + )], ), ( LxMode::Math, @@ -72,18 +97,21 @@ lazy_static! { ( ("d", "x"), VdTokenAnnotation::Variable( - LxVariableAnnotation::SingleVariableIntegralVariableDecl - ) + LxVariableAnnotation::SingleVariableIntegralVariableDecl, + ), ), ], - &[] + &[], ), - ]); + ], + db, + ) } #[test] fn latex_annotations_examples_works() { - let examples = &*LX_ANNOTATIONS_EXAMPLES; // Dereference the lazy_static + let db = &DB::default(); + let examples = lx_annotations_examples(db); expect_test::expect_file!["../../expect-files/annotations/examples.txt"] .assert_eq(&format!("{:#?}", examples)); } diff --git a/crates/visored/visored-lean-transpilation/Cargo.toml b/crates/visored/visored-lean-transpilation/Cargo.toml index f31365a53e..f63cf98f49 100644 --- a/crates/visored/visored-lean-transpilation/Cargo.toml +++ b/crates/visored/visored-lean-transpilation/Cargo.toml @@ -27,6 +27,7 @@ visored-zfs-ty.workspace = true # kernel husky-coword.workspace = true # latex +latex-command.workspace = true latex-ast.workspace = true # lean lean-term.workspace = true @@ -34,6 +35,7 @@ lean-sem-expr.workspace = true # visored visored-sem-expr = { workspace = true, features = ["test_helpers"] } visored-syn-expr = { workspace = true, features = ["test_helpers"] } +visored-annotation = { workspace = true, features = ["test_helpers"] } [lints] workspace = true diff --git a/crates/visored/visored-lean-transpilation/src/tests.rs b/crates/visored/visored-lean-transpilation/src/tests.rs index a8e8b906f7..1ff1d8db06 100644 --- a/crates/visored/visored-lean-transpilation/src/tests.rs +++ b/crates/visored/visored-lean-transpilation/src/tests.rs @@ -11,6 +11,7 @@ use crate::{builder::VdLeanTranspilationBuilder, ToLean}; #[salsa::db( husky_coword::jar::CowordJar, latex_ast::jar::LxAstJar, + latex_command::jar::LxCommandJar, lean_term::jar::LnTermJar, lean_sem_expr::jar::LnSemExprJar, lean_hir_expr::jar::LnHirExprJar, From aa5793a8063ba5592e1ae1a58143fdcb96d4da10 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 18:00:19 -0700 Subject: [PATCH 52/71] make annotations more accessible --- crates/latex/latex-token/src/storage.rs | 8 +- .../src/annotation/space.rs | 7 +- .../src/annotation/token.rs | 7 +- .../visored-annotation/src/annotations.rs | 86 ++++++--- .../src/annotations/builder/sparce.rs | 54 ++++-- .../src/annotations/walker.rs | 171 ------------------ 6 files changed, 113 insertions(+), 220 deletions(-) delete mode 100644 crates/visored/visored-annotation/src/annotations/walker.rs diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index a44cc6032d..0dc12f1215 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -14,8 +14,6 @@ pub struct LxTokenStorage { ranged_tokens: Vec<((usize, usize), TextRange, LxTokenData)>, } -/// # getters - impl std::ops::Index for LxTokenStorage { type Output = LxMathTokenData; @@ -27,7 +25,13 @@ impl std::ops::Index for LxTokenStorage { } } +/// # getters + impl LxTokenStorage { + pub fn ranged_tokens(&self) -> &[((usize, usize), TextRange, LxTokenData)] { + &self.ranged_tokens + } + pub fn whole_token_idx_range(&self) -> LxTokenIdxRange { LxTokenIdxRange::new(0..self.ranged_tokens.len()) } diff --git a/crates/visored/visored-annotation/src/annotation/space.rs b/crates/visored/visored-annotation/src/annotation/space.rs index 47221c71de..9248ffd996 100644 --- a/crates/visored/visored-annotation/src/annotation/space.rs +++ b/crates/visored/visored-annotation/src/annotation/space.rs @@ -1,9 +1,8 @@ #[enum_class::from_variants] -#[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[repr(u8)] pub enum VdSpaceAnnotation { - #[default] - Void, - Apply(LxApplyAnnotation), + Apply(LxApplyAnnotation) = 1, Sever(LxSeverAnnotation), } diff --git a/crates/visored/visored-annotation/src/annotation/token.rs b/crates/visored/visored-annotation/src/annotation/token.rs index 9efb163e0f..5934f1b064 100644 --- a/crates/visored/visored-annotation/src/annotation/token.rs +++ b/crates/visored/visored-annotation/src/annotation/token.rs @@ -1,9 +1,8 @@ #[enum_class::from_variants] -#[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[repr(u8)] pub enum VdTokenAnnotation { - #[default] - Void, - Integral(LxIntegralAnnotation), + Integral(LxIntegralAnnotation) = 1, Variable(LxVariableAnnotation), Differential, } diff --git a/crates/visored/visored-annotation/src/annotations.rs b/crates/visored/visored-annotation/src/annotations.rs index c814acc327..f2fab013a8 100644 --- a/crates/visored/visored-annotation/src/annotations.rs +++ b/crates/visored/visored-annotation/src/annotations.rs @@ -1,5 +1,4 @@ mod builder; -pub mod walker; use crate::{ annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}, @@ -7,36 +6,38 @@ use crate::{ }; use builder::sparce::collect_from_sparse_annotations; use latex_token::storage::LxTokenStorage; -use walker::VdAnnotationsWalker; #[derive(Debug, PartialEq, Eq, Clone)] -pub struct VdAnnotationEntry { +pub struct VdAnnotationRecord { pub start: usize, pub end: usize, pub annotation: A, } -pub type VdTokenAnnotationEntry = VdAnnotationEntry; -pub type VdSpaceAnnotationEntry = VdAnnotationEntry; +pub type VdTokenAnnotationRecord = VdAnnotationRecord; +pub type VdSpaceAnnotationRecord = VdAnnotationRecord; #[derive(Debug, PartialEq, Eq, Clone)] pub struct VdAnnotations { - token_annotation_entries: Vec, - space_annotation_entries: Vec, - token_annotations: Vec, + token_annotation_records: Vec, + space_annotation_records: Vec, + token_annotations: Vec>, + space_annotations: Vec>, } impl VdAnnotations { pub fn new( - token_annotation_entries: Vec, - space_annotation_entries: Vec, + token_annotation_records: Vec, + space_annotation_records: Vec, token_storage: &LxTokenStorage, ) -> Self { - let token_annotations = collect_token_annotations(&token_annotation_entries, token_storage); + let token_annotations = collect_token_annotations(&token_annotation_records, token_storage); + let space_annotations = collect_space_annotations(&space_annotation_records, token_storage); Self { - token_annotation_entries, - space_annotation_entries, + token_annotation_records, + space_annotation_records, token_annotations, + space_annotations, } } @@ -54,25 +55,56 @@ impl VdAnnotations { ) } - pub fn token_annotation_entries(&self) -> &[VdTokenAnnotationEntry] { - &self.token_annotation_entries + pub fn token_annotation_records(&self) -> &[VdTokenAnnotationRecord] { + &self.token_annotation_records } - pub fn space_annotation_entries(&self) -> &[VdSpaceAnnotationEntry] { - &self.space_annotation_entries - } - - pub fn walker(&self) -> VdAnnotationsWalker { - VdAnnotationsWalker::new( - &self.token_annotation_entries, - &self.space_annotation_entries, - ) + pub fn space_annotation_records(&self) -> &[VdSpaceAnnotationRecord] { + &self.space_annotation_records } } fn collect_token_annotations( - token_annotation_entries: &[VdTokenAnnotationEntry], + token_annotation_entries: &[VdTokenAnnotationRecord], + token_storage: &LxTokenStorage, +) -> Vec> { + let mut entry_idx = 0; + token_storage + .ranged_tokens() + .iter() + .map(|&((start, end), _, _)| { + if entry_idx < token_annotation_entries.len() + && token_annotation_entries[entry_idx].start == start + { + assert_eq!(token_annotation_entries[entry_idx].end, end); + entry_idx += 1; + Some(token_annotation_entries[entry_idx - 1].annotation) + } else { + None + } + }) + .collect() +} + +fn collect_space_annotations( + space_annotation_entries: &[VdSpaceAnnotationRecord], token_storage: &LxTokenStorage, -) -> Vec { - todo!() +) -> Vec> { + let mut entry_idx = 0; + token_storage + .ranged_tokens() + .iter() + .map(|&((start, end), _, _)| { + let annotation = if entry_idx < space_annotation_entries.len() + && space_annotation_entries[entry_idx].start == start + { + assert_eq!(space_annotation_entries[entry_idx].end, end); + entry_idx += 1; + Some(space_annotation_entries[entry_idx - 1].annotation) + } else { + None + }; + annotation + }) + .collect() } diff --git a/crates/visored/visored-annotation/src/annotations/builder/sparce.rs b/crates/visored/visored-annotation/src/annotations/builder/sparce.rs index 7611dc444c..111e387f5b 100644 --- a/crates/visored/visored-annotation/src/annotations/builder/sparce.rs +++ b/crates/visored/visored-annotation/src/annotations/builder/sparce.rs @@ -2,7 +2,7 @@ use super::*; use crate::{ annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}, annotations::{ - VdAnnotationEntry, VdAnnotations, VdSpaceAnnotationEntry, VdTokenAnnotationEntry, + VdAnnotationRecord, VdAnnotations, VdSpaceAnnotationRecord, VdTokenAnnotationRecord, }, }; use latex_ast::ast::LxAstArena; @@ -23,28 +23,28 @@ pub(crate) fn collect_from_sparse_annotations<'a>( fn collect_from_sparse_token_annotations<'a>( raw_text: &'a str, token_annotation_iter: impl Iterator, -) -> Vec { +) -> Vec { collect_from_sparse_annotations_aux(raw_text, token_annotation_iter) } fn collect_from_sparse_space_annotations<'a>( raw_text: &'a str, space_annotation_iter: impl Iterator, -) -> Vec { +) -> Vec { collect_from_sparse_annotations_aux(raw_text, space_annotation_iter) } fn collect_from_sparse_annotations_aux<'a, A>( raw_text: &'a str, annotation_iter: impl Iterator, -) -> Vec> { +) -> Vec> { let mut annotations = Vec::new(); for ((prev_s, token_s), annotation) in annotation_iter { let start = prev_s.len(); let end = prev_s.len() + token_s.len(); assert_eq!(&raw_text[..start], prev_s); assert_eq!(&raw_text[start..end], token_s); - annotations.push(VdAnnotationEntry { + annotations.push(VdAnnotationRecord { start, end, annotation, @@ -113,27 +113,27 @@ mod tests { expect![[r#" VdAnnotations { - token_annotations: [ - VdAnnotationEntry { + token_annotation_records: [ + VdAnnotationRecord { start: 0, end: 4, annotation: Integral( SingleVariableIndefiniteIntegralOverReal, ), }, - VdAnnotationEntry { + VdAnnotationRecord { start: 5, end: 6, annotation: Variable( Usage, ), }, - VdAnnotationEntry { + VdAnnotationRecord { start: 6, end: 7, annotation: Differential, }, - VdAnnotationEntry { + VdAnnotationRecord { start: 7, end: 8, annotation: Variable( @@ -141,8 +141,8 @@ mod tests { ), }, ], - space_annotations: [ - VdAnnotationEntry { + space_annotation_records: [ + VdAnnotationRecord { start: 6, end: 7, annotation: Apply( @@ -150,6 +150,36 @@ mod tests { ), }, ], + token_annotations: [ + Some( + Integral( + SingleVariableIndefiniteIntegralOverReal, + ), + ), + Some( + Variable( + Usage, + ), + ), + Some( + Differential, + ), + Some( + Variable( + SingleVariableIntegralVariableDecl, + ), + ), + ], + space_annotations: [ + None, + None, + Some( + Apply( + ScalarDifferentialFormMul, + ), + ), + None, + ], } "#]] .assert_debug_eq(&result); diff --git a/crates/visored/visored-annotation/src/annotations/walker.rs b/crates/visored/visored-annotation/src/annotations/walker.rs deleted file mode 100644 index dda7658f81..0000000000 --- a/crates/visored/visored-annotation/src/annotations/walker.rs +++ /dev/null @@ -1,171 +0,0 @@ -use super::*; -use crate::annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}; - -pub struct VdAnnotationsWalker<'a> { - token_annotations: &'a [VdTokenAnnotationEntry], - space_annotations: &'a [VdSpaceAnnotationEntry], - next_token_annotation_index: usize, - next_space_annotation_index: usize, -} - -impl<'a> VdAnnotationsWalker<'a> { - pub fn new( - token_annotations: &'a [VdTokenAnnotationEntry], - space_annotations: &'a [VdSpaceAnnotationEntry], - ) -> Self { - Self { - token_annotations, - space_annotations, - next_token_annotation_index: 0, - next_space_annotation_index: 0, - } - } -} - -impl<'a> VdAnnotationsWalker<'a> { - pub fn next(&mut self, start: usize, end: usize) -> (VdTokenAnnotation, VdSpaceAnnotation) { - ( - self.next_token_annotation(start, end), - self.next_space_annotation(start, end), - ) - } - - fn next_token_annotation(&mut self, start: usize, end: usize) -> VdTokenAnnotation { - next_annotation_aux( - start, - end, - self.token_annotations, - &mut self.next_token_annotation_index, - ) - } - - fn next_space_annotation(&mut self, start: usize, end: usize) -> VdSpaceAnnotation { - next_annotation_aux( - start, - end, - self.space_annotations, - &mut self.next_space_annotation_index, - ) - } -} - -fn next_annotation_aux( - start: usize, - end: usize, - token_annotations: &[VdAnnotationEntry], - next_token_annotation_index: &mut usize, -) -> A { - if *next_token_annotation_index >= token_annotations.len() { - return A::default(); - } - let start1 = token_annotations[*next_token_annotation_index].start; - if start1 > start { - return A::default(); - } else if start1 == start { - use husky_print_utils::p; - p!(token_annotations[*next_token_annotation_index]); - assert_eq!(end, token_annotations[*next_token_annotation_index].end); - let result = token_annotations[*next_token_annotation_index].annotation; - *next_token_annotation_index += 1; - result - } else { - panic!("Invalid annotation sequence: annotation start ({start1}) is before token start ({start})"); - } -} - -#[cfg(test)] -mod tests { - use latex_ast::ast::{parse_latex_input_into_asts, LxAstArena}; - use latex_prelude::mode::LxMode; - - use super::*; - use crate::annotation::token::{LxIntegralAnnotation, LxVariableAnnotation, VdTokenAnnotation}; - use crate::{ - annotation::space::{LxApplyAnnotation, VdSpaceAnnotation}, - annotations::VdAnnotations, - }; - - fn setup_test_data() -> VdAnnotations { - let db = DB::default(); - - let input = "\\int xdx".to_string(); - - let token_annotations = vec![ - ( - ("", "\\int"), - VdTokenAnnotation::Integral( - LxIntegralAnnotation::SingleVariableIndefiniteIntegralOverReal, - ), - ), - ( - ("\\int ", "x"), - VdTokenAnnotation::Variable(LxVariableAnnotation::Usage), - ), - (("\\int x", "d"), VdTokenAnnotation::Differential), - ( - ("\\int xd", "x"), - VdTokenAnnotation::Variable( - LxVariableAnnotation::SingleVariableIntegralVariableDecl, - ), - ), - ]; - - let space_annotations = vec![( - ("\\int x", "d"), - VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul), - )]; - - let mut token_storage = LxTokenStorage::default(); - let mut ast_arena = LxAstArena::default(); - let asts = parse_latex_input_into_asts( - &db, - &input, - LxMode::Math, - &mut token_storage, - &mut ast_arena, - ); - VdAnnotations::from_sparse( - &input, - token_annotations.into_iter(), - space_annotations.into_iter(), - &token_storage, - ) - } - - #[test] - fn test_walker_next_token_and_space_annotations() { - let annotations = setup_test_data(); - let mut walker = annotations.walker(); - - // Test at offset 0 ("\\int") - assert_eq!( - walker.next(0, 4), - ( - VdTokenAnnotation::Integral( - LxIntegralAnnotation::SingleVariableIndefiniteIntegralOverReal - ), - VdSpaceAnnotation::default() - ) - ); - - // Test at offset 5 ("x" after "\\int ") - assert_eq!( - walker.next(5, 6), - ( - VdTokenAnnotation::Variable(LxVariableAnnotation::Usage), - VdSpaceAnnotation::Void - ) - ); - - // Test at offset 6 ("d" after "x") - assert_eq!( - walker.next(6, 7), - ( - VdTokenAnnotation::Differential, - VdSpaceAnnotation::Apply(LxApplyAnnotation::ScalarDifferentialFormMul) - ) - ); - } - - // Additional tests for next_token_annotation and next_space_annotation can be added here -} From ea39487b59de595862810e417568720564f492ab Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 18:01:02 -0700 Subject: [PATCH 53/71] save --- crates/visored/visored-annotation/src/annotations.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/visored/visored-annotation/src/annotations.rs b/crates/visored/visored-annotation/src/annotations.rs index f2fab013a8..26c3b12484 100644 --- a/crates/visored/visored-annotation/src/annotations.rs +++ b/crates/visored/visored-annotation/src/annotations.rs @@ -21,7 +21,9 @@ pub type VdSpaceAnnotationRecord = VdAnnotationRecord; pub struct VdAnnotations { token_annotation_records: Vec, space_annotation_records: Vec, + /// the ith element is the annotation of the ith token token_annotations: Vec>, + /// the ith element is the annotation of the space before the ith token space_annotations: Vec>, } From 60984c7b6b17d58fe64c1ed9e8ae38db53cdc94b Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 18:02:19 -0700 Subject: [PATCH 54/71] make annotations more accessible --- crates/latex/latex-token/src/idx.rs | 2 +- crates/visored/visored-annotation/src/annotations.rs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/latex/latex-token/src/idx.rs b/crates/latex/latex-token/src/idx.rs index a011ad00ff..c22c330810 100644 --- a/crates/latex/latex-token/src/idx.rs +++ b/crates/latex/latex-token/src/idx.rs @@ -54,7 +54,7 @@ impl LxTokenIdx { Self(index.into()) } - pub(crate) fn index(self) -> usize { + pub fn index(self) -> usize { self.0.index() } } diff --git a/crates/visored/visored-annotation/src/annotations.rs b/crates/visored/visored-annotation/src/annotations.rs index 26c3b12484..3cb1c07f7a 100644 --- a/crates/visored/visored-annotation/src/annotations.rs +++ b/crates/visored/visored-annotation/src/annotations.rs @@ -5,7 +5,7 @@ use crate::{ *, }; use builder::sparce::collect_from_sparse_annotations; -use latex_token::storage::LxTokenStorage; +use latex_token::{idx::LxTokenIdx, storage::LxTokenStorage}; #[derive(Debug, PartialEq, Eq, Clone)] pub struct VdAnnotationRecord { @@ -64,6 +64,14 @@ impl VdAnnotations { pub fn space_annotation_records(&self) -> &[VdSpaceAnnotationRecord] { &self.space_annotation_records } + + pub fn token_annotation(&self, idx: LxTokenIdx) -> Option { + self.token_annotations[idx.index()] + } + + pub fn preceding_space_annotation(&self, idx: LxTokenIdx) -> Option { + self.space_annotations[idx.index()] + } } fn collect_token_annotations( From 99d84a75d465afce2811bb5d16fb273f01e50716 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 19:21:20 -0700 Subject: [PATCH 55/71] save --- crates/visored/visored-syn-expr/src/error.rs | 14 + crates/visored/visored-syn-expr/src/expr.rs | 18 +- crates/visored/visored-syn-expr/src/parser.rs | 35 ++ .../visored-syn-expr/src/parser/accept.rs | 443 ++++++++++++++++++ .../src/parser/disambiguate_token.rs | 11 +- .../visored-syn-expr/src/parser/expr_stack.rs | 228 ++++++++- .../src/parser/incomplete_expr.rs | 14 +- 7 files changed, 748 insertions(+), 15 deletions(-) diff --git a/crates/visored/visored-syn-expr/src/error.rs b/crates/visored/visored-syn-expr/src/error.rs index f20296bc5b..673ee09e66 100644 --- a/crates/visored/visored-syn-expr/src/error.rs +++ b/crates/visored/visored-syn-expr/src/error.rs @@ -1,4 +1,8 @@ +use crate::expr::VdSynExprIdx; +use either::Either; +use latex_token::idx::LxTokenIdx; use thiserror::Error; +use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr}; #[derive(Debug, Error, PartialEq, Eq)] pub enum VdSynExprError { @@ -14,6 +18,16 @@ pub enum OriginalVdSynExprError { Empty, #[error("todo")] Todo, + #[error("no right operand for binary operator")] + NoRightOperandForBinaryOperator { + opr: Either, + }, + #[error("no operand for prefix operator")] + NoOperandForPrefixOperator { + opr: Either, + }, + #[error("unterminated list")] + UnterminatedList { bra_token_idx: LxTokenIdx }, } #[derive(Debug, Error, PartialEq, Eq)] diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index a3d4b2b163..14e7e4f321 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -109,6 +109,7 @@ impl VdSynExprData { } pub enum VdSynExprClass { + Atom, Prefix, Suffix, Separator, @@ -122,7 +123,8 @@ impl ToVdSyn for LxMathAstIdxRange { VdSynExprAstRange::Asts(self.into()), ) } else { - todo!() + let parser = builder.parser(); + parser.parse_asts(self) } } } @@ -171,13 +173,13 @@ mod tests { "\n" "#]], ); - // t( - // "1", - // &[], - // &[], - // &expect![[r#" + t( + "1", + &[], + &[], + &expect![[r#" - // "#]], - // ); + "#]], + ); } } diff --git a/crates/visored/visored-syn-expr/src/parser.rs b/crates/visored/visored-syn-expr/src/parser.rs index 5dcb297966..b3ee887b7f 100644 --- a/crates/visored/visored-syn-expr/src/parser.rs +++ b/crates/visored/visored-syn-expr/src/parser.rs @@ -5,6 +5,10 @@ mod env; mod expr_stack; pub(crate) mod incomplete_expr; +use expr::VdSynExprIdx; +use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; +use range::VdSynExprAstRange; + use self::expr_stack::VdSynExprStack; use crate::builder::VdSynExprBuilder; use crate::*; @@ -13,3 +17,34 @@ pub struct VdSynExprParser<'a, 'db> { builder: &'a mut VdSynExprBuilder<'db>, stack: VdSynExprStack, } + +impl<'a, 'db> VdSynExprParser<'a, 'db> { + fn new(builder: &'a mut VdSynExprBuilder<'db>) -> Self { + Self { + builder, + stack: Default::default(), + } + } +} + +impl<'db> VdSynExprBuilder<'db> { + pub fn parser(&mut self) -> VdSynExprParser<'_, 'db> { + VdSynExprParser::new(self) + } +} + +impl<'a, 'db> VdSynExprParser<'a, 'db> { + pub fn parse_asts(mut self, asts: LxMathAstIdxRange) -> VdSynExprIdx { + for ast in asts { + self.parse_ast(ast); + } + let Self { builder, stack } = self; + let data = stack.finish(); + builder.alloc_expr(data, VdSynExprAstRange::Asts(asts.into())) + } + + fn parse_ast(&mut self, ast: LxMathAstIdx) { + let token = self.disambiguate_token(ast); + self.stack.accept(token); + } +} diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs index 8b13789179..1564215a85 100644 --- a/crates/visored/visored-syn-expr/src/parser/accept.rs +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -1 +1,444 @@ +use super::*; +use disambiguate_token::DisambiguatedToken; +use either::*; +use expr::{VdSynExprClass, VdSynExprData}; +use incomplete_expr::{IncompleteSeparatedListOpr, IncompleteVdSynExprData}; +use latex_token::idx::LxTokenIdx; +use smallvec::smallvec; +use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}; +impl<'a, 'db> VdSynExprParser<'a, 'db> { + pub(crate) fn accept_token(&mut self, token: DisambiguatedToken) { + match token { + DisambiguatedToken::Expr(expr, class) => match class { + VdSynExprClass::Atom => self.accept_atom(expr), + VdSynExprClass::Prefix => todo!(), + VdSynExprClass::Suffix => todo!(), + VdSynExprClass::Separator => todo!(), + }, + DisambiguatedToken::Opr(opr) => self.accept_opr(opr), + DisambiguatedToken::Separator(sep) => self.accept_separator(sep), + } + } + + fn accept_list_end(&mut self, ket: Delimiter, ket_regional_token_idx: RegionalTokenIdx) { + self.reduce(Precedence::ListItem); + let last_incomplete_expr = self.take_last_incomplete_expr().unwrap(); + match last_incomplete_expr { + IncompleteSynExprData::CommaList { + opr, + bra, + bra_regional_token_idx, + mut items, + } => { + if bra != ket { + todo!() + } + self.take_complete_and_push_to_top(|this, finished_expr| { + if let Some(expr) = finished_expr { + items.push(SynCommaListItem::new( + this.context_mut().alloc_expr(expr), + None, + )) + } + match opr { + IncompleteCommaListOpr::UnitOrDelimiteredOrNewTuple => match items.last() { + None => SynExprData::Unit { + lpar_regional_token_idx: bra_regional_token_idx, + rpar_regional_token_idx: ket_regional_token_idx, + }, + Some(last_item) => { + if items.len() == 1 + && last_item.comma_regional_token_idx().is_none() + { + SynExprData::Delimitered { + lpar_regional_token_idx: bra_regional_token_idx, + item: last_item.syn_expr_idx(), + rpar_regional_token_idx: ket_regional_token_idx, + } + } else { + SynExprData::NewTuple { + lpar_regional_token_idx: bra_regional_token_idx, + items, + rpar_regional_token_idx: ket_regional_token_idx, + } + } + } + } + .into(), + IncompleteCommaListOpr::Index { owner } => { + SynExprData::IndexOrCompositionWithList { + owner, + lbox_regional_token_idx: bra_regional_token_idx, + items, + rbox_regional_token_idx: ket_regional_token_idx, + } + .into() + } + IncompleteCommaListOpr::BoxList => SynExprData::List { + lbox_regional_token_idx: bra_regional_token_idx, + items, + rbox_regional_token_idx: ket_regional_token_idx, + } + .into(), + IncompleteCommaListOpr::BoxColonList { + colon_regional_token_idx, + } => SynExprData::BoxColonList { + lbox_regional_token_idx: bra_regional_token_idx, + colon_regional_token_idx, + items, + rbox_regional_token_idx: ket_regional_token_idx, + } + .into(), + IncompleteCommaListOpr::FunctionApplicationOrCall { function } => { + // ad hoc + let generic_arguments: Option = None; + SynExprData::FunctionApplicationOrCall { + function, + template_arguments: generic_arguments, + lpar_regional_token_idx: bra_regional_token_idx, + items, + rpar_regional_token_idx: ket_regional_token_idx, + } + .into() + } + IncompleteCommaListOpr::MethodInstantiation { .. } => { + todo!() + } + IncompleteCommaListOpr::MethodApplicationOrCall { + self_expr, + dot_regional_token_idx, + ident_token, + template_arguments, + } => SynExprData::MethodApplicationOrCall { + self_argument: self_expr, + dot_regional_token_idx, + ident_token, + template_arguments, + lpar_regional_token_idx: bra_regional_token_idx, + items, + rpar_regional_token_idx: ket_regional_token_idx, + } + .into(), + IncompleteCommaListOpr::TemplateInstantiation { template } => { + SynExprData::TemplateInstantiation { + template, + template_arguments: SynTemplateArguments::new( + bra_regional_token_idx, + items, + ket_regional_token_idx, + ), + } + .into() + } + IncompleteCommaListOpr::RitchieArguments { + ritchie_kind_regional_token_idx, + ritchie_kind, + lpar_token, + } => match this.try_parse_option::() { + Ok(Some(light_arrow_token)) => IncompleteSynExprData::Ritchie { + ritchie_kind_regional_token_idx, + ritchie_kind, + lpar_token, + argument_tys: items, + rpar_regional_token_idx: ket_regional_token_idx, + light_arrow_token, + } + .into(), + Ok(None) => todo!(), + Err(_) => todo!(), + }, + } + }) + } + IncompleteSynExprData::CallList { + opr, + lpar_regional_token_idx, + items, + } => match opr { + IncompleteCallListOpr::FunctionCall { + function, + generic_arguments, + } => self.set_complete_expr(SynExprData::FunctionCall { + function, + template_arguments: generic_arguments, + lpar_regional_token_idx, + items, + rpar_regional_token_idx: ket_regional_token_idx, + }), + IncompleteCallListOpr::MethodCall { .. } => todo!(), + }, + _ => { + p!(last_incomplete_expr); + // p!(self.context.path.debug(self.db())); + p!(ket_regional_token_idx); + todo!() + } + } + } + + fn accept_atom(&mut self, atom: SynExprData) { + self.push_top_syn_expr(atom.into()) + } + + fn accept_opr(&mut self, opr: VdOpr) { + match opr { + VdOpr::Binary(opr) => todo!(), + VdOpr::Prefix(opr) => self.accept_prefix_opr(Left(opr)), + VdOpr::Suffix(opr) => self.accept_suffix_opr(Left(opr)), + } + } + + fn accept_prefix_opr(&mut self, opr: Either) { + self.push_top_syn_expr(IncompleteVdSynExprData::Prefix { opr }.into()) + } + + fn accept_suffix_opr(&mut self, opr: Either) { + self.take_complete_and_push_to_top(|slf, top_expr| match top_expr { + Some(expr) => VdSynExprData::Suffix { + opd: slf.builder.alloc_expr(expr, todo!()), + opr, + } + .into(), + None => todo!(), + }) + } + + fn accept_dot_opr(&mut self, dot_regional_token_idx: RegionalTokenIdx) { + self.take_complete_and_push_to_top(|this, finished_expr| match finished_expr { + Some(self_expr) => { + let self_expr = this.context_mut().alloc_expr(self_expr); + match this.try_parse_option::() { + Ok(Some(ident_token)) => match this.try_parse_option::() { + Ok(Some(lpar)) => IncompleteSynExprData::CommaList { + opr: IncompleteCommaListOpr::MethodApplicationOrCall { + self_expr, + dot_regional_token_idx, + ident_token, + template_arguments: None, + }, + bra: Delimiter::Par, + bra_regional_token_idx: lpar.regional_token_idx(), + items: smallvec![], + } + .into(), + Ok(None) => match this.try_parse_option::() { + Ok(Some(langle)) => IncompleteSynExprData::CommaList { + opr: IncompleteCommaListOpr::MethodInstantiation { + self_expr, + dot_regional_token_idx, + ident_token, + }, + bra: Delimiter::TurboFish, + bra_regional_token_idx: langle.regional_token_idx(), + items: smallvec![], + } + .into(), + Ok(None) => SynExprData::Field { + owner: self_expr, + dot_regional_token_idx, + ident_token, + } + .into(), + Err(_) => todo!(), + }, + Err(e) => { + p!(e); + todo!() + } + }, + _ => SynExprData::Err( + OriginalSynExprError::ExpectedIdentAfterDot { + dot_regional_token_idx, + } + .into(), + ) + .into(), + } + } + None => SynExprData::Err( + OriginalSynExprError::ExpectedExprBeforeDot { + dot_regional_token_idx, + } + .into(), + ) + .into(), + }) + } + + fn accept_comma(&mut self, comma_regional_token_idx: RegionalTokenIdx) { + match self.take_complete_expr() { + Some(item) => { + let item = self.context_mut().alloc_expr(item); + match self.last_incomplete_expr_mut() { + Some(expr) => match expr { + IncompleteSynExprData::CommaList { + opr: _, + bra: _, + bra_regional_token_idx: _, + items, + } => { + items.push(SynCommaListItem::new(item, Some(comma_regional_token_idx))) + } + IncompleteSynExprData::CallList { items, .. } => items.push( + SynSimpleOrVariadicCallListItem::new( + item, + CallListSeparator::Comma(comma_regional_token_idx), + ) + .into(), + ), + _ => unreachable!(), + }, + None => unreachable!(), + } + } + None => match self.last_incomplete_expr_mut() { + Some(expr) => match expr { + IncompleteSynExprData::CommaList { + opr: _, + bra: _, + bra_regional_token_idx: _, + items: _, + } => todo!(), + IncompleteSynExprData::CallList { items, .. } => match items.last_mut() { + Some(last_item) => match last_item.separator() { + CallListSeparator::None => last_item + .set_separator(CallListSeparator::Comma(comma_regional_token_idx)), + CallListSeparator::Comma(_) => todo!(), + CallListSeparator::Semicolon(_) => todo!(), + }, + None => todo!(), + }, + _ => unreachable!(), + }, + None => unreachable!(), + }, + } + } + + fn accept_be_pattern(&mut self, be_regional_token_idx: RegionalTokenIdx) { + self.reduce(Precedence::Be); + let src = self.take_complete_expr().unwrap_or(SynExprData::Err( + OriginalSynExprError::ExpectedItemBeforeBe { + be_regional_token_idx, + } + .into(), + )); + let src = self.context_mut().alloc_expr(src); + let end = match self.env() { + Some(env) => match env { + ExprEnvironment::TypeBeforeEq => todo!(), + ExprEnvironment::WithinDelimiteredParameterList(_) => todo!(), + ExprEnvironment::Condition(end) => end, + }, + None => todo!(), + }; + let expr = SynExprData::Be { + src, + be_regional_token_idx, + target: self.parse_be_variables_pattern_expected(end), + }; + self.push_top_syn_expr(expr.into()) + } + + fn accept_binary_opr(&mut self, binary: Either) { + self.reduce(binary.precedence()); + let lopd = self.take_complete_expr().unwrap_or(VdSynExprData::Err( + OriginalSynExprError::NoLeftOperandForBinaryOperator { + binary_regional_token_idx, + } + .into(), + )); + let lopd = self.builder.alloc_expr(lopd, todo!()); + let unfinished_expr = IncompleteVdSynExprData::Binary { + lopd, + punctuation: binary, + punctuation_regional_token_idx: binary_regional_token_idx, + }; + self.push_top_syn_expr(unfinished_expr.into()) + } + + fn accept_list_start(&mut self, bra: Delimiter, bra_regional_token_idx: RegionalTokenIdx) { + self.reduce(Precedence::Application); + if bra == Delimiter::Vert { + let lvert = bra_regional_token_idx; + let closure = self.parse_closure(bra_regional_token_idx); + self.push_top_syn_expr(closure.into()) + } else { + self.take_complete_and_push_to_top(|parser, finished_expr| -> TopSynExpr { + let finished_expr = finished_expr.map(|expr| parser.context_mut().alloc_expr(expr)); + match bra { + Delimiter::Par => match finished_expr { + Some(function) => IncompleteSynExprData::CommaList { + opr: IncompleteCommaListOpr::FunctionApplicationOrCall { function }, + bra, + bra_regional_token_idx, + items: smallvec![], + } + .into(), + None => IncompleteSynExprData::CommaList { + opr: IncompleteCommaListOpr::UnitOrDelimiteredOrNewTuple, + bra, + bra_regional_token_idx, + items: smallvec![], + } + .into(), + }, + Delimiter::Box => IncompleteSynExprData::CommaList { + opr: match finished_expr { + Some(finished_expr) => IncompleteCommaListOpr::Index { + owner: finished_expr, + }, + None => IncompleteCommaListOpr::BoxList, + }, + bra, + bra_regional_token_idx, + items: smallvec![], + } + .into(), + Delimiter::TurboFish => match finished_expr { + Some(template) => IncompleteSynExprData::CommaList { + opr: IncompleteCommaListOpr::TemplateInstantiation { template }, + bra, + bra_regional_token_idx, + items: smallvec![], + } + .into(), + None => todo!(), + }, + Delimiter::NestedCurl => todo!(), + Delimiter::InlineCurl => SynExprData::Err( + OriginalSynExprError::UnexpectedInlineLcurl(bra_regional_token_idx).into(), + ) + .into(), + Delimiter::Vert => { + unreachable!("Handled already") + } + Delimiter::HtmlAngle => { + let function_ident = match parser.try_parse_expected( + OriginalSynExprError::ExpectedFunctionIdentAfterOpeningHtmlBra, + ) { + Ok(function_ident) => function_ident, + Err(e) => return SynExprData::Err(e).into(), + }; + let arguments = match parse_consecutive_vec_map(parser) { + Ok(arguments) => arguments, + Err(e) => return SynExprData::Err(e).into(), + }; + match parser.try_parse_option::() { + Ok(Some(empty_html_ket)) => SynExprData::EmptyHtmlTag { + empty_html_bra_idx: bra_regional_token_idx, + function_ident, + arguments, + empty_html_ket, + } + .into(), + Ok(None) => todo!(), + Err(_) => todo!(), + } + } + } + }) + } + } +} diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs index 7af11c4954..d0d43ba4b4 100644 --- a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -6,7 +6,10 @@ use visored_opr::{ separator::VdSeparator, }; -use super::expr::{VdSynExprClass, VdSynExprIdx}; +use super::{ + expr::{VdSynExprClass, VdSynExprIdx}, + VdSynExprParser, +}; pub struct DisambiguatedMathAst { ast: LxMathAstIdx, @@ -20,3 +23,9 @@ pub enum DisambiguatedToken { LeftDelimiter(VdLeftDelimiter), RightDelimiter(VdRightDelimiter), } + +impl<'a, 'db> VdSynExprParser<'a, 'db> { + pub fn disambiguate_token(&mut self, ast: LxMathAstIdx) -> DisambiguatedToken { + todo!() + } +} diff --git a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs index c666decbfc..5d529bd667 100644 --- a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs +++ b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs @@ -1,7 +1,231 @@ -use super::{expr::VdSynExprData, incomplete_expr::IncompleteVdSynExprData}; -use visored_opr::precedence::VdPrecedence; +use super::{ + error::{OriginalVdSynExprError, VdSynExprResult}, + expr::{VdSynExprData, VdSynExprIdx}, + incomplete_expr::IncompleteVdSynExprData, + VdSynExprParser, +}; +use visored_opr::{delimiter::VdLeftDelimiter, precedence::VdPrecedence}; +#[derive(Default)] pub(crate) struct VdSynExprStack { incomplete_exprs: Vec<(IncompleteVdSynExprData, VdPrecedence)>, complete_expr: Option, } + +impl VdSynExprStack { + pub fn finish(self) -> VdSynExprData { + assert!(self.complete_expr.is_some()); + assert!(self.incomplete_exprs.is_empty()); + self.complete_expr.unwrap() + } +} + +pub(super) enum TopVdSynExpr { + Unfinished(IncompleteVdSynExprData), + Finished(VdSynExprData), +} + +pub(super) enum TopExprRef<'a> { + Incomplete(&'a IncompleteVdSynExprData), + Finished(&'a VdSynExprData), + None, +} + +impl From> for TopVdSynExpr { + fn from(result: VdSynExprResult) -> Self { + Self::Finished(match result { + Ok(data) => data, + Err(e) => VdSynExprData::Err(e), + }) + } +} + +impl From for TopVdSynExpr { + fn from(v: VdSynExprData) -> Self { + Self::Finished(v) + } +} + +impl From for TopVdSynExpr { + fn from(v: IncompleteVdSynExprData) -> Self { + Self::Unfinished(v) + } +} + +impl VdSynExprStack { + pub(super) fn prev_unfinished_expr_precedence(&self) -> Option { + self.incomplete_exprs + .last() + .map(|(_, precedence)| *precedence) + } +} + +impl<'a, 'db> VdSynExprParser<'a, 'db> { + pub(super) fn complete_expr(&self) -> Option<&VdSynExprData> { + self.stack.complete_expr.as_ref() + } + + pub(super) fn incomplete_exprs(&self) -> &[(IncompleteVdSynExprData, VdPrecedence)] { + &self.stack.incomplete_exprs + } + + pub(super) fn take_last_incomplete_expr(&mut self) -> Option { + self.stack.incomplete_exprs.pop().map(|(expr, _)| expr) + } + + fn push_unfinished_expr(&mut self, incomplete_expr: IncompleteVdSynExprData) { + assert!(self.stack.complete_expr.is_none()); + let precedence = incomplete_expr.precedence(); + self.stack + .incomplete_exprs + .push((incomplete_expr, precedence)) + } + + pub(super) fn last_incomplete_expr(&self) -> Option<&IncompleteVdSynExprData> { + self.stack.incomplete_exprs.last().map(|(opr, _)| opr) + } + + pub(super) fn last_incomplete_expr_mut(&mut self) -> Option<&mut IncompleteVdSynExprData> { + self.stack.incomplete_exprs.last_mut().map(|(opr, _)| opr) + } + + /// make `top_expr` the top expression. + /// - if there is already a finished expression, interpret it as a function, + /// and `top_expr` as an argument; + /// - otherwise just adds it in the trivial way + pub(super) fn push_top_syn_expr(&mut self, top_expr: TopVdSynExpr) { + // this is for guaranteeing that application is left associative + if self.complete_expr().is_some() { + self.reduce(VdPrecedence::APPLICATION) + }; + if let Some(function) = self.take_complete_expr() { + todo!() + // self.push_unfinished_expr(IncompleteVdSynExprData::Application { function }); + } + match top_expr { + TopVdSynExpr::Unfinished(unfinished_expr) => self.push_unfinished_expr(unfinished_expr), + TopVdSynExpr::Finished(finished_expr) => self.stack.complete_expr = Some(finished_expr), + } + } + + /// if there's no need for the information of unfinished expressions, call `finished_expr` would be faster + pub(super) fn top_expr<'d>(&'d self) -> TopExprRef<'d> { + if let Some(ref finished_expr) = self.stack.complete_expr { + TopExprRef::Finished(finished_expr) + } else if let Some((unfinished_expr, _)) = self.stack.incomplete_exprs.last() { + TopExprRef::Incomplete(unfinished_expr) + } else { + TopExprRef::None + } + } + + pub(super) fn take_complete_expr(&mut self) -> Option { + std::mem::take(&mut self.stack.complete_expr) + } + + pub(super) fn set_complete_expr(&mut self, expr: VdSynExprData) { + debug_assert!(self.complete_expr().is_none()); + self.stack.complete_expr = Some(expr) + } + + fn reduce_aux( + &mut self, + f: impl Fn(&mut Self, Option, IncompleteVdSynExprData) -> TopVdSynExpr, + ) { + let complete_expr = self.take_complete_expr(); + let Some((incomplete_expr, _)) = self.stack.incomplete_exprs.pop() else { + unreachable!() + }; + let top_expr = f(self, complete_expr, incomplete_expr); + self.push_top_syn_expr(top_expr) + } + + pub(super) fn reduce(&mut self, next_precedence: VdPrecedence) { + while let Some(prev_precedence) = self.stack.prev_unfinished_expr_precedence() { + if prev_precedence < next_precedence { + break; + } + // // curry is right associative + // if prev_precedence == VdPrecedence::Curry && next_precedence == VdPrecedence::Curry { + // break; + // } + match self.stack.incomplete_exprs.pop().unwrap().0 { + IncompleteVdSynExprData::Binary { lopd, opr } => { + let finished_expr = self.take_complete_expr(); + self.stack.complete_expr = Some(match finished_expr { + Some(ropd) => VdSynExprData::Binary { + lopd, + opr, + ropd: self.builder.alloc_expr(ropd, todo!()), + }, + None => VdSynExprData::Err( + OriginalVdSynExprError::NoRightOperandForBinaryOperator { opr }.into(), + ), + }) + } + IncompleteVdSynExprData::Prefix { opr } => { + let finished_expr = self.take_complete_expr(); + self.stack.complete_expr = Some(match finished_expr { + Some(opd) => VdSynExprData::Prefix { + opr, + opd: self.builder.alloc_expr(opd, todo!()), + }, + None => VdSynExprData::Err( + OriginalVdSynExprError::NoOperandForPrefixOperator { opr }.into(), + ), + }) + } + IncompleteVdSynExprData::SeparatedList { bra_token_idx, .. } => { + self.stack.complete_expr = Some(VdSynExprData::Err( + OriginalVdSynExprError::UnterminatedList { bra_token_idx }.into(), + )) + } + IncompleteVdSynExprData::CallList { .. } => todo!(), + } + } + } + + /// use this when the incoming token might change the nature of the top expression + pub(super) fn take_complete_and_push_to_top( + &mut self, + f: impl FnOnce(&mut Self, Option) -> TopVdSynExpr, + ) { + let complete_expr = self.take_complete_expr(); + let top_expr = f(self, complete_expr); + self.push_top_syn_expr(top_expr) + } + + pub(super) fn finish_batch(&mut self) -> Option { + assert!(self.stack.incomplete_exprs.len() == 0); + std::mem::take(&mut self.stack.complete_expr) + .map(|expr| self.builder.alloc_expr(expr, todo!())) + } + + pub(super) fn last_bra(&self) -> Option { + for (unfinished_expr, _) in self.stack.incomplete_exprs.iter().rev() { + match unfinished_expr { + IncompleteVdSynExprData::SeparatedList { bra, .. } => return Some(*bra), + IncompleteVdSynExprData::CallList { .. } => todo!(), + // return Some(Delimiter::Par), + _ => (), + } + } + None + } + + pub(super) fn last_two_bras(&self) -> Vec { + let mut bras = vec![]; + for (unfinished_expr, _) in self.stack.incomplete_exprs.iter().rev() { + match unfinished_expr { + IncompleteVdSynExprData::SeparatedList { bra, .. } => { + bras.push(*bra); + if bras.len() >= 2 { + return bras; + } + } + _ => (), + } + } + bras + } +} diff --git a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs index 289efbfe4b..0078f6d5a7 100644 --- a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs +++ b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs @@ -7,16 +7,21 @@ pub(super) use self::separated_list::*; use super::*; use crate::expr::VdSynExprData; use expr::list_item::VdSynSeparatedListItem; -use visored_opr::{delimiter::VdLeftDelimiter, precedence::VdPrecedence}; +use latex_token::idx::LxTokenIdx; +use visored_opr::{ + delimiter::VdLeftDelimiter, + opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr}, + precedence::VdPrecedence, +}; #[derive(Debug, PartialEq, Eq)] pub(super) enum IncompleteVdSynExprData { Binary { - lopd: VdSynExprData, - opr: VdSynExprData, + lopd: VdSynExprIdx, + opr: Either, }, Prefix { - opr: VdSynExprData, + opr: Either, }, /// list separated by commas /// ```husky @@ -25,6 +30,7 @@ pub(super) enum IncompleteVdSynExprData { SeparatedList { opr: IncompleteSeparatedListOpr, bra: VdLeftDelimiter, + bra_token_idx: LxTokenIdx, items: SmallVec<[VdSynSeparatedListItem; 4]>, }, /// call list includes more separators like `;` From 0776e5fd7007f4ae234a4211df8b64df1fc82d29 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 19:24:23 -0700 Subject: [PATCH 56/71] save --- .../visored-syn-expr/src/parser/accept.rs | 249 +++--------------- 1 file changed, 41 insertions(+), 208 deletions(-) diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs index 1564215a85..f41a2c3aef 100644 --- a/crates/visored/visored-syn-expr/src/parser/accept.rs +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -1,11 +1,16 @@ use super::*; use disambiguate_token::DisambiguatedToken; use either::*; -use expr::{VdSynExprClass, VdSynExprData}; -use incomplete_expr::{IncompleteSeparatedListOpr, IncompleteVdSynExprData}; +use expr::{list_item::VdSynSeparatedListItem, VdSynExprClass, VdSynExprData}; +use incomplete_expr::{IncompleteCallListOpr, IncompleteSeparatedListOpr, IncompleteVdSynExprData}; use latex_token::idx::LxTokenIdx; use smallvec::smallvec; -use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}; +use visored_opr::{ + delimiter::VdRightDelimiter, + opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}, + precedence::VdPrecedence, + separator::VdSeparator, +}; impl<'a, 'db> VdSynExprParser<'a, 'db> { pub(crate) fn accept_token(&mut self, token: DisambiguatedToken) { @@ -21,150 +26,65 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { } } - fn accept_list_end(&mut self, ket: Delimiter, ket_regional_token_idx: RegionalTokenIdx) { - self.reduce(Precedence::ListItem); + fn accept_list_end(&mut self, ket: VdRightDelimiter, ket_token_idx: LxTokenIdx) { + self.reduce(VdPrecedence::LIST_ITEM); let last_incomplete_expr = self.take_last_incomplete_expr().unwrap(); match last_incomplete_expr { - IncompleteSynExprData::CommaList { + IncompleteVdSynExprData::SeparatedList { opr, bra, - bra_regional_token_idx, mut items, } => { if bra != ket { todo!() } - self.take_complete_and_push_to_top(|this, finished_expr| { + self.take_complete_and_push_to_top(|slf, finished_expr| { if let Some(expr) = finished_expr { - items.push(SynCommaListItem::new( - this.context_mut().alloc_expr(expr), + items.push(VdSynSeparatedListItem::new( + slf.builder.alloc_expr(expr, todo!()), None, )) } match opr { - IncompleteCommaListOpr::UnitOrDelimiteredOrNewTuple => match items.last() { - None => SynExprData::Unit { - lpar_regional_token_idx: bra_regional_token_idx, - rpar_regional_token_idx: ket_regional_token_idx, - }, - Some(last_item) => { - if items.len() == 1 - && last_item.comma_regional_token_idx().is_none() - { - SynExprData::Delimitered { - lpar_regional_token_idx: bra_regional_token_idx, - item: last_item.syn_expr_idx(), - rpar_regional_token_idx: ket_regional_token_idx, - } - } else { - SynExprData::NewTuple { - lpar_regional_token_idx: bra_regional_token_idx, - items, - rpar_regional_token_idx: ket_regional_token_idx, + IncompleteSeparatedListOpr::UnitOrDelimiteredOrNewTuple => { + match items.last() { + None => VdSynExprData::Unit { + lpar_token_idx: bra_token_idx, + rpar_token_idx: ket_token_idx, + }, + Some(last_item) => { + if items.len() == 1 + && last_item.comma_regional_token_idx().is_none() + { + VdSynExprData::Delimitered { + lpar_token_idx: bra_token_idx, + item: last_item.syn_expr_idx(), + rpar_token_idx: ket_token_idx, + } + } else { + VdSynExprData::NewTuple { + lpar_token_idx: bra_token_idx, + items, + rpar_regional_token_idx: ket_regional_token_idx, + } } } } - } - .into(), - IncompleteCommaListOpr::Index { owner } => { - SynExprData::IndexOrCompositionWithList { - owner, - lbox_regional_token_idx: bra_regional_token_idx, - items, - rbox_regional_token_idx: ket_regional_token_idx, - } - .into() - } - IncompleteCommaListOpr::BoxList => SynExprData::List { - lbox_regional_token_idx: bra_regional_token_idx, - items, - rbox_regional_token_idx: ket_regional_token_idx, - } - .into(), - IncompleteCommaListOpr::BoxColonList { - colon_regional_token_idx, - } => SynExprData::BoxColonList { - lbox_regional_token_idx: bra_regional_token_idx, - colon_regional_token_idx, - items, - rbox_regional_token_idx: ket_regional_token_idx, - } - .into(), - IncompleteCommaListOpr::FunctionApplicationOrCall { function } => { - // ad hoc - let generic_arguments: Option = None; - SynExprData::FunctionApplicationOrCall { - function, - template_arguments: generic_arguments, - lpar_regional_token_idx: bra_regional_token_idx, - items, - rpar_regional_token_idx: ket_regional_token_idx, - } .into() } - IncompleteCommaListOpr::MethodInstantiation { .. } => { - todo!() - } - IncompleteCommaListOpr::MethodApplicationOrCall { - self_expr, - dot_regional_token_idx, - ident_token, - template_arguments, - } => SynExprData::MethodApplicationOrCall { - self_argument: self_expr, - dot_regional_token_idx, - ident_token, - template_arguments, - lpar_regional_token_idx: bra_regional_token_idx, - items, - rpar_regional_token_idx: ket_regional_token_idx, - } - .into(), - IncompleteCommaListOpr::TemplateInstantiation { template } => { - SynExprData::TemplateInstantiation { - template, - template_arguments: SynTemplateArguments::new( - bra_regional_token_idx, - items, - ket_regional_token_idx, - ), - } - .into() - } - IncompleteCommaListOpr::RitchieArguments { - ritchie_kind_regional_token_idx, - ritchie_kind, - lpar_token, - } => match this.try_parse_option::() { - Ok(Some(light_arrow_token)) => IncompleteSynExprData::Ritchie { - ritchie_kind_regional_token_idx, - ritchie_kind, - lpar_token, - argument_tys: items, - rpar_regional_token_idx: ket_regional_token_idx, - light_arrow_token, - } - .into(), - Ok(None) => todo!(), - Err(_) => todo!(), - }, } }) } - IncompleteSynExprData::CallList { - opr, - lpar_regional_token_idx, - items, - } => match opr { + IncompleteVdSynExprData::CallList { opr, items } => match opr { IncompleteCallListOpr::FunctionCall { function, generic_arguments, - } => self.set_complete_expr(SynExprData::FunctionCall { + } => self.set_complete_expr(VdSynExprData::FunctionCall { function, template_arguments: generic_arguments, - lpar_regional_token_idx, + lpar_token_idx: bra_token_idx, items, - rpar_regional_token_idx: ket_regional_token_idx, + rpar_token_idx: ket_token_idx, }), IncompleteCallListOpr::MethodCall { .. } => todo!(), }, @@ -177,7 +97,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { } } - fn accept_atom(&mut self, atom: SynExprData) { + fn accept_atom(&mut self, atom: VdSynExprData) { self.push_top_syn_expr(atom.into()) } @@ -204,69 +124,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { }) } - fn accept_dot_opr(&mut self, dot_regional_token_idx: RegionalTokenIdx) { - self.take_complete_and_push_to_top(|this, finished_expr| match finished_expr { - Some(self_expr) => { - let self_expr = this.context_mut().alloc_expr(self_expr); - match this.try_parse_option::() { - Ok(Some(ident_token)) => match this.try_parse_option::() { - Ok(Some(lpar)) => IncompleteSynExprData::CommaList { - opr: IncompleteCommaListOpr::MethodApplicationOrCall { - self_expr, - dot_regional_token_idx, - ident_token, - template_arguments: None, - }, - bra: Delimiter::Par, - bra_regional_token_idx: lpar.regional_token_idx(), - items: smallvec![], - } - .into(), - Ok(None) => match this.try_parse_option::() { - Ok(Some(langle)) => IncompleteSynExprData::CommaList { - opr: IncompleteCommaListOpr::MethodInstantiation { - self_expr, - dot_regional_token_idx, - ident_token, - }, - bra: Delimiter::TurboFish, - bra_regional_token_idx: langle.regional_token_idx(), - items: smallvec![], - } - .into(), - Ok(None) => SynExprData::Field { - owner: self_expr, - dot_regional_token_idx, - ident_token, - } - .into(), - Err(_) => todo!(), - }, - Err(e) => { - p!(e); - todo!() - } - }, - _ => SynExprData::Err( - OriginalSynExprError::ExpectedIdentAfterDot { - dot_regional_token_idx, - } - .into(), - ) - .into(), - } - } - None => SynExprData::Err( - OriginalSynExprError::ExpectedExprBeforeDot { - dot_regional_token_idx, - } - .into(), - ) - .into(), - }) - } - - fn accept_comma(&mut self, comma_regional_token_idx: RegionalTokenIdx) { + fn accept_separator(&mut self, separator: VdSeparator) { match self.take_complete_expr() { Some(item) => { let item = self.context_mut().alloc_expr(item); @@ -316,31 +174,6 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { } } - fn accept_be_pattern(&mut self, be_regional_token_idx: RegionalTokenIdx) { - self.reduce(Precedence::Be); - let src = self.take_complete_expr().unwrap_or(SynExprData::Err( - OriginalSynExprError::ExpectedItemBeforeBe { - be_regional_token_idx, - } - .into(), - )); - let src = self.context_mut().alloc_expr(src); - let end = match self.env() { - Some(env) => match env { - ExprEnvironment::TypeBeforeEq => todo!(), - ExprEnvironment::WithinDelimiteredParameterList(_) => todo!(), - ExprEnvironment::Condition(end) => end, - }, - None => todo!(), - }; - let expr = SynExprData::Be { - src, - be_regional_token_idx, - target: self.parse_be_variables_pattern_expected(end), - }; - self.push_top_syn_expr(expr.into()) - } - fn accept_binary_opr(&mut self, binary: Either) { self.reduce(binary.precedence()); let lopd = self.take_complete_expr().unwrap_or(VdSynExprData::Err( From cd4488b826ec45ab8077f90fa8a1416edb9a6982 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 23:02:37 -0700 Subject: [PATCH 57/71] save --- crates/visored/visored-syn-expr/src/parser.rs | 2 +- .../visored-syn-expr/src/parser/accept.rs | 351 +++++++----------- .../src/parser/disambiguate_token.rs | 4 +- 3 files changed, 142 insertions(+), 215 deletions(-) diff --git a/crates/visored/visored-syn-expr/src/parser.rs b/crates/visored/visored-syn-expr/src/parser.rs index b3ee887b7f..f39e1df09e 100644 --- a/crates/visored/visored-syn-expr/src/parser.rs +++ b/crates/visored/visored-syn-expr/src/parser.rs @@ -45,6 +45,6 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { fn parse_ast(&mut self, ast: LxMathAstIdx) { let token = self.disambiguate_token(ast); - self.stack.accept(token); + self.accept_token(token); } } diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs index f41a2c3aef..dd96724577 100644 --- a/crates/visored/visored-syn-expr/src/parser/accept.rs +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -6,7 +6,7 @@ use incomplete_expr::{IncompleteCallListOpr, IncompleteSeparatedListOpr, Incompl use latex_token::idx::LxTokenIdx; use smallvec::smallvec; use visored_opr::{ - delimiter::VdRightDelimiter, + delimiter::{VdLeftDelimiter, VdRightDelimiter}, opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}, precedence::VdPrecedence, separator::VdSeparator, @@ -27,74 +27,73 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { } fn accept_list_end(&mut self, ket: VdRightDelimiter, ket_token_idx: LxTokenIdx) { - self.reduce(VdPrecedence::LIST_ITEM); - let last_incomplete_expr = self.take_last_incomplete_expr().unwrap(); - match last_incomplete_expr { - IncompleteVdSynExprData::SeparatedList { - opr, - bra, - mut items, - } => { - if bra != ket { - todo!() - } - self.take_complete_and_push_to_top(|slf, finished_expr| { - if let Some(expr) = finished_expr { - items.push(VdSynSeparatedListItem::new( - slf.builder.alloc_expr(expr, todo!()), - None, - )) - } - match opr { - IncompleteSeparatedListOpr::UnitOrDelimiteredOrNewTuple => { - match items.last() { - None => VdSynExprData::Unit { - lpar_token_idx: bra_token_idx, - rpar_token_idx: ket_token_idx, - }, - Some(last_item) => { - if items.len() == 1 - && last_item.comma_regional_token_idx().is_none() - { - VdSynExprData::Delimitered { - lpar_token_idx: bra_token_idx, - item: last_item.syn_expr_idx(), - rpar_token_idx: ket_token_idx, - } - } else { - VdSynExprData::NewTuple { - lpar_token_idx: bra_token_idx, - items, - rpar_regional_token_idx: ket_regional_token_idx, - } - } - } - } - .into() - } - } - }) - } - IncompleteVdSynExprData::CallList { opr, items } => match opr { - IncompleteCallListOpr::FunctionCall { - function, - generic_arguments, - } => self.set_complete_expr(VdSynExprData::FunctionCall { - function, - template_arguments: generic_arguments, - lpar_token_idx: bra_token_idx, - items, - rpar_token_idx: ket_token_idx, - }), - IncompleteCallListOpr::MethodCall { .. } => todo!(), - }, - _ => { - p!(last_incomplete_expr); - // p!(self.context.path.debug(self.db())); - p!(ket_regional_token_idx); - todo!() - } - } + todo!() + // self.reduce(VdPrecedence::LIST_ITEM); + // let last_incomplete_expr = self.take_last_incomplete_expr().unwrap(); + // match last_incomplete_expr { + // IncompleteVdSynExprData::SeparatedList { + // opr, + // bra, + // mut items, + // } => { + // if bra != ket { + // todo!() + // } + // self.take_complete_and_push_to_top(|slf, finished_expr| { + // if let Some(expr) = finished_expr { + // items.push(VdSynSeparatedListItem::new( + // slf.builder.alloc_expr(expr, todo!()), + // None, + // )) + // } + // match opr { + // IncompleteSeparatedListOpr::UnitOrDelimiteredOrNewTuple => { + // match items.last() { + // None => VdSynExprData::Unit { + // lpar_token_idx: bra_token_idx, + // rpar_token_idx: ket_token_idx, + // }, + // Some(last_item) => { + // if items.len() == 1 && last_item.comma_token_idx().is_none() { + // VdSynExprData::Delimitered { + // lpar_token_idx: bra_token_idx, + // item: last_item.syn_expr_idx(), + // rpar_token_idx: ket_token_idx, + // } + // } else { + // VdSynExprData::NewTuple { + // lpar_token_idx: bra_token_idx, + // items, + // rpar_token_idx: ket_token_idx, + // } + // } + // } + // } + // .into() + // } + // } + // }) + // } + // IncompleteVdSynExprData::CallList { opr, items } => match opr { + // IncompleteCallListOpr::FunctionCall { + // function, + // generic_arguments, + // } => self.set_complete_expr(VdSynExprData::FunctionCall { + // function, + // template_arguments: generic_arguments, + // lpar_token_idx: bra_token_idx, + // items, + // rpar_token_idx: ket_token_idx, + // }), + // IncompleteCallListOpr::MethodCall { .. } => todo!(), + // }, + // _ => { + // p!(last_incomplete_expr); + // // p!(self.context.path.debug(self.db())); + // p!(ket_token_idx); + // todo!() + // } + // } } fn accept_atom(&mut self, atom: VdSynExprData) { @@ -125,153 +124,81 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { } fn accept_separator(&mut self, separator: VdSeparator) { - match self.take_complete_expr() { - Some(item) => { - let item = self.context_mut().alloc_expr(item); - match self.last_incomplete_expr_mut() { - Some(expr) => match expr { - IncompleteSynExprData::CommaList { - opr: _, - bra: _, - bra_regional_token_idx: _, - items, - } => { - items.push(SynCommaListItem::new(item, Some(comma_regional_token_idx))) - } - IncompleteSynExprData::CallList { items, .. } => items.push( - SynSimpleOrVariadicCallListItem::new( - item, - CallListSeparator::Comma(comma_regional_token_idx), - ) - .into(), - ), - _ => unreachable!(), - }, - None => unreachable!(), - } - } - None => match self.last_incomplete_expr_mut() { - Some(expr) => match expr { - IncompleteSynExprData::CommaList { - opr: _, - bra: _, - bra_regional_token_idx: _, - items: _, - } => todo!(), - IncompleteSynExprData::CallList { items, .. } => match items.last_mut() { - Some(last_item) => match last_item.separator() { - CallListSeparator::None => last_item - .set_separator(CallListSeparator::Comma(comma_regional_token_idx)), - CallListSeparator::Comma(_) => todo!(), - CallListSeparator::Semicolon(_) => todo!(), - }, - None => todo!(), - }, - _ => unreachable!(), - }, - None => unreachable!(), - }, - } + todo!() + // match self.take_complete_expr() { + // Some(item) => { + // let item = self.context_mut().alloc_expr(item); + // match self.last_incomplete_expr_mut() { + // Some(expr) => match expr { + // IncompleteSynExprData::CommaList { + // opr: _, + // bra: _, + // bra_token_idx: _, + // items, + // } => items.push(SynCommaListItem::new(item, Some(comma_token_idx))), + // IncompleteSynExprData::CallList { items, .. } => items.push( + // SynSimpleOrVariadicCallListItem::new( + // item, + // CallListSeparator::Comma(comma_token_idx), + // ) + // .into(), + // ), + // _ => unreachable!(), + // }, + // None => unreachable!(), + // } + // } + // None => match self.last_incomplete_expr_mut() { + // Some(expr) => match expr { + // IncompleteSynExprData::CommaList { + // opr: _, + // bra: _, + // bra_token_idx: _, + // items: _, + // } => todo!(), + // IncompleteSynExprData::CallList { items, .. } => match items.last_mut() { + // Some(last_item) => match last_item.separator() { + // CallListSeparator::None => { + // last_item.set_separator(CallListSeparator::Comma(comma_token_idx)) + // } + // CallListSeparator::Comma(_) => todo!(), + // CallListSeparator::Semicolon(_) => todo!(), + // }, + // None => todo!(), + // }, + // _ => unreachable!(), + // }, + // None => unreachable!(), + // }, + // } } fn accept_binary_opr(&mut self, binary: Either) { - self.reduce(binary.precedence()); - let lopd = self.take_complete_expr().unwrap_or(VdSynExprData::Err( - OriginalSynExprError::NoLeftOperandForBinaryOperator { - binary_regional_token_idx, - } - .into(), - )); - let lopd = self.builder.alloc_expr(lopd, todo!()); - let unfinished_expr = IncompleteVdSynExprData::Binary { - lopd, - punctuation: binary, - punctuation_regional_token_idx: binary_regional_token_idx, - }; - self.push_top_syn_expr(unfinished_expr.into()) + // self.reduce(binary.precedence()); + // let lopd = self.take_complete_expr().unwrap_or(VdSynExprData::Err( + // OriginalSynExprError::NoLeftOperandForBinaryOperator { binary_token_idx }.into(), + // )); + // let lopd = self.builder.alloc_expr(lopd, todo!()); + // let unfinished_expr = IncompleteVdSynExprData::Binary { + // lopd, + // punctuation: binary, + // punctuation_token_idx: binary_token_idx, + // }; + // self.push_top_syn_expr(unfinished_expr.into()) + todo!() } - fn accept_list_start(&mut self, bra: Delimiter, bra_regional_token_idx: RegionalTokenIdx) { - self.reduce(Precedence::Application); - if bra == Delimiter::Vert { - let lvert = bra_regional_token_idx; - let closure = self.parse_closure(bra_regional_token_idx); - self.push_top_syn_expr(closure.into()) - } else { - self.take_complete_and_push_to_top(|parser, finished_expr| -> TopSynExpr { - let finished_expr = finished_expr.map(|expr| parser.context_mut().alloc_expr(expr)); - match bra { - Delimiter::Par => match finished_expr { - Some(function) => IncompleteSynExprData::CommaList { - opr: IncompleteCommaListOpr::FunctionApplicationOrCall { function }, - bra, - bra_regional_token_idx, - items: smallvec![], - } - .into(), - None => IncompleteSynExprData::CommaList { - opr: IncompleteCommaListOpr::UnitOrDelimiteredOrNewTuple, - bra, - bra_regional_token_idx, - items: smallvec![], - } - .into(), - }, - Delimiter::Box => IncompleteSynExprData::CommaList { - opr: match finished_expr { - Some(finished_expr) => IncompleteCommaListOpr::Index { - owner: finished_expr, - }, - None => IncompleteCommaListOpr::BoxList, - }, - bra, - bra_regional_token_idx, - items: smallvec![], - } - .into(), - Delimiter::TurboFish => match finished_expr { - Some(template) => IncompleteSynExprData::CommaList { - opr: IncompleteCommaListOpr::TemplateInstantiation { template }, - bra, - bra_regional_token_idx, - items: smallvec![], - } - .into(), - None => todo!(), - }, - Delimiter::NestedCurl => todo!(), - Delimiter::InlineCurl => SynExprData::Err( - OriginalSynExprError::UnexpectedInlineLcurl(bra_regional_token_idx).into(), - ) - .into(), - Delimiter::Vert => { - unreachable!("Handled already") - } - Delimiter::HtmlAngle => { - let function_ident = match parser.try_parse_expected( - OriginalSynExprError::ExpectedFunctionIdentAfterOpeningHtmlBra, - ) { - Ok(function_ident) => function_ident, - Err(e) => return SynExprData::Err(e).into(), - }; - let arguments = match parse_consecutive_vec_map(parser) { - Ok(arguments) => arguments, - Err(e) => return SynExprData::Err(e).into(), - }; - match parser.try_parse_option::() { - Ok(Some(empty_html_ket)) => SynExprData::EmptyHtmlTag { - empty_html_bra_idx: bra_regional_token_idx, - function_ident, - arguments, - empty_html_ket, - } - .into(), - Ok(None) => todo!(), - Err(_) => todo!(), - } - } - } - }) - } + fn accept_list_start(&mut self, bra: VdLeftDelimiter, bra_token_idx: LxTokenIdx) { + // self.reduce(Precedence::Application); + // self.take_complete_and_push_to_top(|parser, finished_expr| -> TopSynExpr { + // let finished_expr = finished_expr.map(|expr| parser.context_mut().alloc_expr(expr)); + // todo!() + // // match bra { + // // VdLeftDelimiter::Par => todo!(), + // // VdLeftDelimiter::Brace => todo!(), + // // VdLeftDelimiter::Vert => todo!(), + // // } + // }) + todo!() } } diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs index d0d43ba4b4..d619edc351 100644 --- a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -7,7 +7,7 @@ use visored_opr::{ }; use super::{ - expr::{VdSynExprClass, VdSynExprIdx}, + expr::{VdSynExprClass, VdSynExprData, VdSynExprIdx}, VdSynExprParser, }; @@ -17,7 +17,7 @@ pub struct DisambiguatedMathAst { } pub enum DisambiguatedToken { - Expr(VdSynExprIdx, VdSynExprClass), + Expr(VdSynExprData, VdSynExprClass), Opr(VdOpr), Separator(VdSeparator), LeftDelimiter(VdLeftDelimiter), From e3515f97dbaa9b7892a11c7c9ce695b7e7aa08a8 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 23:14:40 -0700 Subject: [PATCH 58/71] save --- crates/visored/visored-hir-expr/src/expr.rs | 2 +- crates/visored/visored-hir-expr/src/tests.rs | 4 +- .../visored-lean-transpilation/src/tests.rs | 8 ++-- crates/visored/visored-opr/src/opr.rs | 21 +++++++--- crates/visored/visored-opr/src/opr/binary.rs | 42 ++++++++++++++++--- crates/visored/visored-opr/src/opr/prefix.rs | 10 ++++- crates/visored/visored-opr/src/opr/suffix.rs | 7 +++- crates/visored/visored-sem-expr/src/expr.rs | 4 +- .../visored-sem-expr/src/helpers/latex_fmt.rs | 4 +- crates/visored/visored-syn-expr/src/error.rs | 6 +-- crates/visored/visored-syn-expr/src/expr.rs | 16 +++---- .../visored-syn-expr/src/helpers/latex_fmt.rs | 6 +-- .../visored-syn-expr/src/parser/accept.rs | 16 +++---- .../src/parser/disambiguate_token.rs | 4 +- .../src/parser/incomplete_expr.rs | 6 +-- 15 files changed, 104 insertions(+), 52 deletions(-) diff --git a/crates/visored/visored-hir-expr/src/expr.rs b/crates/visored/visored-hir-expr/src/expr.rs index dfe556115e..a4993b133c 100644 --- a/crates/visored/visored-hir-expr/src/expr.rs +++ b/crates/visored/visored-hir-expr/src/expr.rs @@ -3,7 +3,7 @@ pub mod application; use crate::*; use application::VdHirApplicationFunction; use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; -use visored_opr::opr::binary::VdBinaryOpr; +use visored_opr::opr::binary::VdBaseBinaryOpr; use visored_sem_expr::expr::{binary::VdSemBinaryDispatch, VdSemExprData, VdSemExprIdx}; use visored_zfs_ty::term::literal::VdZfsLiteral; diff --git a/crates/visored/visored-hir-expr/src/tests.rs b/crates/visored/visored-hir-expr/src/tests.rs index 510e7ca321..35a1a1b5bf 100644 --- a/crates/visored/visored-hir-expr/src/tests.rs +++ b/crates/visored/visored-hir-expr/src/tests.rs @@ -1,4 +1,4 @@ -use visored_opr::opr::binary::VdBinaryOpr; +use visored_opr::opr::binary::VdBaseBinaryOpr; use visored_sem_expr::{ expr::{binary::VdSemBinaryDispatch, literal::VdSemLiteralDispatch, VdSemExprData}, test_helpers::builder::VdSemExprTestBuilder, @@ -34,7 +34,7 @@ fn to_hir_works() { let one_add_one = builder.new_expr_checked( VdSemExprData::Binary { lopd: one, - opr: VdBinaryOpr::Add, + opr: VdBaseBinaryOpr::Add, ropd: one, dispatch: VdSemBinaryDispatch::IntAdd, }, diff --git a/crates/visored/visored-lean-transpilation/src/tests.rs b/crates/visored/visored-lean-transpilation/src/tests.rs index 1ff1d8db06..0385c651c7 100644 --- a/crates/visored/visored-lean-transpilation/src/tests.rs +++ b/crates/visored/visored-lean-transpilation/src/tests.rs @@ -1,5 +1,5 @@ use visored_hir_expr::{builder::VdHirExprBuilder, ToVdHir}; -use visored_opr::opr::binary::VdBinaryOpr; +use visored_opr::opr::binary::VdBaseBinaryOpr; use visored_sem_expr::{ expr::{binary::VdSemBinaryDispatch, literal::VdSemLiteralDispatch, VdSemExprData}, test_helpers::builder::VdSemExprTestBuilder, @@ -39,7 +39,7 @@ fn one_add_one_to_lean_works() { let one_add_one = builder.new_expr_checked( VdSemExprData::Binary { lopd: one, - opr: VdBinaryOpr::Add, + opr: VdBaseBinaryOpr::Add, ropd: one, dispatch: VdSemBinaryDispatch::IntAdd, }, @@ -78,7 +78,7 @@ fn one_add_one_eqs_two_to_lean_works() { let one_add_one = builder.new_expr_checked( VdSemExprData::Binary { lopd: one, - opr: VdBinaryOpr::Add, + opr: VdBaseBinaryOpr::Add, ropd: one, dispatch: VdSemBinaryDispatch::IntAdd, }, @@ -87,7 +87,7 @@ fn one_add_one_eqs_two_to_lean_works() { let one_add_one_eqs_two = builder.new_expr_checked( VdSemExprData::Binary { lopd: one_add_one, - opr: VdBinaryOpr::Eq, + opr: VdBaseBinaryOpr::Eq, ropd: two, dispatch: VdSemBinaryDispatch::TrivialEq, }, diff --git a/crates/visored/visored-opr/src/opr.rs b/crates/visored/visored-opr/src/opr.rs index fee5cc325e..aa67cb0f10 100644 --- a/crates/visored/visored-opr/src/opr.rs +++ b/crates/visored/visored-opr/src/opr.rs @@ -2,11 +2,22 @@ pub mod binary; pub mod prefix; pub mod suffix; -use self::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr}; +use self::{ + binary::{VdBaseBinaryOpr, VdCompositeBinaryOpr}, + prefix::{VdBasePrefixOpr, VdCompositePrefixOpr}, + suffix::{VdBaseSuffixOpr, VdCompositeSuffixOpr}, +}; #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -pub enum VdOpr { - Binary(VdBinaryOpr), - Prefix(VdPrefixOpr), - Suffix(VdSuffixOpr), +pub enum VdBaseOpr { + Prefix(VdBasePrefixOpr), + Suffix(VdBaseSuffixOpr), + Binary(VdBaseBinaryOpr), +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub enum VdCompositeOpr { + Prefix(VdCompositePrefixOpr), + Suffix(VdCompositeSuffixOpr), + Binary(VdCompositeBinaryOpr), } diff --git a/crates/visored/visored-opr/src/opr/binary.rs b/crates/visored/visored-opr/src/opr/binary.rs index f8a9a5683c..4e50825ba4 100644 --- a/crates/visored/visored-opr/src/opr/binary.rs +++ b/crates/visored/visored-opr/src/opr/binary.rs @@ -1,16 +1,16 @@ use crate::precedence::VdPrecedenceRange; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] -pub enum VdBinaryOpr { +pub enum VdBaseBinaryOpr { Add, Eq, } -impl VdBinaryOpr { +impl VdBaseBinaryOpr { pub fn fmt_str(self) -> &'static str { match self { - VdBinaryOpr::Add => "+", - VdBinaryOpr::Eq => "=", + VdBaseBinaryOpr::Add => "+", + VdBaseBinaryOpr::Eq => "=", } } @@ -24,8 +24,38 @@ impl VdBinaryOpr { pub fn latex_code(&self) -> &str { match self { - VdBinaryOpr::Add => "+", - VdBinaryOpr::Eq => "=", + VdBaseBinaryOpr::Add => "+", + VdBaseBinaryOpr::Eq => "=", + } + } +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] +pub enum VdCompositeBinaryOpr { + Add, + Eq, +} + +impl VdCompositeBinaryOpr { + pub fn fmt_str(self) -> &'static str { + match self { + VdCompositeBinaryOpr::Add => "+", + VdCompositeBinaryOpr::Eq => "=", + } + } + + pub fn left_precedence_range(self) -> VdPrecedenceRange { + todo!() + } + + pub fn right_precedence_range(self) -> VdPrecedenceRange { + todo!() + } + + pub fn latex_code(&self) -> &str { + match self { + VdCompositeBinaryOpr::Add => "+", + VdCompositeBinaryOpr::Eq => "=", } } } diff --git a/crates/visored/visored-opr/src/opr/prefix.rs b/crates/visored/visored-opr/src/opr/prefix.rs index 9e4f03da44..dba6c76e13 100644 --- a/crates/visored/visored-opr/src/opr/prefix.rs +++ b/crates/visored/visored-opr/src/opr/prefix.rs @@ -1,9 +1,9 @@ use crate::precedence::VdPrecedenceRange; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] -pub enum VdPrefixOpr {} +pub enum VdBasePrefixOpr {} -impl VdPrefixOpr { +impl VdBasePrefixOpr { pub fn fmt_str(self) -> &'static str { todo!() } @@ -12,3 +12,9 @@ impl VdPrefixOpr { todo!() } } + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] +pub enum VdCompositePrefixOpr { + /// `d/dx` + Differential, +} diff --git a/crates/visored/visored-opr/src/opr/suffix.rs b/crates/visored/visored-opr/src/opr/suffix.rs index fcf72494e0..99205550bc 100644 --- a/crates/visored/visored-opr/src/opr/suffix.rs +++ b/crates/visored/visored-opr/src/opr/suffix.rs @@ -1,9 +1,9 @@ use crate::precedence::VdPrecedenceRange; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] -pub enum VdSuffixOpr {} +pub enum VdBaseSuffixOpr {} -impl VdSuffixOpr { +impl VdBaseSuffixOpr { pub fn fmt_str(self) -> &'static str { todo!() } @@ -12,3 +12,6 @@ impl VdSuffixOpr { todo!() } } + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] +pub enum VdCompositeSuffixOpr {} diff --git a/crates/visored/visored-sem-expr/src/expr.rs b/crates/visored/visored-sem-expr/src/expr.rs index 1e3294875e..6bd639da25 100644 --- a/crates/visored/visored-sem-expr/src/expr.rs +++ b/crates/visored/visored-sem-expr/src/expr.rs @@ -13,7 +13,7 @@ use self::{attach::AttachDispatch, binary::VdSemBinaryDispatch}; use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; use latex_prelude::script::LxScriptKind; use literal::VdSemLiteralDispatch; -use visored_opr::opr::binary::VdBinaryOpr; +use visored_opr::opr::binary::VdBaseBinaryOpr; use visored_zfs_ty::term::literal::VdZfsLiteral; /// It's a tree of both form and meaning @@ -26,7 +26,7 @@ pub enum VdSemExprData { Notation, Binary { lopd: VdSemExprIdx, - opr: VdBinaryOpr, + opr: VdBaseBinaryOpr, ropd: VdSemExprIdx, dispatch: VdSemBinaryDispatch, }, diff --git a/crates/visored/visored-sem-expr/src/helpers/latex_fmt.rs b/crates/visored/visored-sem-expr/src/helpers/latex_fmt.rs index 6240eb15f7..ced749ddcc 100644 --- a/crates/visored/visored-sem-expr/src/helpers/latex_fmt.rs +++ b/crates/visored/visored-sem-expr/src/helpers/latex_fmt.rs @@ -1,5 +1,5 @@ use expr::{binary::VdSemBinaryDispatch, literal::VdSemLiteralDispatch}; -use visored_opr::opr::binary::VdBinaryOpr; +use visored_opr::opr::binary::VdBaseBinaryOpr; use visored_zfs_ty::{menu::vd_zfs_ty_menu, term::literal::VdZfsLiteralData}; use super::*; @@ -159,7 +159,7 @@ fn latex_fmt_works() { let one_add_one = builder.new_expr_checked( VdSemExprData::Binary { lopd: one, - opr: VdBinaryOpr::Add, + opr: VdBaseBinaryOpr::Add, ropd: one, dispatch: VdSemBinaryDispatch::IntAdd, }, diff --git a/crates/visored/visored-syn-expr/src/error.rs b/crates/visored/visored-syn-expr/src/error.rs index 673ee09e66..d68ae172bc 100644 --- a/crates/visored/visored-syn-expr/src/error.rs +++ b/crates/visored/visored-syn-expr/src/error.rs @@ -2,7 +2,7 @@ use crate::expr::VdSynExprIdx; use either::Either; use latex_token::idx::LxTokenIdx; use thiserror::Error; -use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr}; +use visored_opr::opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr}; #[derive(Debug, Error, PartialEq, Eq)] pub enum VdSynExprError { @@ -20,11 +20,11 @@ pub enum OriginalVdSynExprError { Todo, #[error("no right operand for binary operator")] NoRightOperandForBinaryOperator { - opr: Either, + opr: Either, }, #[error("no operand for prefix operator")] NoOperandForPrefixOperator { - opr: Either, + opr: Either, }, #[error("unterminated list")] UnterminatedList { bra_token_idx: LxTokenIdx }, diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 14e7e4f321..7d2cbe79d6 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -20,7 +20,9 @@ use idx_arena::{ use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; use latex_prelude::script::LxScriptKind; use range::VdSynExprAstRange; -use visored_opr::opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}; +use visored_opr::opr::{ + binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr, suffix::VdBaseSuffixOpr, VdBaseOpr, +}; use visored_zfs_ty::term::literal::VdZfsLiteral; /// It's a tree of both form and meaning @@ -30,21 +32,21 @@ pub enum VdSynExprData { literal: VdZfsLiteral, }, Notation, - Opr { - opr: VdOpr, + BaseOpr { + opr: VdBaseOpr, }, Binary { lopd: VdSynExprIdx, - opr: Either, + opr: Either, ropd: VdSynExprIdx, }, Prefix { - opr: Either, + opr: Either, opd: VdSynExprIdx, }, Suffix { opd: VdSynExprIdx, - opr: Either, + opr: Either, }, Attach { base: VdSynExprIdx, @@ -70,7 +72,7 @@ impl VdSynExprData { match *self { VdSynExprData::Literal { literal } => vec![], VdSynExprData::Notation => vec![], - VdSynExprData::Opr { opr } => vec![], + VdSynExprData::BaseOpr { opr } => vec![], VdSynExprData::Binary { lopd, opr, ropd } => match opr { Left(_) => vec![lopd, ropd], Right(opr) => vec![lopd, opr, ropd], diff --git a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs index fc347e97f8..5ebbe7518f 100644 --- a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs +++ b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs @@ -11,7 +11,7 @@ use crate::{ sentence::{VdSynSentenceArenaRef, VdSynSentenceData, VdSynSentenceIdx, VdSynSentenceIdxRange}, }; use either::*; -use visored_opr::opr::binary::VdBinaryOpr; +use visored_opr::opr::binary::VdBaseBinaryOpr; use visored_zfs_ty::{menu::vd_zfs_ty_menu, term::literal::VdZfsLiteralData}; pub struct VdSynExprLaTeXFormatter<'a> { @@ -123,7 +123,7 @@ impl<'a> VdSynExprLaTeXFormatter<'a> { VdSynExprData::VariadicChain => todo!(), VdSynExprData::UniadicArray => todo!(), VdSynExprData::VariadicArray => todo!(), - VdSynExprData::Opr { opr } => todo!(), + VdSynExprData::BaseOpr { opr } => todo!(), VdSynExprData::Err(ref error) => unreachable!("{error}"), } } @@ -147,7 +147,7 @@ fn latex_fmt_works() { let one_add_one = builder.new_expr_checked( VdSynExprData::Binary { lopd: one, - opr: Either::Left(VdBinaryOpr::Add), + opr: Either::Left(VdBaseBinaryOpr::Add), ropd: one, }, "1+1", diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs index dd96724577..230bff0008 100644 --- a/crates/visored/visored-syn-expr/src/parser/accept.rs +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -7,7 +7,7 @@ use latex_token::idx::LxTokenIdx; use smallvec::smallvec; use visored_opr::{ delimiter::{VdLeftDelimiter, VdRightDelimiter}, - opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr, suffix::VdSuffixOpr, VdOpr}, + opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr, suffix::VdBaseSuffixOpr, VdBaseOpr}, precedence::VdPrecedence, separator::VdSeparator, }; @@ -100,19 +100,19 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { self.push_top_syn_expr(atom.into()) } - fn accept_opr(&mut self, opr: VdOpr) { + fn accept_opr(&mut self, opr: VdBaseOpr) { match opr { - VdOpr::Binary(opr) => todo!(), - VdOpr::Prefix(opr) => self.accept_prefix_opr(Left(opr)), - VdOpr::Suffix(opr) => self.accept_suffix_opr(Left(opr)), + VdBaseOpr::Binary(opr) => todo!(), + VdBaseOpr::Prefix(opr) => self.accept_prefix_opr(Left(opr)), + VdBaseOpr::Suffix(opr) => self.accept_suffix_opr(Left(opr)), } } - fn accept_prefix_opr(&mut self, opr: Either) { + fn accept_prefix_opr(&mut self, opr: Either) { self.push_top_syn_expr(IncompleteVdSynExprData::Prefix { opr }.into()) } - fn accept_suffix_opr(&mut self, opr: Either) { + fn accept_suffix_opr(&mut self, opr: Either) { self.take_complete_and_push_to_top(|slf, top_expr| match top_expr { Some(expr) => VdSynExprData::Suffix { opd: slf.builder.alloc_expr(expr, todo!()), @@ -173,7 +173,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { // } } - fn accept_binary_opr(&mut self, binary: Either) { + fn accept_binary_opr(&mut self, binary: Either) { // self.reduce(binary.precedence()); // let lopd = self.take_complete_expr().unwrap_or(VdSynExprData::Err( // OriginalSynExprError::NoLeftOperandForBinaryOperator { binary_token_idx }.into(), diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs index d619edc351..1cd5ef8a21 100644 --- a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -2,7 +2,7 @@ use latex_ast::ast::math::LxMathAstIdx; use visored_annotation::annotation::space::VdSpaceAnnotation; use visored_opr::{ delimiter::{VdLeftDelimiter, VdRightDelimiter}, - opr::VdOpr, + opr::VdBaseOpr, separator::VdSeparator, }; @@ -18,7 +18,7 @@ pub struct DisambiguatedMathAst { pub enum DisambiguatedToken { Expr(VdSynExprData, VdSynExprClass), - Opr(VdOpr), + Opr(VdBaseOpr), Separator(VdSeparator), LeftDelimiter(VdLeftDelimiter), RightDelimiter(VdRightDelimiter), diff --git a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs index 0078f6d5a7..2a5481c01a 100644 --- a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs +++ b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs @@ -10,7 +10,7 @@ use expr::list_item::VdSynSeparatedListItem; use latex_token::idx::LxTokenIdx; use visored_opr::{ delimiter::VdLeftDelimiter, - opr::{binary::VdBinaryOpr, prefix::VdPrefixOpr}, + opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr}, precedence::VdPrecedence, }; @@ -18,10 +18,10 @@ use visored_opr::{ pub(super) enum IncompleteVdSynExprData { Binary { lopd: VdSynExprIdx, - opr: Either, + opr: Either, }, Prefix { - opr: Either, + opr: Either, }, /// list separated by commas /// ```husky From db36f1a4990de6133fd5ab9374a788e5af32d70b Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 23:46:51 -0700 Subject: [PATCH 59/71] feat: parse 1 in visored-syn-expr --- Cargo.lock | 11 +-- Cargo.toml | 2 +- .../abstractions/idx-arena/src/arena_idx.rs | 8 ++- crates/abstractions/idx-arena/src/lib.rs | 2 +- crates/abstractions/salsa/src/jar.rs | 2 +- crates/latex/latex-ast/src/ast.rs | 16 +++++ .../latex/latex-token/src/data/math/digit.rs | 16 +++++ crates/latex/latex-token/src/idx.rs | 7 ++ .../expect-files/husky_lang_jar_packages.txt | 10 +-- .../expect-files/husky_lang_jar_tree.json | 10 +-- .../expect-files/husky_lang_jar_tree.txt | 10 +-- .../husky_lang_jar_tree_trimmed.json | 4 +- .../husky_lang_jar_tree_trimmed.txt | 4 +- .../expect-files/husky_lang_packages.txt | 13 ++-- crates/visored/visored-hir-expr/Cargo.toml | 2 +- crates/visored/visored-hir-expr/src/expr.rs | 4 +- crates/visored/visored-hir-expr/src/tests.rs | 6 +- .../visored-lean-transpilation/Cargo.toml | 2 +- .../visored-lean-transpilation/src/expr.rs | 12 ++-- .../visored-lean-transpilation/src/tests.rs | 8 +-- crates/visored/visored-sem-expr/Cargo.toml | 2 +- crates/visored/visored-sem-expr/src/expr.rs | 4 +- .../visored-sem-expr/src/helpers/latex_fmt.rs | 12 ++-- crates/visored/visored-sem-expr/src/tests.rs | 2 +- crates/visored/visored-syn-expr/Cargo.toml | 3 +- crates/visored/visored-syn-expr/src/expr.rs | 10 +-- .../visored-syn-expr/src/helpers/latex_fmt.rs | 17 +++-- crates/visored/visored-syn-expr/src/parser.rs | 10 +-- .../visored-syn-expr/src/parser/accept.rs | 3 + .../src/parser/disambiguate_token.rs | 71 ++++++++++++++++--- crates/visored/visored-syn-expr/src/tests.rs | 2 +- .../Cargo.toml | 2 +- crates/visored/visored-zfc-ty/src/jar.rs | 2 + .../src/lib.rs | 2 +- crates/visored/visored-zfc-ty/src/menu.rs | 37 ++++++++++ .../visored/visored-zfc-ty/src/refinement.rs | 10 +++ crates/visored/visored-zfc-ty/src/term.rs | 61 ++++++++++++++++ .../src/term/abstract_variable.rs | 10 +-- .../visored-zfc-ty/src/term/abstraction.rs | 19 +++++ .../visored-zfc-ty/src/term/application.rs | 19 +++++ .../src/term/eval.rs | 10 +-- .../src/term/exists.rs | 10 +-- .../src/term/forall.rs | 10 +-- .../src/term/limit.rs | 10 +-- .../visored-zfc-ty/src/term/literal.rs | 30 ++++++++ .../src/term/literal/special_constant.rs | 2 +- .../src/term/stack_variable.rs | 10 +-- .../src/term/symbolic_variable.rs | 4 +- .../src/ty.rs | 6 +- crates/visored/visored-zfs-ty/src/jar.rs | 2 - crates/visored/visored-zfs-ty/src/menu.rs | 37 ---------- .../visored/visored-zfs-ty/src/refinement.rs | 10 --- crates/visored/visored-zfs-ty/src/term.rs | 61 ---------------- .../visored-zfs-ty/src/term/abstraction.rs | 19 ----- .../visored-zfs-ty/src/term/application.rs | 19 ----- .../visored-zfs-ty/src/term/literal.rs | 30 -------- 56 files changed, 415 insertions(+), 302 deletions(-) rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/Cargo.toml (94%) create mode 100644 crates/visored/visored-zfc-ty/src/jar.rs rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/lib.rs (67%) create mode 100644 crates/visored/visored-zfc-ty/src/menu.rs create mode 100644 crates/visored/visored-zfc-ty/src/refinement.rs create mode 100644 crates/visored/visored-zfc-ty/src/term.rs rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/term/abstract_variable.rs (53%) create mode 100644 crates/visored/visored-zfc-ty/src/term/abstraction.rs create mode 100644 crates/visored/visored-zfc-ty/src/term/application.rs rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/term/eval.rs (58%) rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/term/exists.rs (57%) rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/term/forall.rs (57%) rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/term/limit.rs (57%) create mode 100644 crates/visored/visored-zfc-ty/src/term/literal.rs rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/term/literal/special_constant.rs (73%) rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/term/stack_variable.rs (54%) rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/term/symbolic_variable.rs (67%) rename crates/visored/{visored-zfs-ty => visored-zfc-ty}/src/ty.rs (67%) delete mode 100644 crates/visored/visored-zfs-ty/src/jar.rs delete mode 100644 crates/visored/visored-zfs-ty/src/menu.rs delete mode 100644 crates/visored/visored-zfs-ty/src/refinement.rs delete mode 100644 crates/visored/visored-zfs-ty/src/term.rs delete mode 100644 crates/visored/visored-zfs-ty/src/term/abstraction.rs delete mode 100644 crates/visored/visored-zfs-ty/src/term/application.rs delete mode 100644 crates/visored/visored-zfs-ty/src/term/literal.rs diff --git a/Cargo.lock b/Cargo.lock index 8d8bbf3dab..da67eb5b7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12534,7 +12534,7 @@ dependencies = [ "visored-opr", "visored-sem-expr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", ] [[package]] @@ -12561,7 +12561,7 @@ dependencies = [ "visored-opr", "visored-sem-expr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", ] [[package]] @@ -12593,7 +12593,7 @@ dependencies = [ "visored-opr", "visored-sem-expr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", ] [[package]] @@ -12608,6 +12608,7 @@ dependencies = [ "idx-arena", "latex-ast", "latex-command", + "latex-math-letter", "latex-prelude", "latex-token", "salsa", @@ -12616,11 +12617,11 @@ dependencies = [ "visored-annotation", "visored-opr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", ] [[package]] -name = "visored-zfs-ty" +name = "visored-zfc-ty" version = "0.1.0" dependencies = [ "enum-class", diff --git a/Cargo.toml b/Cargo.toml index 5b3df4a9cd..df05ba36b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -435,7 +435,7 @@ visored-lean-transpilation = { path = "crates/visored/visored-lean-transpilation visored-sem-expr = { path = "crates/visored/visored-sem-expr" } visored-syn-expr = { path = "crates/visored/visored-syn-expr" } visored-opr = { path = "crates/visored/visored-opr" } -visored-zfs-ty = { path = "crates/visored/visored-zfs-ty" } +visored-zfc-ty = { path = "crates/visored/visored-zfc-ty" } # vm husky-vm = { path = "crates/vm/husky-vm" } # vmir diff --git a/crates/abstractions/idx-arena/src/arena_idx.rs b/crates/abstractions/idx-arena/src/arena_idx.rs index b0a26b72df..727f00a857 100644 --- a/crates/abstractions/idx-arena/src/arena_idx.rs +++ b/crates/abstractions/idx-arena/src/arena_idx.rs @@ -1,5 +1,5 @@ use crate::*; -use std::num::NonZeroU32; +use std::{num::NonZeroU32, ops::AddAssign}; pub struct ArenaIdx { raw: NonZeroU32, @@ -83,6 +83,12 @@ impl Add for ArenaIdx { } } +impl AddAssign for ArenaIdx { + fn add_assign(&mut self, rhs: usize) { + *self = *self + rhs; + } +} + impl Sub for ArenaIdx { type Output = usize; diff --git a/crates/abstractions/idx-arena/src/lib.rs b/crates/abstractions/idx-arena/src/lib.rs index 0bd9a77914..fd1a822f2d 100644 --- a/crates/abstractions/idx-arena/src/lib.rs +++ b/crates/abstractions/idx-arena/src/lib.rs @@ -106,7 +106,7 @@ impl IntoIterator for ArenaIdxRange { type IntoIter = impl Iterator; fn into_iter(self) -> Self::IntoIter { - self.to_range().into_iter() + self.to_range() } } diff --git a/crates/abstractions/salsa/src/jar.rs b/crates/abstractions/salsa/src/jar.rs index 918bb04597..4eefad7014 100644 --- a/crates/abstractions/salsa/src/jar.rs +++ b/crates/abstractions/salsa/src/jar.rs @@ -193,7 +193,7 @@ pub enum JarIndex { LxAstJar, LxCommandJar, // visored - VdZfsTypeJar, + VdZfcTypeJar, VdOprJar, VdSynExprJar, VdSemExprJar, diff --git a/crates/latex/latex-ast/src/ast.rs b/crates/latex/latex-ast/src/ast.rs index 652c7f1df5..8fd8983bce 100644 --- a/crates/latex/latex-ast/src/ast.rs +++ b/crates/latex/latex-ast/src/ast.rs @@ -52,6 +52,22 @@ pub struct LxAstArenaRef<'a> { rose: LxRoseAstArenaRef<'a>, } +impl<'a> std::ops::Index for LxAstArenaRef<'a> { + type Output = LxMathAstData; + + fn index(&self, index: LxMathAstIdx) -> &Self::Output { + &self.math[index] + } +} + +impl<'a> std::ops::Index for LxAstArenaRef<'a> { + type Output = LxRoseAstData; + + fn index(&self, index: LxRoseAstIdx) -> &Self::Output { + &self.rose[index] + } +} + impl<'a> LxAstArenaRef<'a> { pub fn math(&self) -> LxMathAstArenaRef<'a> { self.math diff --git a/crates/latex/latex-token/src/data/math/digit.rs b/crates/latex/latex-token/src/data/math/digit.rs index 3e8676aab1..e64917de1a 100644 --- a/crates/latex/latex-token/src/data/math/digit.rs +++ b/crates/latex/latex-token/src/data/math/digit.rs @@ -14,6 +14,22 @@ pub enum LxMathDigit { Eight, Nine, } +impl LxMathDigit { + pub fn char(&self) -> char { + match self { + LxMathDigit::Zero => '0', + LxMathDigit::One => '1', + LxMathDigit::Two => '2', + LxMathDigit::Three => '3', + LxMathDigit::Four => '4', + LxMathDigit::Five => '5', + LxMathDigit::Six => '6', + LxMathDigit::Seven => '7', + LxMathDigit::Eight => '8', + LxMathDigit::Nine => '9', + } + } +} impl TryFrom for LxMathDigit { type Error = (); diff --git a/crates/latex/latex-token/src/idx.rs b/crates/latex/latex-token/src/idx.rs index c22c330810..dcee27b276 100644 --- a/crates/latex/latex-token/src/idx.rs +++ b/crates/latex/latex-token/src/idx.rs @@ -8,6 +8,13 @@ pub struct LxTokenIdx(ShiftedU32); #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct LxMathTokenIdx(pub(crate) LxTokenIdx); +impl std::ops::Add for LxMathTokenIdx { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} + impl std::ops::Deref for LxMathTokenIdx { type Target = LxTokenIdx; fn deref(&self) -> &Self::Target { diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt index 645545c652..92cb0ecfe1 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_packages.txt @@ -1311,7 +1311,7 @@ dependencies: [ "visored-opr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", "husky-coword", "latex-ast", "latex-command", @@ -1326,13 +1326,13 @@ dependencies: [ "latex-ast", "visored-opr", - "visored-zfs-ty", + "visored-zfc-ty", "husky-coword", "latex-command", ], }, JarPackageSummary { - name: "visored-zfs-ty", + name: "visored-zfc-ty", dependencies: [], }, JarPackageSummary { @@ -1340,7 +1340,7 @@ dependencies: [ "visored-opr", "visored-sem-expr", - "visored-zfs-ty", + "visored-zfc-ty", "husky-coword", "latex-ast", "latex-command", @@ -1355,7 +1355,7 @@ "lean-term", "visored-hir-expr", "visored-opr", - "visored-zfs-ty", + "visored-zfc-ty", "husky-coword", "latex-ast", "latex-command", diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json index 6e9ffe2881..723c3a0c9e 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.json @@ -1181,7 +1181,7 @@ "visored-opr", "visored-sem-expr", "visored-syn-expr", - "visored-zfs-ty" + "visored-zfc-ty" ], "visored-lean-transpilation": [ "husky-coword", @@ -1194,7 +1194,7 @@ "visored-opr", "visored-sem-expr", "visored-syn-expr", - "visored-zfs-ty" + "visored-zfc-ty" ], "visored-opr": [], "visored-sem-expr": [ @@ -1203,14 +1203,14 @@ "latex-command", "visored-opr", "visored-syn-expr", - "visored-zfs-ty" + "visored-zfc-ty" ], "visored-syn-expr": [ "husky-coword", "latex-ast", "latex-command", "visored-opr", - "visored-zfs-ty" + "visored-zfc-ty" ], - "visored-zfs-ty": [] + "visored-zfc-ty": [] } \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt index 733a8ac984..ab733b5883 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree.txt @@ -1181,7 +1181,7 @@ "visored-opr", "visored-sem-expr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", }, "visored-lean-transpilation": { "husky-coword", @@ -1194,7 +1194,7 @@ "visored-opr", "visored-sem-expr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", }, "visored-opr": {}, "visored-sem-expr": { @@ -1203,14 +1203,14 @@ "latex-command", "visored-opr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", }, "visored-syn-expr": { "husky-coword", "latex-ast", "latex-command", "visored-opr", - "visored-zfs-ty", + "visored-zfc-ty", }, - "visored-zfs-ty": {}, + "visored-zfc-ty": {}, } \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json index 03270dd322..e598a23c75 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.json @@ -196,7 +196,7 @@ "visored-syn-expr": [ "latex-ast", "visored-opr", - "visored-zfs-ty" + "visored-zfc-ty" ], - "visored-zfs-ty": [] + "visored-zfc-ty": [] } \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt index b9a5fcc9cd..ee7e46c695 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_jar_tree_trimmed.txt @@ -196,7 +196,7 @@ "visored-syn-expr": { "latex-ast", "visored-opr", - "visored-zfs-ty", + "visored-zfc-ty", }, - "visored-zfs-ty": {}, + "visored-zfc-ty": {}, } \ No newline at end of file diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 7d2464c873..99aa9d5fae 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4242,7 +4242,7 @@ "visored-annotation", "visored-opr", "visored-syn-expr", - "visored-zfs-ty", + "visored-zfc-ty", "husky-coword", "latex-ast", "latex-command", @@ -4264,6 +4264,7 @@ "husky-tree-utils", "idx-arena", "latex-ast", + "latex-math-letter", "latex-prelude", "latex-token", "salsa", @@ -4271,15 +4272,15 @@ "thiserror", "visored-annotation", "visored-opr", - "visored-zfs-ty", + "visored-zfc-ty", "expect-test", "husky-coword", "latex-command", ], }, PackageSummary { - name: "visored-zfs-ty", - relative_path: "crates/visored/visored-zfs-ty", + name: "visored-zfc-ty", + relative_path: "crates/visored/visored-zfc-ty", dependencies: [ "enum-class", "salsa", @@ -4302,7 +4303,7 @@ "salsa", "visored-opr", "visored-sem-expr", - "visored-zfs-ty", + "visored-zfc-ty", "husky-coword", "latex-ast", "latex-command", @@ -4327,7 +4328,7 @@ "salsa", "visored-hir-expr", "visored-opr", - "visored-zfs-ty", + "visored-zfc-ty", "husky-coword", "latex-ast", "latex-command", diff --git a/crates/visored/visored-hir-expr/Cargo.toml b/crates/visored/visored-hir-expr/Cargo.toml index da0fcc2e12..e444deb212 100644 --- a/crates/visored/visored-hir-expr/Cargo.toml +++ b/crates/visored/visored-hir-expr/Cargo.toml @@ -18,7 +18,7 @@ salsa.workspace = true # visored visored-sem-expr.workspace = true visored-opr.workspace = true -visored-zfs-ty.workspace = true +visored-zfc-ty.workspace = true [dev-dependencies] # latex diff --git a/crates/visored/visored-hir-expr/src/expr.rs b/crates/visored/visored-hir-expr/src/expr.rs index a4993b133c..746df926db 100644 --- a/crates/visored/visored-hir-expr/src/expr.rs +++ b/crates/visored/visored-hir-expr/src/expr.rs @@ -5,11 +5,11 @@ use application::VdHirApplicationFunction; use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; use visored_opr::opr::binary::VdBaseBinaryOpr; use visored_sem_expr::expr::{binary::VdSemBinaryDispatch, VdSemExprData, VdSemExprIdx}; -use visored_zfs_ty::term::literal::VdZfsLiteral; +use visored_zfc_ty::term::literal::VdZfcLiteral; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum VdHirExprData { - Literal(VdZfsLiteral), + Literal(VdZfcLiteral), Variable(VdHirVariable), Application { function: VdHirApplicationFunction, diff --git a/crates/visored/visored-hir-expr/src/tests.rs b/crates/visored/visored-hir-expr/src/tests.rs index 35a1a1b5bf..6d04266b7b 100644 --- a/crates/visored/visored-hir-expr/src/tests.rs +++ b/crates/visored/visored-hir-expr/src/tests.rs @@ -3,7 +3,7 @@ use visored_sem_expr::{ expr::{binary::VdSemBinaryDispatch, literal::VdSemLiteralDispatch, VdSemExprData}, test_helpers::builder::VdSemExprTestBuilder, }; -use visored_zfs_ty::menu::vd_zfs_ty_menu; +use visored_zfc_ty::menu::vd_zfc_ty_menu; use crate::{builder::VdHirExprBuilder, ToVdHir}; @@ -11,7 +11,7 @@ use crate::{builder::VdHirExprBuilder, ToVdHir}; husky_coword::jar::CowordJar, latex_ast::jar::LxAstJar, latex_command::jar::LxCommandJar, - visored_zfs_ty::jar::VdZfsTypeJar, + visored_zfc_ty::jar::VdZfcTypeJar, visored_opr::jar::VdOprJar, visored_sem_expr::jar::VdSemExprJar, visored_syn_expr::jar::VdSynExprJar, @@ -22,7 +22,7 @@ pub(crate) struct DB {} #[test] fn to_hir_works() { let db = &DB::default(); - let menu = vd_zfs_ty_menu(db); + let menu = vd_zfc_ty_menu(db); let mut builder = VdSemExprTestBuilder::new(db); let one = builder.new_expr_checked( VdSemExprData::Literal { diff --git a/crates/visored/visored-lean-transpilation/Cargo.toml b/crates/visored/visored-lean-transpilation/Cargo.toml index f63cf98f49..6d727b82c3 100644 --- a/crates/visored/visored-lean-transpilation/Cargo.toml +++ b/crates/visored/visored-lean-transpilation/Cargo.toml @@ -21,7 +21,7 @@ lean-term.workspace = true # visored visored-hir-expr.workspace = true visored-opr.workspace = true -visored-zfs-ty.workspace = true +visored-zfc-ty.workspace = true [dev-dependencies] # kernel diff --git a/crates/visored/visored-lean-transpilation/src/expr.rs b/crates/visored/visored-lean-transpilation/src/expr.rs index bfa30d9f03..519a5d912d 100644 --- a/crates/visored/visored-lean-transpilation/src/expr.rs +++ b/crates/visored/visored-lean-transpilation/src/expr.rs @@ -4,7 +4,7 @@ use lean_hir_expr::expr::{LnHirExprData, LnHirExprIdx}; use lean_opr::opr::binary::LnBinaryOpr; use lean_term::term::literal::{LnLiteral, LnLiteralData}; use visored_hir_expr::expr::{application::VdHirApplicationFunction, VdHirExprData, VdHirExprIdx}; -use visored_zfs_ty::term::literal::{VdZfsLiteral, VdZfsLiteralData}; +use visored_zfc_ty::term::literal::{VdZfcLiteral, VdZfcLiteralData}; impl ToLean for VdHirExprIdx { type Target = LnHirExprIdx; @@ -51,14 +51,14 @@ impl<'db> VdLeanTranspilationBuilder<'db> { } #[salsa::tracked] -fn to_lean_literal(db: &salsa::Db, literal: VdZfsLiteral) -> LnLiteral { +fn to_lean_literal(db: &salsa::Db, literal: VdZfcLiteral) -> LnLiteral { let data = match literal.data(db) { - VdZfsLiteralData::NaturalNumber(lit) => LnLiteralData::Nat(lit.to_string()), - VdZfsLiteralData::NegativeInteger(_) => todo!(), - VdZfsLiteralData::FiniteDecimalRepresentation(_) => { + VdZfcLiteralData::NaturalNumber(lit) => LnLiteralData::Nat(lit.to_string()), + VdZfcLiteralData::NegativeInteger(_) => todo!(), + VdZfcLiteralData::FiniteDecimalRepresentation(_) => { todo!() } - VdZfsLiteralData::SpecialConstant(vd_zfs_special_constant) => todo!(), + VdZfcLiteralData::SpecialConstant(vd_zfc_special_constant) => todo!(), }; LnLiteral::new(data, db) } diff --git a/crates/visored/visored-lean-transpilation/src/tests.rs b/crates/visored/visored-lean-transpilation/src/tests.rs index 0385c651c7..da32302fd3 100644 --- a/crates/visored/visored-lean-transpilation/src/tests.rs +++ b/crates/visored/visored-lean-transpilation/src/tests.rs @@ -4,7 +4,7 @@ use visored_sem_expr::{ expr::{binary::VdSemBinaryDispatch, literal::VdSemLiteralDispatch, VdSemExprData}, test_helpers::builder::VdSemExprTestBuilder, }; -use visored_zfs_ty::menu::vd_zfs_ty_menu; +use visored_zfc_ty::menu::vd_zfc_ty_menu; use crate::{builder::VdLeanTranspilationBuilder, ToLean}; @@ -15,7 +15,7 @@ use crate::{builder::VdLeanTranspilationBuilder, ToLean}; lean_term::jar::LnTermJar, lean_sem_expr::jar::LnSemExprJar, lean_hir_expr::jar::LnHirExprJar, - visored_zfs_ty::jar::VdZfsTypeJar, + visored_zfc_ty::jar::VdZfcTypeJar, visored_opr::jar::VdOprJar, visored_sem_expr::jar::VdSemExprJar, visored_hir_expr::jar::VdHirExprJar, @@ -27,7 +27,7 @@ pub(crate) struct DB {} #[test] fn one_add_one_to_lean_works() { let db = &DB::default(); - let menu = vd_zfs_ty_menu(db); + let menu = vd_zfc_ty_menu(db); let mut builder = VdSemExprTestBuilder::new(db); let one = builder.new_expr_checked( VdSemExprData::Literal { @@ -59,7 +59,7 @@ fn one_add_one_to_lean_works() { #[test] fn one_add_one_eqs_two_to_lean_works() { let db = &DB::default(); - let menu = vd_zfs_ty_menu(db); + let menu = vd_zfc_ty_menu(db); let mut builder = VdSemExprTestBuilder::new(db); let one = builder.new_expr_checked( VdSemExprData::Literal { diff --git a/crates/visored/visored-sem-expr/Cargo.toml b/crates/visored/visored-sem-expr/Cargo.toml index a0417719cd..1f383dfdd8 100644 --- a/crates/visored/visored-sem-expr/Cargo.toml +++ b/crates/visored/visored-sem-expr/Cargo.toml @@ -21,7 +21,7 @@ latex-prelude.workspace = true # visored visored-annotation.workspace = true visored-opr.workspace = true -visored-zfs-ty.workspace = true +visored-zfc-ty.workspace = true visored-syn-expr.workspace = true [dev-dependencies] diff --git a/crates/visored/visored-sem-expr/src/expr.rs b/crates/visored/visored-sem-expr/src/expr.rs index 6bd639da25..1378bbdf90 100644 --- a/crates/visored/visored-sem-expr/src/expr.rs +++ b/crates/visored/visored-sem-expr/src/expr.rs @@ -14,13 +14,13 @@ use idx_arena::{Arena, ArenaIdx, ArenaIdxRange, ArenaRef}; use latex_prelude::script::LxScriptKind; use literal::VdSemLiteralDispatch; use visored_opr::opr::binary::VdBaseBinaryOpr; -use visored_zfs_ty::term::literal::VdZfsLiteral; +use visored_zfc_ty::term::literal::VdZfcLiteral; /// It's a tree of both form and meaning #[derive(Debug, PartialEq, Eq)] pub enum VdSemExprData { Literal { - literal: VdZfsLiteral, + literal: VdZfcLiteral, dispatch: VdSemLiteralDispatch, }, Notation, diff --git a/crates/visored/visored-sem-expr/src/helpers/latex_fmt.rs b/crates/visored/visored-sem-expr/src/helpers/latex_fmt.rs index ced749ddcc..9f16da68a2 100644 --- a/crates/visored/visored-sem-expr/src/helpers/latex_fmt.rs +++ b/crates/visored/visored-sem-expr/src/helpers/latex_fmt.rs @@ -1,6 +1,6 @@ use expr::{binary::VdSemBinaryDispatch, literal::VdSemLiteralDispatch}; use visored_opr::opr::binary::VdBaseBinaryOpr; -use visored_zfs_ty::{menu::vd_zfs_ty_menu, term::literal::VdZfsLiteralData}; +use visored_zfc_ty::{menu::vd_zfc_ty_menu, term::literal::VdZfcLiteralData}; use super::*; #[cfg(test)] @@ -92,7 +92,7 @@ impl<'a> VdSemExprLaTeXFormatter<'a> { literal, ref dispatch, } => match literal.data(db) { - VdZfsLiteralData::NaturalNumber(s) => { + VdZfcLiteralData::NaturalNumber(s) => { if self .result .chars() @@ -103,11 +103,11 @@ impl<'a> VdSemExprLaTeXFormatter<'a> { } self.result.push_str(s); } - VdZfsLiteralData::NegativeInteger(_) => todo!(), - VdZfsLiteralData::FiniteDecimalRepresentation(_) => { + VdZfcLiteralData::NegativeInteger(_) => todo!(), + VdZfcLiteralData::FiniteDecimalRepresentation(_) => { todo!() } - VdZfsLiteralData::SpecialConstant(vd_zfs_special_constant) => todo!(), + VdZfcLiteralData::SpecialConstant(vd_zfc_special_constant) => todo!(), }, VdSemExprData::Notation => todo!(), VdSemExprData::Binary { @@ -147,7 +147,7 @@ impl<'a> VdSemExprLaTeXFormatter<'a> { #[test] fn latex_fmt_works() { let db = &DB::default(); - let menu = vd_zfs_ty_menu(db); + let menu = vd_zfc_ty_menu(db); let mut builder = VdSemExprTestBuilder::new(db); let one = builder.new_expr_checked( VdSemExprData::Literal { diff --git a/crates/visored/visored-sem-expr/src/tests.rs b/crates/visored/visored-sem-expr/src/tests.rs index 21d330135f..ce824265d2 100644 --- a/crates/visored/visored-sem-expr/src/tests.rs +++ b/crates/visored/visored-sem-expr/src/tests.rs @@ -1,6 +1,6 @@ #[salsa::db( husky_coword::jar::CowordJar, - visored_zfs_ty::jar::VdZfsTypeJar, + visored_zfc_ty::jar::VdZfcTypeJar, visored_opr::jar::VdOprJar, latex_ast::jar::LxAstJar, latex_command::jar::LxCommandJar, diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index 2df93ba11f..ade6de7d3f 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -20,6 +20,7 @@ thiserror.workspace = true salsa.workspace = true idx-arena.workspace = true # latex +latex-math-letter.workspace = true latex-prelude.workspace = true latex-token.workspace = true latex-ast.workspace = true @@ -28,7 +29,7 @@ husky-tree-utils.workspace = true # visored visored-annotation.workspace = true visored-opr.workspace = true -visored-zfs-ty.workspace = true +visored-zfc-ty.workspace = true [dev-dependencies] expect-test.workspace = true diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 7d2cbe79d6..9d3a77ab62 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -19,17 +19,19 @@ use idx_arena::{ }; use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; use latex_prelude::script::LxScriptKind; +use latex_token::idx::LxTokenIdxRange; use range::VdSynExprAstRange; use visored_opr::opr::{ binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr, suffix::VdBaseSuffixOpr, VdBaseOpr, }; -use visored_zfs_ty::term::literal::VdZfsLiteral; +use visored_zfc_ty::term::literal::VdZfcLiteral; /// It's a tree of both form and meaning #[derive(Debug, PartialEq, Eq)] pub enum VdSynExprData { Literal { - literal: VdZfsLiteral, + token_idx_range: LxTokenIdxRange, + literal: VdZfcLiteral, }, Notation, BaseOpr { @@ -70,7 +72,7 @@ pub type VdSynExprArenaRef<'a> = ArenaRef<'a, VdSynExprData>; impl VdSynExprData { pub fn children(&self) -> Vec { match *self { - VdSynExprData::Literal { literal } => vec![], + VdSynExprData::Literal { .. } => vec![], VdSynExprData::Notation => vec![], VdSynExprData::BaseOpr { opr } => vec![], VdSynExprData::Binary { lopd, opr, ropd } => match opr { @@ -180,7 +182,7 @@ mod tests { &[], &[], &expect![[r#" - + "1\n" "#]], ); } diff --git a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs index 5ebbe7518f..7b0624b019 100644 --- a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs +++ b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs @@ -11,8 +11,9 @@ use crate::{ sentence::{VdSynSentenceArenaRef, VdSynSentenceData, VdSynSentenceIdx, VdSynSentenceIdxRange}, }; use either::*; +use latex_token::idx::LxTokenIdxRange; use visored_opr::opr::binary::VdBaseBinaryOpr; -use visored_zfs_ty::{menu::vd_zfs_ty_menu, term::literal::VdZfsLiteralData}; +use visored_zfc_ty::{menu::vd_zfc_ty_menu, term::literal::VdZfcLiteralData}; pub struct VdSynExprLaTeXFormatter<'a> { db: &'a ::salsa::Db, @@ -87,8 +88,8 @@ impl<'a> VdSynExprLaTeXFormatter<'a> { pub fn fmt_expr(&mut self, expr_idx: VdSynExprIdx) { let db = self.db; match self.expr_arena[expr_idx] { - VdSynExprData::Literal { literal } => match literal.data(db) { - VdZfsLiteralData::NaturalNumber(s) => { + VdSynExprData::Literal { literal, .. } => match literal.data(db) { + VdZfcLiteralData::NaturalNumber(s) => { if self .result .chars() @@ -99,11 +100,11 @@ impl<'a> VdSynExprLaTeXFormatter<'a> { } self.result.push_str(s); } - VdZfsLiteralData::NegativeInteger(_) => todo!(), - VdZfsLiteralData::FiniteDecimalRepresentation(_) => { + VdZfcLiteralData::NegativeInteger(_) => todo!(), + VdZfcLiteralData::FiniteDecimalRepresentation(_) => { todo!() } - VdZfsLiteralData::SpecialConstant(vd_zfs_special_constant) => todo!(), + VdZfcLiteralData::SpecialConstant(vd_zfc_special_constant) => todo!(), }, VdSynExprData::Notation => todo!(), VdSynExprData::Binary { @@ -134,13 +135,15 @@ impl<'a> VdSynExprLaTeXFormatter<'a> { } #[test] +#[ignore] fn latex_fmt_works() { let db = &DB::default(); - let menu = vd_zfs_ty_menu(db); + let menu = vd_zfc_ty_menu(db); let mut builder = VdSynExprTestBuilder::new(db); let one = builder.new_expr_checked( VdSynExprData::Literal { literal: menu.one_literal(), + token_idx_range: todo!(), }, "1", ); diff --git a/crates/visored/visored-syn-expr/src/parser.rs b/crates/visored/visored-syn-expr/src/parser.rs index f39e1df09e..cf4a274110 100644 --- a/crates/visored/visored-syn-expr/src/parser.rs +++ b/crates/visored/visored-syn-expr/src/parser.rs @@ -35,16 +35,18 @@ impl<'db> VdSynExprBuilder<'db> { impl<'a, 'db> VdSynExprParser<'a, 'db> { pub fn parse_asts(mut self, asts: LxMathAstIdxRange) -> VdSynExprIdx { - for ast in asts { - self.parse_ast(ast); + let mut next = asts.start(); + let end = asts.end(); + while next < end { + self.parse_ast(&mut next, end); } let Self { builder, stack } = self; let data = stack.finish(); builder.alloc_expr(data, VdSynExprAstRange::Asts(asts.into())) } - fn parse_ast(&mut self, ast: LxMathAstIdx) { - let token = self.disambiguate_token(ast); + fn parse_ast(&mut self, next: &mut LxMathAstIdx, end: LxMathAstIdx) { + let token = self.disambiguate_token(next, end); self.accept_token(token); } } diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs index 230bff0008..cea08f93d2 100644 --- a/crates/visored/visored-syn-expr/src/parser/accept.rs +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -23,6 +23,9 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { }, DisambiguatedToken::Opr(opr) => self.accept_opr(opr), DisambiguatedToken::Separator(sep) => self.accept_separator(sep), + DisambiguatedToken::LeftDelimiter(vd_left_delimiter) => todo!(), + DisambiguatedToken::RightDelimiter(vd_right_delimiter) => todo!(), + DisambiguatedToken::Letter(lx_math_token_idx, lx_math_letter) => todo!(), } } diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs index 1cd5ef8a21..f4613c3cda 100644 --- a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -1,15 +1,20 @@ -use latex_ast::ast::math::LxMathAstIdx; +use super::{ + expr::{VdSynExprClass, VdSynExprData, VdSynExprIdx}, + VdSynExprParser, +}; +use latex_ast::ast::math::{LxMathAstData, LxMathAstIdx}; +use latex_math_letter::LxMathLetter; +use latex_token::{ + data::math::digit::LxMathDigit, + idx::{LxMathTokenIdx, LxTokenIdxRange}, +}; use visored_annotation::annotation::space::VdSpaceAnnotation; use visored_opr::{ delimiter::{VdLeftDelimiter, VdRightDelimiter}, opr::VdBaseOpr, separator::VdSeparator, }; - -use super::{ - expr::{VdSynExprClass, VdSynExprData, VdSynExprIdx}, - VdSynExprParser, -}; +use visored_zfc_ty::term::literal::{VdZfcLiteral, VdZfcLiteralData}; pub struct DisambiguatedMathAst { ast: LxMathAstIdx, @@ -22,10 +27,60 @@ pub enum DisambiguatedToken { Separator(VdSeparator), LeftDelimiter(VdLeftDelimiter), RightDelimiter(VdRightDelimiter), + Letter(LxMathTokenIdx, LxMathLetter), } impl<'a, 'db> VdSynExprParser<'a, 'db> { - pub fn disambiguate_token(&mut self, ast: LxMathAstIdx) -> DisambiguatedToken { - todo!() + pub fn disambiguate_token( + &mut self, + next: &mut LxMathAstIdx, + end: LxMathAstIdx, + ) -> DisambiguatedToken { + let ast_data = &self.builder.ast_arena()[*next]; + match *ast_data { + LxMathAstData::Letter(lx_math_token_idx, lx_math_letter) => { + DisambiguatedToken::Letter(lx_math_token_idx, lx_math_letter) + } + LxMathAstData::Opr(lx_math_token_idx, lx_math_punctuation) => { + todo!() + } + LxMathAstData::Digit(first_token_idx, digit) => { + let mut last_token_idx = first_token_idx; + let mut s = String::new(); + // TODO: handle real number by using a kind variable, literal number kind + while *next < end { + match self.builder.ast_arena()[*next] { + LxMathAstData::Digit(token_idx, digit) => { + last_token_idx = token_idx; + s.push(digit.char()) + } + // TODO: handle real number + _ => break, + } + *next += 1; + } + let expr_data = VdSynExprData::Literal { + token_idx_range: LxTokenIdxRange::new_closed(*first_token_idx, *last_token_idx), + literal: VdZfcLiteral::new( + VdZfcLiteralData::NaturalNumber(s), + self.builder.db(), + ), + }; + DisambiguatedToken::Expr(expr_data, VdSynExprClass::Atom) + } + LxMathAstData::TextEdit { ref buffer } => todo!(), + LxMathAstData::Attach { base, ref scripts } => todo!(), + LxMathAstData::Delimited { + left_delimiter_token_idx, + left_delimiter, + asts, + right_delimiter_token_idx, + right_delimiter, + } => todo!(), + LxMathAstData::Command { + command_token_idx, + command_path, + } => todo!(), + } } } diff --git a/crates/visored/visored-syn-expr/src/tests.rs b/crates/visored/visored-syn-expr/src/tests.rs index add60bbc61..6102bf191a 100644 --- a/crates/visored/visored-syn-expr/src/tests.rs +++ b/crates/visored/visored-syn-expr/src/tests.rs @@ -1,6 +1,6 @@ #[salsa::db( husky_coword::jar::CowordJar, - visored_zfs_ty::jar::VdZfsTypeJar, + visored_zfc_ty::jar::VdZfcTypeJar, latex_ast::jar::LxAstJar, latex_command::jar::LxCommandJar, visored_opr::jar::VdOprJar, diff --git a/crates/visored/visored-zfs-ty/Cargo.toml b/crates/visored/visored-zfc-ty/Cargo.toml similarity index 94% rename from crates/visored/visored-zfs-ty/Cargo.toml rename to crates/visored/visored-zfc-ty/Cargo.toml index a441378e05..89190ee15b 100644 --- a/crates/visored/visored-zfs-ty/Cargo.toml +++ b/crates/visored/visored-zfc-ty/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "visored-zfs-ty" +name = "visored-zfc-ty" version.workspace = true rust-version.workspace = true authors.workspace = true diff --git a/crates/visored/visored-zfc-ty/src/jar.rs b/crates/visored/visored-zfc-ty/src/jar.rs new file mode 100644 index 0000000000..5c2797cbd3 --- /dev/null +++ b/crates/visored/visored-zfc-ty/src/jar.rs @@ -0,0 +1,2 @@ +#[salsa::jar] +pub struct VdZfcTypeJar(crate::term::VdZfcTermId, crate::menu::vd_zfc_ty_menu); diff --git a/crates/visored/visored-zfs-ty/src/lib.rs b/crates/visored/visored-zfc-ty/src/lib.rs similarity index 67% rename from crates/visored/visored-zfs-ty/src/lib.rs rename to crates/visored/visored-zfc-ty/src/lib.rs index 440667b04f..d59cbf292c 100644 --- a/crates/visored/visored-zfs-ty/src/lib.rs +++ b/crates/visored/visored-zfc-ty/src/lib.rs @@ -4,4 +4,4 @@ pub mod refinement; pub mod term; pub mod ty; -use self::jar::VdZfsTypeJar as Jar; +use self::jar::VdZfcTypeJar as Jar; diff --git a/crates/visored/visored-zfc-ty/src/menu.rs b/crates/visored/visored-zfc-ty/src/menu.rs new file mode 100644 index 0000000000..07c72299ad --- /dev/null +++ b/crates/visored/visored-zfc-ty/src/menu.rs @@ -0,0 +1,37 @@ +use crate::term::literal::{VdZfcLiteral, VdZfcLiteralData}; + +#[derive(Debug, PartialEq, Eq)] +pub struct VdZfcTypeMenu { + zero_literal: VdZfcLiteral, + one_literal: VdZfcLiteral, + two_literal: VdZfcLiteral, +} + +impl VdZfcTypeMenu { + fn new(db: &::salsa::Db) -> Self { + Self { + zero_literal: VdZfcLiteral::new(VdZfcLiteralData::NaturalNumber("0".to_string()), db), + one_literal: VdZfcLiteral::new(VdZfcLiteralData::NaturalNumber("1".to_string()), db), + two_literal: VdZfcLiteral::new(VdZfcLiteralData::NaturalNumber("2".to_string()), db), + } + } +} + +impl VdZfcTypeMenu { + pub fn zero_literal(&self) -> VdZfcLiteral { + self.zero_literal + } + + pub fn one_literal(&self) -> VdZfcLiteral { + self.one_literal + } + + pub fn two_literal(&self) -> VdZfcLiteral { + self.two_literal + } +} + +#[salsa::tracked(return_ref)] +pub fn vd_zfc_ty_menu(db: &::salsa::Db) -> VdZfcTypeMenu { + VdZfcTypeMenu::new(db) +} diff --git a/crates/visored/visored-zfc-ty/src/refinement.rs b/crates/visored/visored-zfc-ty/src/refinement.rs new file mode 100644 index 0000000000..9ac041f3d7 --- /dev/null +++ b/crates/visored/visored-zfc-ty/src/refinement.rs @@ -0,0 +1,10 @@ +use crate::{term::VdZfcTermId, ty::VdZfcTypeData}; + +/// It's a term of type A -> B, where $A$ is the base type. +pub struct ZfcTypeRefinement {} + +impl ZfcTypeRefinement { + pub fn new(term: VdZfcTermId, data: &VdZfcTypeData, prev_refinements: &[Self]) -> Self { + todo!() + } +} diff --git a/crates/visored/visored-zfc-ty/src/term.rs b/crates/visored/visored-zfc-ty/src/term.rs new file mode 100644 index 0000000000..a7ba80dfd0 --- /dev/null +++ b/crates/visored/visored-zfc-ty/src/term.rs @@ -0,0 +1,61 @@ +pub mod abstract_variable; +pub mod abstraction; +pub mod application; +pub mod eval; +pub mod exists; +pub mod forall; +pub mod limit; +pub mod literal; +pub mod stack_variable; +pub mod symbolic_variable; + +use self::{ + abstract_variable::{VdZfcAbstractVariable, VdZfcAbstractVariableData}, + abstraction::{VdZfcAbstraction, VdZfcAbstractionData}, + application::{VdZfcApplication, VdZfcApplicationData}, + eval::{VdZfcEval, VdZfcEvalData}, + exists::{VdZfcExists, VdZfcExistsData}, + forall::{VdZfcForAll, VdZfcForAllData}, + limit::{VdZfcLimit, VdZfcLimitData}, + literal::{VdZfcLiteral, VdZfcLiteralData}, + stack_variable::{VdZfcStackVariable, VdZfcStackVariableData}, + symbolic_variable::{VdZfcSymbolicVariable, VdZfcSymbolicVariableData}, +}; +use smallvec::SmallVec; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum VdZfcTerm { + Literal(VdZfcLiteral), + ForAll(VdZfcForAll), + Exists(VdZfcExists), + Limit(VdZfcLimit), + Eval(VdZfcEval), + SymbolicVariable(VdZfcSymbolicVariable), + AbstractVariable(VdZfcAbstractVariable), + StackVariable(VdZfcStackVariable), + Application(VdZfcApplication), + Abstraction(VdZfcAbstraction), +} + +pub type ZfcTerms = SmallVec<[VdZfcTerm; 4]>; + +#[salsa::interned] +pub struct VdZfcTermId { + #[return_ref] + data: VdZfcTermData, +} + +#[enum_class::from_variants] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum VdZfcTermData { + Literal(VdZfcLiteralData), + ForAll(VdZfcForAllData), + Exists(VdZfcExistsData), + Limit(VdZfcLimitData), + Eval(VdZfcEvalData), + SymbolicVariable(VdZfcSymbolicVariableData), + AbstractVariable(VdZfcAbstractVariableData), + StackVariable(VdZfcStackVariableData), + Application(VdZfcApplicationData), + Abstraction(VdZfcAbstractionData), +} diff --git a/crates/visored/visored-zfs-ty/src/term/abstract_variable.rs b/crates/visored/visored-zfc-ty/src/term/abstract_variable.rs similarity index 53% rename from crates/visored/visored-zfs-ty/src/term/abstract_variable.rs rename to crates/visored/visored-zfc-ty/src/term/abstract_variable.rs index 6b4ecd60d6..b2097f71df 100644 --- a/crates/visored/visored-zfs-ty/src/term/abstract_variable.rs +++ b/crates/visored/visored-zfc-ty/src/term/abstract_variable.rs @@ -1,17 +1,17 @@ use super::*; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsAbstractVariable(VdZfsTermId); +pub struct VdZfcAbstractVariable(VdZfcTermId); #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsAbstractVariableData { +pub struct VdZfcAbstractVariableData { // Add appropriate fields here } -impl VdZfsAbstractVariable { - pub fn data(self, db: &::salsa::Db) -> &VdZfsAbstractVariableData { +impl VdZfcAbstractVariable { + pub fn data(self, db: &::salsa::Db) -> &VdZfcAbstractVariableData { match self.0.data(db) { - VdZfsTermData::AbstractVariable(data) => data, + VdZfcTermData::AbstractVariable(data) => data, _ => unreachable!(), } } diff --git a/crates/visored/visored-zfc-ty/src/term/abstraction.rs b/crates/visored/visored-zfc-ty/src/term/abstraction.rs new file mode 100644 index 0000000000..54844cc8db --- /dev/null +++ b/crates/visored/visored-zfc-ty/src/term/abstraction.rs @@ -0,0 +1,19 @@ +use super::{VdZfcTerm, VdZfcTermData, VdZfcTermId, ZfcTerms}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct VdZfcAbstraction(VdZfcTermId); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct VdZfcAbstractionData { + pub parameters: VdZfcTerm, + pub arguments: ZfcTerms, +} + +impl VdZfcAbstraction { + pub fn data(self, db: &::salsa::Db) -> &VdZfcAbstractionData { + match self.0.data(db) { + VdZfcTermData::Abstraction(data) => data, + _ => unreachable!(), + } + } +} diff --git a/crates/visored/visored-zfc-ty/src/term/application.rs b/crates/visored/visored-zfc-ty/src/term/application.rs new file mode 100644 index 0000000000..700d4d9588 --- /dev/null +++ b/crates/visored/visored-zfc-ty/src/term/application.rs @@ -0,0 +1,19 @@ +use super::{VdZfcTerm, VdZfcTermData, VdZfcTermId, ZfcTerms}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct VdZfcApplication(VdZfcTermId); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct VdZfcApplicationData { + pub function: VdZfcTerm, + pub arguments: ZfcTerms, +} + +impl VdZfcApplication { + pub fn data(self, db: &::salsa::Db) -> &VdZfcApplicationData { + match self.0.data(db) { + VdZfcTermData::Application(data) => data, + _ => unreachable!(), + } + } +} diff --git a/crates/visored/visored-zfs-ty/src/term/eval.rs b/crates/visored/visored-zfc-ty/src/term/eval.rs similarity index 58% rename from crates/visored/visored-zfs-ty/src/term/eval.rs rename to crates/visored/visored-zfc-ty/src/term/eval.rs index ec8d8a1c3b..f4f82d2abc 100644 --- a/crates/visored/visored-zfs-ty/src/term/eval.rs +++ b/crates/visored/visored-zfc-ty/src/term/eval.rs @@ -1,17 +1,17 @@ use super::*; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsEval(VdZfsTermId); +pub struct VdZfcEval(VdZfcTermId); #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsEvalData { +pub struct VdZfcEvalData { // Add appropriate fields here } -impl VdZfsEval { - pub fn data(self, db: &::salsa::Db) -> &VdZfsEvalData { +impl VdZfcEval { + pub fn data(self, db: &::salsa::Db) -> &VdZfcEvalData { match self.0.data(db) { - VdZfsTermData::Eval(data) => data, + VdZfcTermData::Eval(data) => data, _ => unreachable!(), } } diff --git a/crates/visored/visored-zfs-ty/src/term/exists.rs b/crates/visored/visored-zfc-ty/src/term/exists.rs similarity index 57% rename from crates/visored/visored-zfs-ty/src/term/exists.rs rename to crates/visored/visored-zfc-ty/src/term/exists.rs index 80631376bf..5228052944 100644 --- a/crates/visored/visored-zfs-ty/src/term/exists.rs +++ b/crates/visored/visored-zfc-ty/src/term/exists.rs @@ -1,17 +1,17 @@ use super::*; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsExists(VdZfsTermId); +pub struct VdZfcExists(VdZfcTermId); #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsExistsData { +pub struct VdZfcExistsData { // Add appropriate fields here } -impl VdZfsExists { - pub fn data(self, db: &::salsa::Db) -> &VdZfsExistsData { +impl VdZfcExists { + pub fn data(self, db: &::salsa::Db) -> &VdZfcExistsData { match self.0.data(db) { - VdZfsTermData::Exists(data) => data, + VdZfcTermData::Exists(data) => data, _ => unreachable!(), } } diff --git a/crates/visored/visored-zfs-ty/src/term/forall.rs b/crates/visored/visored-zfc-ty/src/term/forall.rs similarity index 57% rename from crates/visored/visored-zfs-ty/src/term/forall.rs rename to crates/visored/visored-zfc-ty/src/term/forall.rs index d98943c5e9..bc407c9404 100644 --- a/crates/visored/visored-zfs-ty/src/term/forall.rs +++ b/crates/visored/visored-zfc-ty/src/term/forall.rs @@ -1,17 +1,17 @@ use super::*; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsForAll(VdZfsTermId); +pub struct VdZfcForAll(VdZfcTermId); #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsForAllData { +pub struct VdZfcForAllData { // Add appropriate fields here } -impl VdZfsForAll { - pub fn data(self, db: &::salsa::Db) -> &VdZfsForAllData { +impl VdZfcForAll { + pub fn data(self, db: &::salsa::Db) -> &VdZfcForAllData { match self.0.data(db) { - VdZfsTermData::ForAll(data) => data, + VdZfcTermData::ForAll(data) => data, _ => unreachable!(), } } diff --git a/crates/visored/visored-zfs-ty/src/term/limit.rs b/crates/visored/visored-zfc-ty/src/term/limit.rs similarity index 57% rename from crates/visored/visored-zfs-ty/src/term/limit.rs rename to crates/visored/visored-zfc-ty/src/term/limit.rs index c880f76ae8..7b5b39481f 100644 --- a/crates/visored/visored-zfs-ty/src/term/limit.rs +++ b/crates/visored/visored-zfc-ty/src/term/limit.rs @@ -1,17 +1,17 @@ use super::*; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsLimit(VdZfsTermId); +pub struct VdZfcLimit(VdZfcTermId); #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsLimitData { +pub struct VdZfcLimitData { // Add appropriate fields here } -impl VdZfsLimit { - pub fn data(self, db: &::salsa::Db) -> &VdZfsLimitData { +impl VdZfcLimit { + pub fn data(self, db: &::salsa::Db) -> &VdZfcLimitData { match self.0.data(db) { - VdZfsTermData::Limit(data) => data, + VdZfcTermData::Limit(data) => data, _ => unreachable!(), } } diff --git a/crates/visored/visored-zfc-ty/src/term/literal.rs b/crates/visored/visored-zfc-ty/src/term/literal.rs new file mode 100644 index 0000000000..8cfe7dc46a --- /dev/null +++ b/crates/visored/visored-zfc-ty/src/term/literal.rs @@ -0,0 +1,30 @@ +pub mod special_constant; + +use self::special_constant::VdZfcSpecialConstant; +use super::*; + +#[salsa::as_id] +#[salsa::deref_id] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct VdZfcLiteral(VdZfcTermId); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum VdZfcLiteralData { + NaturalNumber(String), + NegativeInteger(String), + FiniteDecimalRepresentation(String), + SpecialConstant(VdZfcSpecialConstant), +} + +impl VdZfcLiteral { + pub fn data(self, db: &::salsa::Db) -> &VdZfcLiteralData { + match self.0.data(db) { + VdZfcTermData::Literal(data) => data, + _ => unreachable!(), + } + } + + pub fn new(data: VdZfcLiteralData, db: &::salsa::Db) -> Self { + Self(VdZfcTermId::new(db, data.into())) + } +} diff --git a/crates/visored/visored-zfs-ty/src/term/literal/special_constant.rs b/crates/visored/visored-zfc-ty/src/term/literal/special_constant.rs similarity index 73% rename from crates/visored/visored-zfs-ty/src/term/literal/special_constant.rs rename to crates/visored/visored-zfc-ty/src/term/literal/special_constant.rs index 0bc7ffc278..d1e0d0f716 100644 --- a/crates/visored/visored-zfs-ty/src/term/literal/special_constant.rs +++ b/crates/visored/visored-zfc-ty/src/term/literal/special_constant.rs @@ -1,5 +1,5 @@ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum VdZfsSpecialConstant { +pub enum VdZfcSpecialConstant { Pi, Gamma, } diff --git a/crates/visored/visored-zfs-ty/src/term/stack_variable.rs b/crates/visored/visored-zfc-ty/src/term/stack_variable.rs similarity index 54% rename from crates/visored/visored-zfs-ty/src/term/stack_variable.rs rename to crates/visored/visored-zfc-ty/src/term/stack_variable.rs index cd656d1ee1..ebf8f1af90 100644 --- a/crates/visored/visored-zfs-ty/src/term/stack_variable.rs +++ b/crates/visored/visored-zfc-ty/src/term/stack_variable.rs @@ -1,17 +1,17 @@ use super::*; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsStackVariable(VdZfsTermId); +pub struct VdZfcStackVariable(VdZfcTermId); #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsStackVariableData { +pub struct VdZfcStackVariableData { // Add appropriate fields here } -impl VdZfsStackVariable { - pub fn data(self, db: &::salsa::Db) -> &VdZfsStackVariableData { +impl VdZfcStackVariable { + pub fn data(self, db: &::salsa::Db) -> &VdZfcStackVariableData { match self.0.data(db) { - VdZfsTermData::StackVariable(data) => data, + VdZfcTermData::StackVariable(data) => data, _ => unreachable!(), } } diff --git a/crates/visored/visored-zfs-ty/src/term/symbolic_variable.rs b/crates/visored/visored-zfc-ty/src/term/symbolic_variable.rs similarity index 67% rename from crates/visored/visored-zfs-ty/src/term/symbolic_variable.rs rename to crates/visored/visored-zfc-ty/src/term/symbolic_variable.rs index 44ee72bbd8..2b8daf8fa6 100644 --- a/crates/visored/visored-zfs-ty/src/term/symbolic_variable.rs +++ b/crates/visored/visored-zfc-ty/src/term/symbolic_variable.rs @@ -1,9 +1,9 @@ use super::*; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsSymbolicVariable(VdZfsTermId); +pub struct VdZfcSymbolicVariable(VdZfcTermId); #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsSymbolicVariableData { +pub struct VdZfcSymbolicVariableData { // Add appropriate fields here } diff --git a/crates/visored/visored-zfs-ty/src/ty.rs b/crates/visored/visored-zfc-ty/src/ty.rs similarity index 67% rename from crates/visored/visored-zfs-ty/src/ty.rs rename to crates/visored/visored-zfc-ty/src/ty.rs index 3947136d68..ca6d539c59 100644 --- a/crates/visored/visored-zfs-ty/src/ty.rs +++ b/crates/visored/visored-zfc-ty/src/ty.rs @@ -1,11 +1,11 @@ use smallvec::SmallVec; use visored_coword::namae::VdNamae; -pub struct VdZfsType { - data: VdZfsTypeData, +pub struct VdZfcType { + data: VdZfcTypeData, refinements: SmallVec<[(); 2]>, } -pub enum VdZfsTypeData { +pub enum VdZfcTypeData { Named(VdNamae), // TODO: do we need a path here? } diff --git a/crates/visored/visored-zfs-ty/src/jar.rs b/crates/visored/visored-zfs-ty/src/jar.rs deleted file mode 100644 index 4917fc1d39..0000000000 --- a/crates/visored/visored-zfs-ty/src/jar.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[salsa::jar] -pub struct VdZfsTypeJar(crate::term::VdZfsTermId, crate::menu::vd_zfs_ty_menu); diff --git a/crates/visored/visored-zfs-ty/src/menu.rs b/crates/visored/visored-zfs-ty/src/menu.rs deleted file mode 100644 index 42b6baa850..0000000000 --- a/crates/visored/visored-zfs-ty/src/menu.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::term::literal::{VdZfsLiteral, VdZfsLiteralData}; - -#[derive(Debug, PartialEq, Eq)] -pub struct VdZfsTypeMenu { - zero_literal: VdZfsLiteral, - one_literal: VdZfsLiteral, - two_literal: VdZfsLiteral, -} - -impl VdZfsTypeMenu { - fn new(db: &::salsa::Db) -> Self { - Self { - zero_literal: VdZfsLiteral::new(VdZfsLiteralData::NaturalNumber("0".to_string()), db), - one_literal: VdZfsLiteral::new(VdZfsLiteralData::NaturalNumber("1".to_string()), db), - two_literal: VdZfsLiteral::new(VdZfsLiteralData::NaturalNumber("2".to_string()), db), - } - } -} - -impl VdZfsTypeMenu { - pub fn zero_literal(&self) -> VdZfsLiteral { - self.zero_literal - } - - pub fn one_literal(&self) -> VdZfsLiteral { - self.one_literal - } - - pub fn two_literal(&self) -> VdZfsLiteral { - self.two_literal - } -} - -#[salsa::tracked(return_ref)] -pub fn vd_zfs_ty_menu(db: &::salsa::Db) -> VdZfsTypeMenu { - VdZfsTypeMenu::new(db) -} diff --git a/crates/visored/visored-zfs-ty/src/refinement.rs b/crates/visored/visored-zfs-ty/src/refinement.rs deleted file mode 100644 index d6b01fc44d..0000000000 --- a/crates/visored/visored-zfs-ty/src/refinement.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::{term::VdZfsTermId, ty::VdZfsTypeData}; - -/// It's a term of type A -> B, where $A$ is the base type. -pub struct ZfsTypeRefinement {} - -impl ZfsTypeRefinement { - pub fn new(term: VdZfsTermId, data: &VdZfsTypeData, prev_refinements: &[Self]) -> Self { - todo!() - } -} diff --git a/crates/visored/visored-zfs-ty/src/term.rs b/crates/visored/visored-zfs-ty/src/term.rs deleted file mode 100644 index d657daa10e..0000000000 --- a/crates/visored/visored-zfs-ty/src/term.rs +++ /dev/null @@ -1,61 +0,0 @@ -pub mod abstract_variable; -pub mod abstraction; -pub mod application; -pub mod eval; -pub mod exists; -pub mod forall; -pub mod limit; -pub mod literal; -pub mod stack_variable; -pub mod symbolic_variable; - -use self::{ - abstract_variable::{VdZfsAbstractVariable, VdZfsAbstractVariableData}, - abstraction::{VdZfsAbstraction, VdZfsAbstractionData}, - application::{VdZfsApplication, VdZfsApplicationData}, - eval::{VdZfsEval, VdZfsEvalData}, - exists::{VdZfsExists, VdZfsExistsData}, - forall::{VdZfsForAll, VdZfsForAllData}, - limit::{VdZfsLimit, VdZfsLimitData}, - literal::{VdZfsLiteral, VdZfsLiteralData}, - stack_variable::{VdZfsStackVariable, VdZfsStackVariableData}, - symbolic_variable::{VdZfsSymbolicVariable, VdZfsSymbolicVariableData}, -}; -use smallvec::SmallVec; - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum VdZfsTerm { - Literal(VdZfsLiteral), - ForAll(VdZfsForAll), - Exists(VdZfsExists), - Limit(VdZfsLimit), - Eval(VdZfsEval), - SymbolicVariable(VdZfsSymbolicVariable), - AbstractVariable(VdZfsAbstractVariable), - StackVariable(VdZfsStackVariable), - Application(VdZfsApplication), - Abstraction(VdZfsAbstraction), -} - -pub type ZfsTerms = SmallVec<[VdZfsTerm; 4]>; - -#[salsa::interned] -pub struct VdZfsTermId { - #[return_ref] - data: VdZfsTermData, -} - -#[enum_class::from_variants] -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum VdZfsTermData { - Literal(VdZfsLiteralData), - ForAll(VdZfsForAllData), - Exists(VdZfsExistsData), - Limit(VdZfsLimitData), - Eval(VdZfsEvalData), - SymbolicVariable(VdZfsSymbolicVariableData), - AbstractVariable(VdZfsAbstractVariableData), - StackVariable(VdZfsStackVariableData), - Application(VdZfsApplicationData), - Abstraction(VdZfsAbstractionData), -} diff --git a/crates/visored/visored-zfs-ty/src/term/abstraction.rs b/crates/visored/visored-zfs-ty/src/term/abstraction.rs deleted file mode 100644 index 04252d662e..0000000000 --- a/crates/visored/visored-zfs-ty/src/term/abstraction.rs +++ /dev/null @@ -1,19 +0,0 @@ -use super::{VdZfsTerm, VdZfsTermData, VdZfsTermId, ZfsTerms}; - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsAbstraction(VdZfsTermId); - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsAbstractionData { - pub parameters: VdZfsTerm, - pub arguments: ZfsTerms, -} - -impl VdZfsAbstraction { - pub fn data(self, db: &::salsa::Db) -> &VdZfsAbstractionData { - match self.0.data(db) { - VdZfsTermData::Abstraction(data) => data, - _ => unreachable!(), - } - } -} diff --git a/crates/visored/visored-zfs-ty/src/term/application.rs b/crates/visored/visored-zfs-ty/src/term/application.rs deleted file mode 100644 index 7bb4e3acd8..0000000000 --- a/crates/visored/visored-zfs-ty/src/term/application.rs +++ /dev/null @@ -1,19 +0,0 @@ -use super::{VdZfsTerm, VdZfsTermData, VdZfsTermId, ZfsTerms}; - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsApplication(VdZfsTermId); - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsApplicationData { - pub function: VdZfsTerm, - pub arguments: ZfsTerms, -} - -impl VdZfsApplication { - pub fn data(self, db: &::salsa::Db) -> &VdZfsApplicationData { - match self.0.data(db) { - VdZfsTermData::Application(data) => data, - _ => unreachable!(), - } - } -} diff --git a/crates/visored/visored-zfs-ty/src/term/literal.rs b/crates/visored/visored-zfs-ty/src/term/literal.rs deleted file mode 100644 index b88da4149f..0000000000 --- a/crates/visored/visored-zfs-ty/src/term/literal.rs +++ /dev/null @@ -1,30 +0,0 @@ -pub mod special_constant; - -use self::special_constant::VdZfsSpecialConstant; -use super::*; - -#[salsa::as_id] -#[salsa::deref_id] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct VdZfsLiteral(VdZfsTermId); - -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum VdZfsLiteralData { - NaturalNumber(String), - NegativeInteger(String), - FiniteDecimalRepresentation(String), - SpecialConstant(VdZfsSpecialConstant), -} - -impl VdZfsLiteral { - pub fn data(self, db: &::salsa::Db) -> &VdZfsLiteralData { - match self.0.data(db) { - VdZfsTermData::Literal(data) => data, - _ => unreachable!(), - } - } - - pub(crate) fn new(data: VdZfsLiteralData, db: &::salsa::Db) -> Self { - Self(VdZfsTermId::new(db, data.into())) - } -} From 05f4f683d3eb80c6200fac7bf37a30fa2696cb62 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 23:47:22 -0700 Subject: [PATCH 60/71] feat: parse 11 in visored-syn-expr --- crates/visored/visored-syn-expr/src/expr.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 9d3a77ab62..6694f623ab 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -185,5 +185,13 @@ mod tests { "1\n" "#]], ); + t( + "11", + &[], + &[], + &expect![[r#" + "11\n" + "#]], + ); } } From 4cd3dd2c29fdb05ca5c41c9af38a725e2b6d260b Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Thu, 31 Oct 2024 23:52:30 -0700 Subject: [PATCH 61/71] save --- .../expect-files/annotations/examples.txt | 68 +++++++++++++++---- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/crates/visored/visored-annotation/expect-files/annotations/examples.txt b/crates/visored/visored-annotation/expect-files/annotations/examples.txt index 9eda51ee67..8e08d9f278 100644 --- a/crates/visored/visored-annotation/expect-files/annotations/examples.txt +++ b/crates/visored/visored-annotation/expect-files/annotations/examples.txt @@ -1,25 +1,27 @@ [ VdAnnotationsExample { - root_mode: Math, input: "", + root_mode: Math, annotations: VdAnnotations { + token_annotation_records: [], + space_annotation_records: [], token_annotations: [], space_annotations: [], }, }, VdAnnotationsExample { - root_mode: Math, input: "xy", + root_mode: Math, annotations: VdAnnotations { - token_annotations: [ - VdAnnotationEntry { + token_annotation_records: [ + VdAnnotationRecord { start: 0, end: 1, annotation: Variable( Usage, ), }, - VdAnnotationEntry { + VdAnnotationRecord { start: 1, end: 2, annotation: Variable( @@ -27,8 +29,8 @@ ), }, ], - space_annotations: [ - VdAnnotationEntry { + space_annotation_records: [ + VdAnnotationRecord { start: 1, end: 2, annotation: Apply( @@ -36,19 +38,43 @@ ), }, ], + token_annotations: [ + Some( + Variable( + Usage, + ), + ), + Some( + Variable( + Usage, + ), + ), + None, + None, + ], + space_annotations: [ + None, + Some( + Apply( + ScalarMul, + ), + ), + None, + None, + ], }, }, VdAnnotationsExample { - root_mode: Math, input: "dx", + root_mode: Math, annotations: VdAnnotations { - token_annotations: [ - VdAnnotationEntry { + token_annotation_records: [ + VdAnnotationRecord { start: 0, end: 1, annotation: Differential, }, - VdAnnotationEntry { + VdAnnotationRecord { start: 1, end: 2, annotation: Variable( @@ -56,7 +82,25 @@ ), }, ], - space_annotations: [], + space_annotation_records: [], + token_annotations: [ + Some( + Differential, + ), + Some( + Variable( + SingleVariableIntegralVariableDecl, + ), + ), + None, + None, + ], + space_annotations: [ + None, + None, + None, + None, + ], }, }, ] \ No newline at end of file From 0e8578f0f59c50fe0b55de33f4c938ed21dd694e Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 01:57:11 -0700 Subject: [PATCH 62/71] save --- Cargo.lock | 1 + crates/latex/latex-token/src/storage.rs | 19 +++ .../expect-files/husky_lang_packages.txt | 1 + crates/visored/visored-opr/src/delimiter.rs | 10 +- crates/visored/visored-opr/src/precedence.rs | 1 + crates/visored/visored-opr/src/separator.rs | 23 ++- crates/visored/visored-syn-expr/Cargo.toml | 1 + .../visored/visored-syn-expr/src/builder.rs | 54 ++----- crates/visored/visored-syn-expr/src/expr.rs | 60 ++++++-- .../src/helpers/show/display_tree.rs | 29 +--- crates/visored/visored-syn-expr/src/parser.rs | 10 +- .../visored-syn-expr/src/parser/accept.rs | 50 ++++-- .../src/parser/disambiguate_token.rs | 27 +++- .../visored-syn-expr/src/parser/expr_stack.rs | 142 +++++++++++------- .../src/parser/incomplete_expr.rs | 21 ++- crates/visored/visored-syn-expr/src/range.rs | 11 +- .../src/test_helpers/example.rs | 39 +++-- 17 files changed, 299 insertions(+), 200 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da67eb5b7b..3fa328e52a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12604,6 +12604,7 @@ dependencies = [ "enum-class", "expect-test", "husky-coword", + "husky-print-utils", "husky-tree-utils", "idx-arena", "latex-ast", diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index 0dc12f1215..c2cb88b54d 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -25,6 +25,25 @@ impl std::ops::Index for LxTokenStorage { } } +impl std::ops::Index for LxTokenStorage { + type Output = LxRoseTokenData; + + fn index(&self, idx: LxRoseTokenIdx) -> &Self::Output { + match &self.ranged_tokens[idx.0.index()].2 { + LxTokenData::Rose(data) => data, + _ => unreachable!(), + } + } +} + +impl std::ops::Index for LxTokenStorage { + type Output = ((usize, usize), TextRange, LxTokenData); + + fn index(&self, idx: LxTokenIdx) -> &Self::Output { + &self.ranged_tokens[idx.index()] + } +} + /// # getters impl LxTokenStorage { diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 99aa9d5fae..e8f0c63f4a 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4261,6 +4261,7 @@ dependencies: [ "either", "enum-class", + "husky-print-utils", "husky-tree-utils", "idx-arena", "latex-ast", diff --git a/crates/visored/visored-opr/src/delimiter.rs b/crates/visored/visored-opr/src/delimiter.rs index 37b95c9a15..e2ec0d28fa 100644 --- a/crates/visored/visored-opr/src/delimiter.rs +++ b/crates/visored/visored-opr/src/delimiter.rs @@ -1,5 +1,11 @@ #[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum VdLeftDelimiter {} +pub enum VdBaseLeftDelimiter {} #[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum VdRightDelimiter {} +pub enum VdBaseRightDelimiter {} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdCompositeLeftDelimiter {} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdCompositeRightDelimiter {} diff --git a/crates/visored/visored-opr/src/precedence.rs b/crates/visored/visored-opr/src/precedence.rs index 1037fcf0e5..f1d2adae3d 100644 --- a/crates/visored/visored-opr/src/precedence.rs +++ b/crates/visored/visored-opr/src/precedence.rs @@ -3,6 +3,7 @@ pub struct VdPrecedence(u64); impl VdPrecedence { pub const APPLICATION: Self = VdPrecedence(0); + pub const SPACE: Self = VdPrecedence(0); } #[derive(Debug, PartialEq, Eq, Clone, Copy)] diff --git a/crates/visored/visored-opr/src/separator.rs b/crates/visored/visored-opr/src/separator.rs index bc3e8da777..65b8e83a56 100644 --- a/crates/visored/visored-opr/src/separator.rs +++ b/crates/visored/visored-opr/src/separator.rs @@ -1,7 +1,28 @@ +use crate::precedence::VdPrecedence; + #[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum VdSeparator { +pub enum VdBaseSeparator { + Space, Comma, Semicolon, Add, Mul, + Dot, +} +impl VdBaseSeparator { + pub fn precedence(self) -> VdPrecedence { + match self { + VdBaseSeparator::Space => VdPrecedence::SPACE, + VdBaseSeparator::Comma => todo!(), + VdBaseSeparator::Semicolon => todo!(), + VdBaseSeparator::Add => todo!(), + VdBaseSeparator::Mul => todo!(), + VdBaseSeparator::Dot => todo!(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdCompositeSeparator { + Call, } diff --git a/crates/visored/visored-syn-expr/Cargo.toml b/crates/visored/visored-syn-expr/Cargo.toml index ade6de7d3f..f55034c2ef 100644 --- a/crates/visored/visored-syn-expr/Cargo.toml +++ b/crates/visored/visored-syn-expr/Cargo.toml @@ -25,6 +25,7 @@ latex-prelude.workspace = true latex-token.workspace = true latex-ast.workspace = true # utils +husky-print-utils.workspace = true husky-tree-utils.workspace = true # visored visored-annotation.workspace = true diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index 5c6b50ed6c..d69356e0a6 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -10,7 +10,10 @@ use crate::{ sentence::{VdSynSentenceArena, VdSynSentenceData, VdSynSentenceIdx}, }; use either::*; -use latex_ast::ast::{LxAstArena, LxAstArenaRef, LxAstIdxRange}; +use latex_ast::{ + ast::{LxAstArena, LxAstArenaRef, LxAstIdxRange}, + range::LxAstTokenIdxRangeMap, +}; use latex_token::storage::LxTokenStorage; use visored_annotation::annotations::VdAnnotations; @@ -18,15 +21,12 @@ pub(crate) struct VdSynExprBuilder<'db> { db: &'db ::salsa::Db, token_storage: &'db LxTokenStorage, ast_arena: LxAstArenaRef<'db>, + ast_token_idx_range_map: &'db LxAstTokenIdxRangeMap, annotations: &'db VdAnnotations, expr_arena: VdSynExprArena, - expr_range_map: VdSynExprRangeMap, phrase_arena: VdSynPhraseArena, - phrase_range_map: VdSynPhraseRangeMap, clause_arena: VdSynClauseArena, - clause_range_map: VdSynClauseRangeMap, sentence_arena: VdSynSentenceArena, - sentence_range_map: VdSynSentenceRangeMap, } /// # constructor @@ -35,32 +35,24 @@ impl<'db> VdSynExprBuilder<'db> { db: &'db ::salsa::Db, token_storage: &'db LxTokenStorage, ast_arena: &'db LxAstArena, + ast_token_idx_range_map: &'db LxAstTokenIdxRangeMap, annotations: &'db VdAnnotations, ) -> Self { Self { db, token_storage, ast_arena: ast_arena.as_arena_ref(), + ast_token_idx_range_map, annotations, expr_arena: Default::default(), phrase_arena: Default::default(), clause_arena: Default::default(), sentence_arena: Default::default(), - expr_range_map: Default::default(), - phrase_range_map: Default::default(), - clause_range_map: Default::default(), - sentence_range_map: Default::default(), } } - pub(crate) fn alloc_expr( - &mut self, - data: VdSynExprData, - range: VdSynExprAstRange, - ) -> VdSynExprIdx { - let idx = self.expr_arena.alloc_one(data); - self.expr_range_map.insert_next(idx, range); - idx + pub(crate) fn alloc_expr(&mut self, data: VdSynExprData) -> VdSynExprIdx { + self.expr_arena.alloc_one(data) } } @@ -78,6 +70,10 @@ impl<'db> VdSynExprBuilder<'db> { &self.ast_arena } + pub(crate) fn ast_token_idx_range_map(&self) -> &LxAstTokenIdxRangeMap { + &self.ast_token_idx_range_map + } + pub(crate) fn annotations(&self) -> &VdAnnotations { self.annotations } @@ -86,33 +82,17 @@ impl<'db> VdSynExprBuilder<'db> { &self.expr_arena } - pub(crate) fn expr_range_map(&self) -> &VdSynExprRangeMap { - &self.expr_range_map - } - pub(crate) fn phrase_arena(&self) -> &VdSynPhraseArena { &self.phrase_arena } - pub(crate) fn phrase_range_map(&self) -> &VdSynPhraseRangeMap { - &self.phrase_range_map - } - pub(crate) fn clause_arena(&self) -> &VdSynClauseArena { &self.clause_arena } - pub(crate) fn clause_range_map(&self) -> &VdSynClauseRangeMap { - &self.clause_range_map - } - pub(crate) fn sentence_arena(&self) -> &VdSynSentenceArena { &self.sentence_arena } - - pub(crate) fn sentence_range_map(&self) -> &VdSynSentenceRangeMap { - &self.sentence_range_map - } } impl<'db> VdSynExprBuilder<'db> { @@ -129,23 +109,15 @@ impl<'db> VdSynExprBuilder<'db> { self, ) -> ( VdSynExprArena, - VdSynExprRangeMap, VdSynPhraseArena, - VdSynPhraseRangeMap, VdSynClauseArena, - VdSynClauseRangeMap, VdSynSentenceArena, - VdSynSentenceRangeMap, ) { ( self.expr_arena, - self.expr_range_map, self.phrase_arena, - self.phrase_range_map, self.clause_arena, - self.clause_range_map, self.sentence_arena, - self.sentence_range_map, ) } } diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 6694f623ab..7d8424bf4f 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -19,10 +19,15 @@ use idx_arena::{ }; use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; use latex_prelude::script::LxScriptKind; -use latex_token::idx::LxTokenIdxRange; +use latex_token::idx::{LxMathTokenIdx, LxTokenIdxRange}; use range::VdSynExprAstRange; -use visored_opr::opr::{ - binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr, suffix::VdBaseSuffixOpr, VdBaseOpr, +use visored_opr::{ + delimiter::{ + VdBaseLeftDelimiter, VdBaseRightDelimiter, VdCompositeLeftDelimiter, + VdCompositeRightDelimiter, + }, + opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr, suffix::VdBaseSuffixOpr, VdBaseOpr}, + separator::{VdBaseSeparator, VdCompositeSeparator}, }; use visored_zfc_ty::term::literal::VdZfcLiteral; @@ -62,6 +67,24 @@ pub enum VdSynExprData { Err(VdSynExprError), } +#[derive(Debug, PartialEq, Eq)] +pub enum VdSynSeparator { + Base(LxMathTokenIdx, VdBaseSeparator), + Composite(VdSynExprIdx, VdCompositeSeparator), +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdSynLeftDelimiter { + Base(VdBaseLeftDelimiter), + Composite(VdSynExprIdx, VdCompositeLeftDelimiter), +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdSynRightDelimiter { + Base(VdBaseRightDelimiter), + Composite(VdSynExprIdx, VdCompositeRightDelimiter), +} + pub type VdSynExprIdx = ArenaIdx; pub type VdSynExprIdxRange = ArenaIdxRange; pub type VdSynExprArena = Arena; @@ -105,9 +128,21 @@ impl VdSynExprData { pub fn class(&self) -> VdSynExprClass { match *self { - // ad hoc - _ => VdSynExprClass::Separator, - _ => unreachable!(), + VdSynExprData::Literal { + token_idx_range, + literal, + } => VdSynExprClass::Atom, + VdSynExprData::Notation => todo!(), + VdSynExprData::BaseOpr { opr } => todo!(), + VdSynExprData::Binary { lopd, opr, ropd } => todo!(), + VdSynExprData::Prefix { opr, opd } => todo!(), + VdSynExprData::Suffix { opd, opr } => todo!(), + VdSynExprData::Attach { base, ref scripts } => todo!(), + VdSynExprData::UniadicChain => todo!(), + VdSynExprData::VariadicChain => todo!(), + VdSynExprData::UniadicArray => todo!(), + VdSynExprData::VariadicArray => todo!(), + VdSynExprData::Err(ref error) => todo!(), } } } @@ -122,10 +157,7 @@ pub enum VdSynExprClass { impl ToVdSyn for LxMathAstIdxRange { fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> VdSynExprIdx { if self.is_empty() { - builder.alloc_expr( - VdSynExprData::Err(OriginalVdSynExprError::Empty.into()), - VdSynExprAstRange::Asts(self.into()), - ) + builder.alloc_expr(VdSynExprData::Err(OriginalVdSynExprError::Empty.into())) } else { let parser = builder.parser(); parser.parse_asts(self) @@ -193,5 +225,13 @@ mod tests { "11\n" "#]], ); + t( + "1 1", + &[], + &[], + &expect![[r#" + "1 1\n" + "#]], + ); } } diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index 72afa6091c..03b3747d15 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -47,12 +47,12 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { ast_arena: LxAstArenaRef<'a>, ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, expr_arena: VdSynExprArenaRef<'a>, - expr_range_map: &'a VdSynExprRangeMap, phrase_arena: VdSynPhraseArenaRef<'a>, - phrase_range_map: &'a VdSynPhraseRangeMap, clause_arena: VdSynClauseArenaRef<'a>, - clause_range_map: &'a VdSynClauseRangeMap, sentence_arena: VdSynSentenceArenaRef<'a>, + expr_range_map: &'a VdSynExprRangeMap, + phrase_range_map: &'a VdSynPhraseRangeMap, + clause_range_map: &'a VdSynClauseRangeMap, sentence_range_map: &'a VdSynSentenceRangeMap, ) -> Self { Self { @@ -71,29 +71,6 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { sentence_range_map, } } - - #[cfg(feature = "test_helpers")] - pub fn new2( - lx_asts_example: &'a LxAstExample, - builder: &'a VdSynExprBuilder<'a>, - db: &'a salsa::Db, - ) -> Self { - VdSynExprDisplayTreeBuilder::new( - db, - &lx_asts_example.input, - &lx_asts_example.token_storage, - lx_asts_example.ast_arena.as_arena_ref(), - &lx_asts_example.ast_token_idx_range_map, - builder.expr_arena().as_arena_ref(), - &builder.expr_range_map(), - builder.phrase_arena().as_arena_ref(), - &builder.phrase_range_map(), - builder.clause_arena().as_arena_ref(), - &builder.clause_range_map(), - builder.sentence_arena().as_arena_ref(), - &builder.sentence_range_map(), - ) - } } /// # actions diff --git a/crates/visored/visored-syn-expr/src/parser.rs b/crates/visored/visored-syn-expr/src/parser.rs index cf4a274110..d31f203859 100644 --- a/crates/visored/visored-syn-expr/src/parser.rs +++ b/crates/visored/visored-syn-expr/src/parser.rs @@ -41,12 +41,16 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { self.parse_ast(&mut next, end); } let Self { builder, stack } = self; - let data = stack.finish(); - builder.alloc_expr(data, VdSynExprAstRange::Asts(asts.into())) + builder.alloc_expr(stack.finish()) } fn parse_ast(&mut self, next: &mut LxMathAstIdx, end: LxMathAstIdx) { + let range = self.builder.ast_token_idx_range_map()[*next]; + let preceding_space_annotation = self + .builder + .annotations() + .preceding_space_annotation(range.start()); let token = self.disambiguate_token(next, end); - self.accept_token(token); + self.accept_token(preceding_space_annotation, token); } } diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs index cea08f93d2..96d44879c1 100644 --- a/crates/visored/visored-syn-expr/src/parser/accept.rs +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -5,23 +5,28 @@ use expr::{list_item::VdSynSeparatedListItem, VdSynExprClass, VdSynExprData}; use incomplete_expr::{IncompleteCallListOpr, IncompleteSeparatedListOpr, IncompleteVdSynExprData}; use latex_token::idx::LxTokenIdx; use smallvec::smallvec; +use visored_annotation::annotation::space::VdSpaceAnnotation; use visored_opr::{ - delimiter::{VdLeftDelimiter, VdRightDelimiter}, + delimiter::{VdBaseLeftDelimiter, VdBaseRightDelimiter}, opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr, suffix::VdBaseSuffixOpr, VdBaseOpr}, precedence::VdPrecedence, - separator::VdSeparator, + separator::VdBaseSeparator, }; impl<'a, 'db> VdSynExprParser<'a, 'db> { - pub(crate) fn accept_token(&mut self, token: DisambiguatedToken) { + pub(crate) fn accept_token( + &mut self, + preceding_space_annotation: Option, + token: DisambiguatedToken, + ) { match token { DisambiguatedToken::Expr(expr, class) => match class { - VdSynExprClass::Atom => self.accept_atom(expr), + VdSynExprClass::Atom => self.accept_atom(preceding_space_annotation, expr), VdSynExprClass::Prefix => todo!(), VdSynExprClass::Suffix => todo!(), VdSynExprClass::Separator => todo!(), }, - DisambiguatedToken::Opr(opr) => self.accept_opr(opr), + DisambiguatedToken::Opr(opr) => self.accept_opr(preceding_space_annotation, opr), DisambiguatedToken::Separator(sep) => self.accept_separator(sep), DisambiguatedToken::LeftDelimiter(vd_left_delimiter) => todo!(), DisambiguatedToken::RightDelimiter(vd_right_delimiter) => todo!(), @@ -29,7 +34,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { } } - fn accept_list_end(&mut self, ket: VdRightDelimiter, ket_token_idx: LxTokenIdx) { + fn accept_list_end(&mut self, ket: VdBaseRightDelimiter, ket_token_idx: LxTokenIdx) { todo!() // self.reduce(VdPrecedence::LIST_ITEM); // let last_incomplete_expr = self.take_last_incomplete_expr().unwrap(); @@ -99,26 +104,41 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { // } } - fn accept_atom(&mut self, atom: VdSynExprData) { - self.push_top_syn_expr(atom.into()) + fn accept_atom( + &mut self, + preceding_space_annotation: Option, + atom: VdSynExprData, + ) { + self.push_top_syn_expr(preceding_space_annotation, atom.into()) } - fn accept_opr(&mut self, opr: VdBaseOpr) { + fn accept_opr( + &mut self, + preceding_space_annotation: Option, + opr: VdBaseOpr, + ) { match opr { VdBaseOpr::Binary(opr) => todo!(), - VdBaseOpr::Prefix(opr) => self.accept_prefix_opr(Left(opr)), + VdBaseOpr::Prefix(opr) => self.accept_prefix_opr(preceding_space_annotation, Left(opr)), VdBaseOpr::Suffix(opr) => self.accept_suffix_opr(Left(opr)), } } - fn accept_prefix_opr(&mut self, opr: Either) { - self.push_top_syn_expr(IncompleteVdSynExprData::Prefix { opr }.into()) + fn accept_prefix_opr( + &mut self, + preceding_space_annotation: Option, + opr: Either, + ) { + self.push_top_syn_expr( + preceding_space_annotation, + IncompleteVdSynExprData::Prefix { opr }.into(), + ) } fn accept_suffix_opr(&mut self, opr: Either) { self.take_complete_and_push_to_top(|slf, top_expr| match top_expr { Some(expr) => VdSynExprData::Suffix { - opd: slf.builder.alloc_expr(expr, todo!()), + opd: slf.builder.alloc_expr(expr), opr, } .into(), @@ -126,7 +146,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { }) } - fn accept_separator(&mut self, separator: VdSeparator) { + fn accept_separator(&mut self, separator: VdBaseSeparator) { todo!() // match self.take_complete_expr() { // Some(item) => { @@ -191,7 +211,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { todo!() } - fn accept_list_start(&mut self, bra: VdLeftDelimiter, bra_token_idx: LxTokenIdx) { + fn accept_list_start(&mut self, bra: VdBaseLeftDelimiter, bra_token_idx: LxTokenIdx) { // self.reduce(Precedence::Application); // self.take_complete_and_push_to_top(|parser, finished_expr| -> TopSynExpr { // let finished_expr = finished_expr.map(|expr| parser.context_mut().alloc_expr(expr)); diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs index f4613c3cda..36357a708f 100644 --- a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -10,9 +10,9 @@ use latex_token::{ }; use visored_annotation::annotation::space::VdSpaceAnnotation; use visored_opr::{ - delimiter::{VdLeftDelimiter, VdRightDelimiter}, + delimiter::{VdBaseLeftDelimiter, VdBaseRightDelimiter}, opr::VdBaseOpr, - separator::VdSeparator, + separator::VdBaseSeparator, }; use visored_zfc_ty::term::literal::{VdZfcLiteral, VdZfcLiteralData}; @@ -24,9 +24,9 @@ pub struct DisambiguatedMathAst { pub enum DisambiguatedToken { Expr(VdSynExprData, VdSynExprClass), Opr(VdBaseOpr), - Separator(VdSeparator), - LeftDelimiter(VdLeftDelimiter), - RightDelimiter(VdRightDelimiter), + Separator(VdBaseSeparator), + LeftDelimiter(VdBaseLeftDelimiter), + RightDelimiter(VdBaseRightDelimiter), Letter(LxMathTokenIdx, LxMathLetter), } @@ -46,12 +46,29 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { } LxMathAstData::Digit(first_token_idx, digit) => { let mut last_token_idx = first_token_idx; + let mut last_offset_end = self.builder.token_storage()[*first_token_idx].0 .1; let mut s = String::new(); // TODO: handle real number by using a kind variable, literal number kind while *next < end { match self.builder.ast_arena()[*next] { LxMathAstData::Digit(token_idx, digit) => { + let (offset_start, offset_end) = + self.builder.token_storage()[*token_idx].0; + if offset_start != last_offset_end { + break; + } + if let Some(space_annotation) = self + .builder + .annotations() + .preceding_space_annotation(*token_idx) + { + match space_annotation { + VdSpaceAnnotation::Apply(apply_annotation) => todo!(), + VdSpaceAnnotation::Sever(sever_annotation) => todo!(), + } + } last_token_idx = token_idx; + last_offset_end = offset_end; s.push(digit.char()) } // TODO: handle real number diff --git a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs index 5d529bd667..e54a046152 100644 --- a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs +++ b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs @@ -4,7 +4,13 @@ use super::{ incomplete_expr::IncompleteVdSynExprData, VdSynExprParser, }; -use visored_opr::{delimiter::VdLeftDelimiter, precedence::VdPrecedence}; +use crate::expr::VdSynExprClass; +use either::*; +use smallvec::smallvec; +use visored_annotation::annotation::space::VdSpaceAnnotation; +use visored_opr::{ + delimiter::VdBaseLeftDelimiter, precedence::VdPrecedence, separator::VdBaseSeparator, +}; #[derive(Default)] pub(crate) struct VdSynExprStack { @@ -93,14 +99,26 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { /// - if there is already a finished expression, interpret it as a function, /// and `top_expr` as an argument; /// - otherwise just adds it in the trivial way - pub(super) fn push_top_syn_expr(&mut self, top_expr: TopVdSynExpr) { + pub(super) fn push_top_syn_expr( + &mut self, + preceding_space_annotation: Option, + top_expr: TopVdSynExpr, + ) { // this is for guaranteeing that application is left associative if self.complete_expr().is_some() { self.reduce(VdPrecedence::APPLICATION) }; - if let Some(function) = self.take_complete_expr() { - todo!() - // self.push_unfinished_expr(IncompleteVdSynExprData::Application { function }); + if let Some(expr) = self.take_complete_expr() { + match preceding_space_annotation { + Some(annotation) => todo!(), + _ => { + let expr = self.builder.alloc_expr(expr); + self.push_unfinished_expr(IncompleteVdSynExprData::SeparatedList { + separator: VdBaseSeparator::Space, + fragments: smallvec![Left(expr)], + }) + } + } } match top_expr { TopVdSynExpr::Unfinished(unfinished_expr) => self.push_unfinished_expr(unfinished_expr), @@ -128,27 +146,23 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { self.stack.complete_expr = Some(expr) } - fn reduce_aux( - &mut self, - f: impl Fn(&mut Self, Option, IncompleteVdSynExprData) -> TopVdSynExpr, - ) { - let complete_expr = self.take_complete_expr(); - let Some((incomplete_expr, _)) = self.stack.incomplete_exprs.pop() else { - unreachable!() - }; - let top_expr = f(self, complete_expr, incomplete_expr); - self.push_top_syn_expr(top_expr) - } + // fn reduce_aux( + // &mut self, + // f: impl Fn(&mut Self, Option, IncompleteVdSynExprData) -> TopVdSynExpr, + // ) { + // let complete_expr = self.take_complete_expr(); + // let Some((incomplete_expr, _)) = self.stack.incomplete_exprs.pop() else { + // unreachable!() + // }; + // let top_expr = f(self, complete_expr, incomplete_expr); + // self.push_top_syn_expr(top_expr) + // } pub(super) fn reduce(&mut self, next_precedence: VdPrecedence) { while let Some(prev_precedence) = self.stack.prev_unfinished_expr_precedence() { if prev_precedence < next_precedence { break; } - // // curry is right associative - // if prev_precedence == VdPrecedence::Curry && next_precedence == VdPrecedence::Curry { - // break; - // } match self.stack.incomplete_exprs.pop().unwrap().0 { IncompleteVdSynExprData::Binary { lopd, opr } => { let finished_expr = self.take_complete_expr(); @@ -156,7 +170,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { Some(ropd) => VdSynExprData::Binary { lopd, opr, - ropd: self.builder.alloc_expr(ropd, todo!()), + ropd: self.builder.alloc_expr(ropd), }, None => VdSynExprData::Err( OriginalVdSynExprError::NoRightOperandForBinaryOperator { opr }.into(), @@ -164,23 +178,51 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { }) } IncompleteVdSynExprData::Prefix { opr } => { - let finished_expr = self.take_complete_expr(); - self.stack.complete_expr = Some(match finished_expr { + let expr = self.take_complete_expr(); + self.stack.complete_expr = Some(match expr { Some(opd) => VdSynExprData::Prefix { opr, - opd: self.builder.alloc_expr(opd, todo!()), + opd: self.builder.alloc_expr(opd), }, None => VdSynExprData::Err( OriginalVdSynExprError::NoOperandForPrefixOperator { opr }.into(), ), }) } - IncompleteVdSynExprData::SeparatedList { bra_token_idx, .. } => { - self.stack.complete_expr = Some(VdSynExprData::Err( - OriginalVdSynExprError::UnterminatedList { bra_token_idx }.into(), - )) + IncompleteVdSynExprData::SeparatedList { + separator, + fragments, + .. + } => { + let expr = self.take_complete_expr(); + match expr { + Some(expr) => match expr.class() { + VdSynExprClass::Atom => { + match fragments.last() { + Some(fragment) => match fragment { + Left(expr) => todo!(), + Right(separator) => todo!(), + }, + None => unreachable!(), + }; + todo!() + } + VdSynExprClass::Prefix => todo!(), + VdSynExprClass::Suffix => todo!(), + VdSynExprClass::Separator => { + use husky_print_utils::p; + p!(expr); + todo!() + } + }, + None => todo!(), + } + todo!() + // self.stack.complete_expr = Some(VdSynExprData::Err( + // OriginalVdSynExprError::UnterminatedList { bra_token_idx }.into(), + // )) } - IncompleteVdSynExprData::CallList { .. } => todo!(), + IncompleteVdSynExprData::Delimited { bra } => todo!(), } } } @@ -192,40 +234,24 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { ) { let complete_expr = self.take_complete_expr(); let top_expr = f(self, complete_expr); - self.push_top_syn_expr(top_expr) + self.push_top_syn_expr(None, top_expr) } pub(super) fn finish_batch(&mut self) -> Option { assert!(self.stack.incomplete_exprs.len() == 0); - std::mem::take(&mut self.stack.complete_expr) - .map(|expr| self.builder.alloc_expr(expr, todo!())) - } - - pub(super) fn last_bra(&self) -> Option { - for (unfinished_expr, _) in self.stack.incomplete_exprs.iter().rev() { - match unfinished_expr { - IncompleteVdSynExprData::SeparatedList { bra, .. } => return Some(*bra), - IncompleteVdSynExprData::CallList { .. } => todo!(), - // return Some(Delimiter::Par), - _ => (), - } - } - None + std::mem::take(&mut self.stack.complete_expr).map(|expr| self.builder.alloc_expr(expr)) } - pub(super) fn last_two_bras(&self) -> Vec { - let mut bras = vec![]; - for (unfinished_expr, _) in self.stack.incomplete_exprs.iter().rev() { - match unfinished_expr { - IncompleteVdSynExprData::SeparatedList { bra, .. } => { - bras.push(*bra); - if bras.len() >= 2 { - return bras; - } - } - _ => (), - } - } - bras + pub(super) fn last_left_delimiter(&self) -> Option { + todo!() + // for (unfinished_expr, _) in self.stack.incomplete_exprs.iter().rev() { + // match unfinished_expr { + // IncompleteVdSynExprData::SeparatedList { bra, .. } => return Some(*bra), + // IncompleteVdSynExprData::CallList { .. } => todo!(), + // // return Some(Delimiter::Par), + // _ => (), + // } + // } + // None } } diff --git a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs index 2a5481c01a..ab325157e4 100644 --- a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs +++ b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs @@ -6,12 +6,13 @@ pub(super) use self::separated_list::*; use super::*; use crate::expr::VdSynExprData; -use expr::list_item::VdSynSeparatedListItem; +use expr::{list_item::VdSynSeparatedListItem, VdSynLeftDelimiter, VdSynSeparator}; use latex_token::idx::LxTokenIdx; use visored_opr::{ - delimiter::VdLeftDelimiter, + delimiter::VdBaseLeftDelimiter, opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr}, precedence::VdPrecedence, + separator::VdBaseSeparator, }; #[derive(Debug, PartialEq, Eq)] @@ -28,15 +29,11 @@ pub(super) enum IncompleteVdSynExprData { /// A(a, b, c) /// ``` SeparatedList { - opr: IncompleteSeparatedListOpr, - bra: VdLeftDelimiter, - bra_token_idx: LxTokenIdx, - items: SmallVec<[VdSynSeparatedListItem; 4]>, + separator: VdBaseSeparator, + fragments: SmallVec<[Either; 4]>, }, - /// call list includes more separators like `;` - CallList { - opr: IncompleteCallListOpr, - items: SmallVec<[VdSynSeparatedListItem; 4]>, + Delimited { + bra: VdSynLeftDelimiter, }, } @@ -45,8 +42,8 @@ impl IncompleteVdSynExprData { match self { IncompleteVdSynExprData::Binary { opr, .. } => todo!(), IncompleteVdSynExprData::Prefix { opr, .. } => todo!(), - IncompleteVdSynExprData::SeparatedList { .. } - | IncompleteVdSynExprData::CallList { .. } => todo!(), + IncompleteVdSynExprData::SeparatedList { separator, .. } => separator.precedence(), + IncompleteVdSynExprData::Delimited { bra } => todo!(), } } } diff --git a/crates/visored/visored-syn-expr/src/range.rs b/crates/visored/visored-syn-expr/src/range.rs index 13840f4ef1..45e16af1d7 100644 --- a/crates/visored/visored-syn-expr/src/range.rs +++ b/crates/visored/visored-syn-expr/src/range.rs @@ -1,8 +1,7 @@ use latex_ast::ast::{LxAstIdx, LxAstIdxRange}; use crate::{ - clause::VdSynClauseOrderedMap, expr::VdSynExprOrderedMap, phrase::VdSynPhraseOrderedMap, - sentence::VdSynSentenceOrderedMap, + clause::VdSynClauseMap, expr::VdSynExprMap, phrase::VdSynPhraseMap, sentence::VdSynSentenceMap, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -15,7 +14,7 @@ pub type VdSynPhraseRange = LxAstIdxRange; pub type VdSynClauseRange = LxAstIdxRange; pub type VdSynSentenceRange = LxAstIdxRange; -pub type VdSynExprRangeMap = VdSynExprOrderedMap; -pub type VdSynPhraseRangeMap = VdSynPhraseOrderedMap; -pub type VdSynClauseRangeMap = VdSynClauseOrderedMap; -pub type VdSynSentenceRangeMap = VdSynSentenceOrderedMap; +pub type VdSynExprRangeMap = VdSynExprMap; +pub type VdSynPhraseRangeMap = VdSynPhraseMap; +pub type VdSynClauseRangeMap = VdSynClauseMap; +pub type VdSynSentenceRangeMap = VdSynSentenceMap; diff --git a/crates/visored/visored-syn-expr/src/test_helpers/example.rs b/crates/visored/visored-syn-expr/src/test_helpers/example.rs index a2a1f8d0dd..a86ede6dc2 100644 --- a/crates/visored/visored-syn-expr/src/test_helpers/example.rs +++ b/crates/visored/visored-syn-expr/src/test_helpers/example.rs @@ -31,12 +31,12 @@ pub struct VdSynExprExample { pub ast_token_idx_range_map: LxAstTokenIdxRangeMap, pub result: Either, pub expr_arena: VdSynExprArena, - pub expr_range_map: VdSynExprRangeMap, pub phrase_arena: VdSynPhraseArena, - pub phrase_range_map: VdSynPhraseRangeMap, pub clause_arena: VdSynClauseArena, - pub clause_range_map: VdSynClauseRangeMap, pub sentence_arena: VdSynSentenceArena, + pub expr_range_map: VdSynExprRangeMap, + pub phrase_range_map: VdSynPhraseRangeMap, + pub clause_range_map: VdSynClauseRangeMap, pub sentence_range_map: VdSynSentenceRangeMap, } @@ -59,18 +59,15 @@ impl VdSynExprExample { space_annotations.iter().copied(), &token_storage, ); - let mut builder = VdSynExprBuilder::new(db, &token_storage, &ast_arena, &annotations); + let mut builder = VdSynExprBuilder::new( + db, + &token_storage, + &ast_arena, + &ast_token_idx_range_map, + &annotations, + ); let result = asts.to_vd_syn(&mut builder); - let ( - expr_arena, - expr_range_map, - phrase_arena, - phrase_range_map, - clause_arena, - clause_range_map, - sentence_arena, - sentence_range_map, - ) = builder.finish(); + let (expr_arena, phrase_arena, clause_arena, sentence_arena) = builder.finish(); Self { input: input.to_string(), root_mode, @@ -81,13 +78,13 @@ impl VdSynExprExample { ast_token_idx_range_map, result, expr_arena, - expr_range_map, phrase_arena, - phrase_range_map, clause_arena, - clause_range_map, sentence_arena, - sentence_range_map, + expr_range_map: todo!(), + phrase_range_map: todo!(), + clause_range_map: todo!(), + sentence_range_map: todo!(), } } @@ -99,12 +96,12 @@ impl VdSynExprExample { self.ast_arena.as_arena_ref(), &self.ast_token_idx_range_map, self.expr_arena.as_arena_ref(), - &self.expr_range_map, self.phrase_arena.as_arena_ref(), - &self.phrase_range_map, self.clause_arena.as_arena_ref(), - &self.clause_range_map, self.sentence_arena.as_arena_ref(), + &self.expr_range_map, + &self.phrase_range_map, + &self.clause_range_map, &self.sentence_range_map, ); match self.result { From 4a58e61d27bfaf427debe610d169b5f72701cdc1 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 02:26:05 -0700 Subject: [PATCH 63/71] save --- crates/latex/latex-ast/src/range.rs | 8 +- .../visored/visored-syn-expr/src/builder.rs | 4 +- crates/visored/visored-syn-expr/src/error.rs | 33 +++- crates/visored/visored-syn-expr/src/expr.rs | 15 +- .../src/helpers/show/display_tree.rs | 25 +-- crates/visored/visored-syn-expr/src/parser.rs | 2 +- crates/visored/visored-syn-expr/src/range.rs | 183 ++++++++++++++++-- .../src/test_helpers/example.rs | 22 ++- 8 files changed, 253 insertions(+), 39 deletions(-) diff --git a/crates/latex/latex-ast/src/range.rs b/crates/latex/latex-ast/src/range.rs index 14c6e079f4..9680e15ea0 100644 --- a/crates/latex/latex-ast/src/range.rs +++ b/crates/latex/latex-ast/src/range.rs @@ -1,7 +1,7 @@ use latex_token::idx::LxTokenIdxRange; use crate::ast::{ - math::{LxMathAstArenaMap, LxMathAstData, LxMathAstIdx}, + math::{LxMathAstArenaMap, LxMathAstData, LxMathAstIdx, LxMathAstIdxRange}, rose::{LxRoseAstArenaMap, LxRoseAstData, LxRoseAstIdx}, LxAstArena, LxAstArenaMap, LxAstArenaRef, LxAstData, LxAstIdx, }; @@ -28,6 +28,12 @@ impl std::ops::Index for LxAstTokenIdxRangeMap { } } +impl LxAstTokenIdxRangeMap { + pub fn math_asts_token_idx_range(&self, asts: LxMathAstIdxRange) -> LxTokenIdxRange { + todo!() + } +} + pub fn calc_ast_token_idx_range_map(db: &salsa::Db, arena: &LxAstArena) -> LxAstTokenIdxRangeMap { let mut calculator = LxAstTokenIdxRangeCalculator::new(db, arena); calculator.infer_all(); diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index d69356e0a6..d38e1f853f 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -3,8 +3,8 @@ use crate::{ expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx, VdSynExprIdxRange}, phrase::{VdSynPhraseArena, VdSynPhraseData, VdSynPhraseIdx}, range::{ - VdSynClauseRangeMap, VdSynExprAstRange, VdSynExprRangeMap, VdSynPhraseRangeMap, - VdSynSentenceRangeMap, + VdSynClauseTokenIdxRangeMap, VdSynExprTokenIdxRange, VdSynExprTokenIdxRangeMap, + VdSynPhraseTokenIdxRangeMap, VdSynSentenceTokenIdxRangeMap, }, region::VdSynExprRegionData, sentence::{VdSynSentenceArena, VdSynSentenceData, VdSynSentenceIdx}, diff --git a/crates/visored/visored-syn-expr/src/error.rs b/crates/visored/visored-syn-expr/src/error.rs index d68ae172bc..eacca0c110 100644 --- a/crates/visored/visored-syn-expr/src/error.rs +++ b/crates/visored/visored-syn-expr/src/error.rs @@ -1,6 +1,6 @@ use crate::expr::VdSynExprIdx; use either::Either; -use latex_token::idx::LxTokenIdx; +use latex_token::idx::{LxTokenIdx, LxTokenIdxRange}; use thiserror::Error; use visored_opr::opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr}; @@ -15,7 +15,7 @@ pub enum VdSynExprError { #[derive(Debug, Error, PartialEq, Eq)] pub enum OriginalVdSynExprError { #[error("empty")] - Empty, + Empty(LxTokenIdxRange), #[error("todo")] Todo, #[error("no right operand for binary operator")] @@ -38,3 +38,32 @@ pub enum DerivedVdSynExprError { pub type VdSynExprResult = Result; pub type VdSynExprResultRef<'a, T> = Result; + +impl VdSynExprError { + pub fn token_idx_range(&self) -> LxTokenIdxRange { + match self { + VdSynExprError::Original(error) => error.token_idx_range(), + VdSynExprError::Derived(error) => error.token_idx_range(), + } + } +} + +impl OriginalVdSynExprError { + pub fn token_idx_range(&self) -> LxTokenIdxRange { + match *self { + Self::Empty(range) => range, + Self::Todo => todo!(), + Self::NoRightOperandForBinaryOperator { .. } => todo!(), + Self::NoOperandForPrefixOperator { .. } => todo!(), + Self::UnterminatedList { bra_token_idx } => todo!(), + } + } +} + +impl DerivedVdSynExprError { + pub fn token_idx_range(&self) -> LxTokenIdxRange { + match self { + Self::Todo => todo!(), + } + } +} diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 7d8424bf4f..3b0f22dbd0 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -20,7 +20,7 @@ use idx_arena::{ use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; use latex_prelude::script::LxScriptKind; use latex_token::idx::{LxMathTokenIdx, LxTokenIdxRange}; -use range::VdSynExprAstRange; +use range::VdSynExprTokenIdxRange; use visored_opr::{ delimiter::{ VdBaseLeftDelimiter, VdBaseRightDelimiter, VdCompositeLeftDelimiter, @@ -122,7 +122,7 @@ impl VdSynExprData { VdSynExprData::UniadicArray => vec![], // ad hoc VdSynExprData::VariadicArray => vec![], - VdSynExprData::Err(ref error) => vec![], + VdSynExprData::Err(..) => vec![], } } @@ -142,7 +142,7 @@ impl VdSynExprData { VdSynExprData::VariadicChain => todo!(), VdSynExprData::UniadicArray => todo!(), VdSynExprData::VariadicArray => todo!(), - VdSynExprData::Err(ref error) => todo!(), + VdSynExprData::Err(..) => todo!(), } } } @@ -157,7 +157,14 @@ pub enum VdSynExprClass { impl ToVdSyn for LxMathAstIdxRange { fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> VdSynExprIdx { if self.is_empty() { - builder.alloc_expr(VdSynExprData::Err(OriginalVdSynExprError::Empty.into())) + builder.alloc_expr(VdSynExprData::Err( + OriginalVdSynExprError::Empty( + builder + .ast_token_idx_range_map() + .math_asts_token_idx_range(self), + ) + .into(), + )) } else { let parser = builder.parser(); parser.parse_asts(self) diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index 03b3747d15..b7a87f7a79 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -4,8 +4,8 @@ use crate::{ expr::{VdSynExprArenaRef, VdSynExprIdx, VdSynExprIdxRange}, phrase::VdSynPhraseArenaRef, range::{ - VdSynClauseRangeMap, VdSynExprAstRange, VdSynExprRangeMap, VdSynPhraseRangeMap, - VdSynSentenceRangeMap, + VdSynClauseTokenIdxRangeMap, VdSynExprTokenIdxRange, VdSynExprTokenIdxRangeMap, + VdSynPhraseTokenIdxRangeMap, VdSynSentenceTokenIdxRangeMap, }, sentence::VdSynSentenceArenaRef, }; @@ -29,13 +29,13 @@ pub struct VdSynExprDisplayTreeBuilder<'a> { ast_arena: LxAstArenaRef<'a>, ast_token_idx_range_map: &'a LxAstTokenIdxRangeMap, expr_arena: VdSynExprArenaRef<'a>, - expr_range_map: &'a VdSynExprRangeMap, + expr_range_map: &'a VdSynExprTokenIdxRangeMap, phrase_arena: VdSynPhraseArenaRef<'a>, - phrase_range_map: &'a VdSynPhraseRangeMap, + phrase_range_map: &'a VdSynPhraseTokenIdxRangeMap, clause_arena: VdSynClauseArenaRef<'a>, - clause_range_map: &'a VdSynClauseRangeMap, + clause_range_map: &'a VdSynClauseTokenIdxRangeMap, sentence_arena: VdSynSentenceArenaRef<'a>, - sentence_range_map: &'a VdSynSentenceRangeMap, + sentence_range_map: &'a VdSynSentenceTokenIdxRangeMap, } /// # construction @@ -50,10 +50,10 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { phrase_arena: VdSynPhraseArenaRef<'a>, clause_arena: VdSynClauseArenaRef<'a>, sentence_arena: VdSynSentenceArenaRef<'a>, - expr_range_map: &'a VdSynExprRangeMap, - phrase_range_map: &'a VdSynPhraseRangeMap, - clause_range_map: &'a VdSynClauseRangeMap, - sentence_range_map: &'a VdSynSentenceRangeMap, + expr_range_map: &'a VdSynExprTokenIdxRangeMap, + phrase_range_map: &'a VdSynPhraseTokenIdxRangeMap, + clause_range_map: &'a VdSynClauseTokenIdxRangeMap, + sentence_range_map: &'a VdSynSentenceTokenIdxRangeMap, ) -> Self { Self { db, @@ -90,8 +90,9 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { pub fn render_expr(&self, expr: VdSynExprIdx) -> DisplayTree { let expr_range = self.expr_range_map[expr]; let (start, end) = match expr_range { - VdSynExprAstRange::Ast(ast) => self.ast_offset_range(ast), - VdSynExprAstRange::Asts(asts) => self.asts_offset_range(asts), + VdSynExprTokenIdxRange::Standard(token_idx_range) => self + .token_storage + .token_idx_range_offset_range(token_idx_range), }; let value = self.input[start..end].to_string(); DisplayTree::new(value, self.render_exprs(self.expr_arena[expr].children())) diff --git a/crates/visored/visored-syn-expr/src/parser.rs b/crates/visored/visored-syn-expr/src/parser.rs index d31f203859..7a604f5140 100644 --- a/crates/visored/visored-syn-expr/src/parser.rs +++ b/crates/visored/visored-syn-expr/src/parser.rs @@ -7,7 +7,7 @@ pub(crate) mod incomplete_expr; use expr::VdSynExprIdx; use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; -use range::VdSynExprAstRange; +use range::VdSynExprTokenIdxRange; use self::expr_stack::VdSynExprStack; use crate::builder::VdSynExprBuilder; diff --git a/crates/visored/visored-syn-expr/src/range.rs b/crates/visored/visored-syn-expr/src/range.rs index 45e16af1d7..8a9512863a 100644 --- a/crates/visored/visored-syn-expr/src/range.rs +++ b/crates/visored/visored-syn-expr/src/range.rs @@ -1,20 +1,179 @@ -use latex_ast::ast::{LxAstIdx, LxAstIdxRange}; +use latex_token::idx::LxTokenIdxRange; use crate::{ - clause::VdSynClauseMap, expr::VdSynExprMap, phrase::VdSynPhraseMap, sentence::VdSynSentenceMap, + clause::{VdSynClauseArena, VdSynClauseIdx, VdSynClauseMap}, + expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx, VdSynExprMap}, + phrase::{VdSynPhraseArena, VdSynPhraseIdx, VdSynPhraseMap}, + sentence::{VdSynSentenceArena, VdSynSentenceIdx, VdSynSentenceMap}, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VdSynExprAstRange { - Ast(LxAstIdx), - Asts(LxAstIdxRange), +pub enum VdSynExprTokenIdxRange { + Standard(LxTokenIdxRange), } -pub type VdSynPhraseRange = LxAstIdxRange; -pub type VdSynClauseRange = LxAstIdxRange; -pub type VdSynSentenceRange = LxAstIdxRange; +pub type VdSynPhraseTokenIdxRange = LxTokenIdxRange; +pub type VdSynClauseTokenIdxRange = LxTokenIdxRange; +pub type VdSynSentenceTokenIdxRange = LxTokenIdxRange; -pub type VdSynExprRangeMap = VdSynExprMap; -pub type VdSynPhraseRangeMap = VdSynPhraseMap; -pub type VdSynClauseRangeMap = VdSynClauseMap; -pub type VdSynSentenceRangeMap = VdSynSentenceMap; +pub type VdSynExprTokenIdxRangeMap = VdSynExprMap; +pub type VdSynPhraseTokenIdxRangeMap = VdSynPhraseMap; +pub type VdSynClauseTokenIdxRangeMap = VdSynClauseMap; +pub type VdSynSentenceTokenIdxRangeMap = VdSynSentenceMap; + +pub fn calc_expr_range_map( + db: &::salsa::Db, + expr_arena: &VdSynExprArena, + phrase_arena: &VdSynPhraseArena, + clause_arena: &VdSynClauseArena, + sentence_arena: &VdSynSentenceArena, +) -> ( + VdSynExprTokenIdxRangeMap, + VdSynPhraseTokenIdxRangeMap, + VdSynClauseTokenIdxRangeMap, + VdSynSentenceTokenIdxRangeMap, +) { + let mut calculator = + VdSynExprRangeCalculator::new(db, expr_arena, phrase_arena, clause_arena, sentence_arena); + calculator.infer_all_ranges(); + calculator.finish() +} + +struct VdSynExprRangeCalculator<'db> { + db: &'db ::salsa::Db, + expr_arena: &'db VdSynExprArena, + phrase_arena: &'db VdSynPhraseArena, + clause_arena: &'db VdSynClauseArena, + sentence_arena: &'db VdSynSentenceArena, + expr_range_map: VdSynExprTokenIdxRangeMap, + phrase_range_map: VdSynPhraseTokenIdxRangeMap, + clause_range_map: VdSynClauseTokenIdxRangeMap, + sentence_range_map: VdSynSentenceTokenIdxRangeMap, +} + +impl<'db> VdSynExprRangeCalculator<'db> { + fn new( + db: &'db ::salsa::Db, + expr_arena: &'db VdSynExprArena, + phrase_arena: &'db VdSynPhraseArena, + clause_arena: &'db VdSynClauseArena, + sentence_arena: &'db VdSynSentenceArena, + ) -> Self { + Self { + db, + expr_arena, + phrase_arena, + clause_arena, + sentence_arena, + expr_range_map: VdSynExprTokenIdxRangeMap::new(expr_arena), + phrase_range_map: VdSynPhraseTokenIdxRangeMap::new(phrase_arena), + clause_range_map: VdSynClauseTokenIdxRangeMap::new(clause_arena), + sentence_range_map: VdSynSentenceTokenIdxRangeMap::new(sentence_arena), + } + } +} + +impl<'db> VdSynExprRangeCalculator<'db> { + fn infer_all_ranges(&mut self) { + for expr in self.expr_arena.indices() { + self.infer_expr(expr); + } + for phrase in self.phrase_arena.indices() { + self.infer_phrase(phrase); + } + for clause in self.clause_arena.indices() { + self.infer_clause(clause); + } + for sentence in self.sentence_arena.indices() { + self.infer_sentence(sentence); + } + } + + fn infer_expr(&mut self, expr: VdSynExprIdx) { + if self.expr_range_map.has(expr) { + return; + } + let range = self.calc_expr(expr); + self.expr_range_map.insert(expr, range); + } + + fn calc_expr(&mut self, expr: VdSynExprIdx) -> VdSynExprTokenIdxRange { + match self.expr_arena[expr] { + VdSynExprData::Literal { + token_idx_range, + literal, + } => todo!(), + VdSynExprData::Notation => todo!(), + VdSynExprData::BaseOpr { opr } => todo!(), + VdSynExprData::Binary { lopd, opr, ropd } => todo!(), + VdSynExprData::Prefix { opr, opd } => todo!(), + VdSynExprData::Suffix { opd, opr } => todo!(), + VdSynExprData::Attach { base, ref scripts } => todo!(), + VdSynExprData::UniadicChain => todo!(), + VdSynExprData::VariadicChain => todo!(), + VdSynExprData::UniadicArray => todo!(), + VdSynExprData::VariadicArray => todo!(), + VdSynExprData::Err(ref e) => VdSynExprTokenIdxRange::Standard(e.token_idx_range()), + } + } + + fn get_expr(&mut self, expr: VdSynExprIdx) -> VdSynExprTokenIdxRange { + self.infer_expr(expr); + self.expr_range_map[expr] + } + + fn infer_phrase(&mut self, phrase: VdSynPhraseIdx) { + todo!() + } + + fn calc_phrase(&mut self, phrase: VdSynPhraseIdx) -> VdSynPhraseTokenIdxRange { + todo!() + } + + fn get_phrase(&mut self, phrase: VdSynPhraseIdx) -> VdSynPhraseTokenIdxRange { + self.infer_phrase(phrase); + self.phrase_range_map[phrase] + } + + fn infer_clause(&mut self, clause: VdSynClauseIdx) { + todo!() + } + + fn calc_clause(&mut self, clause: VdSynClauseIdx) -> VdSynClauseTokenIdxRange { + todo!() + } + + fn get_clause(&mut self, clause: VdSynClauseIdx) -> VdSynClauseTokenIdxRange { + self.infer_clause(clause); + self.clause_range_map[clause] + } + + fn infer_sentence(&mut self, sentence: VdSynSentenceIdx) { + todo!() + } + + fn calc_sentence(&mut self, sentence: VdSynSentenceIdx) -> VdSynSentenceTokenIdxRange { + todo!() + } + + fn get_sentence(&mut self, sentence: VdSynSentenceIdx) -> VdSynSentenceTokenIdxRange { + self.infer_sentence(sentence); + self.sentence_range_map[sentence] + } + + fn finish( + self, + ) -> ( + VdSynExprTokenIdxRangeMap, + VdSynPhraseTokenIdxRangeMap, + VdSynClauseTokenIdxRangeMap, + VdSynSentenceTokenIdxRangeMap, + ) { + ( + self.expr_range_map, + self.phrase_range_map, + self.clause_range_map, + self.sentence_range_map, + ) + } +} diff --git a/crates/visored/visored-syn-expr/src/test_helpers/example.rs b/crates/visored/visored-syn-expr/src/test_helpers/example.rs index a86ede6dc2..0b166e8f25 100644 --- a/crates/visored/visored-syn-expr/src/test_helpers/example.rs +++ b/crates/visored/visored-syn-expr/src/test_helpers/example.rs @@ -4,7 +4,10 @@ use crate::{ clause::VdSynClauseArena, expr::VdSynExprArena, phrase::VdSynPhraseArena, - range::{VdSynClauseRangeMap, VdSynExprRangeMap, VdSynPhraseRangeMap, VdSynSentenceRangeMap}, + range::{ + VdSynClauseTokenIdxRangeMap, VdSynExprTokenIdxRangeMap, VdSynPhraseTokenIdxRangeMap, + VdSynSentenceTokenIdxRangeMap, + }, sentence::VdSynSentenceArena, }; use expr::VdSynExprIdx; @@ -16,6 +19,7 @@ use latex_ast::{ }; use latex_prelude::mode::LxMode; use latex_token::storage::LxTokenStorage; +use range::calc_expr_range_map; use visored_annotation::{ annotation::{space::VdSpaceAnnotation, token::VdTokenAnnotation}, annotations::VdAnnotations, @@ -34,10 +38,10 @@ pub struct VdSynExprExample { pub phrase_arena: VdSynPhraseArena, pub clause_arena: VdSynClauseArena, pub sentence_arena: VdSynSentenceArena, - pub expr_range_map: VdSynExprRangeMap, - pub phrase_range_map: VdSynPhraseRangeMap, - pub clause_range_map: VdSynClauseRangeMap, - pub sentence_range_map: VdSynSentenceRangeMap, + pub expr_range_map: VdSynExprTokenIdxRangeMap, + pub phrase_range_map: VdSynPhraseTokenIdxRangeMap, + pub clause_range_map: VdSynClauseTokenIdxRangeMap, + pub sentence_range_map: VdSynSentenceTokenIdxRangeMap, } impl VdSynExprExample { @@ -68,6 +72,14 @@ impl VdSynExprExample { ); let result = asts.to_vd_syn(&mut builder); let (expr_arena, phrase_arena, clause_arena, sentence_arena) = builder.finish(); + let (expr_range_map, phrase_range_map, clause_range_map, sentence_range_map) = + calc_expr_range_map( + db, + &expr_arena, + &phrase_arena, + &clause_arena, + &sentence_arena, + ); Self { input: input.to_string(), root_mode, From 46336cd96ddb6b090b7e5e6db57d90df55a7d9c4 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 02:50:29 -0700 Subject: [PATCH 64/71] save --- Cargo.lock | 1 + crates/latex/latex-token/src/storage.rs | 3 +- .../expect-files/husky_lang_packages.txt | 1 + crates/visored/visored-opr/Cargo.toml | 1 + crates/visored/visored-opr/src/precedence.rs | 4 +- crates/visored/visored-opr/src/separator.rs | 20 +++++ .../visored/visored-syn-expr/src/builder.rs | 11 +-- crates/visored/visored-syn-expr/src/expr.rs | 77 ++++++++++--------- .../visored-syn-expr/src/helpers/latex_fmt.rs | 1 + .../src/helpers/show/display_tree.rs | 13 ---- .../src/parser/disambiguate_token.rs | 1 + .../visored-syn-expr/src/parser/expr_stack.rs | 57 ++++++++++---- .../src/parser/incomplete_expr.rs | 4 +- crates/visored/visored-syn-expr/src/range.rs | 1 + .../src/test_helpers/example.rs | 13 ++-- 15 files changed, 128 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3fa328e52a..cf1fd201a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12568,6 +12568,7 @@ dependencies = [ name = "visored-opr" version = "0.1.0" dependencies = [ + "enum-class", "salsa", ] diff --git a/crates/latex/latex-token/src/storage.rs b/crates/latex/latex-token/src/storage.rs index c2cb88b54d..64b562ed12 100644 --- a/crates/latex/latex-token/src/storage.rs +++ b/crates/latex/latex-token/src/storage.rs @@ -55,6 +55,7 @@ impl LxTokenStorage { LxTokenIdxRange::new(0..self.ranged_tokens.len()) } + #[track_caller] pub fn token_offset_range( &self, token_idx: impl std::borrow::Borrow, @@ -63,9 +64,9 @@ impl LxTokenStorage { } pub fn token_idx_range_offset_range(&self, range: LxTokenIdxRange) -> (usize, usize) { - let first = self.token_offset_range(range.start()); match range.last() { Some(last) => { + let first = self.token_offset_range(range.start()); let last = self.token_offset_range(last); (first.0, last.1) } diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index e8f0c63f4a..2e54823d52 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -4252,6 +4252,7 @@ name: "visored-opr", relative_path: "crates/visored/visored-opr", dependencies: [ + "enum-class", "salsa", ], }, diff --git a/crates/visored/visored-opr/Cargo.toml b/crates/visored/visored-opr/Cargo.toml index c86475f9e4..92bdd9a35b 100644 --- a/crates/visored/visored-opr/Cargo.toml +++ b/crates/visored/visored-opr/Cargo.toml @@ -12,6 +12,7 @@ categories.workspace = true keywords.workspace = true [dependencies] +enum-class.workspace = true salsa.workspace = true [lints] diff --git a/crates/visored/visored-opr/src/precedence.rs b/crates/visored/visored-opr/src/precedence.rs index f1d2adae3d..15acf64485 100644 --- a/crates/visored/visored-opr/src/precedence.rs +++ b/crates/visored/visored-opr/src/precedence.rs @@ -2,8 +2,8 @@ pub struct VdPrecedence(u64); impl VdPrecedence { - pub const APPLICATION: Self = VdPrecedence(0); - pub const SPACE: Self = VdPrecedence(0); + pub const APPLICATION: Self = VdPrecedence(1); + pub const SPACE: Self = VdPrecedence(1); } #[derive(Debug, PartialEq, Eq, Clone, Copy)] diff --git a/crates/visored/visored-opr/src/separator.rs b/crates/visored/visored-opr/src/separator.rs index 65b8e83a56..a302cfd5e7 100644 --- a/crates/visored/visored-opr/src/separator.rs +++ b/crates/visored/visored-opr/src/separator.rs @@ -1,5 +1,20 @@ use crate::precedence::VdPrecedence; +#[enum_class::from_variants] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum VdSeparator { + Base(VdBaseSeparator), + Composite(VdCompositeSeparator), +} +impl VdSeparator { + pub fn precedence(self) -> VdPrecedence { + match self { + VdSeparator::Base(sep) => sep.precedence(), + VdSeparator::Composite(sep) => sep.precedence(), + } + } +} + #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum VdBaseSeparator { Space, @@ -26,3 +41,8 @@ impl VdBaseSeparator { pub enum VdCompositeSeparator { Call, } +impl VdCompositeSeparator { + fn precedence(self) -> VdPrecedence { + todo!() + } +} diff --git a/crates/visored/visored-syn-expr/src/builder.rs b/crates/visored/visored-syn-expr/src/builder.rs index d38e1f853f..9d1cfdcfaf 100644 --- a/crates/visored/visored-syn-expr/src/builder.rs +++ b/crates/visored/visored-syn-expr/src/builder.rs @@ -14,7 +14,7 @@ use latex_ast::{ ast::{LxAstArena, LxAstArenaRef, LxAstIdxRange}, range::LxAstTokenIdxRangeMap, }; -use latex_token::storage::LxTokenStorage; +use latex_token::{idx::LxTokenIdxRange, storage::LxTokenStorage}; use visored_annotation::annotations::VdAnnotations; pub(crate) struct VdSynExprBuilder<'db> { @@ -125,11 +125,12 @@ pub trait ToVdSyn { fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> T; } -impl ToVdSyn> for LxAstIdxRange { +impl ToVdSyn> for (LxTokenIdxRange, LxAstIdxRange) { fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> Either { - match self { - LxAstIdxRange::Math(slf) => Either::Left(slf.to_vd_syn(builder)), - LxAstIdxRange::Rose(slf) => Either::Right(todo!()), + let (token_range, asts) = self; + match asts { + LxAstIdxRange::Math(asts) => Either::Left((token_range, asts).to_vd_syn(builder)), + LxAstIdxRange::Rose(asts) => Either::Right(todo!()), } } } diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 3b0f22dbd0..c6251ab7a3 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -27,7 +27,7 @@ use visored_opr::{ VdCompositeRightDelimiter, }, opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr, suffix::VdBaseSuffixOpr, VdBaseOpr}, - separator::{VdBaseSeparator, VdCompositeSeparator}, + separator::{VdBaseSeparator, VdCompositeSeparator, VdSeparator}, }; use visored_zfc_ty::term::literal::VdZfcLiteral; @@ -55,6 +55,10 @@ pub enum VdSynExprData { opd: VdSynExprIdx, opr: Either, }, + SeparatedList { + separator: VdSeparator, + fragments: SmallVec<[Either; 4]>, + }, Attach { base: VdSynExprIdx, // INVARIANCE: at least one of these are some @@ -123,6 +127,7 @@ impl VdSynExprData { // ad hoc VdSynExprData::VariadicArray => vec![], VdSynExprData::Err(..) => vec![], + VdSynExprData::SeparatedList { .. } => todo!(), } } @@ -143,6 +148,7 @@ impl VdSynExprData { VdSynExprData::UniadicArray => todo!(), VdSynExprData::VariadicArray => todo!(), VdSynExprData::Err(..) => todo!(), + VdSynExprData::SeparatedList { .. } => todo!(), } } } @@ -154,21 +160,21 @@ pub enum VdSynExprClass { Separator, } -impl ToVdSyn for LxMathAstIdxRange { +// token idx range is needed because the ast idx range might be empty, +// in which case we need to return an error yet can't determine the token idx range from the ast idx range alone +impl ToVdSyn for (LxTokenIdxRange, LxMathAstIdxRange) { fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> VdSynExprIdx { - if self.is_empty() { + let (token_range, asts) = self; + let a = if asts.is_empty() { builder.alloc_expr(VdSynExprData::Err( - OriginalVdSynExprError::Empty( - builder - .ast_token_idx_range_map() - .math_asts_token_idx_range(self), - ) - .into(), + OriginalVdSynExprError::Empty(token_range).into(), )) } else { let parser = builder.parser(); - parser.parse_asts(self) - } + parser.parse_asts(asts) + }; + todo!(); + a } } @@ -205,33 +211,34 @@ mod tests { space_annotations, db, ); + todo!(); expected.assert_debug_eq(&example.show_display_tree(db)); } - t( - "", - &[], - &[], - &expect![[r#" - "\n" - "#]], - ); - t( - "1", - &[], - &[], - &expect![[r#" - "1\n" - "#]], - ); - t( - "11", - &[], - &[], - &expect![[r#" - "11\n" - "#]], - ); + // t( + // "", + // &[], + // &[], + // &expect![[r#" + // "\n" + // "#]], + // ); + // t( + // "1", + // &[], + // &[], + // &expect![[r#" + // "1\n" + // "#]], + // ); + // t( + // "11", + // &[], + // &[], + // &expect![[r#" + // "11\n" + // "#]], + // ); t( "1 1", &[], diff --git a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs index 7b0624b019..d143a1de6c 100644 --- a/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs +++ b/crates/visored/visored-syn-expr/src/helpers/latex_fmt.rs @@ -126,6 +126,7 @@ impl<'a> VdSynExprLaTeXFormatter<'a> { VdSynExprData::VariadicArray => todo!(), VdSynExprData::BaseOpr { opr } => todo!(), VdSynExprData::Err(ref error) => unreachable!("{error}"), + VdSynExprData::SeparatedList { .. } => todo!(), } } diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index b7a87f7a79..d8c81d64fb 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -142,16 +142,3 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { ) } } - -// impl<'db> VdSynExprBuilder<'db> { -// pub fn display_tree(&self) -> DisplayTree { -// let display_tree_builder = LxAstDisplayTreeBuilder::new( -// self.db, -// self.input, -// self.token_storage, -// self.ast_arena, -// self.ast_token_idx_range_map, -// ); -// display_tree_builder.render_all(self.expr_range_map.keys()) -// } -// } diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs index 36357a708f..e749b1254b 100644 --- a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -37,6 +37,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { end: LxMathAstIdx, ) -> DisambiguatedToken { let ast_data = &self.builder.ast_arena()[*next]; + *next += 1; match *ast_data { LxMathAstData::Letter(lx_math_token_idx, lx_math_letter) => { DisambiguatedToken::Letter(lx_math_token_idx, lx_math_letter) diff --git a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs index e54a046152..1318110743 100644 --- a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs +++ b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs @@ -9,7 +9,9 @@ use either::*; use smallvec::smallvec; use visored_annotation::annotation::space::VdSpaceAnnotation; use visored_opr::{ - delimiter::VdBaseLeftDelimiter, precedence::VdPrecedence, separator::VdBaseSeparator, + delimiter::VdBaseLeftDelimiter, + precedence::VdPrecedence, + separator::{VdBaseSeparator, VdSeparator}, }; #[derive(Default)] @@ -21,7 +23,12 @@ pub(crate) struct VdSynExprStack { impl VdSynExprStack { pub fn finish(self) -> VdSynExprData { assert!(self.complete_expr.is_some()); - assert!(self.incomplete_exprs.is_empty()); + assert!( + self.incomplete_exprs.is_empty(), + "incomplete exprs should be empty but {:?}, complete expr: {:?}", + self.incomplete_exprs, + self.complete_expr + ); self.complete_expr.unwrap() } } @@ -114,7 +121,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { _ => { let expr = self.builder.alloc_expr(expr); self.push_unfinished_expr(IncompleteVdSynExprData::SeparatedList { - separator: VdBaseSeparator::Space, + separator: VdBaseSeparator::Space.into(), fragments: smallvec![Left(expr)], }) } @@ -191,21 +198,36 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { } IncompleteVdSynExprData::SeparatedList { separator, - fragments, - .. + mut fragments, } => { let expr = self.take_complete_expr(); match expr { Some(expr) => match expr.class() { VdSynExprClass::Atom => { - match fragments.last() { - Some(fragment) => match fragment { - Left(expr) => todo!(), - Right(separator) => todo!(), + match fragments.last().expect("fragments are always non-empty") { + Left(_) => match separator { + VdSeparator::Base(base_separator) => match base_separator { + VdBaseSeparator::Space => { + let expr = self.builder.alloc_expr(expr); + fragments.push(Left(expr)); + self.stack.incomplete_exprs.push(( + IncompleteVdSynExprData::SeparatedList { + separator, + fragments, + }, + VdPrecedence::SPACE, + )) + } + VdBaseSeparator::Comma => todo!(), + VdBaseSeparator::Semicolon => todo!(), + VdBaseSeparator::Add => todo!(), + VdBaseSeparator::Mul => todo!(), + VdBaseSeparator::Dot => todo!(), + }, + VdSeparator::Composite(composite_separator) => todo!(), }, - None => unreachable!(), - }; - todo!() + Right(separator) => todo!(), + } } VdSynExprClass::Prefix => todo!(), VdSynExprClass::Suffix => todo!(), @@ -215,12 +237,13 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { todo!() } }, - None => todo!(), + None => { + self.stack.complete_expr = Some(VdSynExprData::SeparatedList { + separator, + fragments, + }) + } } - todo!() - // self.stack.complete_expr = Some(VdSynExprData::Err( - // OriginalVdSynExprError::UnterminatedList { bra_token_idx }.into(), - // )) } IncompleteVdSynExprData::Delimited { bra } => todo!(), } diff --git a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs index ab325157e4..7d1733fa5e 100644 --- a/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs +++ b/crates/visored/visored-syn-expr/src/parser/incomplete_expr.rs @@ -12,7 +12,7 @@ use visored_opr::{ delimiter::VdBaseLeftDelimiter, opr::{binary::VdBaseBinaryOpr, prefix::VdBasePrefixOpr}, precedence::VdPrecedence, - separator::VdBaseSeparator, + separator::{VdBaseSeparator, VdSeparator}, }; #[derive(Debug, PartialEq, Eq)] @@ -29,7 +29,7 @@ pub(super) enum IncompleteVdSynExprData { /// A(a, b, c) /// ``` SeparatedList { - separator: VdBaseSeparator, + separator: VdSeparator, fragments: SmallVec<[Either; 4]>, }, Delimited { diff --git a/crates/visored/visored-syn-expr/src/range.rs b/crates/visored/visored-syn-expr/src/range.rs index 8a9512863a..f2af8fd1a3 100644 --- a/crates/visored/visored-syn-expr/src/range.rs +++ b/crates/visored/visored-syn-expr/src/range.rs @@ -114,6 +114,7 @@ impl<'db> VdSynExprRangeCalculator<'db> { VdSynExprData::UniadicArray => todo!(), VdSynExprData::VariadicArray => todo!(), VdSynExprData::Err(ref e) => VdSynExprTokenIdxRange::Standard(e.token_idx_range()), + VdSynExprData::SeparatedList { .. } => todo!(), } } diff --git a/crates/visored/visored-syn-expr/src/test_helpers/example.rs b/crates/visored/visored-syn-expr/src/test_helpers/example.rs index 0b166e8f25..d977d5bc67 100644 --- a/crates/visored/visored-syn-expr/src/test_helpers/example.rs +++ b/crates/visored/visored-syn-expr/src/test_helpers/example.rs @@ -56,6 +56,7 @@ impl VdSynExprExample { let mut token_storage = LxTokenStorage::default(); let asts = parse_latex_input_into_asts(db, input, root_mode, &mut token_storage, &mut ast_arena); + let whole_token_range = token_storage.whole_token_idx_range(); let ast_token_idx_range_map = calc_ast_token_idx_range_map(db, &ast_arena); let annotations = VdAnnotations::from_sparse( input, @@ -70,8 +71,10 @@ impl VdSynExprExample { &ast_token_idx_range_map, &annotations, ); - let result = asts.to_vd_syn(&mut builder); + let result = (whole_token_range, asts).to_vd_syn(&mut builder); + todo!(); let (expr_arena, phrase_arena, clause_arena, sentence_arena) = builder.finish(); + todo!(); let (expr_range_map, phrase_range_map, clause_range_map, sentence_range_map) = calc_expr_range_map( db, @@ -93,10 +96,10 @@ impl VdSynExprExample { phrase_arena, clause_arena, sentence_arena, - expr_range_map: todo!(), - phrase_range_map: todo!(), - clause_range_map: todo!(), - sentence_range_map: todo!(), + expr_range_map, + phrase_range_map, + clause_range_map, + sentence_range_map, } } From 3665a662aabdd9806bc22f5d2cb904017bd0a00e Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 03:13:18 -0700 Subject: [PATCH 65/71] feat: space separated list of 1s --- crates/visored/visored-opr/src/precedence.rs | 9 ++- crates/visored/visored-syn-expr/src/expr.rs | 68 ++++++++++--------- crates/visored/visored-syn-expr/src/parser.rs | 2 + .../visored-syn-expr/src/parser/accept.rs | 3 + .../src/parser/disambiguate_token.rs | 1 + .../visored-syn-expr/src/parser/expr_stack.rs | 15 ++-- crates/visored/visored-syn-expr/src/range.rs | 34 ++++++++-- .../src/test_helpers/example.rs | 2 - 8 files changed, 85 insertions(+), 49 deletions(-) diff --git a/crates/visored/visored-opr/src/precedence.rs b/crates/visored/visored-opr/src/precedence.rs index 15acf64485..4d8aa9172b 100644 --- a/crates/visored/visored-opr/src/precedence.rs +++ b/crates/visored/visored-opr/src/precedence.rs @@ -2,20 +2,19 @@ pub struct VdPrecedence(u64); impl VdPrecedence { - pub const APPLICATION: Self = VdPrecedence(1); - pub const SPACE: Self = VdPrecedence(1); + pub const SPACE: Self = VdPrecedence(100); } #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum VdPrecedenceRange { Any, Greater(VdPrecedence), - GreaterThan(VdPrecedence), + NoLess(VdPrecedence), } /// # constants impl VdPrecedenceRange { - pub const APPLICATION_SUBEXPR: Self = VdPrecedenceRange::Greater(VdPrecedence::APPLICATION); + pub const SPACE_LEFT: Self = VdPrecedenceRange::NoLess(VdPrecedence::SPACE); } /// # methods @@ -24,7 +23,7 @@ impl VdPrecedenceRange { match self { VdPrecedenceRange::Any => true, VdPrecedenceRange::Greater(p) => precedence > p, - VdPrecedenceRange::GreaterThan(p) => precedence >= p, + VdPrecedenceRange::NoLess(p) => precedence >= p, } } } diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index c6251ab7a3..d9bee34502 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -71,7 +71,7 @@ pub enum VdSynExprData { Err(VdSynExprError), } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum VdSynSeparator { Base(LxMathTokenIdx, VdBaseSeparator), Composite(VdSynExprIdx, VdCompositeSeparator), @@ -127,7 +127,13 @@ impl VdSynExprData { // ad hoc VdSynExprData::VariadicArray => vec![], VdSynExprData::Err(..) => vec![], - VdSynExprData::SeparatedList { .. } => todo!(), + VdSynExprData::SeparatedList { ref fragments, .. } => fragments + .iter() + .filter_map(|fragment| match *fragment { + Left(expr) | Right(VdSynSeparator::Composite(expr, _)) => Some(expr), + Right(VdSynSeparator::Base(_, _)) => None, + }) + .collect(), } } @@ -153,6 +159,7 @@ impl VdSynExprData { } } +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum VdSynExprClass { Atom, Prefix, @@ -165,16 +172,14 @@ pub enum VdSynExprClass { impl ToVdSyn for (LxTokenIdxRange, LxMathAstIdxRange) { fn to_vd_syn(self, builder: &mut VdSynExprBuilder) -> VdSynExprIdx { let (token_range, asts) = self; - let a = if asts.is_empty() { + if asts.is_empty() { builder.alloc_expr(VdSynExprData::Err( OriginalVdSynExprError::Empty(token_range).into(), )) } else { let parser = builder.parser(); parser.parse_asts(asts) - }; - todo!(); - a + } } } @@ -211,40 +216,39 @@ mod tests { space_annotations, db, ); - todo!(); expected.assert_debug_eq(&example.show_display_tree(db)); } - // t( - // "", - // &[], - // &[], - // &expect![[r#" - // "\n" - // "#]], - // ); - // t( - // "1", - // &[], - // &[], - // &expect![[r#" - // "1\n" - // "#]], - // ); - // t( - // "11", - // &[], - // &[], - // &expect![[r#" - // "11\n" - // "#]], - // ); + t( + "", + &[], + &[], + &expect![[r#" + "\n" + "#]], + ); + t( + "1", + &[], + &[], + &expect![[r#" + "1\n" + "#]], + ); + t( + "11", + &[], + &[], + &expect![[r#" + "11\n" + "#]], + ); t( "1 1", &[], &[], &expect![[r#" - "1 1\n" + "1 1\n├─ 1\n└─ 1\n" "#]], ); } diff --git a/crates/visored/visored-syn-expr/src/parser.rs b/crates/visored/visored-syn-expr/src/parser.rs index 7a604f5140..8c78b2416a 100644 --- a/crates/visored/visored-syn-expr/src/parser.rs +++ b/crates/visored/visored-syn-expr/src/parser.rs @@ -8,6 +8,7 @@ pub(crate) mod incomplete_expr; use expr::VdSynExprIdx; use latex_ast::ast::math::{LxMathAstIdx, LxMathAstIdxRange}; use range::VdSynExprTokenIdxRange; +use visored_opr::precedence::{VdPrecedence, VdPrecedenceRange}; use self::expr_stack::VdSynExprStack; use crate::builder::VdSynExprBuilder; @@ -40,6 +41,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { while next < end { self.parse_ast(&mut next, end); } + self.reduce(VdPrecedenceRange::Any); let Self { builder, stack } = self; builder.alloc_expr(stack.finish()) } diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs index 96d44879c1..c0fc1c0525 100644 --- a/crates/visored/visored-syn-expr/src/parser/accept.rs +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -19,6 +19,8 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { preceding_space_annotation: Option, token: DisambiguatedToken, ) { + use husky_print_utils::p; + p!(self.stack, token); match token { DisambiguatedToken::Expr(expr, class) => match class { VdSynExprClass::Atom => self.accept_atom(preceding_space_annotation, expr), @@ -32,6 +34,7 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { DisambiguatedToken::RightDelimiter(vd_right_delimiter) => todo!(), DisambiguatedToken::Letter(lx_math_token_idx, lx_math_letter) => todo!(), } + p!(self.stack); } fn accept_list_end(&mut self, ket: VdBaseRightDelimiter, ket_token_idx: LxTokenIdx) { diff --git a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs index e749b1254b..14b2cc0bc6 100644 --- a/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs +++ b/crates/visored/visored-syn-expr/src/parser/disambiguate_token.rs @@ -21,6 +21,7 @@ pub struct DisambiguatedMathAst { preceding_space_annotation: Option, } +#[derive(Debug)] pub enum DisambiguatedToken { Expr(VdSynExprData, VdSynExprClass), Opr(VdBaseOpr), diff --git a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs index 1318110743..2e9dc88380 100644 --- a/crates/visored/visored-syn-expr/src/parser/expr_stack.rs +++ b/crates/visored/visored-syn-expr/src/parser/expr_stack.rs @@ -10,11 +10,11 @@ use smallvec::smallvec; use visored_annotation::annotation::space::VdSpaceAnnotation; use visored_opr::{ delimiter::VdBaseLeftDelimiter, - precedence::VdPrecedence, + precedence::{VdPrecedence, VdPrecedenceRange}, separator::{VdBaseSeparator, VdSeparator}, }; -#[derive(Default)] +#[derive(Default, Debug)] pub(crate) struct VdSynExprStack { incomplete_exprs: Vec<(IncompleteVdSynExprData, VdPrecedence)>, complete_expr: Option, @@ -113,7 +113,12 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { ) { // this is for guaranteeing that application is left associative if self.complete_expr().is_some() { - self.reduce(VdPrecedence::APPLICATION) + match preceding_space_annotation { + Some(annotation) => todo!(), + _ => { + self.reduce(VdPrecedenceRange::SPACE_LEFT); + } + } }; if let Some(expr) = self.take_complete_expr() { match preceding_space_annotation { @@ -165,9 +170,9 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { // self.push_top_syn_expr(top_expr) // } - pub(super) fn reduce(&mut self, next_precedence: VdPrecedence) { + pub(super) fn reduce(&mut self, precedence_range: VdPrecedenceRange) { while let Some(prev_precedence) = self.stack.prev_unfinished_expr_precedence() { - if prev_precedence < next_precedence { + if !precedence_range.include(prev_precedence) { break; } match self.stack.incomplete_exprs.pop().unwrap().0 { diff --git a/crates/visored/visored-syn-expr/src/range.rs b/crates/visored/visored-syn-expr/src/range.rs index f2af8fd1a3..9df6e88061 100644 --- a/crates/visored/visored-syn-expr/src/range.rs +++ b/crates/visored/visored-syn-expr/src/range.rs @@ -1,16 +1,25 @@ -use latex_token::idx::LxTokenIdxRange; - use crate::{ clause::{VdSynClauseArena, VdSynClauseIdx, VdSynClauseMap}, - expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx, VdSynExprMap}, + expr::{VdSynExprArena, VdSynExprData, VdSynExprIdx, VdSynExprMap, VdSynSeparator}, phrase::{VdSynPhraseArena, VdSynPhraseIdx, VdSynPhraseMap}, sentence::{VdSynSentenceArena, VdSynSentenceIdx, VdSynSentenceMap}, }; +use either::*; +use latex_token::idx::LxTokenIdxRange; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VdSynExprTokenIdxRange { Standard(LxTokenIdxRange), } +impl VdSynExprTokenIdxRange { + fn join(self, other: VdSynExprTokenIdxRange) -> Self { + match (self, other) { + (VdSynExprTokenIdxRange::Standard(slf), VdSynExprTokenIdxRange::Standard(other)) => { + VdSynExprTokenIdxRange::Standard(slf.join(other)) + } + } + } +} pub type VdSynPhraseTokenIdxRange = LxTokenIdxRange; pub type VdSynClauseTokenIdxRange = LxTokenIdxRange; @@ -102,7 +111,7 @@ impl<'db> VdSynExprRangeCalculator<'db> { VdSynExprData::Literal { token_idx_range, literal, - } => todo!(), + } => VdSynExprTokenIdxRange::Standard(token_idx_range), VdSynExprData::Notation => todo!(), VdSynExprData::BaseOpr { opr } => todo!(), VdSynExprData::Binary { lopd, opr, ropd } => todo!(), @@ -114,7 +123,22 @@ impl<'db> VdSynExprRangeCalculator<'db> { VdSynExprData::UniadicArray => todo!(), VdSynExprData::VariadicArray => todo!(), VdSynExprData::Err(ref e) => VdSynExprTokenIdxRange::Standard(e.token_idx_range()), - VdSynExprData::SeparatedList { .. } => todo!(), + VdSynExprData::SeparatedList { ref fragments, .. } => { + // use the first and the last fragment's range + let mut t = |fragment: Either| match fragment { + Left(expr) | Right(VdSynSeparator::Composite(expr, _)) => self.get_expr(expr), + Right(VdSynSeparator::Base(lx_math_token_idx, _)) => { + VdSynExprTokenIdxRange::Standard(LxTokenIdxRange::new_single( + *lx_math_token_idx, + )) + } + }; + let first = *fragments.first().expect("fragments are always non-empty"); + let last = *fragments.last().expect("fragments are always non-empty"); + let first_range = t(first); + let last_range = t(last); + first_range.join(last_range) + } } } diff --git a/crates/visored/visored-syn-expr/src/test_helpers/example.rs b/crates/visored/visored-syn-expr/src/test_helpers/example.rs index d977d5bc67..d527354394 100644 --- a/crates/visored/visored-syn-expr/src/test_helpers/example.rs +++ b/crates/visored/visored-syn-expr/src/test_helpers/example.rs @@ -72,9 +72,7 @@ impl VdSynExprExample { &annotations, ); let result = (whole_token_range, asts).to_vd_syn(&mut builder); - todo!(); let (expr_arena, phrase_arena, clause_arena, sentence_arena) = builder.finish(); - todo!(); let (expr_range_map, phrase_range_map, clause_range_map, sentence_range_map) = calc_expr_range_map( db, From f472c2d7e5f66c9b0f683451f2f7fb2c523beef4 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 03:33:47 -0700 Subject: [PATCH 66/71] save --- Cargo.toml | 2 +- crates/visored/visored-syn-expr/src/expr.rs | 12 +++++++----- crates/visored/visored-syn-expr/src/parser/accept.rs | 3 --- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index df05ba36b9..1dd023f026 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,7 +119,7 @@ ena = "0.14.2" env_logger = "0.10" env_proxy = "0.4" epaint = { git = "https://github.com/xiyuzhai-husky-lang/egui.git" } -expect-test = "1.4.1" +expect-test = "1.5.0" futures = "0.3.28" futures-util = { version = "0.3", default-features = false, features = [ "sink", diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index d9bee34502..c378c01d31 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -216,7 +216,7 @@ mod tests { space_annotations, db, ); - expected.assert_debug_eq(&example.show_display_tree(db)); + expected.assert_eq(&example.show_display_tree(db)); } t( @@ -224,7 +224,7 @@ mod tests { &[], &[], &expect![[r#" - "\n" + "#]], ); t( @@ -232,7 +232,7 @@ mod tests { &[], &[], &expect![[r#" - "1\n" + 1 "#]], ); t( @@ -240,7 +240,7 @@ mod tests { &[], &[], &expect![[r#" - "11\n" + 11 "#]], ); t( @@ -248,7 +248,9 @@ mod tests { &[], &[], &expect![[r#" - "1 1\n├─ 1\n└─ 1\n" + 1 1 + ├─ 1 + └─ 1 "#]], ); } diff --git a/crates/visored/visored-syn-expr/src/parser/accept.rs b/crates/visored/visored-syn-expr/src/parser/accept.rs index c0fc1c0525..96d44879c1 100644 --- a/crates/visored/visored-syn-expr/src/parser/accept.rs +++ b/crates/visored/visored-syn-expr/src/parser/accept.rs @@ -19,8 +19,6 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { preceding_space_annotation: Option, token: DisambiguatedToken, ) { - use husky_print_utils::p; - p!(self.stack, token); match token { DisambiguatedToken::Expr(expr, class) => match class { VdSynExprClass::Atom => self.accept_atom(preceding_space_annotation, expr), @@ -34,7 +32,6 @@ impl<'a, 'db> VdSynExprParser<'a, 'db> { DisambiguatedToken::RightDelimiter(vd_right_delimiter) => todo!(), DisambiguatedToken::Letter(lx_math_token_idx, lx_math_letter) => todo!(), } - p!(self.stack); } fn accept_list_end(&mut self, ket: VdBaseRightDelimiter, ket_token_idx: LxTokenIdx) { From 4586df6d9d39ebb08a9b1e2cb0467c35571bc97d Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 03:50:00 -0700 Subject: [PATCH 67/71] circumvent expect test bug --- crates/visored/visored-syn-expr/src/expr.rs | 33 ++++----------------- scripts/update_expect_quick.sh | 2 +- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index c378c01d31..cd337f2405 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -216,33 +216,13 @@ mod tests { space_annotations, db, ); - expected.assert_eq(&example.show_display_tree(db)); + // needs to trim to avoid expect test bug + expected.assert_eq(&example.show_display_tree(db).trim_end_matches('\n')); } - t( - "", - &[], - &[], - &expect![[r#" - - "#]], - ); - t( - "1", - &[], - &[], - &expect![[r#" - 1 - "#]], - ); - t( - "11", - &[], - &[], - &expect![[r#" - 11 - "#]], - ); + t("", &[], &[], &expect![""]); + t("1", &[], &[], &expect!["1"]); + t("11", &[], &[], &expect!["11"]); t( "1 1", &[], @@ -250,8 +230,7 @@ mod tests { &expect![[r#" 1 1 ├─ 1 - └─ 1 - "#]], + └─ 1"#]], ); } } diff --git a/scripts/update_expect_quick.sh b/scripts/update_expect_quick.sh index a0a72b2bfb..bb614fed4f 100755 --- a/scripts/update_expect_quick.sh +++ b/scripts/update_expect_quick.sh @@ -35,7 +35,7 @@ OVERALL_SUCCESS=true # Loop through each target and run tests for TARGET in $MAKE_QUICK_TARGETS; do cargo build -p $TARGET --tests - if ! UPDATE_EXPECT=1 cargo test -p "$TARGET" -j 1 -- --nocapture; then + if ! UPDATE_EXPECT=1 cargo test -p "$TARGET" -j 1 -- --nocapture && cargo test -p "$TARGET"; then scripts/play_update_expect_failure_music.sh exit 1 # Exit the script with an error code fi From 0f6f9c10c2ed762e16be1c45a216386a253c4e94 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 03:50:57 -0700 Subject: [PATCH 68/71] save --- scripts/update_expect_quick.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/update_expect_quick.sh b/scripts/update_expect_quick.sh index bb614fed4f..46fd24590e 100755 --- a/scripts/update_expect_quick.sh +++ b/scripts/update_expect_quick.sh @@ -29,9 +29,6 @@ UPDATE_EXPECT=1 cargo test -p husky-jar-utils # Check the Rust project, including tests cargo check --tests -# Initialize a variable to track overall success -OVERALL_SUCCESS=true - # Loop through each target and run tests for TARGET in $MAKE_QUICK_TARGETS; do cargo build -p $TARGET --tests From 9c5360154d66148ccd09ae305eb1d7a0bc4fc128 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 03:53:13 -0700 Subject: [PATCH 69/71] better --- crates/visored/visored-syn-expr/src/expr.rs | 22 ++++++++++++------- .../src/helpers/show/display_tree.rs | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index cd337f2405..3e349a2cdc 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -216,21 +216,27 @@ mod tests { space_annotations, db, ); - // needs to trim to avoid expect test bug - expected.assert_eq(&example.show_display_tree(db).trim_end_matches('\n')); + expected.assert_eq(&example.show_display_tree(db)); } - t("", &[], &[], &expect![""]); - t("1", &[], &[], &expect!["1"]); - t("11", &[], &[], &expect!["11"]); + t("", &[], &[], &expect![[r#" + "" + "#]]); + t("1", &[], &[], &expect![[r#" + "1" + "#]]); + t("11", &[], &[], &expect![[r#" + "11" + "#]]); t( "1 1", &[], &[], &expect![[r#" - 1 1 - ├─ 1 - └─ 1"#]], + "1 1" + ├─ "1" + └─ "1" + "#]], ); } } diff --git a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs index d8c81d64fb..ebfeae02e9 100644 --- a/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs +++ b/crates/visored/visored-syn-expr/src/helpers/show/display_tree.rs @@ -94,7 +94,7 @@ impl<'a> VdSynExprDisplayTreeBuilder<'a> { .token_storage .token_idx_range_offset_range(token_idx_range), }; - let value = self.input[start..end].to_string(); + let value = format!("{:?}", &self.input[start..end]); DisplayTree::new(value, self.render_exprs(self.expr_arena[expr].children())) } From d7e8f0ff83cc57dc1142f1d684a8564053a4c4f9 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 12:34:01 -0700 Subject: [PATCH 70/71] intro `Offset` type, #346 --- Cargo.lock | 4 + .../shifted-unsigned-int/src/lib.rs | 55 +++++- crates/devtime/husky-trace/src/token.rs | 6 +- crates/hir/husky-hir-eager-expr/Cargo.toml | 2 + .../src/helpers/codespan.rs | 3 +- crates/lex/husky-text/src/index.rs | 58 +++--- crates/lex/husky-text/src/lib.rs | 4 +- crates/lex/husky-token/src/lex/lexer.rs | 4 +- crates/lex/husky-token/src/verse/builder.rs | 4 +- .../protocols/husky-text-protocol/Cargo.toml | 2 + .../husky-text-protocol/src/change.rs | 2 +- .../protocols/husky-text-protocol/src/lib.rs | 1 + .../husky-text-protocol/src/line_map.rs | 126 ++++++------- .../husky-text-protocol/src/line_map/lsp.rs | 7 +- .../husky-text-protocol/src/offset.rs | 167 ++++++++++++++++++ .../husky-text-protocol/src/position.rs | 14 +- .../src/position/column.rs | 47 ++++- .../husky-text-protocol/src/position/line.rs | 10 ++ crates/semantics/husky-sem-expr/Cargo.toml | 1 + .../husky-sem-expr/src/helpers/codespan.rs | 3 +- crates/syntax/husky-syn-expr/Cargo.toml | 2 + .../husky-syn-expr/src/helpers/codespan.rs | 5 +- .../toml/husky-toml-token/src/line_group.rs | 4 +- .../expect-files/husky_lang_packages.txt | 4 + crates/visored/visored-syn-expr/src/expr.rs | 27 ++- 25 files changed, 434 insertions(+), 128 deletions(-) create mode 100644 crates/protocols/husky-text-protocol/src/offset.rs diff --git a/Cargo.lock b/Cargo.lock index cf1fd201a5..02c5925eb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5489,6 +5489,7 @@ dependencies = [ "husky-syn-expr", "husky-term-prelude", "husky-text", + "husky-text-protocol", "husky-token", "husky-token-data", "husky-vfs", @@ -6435,6 +6436,7 @@ dependencies = [ "husky-syn-opr", "husky-term-prelude", "husky-text", + "husky-text-protocol", "husky-token", "husky-token-data", "husky-toml-ast", @@ -6884,6 +6886,7 @@ dependencies = [ "husky-syn-opr", "husky-term-prelude", "husky-text", + "husky-text-protocol", "husky-token", "husky-token-data", "husky-toml-ast", @@ -6983,6 +6986,7 @@ dependencies = [ "lsp-types 0.95.0", "rustc-hash 2.0.0", "serde", + "shifted-unsigned-int", ] [[package]] diff --git a/crates/abstractions/shifted-unsigned-int/src/lib.rs b/crates/abstractions/shifted-unsigned-int/src/lib.rs index b802e99d9e..0087f41ef7 100644 --- a/crates/abstractions/shifted-unsigned-int/src/lib.rs +++ b/crates/abstractions/shifted-unsigned-int/src/lib.rs @@ -1,7 +1,7 @@ #![feature(nonzero_ops)] use std::{ num::{NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize}, - ops::{Add, AddAssign, Sub}, + ops::{Add, AddAssign, Sub, SubAssign}, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -88,6 +88,13 @@ impl ShiftedU32 { } } +impl From for ShiftedU32 { + fn from(value: i32) -> Self { + let value: u32 = value.try_into().unwrap(); + ShiftedU32::from(value) + } +} + impl From for ShiftedU32 { fn from(value: u32) -> Self { debug_assert!(value + 1 < u32::MAX as u32); @@ -120,12 +127,6 @@ impl ShiftedU32 { } } -impl AddAssign for ShiftedU32 { - fn add_assign(&mut self, rhs: u32) { - self.0 = unsafe { NonZeroU32::new_unchecked(self.0.get() + rhs) } - } -} - impl Add for ShiftedU32 { type Output = Self; @@ -156,6 +157,46 @@ impl Sub for ShiftedU32 { } } +impl Sub for ShiftedU32 { + type Output = Self; + fn sub(self, rhs: u32) -> Self::Output { + self.checked_sub(rhs).unwrap() + } +} + +impl Sub for ShiftedU32 { + type Output = Self; + fn sub(self, rhs: i32) -> Self::Output { + self.checked_sub(rhs.try_into().unwrap()).unwrap() + } +} + +impl AddAssign for ShiftedU32 { + fn add_assign(&mut self, rhs: u32) { + self.0 = unsafe { NonZeroU32::new_unchecked(self.0.get() + rhs) } + } +} + +impl AddAssign for ShiftedU32 { + fn add_assign(&mut self, rhs: usize) { + let rhs: u32 = rhs.try_into().unwrap(); + *self += rhs + } +} + +impl SubAssign for ShiftedU32 { + fn sub_assign(&mut self, rhs: u32) { + *self = *self - rhs; + } +} + +impl SubAssign for ShiftedU32 { + fn sub_assign(&mut self, rhs: usize) { + let rhs: u32 = rhs.try_into().unwrap(); + *self -= rhs; + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(from = "usize", into = "usize"))] diff --git a/crates/devtime/husky-trace/src/token.rs b/crates/devtime/husky-trace/src/token.rs index 557553de53..ec01c9b5f6 100644 --- a/crates/devtime/husky-trace/src/token.rs +++ b/crates/devtime/husky-trace/src/token.rs @@ -92,13 +92,13 @@ where let spaces_before: u32 = if token_idx.index() > 0 { let prev_text_range = self.ranged_token_sheet.token_text_range(token_idx - 1); if prev_text_range.start.line == text_range.end.line { - text_range.start.col.0 - prev_text_range.end.col.0 + text_range.start.col.index32() - prev_text_range.end.col.index32() } else { self.new_line_if_tokens_data_is_empty(); - text_range.start.col.0 + text_range.start.col.index32() } } else { - text_range.start.col.0 + text_range.start.col.index32() }; self.tokens_data.push(TraceViewTokenData::new( text.to_string(), diff --git a/crates/hir/husky-hir-eager-expr/Cargo.toml b/crates/hir/husky-hir-eager-expr/Cargo.toml index 52e547fa18..c01872b676 100644 --- a/crates/hir/husky-hir-eager-expr/Cargo.toml +++ b/crates/hir/husky-hir-eager-expr/Cargo.toml @@ -32,6 +32,8 @@ husky-text.workspace = true husky-token-data.workspace = true husky-token.workspace = true husky-regional-token.workspace = true +# protocols +husky-text-protocol.workspace = true # semantics husky-sem-expr.workspace = true husky-sem-opr.workspace = true diff --git a/crates/hir/husky-hir-eager-expr/src/helpers/codespan.rs b/crates/hir/husky-hir-eager-expr/src/helpers/codespan.rs index 7db899fe64..cd4813a21d 100644 --- a/crates/hir/husky-hir-eager-expr/src/helpers/codespan.rs +++ b/crates/hir/husky-hir-eager-expr/src/helpers/codespan.rs @@ -17,6 +17,7 @@ use husky_sem_expr::{ SemExprIdx, SemExprRegionData, }; use husky_text::{HasText, Text}; +use husky_text_protocol::offset::OffsetRange; use husky_token::{RangedTokenSheet, TokenDb}; use husky_vfs::path::module_path::ModulePath; @@ -145,7 +146,7 @@ impl<'a> HirEagerExprCodespanEmitter<'a> { /// # getters impl<'a> HirEagerExprCodespanEmitter<'a> { - fn expr_offset_range(&self, expr: HirEagerExprIdx) -> std::ops::Range { + fn expr_offset_range(&self, expr: HirEagerExprIdx) -> OffsetRange { let expr: SemExprIdx = self .hir_eager_expr_source_map_data .hir_eager_to_sem_expr_idx(expr); diff --git a/crates/lex/husky-text/src/index.rs b/crates/lex/husky-text/src/index.rs index 0653af6351..6a5bab3ae7 100644 --- a/crates/lex/husky-text/src/index.rs +++ b/crates/lex/husky-text/src/index.rs @@ -91,33 +91,35 @@ pub(crate) fn text_ref_index_by_range_to_text_position_works() { t("hello", ..((0, 3).into()), "hel"); } -impl<'a> RefIndex<'a, RangeInclusive> for Text<'a> { - type Output = str; +// TODO: inclusive text position might be tricky because a char might span multiple bytes - fn ref_index(self, range_inclusive: RangeInclusive) -> &'a Self::Output { - &self.raw[self.line_map.offset_range_inclusive(range_inclusive)] - } -} +// impl<'a> RefIndex<'a, RangeInclusive> for Text<'a> { +// type Output = str; -#[test] -pub(crate) fn text_ref_index_by_range_inclusive_text_position_works() { - let t = run_index_test_on_text::>; - t("hello", ((0, 0).into())..=((0, 2).into()), "hel"); -} +// fn ref_index(self, range_inclusive: RangeInclusive) -> &'a Self::Output { +// &self.raw[self.line_map.offset_range_inclusive(range_inclusive)] +// } +// } -impl<'a> RefIndex<'a, RangeToInclusive> for Text<'a> { - type Output = str; +// #[test] +// pub(crate) fn text_ref_index_by_range_inclusive_text_position_works() { +// let t = run_index_test_on_text::>; +// t("hello", ((0, 0).into())..=((0, 2).into()), "hel"); +// } - fn ref_index(self, range_to_inclusive: RangeToInclusive) -> &'a Self::Output { - &self.raw[self.line_map.offset_range_to_inclusive(range_to_inclusive)] - } -} +// impl<'a> RefIndex<'a, RangeToInclusive> for Text<'a> { +// type Output = str; -#[test] -pub(crate) fn text_ref_index_by_range_to_inclusive_text_position_works() { - let t = run_index_test_on_text::>; - t("hello", ..=((0, 2).into()), "hel"); -} +// fn ref_index(self, range_to_inclusive: RangeToInclusive) -> &'a Self::Output { +// &self.raw[self.line_map.offset_range_to_inclusive(range_to_inclusive)] +// } +// } + +// #[test] +// pub(crate) fn text_ref_index_by_range_to_inclusive_text_position_works() { +// let t = run_index_test_on_text::>; +// t("hello", ..=((0, 2).into()), "hel"); +// } impl<'a> RefIndex<'a, TextLine> for Text<'a> { type Output = str; @@ -213,9 +215,9 @@ pub(crate) fn text_ref_index_by_range_to_inclusive_text_line_works() { t("hello\nworld\n", ..=(2.into()), "hello\nworld\n"); } -#[test] -#[should_panic] -pub(crate) fn text_ref_index_by_range_to_inclusive_text_line_fails() { - let t = run_index_test_on_text::>; - t("hello\nworld", ..=(2.into()), "hello\nworld"); -} +// #[test] +// #[should_panic] +// pub(crate) fn text_ref_index_by_range_to_inclusive_text_line_fails() { +// let t = run_index_test_on_text::>; +// t("hello\nworld", ..=(2.into()), "hello\nworld"); +// } diff --git a/crates/lex/husky-text/src/lib.rs b/crates/lex/husky-text/src/lib.rs index ec912973a5..fc37ddac24 100644 --- a/crates/lex/husky-text/src/lib.rs +++ b/crates/lex/husky-text/src/lib.rs @@ -8,7 +8,7 @@ mod tests; use self::jar::*; #[cfg(test)] use self::tests::*; -use husky_text_protocol::{line_map::*, position::TextLine, range::*}; +use husky_text_protocol::{line_map::*, offset::OffsetRange, position::TextLine, range::*}; use husky_vfs::path::module_path::ModulePath; use line_map::module_text_line_map; @@ -50,7 +50,7 @@ impl<'a> std::fmt::Debug for Text<'a> { } impl<'a> Text<'a> { - pub fn offset_range(self, range: TextRange) -> std::ops::Range { + pub fn offset_range(self, range: TextRange) -> OffsetRange { self.line_map.offset_range(range) } } diff --git a/crates/lex/husky-token/src/lex/lexer.rs b/crates/lex/husky-token/src/lex/lexer.rs index aa8cfa0b08..afa47b8dd9 100644 --- a/crates/lex/husky-token/src/lex/lexer.rs +++ b/crates/lex/husky-token/src/lex/lexer.rs @@ -109,8 +109,8 @@ impl<'token> Tokenizer<'token> { if token_range.start.col <= ranged_pretoken.range.start.col { break; } - if token_range.start.col.0 - == ranged_pretoken.range.start.col.0 + INDENT_INCR + if token_range.start.col + == ranged_pretoken.range.start.col + INDENT_INCR { token_data = match i { 0 => TokenData::Error( diff --git a/crates/lex/husky-token/src/verse/builder.rs b/crates/lex/husky-token/src/verse/builder.rs index 9ea6f215a1..188cb8e43e 100644 --- a/crates/lex/husky-token/src/verse/builder.rs +++ b/crates/lex/husky-token/src/verse/builder.rs @@ -55,7 +55,7 @@ impl<'a> TokenVersesBuilder<'a> { let mut state = TokenVerseProductionState::None; while *i < line_starts.len() { let line0_start = line_starts[*i]; - let line0_indent = token_ranges[line0_start].start.col.0; + let line0_indent = token_ranges[line0_start].start.col.index32(); if line0_indent < indent0 { return token_verse_starts; } @@ -80,7 +80,7 @@ impl<'a> TokenVersesBuilder<'a> { while j < line_starts.len() { let line1_start = line_starts[j]; let line1_start_token_data = &tokens_data[line1_start]; - let line1_indent = token_ranges[line1_start].start.col.0; + let line1_indent = token_ranges[line1_start].start.col.index32(); let control_flow = if line1_indent > line0_indent { // detected an indentation match tokens_data[line1_start - 1] { diff --git a/crates/protocols/husky-text-protocol/Cargo.toml b/crates/protocols/husky-text-protocol/Cargo.toml index 572dd5c3ec..5b6d68d700 100644 --- a/crates/protocols/husky-text-protocol/Cargo.toml +++ b/crates/protocols/husky-text-protocol/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" serde.workspace = true lsp-types = { workspace = true, optional = true } rustc-hash.workspace = true +# abstractions +shifted-unsigned-int.workspace = true [features] lsp_support = ["dep:lsp-types"] diff --git a/crates/protocols/husky-text-protocol/src/change.rs b/crates/protocols/husky-text-protocol/src/change.rs index 3039ee4d25..305df54485 100644 --- a/crates/protocols/husky-text-protocol/src/change.rs +++ b/crates/protocols/husky-text-protocol/src/change.rs @@ -31,7 +31,7 @@ pub fn apply_document_changes( } valid_range = LineMapValidRange::UpToLineExclusive(range.start.line); let range = LineMap::string_range(&line_map, range); - old_text.replace_range(range, &change.text); + old_text.replace_range(range.raw(), &change.text); } None => { *old_text = change.text; diff --git a/crates/protocols/husky-text-protocol/src/lib.rs b/crates/protocols/husky-text-protocol/src/lib.rs index a57797fb03..79818a7b99 100644 --- a/crates/protocols/husky-text-protocol/src/lib.rs +++ b/crates/protocols/husky-text-protocol/src/lib.rs @@ -4,6 +4,7 @@ pub mod change; pub mod char; pub mod line_map; +pub mod offset; pub mod paragraph; pub mod position; pub mod range; diff --git a/crates/protocols/husky-text-protocol/src/line_map.rs b/crates/protocols/husky-text-protocol/src/line_map.rs index d602d91082..7fc1c2138d 100644 --- a/crates/protocols/husky-text-protocol/src/line_map.rs +++ b/crates/protocols/husky-text-protocol/src/line_map.rs @@ -3,8 +3,10 @@ mod lsp; mod text_bytes_len; mod wide_char; +use offset::{OffsetRange, OffsetRangeFrom, OffsetRangeTo}; pub use text_bytes_len::*; +use crate::offset::Offset; use crate::*; use rustc_hash::FxHashMap; use std::{ @@ -17,7 +19,7 @@ use wide_char::*; pub struct LineMap { text_len_in_bytes: usize, /// Offset the the beginning of each line, zero-based - pub(crate) newlines: Vec, + pub(crate) newlines: Vec, /// List of non-ASCII characters on each line pub(crate) wide_chars_line_map: FxHashMap>, } @@ -36,7 +38,7 @@ impl LineMap { let mut wide_chars_line_map = FxHashMap::default(); let mut wide_chars_buffer: Vec = vec![]; - let mut newlines: Vec = vec![0]; + let mut newlines: Vec = vec![0.into()]; let mut curr_row: usize = 0; let mut curr_col: u32 = 0; let mut line = 0; @@ -44,7 +46,7 @@ impl LineMap { let c_len = c.text_bytes_len(); curr_row += c_len as usize; if c == '\n' { - newlines.push(curr_row); + newlines.push(Offset::from(curr_row)); // Save any utf-16 characters seen in the previous line if !wide_chars_buffer.is_empty() { @@ -84,7 +86,7 @@ impl LineMap { (0.into())..(self.newlines.len().into()) } - pub fn position_from_offset(&self, offset: usize) -> TextPosition { + pub fn position_from_offset(&self, offset: Offset) -> TextPosition { let row = self.newlines.partition_point(|&it| it <= offset) - 1; let line_start_offset = self.newlines[row]; let col = offset - line_start_offset; @@ -95,110 +97,116 @@ impl LineMap { } /// the byte index - pub fn offset(&self, pos: TextPosition) -> usize { - if pos.line.0 as usize == self.newlines.len() { - if pos.col.0 == 0 { - return self.text_len_in_bytes; + pub fn offset(&self, pos: TextPosition) -> Offset { + if pos.line.index() == self.newlines.len() { + if pos.col.index32() == 0 { + return self.text_len_in_bytes.into(); } else { todo!() } } - self.newlines[pos.line.0 as usize] + usize::from(pos.col.0 as usize) + self.newlines[pos.line.index()] + usize::from(pos.col.index()) } - pub fn offset_range(&self, range: impl Into) -> Range { + pub fn offset_range(&self, range: impl Into) -> OffsetRange { let range = range.into(); - self.offset(range.start)..self.offset(range.end) + (self.offset(range.start)..self.offset(range.end)).into() } - pub fn offset_range_from(&self, range_from: RangeFrom) -> RangeFrom { - self.offset(range_from.start).. + pub fn offset_range_from(&self, range_from: RangeFrom) -> OffsetRangeFrom { + (self.offset(range_from.start)..).into() } - pub fn offset_range_to(&self, range_to: RangeTo) -> RangeTo { - ..self.offset(range_to.end) + pub fn offset_range_to(&self, range_to: RangeTo) -> OffsetRangeTo { + (..self.offset(range_to.end)).into() } - pub fn offset_range_inclusive( - &self, - range_inclusive: RangeInclusive, - ) -> RangeInclusive { - self.offset(*range_inclusive.start())..=self.offset(*range_inclusive.end()) - } - - pub fn offset_range_to_inclusive( - &self, - range_to_inclusive: RangeToInclusive, - ) -> RangeToInclusive { - ..=self.offset(range_to_inclusive.end) - } - - pub fn text_line_offset_range(&self, line: TextLine) -> Range { + // pub fn offset_range_inclusive( + // &self, + // range_inclusive: RangeInclusive, + // ) -> OffsetRangeInclusive { + // (self.offset(*range_inclusive.start())..=self.offset(*range_inclusive.end())).into() + // } + + // pub fn offset_range_to_inclusive( + // &self, + // range_to_inclusive: RangeToInclusive, + // ) -> OffsetRangeToInclusive { + // (..=self.offset(range_to_inclusive.end)).into() + // } + + pub fn text_line_offset_range(&self, line: TextLine) -> OffsetRange { let end = if (line.0 as usize) < self.newlines.len() { self.offset(TextPosition { line: line + 1, col: 0.into(), }) } else { - self.text_len_in_bytes + self.text_len_in_bytes.into() }; - self.offset(TextPosition { + (self.offset(TextPosition { line, col: 0.into(), - })..end + })..end) + .into() } - pub fn text_line_range_offset_range(&self, text_line_range: Range) -> Range { - self.offset(TextPosition { + pub fn text_line_range_offset_range(&self, text_line_range: Range) -> OffsetRange { + (self.offset(TextPosition { line: text_line_range.start, col: 0.into(), })..self.offset(TextPosition { line: text_line_range.end, col: 0.into(), - }) + })) + .into() } pub fn offset_range_from_text_line( &self, range_from_text_line: RangeFrom, - ) -> RangeFrom { - self.offset(TextPosition { + ) -> OffsetRangeFrom { + (self.offset(TextPosition { line: range_from_text_line.start, col: 0.into(), - }).. + })..) + .into() } pub fn offset_range_to_text_line( &self, range_from_text_line: RangeTo, - ) -> RangeTo { - ..self.offset(TextPosition { + ) -> OffsetRangeTo { + (..self.offset(TextPosition { line: range_from_text_line.end, col: 0.into(), - }) + })) + .into() } pub fn offset_range_inclusive_text_line( &self, range_from_text_line: RangeInclusive, - ) -> Range { - self.offset(TextPosition { + ) -> OffsetRange { + (self.offset(TextPosition { line: *range_from_text_line.start(), col: 0.into(), })..self.offset(TextPosition { line: *range_from_text_line.end() + 1, col: 0.into(), - }) + })) + .into() } pub fn offset_range_to_inclusive_text_line( &self, range_from_text_line: RangeToInclusive, - ) -> RangeTo { - ..self.offset(TextPosition { + ) -> OffsetRangeTo { + (..self.offset(TextPosition { line: range_from_text_line.end + 1, col: 0.into(), - }) + })) + .into() } pub fn to_utf16(&self, line_col: TextPosition) -> TextPositionUtf16 { @@ -217,24 +225,24 @@ impl LineMap { } } - pub fn lines(&self, range: Range) -> impl Iterator> + '_ { - let lo = self.newlines.partition_point(|&it| it < range.start); - let hi = self.newlines.partition_point(|&it| it <= range.end); - let all = iter::once(range.start) + pub fn lines(&self, range: OffsetRange) -> impl Iterator + '_ { + let lo = self.newlines.partition_point(|&it| it < range.start()); + let hi = self.newlines.partition_point(|&it| it <= range.end()); + let all = iter::once(range.start()) .chain(self.newlines[lo..hi].iter().copied()) - .chain(iter::once(range.end)); + .chain(iter::once(range.end())); all.clone() .zip(all.skip(1)) - .map(|(lo, hi)| lo..hi) + .map(|(lo, hi)| -> OffsetRange { (lo..hi).into() }) .filter(|it| !it.is_empty()) } fn utf8_to_utf16_col(&self, line: TextLine, col: TextColumn) -> TextColumn { - let mut res = col.0; + let mut res = col.index32(); if let Some(utf16_chars) = self.wide_chars_line_map.get(&line) { for c in utf16_chars { - if c.end <= col.0 { + if c.end <= col.index32() { res -= c.len() - c.len_utf16(); } else { // From here on, all utf16 characters come *after* the character we are mapping, @@ -243,14 +251,14 @@ impl LineMap { } } } - TextColumn(res) + TextColumn::from(res) } fn utf16_to_utf8_col(&self, line: TextLine, mut col: TextColumn) -> TextColumn { if let Some(utf16_chars) = self.wide_chars_line_map.get(&line) { for c in utf16_chars { - if col.0 > c.start { - col.0 += c.len() - c.len_utf16(); + if col.index32() > c.start { + col += c.len() - c.len_utf16(); } else { // From here on, all utf16 characters come *after* the character we are mapping, // so we don't need to take them into account diff --git a/crates/protocols/husky-text-protocol/src/line_map/lsp.rs b/crates/protocols/husky-text-protocol/src/line_map/lsp.rs index 8b2ab65be4..0fdc55d1ce 100644 --- a/crates/protocols/husky-text-protocol/src/line_map/lsp.rs +++ b/crates/protocols/husky-text-protocol/src/line_map/lsp.rs @@ -3,16 +3,17 @@ use super::*; #[cfg(feature = "lsp_support")] impl LineMap { #[cfg(feature = "lsp_support")] - pub fn string_range(line_map: &LineMap, range: lsp_types::Range) -> std::ops::Range { + pub fn string_range(line_map: &LineMap, range: lsp_types::Range) -> OffsetRange { let start = line_map.lsp_offset(range.start); let end = line_map.lsp_offset(range.end); - start..end + (start..end).into() } - pub(crate) fn lsp_offset(&self, position: lsp_types::Position) -> usize { + pub(crate) fn lsp_offset(&self, position: lsp_types::Position) -> Offset { let line_col = TextPosition { line: position.line.into(), col: position.character.into(), + // TODO: what is this? // match line_map.encoding // OffsetEncoding::Utf8 => LineCol { // line: position.line as u32, diff --git a/crates/protocols/husky-text-protocol/src/offset.rs b/crates/protocols/husky-text-protocol/src/offset.rs new file mode 100644 index 0000000000..a4dc397671 --- /dev/null +++ b/crates/protocols/husky-text-protocol/src/offset.rs @@ -0,0 +1,167 @@ +use std::ops::{Range, RangeFrom, RangeInclusive, RangeTo, RangeToInclusive}; + +use serde::{Deserialize, Serialize}; +use shifted_unsigned_int::ShiftedU32; + +/// Character byte offset (zero-based). The meaning of this +/// offset is determined by the negotiated `PositionEncodingKind`. +#[derive( + Debug, PartialEq, Default, Eq, PartialOrd, Ord, Clone, Copy, Hash, Serialize, Deserialize, +)] +#[serde(transparent)] +pub struct Offset(ShiftedU32); + +impl Offset { + pub fn index(self) -> usize { + self.0.index() + } +} + +impl From for Offset { + fn from(raw: i32) -> Self { + Offset(ShiftedU32::from(raw)) + } +} + +impl From for Offset { + fn from(raw: usize) -> Self { + Offset(ShiftedU32::from(raw)) + } +} + +impl From for Offset { + fn from(raw: u32) -> Self { + Offset(ShiftedU32::from(raw)) + } +} + +impl std::ops::Add for Offset { + type Output = Offset; + + fn add(self, rhs: usize) -> Self::Output { + Offset(self.0 + rhs) + } +} + +impl std::ops::AddAssign for Offset { + fn add_assign(&mut self, rhs: usize) { + self.0 += rhs; + } +} + +impl std::ops::Sub for Offset { + type Output = usize; + + fn sub(self, rhs: Self) -> Self::Output { + self.index() - rhs.index() + } +} + +/// # Offset Range + +#[derive( + Debug, PartialEq, Default, Eq, PartialOrd, Ord, Clone, Copy, Hash, Serialize, Deserialize, +)] +pub struct OffsetRange { + start: Offset, + end: Offset, +} + +impl From> for OffsetRange { + fn from(range: Range) -> Self { + Self { + start: range.start, + end: range.end, + } + } +} + +impl Into> for OffsetRange { + fn into(self) -> Range { + self.raw() + } +} + +impl OffsetRange { + pub fn new(start: Offset, end: Offset) -> Self { + Self { start, end } + } + + pub(crate) fn raw(self) -> Range { + self.start.index()..self.end.index() + } +} + +impl OffsetRange { + pub fn start(self) -> Offset { + self.start + } + + pub fn end(self) -> Offset { + self.end + } + + pub fn is_empty(self) -> bool { + self.start == self.end + } +} + +impl std::ops::Index for str { + type Output = str; + + fn index(&self, range: OffsetRange) -> &Self::Output { + &self[range.start.index()..range.end.index()] + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct OffsetRangeFrom { + start: Offset, +} + +impl From> for OffsetRangeFrom { + fn from(range: RangeFrom) -> Self { + Self { start: range.start } + } +} + +impl std::ops::Index for str { + type Output = str; + + fn index(&self, range: OffsetRangeFrom) -> &Self::Output { + &self[range.start.index()..] + } +} + +#[derive( + Debug, PartialEq, Default, Eq, PartialOrd, Ord, Clone, Copy, Hash, Serialize, Deserialize, +)] +#[serde(transparent)] +pub struct OffsetRangeTo { + end: Offset, +} + +impl From> for OffsetRangeTo { + fn from(range: RangeTo) -> Self { + Self { end: range.end } + } +} + +impl OffsetRangeTo { + pub fn end(self) -> Offset { + self.end + } + + pub fn raw(self) -> RangeTo { + ..self.end.index() + } +} + +impl std::ops::Index for str { + type Output = str; + + fn index(&self, range: OffsetRangeTo) -> &Self::Output { + &self[range.raw()] + } +} diff --git a/crates/protocols/husky-text-protocol/src/position.rs b/crates/protocols/husky-text-protocol/src/position.rs index 3739c9736c..f6f1b12c94 100644 --- a/crates/protocols/husky-text-protocol/src/position.rs +++ b/crates/protocols/husky-text-protocol/src/position.rs @@ -86,20 +86,20 @@ impl TextPosition { self.line.0 } pub fn j(&self) -> u32 { - self.col.0 + self.col.index32() } pub fn to_left(&self, x: u32) -> TextPosition { Self { line: self.line, - col: TextColumn(self.col.0 - x), + col: self.col - x, } } pub fn to_right(&self, x: u32) -> TextPosition { Self { line: self.line, - col: TextColumn(self.col.0 + x), + col: self.col + x, } } @@ -113,13 +113,17 @@ impl TextPosition { impl std::fmt::Display for TextPosition { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_fmt(format_args!("{:?}:{:?}", self.line.0 + 1, self.col.0 + 1)) + f.write_fmt(format_args!( + "{:?}:{:?}", + self.line.index() + 1, + self.col.index() + 1 + )) } } #[cfg(feature = "lsp_support")] impl From for lsp_types::Position { fn from(val: TextPosition) -> Self { - lsp_types::Position::new(val.line.0, val.col.0) + lsp_types::Position::new(val.line.index32(), val.col.index32()) } } diff --git a/crates/protocols/husky-text-protocol/src/position/column.rs b/crates/protocols/husky-text-protocol/src/position/column.rs index 34d537ccc5..8e41226a3f 100644 --- a/crates/protocols/husky-text-protocol/src/position/column.rs +++ b/crates/protocols/husky-text-protocol/src/position/column.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use shifted_unsigned_int::ShiftedU32; /// Character byte offset on a line in a document (zero-based). The meaning of this /// offset is determined by the negotiated `PositionEncodingKind`. @@ -8,24 +9,62 @@ use serde::{Deserialize, Serialize}; #[derive( Debug, PartialEq, Default, Eq, PartialOrd, Ord, Clone, Copy, Hash, Serialize, Deserialize, )] -pub struct TextColumn(pub u32); +#[serde(transparent)] +pub struct TextColumn(ShiftedU32); + +impl std::ops::Add for TextColumn { + type Output = Self; + fn add(self, rhs: u32) -> Self::Output { + TextColumn(self.0 + rhs) + } +} + +impl std::ops::Sub for TextColumn { + type Output = Self; + fn sub(self, rhs: u32) -> Self::Output { + TextColumn(self.0 - rhs) + } +} + +impl std::ops::AddAssign for TextColumn { + fn add_assign(&mut self, rhs: u32) { + self.0 += rhs; + } +} + +impl std::ops::SubAssign for TextColumn { + fn sub_assign(&mut self, rhs: u32) { + self.0 -= rhs; + } +} + +impl TextColumn { + pub fn index(&self) -> usize { + self.0.index() + } + + pub fn index32(&self) -> u32 { + self.0.into() + } +} impl From for TextColumn { fn from(raw: u32) -> Self { - TextColumn(raw) + TextColumn(ShiftedU32::from(raw)) } } impl From for TextColumn { fn from(raw: usize) -> Self { - TextColumn(>::try_into(raw).expect("success")) + TextColumn(ShiftedU32::from(raw)) } } impl From for TextColumn { fn from(raw: i32) -> Self { assert!(raw >= 0); - TextColumn(raw as u32) + let raw: u32 = raw.try_into().unwrap(); + TextColumn(ShiftedU32::from(raw)) } } diff --git a/crates/protocols/husky-text-protocol/src/position/line.rs b/crates/protocols/husky-text-protocol/src/position/line.rs index 286be56321..dc609708d8 100644 --- a/crates/protocols/husky-text-protocol/src/position/line.rs +++ b/crates/protocols/husky-text-protocol/src/position/line.rs @@ -4,6 +4,16 @@ use serde::{Deserialize, Serialize}; #[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash, Serialize, Deserialize)] pub struct TextLine(pub u32); +impl TextLine { + pub fn index(self) -> usize { + self.0 as usize + } + + pub(crate) fn index32(&self) -> u32 { + self.0 + } +} + impl TextLine { pub fn to_next_line(&self) -> Self { Self(self.0 + 1) diff --git a/crates/semantics/husky-sem-expr/Cargo.toml b/crates/semantics/husky-sem-expr/Cargo.toml index d33ed5ef5c..ad5566a5df 100644 --- a/crates/semantics/husky-sem-expr/Cargo.toml +++ b/crates/semantics/husky-sem-expr/Cargo.toml @@ -36,6 +36,7 @@ husky-token-data.workspace = true husky-token.workspace = true husky-regional-token.workspace = true # protocol +husky-text-protocol.workspace = true husky-visual-protocol.workspace = true # semantics husky-sem-opr.workspace = true diff --git a/crates/semantics/husky-sem-expr/src/helpers/codespan.rs b/crates/semantics/husky-sem-expr/src/helpers/codespan.rs index 7624d82f21..9f153dc715 100644 --- a/crates/semantics/husky-sem-expr/src/helpers/codespan.rs +++ b/crates/semantics/husky-sem-expr/src/helpers/codespan.rs @@ -11,6 +11,7 @@ use husky_entity_path::region::RegionPath; use husky_entity_tree::helpers::tokra_region::HasRegionalTokenIdxBase; use husky_regional_token::RegionalTokenIdxBase; use husky_text::{HasText, Text}; +use husky_text_protocol::offset::OffsetRange; use husky_token::{RangedTokenSheet, TokenDb}; use husky_vfs::path::module_path::ModulePath; use salsa::DisplayWithDb; @@ -133,7 +134,7 @@ impl<'a> SemExprCodespanEmitter<'a> { /// # getters impl<'a> SemExprCodespanEmitter<'a> { - fn expr_offset_range(&self, expr: SemExprIdx) -> std::ops::Range { + fn expr_offset_range(&self, expr: SemExprIdx) -> OffsetRange { let token_idx_range = self.sem_expr_range_region_data[expr].token_idx_range(self.regional_token_idx_base); let text_range = self.ranged_token_sheet.tokens_text_range(token_idx_range); diff --git a/crates/syntax/husky-syn-expr/Cargo.toml b/crates/syntax/husky-syn-expr/Cargo.toml index 62f1b3bb7a..d249b0de3d 100644 --- a/crates/syntax/husky-syn-expr/Cargo.toml +++ b/crates/syntax/husky-syn-expr/Cargo.toml @@ -33,6 +33,8 @@ husky-entity-kind.workspace = true husky-term-prelude.workspace = true husky-opr.workspace = true husky-expr = { path = "../../kernel/husky-expr" } +# protocols +husky-text-protocol.workspace = true # syntax husky-entity-tree.workspace = true husky-item-decl-ast.workspace = true diff --git a/crates/syntax/husky-syn-expr/src/helpers/codespan.rs b/crates/syntax/husky-syn-expr/src/helpers/codespan.rs index 87e4872157..16dcb4fb0f 100644 --- a/crates/syntax/husky-syn-expr/src/helpers/codespan.rs +++ b/crates/syntax/husky-syn-expr/src/helpers/codespan.rs @@ -17,6 +17,7 @@ use husky_entity_tree::{ }; use husky_regional_token::RegionalTokenIdxBase; use husky_text::{HasText, Text}; +use husky_text_protocol::offset::OffsetRange; use husky_token::{RangedTokenSheet, TokenDb}; use husky_vfs::path::module_path::ModulePath; use salsa::DisplayWithDb; @@ -169,14 +170,14 @@ impl<'a> SynExprCodespanEmitter<'a> { /// # getters impl<'a> SynExprCodespanEmitter<'a> { - fn expr_offset_range(&self, expr: SynExprIdx) -> std::ops::Range { + fn expr_offset_range(&self, expr: SynExprIdx) -> OffsetRange { let token_idx_range = self.syn_expr_range_region[expr].token_idx_range(self.regional_token_idx_base); let text_range = self.ranged_token_sheet.tokens_text_range(token_idx_range); self.text.offset_range(text_range) } - fn pattern_offset_range(&self, pattern: SynPatternIdx) -> std::ops::Range { + fn pattern_offset_range(&self, pattern: SynPatternIdx) -> OffsetRange { let token_idx_range = self.syn_expr_range_region[pattern].token_idx_range(self.regional_token_idx_base); let text_range = self.ranged_token_sheet.tokens_text_range(token_idx_range); diff --git a/crates/toml/husky-toml-token/src/line_group.rs b/crates/toml/husky-toml-token/src/line_group.rs index 130183c311..2e16b285f0 100644 --- a/crates/toml/husky-toml-token/src/line_group.rs +++ b/crates/toml/husky-toml-token/src/line_group.rs @@ -6,14 +6,14 @@ pub(crate) fn produce_line_group_starts(tokens: &[TomlToken]) -> Vec { let mut line_group_starts = vec![]; while i < line_starts.len() { let line_start0 = line_starts[i]; - let line_indent0 = tokens[line_start0].range().start.col.0; + let line_indent0 = tokens[line_start0].range().start.col.index32(); line_group_starts.push(line_start0); i = { let mut j = i + 1; while j < line_starts.len() { let line_start1 = line_starts[j]; let line_start_token = &tokens[line_start1]; - let line_indent1 = line_start_token.range().start.col.0; + let line_indent1 = line_start_token.range().start.col.index32(); if line_indent1 > line_indent0 { // detect an indentation j += 1; diff --git a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt index 2e54823d52..eb47c98480 100644 --- a/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt +++ b/crates/utils/husky-jar-utils/expect-files/husky_lang_packages.txt @@ -738,6 +738,7 @@ "lsp-types", "rustc-hash", "serde", + "shifted-unsigned-int", ], }, PackageSummary { @@ -1286,6 +1287,7 @@ "husky-syn-opr", "husky-term-prelude", "husky-text", + "husky-text-protocol", "husky-token", "husky-token-data", "husky-vfs", @@ -1690,6 +1692,7 @@ "husky-syn-expr", "husky-term-prelude", "husky-text", + "husky-text-protocol", "husky-token", "husky-token-data", "husky-vfs", @@ -1810,6 +1813,7 @@ "husky-syn-opr", "husky-term-prelude", "husky-text", + "husky-text-protocol", "husky-token", "husky-token-data", "husky-vfs", diff --git a/crates/visored/visored-syn-expr/src/expr.rs b/crates/visored/visored-syn-expr/src/expr.rs index 3e349a2cdc..d143a3b279 100644 --- a/crates/visored/visored-syn-expr/src/expr.rs +++ b/crates/visored/visored-syn-expr/src/expr.rs @@ -219,15 +219,30 @@ mod tests { expected.assert_eq(&example.show_display_tree(db)); } - t("", &[], &[], &expect![[r#" + t( + "", + &[], + &[], + &expect![[r#" "" - "#]]); - t("1", &[], &[], &expect![[r#" + "#]], + ); + t( + "1", + &[], + &[], + &expect![[r#" "1" - "#]]); - t("11", &[], &[], &expect![[r#" + "#]], + ); + t( + "11", + &[], + &[], + &expect![[r#" "11" - "#]]); + "#]], + ); t( "1 1", &[], From c9be034b2fd00a2af1d1fbadfa6ed3b6a6672536 Mon Sep 17 00:00:00 2001 From: Xiyu Zhai Date: Fri, 1 Nov 2024 13:47:48 -0700 Subject: [PATCH 71/71] rename for clarity, #346 --- .../src/helpers/codespan.rs | 4 +- crates/lex/husky-text/src/lib.rs | 4 +- .../husky-text-protocol/src/line_map.rs | 39 ++++----- .../husky-text-protocol/src/line_map/lsp.rs | 4 +- .../husky-text-protocol/src/offset.rs | 80 +++++++++---------- .../husky-sem-expr/src/helpers/codespan.rs | 4 +- .../husky-syn-expr/src/helpers/codespan.rs | 6 +- 7 files changed, 72 insertions(+), 69 deletions(-) diff --git a/crates/hir/husky-hir-eager-expr/src/helpers/codespan.rs b/crates/hir/husky-hir-eager-expr/src/helpers/codespan.rs index cd4813a21d..34039abed0 100644 --- a/crates/hir/husky-hir-eager-expr/src/helpers/codespan.rs +++ b/crates/hir/husky-hir-eager-expr/src/helpers/codespan.rs @@ -17,7 +17,7 @@ use husky_sem_expr::{ SemExprIdx, SemExprRegionData, }; use husky_text::{HasText, Text}; -use husky_text_protocol::offset::OffsetRange; +use husky_text_protocol::offset::TextOffsetRange; use husky_token::{RangedTokenSheet, TokenDb}; use husky_vfs::path::module_path::ModulePath; @@ -146,7 +146,7 @@ impl<'a> HirEagerExprCodespanEmitter<'a> { /// # getters impl<'a> HirEagerExprCodespanEmitter<'a> { - fn expr_offset_range(&self, expr: HirEagerExprIdx) -> OffsetRange { + fn expr_offset_range(&self, expr: HirEagerExprIdx) -> TextOffsetRange { let expr: SemExprIdx = self .hir_eager_expr_source_map_data .hir_eager_to_sem_expr_idx(expr); diff --git a/crates/lex/husky-text/src/lib.rs b/crates/lex/husky-text/src/lib.rs index fc37ddac24..6fc1fef165 100644 --- a/crates/lex/husky-text/src/lib.rs +++ b/crates/lex/husky-text/src/lib.rs @@ -8,7 +8,7 @@ mod tests; use self::jar::*; #[cfg(test)] use self::tests::*; -use husky_text_protocol::{line_map::*, offset::OffsetRange, position::TextLine, range::*}; +use husky_text_protocol::{line_map::*, offset::TextOffsetRange, position::TextLine, range::*}; use husky_vfs::path::module_path::ModulePath; use line_map::module_text_line_map; @@ -50,7 +50,7 @@ impl<'a> std::fmt::Debug for Text<'a> { } impl<'a> Text<'a> { - pub fn offset_range(self, range: TextRange) -> OffsetRange { + pub fn offset_range(self, range: TextRange) -> TextOffsetRange { self.line_map.offset_range(range) } } diff --git a/crates/protocols/husky-text-protocol/src/line_map.rs b/crates/protocols/husky-text-protocol/src/line_map.rs index 7fc1c2138d..d955130dd9 100644 --- a/crates/protocols/husky-text-protocol/src/line_map.rs +++ b/crates/protocols/husky-text-protocol/src/line_map.rs @@ -3,10 +3,10 @@ mod lsp; mod text_bytes_len; mod wide_char; -use offset::{OffsetRange, OffsetRangeFrom, OffsetRangeTo}; +use offset::{TextOffsetRange, TextOffsetRangeFrom, TextOffsetRangeTo}; pub use text_bytes_len::*; -use crate::offset::Offset; +use crate::offset::TextOffset; use crate::*; use rustc_hash::FxHashMap; use std::{ @@ -19,7 +19,7 @@ use wide_char::*; pub struct LineMap { text_len_in_bytes: usize, /// Offset the the beginning of each line, zero-based - pub(crate) newlines: Vec, + pub(crate) newlines: Vec, /// List of non-ASCII characters on each line pub(crate) wide_chars_line_map: FxHashMap>, } @@ -38,7 +38,7 @@ impl LineMap { let mut wide_chars_line_map = FxHashMap::default(); let mut wide_chars_buffer: Vec = vec![]; - let mut newlines: Vec = vec![0.into()]; + let mut newlines: Vec = vec![0.into()]; let mut curr_row: usize = 0; let mut curr_col: u32 = 0; let mut line = 0; @@ -46,7 +46,7 @@ impl LineMap { let c_len = c.text_bytes_len(); curr_row += c_len as usize; if c == '\n' { - newlines.push(Offset::from(curr_row)); + newlines.push(TextOffset::from(curr_row)); // Save any utf-16 characters seen in the previous line if !wide_chars_buffer.is_empty() { @@ -86,7 +86,7 @@ impl LineMap { (0.into())..(self.newlines.len().into()) } - pub fn position_from_offset(&self, offset: Offset) -> TextPosition { + pub fn position_from_offset(&self, offset: TextOffset) -> TextPosition { let row = self.newlines.partition_point(|&it| it <= offset) - 1; let line_start_offset = self.newlines[row]; let col = offset - line_start_offset; @@ -97,7 +97,7 @@ impl LineMap { } /// the byte index - pub fn offset(&self, pos: TextPosition) -> Offset { + pub fn offset(&self, pos: TextPosition) -> TextOffset { if pos.line.index() == self.newlines.len() { if pos.col.index32() == 0 { return self.text_len_in_bytes.into(); @@ -108,16 +108,16 @@ impl LineMap { self.newlines[pos.line.index()] + usize::from(pos.col.index()) } - pub fn offset_range(&self, range: impl Into) -> OffsetRange { + pub fn offset_range(&self, range: impl Into) -> TextOffsetRange { let range = range.into(); (self.offset(range.start)..self.offset(range.end)).into() } - pub fn offset_range_from(&self, range_from: RangeFrom) -> OffsetRangeFrom { + pub fn offset_range_from(&self, range_from: RangeFrom) -> TextOffsetRangeFrom { (self.offset(range_from.start)..).into() } - pub fn offset_range_to(&self, range_to: RangeTo) -> OffsetRangeTo { + pub fn offset_range_to(&self, range_to: RangeTo) -> TextOffsetRangeTo { (..self.offset(range_to.end)).into() } @@ -135,7 +135,7 @@ impl LineMap { // (..=self.offset(range_to_inclusive.end)).into() // } - pub fn text_line_offset_range(&self, line: TextLine) -> OffsetRange { + pub fn text_line_offset_range(&self, line: TextLine) -> TextOffsetRange { let end = if (line.0 as usize) < self.newlines.len() { self.offset(TextPosition { line: line + 1, @@ -151,7 +151,10 @@ impl LineMap { .into() } - pub fn text_line_range_offset_range(&self, text_line_range: Range) -> OffsetRange { + pub fn text_line_range_offset_range( + &self, + text_line_range: Range, + ) -> TextOffsetRange { (self.offset(TextPosition { line: text_line_range.start, col: 0.into(), @@ -165,7 +168,7 @@ impl LineMap { pub fn offset_range_from_text_line( &self, range_from_text_line: RangeFrom, - ) -> OffsetRangeFrom { + ) -> TextOffsetRangeFrom { (self.offset(TextPosition { line: range_from_text_line.start, col: 0.into(), @@ -176,7 +179,7 @@ impl LineMap { pub fn offset_range_to_text_line( &self, range_from_text_line: RangeTo, - ) -> OffsetRangeTo { + ) -> TextOffsetRangeTo { (..self.offset(TextPosition { line: range_from_text_line.end, col: 0.into(), @@ -187,7 +190,7 @@ impl LineMap { pub fn offset_range_inclusive_text_line( &self, range_from_text_line: RangeInclusive, - ) -> OffsetRange { + ) -> TextOffsetRange { (self.offset(TextPosition { line: *range_from_text_line.start(), col: 0.into(), @@ -201,7 +204,7 @@ impl LineMap { pub fn offset_range_to_inclusive_text_line( &self, range_from_text_line: RangeToInclusive, - ) -> OffsetRangeTo { + ) -> TextOffsetRangeTo { (..self.offset(TextPosition { line: range_from_text_line.end + 1, col: 0.into(), @@ -225,7 +228,7 @@ impl LineMap { } } - pub fn lines(&self, range: OffsetRange) -> impl Iterator + '_ { + pub fn lines(&self, range: TextOffsetRange) -> impl Iterator + '_ { let lo = self.newlines.partition_point(|&it| it < range.start()); let hi = self.newlines.partition_point(|&it| it <= range.end()); let all = iter::once(range.start()) @@ -234,7 +237,7 @@ impl LineMap { all.clone() .zip(all.skip(1)) - .map(|(lo, hi)| -> OffsetRange { (lo..hi).into() }) + .map(|(lo, hi)| -> TextOffsetRange { (lo..hi).into() }) .filter(|it| !it.is_empty()) } diff --git a/crates/protocols/husky-text-protocol/src/line_map/lsp.rs b/crates/protocols/husky-text-protocol/src/line_map/lsp.rs index 0fdc55d1ce..f51db753dc 100644 --- a/crates/protocols/husky-text-protocol/src/line_map/lsp.rs +++ b/crates/protocols/husky-text-protocol/src/line_map/lsp.rs @@ -3,13 +3,13 @@ use super::*; #[cfg(feature = "lsp_support")] impl LineMap { #[cfg(feature = "lsp_support")] - pub fn string_range(line_map: &LineMap, range: lsp_types::Range) -> OffsetRange { + pub fn string_range(line_map: &LineMap, range: lsp_types::Range) -> TextOffsetRange { let start = line_map.lsp_offset(range.start); let end = line_map.lsp_offset(range.end); (start..end).into() } - pub(crate) fn lsp_offset(&self, position: lsp_types::Position) -> Offset { + pub(crate) fn lsp_offset(&self, position: lsp_types::Position) -> TextOffset { let line_col = TextPosition { line: position.line.into(), col: position.character.into(), diff --git a/crates/protocols/husky-text-protocol/src/offset.rs b/crates/protocols/husky-text-protocol/src/offset.rs index a4dc397671..2e66508e68 100644 --- a/crates/protocols/husky-text-protocol/src/offset.rs +++ b/crates/protocols/husky-text-protocol/src/offset.rs @@ -9,47 +9,47 @@ use shifted_unsigned_int::ShiftedU32; Debug, PartialEq, Default, Eq, PartialOrd, Ord, Clone, Copy, Hash, Serialize, Deserialize, )] #[serde(transparent)] -pub struct Offset(ShiftedU32); +pub struct TextOffset(ShiftedU32); -impl Offset { +impl TextOffset { pub fn index(self) -> usize { self.0.index() } } -impl From for Offset { +impl From for TextOffset { fn from(raw: i32) -> Self { - Offset(ShiftedU32::from(raw)) + TextOffset(ShiftedU32::from(raw)) } } -impl From for Offset { +impl From for TextOffset { fn from(raw: usize) -> Self { - Offset(ShiftedU32::from(raw)) + TextOffset(ShiftedU32::from(raw)) } } -impl From for Offset { +impl From for TextOffset { fn from(raw: u32) -> Self { - Offset(ShiftedU32::from(raw)) + TextOffset(ShiftedU32::from(raw)) } } -impl std::ops::Add for Offset { - type Output = Offset; +impl std::ops::Add for TextOffset { + type Output = TextOffset; fn add(self, rhs: usize) -> Self::Output { - Offset(self.0 + rhs) + TextOffset(self.0 + rhs) } } -impl std::ops::AddAssign for Offset { +impl std::ops::AddAssign for TextOffset { fn add_assign(&mut self, rhs: usize) { self.0 += rhs; } } -impl std::ops::Sub for Offset { +impl std::ops::Sub for TextOffset { type Output = usize; fn sub(self, rhs: Self) -> Self::Output { @@ -62,13 +62,13 @@ impl std::ops::Sub for Offset { #[derive( Debug, PartialEq, Default, Eq, PartialOrd, Ord, Clone, Copy, Hash, Serialize, Deserialize, )] -pub struct OffsetRange { - start: Offset, - end: Offset, +pub struct TextOffsetRange { + start: TextOffset, + end: TextOffset, } -impl From> for OffsetRange { - fn from(range: Range) -> Self { +impl From> for TextOffsetRange { + fn from(range: Range) -> Self { Self { start: range.start, end: range.end, @@ -76,14 +76,14 @@ impl From> for OffsetRange { } } -impl Into> for OffsetRange { +impl Into> for TextOffsetRange { fn into(self) -> Range { self.raw() } } -impl OffsetRange { - pub fn new(start: Offset, end: Offset) -> Self { +impl TextOffsetRange { + pub fn new(start: TextOffset, end: TextOffset) -> Self { Self { start, end } } @@ -92,12 +92,12 @@ impl OffsetRange { } } -impl OffsetRange { - pub fn start(self) -> Offset { +impl TextOffsetRange { + pub fn start(self) -> TextOffset { self.start } - pub fn end(self) -> Offset { + pub fn end(self) -> TextOffset { self.end } @@ -106,30 +106,30 @@ impl OffsetRange { } } -impl std::ops::Index for str { +impl std::ops::Index for str { type Output = str; - fn index(&self, range: OffsetRange) -> &Self::Output { + fn index(&self, range: TextOffsetRange) -> &Self::Output { &self[range.start.index()..range.end.index()] } } #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize)] #[serde(transparent)] -pub struct OffsetRangeFrom { - start: Offset, +pub struct TextOffsetRangeFrom { + start: TextOffset, } -impl From> for OffsetRangeFrom { - fn from(range: RangeFrom) -> Self { +impl From> for TextOffsetRangeFrom { + fn from(range: RangeFrom) -> Self { Self { start: range.start } } } -impl std::ops::Index for str { +impl std::ops::Index for str { type Output = str; - fn index(&self, range: OffsetRangeFrom) -> &Self::Output { + fn index(&self, range: TextOffsetRangeFrom) -> &Self::Output { &self[range.start.index()..] } } @@ -138,18 +138,18 @@ impl std::ops::Index for str { Debug, PartialEq, Default, Eq, PartialOrd, Ord, Clone, Copy, Hash, Serialize, Deserialize, )] #[serde(transparent)] -pub struct OffsetRangeTo { - end: Offset, +pub struct TextOffsetRangeTo { + end: TextOffset, } -impl From> for OffsetRangeTo { - fn from(range: RangeTo) -> Self { +impl From> for TextOffsetRangeTo { + fn from(range: RangeTo) -> Self { Self { end: range.end } } } -impl OffsetRangeTo { - pub fn end(self) -> Offset { +impl TextOffsetRangeTo { + pub fn end(self) -> TextOffset { self.end } @@ -158,10 +158,10 @@ impl OffsetRangeTo { } } -impl std::ops::Index for str { +impl std::ops::Index for str { type Output = str; - fn index(&self, range: OffsetRangeTo) -> &Self::Output { + fn index(&self, range: TextOffsetRangeTo) -> &Self::Output { &self[range.raw()] } } diff --git a/crates/semantics/husky-sem-expr/src/helpers/codespan.rs b/crates/semantics/husky-sem-expr/src/helpers/codespan.rs index 9f153dc715..3f0c6bdd1e 100644 --- a/crates/semantics/husky-sem-expr/src/helpers/codespan.rs +++ b/crates/semantics/husky-sem-expr/src/helpers/codespan.rs @@ -11,7 +11,7 @@ use husky_entity_path::region::RegionPath; use husky_entity_tree::helpers::tokra_region::HasRegionalTokenIdxBase; use husky_regional_token::RegionalTokenIdxBase; use husky_text::{HasText, Text}; -use husky_text_protocol::offset::OffsetRange; +use husky_text_protocol::offset::TextOffsetRange; use husky_token::{RangedTokenSheet, TokenDb}; use husky_vfs::path::module_path::ModulePath; use salsa::DisplayWithDb; @@ -134,7 +134,7 @@ impl<'a> SemExprCodespanEmitter<'a> { /// # getters impl<'a> SemExprCodespanEmitter<'a> { - fn expr_offset_range(&self, expr: SemExprIdx) -> OffsetRange { + fn expr_offset_range(&self, expr: SemExprIdx) -> TextOffsetRange { let token_idx_range = self.sem_expr_range_region_data[expr].token_idx_range(self.regional_token_idx_base); let text_range = self.ranged_token_sheet.tokens_text_range(token_idx_range); diff --git a/crates/syntax/husky-syn-expr/src/helpers/codespan.rs b/crates/syntax/husky-syn-expr/src/helpers/codespan.rs index 16dcb4fb0f..960cf0784b 100644 --- a/crates/syntax/husky-syn-expr/src/helpers/codespan.rs +++ b/crates/syntax/husky-syn-expr/src/helpers/codespan.rs @@ -17,7 +17,7 @@ use husky_entity_tree::{ }; use husky_regional_token::RegionalTokenIdxBase; use husky_text::{HasText, Text}; -use husky_text_protocol::offset::OffsetRange; +use husky_text_protocol::offset::TextOffsetRange; use husky_token::{RangedTokenSheet, TokenDb}; use husky_vfs::path::module_path::ModulePath; use salsa::DisplayWithDb; @@ -170,14 +170,14 @@ impl<'a> SynExprCodespanEmitter<'a> { /// # getters impl<'a> SynExprCodespanEmitter<'a> { - fn expr_offset_range(&self, expr: SynExprIdx) -> OffsetRange { + fn expr_offset_range(&self, expr: SynExprIdx) -> TextOffsetRange { let token_idx_range = self.syn_expr_range_region[expr].token_idx_range(self.regional_token_idx_base); let text_range = self.ranged_token_sheet.tokens_text_range(token_idx_range); self.text.offset_range(text_range) } - fn pattern_offset_range(&self, pattern: SynPatternIdx) -> OffsetRange { + fn pattern_offset_range(&self, pattern: SynPatternIdx) -> TextOffsetRange { let token_idx_range = self.syn_expr_range_region[pattern].token_idx_range(self.regional_token_idx_base); let text_range = self.ranged_token_sheet.tokens_text_range(token_idx_range);