API improvements (#1223)

- add `TerminalKindExtensions.is_identifier()` API to distinguish terminals like Solidity's `Identifier` and Yul's `YulIdentifier`. - split `parser/Parser.supportedVersions()` into a new `utils/LanguageFacts` API, with `allVersions()`, `earliestVersion()`, and `latestVersion()` methods. - rename `Query.parse()` to `Query.create()`, and provide exact `TextRange` for any errors it returns. - split `Parser.parse()` API into `parse_file_contents()` and `parse_nonterminal()`.
NomicFoundation · Jan 24, 2025 · 3e85a14 · 3e85a14
1 parent be914d9
commit 3e85a14
Show file tree

Hide file tree

Showing 167 changed files with 10,085 additions and 9,730 deletions.
diff --git a/.changeset/fluffy-elephants-beg.md b/.changeset/fluffy-elephants-beg.md
@@ -2,4 +2,4 @@
 "@nomicfoundation/slang": minor
 ---
 
-rename `parser/Parser.supportedVersions()` API to `utils/LanguageFacts.supportedVersions()`.
+split `parser/Parser.supportedVersions()` into a new `utils/LanguageFacts` API, with `allVersions()`, `earliestVersion()`, and `latestVersion()` methods.
diff --git a/.changeset/new-bikes-hear.md b/.changeset/new-bikes-hear.md
@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+split `Parser.parse()` API into `parse_file_contents()` and `parse_nonterminal()`.
diff --git a/.changeset/shiny-feet-shop.md b/.changeset/shiny-feet-shop.md
@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+rename `Query.parse()` to `Query.create()`, and provide exact `TextRange` for any errors it returns.
diff --git a/.changeset/tame-tools-walk.md b/.changeset/tame-tools-walk.md
@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+add `TerminalKindExtensions.is_identifier()` API to distinguish terminals like Solidity's `Identifier` and Yul's `YulIdentifier`.
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/compilation/file.rs b/crates/codegen/runtime/cargo/crate/src/runtime/compilation/file.rs
@@ -4,20 +4,25 @@ use std::rc::Rc;
 use metaslang_cst::text_index::TextIndex;
 
 use crate::cst::{Cursor, NonterminalNode};
+use crate::parser::{ParseError, ParseOutput};
 
 #[derive(Clone)]
 pub struct File {
     id: String,
     tree: Rc<NonterminalNode>,
+    errors: Vec<ParseError>,
 
     resolved_imports: BTreeMap<usize, String>,
 }
 
 impl File {
-    pub(super) fn new(id: String, tree: Rc<NonterminalNode>) -> Self {
+    pub(super) fn create(id: String, parse_output: ParseOutput) -> Self {
+        let ParseOutput { tree, errors } = parse_output;
+
         Self {
             id,
             tree,
+            errors,
 
             resolved_imports: BTreeMap::new(),
         }
@@ -31,8 +36,12 @@ impl File {
         &self.tree
     }
 
+    pub fn errors(&self) -> &Vec<ParseError> {
+        &self.errors
+    }
+
     pub fn create_tree_cursor(&self) -> Cursor {
-        Rc::clone(&self.tree).cursor_with_offset(TextIndex::ZERO)
+        Rc::clone(&self.tree).create_cursor(TextIndex::ZERO)
     }
 
     pub(super) fn resolve_import(&mut self, import_path: &Cursor, destination_file_id: String) {

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/compilation/internal_builder.rs b/crates/codegen/runtime/cargo/crate/src/runtime/compilation/internal_builder.rs
@@ -39,11 +39,11 @@ impl InternalCompilationBuilder {
             };
         }
 
-        let parse_output = self.parser.parse(Parser::ROOT_KIND, contents);
+        let parse_output = self.parser.parse_file_contents(contents);
 
         let import_paths = self.imports.extract(parse_output.create_tree_cursor());
 
-        let file = File::new(id.clone(), Rc::clone(parse_output.tree()));
+        let file = File::create(id.clone(), parse_output);
         self.files.insert(id, file);
 
         AddFileResponse { import_paths }
@@ -72,7 +72,7 @@ impl InternalCompilationBuilder {
             .map(|(id, file)| (id.to_owned(), Rc::new(file.to_owned())))
             .collect();
 
-        CompilationUnit::new(language_version, files)
+        CompilationUnit::create(language_version, files)
     }
 }
 

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/compilation/unit.rs b/crates/codegen/runtime/cargo/crate/src/runtime/compilation/unit.rs
@@ -17,7 +17,7 @@ pub struct CompilationUnit {
 }
 
 impl CompilationUnit {
-    pub(super) fn new(language_version: Version, files: BTreeMap<String, Rc<File>>) -> Self {
+    pub(super) fn create(language_version: Version, files: BTreeMap<String, Rc<File>>) -> Self {
         Self {
             language_version,
             files,

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/cst/generated/terminal_kind.rs b/crates/codegen/runtime/cargo/crate/src/runtime/cst/generated/terminal_kind.rs
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/cst/terminal_kind.rs.jinja2 b/crates/codegen/runtime/cargo/crate/src/runtime/cst/terminal_kind.rs.jinja2
@@ -44,20 +44,27 @@ pub enum TerminalKind {
 }
 
 impl crate::cst::TerminalKindExtensions for TerminalKind {
-    fn is_trivia(&self) -> bool {
-        {%- if rendering_in_stubs -%}
-            false
-        {%- else -%}
+    {%- if not rendering_in_stubs -%}
+        fn is_identifier(self) -> bool {
             matches!(
                 self,
-                {%- for variant in model.kinds.trivia_scanner_names -%}
+                {%- for variant in model.kinds.identifier_terminals -%}
                     | Self::{{ variant }}
                 {%- endfor -%}
             )
-        {%- endif -%}
-    }
+        }
 
-    fn is_valid(&self) -> bool {
-        !matches!(self, Self::UNRECOGNIZED | Self::MISSING)
-    }
+        fn is_trivia(self) -> bool {
+            matches!(
+                self,
+                {%- for variant in model.kinds.trivia_terminals -%}
+                    | Self::{{ variant }}
+                {%- endfor -%}
+            )
+        }
+
+        fn is_valid(self) -> bool {
+            !matches!(self, Self::UNRECOGNIZED | Self::MISSING)
+        }
+    {%- endif -%}
 }
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/generated/parser.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/generated/parser.rs
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parse_error.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parse_error.rs
@@ -25,7 +25,7 @@ impl ParseError {
 }
 
 impl ParseError {
-    pub(crate) fn new(
+    pub(crate) fn create(
         text_range: TextRange,
         mut terminals_that_would_have_allowed_more_progress: Vec<TerminalKind>,
     ) -> Self {

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parse_output.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parse_output.rs
@@ -24,6 +24,6 @@ impl ParseOutput {
 
     /// Creates a cursor that starts at the root of the parse tree.
     pub fn create_tree_cursor(&self) -> Cursor {
-        Rc::clone(&self.tree).cursor_with_offset(TextIndex::ZERO)
+        Rc::clone(&self.tree).create_cursor(TextIndex::ZERO)
     }
 }
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser.rs.jinja2 b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser.rs.jinja2
@@ -48,10 +48,8 @@ pub enum ParserInitializationError {
 }
 
 impl Parser {
-    pub const ROOT_KIND: NonterminalKind = NonterminalKind::{{ model.kinds.root_kind }};
-
     pub fn create(language_version: Version) -> std::result::Result<Self, ParserInitializationError> {
-        if LanguageFacts::SUPPORTED_VERSIONS.binary_search(&language_version).is_ok() {
+        if LanguageFacts::ALL_VERSIONS.binary_search(&language_version).is_ok() {
             Ok(Self {
                 {%- if not rendering_in_stubs -%}
                     {%- for version in model.breaking_language_versions %}
@@ -70,7 +68,25 @@ impl Parser {
         &self.language_version
     }
 
-    {%- if not rendering_in_stubs -%}
+    pub fn parse_file_contents(&self, input: &str) -> ParseOutput {
+        self.parse_nonterminal(NonterminalKind::{{ model.kinds.root_kind }}, input)
+    }
+
+    {%- if rendering_in_stubs -%}
+
+        pub fn parse_nonterminal(&self, kind: NonterminalKind, input: &str) -> ParseOutput {
+            unreachable!("Attempting to parse in stubs: {kind}: {input}")
+        }
+
+    {%- else -%}
+
+        pub fn parse_nonterminal(&self, kind: NonterminalKind, input: &str) -> ParseOutput {
+            match kind {
+                {%- for parser_name, _ in model.parser.parser_functions -%}
+                    NonterminalKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input, kind),
+                {%- endfor -%}
+            }
+        }
 
         /********************************************
         *         Parser Functions
@@ -101,18 +117,6 @@ impl Parser {
         {% endfor %}
 
     {% endif %}
-
-    pub fn parse(&self, kind: NonterminalKind, input: &str) -> ParseOutput {
-        {%- if rendering_in_stubs -%}
-            unreachable!("Attempting to parse in stubs: {kind}: {input}")
-        {%- else -%}
-            match kind {
-                {%- for parser_name, _ in model.parser.parser_functions -%}
-                    NonterminalKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input, kind),
-                {%- endfor -%}
-            }
-        {%- endif -%}
-    }
 }
 
 impl Lexer for Parser {

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs
@@ -138,7 +138,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize {
         .flat_map(|edge| {
             edge.node
                 .clone()
-                .cursor_with_offset(TextIndex::ZERO)
+                .create_cursor(TextIndex::ZERO)
                 .remaining_nodes()
         })
         .filter_map(|edge| match edge.node {

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs
@@ -89,8 +89,8 @@ where
                 let node = Node::terminal(kind, input.to_string());
                 children.push(Edge::anonymous(node));
                 ParseOutput {
-                    tree: Rc::new(NonterminalNode::new(topmost_kind, children)),
-                    errors: vec![ParseError::new(
+                    tree: NonterminalNode::create(topmost_kind, children),
+                    errors: vec![ParseError::create(
                         start..start + input.into(),
                         no_match.expected_terminals,
                     )],
@@ -147,13 +147,13 @@ where
 
                     let start_index = stream.text_index_at(start);
                     let mut errors = stream.into_errors();
-                    errors.push(ParseError::new(
+                    errors.push(ParseError::create(
                         start_index..input.into(),
                         expected_terminals,
                     ));
 
                     ParseOutput {
-                        tree: Rc::new(NonterminalNode::new(topmost_node.kind, new_children)),
+                        tree: NonterminalNode::create(topmost_node.kind, new_children),
                         errors,
                     }
                 } else {
@@ -164,7 +164,7 @@ where
                     debug_assert_eq!(
                         errors.is_empty(),
                         Rc::clone(&tree)
-                            .cursor_with_offset(TextIndex::ZERO)
+                            .create_cursor(TextIndex::ZERO)
                             .remaining_nodes()
                             .all(|edge| edge
                                 .as_terminal()

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs
@@ -132,7 +132,7 @@ impl Match {
             .flat_map(|edge| {
                 edge.node
                     .clone()
-                    .cursor_with_offset(TextIndex::ZERO)
+                    .create_cursor(TextIndex::ZERO)
                     .remaining_nodes()
             })
             .all(|edge| {
@@ -213,7 +213,7 @@ impl IncompleteMatch {
             .flat_map(|edge| {
                 edge.node
                     .clone()
-                    .cursor_with_offset(TextIndex::ZERO)
+                    .create_cursor(TextIndex::ZERO)
                     .remaining_nodes()
             })
             .try_fold(0u8, |mut acc, edge| {

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs
@@ -88,7 +88,10 @@ impl ParserResult {
 
             let skipped = input.content(skipped_range.utf8());
 
-            input.emit(ParseError::new(skipped_range, expected_terminals.clone()));
+            input.emit(ParseError::create(
+                skipped_range,
+                expected_terminals.clone(),
+            ));
 
             ParserResult::SkippedUntil(SkippedUntil {
                 nodes,

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs
@@ -60,7 +60,7 @@ impl SeparatedHelper {
                                 kind,
                                 input.content(skipped_range.utf8()),
                             )));
-                            input.emit(ParseError::new(
+                            input.emit(ParseError::create(
                                 skipped_range,
                                 incomplete.expected_terminals,
                             ));

diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/utils/generated/language_facts.rs b/crates/codegen/runtime/cargo/crate/src/runtime/utils/generated/language_facts.rs
diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/utils/language_facts.rs.jinja2 b/crates/codegen/runtime/cargo/crate/src/runtime/utils/language_facts.rs.jinja2
@@ -3,11 +3,17 @@ use semver::Version;
 pub struct LanguageFacts;
 
 impl LanguageFacts {
-    pub const NAME: &'static str = "{{ model.language_name }}";
-
-    pub const SUPPORTED_VERSIONS: &'static [Version] = &[
+    pub const ALL_VERSIONS: &'static [Version] = &[
         {% for version in model.all_language_versions %}
             Version::new({{ version | split(pat=".") | join(sep=", ") }}),
         {% endfor %}
     ];
+
+    pub const EARLIEST_VERSION: Version = Version::new(
+        {{ model.all_language_versions | first() | split(pat=".") | join(sep=", ") }}
+    );
+
+    pub const LATEST_VERSION: Version = Version::new(
+        {{ model.all_language_versions | last() | split(pat=".") | join(sep=", ") }}
+    );
 }
diff --git a/crates/codegen/runtime/cargo/wasm/src/runtime/config.json.jinja2 b/crates/codegen/runtime/cargo/wasm/src/runtime/config.json.jinja2
@@ -170,16 +170,6 @@
         "as_getter": true
       }
     },
-    "nomic-foundation:slang:parser:parse-error.text-range()": {
-      "Function": {
-        "as_getter": true
-      }
-    },
-    "nomic-foundation:slang:parser:parse-error.message()": {
-      "Function": {
-        "as_getter": true
-      }
-    },
     "nomic-foundation:slang:parser:parse-output.tree()": {
       "Function": {
         "as_getter": true