Merge pull request #4 from Michael-F-Bryan/bump-tree-sitter

Bump the tree-sitter version to `0.22.5` and add a `xtask codegen [ast]` sub-command

.github/workflows/ci.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Install Tree Sitter CLI
         uses: taiki-e/install-action@v2
         with:
-          tool: tree-sitter-cli
+          tool: tree-sitter-cli@0.22.5
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
       - name: Ensure up to date

Cargo.toml

@@ -21,7 +21,7 @@ serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.114"
 tracing = { version = "0.1.40", features = ["async-await"] }
 tokio = { version = "1.36.0", features = ["macros", "rt-multi-thread", "net", "io-std"] }
-tree-sitter = ">=0.22.2"
+tree-sitter = ">=0.22.5"
 
 [profile.dist]
 inherits = "release"

crates/xtask/Cargo.toml

@@ -12,3 +12,12 @@ rust-version.workspace = true
 
 [dependencies]
 clap = { version = "4", features = ["derive", "env"] }
+color-eyre = { workspace = true }
+heck = "0.5.0"
+once_cell = "1.19.0"
+prettyplease = "0.2.16"
+proc-macro2 = "1.0.79"
+quote = "1.0.35"
+serde = { workspace = true }
+serde_json = { workspace = true }
+syn = "2.0.55"

tree-sitter-wit/bindings/rust/codegen.rs → crates/xtask/src/codegen/ast.rs

@@ -1,11 +1,60 @@
-//! Automatically generate a strongly-typed AST based on [`crate::NODE_TYPES`].
-
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, path::PathBuf};
 
+use clap::Parser;
+use color_eyre::{eyre::Context, Report};
 use heck::ToPascalCase;
+use once_cell::sync::Lazy;
 use proc_macro2::{Ident, TokenStream};
 use quote::{format_ident, quote};
 
+use crate::utils;
+
+static NODE_TYPES_PATH: Lazy<PathBuf> = Lazy::new(|| {
+    utils::project_root()
+        .join("tree-sitter-wit")
+        .join("src")
+        .join("node-types.json")
+});
+static AST_GENERATED_PATH: Lazy<PathBuf> = Lazy::new(|| {
+    utils::project_root()
+        .join("crates")
+        .join("wit-compiler")
+        .join("src")
+        .join("ast")
+        .join("generated.rs")
+});
+
+#[derive(Debug, Clone, Parser)]
+pub struct Ast {
+    #[clap(short, long, default_value = NODE_TYPES_PATH.as_os_str())]
+    node_types: PathBuf,
+    #[clap(short, long, default_value = AST_GENERATED_PATH.as_os_str())]
+    out: PathBuf,
+}
+
+impl Ast {
+    pub fn generate(self) -> Result<(), Report> {
+        let Ast { node_types, out } = self;
+
+        let node_types = std::fs::read_to_string(&node_types)
+            .with_context(|| format!("Unable to read \"{}\"", node_types.display()))?;
+
+        let tokens = generate_ast(&node_types);
+        let src = utils::format_rust(tokens);
+
+        utils::ensure_file_contents(out, src)
+    }
+}
+
+impl Default for Ast {
+    fn default() -> Self {
+        Ast {
+            node_types: NODE_TYPES_PATH.clone(),
+            out: AST_GENERATED_PATH.clone(),
+        }
+    }
+}
+
 pub(crate) fn generate_ast(node_types: &str) -> TokenStream {
     let node_types: Vec<NodeType> = serde_json::from_str(node_types).unwrap();
 
@@ -536,3 +585,18 @@ const TOKENS: &[Token] = &[
         kind: TokenKind::Punctuation,
     },
 ];
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::utils;
+
+    #[test]
+    fn ast_is_up_to_date() {
+        let node_types = std::fs::read_to_string(&*NODE_TYPES_PATH).unwrap();
+        let tokens = generate_ast(&node_types);
+        let src = utils::format_rust(tokens);
+        let ast_rs = utils::project_root().join("crates/wit-compiler/src/ast/generated.rs");
+        utils::ensure_file_contents(ast_rs, src).unwrap();
+    }
+}

crates/xtask/src/codegen/mod.rs

@@ -0,0 +1,45 @@
+mod ast;
+
+use clap::Parser;
+use color_eyre::Report;
+
+use crate::codegen::ast::Ast;
+
+#[derive(Debug, Clone, Parser)]
+#[clap(subcommand_value_name = "TARGET", subcommand_help_heading = "Targets")]
+pub struct Codegen {
+    #[clap(subcommand)]
+    target: Option<Target>,
+}
+
+impl Codegen {
+    pub fn run(self) -> Result<(), Report> {
+        let Codegen { target } = self;
+
+        match target {
+            Some(target) => target.generate(),
+            None => run_all_generators(),
+        }
+    }
+}
+
+/// Run all code generators using the default settings.
+fn run_all_generators() -> Result<(), Report> {
+    Ast::default().generate()?;
+
+    Ok(())
+}
+
+#[derive(Debug, Clone, Parser)]
+enum Target {
+    /// Generate strongly-typed AST nodes.
+    Ast(Ast),
+}
+
+impl Target {
+    fn generate(self) -> Result<(), Report> {
+        match self {
+            Target::Ast(a) => a.generate(),
+        }
+    }
+}

crates/xtask/src/main.rs

@@ -1,10 +1,24 @@
+mod codegen;
+mod utils;
+
 use clap::Parser;
+use color_eyre::Report;
+
+use crate::codegen::Codegen;
+
+fn main() -> Result<(), Report> {
+    color_eyre::install()?;
 
-fn main() {
-    let _ = Cmd::parse();
-    todo!();
+    let cmd = Cmd::parse();
+
+    match cmd {
+        Cmd::Codegen(c) => c.run(),
+    }
 }
 
 #[derive(Parser, Debug)]
 #[command(author, version)]
-enum Cmd {}
+enum Cmd {
+    /// Run code generation.
+    Codegen(Codegen),
+}

crates/xtask/src/utils.rs

@@ -0,0 +1,70 @@
+#![allow(dead_code)] // used during testing
+
+use std::path::Path;
+
+use color_eyre::eyre::{Context, Report};
+use quote::ToTokens;
+
+/// Format some Rust tokens.
+///
+/// # Panics
+///
+/// It is assumed that the tokens would parse as a Rust file.
+pub fn format_rust(contents: impl ToTokens) -> String {
+    let contents =
+        syn::parse2(contents.to_token_stream()).expect("Unable to parse the tokens as a syn::File");
+    prettyplease::unparse(&contents)
+}
+
+/// Check that a particular file has the desired contents.
+///
+/// If the file is missing or outdated, this function will update the file and
+/// error out.
+pub fn ensure_file_contents(
+    path: impl AsRef<Path>,
+    contents: impl AsRef<str>,
+) -> Result<(), Report> {
+    let path = path.as_ref();
+    let contents = normalize_newlines(contents.as_ref());
+
+    if let Ok(old_contents) = std::fs::read_to_string(path) {
+        if contents == normalize_newlines(&old_contents) {
+            // File is already up to date
+            return Ok(());
+        }
+    }
+
+    let display_path = path.strip_prefix(project_root()).unwrap_or(path);
+
+    eprintln!(
+        "\"{}\" was not up-to-date, updating...",
+        display_path.display()
+    );
+
+    if std::env::var("CI").is_ok() {
+        eprintln!("Note: run codegen locally and commit the updated files");
+    }
+
+    if let Some(parent) = path.parent() {
+        let _ = std::fs::create_dir_all(parent);
+    }
+    std::fs::write(path, contents)
+        .wrap_err_with(|| format!("Unable to save to \"{}\"", path.display()))?;
+
+    color_eyre::eyre::bail!(
+        "\"{}\" was not up to date and has been updated. Please re-run the tests.",
+        display_path.display()
+    );
+}
+
+fn normalize_newlines(s: &str) -> String {
+    s.replace("\r\n", "\n")
+}
+
+/// Get the root directory for this repository.
+pub fn project_root() -> &'static Path {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .ancestors()
+        .find(|p| p.join(".git").exists())
+        .unwrap()
+}

tree-sitter-wit/Cargo.toml

@@ -22,12 +22,3 @@ tree-sitter = { workspace = true }
 
 [build-dependencies]
 cc = "1.0.87"
-
-[dev-dependencies]
-heck = "0.5.0"
-prettyplease = "0.2.16"
-proc-macro2 = "1.0.79"
-quote = "1.0.35"
-serde = { workspace = true }
-serde_json = { workspace = true }
-syn = "2.0.55"