[fud2] Additional Syntax to Rhai DSL (#2203)

## Description
This pull request adds new syntax for defining ops which punts most of
the work of the DSL to Rhai. That is, the PR defines new syntax to
define ops in Rhai by directly associating a sequence of shell commands
with ops. Variables and setups can be done using Rhai's scripting
instead of variables and multiple rules in ninja. This leaves only very
simple, one rule for each op, ninja files.

As currently written, this style of op creation may be easier to write
and read, however, it may also generate less readable ninja files.

## Example of New Syntax
```
export const verilog_state = state("verilog", ["sv", "v"]);
export const calyx_state = state("calyx", ["futil"]);
export const verilog_noverify = state("verilog-noverify", ["sv"]);

fn calyx_setup() {
    // Rhai syntax for object maps.
    #{ 
        calyx_base: config("calyx.base"), 
        calyx_exe: config_or("calyx.exe", "${calyx_base}/target/debug/calyx"),
        args: config_or("calyx.args", ""),
    }
}

defop calyx_to_verilog(c: calyx_state) >> v: verilog_state {
    let s = calyx_setup();
    shell("${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} ${c} > ${v}");
}

defop calyx_noverify(c: calyx_state) >> v: verilog_noverify {
    let s = calyx_setup();
    shell("${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} --disable-verify ${c} > ${v}");
}
```
The strings passed to `shell` can be constructed with a function to
remove some repetition if desired.

While this style of creating ops does clash with the current style, they
should still be able to coexist.

## TODO
- [x] Implement `defop` syntax.
- [x] Implement `shell` function.
- [x] Implement `config` and `config_or` function.
- [x] Make the two types of substitution more distinct.

fud2/Cargo.toml

@@ -33,3 +33,4 @@ path = "src/main.rs"
 [dev-dependencies]
 insta = "1.36.0"
 itertools.workspace = true
+figment = { version = "0.10.12", features = ["toml"] }

fud2/fud-core/src/cli.rs

@@ -314,7 +314,26 @@ fn get_resource(driver: &Driver, cmd: GetResource) -> anyhow::Result<()> {
     bail!("unknown resource file {}", cmd.filename);
 }
 
-pub fn cli(driver: &Driver) -> anyhow::Result<()> {
+/// Given the name of a Driver, returns a config based on that name and CLI arguments.
+pub fn config_from_cli(name: &str) -> anyhow::Result<figment::Figment> {
+    let args: FakeArgs = argh::from_env();
+    let mut config = config::load_config(name);
+
+    // Use `--set` arguments to override configuration values.
+    for set in args.set {
+        let mut parts = set.splitn(2, '=');
+        let key = parts.next().unwrap();
+        let value = parts
+            .next()
+            .ok_or(anyhow!("--set arguments must be in key=value form"))?;
+        let dict = figment::util::nest(key, value.into());
+        config = config.merge(figment::providers::Serialized::defaults(dict));
+    }
+
+    Ok(config)
+}
+
+pub fn cli(driver: &Driver, config: &figment::Figment) -> anyhow::Result<()> {
     let args: FakeArgs = argh::from_env();
 
     // Configure logging.
@@ -345,7 +364,7 @@ pub fn cli(driver: &Driver) -> anyhow::Result<()> {
     let plan = driver.plan(req).ok_or(anyhow!("could not find path"))?;
 
     // Configure.
-    let mut run = Run::new(driver, plan);
+    let mut run = Run::new(driver, plan, config.clone());
 
     // Override some global config options.
     if let Some(keep) = args.keep {
@@ -355,19 +374,6 @@ pub fn cli(driver: &Driver) -> anyhow::Result<()> {
         run.global_config.verbose = verbose;
     }
 
-    // Use `--set` arguments to override configuration values.
-    for set in args.set {
-        let mut parts = set.splitn(2, '=');
-        let key = parts.next().unwrap();
-        let value = parts
-            .next()
-            .ok_or(anyhow!("--set arguments must be in key=value form"))?;
-        let dict = figment::util::nest(key, value.into());
-        run.config_data = run
-            .config_data
-            .merge(figment::providers::Serialized::defaults(dict));
-    }
-
     // Execute.
     match args.mode {
         Mode::ShowPlan => run.show(),

fud2/fud-core/src/exec/driver.rs

@@ -1,5 +1,5 @@
 use super::{OpRef, Operation, Request, Setup, SetupRef, State, StateRef};
-use crate::{config, run, script, utils};
+use crate::{run, script, utils};
 use camino::{Utf8Path, Utf8PathBuf};
 use cranelift_entity::PrimaryMap;
 use rand::distributions::{Alphanumeric, DistString};
@@ -439,15 +439,11 @@ impl DriverBuilder {
     }
 
     /// Load any plugin scripts specified in the configuration file.
-    pub fn load_plugins(mut self) -> Self {
+    pub fn load_plugins(mut self, config_data: &figment::Figment) -> Self {
         // pull out things from self that we need
         let plugin_dir = self.scripts_dir.take();
         let plugin_files = self.script_files.take();
 
-        // TODO: Let's try to avoid loading/parsing the configuration file here and
-        // somehow reusing it from wherever we do that elsewhere.
-        let config = config::load_config(&self.name);
-
         let mut runner = script::ScriptRunner::new(self);
 
         // add system plugins
@@ -469,7 +465,7 @@ impl DriverBuilder {
 
         // add user plugins defined in config
         if let Ok(plugins) =
-            config.extract_inner::<Vec<std::path::PathBuf>>("plugins")
+            config_data.extract_inner::<Vec<std::path::PathBuf>>("plugins")
         {
             runner.add_files(plugins.into_iter());
         }

fud2/fud-core/src/run.rs

@@ -87,6 +87,79 @@ impl EmitBuild for EmitBuildFn {
     }
 }
 
+/// A config variable.
+#[derive(Debug, Clone)]
+pub enum ConfigVar {
+    /// The key for the config variable.
+    Required(String),
+    /// The key for the config variable followed by the value it should be if the key is not found.
+    Optional(String, String),
+}
+
+/// The data required to emit a single, simple op whose body is composed of an ordered set of
+/// commands.
+pub struct RulesOp {
+    /// The name of the rule generated.
+    pub rule_name: String,
+
+    /// The shell commands emitted by to the generated rule.
+    /// Each of these must be run in order in a context where the variables in each cmd are
+    /// supplied. In particular, this means that variables of the form "$[i|o]<digit>" are in
+    /// scope.
+    pub cmds: Vec<String>,
+
+    /// Variables to be emitted
+    pub config_vars: Vec<ConfigVar>,
+}
+
+/// Given the `index` of a file in the list of input/output files and if the file is an
+/// `input`, returns a valid Ninja variable name.
+pub fn io_file_var_name(index: usize, input: bool) -> String {
+    if input {
+        format!("i{}", index)
+    } else {
+        format!("o{}", index)
+    }
+}
+
+impl EmitBuild for RulesOp {
+    fn build(
+        &self,
+        emitter: &mut StreamEmitter,
+        inputs: &[&str],
+        outputs: &[&str],
+    ) -> EmitResult {
+        // Write the Ninja file.
+        let cmd = self.cmds.join(" && ");
+        emitter.rule(&self.rule_name, &cmd)?;
+        let in_vars = inputs
+            .iter()
+            .enumerate()
+            .map(|(k, &v)| (io_file_var_name(k, true), v));
+        let out_vars = outputs
+            .iter()
+            .enumerate()
+            .map(|(k, &v)| (io_file_var_name(k, false), v));
+        let vars: Vec<_> = in_vars.chain(out_vars).collect();
+
+        for var in &self.config_vars {
+            match var {
+                ConfigVar::Required(k) => emitter.config_var(k, k)?,
+                ConfigVar::Optional(k, d) => emitter.config_var_or(k, k, d)?,
+            }
+        }
+
+        emitter.build_cmd_with_args(
+            outputs,
+            &self.rule_name,
+            inputs,
+            &[],
+            &vars,
+        )?;
+        Ok(())
+    }
+}
+
 // TODO make this unnecessary...
 /// A simple `build` emitter that just runs a Ninja rule.
 pub struct EmitRuleBuild {
@@ -126,9 +199,12 @@ pub struct Run<'a> {
 }
 
 impl<'a> Run<'a> {
-    pub fn new(driver: &'a Driver, plan: Plan) -> Self {
-        let config_data = config::load_config(&driver.name);
-        Self::with_config(driver, plan, config_data)
+    pub fn new(
+        driver: &'a Driver,
+        plan: Plan,
+        config: figment::Figment,
+    ) -> Self {
+        Self::with_config(driver, plan, config)
     }
 
     pub fn with_config(
@@ -507,6 +583,26 @@ impl<W: Write> Emitter<W> {
         Ok(())
     }
 
+    /// Emit a Ninja build command with variable list.
+    ///
+    /// Here `variables` is an association list, the first element of each tuple a key and the
+    /// second a value.
+    pub fn build_cmd_with_args(
+        &mut self,
+        targets: &[&str],
+        rule: &str,
+        deps: &[&str],
+        implicit_deps: &[&str],
+        variables: &[(String, &str)],
+    ) -> std::io::Result<()> {
+        self.build_cmd(targets, rule, deps, implicit_deps)?;
+        for (key, value) in variables {
+            self.arg(key, value)?;
+        }
+        writeln!(self.out)?;
+        Ok(())
+    }
+
     /// Emit a Ninja comment.
     pub fn comment(&mut self, text: &str) -> std::io::Result<()> {
         writeln!(self.out, "# {}", text)?;

fud2/fud-core/src/script/error.rs

@@ -10,13 +10,40 @@ pub(super) struct RhaiSystemError {
 
 #[derive(Debug)]
 pub(super) enum RhaiSystemErrorKind {
-    ErrorSetupRef(String),
+    SetupRef(String),
+    StateRef(String),
+    BeganOp(String, String),
+    NoOp,
 }
 
 impl RhaiSystemError {
     pub(super) fn setup_ref(v: rhai::Dynamic) -> Self {
         Self {
-            kind: RhaiSystemErrorKind::ErrorSetupRef(v.to_string()),
+            kind: RhaiSystemErrorKind::SetupRef(v.to_string()),
+            position: rhai::Position::NONE,
+        }
+    }
+
+    pub(super) fn state_ref(v: rhai::Dynamic) -> Self {
+        Self {
+            kind: RhaiSystemErrorKind::StateRef(v.to_string()),
+            position: rhai::Position::NONE,
+        }
+    }
+
+    pub(super) fn began_op(old_name: &str, new_name: &str) -> Self {
+        Self {
+            kind: RhaiSystemErrorKind::BeganOp(
+                old_name.to_string(),
+                new_name.to_string(),
+            ),
+            position: rhai::Position::NONE,
+        }
+    }
+
+    pub(super) fn no_op() -> Self {
+        Self {
+            kind: RhaiSystemErrorKind::NoOp,
             position: rhai::Position::NONE,
         }
     }
@@ -30,9 +57,18 @@ impl RhaiSystemError {
 impl Display for RhaiSystemError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match &self.kind {
-            RhaiSystemErrorKind::ErrorSetupRef(v) => {
+            RhaiSystemErrorKind::SetupRef(v) => {
                 write!(f, "Unable to construct SetupRef: `{v:?}`")
             }
+            RhaiSystemErrorKind::StateRef(v) => {
+                write!(f, "Unable to construct StateRef: `{v:?}`")
+            }
+            RhaiSystemErrorKind::BeganOp(old_name, new_name) => {
+                write!(f, "Unable to build two ops at once: trying to build `{new_name:?}` but already building `{old_name:?}`")
+            }
+            RhaiSystemErrorKind::NoOp => {
+                write!(f, "Unable to find current op being built. Consider calling start_op_stmts earlier in the program.")
+            }
         }
     }
 }