From 9f7782c2cbc58dff3796fcb264a85454ee4efd4a Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Tue, 9 Jul 2024 16:19:06 -0400 Subject: [PATCH 01/30] create function to begin listening for op body --- fud2/fud-core/src/script/error.rs | 31 +++++++++++++-- fud2/fud-core/src/script/plugin.rs | 64 ++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/fud2/fud-core/src/script/error.rs b/fud2/fud-core/src/script/error.rs index bf2374c0b5..5ebba833de 100644 --- a/fud2/fud-core/src/script/error.rs +++ b/fud2/fud-core/src/script/error.rs @@ -10,13 +10,32 @@ pub(super) struct RhaiSystemError { #[derive(Debug)] pub(super) enum RhaiSystemErrorKind { - ErrorSetupRef(String), + SetupRef(String), + StateRef(String), + BeganOp(String, String), } impl RhaiSystemError { pub(super) fn setup_ref(v: rhai::Dynamic) -> Self { Self { - kind: RhaiSystemErrorKind::ErrorSetupRef(v.to_string()), + kind: RhaiSystemErrorKind::SetupRef(v.to_string()), + position: rhai::Position::NONE, + } + } + + pub(super) fn state_ref(v: rhai::Dynamic) -> Self { + Self { + kind: RhaiSystemErrorKind::StateRef(v.to_string()), + position: rhai::Position::NONE, + } + } + + pub(super) fn began_op(old_name: &str, new_name: &str) -> Self { + Self { + kind: RhaiSystemErrorKind::BeganOp( + old_name.to_string(), + new_name.to_string(), + ), position: rhai::Position::NONE, } } @@ -30,9 +49,15 @@ impl RhaiSystemError { impl Display for RhaiSystemError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.kind { - RhaiSystemErrorKind::ErrorSetupRef(v) => { + RhaiSystemErrorKind::SetupRef(v) => { write!(f, "Unable to construct SetupRef: `{v:?}`") } + RhaiSystemErrorKind::StateRef(v) => { + write!(f, "Unable to construct StateRef: `{v:?}`") + } + RhaiSystemErrorKind::BeganOp(old_name, new_name) => { + write!(f, "Cannot build two ops at once: trying to build `{new_name:?}` but already building `{old_name:?}`") + } } } } diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 4c2df65b72..1c2eecea91 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -16,12 +16,18 @@ use super::{ resolver::Resolver, }; +// The name, input states, and output states of an op +type OpSig = (String, Vec, Vec); + #[derive(Clone)] struct ScriptContext { builder: Rc>, path: Rc, ast: Rc, setups: Rc>>, + + /// An op currently being built. `None` means no op is currently being built. + cur_op: Rc>>, } impl ScriptContext { @@ -72,6 +78,47 @@ impl ScriptContext { *self.setups.borrow().get(fnptr.fn_name()).unwrap() } + + /// Begins building an op. This fails if an op is already being built or `input` or `output` + /// are not arrays of `StateRef`. + fn begin_op( + &self, + ctx: &rhai::NativeCallContext, + name: &str, + inputs: rhai::Array, + outputs: rhai::Array, + ) -> RhaiResult<()> { + let inputs = inputs + .into_iter() + .map(|i| match i.clone().try_cast::() { + Some(state) => Ok(state), + None => Err(RhaiSystemError::state_ref(i) + .with_pos(ctx.position()) + .into()), + }) + .collect::>>()?; + let outputs = outputs + .into_iter() + .map(|i| match i.clone().try_cast::() { + Some(state) => Ok(state), + None => Err(RhaiSystemError::state_ref(i) + .with_pos(ctx.position()) + .into()), + }) + .collect::>>()?; + let mut cur_op = self.cur_op.borrow_mut(); + match *cur_op { + None => { + *cur_op = Some((name.to_string(), inputs, outputs)); + RhaiResult::Ok(()) + } + Some((ref old_name, _, _)) => { + Err(RhaiSystemError::began_op(old_name, name) + .with_pos(ctx.position()) + .into()) + } + } + } } pub struct ScriptRunner { @@ -240,12 +287,28 @@ impl ScriptRunner { ); } + /// Registers a Rhai function which starts the parser listening for shell commands, how an op + /// does its transformation. + fn reg_start_op_stmts(&mut self, sctx: ScriptContext) { + self.engine.register_fn( + "start_op_stmts", + move |ctx: rhai::NativeCallContext, + name: &str, + inputs: rhai::Array, + outputs: rhai::Array| + -> RhaiResult<_> { + sctx.begin_op(&ctx, name, inputs, outputs) + }, + ); + } + fn script_context(&self, path: PathBuf) -> ScriptContext { ScriptContext { builder: Rc::clone(&self.builder), path: Rc::new(path), ast: Rc::new(self.rhai_functions.clone()), setups: Rc::clone(&self.setups), + cur_op: Rc::new(None.into()), } } @@ -253,6 +316,7 @@ impl ScriptRunner { let sctx = self.script_context(path.to_path_buf()); self.reg_rule(sctx.clone()); self.reg_op(sctx.clone()); + self.reg_start_op_stmts(sctx.clone()); self.engine .module_resolver() From 1c9afbd35a4d035ceda1e5d31c0a4d07fec311cf Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Tue, 9 Jul 2024 21:31:58 -0400 Subject: [PATCH 02/30] implement adding shells to builds --- fud2/fud-core/src/script/error.rs | 13 ++++++++++- fud2/fud-core/src/script/plugin.rs | 37 ++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/fud2/fud-core/src/script/error.rs b/fud2/fud-core/src/script/error.rs index 5ebba833de..95b570a3db 100644 --- a/fud2/fud-core/src/script/error.rs +++ b/fud2/fud-core/src/script/error.rs @@ -13,6 +13,7 @@ pub(super) enum RhaiSystemErrorKind { SetupRef(String), StateRef(String), BeganOp(String, String), + NoOp(String), } impl RhaiSystemError { @@ -40,6 +41,13 @@ impl RhaiSystemError { } } + pub(super) fn no_op(cmd: &str) -> Self { + Self { + kind: RhaiSystemErrorKind::NoOp(cmd.to_string()), + position: rhai::Position::NONE, + } + } + pub(super) fn with_pos(mut self, p: rhai::Position) -> Self { self.position = p; self @@ -56,7 +64,10 @@ impl Display for RhaiSystemError { write!(f, "Unable to construct StateRef: `{v:?}`") } RhaiSystemErrorKind::BeganOp(old_name, new_name) => { - write!(f, "Cannot build two ops at once: trying to build `{new_name:?}` but already building `{old_name:?}`") + write!(f, "Unable to build two ops at once: trying to build `{new_name:?}` but already building `{old_name:?}`") + } + RhaiSystemErrorKind::NoOp(cmd) => { + write!(f, "No op exists: unable to add cmd `{cmd:?}`") } } } diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 1c2eecea91..c952cb00b4 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -16,8 +16,8 @@ use super::{ resolver::Resolver, }; -// The name, input states, and output states of an op -type OpSig = (String, Vec, Vec); +// The name, input states, output states, and shell commands to run of an op +type OpSig = (String, Vec, Vec, Vec); #[derive(Clone)] struct ScriptContext { @@ -109,16 +109,34 @@ impl ScriptContext { let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { None => { - *cur_op = Some((name.to_string(), inputs, outputs)); + *cur_op = Some((name.to_string(), inputs, outputs, vec![])); RhaiResult::Ok(()) } - Some((ref old_name, _, _)) => { + Some((ref old_name, _, _, _)) => { Err(RhaiSystemError::began_op(old_name, name) .with_pos(ctx.position()) .into()) } } } + + /// Adds a shell command to the `cur_op` or returns an error if `cur_op` is `None`. + fn add_shell( + &self, + ctx: &rhai::NativeCallContext, + cmd: String, + ) -> RhaiResult<()> { + let mut cur_op = self.cur_op.borrow_mut(); + match *cur_op { + Some(ref mut op_sig) => { + op_sig.3.push(cmd); + Ok(()) + } + None => Err(RhaiSystemError::no_op(&cmd) + .with_pos(ctx.position()) + .into()), + } + } } pub struct ScriptRunner { @@ -302,6 +320,16 @@ impl ScriptRunner { ); } + /// Registers a Rhai function which adds shell commands to be used by an op. + fn reg_shell(&mut self, sctx: ScriptContext) { + self.engine.register_fn( + "shell", + move |ctx: rhai::NativeCallContext, cmd: &str| -> RhaiResult<_> { + sctx.add_shell(&ctx, cmd.to_string()) + }, + ); + } + fn script_context(&self, path: PathBuf) -> ScriptContext { ScriptContext { builder: Rc::clone(&self.builder), @@ -317,6 +345,7 @@ impl ScriptRunner { self.reg_rule(sctx.clone()); self.reg_op(sctx.clone()); self.reg_start_op_stmts(sctx.clone()); + self.reg_shell(sctx.clone()); self.engine .module_resolver() From e9fe9e9981e21f1b21fb81ab6589fc4dea3e8f5c Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Wed, 10 Jul 2024 11:47:56 -0400 Subject: [PATCH 03/30] add function to emit built op --- fud2/fud-core/src/run.rs | 14 +++ fud2/fud-core/src/script/error.rs | 10 +- fud2/fud-core/src/script/plugin.rs | 142 +++++++++++++++++++++++++++-- 3 files changed, 152 insertions(+), 14 deletions(-) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index afb44e34b4..498b9b6b52 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -87,6 +87,20 @@ impl EmitBuild for EmitBuildFn { } } +pub type EmitBuildClosure = + Box EmitResult>; + +impl EmitBuild for EmitBuildClosure { + fn build( + &self, + emitter: &mut StreamEmitter, + input: &[&str], + output: &[&str], + ) -> EmitResult { + self(emitter, input, output) + } +} + // TODO make this unnecessary... /// A simple `build` emitter that just runs a Ninja rule. pub struct EmitRuleBuild { diff --git a/fud2/fud-core/src/script/error.rs b/fud2/fud-core/src/script/error.rs index 95b570a3db..4bd84f8629 100644 --- a/fud2/fud-core/src/script/error.rs +++ b/fud2/fud-core/src/script/error.rs @@ -13,7 +13,7 @@ pub(super) enum RhaiSystemErrorKind { SetupRef(String), StateRef(String), BeganOp(String, String), - NoOp(String), + NoOp, } impl RhaiSystemError { @@ -41,9 +41,9 @@ impl RhaiSystemError { } } - pub(super) fn no_op(cmd: &str) -> Self { + pub(super) fn no_op() -> Self { Self { - kind: RhaiSystemErrorKind::NoOp(cmd.to_string()), + kind: RhaiSystemErrorKind::NoOp, position: rhai::Position::NONE, } } @@ -66,8 +66,8 @@ impl Display for RhaiSystemError { RhaiSystemErrorKind::BeganOp(old_name, new_name) => { write!(f, "Unable to build two ops at once: trying to build `{new_name:?}` but already building `{old_name:?}`") } - RhaiSystemErrorKind::NoOp(cmd) => { - write!(f, "No op exists: unable to add cmd `{cmd:?}`") + RhaiSystemErrorKind::NoOp => { + write!(f, "Unable to find current op being built. Consider calling start_op_stmts earlier in the program.") } } } diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index c952cb00b4..00f0b13601 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -1,9 +1,10 @@ use crate::{ exec::{OpRef, SetupRef, StateRef}, + run::EmitBuildClosure, DriverBuilder, }; use std::{ - cell::RefCell, + cell::{RefCell, RefMut}, collections::HashMap, path::{Path, PathBuf}, rc::Rc, @@ -17,7 +18,12 @@ use super::{ }; // The name, input states, output states, and shell commands to run of an op -type OpSig = (String, Vec, Vec, Vec); +type OpSig = ( + String, + Vec<(String, StateRef)>, + Vec<(String, StateRef)>, + Vec, +); #[derive(Clone)] struct ScriptContext { @@ -85,7 +91,9 @@ impl ScriptContext { &self, ctx: &rhai::NativeCallContext, name: &str, + input_names: rhai::Array, inputs: rhai::Array, + output_names: rhai::Array, outputs: rhai::Array, ) -> RhaiResult<()> { let inputs = inputs @@ -97,6 +105,15 @@ impl ScriptContext { .into()), }) .collect::>>()?; + let input_names = input_names + .into_iter() + .map(|i| match i.clone().try_cast::() { + Some(state) => Ok(state), + None => Err(RhaiSystemError::state_ref(i) + .with_pos(ctx.position()) + .into()), + }) + .collect::>>()?; let outputs = outputs .into_iter() .map(|i| match i.clone().try_cast::() { @@ -106,11 +123,28 @@ impl ScriptContext { .into()), }) .collect::>>()?; + let output_names = output_names + .into_iter() + .map(|i| match i.clone().try_cast::() { + Some(state) => Ok(state), + None => Err(RhaiSystemError::state_ref(i) + .with_pos(ctx.position()) + .into()), + }) + .collect::>>()?; + let input_tuples = input_names.into_iter().zip(inputs).collect(); + let output_tuples = output_names.into_iter().zip(outputs).collect(); + let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { None => { - *cur_op = Some((name.to_string(), inputs, outputs, vec![])); - RhaiResult::Ok(()) + *cur_op = Some(( + name.to_string(), + input_tuples, + output_tuples, + vec![], + )); + Ok(()) } Some((ref old_name, _, _, _)) => { Err(RhaiSystemError::began_op(old_name, name) @@ -120,7 +154,8 @@ impl ScriptContext { } } - /// Adds a shell command to the `cur_op` or returns an error if `cur_op` is `None`. + /// Adds a shell command to the `cur_op`.Returns and error if `begin_op` has not been called + /// before this `end_op` and after any previous `end_op` fn add_shell( &self, ctx: &rhai::NativeCallContext, @@ -132,9 +167,76 @@ impl ScriptContext { op_sig.3.push(cmd); Ok(()) } - None => Err(RhaiSystemError::no_op(&cmd) - .with_pos(ctx.position()) - .into()), + None => { + Err(RhaiSystemError::no_op().with_pos(ctx.position()).into()) + } + } + } + + /// If `{$ident}` is a substring of `cmd` and `("ident", "value")` is in `mapping`, returns a + /// string identical to `cmd` except for all instances `{$ident}` replaced with `value`. + fn substitute_shell_text(cmd: &str, mapping: &[(&str, &str)]) -> String { + let mut cur = cmd.to_string(); + for (i, v) in mapping { + let substr = format!("{{${}}}", i); + cur = cur.replace(&substr, v); + } + cur + } + + /// Collects an op currently being built and adds it to `bld`. Returns and error if `begin_op` + /// has not been called before this `end_op` and after any previous `end_op`. + fn end_op( + &self, + ctx: &rhai::NativeCallContext, + mut bld: RefMut, + ) -> RhaiResult<()> { + let mut cur_op = self.cur_op.borrow_mut(); + match *cur_op { + Some((ref name, ref input_tuples, ref output_tuples, ref cmds)) => { + // Create the emitter. + let input_tuples_c = input_tuples.clone(); + let output_tuples_c = output_tuples.clone(); + let cmds = cmds.clone(); + let name_c = name.clone(); + let f: EmitBuildClosure = + Box::new(move |e, inputs, outputs| { + // Subsitute variables in shell commands with actual values. + let input_names = input_tuples_c.iter().map(|(s, _)| s); + let output_names = + output_tuples_c.iter().map(|(s, _)| s); + let mapping: Vec<_> = input_names + .into_iter() + .chain(output_names) + .map(|s| s.as_str()) + .zip(inputs.iter().chain(outputs.iter()).copied()) + .collect(); + let cmds: Vec<_> = cmds + .iter() + .map(|c| Self::substitute_shell_text(c, &mapping)) + .collect(); + + // Write the Ninja file. + let cmd = cmds.join(" && "); + e.rule(&name_c, &cmd)?; + e.build_cmd(inputs, &name_c, outputs, &[])?; + Ok(()) + }); + + // Add the op. + let inputs: Vec<_> = + input_tuples.iter().map(|(_, s)| *s).collect(); + let outputs: Vec<_> = + output_tuples.iter().map(|(_, s)| *s).collect(); + bld.add_op(name, &[], &inputs, &outputs, f); + + // Now no op is being built. + *cur_op = None; + Ok(()) + } + None => { + Err(RhaiSystemError::no_op().with_pos(ctx.position()).into()) + } } } } @@ -312,10 +414,19 @@ impl ScriptRunner { "start_op_stmts", move |ctx: rhai::NativeCallContext, name: &str, + input_names: rhai::Array, inputs: rhai::Array, + output_names: rhai::Array, outputs: rhai::Array| -> RhaiResult<_> { - sctx.begin_op(&ctx, name, inputs, outputs) + sctx.begin_op( + &ctx, + name, + input_names, + inputs, + output_names, + outputs, + ) }, ); } @@ -330,6 +441,18 @@ impl ScriptRunner { ); } + /// Registers a Rhai function which stops the parser listening to shell commands and adds the + /// created op to `self.builder`. + fn reg_end_op_stmts(&mut self, sctx: ScriptContext) { + let bld = Rc::clone(&self.builder); + self.engine.register_fn( + "end_op_stmts", + move |ctx: rhai::NativeCallContext| -> RhaiResult<_> { + sctx.end_op(&ctx, bld.borrow_mut()) + }, + ); + } + fn script_context(&self, path: PathBuf) -> ScriptContext { ScriptContext { builder: Rc::clone(&self.builder), @@ -346,6 +469,7 @@ impl ScriptRunner { self.reg_op(sctx.clone()); self.reg_start_op_stmts(sctx.clone()); self.reg_shell(sctx.clone()); + self.reg_end_op_stmts(sctx.clone()); self.engine .module_resolver() From 11d7efe67a728c766d9c429832f90d9e7a959da2 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Wed, 10 Jul 2024 22:41:15 -0400 Subject: [PATCH 04/30] add custom syntax for defining op using shell --- fud2/fud-core/src/script/plugin.rs | 264 +++++++++++++++++++++++++---- 1 file changed, 235 insertions(+), 29 deletions(-) diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 00f0b13601..0d161f17f5 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -1,3 +1,5 @@ +use rhai::{Dynamic, ImmutableString, ParseError, Position}; + use crate::{ exec::{OpRef, SetupRef, StateRef}, run::EmitBuildClosure, @@ -89,7 +91,7 @@ impl ScriptContext { /// are not arrays of `StateRef`. fn begin_op( &self, - ctx: &rhai::NativeCallContext, + pos: Position, name: &str, input_names: rhai::Array, inputs: rhai::Array, @@ -100,36 +102,28 @@ impl ScriptContext { .into_iter() .map(|i| match i.clone().try_cast::() { Some(state) => Ok(state), - None => Err(RhaiSystemError::state_ref(i) - .with_pos(ctx.position()) - .into()), + None => Err(RhaiSystemError::state_ref(i).with_pos(pos).into()), }) .collect::>>()?; let input_names = input_names .into_iter() .map(|i| match i.clone().try_cast::() { Some(state) => Ok(state), - None => Err(RhaiSystemError::state_ref(i) - .with_pos(ctx.position()) - .into()), + None => Err(RhaiSystemError::state_ref(i).with_pos(pos).into()), }) .collect::>>()?; let outputs = outputs .into_iter() .map(|i| match i.clone().try_cast::() { Some(state) => Ok(state), - None => Err(RhaiSystemError::state_ref(i) - .with_pos(ctx.position()) - .into()), + None => Err(RhaiSystemError::state_ref(i).with_pos(pos).into()), }) .collect::>>()?; let output_names = output_names .into_iter() .map(|i| match i.clone().try_cast::() { Some(state) => Ok(state), - None => Err(RhaiSystemError::state_ref(i) - .with_pos(ctx.position()) - .into()), + None => Err(RhaiSystemError::state_ref(i).with_pos(pos).into()), }) .collect::>>()?; let input_tuples = input_names.into_iter().zip(inputs).collect(); @@ -148,7 +142,7 @@ impl ScriptContext { } Some((ref old_name, _, _, _)) => { Err(RhaiSystemError::began_op(old_name, name) - .with_pos(ctx.position()) + .with_pos(pos) .into()) } } @@ -156,20 +150,14 @@ impl ScriptContext { /// Adds a shell command to the `cur_op`.Returns and error if `begin_op` has not been called /// before this `end_op` and after any previous `end_op` - fn add_shell( - &self, - ctx: &rhai::NativeCallContext, - cmd: String, - ) -> RhaiResult<()> { + fn add_shell(&self, pos: Position, cmd: String) -> RhaiResult<()> { let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { Some(ref mut op_sig) => { op_sig.3.push(cmd); Ok(()) } - None => { - Err(RhaiSystemError::no_op().with_pos(ctx.position()).into()) - } + None => Err(RhaiSystemError::no_op().with_pos(pos).into()), } } @@ -188,7 +176,7 @@ impl ScriptContext { /// has not been called before this `end_op` and after any previous `end_op`. fn end_op( &self, - ctx: &rhai::NativeCallContext, + pos: Position, mut bld: RefMut, ) -> RhaiResult<()> { let mut cur_op = self.cur_op.borrow_mut(); @@ -234,9 +222,68 @@ impl ScriptContext { *cur_op = None; Ok(()) } - None => { - Err(RhaiSystemError::no_op().with_pos(ctx.position()).into()) - } + None => Err(RhaiSystemError::no_op().with_pos(pos).into()), + } + } +} + +/// All nodes in the parsing state machine. +#[derive(Copy, Clone)] +enum ParseNode { + DefopS, + IdentS, + OpenParenS, + IdentP1, + ColonP1, + ExprP1, + CommaP1, + CloseParenP1, + ArrowS, + IdentP2, + ColonP2, + ExprP2, + CommaP2, + BlockS, +} + +/// All of the state of the parser. +#[derive(Clone)] +struct ParseState { + node: ParseNode, + inputs: usize, + outputs: usize, +} + +impl ParseState { + pub fn new() -> Self { + Self { + node: ParseNode::DefopS, + inputs: 0, + outputs: 0, + } + } + + pub fn node(&self, node: ParseNode) -> Self { + Self { + node, + inputs: self.inputs, + outputs: self.outputs, + } + } + + pub fn inputs(&self, inputs: usize) -> Self { + Self { + node: self.node, + inputs, + outputs: self.outputs, + } + } + + pub fn outputs(&self, outputs: usize) -> Self { + Self { + node: self.node, + inputs: self.inputs, + outputs, } } } @@ -261,6 +308,7 @@ impl ScriptRunner { this.reg_state(); this.reg_get_state(); this.reg_get_setup(); + this.reg_defop_syntax_nop(); this } @@ -420,7 +468,7 @@ impl ScriptRunner { outputs: rhai::Array| -> RhaiResult<_> { sctx.begin_op( - &ctx, + ctx.position(), name, input_names, inputs, @@ -436,7 +484,7 @@ impl ScriptRunner { self.engine.register_fn( "shell", move |ctx: rhai::NativeCallContext, cmd: &str| -> RhaiResult<_> { - sctx.add_shell(&ctx, cmd.to_string()) + sctx.add_shell(ctx.position(), cmd.to_string()) }, ); } @@ -448,7 +496,164 @@ impl ScriptRunner { self.engine.register_fn( "end_op_stmts", move |ctx: rhai::NativeCallContext| -> RhaiResult<_> { - sctx.end_op(&ctx, bld.borrow_mut()) + sctx.end_op(ctx.position(), bld.borrow_mut()) + }, + ); + } + + /// A parse function to add custom syntax for defining ops to rhai. + fn parse_defop( + symbols: &[ImmutableString], + look_ahead: &str, + state: &mut Dynamic, + ) -> Result, ParseError> { + if symbols.len() == 1 { + *state = Dynamic::from(ParseState::new()); + } + let s = state.clone_cast::(); + match s.node { + ParseNode::DefopS => { + *state = Dynamic::from(s.node(ParseNode::IdentS)); + Ok(Some("$ident$".into())) + } + ParseNode::IdentS => { + *state = Dynamic::from(s.node(ParseNode::OpenParenS)); + Ok(Some("(".into())) + } + ParseNode::OpenParenS => { + *state = Dynamic::from(s.node(ParseNode::IdentP1)); + Ok(Some("$ident$".into())) + } + ParseNode::IdentP1 => { + *state = Dynamic::from( + s.node(ParseNode::ColonP1).inputs(s.inputs + 1), + ); + Ok(Some(":".into())) + } + ParseNode::ColonP1 => { + *state = Dynamic::from(s.node(ParseNode::ExprP1)); + Ok(Some("$expr$".into())) + } + ParseNode::ExprP1 => { + if look_ahead == "," { + *state = Dynamic::from(s.node(ParseNode::CommaP1)); + Ok(Some(",".into())) + } else { + *state = Dynamic::from(s.node(ParseNode::CloseParenP1)); + Ok(Some(")".into())) + } + } + ParseNode::CommaP1 => { + *state = Dynamic::from(s.node(ParseNode::IdentP1)); + Ok(Some("$ident$".into())) + } + ParseNode::CloseParenP1 => { + *state = Dynamic::from(s.node(ParseNode::ArrowS)); + Ok(Some(">>".into())) + } + ParseNode::ArrowS => { + *state = Dynamic::from(s.node(ParseNode::IdentP2)); + Ok(Some("$ident$".into())) + } + ParseNode::IdentP2 => { + *state = Dynamic::from( + s.node(ParseNode::ColonP2).outputs(s.outputs + 1), + ); + Ok(Some(":".into())) + } + ParseNode::ColonP2 => { + *state = Dynamic::from(s.node(ParseNode::ExprP2)); + Ok(Some("$expr$".into())) + } + ParseNode::ExprP2 => { + if look_ahead == "," { + *state = Dynamic::from(s.node(ParseNode::CommaP2)); + Ok(Some(",".into())) + } else { + *state = Dynamic::from(s.node(ParseNode::BlockS)); + Ok(Some("$block$".into())) + } + } + ParseNode::CommaP2 => { + *state = Dynamic::from(s.node(ParseNode::IdentP2)); + Ok(Some("$ident$".into())) + } + ParseNode::BlockS => Ok(None), + } + } + + /// Registers custom syntax for defining op without actually defining the op. + fn reg_defop_syntax_nop(&mut self) { + self.engine.register_custom_syntax_with_state_raw( + "defop", + Self::parse_defop, + false, + move |_context, _inputs, _state| Ok(().into()), + ); + } + + /// Registers a custom syntax for creating ops using `start_op_stmts` and `end_op_stmts`. + fn reg_defop_syntax(&mut self, sctx: ScriptContext) { + let bld = Rc::clone(&self.builder); + self.engine.register_custom_syntax_with_state_raw( + "defop", + Self::parse_defop, + false, + move |context, inputs, state| { + let state = state.clone_cast::(); + + // Collect name of op and input/output states. + let op_name = + inputs.first().unwrap().get_string_value().unwrap(); + let input_names: Vec<_> = inputs + .iter() + .skip(1) + .step_by(2) + .take(state.inputs) + .map(|n| { + Dynamic::from(n.get_string_value().unwrap().to_string()) + }) + .collect(); + let input_states = inputs + .iter() + .skip(2) + .step_by(2) + .take(state.inputs) + .map(|s| s.eval_with_context(context)) + .collect::>>()?; + let output_names: Vec<_> = inputs + .iter() + .skip(1 + 2 * state.inputs) + .step_by(2) + .take(state.inputs) + .map(|n| { + Dynamic::from(n.get_string_value().unwrap().to_string()) + }) + .collect(); + let output_states: Vec<_> = inputs + .iter() + .skip(2 + 2 * state.inputs) + .step_by(2) + .take(state.inputs) + .map(|s| s.eval_with_context(context)) + .collect::>>()?; + let body = inputs.last().unwrap(); + + // Position to note error. + let op_pos = inputs.first().unwrap().position(); + + // Begin listening for `shell` functions. Execute definition body and collect into + // an op. + sctx.begin_op( + op_pos, + op_name, + input_names, + input_states, + output_names, + output_states, + )?; + let _ = body.eval_with_context(context); + sctx.end_op(op_pos, bld.borrow_mut()).map(Dynamic::from) }, ); } @@ -470,6 +675,7 @@ impl ScriptRunner { self.reg_start_op_stmts(sctx.clone()); self.reg_shell(sctx.clone()); self.reg_end_op_stmts(sctx.clone()); + self.reg_defop_syntax(sctx.clone()); self.engine .module_resolver() From e0844a1f14471491b24bc9a8511e863c649849c2 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 11 Jul 2024 12:23:54 -0400 Subject: [PATCH 05/30] implement config and config_or --- fud2/fud-core/src/cli.rs | 36 ++++++++++++++++++------------ fud2/fud-core/src/exec/driver.rs | 4 ++++ fud2/fud-core/src/run.rs | 3 +-- fud2/fud-core/src/script/error.rs | 11 +++++++++ fud2/fud-core/src/script/plugin.rs | 33 +++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 16 deletions(-) diff --git a/fud2/fud-core/src/cli.rs b/fud2/fud-core/src/cli.rs index ef64f808da..2a122b5ca2 100644 --- a/fud2/fud-core/src/cli.rs +++ b/fud2/fud-core/src/cli.rs @@ -1,8 +1,8 @@ -use crate::config; use crate::exec::{ Driver, EnumeratePlanner, Request, SingleOpOutputPlanner, StateRef, }; use crate::run::Run; +use crate::{config, DriverBuilder}; use anyhow::{anyhow, bail}; use argh::FromArgs; use camino::Utf8PathBuf; @@ -280,6 +280,27 @@ fn get_resource(driver: &Driver, cmd: GetResource) -> anyhow::Result<()> { bail!("unknown resource file {}", cmd.filename); } +pub fn override_config_from_cli( + mut builder: DriverBuilder, +) -> anyhow::Result { + let args: FakeArgs = argh::from_env(); + + // Use `--set` arguments to override configuration values. + for set in args.set { + let mut parts = set.splitn(2, '='); + let key = parts.next().unwrap(); + let value = parts + .next() + .ok_or(anyhow!("--set arguments must be in key=value form"))?; + let dict = figment::util::nest(key, value.into()); + builder.config_data = builder + .config_data + .merge(figment::providers::Serialized::defaults(dict)); + } + + Ok(builder) +} + pub fn cli(driver: &Driver) -> anyhow::Result<()> { let args: FakeArgs = argh::from_env(); @@ -321,19 +342,6 @@ pub fn cli(driver: &Driver) -> anyhow::Result<()> { run.global_config.verbose = verbose; } - // Use `--set` arguments to override configuration values. - for set in args.set { - let mut parts = set.splitn(2, '='); - let key = parts.next().unwrap(); - let value = parts - .next() - .ok_or(anyhow!("--set arguments must be in key=value form"))?; - let dict = figment::util::nest(key, value.into()); - run.config_data = run - .config_data - .merge(figment::providers::Serialized::defaults(dict)); - } - // Execute. match args.mode { Mode::ShowPlan => run.show(), diff --git a/fud2/fud-core/src/exec/driver.rs b/fud2/fud-core/src/exec/driver.rs index da7c1a1bd3..665837561a 100644 --- a/fud2/fud-core/src/exec/driver.rs +++ b/fud2/fud-core/src/exec/driver.rs @@ -16,6 +16,7 @@ pub struct Driver { pub ops: PrimaryMap, pub rsrc_dir: Option, pub rsrc_files: Option, + pub config_data: figment::Figment, } impl Driver { @@ -271,6 +272,7 @@ pub struct DriverBuilder { rsrc_files: Option, scripts_dir: Option, script_files: Option, + pub config_data: figment::Figment, } #[derive(Debug)] @@ -305,6 +307,7 @@ impl DriverBuilder { rsrc_files: None, scripts_dir: None, script_files: None, + config_data: config::load_config(name), } } @@ -465,6 +468,7 @@ impl DriverBuilder { ops: self.ops, rsrc_dir: self.rsrc_dir, rsrc_files: self.rsrc_files, + config_data: self.config_data, } } } diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index 498b9b6b52..c304cfd739 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -141,8 +141,7 @@ pub struct Run<'a> { impl<'a> Run<'a> { pub fn new(driver: &'a Driver, plan: Plan) -> Self { - let config_data = config::load_config(&driver.name); - Self::with_config(driver, plan, config_data) + Self::with_config(driver, plan, driver.config_data.clone()) } pub fn with_config( diff --git a/fud2/fud-core/src/script/error.rs b/fud2/fud-core/src/script/error.rs index 4bd84f8629..26c0d47a88 100644 --- a/fud2/fud-core/src/script/error.rs +++ b/fud2/fud-core/src/script/error.rs @@ -14,6 +14,7 @@ pub(super) enum RhaiSystemErrorKind { StateRef(String), BeganOp(String, String), NoOp, + ConfigNotFound(String), } impl RhaiSystemError { @@ -48,6 +49,13 @@ impl RhaiSystemError { } } + pub(super) fn config_not_found(key: &str) -> Self { + Self { + kind: RhaiSystemErrorKind::ConfigNotFound(key.to_string()), + position: rhai::Position::NONE, + } + } + pub(super) fn with_pos(mut self, p: rhai::Position) -> Self { self.position = p; self @@ -69,6 +77,9 @@ impl Display for RhaiSystemError { RhaiSystemErrorKind::NoOp => { write!(f, "Unable to find current op being built. Consider calling start_op_stmts earlier in the program.") } + RhaiSystemErrorKind::ConfigNotFound(v) => { + write!(f, "Unable to find config value: `{v:?}`") + } } } } diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 0d161f17f5..e0622cb418 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -309,6 +309,8 @@ impl ScriptRunner { this.reg_get_state(); this.reg_get_setup(); this.reg_defop_syntax_nop(); + this.reg_config(); + this.reg_config_or(); this } @@ -489,6 +491,37 @@ impl ScriptRunner { ); } + /// Registers a Rhai function for getting values from the config file. + fn reg_config(&mut self) { + let bld = Rc::clone(&self.builder); + self.engine.register_fn( + "config", + move |ctx: rhai::NativeCallContext, key: &str| -> RhaiResult<_> { + bld.borrow() + .config_data + .extract_inner::(key) + .or(Err(RhaiSystemError::config_not_found(key) + .with_pos(ctx.position()) + .into())) + }, + ); + } + + /// Registers a Rhai function for getting values from the config file or using a provided + /// string if the key is not found. + fn reg_config_or(&mut self) { + let bld = Rc::clone(&self.builder); + self.engine.register_fn( + "config_or", + move |key: &str, default: &str| -> RhaiResult<_> { + bld.borrow() + .config_data + .extract_inner::(key) + .or(Ok(default.to_string())) + }, + ); + } + /// Registers a Rhai function which stops the parser listening to shell commands and adds the /// created op to `self.builder`. fn reg_end_op_stmts(&mut self, sctx: ScriptContext) { From 155412ed3463d35b072bca9bb72c569564d8168f Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 11 Jul 2024 12:49:11 -0400 Subject: [PATCH 06/30] read configs from cli --- fud2/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fud2/src/main.rs b/fud2/src/main.rs index 4f05686a71..5fbd9693de 100644 --- a/fud2/src/main.rs +++ b/fud2/src/main.rs @@ -36,6 +36,9 @@ fn main() -> anyhow::Result<()> { }); } + // Get config values from cli. + bld = cli::override_config_from_cli(bld)?; + #[cfg(feature = "migrate_to_scripts")] { bld = bld.load_plugins(); From b35bb30eff8ea9b2323dc49e3da8e38eb04b9ff1 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 11 Jul 2024 15:38:58 -0400 Subject: [PATCH 07/30] change syntax for shell to #ident --- fud2/fud-core/src/script/plugin.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index e0622cb418..f3e328bf94 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -161,12 +161,12 @@ impl ScriptContext { } } - /// If `{$ident}` is a substring of `cmd` and `("ident", "value")` is in `mapping`, returns a - /// string identical to `cmd` except for all instances `{$ident}` replaced with `value`. + /// If `#ident` is a substring of `cmd` and `("ident", "value")` is in `mapping`, returns a + /// string identical to `cmd` except for all instances `#ident` replaced with `value`. fn substitute_shell_text(cmd: &str, mapping: &[(&str, &str)]) -> String { let mut cur = cmd.to_string(); for (i, v) in mapping { - let substr = format!("{{${}}}", i); + let substr = format!("#{}", i); cur = cur.replace(&substr, v); } cur From ed1e7e39cf3fe141dfa1eadcb21cc4dbb8ac1ab1 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 11 Jul 2024 23:54:09 -0400 Subject: [PATCH 08/30] tiny bug fixes - swapped inputs and outputs in build command - disregarding error messages inside defops --- fud2/fud-core/src/script/plugin.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 26dd6368ff..2691a80ce6 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -207,7 +207,7 @@ impl ScriptContext { // Write the Ninja file. let cmd = cmds.join(" && "); e.rule(&name_c, &cmd)?; - e.build_cmd(inputs, &name_c, outputs, &[])?; + e.build_cmd(outputs, &name_c, inputs, &[])?; Ok(()) }); @@ -687,7 +687,7 @@ impl ScriptRunner { output_names, output_states, )?; - let _ = body.eval_with_context(context); + let _ = body.eval_with_context(context)?; sctx.end_op(op_pos, bld.borrow_mut()).map(Dynamic::from) }, ); From 71c122b2464966e562220938d0dbf9c4a37fb9b7 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Tue, 16 Jul 2024 06:42:57 -0400 Subject: [PATCH 09/30] remove config_data field from driver --- Cargo.lock | 1 + fud2/Cargo.toml | 1 + fud2/fud-core/src/cli.rs | 18 ++++++++---------- fud2/fud-core/src/exec/driver.rs | 8 ++------ fud2/fud-core/src/run.rs | 8 ++++++-- fud2/fud-core/src/script/plugin.rs | 20 ++++++++++---------- fud2/src/main.rs | 6 +++--- fud2/tests/tests.rs | 3 ++- 8 files changed, 33 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 01c3e652d3..70084d1971 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1189,6 +1189,7 @@ name = "fud" version = "0.0.2" dependencies = [ "anyhow", + "figment", "fud-core", "include_dir", "insta", diff --git a/fud2/Cargo.toml b/fud2/Cargo.toml index 6d79ed550b..6ad0e7d356 100644 --- a/fud2/Cargo.toml +++ b/fud2/Cargo.toml @@ -33,3 +33,4 @@ path = "src/main.rs" [dev-dependencies] insta = "1.36.0" itertools.workspace = true +figment = { version = "0.10.12", features = ["toml"] } diff --git a/fud2/fud-core/src/cli.rs b/fud2/fud-core/src/cli.rs index 2a122b5ca2..d3086428f2 100644 --- a/fud2/fud-core/src/cli.rs +++ b/fud2/fud-core/src/cli.rs @@ -1,8 +1,8 @@ +use crate::config; use crate::exec::{ Driver, EnumeratePlanner, Request, SingleOpOutputPlanner, StateRef, }; use crate::run::Run; -use crate::{config, DriverBuilder}; use anyhow::{anyhow, bail}; use argh::FromArgs; use camino::Utf8PathBuf; @@ -280,10 +280,10 @@ fn get_resource(driver: &Driver, cmd: GetResource) -> anyhow::Result<()> { bail!("unknown resource file {}", cmd.filename); } -pub fn override_config_from_cli( - mut builder: DriverBuilder, -) -> anyhow::Result { +/// Given the name of a Driver, returns a config based on that name and CLI arguments. +pub fn config_from_cli(name: &str) -> anyhow::Result { let args: FakeArgs = argh::from_env(); + let mut config = config::load_config(name); // Use `--set` arguments to override configuration values. for set in args.set { @@ -293,15 +293,13 @@ pub fn override_config_from_cli( .next() .ok_or(anyhow!("--set arguments must be in key=value form"))?; let dict = figment::util::nest(key, value.into()); - builder.config_data = builder - .config_data - .merge(figment::providers::Serialized::defaults(dict)); + config = config.merge(figment::providers::Serialized::defaults(dict)); } - Ok(builder) + Ok(config) } -pub fn cli(driver: &Driver) -> anyhow::Result<()> { +pub fn cli(driver: &Driver, config: &figment::Figment) -> anyhow::Result<()> { let args: FakeArgs = argh::from_env(); // Configure logging. @@ -332,7 +330,7 @@ pub fn cli(driver: &Driver) -> anyhow::Result<()> { let plan = driver.plan(req).ok_or(anyhow!("could not find path"))?; // Configure. - let mut run = Run::new(driver, plan); + let mut run = Run::new(driver, plan, config.clone()); // Override some global config options. if let Some(keep) = args.keep { diff --git a/fud2/fud-core/src/exec/driver.rs b/fud2/fud-core/src/exec/driver.rs index 665837561a..c3078b1612 100644 --- a/fud2/fud-core/src/exec/driver.rs +++ b/fud2/fud-core/src/exec/driver.rs @@ -16,7 +16,6 @@ pub struct Driver { pub ops: PrimaryMap, pub rsrc_dir: Option, pub rsrc_files: Option, - pub config_data: figment::Figment, } impl Driver { @@ -272,7 +271,6 @@ pub struct DriverBuilder { rsrc_files: Option, scripts_dir: Option, script_files: Option, - pub config_data: figment::Figment, } #[derive(Debug)] @@ -307,7 +305,6 @@ impl DriverBuilder { rsrc_files: None, scripts_dir: None, script_files: None, - config_data: config::load_config(name), } } @@ -422,7 +419,7 @@ impl DriverBuilder { } /// Load any plugin scripts specified in the configuration file. - pub fn load_plugins(mut self) -> Self { + pub fn load_plugins(mut self, config_data: &figment::Figment) -> Self { // pull out things from self that we need let plugin_dir = self.scripts_dir.take(); let plugin_files = self.script_files.take(); @@ -431,7 +428,7 @@ impl DriverBuilder { // somehow reusing it from wherever we do that elsewhere. let config = config::load_config(&self.name); - let mut runner = script::ScriptRunner::new(self); + let mut runner = script::ScriptRunner::new(self, config_data.clone()); // add system plugins if let Some(plugin_dir) = plugin_dir { @@ -468,7 +465,6 @@ impl DriverBuilder { ops: self.ops, rsrc_dir: self.rsrc_dir, rsrc_files: self.rsrc_files, - config_data: self.config_data, } } } diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index c304cfd739..9723a38afa 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -140,8 +140,12 @@ pub struct Run<'a> { } impl<'a> Run<'a> { - pub fn new(driver: &'a Driver, plan: Plan) -> Self { - Self::with_config(driver, plan, driver.config_data.clone()) + pub fn new( + driver: &'a Driver, + plan: Plan, + config: figment::Figment, + ) -> Self { + Self::with_config(driver, plan, config) } pub fn with_config( diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 2691a80ce6..6818f66aab 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -294,16 +294,18 @@ pub struct ScriptRunner { rhai_functions: rhai::AST, resolver: Option, setups: Rc>>, + config_data: figment::Figment, } impl ScriptRunner { - pub fn new(builder: DriverBuilder) -> Self { + pub fn new(builder: DriverBuilder, config_data: figment::Figment) -> Self { let mut this = Self { builder: Rc::new(RefCell::new(builder)), engine: rhai::Engine::new(), rhai_functions: rhai::AST::empty(), resolver: Some(Resolver::default()), setups: Rc::default(), + config_data, }; this.reg_state(); this.reg_get_state(); @@ -495,16 +497,15 @@ impl ScriptRunner { /// Registers a Rhai function for getting values from the config file. fn reg_config(&mut self) { - let bld = Rc::clone(&self.builder); + let config_data = self.config_data.clone(); self.engine.register_fn( "config", move |ctx: rhai::NativeCallContext, key: &str| -> RhaiResult<_> { - bld.borrow() - .config_data - .extract_inner::(key) - .or(Err(RhaiSystemError::config_not_found(key) + config_data.extract_inner::(key).or(Err( + RhaiSystemError::config_not_found(key) .with_pos(ctx.position()) - .into())) + .into(), + )) }, ); } @@ -512,12 +513,11 @@ impl ScriptRunner { /// Registers a Rhai function for getting values from the config file or using a provided /// string if the key is not found. fn reg_config_or(&mut self) { - let bld = Rc::clone(&self.builder); + let config_data = self.config_data.clone(); self.engine.register_fn( "config_or", move |key: &str, default: &str| -> RhaiResult<_> { - bld.borrow() - .config_data + config_data .extract_inner::(key) .or(Ok(default.to_string())) }, diff --git a/fud2/src/main.rs b/fud2/src/main.rs index 5fbd9693de..419939e729 100644 --- a/fud2/src/main.rs +++ b/fud2/src/main.rs @@ -37,13 +37,13 @@ fn main() -> anyhow::Result<()> { } // Get config values from cli. - bld = cli::override_config_from_cli(bld)?; + let config = cli::config_from_cli(&bld.name)?; #[cfg(feature = "migrate_to_scripts")] { - bld = bld.load_plugins(); + bld = bld.load_plugins(&config); } let driver = bld.build(); - cli::cli(&driver) + cli::cli(&driver, &config) } diff --git a/fud2/tests/tests.rs b/fud2/tests/tests.rs index 126f9345a6..fa17eb3c53 100644 --- a/fud2/tests/tests.rs +++ b/fud2/tests/tests.rs @@ -16,8 +16,9 @@ fn test_driver() -> Driver { #[cfg(feature = "migrate_to_scripts")] fn test_driver() -> Driver { let mut bld = DriverBuilder::new("fud2-plugins"); + let config = figment::Figment::new(); bld.scripts_dir(manifest_dir_macros::directory_path!("scripts")); - bld.load_plugins().build() + bld.load_plugins(&config).build() } trait InstaTest: Sized { From 5ce36a373805ff1de68fa1558db6e4018e5e715b Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Tue, 16 Jul 2024 07:56:22 -0400 Subject: [PATCH 10/30] use ninja varaibles for input/output files --- fud2/fud-core/src/exec/driver.rs | 8 +-- fud2/fud-core/src/script/plugin.rs | 104 ++++++----------------------- 2 files changed, 23 insertions(+), 89 deletions(-) diff --git a/fud2/fud-core/src/exec/driver.rs b/fud2/fud-core/src/exec/driver.rs index c3078b1612..542946f1b7 100644 --- a/fud2/fud-core/src/exec/driver.rs +++ b/fud2/fud-core/src/exec/driver.rs @@ -1,5 +1,5 @@ use super::{OpRef, Operation, Request, Setup, SetupRef, State, StateRef}; -use crate::{config, run, script, utils}; +use crate::{run, script, utils}; use camino::{Utf8Path, Utf8PathBuf}; use cranelift_entity::PrimaryMap; use rand::distributions::{Alphanumeric, DistString}; @@ -424,10 +424,6 @@ impl DriverBuilder { let plugin_dir = self.scripts_dir.take(); let plugin_files = self.script_files.take(); - // TODO: Let's try to avoid loading/parsing the configuration file here and - // somehow reusing it from wherever we do that elsewhere. - let config = config::load_config(&self.name); - let mut runner = script::ScriptRunner::new(self, config_data.clone()); // add system plugins @@ -449,7 +445,7 @@ impl DriverBuilder { // add user plugins defined in config if let Ok(plugins) = - config.extract_inner::>("plugins") + config_data.extract_inner::>("plugins") { runner.add_files(plugins.into_iter()); } diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 6818f66aab..c110c8079a 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -20,12 +20,7 @@ use super::{ }; // The name, input states, output states, and shell commands to run of an op -type OpSig = ( - String, - Vec<(String, StateRef)>, - Vec<(String, StateRef)>, - Vec, -); +type OpSig = (String, Vec, Vec, Vec); #[derive(Clone)] struct ScriptContext { @@ -93,9 +88,7 @@ impl ScriptContext { &self, pos: Position, name: &str, - input_names: rhai::Array, inputs: rhai::Array, - output_names: rhai::Array, outputs: rhai::Array, ) -> RhaiResult<()> { let inputs = inputs @@ -105,13 +98,6 @@ impl ScriptContext { None => Err(RhaiSystemError::state_ref(i).with_pos(pos).into()), }) .collect::>>()?; - let input_names = input_names - .into_iter() - .map(|i| match i.clone().try_cast::() { - Some(state) => Ok(state), - None => Err(RhaiSystemError::state_ref(i).with_pos(pos).into()), - }) - .collect::>>()?; let outputs = outputs .into_iter() .map(|i| match i.clone().try_cast::() { @@ -119,25 +105,11 @@ impl ScriptContext { None => Err(RhaiSystemError::state_ref(i).with_pos(pos).into()), }) .collect::>>()?; - let output_names = output_names - .into_iter() - .map(|i| match i.clone().try_cast::() { - Some(state) => Ok(state), - None => Err(RhaiSystemError::state_ref(i).with_pos(pos).into()), - }) - .collect::>>()?; - let input_tuples = input_names.into_iter().zip(inputs).collect(); - let output_tuples = output_names.into_iter().zip(outputs).collect(); let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { None => { - *cur_op = Some(( - name.to_string(), - input_tuples, - output_tuples, - vec![], - )); + *cur_op = Some((name.to_string(), inputs, outputs, vec![])); Ok(()) } Some((ref old_name, _, _, _)) => { @@ -161,17 +133,6 @@ impl ScriptContext { } } - /// If `#ident` is a substring of `cmd` and `("ident", "value")` is in `mapping`, returns a - /// string identical to `cmd` except for all instances `#ident` replaced with `value`. - fn substitute_shell_text(cmd: &str, mapping: &[(&str, &str)]) -> String { - let mut cur = cmd.to_string(); - for (i, v) in mapping { - let substr = format!("#{}", i); - cur = cur.replace(&substr, v); - } - cur - } - /// Collects an op currently being built and adds it to `bld`. Returns and error if `begin_op` /// has not been called before this `end_op` and after any previous `end_op`. fn end_op( @@ -181,29 +142,12 @@ impl ScriptContext { ) -> RhaiResult<()> { let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { - Some((ref name, ref input_tuples, ref output_tuples, ref cmds)) => { + Some((ref name, ref input_states, ref output_states, ref cmds)) => { // Create the emitter. - let input_tuples_c = input_tuples.clone(); - let output_tuples_c = output_tuples.clone(); let cmds = cmds.clone(); let name_c = name.clone(); let f: EmitBuildClosure = Box::new(move |e, inputs, outputs| { - // Subsitute variables in shell commands with actual values. - let input_names = input_tuples_c.iter().map(|(s, _)| s); - let output_names = - output_tuples_c.iter().map(|(s, _)| s); - let mapping: Vec<_> = input_names - .into_iter() - .chain(output_names) - .map(|s| s.as_str()) - .zip(inputs.iter().chain(outputs.iter()).copied()) - .collect(); - let cmds: Vec<_> = cmds - .iter() - .map(|c| Self::substitute_shell_text(c, &mapping)) - .collect(); - // Write the Ninja file. let cmd = cmds.join(" && "); e.rule(&name_c, &cmd)?; @@ -212,11 +156,7 @@ impl ScriptContext { }); // Add the op. - let inputs: Vec<_> = - input_tuples.iter().map(|(_, s)| *s).collect(); - let outputs: Vec<_> = - output_tuples.iter().map(|(_, s)| *s).collect(); - bld.add_op(name, &[], &inputs, &outputs, f); + bld.add_op(name, &[], input_states, output_states, f); // Now no op is being built. *cur_op = None; @@ -468,19 +408,10 @@ impl ScriptRunner { "start_op_stmts", move |ctx: rhai::NativeCallContext, name: &str, - input_names: rhai::Array, inputs: rhai::Array, - output_names: rhai::Array, outputs: rhai::Array| -> RhaiResult<_> { - sctx.begin_op( - ctx.position(), - name, - input_names, - inputs, - output_names, - outputs, - ) + sctx.begin_op(ctx.position(), name, inputs, outputs) }, ); } @@ -633,7 +564,7 @@ impl ScriptRunner { self.engine.register_custom_syntax_with_state_raw( "defop", Self::parse_defop, - false, + true, move |context, inputs, state| { let state = state.clone_cast::(); @@ -674,19 +605,26 @@ impl ScriptRunner { .collect::>>()?; let body = inputs.last().unwrap(); + for (i, name) in input_names.clone().into_iter().enumerate() { + context.scope_mut().set_value( + name.into_string().unwrap(), + format!("$in[{}]", i), + ); + } + + for (i, name) in output_names.clone().into_iter().enumerate() { + context.scope_mut().set_value( + name.into_string().unwrap(), + format!("$out[{}]", i), + ); + } + // Position to note error. let op_pos = inputs.first().unwrap().position(); // Begin listening for `shell` functions. Execute definition body and collect into // an op. - sctx.begin_op( - op_pos, - op_name, - input_names, - input_states, - output_names, - output_states, - )?; + sctx.begin_op(op_pos, op_name, input_states, output_states)?; let _ = body.eval_with_context(context)?; sctx.end_op(op_pos, bld.borrow_mut()).map(Dynamic::from) }, From ad8f27943f79319eb0f7ffa2e42ae953bde8fb53 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Tue, 16 Jul 2024 08:00:40 -0400 Subject: [PATCH 11/30] rewrite axi and calyx scripts --- fud2/fud-core/src/script/plugin.rs | 4 +- fud2/scripts/axi.rhai | 84 +++++++++++------------------- fud2/scripts/calyx.rhai | 28 +++++++++- 3 files changed, 58 insertions(+), 58 deletions(-) diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 26dd6368ff..2691a80ce6 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -207,7 +207,7 @@ impl ScriptContext { // Write the Ninja file. let cmd = cmds.join(" && "); e.rule(&name_c, &cmd)?; - e.build_cmd(inputs, &name_c, outputs, &[])?; + e.build_cmd(outputs, &name_c, inputs, &[])?; Ok(()) }); @@ -687,7 +687,7 @@ impl ScriptRunner { output_names, output_states, )?; - let _ = body.eval_with_context(context); + let _ = body.eval_with_context(context)?; sctx.end_op(op_pos, bld.borrow_mut()).map(Dynamic::from) }, ); diff --git a/fud2/scripts/axi.rhai b/fud2/scripts/axi.rhai index 9505c9ffc6..462424924f 100644 --- a/fud2/scripts/axi.rhai +++ b/fud2/scripts/axi.rhai @@ -2,43 +2,14 @@ import "calyx" as c; export const yxi = state("yxi", ["yxi"]); -fn yxi_setup(e) { - e.config_var_or("yxi", "yxi", "$calyx-base/target/debug/yxi"); - e.rule("yxi", "$yxi -l $calyx-base $in > $out"); +fn yxi_pass(in_str, out_str) { + let s = c::calyx_setup(); + let yxi_bin = config_or("yxi", `${s.calyx_base}/target/debug/yxi`); + shell(`${yxi_bin} -l ${s.calyx_base} ${in_str} > ${out_str}`); } -op( - "calyx-to-yxi", - [c::calyx_setup, yxi_setup], - c::calyx_state, - yxi, - |e, input, output| { - e.build_cmd([output], "yxi", [input], []); - }, -); - -fn wrapper_setup(e) { - // Define a `gen-axi` rule that invokes our Python code generator program. - // For now point to standalone axi-generator.py. Can maybe turn this into a rsrc file? - let dynamic = - e.config_constrained_or("dynamic", ["true", "false"], "false"); - let generator_path = if dynamic == "true" { - "$calyx-base/yxi/axi-calyx/dynamic-axi-generator.py" - } else { - "$calyx-base/yxi/axi-calyx/axi-generator.py" - }; - e.config_var_or("axi-generator", "axi.generator", generator_path); - e.config_var_or("python", "python", "python3"); - - e.rule("gen-axi", "$python $axi-generator $in > $out"); - - // Define a simple `combine` rule that just concatenates any numer of files. - e.rule("combine", "cat $in > $out"); - - e.rule( - "remove-imports", - "sed '1,/component main/{/component main/!d; }' $in > $out", - ); +defop calyx_to_yxi(i: c::calyx_state) >> o: yxi { + yxi_pass("#i", "#o"); } /// Replace the extension in `path` with `new_ext` @@ -51,29 +22,32 @@ fn replace_ext(path, new_ext) { } } -fn axi_wrapped_op(e, input, output) { - let file_name = input.split("/")[-1]; - let tmp_yxi = replace_ext(file_name, "yxi"); +defop axi_wrapped(i: c::calyx_state) >> o: c::calyx_state { + let file_name = "input"; + let tmp_yxi = replace_ext(file_name, "yxi"); - e.build_cmd([tmp_yxi], "yxi", [input], []); + yxi_pass("#i", tmp_yxi); - let refified_calyx = replace_ext(`refified_${file_name}`, "futil"); - e.build_cmd([refified_calyx], "calyx-pass", [input], []); - e.arg("pass", "external-to-ref"); + let refified_calyx = replace_ext(`refified_${file_name}`, "futil"); + c::calyx_pass("external-to-ref", "", "#i", refified_calyx); - let axi_wrapper = "axi_wrapper.futil"; - e.build_cmd([axi_wrapper], "gen-axi", [tmp_yxi], []); + // For now point to standalone axi-generator.py. Can maybe turn this into a rsrc file? + let s = c::calyx_setup(); + let dynamic = config_or("dynamic", "false"); + let generator_path = if dynamic == "true" { + `{s.calyx_base}/yxi/axi-calyx/dynamic-axi-generator.py` + } else { + `${s.calyx_base}/yxi/axi-calyx/axi-generator.py` + }; - let no_imports_calyx = `no_imports_${refified_calyx}`; - e.build_cmd([no_imports_calyx], "remove-imports", [refified_calyx], []); + let axi_wrapper = "axi_wrapper.futil"; + let axi_generator = config_or("axi.generator", generator_path); + let python = config_or("python", "python3"); - e.build_cmd([output], "combine", [axi_wrapper, no_imports_calyx], []); -} + shell(`${python} ${axi_generator} ${tmp_yxi} > ${axi_wrapper}`); + + let no_imports_calyx = `no_imports_${refified_calyx}`; + shell(`sed '1,/component main/{/component main/!d; }' ${refified_calyx} > ${no_imports_calyx}`); -op( - "axi-wrapped", - [c::calyx_setup, yxi_setup, wrapper_setup], - c::calyx_state, - c::calyx_state, - axi_wrapped_op -); + shell(`cat ${axi_wrapper} ${no_imports_calyx} > #o`); +} diff --git a/fud2/scripts/calyx.rhai b/fud2/scripts/calyx.rhai index 83162e51cf..42c13b5158 100644 --- a/fud2/scripts/calyx.rhai +++ b/fud2/scripts/calyx.rhai @@ -1,8 +1,34 @@ export const verilog_state = state("verilog", ["sv", "v"]); -export const verilog_noverify = state("verilog-noverify", ["sv"]); export const calyx_state = state("calyx", ["futil"]); +export const verilog_noverify = state("verilog-noverify", ["sv"]); export let calyx_setup = calyx_setup; +export let calyx_pass = calyx_pass; + +fn calyx_setup() { + let calyx_base = config("calyx.base"); + #{ + calyx_base: calyx_base, + calyx_exe: config_or("calyx.exe", `${calyx_base}/target/debug/calyx`), + args: config_or("calyx.args", ""), + } +} + +defop calyx_to_verilog(c: calyx_state) >> v: verilog_state { + let s = calyx_setup(); + shell(`${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} #c > #v`); +} + +defop calyx_noverify(c: calyx_state) >> v: verilog_noverify { + let s = calyx_setup(); + shell(`${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} --disable-verify #c > #v`); +} + +fn calyx_pass(pass, args, in_str, out_str) { + let s = calyx_setup(); + shell(`${s.calyx_exe} -l ${s.calyx_base} -p ${pass} ${args} ${in_str} > ${out_str}`); +} + fn calyx_setup(e) { e.config_var("calyx-base", "calyx.base"); e.config_var_or("calyx-exe", "calyx.exe", "$calyx-base/target/debug/calyx"); From d155cd7a65cd55aa3f6a5c6a73082fcc6a582f72 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 18 Jul 2024 15:20:02 -0400 Subject: [PATCH 12/30] use variables instead of hardcoding names --- fud2/fud-core/src/run.rs | 31 ++++++++++++++++++++++++++++ fud2/fud-core/src/script/plugin.rs | 33 ++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index 9723a38afa..a007e652c0 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -524,6 +524,37 @@ impl Emitter { Ok(()) } + /// Emit a Ninja build command with variable list. + pub fn build_cmd_with_vars( + &mut self, + targets: &[&str], + rule: &str, + deps: &[&str], + implicit_deps: &[&str], + variables: &[(String, &str)], + ) -> std::io::Result<()> { + write!(self.out, "build")?; + for target in targets { + write!(self.out, " {}", target)?; + } + write!(self.out, ": {}", rule)?; + for dep in deps { + write!(self.out, " {}", dep)?; + } + if !implicit_deps.is_empty() { + write!(self.out, " |")?; + for dep in implicit_deps { + write!(self.out, " {}", dep)?; + } + } + writeln!(self.out)?; + for (key, value) in variables { + writeln!(self.out, " {} = {}", key, value)?; + } + writeln!(self.out)?; + Ok(()) + } + /// Emit a Ninja comment. pub fn comment(&mut self, text: &str) -> std::io::Result<()> { writeln!(self.out, "# {}", text)?; diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index c110c8079a..944edd7b50 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -151,7 +151,23 @@ impl ScriptContext { // Write the Ninja file. let cmd = cmds.join(" && "); e.rule(&name_c, &cmd)?; - e.build_cmd(outputs, &name_c, inputs, &[])?; + let in_vars = inputs + .iter() + .enumerate() + .map(|(k, &v)| (format!("i{}", k + 1), v)); + let out_vars = outputs + .iter() + .enumerate() + .map(|(k, &v)| (format!("o{}", k + 1), v)); + let vars: Vec<_> = in_vars.chain(out_vars).collect(); + + e.build_cmd_with_vars( + outputs, + &name_c, + inputs, + &[], + &vars, + )?; Ok(()) }); @@ -605,17 +621,19 @@ impl ScriptRunner { .collect::>>()?; let body = inputs.last().unwrap(); + let orig_scope_size = context.scope().len(); + for (i, name) in input_names.clone().into_iter().enumerate() { - context.scope_mut().set_value( + context.scope_mut().push( name.into_string().unwrap(), - format!("$in[{}]", i), + format!("$i{}", i + 1), ); } for (i, name) in output_names.clone().into_iter().enumerate() { - context.scope_mut().set_value( + context.scope_mut().push( name.into_string().unwrap(), - format!("$out[{}]", i), + format!("$o{}", i + 1), ); } @@ -626,7 +644,10 @@ impl ScriptRunner { // an op. sctx.begin_op(op_pos, op_name, input_states, output_states)?; let _ = body.eval_with_context(context)?; - sctx.end_op(op_pos, bld.borrow_mut()).map(Dynamic::from) + let res = + sctx.end_op(op_pos, bld.borrow_mut()).map(Dynamic::from); + context.scope_mut().rewind(orig_scope_size); + res }, ); } From 34e38fc461c03e137c93055a5e6a1b86e303b08d Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 18 Jul 2024 18:42:24 -0400 Subject: [PATCH 13/30] factor out genation of ninja varaibles name --- fud2/fud-core/src/script/plugin.rs | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 944edd7b50..89664db4c1 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -151,14 +151,14 @@ impl ScriptContext { // Write the Ninja file. let cmd = cmds.join(" && "); e.rule(&name_c, &cmd)?; - let in_vars = inputs - .iter() - .enumerate() - .map(|(k, &v)| (format!("i{}", k + 1), v)); - let out_vars = outputs - .iter() - .enumerate() - .map(|(k, &v)| (format!("o{}", k + 1), v)); + let in_vars = + inputs.iter().enumerate().map(|(k, &v)| { + (ScriptRunner::io_file_var_name(k, true), v) + }); + let out_vars = + outputs.iter().enumerate().map(|(k, &v)| { + (ScriptRunner::io_file_var_name(k, false), v) + }); let vars: Vec<_> = in_vars.chain(out_vars).collect(); e.build_cmd_with_vars( @@ -574,6 +574,16 @@ impl ScriptRunner { ); } + /// Given the `index` of a file in the list of input/output files and if the file is an + /// `input`, returns a valid Ninja variable name. + fn io_file_var_name(index: usize, input: bool) -> String { + if input { + format!("i{}", index) + } else { + format!("o{}", index) + } + } + /// Registers a custom syntax for creating ops using `start_op_stmts` and `end_op_stmts`. fn reg_defop_syntax(&mut self, sctx: ScriptContext) { let bld = Rc::clone(&self.builder); @@ -626,14 +636,14 @@ impl ScriptRunner { for (i, name) in input_names.clone().into_iter().enumerate() { context.scope_mut().push( name.into_string().unwrap(), - format!("$i{}", i + 1), + Self::io_file_var_name(i, true), ); } for (i, name) in output_names.clone().into_iter().enumerate() { context.scope_mut().push( name.into_string().unwrap(), - format!("$o{}", i + 1), + Self::io_file_var_name(i, false), ); } From 56118172ed924fb9c58fbdd3c786e78eda86b9fe Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 18 Jul 2024 19:07:08 -0400 Subject: [PATCH 14/30] replace closure with struct --- fud2/fud-core/src/run.rs | 49 ++++++++++++++++++++++++++---- fud2/fud-core/src/script/plugin.rs | 44 +++------------------------ 2 files changed, 48 insertions(+), 45 deletions(-) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index a007e652c0..4f378a09ae 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -87,17 +87,54 @@ impl EmitBuild for EmitBuildFn { } } -pub type EmitBuildClosure = - Box EmitResult>; +/// The data required to emit a single op. +pub struct OpEmitData { + /// The name of the op. + pub op_name: String, + /// The commands run whenever the op run. Each of these must be run in order in a context where + /// the variables in each cmd are supplied. In particular, this means that variables of the + /// form "$[i|o]" are in scope. + pub cmds: Vec, +} + +/// Given the `index` of a file in the list of input/output files and if the file is an +/// `input`, returns a valid Ninja variable name. +pub fn io_file_var_name(index: usize, input: bool) -> String { + if input { + format!("i{}", index) + } else { + format!("o{}", index) + } +} -impl EmitBuild for EmitBuildClosure { +impl EmitBuild for OpEmitData { fn build( &self, emitter: &mut StreamEmitter, - input: &[&str], - output: &[&str], + inputs: &[&str], + outputs: &[&str], ) -> EmitResult { - self(emitter, input, output) + // Write the Ninja file. + let cmd = self.cmds.join(" && "); + emitter.rule(&self.op_name, &cmd)?; + let in_vars = inputs + .iter() + .enumerate() + .map(|(k, &v)| (io_file_var_name(k, true), v)); + let out_vars = outputs + .iter() + .enumerate() + .map(|(k, &v)| (io_file_var_name(k, false), v)); + let vars: Vec<_> = in_vars.chain(out_vars).collect(); + + emitter.build_cmd_with_vars( + outputs, + &self.op_name, + inputs, + &[], + &vars, + )?; + Ok(()) } } diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 89664db4c1..c79b1f0cba 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -2,7 +2,6 @@ use rhai::{Dynamic, ImmutableString, ParseError, Position}; use crate::{ exec::{SetupRef, StateRef}, - run::EmitBuildClosure, DriverBuilder, }; use std::{ @@ -145,34 +144,11 @@ impl ScriptContext { Some((ref name, ref input_states, ref output_states, ref cmds)) => { // Create the emitter. let cmds = cmds.clone(); - let name_c = name.clone(); - let f: EmitBuildClosure = - Box::new(move |e, inputs, outputs| { - // Write the Ninja file. - let cmd = cmds.join(" && "); - e.rule(&name_c, &cmd)?; - let in_vars = - inputs.iter().enumerate().map(|(k, &v)| { - (ScriptRunner::io_file_var_name(k, true), v) - }); - let out_vars = - outputs.iter().enumerate().map(|(k, &v)| { - (ScriptRunner::io_file_var_name(k, false), v) - }); - let vars: Vec<_> = in_vars.chain(out_vars).collect(); - - e.build_cmd_with_vars( - outputs, - &name_c, - inputs, - &[], - &vars, - )?; - Ok(()) - }); + let op_name = name.clone(); + let op_emitter = crate::run::OpEmitData { op_name, cmds }; // Add the op. - bld.add_op(name, &[], input_states, output_states, f); + bld.add_op(name, &[], input_states, output_states, op_emitter); // Now no op is being built. *cur_op = None; @@ -574,16 +550,6 @@ impl ScriptRunner { ); } - /// Given the `index` of a file in the list of input/output files and if the file is an - /// `input`, returns a valid Ninja variable name. - fn io_file_var_name(index: usize, input: bool) -> String { - if input { - format!("i{}", index) - } else { - format!("o{}", index) - } - } - /// Registers a custom syntax for creating ops using `start_op_stmts` and `end_op_stmts`. fn reg_defop_syntax(&mut self, sctx: ScriptContext) { let bld = Rc::clone(&self.builder); @@ -636,14 +602,14 @@ impl ScriptRunner { for (i, name) in input_names.clone().into_iter().enumerate() { context.scope_mut().push( name.into_string().unwrap(), - Self::io_file_var_name(i, true), + crate::run::io_file_var_name(i, true), ); } for (i, name) in output_names.clone().into_iter().enumerate() { context.scope_mut().push( name.into_string().unwrap(), - Self::io_file_var_name(i, false), + crate::run::io_file_var_name(i, false), ); } From 54ce0dc96826be475a34b3dd22b4bcc2c65828dc Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 18 Jul 2024 19:39:58 -0400 Subject: [PATCH 15/30] name tuple fields --- fud2/fud-core/src/script/plugin.rs | 42 +++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index c79b1f0cba..7327e921ba 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -18,8 +18,17 @@ use super::{ resolver::Resolver, }; -// The name, input states, output states, and shell commands to run of an op -type OpSig = (String, Vec, Vec, Vec); +/// The signature and implementation of an operation. +struct OpSig { + /// Operation name. + name: String, + /// Inputs states of the op. + input_states: Vec, + /// Output states of the op. + output_states: Vec, + /// An ordered list of the commands run when this op is required. + cmds: Vec, +} #[derive(Clone)] struct ScriptContext { @@ -108,14 +117,22 @@ impl ScriptContext { let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { None => { - *cur_op = Some((name.to_string(), inputs, outputs, vec![])); + *cur_op = Some(OpSig { + name: name.to_string(), + input_states: inputs, + output_states: outputs, + cmds: vec![], + }); Ok(()) } - Some((ref old_name, _, _, _)) => { - Err(RhaiSystemError::began_op(old_name, name) - .with_pos(pos) - .into()) - } + Some(OpSig { + name: ref old_name, + input_states: _, + output_states: _, + cmds: _, + }) => Err(RhaiSystemError::began_op(old_name, name) + .with_pos(pos) + .into()), } } @@ -125,7 +142,7 @@ impl ScriptContext { let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { Some(ref mut op_sig) => { - op_sig.3.push(cmd); + op_sig.cmds.push(cmd); Ok(()) } None => Err(RhaiSystemError::no_op().with_pos(pos).into()), @@ -141,7 +158,12 @@ impl ScriptContext { ) -> RhaiResult<()> { let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { - Some((ref name, ref input_states, ref output_states, ref cmds)) => { + Some(OpSig { + ref name, + ref input_states, + ref output_states, + ref cmds, + }) => { // Create the emitter. let cmds = cmds.clone(); let op_name = name.clone(); From 2e9194816c529dc70d052e47d460955fec2acb52 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 18 Jul 2024 19:56:32 -0400 Subject: [PATCH 16/30] rename OpSig --- fud2/fud-core/src/script/plugin.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 7327e921ba..0a468600f4 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -19,7 +19,7 @@ use super::{ }; /// The signature and implementation of an operation. -struct OpSig { +struct RhaiOp { /// Operation name. name: String, /// Inputs states of the op. @@ -38,7 +38,7 @@ struct ScriptContext { setups: Rc>>, /// An op currently being built. `None` means no op is currently being built. - cur_op: Rc>>, + cur_op: Rc>>, } impl ScriptContext { @@ -117,7 +117,7 @@ impl ScriptContext { let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { None => { - *cur_op = Some(OpSig { + *cur_op = Some(RhaiOp { name: name.to_string(), input_states: inputs, output_states: outputs, @@ -125,7 +125,7 @@ impl ScriptContext { }); Ok(()) } - Some(OpSig { + Some(RhaiOp { name: ref old_name, input_states: _, output_states: _, @@ -158,7 +158,7 @@ impl ScriptContext { ) -> RhaiResult<()> { let mut cur_op = self.cur_op.borrow_mut(); match *cur_op { - Some(OpSig { + Some(RhaiOp { ref name, ref input_states, ref output_states, From 41c98afb9e3c21acb4b75442270f410ac10152a9 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 18 Jul 2024 19:57:17 -0400 Subject: [PATCH 17/30] improve docs --- fud2/fud-core/src/script/plugin.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 0a468600f4..b97343fa62 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -18,7 +18,7 @@ use super::{ resolver::Resolver, }; -/// The signature and implementation of an operation. +/// The signature and implementation of an operation specified in Rhai. struct RhaiOp { /// Operation name. name: String, From 0475ce81759a79908657d072e910f1b9b661793c Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 18 Jul 2024 22:26:53 -0400 Subject: [PATCH 18/30] use Ninja variables for configs --- fud2/fud-core/src/exec/driver.rs | 2 +- fud2/fud-core/src/run.rs | 20 ++++++++ fud2/fud-core/src/script/error.rs | 11 ----- fud2/fud-core/src/script/plugin.rs | 73 +++++++++++++++++++++--------- fud2/scripts/calyx.rhai | 4 +- 5 files changed, 75 insertions(+), 35 deletions(-) diff --git a/fud2/fud-core/src/exec/driver.rs b/fud2/fud-core/src/exec/driver.rs index 542946f1b7..61d3f82e95 100644 --- a/fud2/fud-core/src/exec/driver.rs +++ b/fud2/fud-core/src/exec/driver.rs @@ -424,7 +424,7 @@ impl DriverBuilder { let plugin_dir = self.scripts_dir.take(); let plugin_files = self.script_files.take(); - let mut runner = script::ScriptRunner::new(self, config_data.clone()); + let mut runner = script::ScriptRunner::new(self); // add system plugins if let Some(plugin_dir) = plugin_dir { diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index 4f378a09ae..4416aef1af 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -87,14 +87,27 @@ impl EmitBuild for EmitBuildFn { } } +/// A config variable. +#[derive(Debug, Clone)] +pub enum ConfigVar { + /// The key for the config variable. + Var(String), + /// The key for the config variable followed by the value it should be if the key is not found. + VarOr(String, String), +} + /// The data required to emit a single op. pub struct OpEmitData { /// The name of the op. pub op_name: String, + /// The commands run whenever the op run. Each of these must be run in order in a context where /// the variables in each cmd are supplied. In particular, this means that variables of the /// form "$[i|o]" are in scope. pub cmds: Vec, + + /// Variables to be emitted + pub config_vars: Vec, } /// Given the `index` of a file in the list of input/output files and if the file is an @@ -127,6 +140,13 @@ impl EmitBuild for OpEmitData { .map(|(k, &v)| (io_file_var_name(k, false), v)); let vars: Vec<_> = in_vars.chain(out_vars).collect(); + for var in &self.config_vars { + match var { + ConfigVar::Var(k) => emitter.config_var(k, k)?, + ConfigVar::VarOr(k, d) => emitter.config_var_or(k, k, d)?, + } + } + emitter.build_cmd_with_vars( outputs, &self.op_name, diff --git a/fud2/fud-core/src/script/error.rs b/fud2/fud-core/src/script/error.rs index 26c0d47a88..4bd84f8629 100644 --- a/fud2/fud-core/src/script/error.rs +++ b/fud2/fud-core/src/script/error.rs @@ -14,7 +14,6 @@ pub(super) enum RhaiSystemErrorKind { StateRef(String), BeganOp(String, String), NoOp, - ConfigNotFound(String), } impl RhaiSystemError { @@ -49,13 +48,6 @@ impl RhaiSystemError { } } - pub(super) fn config_not_found(key: &str) -> Self { - Self { - kind: RhaiSystemErrorKind::ConfigNotFound(key.to_string()), - position: rhai::Position::NONE, - } - } - pub(super) fn with_pos(mut self, p: rhai::Position) -> Self { self.position = p; self @@ -77,9 +69,6 @@ impl Display for RhaiSystemError { RhaiSystemErrorKind::NoOp => { write!(f, "Unable to find current op being built. Consider calling start_op_stmts earlier in the program.") } - RhaiSystemErrorKind::ConfigNotFound(v) => { - write!(f, "Unable to find config value: `{v:?}`") - } } } } diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index b97343fa62..911d6f1724 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -28,6 +28,8 @@ struct RhaiOp { output_states: Vec, /// An ordered list of the commands run when this op is required. cmds: Vec, + /// A list of the values required from the config. + config_vars: Vec, } #[derive(Clone)] @@ -122,6 +124,7 @@ impl ScriptContext { input_states: inputs, output_states: outputs, cmds: vec![], + config_vars: vec![], }); Ok(()) } @@ -130,6 +133,7 @@ impl ScriptContext { input_states: _, output_states: _, cmds: _, + config_vars: _, }) => Err(RhaiSystemError::began_op(old_name, name) .with_pos(pos) .into()), @@ -149,6 +153,23 @@ impl ScriptContext { } } + /// Adds a config var. Returns an error if `begin_op` has not been called + /// before this `end_op` and after any previous `end_op`. + fn add_config_var( + &self, + pos: Position, + var: crate::run::ConfigVar, + ) -> RhaiResult<()> { + let mut cur_op = self.cur_op.borrow_mut(); + match *cur_op { + Some(ref mut op_sig) => { + op_sig.config_vars.push(var); + Ok(()) + } + None => Err(RhaiSystemError::no_op().with_pos(pos).into()), + } + } + /// Collects an op currently being built and adds it to `bld`. Returns and error if `begin_op` /// has not been called before this `end_op` and after any previous `end_op`. fn end_op( @@ -163,11 +184,17 @@ impl ScriptContext { ref input_states, ref output_states, ref cmds, + ref config_vars, }) => { // Create the emitter. let cmds = cmds.clone(); let op_name = name.clone(); - let op_emitter = crate::run::OpEmitData { op_name, cmds }; + let config_vars = config_vars.clone(); + let op_emitter = crate::run::OpEmitData { + op_name, + cmds, + config_vars, + }; // Add the op. bld.add_op(name, &[], input_states, output_states, op_emitter); @@ -248,25 +275,21 @@ pub struct ScriptRunner { rhai_functions: rhai::AST, resolver: Option, setups: Rc>>, - config_data: figment::Figment, } impl ScriptRunner { - pub fn new(builder: DriverBuilder, config_data: figment::Figment) -> Self { + pub fn new(builder: DriverBuilder) -> Self { let mut this = Self { builder: Rc::new(RefCell::new(builder)), engine: rhai::Engine::new(), rhai_functions: rhai::AST::empty(), resolver: Some(Resolver::default()), setups: Rc::default(), - config_data, }; this.reg_state(); this.reg_get_state(); this.reg_get_setup(); this.reg_defop_syntax_nop(); - this.reg_config(); - this.reg_config_or(); this } @@ -441,30 +464,36 @@ impl ScriptRunner { } /// Registers a Rhai function for getting values from the config file. - fn reg_config(&mut self) { - let config_data = self.config_data.clone(); + fn reg_config(&mut self, sctx: ScriptContext) { self.engine.register_fn( "config", move |ctx: rhai::NativeCallContext, key: &str| -> RhaiResult<_> { - config_data.extract_inner::(key).or(Err( - RhaiSystemError::config_not_found(key) - .with_pos(ctx.position()) - .into(), - )) + sctx.add_config_var( + ctx.position(), + crate::run::ConfigVar::Var(key.to_string()), + )?; + Ok(format!("${{{}}}", key)) }, ); } /// Registers a Rhai function for getting values from the config file or using a provided /// string if the key is not found. - fn reg_config_or(&mut self) { - let config_data = self.config_data.clone(); + fn reg_config_or(&mut self, sctx: ScriptContext) { self.engine.register_fn( "config_or", - move |key: &str, default: &str| -> RhaiResult<_> { - config_data - .extract_inner::(key) - .or(Ok(default.to_string())) + move |ctx: rhai::NativeCallContext, + key: &str, + default: &str| + -> RhaiResult<_> { + sctx.add_config_var( + ctx.position(), + crate::run::ConfigVar::VarOr( + key.to_string(), + default.to_string(), + ), + )?; + Ok(format!("${{{}}}", key)) }, ); } @@ -624,14 +653,14 @@ impl ScriptRunner { for (i, name) in input_names.clone().into_iter().enumerate() { context.scope_mut().push( name.into_string().unwrap(), - crate::run::io_file_var_name(i, true), + format!("${}", crate::run::io_file_var_name(i, true)), ); } for (i, name) in output_names.clone().into_iter().enumerate() { context.scope_mut().push( name.into_string().unwrap(), - crate::run::io_file_var_name(i, false), + format!("${}", crate::run::io_file_var_name(i, false)), ); } @@ -668,6 +697,8 @@ impl ScriptRunner { self.reg_shell(sctx.clone()); self.reg_end_op_stmts(sctx.clone()); self.reg_defop_syntax(sctx.clone()); + self.reg_config(sctx.clone()); + self.reg_config_or(sctx.clone()); self.engine .module_resolver() diff --git a/fud2/scripts/calyx.rhai b/fud2/scripts/calyx.rhai index 42c13b5158..ab7c3089bb 100644 --- a/fud2/scripts/calyx.rhai +++ b/fud2/scripts/calyx.rhai @@ -16,12 +16,12 @@ fn calyx_setup() { defop calyx_to_verilog(c: calyx_state) >> v: verilog_state { let s = calyx_setup(); - shell(`${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} #c > #v`); + shell(`${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} ${c} > ${v}`); } defop calyx_noverify(c: calyx_state) >> v: verilog_noverify { let s = calyx_setup(); - shell(`${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} --disable-verify #c > #v`); + shell(`${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} --disable-verify ${c} > ${v}`); } fn calyx_pass(pass, args, in_str, out_str) { From 18f2bba8f3719770da3dd4c5dd91705506f7b3e2 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 18 Jul 2024 22:28:35 -0400 Subject: [PATCH 19/30] revert rhai scripts --- fud2/scripts/axi.rhai | 84 +++++++++++++++++++++++++++-------------- fud2/scripts/calyx.rhai | 25 ------------ 2 files changed, 55 insertions(+), 54 deletions(-) diff --git a/fud2/scripts/axi.rhai b/fud2/scripts/axi.rhai index 462424924f..9505c9ffc6 100644 --- a/fud2/scripts/axi.rhai +++ b/fud2/scripts/axi.rhai @@ -2,14 +2,43 @@ import "calyx" as c; export const yxi = state("yxi", ["yxi"]); -fn yxi_pass(in_str, out_str) { - let s = c::calyx_setup(); - let yxi_bin = config_or("yxi", `${s.calyx_base}/target/debug/yxi`); - shell(`${yxi_bin} -l ${s.calyx_base} ${in_str} > ${out_str}`); +fn yxi_setup(e) { + e.config_var_or("yxi", "yxi", "$calyx-base/target/debug/yxi"); + e.rule("yxi", "$yxi -l $calyx-base $in > $out"); } -defop calyx_to_yxi(i: c::calyx_state) >> o: yxi { - yxi_pass("#i", "#o"); +op( + "calyx-to-yxi", + [c::calyx_setup, yxi_setup], + c::calyx_state, + yxi, + |e, input, output| { + e.build_cmd([output], "yxi", [input], []); + }, +); + +fn wrapper_setup(e) { + // Define a `gen-axi` rule that invokes our Python code generator program. + // For now point to standalone axi-generator.py. Can maybe turn this into a rsrc file? + let dynamic = + e.config_constrained_or("dynamic", ["true", "false"], "false"); + let generator_path = if dynamic == "true" { + "$calyx-base/yxi/axi-calyx/dynamic-axi-generator.py" + } else { + "$calyx-base/yxi/axi-calyx/axi-generator.py" + }; + e.config_var_or("axi-generator", "axi.generator", generator_path); + e.config_var_or("python", "python", "python3"); + + e.rule("gen-axi", "$python $axi-generator $in > $out"); + + // Define a simple `combine` rule that just concatenates any numer of files. + e.rule("combine", "cat $in > $out"); + + e.rule( + "remove-imports", + "sed '1,/component main/{/component main/!d; }' $in > $out", + ); } /// Replace the extension in `path` with `new_ext` @@ -22,32 +51,29 @@ fn replace_ext(path, new_ext) { } } -defop axi_wrapped(i: c::calyx_state) >> o: c::calyx_state { - let file_name = "input"; - let tmp_yxi = replace_ext(file_name, "yxi"); +fn axi_wrapped_op(e, input, output) { + let file_name = input.split("/")[-1]; + let tmp_yxi = replace_ext(file_name, "yxi"); - yxi_pass("#i", tmp_yxi); + e.build_cmd([tmp_yxi], "yxi", [input], []); - let refified_calyx = replace_ext(`refified_${file_name}`, "futil"); - c::calyx_pass("external-to-ref", "", "#i", refified_calyx); + let refified_calyx = replace_ext(`refified_${file_name}`, "futil"); + e.build_cmd([refified_calyx], "calyx-pass", [input], []); + e.arg("pass", "external-to-ref"); - // For now point to standalone axi-generator.py. Can maybe turn this into a rsrc file? - let s = c::calyx_setup(); - let dynamic = config_or("dynamic", "false"); - let generator_path = if dynamic == "true" { - `{s.calyx_base}/yxi/axi-calyx/dynamic-axi-generator.py` - } else { - `${s.calyx_base}/yxi/axi-calyx/axi-generator.py` - }; + let axi_wrapper = "axi_wrapper.futil"; + e.build_cmd([axi_wrapper], "gen-axi", [tmp_yxi], []); - let axi_wrapper = "axi_wrapper.futil"; - let axi_generator = config_or("axi.generator", generator_path); - let python = config_or("python", "python3"); + let no_imports_calyx = `no_imports_${refified_calyx}`; + e.build_cmd([no_imports_calyx], "remove-imports", [refified_calyx], []); - shell(`${python} ${axi_generator} ${tmp_yxi} > ${axi_wrapper}`); - - let no_imports_calyx = `no_imports_${refified_calyx}`; - shell(`sed '1,/component main/{/component main/!d; }' ${refified_calyx} > ${no_imports_calyx}`); - - shell(`cat ${axi_wrapper} ${no_imports_calyx} > #o`); + e.build_cmd([output], "combine", [axi_wrapper, no_imports_calyx], []); } + +op( + "axi-wrapped", + [c::calyx_setup, yxi_setup, wrapper_setup], + c::calyx_state, + c::calyx_state, + axi_wrapped_op +); diff --git a/fud2/scripts/calyx.rhai b/fud2/scripts/calyx.rhai index ab7c3089bb..d150ea8994 100644 --- a/fud2/scripts/calyx.rhai +++ b/fud2/scripts/calyx.rhai @@ -3,31 +3,6 @@ export const calyx_state = state("calyx", ["futil"]); export const verilog_noverify = state("verilog-noverify", ["sv"]); export let calyx_setup = calyx_setup; -export let calyx_pass = calyx_pass; - -fn calyx_setup() { - let calyx_base = config("calyx.base"); - #{ - calyx_base: calyx_base, - calyx_exe: config_or("calyx.exe", `${calyx_base}/target/debug/calyx`), - args: config_or("calyx.args", ""), - } -} - -defop calyx_to_verilog(c: calyx_state) >> v: verilog_state { - let s = calyx_setup(); - shell(`${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} ${c} > ${v}`); -} - -defop calyx_noverify(c: calyx_state) >> v: verilog_noverify { - let s = calyx_setup(); - shell(`${s.calyx_exe} -l ${s.calyx_base} -b verilog ${s.args} --disable-verify ${c} > ${v}`); -} - -fn calyx_pass(pass, args, in_str, out_str) { - let s = calyx_setup(); - shell(`${s.calyx_exe} -l ${s.calyx_base} -p ${pass} ${args} ${in_str} > ${out_str}`); -} fn calyx_setup(e) { e.config_var("calyx-base", "calyx.base"); From 9206e320e1ec84091814a1cd64a682cd9905124b Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Thu, 25 Jul 2024 07:06:20 -0400 Subject: [PATCH 20/30] revert calyx.rhai --- fud2/scripts/calyx.rhai | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fud2/scripts/calyx.rhai b/fud2/scripts/calyx.rhai index d150ea8994..83162e51cf 100644 --- a/fud2/scripts/calyx.rhai +++ b/fud2/scripts/calyx.rhai @@ -1,9 +1,8 @@ export const verilog_state = state("verilog", ["sv", "v"]); -export const calyx_state = state("calyx", ["futil"]); export const verilog_noverify = state("verilog-noverify", ["sv"]); +export const calyx_state = state("calyx", ["futil"]); export let calyx_setup = calyx_setup; - fn calyx_setup(e) { e.config_var("calyx-base", "calyx.base"); e.config_var_or("calyx-exe", "calyx.exe", "$calyx-base/target/debug/calyx"); From 2fc8e22973a220f9aabc581f16d9bef2a246505e Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Fri, 26 Jul 2024 01:19:39 -0400 Subject: [PATCH 21/30] add function for testing rhai scripts --- fud2/tests/tests.rs | 56 +++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/fud2/tests/tests.rs b/fud2/tests/tests.rs index fa17eb3c53..d5ec101d64 100644 --- a/fud2/tests/tests.rs +++ b/fud2/tests/tests.rs @@ -1,6 +1,6 @@ use fud_core::{ config::default_config, - exec::{Plan, Request, SingleOpOutputPlanner, IO}, + exec::{FindPlan, Plan, Request, SingleOpOutputPlanner, IO}, run::Run, Driver, DriverBuilder, }; @@ -21,6 +21,18 @@ fn test_driver() -> Driver { bld.load_plugins(&config).build() } +fn driver_from_path(path: &str) -> Driver { + let mut bld = DriverBuilder::new("fud2-plugins"); + let config = figment::Figment::new(); + let path = format!( + "{}/{}", + manifest_dir_macros::directory_path!("tests/scripts"), + path + ); + bld.scripts_dir(&path); + bld.load_plugins(&config).build() +} + trait InstaTest: Sized { /// Get a human-readable description of Self fn desc(&self, driver: &Driver) -> String; @@ -124,23 +136,36 @@ impl InstaTest for Request { } } -fn request( +fn request_with_planner( driver: &Driver, - start: &str, - end: &str, + start: &[&str], + end: &[&str], through: &[&str], + planner: impl FindPlan + 'static, ) -> Request { fud_core::exec::Request { start_files: vec![], - start_states: vec![driver.get_state(start).unwrap()], + start_states: start + .iter() + .map(|s| driver.get_state(s).unwrap()) + .collect(), end_files: vec![], - end_states: vec![driver.get_state(end).unwrap()], + end_states: end.iter().map(|s| driver.get_state(s).unwrap()).collect(), through: through.iter().map(|s| driver.get_op(s).unwrap()).collect(), workdir: ".".into(), - planner: Box::new(SingleOpOutputPlanner {}), + planner: Box::new(planner), } } +fn request( + driver: &Driver, + start: &[&str], + end: &[&str], + through: &[&str], +) -> Request { + request_with_planner(driver, start, end, through, SingleOpOutputPlanner {}) +} + #[test] fn all_ops() { let driver = test_driver(); @@ -200,13 +225,14 @@ fn list_ops() { #[test] fn calyx_to_verilog() { let driver = test_driver(); - request(&driver, "calyx", "verilog", &[]).test(&driver); + request(&driver, &["calyx"], &["verilog"], &[]).test(&driver); } #[test] fn calyx_via_firrtl() { let driver = test_driver(); - request(&driver, "calyx", "verilog-refmem", &["firrtl"]).test(&driver); + request(&driver, &["calyx"], &["verilog-refmem"], &["firrtl"]) + .test(&driver); } #[test] @@ -214,7 +240,7 @@ fn sim_tests() { let driver = test_driver(); for dest in &["dat", "vcd"] { for sim in &["icarus", "verilator"] { - request(&driver, "calyx", dest, &[sim]).test(&driver); + request(&driver, &["calyx"], &[dest], &[sim]).test(&driver); } } } @@ -222,21 +248,21 @@ fn sim_tests() { #[test] fn cider_tests() { let driver = test_driver(); - request(&driver, "calyx", "dat", &["interp"]).test(&driver); - request(&driver, "calyx", "debug", &[]).test(&driver); + request(&driver, &["calyx"], &["dat"], &["interp"]).test(&driver); + request(&driver, &["calyx"], &["debug"], &[]).test(&driver); } #[test] fn xrt_tests() { let driver = test_driver(); - request(&driver, "calyx", "dat", &["xrt"]).test(&driver); - request(&driver, "calyx", "vcd", &["xrt-trace"]).test(&driver); + request(&driver, &["calyx"], &["dat"], &["xrt"]).test(&driver); + request(&driver, &["calyx"], &["vcd"], &["xrt-trace"]).test(&driver); } #[test] fn frontend_tests() { let driver = test_driver(); for frontend in &["dahlia", "mrxl"] { - request(&driver, frontend, "calyx", &[]).test(&driver); + request(&driver, &[frontend], &["calyx"], &[]).test(&driver); } } From 890d918cc4b030c17e78000d0e04ef960b816e62 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Sun, 28 Jul 2024 21:56:34 -0400 Subject: [PATCH 22/30] add enumerate planner --- fud2/fud-core/src/exec/planner.rs | 150 +++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 5 deletions(-) diff --git a/fud2/fud-core/src/exec/planner.rs b/fud2/fud-core/src/exec/planner.rs index 49adc1dada..b6ea6dce24 100644 --- a/fud2/fud-core/src/exec/planner.rs +++ b/fud2/fud-core/src/exec/planner.rs @@ -24,20 +24,160 @@ pub trait FindPlan: std::fmt::Debug { #[derive(Debug, Default)] pub struct EnumeratePlanner {} impl EnumeratePlanner { + /// The max number of ops in a searched for plan. + const MAX_PLAN_LEN: u32 = 7; + pub fn new() -> Self { EnumeratePlanner {} } + + /// Returns `true` if executing `plan` will take `start` to `end`, going through all ops in `through`. + /// + /// This function assumes all required inputs to `plan` exist or will be generated by `plan`. + fn valid_plan(plan: &[Step], end: &[StateRef], through: &[OpRef]) -> bool { + // Check all states in `end` are created. + let end_created = end + .iter() + .all(|s| plan.iter().any(|(_, states)| states.contains(s))); + + // FIXME: Currently this checks that an outputs of an op specified by though is used. + // However, it's possible that the only use of this output by another op whose outputs + // are all unused. This means the plan doesn't actually use the specified op. but this + // code reports it would. + let through_used = through.iter().all(|t| { + plan.iter() + .any(|(op, used_states)| op == t && !used_states.is_empty()) + }); + + end_created && through_used + } + + /// A recursive function to generate all sequences prefixed by `plan` and containing `len` more + /// `Steps`. Returns a sequence such that applying `valid_plan` to the sequence results in `true. + /// If no such sequence exists, then `None` is returned. + /// + /// `start` is the base inputs which can be used for ops. + /// `end` is the states to be generated by the return sequence of ops. + /// `ops` contains all usable operations to construct `Step`s from. + fn try_paths_of_length( + plan: &mut Vec, + len: u32, + start: &[StateRef], + end: &[StateRef], + through: &[OpRef], + ops: &PrimaryMap, + ) -> Option> { + // Base case of the recursion. As `len == 0`, the algorithm reduces to applying `good` to + // `plan. + if len == 0 { + return if Self::valid_plan(plan, end, through) { + Some(plan.clone()) + } else { + None + }; + } + + // Try adding every op to the back of the current `plan`. Then recurse on the subproblem. + for op_ref in ops.keys() { + // Check previous ops in the plan to see if any generated an input to `op_ref`. + let all_generated = ops[op_ref].input.iter().all(|input| { + // Check the outputs of ops earlier in the plan can be used as inputs to `op_ref`. + // `plan` is reversed so the latest versions of states are used. + plan.iter_mut().rev().any(|(o, _used_outputs)| + ops[*o].output.contains(input) + ) + // As well as being generated in `plan`, `input` could be given in `start`. + || start.contains(input) + }); + + // If this op cannot be uesd in the `plan` try a different one. + if !all_generated { + continue; + } + + // Mark all used outputs. + let used_outputs_idxs: Vec<_> = ops[op_ref] + .input + .iter() + .filter_map(|input| { + // Get indicies of `Step`s whose `used_outputs` must be modified. + plan.iter() + .rev() + .position(|(o, used_outputs)| { + // `op_ref`'s op now uses the input of the previous op in the plan. + // This should be noted in `used_outputs`. + !used_outputs.contains(input) + && ops[*o].output.contains(input) + }) + .map(|i| (input, plan.len() - i - 1)) + }) + .collect(); + + for &(&input, i) in &used_outputs_idxs { + plan[i].1.push(input); + } + + // Mark all outputs in `end` as used because they are used (or at least requested) by + // `end`. + let outputs = ops[op_ref].output.clone().into_iter(); + let used_outputs = + outputs.filter(|s| end.contains(s)).collect::>(); + + // Recurse! Now that `len` has been reduced by one, see if this new problem has a + // solution. + plan.push((op_ref, used_outputs)); + if let Some(plan) = Self::try_paths_of_length( + plan, + len - 1, + start, + end, + through, + ops, + ) { + return Some(plan); + } + + // The investigated plan didn't work. + // Pop off the attempted element. + plan.pop(); + + // Revert modifications to `used_outputs`. + for &(_, i) in &used_outputs_idxs { + plan[i].1.pop(); + } + } + + // No sequence of `Step`s found :(. + None + } + /// Returns a sequence of `Step`s to transform `start` to `end`. The `Step`s are guaranteed to /// contain all ops in `through`. If no such sequence exists, `None` is returned. /// /// `ops` is a complete list of operations. fn find_plan( - _start: &[StateRef], - _end: &[StateRef], - _through: &[OpRef], - _ops: &PrimaryMap, + start: &[StateRef], + end: &[StateRef], + through: &[OpRef], + ops: &PrimaryMap, ) -> Option> { - todo!() + // Try all sequences of ops up to `MAX_PATH_LEN`. At that point, the computation starts to + // become really big. + for len in 1..=Self::MAX_PLAN_LEN { + if let Some(plan) = Self::try_paths_of_length( + &mut vec![], + len, + start, + end, + through, + ops, + ) { + return Some(plan); + } + } + + // No sequence of `Step`s found :(. + None } } From 346bc26f98bfec1aa6e3116d05bf15182ae6ff69 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Mon, 29 Jul 2024 01:53:06 -0400 Subject: [PATCH 23/30] bug fix and tests --- fud2/fud-core/src/script/plugin.rs | 9 +-- fud2/tests/scripts/defop/test.rhai | 26 +++++++ ...alyx_through_firrtl_to_verilog-refmem.snap | 49 ++++++++++++ ...sts__test@calyx_through_icarus_to_dat.snap | 50 ++++++++++++ ...sts__test@calyx_through_icarus_to_vcd.snap | 49 ++++++++++++ ...sts__test@calyx_through_interp_to_dat.snap | 60 +++++++++++++++ ...__test@calyx_through_verilator_to_dat.snap | 54 +++++++++++++ ...__test@calyx_through_verilator_to_vcd.snap | 53 +++++++++++++ ...__test@calyx_through_xrt-trace_to_vcd.snap | 76 +++++++++++++++++++ .../tests__test@calyx_through_xrt_to_dat.snap | 76 +++++++++++++++++++ .../snapshots/tests__test@calyx_to_debug.snap | 59 ++++++++++++++ .../tests__test@calyx_to_verilog.snap | 23 ++++++ .../tests__test@dahlia_to_calyx.snap | 15 ++++ .../snapshots/tests__test@mrxl_to_calyx.snap | 15 ++++ .../snapshots/tests__test@state0_state1.snap | 16 ++++ ...state0_state1_state2_to_state3_state4.snap | 20 +++++ .../tests__test@state0_state1_to_state2.snap | 17 +++++ .../tests__test@state0_to_state1.snap | 16 ++++ .../tests__test@state0_to_state2_state1.snap | 18 +++++ fud2/tests/tests.rs | 75 +++++++++++++++--- 20 files changed, 759 insertions(+), 17 deletions(-) create mode 100644 fud2/tests/scripts/defop/test.rhai create mode 100644 fud2/tests/snapshots/tests__test@calyx_through_firrtl_to_verilog-refmem.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_through_icarus_to_dat.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_through_icarus_to_vcd.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_through_interp_to_dat.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_through_verilator_to_dat.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_through_verilator_to_vcd.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_through_xrt-trace_to_vcd.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_through_xrt_to_dat.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_to_debug.snap create mode 100644 fud2/tests/snapshots/tests__test@calyx_to_verilog.snap create mode 100644 fud2/tests/snapshots/tests__test@dahlia_to_calyx.snap create mode 100644 fud2/tests/snapshots/tests__test@mrxl_to_calyx.snap create mode 100644 fud2/tests/snapshots/tests__test@state0_state1.snap create mode 100644 fud2/tests/snapshots/tests__test@state0_state1_state2_to_state3_state4.snap create mode 100644 fud2/tests/snapshots/tests__test@state0_state1_to_state2.snap create mode 100644 fud2/tests/snapshots/tests__test@state0_to_state1.snap create mode 100644 fud2/tests/snapshots/tests__test@state0_to_state2_state1.snap diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 911d6f1724..345ea56917 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -209,7 +209,7 @@ impl ScriptContext { } /// All nodes in the parsing state machine. -#[derive(Copy, Clone)] +#[derive(Debug, Copy, Clone)] enum ParseNode { DefopS, IdentS, @@ -228,7 +228,7 @@ enum ParseNode { } /// All of the state of the parser. -#[derive(Clone)] +#[derive(Debug, Clone)] struct ParseState { node: ParseNode, inputs: usize, @@ -610,7 +610,6 @@ impl ScriptRunner { true, move |context, inputs, state| { let state = state.clone_cast::(); - // Collect name of op and input/output states. let op_name = inputs.first().unwrap().get_string_value().unwrap(); @@ -634,7 +633,7 @@ impl ScriptRunner { .iter() .skip(1 + 2 * state.inputs) .step_by(2) - .take(state.inputs) + .take(state.outputs) .map(|n| { Dynamic::from(n.get_string_value().unwrap().to_string()) }) @@ -643,7 +642,7 @@ impl ScriptRunner { .iter() .skip(2 + 2 * state.inputs) .step_by(2) - .take(state.inputs) + .take(state.outputs) .map(|s| s.eval_with_context(context)) .collect::>>()?; let body = inputs.last().unwrap(); diff --git a/fud2/tests/scripts/defop/test.rhai b/fud2/tests/scripts/defop/test.rhai new file mode 100644 index 0000000000..5e8a9dddb8 --- /dev/null +++ b/fud2/tests/scripts/defop/test.rhai @@ -0,0 +1,26 @@ +const state0 = state("state0", []); +const state1 = state("state1", []); +const state2 = state("state2", []); +const state3 = state("state3", []); +const state4 = state("state4", []); + +// simple defop +defop t0(s0: state0) >> s1: state1 { + shell("echo uwu"); +} + +// defop combining states +defop t1(s0: state0, s1: state1) >> s2: state2 { + shell("echo owo"); +} + +// defop splitting states +defop t2(s0: state0) >> s1: state1, s2: state2 { + shell("echo -_-"); +} + +// testing input and output variables +defop t3(s0: state0, s1: state1, s2: state2) >> s3: state3, s4: state4 { + shell(`echo inputs ${s0} ${s1} ${s2}`); + shell(`echo outputs ${s3} ${s4}`); +} diff --git a/fud2/tests/snapshots/tests__test@calyx_through_firrtl_to_verilog-refmem.snap b/fud2/tests/snapshots/tests__test@calyx_through_firrtl_to_verilog-refmem.snap new file mode 100644 index 0000000000..5b22cb0ebf --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_through_firrtl_to_verilog-refmem.snap @@ -0,0 +1,49 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> verilog-refmem through firrtl" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +rule ref-to-external + command = sed 's/ref /@external /g' $in > $out +rule external-to-ref + command = sed 's/@external([0-9]*)/ref/g' $in | sed 's/@external/ref/g' > $out +gen-testbench-script = $calyx-base/tools/firrtl/generate-testbench.py +build memories.sv: get-rsrc +rule generate-refmem-testbench + command = python3 $gen-testbench-script $in > $out +rule dummy + command = sh -c 'cat $$0' $in > $out + +firrtl-exe = /test/bin/firrtl +rule firrtl + command = $firrtl-exe -i $in -o $out -X sverilog +build primitives-for-firrtl.sv: get-rsrc +rule add-verilog-primitives + command = cat primitives-for-firrtl.sv $in > $out + +build external.futil: ref-to-external _from_stdin_calyx.futil +build ref.futil: external-to-ref _from_stdin_calyx.futil +build memory-info.json: yxi external.futil +build tb.sv: generate-refmem-testbench memory-info.json +build tmp-out.fir: calyx ref.futil + backend = firrtl + args = --emit-primitive-extmodules +build firrtl.fir: dummy tmp-out.fir tb.sv +build partial.sv: firrtl firrtl.fir +build _to_stdout_verilog-refmem.sv: add-verilog-primitives partial.sv | primitives-for-firrtl.sv + +default _to_stdout_verilog-refmem.sv diff --git a/fud2/tests/snapshots/tests__test@calyx_through_icarus_to_dat.snap b/fud2/tests/snapshots/tests__test@calyx_through_icarus_to_dat.snap new file mode 100644 index 0000000000..2a1a35ed98 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_through_icarus_to_dat.snap @@ -0,0 +1,50 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> dat through icarus" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +python = python3 +build json-dat.py: get-rsrc +rule hex-data + command = $python json-dat.py --from-json $in $out +rule json-data + command = $python json-dat.py --to-json $out $in +sim_data = /test/data.json +datadir = sim_data +build $datadir: hex-data $sim_data | json-dat.py +rule sim-run + command = ./$bin +DATA=$datadir +CYCLE_LIMIT=$cycle-limit $args > $out +cycle-limit = 500000000 + +build tb.sv: get-rsrc + +iverilog = iverilog +rule icarus-compile-standalone-tb + command = $iverilog -g2012 -o $out tb.sv $in +rule icarus-compile-custom-tb + command = $iverilog -g2012 -o $out tb.sv memories.sv $in + +build verilog-noverify.sv: calyx _from_stdin_calyx.futil + backend = verilog + args = --disable-verify +build sim.exe: icarus-compile-standalone-tb verilog-noverify.sv | tb.sv +build sim.log: sim-run sim.exe $datadir + bin = sim.exe + args = +NOTRACE=1 +build _to_stdout_dat.json: json-data $datadir sim.log | json-dat.py + +default _to_stdout_dat.json diff --git a/fud2/tests/snapshots/tests__test@calyx_through_icarus_to_vcd.snap b/fud2/tests/snapshots/tests__test@calyx_through_icarus_to_vcd.snap new file mode 100644 index 0000000000..85827059d6 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_through_icarus_to_vcd.snap @@ -0,0 +1,49 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> vcd through icarus" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +python = python3 +build json-dat.py: get-rsrc +rule hex-data + command = $python json-dat.py --from-json $in $out +rule json-data + command = $python json-dat.py --to-json $out $in +sim_data = /test/data.json +datadir = sim_data +build $datadir: hex-data $sim_data | json-dat.py +rule sim-run + command = ./$bin +DATA=$datadir +CYCLE_LIMIT=$cycle-limit $args > $out +cycle-limit = 500000000 + +build tb.sv: get-rsrc + +iverilog = iverilog +rule icarus-compile-standalone-tb + command = $iverilog -g2012 -o $out tb.sv $in +rule icarus-compile-custom-tb + command = $iverilog -g2012 -o $out tb.sv memories.sv $in + +build verilog-noverify.sv: calyx _from_stdin_calyx.futil + backend = verilog + args = --disable-verify +build sim.exe: icarus-compile-standalone-tb verilog-noverify.sv | tb.sv +build sim.log _to_stdout_vcd.vcd: sim-run sim.exe $datadir + bin = sim.exe + args = +NOTRACE=0 +OUT=_to_stdout_vcd.vcd + +default _to_stdout_vcd.vcd diff --git a/fud2/tests/snapshots/tests__test@calyx_through_interp_to_dat.snap b/fud2/tests/snapshots/tests__test@calyx_through_interp_to_dat.snap new file mode 100644 index 0000000000..713d53f60d --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_through_interp_to_dat.snap @@ -0,0 +1,60 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> dat through interp" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +python = python3 +build json-dat.py: get-rsrc +rule hex-data + command = $python json-dat.py --from-json $in $out +rule json-data + command = $python json-dat.py --to-json $out $in +sim_data = /test/data.json +datadir = sim_data +build $datadir: hex-data $sim_data | json-dat.py +rule sim-run + command = ./$bin +DATA=$datadir +CYCLE_LIMIT=$cycle-limit $args > $out +cycle-limit = 500000000 + +build tb.sv: get-rsrc + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +cider-exe = $calyx-base/target/debug/cider +cider-converter = $calyx-base/target/debug/cider-data-converter +rule cider + command = $cider-exe -l $calyx-base --raw --data data.json $in > $out +rule cider-debug + command = $cider-exe -l $calyx-base --data data.json $in debug || true + pool = console +build interp-dat.py: get-rsrc +python = python3 +rule dat-to-interp + command = $python interp-dat.py --to-interp $in +rule interp-to-dat + command = $python interp-dat.py --from-interp $in $sim_data > $out +build data.json: dat-to-interp $sim_data | interp-dat.py +rule run-cider + command = $cider-exe -l $calyx-base --data data.dump $in flat > $out +rule dump-to-interp + command = $cider-converter --to cider $in > $out +rule interp-to-dump + command = $cider-converter --to json $in > $out +build data.dump: dump-to-interp $sim_data | $cider-converter + +build interp_out.json: cider _from_stdin_calyx.futil | data.json +build _to_stdout_dat.json: interp-to-dat interp_out.json | $sim_data interp-dat.py + +default _to_stdout_dat.json diff --git a/fud2/tests/snapshots/tests__test@calyx_through_verilator_to_dat.snap b/fud2/tests/snapshots/tests__test@calyx_through_verilator_to_dat.snap new file mode 100644 index 0000000000..184bb8a24f --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_through_verilator_to_dat.snap @@ -0,0 +1,54 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> dat through verilator" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +python = python3 +build json-dat.py: get-rsrc +rule hex-data + command = $python json-dat.py --from-json $in $out +rule json-data + command = $python json-dat.py --to-json $out $in +sim_data = /test/data.json +datadir = sim_data +build $datadir: hex-data $sim_data | json-dat.py +rule sim-run + command = ./$bin +DATA=$datadir +CYCLE_LIMIT=$cycle-limit $args > $out +cycle-limit = 500000000 + +build tb.sv: get-rsrc + +verilator = verilator +cycle-limit = 500000000 +rule verilator-compile-standalone-tb + command = $verilator $in tb.sv --trace --binary --top-module TOP -fno-inline -Mdir $out-dir +rule verilator-compile-custom-tb + command = $verilator $in tb.sv memories.sv --trace --binary --top-module TOP -fno-inline -Mdir $out-dir +rule cp + command = cp $in $out + +build verilog.sv: calyx _from_stdin_calyx.futil + backend = verilog +build verilator-out/VTOP: verilator-compile-standalone-tb verilog.sv | tb.sv + out-dir = verilator-out +build sim.exe: cp verilator-out/VTOP +build sim.log: sim-run sim.exe $datadir + bin = sim.exe + args = +NOTRACE=1 +build _to_stdout_dat.json: json-data $datadir sim.log | json-dat.py + +default _to_stdout_dat.json diff --git a/fud2/tests/snapshots/tests__test@calyx_through_verilator_to_vcd.snap b/fud2/tests/snapshots/tests__test@calyx_through_verilator_to_vcd.snap new file mode 100644 index 0000000000..07ab0b6544 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_through_verilator_to_vcd.snap @@ -0,0 +1,53 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> vcd through verilator" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +python = python3 +build json-dat.py: get-rsrc +rule hex-data + command = $python json-dat.py --from-json $in $out +rule json-data + command = $python json-dat.py --to-json $out $in +sim_data = /test/data.json +datadir = sim_data +build $datadir: hex-data $sim_data | json-dat.py +rule sim-run + command = ./$bin +DATA=$datadir +CYCLE_LIMIT=$cycle-limit $args > $out +cycle-limit = 500000000 + +build tb.sv: get-rsrc + +verilator = verilator +cycle-limit = 500000000 +rule verilator-compile-standalone-tb + command = $verilator $in tb.sv --trace --binary --top-module TOP -fno-inline -Mdir $out-dir +rule verilator-compile-custom-tb + command = $verilator $in tb.sv memories.sv --trace --binary --top-module TOP -fno-inline -Mdir $out-dir +rule cp + command = cp $in $out + +build verilog.sv: calyx _from_stdin_calyx.futil + backend = verilog +build verilator-out/VTOP: verilator-compile-standalone-tb verilog.sv | tb.sv + out-dir = verilator-out +build sim.exe: cp verilator-out/VTOP +build sim.log _to_stdout_vcd.vcd: sim-run sim.exe $datadir + bin = sim.exe + args = +NOTRACE=0 +OUT=_to_stdout_vcd.vcd + +default _to_stdout_vcd.vcd diff --git a/fud2/tests/snapshots/tests__test@calyx_through_xrt-trace_to_vcd.snap b/fud2/tests/snapshots/tests__test@calyx_through_xrt-trace_to_vcd.snap new file mode 100644 index 0000000000..6531aaf4f2 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_through_xrt-trace_to_vcd.snap @@ -0,0 +1,76 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> vcd through xrt-trace" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +vivado-dir = /test/xilinx/vivado +vitis-dir = /test/xilinx/vitis +build gen_xo.tcl: get-rsrc +build get-ports.py: get-rsrc +python = python3 +rule gen-xo + command = $vivado-dir/bin/vivado -mode batch -source gen_xo.tcl -tclargs $out `$python get-ports.py kernel.xml` + pool = console +xilinx-mode = hw_emu +platform = xilinx_u50_gen3x16_xdma_201920_3 +rule compile-xclbin + command = $vitis-dir/bin/v++ -g -t $xilinx-mode --platform $platform --save-temps --profile.data all:all:all --profile.exec all:all:all -lo $out $in + pool = console + +python = python3 +build json-dat.py: get-rsrc +rule hex-data + command = $python json-dat.py --from-json $in $out +rule json-data + command = $python json-dat.py --to-json $out $in +sim_data = /test/data.json +datadir = sim_data +build $datadir: hex-data $sim_data | json-dat.py +rule sim-run + command = ./$bin +DATA=$datadir +CYCLE_LIMIT=$cycle-limit $args > $out +cycle-limit = 500000000 + +build tb.sv: get-rsrc + +rule emconfig + command = $vitis-dir/bin/emconfigutil --platform $platform +build emconfig.json: emconfig +xrt-dir = /test/xilinx/xrt +rule xclrun + command = bash -c 'source $vitis-dir/settings64.sh ; source $xrt-dir/setup.sh ; XRT_INI_PATH=$xrt_ini EMCONFIG_PATH=. XCL_EMULATION_MODE=$xilinx-mode $python -m fud.xclrun --out $out $in' + pool = console +rule echo + command = echo $contents > $out +build pre_sim.tcl: echo | + contents = open_vcd\\nlog_vcd *\\n +build post_sim.tcl: echo | + contents = close_vcd\\n + +build main.sv: calyx _from_stdin_calyx.futil + backend = verilog + args = --synthesis -p external +build toplevel.v: calyx _from_stdin_calyx.futil + backend = xilinx +build kernel.xml: calyx _from_stdin_calyx.futil + backend = xilinx-xml +build xo.xo: gen-xo | main.sv toplevel.v kernel.xml gen_xo.tcl get-ports.py +build xclbin.xclbin: compile-xclbin xo.xo +build xrt_trace.ini: get-rsrc +build _to_stdout_vcd.vcd: xclrun xclbin.xclbin $sim_data | emconfig.json pre_sim.tcl post_sim.tcl xrt_trace.ini + xrt_ini = xrt_trace.ini + +default _to_stdout_vcd.vcd diff --git a/fud2/tests/snapshots/tests__test@calyx_through_xrt_to_dat.snap b/fud2/tests/snapshots/tests__test@calyx_through_xrt_to_dat.snap new file mode 100644 index 0000000000..5aa1580b4b --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_through_xrt_to_dat.snap @@ -0,0 +1,76 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> dat through xrt" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +vivado-dir = /test/xilinx/vivado +vitis-dir = /test/xilinx/vitis +build gen_xo.tcl: get-rsrc +build get-ports.py: get-rsrc +python = python3 +rule gen-xo + command = $vivado-dir/bin/vivado -mode batch -source gen_xo.tcl -tclargs $out `$python get-ports.py kernel.xml` + pool = console +xilinx-mode = hw_emu +platform = xilinx_u50_gen3x16_xdma_201920_3 +rule compile-xclbin + command = $vitis-dir/bin/v++ -g -t $xilinx-mode --platform $platform --save-temps --profile.data all:all:all --profile.exec all:all:all -lo $out $in + pool = console + +python = python3 +build json-dat.py: get-rsrc +rule hex-data + command = $python json-dat.py --from-json $in $out +rule json-data + command = $python json-dat.py --to-json $out $in +sim_data = /test/data.json +datadir = sim_data +build $datadir: hex-data $sim_data | json-dat.py +rule sim-run + command = ./$bin +DATA=$datadir +CYCLE_LIMIT=$cycle-limit $args > $out +cycle-limit = 500000000 + +build tb.sv: get-rsrc + +rule emconfig + command = $vitis-dir/bin/emconfigutil --platform $platform +build emconfig.json: emconfig +xrt-dir = /test/xilinx/xrt +rule xclrun + command = bash -c 'source $vitis-dir/settings64.sh ; source $xrt-dir/setup.sh ; XRT_INI_PATH=$xrt_ini EMCONFIG_PATH=. XCL_EMULATION_MODE=$xilinx-mode $python -m fud.xclrun --out $out $in' + pool = console +rule echo + command = echo $contents > $out +build pre_sim.tcl: echo | + contents = open_vcd\\nlog_vcd *\\n +build post_sim.tcl: echo | + contents = close_vcd\\n + +build main.sv: calyx _from_stdin_calyx.futil + backend = verilog + args = --synthesis -p external +build toplevel.v: calyx _from_stdin_calyx.futil + backend = xilinx +build kernel.xml: calyx _from_stdin_calyx.futil + backend = xilinx-xml +build xo.xo: gen-xo | main.sv toplevel.v kernel.xml gen_xo.tcl get-ports.py +build xclbin.xclbin: compile-xclbin xo.xo +build xrt.ini: get-rsrc +build _to_stdout_dat.json: xclrun xclbin.xclbin $sim_data | emconfig.json xrt.ini + xrt_ini = xrt.ini + +default _to_stdout_dat.json diff --git a/fud2/tests/snapshots/tests__test@calyx_to_debug.snap b/fud2/tests/snapshots/tests__test@calyx_to_debug.snap new file mode 100644 index 0000000000..8c65bdb7e6 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_to_debug.snap @@ -0,0 +1,59 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> debug" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +python = python3 +build json-dat.py: get-rsrc +rule hex-data + command = $python json-dat.py --from-json $in $out +rule json-data + command = $python json-dat.py --to-json $out $in +sim_data = /test/data.json +datadir = sim_data +build $datadir: hex-data $sim_data | json-dat.py +rule sim-run + command = ./$bin +DATA=$datadir +CYCLE_LIMIT=$cycle-limit $args > $out +cycle-limit = 500000000 + +build tb.sv: get-rsrc + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +cider-exe = $calyx-base/target/debug/cider +cider-converter = $calyx-base/target/debug/cider-data-converter +rule cider + command = $cider-exe -l $calyx-base --raw --data data.json $in > $out +rule cider-debug + command = $cider-exe -l $calyx-base --data data.json $in debug || true + pool = console +build interp-dat.py: get-rsrc +python = python3 +rule dat-to-interp + command = $python interp-dat.py --to-interp $in +rule interp-to-dat + command = $python interp-dat.py --from-interp $in $sim_data > $out +build data.json: dat-to-interp $sim_data | interp-dat.py +rule run-cider + command = $cider-exe -l $calyx-base --data data.dump $in flat > $out +rule dump-to-interp + command = $cider-converter --to cider $in > $out +rule interp-to-dump + command = $cider-converter --to json $in > $out +build data.dump: dump-to-interp $sim_data | $cider-converter + +build _to_stdout_debug: cider-debug _from_stdin_calyx.futil | data.json + +default _to_stdout_debug diff --git a/fud2/tests/snapshots/tests__test@calyx_to_verilog.snap b/fud2/tests/snapshots/tests__test@calyx_to_verilog.snap new file mode 100644 index 0000000000..af413d8098 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@calyx_to_verilog.snap @@ -0,0 +1,23 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: calyx -> verilog" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +calyx-base = /test/calyx +calyx-exe = $calyx-base/target/debug/calyx +args = +rule calyx + command = $calyx-exe -l $calyx-base -b $backend $args $in > $out +rule calyx-pass + command = $calyx-exe -l $calyx-base -p $pass $args $in > $out +flags = -p none +rule calyx-with-flags + command = $calyx-exe -l $calyx-base $flags $args $in > $out + +build _to_stdout_verilog.sv: calyx _from_stdin_calyx.futil + backend = verilog + +default _to_stdout_verilog.sv diff --git a/fud2/tests/snapshots/tests__test@dahlia_to_calyx.snap b/fud2/tests/snapshots/tests__test@dahlia_to_calyx.snap new file mode 100644 index 0000000000..9906ddd097 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@dahlia_to_calyx.snap @@ -0,0 +1,15 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: dahlia -> calyx" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +dahlia-exe = /test/bin/dahlia +rule dahlia-to-calyx + command = $dahlia-exe -b calyx --lower -l error $in -o $out + +build _to_stdout_calyx.futil: dahlia-to-calyx _from_stdin_dahlia.fuse + +default _to_stdout_calyx.futil diff --git a/fud2/tests/snapshots/tests__test@mrxl_to_calyx.snap b/fud2/tests/snapshots/tests__test@mrxl_to_calyx.snap new file mode 100644 index 0000000000..fbd9d680b1 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@mrxl_to_calyx.snap @@ -0,0 +1,15 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: mrxl -> calyx" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +mrxl-exe = mrxl +rule mrxl-to-calyx + command = $mrxl-exe $in > $out + +build _to_stdout_calyx.futil: mrxl-to-calyx _from_stdin_mrxl.mrxl + +default _to_stdout_calyx.futil diff --git a/fud2/tests/snapshots/tests__test@state0_state1.snap b/fud2/tests/snapshots/tests__test@state0_state1.snap new file mode 100644 index 0000000000..b2cd496a36 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@state0_state1.snap @@ -0,0 +1,16 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: state0 -> state1" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +rule t0 + command = echo uwu +build _to_stdout_state1: t0 _from_stdin_state0 + i0 = _from_stdin_state0 + o0 = _to_stdout_state1 + + +default _to_stdout_state1 diff --git a/fud2/tests/snapshots/tests__test@state0_state1_state2_to_state3_state4.snap b/fud2/tests/snapshots/tests__test@state0_state1_state2_to_state3_state4.snap new file mode 100644 index 0000000000..01c59e594a --- /dev/null +++ b/fud2/tests/snapshots/tests__test@state0_state1_state2_to_state3_state4.snap @@ -0,0 +1,20 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: state0 state1 state2 -> state3 state4" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +rule t3 + command = echo inputs $i0 $i1 $i2 && echo outputs $o0 $o1 +build _to_stdout_state3 _to_stdout_state4: t3 _from_stdin_state0 _from_stdin_state1 _from_stdin_state2 + i0 = _from_stdin_state0 + i1 = _from_stdin_state1 + i2 = _from_stdin_state2 + o0 = _to_stdout_state3 + o1 = _to_stdout_state4 + + +default _to_stdout_state3 +default _to_stdout_state4 diff --git a/fud2/tests/snapshots/tests__test@state0_state1_to_state2.snap b/fud2/tests/snapshots/tests__test@state0_state1_to_state2.snap new file mode 100644 index 0000000000..1ca3828b59 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@state0_state1_to_state2.snap @@ -0,0 +1,17 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: state0 state1 -> state2" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +rule t1 + command = echo owo +build _to_stdout_state2: t1 _from_stdin_state0 _from_stdin_state1 + i0 = _from_stdin_state0 + i1 = _from_stdin_state1 + o0 = _to_stdout_state2 + + +default _to_stdout_state2 diff --git a/fud2/tests/snapshots/tests__test@state0_to_state1.snap b/fud2/tests/snapshots/tests__test@state0_to_state1.snap new file mode 100644 index 0000000000..b2cd496a36 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@state0_to_state1.snap @@ -0,0 +1,16 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: state0 -> state1" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +rule t0 + command = echo uwu +build _to_stdout_state1: t0 _from_stdin_state0 + i0 = _from_stdin_state0 + o0 = _to_stdout_state1 + + +default _to_stdout_state1 diff --git a/fud2/tests/snapshots/tests__test@state0_to_state2_state1.snap b/fud2/tests/snapshots/tests__test@state0_to_state2_state1.snap new file mode 100644 index 0000000000..ff06d4ce3b --- /dev/null +++ b/fud2/tests/snapshots/tests__test@state0_to_state2_state1.snap @@ -0,0 +1,18 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: state0 -> state2 state1" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +rule t2 + command = echo -_- +build _to_stdout_state1 _to_stdout_state2: t2 _from_stdin_state0 + i0 = _from_stdin_state0 + o0 = _to_stdout_state1 + o1 = _to_stdout_state2 + + +default _to_stdout_state2 +default _to_stdout_state1 diff --git a/fud2/tests/tests.rs b/fud2/tests/tests.rs index d5ec101d64..529bc94c96 100644 --- a/fud2/tests/tests.rs +++ b/fud2/tests/tests.rs @@ -1,6 +1,8 @@ use fud_core::{ config::default_config, - exec::{FindPlan, Plan, Request, SingleOpOutputPlanner, IO}, + exec::{ + EnumeratePlanner, FindPlan, Plan, Request, SingleOpOutputPlanner, IO, + }, run::Run, Driver, DriverBuilder, }; @@ -104,11 +106,17 @@ impl InstaTest for Plan { impl InstaTest for Request { fn desc(&self, driver: &Driver) -> String { - let mut desc = format!( - "emit request: {} -> {}", - driver.states[self.start_states[0]].name, - driver.states[self.end_states[0]].name - ); + let start_str = self + .start_states + .iter() + .map(|&state| &driver.states[state].name) + .join(" "); + let end_str = &self + .end_states + .iter() + .map(|&state| &driver.states[state].name) + .join(" "); + let mut desc = format!("emit request: {} -> {}", start_str, end_str); if !self.through.is_empty() { desc.push_str(" through"); for op in &self.through { @@ -120,13 +128,26 @@ impl InstaTest for Request { } fn slug(&self, driver: &Driver) -> String { - let mut desc = driver.states[self.start_states[0]].name.to_string(); - for op in &self.through { - desc.push('_'); - desc.push_str(&driver.ops[*op].name); + let mut desc = self + .start_states + .iter() + .map(|&state| &driver.states[state].name) + .join("_"); + if !self.through.is_empty() { + desc.push_str("_through"); + for op in &self.through { + desc.push('_'); + desc.push_str(&driver.ops[*op].name); + } } - desc.push('_'); - desc.push_str(&driver.states[self.end_states[0]].name); + desc.push_str("_to_"); + desc.push_str( + &self + .end_states + .iter() + .map(|&state| &driver.states[state].name) + .join("_"), + ); desc } @@ -266,3 +287,33 @@ fn frontend_tests() { request(&driver, &[frontend], &["calyx"], &[]).test(&driver); } } + +#[test] +fn simple_defops() { + let driver = driver_from_path("defop"); + request(&driver, &["state0"], &["state1"], &[]).test(&driver); + request_with_planner( + &driver, + &["state0", "state1"], + &["state2"], + &[], + EnumeratePlanner {}, + ) + .test(&driver); + request_with_planner( + &driver, + &["state0"], + &["state2", "state1"], + &[], + EnumeratePlanner {}, + ) + .test(&driver); + request_with_planner( + &driver, + &["state0", "state1", "state2"], + &["state3", "state4"], + &[], + EnumeratePlanner {}, + ) + .test(&driver); +} From 838bcb6856b19971cd06950e5a0c3bf2b9a609e9 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Mon, 29 Jul 2024 02:51:21 -0400 Subject: [PATCH 24/30] add basic tests for config and config_or --- fud2/tests/scripts/defop/test.rhai | 12 +++++++++ ...sts__test@state0_through_t4_to_state1.snap | 17 ++++++++++++ ...sts__test@state0_through_t5_to_state1.snap | 17 ++++++++++++ .../tests__test@state0_to_state1-2.snap | 16 +++++++++++ fud2/tests/tests.rs | 27 ++++++++++++++++--- 5 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 fud2/tests/snapshots/tests__test@state0_through_t4_to_state1.snap create mode 100644 fud2/tests/snapshots/tests__test@state0_through_t5_to_state1.snap create mode 100644 fud2/tests/snapshots/tests__test@state0_to_state1-2.snap diff --git a/fud2/tests/scripts/defop/test.rhai b/fud2/tests/scripts/defop/test.rhai index 5e8a9dddb8..676a403d8d 100644 --- a/fud2/tests/scripts/defop/test.rhai +++ b/fud2/tests/scripts/defop/test.rhai @@ -24,3 +24,15 @@ defop t3(s0: state0, s1: state1, s2: state2) >> s3: state3, s4: state4 { shell(`echo inputs ${s0} ${s1} ${s2}`); shell(`echo outputs ${s3} ${s4}`); } + +// testing config works +defop t4(s0: state0) >> s1: state1 { + let c0 = config("c0"); + shell(`echo ${c0}`); +} + +// testing config_or works +defop t5(s0: state0) >> s1: state1 { + let c0 = config_or("this-config-better-not-exist", "gholdengo"); + shell(`echo ${c0}`); +} diff --git a/fud2/tests/snapshots/tests__test@state0_through_t4_to_state1.snap b/fud2/tests/snapshots/tests__test@state0_through_t4_to_state1.snap new file mode 100644 index 0000000000..2541b82a8a --- /dev/null +++ b/fud2/tests/snapshots/tests__test@state0_through_t4_to_state1.snap @@ -0,0 +1,17 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: state0 -> state1 through t4" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +rule t4 + command = echo ${c0} +c0 = v1 +build _to_stdout_state1: t4 _from_stdin_state0 + i0 = _from_stdin_state0 + o0 = _to_stdout_state1 + + +default _to_stdout_state1 diff --git a/fud2/tests/snapshots/tests__test@state0_through_t5_to_state1.snap b/fud2/tests/snapshots/tests__test@state0_through_t5_to_state1.snap new file mode 100644 index 0000000000..b50dc74348 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@state0_through_t5_to_state1.snap @@ -0,0 +1,17 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: state0 -> state1 through t5" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +rule t5 + command = echo ${this-config-better-not-exist} +this-config-better-not-exist = gholdengo +build _to_stdout_state1: t5 _from_stdin_state0 + i0 = _from_stdin_state0 + o0 = _to_stdout_state1 + + +default _to_stdout_state1 diff --git a/fud2/tests/snapshots/tests__test@state0_to_state1-2.snap b/fud2/tests/snapshots/tests__test@state0_to_state1-2.snap new file mode 100644 index 0000000000..b2cd496a36 --- /dev/null +++ b/fud2/tests/snapshots/tests__test@state0_to_state1-2.snap @@ -0,0 +1,16 @@ +--- +source: fud2/tests/tests.rs +description: "emit request: state0 -> state1" +--- +build-tool = fud2 +rule get-rsrc + command = $build-tool get-rsrc $out + +rule t0 + command = echo uwu +build _to_stdout_state1: t0 _from_stdin_state0 + i0 = _from_stdin_state0 + o0 = _to_stdout_state1 + + +default _to_stdout_state1 diff --git a/fud2/tests/tests.rs b/fud2/tests/tests.rs index 529bc94c96..9a17621e72 100644 --- a/fud2/tests/tests.rs +++ b/fud2/tests/tests.rs @@ -1,3 +1,4 @@ +use figment::providers::Format as _; use fud_core::{ config::default_config, exec::{ @@ -23,9 +24,11 @@ fn test_driver() -> Driver { bld.load_plugins(&config).build() } -fn driver_from_path(path: &str) -> Driver { +fn driver_from_path_with_config( + path: &str, + config: figment::Figment, +) -> Driver { let mut bld = DriverBuilder::new("fud2-plugins"); - let config = figment::Figment::new(); let path = format!( "{}/{}", manifest_dir_macros::directory_path!("tests/scripts"), @@ -35,6 +38,10 @@ fn driver_from_path(path: &str) -> Driver { bld.load_plugins(&config).build() } +fn driver_from_path(path: &str) -> Driver { + driver_from_path_with_config(path, figment::Figment::new()) +} + trait InstaTest: Sized { /// Get a human-readable description of Self fn desc(&self, driver: &Driver) -> String; @@ -90,7 +97,8 @@ impl InstaTest for Plan { .merge(("xilinx.vivado", "/test/xilinx/vivado")) .merge(("xilinx.vitis", "/test/xilinx/vitis")) .merge(("xilinx.xrt", "/test/xilinx/xrt")) - .merge(("dahlia", "/test/bin/dahlia")); + .merge(("dahlia", "/test/bin/dahlia")) + .merge(("c0", "v1")); let run = Run::with_config(driver, self, config); let mut buf = vec![]; run.emit(&mut buf).unwrap(); @@ -317,3 +325,16 @@ fn simple_defops() { ) .test(&driver); } + +#[test] +fn config() { + let config = figment::Figment::from({ + let source = r#" + c0 = "v0" + "#; + figment::providers::Toml::string(source) + }); + let driver = driver_from_path_with_config("defop", config); + request(&driver, &["state0"], &["state1"], &["t4"]).test(&driver); + request(&driver, &["state0"], &["state1"], &["t5"]).test(&driver); +} From 6b0303008e1fd391c5b9ac4228603de6af34057d Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Fri, 2 Aug 2024 08:54:21 -0400 Subject: [PATCH 25/30] improve name and doc of struct for emiting ops --- fud2/fud-core/src/run.rs | 7 ++++--- fud2/fud-core/src/script/plugin.rs | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index 4416aef1af..2d7231773f 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -96,8 +96,9 @@ pub enum ConfigVar { VarOr(String, String), } -/// The data required to emit a single op. -pub struct OpEmitData { +/// The data required to emit a single, simple op whose body is composed of an ordered set of +/// commands. +pub struct OrderedCommandOp { /// The name of the op. pub op_name: String, @@ -120,7 +121,7 @@ pub fn io_file_var_name(index: usize, input: bool) -> String { } } -impl EmitBuild for OpEmitData { +impl EmitBuild for OrderedCommandOp { fn build( &self, emitter: &mut StreamEmitter, diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 345ea56917..a8a1dcaa61 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -190,7 +190,7 @@ impl ScriptContext { let cmds = cmds.clone(); let op_name = name.clone(); let config_vars = config_vars.clone(); - let op_emitter = crate::run::OpEmitData { + let op_emitter = crate::run::OrderedCommandOp { op_name, cmds, config_vars, From 2e4e92df91dcfc22b1cdef43217b24212051f00d Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Fri, 2 Aug 2024 09:27:22 -0400 Subject: [PATCH 26/30] being wishy-washy on names --- fud2/fud-core/src/run.rs | 4 ++-- fud2/fud-core/src/script/plugin.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index 2d7231773f..e6050e0cd0 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -98,7 +98,7 @@ pub enum ConfigVar { /// The data required to emit a single, simple op whose body is composed of an ordered set of /// commands. -pub struct OrderedCommandOp { +pub struct RulesOp { /// The name of the op. pub op_name: String, @@ -121,7 +121,7 @@ pub fn io_file_var_name(index: usize, input: bool) -> String { } } -impl EmitBuild for OrderedCommandOp { +impl EmitBuild for RulesOp { fn build( &self, emitter: &mut StreamEmitter, diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index a8a1dcaa61..9bdb7b7643 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -190,7 +190,7 @@ impl ScriptContext { let cmds = cmds.clone(); let op_name = name.clone(); let config_vars = config_vars.clone(); - let op_emitter = crate::run::OrderedCommandOp { + let op_emitter = crate::run::RulesOp { op_name, cmds, config_vars, From be95dcfb9fe998212ae5d55afa26b8600edecbfd Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Fri, 2 Aug 2024 09:42:58 -0400 Subject: [PATCH 27/30] name clarification and improved docs --- fud2/fud-core/src/run.rs | 23 ++++++++++++----------- fud2/fud-core/src/script/plugin.rs | 6 +++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index e6050e0cd0..86456b5781 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -91,20 +91,21 @@ impl EmitBuild for EmitBuildFn { #[derive(Debug, Clone)] pub enum ConfigVar { /// The key for the config variable. - Var(String), + Required(String), /// The key for the config variable followed by the value it should be if the key is not found. - VarOr(String, String), + Optional(String, String), } /// The data required to emit a single, simple op whose body is composed of an ordered set of /// commands. pub struct RulesOp { - /// The name of the op. - pub op_name: String, + /// The name of the rule generated. + pub rule_name: String, - /// The commands run whenever the op run. Each of these must be run in order in a context where - /// the variables in each cmd are supplied. In particular, this means that variables of the - /// form "$[i|o]" are in scope. + /// The shell commands emitted by to the generated rule. + /// Each of these must be run in order in a context where the variables in each cmd are + /// supplied. In particular, this means that variables of the form "$[i|o]" are in + /// scope. pub cmds: Vec, /// Variables to be emitted @@ -130,7 +131,7 @@ impl EmitBuild for RulesOp { ) -> EmitResult { // Write the Ninja file. let cmd = self.cmds.join(" && "); - emitter.rule(&self.op_name, &cmd)?; + emitter.rule(&self.rule_name, &cmd)?; let in_vars = inputs .iter() .enumerate() @@ -143,14 +144,14 @@ impl EmitBuild for RulesOp { for var in &self.config_vars { match var { - ConfigVar::Var(k) => emitter.config_var(k, k)?, - ConfigVar::VarOr(k, d) => emitter.config_var_or(k, k, d)?, + ConfigVar::Required(k) => emitter.config_var(k, k)?, + ConfigVar::Optional(k, d) => emitter.config_var_or(k, k, d)?, } } emitter.build_cmd_with_vars( outputs, - &self.op_name, + &self.rule_name, inputs, &[], &vars, diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 9bdb7b7643..632e24db80 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -191,7 +191,7 @@ impl ScriptContext { let op_name = name.clone(); let config_vars = config_vars.clone(); let op_emitter = crate::run::RulesOp { - op_name, + rule_name: op_name, cmds, config_vars, }; @@ -470,7 +470,7 @@ impl ScriptRunner { move |ctx: rhai::NativeCallContext, key: &str| -> RhaiResult<_> { sctx.add_config_var( ctx.position(), - crate::run::ConfigVar::Var(key.to_string()), + crate::run::ConfigVar::Required(key.to_string()), )?; Ok(format!("${{{}}}", key)) }, @@ -488,7 +488,7 @@ impl ScriptRunner { -> RhaiResult<_> { sctx.add_config_var( ctx.position(), - crate::run::ConfigVar::VarOr( + crate::run::ConfigVar::Optional( key.to_string(), default.to_string(), ), From b59bc02ab21382efd19ed0f096f2f2d2cc6e9a6c Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Fri, 2 Aug 2024 09:46:10 -0400 Subject: [PATCH 28/30] improve docs for build_cmd_with_vars --- fud2/fud-core/src/run.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index 86456b5781..cee491ac89 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -584,6 +584,9 @@ impl Emitter { } /// Emit a Ninja build command with variable list. + /// + /// Here `variables` is an association list, the first element of each tuple a key and the + /// second a value. pub fn build_cmd_with_vars( &mut self, targets: &[&str], From 4a24316d35716ef67ffdd2d08a3990fda3e8bc78 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Fri, 2 Aug 2024 10:44:48 -0400 Subject: [PATCH 29/30] more name bikesheadding --- fud2/fud-core/src/run.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index cee491ac89..b58f54f79b 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -149,7 +149,7 @@ impl EmitBuild for RulesOp { } } - emitter.build_cmd_with_vars( + emitter.build_cmd_with_bindings( outputs, &self.rule_name, inputs, @@ -587,7 +587,7 @@ impl Emitter { /// /// Here `variables` is an association list, the first element of each tuple a key and the /// second a value. - pub fn build_cmd_with_vars( + pub fn build_cmd_with_bindings( &mut self, targets: &[&str], rule: &str, From 5111ad6e5ec54edc39a65483c1dd37691f4f4c23 Mon Sep 17 00:00:00 2001 From: jeremy ku-benjet Date: Fri, 2 Aug 2024 11:07:42 -0400 Subject: [PATCH 30/30] more improvements to build_cmd_with_args --- fud2/fud-core/src/run.rs | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index b58f54f79b..c0978b9fa8 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -149,7 +149,7 @@ impl EmitBuild for RulesOp { } } - emitter.build_cmd_with_bindings( + emitter.build_cmd_with_args( outputs, &self.rule_name, inputs, @@ -587,7 +587,7 @@ impl Emitter { /// /// Here `variables` is an association list, the first element of each tuple a key and the /// second a value. - pub fn build_cmd_with_bindings( + pub fn build_cmd_with_args( &mut self, targets: &[&str], rule: &str, @@ -595,23 +595,9 @@ impl Emitter { implicit_deps: &[&str], variables: &[(String, &str)], ) -> std::io::Result<()> { - write!(self.out, "build")?; - for target in targets { - write!(self.out, " {}", target)?; - } - write!(self.out, ": {}", rule)?; - for dep in deps { - write!(self.out, " {}", dep)?; - } - if !implicit_deps.is_empty() { - write!(self.out, " |")?; - for dep in implicit_deps { - write!(self.out, " {}", dep)?; - } - } - writeln!(self.out)?; + self.build_cmd(targets, rule, deps, implicit_deps)?; for (key, value) in variables { - writeln!(self.out, " {} = {}", key, value)?; + self.arg(key, value)?; } writeln!(self.out)?; Ok(())