From d8c86b4bb7b057061dea98e7607ecc284689a8b2 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 19 Jul 2024 06:29:42 -0700 Subject: [PATCH 01/18] bump v to 0.6.9 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d089ad2..104b86c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1624,7 +1624,7 @@ dependencies = [ [[package]] name = "kit" -version = "0.6.8" +version = "0.6.9" dependencies = [ "anyhow", "base64 0.21.7", diff --git a/Cargo.toml b/Cargo.toml index 9a04d371..9d985ec6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kit" -version = "0.6.8" +version = "0.6.9" edition = "2021" [build-dependencies] From b34391d523944b86466830b4058ea2baa782eb4e Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 19 Jul 2024 06:30:29 -0700 Subject: [PATCH 02/18] setup: relax npm major v check to `>=` from `==` --- src/setup/mod.rs | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/setup/mod.rs b/src/setup/mod.rs index 184f2078..e61f1f1e 100644 --- a/src/setup/mod.rs +++ b/src/setup/mod.rs @@ -13,7 +13,7 @@ use crate::build::run_command; const FETCH_NVM_VERSION: &str = "v0.39.7"; const REQUIRED_NODE_MAJOR: u32 = 20; const MINIMUM_NODE_MINOR: u32 = 0; -const REQUIRED_NPM_MAJOR: u32 = 9; +const MINIMUM_NPM_MAJOR: u32 = 9; const MINIMUM_NPM_MINOR: u32 = 0; pub const REQUIRED_PY_MAJOR: u32 = 3; pub const MINIMUM_PY_MINOR: u32 = 10; @@ -39,7 +39,7 @@ impl std::fmt::Display for Dependency { Dependency::Anvil => write!(f, "anvil"), Dependency::Forge => write!(f, "forge"), Dependency::Nvm => write!(f, "nvm {}", FETCH_NVM_VERSION), - Dependency::Npm => write!(f, "npm {}.{}", REQUIRED_NPM_MAJOR, MINIMUM_NPM_MINOR), + Dependency::Npm => write!(f, "npm {}.{}", MINIMUM_NPM_MAJOR, MINIMUM_NPM_MINOR), Dependency::Node => write!(f, "node {}.{}", REQUIRED_NODE_MAJOR, MINIMUM_NODE_MINOR), Dependency::Rust => write!(f, "rust"), Dependency::RustNightly => write!(f, "rust nightly"), @@ -129,18 +129,7 @@ fn is_npm_version_correct(node_version: String, required_version: (u32, u32)) -> .collect::>(); let version = version.last().unwrap_or_else(|| &""); Ok(parse_version(version) - .and_then(|v| Some(compare_versions(v, required_version))) - .unwrap_or(false)) -} - -#[instrument(level = "trace", skip_all)] -fn is_version_correct(cmd: &str, required_version: (u32, u32)) -> Result { - let output = Command::new(cmd).arg("--version").output()?.stdout; - - let version = String::from_utf8_lossy(&output); - - Ok(parse_version(version.trim()) - .and_then(|v| Some(compare_versions(v, required_version))) + .and_then(|v| Some(compare_versions_min_major(v, required_version))) .unwrap_or(false)) } @@ -229,8 +218,8 @@ fn call_cargo(arg: &str, verbose: bool) -> Result<()> { Ok(()) } -fn compare_versions(installed_version: (u32, u32), required_version: (u32, u32)) -> bool { - installed_version.0 == required_version.0 && installed_version.1 >= required_version.1 +fn compare_versions_min_major(installed_version: (u32, u32), required_version: (u32, u32)) -> bool { + installed_version.0 >= required_version.0 && installed_version.1 >= required_version.1 } fn parse_version(version_str: &str) -> Option<(u32, u32)> { @@ -385,7 +374,7 @@ pub fn check_js_deps() -> Result> { None => missing_deps.extend_from_slice(&[Dependency::Node, Dependency::Npm]), Some(vn) => { if !is_command_installed("npm")? - || !is_npm_version_correct(vn, (REQUIRED_NPM_MAJOR, MINIMUM_NPM_MINOR))? + || !is_npm_version_correct(vn, (MINIMUM_NPM_MAJOR, MINIMUM_NPM_MINOR))? { missing_deps.push(Dependency::Npm); } From 888e7691f85c59a347dc2284d81300e2887e7ec5 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 19 Jul 2024 15:00:56 -0700 Subject: [PATCH 03/18] add more helpful error messaging for case when CWD DNE --- src/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index f211b2b5..db9893ea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -972,7 +972,9 @@ async fn main() -> Result<()> { color_eyre::config::HookBuilder::default() .display_env_section(false) .install()?; - let current_dir = env::current_dir()?.into_os_string(); + let current_dir = env::current_dir() + .with_suggestion(|| "Could not fetch CWD. Does CWD exist?")? + .into_os_string(); let mut app = make_app(¤t_dir).await?; let usage = app.render_usage(); From 3d0ea5db872798bef1ed9a2fc3dfbba95b288c8f Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 19 Jul 2024 21:31:12 -0700 Subject: [PATCH 04/18] build: add `--local-dependency` flag --- src/build/mod.rs | 153 ++++++++++++++++++++++++--------- src/build_start_package/mod.rs | 8 +- src/main.rs | 46 +++++++++- src/run_tests/mod.rs | 12 +++ 4 files changed, 175 insertions(+), 44 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index 4a610aa1..b1ff8972 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -202,7 +202,7 @@ fn extract_worlds_from_files(directory: &Path) -> Vec { worlds } -fn get_world_or_default(directory: &Path, default_world: String) -> String { +fn get_world_or_default(directory: &Path, default_world: &str) -> String { let worlds = extract_worlds_from_files(directory); if worlds.len() == 1 { return worlds[0].clone(); @@ -211,7 +211,7 @@ fn get_world_or_default(directory: &Path, default_world: String) -> String { "Found {} worlds in {directory:?}; defaulting to {default_world}", worlds.len() ); - default_world + default_world.to_string() } #[instrument(level = "trace", skip_all)] @@ -316,7 +316,7 @@ fn get_file_modified_time(file_path: &Path) -> Result { async fn compile_javascript_wasm_process( process_dir: &Path, valid_node: Option, - world: String, + world: &str, verbose: bool, ) -> Result<()> { info!( @@ -369,7 +369,7 @@ async fn compile_javascript_wasm_process( async fn compile_python_wasm_process( process_dir: &Path, python: &str, - world: String, + world: &str, verbose: bool, ) -> Result<()> { info!("Compiling Python Kinode process in {:?}...", process_dir); @@ -583,8 +583,10 @@ async fn compile_package_and_ui( skip_deps_check: bool, features: &str, url: Option, - default_world: Option, + default_world: Option<&str>, download_from: Option<&str>, + local_dependencies: Vec, + force: bool, verbose: bool, ) -> Result<()> { compile_and_copy_ui(package_dir, valid_node, verbose).await?; @@ -595,6 +597,8 @@ async fn compile_package_and_ui( url, default_world, download_from, + local_dependencies, + force, verbose, ) .await?; @@ -651,10 +655,10 @@ async fn compile_package_item( } else if is_py_process { let python = get_python_version(None, None)? .ok_or_else(|| eyre!("kit requires Python 3.10 or newer"))?; - compile_python_wasm_process(&path, &python, world, verbose).await?; + compile_python_wasm_process(&path, &python, &world, verbose).await?; } else if is_js_process { let valid_node = get_newest_valid_node_version(None, None)?; - compile_javascript_wasm_process(&path, valid_node, world, verbose).await?; + compile_javascript_wasm_process(&path, valid_node, &world, verbose).await?; } } Ok(()) @@ -665,19 +669,71 @@ async fn fetch_dependencies( dependencies: &Vec, apis: &mut HashMap>, wasm_paths: &mut HashSet, - url: String, + url: Option, download_from: Option<&str>, + local_dependencies: Vec, + features: &str, + default_world: Option<&str>, + force: bool, + verbose: bool, ) -> Result<()> { + for local_dependency in &local_dependencies { + // build dependency + Box::pin(execute( + local_dependency, + true, + false, + true, + features, + url.clone(), + download_from, + default_world, + vec![], // TODO: what about deps-of-deps? + verbose, + force, + )).await?; + for entry in local_dependency.join("api").read_dir()? { + let entry = entry?; + let path = entry.path(); + let maybe_ext = path.extension().and_then(|s| s.to_str()); + if Some("wit") == maybe_ext { + let file_name = path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or_default(); + let wit_contents = fs::read(&path)?; + apis.insert(file_name.into(), wit_contents); + } + } + for entry in local_dependency.join("target").join("api").read_dir()? { + let entry = entry?; + let path = entry.path(); + let maybe_ext = path.extension().and_then(|s| s.to_str()); + if Some("wasm") == maybe_ext { + wasm_paths.insert(path); + } + } + } + let Some(ref url) = url else { + return Ok(()); + }; + let local_dependencies: HashSet<&str> = local_dependencies + .iter() + .map(|p| p.file_name().and_then(|f| f.to_str()).unwrap()) + .collect(); for dependency in dependencies { - if dependency.parse::().is_err() { + let Ok(dep) = dependency.parse::() else { return Err(eyre!( "Dependencies must be PackageIds (e.g. `package:publisher.os`); given {dependency}.", )); }; + if local_dependencies.contains(dep.package()) { + continue; + } let Some(zip_dir) = view_api::execute( None, Some(dependency), - &url, + url, download_from, false, ).await? else { @@ -823,8 +879,10 @@ async fn compile_package( skip_deps_check: bool, features: &str, url: Option, - default_world: Option, + default_world: Option<&str>, download_from: Option<&str>, + local_dependencies: Vec, + force: bool, verbose: bool, ) -> Result<()> { let metadata = read_metadata(package_dir)?; @@ -879,16 +937,21 @@ async fn compile_package( if dependencies.is_empty() { continue; } - let Some(ref url) = url else { - // TODO: can we use kit-cached deps? - return Err(eyre!("Need a node to be able to fetch dependencies")); - }; + //let Some(ref url) = url else { + // // TODO: can we use kit-cached deps? + // return Err(eyre!("Need a node to be able to fetch dependencies")); + //}; fetch_dependencies( dependencies, &mut apis, &mut wasm_paths, url.clone(), download_from, + local_dependencies.clone(), + features, + default_world, + force, + verbose, ).await?; } } @@ -896,9 +959,9 @@ async fn compile_package( } let wit_world = default_world.unwrap_or_else(|| match metadata.properties.wit_version { - None => DEFAULT_WORLD_0_7_0.to_string(), - Some(0) | _ => DEFAULT_WORLD_0_8_0.to_string(), - }); + None => DEFAULT_WORLD_0_7_0, + Some(0) | _ => DEFAULT_WORLD_0_8_0, + }).to_string(); let mut tasks = tokio::task::JoinSet::new(); let features = features.to_string(); @@ -971,7 +1034,9 @@ pub async fn execute( features: &str, url: Option, download_from: Option<&str>, - default_world: Option, + default_world: Option<&str>, + local_dependencies: Vec, + force: bool, verbose: bool, ) -> Result<()> { if !package_dir.join("pkg").exists() { @@ -986,26 +1051,28 @@ pub async fn execute( .with_suggestion(|| "Please re-run targeting a package.")); } let build_with_features_path = package_dir.join("target").join("build_with_features.txt"); - let old_features = fs::read_to_string(&build_with_features_path).ok(); - if old_features == Some(features.to_string()) - && package_dir.join("Cargo.lock").exists() - && package_dir.join("pkg").exists() - && package_dir.join("pkg").join("api.zip").exists() - && file_with_extension_exists(&package_dir.join("pkg"), "wasm") - { - let (source_time, build_time) = get_most_recent_modified_time( - package_dir, - &HashSet::from(["Cargo.lock", "api.zip"]), - &HashSet::from(["wasm"]), - &HashSet::from(["target"]), - )?; - if let Some(source_time) = source_time { - if let Some(build_time) = build_time { - if build_time.duration_since(source_time).is_ok() { - // build_time - source_time >= 0 - // -> current build is up-to-date: don't rebuild - info!("Build up-to-date."); - return Ok(()); + if !force { + let old_features = fs::read_to_string(&build_with_features_path).ok(); + if old_features == Some(features.to_string()) + && package_dir.join("Cargo.lock").exists() + && package_dir.join("pkg").exists() + && package_dir.join("pkg").join("api.zip").exists() + && file_with_extension_exists(&package_dir.join("pkg"), "wasm") + { + let (source_time, build_time) = get_most_recent_modified_time( + package_dir, + &HashSet::from(["Cargo.lock", "api.zip"]), + &HashSet::from(["wasm"]), + &HashSet::from(["target"]), + )?; + if let Some(source_time) = source_time { + if let Some(build_time) = build_time { + if build_time.duration_since(source_time).is_ok() { + // build_time - source_time >= 0 + // -> current build is up-to-date: don't rebuild + info!("Build up-to-date."); + return Ok(()); + } } } } @@ -1023,8 +1090,10 @@ pub async fn execute( skip_deps_check, features, url, - default_world, + default_world.clone(), download_from, + local_dependencies, + force, verbose, ) .await @@ -1038,6 +1107,8 @@ pub async fn execute( url, default_world, download_from, + local_dependencies, + force, verbose, ) .await; @@ -1058,6 +1129,8 @@ pub async fn execute( url, default_world, download_from, + local_dependencies, + force, verbose, ) .await diff --git a/src/build_start_package/mod.rs b/src/build_start_package/mod.rs index 4f9ab63e..4b2d2a6d 100644 --- a/src/build_start_package/mod.rs +++ b/src/build_start_package/mod.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use color_eyre::Result; use tracing::instrument; @@ -15,7 +15,9 @@ pub async fn execute( skip_deps_check: bool, features: &str, download_from: Option<&str>, - default_world: Option, + default_world: Option<&str>, + local_dependencies: Vec, + force: bool, verbose: bool, ) -> Result<()> { build::execute( @@ -27,6 +29,8 @@ pub async fn execute( Some(url.into()), download_from, default_world, + local_dependencies, + force, verbose, ) .await?; diff --git a/src/main.rs b/src/main.rs index db9893ea..2ce0fb8b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -193,6 +193,12 @@ async fn execute( .get_one::("NODE") .and_then(|s: &String| Some(s.as_str())); let default_world = matches.get_one::("WORLD"); + let local_dependencies: Vec = matches + .get_many::("PATH") + .unwrap_or_default() + .map(|s| PathBuf::from(s)) + .collect(); + let force = matches.get_one::("FORCE").unwrap(); let verbose = matches.get_one::("VERBOSE").unwrap(); build::execute( @@ -203,7 +209,9 @@ async fn execute( &features, url, download_from, - default_world.cloned(), + default_world.map(|w| w.as_str()), + local_dependencies, + *force, *verbose, ) .await @@ -229,6 +237,12 @@ async fn execute( .get_one::("NODE") .and_then(|s: &String| Some(s.as_str())); let default_world = matches.get_one::("WORLD"); + let local_dependencies: Vec = matches + .get_many::("PATH") + .unwrap_or_default() + .map(|s| PathBuf::from(s)) + .collect(); + let force = matches.get_one::("FORCE").unwrap(); let verbose = matches.get_one::("VERBOSE").unwrap(); build_start_package::execute( @@ -239,7 +253,9 @@ async fn execute( *skip_deps_check, &features, download_from, - default_world.cloned(), + default_world.map(|w| w.as_str()), + local_dependencies, + *force, *verbose, ) .await @@ -617,6 +633,19 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .long("world") .help("Fallback WIT world name") ) + .arg(Arg::new("PATH") + .action(ArgAction::Append) + .short('l') + .long("local-dependency") + .help("Path to local dependency package (can specify multiple times)") + ) + .arg(Arg::new("FORCE") + .action(ArgAction::SetTrue) + .short('f') + .long("force") + .help("Force a rebuild") + .required(false) + ) .arg(Arg::new("VERBOSE") .action(ArgAction::SetTrue) .short('v') @@ -655,6 +684,12 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .help("Fallback WIT world name") .required(false) ) + .arg(Arg::new("PATH") + .action(ArgAction::Append) + .short('l') + .long("local-dependency") + .help("Path to local dependency package (can specify multiple times)") + ) .arg(Arg::new("NO_UI") .action(ArgAction::SetTrue) .long("no-ui") @@ -680,6 +715,13 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .help("Pass these comma-delimited feature flags to Rust cargo builds") .required(false) ) + .arg(Arg::new("FORCE") + .action(ArgAction::SetTrue) + .short('f') + .long("force") + .help("Force a rebuild") + .required(false) + ) .arg(Arg::new("VERBOSE") .action(ArgAction::SetTrue) .short('v') diff --git a/src/run_tests/mod.rs b/src/run_tests/mod.rs index 4ead8efd..4188f614 100644 --- a/src/run_tests/mod.rs +++ b/src/run_tests/mod.rs @@ -271,6 +271,12 @@ async fn build_packages( persist_home: &bool, runtime_path: &Path, ) -> Result<(Vec, Vec)> { + let dependency_package_paths: Vec = test + .dependency_package_paths + .iter() + .cloned() + .map(|p| test_dir_path.join(p).canonicalize().unwrap()) + .collect(); let setup_packages: Vec = test .setup_packages .iter() @@ -344,6 +350,8 @@ async fn build_packages( Some(url.clone()), None, None, + dependency_package_paths.clone(), + false, false, ).await?; start_package::execute(&path, &url).await?; @@ -359,6 +367,8 @@ async fn build_packages( Some(url.clone()), None, None, + dependency_package_paths.clone(), + false, false, ).await?; } @@ -372,6 +382,8 @@ async fn build_packages( Some(url.clone()), None, None, + dependency_package_paths.clone(), + false, false, ).await?; } From 9a0ceb212d5ee0846888998147a571ff1b299d8c Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Tue, 23 Jul 2024 20:29:16 -0700 Subject: [PATCH 05/18] build: allow depending on apis exported by self --- src/build/mod.rs | 192 +++++++++++++++++++++------------ src/build_start_package/mod.rs | 1 + src/main.rs | 1 + src/run_tests/mod.rs | 3 + 4 files changed, 129 insertions(+), 68 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index b1ff8972..9d091621 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -588,6 +588,7 @@ async fn compile_package_and_ui( local_dependencies: Vec, force: bool, verbose: bool, + ignore_deps: bool, ) -> Result<()> { compile_and_copy_ui(package_dir, valid_node, verbose).await?; compile_package( @@ -600,6 +601,7 @@ async fn compile_package_and_ui( local_dependencies, force, verbose, + ignore_deps, ) .await?; Ok(()) @@ -664,19 +666,65 @@ async fn compile_package_item( Ok(()) } +#[instrument(level = "trace", skip_all)] +fn fetch_local_built_dependency( + apis: &mut HashMap>, + wasm_paths: &mut HashSet, + local_dependency: &Path, +) -> Result<()> { + for entry in local_dependency.join("api").read_dir()? { + let entry = entry?; + let path = entry.path(); + let maybe_ext = path.extension().and_then(|s| s.to_str()); + if Some("wit") == maybe_ext { + let file_name = path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or_default(); + let wit_contents = fs::read(&path)?; + apis.insert(file_name.into(), wit_contents); + } + } + for entry in local_dependency.join("target").join("api").read_dir()? { + let entry = entry?; + let path = entry.path(); + let maybe_ext = path.extension().and_then(|s| s.to_str()); + if Some("wasm") == maybe_ext { + wasm_paths.insert(path); + } + } + Ok(()) +} + #[instrument(level = "trace", skip_all)] async fn fetch_dependencies( + package_dir: &Path, dependencies: &Vec, apis: &mut HashMap>, wasm_paths: &mut HashSet, url: Option, download_from: Option<&str>, - local_dependencies: Vec, + mut local_dependencies: Vec, features: &str, default_world: Option<&str>, force: bool, verbose: bool, ) -> Result<()> { + Box::pin(execute( + package_dir, + true, + false, + true, + features, + url.clone(), + download_from, + default_world, + vec![], // TODO: what about deps-of-deps? + force, + verbose, + true, + )).await?; + fetch_local_built_dependency(apis, wasm_paths, package_dir)?; for local_dependency in &local_dependencies { // build dependency Box::pin(execute( @@ -689,34 +737,16 @@ async fn fetch_dependencies( download_from, default_world, vec![], // TODO: what about deps-of-deps? - verbose, force, + verbose, + false, )).await?; - for entry in local_dependency.join("api").read_dir()? { - let entry = entry?; - let path = entry.path(); - let maybe_ext = path.extension().and_then(|s| s.to_str()); - if Some("wit") == maybe_ext { - let file_name = path - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or_default(); - let wit_contents = fs::read(&path)?; - apis.insert(file_name.into(), wit_contents); - } - } - for entry in local_dependency.join("target").join("api").read_dir()? { - let entry = entry?; - let path = entry.path(); - let maybe_ext = path.extension().and_then(|s| s.to_str()); - if Some("wasm") == maybe_ext { - wasm_paths.insert(path); - } - } + fetch_local_built_dependency(apis, wasm_paths, &local_dependency)?; } let Some(ref url) = url else { return Ok(()); }; + local_dependencies.push(package_dir.into()); let local_dependencies: HashSet<&str> = local_dependencies .iter() .map(|p| p.file_name().and_then(|f| f.to_str()).unwrap()) @@ -884,6 +914,7 @@ async fn compile_package( local_dependencies: Vec, force: bool, verbose: bool, + ignore_deps: bool, ) -> Result<()> { let metadata = read_metadata(package_dir)?; let mut checked_rust = false; @@ -891,6 +922,7 @@ async fn compile_package( let mut checked_js = false; let mut apis = HashMap::new(); let mut wasm_paths = HashSet::new(); + let mut dependencies = HashSet::new(); for entry in package_dir.read_dir()? { let entry = entry?; let path = entry.path(); @@ -933,31 +965,49 @@ async fn compile_package( } // fetch dependency apis: to be used in build - if let Some(ref dependencies) = metadata.properties.dependencies { - if dependencies.is_empty() { - continue; - } - //let Some(ref url) = url else { - // // TODO: can we use kit-cached deps? - // return Err(eyre!("Need a node to be able to fetch dependencies")); - //}; - fetch_dependencies( - dependencies, - &mut apis, - &mut wasm_paths, - url.clone(), - download_from, - local_dependencies.clone(), - features, - default_world, - force, - verbose, - ).await?; + if let Some(ref deps) = metadata.properties.dependencies { + dependencies.extend(deps); + //if dependencies.is_empty() { + // continue; + //} + ////let Some(ref url) = url else { + //// // TODO: can we use kit-cached deps? + //// return Err(eyre!("Need a node to be able to fetch dependencies")); + ////}; + //fetch_dependencies( + // package_dir, + // dependencies, + // &mut apis, + // &mut wasm_paths, + // url.clone(), + // download_from, + // local_dependencies.clone(), + // features, + // default_world, + // force, + // verbose, + //).await?; } } } } + if !ignore_deps && !dependencies.is_empty() { + fetch_dependencies( + package_dir, + &dependencies.iter().map(|s| s.to_string()).collect(), + &mut apis, + &mut wasm_paths, + url.clone(), + download_from, + local_dependencies.clone(), + features, + default_world, + force, + verbose, + ).await?; + } + let wit_world = default_world.unwrap_or_else(|| match metadata.properties.wit_version { None => DEFAULT_WORLD_0_7_0, Some(0) | _ => DEFAULT_WORLD_0_8_0, @@ -987,31 +1037,33 @@ async fn compile_package( copy_dir(&api_dir, &target_api_dir)?; } - // find non-standard imports/exports -> compositions - let (importers, exporters) = find_non_standard(package_dir, wasm_paths)?; - - // compose - for (import, import_paths) in importers { - let Some(export_path) = exporters.get(&import) else { - return Err(eyre!( - "Processes {import_paths:?} required export {import} not found in `pkg/`.", - )); - }; - let export_path = export_path.to_str().unwrap(); - for import_path in import_paths { - let import_path_str = import_path.to_str().unwrap(); - run_command( - Command::new("wasm-tools") - .args([ - "compose", - import_path_str, - "-d", - export_path, - "-o", - import_path_str, - ]), - false, - )?; + if !ignore_deps { + // find non-standard imports/exports -> compositions + let (importers, exporters) = find_non_standard(package_dir, wasm_paths)?; + + // compose + for (import, import_paths) in importers { + let Some(export_path) = exporters.get(&import) else { + return Err(eyre!( + "Processes {import_paths:?} required export {import} not found in `pkg/`.", + )); + }; + let export_path = export_path.to_str().unwrap(); + for import_path in import_paths { + let import_path_str = import_path.to_str().unwrap(); + run_command( + Command::new("wasm-tools") + .args([ + "compose", + import_path_str, + "-d", + export_path, + "-o", + import_path_str, + ]), + false, + )?; + } } } @@ -1038,6 +1090,7 @@ pub async fn execute( local_dependencies: Vec, force: bool, verbose: bool, + ignore_deps: bool, // for internal use; may cause problems when adding recursive deps ) -> Result<()> { if !package_dir.join("pkg").exists() { if Some(".DS_Store") == package_dir.file_name().and_then(|s| s.to_str()) { @@ -1095,6 +1148,7 @@ pub async fn execute( local_dependencies, force, verbose, + ignore_deps, ) .await } @@ -1110,6 +1164,7 @@ pub async fn execute( local_dependencies, force, verbose, + ignore_deps, ) .await; } @@ -1132,6 +1187,7 @@ pub async fn execute( local_dependencies, force, verbose, + ignore_deps, ) .await } diff --git a/src/build_start_package/mod.rs b/src/build_start_package/mod.rs index 4b2d2a6d..a0fc50dd 100644 --- a/src/build_start_package/mod.rs +++ b/src/build_start_package/mod.rs @@ -32,6 +32,7 @@ pub async fn execute( local_dependencies, force, verbose, + false, ) .await?; start_package::execute(package_dir, url).await?; diff --git a/src/main.rs b/src/main.rs index 2ce0fb8b..3934ab2a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -213,6 +213,7 @@ async fn execute( local_dependencies, *force, *verbose, + false, ) .await } diff --git a/src/run_tests/mod.rs b/src/run_tests/mod.rs index 4188f614..0d8b5f5e 100644 --- a/src/run_tests/mod.rs +++ b/src/run_tests/mod.rs @@ -353,6 +353,7 @@ async fn build_packages( dependency_package_paths.clone(), false, false, + false, ).await?; start_package::execute(&path, &url).await?; } @@ -370,6 +371,7 @@ async fn build_packages( dependency_package_paths.clone(), false, false, + false, ).await?; } for test_package_path in &test_package_paths { @@ -385,6 +387,7 @@ async fn build_packages( dependency_package_paths.clone(), false, false, + false, ).await?; } From e4f55c78bf4b9a44ad9ed5560df6c20848152b83 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Tue, 23 Jul 2024 20:29:57 -0700 Subject: [PATCH 06/18] new: change file_transfer to use api call to spawn worker --- .../rust/no-ui/file_transfer/Cargo.toml_ | 1 + .../api/{package_name}:{publisher}-v0.wit | 24 ++- .../no-ui/file_transfer/download/src/lib.rs | 8 +- .../rust/no-ui/file_transfer/metadata.json | 4 +- .../no-ui/file_transfer/worker/src/lib.rs | 177 +++++++++++------- .../file_transfer/{package_name}/src/lib.rs | 112 +++++------ .../{package_name}_api/Cargo.toml_ | 18 ++ .../{package_name}_api/src/lib.rs | 75 ++++++++ 8 files changed, 271 insertions(+), 148 deletions(-) create mode 100644 src/new/templates/rust/no-ui/file_transfer/{package_name}_api/Cargo.toml_ create mode 100644 src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs diff --git a/src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ b/src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ index 02d3f899..b575d19d 100644 --- a/src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ +++ b/src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ @@ -2,6 +2,7 @@ resolver = "2" members = [ "{package_name}", + "{package_name}_api", "worker", "download", "list_files" diff --git a/src/new/templates/rust/no-ui/file_transfer/api/{package_name}:{publisher}-v0.wit b/src/new/templates/rust/no-ui/file_transfer/api/{package_name}:{publisher}-v0.wit index 3ae9a97c..8314f747 100644 --- a/src/new/templates/rust/no-ui/file_transfer/api/{package_name}:{publisher}-v0.wit +++ b/src/new/templates/rust/no-ui/file_transfer/api/{package_name}:{publisher}-v0.wit @@ -9,13 +9,11 @@ interface {package_name_kebab} { variant response { list-files(list), - download, - done, - started, + download(result<_, string>), + progress(result<_, string>), } variant worker-request { - initialize(initialize-request), chunk(chunk-request), size(u64), } @@ -23,6 +21,7 @@ interface {package_name_kebab} { record download-request { name: string, target: address, + is-requestor: bool, } record progress-request { @@ -35,16 +34,23 @@ interface {package_name_kebab} { size: u64, } - record initialize-request { - name: string, - target-worker: option
, - } - record chunk-request { name: string, offset: u64, length: u64, } + + start-download: func( + our: address, + source: address, + name: string, + target: address, + is-requestor: bool, + ) -> result<_, string>; +} + +world {package_name_kebab}-{publisher_dotted_kebab}-api-v0 { + export {package_name_kebab}; } world {package_name_kebab}-{publisher_dotted_kebab}-v0 { diff --git a/src/new/templates/rust/no-ui/file_transfer/download/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/download/src/lib.rs index 7f35787f..ee5b3d95 100644 --- a/src/new/templates/rust/no-ui/file_transfer/download/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/download/src/lib.rs @@ -54,10 +54,12 @@ fn init(our: Address) { .body(TransferRequest::Download(DownloadRequest { name: name.into(), target: target.clone().into(), + is_requestor: true, })) - .send() + .send_and_await_response(5) { - Ok(_) => {} - Err(e) => println!("download failed: {e:?}"), + Ok(Ok(_)) => {} + Ok(Err(e)) => println!("download failed: {e:?}"), + Err(e) => println!("download failed; SendError: {e:?}"), } } diff --git a/src/new/templates/rust/no-ui/file_transfer/metadata.json b/src/new/templates/rust/no-ui/file_transfer/metadata.json index 940b56a7..259d2a1e 100644 --- a/src/new/templates/rust/no-ui/file_transfer/metadata.json +++ b/src/new/templates/rust/no-ui/file_transfer/metadata.json @@ -11,7 +11,9 @@ "0.1.0": "" }, "wit_version": 0, - "dependencies": [] + "dependencies": [ + "{package_name}:{publisher}" + ] }, "external_url": "", "animation_url": "" diff --git a/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs index 7ffb32c1..fcf77c59 100644 --- a/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs @@ -1,5 +1,5 @@ use crate::kinode::process::standard::{ProcessId as WitProcessId}; -use crate::kinode::process::{package_name}::{Address as WitAddress, Request as TransferRequest, Response as TransferResponse, WorkerRequest, ProgressRequest, InitializeRequest, ChunkRequest}; +use crate::kinode::process::{package_name}::{Address as WitAddress, Request as TransferRequest, Response as TransferResponse, WorkerRequest, DownloadRequest, ProgressRequest, ChunkRequest}; use kinode_process_lib::{ await_message, call_init, get_blob, println, vfs::{open_dir, open_file, Directory, File, SeekFrom}, @@ -13,24 +13,13 @@ wit_bindgen::generate!({ additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); -impl From
for WitAddress { - fn from(address: Address) -> Self { - WitAddress { - node: address.node, - process: address.process.into(), - } - } +#[derive(Debug, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto)] +#[serde(untagged)] // untagged as a meta-type for all incoming responses +enum Req { + TransferRequest(TransferRequest), + WorkerRequest(WorkerRequest), } -impl From for WitProcessId { - fn from(process: ProcessId) -> Self { - WitProcessId { - process_name: process.process_name, - package_name: process.package_name, - publisher_node: process.publisher_node, - } - } -} impl From for Address { fn from(address: WitAddress) -> Self { Address { @@ -52,77 +41,89 @@ impl From for ProcessId { const CHUNK_SIZE: u64 = 1048576; // 1MB -fn handle_message( - our: &Address, - message: &Message, +fn handle_transfer_request( + request: &TransferRequest, file: &mut Option, files_dir: &Directory, - size: &mut Option, ) -> anyhow::Result { - if !message.is_request() { - return Err(anyhow::anyhow!("unexpected Response: {:?}", message)); - } - - match message.body().try_into()? { - WorkerRequest::Initialize(InitializeRequest { + match request { + TransferRequest::Download(DownloadRequest { name, - target_worker, + target, + is_requestor, }) => { - // initialize command from main process, - // sets up worker, matches on if it's a sender or receiver. - // target_worker = None, we are receiver, else sender. + Response::new() + .body(TransferResponse::Download(Ok(()))) + .send()?; // open/create empty file in both cases. let mut active_file = open_file(&format!("{}/{}", files_dir.path, &name), true, None)?; - match target_worker { - Some(target_worker) => { - let target_worker: Address = target_worker.into(); - // we have a target, chunk the data, and send it. - let size = active_file.metadata()?.len; - let num_chunks = (size as f64 / CHUNK_SIZE as f64).ceil() as u64; + if *is_requestor { + *file = Some(active_file); + Request::new() + .expects_response(5) + .body(TransferRequest::Download(DownloadRequest { + name: name.to_string(), + target: target.clone(), + is_requestor: false, + })) + .target::
(target.clone().into()) + .send()?; + } else { + // we are sender: chunk the data, and send it. + let size = active_file.metadata()?.len; + let num_chunks = (size as f64 / CHUNK_SIZE as f64).ceil() as u64; - // give the receiving worker a size request so it can track it's progress! - Request::new() - .body(WorkerRequest::Size(size)) - .target(target_worker.clone()) - .send()?; + // give receiving worker file size so it can track download progress + Request::new() + .body(WorkerRequest::Size(size)) + .target(target.clone()) + .send()?; - active_file.seek(SeekFrom::Start(0))?; + active_file.seek(SeekFrom::Start(0))?; - for i in 0..num_chunks { - let offset = i * CHUNK_SIZE; - let length = CHUNK_SIZE.min(size - offset); + for i in 0..num_chunks { + let offset = i * CHUNK_SIZE; + let length = CHUNK_SIZE.min(size - offset); - let mut buffer = vec![0; length as usize]; - active_file.read_at(&mut buffer)?; + let mut buffer = vec![0; length as usize]; + active_file.read_at(&mut buffer)?; - Request::new() - .body(WorkerRequest::Chunk(ChunkRequest { - name: name.clone(), - offset, - length, - })) - .target(target_worker.clone()) - .blob_bytes(buffer) - .send()?; - } - return Ok(true); - } - None => { - // waiting for response, store created empty file. - *file = Some(active_file); - Response::new().body(TransferResponse::Started).send()?; + Request::new() + .body(WorkerRequest::Chunk(ChunkRequest { + name: name.clone(), + offset, + length, + })) + .target(target.clone()) + .blob_bytes(buffer) + .send()?; } + return Ok(true); } } - // someone sending a chunk to us! + TransferRequest::ListFiles | TransferRequest::Progress(_) => { + return Err(anyhow::anyhow!("worker: unexpected TransferRequest: {request:?}")); + } + } + Ok(false) +} + +fn handle_worker_request( + our: &Address, + request: &WorkerRequest, + file: &mut Option, + size: &mut Option, +) -> anyhow::Result { + match request { WorkerRequest::Chunk(ChunkRequest { name, offset, length, }) => { + // someone sending a chunk to us let file = match file { Some(file) => file, None => { @@ -139,9 +140,8 @@ fn handle_message( } }; - // file.seek(SeekFrom::Start(offset))?; seek not necessary if the sends come in order. file.write_all(&bytes)?; - // if sender has sent us a size, give a progress update to main transfer! + // if sender has sent us a size, give a progress update to main transfer if let Some(size) = size { let progress = ((offset + length) as f64 / *size as f64 * 100.0) as u64; @@ -152,8 +152,9 @@ fn handle_message( }; Request::new() + .expects_response(5) .body(TransferRequest::Progress(ProgressRequest { - name, + name: name.to_string(), progress, })) .target(&main_app) @@ -165,13 +166,51 @@ fn handle_message( } } WorkerRequest::Size(incoming_size) => { - *size = Some(incoming_size); + *size = Some(*incoming_size); } } + Ok(false) +} +fn handle_transfer_response( + message: &Message, +) -> anyhow::Result { + match message.body().try_into()? { + TransferResponse::ListFiles(_) => {} + TransferResponse::Download(result) | TransferResponse::Progress(result) => { + if result.is_err() { + return Err(anyhow::anyhow!("{}", result.unwrap_err())); + } + } + } Ok(false) } +fn handle_message( + our: &Address, + message: &Message, + file: &mut Option, + files_dir: &Directory, + size: &mut Option, +) -> anyhow::Result { + if !message.is_request() { + return handle_transfer_response(message); + } + return Ok(match message.body().try_into()? { + Req::TransferRequest(ref tr) => handle_transfer_request( + tr, + file, + files_dir, + )?, + Req::WorkerRequest(ref wr) => handle_worker_request( + our, + wr, + file, + size, + )?, + }); +} + call_init!(init); fn init(our: Address) { println!("worker: begin"); diff --git a/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs index bd19c864..d0e9eea9 100644 --- a/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs @@ -1,9 +1,9 @@ use crate::kinode::process::standard::{ProcessId as WitProcessId}; -use crate::kinode::process::{package_name}::{Address as WitAddress, Request as TransferRequest, Response as TransferResponse, WorkerRequest, DownloadRequest, ProgressRequest, FileInfo, InitializeRequest}; +use crate::kinode::process::{package_name}::{start_download, Address as WitAddress, Request as TransferRequest, Response as TransferResponse, DownloadRequest, ProgressRequest, FileInfo}; use kinode_process_lib::{ - await_message, call_init, our_capabilities, println, spawn, + await_message, call_init, println, vfs::{create_drive, metadata, open_dir, Directory, FileType}, - Address, Message, OnExit, ProcessId, Request, Response, + Address, Message, ProcessId, Response, }; wit_bindgen::generate!({ @@ -31,24 +31,6 @@ impl From for WitProcessId { } } } -impl From for Address { - fn from(address: WitAddress) -> Self { - Address { - node: address.node, - process: address.process.into(), - } - } -} - -impl From for ProcessId { - fn from(process: WitProcessId) -> Self { - ProcessId { - process_name: process.process_name, - package_name: process.package_name, - publisher_node: process.publisher_node, - } - } -} fn ls_files(files_dir: &Directory) -> anyhow::Result> { let entries = files_dir.read()?; @@ -77,58 +59,52 @@ fn handle_transfer_request( match message.body().try_into()? { TransferRequest::ListFiles => { let files = ls_files(files_dir)?; - Response::new() .body(TransferResponse::ListFiles(files)) .send()?; } - TransferRequest::Download(DownloadRequest { name, target }) => { - // spin up a worker, initialize based on whether it's a downloader or a sender. - let our_worker = spawn( - None, - &format!("{}/pkg/worker.wasm", our.package_id()), - OnExit::None, - our_capabilities(), - vec![], - false, - )?; - - let our_worker_address = Address { - node: our.node.clone(), - process: our_worker, - }; - - if message.source().node == our.node { - // we want to download a file - let _resp = Request::new() - .body(WorkerRequest::Initialize(InitializeRequest { - name: name.clone(), - target_worker: None, - })) - .target(&our_worker_address) - .send_and_await_response(5)??; - - // send our initialized worker address to the other node - Request::new() - .body(TransferRequest::Download(DownloadRequest { - name: name.clone(), - target: our_worker_address.into(), - })) - .target::
(target.clone().into()) - .send()?; - } else { - // they want to download a file - Request::new() - .body(WorkerRequest::Initialize(InitializeRequest { - name: name.clone(), - target_worker: Some(target), - })) - .target(&our_worker_address) - .send()?; + TransferRequest::Download(DownloadRequest { ref name, ref target, is_requestor }) => { + match start_download( + &our.clone().into(), + &message.source().clone().into(), + name, + target, + is_requestor, + ) { + Ok(_) => {} + Err(e) => return Err(anyhow::anyhow!("{e}")), } } TransferRequest::Progress(ProgressRequest { name, progress }) => { println!("{} progress: {}%", name, progress); + Response::new() + .body(TransferResponse::Progress(Ok(()))) + .send()?; + } + } + + Ok(()) +} + +fn handle_transfer_response(message: &Message) -> anyhow::Result<()> { + match message.body().try_into()? { + TransferResponse::ListFiles(ref files) => { + println!( + "{}", + files.iter(). + fold(format!("{} available files:\nFile\t\tSize (bytes)\n", message.source()), |mut msg, file| { + msg.push_str(&format!( + "{}\t\t{}", file.name.split('/').last().unwrap(), + file.size, + )); + msg + }) + ); + } + TransferResponse::Download(result) | TransferResponse::Progress(result) => { + if result.is_err() { + return Err(anyhow::anyhow!("{}", result.unwrap_err())); + } } } @@ -140,7 +116,11 @@ fn handle_message( message: &Message, files_dir: &Directory, ) -> anyhow::Result<()> { - handle_transfer_request(our, message, files_dir) + if message.is_request() { + handle_transfer_request(our, message, files_dir) + } else { + handle_transfer_response(message) + } } call_init!(init); diff --git a/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/Cargo.toml_ b/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/Cargo.toml_ new file mode 100644 index 00000000..9231047c --- /dev/null +++ b/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/Cargo.toml_ @@ -0,0 +1,18 @@ +[package] +name = "{package_name}_api" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +kinode_process_lib = { git = "https://github.com/kinode-dao/process_lib", tag = "v0.8.3" } +process_macros = { git = "https://github.com/kinode-dao/process_macros", rev = "626e501" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +wit-bindgen = "0.24.0" + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "kinode:process" diff --git a/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs new file mode 100644 index 00000000..f09eb8f0 --- /dev/null +++ b/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs @@ -0,0 +1,75 @@ +use crate::exports::kinode::process::{package_name}::{DownloadRequest, Guest, Request as TransferRequest, Response as TransferResponse}; +use crate::kinode::process::standard::{Address as WitAddress}; +use kinode_process_lib::{our_capabilities, spawn, Address, OnExit, Request, Response}; + +wit_bindgen::generate!({ + path: "target/wit", + world: "{package_name_kebab}-{publisher_dotted_kebab}-api-v0", + generate_unused_types: true, + additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], +}); + +fn start_download( + our: &WitAddress, + source: &WitAddress, + name: &str, + target: &WitAddress, + is_requestor: bool, +) -> anyhow::Result<()> { + // spin up a worker, initialize based on whether it's a downloader or a sender. + let our_worker = spawn( + None, + &format!( + "{}:{}/pkg/worker.wasm", + our.process.package_name, + our.process.publisher_node, + ), + OnExit::None, + our_capabilities(), + vec![], + false, + )?; + + let target = if is_requestor { + target + } else { + source + }; + let our_worker_address = Address { + node: our.node.clone(), + process: our_worker, + }; + + Response::new() + .body(TransferResponse::Download(Ok(()))) + .send()?; + + Request::new() + .expects_response(5) + .body(TransferRequest::Download(DownloadRequest { + name: name.to_string(), + target: target.clone(), + is_requestor, + })) + .target(&our_worker_address) + .send()?; + + Ok(()) +} + +struct Api; +impl Guest for Api { + fn start_download( + our: WitAddress, + source: WitAddress, + name: String, + target: WitAddress, + is_requestor: bool, + ) -> Result<(), String> { + match start_download(&our, &source, &name, &target, is_requestor) { + Ok(result) => Ok(result), + Err(e) => Err(format!("{e:?}")), + } + } +} +export!(Api); From 8b14873aacc92a98174fed110f3ad90e4869efeb Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 24 Jul 2024 13:28:07 -0700 Subject: [PATCH 07/18] new: tighten up the file_transfer_worker api --- .../api/{package_name}:{publisher}-v0.wit | 42 +++++-- .../no-ui/file_transfer/download/src/lib.rs | 8 +- .../no-ui/file_transfer/worker/src/lib.rs | 107 +++++++++--------- .../file_transfer/{package_name}/src/lib.rs | 74 ++++++++---- .../{package_name}_api/src/lib.rs | 21 ++-- 5 files changed, 147 insertions(+), 105 deletions(-) diff --git a/src/new/templates/rust/no-ui/file_transfer/api/{package_name}:{publisher}-v0.wit b/src/new/templates/rust/no-ui/file_transfer/api/{package_name}:{publisher}-v0.wit index 8314f747..e0e1a9d1 100644 --- a/src/new/templates/rust/no-ui/file_transfer/api/{package_name}:{publisher}-v0.wit +++ b/src/new/templates/rust/no-ui/file_transfer/api/{package_name}:{publisher}-v0.wit @@ -1,19 +1,41 @@ interface {package_name_kebab} { + variant request { + list-files, + } + + variant response { + list-files(list), + } + + record file-info { + name: string, + size: u64, + } +} + +interface file-transfer-worker { use standard.{address}; + /// external-facing requests variant request { - list-files, + /// download starts a download. + /// * used by requestor to start whole process + /// * used by provider to spin up worker to serve request download(download-request), + /// progress is from worker to parent + /// * acks not required, but provided for completeness progress(progress-request), } variant response { - list-files(list), download(result<_, string>), - progress(result<_, string>), + /// ack: not required, but provided for completeness + progress, } - variant worker-request { + /// requests used between workers to transfer the file + /// parent will not receive these, so need not handle them + variant internal-request { chunk(chunk-request), size(u64), } @@ -29,17 +51,14 @@ interface {package_name_kebab} { progress: u64, } - record file-info { - name: string, - size: u64, - } - record chunk-request { name: string, offset: u64, length: u64, } + /// easiest way to use file-transfer-worker + /// handle file-transfer-worker::request by calling this helper function start-download: func( our: address, source: address, @@ -49,11 +68,12 @@ interface {package_name_kebab} { ) -> result<_, string>; } -world {package_name_kebab}-{publisher_dotted_kebab}-api-v0 { - export {package_name_kebab}; +world file-transfer-worker-api-v0 { + export file-transfer-worker; } world {package_name_kebab}-{publisher_dotted_kebab}-v0 { import {package_name_kebab}; + import file-transfer-worker; include process-v0; } diff --git a/src/new/templates/rust/no-ui/file_transfer/download/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/download/src/lib.rs index ee5b3d95..b728e452 100644 --- a/src/new/templates/rust/no-ui/file_transfer/download/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/download/src/lib.rs @@ -1,5 +1,5 @@ -use crate::kinode::process::standard::{ProcessId as WitProcessId}; -use crate::kinode::process::{package_name}::{Address as WitAddress, Request as TransferRequest, DownloadRequest}; +use crate::kinode::process::file_transfer_worker::{DownloadRequest, Request as WorkerRequest}; +use crate::kinode::process::standard::{Address as WitAddress, ProcessId as WitProcessId}; use kinode_process_lib::{ await_next_message_body, call_init, println, Address, ProcessId, Request, }; @@ -40,7 +40,7 @@ fn init(our: Address) { let args = String::from_utf8(body).unwrap_or_default(); let Some((name, who)) = args.split_once(" ") else { println!("usage: download:{package_name}:{publisher} file_name who"); - return + return; }; let our: Address = format!("{}@{package_name}:{package_name}:{publisher}", our.node()) .parse() @@ -51,7 +51,7 @@ fn init(our: Address) { .unwrap(); match Request::to(our) - .body(TransferRequest::Download(DownloadRequest { + .body(WorkerRequest::Download(DownloadRequest { name: name.into(), target: target.clone().into(), is_requestor: true, diff --git a/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs index fcf77c59..5ec07157 100644 --- a/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs @@ -1,5 +1,9 @@ -use crate::kinode::process::standard::{ProcessId as WitProcessId}; -use crate::kinode::process::{package_name}::{Address as WitAddress, Request as TransferRequest, Response as TransferResponse, WorkerRequest, DownloadRequest, ProgressRequest, ChunkRequest}; +use crate::kinode::process::file_transfer_worker::{ + ChunkRequest, DownloadRequest, InternalRequest, ProgressRequest, Request as WorkerRequest, + Response as WorkerResponse, +}; +use crate::kinode::process::standard::{Address as WitAddress, ProcessId as WitProcessId}; +//use crate::kinode::process::{package_name}::{Request as TransferRequest, Response as TransferResponse, WorkerRequest, DownloadRequest, ProgressRequest, ChunkRequest}; use kinode_process_lib::{ await_message, call_init, get_blob, println, vfs::{open_dir, open_file, Directory, File, SeekFrom}, @@ -14,10 +18,14 @@ wit_bindgen::generate!({ }); #[derive(Debug, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto)] -#[serde(untagged)] // untagged as a meta-type for all incoming responses -enum Req { - TransferRequest(TransferRequest), +#[serde(untagged)] // untagged as a meta-type for all incoming messages +enum Msg { + // requests WorkerRequest(WorkerRequest), + InternalRequest(InternalRequest), + + // responses + WorkerResponse(WorkerResponse), } impl From for Address { @@ -41,30 +49,29 @@ impl From for ProcessId { const CHUNK_SIZE: u64 = 1048576; // 1MB -fn handle_transfer_request( - request: &TransferRequest, +fn handle_worker_request( + request: &WorkerRequest, file: &mut Option, files_dir: &Directory, ) -> anyhow::Result { match request { - TransferRequest::Download(DownloadRequest { + WorkerRequest::Download(DownloadRequest { name, target, is_requestor, }) => { Response::new() - .body(TransferResponse::Download(Ok(()))) + .body(WorkerResponse::Download(Ok(()))) .send()?; // open/create empty file in both cases. - let mut active_file = - open_file(&format!("{}/{}", files_dir.path, &name), true, None)?; + let mut active_file = open_file(&format!("{}/{}", files_dir.path, &name), true, None)?; if *is_requestor { *file = Some(active_file); Request::new() .expects_response(5) - .body(TransferRequest::Download(DownloadRequest { + .body(WorkerRequest::Download(DownloadRequest { name: name.to_string(), target: target.clone(), is_requestor: false, @@ -78,7 +85,7 @@ fn handle_transfer_request( // give receiving worker file size so it can track download progress Request::new() - .body(WorkerRequest::Size(size)) + .body(InternalRequest::Size(size)) .target(target.clone()) .send()?; @@ -92,7 +99,7 @@ fn handle_transfer_request( active_file.read_at(&mut buffer)?; Request::new() - .body(WorkerRequest::Chunk(ChunkRequest { + .body(InternalRequest::Chunk(ChunkRequest { name: name.clone(), offset, length, @@ -104,21 +111,23 @@ fn handle_transfer_request( return Ok(true); } } - TransferRequest::ListFiles | TransferRequest::Progress(_) => { - return Err(anyhow::anyhow!("worker: unexpected TransferRequest: {request:?}")); + WorkerRequest::Progress(_) => { + return Err(anyhow::anyhow!( + "worker: got unexpected WorkerRequest::Progress", + )); } } Ok(false) } -fn handle_worker_request( +fn handle_internal_request( our: &Address, - request: &WorkerRequest, + request: &InternalRequest, file: &mut Option, size: &mut Option, ) -> anyhow::Result { match request { - WorkerRequest::Chunk(ChunkRequest { + InternalRequest::Chunk(ChunkRequest { name, offset, length, @@ -128,7 +137,7 @@ fn handle_worker_request( Some(file) => file, None => { return Err(anyhow::anyhow!( - "{package_name} worker: receive error: no file initialized" + "worker: receive error: no file initialized" )); } }; @@ -136,7 +145,7 @@ fn handle_worker_request( let bytes = match get_blob() { Some(blob) => blob.bytes, None => { - return Err(anyhow::anyhow!("{package_name} worker: receive error: no blob")); + return Err(anyhow::anyhow!("worker: receive error: no blob")); } }; @@ -153,7 +162,7 @@ fn handle_worker_request( Request::new() .expects_response(5) - .body(TransferRequest::Progress(ProgressRequest { + .body(WorkerRequest::Progress(ProgressRequest { name: name.to_string(), progress, })) @@ -165,23 +174,21 @@ fn handle_worker_request( } } } - WorkerRequest::Size(incoming_size) => { + InternalRequest::Size(incoming_size) => { *size = Some(*incoming_size); } } Ok(false) } -fn handle_transfer_response( - message: &Message, -) -> anyhow::Result { - match message.body().try_into()? { - TransferResponse::ListFiles(_) => {} - TransferResponse::Download(result) | TransferResponse::Progress(result) => { +fn handle_worker_response(response: &WorkerResponse) -> anyhow::Result { + match response { + WorkerResponse::Download(ref result) => { if result.is_err() { - return Err(anyhow::anyhow!("{}", result.unwrap_err())); + return Err(anyhow::anyhow!("{}", result.as_ref().unwrap_err())); } } + WorkerResponse::Progress => {} } Ok(false) } @@ -193,21 +200,13 @@ fn handle_message( files_dir: &Directory, size: &mut Option, ) -> anyhow::Result { - if !message.is_request() { - return handle_transfer_response(message); - } return Ok(match message.body().try_into()? { - Req::TransferRequest(ref tr) => handle_transfer_request( - tr, - file, - files_dir, - )?, - Req::WorkerRequest(ref wr) => handle_worker_request( - our, - wr, - file, - size, - )?, + // requests + Msg::WorkerRequest(ref wr) => handle_worker_request(wr, file, files_dir)?, + Msg::InternalRequest(ref ir) => handle_internal_request(our, ir, file, size)?, + + // responses + Msg::WorkerResponse(ref wr) => handle_worker_response(wr)?, }); } @@ -225,20 +224,16 @@ fn init(our: Address) { loop { match await_message() { Err(send_error) => println!("worker: got SendError: {send_error}"), - Ok(ref message) => match handle_message( - &our, - message, - &mut file, - &files_dir, - &mut size, - ) { - Ok(exit) => { - if exit { - println!("worker: done: exiting, took {:?}", start.elapsed()); - break; + Ok(ref message) => { + match handle_message(&our, message, &mut file, &files_dir, &mut size) { + Ok(exit) => { + if exit { + println!("worker: done: exiting, took {:?}", start.elapsed()); + break; + } } + Err(e) => println!("worker: got error while handling message: {e:?}"), } - Err(e) => println!("worker: got error while handling message: {e:?}"), } } } diff --git a/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs index d0e9eea9..85cc9ffd 100644 --- a/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs @@ -1,5 +1,6 @@ -use crate::kinode::process::standard::{ProcessId as WitProcessId}; -use crate::kinode::process::{package_name}::{start_download, Address as WitAddress, Request as TransferRequest, Response as TransferResponse, DownloadRequest, ProgressRequest, FileInfo}; +use crate::kinode::process::standard::{Address as WitAddress, ProcessId as WitProcessId}; +use crate::kinode::process::file_transfer_worker::{start_download, Request as WorkerRequest, Response as WorkerResponse, DownloadRequest, ProgressRequest}; +use crate::kinode::process::{package_name}::{Request as TransferRequest, Response as TransferResponse, FileInfo}; use kinode_process_lib::{ await_message, call_init, println, vfs::{create_drive, metadata, open_dir, Directory, FileType}, @@ -13,6 +14,18 @@ wit_bindgen::generate!({ additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); +#[derive(Debug, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto)] +#[serde(untagged)] // untagged as a meta-type for all incoming messages +enum Msg { + // requests + TransferRequest(TransferRequest), + WorkerRequest(WorkerRequest), + + // responses + TransferResponse(TransferResponse), + WorkerResponse(WorkerResponse), +} + impl From
for WitAddress { fn from(address: Address) -> Self { WitAddress { @@ -47,52 +60,59 @@ fn ls_files(files_dir: &Directory) -> anyhow::Result> { _ => None, }) .collect(); - Ok(files) } fn handle_transfer_request( - our: &Address, - message: &Message, + request: &TransferRequest, files_dir: &Directory, ) -> anyhow::Result<()> { - match message.body().try_into()? { + match request { TransferRequest::ListFiles => { let files = ls_files(files_dir)?; Response::new() .body(TransferResponse::ListFiles(files)) .send()?; } - TransferRequest::Download(DownloadRequest { ref name, ref target, is_requestor }) => { + } + Ok(()) +} + +fn handle_worker_request( + our: &Address, + source: &Address, + request: &WorkerRequest, +) -> anyhow::Result<()> { + match request { + WorkerRequest::Download(DownloadRequest { ref name, ref target, is_requestor }) => { match start_download( &our.clone().into(), - &message.source().clone().into(), + &source.clone().into(), name, target, - is_requestor, + *is_requestor, ) { Ok(_) => {} Err(e) => return Err(anyhow::anyhow!("{e}")), } } - TransferRequest::Progress(ProgressRequest { name, progress }) => { + WorkerRequest::Progress(ProgressRequest { name, progress }) => { println!("{} progress: {}%", name, progress); Response::new() - .body(TransferResponse::Progress(Ok(()))) + .body(WorkerResponse::Progress) .send()?; } } - Ok(()) } -fn handle_transfer_response(message: &Message) -> anyhow::Result<()> { - match message.body().try_into()? { +fn handle_transfer_response(source: &Address, response: &TransferResponse) -> anyhow::Result<()> { + match response { TransferResponse::ListFiles(ref files) => { println!( "{}", files.iter(). - fold(format!("{} available files:\nFile\t\tSize (bytes)\n", message.source()), |mut msg, file| { + fold(format!("{source} available files:\nFile\t\tSize (bytes)\n"), |mut msg, file| { msg.push_str(&format!( "{}\t\t{}", file.name.split('/').last().unwrap(), file.size, @@ -101,13 +121,19 @@ fn handle_transfer_response(message: &Message) -> anyhow::Result<()> { }) ); } - TransferResponse::Download(result) | TransferResponse::Progress(result) => { + } + Ok(()) +} + +fn handle_worker_response(response: &WorkerResponse) -> anyhow::Result<()> { + match response { + WorkerResponse::Download(ref result) => { if result.is_err() { - return Err(anyhow::anyhow!("{}", result.unwrap_err())); + return Err(anyhow::anyhow!("{}", result.as_ref().unwrap_err())); } } + WorkerResponse::Progress => {} } - Ok(()) } @@ -116,10 +142,14 @@ fn handle_message( message: &Message, files_dir: &Directory, ) -> anyhow::Result<()> { - if message.is_request() { - handle_transfer_request(our, message, files_dir) - } else { - handle_transfer_response(message) + match message.body().try_into()? { + // requests + Msg::TransferRequest(ref tr) => handle_transfer_request(tr, files_dir), + Msg::WorkerRequest(ref wr) => handle_worker_request(our, message.source(), wr), + + // responses + Msg::TransferResponse(ref tr) => handle_transfer_response(message.source(), tr), + Msg::WorkerResponse(ref wr) => handle_worker_response(wr), } } diff --git a/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs index f09eb8f0..81f2a65c 100644 --- a/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs @@ -1,10 +1,12 @@ -use crate::exports::kinode::process::{package_name}::{DownloadRequest, Guest, Request as TransferRequest, Response as TransferResponse}; -use crate::kinode::process::standard::{Address as WitAddress}; +use crate::exports::kinode::process::file_transfer_worker::{ + DownloadRequest, Guest, Request as WorkerRequest, Response as WorkerResponse, +}; +use crate::kinode::process::standard::Address as WitAddress; use kinode_process_lib::{our_capabilities, spawn, Address, OnExit, Request, Response}; wit_bindgen::generate!({ path: "target/wit", - world: "{package_name_kebab}-{publisher_dotted_kebab}-api-v0", + world: "file-transfer-worker-api-v0", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -21,8 +23,7 @@ fn start_download( None, &format!( "{}:{}/pkg/worker.wasm", - our.process.package_name, - our.process.publisher_node, + our.process.package_name, our.process.publisher_node, ), OnExit::None, our_capabilities(), @@ -30,23 +31,19 @@ fn start_download( false, )?; - let target = if is_requestor { - target - } else { - source - }; + let target = if is_requestor { target } else { source }; let our_worker_address = Address { node: our.node.clone(), process: our_worker, }; Response::new() - .body(TransferResponse::Download(Ok(()))) + .body(WorkerResponse::Download(Ok(()))) .send()?; Request::new() .expects_response(5) - .body(TransferRequest::Download(DownloadRequest { + .body(WorkerRequest::Download(DownloadRequest { name: name.to_string(), target: target.clone(), is_requestor, From ab49e3b50d48a1fa5c6aa017e2ae929c29c643d4 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 24 Jul 2024 13:37:00 -0700 Subject: [PATCH 08/18] new: move some file_transfer stuff around --- src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ | 4 ++-- .../{worker => file_transfer_worker}/Cargo.toml_ | 2 +- .../file_transfer/{worker => file_transfer_worker}/src/lib.rs | 0 .../Cargo.toml_ | 2 +- .../src/lib.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename src/new/templates/rust/no-ui/file_transfer/{worker => file_transfer_worker}/Cargo.toml_ (93%) rename src/new/templates/rust/no-ui/file_transfer/{worker => file_transfer_worker}/src/lib.rs (100%) rename src/new/templates/rust/no-ui/file_transfer/{{package_name}_api => file_transfer_worker_api}/Cargo.toml_ (92%) rename src/new/templates/rust/no-ui/file_transfer/{{package_name}_api => file_transfer_worker_api}/src/lib.rs (97%) diff --git a/src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ b/src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ index b575d19d..0a734fee 100644 --- a/src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ +++ b/src/new/templates/rust/no-ui/file_transfer/Cargo.toml_ @@ -2,8 +2,8 @@ resolver = "2" members = [ "{package_name}", - "{package_name}_api", - "worker", + "file_transfer_worker_api", + "file_transfer_worker", "download", "list_files" ] diff --git a/src/new/templates/rust/no-ui/file_transfer/worker/Cargo.toml_ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/Cargo.toml_ similarity index 93% rename from src/new/templates/rust/no-ui/file_transfer/worker/Cargo.toml_ rename to src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/Cargo.toml_ index 4145e57d..ec9836f6 100644 --- a/src/new/templates/rust/no-ui/file_transfer/worker/Cargo.toml_ +++ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/Cargo.toml_ @@ -1,5 +1,5 @@ [package] -name = "worker" +name = "file_transfer_worker" version = "0.1.0" edition = "2021" diff --git a/src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs similarity index 100% rename from src/new/templates/rust/no-ui/file_transfer/worker/src/lib.rs rename to src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs diff --git a/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/Cargo.toml_ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker_api/Cargo.toml_ similarity index 92% rename from src/new/templates/rust/no-ui/file_transfer/{package_name}_api/Cargo.toml_ rename to src/new/templates/rust/no-ui/file_transfer/file_transfer_worker_api/Cargo.toml_ index 9231047c..2e75c860 100644 --- a/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/Cargo.toml_ +++ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker_api/Cargo.toml_ @@ -1,5 +1,5 @@ [package] -name = "{package_name}_api" +name = "file_transfer_worker_api" version = "0.1.0" edition = "2021" diff --git a/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker_api/src/lib.rs similarity index 97% rename from src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs rename to src/new/templates/rust/no-ui/file_transfer/file_transfer_worker_api/src/lib.rs index 81f2a65c..a2ed498c 100644 --- a/src/new/templates/rust/no-ui/file_transfer/{package_name}_api/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker_api/src/lib.rs @@ -22,7 +22,7 @@ fn start_download( let our_worker = spawn( None, &format!( - "{}:{}/pkg/worker.wasm", + "{}:{}/pkg/file_transfer_worker.wasm", our.process.package_name, our.process.publisher_node, ), OnExit::None, From 09cca773da8943e753d4c095daeea928305b710c Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 24 Jul 2024 13:38:09 -0700 Subject: [PATCH 09/18] build: remove commented-out code --- src/build/mod.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index 9d091621..768e6b1a 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -967,26 +967,6 @@ async fn compile_package( // fetch dependency apis: to be used in build if let Some(ref deps) = metadata.properties.dependencies { dependencies.extend(deps); - //if dependencies.is_empty() { - // continue; - //} - ////let Some(ref url) = url else { - //// // TODO: can we use kit-cached deps? - //// return Err(eyre!("Need a node to be able to fetch dependencies")); - ////}; - //fetch_dependencies( - // package_dir, - // dependencies, - // &mut apis, - // &mut wasm_paths, - // url.clone(), - // download_from, - // local_dependencies.clone(), - // features, - // default_world, - // force, - // verbose, - //).await?; } } } From 125106757ed2684289a74da55f30f3f8d3dae8b6 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 24 Jul 2024 14:03:18 -0700 Subject: [PATCH 10/18] new: remove old pkg/metadata.json --- src/new/templates/rust/no-ui/file_transfer/pkg/metadata.json | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/new/templates/rust/no-ui/file_transfer/pkg/metadata.json diff --git a/src/new/templates/rust/no-ui/file_transfer/pkg/metadata.json b/src/new/templates/rust/no-ui/file_transfer/pkg/metadata.json deleted file mode 100644 index af3632bc..00000000 --- a/src/new/templates/rust/no-ui/file_transfer/pkg/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "package": "{package_name}", - "publisher": "{publisher}", - "version": [0, 1, 0] -} From 577201cb381c5ff4c403364bb84a5f9e663cfb78 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 24 Jul 2024 16:43:30 -0700 Subject: [PATCH 11/18] new: simplify some error handling --- .../rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs | 4 ++-- .../rust/no-ui/file_transfer/{package_name}/src/lib.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs index 5ec07157..76b7e995 100644 --- a/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs @@ -184,8 +184,8 @@ fn handle_internal_request( fn handle_worker_response(response: &WorkerResponse) -> anyhow::Result { match response { WorkerResponse::Download(ref result) => { - if result.is_err() { - return Err(anyhow::anyhow!("{}", result.as_ref().unwrap_err())); + if let Err(e) = result { + return Err(anyhow::anyhow!("{e}")); } } WorkerResponse::Progress => {} diff --git a/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs index 85cc9ffd..1f3c7879 100644 --- a/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/{package_name}/src/lib.rs @@ -128,8 +128,8 @@ fn handle_transfer_response(source: &Address, response: &TransferResponse) -> an fn handle_worker_response(response: &WorkerResponse) -> anyhow::Result<()> { match response { WorkerResponse::Download(ref result) => { - if result.is_err() { - return Err(anyhow::anyhow!("{}", result.as_ref().unwrap_err())); + if let Err(e) = result { + return Err(anyhow::anyhow!("{e}")) } } WorkerResponse::Progress => {} From 58f724abc853f6731df1f3bd959a5839b68fdfb7 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 24 Jul 2024 16:47:15 -0700 Subject: [PATCH 12/18] build: add `--add-to-api` flag (hacky) --- src/build/mod.rs | 29 +++++++++++++++++++++++++++++ src/build_start_package/mod.rs | 2 ++ src/main.rs | 28 ++++++++++++++++++++++++++-- src/run_tests/mod.rs | 3 +++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index 768e6b1a..c021e3b4 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -586,6 +586,7 @@ async fn compile_package_and_ui( default_world: Option<&str>, download_from: Option<&str>, local_dependencies: Vec, + add_paths_to_api: Vec, force: bool, verbose: bool, ignore_deps: bool, @@ -599,6 +600,7 @@ async fn compile_package_and_ui( default_world, download_from, local_dependencies, + add_paths_to_api, force, verbose, ignore_deps, @@ -720,6 +722,7 @@ async fn fetch_dependencies( download_from, default_world, vec![], // TODO: what about deps-of-deps? + vec![], force, verbose, true, @@ -737,6 +740,7 @@ async fn fetch_dependencies( download_from, default_world, vec![], // TODO: what about deps-of-deps? + vec![], force, verbose, false, @@ -912,6 +916,7 @@ async fn compile_package( default_world: Option<&str>, download_from: Option<&str>, local_dependencies: Vec, + add_paths_to_api: Vec, force: bool, verbose: bool, ignore_deps: bool, @@ -1015,6 +1020,8 @@ async fn compile_package( let target_api_dir = package_dir.join("target").join("api"); if api_dir.exists() { copy_dir(&api_dir, &target_api_dir)?; + } else if !target_api_dir.exists() { + fs::create_dir_all(&target_api_dir)?; } if !ignore_deps { @@ -1049,6 +1056,24 @@ async fn compile_package( // zip & place API inside of pkg/ to publish API if target_api_dir.exists() { + for path in add_paths_to_api { + let path = if path.exists() { + path + } else { + package_dir.join(path).canonicalize().unwrap_or_default() + }; + if !path.exists() { + warn!("Given path to add to API does not exist: {path:?}"); + continue; + } + if let Err(e) = fs::copy( + &path, + target_api_dir.join(path.file_name().and_then(|f| f.to_str()).unwrap()), + ) { + warn!("Could not add path {path:?} to API: {e:?}"); + } + } + let zip_path = package_dir.join("pkg").join("api.zip"); let zip_path = zip_path.to_str().unwrap(); zip_directory(&target_api_dir, zip_path)?; @@ -1068,6 +1093,7 @@ pub async fn execute( download_from: Option<&str>, default_world: Option<&str>, local_dependencies: Vec, + add_paths_to_api: Vec, force: bool, verbose: bool, ignore_deps: bool, // for internal use; may cause problems when adding recursive deps @@ -1126,6 +1152,7 @@ pub async fn execute( default_world.clone(), download_from, local_dependencies, + add_paths_to_api, force, verbose, ignore_deps, @@ -1142,6 +1169,7 @@ pub async fn execute( default_world, download_from, local_dependencies, + add_paths_to_api, force, verbose, ignore_deps, @@ -1165,6 +1193,7 @@ pub async fn execute( default_world, download_from, local_dependencies, + add_paths_to_api, force, verbose, ignore_deps, diff --git a/src/build_start_package/mod.rs b/src/build_start_package/mod.rs index a0fc50dd..3202c07c 100644 --- a/src/build_start_package/mod.rs +++ b/src/build_start_package/mod.rs @@ -17,6 +17,7 @@ pub async fn execute( download_from: Option<&str>, default_world: Option<&str>, local_dependencies: Vec, + add_paths_to_api: Vec, force: bool, verbose: bool, ) -> Result<()> { @@ -30,6 +31,7 @@ pub async fn execute( download_from, default_world, local_dependencies, + add_paths_to_api, force, verbose, false, diff --git a/src/main.rs b/src/main.rs index 3934ab2a..b2bcd364 100644 --- a/src/main.rs +++ b/src/main.rs @@ -194,6 +194,11 @@ async fn execute( .and_then(|s: &String| Some(s.as_str())); let default_world = matches.get_one::("WORLD"); let local_dependencies: Vec = matches + .get_many::("DEPENDENCY_PACKAGE_PATH") + .unwrap_or_default() + .map(|s| PathBuf::from(s)) + .collect(); + let add_paths_to_api: Vec = matches .get_many::("PATH") .unwrap_or_default() .map(|s| PathBuf::from(s)) @@ -211,6 +216,7 @@ async fn execute( download_from, default_world.map(|w| w.as_str()), local_dependencies, + add_paths_to_api, *force, *verbose, false, @@ -239,6 +245,11 @@ async fn execute( .and_then(|s: &String| Some(s.as_str())); let default_world = matches.get_one::("WORLD"); let local_dependencies: Vec = matches + .get_many::("DEPENDENCY_PACKAGE_PATH") + .unwrap_or_default() + .map(|s| PathBuf::from(s)) + .collect(); + let add_paths_to_api: Vec = matches .get_many::("PATH") .unwrap_or_default() .map(|s| PathBuf::from(s)) @@ -256,6 +267,7 @@ async fn execute( download_from, default_world.map(|w| w.as_str()), local_dependencies, + add_paths_to_api, *force, *verbose, ) @@ -634,12 +646,18 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .long("world") .help("Fallback WIT world name") ) - .arg(Arg::new("PATH") + .arg(Arg::new("DEPENDENCY_PACKAGE_PATH") .action(ArgAction::Append) .short('l') .long("local-dependency") .help("Path to local dependency package (can specify multiple times)") ) + .arg(Arg::new("PATH") + .action(ArgAction::Append) + .short('a') + .long("add-to-api") + .help("Path to file to add to api.zip (can specify multiple times)") + ) .arg(Arg::new("FORCE") .action(ArgAction::SetTrue) .short('f') @@ -685,12 +703,18 @@ async fn make_app(current_dir: &std::ffi::OsString) -> Result { .help("Fallback WIT world name") .required(false) ) - .arg(Arg::new("PATH") + .arg(Arg::new("DEPENDENCY_PACKAGE_PATH") .action(ArgAction::Append) .short('l') .long("local-dependency") .help("Path to local dependency package (can specify multiple times)") ) + .arg(Arg::new("PATH") + .action(ArgAction::Append) + .short('a') + .long("add-to-api") + .help("Path to file to add to api.zip (can specify multiple times)") + ) .arg(Arg::new("NO_UI") .action(ArgAction::SetTrue) .long("no-ui") diff --git a/src/run_tests/mod.rs b/src/run_tests/mod.rs index 0d8b5f5e..12e89622 100644 --- a/src/run_tests/mod.rs +++ b/src/run_tests/mod.rs @@ -351,6 +351,7 @@ async fn build_packages( None, None, dependency_package_paths.clone(), + vec![], // TODO false, false, false, @@ -369,6 +370,7 @@ async fn build_packages( None, None, dependency_package_paths.clone(), + vec![], // TODO false, false, false, @@ -385,6 +387,7 @@ async fn build_packages( None, None, dependency_package_paths.clone(), + vec![], // TODO false, false, false, From b8b31b2b05c548733312ef93357598981ba4dc5f Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 25 Jul 2024 18:10:09 -0700 Subject: [PATCH 13/18] build: properly place non-exporter .wasm files from api.zip into pkg/ --- src/build/mod.rs | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index c021e3b4..749c9666 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -12,7 +12,7 @@ use color_eyre::{ }; use fs_err as fs; use serde::{Deserialize, Serialize}; -use tracing::{info, instrument, warn}; +use tracing::{debug, info, instrument, warn}; use kinode_process_lib::{PackageId, kernel_types::Erc721Metadata}; @@ -712,7 +712,7 @@ async fn fetch_dependencies( force: bool, verbose: bool, ) -> Result<()> { - Box::pin(execute( + if let Err(e) = Box::pin(execute( package_dir, true, false, @@ -726,8 +726,15 @@ async fn fetch_dependencies( force, verbose, true, - )).await?; - fetch_local_built_dependency(apis, wasm_paths, package_dir)?; + )).await { + debug!("Failed to build self as dependency: {e:?}"); + } else if let Err(e) = fetch_local_built_dependency( + apis, + wasm_paths, + package_dir, + ) { + debug!("Failed to fetch self as dependency: {e:?}"); + }; for local_dependency in &local_dependencies { // build dependency Box::pin(execute( @@ -866,7 +873,11 @@ fn get_imports_exports_from_wasm( fn find_non_standard( package_dir: &Path, wasm_paths: HashSet, -) -> Result<(HashMap>, HashMap)> { +) -> Result<( + HashMap>, + HashMap, + HashSet, +)> { let mut imports = HashMap::new(); let mut exports = HashMap::new(); @@ -878,11 +889,15 @@ fn find_non_standard( } get_imports_exports_from_wasm(&path, &mut imports, &mut exports, true)?; } - for wasm_path in wasm_paths { - get_imports_exports_from_wasm(&wasm_path, &mut imports, &mut exports, false)?; + for wasm_path in &wasm_paths { + get_imports_exports_from_wasm(wasm_path, &mut imports, &mut exports, false)?; } - Ok((imports, exports)) + let others = wasm_paths + .difference(&exports.values().map(|p| p.clone()).collect()) + .map(|p| p.clone()) + .collect(); + Ok((imports, exports, others)) } /// package dir looks like: @@ -1026,7 +1041,8 @@ async fn compile_package( if !ignore_deps { // find non-standard imports/exports -> compositions - let (importers, exporters) = find_non_standard(package_dir, wasm_paths)?; + let (importers, exporters, others) = find_non_standard(package_dir, wasm_paths)?; + println!("{importers:?} {exporters:?} {others:?}"); // compose for (import, import_paths) in importers { @@ -1052,6 +1068,16 @@ async fn compile_package( )?; } } + + // copy others into pkg/ + for path in &others { + fs::copy( + path, + package_dir + .join("pkg") + .join(path.file_name().and_then(|f| f.to_str()).unwrap()) + )?; + } } // zip & place API inside of pkg/ to publish API From 4214f40507101396ab893a51dbde530de42962e6 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Thu, 25 Jul 2024 21:28:19 -0700 Subject: [PATCH 14/18] new: generalize `file_transfer_worker`s progress updating --- .../file_transfer_worker/src/lib.rs | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs index 76b7e995..8d64a9b9 100644 --- a/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs @@ -121,10 +121,10 @@ fn handle_worker_request( } fn handle_internal_request( - our: &Address, request: &InternalRequest, file: &mut Option, size: &mut Option, + parent: &Option
, ) -> anyhow::Result { match request { InternalRequest::Chunk(ChunkRequest { @@ -150,23 +150,21 @@ fn handle_internal_request( }; file.write_all(&bytes)?; + // if sender has sent us a size, give a progress update to main transfer + let Some(ref parent) = parent else { + return Ok(false); + }; if let Some(size) = size { let progress = ((offset + length) as f64 / *size as f64 * 100.0) as u64; - // send update to main process - let main_app = Address { - node: our.node.clone(), - process: "{package_name}:{package_name}:{publisher}".parse()?, - }; - Request::new() .expects_response(5) .body(WorkerRequest::Progress(ProgressRequest { name: name.to_string(), progress, })) - .target(&main_app) + .target(parent) .send()?; if progress >= 100 { @@ -199,11 +197,15 @@ fn handle_message( file: &mut Option, files_dir: &Directory, size: &mut Option, + parent: &mut Option
, ) -> anyhow::Result { return Ok(match message.body().try_into()? { // requests - Msg::WorkerRequest(ref wr) => handle_worker_request(wr, file, files_dir)?, - Msg::InternalRequest(ref ir) => handle_internal_request(our, ir, file, size)?, + Msg::WorkerRequest(ref wr) => { + *parent = Some(message.source().clone()); + handle_worker_request(wr, file, files_dir)? + } + Msg::InternalRequest(ref ir) => handle_internal_request(ir, file, size, parent)?, // responses Msg::WorkerResponse(ref wr) => handle_worker_response(wr)?, @@ -220,12 +222,13 @@ fn init(our: Address) { let mut file: Option = None; let mut size: Option = None; + let mut parent: Option
= None; loop { match await_message() { Err(send_error) => println!("worker: got SendError: {send_error}"), Ok(ref message) => { - match handle_message(&our, message, &mut file, &files_dir, &mut size) { + match handle_message(&our, message, &mut file, &files_dir, &mut size, &mut parent) { Ok(exit) => { if exit { println!("worker: done: exiting, took {:?}", start.elapsed()); From 76380492ac93f701a837763968fdff24aaef36c6 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 26 Jul 2024 13:21:27 -0700 Subject: [PATCH 15/18] new: remove a commented-out loc --- .../rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs index 8d64a9b9..29a3a69c 100644 --- a/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs @@ -3,7 +3,6 @@ use crate::kinode::process::file_transfer_worker::{ Response as WorkerResponse, }; use crate::kinode::process::standard::{Address as WitAddress, ProcessId as WitProcessId}; -//use crate::kinode::process::{package_name}::{Request as TransferRequest, Response as TransferResponse, WorkerRequest, DownloadRequest, ProgressRequest, ChunkRequest}; use kinode_process_lib::{ await_message, call_init, get_blob, println, vfs::{open_dir, open_file, Directory, File, SeekFrom}, From c3a6b320c0aec6db183117dfb1fbece0303b41ce Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 26 Jul 2024 14:46:17 -0700 Subject: [PATCH 16/18] build: remove debugging println --- src/build/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index 749c9666..38382f12 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -1042,7 +1042,6 @@ async fn compile_package( if !ignore_deps { // find non-standard imports/exports -> compositions let (importers, exporters, others) = find_non_standard(package_dir, wasm_paths)?; - println!("{importers:?} {exporters:?} {others:?}"); // compose for (import, import_paths) in importers { From 884fd2bdfe1fd55f5288947788406b68160db75c Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 26 Jul 2024 14:52:36 -0700 Subject: [PATCH 17/18] new: remove unused arg from file_transfer_worker handle_message --- .../rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs index 29a3a69c..f5dc1ed9 100644 --- a/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs +++ b/src/new/templates/rust/no-ui/file_transfer/file_transfer_worker/src/lib.rs @@ -191,7 +191,6 @@ fn handle_worker_response(response: &WorkerResponse) -> anyhow::Result { } fn handle_message( - our: &Address, message: &Message, file: &mut Option, files_dir: &Directory, @@ -227,7 +226,7 @@ fn init(our: Address) { match await_message() { Err(send_error) => println!("worker: got SendError: {send_error}"), Ok(ref message) => { - match handle_message(&our, message, &mut file, &files_dir, &mut size, &mut parent) { + match handle_message(message, &mut file, &files_dir, &mut size, &mut parent) { Ok(exit) => { if exit { println!("worker: done: exiting, took {:?}", start.elapsed()); From 4479da36825b6cf713450f463b0ccd8fa4693ede Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 26 Jul 2024 18:57:17 -0700 Subject: [PATCH 18/18] build: dont include deps twice --- src/build/mod.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/build/mod.rs b/src/build/mod.rs index 38382f12..7f855ffd 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -841,9 +841,7 @@ fn get_imports_exports_from_wasm( } for wit_export in wit_exports { if exports.contains_key(&wit_export) { - return Err(eyre!( - "found multiple exporters of {wit_export}: {path:?} & {exports:?}", - )); + warn!("found multiple exporters of {wit_export}: {path:?} & {exports:?}"); } let path = if should_move_export { let file_name = path @@ -872,7 +870,7 @@ fn get_imports_exports_from_wasm( #[instrument(level = "trace", skip_all)] fn find_non_standard( package_dir: &Path, - wasm_paths: HashSet, + wasm_paths: &mut HashSet, ) -> Result<( HashMap>, HashMap, @@ -884,12 +882,21 @@ fn find_non_standard( for entry in package_dir.join("pkg").read_dir()? { let entry = entry?; let path = entry.path(); + if wasm_paths.contains(&path) { + continue; + } if !(path.is_file() && Some("wasm") == path.extension().and_then(|e| e.to_str())) { continue; } get_imports_exports_from_wasm(&path, &mut imports, &mut exports, true)?; } - for wasm_path in &wasm_paths { + for export_path in exports.values() { + if wasm_paths.contains(export_path) { + // we already have it; don't include it twice + wasm_paths.remove(export_path); + } + } + for wasm_path in wasm_paths.iter() { get_imports_exports_from_wasm(wasm_path, &mut imports, &mut exports, false)?; } @@ -1041,7 +1048,7 @@ async fn compile_package( if !ignore_deps { // find non-standard imports/exports -> compositions - let (importers, exporters, others) = find_non_standard(package_dir, wasm_paths)?; + let (importers, exporters, others) = find_non_standard(package_dir, &mut wasm_paths)?; // compose for (import, import_paths) in importers {