diff --git a/Cargo.lock b/Cargo.lock index 837f92161..581a9fe63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,7 +250,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "benchmark-example-node" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-node-api", "eyre", @@ -263,7 +263,7 @@ dependencies = [ [[package]] name = "benchmark-example-sink" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-node-api", "eyre", @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "communication-layer-pub-sub" -version = "0.2.1" +version = "0.2.2" dependencies = [ "flume", "zenoh", @@ -521,7 +521,7 @@ dependencies = [ [[package]] name = "communication-layer-request-reply" -version = "0.2.1" +version = "0.2.2" [[package]] name = "concurrent-queue" @@ -817,7 +817,7 @@ dependencies = [ [[package]] name = "dora-cli" -version = "0.2.1" +version = "0.2.2" dependencies = [ "atty", "clap 4.0.3", @@ -841,7 +841,7 @@ dependencies = [ [[package]] name = "dora-coordinator" -version = "0.2.1" +version = "0.2.2" dependencies = [ "ctrlc", "dora-core", @@ -864,7 +864,7 @@ dependencies = [ [[package]] name = "dora-core" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-message", "eyre", @@ -880,7 +880,7 @@ dependencies = [ [[package]] name = "dora-daemon" -version = "0.2.1" +version = "0.2.2" dependencies = [ "async-trait", "bincode", @@ -906,7 +906,7 @@ dependencies = [ [[package]] name = "dora-download" -version = "0.2.1" +version = "0.2.2" dependencies = [ "eyre", "reqwest", @@ -931,7 +931,7 @@ dependencies = [ [[package]] name = "dora-message" -version = "0.2.1" +version = "0.2.2" dependencies = [ "capnp", "capnpc", @@ -941,7 +941,7 @@ dependencies = [ [[package]] name = "dora-metrics" -version = "0.2.1" +version = "0.2.2" dependencies = [ "futures", "opentelemetry 0.17.0", @@ -952,7 +952,7 @@ dependencies = [ [[package]] name = "dora-node-api" -version = "0.2.1" +version = "0.2.2" dependencies = [ "bincode", "capnp", @@ -974,7 +974,7 @@ dependencies = [ [[package]] name = "dora-node-api-c" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-node-api", "eyre", @@ -984,7 +984,7 @@ dependencies = [ [[package]] name = "dora-node-api-cxx" -version = "0.2.1" +version = "0.2.2" dependencies = [ "cxx", "cxx-build", @@ -994,7 +994,7 @@ dependencies = [ [[package]] name = "dora-node-api-python" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-node-api", "dora-operator-api-python", @@ -1007,7 +1007,7 @@ dependencies = [ [[package]] name = "dora-operator-api" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-operator-api-macros", "dora-operator-api-types", @@ -1015,14 +1015,14 @@ dependencies = [ [[package]] name = "dora-operator-api-c" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-operator-api-types", ] [[package]] name = "dora-operator-api-cxx" -version = "0.2.1" +version = "0.2.2" dependencies = [ "cxx", "cxx-build", @@ -1031,7 +1031,7 @@ dependencies = [ [[package]] name = "dora-operator-api-macros" -version = "0.2.1" +version = "0.2.2" dependencies = [ "proc-macro2", "quote", @@ -1040,7 +1040,7 @@ dependencies = [ [[package]] name = "dora-operator-api-python" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-node-api", "eyre", @@ -1051,14 +1051,14 @@ dependencies = [ [[package]] name = "dora-operator-api-types" -version = "0.2.1" +version = "0.2.2" dependencies = [ "safer-ffi", ] [[package]] name = "dora-runtime" -version = "0.2.1" +version = "0.2.2" dependencies = [ "clap 4.0.3", "dora-core", @@ -1086,7 +1086,7 @@ dependencies = [ [[package]] name = "dora-tracing" -version = "0.2.1" +version = "0.2.2" dependencies = [ "eyre", "opentelemetry 0.18.0", @@ -3097,7 +3097,7 @@ dependencies = [ [[package]] name = "rust-dataflow-example-node" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-node-api", "eyre", @@ -3108,14 +3108,14 @@ dependencies = [ [[package]] name = "rust-dataflow-example-operator" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-operator-api", ] [[package]] name = "rust-dataflow-example-sink" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dora-node-api", "eyre", @@ -3400,7 +3400,7 @@ dependencies = [ [[package]] name = "shared-memory-server" -version = "0.2.1" +version = "0.2.2" dependencies = [ "bincode", "eyre", @@ -4804,7 +4804,7 @@ dependencies = [ [[package]] name = "zenoh-logger" -version = "0.2.1" +version = "0.2.2" dependencies = [ "zenoh", ] diff --git a/Cargo.toml b/Cargo.toml index f5de7e860..ea907ef70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,28 +24,28 @@ members = [ [workspace.package] # Make sure to also bump `apis/node/python/__init__.py` version. -version = "0.2.1" +version = "0.2.2" description = "`dora` goal is to be a low latency, composable, and distributed data flow." documentation = "https://dora.carsmos.ai/dora/" license = "Apache-2.0" [workspace.dependencies] -dora-node-api = { version = "0.2.1", path = "apis/rust/node", default-features = false } -dora-operator-api = { version = "0.2.1", path = "apis/rust/operator", default-features = false } -dora-operator-api-macros = { version = "0.2.1", path = "apis/rust/operator/macros" } -dora-operator-api-types = { version = "0.2.1", path = "apis/rust/operator/types" } -dora-operator-api-python = { version = "0.2.1", path = "apis/python/operator" } -dora-operator-api-c = { version = "0.2.1", path = "apis/c/operator" } -dora-node-api-c = { version = "0.2.1", path = "apis/c/node" } -dora-core = { version = "0.2.1", path = "libraries/core" } -dora-tracing = { version = "0.2.1", path = "libraries/extensions/telemetry/tracing" } -dora-metrics = { version = "0.2.1", path = "libraries/extensions/telemetry/metrics" } -dora-download = { version = "0.2.1", path = "libraries/extensions/download" } -shared-memory-server = { version = "0.2.1", path = "libraries/shared-memory-server" } -communication-layer-request-reply = { version = "0.2.1", path = "libraries/communication-layer/request-reply" } -dora-message = { version = "0.2.1", path = "libraries/message" } -dora-runtime = { version = "0.2.1", path = "binaries/runtime" } -dora-daemon = { version = "0.2.1", path = "binaries/daemon" } +dora-node-api = { version = "0.2.2", path = "apis/rust/node", default-features = false } +dora-operator-api = { version = "0.2.2", path = "apis/rust/operator", default-features = false } +dora-operator-api-macros = { version = "0.2.2", path = "apis/rust/operator/macros" } +dora-operator-api-types = { version = "0.2.2", path = "apis/rust/operator/types" } +dora-operator-api-python = { version = "0.2.2", path = "apis/python/operator" } +dora-operator-api-c = { version = "0.2.2", path = "apis/c/operator" } +dora-node-api-c = { version = "0.2.2", path = "apis/c/node" } +dora-core = { version = "0.2.2", path = "libraries/core" } +dora-tracing = { version = "0.2.2", path = "libraries/extensions/telemetry/tracing" } +dora-metrics = { version = "0.2.2", path = "libraries/extensions/telemetry/metrics" } +dora-download = { version = "0.2.2", path = "libraries/extensions/download" } +shared-memory-server = { version = "0.2.2", path = "libraries/shared-memory-server" } +communication-layer-request-reply = { version = "0.2.2", path = "libraries/communication-layer/request-reply" } +dora-message = { version = "0.2.2", path = "libraries/message" } +dora-runtime = { version = "0.2.2", path = "binaries/runtime" } +dora-daemon = { version = "0.2.2", path = "binaries/daemon" } [package] name = "dora-examples" diff --git a/Changelog.md b/Changelog.md index 7cf78e8e2..d6f4f5511 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,25 @@ # Changelog +## v0.2.2 (2023-04-01) + +### Features + +* Make queue length configurable through the dataflow file by @phil-opp in https://github.com/dora-rs/dora/pull/231 +* Hot reloading Python Operator by @haixuanTao in https://github.com/dora-rs/dora/pull/239 +* Synchronize node and operator start by @phil-opp in https://github.com/dora-rs/dora/pull/236 +* Add opentelemetry capability at runtime instead of compile time by @haixuanTao in https://github.com/dora-rs/dora/pull/234 + +### Others + +* Wait on events and messages simultaneously to prevent queue buildup by @phil-opp in https://github.com/dora-rs/dora/pull/235 +* Fix looping in daemon listener loop by @phil-opp in https://github.com/dora-rs/dora/pull/244 +* Validate shell command as source and url source by @haixuanTao in https://github.com/dora-rs/dora/pull/243 +* Push error into the `init_done` channel for debugging context by @haixuanTao in https://github.com/dora-rs/dora/pull/238 +* Option communication config by @haixuanTao in https://github.com/dora-rs/dora/pull/241 +* Validate yaml when reading by @haixuanTao in https://github.com/dora-rs/dora/pull/237 + +**Full Changelog**: https://github.com/dora-rs/dora/compare/v0.2.1...v0.2.2 + ## v0.2.1 (2023-03-22) ### Features diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py index 0674c0190..24b0da0ea 100644 --- a/apis/python/node/dora/__init__.py +++ b/apis/python/node/dora/__init__.py @@ -3,7 +3,7 @@ from .dora import * __author__ = "Dora-rs Authors" -__version__ = "0.2.1" +__version__ = "0.2.2" class DoraStatus(Enum): diff --git a/binaries/daemon/src/spawn.rs b/binaries/daemon/src/spawn.rs index 271b74a5d..b079783d0 100644 --- a/binaries/daemon/src/spawn.rs +++ b/binaries/daemon/src/spawn.rs @@ -5,15 +5,13 @@ use crate::{ use dora_core::{ config::NodeRunConfig, daemon_messages::{DaemonCommunicationConfig, DataflowId, NodeConfig, RuntimeConfig}, - descriptor::{resolve_path, source_is_url, OperatorSource, ResolvedNode}, + descriptor::{resolve_path, source_is_url, OperatorSource, ResolvedNode, SHELL_SOURCE}, }; use dora_download::download_file; use eyre::WrapErr; use std::{env::consts::EXE_EXTENSION, path::Path, process::Stdio}; use tokio::sync::mpsc; -const SHELL_SOURCE: &str = "shell"; - pub async fn spawn_node( dataflow_id: DataflowId, working_dir: &Path, diff --git a/libraries/core/src/descriptor/mod.rs b/libraries/core/src/descriptor/mod.rs index cd5b52459..3f506aeb9 100644 --- a/libraries/core/src/descriptor/mod.rs +++ b/libraries/core/src/descriptor/mod.rs @@ -15,11 +15,13 @@ pub use visualize::collect_dora_timers; mod validate; mod visualize; +pub const SHELL_SOURCE: &str = "shell"; #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Descriptor { // see https://github.com/dtolnay/serde-yaml/issues/298 + #[serde(default)] #[serde(with = "serde_yaml::with::singleton_map")] pub communication: Option, pub nodes: Vec, diff --git a/libraries/core/src/descriptor/validate.rs b/libraries/core/src/descriptor/validate.rs index 8ce90e995..8105d3bac 100644 --- a/libraries/core/src/descriptor/validate.rs +++ b/libraries/core/src/descriptor/validate.rs @@ -10,8 +10,9 @@ use std::{ path::{Path, PathBuf}, process::Command, }; +use tracing::info; -use super::Descriptor; +use super::{Descriptor, SHELL_SOURCE}; const VERSION: &str = env!("CARGO_PKG_VERSION"); pub fn check_dataflow( @@ -27,28 +28,31 @@ pub fn check_dataflow( // check that nodes and operators exist for node in &nodes { match &node.kind { - descriptor::CoreNodeKind::Custom(node) => { - let path = if source_is_url(&node.source) { - todo!("check URL"); - } else { - let raw = Path::new(&node.source); - if raw.extension().is_none() { - raw.with_extension(EXE_EXTENSION) + descriptor::CoreNodeKind::Custom(node) => match node.source.as_str() { + SHELL_SOURCE => (), + source => { + if source_is_url(source) { + info!("{source} is a URL."); // TODO: Implement url check. } else { - raw.to_owned() - } - }; - base.join(&path) - .canonicalize() - .wrap_err_with(|| format!("no node exists at `{}`", path.display()))?; - } + let raw = Path::new(source); + let path = if raw.extension().is_none() { + raw.with_extension(EXE_EXTENSION) + } else { + raw.to_owned() + }; + base.join(&path) + .canonicalize() + .wrap_err_with(|| format!("no node exists at `{}`", path.display()))?; + }; + } + }, descriptor::CoreNodeKind::Runtime(node) => { for operator_definition in &node.operators { match &operator_definition.config.source { OperatorSource::SharedLibrary(path) => { has_shared_lib_operator = true; if source_is_url(path) { - todo!("check URL"); + info!("{path} is a URL."); // TODO: Implement url check. } else { let path = adjust_shared_library_path(Path::new(&path))?; if !base.join(&path).exists() { @@ -59,14 +63,14 @@ pub fn check_dataflow( OperatorSource::Python(path) => { has_python_operator = true; if source_is_url(path) { - todo!("check URL"); + info!("{path} is a URL."); // TODO: Implement url check. } else if !base.join(path).exists() { bail!("no Python library at `{path}`"); } } OperatorSource::Wasm(path) => { if source_is_url(path) { - todo!("check URL"); + info!("{path} is a URL."); // TODO: Implement url check. } else if !base.join(path).exists() { bail!("no WASM library at `{path}`"); }