From 649886116fe4e1383fd34b1ee96b7403e0fe970a Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 28 Mar 2024 08:13:19 -0400 Subject: [PATCH 1/6] Initial working version of update-markdown - Create new helper method to deal with possible git-url for YAML files. - Implement directory walking. --- Cargo.lock | 2 ++ Cargo.toml | 1 + crates/weaver_semconv_gen/Cargo.toml | 1 + crates/weaver_semconv_gen/src/lib.rs | 26 ++++++++++++++++++++ src/registry/update_markdown.rs | 36 +++++++++++++++++++++++++--- 5 files changed, 63 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fe13031..7410c9e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4016,6 +4016,7 @@ dependencies = [ "serde_yaml", "tantivy", "tui-textarea", + "walkdir", "weaver_cache", "weaver_forge", "weaver_logger", @@ -4139,6 +4140,7 @@ dependencies = [ "similar", "thiserror", "walkdir", + "weaver_cache", "weaver_logger", "weaver_resolved_schema", "weaver_resolver", diff --git a/Cargo.toml b/Cargo.toml index f40357ca..72df2439 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ crossterm = "0.27.0" ratatui = "0.26.1" tui-textarea = "0.4.0" tantivy = "0.21.1" +walkdir = "2.5.0" # workspace dependencies serde.workspace = true diff --git a/crates/weaver_semconv_gen/Cargo.toml b/crates/weaver_semconv_gen/Cargo.toml index adc04e05..f3867cb9 100644 --- a/crates/weaver_semconv_gen/Cargo.toml +++ b/crates/weaver_semconv_gen/Cargo.toml @@ -9,6 +9,7 @@ edition.workspace = true rust-version.workspace = true [dependencies] +weaver_cache = { path = "../weaver_cache" } weaver_logger = { path = "../weaver_logger" } weaver_resolver = { path = "../weaver_resolver" } weaver_resolved_schema = { path = "../weaver_resolved_schema" } diff --git a/crates/weaver_semconv_gen/src/lib.rs b/crates/weaver_semconv_gen/src/lib.rs index 1e193afe..adf872a8 100644 --- a/crates/weaver_semconv_gen/src/lib.rs +++ b/crates/weaver_semconv_gen/src/lib.rs @@ -6,6 +6,7 @@ use std::fs; use weaver_logger::Logger; +use weaver_cache::Cache; use weaver_resolved_schema::attribute::{Attribute, AttributeRef}; use weaver_resolved_schema::registry::{Group, Registry}; use weaver_resolved_schema::ResolvedTelemetrySchema; @@ -214,6 +215,31 @@ impl ResolvedSemconvRegistry { Ok(lookup) } + /// Resolve semconv registry (possibly from git), and make it available for rendering. + pub fn try_from_url( + // Local or GIT URL of semconv registry. + registry: String, + // Optional path where YAML files are located (default: model) + registry_sub_dir: Option, + cache: &Cache, + log: impl Logger + Clone + Sync, + ) -> Result { + let registry_id = "semantic_conventions"; + let mut registry = SchemaResolver::load_semconv_registry( + registry_id, + registry, + registry_sub_dir, + cache, + log.clone(), + )?; + let schema = SchemaResolver::resolve_semantic_convention_registry(&mut registry, log)?; + let lookup = ResolvedSemconvRegistry { + schema, + registry_id: registry_id.into(), + }; + Ok(lookup) + } + fn my_registry(&self) -> Option<&Registry> { self.schema.registry(self.registry_id.as_str()) } diff --git a/src/registry/update_markdown.rs b/src/registry/update_markdown.rs index 2b133966..b57ddc74 100644 --- a/src/registry/update_markdown.rs +++ b/src/registry/update_markdown.rs @@ -6,6 +6,7 @@ use clap::Args; use weaver_cache::Cache; use weaver_logger::Logger; +use weaver_semconv_gen::{ResolvedSemconvRegistry, update_markdown}; /// Parameters for the `registry update-markdown` sub-command #[derive(Debug, Args)] @@ -25,13 +26,42 @@ pub struct RegistryUpdateMarkdownArgs { /// registry is located #[arg(short = 'd', long, default_value = "model")] pub registry_git_sub_dir: Option, + + /// Whether or not to run updates in dry-run mode. + #[arg(long, default_value="false")] + pub dry_run: bool, } /// Update markdown files. pub(crate) fn command( - _log: impl Logger + Sync + Clone, - _cache: &Cache, + log: impl Logger + Sync + Clone, + cache: &Cache, args: &RegistryUpdateMarkdownArgs, ) { - println!("Args: {:#?}", args); + let registry = + ResolvedSemconvRegistry::try_from_url(args.registry.clone(), args.registry_git_sub_dir.clone(), cache, log.clone()) + .unwrap_or_else(|e| { + panic!("Failed to resolve the semantic convention registry.\n{e}"); + }); + log.success("Registry resolved succesfully"); + fn is_markdown(entry: &walkdir::DirEntry) -> bool { + let path = entry.path(); + let extension = path.extension().unwrap_or_else(|| std::ffi::OsStr::new("")); + path.is_file() && extension == "md" + } + let operation = if args.dry_run { "Validating" } else { "Updating" }; + for entry in + walkdir::WalkDir::new(args.markdown_dir.clone()) + .into_iter() + .filter_map(|e| { + match e { + Ok(v) if is_markdown(&v) => Some(v), + _ => None, + } + }) { + log.info(&format!("{}: ${}", operation, entry.path().display())); + if let Err(error) = update_markdown(&entry.path().display().to_string(), ®istry, args.dry_run) { + log.error(&format!("{error}")); + } + } } From b6dcbb7ec0bd915e1b73dceeb7ee66bf84ea3f09 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 28 Mar 2024 08:21:50 -0400 Subject: [PATCH 2/6] Clean up style and ensure command lline failure --- Cargo.toml | 2 +- crates/weaver_semconv_gen/src/lib.rs | 2 +- src/registry/update_markdown.rs | 45 +++++++++++++++++----------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 72df2439..6b41e1b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,12 +64,12 @@ crossterm = "0.27.0" ratatui = "0.26.1" tui-textarea = "0.4.0" tantivy = "0.21.1" -walkdir = "2.5.0" # workspace dependencies serde.workspace = true serde_yaml.workspace = true serde_json.workspace = true +walkdir.workspace = true [package.metadata.cargo-machete] # force cargo machete to ignore the following crates diff --git a/crates/weaver_semconv_gen/src/lib.rs b/crates/weaver_semconv_gen/src/lib.rs index adf872a8..84301f1c 100644 --- a/crates/weaver_semconv_gen/src/lib.rs +++ b/crates/weaver_semconv_gen/src/lib.rs @@ -5,8 +5,8 @@ //! poorly porting the code into RUST. We expect to optimise and improve things over time. use std::fs; -use weaver_logger::Logger; use weaver_cache::Cache; +use weaver_logger::Logger; use weaver_resolved_schema::attribute::{Attribute, AttributeRef}; use weaver_resolved_schema::registry::{Group, Registry}; use weaver_resolved_schema::ResolvedTelemetrySchema; diff --git a/src/registry/update_markdown.rs b/src/registry/update_markdown.rs index b57ddc74..fc335100 100644 --- a/src/registry/update_markdown.rs +++ b/src/registry/update_markdown.rs @@ -6,7 +6,7 @@ use clap::Args; use weaver_cache::Cache; use weaver_logger::Logger; -use weaver_semconv_gen::{ResolvedSemconvRegistry, update_markdown}; +use weaver_semconv_gen::{update_markdown, ResolvedSemconvRegistry}; /// Parameters for the `registry update-markdown` sub-command #[derive(Debug, Args)] @@ -28,7 +28,7 @@ pub struct RegistryUpdateMarkdownArgs { pub registry_git_sub_dir: Option, /// Whether or not to run updates in dry-run mode. - #[arg(long, default_value="false")] + #[arg(long, default_value = "false")] pub dry_run: bool, } @@ -38,30 +38,41 @@ pub(crate) fn command( cache: &Cache, args: &RegistryUpdateMarkdownArgs, ) { - let registry = - ResolvedSemconvRegistry::try_from_url(args.registry.clone(), args.registry_git_sub_dir.clone(), cache, log.clone()) - .unwrap_or_else(|e| { - panic!("Failed to resolve the semantic convention registry.\n{e}"); - }); + let registry = ResolvedSemconvRegistry::try_from_url( + args.registry.clone(), + args.registry_git_sub_dir.clone(), + cache, + log.clone(), + ) + .unwrap_or_else(|e| { + panic!("Failed to resolve the semantic convention registry.\n{e}"); + }); log.success("Registry resolved succesfully"); fn is_markdown(entry: &walkdir::DirEntry) -> bool { let path = entry.path(); let extension = path.extension().unwrap_or_else(|| std::ffi::OsStr::new("")); path.is_file() && extension == "md" } - let operation = if args.dry_run { "Validating" } else { "Updating" }; - for entry in - walkdir::WalkDir::new(args.markdown_dir.clone()) - .into_iter() - .filter_map(|e| { - match e { + let operation = if args.dry_run { + "Validating" + } else { + "Updating" + }; + let mut has_error = false; + for entry in walkdir::WalkDir::new(args.markdown_dir.clone()) + .into_iter() + .filter_map(|e| match e { Ok(v) if is_markdown(&v) => Some(v), _ => None, - } - }) { + }) + { log.info(&format!("{}: ${}", operation, entry.path().display())); - if let Err(error) = update_markdown(&entry.path().display().to_string(), ®istry, args.dry_run) { + if let Err(error) = + update_markdown(&entry.path().display().to_string(), ®istry, args.dry_run) + { + has_error = true; log.error(&format!("{error}")); } - } + } + panic!("weaver registry update-markdown failed.") } From bd27a4e18b320e9a325d1809bc8888a9fa7fd692 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 28 Mar 2024 08:33:24 -0400 Subject: [PATCH 3/6] Update external types to allow cache --- crates/weaver_semconv_gen/allowed-external-types.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/weaver_semconv_gen/allowed-external-types.toml b/crates/weaver_semconv_gen/allowed-external-types.toml index a3eb6a1f..782de5f2 100644 --- a/crates/weaver_semconv_gen/allowed-external-types.toml +++ b/crates/weaver_semconv_gen/allowed-external-types.toml @@ -6,4 +6,5 @@ allowed_external_types = [ "weaver_semconv::Error", "weaver_resolver::Error", "weaver_logger::Logger", + "weaver_cache::Cache", ] From 980ba60a218692378411deca10b321e6e2f49bfe Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 28 Mar 2024 13:56:00 -0400 Subject: [PATCH 4/6] Update src/registry/update_markdown.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Laurent QuĂ©rel --- src/registry/update_markdown.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registry/update_markdown.rs b/src/registry/update_markdown.rs index fc335100..2f42c70a 100644 --- a/src/registry/update_markdown.rs +++ b/src/registry/update_markdown.rs @@ -47,7 +47,7 @@ pub(crate) fn command( .unwrap_or_else(|e| { panic!("Failed to resolve the semantic convention registry.\n{e}"); }); - log.success("Registry resolved succesfully"); + log.success("Registry resolved successfully"); fn is_markdown(entry: &walkdir::DirEntry) -> bool { let path = entry.path(); let extension = path.extension().unwrap_or_else(|| std::ffi::OsStr::new("")); From 6f35bf6333fde8af8111026ea95234db81b9f481 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 28 Mar 2024 13:57:04 -0400 Subject: [PATCH 5/6] Fix command to only panic on failure --- src/registry/update_markdown.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/registry/update_markdown.rs b/src/registry/update_markdown.rs index 2f42c70a..28355b7c 100644 --- a/src/registry/update_markdown.rs +++ b/src/registry/update_markdown.rs @@ -74,5 +74,7 @@ pub(crate) fn command( log.error(&format!("{error}")); } } - panic!("weaver registry update-markdown failed.") + if has_error { + panic!("weaver registry update-markdown failed."); + } } From 7e1039ceaae7ad869d333f5ee13558126e0a5ef0 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 28 Mar 2024 19:03:17 -0400 Subject: [PATCH 6/6] Fix cargo format --- src/registry/update_markdown.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registry/update_markdown.rs b/src/registry/update_markdown.rs index 28355b7c..cf6f59db 100644 --- a/src/registry/update_markdown.rs +++ b/src/registry/update_markdown.rs @@ -75,6 +75,6 @@ pub(crate) fn command( } } if has_error { - panic!("weaver registry update-markdown failed."); + panic!("weaver registry update-markdown failed."); } }