Skip to content

Commit

Permalink
feat(semconv): add support for both local path and git url in registr…
Browse files Browse the repository at this point in the history
…y commands.
  • Loading branch information
lquerel committed Apr 2, 2024
1 parent 36dfe67 commit 355c59f
Show file tree
Hide file tree
Showing 14 changed files with 195 additions and 128 deletions.
167 changes: 80 additions & 87 deletions crates/weaver_resolver/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ use weaver_logger::Logger;
use weaver_resolved_schema::catalog::Catalog;
use weaver_resolved_schema::registry::Constraint;
use weaver_resolved_schema::ResolvedTelemetrySchema;
use weaver_schema::{SemConvImport, TelemetrySchema};
use weaver_schema::TelemetrySchema;
use weaver_semconv::path::RegistryPath;
use weaver_semconv::{ResolverConfig, SemConvRegistry, SemConvSpec, SemConvSpecWithProvenance};
use weaver_version::VersionChanges;

Expand Down Expand Up @@ -301,7 +302,7 @@ impl SchemaResolver {
) -> Result<SemConvRegistry, Error> {
Self::semconv_registry_from_imports(
registry_id,
&[SemConvImport::GitUrl {
&[RegistryPath::GitUrl {
git_url: registry_git_url,
path,
}],
Expand All @@ -314,20 +315,11 @@ impl SchemaResolver {
/// Loads a semantic convention registry from the given Git URL.
pub fn load_semconv_registry(
registry_id: &str,
registry_git_url: String,
path: Option<String>,
registry_path: RegistryPath,
cache: &Cache,
log: impl Logger + Clone + Sync,
) -> Result<SemConvRegistry, Error> {
Self::load_semconv_registry_from_imports(
registry_id,
&[SemConvImport::GitUrl {
git_url: registry_git_url,
path,
}],
cache,
log.clone(),
)
Self::load_semconv_registry_from_imports(registry_id, &[registry_path], cache, log.clone())
}

/// Loads a telemetry schema from the given URL or path.
Expand Down Expand Up @@ -401,7 +393,7 @@ impl SchemaResolver {
/// Loads a semantic convention registry from the given semantic convention imports.
pub fn load_semconv_registry_from_imports(
registry_id: &str,
imports: &[SemConvImport],
imports: &[RegistryPath],
cache: &Cache,
log: impl Logger + Clone + Sync,
) -> Result<SemConvRegistry, Error> {
Expand All @@ -422,7 +414,7 @@ impl SchemaResolver {
/// Loads a semantic convention registry from the given semantic convention imports.
pub fn semconv_registry_from_imports(
registry_id: &str,
imports: &[SemConvImport],
imports: &[RegistryPath],
resolver_config: ResolverConfig,
cache: &Cache,
log: impl Logger + Clone + Sync,
Expand Down Expand Up @@ -545,7 +537,7 @@ impl SchemaResolver {
/// Creates a semantic convention registry from the given telemetry schema.
fn create_semantic_convention_registry(
registry_id: &str,
sem_convs: &[SemConvImport],
sem_convs: &[RegistryPath],
cache: &Cache,
log: impl Logger + Sync,
) -> Result<SemConvRegistry, Error> {
Expand Down Expand Up @@ -600,90 +592,91 @@ impl SchemaResolver {
Ok(sem_conv_catalog)
}

/// Imports the semantic convention specifications from the given import declaration.
/// Imports the semantic convention specifications from the given registry path.
/// This function returns a vector of results because the import declaration can be a
/// URL or a git URL (containing potentially multiple semantic convention specifications).
fn import_sem_conv_specs(
import_decl: &SemConvImport,
registry_path: &RegistryPath,
cache: &Cache,
) -> Vec<Result<(String, SemConvSpec), Error>> {
match import_decl {
SemConvImport::Url { url } => {
let spec = SemConvRegistry::load_sem_conv_spec_from_url(url).map_err(|e| {
Error::SemConvError {
message: e.to_string(),
match registry_path {
RegistryPath::Local { path } => Self::import_semconv_from_local_path(path.into(), path),
RegistryPath::GitUrl { git_url, path } => {
match cache.git_repo(git_url.clone(), path.clone()) {
Ok(local_git_repo) => {
Self::import_semconv_from_local_path(local_git_repo, git_url)
}
});
vec![spec]
}
SemConvImport::GitUrl { git_url, path } => {
fn is_hidden(entry: &DirEntry) -> bool {
entry
.file_name()
.to_str()
.map(|s| s.starts_with('.'))
.unwrap_or(false)
}
fn is_semantic_convention_file(entry: &DirEntry) -> bool {
let path = entry.path();
let extension = path.extension().unwrap_or_else(|| std::ffi::OsStr::new(""));
let file_name = path.file_name().unwrap_or_else(|| std::ffi::OsStr::new(""));
path.is_file()
&& (extension == "yaml" || extension == "yml")
&& file_name != "schema-next.yaml"
Err(e) => vec![Err(Error::SemConvError {
message: e.to_string(),
})],
}
}
}
}

let mut result = vec![];
let git_repo = cache.git_repo(git_url.clone(), path.clone()).map_err(|e| {
Error::SemConvError {
message: e.to_string(),
}
});

if let Ok(git_repo) = git_repo {
// Loads the semantic convention specifications from the git repo.
// All yaml files are recursively loaded from the given path.
for entry in walkdir::WalkDir::new(git_repo.clone())
.into_iter()
.filter_entry(|e| !is_hidden(e))
{
match entry {
Ok(entry) => {
if is_semantic_convention_file(&entry) {
let spec =
SemConvRegistry::load_sem_conv_spec_from_file(entry.path())
.map_err(|e| Error::SemConvError {
message: e.to_string(),
});
result.push(match spec {
Ok((path, spec)) => {
// Replace the local path with the git URL combined with the relative path
// of the semantic convention file.
let prefix = git_repo
.to_str()
.map(|s| s.to_owned())
.unwrap_or_default();
let path = format!(
"{}/{}",
git_url,
&path[prefix.len() + 1..]
);
Ok((path, spec))
}
Err(e) => Err(e),
});
}
}
Err(e) => result.push(Err(Error::SemConvError {
/// Imports the semantic convention specifications from the given local path.
///
/// # Arguments
/// * `local_path` - The local path containing the semantic convention files.
/// * `registry_path_repr` - The representation of the registry path (URL or path).
fn import_semconv_from_local_path(
local_path: PathBuf,
registry_path_repr: &str,
) -> Vec<Result<(String, SemConvSpec), Error>> {
fn is_hidden(entry: &DirEntry) -> bool {
entry
.file_name()
.to_str()
.map(|s| s.starts_with('.'))
.unwrap_or(false)
}
fn is_semantic_convention_file(entry: &DirEntry) -> bool {
let path = entry.path();
let extension = path.extension().unwrap_or_else(|| std::ffi::OsStr::new(""));
let file_name = path.file_name().unwrap_or_else(|| std::ffi::OsStr::new(""));
path.is_file()
&& (extension == "yaml" || extension == "yml")
&& file_name != "schema-next.yaml"
}

let mut result = vec![];

// Loads the semantic convention specifications from the git repo.
// All yaml files are recursively loaded from the given path.
for entry in walkdir::WalkDir::new(local_path.clone())
.into_iter()
.filter_entry(|e| !is_hidden(e))
{
match entry {
Ok(entry) => {
if is_semantic_convention_file(&entry) {
let spec = SemConvRegistry::load_sem_conv_spec_from_file(entry.path())
.map_err(|e| Error::SemConvError {
message: e.to_string(),
})),
}
});
result.push(match spec {
Ok((path, spec)) => {
// Replace the local path with the git URL combined with the relative path
// of the semantic convention file.
let prefix = local_path
.to_str()
.map(|s| s.to_owned())
.unwrap_or_default();
let path =
format!("{}/{}", registry_path_repr, &path[prefix.len() + 1..]);
Ok((path, spec))
}
Err(e) => Err(e),
});
}
}

result
Err(e) => result.push(Err(Error::SemConvError {
message: e.to_string(),
})),
}
}

result
}
}

Expand Down
5 changes: 3 additions & 2 deletions crates/weaver_schema/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use std::path::Path;

use serde::{Deserialize, Serialize};
use url::Url;
use weaver_semconv::path::RegistryPath;

use weaver_semconv::SemConvRegistry;
use weaver_version::Versions;
Expand Down Expand Up @@ -86,7 +87,7 @@ pub struct TelemetrySchema {
/// The semantic conventions that are imported by the current schema (optional).
#[serde(default)]
#[serde(skip_serializing_if = "Vec::is_empty")]
pub semantic_conventions: Vec<SemConvImport>,
pub semantic_conventions: Vec<RegistryPath>,
/// Definition of the telemetry schema for an application or a library.
#[serde(skip_serializing_if = "Option::is_none")]
pub schema: Option<SchemaSpec>,
Expand Down Expand Up @@ -209,7 +210,7 @@ impl TelemetrySchema {

/// Returns the semantic conventions for the schema and its parent schemas.
#[must_use]
pub fn merged_semantic_conventions(&self) -> Vec<SemConvImport> {
pub fn merged_semantic_conventions(&self) -> Vec<RegistryPath> {
let mut result = vec![];
if let Some(parent_schema) = self.parent_schema.as_ref() {
result.extend(parent_schema.merged_semantic_conventions().iter().cloned());
Expand Down
1 change: 1 addition & 0 deletions crates/weaver_semconv/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ use crate::metric::MetricSpec;
pub mod attribute;
pub mod group;
pub mod metric;
pub mod path;
pub mod stability;

/// An error that can occur while loading a semantic convention registry.
Expand Down
23 changes: 23 additions & 0 deletions crates/weaver_semconv/src/path.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// SPDX-License-Identifier: Apache-2.0

//! Semantic convention registry path.
use serde::{Deserialize, Serialize};

/// A semantic convention registry path.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RegistryPath {
/// A local path to the semantic convention registry.
Local {
/// The local path to the semantic convention directory.
path: String,
},
/// A git URL to the semantic convention registry.
GitUrl {
/// The git URL of the semantic convention git repo.
git_url: String,
/// An optional path to the semantic convention directory containing
/// the semantic convention files.
path: Option<String>,
},
}
15 changes: 4 additions & 11 deletions crates/weaver_semconv_gen/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use weaver_resolved_schema::attribute::{Attribute, AttributeRef};
use weaver_resolved_schema::registry::{Group, Registry};
use weaver_resolved_schema::ResolvedTelemetrySchema;
use weaver_resolver::SchemaResolver;
use weaver_semconv::path::RegistryPath;
use weaver_semconv::SemConvRegistry;

mod diff;
Expand Down Expand Up @@ -217,21 +218,13 @@ impl ResolvedSemconvRegistry {

/// Resolve semconv registry (possibly from git), and make it available for rendering.
pub fn try_from_url(
// Local or GIT URL of semconv registry.
registry: String,
// Optional path where YAML files are located (default: model)
registry_sub_dir: Option<String>,
registry_path: RegistryPath,
cache: &Cache,
log: impl Logger + Clone + Sync,
) -> Result<ResolvedSemconvRegistry, Error> {
let registry_id = "semantic_conventions";
let mut registry = SchemaResolver::load_semconv_registry(
registry_id,
registry,
registry_sub_dir,
cache,
log.clone(),
)?;
let mut registry =
SchemaResolver::load_semconv_registry(registry_id, registry_path, cache, log.clone())?;
let schema = SchemaResolver::resolve_semantic_convention_registry(&mut registry, log)?;
let lookup = ResolvedSemconvRegistry {
schema,
Expand Down
9 changes: 5 additions & 4 deletions src/registry/check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

//! Check a semantic convention registry.
use crate::registry::{semconv_registry_path_from, RegistryPath};
use clap::Args;
use weaver_cache::Cache;
use weaver_logger::Logger;
Expand All @@ -18,7 +19,7 @@ pub struct RegistryCheckArgs {
long,
default_value = "https://github.com/open-telemetry/semantic-conventions.git"
)]
pub registry: String,
pub registry: RegistryPath,

/// Optional path in the Git repository where the semantic convention
/// registry is located
Expand All @@ -36,8 +37,7 @@ pub(crate) fn command(log: impl Logger + Sync + Clone, cache: &Cache, args: &Reg
// No parsing errors should be observed.
let semconv_specs = SchemaResolver::load_semconv_registry(
registry_id,
args.registry.to_string(),
args.registry_git_sub_dir.clone(),
semconv_registry_path_from(&args.registry, &args.registry_git_sub_dir),
cache,
log.clone(),
)
Expand All @@ -47,7 +47,8 @@ pub(crate) fn command(log: impl Logger + Sync + Clone, cache: &Cache, args: &Reg

// Resolve the semantic convention registry.
let mut attr_catalog = AttributeCatalog::default();
let _ = resolve_semconv_registry(&mut attr_catalog, &args.registry, &semconv_specs)
let registry_path = args.registry.to_string();
let _ = resolve_semconv_registry(&mut attr_catalog, &registry_path, &semconv_specs)
.unwrap_or_else(|e| {
panic!("Failed to resolve the semantic convention registry.\n{e}");
});
Expand Down
6 changes: 3 additions & 3 deletions src/registry/generate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
use clap::Args;
use std::path::PathBuf;

use crate::registry::{semconv_registry_path_from, RegistryPath};
use weaver_cache::Cache;
use weaver_forge::debug::print_dedup_errors;
use weaver_forge::registry::TemplateRegistry;
Expand Down Expand Up @@ -34,7 +35,7 @@ pub struct RegistryGenerateArgs {
long,
default_value = "https://github.com/open-telemetry/semantic-conventions.git"
)]
pub registry: String,
pub registry: RegistryPath,

/// Optional path in the Git repository where the semantic convention
/// registry is located
Expand All @@ -58,8 +59,7 @@ pub(crate) fn command(
// Load the semantic convention registry into a local cache.
let mut registry = SchemaResolver::load_semconv_registry(
registry_id,
args.registry.to_string(),
args.registry_git_sub_dir.clone(),
semconv_registry_path_from(&args.registry, &args.registry_git_sub_dir),
cache,
logger.clone(),
)
Expand Down
Loading

0 comments on commit 355c59f

Please sign in to comment.