refactor(duvet-core): various fixes needed for integration into main …

…crate (#129)

* refactor: improve IETF parser

* refactor(duvet-core): various fixes needed for integration into main crate

.gitattributes

@@ -1 +1,2 @@
 integration/snapshots/*.snap filter=lfs diff=lfs merge=lfs -text
+**/snapshots.tar.gz filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -4,3 +4,4 @@ Cargo.lock
 target
 .history
 *.snap.new
+/duvet/src/specification/ietf/snapshots

Cargo.toml

@@ -7,11 +7,6 @@ members = [
 ]
 resolver = "2"
 
-[profile.release]
-lto = true
-codegen-units = 1
-incremental = false
-
 [profile.bench]
 lto = true
 codegen-units = 1

duvet-core/Cargo.toml

@@ -13,6 +13,7 @@ http = ["dep:http", "reqwest"]
 testing = ["tracing-subscriber"]
 
 [dependencies]
+anyhow = "1"
 blake3 = "1"
 bytes = "1"
 duvet-macros = { version = "0.1", path = "../duvet-macros" }
@@ -24,6 +25,7 @@ miette = { version = "7", features = ["fancy"] }
 once_cell = "1"
 reqwest = { version = "0.12", optional = true }
 serde = { version = "1", features = ["derive", "rc"] }
+serde_json = "1"
 tokio = { version = "1", features = ["fs", "sync"] }
 tokio-util = "0.7"
 toml_edit = { version = "0.22", features = ["parse", "serde"] }

duvet-core/src/diagnostic.rs

@@ -96,6 +96,24 @@ impl Diagnostic for Error {
     }
 }
 
+impl From<anyhow::Error> for Error {
+    fn from(value: anyhow::Error) -> Self {
+        Report::msg(value).into()
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(value: std::io::Error) -> Self {
+        Report::msg(value).into()
+    }
+}
+
+impl From<serde_json::Error> for Error {
+    fn from(value: serde_json::Error) -> Self {
+        Report::msg(value).into()
+    }
+}
+
 impl From<Report> for Error {
     fn from(err: Report) -> Self {
         Self(Arc::new(err))
@@ -210,7 +228,7 @@ impl From<Vec<Error>> for Set {
 impl fmt::Display for Set {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         for error in self.errors.iter() {
-            writeln!(f, "{}", error)?;
+            writeln!(f, "{:?}", error)?;
         }
         Ok(())
     }
@@ -227,12 +245,3 @@ impl StdError for Set {
         Some(&self.main)
     }
 }
-
-/*
-impl Diagnostic for Set {
-    fn related<'a>(&'a self) -> Option<Box<dyn Iterator<Item = &'a dyn Diagnostic> + 'a>> {
-        let iter = self.errors.iter().map(|e| e as &dyn Diagnostic);
-        Some(Box::new(iter))
-    }
-}
-*/

duvet-core/src/env.rs

@@ -0,0 +1,32 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{diagnostic::IntoDiagnostic, path::Path, Result};
+use core::cell::RefCell;
+use once_cell::sync::Lazy;
+use std::sync::Arc;
+
+static GLOBAL_ARGS: Lazy<Arc<[String]>> = Lazy::new(|| std::env::args().collect());
+static GLOBAL_DIR: Lazy<Result<Path>> =
+    Lazy::new(|| std::env::current_dir().map(|v| v.into()).into_diagnostic());
+
+thread_local! {
+    static ARGS: RefCell<Arc<[String]>> = RefCell::new(GLOBAL_ARGS.clone());
+    static DIR: RefCell<Result<Path>> = RefCell::new(GLOBAL_DIR.clone());
+}
+
+pub fn args() -> Arc<[String]> {
+    ARGS.with(|current| current.borrow().clone())
+}
+
+pub fn set_args(args: Arc<[String]>) {
+    ARGS.with(|current| *current.borrow_mut() = args);
+}
+
+pub fn current_dir() -> Result<Path> {
+    DIR.with(|current| current.borrow().clone())
+}
+
+pub fn set_current_dir(dir: Path) {
+    DIR.with(|current| *current.borrow_mut() = Ok(dir));
+}

duvet-core/src/file.rs

@@ -91,6 +91,26 @@ impl SourceFile {
             .await
     }
 
+    pub async fn as_json<T>(&self) -> crate::Result<Arc<T>>
+    where
+        T: 'static + Send + Sync + serde::de::DeserializeOwned,
+    {
+        let path = self.path.clone();
+        let contents = self.contents.clone();
+        // TODO can we get better errors by mapping string ranges?
+        crate::Cache::current()
+            .get_or_init(*self.hash(), move || {
+                crate::Query::from(
+                    serde_json::from_slice(contents.data())
+                        .map(Arc::new)
+                        .into_diagnostic()
+                        .wrap_err(path)
+                        .map_err(|err| err.into()),
+                )
+            })
+            .await
+    }
+
     pub fn substr(&self, v: &str) -> Option<Slice<SourceFile>> {
         unsafe {
             let beginning = self.as_bytes().as_ptr();
@@ -164,7 +184,7 @@ impl SourceCode for SourceFile {
 }
 
 #[derive(Clone, PartialEq, PartialOrd, Hash, Eq, Ord)]
-pub struct Slice<File> {
+pub struct Slice<File = SourceFile> {
     file: File,
     start: usize,
     end: usize,

duvet-core/src/glob.rs

@@ -6,25 +6,39 @@ use globset as g;
 use serde::de;
 use std::{str::FromStr, sync::Arc};
 
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct Glob {
-    set: Arc<g::GlobSet>,
+    set: Arc<(g::GlobSet, Vec<String>)>,
+}
+
+impl fmt::Debug for Glob {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let list = &self.set.1;
+        if list.len() == 1 {
+            list[0].fmt(f)
+        } else {
+            list.fmt(f)
+        }
+    }
 }
 
 impl Glob {
     pub fn is_match<P: AsRef<std::path::Path>>(&self, path: &P) -> bool {
-        self.set.is_match(path)
+        self.set.0.is_match(path)
     }
 
     pub fn try_from_iter<T: IntoIterator<Item = I>, I: AsRef<str>>(
         iter: T,
     ) -> Result<Glob, g::Error> {
         let mut builder = g::GlobSetBuilder::new();
+        let mut display = vec![];
         for item in iter {
-            builder.add(g::Glob::new(item.as_ref())?);
+            let value = format_value(item.as_ref());
+            builder.add(g::Glob::new(&value)?);
+            display.push(value);
         }
         let set = builder.build()?;
-        let set = Arc::new(set);
+        let set = Arc::new((set, display));
         Ok(Self { set })
     }
 }
@@ -37,6 +51,14 @@ impl FromStr for Glob {
     }
 }
 
+impl TryFrom<&str> for Glob {
+    type Error = g::Error;
+
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        value.parse()
+    }
+}
+
 impl<'de> de::Deserialize<'de> for Glob {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
@@ -67,12 +89,23 @@ impl<'de> de::Visitor<'de> for StringOrList {
         S: de::SeqAccess<'de>,
     {
         let mut builder = g::GlobSetBuilder::new();
+        let mut display = vec![];
         while let Some(value) = seq.next_element()? {
-            let item = g::Glob::new(value).map_err(serde::de::Error::custom)?;
+            let value = format_value(value);
+            let item = g::Glob::new(&value).map_err(serde::de::Error::custom)?;
             builder.add(item);
+            display.push(value);
         }
         let set = builder.build().map_err(serde::de::Error::custom)?;
-        let set = Arc::new(set);
+        let set = Arc::new((set, display));
         Ok(Glob { set })
     }
 }
+
+fn format_value(v: &str) -> String {
+    if v.starts_with("**/") || v.starts_with('/') {
+        v.to_string()
+    } else {
+        format!("**/{v}")
+    }
+}

duvet-core/src/hash.rs

@@ -9,8 +9,9 @@ pub struct Hash([u8; HASH_LEN]);
 
 impl fmt::Debug for Hash {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "0x")?;
         for byte in &self.0 {
-            write!(f, "{byte:#02x}")?;
+            write!(f, "{byte:02x}")?;
         }
         Ok(())
     }

duvet-core/src/http.rs

@@ -13,14 +13,31 @@ use std::sync::Arc;
 pub use http::response::Parts;
 pub use reqwest::Client;
 
+fn default_headers() -> reqwest::header::HeaderMap {
+    let mut map = reqwest::header::HeaderMap::new();
+
+    map.insert("accept", "text/plain".parse().unwrap());
+
+    map
+}
+
 pub fn client() -> Query<Client> {
     #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
     struct Q;
 
-    // TODO configure the client more
-    //     - User-Agent headers
-    //     - Accept headers?
-    Cache::current().get_or_init(Q, || Query::from(Client::builder().build().unwrap()))
+    Cache::current().get_or_init(Q, || {
+        Query::from(
+            Client::builder()
+                .user_agent(concat!(
+                    env!("CARGO_PKG_NAME"),
+                    "/",
+                    env!("CARGO_PKG_VERSION")
+                ))
+                .default_headers(default_headers())
+                .build()
+                .unwrap(),
+        )
+    })
 }
 
 pub fn get_full<U>(url: U) -> Query<Result<(Arc<Parts>, Contents)>>

duvet-core/src/lib.rs

@@ -1,13 +1,26 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+#[macro_export]
+macro_rules! ensure {
+    ($cond:expr) => {
+        ensure!($cond, ());
+    };
+    ($cond:expr, $otherwise:expr) => {
+        if !($cond) {
+            return $otherwise;
+        }
+    };
+}
+
 #[cfg(any(test, feature = "testing"))]
 pub mod testing;
 
 mod cache;
 pub mod contents;
 pub mod diagnostic;
 pub mod dir;
+pub mod env;
 pub mod file;
 pub mod glob;
 pub mod hash;
@@ -24,4 +37,4 @@ pub use cache::Cache;
 pub use duvet_macros::*;
 pub use query::Query;
 
-pub type Result<T, E = diagnostic::Error> = core::result::Result<T, E>;
+pub type Result<T = (), E = diagnostic::Error> = core::result::Result<T, E>;