Merge remote-tracking branch 'origin/main' into v0.6.5

CHANGELOG.md

@@ -12,6 +12,7 @@ tree, and subtasks are marked with indentation.
 - Refactored Pipelines API to expose more functionality
   - PR: <https://github.com/iesahin/xvc/pull/243>
   - Exposed Git operations in the API
+- Fixed `xvc pipeline step update` clobbering `--when` option
 
 - Added `xvc pipeline step list` command to list pipeline steps
 

book/src/ref/xvc-pipeline-dag.md

@@ -42,14 +42,19 @@ $ xvc pipeline step new --step-name train --command "echo 'train'"
 $ xvc pipeline step dependency --step-name train --step preprocess
 
 ```
+It's not very readable but you can supply the result directly to dot and get a more useful output.
 
 ```console
 $ xvc pipeline dag
 digraph pipeline{n0[shape=box;label="preprocess";];n1[shape=box;label="train";];n0[shape=box;label="preprocess";];n0->n1;}
 
 ```
 
-When you add a dependency between two steps, the graph shows it as a node.
+The output after `dot -Tsvg` is:
+
+![pipeline-1](/images/xvc-pipeline-dag-pipeline-1.svg)
+
+When you add a dependency between two steps, the graph shows it as a node. For example, 
 
 ```console
 $ xvc pipeline step dependency --step-name preprocess --glob 'data/*'
@@ -59,6 +64,8 @@ digraph pipeline{n0[shape=box;label="preprocess";];n1[shape=folder;label="data/*
 
 ```
 
+![pipeline-2](/images/xvc-pipeline-dag-pipeline-2.svg)
+
 You can use `--mermaid` option to get a [mermaid.js](https://mermaid.js.org) diagram.
 
 ```

core/Cargo.toml

@@ -79,9 +79,9 @@ uuid = { version = "^1.5", features = ["serde", "v4", "fast-rng"] }
 hex = { version = "^0.4", features = ["serde"] }
 cached = "^0.46"
 derive_more = "^0.99"
-itertools = "^0.11"
+itertools = "^0.12"
 
 [dev-dependencies]
+xvc-test-helper = { version = "^0.6", path = "../test_helper/" }
 proptest = "^1.3"
 test-case = "^3.2"
-xvc-test-helper = { path = "../test_helper/" }

file/Cargo.toml

@@ -93,5 +93,5 @@ parse-size = "^1.0"
 
 
 [dev-dependencies]
-xvc-test-helper = { path = "../test_helper/" }
+xvc-test-helper = { version = "^0.6", path = "../test_helper/" }
 shellfn = "^0.1"

lib/Cargo.toml

@@ -113,4 +113,4 @@ jwalk = "^0.8"
 proptest = "^1.3"
 shellfn = "^0.1"
 test-case = "^3.2"
-xvc-test-helper = { path = "../test_helper/" }
+xvc-test-helper = { version = "*", path = "../test_helper/" }

pipeline/Cargo.toml

@@ -92,5 +92,5 @@ itertools = "^0.12"
 derive_more = "^0.99"
 
 [dev-dependencies]
-xvc-test-helper = { path = "../test_helper/" }
+xvc-test-helper = { version = "^0.6", path = "../test_helper/" }
 test-case = "^3.2"

pipeline/src/pipeline/api/dag.rs

@@ -252,8 +252,9 @@ fn dependency_graph_stmts(
 
     for (xe, step) in pipeline_steps.iter().sorted() {
         let step_identity = short_id(id_from_string(&step.name)?)?;
-        let step_deps = all_deps.children_of(xe)?;
-        let step_outs = all_outs.children_of(xe)?;
+        let step_deps: HStore<XvcDependency> =
+            all_deps.children_of(xe)?.into_iter().sorted().collect();
+        let step_outs: HStore<XvcOutput> = all_outs.children_of(xe)?.into_iter().sorted().collect();
 
         stmts = stmts.add_node(
             step_identity.clone(),

storage/Cargo.toml

@@ -79,19 +79,22 @@ lazy_static = "^1.4"
 uuid = { version = "^1.5", features = ["serde", "v4", "fast-rng"] }
 hex = { version = "^0.4", features = ["serde"] }
 url = { version = "^2.4", features = ["serde"] }
-itertools = "^0.11"
+itertools = "^0.12"
 derive_more = "^0.99"
 tempfile = "^3.8"
 
 ## Networking & Async
 
-## Although we don't use openssl directly, we use its vendored version for cross compilation
-openssl = { version = "^0.10", features = ["vendored"] }
 
 tokio = { version = "^1.32", optional = true, features = ["rt-multi-thread"] }
 rust-s3 = { version = "^0.33", optional = true }
 futures = { version = "^0.3", optional = true }
 
+# Although we don't use openssl directly, we use its vendored version for cross compilation
+# Cannot compile on Windows with vendored
+[target.'cfg(target_os = "linux")'.dependencies]
+openssl = { version = "^0.10", features = ["vendored"] }
+
 
 [features]
 default = ["s3", "minio", "gcs", "wasabi", "r2", "digital-ocean"]
@@ -104,7 +107,7 @@ digital-ocean = ["dep:rust-s3", "dep:futures", "dep:tokio"]
 
 
 [dev-dependencies]
-xvc-test-helper = { path = "../test_helper/" }
+xvc-test-helper = { version = "^0.6", path = "../test_helper/" }
 shellfn = "^0.1"
 
 [package.metadata.cargo-udeps.ignore]

walker/Cargo.toml

@@ -37,11 +37,11 @@ anyhow = "^1.0"
 log = "^0.4"
 
 ## Misc
-itertools = "^0.11"
+itertools = "^0.12"
 regex = "^1.10"
 
 [dev-dependencies]
-xvc-test-helper = { path = "../test_helper/" }
+xvc-test-helper = { path = "../test_helper/", version = "^0.6" }
 test-case = "^3.2"
 
 [package.metadata.cargo-udeps.ignore]

workflow_tests/Cargo.toml

@@ -66,6 +66,8 @@ test-generic-rsync = []
 
 
 [dev-dependencies]
+xvc-test-helper = { version = "0.6.0", path = "../test_helper" }
+
 proptest = "^1.3"
 test-case = "^3.2"
 globset = "^0.4"

workflow_tests/tests/z_test_docs.rs

@@ -60,6 +60,34 @@ fn link_to_docs() -> Result<()> {
 
     let book_dirs_and_filters = book_dirs_and_filters;
 
+    let mut book_dirs_and_filters = vec![];
+    if trycmd_tests.contains("intro") {
+        book_dirs_and_filters.push(("intro", r".*"));
+    }
+    if trycmd_tests.contains("start") {
+        book_dirs_and_filters.push(("start", r".*"));
+    }
+    if trycmd_tests.contains("how-to") || trycmd_tests.contains("howto") {
+        book_dirs_and_filters.push(("how-to", r".*"));
+    }
+
+    if trycmd_tests.contains("storage") {
+        book_dirs_and_filters.push(("ref", r"xvc-storage.*"));
+    }
+    if trycmd_tests.contains("file") {
+        book_dirs_and_filters.push(("ref", r"xvc-file.*"));
+    }
+
+    if trycmd_tests.contains("pipeline") {
+        book_dirs_and_filters.push(("ref", r"xvc-pipeline.*"));
+    }
+
+    if trycmd_tests.contains("core") {
+        book_dirs_and_filters.push(("ref", r"^xvc-[^psf].*"))
+    }
+
+    let book_dirs_and_filters = book_dirs_and_filters;
+
     let template_dir_root = Path::new("templates");
 
     // This is a directory that we create to keep testing artifacts outside the code