diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 73c337da7..2588bd7e6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -60,7 +60,7 @@ jobs: - name: Install required packages run: | - brew install s3cmd + brew install --head s3cmd # 2.3.0 has a bug with Python 3.12 brew install ripgrep brew install tree brew install lsd diff --git a/CHANGELOG.md b/CHANGELOG.md index d19df745a..1f65d799e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ tree, and subtasks are marked with indentation. ## Unreleased +- Refactored Pipelines API to expose more functionality + - PR: + - Exposed Git operations in the API + ## 0.6.3 (2023-11-02) - Updated README and added its commands to the tests diff --git a/README.md b/README.md index e098db029..b1c898a78 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ $ cargo install xvc ## 🏃🏾 Quicktart -Xvc tracks your files and directories on top of Git. To start run the following command in the repository. +Xvc seamlessly monitors your files and directories on top of Git. To commence, execute the following command within the repository: ```console $ git init # if you're not already in a Git repository @@ -46,17 +46,15 @@ Initialized empty Git repository in [CWD]/.git/ $ xvc init ``` -It initializes the metafiles in `.xvc/` directory and adds `.xvcignore` file for paths you may want to hide from Xvc. +This command initializes the `.xvc/` directory and adds a `.xvcignore` file for specifying paths you wish to conceal from Xvc. -Add your data files and directories for tracking. +Include your data files and directories for tracking: ```shell $ xvc file track my-data/ --as symlink ``` -The command calculates data content hashes (with BLAKE-3, by default) and records them. -It commits these changes to Git. -It also copies these files to content-addressed directories under `.xvc/b3` and creates read-only symbolic links to them. +This command calculates content hashes for data (using BLAKE-3, by default) and logs them. The changes are committed to Git, and the files are copied to content-addressed directories within `.xvc/b3`. Additionally, read-only symbolic links to these directories are created. You can specify different [recheck (checkout) methods](https://docs.xvc.dev/ref/xvc-file-recheck/) for files and directories, depending on your use case. If you need to track model files that change frequently, you can set recheck method `--as copy` (the default). @@ -89,6 +87,8 @@ $ xvc file bring my-data/ --from my-remote ``` +This approach ensures convenient access to files from the shared storage when needed. + You don't have to reconfigure the storage after cloning, but you need to have valid credentials as environment variables to access the storage. Xvc never stores any credentials. diff --git a/book/src/ref/xvc-file-copy.md b/book/src/ref/xvc-file-copy.md index 149098c73..667cd73f7 100644 --- a/book/src/ref/xvc-file-copy.md +++ b/book/src/ref/xvc-file-copy.md @@ -163,9 +163,9 @@ FH 19 [..] c85f3e81 c85f3e81 another-set/data3.txt FH 19 [..] c85f3e81 c85f3e81 another-set/data2.txt FH 19 [..] c85f3e81 c85f3e81 another-set/data.txt DX 160 [..] another-set -FX 130 [..] ac46bf74 .xvcignore -FX [..] .gitignore -Total #: 11 Workspace Size: [..] Cached Size: 19 +FX 141 [..] 3054b812 .xvcignore +FX 529 [..] [..] .gitignore +Total #: 11 Workspace Size: 1105 Cached Size: 19 ``` diff --git a/book/src/ref/xvc-file-list.md b/book/src/ref/xvc-file-list.md index d77a0fd0f..ed2e49c0a 100644 --- a/book/src/ref/xvc-file-list.md +++ b/book/src/ref/xvc-file-list.md @@ -127,7 +127,7 @@ Now it lists all files and directories. ```console $ xvc file list --sort name-asc FX 107 [..] ce9fcf30 .gitignore -FX 130 [..] ac46bf74 .xvcignore +FX 141 [..] 3054b812 .xvcignore DX 224 [..] dir-0001 FX 2001 [..] 1953f05d dir-0001/file-0001.bin FX 2002 [..] 7e807161 dir-0001/file-0002.bin @@ -157,8 +157,8 @@ FX 2001 [..] 1953f05d dir-0005/file-0001.bin FX 2002 [..] 7e807161 dir-0005/file-0002.bin FX 2003 [..] d2432259 dir-0005/file-0003.bin FX 2004 [..] 63535612 dir-0005/file-0004.bin -FX 2005 [..] 447933dc dir-0005/file-0005.bin -Total #: 32 Workspace Size: 51432 Cached Size: 0 +FX 2005 [..] 447933dc dir-0005/file-0005.bin +Total #: 32 Workspace Size: 51443 Cached Size: 0 ``` diff --git a/book/src/ref/xvc-file-move.md b/book/src/ref/xvc-file-move.md index 3ea24cbd2..b6427577c 100644 --- a/book/src/ref/xvc-file-move.md +++ b/book/src/ref/xvc-file-move.md @@ -125,9 +125,9 @@ $ xvc file list XH c85f3e81 data6.txt FH 19 [..] c85f3e81 c85f3e81 another-set/data4.txt DX 96 [..] another-set -FX 130 [..] ac46bf74 .xvcignore +FX 141 [..] 3054b812 .xvcignore FX [..] .gitignore -Total #: 5 Workspace Size: [..] Cached Size: 19 +Total #: 5 Workspace Size: 785 Cached Size: 19 ``` diff --git a/book/src/ref/xvc-file-remove.md b/book/src/ref/xvc-file-remove.md index 748ca5cfc..d93a8b8c4 100644 --- a/book/src/ref/xvc-file-remove.md +++ b/book/src/ref/xvc-file-remove.md @@ -58,9 +58,9 @@ $ xvc file track 'd*.txt' $ xvc file list FC [..] c85f3e81 c85f3e81 data.txt -FX [..] ac46bf74 .xvcignore -FX [..] .gitignore -Total #: 3 Workspace Size: [..] Cached Size: 19 +FX 141 [..] 3054b812 .xvcignore +FX 190 [..] [..] .gitignore +Total #: 3 Workspace Size: 350 Cached Size: 19 $ tree .xvc/b3/ @@ -118,9 +118,9 @@ $ xvc file carry-in --force data.txt $ xvc file list FC [..] c85f3e81 c85f3e81 data.txt -FX [..] ac46bf74 .xvcignore -FX [..] [..] .gitignore -Total #: 3 Workspace Size: [..] Cached Size: 19 +FX 141 [..] 3054b812 .xvcignore +FX 190 [..] [..] .gitignore +Total #: 3 Workspace Size: 350 Cached Size: 19 $ tree .xvc/b3/ @@ -157,9 +157,9 @@ $ tree .xvc/b3/ $ xvc file list FC [..] 6602cff6 6602cff6 data.txt -FX [..] ac46bf74 .xvcignore -FX [..] [..] .gitignore -Total #: 3 Workspace Size: [..] Cached Size: 19 +FX 141 [..] 3054b812 .xvcignore +FX 190 [..] [..] .gitignore +Total #: 3 Workspace Size: 350 Cached Size: 19 $ xvc file remove --from-cache --only-version c85-f3e data.txt @@ -281,9 +281,9 @@ $ xvc file copy data.txt data2.txt --as symlink $ xvc file list SS [..] [..] 4a2e9d7c data2.txt FC 1024 [..] 4a2e9d7c 4a2e9d7c data.txt -FX 130 [..] ac46bf74 .xvcignore -FX [..] .gitignore -Total #: 4 Workspace Size: [..] Cached Size: 1024 +FX 141 [..] 3054b812 .xvcignore +FX 274 [..] [..] .gitignore +Total #: 4 Workspace Size: 1621 Cached Size: 1024 $ xvc file remove --from-cache data.txt diff --git a/book/src/ref/xvc-file-untrack.md b/book/src/ref/xvc-file-untrack.md index 127589e41..3ce2b8e9e 100644 --- a/book/src/ref/xvc-file-untrack.md +++ b/book/src/ref/xvc-file-untrack.md @@ -35,9 +35,9 @@ $ xvc file track 'd*.txt' $ xvc file list FC 19 [..] c85f3e81 c85f3e81 data.txt -FX 130 [..] [..] .xvcignore +FX 141 [..] 3054b812 .xvcignore FX [..] .gitignore -Total #: 3 Workspace Size: [..] Cached Size: 19 +Total #: 3 Workspace Size: 350 Cached Size: 19 ``` diff --git a/book/src/ref/xvc-pipeline-dag.md b/book/src/ref/xvc-pipeline-dag.md index 072a8bcb5..f905fa85c 100644 --- a/book/src/ref/xvc-pipeline-dag.md +++ b/book/src/ref/xvc-pipeline-dag.md @@ -9,10 +9,10 @@ Generate a dot or mermaid diagram for the pipeline Usage: xvc pipeline dag [OPTIONS] Options: - -n, --name Name of the pipeline to generate the diagram - --file Output file. Writes to stdout if not set - --format Format for graph. Either dot or mermaid [default: dot] - -h, --help Print help + -p, --pipeline-name Name of the pipeline to generate the diagram + --file Output file. Writes to stdout if not set + --format Format for graph. Either dot or mermaid [default: dot] + -h, --help Print help ``` diff --git a/book/src/ref/xvc-pipeline-delete.md b/book/src/ref/xvc-pipeline-delete.md index fb1d977c1..b61607020 100644 --- a/book/src/ref/xvc-pipeline-delete.md +++ b/book/src/ref/xvc-pipeline-delete.md @@ -6,10 +6,10 @@ $ xvc pipeline delete --help Delete a pipeline -Usage: xvc pipeline delete --name +Usage: xvc pipeline delete --pipeline-name Options: - -n, --name Name or GUID of the pipeline to be deleted - -h, --help Print help + -p, --pipeline-name Name or GUID of the pipeline to be deleted + -h, --help Print help ``` diff --git a/book/src/ref/xvc-pipeline-export.md b/book/src/ref/xvc-pipeline-export.md index e489133d0..c6fdb3ba9 100644 --- a/book/src/ref/xvc-pipeline-export.md +++ b/book/src/ref/xvc-pipeline-export.md @@ -9,10 +9,10 @@ Export the pipeline to a YAML or JSON file to edit Usage: xvc pipeline export [OPTIONS] Options: - -n, --name Name of the pipeline to export - --file File to write the pipeline. Writes to stdout if not set - --format Output format. One of json or yaml. If not set, the format is guessed from the file extension. If the file extension is not set, json is used as default - -h, --help Print help + -p, --pipeline-name Name of the pipeline to export + --file File to write the pipeline. Writes to stdout if not set + --format Output format. One of json or yaml. If not set, the format is guessed from the file extension. If the file extension is not set, json is used as default + -h, --help Print help ``` diff --git a/book/src/ref/xvc-pipeline-import.md b/book/src/ref/xvc-pipeline-import.md index 56b542867..783bb9580 100644 --- a/book/src/ref/xvc-pipeline-import.md +++ b/book/src/ref/xvc-pipeline-import.md @@ -9,11 +9,11 @@ Import the pipeline from a file Usage: xvc pipeline import [OPTIONS] Options: - -n, --name Name of the pipeline to import. If not set, the name from the file is used - --file File to read the pipeline. Use stdin if not specified - --format Input format. One of json or yaml. If not set, the format is guessed from the file extension. If the file extension is not set, json is used as default - --overwrite Overwrite the pipeline even if the name already exists - -h, --help Print help + -p, --pipeline-name Name of the pipeline to import. If not set, the name from the file is used + --file File to read the pipeline. Use stdin if not specified + --format Input format. One of json or yaml. If not set, the format is guessed from the file extension. If the file extension is not set, json is used as default + --overwrite Overwrite the pipeline even if the name already exists + -h, --help Print help ``` @@ -248,7 +248,7 @@ You can specify a new name for the pipeline and it will override the name set in This way you can edit and import similar pipelines with minor differences. ```console -$ xvc pipeline import --name another-pipeline --file pipeline.yaml +$ xvc pipeline import --pipeline-name another-pipeline --file pipeline.yaml ``` diff --git a/book/src/ref/xvc-pipeline-new.md b/book/src/ref/xvc-pipeline-new.md index 55b9d8f9e..23cbf4094 100644 --- a/book/src/ref/xvc-pipeline-new.md +++ b/book/src/ref/xvc-pipeline-new.md @@ -6,12 +6,12 @@ $ xvc pipeline new --help Create a new pipeline -Usage: xvc pipeline new [OPTIONS] --name +Usage: xvc pipeline new [OPTIONS] --pipeline-name Options: - -n, --name Name of the pipeline this command applies to - -w, --workdir Default working directory - -h, --help Print help + -p, --pipeline-name Name of the pipeline this command applies to + -w, --workdir Default working directory + -h, --help Print help ``` @@ -28,7 +28,8 @@ $ xvc init You can create a new pipeline with a name. ```console -$ xvc pipeline new --name my-pipeline +$ xvc pipeline new --pipeline-name my-pipeline + ``` By default it will run the commands in the repository root. @@ -49,7 +50,7 @@ If you want to define a pipeline specific to a directory, you can set the workin ```console $ xvc-test-helper create-directory-tree --directories 1 --files 3 --seed 20230215 -$ xvc pipeline new --name another-pipeline --workdir dir-0001 +$ xvc pipeline new --pipeline-name another-pipeline --workdir dir-0001 ``` diff --git a/book/src/ref/xvc-pipeline-run.md b/book/src/ref/xvc-pipeline-run.md index b89bb8060..ed7502766 100644 --- a/book/src/ref/xvc-pipeline-run.md +++ b/book/src/ref/xvc-pipeline-run.md @@ -9,8 +9,8 @@ Run a pipeline Usage: xvc pipeline run [OPTIONS] Options: - -n, --name Name of the pipeline to run - -h, --help Print help + -p, --pipeline-name Name of the pipeline to run + -h, --help Print help ``` @@ -74,12 +74,14 @@ $ xvc pipeline run You can run a specific pipeline by specifying its name with `--name` option. ```console -$ xvc pipeline new --name my-pipeline -$ xvc pipeline --name my-pipeline step new --step-name my-hello --command "echo 'hello from my-pipeline'" +$ xvc pipeline new --pipeline-name my-pipeline + +$ xvc pipeline --pipeline-name my-pipeline step new --step-name my-hello --command "echo 'hello from my-pipeline'" + ``` ```console -$ xvc pipeline run --name my-pipeline +$ xvc pipeline run --pipeline-name my-pipeline [OUT] [my-hello] hello from my-pipeline [DONE] my-hello (echo 'hello from my-pipeline') diff --git a/book/src/ref/xvc-pipeline-step-dependency-generic.md b/book/src/ref/xvc-pipeline-step-dependency-generic.md index 2cf876cca..fd7a27faa 100644 --- a/book/src/ref/xvc-pipeline-step-dependency-generic.md +++ b/book/src/ref/xvc-pipeline-step-dependency-generic.md @@ -22,7 +22,7 @@ $ xvc pipeline step dependency --step-name morning-message --generic 'date +%F' The step is invalidated when the date changes and the step is run again. -```console +```console,ignore $ xvc pipeline run [OUT] [morning-message] Good Morning! @@ -34,6 +34,9 @@ The step won't run until tomorrow, when `date +%F` changes. ```console $ xvc pipeline run +[OUT] [morning-message] Good Morning! + +[DONE] morning-message (echo 'Good Morning!') ``` diff --git a/book/src/ref/xvc-pipeline-update.md b/book/src/ref/xvc-pipeline-update.md index e8207363d..8f8d968ce 100644 --- a/book/src/ref/xvc-pipeline-update.md +++ b/book/src/ref/xvc-pipeline-update.md @@ -4,15 +4,15 @@ ```console $ xvc pipeline update --help -Rename, change dir or set a pipeline as default +Update the name and other attributes of a pipeline Usage: xvc pipeline update [OPTIONS] Options: - -n, --name Name of the pipeline this command applies to - --rename Rename the pipeline to - --workdir Set the working directory - --set-default set this pipeline default - -h, --help Print help + -p, --pipeline-name Name of the pipeline this command applies to + --rename Rename the pipeline to + --workdir Set the working directory + --set-default set this pipeline default + -h, --help Print help ``` diff --git a/book/src/ref/xvc-pipeline.md b/book/src/ref/xvc-pipeline.md index ee6ec962d..697d003c2 100644 --- a/book/src/ref/xvc-pipeline.md +++ b/book/src/ref/xvc-pipeline.md @@ -10,7 +10,7 @@ Usage: xvc pipeline [OPTIONS] Commands: new Create a new pipeline - update Rename, change dir or set a pipeline as default + update Update the name and other attributes of a pipeline delete Delete a pipeline run Run a pipeline list List all pipelines @@ -21,7 +21,7 @@ Commands: help Print this message or the help of the given subcommand(s) Options: - -n, --name Name of the pipeline this command applies - -h, --help Print help + -p, --pipeline-name Name of the pipeline this command applies to + -h, --help Print help ``` diff --git a/core/src/lib.rs b/core/src/lib.rs index 8fc35666d..c3aba45d7 100755 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -37,6 +37,9 @@ pub use types::xvcpath::XvcCachePath; pub use types::xvcpath::XvcPath; pub use types::xvcroot::XvcRoot; +pub use walker::AbsolutePath; +pub use xvc_walker as walker; + pub use error::Error; pub use error::Result; diff --git a/file/src/copy/mod.rs b/file/src/copy/mod.rs index 6a3362710..afdb127e5 100644 --- a/file/src/copy/mod.rs +++ b/file/src/copy/mod.rs @@ -309,11 +309,9 @@ pub(crate) fn recheck_destination( Ok(()) } -pub(crate) fn cmd_copy( - output_snd: &XvcOutputSender, - xvc_root: &XvcRoot, - opts: CopyCLI, -) -> Result<()> { +/// Entry point for `xvc file copy` command. +/// Copies a file (and its records) to a new location in the repository +pub fn cmd_copy(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: CopyCLI) -> Result<()> { // Get all files to copy let stored_metadata_store = xvc_root.load_store::()?; diff --git a/file/src/lib.rs b/file/src/lib.rs index f3bac0058..e8b73fdd8 100755 --- a/file/src/lib.rs +++ b/file/src/lib.rs @@ -22,22 +22,28 @@ pub mod send; pub mod track; pub mod untrack; +pub use bring::cmd_bring; +pub use carry_in::cmd_carry_in; +pub use copy::cmd_copy; +pub use hash::cmd_hash; +pub use list::cmd_list; +pub use mv::cmd_move; +pub use recheck::cmd_recheck; +pub use remove::cmd_remove; +pub use send::cmd_send; +pub use track::cmd_track; +pub use untrack::cmd_untrack; + use crate::error::{Error, Result}; -use carry_in::CarryInCLI; use clap::Subcommand; -use copy::CopyCLI; use crossbeam::thread; use crossbeam_channel::bounded; -use list::ListCLI; use log::{debug, error, info, warn, LevelFilter}; -use mv::MoveCLI; -use remove::RemoveCLI; use std::io; use std::io::Write; use std::path::Path; use std::path::PathBuf; -use untrack::UntrackCLI; use xvc_config::XvcConfigInitParams; use xvc_config::XvcVerbosity; use xvc_core::default_project_config; @@ -48,11 +54,17 @@ use xvc_logging::{setup_logging, watch}; use xvc_logging::{XvcOutputLine, XvcOutputSender}; use xvc_walker::AbsolutePath; -use bring::BringCLI; -use hash::HashCLI; -use recheck::RecheckCLI; -use send::SendCLI; -use track::TrackCLI; +pub use bring::BringCLI; +pub use carry_in::CarryInCLI; +pub use copy::CopyCLI; +pub use hash::HashCLI; +pub use list::ListCLI; +pub use mv::MoveCLI; +pub use recheck::RecheckCLI; +pub use remove::RemoveCLI; +pub use send::SendCLI; +pub use track::TrackCLI; +pub use untrack::UntrackCLI; use clap::Parser; @@ -156,53 +168,53 @@ pub fn run( ) -> Result<()> { watch!(opts); match opts.subcommand { - XvcFileSubCommand::Track(opts) => track::cmd_track( + XvcFileSubCommand::Track(opts) => cmd_track( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::Hash(opts) => hash::cmd_hash(output_snd, xvc_root, opts), - XvcFileSubCommand::CarryIn(opts) => carry_in::cmd_carry_in( + XvcFileSubCommand::Hash(opts) => cmd_hash(output_snd, xvc_root, opts), + XvcFileSubCommand::CarryIn(opts) => cmd_carry_in( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::Recheck(opts) => recheck::cmd_recheck( + XvcFileSubCommand::Recheck(opts) => cmd_recheck( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::List(opts) => list::cmd_list( + XvcFileSubCommand::List(opts) => cmd_list( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::Send(opts) => send::cmd_send( + XvcFileSubCommand::Send(opts) => cmd_send( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::Bring(opts) => bring::cmd_bring( + XvcFileSubCommand::Bring(opts) => cmd_bring( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::Copy(opts) => copy::cmd_copy( + XvcFileSubCommand::Copy(opts) => cmd_copy( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::Move(opts) => mv::cmd_move( + XvcFileSubCommand::Move(opts) => cmd_move( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::Untrack(opts) => untrack::cmd_untrack( + XvcFileSubCommand::Untrack(opts) => cmd_untrack( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, ), - XvcFileSubCommand::Remove(opts) => remove::cmd_remove( + XvcFileSubCommand::Remove(opts) => cmd_remove( output_snd, xvc_root.ok_or(Error::RequiresXvcRepository)?, opts, diff --git a/file/src/mv/mod.rs b/file/src/mv/mod.rs index db41e6169..b1c87a107 100644 --- a/file/src/mv/mod.rs +++ b/file/src/mv/mod.rs @@ -158,11 +158,7 @@ pub fn get_move_source_dest_store( } /// Entry point for `xvc file move` -pub(crate) fn cmd_move( - output_snd: &XvcOutputSender, - xvc_root: &XvcRoot, - opts: MoveCLI, -) -> Result<()> { +pub fn cmd_move(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: MoveCLI) -> Result<()> { // Get all files to move let stored_metadata_store = xvc_root.load_store::()?; let stored_xvc_path_store = xvc_root.load_store::()?; diff --git a/file/src/remove/mod.rs b/file/src/remove/mod.rs index b08989bdc..f4699aaa4 100644 --- a/file/src/remove/mod.rs +++ b/file/src/remove/mod.rs @@ -49,11 +49,8 @@ pub struct RemoveCLI { targets: Vec, } -pub(crate) fn cmd_remove( - output_snd: &XvcOutputSender, - xvc_root: &XvcRoot, - opts: RemoveCLI, -) -> Result<()> { +/// Removes a file from XVC cache or storage +pub fn cmd_remove(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: RemoveCLI) -> Result<()> { if !opts.from_cache && opts.from_storage.is_none() { return Err(anyhow::anyhow!( "At least one of --from-cache or --from-storage must be specified" diff --git a/lib/src/api.rs b/lib/src/api.rs new file mode 100644 index 000000000..38df23975 --- /dev/null +++ b/lib/src/api.rs @@ -0,0 +1,71 @@ +//! The `xvc` API. + +pub use crate::error::{Error, Result}; + +pub use xvc_config as config; +pub use xvc_core as core; +pub use xvc_ecs as ecs; +pub use xvc_file as file; +pub use xvc_logging as logging; +pub use xvc_pipeline as pipeline; +pub use xvc_storage as storage; + +pub use xvc_logging::debug; +pub use xvc_logging::error; +pub use xvc_logging::info; +pub use xvc_logging::panic; +pub use xvc_logging::trace; +pub use xvc_logging::warn; +pub use xvc_logging::watch; + +pub use xvc_config::XvcConfig; +pub use xvc_config::XvcConfigInitParams; +pub use xvc_config::XvcConfigOptionSource; + +pub use xvc_core::AbsolutePath; + +pub use xvc_file::BringCLI as XvcFileBringCLI; +pub use xvc_file::CarryInCLI as XvcFileCarryInCLI; +pub use xvc_file::CopyCLI as XvcFileCopyCLI; +pub use xvc_file::HashCLI as XvcFileHashCLI; +pub use xvc_file::ListCLI as XvcFileListCLI; +pub use xvc_file::MoveCLI as XvcFileMoveCLI; +pub use xvc_file::RemoveCLI as XvcFileRemoveCLI; +pub use xvc_file::SendCLI as XvcFileSendCLI; +pub use xvc_file::TrackCLI as XvcFileTrackCLI; +pub use xvc_file::UntrackCLI as XvcFileUntrackCLI; +pub use xvc_file::XvcFileCLI; + +pub use xvc_file::cmd_carry_in as file_carry_in; +pub use xvc_file::cmd_copy as file_copy; +pub use xvc_file::cmd_hash as file_hash; +pub use xvc_file::cmd_list as file_list; +pub use xvc_file::cmd_move as file_move; +pub use xvc_file::cmd_remove as file_remove; +pub use xvc_file::cmd_send as file_send; +pub use xvc_file::cmd_track as file_track; +pub use xvc_file::cmd_untrack as file_untrack; + +pub use xvc_pipeline::cmd_dag as pipeline_dag; +pub use xvc_pipeline::cmd_delete as pipeline_delete; +pub use xvc_pipeline::cmd_export as pipeline_export; +pub use xvc_pipeline::cmd_import as pipeline_import; +pub use xvc_pipeline::cmd_list as pipeline_list; +pub use xvc_pipeline::cmd_new as pipeline_new; +pub use xvc_pipeline::cmd_run as pipeline_run; +pub use xvc_pipeline::cmd_update as pipeline_update; + +pub use xvc_pipeline::cmd_step_dependency as pipeline_step_dependency; +pub use xvc_pipeline::cmd_step_new as pipeline_step_new; +pub use xvc_pipeline::cmd_step_output as pipeline_step_output; +pub use xvc_pipeline::cmd_step_show as pipeline_step_show; +pub use xvc_pipeline::cmd_step_update as pipeline_step_update; + +pub use crate::git::exec_git; +pub use crate::git::get_absolute_git_command; +pub use crate::git::git_auto_commit; +pub use crate::git::git_auto_stage; +pub use crate::git::git_checkout_ref; +pub use crate::git::handle_git_automation; +pub use crate::git::stash_user_staged_files; +pub use crate::git::unstash_user_staged_files; diff --git a/lib/src/cli/mod.rs b/lib/src/cli/mod.rs index 05425a93d..423d4f9f2 100644 --- a/lib/src/cli/mod.rs +++ b/lib/src/cli/mod.rs @@ -512,7 +512,7 @@ fn git_checkout_ref( /// This receives `xvc_root` ownership because as a final operation, it must drop the root to /// record the last entity counter before commit. -fn handle_git_automation( +pub fn handle_git_automation( output_snd: &XvcOutputSender, xvc_root: XvcRoot, to_branch: Option<&str>, diff --git a/lib/src/error.rs b/lib/src/error.rs index c1016b641..4f730b7d5 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -90,8 +90,10 @@ pub enum Error { #[from] source: serde_yaml::Error, }, - #[error("[E2004] Requires xvc repository.")] + + #[error("This command requires Xvc repository. Please use xvc init first.")] RequiresXvcRepository, + #[error("I/O Error: {source}")] IoError { #[from] diff --git a/lib/src/git.rs b/lib/src/git.rs new file mode 100644 index 000000000..1a06e6299 --- /dev/null +++ b/lib/src/git.rs @@ -0,0 +1,231 @@ +//! Git operations for Xvc repository commands +use std::{ffi::OsString, path::PathBuf, str::FromStr}; + +use subprocess::Exec; +use xvc_core::XvcRoot; +use xvc_logging::{debug, watch, XvcOutputSender}; + +use crate::{Error, Result}; + +pub fn get_absolute_git_command(git_command: &str) -> Result { + let git_cmd_path = PathBuf::from(git_command); + let git_cmd = if git_cmd_path.is_absolute() { + git_command.to_string() + } else { + let cmd_path = which::which(git_command)?; + cmd_path.to_string_lossy().to_string() + }; + Ok(git_cmd) +} + +pub fn exec_git(git_command: &str, xvc_directory: &str, args_str_vec: &[&str]) -> Result { + let mut args = vec!["-C", xvc_directory]; + args.extend(args_str_vec); + let args: Vec = args + .iter() + .map(|s| OsString::from_str(s).unwrap()) + .collect(); + watch!(args); + let proc_res = Exec::cmd(git_command).args(&args).capture()?; + + match proc_res.exit_status { + subprocess::ExitStatus::Exited(0) => Ok(proc_res.stdout_str()), + subprocess::ExitStatus::Exited(_) => Err(Error::GitProcessError { + stdout: proc_res.stdout_str(), + stderr: proc_res.stderr_str(), + }), + subprocess::ExitStatus::Signaled(_) + | subprocess::ExitStatus::Other(_) + | subprocess::ExitStatus::Undetermined => Err(Error::GitProcessError { + stdout: proc_res.stdout_str(), + stderr: proc_res.stderr_str(), + }), + } +} + +pub fn stash_user_staged_files( + output_snd: &XvcOutputSender, + git_command: &str, + xvc_directory: &str, +) -> Result { + // Do we have user staged files? + let git_diff_staged_out = exec_git( + git_command, + xvc_directory, + &["diff", "--name-only", "--cached"], + )?; + + watch!(git_diff_staged_out); + + // If so stash them + if !git_diff_staged_out.trim().is_empty() { + debug!( + output_snd, + "Stashing user staged files: {git_diff_staged_out}" + ); + let stash_out = exec_git(git_command, xvc_directory, &["stash", "push", "--staged"])?; + debug!(output_snd, "Stashed user staged files: {stash_out}"); + } + + Ok(git_diff_staged_out) +} + +pub fn unstash_user_staged_files( + output_snd: &XvcOutputSender, + git_command: &str, + xvc_directory: &str, +) -> Result<()> { + let res_git_stash_pop = exec_git(git_command, xvc_directory, &["stash", "pop", "--index"])?; + debug!( + output_snd, + "Unstashed user staged files: {res_git_stash_pop}" + ); + Ok(()) +} + +pub fn git_checkout_ref( + output_snd: &XvcOutputSender, + xvc_root: &XvcRoot, + from_ref: String, +) -> Result<()> { + let xvc_directory = xvc_root.as_path().to_str().unwrap(); + let git_command_option = xvc_root.config().get_str("git.command")?.option; + let git_command = get_absolute_git_command(&git_command_option)?; + + let git_diff_staged_out = stash_user_staged_files(output_snd, &git_command, xvc_directory)?; + exec_git(&git_command, xvc_directory, &["checkout", &from_ref])?; + + if !git_diff_staged_out.trim().is_empty() { + debug!("Unstashing user staged files: {git_diff_staged_out}"); + unstash_user_staged_files(output_snd, &git_command, xvc_directory)?; + } + Ok(()) +} + +/// This receives `xvc_root` ownership because as a final operation, it must drop the root to +/// record the last entity counter before commit. +pub fn handle_git_automation( + output_snd: &XvcOutputSender, + xvc_root: XvcRoot, + to_branch: Option<&str>, + xvc_cmd: &str, +) -> Result<()> { + let xvc_root_dir = xvc_root.as_path().to_path_buf(); + let xvc_root_str = xvc_root_dir.to_str().unwrap(); + let use_git = xvc_root.config().get_bool("git.use_git")?.option; + let auto_commit = xvc_root.config().get_bool("git.auto_commit")?.option; + let auto_stage = xvc_root.config().get_bool("git.auto_stage")?.option; + let git_command_str = xvc_root.config().get_str("git.command")?.option; + let git_command = get_absolute_git_command(&git_command_str)?; + let xvc_dir = xvc_root.xvc_dir().clone(); + let xvc_dir_str = xvc_dir.to_str().unwrap(); + + // we drop here to record the final state + drop(xvc_root); + + if use_git { + if auto_commit { + git_auto_commit( + output_snd, + &git_command, + xvc_root_str, + xvc_dir_str, + xvc_cmd, + to_branch, + )?; + } else if auto_stage { + git_auto_stage(output_snd, &git_command, xvc_root_str, xvc_dir_str)?; + } + } + + Ok(()) +} + +pub fn git_auto_commit( + output_snd: &XvcOutputSender, + git_command: &str, + xvc_root_str: &str, + xvc_dir_str: &str, + xvc_cmd: &str, + to_branch: Option<&str>, +) -> Result<()> { + debug!(output_snd, "Using Git: {git_command}"); + + let git_diff_staged_out = stash_user_staged_files(output_snd, git_command, xvc_root_str)?; + + if let Some(branch) = to_branch { + debug!(output_snd, "Checking out branch {branch}"); + exec_git(git_command, xvc_root_str, &["checkout", "-b", branch])?; + } + + // Add and commit `.xvc` + match exec_git( + git_command, + xvc_root_str, + // We check the output of the git add command to see if there were any files added. + // "--verbose" is required to get the output we need. + &[ + "add", + "--verbose", + &xvc_dir_str, + "*.gitignore", + "*.xvcignore", + ], + ) { + Ok(git_add_output) => { + watch!(git_add_output); + if git_add_output.trim().is_empty() { + debug!(output_snd, "No files to commit"); + return Ok(()); + } else { + match exec_git( + git_command, + xvc_root_str, + &[ + "commit", + "-m", + &format!("Xvc auto-commit after '{xvc_cmd}'"), + ], + ) { + Ok(res_git_commit) => { + debug!(output_snd, "Committing .xvc/ to git: {res_git_commit}"); + } + Err(e) => { + debug!(output_snd, "Error committing .xvc/ to git: {e}"); + return Err(e); + } + } + } + } + Err(e) => { + debug!(output_snd, "Error adding .xvc/ to git: {e}"); + return Err(e); + } + } + + // Pop the stash if there were files we stashed + + if !git_diff_staged_out.trim().is_empty() { + debug!( + output_snd, + "Unstashing user staged files: {git_diff_staged_out}" + ); + unstash_user_staged_files(output_snd, git_command, xvc_root_str)?; + } + Ok(()) +} + +pub fn git_auto_stage( + output_snd: &XvcOutputSender, + git_command: &str, + xvc_root_str: &str, + xvc_dir_str: &str, +) -> Result<()> { + let res_git_add = exec_git( + git_command, + xvc_root_str, + &["add", &xvc_dir_str, "*.gitignore", "*.xvcignore"], + )?; + debug!(output_snd, "Staging .xvc/ to git: {res_git_add}"); + Ok(()) +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 843e9dcaf..64bb7ff12 100755 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -3,18 +3,12 @@ //! The main dispatching functions for the entire XVC CLI pub mod cli; pub mod error; +pub mod git; pub mod init; -pub use xvc_config as config; -pub use xvc_core as core; -pub use xvc_ecs as ecs; -pub use xvc_file as file; -pub use xvc_logging as logging; -pub use xvc_pipeline as pipeline; +pub mod api; -pub use xvc_logging::watch; - -pub use crate::error::Result; +pub use api::*; /// Adds `xvc` as the first elements to `args` and calls [cli::dispatch] after parsing them. pub fn dispatch(args: Vec<&str>) -> Result<()> { diff --git a/pipeline/src/lib.rs b/pipeline/src/lib.rs index 191bde1f3..11f85d2ee 100755 --- a/pipeline/src/lib.rs +++ b/pipeline/src/lib.rs @@ -10,19 +10,25 @@ mod pipeline; pub use crate::pipeline::api::{ dag::cmd_dag, delete::cmd_delete, export::cmd_export, import::cmd_import, list::cmd_list, - new::cmd_new, run::cmd_run, step_new::cmd_step_new, step_output::cmd_step_output, - step_show::cmd_step_show, step_update::cmd_step_update, update::cmd_update, + new::cmd_new, run::cmd_run, step_dependency::cmd_step_dependency, step_new::cmd_step_new, + step_output::cmd_step_output, step_show::cmd_step_show, step_update::cmd_step_update, + update::cmd_update, }; use clap::Parser; -use pipeline::api::step_dependency::XvcDependencyList; +use pipeline::api::dag::DagCLI; +use pipeline::api::delete::DeleteCLI; +use pipeline::api::export::ExportCLI; +use pipeline::api::import::ImportCLI; +use pipeline::api::new::NewCLI; +use pipeline::api::update::UpdateCLI; pub use pipeline::deps; -use pipeline::schema::XvcSchemaSerializationFormat; +use pipeline::step::handle_step_cli; +use pipeline::step::StepCLI; use serde::{Deserialize, Serialize}; use std::io::BufRead; -use std::path::PathBuf; use std::str::FromStr; use xvc_config::{conf, FromConfigKey, UpdateFromXvcConfig, XvcConfig}; use xvc_ecs::XvcStore; @@ -32,8 +38,6 @@ use xvc_core::XvcPath; use xvc_core::XvcRoot; use xvc_ecs::{self, persist, XvcEntity}; -use crate::pipeline::api::dag::XvcPipelineDagFormat; - use crate::error::{Error, Result}; pub use crate::pipeline::command::CommandProcess; pub use crate::pipeline::command::XvcStepCommand; @@ -45,13 +49,15 @@ pub use crate::pipeline::schema::XvcStepSchema; pub use crate::pipeline::step::XvcStep; use crate::pipeline::XvcStepInvalidate; +pub use crate::pipeline::api::run::RunCLI; + /// Pipeline management commands #[derive(Debug, Parser)] #[command(name = "pipeline")] pub struct PipelineCLI { - /// Name of the pipeline this command applies + /// Name of the pipeline this command applies to #[arg(long, short)] - pub name: Option, + pub pipeline_name: Option, /// Subcommand to run #[command(subcommand)] pub subcommand: PipelineSubCommand, @@ -63,303 +69,41 @@ pub struct PipelineCLI { #[allow(clippy::large_enum_variant)] pub enum PipelineSubCommand { /// Create a new pipeline - #[command()] - New { - /// Name of the pipeline this command applies to - #[arg(long, short)] - name: String, + New(NewCLI), - /// Default working directory - #[arg(short, long)] - workdir: Option, - }, - - /// Rename, change dir or set a pipeline as default - #[command()] - Update { - /// Name of the pipeline this command applies to - #[arg(long, short)] - name: Option, - - /// Rename the pipeline to - #[arg(long)] - rename: Option, - - /// Set the working directory - #[arg(long)] - workdir: Option, - - /// Set this pipeline as default - #[arg(long, help = "set this pipeline default")] - set_default: bool, - }, + /// Update the name and other attributes of a pipeline + Update(UpdateCLI), /// Delete a pipeline - #[command(about = "Delete a pipeline")] - Delete { - /// Name or GUID of the pipeline to be deleted - #[arg(long, short)] - name: String, - }, + Delete(DeleteCLI), /// Run a pipeline - #[command(about = "Run a pipeline")] - Run { - /// Name of the pipeline to run - #[arg(long, short)] - name: Option, - }, + Run(RunCLI), /// List all pipelines - #[command()] List, /// Generate a dot or mermaid diagram for the pipeline - #[command()] - Dag { - /// Name of the pipeline to generate the diagram - #[arg(long, short)] - name: Option, - - /// Output file. Writes to stdout if not set. - #[arg(long)] - file: Option, - - /// Format for graph. Either dot or mermaid. - #[arg(long, default_value = "dot")] - format: XvcPipelineDagFormat, - }, + Dag(DagCLI), /// Export the pipeline to a YAML or JSON file to edit - #[command()] - Export { - /// Name of the pipeline to export - #[arg(long, short)] - name: Option, - - /// File to write the pipeline. Writes to stdout if not set. - #[arg(long)] - file: Option, - - /// Output format. One of json or yaml. If not set, the format is - /// guessed from the file extension. If the file extension is not set, - /// json is used as default. - #[arg(long)] - format: Option, - }, + Export(ExportCLI), /// Import the pipeline from a file - #[command()] - Import { - /// Name of the pipeline to import. - /// If not set, the name from the file is used. - #[arg(long, short)] - name: Option, - - /// File to read the pipeline. Use stdin if not specified. - #[arg(long)] - file: Option, - - /// Input format. One of json or yaml. If not set, the format is - /// guessed from the file extension. If the file extension is not set, - /// json is used as default. - #[arg(long)] - format: Option, - - /// Overwrite the pipeline even if the name already exists - #[arg(long)] - overwrite: bool, - }, + Import(ImportCLI), /// Step creation, dependency, output commands #[command()] Step(StepCLI), } -/// Step creation, dependency, output commands -#[derive(Debug, Clone, Parser)] -#[command(name = "step")] -pub struct StepCLI { - /// Step subcommand - #[command(subcommand)] - pub subcommand: StepSubCommand, -} - -/// Step management subcommands -#[derive(Debug, Clone, Parser)] -#[command()] -pub enum StepSubCommand { - /// Add a new step - #[command()] - New { - /// Name of the new step - #[arg(long, short)] - step_name: String, - - /// Step command to run - #[arg(long, short)] - command: String, - - /// When to run the command. One of always, never, by_dependencies (default). - /// This is used to freeze or invalidate a step manually. - #[arg(long)] - when: Option, - }, - - /// Update a step's command or when options. - #[command(about = "Update step options")] - Update { - /// Name of the step to update. The step should already be defined. - #[arg(long, short)] - step_name: String, - - /// Step command to run - #[arg(long, short)] - command: Option, - - /// When to run the command. One of always, never, by_dependencies (default). - /// This is used to freeze or invalidate a step manually. - #[arg(long)] - when: Option, - }, - - /// Add a dependency to a step - #[command()] - Dependency { - /// Name of the step to add the dependency to - #[arg(long, short)] - step_name: String, - - /// Add a generic command output as a dependency. Can be used multiple times. - /// Please delimit the command with ' ' to avoid shell expansion. - #[arg(long = "generic")] - generics: Option>, - - /// Add a URL dependency to the step. Can be used multiple times. - #[arg(long = "url")] - urls: Option>, - - /// Add a file dependency to the step. Can be used multiple times. - #[arg(long = "file")] - files: Option>, - - /// Add a step dependency to a step. Can be used multiple times. - /// Steps are referred with their names. - #[arg(long = "step")] - steps: Option>, - - /// Add a glob items dependency to the step. - /// - /// You can depend on multiple files and directories with this dependency. - /// - /// The difference between this and the glob option is that this option keeps track of all - /// matching files, but glob only keeps track of the matched files' digest. When you want - /// to use ${XVC_GLOB_ITEMS}, ${XVC_ADDED_GLOB_ITEMS}, or ${XVC_REMOVED_GLOB_ITEMS} - /// environment variables in the step command, use the glob-items dependency. Otherwise, - /// you can use the glob option to save disk space. - #[arg(long = "glob_items", aliases=&["glob-items", "glob-i"])] - glob_items: Option>, - - /// Add a glob dependency to the step. Can be used multiple times. - /// - /// You can depend on multiple files and directories with this dependency. - /// - /// The difference between this and the glob-items option is that the glob-items option - /// keeps track of all matching files individually, but this option only keeps track of the - /// matched files' digest. This dependency uses considerably less disk space. - #[arg(long = "glob", aliases=&["globs"])] - globs: Option>, - - /// Add a parameter dependency to the step in the form filename.yaml::model.units . Can be used multiple times. - #[arg(long = "param", aliases = &["params"])] - params: Option>, - - /// Add a regex dependency in the form filename.txt:/^regex/ . Can be used multiple times. - /// - /// The difference between this and the regex option is that the regex-items option keeps - /// track of all matching lines, but regex only keeps track of the matched lines' digest. - /// When you want to use ${XVC_REGEX_ITEMS}, ${XVC_ADDED_REGEX_ITEMS}, - /// ${XVC_REMOVED_REGEX_ITEMS} environment variables in the step command, use the regex - /// option. Otherwise, you can use the regex-digest option to save disk space. - #[arg( - long = "regex_items", - aliases = &["regex-items", "regexp_items", "regexp-items"], - )] - regex_items: Option>, - - /// Add a regex dependency in the form filename.txt:/^regex/ . Can be used multiple times. - /// - /// The difference between this and the regex option is that the regex option keeps track - /// of all matching lines that can be used in the step command. This option only keeps - /// track of the matched lines' digest. - #[arg( - long = "regex", - aliases = &["regexp"], - )] - regexes: Option>, - - /// Add a line dependency in the form filename.txt::123-234 - /// - /// The difference between this and the lines option is that the line-items option keeps - /// track of all matching lines that can be used in the step command. This option only - /// keeps track of the matched lines' digest. When you want to use ${XVC_ALL_LINE_ITEMS}, - /// ${XVC_ADDED_LINE_ITEMS}, ${XVC_CHANGED_LINE_ITEMS} options in the step command, use the - /// line option. Otherwise, you can use the lines option to save disk space. - #[arg( - long = "line_items", - aliases = &["line-items", "line-i"], - )] - line_items: Option>, - - /// Add a line digest dependency in the form filename.txt::123-234 - /// - /// The difference between this and the line-items dependency is that the line option keeps - /// track of all matching lines that can be used in the step command. This option only - /// keeps track of the matched lines' digest. If you don't need individual lines to be - /// kept, use this option to save space. - #[arg( - long = "lines", - aliases = &["line"], - )] - lines: Option>, - }, - - /// Add an output to a step - #[command()] - Output { - /// Name of the step to add the output to - #[arg(long, short)] - step_name: String, - - /// Add a file output to the step. Can be used multiple times. - #[arg(long = "output-file")] - files: Option>, - - /// Add a metric output to the step. Can be used multiple times. - #[arg(long = "output-metric")] - metrics: Option>, - - /// Add an image output to the step. Can be used multiple times. - #[arg(long = "output-image")] - images: Option>, - }, - - /// Print step configuration - #[command()] - Show { - /// Name of the step to show - #[arg(long, short)] - step_name: String, - }, -} - impl UpdateFromXvcConfig for PipelineCLI { fn update_from_conf(self, conf: &XvcConfig) -> xvc_config::error::Result> { let default_pipeline = XvcPipeline::from_conf(conf); - let name = Some(self.name.clone().unwrap_or(default_pipeline.name)); + let name = Some(self.pipeline_name.clone().unwrap_or(default_pipeline.name)); watch!(name); Ok(Box::new(Self { - name, + pipeline_name: name, subcommand: self.subcommand.clone(), })) } @@ -449,103 +193,19 @@ pub fn cmd_pipeline( let conf = xvc_root.config(); let command = command.update_from_conf(conf)?; // This should already be filled from the conf if not given - let pipeline_name = command.name.unwrap(); + let pipeline_name = command.pipeline_name.unwrap(); match command.subcommand { - PipelineSubCommand::Run { name } => { - let pipeline_name = name.unwrap_or(pipeline_name); - cmd_run(output_snd, xvc_root, pipeline_name) - } - - PipelineSubCommand::New { name, workdir } => cmd_new(xvc_root, &name, workdir), - PipelineSubCommand::Update { - name, - rename, - workdir, - set_default, - } => cmd_update( - xvc_root, - name.unwrap_or(pipeline_name), - rename, - workdir, - set_default, - ), + PipelineSubCommand::Run(opts) => cmd_run(output_snd, xvc_root, opts), + PipelineSubCommand::New(opts) => cmd_new(xvc_root, opts), + PipelineSubCommand::Update(opts) => cmd_update(xvc_root, opts), PipelineSubCommand::List => cmd_list(output_snd, xvc_root), - PipelineSubCommand::Delete { name } => cmd_delete(xvc_root, name), - PipelineSubCommand::Export { name, file, format } => { - let pipeline_name = name.unwrap_or(pipeline_name); - cmd_export(output_snd, xvc_root, pipeline_name, file, format) - } - PipelineSubCommand::Dag { name, file, format } => { - let pipeline_name = name.unwrap_or(pipeline_name); - cmd_dag(output_snd, xvc_root, pipeline_name, file, format) - } - PipelineSubCommand::Import { - name, - file, - format, - overwrite, - } => { - let pipeline_name = name.unwrap_or(pipeline_name); - cmd_import(input, xvc_root, pipeline_name, file, format, overwrite) - } + PipelineSubCommand::Delete(opts) => cmd_delete(xvc_root, opts), + PipelineSubCommand::Export(opts) => cmd_export(output_snd, xvc_root, opts), + PipelineSubCommand::Dag(opts) => cmd_dag(output_snd, xvc_root, opts), + PipelineSubCommand::Import(opts) => cmd_import(input, xvc_root, opts), PipelineSubCommand::Step(step_cli) => { handle_step_cli(output_snd, xvc_root, &pipeline_name, step_cli) } } } -/// Dispatch `xvc pipeline step` subcommands. -pub fn handle_step_cli( - output_snd: &XvcOutputSender, - xvc_root: &XvcRoot, - pipeline_name: &str, - command: StepCLI, -) -> Result<()> { - match command.subcommand { - StepSubCommand::New { - step_name, - command, - when: changed, - } => cmd_step_new(xvc_root, pipeline_name, step_name, command, changed), - StepSubCommand::Update { - step_name, - command, - when: changed, - } => cmd_step_update(xvc_root, pipeline_name, step_name, command, changed), - - StepSubCommand::Dependency { - step_name, - generics, - urls, - files, - glob_items, - globs, - params, - steps, - regex_items, - regexes, - line_items, - lines, - } => XvcDependencyList::new(output_snd, xvc_root, pipeline_name, &step_name)? - .files(files)? - .glob_items(glob_items)? - .globs(globs)? - .params(params)? - .steps(steps)? - .generic_commands(generics)? - .regexes(regexes)? - .regex_items(regex_items)? - .lines(lines)? - .line_items(line_items)? - .urls(urls)? - .record(), - StepSubCommand::Output { - step_name, - files, - metrics, - images, - } => cmd_step_output(xvc_root, pipeline_name, step_name, files, metrics, images), - StepSubCommand::Show { step_name } => cmd_step_show(xvc_root, pipeline_name, step_name), - } -} - diff --git a/pipeline/src/pipeline/api/dag.rs b/pipeline/src/pipeline/api/dag.rs index d48abf571..e27dd1935 100644 --- a/pipeline/src/pipeline/api/dag.rs +++ b/pipeline/src/pipeline/api/dag.rs @@ -1,7 +1,9 @@ +use clap::Parser; use itertools::Itertools; use petgraph::graphmap::DiGraphMap; use tabbycat::attributes::{color, label, shape, Color, Shape}; use tabbycat::{AttrList, Edge, GraphBuilder, Identity, StmtList}; +use xvc_config::FromConfigKey; use xvc_core::{all_paths_and_metadata, XvcPath, XvcRoot}; use xvc_ecs::{HStore, R1NStore, XvcEntity}; use xvc_logging::{output, watch, XvcOutputSender}; @@ -20,6 +22,22 @@ use crate::{ XvcDependency, XvcOutput, XvcPipeline, XvcPipelineRunDir, XvcStep, }; +#[derive(Debug, Clone, Parser)] +#[command(name = "dag")] +pub struct DagCLI { + /// Name of the pipeline to generate the diagram + #[arg(long, short)] + pipeline_name: Option, + + /// Output file. Writes to stdout if not set. + #[arg(long)] + file: Option, + + /// Format for graph. Either dot or mermaid. + #[arg(long, default_value = "dot")] + format: XvcPipelineDagFormat, +} + #[derive(Debug, Clone, Eq, PartialEq, EnumString, Display, IntoStaticStr, Default)] #[strum(serialize_all = "lowercase")] pub enum XvcPipelineDagFormat { @@ -30,13 +48,11 @@ pub enum XvcPipelineDagFormat { /// Entry point for `xvc pipeline dag` command. /// Create a graph of the pipeline and output it in the specified format. -pub fn cmd_dag( - output_snd: &XvcOutputSender, - xvc_root: &XvcRoot, - pipeline_name: String, - file: Option, - format: XvcPipelineDagFormat, -) -> Result<()> { +pub fn cmd_dag(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: DagCLI) -> Result<()> { + let pipeline = XvcPipeline::from_conf(xvc_root.config()); + let pipeline_name = opts.pipeline_name.unwrap_or(pipeline.name); + let file = opts.file; + let format = opts.format; let _conf = xvc_root.config(); let (pipeline_e, _) = XvcPipeline::from_name(xvc_root, &pipeline_name)?; @@ -316,3 +332,4 @@ fn make_mermaid_graph( Ok(res_string) } + diff --git a/pipeline/src/pipeline/api/delete.rs b/pipeline/src/pipeline/api/delete.rs index d249ef29d..cd80955ae 100644 --- a/pipeline/src/pipeline/api/delete.rs +++ b/pipeline/src/pipeline/api/delete.rs @@ -1,13 +1,24 @@ use crate::error::{Error, Result}; +use clap::Parser; use xvc_config::FromConfigKey; use xvc_core::XvcRoot; use crate::XvcPipeline; +/// Delete a pipeline +#[derive(Debug, Clone, Parser)] +#[command(name = "delete")] +pub struct DeleteCLI { + /// Name or GUID of the pipeline to be deleted + #[arg(long, short)] + pipeline_name: String, +} + /// Entry point for `xvc pipeline delete` command. /// It deletes the pipeline with the given name. /// It is not possible to delete the default pipeline. -pub fn cmd_delete(xvc_root: &XvcRoot, name: String) -> Result<()> { +pub fn cmd_delete(xvc_root: &XvcRoot, opts: DeleteCLI) -> Result<()> { + let name = opts.pipeline_name; let conf = xvc_root.config(); let default_pipeline = XvcPipeline::from_conf(conf); if name == default_pipeline.name { diff --git a/pipeline/src/pipeline/api/export.rs b/pipeline/src/pipeline/api/export.rs index 0515248c6..d84c0bcaa 100644 --- a/pipeline/src/pipeline/api/export.rs +++ b/pipeline/src/pipeline/api/export.rs @@ -1,7 +1,9 @@ use crate::error::{Error, Result}; +use clap::Parser; use itertools::Itertools; use std::{fs, path::PathBuf}; +use xvc_config::FromConfigKey; use xvc_core::{ util::serde::{to_json, to_yaml}, @@ -16,19 +18,35 @@ use crate::{ XvcStepCommand, XvcStepSchema, }; +#[derive(Debug, Clone, Parser)] +#[command(name = "export")] +pub struct ExportCLI { + /// Name of the pipeline to export + #[arg(long, short)] + pipeline_name: Option, + + /// File to write the pipeline. Writes to stdout if not set. + #[arg(long)] + file: Option, + + /// Output format. One of json or yaml. If not set, the format is + /// guessed from the file extension. If the file extension is not set, + /// json is used as default. + #[arg(long)] + format: Option, +} + /// Entry point for `xvc pipeline export` command. /// Export the pipeline and all its steps to a file. /// The file format is determined by the `format` parameter. /// If `file` is None, prints to stdout. /// If `name` is None, uses the default pipeline name from the config. /// If `format` is None, uses the default format from [XvcSchemaSerializationFormat::default()] -pub fn cmd_export( - output_snd: &XvcOutputSender, - xvc_root: &XvcRoot, - name: String, - file: Option, - format: Option, -) -> Result<()> { +pub fn cmd_export(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: ExportCLI) -> Result<()> { + let pipeline = XvcPipeline::from_conf(xvc_root.config()); + let name = opts.pipeline_name.unwrap_or(pipeline.name); + let file = opts.file; + let format = opts.format; let mut p_res: Result<(XvcEntity, XvcPipeline)> = Err(Error::CannotFindPipeline { name: name.clone() }); diff --git a/pipeline/src/pipeline/api/import.rs b/pipeline/src/pipeline/api/import.rs index 3258b4e18..8d2d0567a 100644 --- a/pipeline/src/pipeline/api/import.rs +++ b/pipeline/src/pipeline/api/import.rs @@ -1,6 +1,8 @@ use crate::error::{Error, Result}; +use clap::Parser; use log::warn; use std::{fs, io::BufRead, path::PathBuf}; +use xvc_config::FromConfigKey; use xvc_core::XvcRoot; use xvc_ecs::{R11Store, R1NStore}; @@ -10,20 +12,41 @@ use crate::{ XvcStepCommand, }; +#[derive(Debug, Clone, Parser)] +#[command(name = "import")] +pub struct ImportCLI { + /// Name of the pipeline to import. + /// If not set, the name from the file is used. + #[arg(long, short)] + pipeline_name: Option, + + /// File to read the pipeline. Use stdin if not specified. + #[arg(long)] + file: Option, + + /// Input format. One of json or yaml. If not set, the format is + /// guessed from the file extension. If the file extension is not set, + /// json is used as default. + #[arg(long)] + format: Option, + + /// Overwrite the pipeline even if the name already exists + #[arg(long)] + overwrite: bool, +} + /// Entry point for `xvc pipeline import` command. /// Reads a pipeline definition in JSON or YAML formats and creates/updates it. /// If `name` is None, uses the pipeline name from the file. /// If `file` is None, reads from stdin. /// If `format` is None, uses the file extension to determine the format. /// If `overwrite` is true, overwrites the pipeline if it already exists. -pub fn cmd_import( - input: R, - xvc_root: &XvcRoot, - pipeline_name: String, - file: Option, - format: Option, - overwrite: bool, -) -> Result<()> { +pub fn cmd_import(input: R, xvc_root: &XvcRoot, opts: ImportCLI) -> Result<()> { + let pipeline = XvcPipeline::from_conf(xvc_root.config()); + let pipeline_name = opts.pipeline_name.unwrap_or(pipeline.name); + let file = opts.file; + let format = opts.format; + let overwrite = opts.overwrite; let (content, format) = match file { None => { if let Some(format) = format { diff --git a/pipeline/src/pipeline/api/new.rs b/pipeline/src/pipeline/api/new.rs index 66edd264a..607e5dd2d 100644 --- a/pipeline/src/pipeline/api/new.rs +++ b/pipeline/src/pipeline/api/new.rs @@ -1,19 +1,33 @@ use std::path::PathBuf; use crate::error::Result; +use clap::Parser; use xvc_core::{XvcPath, XvcRoot}; use xvc_ecs::error::Error as EcsError; use xvc_ecs::R11Store; use crate::{XvcPipeline, XvcPipelineRunDir}; +/// Create a new pipeline +#[derive(Debug, Clone, Parser)] +#[command(name = "new")] +pub struct NewCLI { + /// Name of the pipeline this command applies to + #[arg(long, short)] + pipeline_name: String, + + /// Default working directory + #[arg(short, long)] + workdir: Option, +} + /// Entry point for `xvc pipeline new` command. /// It creates a new pipeline with the given name. /// If `workdir` is None, uses the default workdir. -pub fn cmd_new(xvc_root: &XvcRoot, name: &str, workdir: Option) -> Result<()> { +pub fn cmd_new(xvc_root: &XvcRoot, opts: NewCLI) -> Result<()> { Ok( xvc_root.with_r11store_mut(|rs: &mut R11Store| { - let name = name.to_owned(); + let name = opts.pipeline_name.clone(); if rs.left.iter().any(|(_, p)| p.name == name) { Err(EcsError::KeyAlreadyFound { key: name, @@ -24,7 +38,7 @@ pub fn cmd_new(xvc_root: &XvcRoot, name: &str, workdir: Option) -> Resu let pipeline = XvcPipeline { name }; let p_e = xvc_root.new_entity(); rs.left.insert(p_e, pipeline); - if let Some(wd) = &workdir { + if let Some(wd) = &opts.workdir { let conf = xvc_root.config(); let current_dir = conf.current_dir()?; let run_dir = XvcPipelineRunDir { @@ -37,3 +51,4 @@ pub fn cmd_new(xvc_root: &XvcRoot, name: &str, workdir: Option) -> Resu })?, ) } + diff --git a/pipeline/src/pipeline/api/run.rs b/pipeline/src/pipeline/api/run.rs index ecb2cd61d..6b3de4e6f 100644 --- a/pipeline/src/pipeline/api/run.rs +++ b/pipeline/src/pipeline/api/run.rs @@ -1,19 +1,29 @@ -use crate::error::Result; +use crate::{error::Result, XvcPipeline}; +use clap::Parser; +use xvc_config::FromConfigKey; use xvc_core::XvcRoot; use xvc_logging::XvcOutputSender; use crate::pipeline::the_grand_pipeline_loop; +/// Run a pipeline +#[derive(Debug, Clone, Parser)] +#[command(name = "run")] +pub struct RunCLI { + /// Name of the pipeline to run + #[arg(long, short)] + pipeline_name: Option, +} + /// Entry point for `xvc pipeline run` command. /// /// It loads an [`XvcPipeline`] with the name and runs [`the_grand_pipeline_loop`] /// with it. -pub fn cmd_run( - output_snd: &XvcOutputSender, - xvc_root: &XvcRoot, - pipeline_name: String, -) -> Result<()> { +pub fn cmd_run(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: RunCLI) -> Result<()> { + let config = xvc_root.config(); + let default_pipeline = XvcPipeline::from_conf(config); + let pipeline_name = opts.pipeline_name.unwrap_or(default_pipeline.name); the_grand_pipeline_loop(output_snd, xvc_root, pipeline_name) } diff --git a/pipeline/src/pipeline/api/step_dependency.rs b/pipeline/src/pipeline/api/step_dependency.rs index a4ebd4cb2..42ea42e14 100644 --- a/pipeline/src/pipeline/api/step_dependency.rs +++ b/pipeline/src/pipeline/api/step_dependency.rs @@ -12,6 +12,7 @@ use crate::pipeline::deps::regex_items::RegexItemsDep; use crate::pipeline::deps::step::StepDep; use crate::pipeline::deps::url::UrlDigestDep; use crate::pipeline::deps::ParamDep; +use crate::pipeline::step::StepSubCommand; use regex::Regex; use url::Url; @@ -25,6 +26,47 @@ use crate::{pipeline::deps, XvcDependency, XvcParamFormat, XvcPipeline, XvcStep} /// Entry point for `xvc pipeline step dependency` command. /// Add a set of new dependencies to the given step in the pipeline. /// +/// Unlike other entry points, this receives the options directly to avoid a long list of +/// parameters. +pub fn cmd_step_dependency( + output_snd: &XvcOutputSender, + xvc_root: &XvcRoot, + pipeline_name: &str, + cmd_opts: StepSubCommand, +) -> Result<()> { + if let StepSubCommand::Dependency { + step_name, + generics, + urls, + files, + steps, + glob_items, + globs, + params, + regex_items, + regexes, + line_items, + lines, + } = cmd_opts + { + XvcDependencyList::new(output_snd, xvc_root, pipeline_name, &step_name)? + .files(files)? + .glob_items(glob_items)? + .globs(globs)? + .params(params)? + .steps(steps)? + .generic_commands(generics)? + .regexes(regexes)? + .regex_items(regex_items)? + .lines(lines)? + .line_items(line_items)? + .urls(urls)? + .record() + } else { + Err(anyhow::anyhow!("This method is only for StepSubCommand::Dependency").into()) + } +} +/// /// Parses dependencies using member functions, in order to avoid a single function with a lot of parameters. pub struct XvcDependencyList<'a> { output_snd: &'a XvcOutputSender, @@ -358,3 +400,4 @@ impl<'a> XvcDependencyList<'a> { Ok(()) } } + diff --git a/pipeline/src/pipeline/api/update.rs b/pipeline/src/pipeline/api/update.rs index f27b7612f..9e46f0f31 100644 --- a/pipeline/src/pipeline/api/update.rs +++ b/pipeline/src/pipeline/api/update.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use crate::error::Result; +use clap::Parser; use xvc_core::error::Error as CoreError; use xvc_core::{XvcPath, XvcRoot}; use xvc_ecs::error::Error as EcsError; @@ -8,16 +9,35 @@ use xvc_ecs::R11Store; use crate::{XvcPipeline, XvcPipelineRunDir}; +/// Rename, change dir or set a pipeline as default +#[derive(Debug, Clone, Parser)] +#[command(name = "update")] +pub struct UpdateCLI { + /// Name of the pipeline this command applies to + #[arg(long, short)] + pipeline_name: Option, + + /// Rename the pipeline to + #[arg(long)] + rename: Option, + + /// Set the working directory + #[arg(long)] + workdir: Option, + + /// Set this pipeline as default + #[arg(long, help = "set this pipeline default")] + set_default: bool, +} + /// Entry point for `xvc pipeline update` command. /// Can rename the pipeline, change the working directory or set the pipeline as /// default. -pub fn cmd_update( - xvc_root: &XvcRoot, - name: String, - rename: Option, - workdir: Option, - default: bool, -) -> Result<()> { +pub fn cmd_update(xvc_root: &XvcRoot, opts: UpdateCLI) -> Result<()> { + let name = opts.pipeline_name.expect("Pipeline name is required"); + let rename = opts.rename; + let workdir = opts.workdir; + let default = opts.set_default; Ok( xvc_root.with_r11store_mut(|rs: &mut R11Store| { let name = name.to_owned(); diff --git a/pipeline/src/pipeline/command.rs b/pipeline/src/pipeline/command.rs index be7b70a9a..0bb138762 100644 --- a/pipeline/src/pipeline/command.rs +++ b/pipeline/src/pipeline/command.rs @@ -42,18 +42,30 @@ impl AsRef for XvcStepCommand { /// processes. #[derive(Debug)] pub struct CommandProcess { + /// Environment variables injected to the shell that runs the command. This is used to pass + /// added, removed items in certain dependency types. pub environment: HashMap, + /// The step that this command belongs to pub step: XvcStep, + /// The command to run pub step_command: XvcStepCommand, + /// When we started running the command pub birth: Option, + /// The process that runs the command pub process: Option, + /// Channel to send stdout to pub stdout_sender: Sender, + /// Channel to send stderr to pub stderr_sender: Sender, + /// Channel to receive stdout from pub stdout_receiver: Receiver, + /// Channel to receive stderr from pub stderr_receiver: Receiver, } impl CommandProcess { + /// Create a new CommandProcess by creating channels and setting other variables to their + /// default values. pub fn new(step: &XvcStep, step_command: &XvcStepCommand) -> Self { let (stdout_sender, stdout_receiver) = crossbeam_channel::bounded(CHANNEL_CAPACITY); let (stderr_sender, stderr_receiver) = crossbeam_channel::bounded(CHANNEL_CAPACITY); @@ -70,6 +82,7 @@ impl CommandProcess { } } + /// Add an environment variable to inject to the shell that runs the command. pub fn add_environment_variable(&mut self, key: &str, value: &str) -> Result<&mut Self> { watch!(self); self.environment.insert(key.to_owned(), value.to_owned()); @@ -77,6 +90,8 @@ impl CommandProcess { Ok(self) } + /// Start executing the command in a shell. Updates birth and process variables after + /// detaching. pub fn run(&mut self) -> Result<()> { watch!(self.environment); let process = sp::Exec::shell(self.step_command.command.clone()) @@ -96,6 +111,7 @@ impl CommandProcess { Ok(()) } + /// Collects the output from process and sends to output channels. pub fn update_output_channels(&mut self) -> Result<()> { if let Some(p) = &self.process { if let Some(mut stdout) = p.stdout.as_ref() { diff --git a/pipeline/src/pipeline/step.rs b/pipeline/src/pipeline/step.rs index 51772f1af..749430eac 100644 --- a/pipeline/src/pipeline/step.rs +++ b/pipeline/src/pipeline/step.rs @@ -1,11 +1,227 @@ #![allow(clippy::enum_variant_names)] use crate::error::{Error, Result}; -use crate::XvcPipeline; +use crate::{ + cmd_step_dependency, cmd_step_new, cmd_step_output, cmd_step_show, cmd_step_update, XvcPipeline, +}; +use clap::Parser; use sad_machine::state_machine; use serde::{Deserialize, Serialize}; use xvc_core::XvcRoot; use xvc_ecs::{persist, XvcEntity}; +use xvc_logging::XvcOutputSender; + +use super::XvcStepInvalidate; + +/// Step creation, dependency, output commands +#[derive(Debug, Clone, Parser)] +#[command(name = "step")] +pub struct StepCLI { + /// Step subcommand + #[command(subcommand)] + pub subcommand: StepSubCommand, +} + +/// Step management subcommands +#[derive(Debug, Clone, Parser)] +#[command()] +pub enum StepSubCommand { + /// Add a new step + #[command()] + New { + /// Name of the new step + #[arg(long, short)] + step_name: String, + + /// Step command to run + #[arg(long, short)] + command: String, + + /// When to run the command. One of always, never, by_dependencies (default). + /// This is used to freeze or invalidate a step manually. + #[arg(long)] + when: Option, + }, + + /// Update a step's command or when options. + #[command(about = "Update step options")] + Update { + /// Name of the step to update. The step should already be defined. + #[arg(long, short)] + step_name: String, + + /// Step command to run + #[arg(long, short)] + command: Option, + + /// When to run the command. One of always, never, by_dependencies (default). + /// This is used to freeze or invalidate a step manually. + #[arg(long)] + when: Option, + }, + + /// Add a dependency to a step + #[command()] + Dependency { + /// Name of the step to add the dependency to + #[arg(long, short)] + step_name: String, + + /// Add a generic command output as a dependency. Can be used multiple times. + /// Please delimit the command with ' ' to avoid shell expansion. + #[arg(long = "generic")] + generics: Option>, + + /// Add a URL dependency to the step. Can be used multiple times. + #[arg(long = "url")] + urls: Option>, + + /// Add a file dependency to the step. Can be used multiple times. + #[arg(long = "file")] + files: Option>, + + /// Add a step dependency to a step. Can be used multiple times. + /// Steps are referred with their names. + #[arg(long = "step")] + steps: Option>, + + /// Add a glob items dependency to the step. + /// + /// You can depend on multiple files and directories with this dependency. + /// + /// The difference between this and the glob option is that this option keeps track of all + /// matching files, but glob only keeps track of the matched files' digest. When you want + /// to use ${XVC_GLOB_ITEMS}, ${XVC_ADDED_GLOB_ITEMS}, or ${XVC_REMOVED_GLOB_ITEMS} + /// environment variables in the step command, use the glob-items dependency. Otherwise, + /// you can use the glob option to save disk space. + #[arg(long = "glob_items", aliases=&["glob-items", "glob-i"])] + glob_items: Option>, + + /// Add a glob dependency to the step. Can be used multiple times. + /// + /// You can depend on multiple files and directories with this dependency. + /// + /// The difference between this and the glob-items option is that the glob-items option + /// keeps track of all matching files individually, but this option only keeps track of the + /// matched files' digest. This dependency uses considerably less disk space. + #[arg(long = "glob", aliases=&["globs"])] + globs: Option>, + + /// Add a parameter dependency to the step in the form filename.yaml::model.units . Can be used multiple times. + #[arg(long = "param", aliases = &["params"])] + params: Option>, + + /// Add a regex dependency in the form filename.txt:/^regex/ . Can be used multiple times. + /// + /// The difference between this and the regex option is that the regex-items option keeps + /// track of all matching lines, but regex only keeps track of the matched lines' digest. + /// When you want to use ${XVC_REGEX_ITEMS}, ${XVC_ADDED_REGEX_ITEMS}, + /// ${XVC_REMOVED_REGEX_ITEMS} environment variables in the step command, use the regex + /// option. Otherwise, you can use the regex-digest option to save disk space. + #[arg( + long = "regex_items", + aliases = &["regex-items", "regexp_items", "regexp-items"], + )] + regex_items: Option>, + + /// Add a regex dependency in the form filename.txt:/^regex/ . Can be used multiple times. + /// + /// The difference between this and the regex option is that the regex option keeps track + /// of all matching lines that can be used in the step command. This option only keeps + /// track of the matched lines' digest. + #[arg( + long = "regex", + aliases = &["regexp"], + )] + regexes: Option>, + + /// Add a line dependency in the form filename.txt::123-234 + /// + /// The difference between this and the lines option is that the line-items option keeps + /// track of all matching lines that can be used in the step command. This option only + /// keeps track of the matched lines' digest. When you want to use ${XVC_ALL_LINE_ITEMS}, + /// ${XVC_ADDED_LINE_ITEMS}, ${XVC_CHANGED_LINE_ITEMS} options in the step command, use the + /// line option. Otherwise, you can use the lines option to save disk space. + #[arg( + long = "line_items", + aliases = &["line-items", "line-i"], + )] + line_items: Option>, + + /// Add a line digest dependency in the form filename.txt::123-234 + /// + /// The difference between this and the line-items dependency is that the line option keeps + /// track of all matching lines that can be used in the step command. This option only + /// keeps track of the matched lines' digest. If you don't need individual lines to be + /// kept, use this option to save space. + #[arg( + long = "lines", + aliases = &["line"], + )] + lines: Option>, + }, + + /// Add an output to a step + #[command()] + Output { + /// Name of the step to add the output to + #[arg(long, short)] + step_name: String, + + /// Add a file output to the step. Can be used multiple times. + #[arg(long = "output-file")] + files: Option>, + + /// Add a metric output to the step. Can be used multiple times. + #[arg(long = "output-metric")] + metrics: Option>, + + /// Add an image output to the step. Can be used multiple times. + #[arg(long = "output-image")] + images: Option>, + }, + + /// Print step configuration + #[command()] + Show { + /// Name of the step to show + #[arg(long, short)] + step_name: String, + }, +} + +/// Dispatch `xvc pipeline step` subcommands. +pub fn handle_step_cli( + output_snd: &XvcOutputSender, + xvc_root: &XvcRoot, + pipeline_name: &str, + command: StepCLI, +) -> Result<()> { + match command.subcommand { + StepSubCommand::New { + step_name, + command, + when: changed, + } => cmd_step_new(xvc_root, pipeline_name, step_name, command, changed), + StepSubCommand::Update { + step_name, + command, + when: changed, + } => cmd_step_update(xvc_root, pipeline_name, step_name, command, changed), + + dep_opts @ StepSubCommand::Dependency { .. } => { + cmd_step_dependency(output_snd, xvc_root, pipeline_name, dep_opts) + } + + StepSubCommand::Output { + step_name, + files, + metrics, + images, + } => cmd_step_output(xvc_root, pipeline_name, step_name, files, metrics, images), + StepSubCommand::Show { step_name } => cmd_step_show(xvc_root, pipeline_name, step_name), + } +} /// A step (stage) in a pipeline. #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, Ord, PartialOrd)] @@ -203,3 +419,4 @@ state_machine! { } } + diff --git a/workflow_tests/tests/test_pipeline.rs b/workflow_tests/tests/test_pipeline.rs index 8a8cfdbfe..23416f0ac 100644 --- a/workflow_tests/tests/test_pipeline.rs +++ b/workflow_tests/tests/test_pipeline.rs @@ -14,17 +14,29 @@ fn test_pipeline() -> Result<()> { common::run_xvc(Some(&xvc_root), &c, XvcVerbosity::Trace) }; - x(&["new", "--name", "pipeline-1"])?; - x(&["new", "--name", "pipeline-2", "--workdir", "pipeline-2"])?; - x(&["update", "--name", "pipeline-1", "--rename", "pipeline-old"])?; + x(&["new", "--pipeline-name", "pipeline-1"])?; + x(&[ + "new", + "--pipeline-name", + "pipeline-2", + "--workdir", + "pipeline-2", + ])?; + x(&[ + "update", + "--pipeline-name", + "pipeline-1", + "--rename", + "pipeline-old", + ])?; let pipelines_1 = x(&["list"])?; assert!(pipelines_1.contains("pipeline-old")); - x(&["delete", "--name", "pipeline-old"])?; + x(&["delete", "--pipeline-name", "pipeline-old"])?; let pipelines_2 = x(&["list"])?; assert!(!pipelines_2.contains("pipeline-old")); let res = x(&[ - "-n", + "-p", "pipeline-2", "step", "new", @@ -37,7 +49,7 @@ fn test_pipeline() -> Result<()> { watch!(res); x(&[ - "-n", + "-p", "pipeline-2", "step", "new", @@ -49,7 +61,7 @@ fn test_pipeline() -> Result<()> { "always", ])?; x(&[ - "-n", + "-p", "pipeline-2", "step", "dependency", @@ -60,7 +72,7 @@ fn test_pipeline() -> Result<()> { ])?; x(&[ - "-n", + "-p", "pipeline-2", "step", "dependency", @@ -71,7 +83,7 @@ fn test_pipeline() -> Result<()> { ])?; x(&[ - "-n", + "-p", "pipeline-2", "step", "dependency", @@ -82,7 +94,7 @@ fn test_pipeline() -> Result<()> { ])?; x(&[ - "-n", + "-p", "pipeline-2", "step", "dependency", @@ -93,7 +105,7 @@ fn test_pipeline() -> Result<()> { ])?; x(&[ - "-n", + "-p", "pipeline-2", "step", "dependency", @@ -104,7 +116,7 @@ fn test_pipeline() -> Result<()> { ])?; x(&[ - "-n", + "-p", "pipeline-2", "step", "dependency", diff --git a/workflow_tests/tests/test_storage_new_r2.rs b/workflow_tests/tests/test_storage_new_r2.rs index de9e0b793..bc56ef616 100644 --- a/workflow_tests/tests/test_storage_new_r2.rs +++ b/workflow_tests/tests/test_storage_new_r2.rs @@ -120,7 +120,7 @@ fn create_directory_hierarchy() -> Result { create_directory_tree(&temp_dir, 10, 10, 1000, Some(47))?; // root/dir1 may have another tree let level_1 = &temp_dir.join(&PathBuf::from("dir-0001")); - create_directory_tree(&level_1, 10, 10, 1000, Some(47))?; + create_directory_tree(level_1, 10, 10, 1000, Some(47))?; Ok(temp_dir) } @@ -147,7 +147,7 @@ fn test_storage_new_r2() -> Result<()> { let account_id = env::var("R2_ACCOUNT_ID")?; let account_id = account_id.trim(); - let config_file_name = write_s3cmd_config(&account_id, &access_key, &secret_key)?; + let config_file_name = write_s3cmd_config(account_id, access_key, secret_key)?; watch!(config_file_name); let s3cmd = |cmd: &str, append: &str| -> String { @@ -179,7 +179,7 @@ fn test_storage_new_r2() -> Result<()> { &format!("| rg {storage_prefix} | rg {XVC_STORAGE_GUID_FILENAME}"), ); watch!(s3_bucket_list); - assert!(s3_bucket_list.len() > 0); + assert!(!s3_bucket_list.is_empty()); let the_file = "file-0000.bin"; diff --git a/workflow_tests/tests/z_test_docs.rs b/workflow_tests/tests/z_test_docs.rs index a04b16d9e..d1a078fdf 100644 --- a/workflow_tests/tests/z_test_docs.rs +++ b/workflow_tests/tests/z_test_docs.rs @@ -48,9 +48,8 @@ fn link_to_docs() -> Result<()> { book_dirs_and_filters } else { // If not defined, make all tests - // // If not defined, make all tests vec![ - ("intro", ".*"), + ("intro", r".*"), ("ref", r".*"), ("start", r".*"), ("how-to", r".*"), @@ -185,7 +184,7 @@ fn z_doc_tests() -> Result<()> { .register_bin("zsh", which::which("zsh")) .register_bin("dot", which::which("dot")) .case("docs/*/*.md") - .timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(120)) // We skip this for the time being. .skip("docs/start/ml.md");