Refactor pipeline CLI (#243)

.github/workflows/rust.yml

@@ -60,7 +60,7 @@ jobs:
 
       - name: Install required packages
         run: |
-          brew install s3cmd
+          brew install --head s3cmd # 2.3.0 has a bug with Python 3.12
           brew install ripgrep
           brew install tree
           brew install lsd

CHANGELOG.md

@@ -5,6 +5,10 @@ tree, and subtasks are marked with indentation.
 
 ## Unreleased
 
+- Refactored Pipelines API to expose more functionality
+  - PR: <https://github.com/iesahin/xvc/pull/243>
+  - Exposed Git operations in the API
+
 ## 0.6.3 (2023-11-02)
 
 - Updated README and added its commands to the tests

README.md

@@ -37,7 +37,7 @@ $ cargo install xvc
 
 ## 🏃🏾 Quicktart
 
-Xvc tracks your files and directories on top of Git. To start run the following command in the repository.
+Xvc seamlessly monitors your files and directories on top of Git. To commence, execute the following command within the repository:
 
 ```console
 $ git init # if you're not already in a Git repository
@@ -46,17 +46,15 @@ Initialized empty Git repository in [CWD]/.git/
 $ xvc init
 ```
 
-It initializes the metafiles in `.xvc/` directory and adds `.xvcignore` file for paths you may want to hide from Xvc.
+This command initializes the `.xvc/` directory and adds a `.xvcignore` file for specifying paths you wish to conceal from Xvc.
 
-Add your data files and directories for tracking.
+Include your data files and directories for tracking:
 
 ```shell
 $ xvc file track my-data/ --as symlink
 ```
 
-The command calculates data content hashes (with BLAKE-3, by default) and records them.
-It commits these changes to Git.
-It also copies these files to content-addressed directories under `.xvc/b3` and creates read-only symbolic links to them.
+This command calculates content hashes for data (using BLAKE-3, by default) and logs them. The changes are committed to Git, and the files are copied to content-addressed directories within `.xvc/b3`. Additionally, read-only symbolic links to these directories are created. 
 
 You can specify different [recheck (checkout) methods](https://docs.xvc.dev/ref/xvc-file-recheck/) for files and directories, depending on your use case.
 If you need to track model files that change frequently, you can set recheck method `--as copy` (the default).
@@ -89,6 +87,8 @@ $ xvc file bring my-data/ --from my-remote
 
 ```
 
+This approach ensures convenient access to files from the shared storage when needed.
+
 You don't have to reconfigure the storage after cloning, but you need to have valid credentials as environment variables
 to access the storage.
 Xvc never stores any credentials.

book/src/ref/xvc-file-copy.md

@@ -163,9 +163,9 @@ FH          19 [..] c85f3e81 c85f3e81 another-set/data3.txt
 FH          19 [..] c85f3e81 c85f3e81 another-set/data2.txt
 FH          19 [..] c85f3e81 c85f3e81 another-set/data.txt
 DX         160 [..]                   another-set
-FX         130 [..]          ac46bf74 .xvcignore
-FX         [..] .gitignore
-Total #: 11 Workspace Size:        [..] Cached Size:          19
+FX         141 [..]          3054b812 .xvcignore
+FX         529 [..]          [..] .gitignore
+Total #: 11 Workspace Size:        1105 Cached Size:          19
 
 
 ```

book/src/ref/xvc-file-list.md

@@ -127,7 +127,7 @@ Now it lists all files and directories.
 ```console
 $ xvc file list --sort name-asc
 FX         107 [..]          ce9fcf30 .gitignore
-FX         130 [..]          ac46bf74 .xvcignore
+FX         141 [..]          3054b812 .xvcignore
 DX         224 [..]                   dir-0001
 FX        2001 [..]          1953f05d dir-0001/file-0001.bin
 FX        2002 [..]          7e807161 dir-0001/file-0002.bin
@@ -157,8 +157,8 @@ FX        2001 [..]          1953f05d dir-0005/file-0001.bin
 FX        2002 [..]          7e807161 dir-0005/file-0002.bin
 FX        2003 [..]          d2432259 dir-0005/file-0003.bin
 FX        2004 [..]          63535612 dir-0005/file-0004.bin
-FX        2005 [..]     447933dc dir-0005/file-0005.bin
-Total #: 32 Workspace Size:       51432 Cached Size:           0
+FX        2005 [..]          447933dc dir-0005/file-0005.bin
+Total #: 32 Workspace Size:       51443 Cached Size:           0
 
 
 ```

book/src/ref/xvc-file-move.md

@@ -125,9 +125,9 @@ $ xvc file list
 XH                                 c85f3e81          data6.txt
 FH          19 [..] c85f3e81 c85f3e81 another-set/data4.txt
 DX          96 [..]                   another-set
-FX         130 [..]          ac46bf74 .xvcignore
+FX         141 [..]          3054b812 .xvcignore
 FX         [..] .gitignore
-Total #: 5 Workspace Size:         [..] Cached Size:          19
+Total #: 5 Workspace Size:         785 Cached Size:          19
 
 
 ```

book/src/ref/xvc-file-remove.md

@@ -58,9 +58,9 @@ $ xvc file track 'd*.txt'
 
 $ xvc file list
 FC        [..] c85f3e81 c85f3e81 data.txt
-FX        [..]          ac46bf74 .xvcignore
-FX        [..] .gitignore
-Total #: 3 Workspace Size:         [..] Cached Size:          19
+FX         141 [..]          3054b812 .xvcignore
+FX         190 [..]          [..] .gitignore
+Total #: 3 Workspace Size:         350 Cached Size:          19
 
 
 $ tree .xvc/b3/
@@ -118,9 +118,9 @@ $ xvc file carry-in --force data.txt
 
 $ xvc file list
 FC         [..] c85f3e81 c85f3e81 data.txt
-FX         [..]          ac46bf74 .xvcignore
-FX         [..]          [..] .gitignore
-Total #: 3 Workspace Size:         [..] Cached Size:          19
+FX         141 [..]          3054b812 .xvcignore
+FX         190 [..]          [..] .gitignore
+Total #: 3 Workspace Size:         350 Cached Size:          19
 
 
 $ tree .xvc/b3/
@@ -157,9 +157,9 @@ $ tree .xvc/b3/
 
 $ xvc file list
 FC         [..] 6602cff6 6602cff6 data.txt
-FX         [..]          ac46bf74 .xvcignore
-FX         [..]          [..] .gitignore
-Total #: 3 Workspace Size:         [..] Cached Size:          19
+FX         141 [..]          3054b812 .xvcignore
+FX         190 [..]          [..] .gitignore
+Total #: 3 Workspace Size:         350 Cached Size:          19
 
 
 $ xvc file remove --from-cache --only-version c85-f3e data.txt
@@ -281,9 +281,9 @@ $ xvc file copy data.txt data2.txt --as symlink
 $ xvc file list
 SS        [..] [..] 4a2e9d7c          data2.txt
 FC        1024 [..] 4a2e9d7c 4a2e9d7c data.txt
-FX         130 [..]          ac46bf74 .xvcignore
-FX         [..] .gitignore
-Total #: 4 Workspace Size:        [..] Cached Size:        1024
+FX         141 [..]          3054b812 .xvcignore
+FX         274 [..]          [..] .gitignore
+Total #: 4 Workspace Size:        1621 Cached Size:        1024
 
 
 $ xvc file remove --from-cache data.txt

book/src/ref/xvc-file-untrack.md

@@ -35,9 +35,9 @@ $ xvc file track 'd*.txt'
 
 $ xvc file list
 FC          19 [..] c85f3e81 c85f3e81 data.txt
-FX         130 [..]          [..] .xvcignore
+FX         141 [..]          3054b812 .xvcignore
 FX         [..] .gitignore
-Total #: 3 Workspace Size:         [..] Cached Size:          19
+Total #: 3 Workspace Size:         350 Cached Size:          19
 
 
 ```

book/src/ref/xvc-pipeline-dag.md

@@ -9,10 +9,10 @@ Generate a dot or mermaid diagram for the pipeline
 Usage: xvc pipeline dag [OPTIONS]
 
 Options:
-  -n, --name <NAME>      Name of the pipeline to generate the diagram
-      --file <FILE>      Output file. Writes to stdout if not set
-      --format <FORMAT>  Format for graph. Either dot or mermaid [default: dot]
-  -h, --help             Print help
+  -p, --pipeline-name <PIPELINE_NAME>  Name of the pipeline to generate the diagram
+      --file <FILE>                    Output file. Writes to stdout if not set
+      --format <FORMAT>                Format for graph. Either dot or mermaid [default: dot]
+  -h, --help                           Print help
 
 ```
 

book/src/ref/xvc-pipeline-delete.md

@@ -6,10 +6,10 @@
 $ xvc pipeline delete --help
 Delete a pipeline
 
-Usage: xvc pipeline delete --name <NAME>
+Usage: xvc pipeline delete --pipeline-name <PIPELINE_NAME>
 
 Options:
-  -n, --name <NAME>  Name or GUID of the pipeline to be deleted
-  -h, --help         Print help
+  -p, --pipeline-name <PIPELINE_NAME>  Name or GUID of the pipeline to be deleted
+  -h, --help                           Print help
 
 ```

book/src/ref/xvc-pipeline-export.md

@@ -9,10 +9,10 @@ Export the pipeline to a YAML or JSON file to edit
 Usage: xvc pipeline export [OPTIONS]
 
 Options:
-  -n, --name <NAME>      Name of the pipeline to export
-      --file <FILE>      File to write the pipeline. Writes to stdout if not set
-      --format <FORMAT>  Output format. One of json or yaml. If not set, the format is guessed from the file extension. If the file extension is not set, json is used as default
-  -h, --help             Print help
+  -p, --pipeline-name <PIPELINE_NAME>  Name of the pipeline to export
+      --file <FILE>                    File to write the pipeline. Writes to stdout if not set
+      --format <FORMAT>                Output format. One of json or yaml. If not set, the format is guessed from the file extension. If the file extension is not set, json is used as default
+  -h, --help                           Print help
 
 ```
 

book/src/ref/xvc-pipeline-import.md

@@ -9,11 +9,11 @@ Import the pipeline from a file
 Usage: xvc pipeline import [OPTIONS]
 
 Options:
-  -n, --name <NAME>      Name of the pipeline to import. If not set, the name from the file is used
-      --file <FILE>      File to read the pipeline. Use stdin if not specified
-      --format <FORMAT>  Input format. One of json or yaml. If not set, the format is guessed from the file extension. If the file extension is not set, json is used as default
-      --overwrite        Overwrite the pipeline even if the name already exists
-  -h, --help             Print help
+  -p, --pipeline-name <PIPELINE_NAME>  Name of the pipeline to import. If not set, the name from the file is used
+      --file <FILE>                    File to read the pipeline. Use stdin if not specified
+      --format <FORMAT>                Input format. One of json or yaml. If not set, the format is guessed from the file extension. If the file extension is not set, json is used as default
+      --overwrite                      Overwrite the pipeline even if the name already exists
+  -h, --help                           Print help
 
 ```
 
@@ -248,7 +248,7 @@ You can specify a new name for the pipeline and it will override the name set in
 This way you can edit and import similar pipelines with minor differences.
 
 ```console
-$ xvc pipeline import --name another-pipeline --file pipeline.yaml
+$ xvc pipeline import --pipeline-name another-pipeline --file pipeline.yaml
 
 ```
 

book/src/ref/xvc-pipeline-new.md

@@ -6,12 +6,12 @@
 $ xvc pipeline new --help
 Create a new pipeline
 
-Usage: xvc pipeline new [OPTIONS] --name <NAME>
+Usage: xvc pipeline new [OPTIONS] --pipeline-name <PIPELINE_NAME>
 
 Options:
-  -n, --name <NAME>        Name of the pipeline this command applies to
-  -w, --workdir <WORKDIR>  Default working directory
-  -h, --help               Print help
+  -p, --pipeline-name <PIPELINE_NAME>  Name of the pipeline this command applies to
+  -w, --workdir <WORKDIR>              Default working directory
+  -h, --help                           Print help
 
 ```
 
@@ -28,7 +28,8 @@ $ xvc init
 You can create a new pipeline with a name.
 
 ```console
-$ xvc pipeline new --name my-pipeline
+$ xvc pipeline new --pipeline-name my-pipeline
+
 ```
 
 By default it will run the commands in the repository root.
@@ -49,7 +50,7 @@ If you want to define a pipeline specific to a directory, you can set the workin
 
 ```console
 $ xvc-test-helper create-directory-tree --directories 1 --files 3  --seed 20230215
-$ xvc pipeline new --name another-pipeline --workdir dir-0001
+$ xvc pipeline new --pipeline-name another-pipeline --workdir dir-0001
 
 ```
 

book/src/ref/xvc-pipeline-run.md

@@ -9,8 +9,8 @@ Run a pipeline
 Usage: xvc pipeline run [OPTIONS]
 
 Options:
-  -n, --name <NAME>  Name of the pipeline to run
-  -h, --help         Print help
+  -p, --pipeline-name <PIPELINE_NAME>  Name of the pipeline to run
+  -h, --help                           Print help
 
 ```
 
@@ -74,12 +74,14 @@ $ xvc pipeline run
 You can run a specific pipeline by specifying its name with `--name` option.
 
 ```console
-$ xvc pipeline new --name my-pipeline
-$ xvc pipeline --name my-pipeline step new --step-name my-hello --command "echo 'hello from my-pipeline'"
+$ xvc pipeline new --pipeline-name my-pipeline
+
+$ xvc pipeline --pipeline-name my-pipeline step new --step-name my-hello --command "echo 'hello from my-pipeline'"
+
 ```
 
 ```console
-$ xvc pipeline run --name my-pipeline
+$ xvc pipeline run --pipeline-name my-pipeline
 [OUT] [my-hello] hello from my-pipeline
  
 [DONE] my-hello (echo 'hello from my-pipeline')

book/src/ref/xvc-pipeline-step-dependency-generic.md

@@ -22,7 +22,7 @@ $ xvc  pipeline step dependency --step-name morning-message --generic 'date +%F'
 
 The step is invalidated when the date changes and the step is run again.
 
-```console
+```console,ignore
 $ xvc pipeline run
 [OUT] [morning-message] Good Morning!
  
@@ -34,6 +34,9 @@ The step won't run until tomorrow, when `date +%F` changes.
 
 ```console
 $ xvc pipeline run
+[OUT] [morning-message] Good Morning!
+ 
+[DONE] morning-message (echo 'Good Morning!')
 
 ```
 

book/src/ref/xvc-pipeline-update.md

@@ -4,15 +4,15 @@
 
 ```console
 $ xvc pipeline update --help
-Rename, change dir or set a pipeline as default
+Update the name and other attributes of a pipeline
 
 Usage: xvc pipeline update [OPTIONS]
 
 Options:
-  -n, --name <NAME>        Name of the pipeline this command applies to
-      --rename <RENAME>    Rename the pipeline to
-      --workdir <WORKDIR>  Set the working directory
-      --set-default        set this pipeline default
-  -h, --help               Print help
+  -p, --pipeline-name <PIPELINE_NAME>  Name of the pipeline this command applies to
+      --rename <RENAME>                Rename the pipeline to
+      --workdir <WORKDIR>              Set the working directory
+      --set-default                    set this pipeline default
+  -h, --help                           Print help
 
 ```

book/src/ref/xvc-pipeline.md

@@ -10,7 +10,7 @@ Usage: xvc pipeline [OPTIONS] <COMMAND>
 
 Commands:
   new     Create a new pipeline
-  update  Rename, change dir or set a pipeline as default
+  update  Update the name and other attributes of a pipeline
   delete  Delete a pipeline
   run     Run a pipeline
   list    List all pipelines
@@ -21,7 +21,7 @@ Commands:
   help    Print this message or the help of the given subcommand(s)
 
 Options:
-  -n, --name <NAME>  Name of the pipeline this command applies
-  -h, --help         Print help
+  -p, --pipeline-name <PIPELINE_NAME>  Name of the pipeline this command applies to
+  -h, --help                           Print help
 
 ```

core/src/lib.rs

@@ -37,6 +37,9 @@ pub use types::xvcpath::XvcCachePath;
 pub use types::xvcpath::XvcPath;
 pub use types::xvcroot::XvcRoot;
 
+pub use walker::AbsolutePath;
+pub use xvc_walker as walker;
+
 pub use error::Error;
 pub use error::Result;
 

file/src/copy/mod.rs

@@ -309,11 +309,9 @@ pub(crate) fn recheck_destination(
     Ok(())
 }
 
-pub(crate) fn cmd_copy(
-    output_snd: &XvcOutputSender,
-    xvc_root: &XvcRoot,
-    opts: CopyCLI,
-) -> Result<()> {
+/// Entry point for `xvc file copy` command.
+/// Copies a file (and its records) to a new location in the repository
+pub fn cmd_copy(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: CopyCLI) -> Result<()> {
     // Get all files to copy
 
     let stored_metadata_store = xvc_root.load_store::<XvcMetadata>()?;