feat(commands): Add option stdin_command to be used in CLI and config…

… file (#266)

Adds the option `stdin_command` to `BackupOptions`. This options starts
the given command and reads its output as stdin if `-` is specified as
backup source.

see rustic-rs/rustic#662

---------

Co-authored-by: simonsan <[email protected]>

crates/core/src/backend.rs

@@ -1,5 +1,6 @@
 //! Module for backend related functionality.
 pub(crate) mod cache;
+pub(crate) mod childstdout;
 pub(crate) mod decrypt;
 pub(crate) mod dry_run;
 pub(crate) mod hotcold;
@@ -22,7 +23,7 @@ use mockall::mock;
 use serde_derive::{Deserialize, Serialize};
 
 use crate::{
-    backend::node::Node,
+    backend::node::{Metadata, Node, NodeType},
     error::{BackendAccessErrorKind, RusticErrorKind},
     id::Id,
     RusticResult,
@@ -424,6 +425,18 @@ pub struct ReadSourceEntry<O> {
     pub open: Option<O>,
 }
 
+impl<O> ReadSourceEntry<O> {
+    fn from_path(path: PathBuf, open: Option<O>) -> RusticResult<Self> {
+        let node = Node::new_node(
+            path.file_name()
+                .ok_or_else(|| BackendAccessErrorKind::PathNotAllowed(path.clone()))?,
+            NodeType::File,
+            Metadata::default(),
+        );
+        Ok(Self { path, node, open })
+    }
+}
+
 /// Trait for backends that can read and open sources.
 /// This trait is implemented by all backends that can read data and open from a source.
 pub trait ReadSourceOpen {
@@ -442,6 +455,14 @@ pub trait ReadSourceOpen {
     fn open(self) -> RusticResult<Self::Reader>;
 }
 
+/// blanket implementation for readers
+impl<T: Read + Send + 'static> ReadSourceOpen for T {
+    type Reader = T;
+    fn open(self) -> RusticResult<Self::Reader> {
+        Ok(self)
+    }
+}
+
 /// Trait for backends that can read from a source.
 ///
 /// This trait is implemented by all backends that can read data from a source.

crates/core/src/backend/childstdout.rs

@@ -0,0 +1,78 @@
+use std::{
+    iter::{once, Once},
+    path::PathBuf,
+    process::{Child, ChildStdout, Command, Stdio},
+    sync::Mutex,
+};
+
+use crate::{
+    backend::{ReadSource, ReadSourceEntry},
+    error::{RepositoryErrorKind, RusticResult},
+    CommandInput,
+};
+
+/// The `ChildStdoutSource` is a `ReadSource` when spawning a child process and reading its stdout
+#[derive(Debug)]
+pub struct ChildStdoutSource {
+    /// The path of the stdin entry.
+    path: PathBuf,
+    /// The child process
+    ///
+    /// # Note
+    ///
+    /// This is in a Mutex as we want to take out `ChildStdout`
+    /// in the `entries` method - but this method only gets a
+    /// reference of self.
+    process: Mutex<Child>,
+    /// the command which is called
+    command: CommandInput,
+}
+
+impl ChildStdoutSource {
+    /// Creates a new `ChildSource`.
+    pub fn new(cmd: &CommandInput, path: PathBuf) -> RusticResult<Self> {
+        let process = Command::new(cmd.command())
+            .args(cmd.args())
+            .stdout(Stdio::piped())
+            .spawn()
+            .map_err(|err| {
+                RepositoryErrorKind::CommandExecutionFailed(
+                    "stdin-command".into(),
+                    "call".into(),
+                    err,
+                )
+                .into()
+            });
+
+        let process = cmd.on_failure().display_result(process)?;
+
+        Ok(Self {
+            path,
+            process: Mutex::new(process),
+            command: cmd.clone(),
+        })
+    }
+
+    /// Finishes the `ChildSource`
+    pub fn finish(self) -> RusticResult<()> {
+        let status = self.process.lock().unwrap().wait();
+        self.command
+            .on_failure()
+            .handle_status(status, "stdin-command", "call")?;
+        Ok(())
+    }
+}
+
+impl ReadSource for ChildStdoutSource {
+    type Open = ChildStdout;
+    type Iter = Once<RusticResult<ReadSourceEntry<ChildStdout>>>;
+
+    fn size(&self) -> RusticResult<Option<u64>> {
+        Ok(None)
+    }
+
+    fn entries(&self) -> Self::Iter {
+        let open = self.process.lock().unwrap().stdout.take();
+        once(ReadSourceEntry::from_path(self.path.clone(), open))
+    }
+}

crates/core/src/backend/stdin.rs

@@ -1,51 +1,33 @@
-use std::{io::stdin, path::PathBuf};
+use std::{
+    io::{stdin, Stdin},
+    iter::{once, Once},
+    path::PathBuf,
+};
 
 use crate::{
-    backend::{
-        node::{Metadata, Node, NodeType},
-        ReadSource, ReadSourceEntry, ReadSourceOpen,
-    },
+    backend::{ReadSource, ReadSourceEntry},
     error::RusticResult,
 };
 
 /// The `StdinSource` is a `ReadSource` for stdin.
 #[derive(Debug, Clone)]
 pub struct StdinSource {
-    /// Whether we have already yielded the stdin entry.
-    finished: bool,
     /// The path of the stdin entry.
     path: PathBuf,
 }
 
 impl StdinSource {
     /// Creates a new `StdinSource`.
     pub const fn new(path: PathBuf) -> Self {
-        Self {
-            finished: false,
-            path,
-        }
-    }
-}
-
-/// The `OpenStdin` is a `ReadSourceOpen` for stdin.
-#[derive(Debug, Copy, Clone)]
-pub struct OpenStdin();
-
-impl ReadSourceOpen for OpenStdin {
-    /// The reader type.
-    type Reader = std::io::Stdin;
-
-    /// Opens stdin.
-    fn open(self) -> RusticResult<Self::Reader> {
-        Ok(stdin())
+        Self { path }
     }
 }
 
 impl ReadSource for StdinSource {
     /// The open type.
-    type Open = OpenStdin;
+    type Open = Stdin;
     /// The iterator type.
-    type Iter = Self;
+    type Iter = Once<RusticResult<ReadSourceEntry<Stdin>>>;
 
     /// Returns the size of the source.
     fn size(&self) -> RusticResult<Option<u64>> {
@@ -54,27 +36,7 @@ impl ReadSource for StdinSource {
 
     /// Returns an iterator over the source.
     fn entries(&self) -> Self::Iter {
-        self.clone()
-    }
-}
-
-impl Iterator for StdinSource {
-    type Item = RusticResult<ReadSourceEntry<OpenStdin>>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.finished {
-            return None;
-        }
-        self.finished = true;
-
-        Some(Ok(ReadSourceEntry {
-            path: self.path.clone(),
-            node: Node::new_node(
-                self.path.file_name().unwrap(),
-                NodeType::File,
-                Metadata::default(),
-            ),
-            open: Some(OpenStdin()),
-        }))
+        let open = Some(stdin());
+        once(ReadSourceEntry::from_path(self.path.clone(), open))
     }
 }

crates/core/src/commands/backup.rs

@@ -11,6 +11,7 @@ use serde_with::{serde_as, DisplayFromStr};
 use crate::{
     archiver::{parent::Parent, Archiver},
     backend::{
+        childstdout::ChildStdoutSource,
         dry_run::DryRunBackend,
         ignore::{LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions},
         stdin::StdinSource,
@@ -23,6 +24,7 @@ use crate::{
         PathList, SnapshotFile,
     },
     repository::{IndexedIds, IndexedTree, Repository},
+    CommandInput,
 };
 
 #[cfg(feature = "clap")]
@@ -141,6 +143,10 @@ pub struct BackupOptions {
     #[cfg_attr(feature = "merge", merge(skip))]
     pub stdin_filename: String,
 
+    /// Call the given command and use its output as stdin
+    #[cfg_attr(feature = "clap", clap(long, value_name = "COMMAND"))]
+    pub stdin_command: Option<CommandInput>,
+
     /// Manually set backup path in snapshot
     #[cfg_attr(feature = "clap", clap(long, value_name = "PATH", value_hint = ValueHint::DirPath))]
     pub as_path: Option<PathBuf>,
@@ -246,15 +252,29 @@ pub(crate) fn backup<P: ProgressBars, S: IndexedIds>(
 
     let snap = if backup_stdin {
         let path = &backup_path[0];
-        let src = StdinSource::new(path.clone());
-        archiver.archive(
-            &src,
-            path,
-            as_path.as_ref(),
-            opts.parent_opts.skip_identical_parent,
-            opts.no_scan,
-            &p,
-        )?
+        if let Some(command) = &opts.stdin_command {
+            let src = ChildStdoutSource::new(command, path.clone())?;
+            let res = archiver.archive(
+                &src,
+                path,
+                as_path.as_ref(),
+                opts.parent_opts.skip_identical_parent,
+                opts.no_scan,
+                &p,
+            )?;
+            src.finish()?;
+            res
+        } else {
+            let src = StdinSource::new(path.clone());
+            archiver.archive(
+                &src,
+                path,
+                as_path.as_ref(),
+                opts.parent_opts.skip_identical_parent,
+                opts.no_scan,
+                &p,
+            )?
+        }
     } else {
         let src = LocalSource::new(
             opts.ignore_save_opts,

crates/core/src/error.rs

@@ -364,6 +364,8 @@ pub enum BackendAccessErrorKind {
     RemovingDataFromBackendFailed,
     /// failed to list files on Backend
     ListingFilesOnBackendFailed,
+    /// Path is not allowed: `{0:?}`
+    PathNotAllowed(PathBuf),
 }
 
 /// [`ConfigFileErrorKind`] describes the errors that can be returned for `ConfigFile`s

crates/core/src/repository/command_input.rs

@@ -165,20 +165,32 @@ pub enum OnFailure {
 
 impl OnFailure {
     fn eval<T>(self, res: RusticResult<T>) -> RusticResult<Option<T>> {
-        match res {
-            Err(err) => match self {
+        let res = self.display_result(res);
+        match (res, self) {
+            (Err(err), Self::Error) => Err(err),
+            (Err(_), _) => Ok(None),
+            (Ok(res), _) => Ok(Some(res)),
+        }
+    }
+
+    /// Displays a result depending on the defined error handling which still yielding the same result
+    /// # Note
+    ///
+    /// This can be used where an error might occur, but in that
+    /// case we have to abort.
+    pub fn display_result<T>(self, res: RusticResult<T>) -> RusticResult<T> {
+        if let Err(err) = &res {
+            match self {
                 Self::Error => {
                     error!("{err}");
-                    Err(err)
                 }
                 Self::Warn => {
                     warn!("{err}");
-                    Ok(None)
                 }
-                Self::Ignore => Ok(None),
-            },
-            Ok(res) => Ok(Some(res)),
+                Self::Ignore => {}
+            }
         }
+        res
     }
 
     /// Handle a status of a called command depending on the defined error handling

crates/core/tests/integration.rs

@@ -384,6 +384,36 @@ fn test_backup_dry_run_with_tar_gz_passes(
     Ok(())
 }
 
+#[rstest]
+fn test_backup_stdin_command(
+    set_up_repo: Result<RepoOpen>,
+    insta_snapshotfile_redaction: Settings,
+) -> Result<()> {
+    // Fixtures
+    let repo = set_up_repo?.to_indexed_ids()?;
+    let paths = PathList::from_string("-")?;
+
+    let cmd: CommandInput = "echo test".parse()?;
+    let opts = BackupOptions::default()
+        .stdin_filename("test")
+        .stdin_command(cmd);
+    // backup data from cmd
+    let snapshot = repo.backup(&opts, &paths, SnapshotFile::default())?;
+    insta_snapshotfile_redaction.bind(|| {
+        assert_with_win("stdin-command-summary", &snapshot);
+    });
+
+    // re-read index
+    let repo = repo.to_indexed()?;
+
+    // check content
+    let node = repo.node_from_snapshot_path("latest:test", |_| true)?;
+    let mut content = Vec::new();
+    repo.dump(&node, &mut content)?;
+    assert_eq!(content, b"test\n");
+    Ok(())
+}
+
 #[rstest]
 fn test_ls(
     tar_gz_testdata: Result<TestSource>,