From ae66312b5338913b4670888b1fba009061b4bf77 Mon Sep 17 00:00:00 2001 From: amribm Date: Fri, 13 Sep 2024 10:27:50 +0530 Subject: [PATCH 01/22] feat(watcher): change for add poll watcher support --- src/app.rs | 35 ++++++++++++++--- src/cli.rs | 25 +++++++++++++ src/config/watcher.rs | 87 +++++++++++++++++++++++++++++++------------ 3 files changed, 118 insertions(+), 29 deletions(-) diff --git a/src/app.rs b/src/app.rs index e27d68e5a6cab..16238dac49ff8 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,5 +1,10 @@ #![allow(missing_docs)] -use std::{num::NonZeroUsize, path::PathBuf, process::ExitStatus, time::Duration}; +use std::{ + num::{NonZeroU64, NonZeroUsize}, + path::PathBuf, + process::ExitStatus, + time::Duration, +}; use exitcode::ExitCode; use futures::StreamExt; @@ -12,7 +17,7 @@ use crate::extra_context::ExtraContext; #[cfg(feature = "api")] use crate::{api, internal_events::ApiStarted}; use crate::{ - cli::{handle_config_errors, LogFormat, Opts, RootOpts}, + cli::{handle_config_errors, LogFormat, Opts, RootOpts, WatchConfigMethod}, config::{self, Config, ConfigPath}, heartbeat, internal_events::{VectorConfigLoadError, VectorQuit, VectorStarted, VectorStopped}, @@ -59,9 +64,18 @@ impl ApplicationConfig { let graceful_shutdown_duration = (!opts.no_graceful_shutdown_limit) .then(|| Duration::from_secs(u64::from(opts.graceful_shutdown_limit_secs))); + let watcher_conf = if opts.watch_config { + Some(watcher_config( + opts.watch_config_method, + opts.watch_config_poll_interval_seconds, + )) + } else { + None + }; + let config = load_configs( &config_paths, - opts.watch_config, + watcher_conf, opts.require_healthy, opts.allow_empty_config, graceful_shutdown_duration, @@ -466,7 +480,7 @@ pub fn build_runtime(threads: Option, thread_name: &str) -> Result, require_healthy: Option, allow_empty_config: bool, graceful_shutdown_duration: Option, @@ -474,9 +488,10 @@ pub async fn load_configs( ) -> Result { let config_paths = config::process_paths(config_paths).ok_or(exitcode::CONFIG)?; - if watch_config { + if watcher_conf.is_some() { // Start listening for config changes immediately. config::watcher::spawn_thread( + watcher_conf.unwrap(), signal_handler.clone_tx(), config_paths.iter().map(Into::into), None, @@ -526,3 +541,13 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) ); info!(message = "Log level is enabled.", level = ?level); } + +pub fn watcher_config( + method: WatchConfigMethod, + interval: NonZeroU64, +) -> config::watcher::WatcherConfig { + match method { + WatchConfigMethod::Inotify => config::watcher::WatcherConfig::RecommendedWatcher, + WatchConfigMethod::Poll => config::watcher::WatcherConfig::PollWatcher(interval.into()), + } +} diff --git a/src/cli.rs b/src/cli.rs index e94911b1dfaed..a239371e4c354 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -152,6 +152,23 @@ pub struct RootOpts { #[arg(short, long, env = "VECTOR_WATCH_CONFIG")] pub watch_config: bool, + /// Method for watching config + /// + /// By default `vector` use `inotify` which is recommended for linux based system + /// `poll` watcher can be used where inotify not work. ie, attaching config using NFS + #[arg(long, default_value = "inotify", env = "VECTOR_WATCH_METHOD")] + pub watch_config_method: WatchConfigMethod, + + /// poll for changes in configuration file on given interval + /// + /// This config is only applicable if poll is enabled in `--watch-config-method` + #[arg( + long, + env = "VECTOR_WATCH_CONFIG_POLL_INTERVAL_SECONDS", + default_value = "30" + )] + pub watch_config_poll_interval_seconds: NonZeroU64, + /// Set the internal log rate limit #[arg( short, @@ -354,6 +371,14 @@ pub enum LogFormat { Json, } +#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] +pub enum WatchConfigMethod { + /// recommended for linux based systems + Inotify, + /// works for EFS/NFS like network storage systems + Poll, +} + pub fn handle_config_errors(errors: Vec) -> exitcode::ExitCode { for error in errors { error!(message = "Configuration error.", %error); diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 584dcc3bd64fe..482d51f0a1a47 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -4,7 +4,10 @@ use std::{ thread, }; -use notify::{recommended_watcher, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; +use notify::{ + recommended_watcher, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, + Watcher as NotifyWatcher, +}; use crate::Error; @@ -19,11 +22,40 @@ const CONFIG_WATCH_DELAY: std::time::Duration = std::time::Duration::from_secs(1 const RETRY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +pub enum WatcherConfig { + /// recommended watcher for os, usally inotify for linux based systems + RecommendedWatcher, + /// poll based watcher. for watching files from NFS. + PollWatcher(u64), +} + +enum Watcher { + /// recommended watcher for os, usally inotify for linux based systems + RecommendedWatcher(RecommendedWatcher), + /// poll based watcher. for watching files from NFS. + PollWatcher(PollWatcher), +} + +impl Watcher { + fn add_paths(&mut self, config_paths: &[PathBuf]) -> Result<(), Error> { + let watcher: &mut dyn NotifyWatcher = match self { + &mut Watcher::RecommendedWatcher(ref mut w) => w, + &mut Watcher::PollWatcher(ref mut w) => w, + }; + + for path in config_paths { + watcher.watch(path, RecursiveMode::Recursive)?; + } + Ok(()) + } +} + /// Sends a ReloadFromDisk on config_path changes. /// Accumulates file changes until no change for given duration has occurred. /// Has best effort guarantee of detecting all file changes from the end of /// this function until the main thread stops. pub fn spawn_thread<'a>( + watcher_conf: WatcherConfig, signal_tx: crate::signal::SignalTx, config_paths: impl IntoIterator + 'a, delay: impl Into>, @@ -33,7 +65,7 @@ pub fn spawn_thread<'a>( // Create watcher now so not to miss any changes happening between // returning from this function and the thread starting. - let mut watcher = Some(create_watcher(&config_paths)?); + let mut watcher = Some(create_watcher(&watcher_conf, &config_paths)?); info!("Watching configuration files."); @@ -53,7 +85,7 @@ pub fn spawn_thread<'a>( // We need to read paths to resolve any inode changes that may have happened. // And we need to do it before raising sighup to avoid missing any change. - if let Err(error) = add_paths(&mut watcher, &config_paths) { + if let Err(error) = watcher.add_paths(&config_paths) { error!(message = "Failed to read files to watch.", %error); break; } @@ -72,7 +104,7 @@ pub fn spawn_thread<'a>( thread::sleep(RETRY_TIMEOUT); - watcher = create_watcher(&config_paths) + watcher = create_watcher(&watcher_conf, &config_paths) .map_err(|error| error!(message = "Failed to create file watcher.", %error)) .ok(); @@ -91,26 +123,28 @@ pub fn spawn_thread<'a>( } fn create_watcher( + watcher_conf: &WatcherConfig, config_paths: &[PathBuf], -) -> Result< - ( - RecommendedWatcher, - Receiver>, - ), - Error, -> { +) -> Result<(Watcher, Receiver>), Error> { info!("Creating configuration file watcher."); - let (sender, receiver) = channel(); - let mut watcher = recommended_watcher(sender)?; - add_paths(&mut watcher, config_paths)?; - Ok((watcher, receiver)) -} -fn add_paths(watcher: &mut RecommendedWatcher, config_paths: &[PathBuf]) -> Result<(), Error> { - for path in config_paths { - watcher.watch(path, RecursiveMode::Recursive)?; + let (sender, receiver) = channel(); + match watcher_conf { + WatcherConfig::RecommendedWatcher => { + let recommended_watcher = recommended_watcher(sender)?; + let mut watcher = Watcher::RecommendedWatcher(recommended_watcher); + watcher.add_paths(config_paths)?; + Ok((watcher, receiver)) + } + WatcherConfig::PollWatcher(interval) => { + let config = + notify::Config::default().with_poll_interval(Duration::from_secs(*interval)); + let poll_watcher = PollWatcher::new(sender, config)?; + let mut watcher = Watcher::PollWatcher(poll_watcher); + watcher.add_paths(config_paths)?; + Ok((watcher, receiver)) + } } - Ok(()) } #[cfg(all(test, unix, not(target_os = "macos")))] // https://github.com/vectordotdev/vector/issues/5000 @@ -140,12 +174,13 @@ mod tests { let delay = Duration::from_secs(3); let dir = temp_dir().to_path_buf(); let file_path = dir.join("vector.toml"); + let watcher_conf = WatcherConfig::RecommendedWatcher; std::fs::create_dir(&dir).unwrap(); let mut file = File::create(&file_path).unwrap(); let (signal_tx, signal_rx) = broadcast::channel(128); - spawn_thread(signal_tx, &[dir], delay).unwrap(); + spawn_thread(watcher_conf, signal_tx, &[dir], delay).unwrap(); if !test(&mut file, delay * 5, signal_rx).await { panic!("Test timed out"); @@ -159,9 +194,10 @@ mod tests { let delay = Duration::from_secs(3); let file_path = temp_file(); let mut file = File::create(&file_path).unwrap(); + let watcher_conf = WatcherConfig::RecommendedWatcher; let (signal_tx, signal_rx) = broadcast::channel(128); - spawn_thread(signal_tx, &[file_path], delay).unwrap(); + spawn_thread(watcher_conf, signal_tx, &[file_path], delay).unwrap(); if !test(&mut file, delay * 5, signal_rx).await { panic!("Test timed out"); @@ -179,8 +215,10 @@ mod tests { let mut file = File::create(&file_path).unwrap(); std::os::unix::fs::symlink(&file_path, &sym_file).unwrap(); + let watcher_conf = WatcherConfig::RecommendedWatcher; + let (signal_tx, signal_rx) = broadcast::channel(128); - spawn_thread(signal_tx, &[sym_file], delay).unwrap(); + spawn_thread(watcher_conf, signal_tx, &[sym_file], delay).unwrap(); if !test(&mut file, delay * 5, signal_rx).await { panic!("Test timed out"); @@ -195,12 +233,13 @@ mod tests { let dir = temp_dir().to_path_buf(); let sub_dir = dir.join("sources"); let file_path = sub_dir.join("input.toml"); + let watcher_conf = WatcherConfig::RecommendedWatcher; std::fs::create_dir_all(&sub_dir).unwrap(); let mut file = File::create(&file_path).unwrap(); let (signal_tx, signal_rx) = broadcast::channel(128); - spawn_thread(signal_tx, &[sub_dir], delay).unwrap(); + spawn_thread(watcher_conf, signal_tx, &[sub_dir], delay).unwrap(); if !test(&mut file, delay * 5, signal_rx).await { panic!("Test timed out"); From 1802ece06a97c9d79575f23872db56a3f1a23882 Mon Sep 17 00:00:00 2001 From: amribm Date: Fri, 13 Sep 2024 10:45:54 +0530 Subject: [PATCH 02/22] fix(watcher): fix spelling --- src/config/watcher.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 482d51f0a1a47..3dcb5f5d02740 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -23,14 +23,14 @@ const CONFIG_WATCH_DELAY: std::time::Duration = std::time::Duration::from_secs(1 const RETRY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); pub enum WatcherConfig { - /// recommended watcher for os, usally inotify for linux based systems + /// recommended watcher for os, usually inotify for linux based systems RecommendedWatcher, /// poll based watcher. for watching files from NFS. PollWatcher(u64), } enum Watcher { - /// recommended watcher for os, usally inotify for linux based systems + /// recommended watcher for os, usually inotify for linux based systems RecommendedWatcher(RecommendedWatcher), /// poll based watcher. for watching files from NFS. PollWatcher(PollWatcher), From ed86d4a2d179583b75989e9c3330428763f40d8a Mon Sep 17 00:00:00 2001 From: amribm Date: Fri, 13 Sep 2024 23:14:14 +0530 Subject: [PATCH 03/22] Add(watcher): change for PR --- src/app.rs | 4 ++-- src/config/watcher.rs | 45 +++++++++++++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/app.rs b/src/app.rs index 16238dac49ff8..a546b92a9e0cc 100644 --- a/src/app.rs +++ b/src/app.rs @@ -488,10 +488,10 @@ pub async fn load_configs( ) -> Result { let config_paths = config::process_paths(config_paths).ok_or(exitcode::CONFIG)?; - if watcher_conf.is_some() { + if let Some(watcher_conf) = watcher_conf { // Start listening for config changes immediately. config::watcher::spawn_thread( - watcher_conf.unwrap(), + watcher_conf, signal_handler.clone_tx(), config_paths.iter().map(Into::into), None, diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 3dcb5f5d02740..ce7ba0dfe06ff 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -1,13 +1,13 @@ -use std::{path::PathBuf, time::Duration}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; use std::{ sync::mpsc::{channel, Receiver}, thread, }; -use notify::{ - recommended_watcher, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, - Watcher as NotifyWatcher, -}; +use notify::{recommended_watcher, EventKind, RecursiveMode}; use crate::Error; @@ -31,25 +31,42 @@ pub enum WatcherConfig { enum Watcher { /// recommended watcher for os, usually inotify for linux based systems - RecommendedWatcher(RecommendedWatcher), + RecommendedWatcher(notify::RecommendedWatcher), /// poll based watcher. for watching files from NFS. - PollWatcher(PollWatcher), + PollWatcher(notify::PollWatcher), } impl Watcher { fn add_paths(&mut self, config_paths: &[PathBuf]) -> Result<(), Error> { - let watcher: &mut dyn NotifyWatcher = match self { - &mut Watcher::RecommendedWatcher(ref mut w) => w, - &mut Watcher::PollWatcher(ref mut w) => w, - }; - for path in config_paths { - watcher.watch(path, RecursiveMode::Recursive)?; + self.watch(path, RecursiveMode::Recursive)?; + } + Ok(()) + } + + fn watch(&mut self, path: &Path, recursive_mode: RecursiveMode) -> Result<(), Error> { + match self { + &mut Watcher::RecommendedWatcher(ref mut watcher) => { + watcher.watch(path, recursive_mode)? + } + &mut Watcher::PollWatcher(ref mut watcher) => watcher.watch(path, recursive_mode)?, } Ok(()) } } +// impl From for Watcher { +// fn from(recommended_watcher: notify::RecommendedWatcher) -> Self { +// Watcher::RecommendedWatcher(recommended_watcher) +// } +// } + +// impl From for Watcher { +// fn from(poll_watcher: notify::PollWatcher) -> Self { +// Watcher::PollWatcher(poll_watcher) +// } +// } + /// Sends a ReloadFromDisk on config_path changes. /// Accumulates file changes until no change for given duration has occurred. /// Has best effort guarantee of detecting all file changes from the end of @@ -139,7 +156,7 @@ fn create_watcher( WatcherConfig::PollWatcher(interval) => { let config = notify::Config::default().with_poll_interval(Duration::from_secs(*interval)); - let poll_watcher = PollWatcher::new(sender, config)?; + let poll_watcher = notify::PollWatcher::new(sender, config)?; let mut watcher = Watcher::PollWatcher(poll_watcher); watcher.add_paths(config_paths)?; Ok((watcher, receiver)) From 4bee121c4fbe8b35a3188be54b431a988a77b72e Mon Sep 17 00:00:00 2001 From: Ameer Ibrahim <99497445+amribm@users.noreply.github.com> Date: Sun, 15 Sep 2024 14:35:18 +0530 Subject: [PATCH 04/22] Update src/config/watcher.rs Co-authored-by: Jorge Hermo --- src/config/watcher.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/config/watcher.rs b/src/config/watcher.rs index ce7ba0dfe06ff..4f19903f585ed 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -47,11 +47,10 @@ impl Watcher { fn watch(&mut self, path: &Path, recursive_mode: RecursiveMode) -> Result<(), Error> { match self { &mut Watcher::RecommendedWatcher(ref mut watcher) => { - watcher.watch(path, recursive_mode)? + watcher.watch(path, recursive_mode) } - &mut Watcher::PollWatcher(ref mut watcher) => watcher.watch(path, recursive_mode)?, + &mut Watcher::PollWatcher(ref mut watcher) => watcher.watch(path, recursive_mode), } - Ok(()) } } From da736007322ca906f6422f4caed2c357c3556f1d Mon Sep 17 00:00:00 2001 From: amribm Date: Fri, 20 Sep 2024 11:24:55 +0530 Subject: [PATCH 05/22] Add(watcher): PR changes --- src/config/watcher.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 4f19903f585ed..45ce47b8f5d30 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -45,12 +45,16 @@ impl Watcher { } fn watch(&mut self, path: &Path, recursive_mode: RecursiveMode) -> Result<(), Error> { + use notify::Watcher as NotifyWatcher; match self { - &mut Watcher::RecommendedWatcher(ref mut watcher) => { - watcher.watch(path, recursive_mode) + Watcher::RecommendedWatcher(ref mut watcher) => { + watcher.watch(path, recursive_mode)?; + } + Watcher::PollWatcher(ref mut watcher) => { + watcher.watch(path, recursive_mode)?; } - &mut Watcher::PollWatcher(ref mut watcher) => watcher.watch(path, recursive_mode), } + Ok(()) } } @@ -145,22 +149,20 @@ fn create_watcher( info!("Creating configuration file watcher."); let (sender, receiver) = channel(); - match watcher_conf { + let mut watcher = match watcher_conf { WatcherConfig::RecommendedWatcher => { let recommended_watcher = recommended_watcher(sender)?; - let mut watcher = Watcher::RecommendedWatcher(recommended_watcher); - watcher.add_paths(config_paths)?; - Ok((watcher, receiver)) + Watcher::RecommendedWatcher(recommended_watcher) } WatcherConfig::PollWatcher(interval) => { let config = notify::Config::default().with_poll_interval(Duration::from_secs(*interval)); let poll_watcher = notify::PollWatcher::new(sender, config)?; - let mut watcher = Watcher::PollWatcher(poll_watcher); - watcher.add_paths(config_paths)?; - Ok((watcher, receiver)) + Watcher::PollWatcher(poll_watcher) } - } + }; + watcher.add_paths(config_paths)?; + Ok((watcher, receiver)) } #[cfg(all(test, unix, not(target_os = "macos")))] // https://github.com/vectordotdev/vector/issues/5000 From e31bec08eb9f8344b7985adb3ea78112494b2fa8 Mon Sep 17 00:00:00 2001 From: amribm Date: Mon, 23 Sep 2024 17:08:21 +0530 Subject: [PATCH 06/22] add(watcher): PR changes --- src/config/watcher.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 45ce47b8f5d30..4f33d03da6b3b 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -47,10 +47,10 @@ impl Watcher { fn watch(&mut self, path: &Path, recursive_mode: RecursiveMode) -> Result<(), Error> { use notify::Watcher as NotifyWatcher; match self { - Watcher::RecommendedWatcher(ref mut watcher) => { + Watcher::RecommendedWatcher(watcher) => { watcher.watch(path, recursive_mode)?; } - Watcher::PollWatcher(ref mut watcher) => { + Watcher::PollWatcher(watcher) => { watcher.watch(path, recursive_mode)?; } } From d07c4b3f0816189de1029bac072e750a54a1ef34 Mon Sep 17 00:00:00 2001 From: amribm Date: Fri, 27 Sep 2024 17:51:40 +0530 Subject: [PATCH 07/22] Add: ChangeLog entry --- changelog.d/poll-watcher-support.enhancement.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/poll-watcher-support.enhancement.md diff --git a/changelog.d/poll-watcher-support.enhancement.md b/changelog.d/poll-watcher-support.enhancement.md new file mode 100644 index 0000000000000..f75ef41bf2def --- /dev/null +++ b/changelog.d/poll-watcher-support.enhancement.md @@ -0,0 +1,3 @@ +`Poll Watcher` support for watching config files. + +authors: amribm From 080563441112a9dcaf245f7feec0e2ff1c8af145 Mon Sep 17 00:00:00 2001 From: Ameer Ibrahim <99497445+amribm@users.noreply.github.com> Date: Sat, 28 Sep 2024 10:59:00 +0530 Subject: [PATCH 08/22] Update src/cli.rs Co-authored-by: Pavlos Rontidis --- src/cli.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index a239371e4c354..2fa232ff1d332 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -152,12 +152,13 @@ pub struct RootOpts { #[arg(short, long, env = "VECTOR_WATCH_CONFIG")] pub watch_config: bool, - /// Method for watching config - /// - /// By default `vector` use `inotify` which is recommended for linux based system - /// `poll` watcher can be used where inotify not work. ie, attaching config using NFS - #[arg(long, default_value = "inotify", env = "VECTOR_WATCH_METHOD")] - pub watch_config_method: WatchConfigMethod, +/// Method for configuration watching. +/// +/// By default, `vector` uses `inotify`, which is recommended for Linux-based systems. +/// The `poll` watcher can be used in cases where `inotify` doesn't work, e.g., when attaching the configuration via NFS. +#[arg(long, default_value = "inotify", env = "VECTOR_WATCH_METHOD")] +pub watch_config_method: WatchConfigMethod, + /// poll for changes in configuration file on given interval /// From 524fb0a7d8ea6f33a355520c9950f77444ad95bf Mon Sep 17 00:00:00 2001 From: Ameer Ibrahim <99497445+amribm@users.noreply.github.com> Date: Sat, 28 Sep 2024 10:59:45 +0530 Subject: [PATCH 09/22] Update src/config/watcher.rs Co-authored-by: Pavlos Rontidis --- src/config/watcher.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 4f33d03da6b3b..69668f9b9e4de 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -23,9 +23,9 @@ const CONFIG_WATCH_DELAY: std::time::Duration = std::time::Duration::from_secs(1 const RETRY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); pub enum WatcherConfig { - /// recommended watcher for os, usually inotify for linux based systems + /// Recommended watcher for the current OS, usually `inotify` for Linux-based systems. RecommendedWatcher, - /// poll based watcher. for watching files from NFS. + /// Poll-based watcher, typically used for watching files on NFS. PollWatcher(u64), } From ad936cc8d10bdd35c50c6aebb57e64e04452f934 Mon Sep 17 00:00:00 2001 From: Ameer Ibrahim <99497445+amribm@users.noreply.github.com> Date: Sat, 28 Sep 2024 11:00:00 +0530 Subject: [PATCH 10/22] Update src/cli.rs Co-authored-by: Pavlos Rontidis --- src/cli.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 2fa232ff1d332..96b8cb9f83df4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -160,9 +160,9 @@ pub struct RootOpts { pub watch_config_method: WatchConfigMethod, - /// poll for changes in configuration file on given interval - /// - /// This config is only applicable if poll is enabled in `--watch-config-method` +/// Poll for changes in the configuration file at the given interval. +/// +/// This setting is only applicable if `Poll` is set in `--watch-config-method`. #[arg( long, env = "VECTOR_WATCH_CONFIG_POLL_INTERVAL_SECONDS", From 3f1310b6ac5373473546282f6ce42f6e0ecb428b Mon Sep 17 00:00:00 2001 From: amribm Date: Tue, 1 Oct 2024 21:02:37 +0530 Subject: [PATCH 11/22] chore(watcher): rename platform specific `inotify` with `recommended` and docs update --- src/app.rs | 2 +- src/cli.rs | 28 +++++++++++++++------------- website/cue/reference/cli.cue | 16 ++++++++++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/app.rs b/src/app.rs index a546b92a9e0cc..a9be105afc3df 100644 --- a/src/app.rs +++ b/src/app.rs @@ -547,7 +547,7 @@ pub fn watcher_config( interval: NonZeroU64, ) -> config::watcher::WatcherConfig { match method { - WatchConfigMethod::Inotify => config::watcher::WatcherConfig::RecommendedWatcher, + WatchConfigMethod::Recommended => config::watcher::WatcherConfig::RecommendedWatcher, WatchConfigMethod::Poll => config::watcher::WatcherConfig::PollWatcher(interval.into()), } } diff --git a/src/cli.rs b/src/cli.rs index 96b8cb9f83df4..16ab6a6a52ad3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -152,17 +152,19 @@ pub struct RootOpts { #[arg(short, long, env = "VECTOR_WATCH_CONFIG")] pub watch_config: bool, -/// Method for configuration watching. -/// -/// By default, `vector` uses `inotify`, which is recommended for Linux-based systems. -/// The `poll` watcher can be used in cases where `inotify` doesn't work, e.g., when attaching the configuration via NFS. -#[arg(long, default_value = "inotify", env = "VECTOR_WATCH_METHOD")] -pub watch_config_method: WatchConfigMethod, - - -/// Poll for changes in the configuration file at the given interval. -/// -/// This setting is only applicable if `Poll` is set in `--watch-config-method`. + /// Method for configuration watching. + /// + /// By default, `vector` uses recommended watcher for host OS + /// - `inotify` for Linux-based systems. + /// - `kqueue` for unix/macos + /// - `ReadDirectoryChangesWatcher` for windows + /// The `poll` watcher can be used in cases where `inotify` doesn't work, e.g., when attaching the configuration via NFS. + #[arg(long, default_value = "inotify", env = "VECTOR_WATCH_METHOD")] + pub watch_config_method: WatchConfigMethod, + + /// Poll for changes in the configuration file at the given interval. + /// + /// This setting is only applicable if `Poll` is set in `--watch-config-method`. #[arg( long, env = "VECTOR_WATCH_CONFIG_POLL_INTERVAL_SECONDS", @@ -374,8 +376,8 @@ pub enum LogFormat { #[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] pub enum WatchConfigMethod { - /// recommended for linux based systems - Inotify, + ///recommended watcher for host OS + Recommended, /// works for EFS/NFS like network storage systems Poll, } diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index 0a53739aa0363..cc91f558f97cc 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -624,6 +624,22 @@ cli: { description: "Watch for changes in the configuration file and reload accordingly" type: bool: default: false } + VECTOR_WATCH_CONFIG_METHOD: { + description: """ + Method for watching config files. + + `recommend` - recommended event based watcher for host OS + `poll` - `poll` watcher can be used in cases where event based watcher doesn't work, e.g., when attaching the configuration via NFS. + """ + type: text: default: recommended + } + VECTOR_WATCH_CONFIG_POLL_INTERVAL_SECONDS: { + description: """ + Poll for config changes at given interval + only applicable if `poll` is set in `--watch-config-method` + """ + type: bool: default: false + } VECTOR_INTERNAL_LOG_RATE_LIMIT: { description: "Set the internal log rate limit. This limits Vector from emitting identical logs more than once over the given number of seconds." type: uint: { From 46c57635573fa491c4e0c3f9d968add64d542ea5 Mon Sep 17 00:00:00 2001 From: amribm Date: Tue, 1 Oct 2024 21:11:52 +0530 Subject: [PATCH 12/22] chore(watcher): remove commented code --- src/config/watcher.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 69668f9b9e4de..03a54ac37e58a 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -58,18 +58,6 @@ impl Watcher { } } -// impl From for Watcher { -// fn from(recommended_watcher: notify::RecommendedWatcher) -> Self { -// Watcher::RecommendedWatcher(recommended_watcher) -// } -// } - -// impl From for Watcher { -// fn from(poll_watcher: notify::PollWatcher) -> Self { -// Watcher::PollWatcher(poll_watcher) -// } -// } - /// Sends a ReloadFromDisk on config_path changes. /// Accumulates file changes until no change for given duration has occurred. /// Has best effort guarantee of detecting all file changes from the end of From 56fdf20716006dff5f1d150d1a428f839df68da4 Mon Sep 17 00:00:00 2001 From: Ameer Ibrahim <99497445+amribm@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:20:40 +0530 Subject: [PATCH 13/22] Update src/config/watcher.rs Co-authored-by: Pavlos Rontidis --- src/config/watcher.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 03a54ac37e58a..bc28406f4a7ed 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -22,10 +22,11 @@ const CONFIG_WATCH_DELAY: std::time::Duration = std::time::Duration::from_secs(1 const RETRY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +/// Refer to [`crate::cli::WatchConfigMethod`] for details. pub enum WatcherConfig { - /// Recommended watcher for the current OS, usually `inotify` for Linux-based systems. + /// Recommended watcher for the current OS. RecommendedWatcher, - /// Poll-based watcher, typically used for watching files on NFS. + /// A poll-based watcher that checks for file changes at regular intervals. PollWatcher(u64), } From 94420b857354f7909f6ad3d9ca61167bf7c23350 Mon Sep 17 00:00:00 2001 From: Ameer Ibrahim <99497445+amribm@users.noreply.github.com> Date: Wed, 16 Oct 2024 09:47:03 +0530 Subject: [PATCH 14/22] Update src/cli.rs Co-authored-by: Pavlos Rontidis --- src/cli.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 16ab6a6a52ad3..9634e8af64567 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -376,9 +376,10 @@ pub enum LogFormat { #[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] pub enum WatchConfigMethod { - ///recommended watcher for host OS + /// Recommended watcher for the current OS, usually `inotify` for Linux-based systems. Recommended, - /// works for EFS/NFS like network storage systems + /// Poll-based watcher, typically used for watching files on EFS/NFS-like network storage systems. + /// The interval is determined by [`RootOpts::watch_config_poll_interval_seconds`]. Poll, } From 8a38f2f6289fdbd1147c642381da8d7f59849fcd Mon Sep 17 00:00:00 2001 From: amribm Date: Wed, 16 Oct 2024 10:39:43 +0530 Subject: [PATCH 15/22] add: PR changes --- changelog.d/poll-watcher-support.enhancement.md | 2 +- src/cli.rs | 6 +++++- website/cue/reference/cli.cue | 17 ++++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/changelog.d/poll-watcher-support.enhancement.md b/changelog.d/poll-watcher-support.enhancement.md index f75ef41bf2def..c8dd9f6673bcc 100644 --- a/changelog.d/poll-watcher-support.enhancement.md +++ b/changelog.d/poll-watcher-support.enhancement.md @@ -1,3 +1,3 @@ -`Poll Watcher` support for watching config files. +Support for watching config file changes by polling at certain interval authors: amribm diff --git a/src/cli.rs b/src/cli.rs index 9634e8af64567..be2538a3c82f4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -159,7 +159,11 @@ pub struct RootOpts { /// - `kqueue` for unix/macos /// - `ReadDirectoryChangesWatcher` for windows /// The `poll` watcher can be used in cases where `inotify` doesn't work, e.g., when attaching the configuration via NFS. - #[arg(long, default_value = "inotify", env = "VECTOR_WATCH_METHOD")] + #[arg( + long, + default_value = "recommended", + env = "VECTOR_WATCH_CONFIG_METHOD" + )] pub watch_config_method: WatchConfigMethod, /// Poll for changes in the configuration file at the given interval. diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index cc91f558f97cc..974e46fa83791 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -157,6 +157,18 @@ cli: { env_var: "VECTOR_GRACEFUL_SHUTDOWN_LIMIT_SECS" type: "integer" } + "watch-config-poll-interval-seconds": { + description: env_vars.VECTOR_WATCH_CONFIG_POLL_INTERVAL_SECONDS.description + env_var: "VECTOR_WATCH_CONFIG_POLL_INTERVAL_SECONDS" + default: env_vars.VECTOR_GRACEFUL_SHUTDOWN_LIMIT_SECS.type.uint.default + type: "integer" + } + "watch-config-method": { + description: env_vars.VECTOR_WATCH_CONFIG_METHOD.description + env_var: "VECTOR_WATCH_CONFIG_METHOD" + default: env_vars.VECTOR_WATCH_CONFIG_METHOD.default + type: "string" + } } // Reusable options @@ -638,7 +650,10 @@ cli: { Poll for config changes at given interval only applicable if `poll` is set in `--watch-config-method` """ - type: bool: default: false + type: uint: { + default: 30 + unit: "seconds" + } } VECTOR_INTERNAL_LOG_RATE_LIMIT: { description: "Set the internal log rate limit. This limits Vector from emitting identical logs more than once over the given number of seconds." From 28225551d750435021945fedc7ec71f3df5b36f9 Mon Sep 17 00:00:00 2001 From: amribm Date: Wed, 16 Oct 2024 11:54:31 +0530 Subject: [PATCH 16/22] Add: documentation for config watching --- website/content/en/docs/administration/management.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/website/content/en/docs/administration/management.md b/website/content/en/docs/administration/management.md index 735dd62ec112e..99fb41a3b91ad 100644 --- a/website/content/en/docs/administration/management.md +++ b/website/content/en/docs/administration/management.md @@ -218,6 +218,13 @@ As you can see above, many administrative interfaces for Vector enable you to tr You can make Vector automatically reload itself when its [configuration file][configuration] changes by setting the `--watch-config` or `-w` [flag][watch_config] when you first start your Vector instance. +Additionally you can add method for watching config change by setting the `--watch-config-method` to `recommended` or `poll`. + +`recommended` is default and it uses file event listener for file change events. +`poll` can used where the event listener won't work. eg., attaching config files by NFS/EFS. which polls for file changes on certain interval. + +you can set the poll interval by setting flag `--watch-config-poll-interval-seconds`. which defaults to `30`. + ## How it works Running Vector instances accept the IPC [signals](#signals) and produce the [exit codes](#exit-codes) listed below. From 72a68ea8f97b99c54f9ffe9f2b5fa5321fa781f0 Mon Sep 17 00:00:00 2001 From: Ameer Ibrahim <99497445+amribm@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:32:03 +0530 Subject: [PATCH 17/22] Update website/content/en/docs/administration/management.md Co-authored-by: Jorge Hermo --- website/content/en/docs/administration/management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/en/docs/administration/management.md b/website/content/en/docs/administration/management.md index 99fb41a3b91ad..32898ad7b85e8 100644 --- a/website/content/en/docs/administration/management.md +++ b/website/content/en/docs/administration/management.md @@ -223,7 +223,7 @@ Additionally you can add method for watching config change by setting the `--wat `recommended` is default and it uses file event listener for file change events. `poll` can used where the event listener won't work. eg., attaching config files by NFS/EFS. which polls for file changes on certain interval. -you can set the poll interval by setting flag `--watch-config-poll-interval-seconds`. which defaults to `30`. +You can set the poll interval by setting flag `--watch-config-poll-interval-seconds`. which defaults to `30`. ## How it works From b7accf4e14fc472776c35cd7a2ea3b93f5fa007d Mon Sep 17 00:00:00 2001 From: Ameer Ibrahim <99497445+amribm@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:20:45 +0530 Subject: [PATCH 18/22] Update website/content/en/docs/administration/management.md Co-authored-by: Jorge Hermo --- website/content/en/docs/administration/management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/en/docs/administration/management.md b/website/content/en/docs/administration/management.md index 32898ad7b85e8..c1eb175139124 100644 --- a/website/content/en/docs/administration/management.md +++ b/website/content/en/docs/administration/management.md @@ -221,7 +221,7 @@ You can make Vector automatically reload itself when its [configuration file][co Additionally you can add method for watching config change by setting the `--watch-config-method` to `recommended` or `poll`. `recommended` is default and it uses file event listener for file change events. -`poll` can used where the event listener won't work. eg., attaching config files by NFS/EFS. which polls for file changes on certain interval. +`poll` can used where the event listener won't work, eg. attaching config files by NFS/EFS, which will poll for file changes on certain interval. You can set the poll interval by setting flag `--watch-config-poll-interval-seconds`. which defaults to `30`. From 602ff3fbe1a38cfc22a6459f8e92fd6775174f71 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 22 Oct 2024 14:22:57 -0400 Subject: [PATCH 19/22] attempt to fix failing style check --- changelog.d/poll-watcher-support.enhancement.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/poll-watcher-support.enhancement.md b/changelog.d/poll-watcher-support.enhancement.md index c8dd9f6673bcc..5f87b659c92e2 100644 --- a/changelog.d/poll-watcher-support.enhancement.md +++ b/changelog.d/poll-watcher-support.enhancement.md @@ -1,3 +1,3 @@ -Support for watching config file changes by polling at certain interval +Support for watching config file changes by polling at certain interval. authors: amribm From 6a914ca84f521741d69ec56399a8137e8244e4fd Mon Sep 17 00:00:00 2001 From: amribm Date: Wed, 23 Oct 2024 16:21:07 +0530 Subject: [PATCH 20/22] add: fixing clippy issues --- src/cli.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli.rs b/src/cli.rs index be2538a3c82f4..6ab179ddfaff0 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -158,6 +158,7 @@ pub struct RootOpts { /// - `inotify` for Linux-based systems. /// - `kqueue` for unix/macos /// - `ReadDirectoryChangesWatcher` for windows + /// /// The `poll` watcher can be used in cases where `inotify` doesn't work, e.g., when attaching the configuration via NFS. #[arg( long, From f38ad118d33bc76478540996cd5053230f758824 Mon Sep 17 00:00:00 2001 From: amribm Date: Fri, 25 Oct 2024 14:29:54 +0530 Subject: [PATCH 21/22] fix(docs): docs fixes --- website/cue/reference/cli.cue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index 974e46fa83791..776e9484819ca 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -166,7 +166,7 @@ cli: { "watch-config-method": { description: env_vars.VECTOR_WATCH_CONFIG_METHOD.description env_var: "VECTOR_WATCH_CONFIG_METHOD" - default: env_vars.VECTOR_WATCH_CONFIG_METHOD.default + default: env_vars.VECTOR_WATCH_CONFIG_METHOD.type.string.default type: "string" } } @@ -643,7 +643,7 @@ cli: { `recommend` - recommended event based watcher for host OS `poll` - `poll` watcher can be used in cases where event based watcher doesn't work, e.g., when attaching the configuration via NFS. """ - type: text: default: recommended + type: string: default: "recommended" } VECTOR_WATCH_CONFIG_POLL_INTERVAL_SECONDS: { description: """ From 14c0e728c0a9552b64b843e1c677fc6d88fd13a9 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Mon, 11 Nov 2024 08:22:21 -0500 Subject: [PATCH 22/22] cuefmt Signed-off-by: Jesse Szwedko --- website/cue/reference/cli.cue | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index 776e9484819ca..6ce8705fd6cf2 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -638,18 +638,18 @@ cli: { } VECTOR_WATCH_CONFIG_METHOD: { description: """ - Method for watching config files. + Method for watching config files. - `recommend` - recommended event based watcher for host OS - `poll` - `poll` watcher can be used in cases where event based watcher doesn't work, e.g., when attaching the configuration via NFS. - """ + `recommend` - recommended event based watcher for host OS + `poll` - `poll` watcher can be used in cases where event based watcher doesn't work, e.g., when attaching the configuration via NFS. + """ type: string: default: "recommended" } VECTOR_WATCH_CONFIG_POLL_INTERVAL_SECONDS: { description: """ - Poll for config changes at given interval - only applicable if `poll` is set in `--watch-config-method` - """ + Poll for config changes at given interval + only applicable if `poll` is set in `--watch-config-method` + """ type: uint: { default: 30 unit: "seconds"