feat: add more variables for alerts (openobserve#1973)

Cargo.toml

@@ -17,7 +17,7 @@ keywords = [
 license = "AGPL-3.0"
 name = "openobserve"
 repository = "https://github.com/openobserve/openobserve/"
-version = "0.7.1"
+version = "0.8.0"
 publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

src/common/infra/config.rs

@@ -29,7 +29,7 @@ use vector_enrichment::TableRegistry;
 
 use crate::common::{
     meta::{
-        alert::{AlertDestination, AlertList, DestinationTemplate, Trigger, TriggerTimer},
+        alerts,
         functions::{StreamFunctionsList, Transform},
         maxmind::MaxmindClient,
         organization::OrganizationSetting,
@@ -158,12 +158,13 @@ pub static METRIC_CLUSTER_MAP: Lazy<Arc<RwAHashMap<String, Vec<String>>>> =
     Lazy::new(|| Arc::new(tokio::sync::RwLock::new(AHashMap::new())));
 pub static METRIC_CLUSTER_LEADER: Lazy<Arc<RwAHashMap<String, ClusterLeader>>> =
     Lazy::new(|| Arc::new(tokio::sync::RwLock::new(AHashMap::new())));
-pub static STREAM_ALERTS: Lazy<RwHashMap<String, AlertList>> = Lazy::new(DashMap::default);
-pub static TRIGGERS: Lazy<RwHashMap<String, Trigger>> = Lazy::new(DashMap::default);
-pub static TRIGGERS_IN_PROCESS: Lazy<RwHashMap<String, TriggerTimer>> = Lazy::new(DashMap::default);
-pub static ALERTS_TEMPLATES: Lazy<RwHashMap<String, DestinationTemplate>> =
+pub static STREAM_ALERTS: Lazy<RwAHashMap<String, Vec<alerts::Alert>>> =
     Lazy::new(Default::default);
-pub static ALERTS_DESTINATIONS: Lazy<RwHashMap<String, AlertDestination>> =
+pub static TRIGGERS: Lazy<RwAHashMap<String, alerts::triggers::Trigger>> =
+    Lazy::new(Default::default);
+pub static ALERTS_TEMPLATES: Lazy<RwHashMap<String, alerts::templates::Template>> =
+    Lazy::new(Default::default);
+pub static ALERTS_DESTINATIONS: Lazy<RwHashMap<String, alerts::destinations::Destination>> =
     Lazy::new(Default::default);
 pub static SYSLOG_ROUTES: Lazy<RwHashMap<String, SyslogRoute>> = Lazy::new(Default::default);
 pub static SYSLOG_ENABLED: Lazy<Arc<RwLock<bool>>> = Lazy::new(|| Arc::new(RwLock::new(false)));
@@ -382,7 +383,7 @@ pub struct Common {
     #[env_config(name = "ZO_DEFAULT_SCRAPE_INTERVAL", default = 15)]
     // Default scrape_interval value 15s
     pub default_scrape_interval: u32,
-    // logger timestamp local setup
+    // logger timestamp local setup, eg: %Y-%m-%dT%H:%M:%S
     #[env_config(name = "ZO_LOG_LOCAL_TIME_FORMAT", default = "")]
     pub log_local_time_format: String,
     #[env_config(name = "ZO_CIRCUIT_BREAKER_ENABLE", default = false)]
@@ -416,6 +417,8 @@ pub struct Limit {
     pub query_timeout: u64,
     #[env_config(name = "ZO_INGEST_ALLOWED_UPTO", default = 5)] // in hours - in past
     pub ingest_allowed_upto: i64,
+    #[env_config(name = "ZO_IGNORE_FILE_RETENTION_BY_STREAM", default = false)]
+    pub ignore_file_retention_by_stream: bool,
     #[env_config(name = "ZO_LOGS_FILE_RETENTION", default = "hourly")]
     pub logs_file_retention: String,
     #[env_config(name = "ZO_TRACES_FILE_RETENTION", default = "hourly")]

src/common/infra/db/dynamo.rs

@@ -28,9 +28,11 @@ use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use tokio::sync::{mpsc, OnceCell};
 
-use super::Event;
-use super::Stats;
-use crate::common::infra::{config::CONFIG, errors::*};
+use crate::common::infra::{
+    config::CONFIG,
+    db::{Event, Stats},
+    errors::*,
+};
 
 static DB: OnceCell<DynamoDb> = OnceCell::const_new();
 static DB_CLIENT: OnceCell<Client> = OnceCell::const_new();

src/common/infra/db/etcd.rs

@@ -32,8 +32,12 @@ use tokio::{
     time,
 };
 
-use super::{Event, EventData};
-use crate::common::infra::{cluster, config::CONFIG, errors::*};
+use crate::common::infra::{
+    cluster,
+    config::CONFIG,
+    db::{Event, EventData},
+    errors::*,
+};
 
 static ETCD_CLIENT: OnceCell<etcd_client::Client> = OnceCell::const_new();
 

src/common/infra/db/sled.rs

@@ -20,8 +20,12 @@ use once_cell::sync::Lazy;
 use std::sync::Arc;
 use tokio::{sync::mpsc, task::JoinHandle};
 
-use super::{Event, EventData};
-use crate::common::infra::{cluster, config::CONFIG, errors::*};
+use crate::common::infra::{
+    cluster,
+    config::CONFIG,
+    db::{Event, EventData},
+    errors::*,
+};
 
 pub static SLED_CLIENT: Lazy<Option<::sled::Db>> = Lazy::new(connect_sled);
 

src/common/infra/metrics.rs

@@ -493,12 +493,12 @@ pub static META_NUM_DASHBOARDS: Lazy<IntGaugeVec> = Lazy::new(|| {
     .expect("Metric created")
 });
 
-pub static OO_MEM_USAGE: Lazy<IntGaugeVec> = Lazy::new(|| {
+pub static MEMORY_USAGE: Lazy<IntGaugeVec> = Lazy::new(|| {
     IntGaugeVec::new(
-        Opts::new("mem_usage", "Openobserve memory usage")
+        Opts::new("memory_usage", "Process memory usage")
             .namespace(NAMESPACE)
             .const_labels(create_const_labels()),
-        &["organization"],
+        &[],
     )
     .expect("Metric created")
 });
@@ -631,7 +631,7 @@ fn register_metrics(registry: &Registry) {
         .register(Box::new(META_NUM_DASHBOARDS.clone()))
         .expect("Metric registered");
     registry
-        .register(Box::new(OO_MEM_USAGE.clone()))
+        .register(Box::new(MEMORY_USAGE.clone()))
         .expect("Metric registered");
 }
 

src/common/infra/storage/local.rs

@@ -23,8 +23,11 @@ use object_store::{
 use std::ops::Range;
 use tokio::io::AsyncWrite;
 
-use super::{format_key, CONCURRENT_REQUESTS};
-use crate::common::infra::{config::CONFIG, metrics};
+use crate::common::infra::{
+    config::CONFIG,
+    metrics,
+    storage::{format_key, CONCURRENT_REQUESTS},
+};
 
 pub struct Local {
     client: LimitStore<Box<dyn object_store::ObjectStore>>,

src/common/infra/storage/remote.rs

@@ -23,8 +23,11 @@ use object_store::{
 use std::ops::Range;
 use tokio::io::AsyncWrite;
 
-use super::{format_key, CONCURRENT_REQUESTS};
-use crate::common::infra::{config::CONFIG, metrics};
+use crate::common::infra::{
+    config::CONFIG,
+    metrics,
+    storage::{format_key, CONCURRENT_REQUESTS},
+};
 
 pub struct Remote {
     client: LimitStore<Box<dyn object_store::ObjectStore>>,

src/common/infra/wal.rs

@@ -333,7 +333,7 @@ impl RwFile {
 
         let time_now: DateTime<Utc> = Utc::now();
         let level_duration = partition_time_level.unwrap_or_default().duration();
-        let ttl = if level_duration > 0 {
+        let ttl = if !CONFIG.limit.ignore_file_retention_by_stream && level_duration > 0 {
             let time_end_day = Utc
                 .with_ymd_and_hms(
                     time_now.year(),