From 01b30f49a6005b345b8748424241fcba27fabba3 Mon Sep 17 00:00:00 2001
From: Teppo Kurki <teppo.kurki@iki.fi>
Date: Tue, 17 Sep 2024 22:25:47 +0300
Subject: [PATCH] feature: add filteringrules

Add more flexible filteringRules so that you can use both
allow and ignore logic in determining what should or should not
be written to the database.

Fixes #27.
---
 src/HistoryAPI.ts  |  2 +-
 src/influx.ts      | 56 +++++++++++++++++++++++++++++++++++++++++++++-
 src/plugin.test.ts |  2 ++
 src/query.ts       |  1 +
 4 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/src/HistoryAPI.ts b/src/HistoryAPI.ts
index 63142cf..ae965cc 100644
--- a/src/HistoryAPI.ts
+++ b/src/HistoryAPI.ts
@@ -111,7 +111,7 @@ interface SimpleRequest {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type ValuesResultRow = any[]
 
-function getPositions(
+export function getPositions(
   v1Client: InfluxV1,
   context: string,
   from: ZonedDateTime,
diff --git a/src/influx.ts b/src/influx.ts
index e7e8ec4..eeabccd 100644
--- a/src/influx.ts
+++ b/src/influx.ts
@@ -25,6 +25,25 @@ import { S2 } from 's2-geometry'
 export const SELF_TAG_NAME = 'self'
 export const SELF_TAG_VALUE = 'true'
 
+interface FilteringRule {
+  /**
+   * @title Allow rule
+   * @description Check to use this rule for picking data for writing, otherwise this rule will cause data to be ignored.
+   */
+  allow: boolean
+  /**
+   * @title Path
+   * @description Literal value or JS regular expression.
+   */
+  path: string
+
+  /**
+   * @title Source
+   * @description Literal value or JS regular expression. You can copypaste values from server's Data Browser
+   */
+  source: string
+}
+
 export interface SKInfluxConfig {
   /**
    * Url of the InfluxDb 2 server
@@ -55,6 +74,13 @@ export interface SKInfluxConfig {
    */
   onlySelf: boolean
 
+  /**
+   * @title Filtering Rules
+   * @default []
+   * @description Filtering rules for allowing and/or ignoring data for writing. First matching rule decides whether to allow writing or ignore and not write the value. If there are rules but none of them match the value will be written. Adding rules disables ignoredPaths and ignoredValues. To pick some values add allow rules for them and an "ignore all" rule at the end with .* in either path or source field. Activate Debug logging in plugin configuration to get logging on server startup about ignored and allowed data.
+   */
+  filteringRules: FilteringRule[]
+
   /**
    * @title Ignored paths
    * @default []
@@ -116,6 +142,7 @@ export class SKInflux {
   private ignoreStatusForPathSources: {
     [path: string]: boolean
   } = {}
+  private filteringRules: FilteringRule[]
   private ignoredPaths: string[]
   private ignoredSources: string[]
   private useSKTimestamp: boolean
@@ -130,7 +157,8 @@ export class SKInflux {
   private resolution: number
 
   constructor(config: SKInfluxConfig, private logging: Logging, triggerStatusUpdate: () => void) {
-    const { org, bucket, url, onlySelf, ignoredPaths, ignoredSources, resolution, useSKTimestamp } = config
+    const { org, bucket, url, onlySelf, ignoredPaths, ignoredSources, resolution, useSKTimestamp, filteringRules } =
+      config
     this.influx = new InfluxDB(config)
     this.org = org
     this.bucket = bucket
@@ -138,6 +166,7 @@ export class SKInflux {
     this.onlySelf = onlySelf
     this.ignoredPaths = ignoredPaths
     this.ignoredSources = ignoredSources
+    this.filteringRules = filteringRules || []
     this.useSKTimestamp = useSKTimestamp
     this.resolution = resolution
     this.writeApi = this.influx.getWriteApi(org, bucket, 'ms', {
@@ -284,6 +313,31 @@ export class SKInflux {
   }
 
   ignoreStatusByConfig(path: string, sourceRef: string): boolean {
+    if (this.filteringRules?.length > 0) {
+      return this.ignoreStatusByRule(path, sourceRef)
+    } else {
+      return this.ignoreStatusByIgnoredPathOrSource(path, sourceRef)
+    }
+  }
+
+  ignoreStatusByRule(path: string, sourceRef: string): boolean {
+    const firstMatchingRule = this.filteringRules.find((rule) => {
+      return (
+        (rule.path === undefined || rule.path.length === 0 || new RegExp(rule.path).test(path)) &&
+        (rule.source === undefined || rule.source.length === 0 || new RegExp(rule.source).test(sourceRef))
+      )
+    })
+    //ignore if there is a matching rule AND the rule is not an allow rule
+    const isIgnored = firstMatchingRule !== undefined && !firstMatchingRule.allow
+    this.logging.debug(
+      `${path} from ${sourceRef} will be ${isIgnored ? 'ignored' : 'written'}, matching rule is ${JSON.stringify(
+        firstMatchingRule,
+      )}`,
+    )
+    return isIgnored
+  }
+
+  ignoreStatusByIgnoredPathOrSource(path: string, sourceRef: string): boolean {
     try {
       const ignoredByPath =
         this.ignoredPaths?.reduce<boolean>((acc, ignoredPathExp) => {
diff --git a/src/plugin.test.ts b/src/plugin.test.ts
index 89cecad..fd05672 100644
--- a/src/plugin.test.ts
+++ b/src/plugin.test.ts
@@ -63,6 +63,7 @@ describe('Plugin', () => {
             flushInterval: 10,
             maxRetries: 1,
           },
+          filteringRules: [],
           ignoredPaths: [],
           ignoredSources: [],
           useSKTimestamp: false,
@@ -255,6 +256,7 @@ describe('Plugin', () => {
             flushInterval: 10,
             maxRetries: 1,
           },
+          filteringRules: [],
           ignoredPaths: [],
           ignoredSources: [],
           useSKTimestamp: true, // <===============
diff --git a/src/query.ts b/src/query.ts
index a641c21..ba990ce 100644
--- a/src/query.ts
+++ b/src/query.ts
@@ -20,6 +20,7 @@ const skinflux = new SKInflux(
     bucket: process.env.BUCKET || '!!!',
     org: process.env.ORG || '!!!',
     writeOptions: {},
+    filteringRules: [],
     ignoredPaths: [],
     ignoredSources: [],
     useSKTimestamp: false,