Add regex filtering support for domains on the Query Log (new config …

…option webserver.api.excludeRegex) Signed-off-by: DL6ER <[email protected]>
pi-hole · Aug 1, 2023 · 8686c4c · 8686c4c
1 parent 4ac9f12
commit 8686c4c
Show file tree

Hide file tree

Showing 5 changed files with 96 additions and 3 deletions.
diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml
@@ -397,6 +397,10 @@ components:
                       type: array
                       items:
                         type: string
+                    excludeRegex:
+                      type: array
+                      items:
+                        type: string
                     maxHistory:
                       type: integer
                     allow_destructive:
@@ -656,6 +660,7 @@ components:
               totp_secret: ''
               excludeClients: [ '1.2.3.4', 'localhost', 'fe80::345' ]
               excludeDomains: [ 'google.de', 'pi-hole.net' ]
+              excludeRegex: [ '\.fishy-domain\.com$' ]
               maxHistory: 86400
               allow_destructive: true
               temp:

diff --git a/src/api/queries.c b/src/api/queries.c
@@ -19,7 +19,7 @@
 #include "database/aliasclients.h"
 // get_memdb()
 #include "database/query-table.h"
-
+#include "regex.h"
 // dbopen(false, ), dbclose()
 #include "database/common.h"
 
@@ -340,6 +340,48 @@ int api_queries(struct ftl_conn *api)
 			add_querystr_string(api, querystr, "q.dnssec=", ":dnssec", &where);
 	}
 
+	// Regex filtering?
+	const size_t regex_filters = cJSON_GetArraySize(config.webserver.api.excludeRegex.v.json);
+	regex_t *regex = NULL;
+	if(regex_filters > 0)
+	{
+		// Allocate memory for regex array
+		regex = calloc(regex_filters, sizeof(regex_t));
+		if(regex == NULL)
+		{
+			return send_json_error(api, 500,
+			                       "internal_error",
+			                       "Internal server error, failed to allocate memory",
+			                       NULL);
+		}
+
+		// Compile regexes
+		for(size_t i = 0; i < regex_filters; i++)
+		{
+			// Iterate over regexes
+			cJSON *filter = NULL;
+			cJSON_ArrayForEach(filter, config.webserver.api.excludeRegex.v.json)
+			{
+				// Skip non-string, invalid and empty values
+				if(!cJSON_IsString(filter) || filter->valuestring == NULL || strlen(filter->valuestring) == 0)
+					continue;
+
+				// Compile regex
+				int rc = regcomp(&regex[i], filter->valuestring, REG_EXTENDED);
+				if(rc != 0)
+				{
+					// Failed to compile regex
+					char errbuf[1024];
+					regerror(rc, &regex[i], errbuf, sizeof(errbuf));
+					return send_json_error(api, 400,
+							"bad_request",
+							"Failed to compile regex",
+							errbuf);
+				}
+			}
+		}
+	}
+
 	// Get connection to in-memory database
 	sqlite3 *db = get_memdb();
 
@@ -629,6 +671,31 @@ int api_queries(struct ftl_conn *api)
 			break;
 		}
 
+		// Apply possible regex filters to Query Log domains
+		const char *domain = (const char*)sqlite3_column_text(read_stmt, 4); // d.domain
+		if(regex_filters > 0)
+		{
+			bool match = false;
+			// Iterate over all regex filters
+			for(size_t i = 0; i < regex_filters; i++)
+			{
+				// Check if the domain matches the regex
+				if(regexec(&regex[i], domain, 0, NULL, 0) == 0)
+				{
+					// Domain matches, so we can stop here
+					match = true;
+					break;
+				}
+			}
+			if(match)
+			{
+				// Domain matches, so we can skip it and adjust
+				// the counter
+				recordsCounted--;
+				continue;
+			}
+		}
+
 		// Build item object
 		cJSON *item = JSON_NEW_OBJECT();
 		queriesData query = { 0 };
@@ -643,7 +710,7 @@ int api_queries(struct ftl_conn *api)
 		JSON_COPY_STR_TO_OBJECT(item, "type", get_query_type_str(query.type, &query, buffer));
 		JSON_REF_STR_IN_OBJECT(item, "status", get_query_status_str(query.status));
 		JSON_REF_STR_IN_OBJECT(item, "dnssec", get_query_dnssec_str(query.dnssec));
-		JSON_COPY_STR_TO_OBJECT(item, "domain", sqlite3_column_text(read_stmt, 4)); // d.domain
+		JSON_COPY_STR_TO_OBJECT(item, "domain", domain);
 
 		if(sqlite3_column_type(read_stmt, 5) == SQLITE_TEXT &&
 		   sqlite3_column_bytes(read_stmt, 5) > 0)
@@ -729,7 +796,7 @@ int api_queries(struct ftl_conn *api)
 	// DataTables specific properties
 	const unsigned long recordsTotal = disk ? disk_dbnum : mem_dbnum;
 	JSON_ADD_NUMBER_TO_OBJECT(json, "recordsTotal", recordsTotal);
-	JSON_ADD_NUMBER_TO_OBJECT(json, "recordsFiltered", filtering ? recordsCounted : recordsTotal);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "recordsFiltered", filtering || regex_filters > 0 ? recordsCounted : recordsTotal);
 	JSON_ADD_NUMBER_TO_OBJECT(json, "draw", draw);
 
 	// Finalize statements
@@ -743,5 +810,11 @@ int api_queries(struct ftl_conn *api)
 		                       message);
 	}
 
+	// Free regex memory if allocated
+	if(regex_filters > 0)
+	{
+		free(regex);
+	}
+
 	JSON_SEND_OBJECT(json);
 }
diff --git a/src/config/config.c b/src/config/config.c
@@ -929,6 +929,13 @@ void initConfig(struct config *conf)
 	conf->webserver.api.excludeDomains.t = CONF_JSON_STRING_ARRAY;
 	conf->webserver.api.excludeDomains.d.json = cJSON_CreateArray();
 
+	conf->webserver.api.excludeRegex.k = "webserver.api.excludeRegex";
+	conf->webserver.api.excludeRegex.h = "Array of regular expressions to be excluded from certain API responses\n Example: [ \"(^|\\.)\\.google\\.de$\", \"\\.pi-hole\\.net$\" ]";
+	conf->webserver.api.excludeRegex.a = cJSON_CreateStringReference("array of regular expressions");
+	conf->webserver.api.excludeRegex.t = CONF_JSON_STRING_ARRAY;
+	conf->webserver.api.excludeRegex.f = FLAG_RESTART_DNSMASQ | FLAG_ADVANCED_SETTING;
+	conf->webserver.api.excludeRegex.d.json = cJSON_CreateArray();
+
 	conf->webserver.api.maxHistory.k = "webserver.api.maxHistory";
 	conf->webserver.api.maxHistory.h = "How much history should be imported from the database and returned by the API [seconds]? (max 24*60*60 = 86400)";
 	conf->webserver.api.maxHistory.t = CONF_UINT;

diff --git a/src/config/config.h b/src/config/config.h
@@ -224,6 +224,7 @@ struct config {
 			struct conf_item totp_secret; // This is a write-only item
 			struct conf_item excludeClients;
 			struct conf_item excludeDomains;
+			struct conf_item excludeRegex;
 			struct conf_item maxHistory;
 			struct conf_item allow_destructive;
 			struct {

diff --git a/test/pihole.toml b/test/pihole.toml
@@ -622,6 +622,13 @@
     #     array of IP addresses and/or hostnames
     excludeDomains = []
 
+    # Array of regular expressions to be excluded from certain API responses
+    # Example: [ "(^|\.)\.google\.de$", "\.pi-hole\.net$" ]
+    #
+    # Possible values are:
+    #     array of regular expressions
+    excludeRegex = []
+
     # How much history should be imported from the database [seconds]? (max 24*60*60 =
     # 86400)
     maxHistory = 86400