Merge pull request #1567 from pi-hole/development

Pi-hole FTL v5.23

.github/workflows/build.yml

@@ -24,7 +24,7 @@ jobs:
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.4.0
+        uses: actions/checkout@v3.5.2
       -
         name: "Calculate required variables"
         id: variables
@@ -90,7 +90,7 @@ jobs:
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.4.0
+        uses: actions/checkout@v3.5.2
       -
         name: "Fix ownership of repository"
         run: chown -R root .
@@ -133,7 +133,7 @@ jobs:
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.4.0
+        uses: actions/checkout@v3.5.2
       -
         name: Get Binaries built in previous jobs
         uses: actions/[email protected]

.github/workflows/codespell.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
     -
       name: Checkout repository
-      uses: actions/checkout@v3.4.0
+      uses: actions/checkout@v3.5.2
     -
       name: Spell-Checking
       uses: codespell-project/actions-codespell@master

.github/workflows/stale.yml

@@ -2,25 +2,44 @@ name: Mark stale issues
 
 on:
   schedule:
-  - cron: '0 8 * * *'
+    - cron: '0 8 * * *'
   workflow_dispatch:
+  issue_comment:
 
-jobs:
-  stale:
+env:
+  stale_label: stale
 
+jobs:
+  stale_action:
+    if: github.event_name != 'issue_comment'
     runs-on: ubuntu-latest
     permissions:
       issues: write
 
     steps:
-    - uses: actions/stale@v7.0.0
+    - uses: actions/stale@v8.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 30
         days-before-close: 5
         stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.'
-        stale-issue-label: 'stale'
+        stale-issue-label: $stale_label
         exempt-issue-labels: 'Fixed in next release, Bug, Bug:Confirmed, Bugfix in progress, documentation needed, internal'
         exempt-all-issue-assignees: true
         operations-per-run: 300
         close-issue-reason: 'not_planned'
+
+  remove_stale: # trigger "stale" removal immediately when stale issues are commented on
+    if: github.event_name == 'issue_comment'
+    permissions:
+      contents: read #  for actions/checkout
+      issues: write #  to edit issues label
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      - name: Remove 'stale' label
+        run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

.github/workflows/stale_pr.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v7.0.0
+      - uses: actions/stale@v8.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # Do not automatically mark PR/issue as stale

.github/workflows/sync-back-to-dev.yml

@@ -11,7 +11,7 @@ jobs:
     name: Syncing branches
     steps:
       - name: Checkout
-        uses: actions/checkout@v3.4.0
+        uses: actions/checkout@v3.5.2
       - name: Opening pull request
         run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
         env:

deploy.sh

@@ -42,11 +42,18 @@ for dir in "${path[@]}"; do
         ls -1"
     )"
 
-    # Only try to create the subdir if does not already exist
-    if [[ "${dir_content[*]}" =~ "${dir}" ]]; then
-        echo "Dir: ${old_path}/${dir} already exists"
-    else
-        echo "Creating dir: ${old_path}/${dir}"
+    # Loop over the dir content and check if this exact dir already exists
+    path_exists=0
+    for content in "${dir_content[@]}"; do
+        if [[ "${content}" == "${dir}" ]]; then
+            echo "Dir: ${old_path}/${dir} already exists"
+            path_exists=1
+        fi
+    done
+
+    # If the dir does not exist, create it
+    if [[ "${path_exists}" -eq 0 ]]; then
+        echo "Dir: ${old_path}/${dir} does not exist. Creating it."
         sftp -b - "${USER}"@"${HOST}" <<< "cd ${old_path}
         -mkdir ${dir}"
     fi

patch/lua/0001-add-pihole-library.patch

@@ -48,9 +48,9 @@ index 454ce12f..a363925c 100644
 +  ftl_lua_init(L);
 +  /***************************************************/
 +
-   if (!runargs(L, argv, script))  /* execute arguments -e and -l */
+   if (!runargs(L, argv, optlim))  /* execute arguments -e and -l */
      return 0;  /* something failed */
-   if (script < argc &&  /* execute main script (if there is one) */
+   if (script > 0) {  /* execute main script (if there is one) */
 @@ -616,7 +622,9 @@ static int pmain (lua_State *L) {
  }
 

src/CMakeLists.txt

@@ -116,8 +116,6 @@ set(sources
         daemon.h
         datastructure.c
         datastructure.h
-        dhcp-discover.c
-        dhcp-discover.h
         dnsmasq_interface.c
         dnsmasq_interface.h
         edns0.c
@@ -180,6 +178,7 @@ add_executable(pihole-FTL
         $<TARGET_OBJECTS:lua>
         $<TARGET_OBJECTS:tre-regex>
         $<TARGET_OBJECTS:syscalls>
+        $<TARGET_OBJECTS:tools>
         )
 if(STATIC STREQUAL "true")
     set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON)
@@ -253,3 +252,4 @@ add_subdirectory(lua)
 add_subdirectory(lua/scripts)
 add_subdirectory(tre-regex)
 add_subdirectory(syscalls)
+add_subdirectory(tools)

src/args.c

@@ -32,8 +32,12 @@
 #include "shmem.h"
 // LUA dependencies
 #include "lua/ftl_lua.h"
+// gravity_parseList()
+#include "tools/gravity-parseList.h"
 // run_dhcp_discover()
-#include "dhcp-discover.h"
+#include "tools/dhcp-discover.h"
+// run_arp_scan()
+#include "tools/arp-scan.h"
 // defined in dnsmasq.c
 extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal);
 
@@ -124,7 +128,15 @@ static const char __attribute__ ((pure)) *cli_color(const char *color)
 	return is_term() ? color : "";
 }
 
-static inline bool strEndsWith(const char *input, const char *end){
+// Go back to beginning of line and erase to end of line if STDOUT is a terminal
+const char __attribute__ ((pure)) *cli_over(void)
+{
+	// \x1b[K is the ANSI escape sequence for "erase to end of line"
+	return is_term() ? "\r\x1b[K" : "\r";
+}
+
+static inline bool strEndsWith(const char *input, const char *end)
+{
 	return strcmp(input + strlen(input) - strlen(end), end) == 0;
 }
 
@@ -163,6 +175,39 @@ void parse_args(int argc, char* argv[])
 	   (argc > 1 && strEndsWith(argv[1], ".db")))
 			exit(sqlite3_shell_main(argc, argv));
 
+	// If the first argument is "gravity" (e.g., /usr/bin/pihole-FTL gravity),
+	// we offer some specialized gravity tools
+	if(argc > 1 && strcmp(argv[1], "gravity") == 0)
+	{
+		// pihole-FTL gravity parseList <infile> <outfile> <adlistID>
+		if(argc == 6 && strcmp(argv[2], "parseList") == 0)
+		{
+			// Parse the given list and write the result to the given file
+			exit(gravity_parseList(argv[3], argv[4], argv[5]));
+		}
+
+		printf("Incorrect usage of pihole-FTL gravity subcommand\n");
+		exit(EXIT_FAILURE);
+	}
+
+	// DHCP discovery mode
+	if(argc > 1 && strcmp(argv[1], "dhcp-discover") == 0)
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		exit(run_dhcp_discover());
+	}
+
+	// ARP scanning mode
+	if(argc > 1 && strcmp(argv[1], "arp-scan") == 0)
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		const bool scan_all = argc > 2 && strcmp(argv[2], "-a") == 0;
+		const bool extreme_mode = argc > 2 && strcmp(argv[2], "-x") == 0;
+		exit(run_arp_scan(scan_all, extreme_mode));
+	}
+
 	// start from 1, as argv[0] is the executable name
 	for(int i = 1; i < argc; i++)
 	{
@@ -415,14 +460,6 @@ void parse_args(int argc, char* argv[])
 			}
 		}
 
-		// Regex test mode
-		if(strcmp(argv[i], "dhcp-discover") == 0)
-		{
-			// Enable stdout printing
-			cli_mode = true;
-			exit(run_dhcp_discover());
-		}
-
 		// List of implemented arguments
 		if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "help") == 0 || strcmp(argv[i], "--help") == 0)
 		{
@@ -495,13 +532,19 @@ void parse_args(int argc, char* argv[])
 
 			printf("%sDebugging and special use:%s\n", yellow, normal);
 			printf("\t%sd%s, %sdebug%s            Enter debugging mode\n", green, normal, green, normal);
-			printf("\t%stest%s                Don't start pihole-FTL but\n", green, normal);
-			printf("\t                    instead quit immediately\n");
+			printf("\t%stest%s                Don't start pihole-FTL but instead\n", green, normal);
+			printf("\t                    quit immediately\n");
 			printf("\t%s-f%s, %sno-daemon%s       Don't go into daemon mode\n\n", green, normal, green, normal);
 
 			printf("%sOther:%s\n", yellow, normal);
 			printf("\t%sdhcp-discover%s       Discover DHCP servers in the local\n", green, normal);
 			printf("\t                    network\n");
+			printf("\t%sarp-scan %s[-a/-x]%s    Use ARP to scan local network for\n", green, cyan, normal);
+			printf("\t                    possible IP conflicts\n");
+			printf("\t                    Append %s-a%s to force scan on all\n", cyan, normal);
+			printf("\t                    interfaces\n");
+			printf("\t                    Append %s-x%s to force scan on all\n", cyan, normal);
+			printf("\t                    interfaces and scan 10x more often\n");
 			printf("\t%s-h%s, %shelp%s            Display this help and exit\n\n", green, normal, green, normal);
 			exit(EXIT_SUCCESS);
 		}

src/args.h

@@ -23,6 +23,7 @@ const char *cli_qst(void) __attribute__ ((const));
 const char *cli_done(void) __attribute__ ((pure));
 const char *cli_bold(void) __attribute__ ((pure));
 const char *cli_normal(void) __attribute__ ((pure));
+const char *cli_over(void) __attribute__ ((pure));
 
 // defined in dnsmasq_interface.c
 int check_struct_sizes(void);

src/capabilities.c

@@ -21,6 +21,47 @@ static const unsigned int capabilityIDs[]   = { CAP_CHOWN ,  CAP_DAC_OVERRIDE ,
 static const char*        capabilityNames[] = {"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID", "CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE", "CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW", "CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT", "CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE", "CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE", "CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP"};
 static const unsigned int numCaps = sizeof(capabilityIDs) / sizeof(*capabilityIDs);
 
+bool check_capability(const unsigned int cap)
+{
+	// First assume header version 1
+	int capsize = 1; // VFS_CAP_U32_1
+	cap_user_data_t data = NULL;
+	cap_user_header_t hdr = calloc(sizeof(*hdr), capsize);
+
+	// Determine capabilities version used by the current kernel
+	capget(hdr, NULL);
+
+	// Check version
+	if (hdr->version != LINUX_CAPABILITY_VERSION_1)
+	{
+		// If unknown version, use largest supported version (3)
+		// Version 2 is deprecated according to linux/capability.h
+		if (hdr->version != LINUX_CAPABILITY_VERSION_2)
+		{
+			hdr->version = LINUX_CAPABILITY_VERSION_3;
+			capsize = 2; // VFS_CAP_U32_3
+		}
+		else
+		{
+			// Use version 2
+			capsize = 2; // VFS_CAP_U32_2
+		}
+	}
+
+	// Get current capabilities
+	data = calloc(sizeof(*data), capsize);
+	capget(hdr, data);
+
+	// Check if the capability is available
+	const bool available = ((data->permitted & (1 << cap)) && (data->effective & (1 << cap)));
+
+	// Free memory
+	free(hdr);
+	free(data);
+
+	return available;
+}
+
 bool check_capabilities(void)
 {
 	// First assume header version 1

src/capabilities.h

@@ -10,6 +10,7 @@
 #ifndef CAPABILITIES_H
 #define CAPABILITIES_H
 
+bool check_capability(const unsigned int cap);
 bool check_capabilities(void);
 
 #endif //CAPABILITIES_H

src/config.c

@@ -837,9 +837,6 @@ static char *parse_FTLconf(FILE *fp, const char *key)
 	// Go to beginning of file
 	fseek(fp, 0L, SEEK_SET);
 
-	if(config.debug & DEBUG_EXTRA)
-		logg("initial: conflinebuffer = %p, keystr = %p, size = %zu", conflinebuffer, keystr, size);
-
 	// Set size to zero if conflinebuffer is not available here
 	// This causes getline() to allocate memory for the buffer itself
 	if(conflinebuffer == NULL && size != 0)
@@ -848,11 +845,6 @@ static char *parse_FTLconf(FILE *fp, const char *key)
 	errno = 0;
 	while(getline(&conflinebuffer, &size, fp) != -1)
 	{
-		if(config.debug & DEBUG_EXTRA)
-		{
-			logg("conflinebuffer = %p, keystr = %p, size = %zu", conflinebuffer, keystr, size);
-			logg("  while reading line \"%s\" looking for \"%s\"", conflinebuffer, keystr);
-		}
 		// Check if memory allocation failed
 		if(conflinebuffer == NULL)
 			break;