Add ENV variable for ConfigMap for adlists

README.md

@@ -148,6 +148,7 @@ There are other environment variables if you want to customize various things in
 | `WEB_UID` | `33` | Number | Overrides image's default www-data user id to match a host user id<br/>**IMPORTANT**: id must not already be in use inside the container! (Make sure it is different to `PIHOLE_UID` if you are using that, also)|
 | `WEB_GID` | `33` | Number | Overrides image's default www-data group id to match a host group id<br/>**IMPORTANT**: id must not already be in use inside the container! (Make sure it is different to `PIHOLE_GID` if you are using that, also)|
 | `WEBLOGS_STDOUT` | 0 | 0&vert;1 | 0 logs to defined files, 1 redirect access and error logs to stdout |
+| `CONFIGMAP_ADLISTS` | unset | `<"true"\|"false">` | Environmental variable to indicate that a ConfigMap was used to supply adlists. |
 
 ## Deprecated environment variables:
 While these may still work, they are likely to be removed in a future version. Where applicable, alternative variable names are indicated. Please review the table above for usage of the alternative variables

src/s6/debian-root/usr/local/bin/_startup.sh

@@ -8,6 +8,9 @@ fi
 # shellcheck source=/dev/null
 . /usr/local/bin/bash_functions.sh
 
+# Experimental feature to allow for declarative adlists in kubernetes
+. /usr/local/bin/configmap_adlists.sh
+
 # shellcheck source=/dev/null
 SKIP_INSTALL=true . /etc/.pihole/automated\ install/basic-install.sh
 
@@ -44,6 +47,7 @@ setup_lighttpd_bind
 # Misc Setup
 # ===========================
 installCron
+[[ -n "${CONFIGMAP_ADLISTS}" && "${CONFIGMAP_ADLISTS}" == "true" ]] && echo "  [i] Using configMap for adlists" && configMap_adlists
 setup_blocklists
 
 # FTL setup

src/s6/debian-root/usr/local/bin/configmap_adlists.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+export LC_ALL=C
+
+basename="pihole"
+PIHOLE_COMMAND="/usr/local/bin/${basename}"
+
+piholeDir="/etc/${basename}"
+
+adListFile="${piholeDir}/adlists.list"
+
+domainsExtension="domains"
+
+# Set up tmp dir variable in case it's not configured
+: "${GRAVITY_TMPDIR:=/tmp}"
+
+if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then
+  echo -e "  ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}"
+  GRAVITY_TMPDIR="/tmp"
+fi
+
+gravityDBfile_default="${piholeDir}/gravity.db"
+GRAVITYDB="${gravityDBfile_default}"
+
+# Set this only after sourcing pihole-FTL.conf as the gravity database path may
+# have changed
+gravityDBfile="${GRAVITYDB}"
+gravityTEMPfile="${GRAVITYDB}_temp"
+gravityDIR="$(dirname -- "${gravityDBfile}")"
+gravityOLDfile="${gravityDIR}/gravity_old.db"
+
+
+configMap_adlists() {
+    echo "  [i] Deleting existing adlists from gravity"
+
+    # Experimental feature to clean out domains from gravity to allow a kubernetes ConfigMap to manage them
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM gravity;"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist;"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist_by_group;"
+
+    echo "  [i] Finished clearing out adlists from gravity"
+
+    # Migrate list files to new database
+    if [ -e "${adListFile}" ]; then
+        # Store adlist domains in database
+        echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
+        database_table_from_file "adlist" "${adListFile}"
+    fi
+
+}
+
+# Import domains from file and store them in the specified database table
+database_table_from_file() {
+  # Define locals
+  local table src backup_path backup_file tmpFile list_type
+  table="${1}"
+  src="${2}"
+  backup_path="${piholeDir}/migration_backup"
+  backup_file="${backup_path}/$(basename "${2}")"
+  # Create a temporary file. We don't use '--suffix' here because not all
+  # implementations of mktemp support it, e.g. on Alpine
+  tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")"
+  mv "${tmpFile}" "${tmpFile%.*}.gravity"
+
+  local timestamp
+  timestamp="$(date --utc +'%s')"
+
+  local rowid
+  declare -i rowid
+  rowid=1
+
+  # Special handling for domains to be imported into the common domainlist table
+  if [[ "${table}" == "whitelist" ]]; then
+    list_type="0"
+    table="domainlist"
+  elif [[ "${table}" == "blacklist" ]]; then
+    list_type="1"
+    table="domainlist"
+  elif [[ "${table}" == "regex" ]]; then
+    list_type="3"
+    table="domainlist"
+  fi
+
+  # Get MAX(id) from domainlist when INSERTing into this table
+  if [[ "${table}" == "domainlist" ]]; then
+    rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")"
+    if [[ -z "$rowid" ]]; then
+      rowid=0
+    fi
+    rowid+=1
+  fi
+
+  # Loop over all domains in ${src} file
+  # Read file line by line
+  grep -v '^ *#' < "${src}" | while IFS= read -r domain
+  do
+    # Only add non-empty lines
+    if [[ -n "${domain}" ]]; then
+      if [[ "${table}" == "domain_audit" ]]; then
+        # domain_audit table format (no enable or modified fields)
+        echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
+      elif [[ "${table}" == "adlist" ]]; then
+        # Adlist table format
+        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0" >> "${tmpFile}"
+      else
+        # White-, black-, and regexlist table format
+        echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}"
+      fi
+      rowid+=1
+    fi
+  done
+
+  # Store domains in database table specified by ${table}
+  # Use printf as .mode and .import need to be on separate lines
+  # see https://unix.stackexchange.com/a/445615/83260
+  output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to fill table ${table}${list_type} in database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+
+  # Move source file to backup directory, create directory if not existing
+  mkdir -p "${backup_path}"
+  mv "${src}" "${backup_file}" 2> /dev/null || \
+    echo -e "  ${CROSS} Unable to backup ${src} to ${backup_path}"
+
+  # Delete tmpFile
+  rm "${tmpFile}" > /dev/null 2>&1 || \
+    echo -e "  ${CROSS} Unable to remove ${tmpFile}"
+}
+
+# Clean up after Gravity upon exit or cancellation
+gravity_Cleanup() {
+  local error="${1:-}"
+
+  str="Cleaning up stray matter"
+  echo -ne "  ${INFO} ${str}..."
+
+  # Delete tmp content generated by Gravity
+  rm ${piholeDir}/pihole.*.txt 2> /dev/null
+  rm ${piholeDir}/*.tmp 2> /dev/null
+  # listCurlBuffer location
+  rm "${GRAVITY_TMPDIR}"/*.phgpb 2> /dev/null
+  # invalid_domains location
+  rm "${GRAVITY_TMPDIR}"/*.ph-non-domains 2> /dev/null
+
+  # Ensure this function only runs when gravity_SetDownloadOptions() has completed
+  if [[ "${gravity_Blackbody:-}" == true ]]; then
+    # Remove any unused .domains files
+    for file in "${piholeDir}"/*."${domainsExtension}"; do
+      # If list is not in active array, then remove it
+      if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then
+        rm -f "${file}" 2> /dev/null || \
+          echo -e "  ${CROSS} Failed to remove ${file##*/}"
+      fi
+    done
+  fi
+
+  echo -e "${OVER}  ${TICK} ${str}"
+
+  # Only restart DNS service if offline
+  if ! pgrep pihole-FTL &> /dev/null; then
+    "${PIHOLE_COMMAND}" restartdns
+    dnsWasOffline=true
+  fi
+
+  # Print Pi-hole status if an error occurred
+  if [[ -n "${error}" ]]; then
+    "${PIHOLE_COMMAND}" status
+    exit 1
+  fi
+}