From 6335fb6b416373b6a2a7329a9d830f34e99d70a1 Mon Sep 17 00:00:00 2001 From: gattjoe Date: Tue, 14 May 2024 16:11:07 -0400 Subject: [PATCH] Add ENV variable for ConfigMap for adlists Signed-off-by: gattjoe --- README.md | 1 + src/s6/debian-root/usr/local/bin/_startup.sh | 4 + .../usr/local/bin/configmap_adlists.sh | 173 ++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 src/s6/debian-root/usr/local/bin/configmap_adlists.sh diff --git a/README.md b/README.md index a194e955e..0ed1b374f 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ There are other environment variables if you want to customize various things in | `WEB_UID` | `33` | Number | Overrides image's default www-data user id to match a host user id
**IMPORTANT**: id must not already be in use inside the container! (Make sure it is different to `PIHOLE_UID` if you are using that, also)| | `WEB_GID` | `33` | Number | Overrides image's default www-data group id to match a host group id
**IMPORTANT**: id must not already be in use inside the container! (Make sure it is different to `PIHOLE_GID` if you are using that, also)| | `WEBLOGS_STDOUT` | 0 | 0|1 | 0 logs to defined files, 1 redirect access and error logs to stdout | +| `CONFIGMAP_ADLISTS` | unset | `<"true"\|"false">` | Environmental variable to indicate that a ConfigMap was used to supply adlists. | ## Deprecated environment variables: While these may still work, they are likely to be removed in a future version. Where applicable, alternative variable names are indicated. Please review the table above for usage of the alternative variables diff --git a/src/s6/debian-root/usr/local/bin/_startup.sh b/src/s6/debian-root/usr/local/bin/_startup.sh index ad2a92c89..f202f38f7 100755 --- a/src/s6/debian-root/usr/local/bin/_startup.sh +++ b/src/s6/debian-root/usr/local/bin/_startup.sh @@ -8,6 +8,9 @@ fi # shellcheck source=/dev/null . /usr/local/bin/bash_functions.sh +# Experimental feature to allow for declarative adlists in kubernetes +. /usr/local/bin/configmap_adlists.sh + # shellcheck source=/dev/null SKIP_INSTALL=true . /etc/.pihole/automated\ install/basic-install.sh @@ -44,6 +47,7 @@ setup_lighttpd_bind # Misc Setup # =========================== installCron +[[ -n "${CONFIGMAP_ADLISTS}" && "${CONFIGMAP_ADLISTS}" == "true" ]] && echo " [i] Using configMap for adlists" && configMap_adlists setup_blocklists # FTL setup diff --git a/src/s6/debian-root/usr/local/bin/configmap_adlists.sh b/src/s6/debian-root/usr/local/bin/configmap_adlists.sh new file mode 100644 index 000000000..ddb28c1b6 --- /dev/null +++ b/src/s6/debian-root/usr/local/bin/configmap_adlists.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +export LC_ALL=C + +basename="pihole" +PIHOLE_COMMAND="/usr/local/bin/${basename}" + +piholeDir="/etc/${basename}" + +adListFile="${piholeDir}/adlists.list" + +domainsExtension="domains" + +# Set up tmp dir variable in case it's not configured +: "${GRAVITY_TMPDIR:=/tmp}" + +if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then + echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" + GRAVITY_TMPDIR="/tmp" +fi + +gravityDBfile_default="${piholeDir}/gravity.db" +GRAVITYDB="${gravityDBfile_default}" + +# Set this only after sourcing pihole-FTL.conf as the gravity database path may +# have changed +gravityDBfile="${GRAVITYDB}" +gravityTEMPfile="${GRAVITYDB}_temp" +gravityDIR="$(dirname -- "${gravityDBfile}")" +gravityOLDfile="${gravityDIR}/gravity_old.db" + + +configMap_adlists() { + echo " [i] Deleting existing adlists from gravity" + + # Experimental feature to clean out domains from gravity to allow a kubernetes ConfigMap to manage them + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM gravity;" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist;" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist_by_group;" + + echo " [i] Finished clearing out adlists from gravity" + + # Migrate list files to new database + if [ -e "${adListFile}" ]; then + # Store adlist domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlist" "${adListFile}" + fi + +} + +# Import domains from file and store them in the specified database table +database_table_from_file() { + # Define locals + local table src backup_path backup_file tmpFile list_type + table="${1}" + src="${2}" + backup_path="${piholeDir}/migration_backup" + backup_file="${backup_path}/$(basename "${2}")" + # Create a temporary file. We don't use '--suffix' here because not all + # implementations of mktemp support it, e.g. on Alpine + tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")" + mv "${tmpFile}" "${tmpFile%.*}.gravity" + + local timestamp + timestamp="$(date --utc +'%s')" + + local rowid + declare -i rowid + rowid=1 + + # Special handling for domains to be imported into the common domainlist table + if [[ "${table}" == "whitelist" ]]; then + list_type="0" + table="domainlist" + elif [[ "${table}" == "blacklist" ]]; then + list_type="1" + table="domainlist" + elif [[ "${table}" == "regex" ]]; then + list_type="3" + table="domainlist" + fi + + # Get MAX(id) from domainlist when INSERTing into this table + if [[ "${table}" == "domainlist" ]]; then + rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + if [[ -z "$rowid" ]]; then + rowid=0 + fi + rowid+=1 + fi + + # Loop over all domains in ${src} file + # Read file line by line + grep -v '^ *#' < "${src}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ -n "${domain}" ]]; then + if [[ "${table}" == "domain_audit" ]]; then + # domain_audit table format (no enable or modified fields) + echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" + elif [[ "${table}" == "adlist" ]]; then + # Adlist table format + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0" >> "${tmpFile}" + else + # White-, black-, and regexlist table format + echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}" + fi + rowid+=1 + fi + done + + # Store domains in database table specified by ${table} + # Use printf as .mode and .import need to be on separate lines + # see https://unix.stackexchange.com/a/445615/83260 + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to fill table ${table}${list_type} in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Move source file to backup directory, create directory if not existing + mkdir -p "${backup_path}" + mv "${src}" "${backup_file}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to backup ${src} to ${backup_path}" + + # Delete tmpFile + rm "${tmpFile}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${tmpFile}" +} + +# Clean up after Gravity upon exit or cancellation +gravity_Cleanup() { + local error="${1:-}" + + str="Cleaning up stray matter" + echo -ne " ${INFO} ${str}..." + + # Delete tmp content generated by Gravity + rm ${piholeDir}/pihole.*.txt 2> /dev/null + rm ${piholeDir}/*.tmp 2> /dev/null + # listCurlBuffer location + rm "${GRAVITY_TMPDIR}"/*.phgpb 2> /dev/null + # invalid_domains location + rm "${GRAVITY_TMPDIR}"/*.ph-non-domains 2> /dev/null + + # Ensure this function only runs when gravity_SetDownloadOptions() has completed + if [[ "${gravity_Blackbody:-}" == true ]]; then + # Remove any unused .domains files + for file in "${piholeDir}"/*."${domainsExtension}"; do + # If list is not in active array, then remove it + if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then + rm -f "${file}" 2> /dev/null || \ + echo -e " ${CROSS} Failed to remove ${file##*/}" + fi + done + fi + + echo -e "${OVER} ${TICK} ${str}" + + # Only restart DNS service if offline + if ! pgrep pihole-FTL &> /dev/null; then + "${PIHOLE_COMMAND}" restartdns + dnsWasOffline=true + fi + + # Print Pi-hole status if an error occurred + if [[ -n "${error}" ]]; then + "${PIHOLE_COMMAND}" status + exit 1 + fi +}