fix: Prevent one redis node going down causing degradation

CHANGELOG.md

@@ -6,6 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-or-later
 
 # Changelog
 
+# 4.43.5
+
+* Fix: Added check to ensure the redis servers are still available.
+
 # 4.43.4
 
 * Fix logic error when setting censored key

horde/apis/v2/base.py

@@ -18,7 +18,6 @@
 import horde.apis.limiter_api as lim
 import horde.classes.base.stats as stats
 from horde import exceptions as e
-from horde import horde_redis as hr
 from horde.apis.models.v2 import Models, Parsers
 from horde.argparser import args
 from horde.classes.base import settings
@@ -33,6 +32,7 @@
 from horde.database import functions as database
 from horde.detection import prompt_checker
 from horde.flask import HORDE, cache, db
+from horde.horde_redis import horde_redis as hr
 from horde.image import ensure_source_image_uploaded
 from horde.limiter import limiter
 from horde.logger import logger

horde/classes/base/user.py

@@ -12,12 +12,12 @@
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.ext.hybrid import hybrid_property
 
-from horde import horde_redis as hr
 from horde import vars as hv
 from horde.countermeasures import CounterMeasures
 from horde.discord import send_problem_user_notification
 from horde.enums import UserRecordTypes, UserRoleTypes
 from horde.flask import SQLITE_MODE, db
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.patreon import patrons
 from horde.suspicions import SUSPICION_LOGS, Suspicions

horde/classes/base/waiting_prompt.py

@@ -11,13 +11,13 @@
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.sql import expression
 
-from horde import horde_redis as hr
 from horde import vars as hv
 from horde.bridge_reference import check_bridge_capability
 from horde.classes.base.processing_generation import ProcessingGeneration
 from horde.classes.kobold.processing_generation import TextProcessingGeneration
 from horde.classes.stable.processing_generation import ImageProcessingGeneration
 from horde.flask import SQLITE_MODE, db
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.utils import get_db_uuid, get_expiry_date, get_extra_slow_expiry_date
 

horde/classes/base/worker.py

@@ -9,11 +9,11 @@
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.ext.hybrid import hybrid_property
 
-from horde import horde_redis as hr
 from horde import vars as hv
 from horde.classes.base import settings
 from horde.discord import send_pause_notification
 from horde.flask import SQLITE_MODE, db
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.suspicions import SUSPICION_LOGS, Suspicions
 from horde.utils import get_db_uuid, is_profane, sanitize_string

horde/classes/kobold/worker.py

@@ -8,12 +8,12 @@
 from sqlalchemy.dialects.postgresql import UUID
 
 from horde import exceptions as e
-from horde import horde_redis as hr
 from horde.bridge_reference import (
     is_backed_validated,
 )
 from horde.classes.base.worker import Worker
 from horde.flask import SQLITE_MODE, db
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.model_reference import model_reference
 from horde.utils import sanitize_string

horde/classes/stable/interrogation.py

@@ -9,10 +9,10 @@
 from sqlalchemy import JSON, Enum
 from sqlalchemy.dialects.postgresql import JSONB, UUID
 
-from horde import horde_redis as hr
 from horde.consts import KNOWN_POST_PROCESSORS
 from horde.enums import State
 from horde.flask import SQLITE_MODE, db
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.r2 import generate_procgen_download_url, generate_procgen_upload_url
 from horde.utils import get_db_uuid, get_expiry_date, get_interrogation_form_expiry_date

horde/consts.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-HORDE_VERSION = "4.43.4"
+HORDE_VERSION = "4.43.5"
 
 WHITELISTED_SERVICE_IPS = {
     "212.227.227.178",  # Turing Bot

horde/countermeasures.py

@@ -19,26 +19,16 @@
 )
 
 ip_r = None
-logger.init("IP Address Cache", status="Connecting")
-if is_redis_up():
-    ip_r = get_ipaddr_db()
-    logger.init_ok("IP Address Cache", status="Connected")
-else:
-    logger.init_err("IP Address Cache", status="Failed")
 ip_s_r = None
-logger.init("IP Suspicion Cache", status="Connecting")
-if is_redis_up():
-    ip_s_r = get_ipaddr_suspicion_db()
-    logger.init_ok("IP Suspicion Cache", status="Connected")
-else:
-    logger.init_err("IP Suspicion Cache", status="Failed")
 ip_t_r = None
-logger.init("IP Timeout Cache", status="Connecting")
+logger.init("IP Caches", status="Connecting")
 if is_redis_up():
+    ip_r = get_ipaddr_db()
+    ip_s_r = get_ipaddr_suspicion_db()
     ip_t_r = get_ipaddr_timeout_db()
-    logger.init_ok("IP Timeout Cache", status="Connected")
+    logger.init_ok("IP Caches", status="Connected")
 else:
-    logger.init_err("IP Timeout Cache", status="Failed")
+    logger.init_err("IP Caches", status="Failed")
 
 test_timeout = 0
 

horde/database/functions.py

@@ -13,7 +13,6 @@
 from sqlalchemy.orm import noload
 
 import horde.classes.base.stats as stats
-from horde import horde_redis as hr
 from horde import vars as hv
 from horde.bridge_reference import (
     check_bridge_capability,
@@ -34,6 +33,7 @@
 from horde.database.classes import FakeWPRow
 from horde.enums import State
 from horde.flask import SQLITE_MODE, db
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.model_reference import model_reference
 from horde.utils import hash_api_key, validate_regex

horde/database/text_functions.py

@@ -10,7 +10,6 @@
 from sqlalchemy.orm import noload
 
 import horde.classes.base.stats as stats
-from horde import horde_redis as hr
 from horde.bridge_reference import (
     is_backed_validated,
 )
@@ -22,6 +21,7 @@
 from horde.classes.kobold.waiting_prompt import TextWaitingPrompt
 from horde.database.functions import query_prioritized_wps
 from horde.flask import SQLITE_MODE, db
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.model_reference import model_reference
 

horde/database/threads.py

@@ -9,7 +9,6 @@
 import patreon
 from sqlalchemy import func, or_
 
-from horde import horde_redis as hr
 from horde.argparser import args
 from horde.classes.base.user import User
 from horde.classes.kobold.processing_generation import TextProcessingGeneration
@@ -30,6 +29,7 @@
 )
 from horde.enums import State
 from horde.flask import HORDE, SQLITE_MODE, db
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.patreon import patrons
 from horde.r2 import delete_source_image

horde/detection.py

@@ -13,7 +13,7 @@
 from horde.argparser import args
 from horde.database.functions import compile_regex_filter, retrieve_regex_replacements
 from horde.flask import HORDE, SQLITE_MODE  # Local Testing
-from horde.horde_redis import horde_r_get
+from horde.horde_redis import horde_redis as hr
 from horde.logger import logger
 from horde.model_reference import model_reference
 
@@ -82,7 +82,7 @@ def refresh_regex(self):
             with HORDE.app_context():
                 stored_replacements = retrieve_regex_replacements(filter_type=10)
         else:
-            cached_replacements = horde_r_get("cached_regex_replacements")
+            cached_replacements = hr.horde_r_get("cached_regex_replacements")
             if not cached_replacements:
                 logger.warning("No cached regex replacements found in redis! Check threads!")
                 stored_replacements = []
@@ -97,7 +97,7 @@ def refresh_regex(self):
                 with HORDE.app_context():
                     stored_filter = compile_regex_filter(_id)
             else:
-                stored_filter = horde_r_get(filter_id)
+                stored_filter = hr.horde_r_get(filter_id)
             # Ensure we don't get catch-all regex
             if not stored_filter:
                 continue