Fix example Docker Compose environment

Pull Horde's custom postgres image

Build aihorde Docker image in two stages to minimise rebuild time

Establish minimal necessary configuration to bring up a working composition

Fix pre-commit config so black and ruff hooks run

Catch and report exceptions in is_redis_up()

Add missing final EOL markers to some files (per black)

Update documentation

.pre-commit-config.yaml

@@ -20,9 +20,7 @@ repos:
     rev: v0.4.3
     hooks:
     -    id: ruff
-repos:
-  - repo: https://github.com/fsfe/reuse-tool
+-   repo: https://github.com/fsfe/reuse-tool
     rev: v4.0.3
     hooks:
       - id: reuse
-

CONTRIBUTING.md

@@ -12,6 +12,8 @@ SPDX-License-Identifier: AGPL-3.0-or-later
 * start server with `python server.py  -vvvvi --horde stable`
 * You can now connect to http://localhost:7001
 
+To run the AI Horde with Docker or Docker Compose, see the [README_docker.md](README_docker.md).
+
 # How to contribute to the AI Horde code
 
 We are happy you have ideas to improve this service and we welcome all contributors.
@@ -85,4 +87,4 @@ Note that the pre-commit will not complain if you forget to add your copyright n
 
 # Code of Conduct
 
-We expect all contributors to follow the [Anarchist code of conduct](https://wiki.dbzer0.com/the-anarchist-code-of-conduct/). Do not drive away other contributors due to intended or unintended on bigotry.
+We expect all contributors to follow the [Anarchist code of conduct](https://wiki.dbzer0.com/the-anarchist-code-of-conduct/). Do not drive away other contributors due to intended or unintended on bigotry.

Dockerfile

@@ -1,25 +1,52 @@
 # SPDX-FileCopyrightText: 2024 Tazlin
+# SPDX-FileCopyrightText: 2024 ceruleandeep
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 # Use a slim base image for Python 3.10
-FROM python:3.10-slim
+FROM python:3.10-slim AS python
+
+
+##
+## BUILD STAGE
+##
+FROM python AS python-build-stage
 
 # Install Git
 RUN apt-get update && apt-get install -y git
 
-# Set the working directory
+RUN --mount=type=cache,target=/root/.cache pip install --upgrade pip
+
+# Build dependencies
+COPY ./requirements.txt .
+RUN --mount=type=cache,target=/root/.cache \
+  pip wheel --wheel-dir /usr/src/app/wheels \
+  -r requirements.txt
+
+
+##
+## RUN STAGE
+##
+FROM python AS python-run-stage
+
+# git is required in the run stage because one dependency is not available in PyPI
+RUN apt-get update && apt-get install -y git
+
+RUN --mount=type=cache,target=/root/.cache pip install --upgrade pip
+
+# Install dependencies
+COPY --from=python-build-stage /usr/src/app/wheels /wheels/
+COPY ./requirements.txt .
+RUN pip install --no-cache-dir --no-index --find-links=/wheels/ \
+  -r requirements.txt \
+	&& rm -rf /wheels/
+
 WORKDIR /app
 
-# Copy the source code to the container
 COPY . /app
 
-# Install the dependencies
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --no-cache-dir --prefer-binary -r requirements.txt
-
 # Set the environment variables
-ENV PROFILE=
+ENV PROFILE=""
 
 # Set the command to run when the container starts
 CMD ["python", "server.py", "-vvvvi", "--horde", "stable"]

README_docker.md

@@ -22,16 +22,29 @@ Run the following command in your project root folder (the folder where your Doc
 docker build -t aihorde:latest .
 ```
 
-## with Docker-compose
+## with Docker Compose
 
-Create `.env_docker` file to deliver access information of services used together such as Redis and Postgres.
+[docker-compose.yaml](docker-compose.yaml) is provided to run the AI-Horde with Redis and Postgres.
 
-Copy the `.env_template` file in the root folder to create the .env_docker file.
+Copy the `.env_template` file in the root folder to create the `.env_docker` file.
 
-[docker-compose.yaml](docker-compose.yaml) Change the file as needed.
+```bash
+cp .env_template .env_docker
+```
+
+To use the supplied `.env_template` with the supplied `docker-compose.yaml`, you will need to set:
+
+```bash
+# .env_docker
+REDIS_IP="redis"
+REDIS_SERVERS='["redis"]'
+USE_SQLITE=0
+POSTGRES_URL="postgres"
+```
 
+Then run the following command in your project root folder:
 
 ```bash
 # run in background
-docker-compose up -d
+docker compose up --build -d
 ```

docker-compose.yaml

@@ -1,33 +1,43 @@
 # SPDX-FileCopyrightText: 2024 Tazlin
+# SPDX-FileCopyrightText: 2024 ceruleandeep
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-version: '3'
 services:
   aihorde:
+    build:
+      context: .
+      dockerfile: Dockerfile
     image: aihorde:latest
     container_name: aihorde
     ports:
       - "7001:7001"
     # The port number written in front of the colon (:) is the port number to be exposed to the outside, so if you change it, you can access it with localhost:{changePort}.
     environment:
-      - PROFILE=docker # If you write a profile, the .env_{PROFILE} file is read.
+      # Flask obtains its environment variables from the .env file.
+      # If you set a profile, the .env_{PROFILE} file is read instead.
+      - PROFILE=docker
     volumes:
-      - .env_docker:/app/.env_docker # You can replace the local pre-built .env file with the container's file.
+      # .env_{PROFILE} is copied into the image when it is built.
+      # So that you can change the environment variables without rebuilding the image, mount the .env file.
+      - .env_docker:/app/.env_docker
+      # Likewise, you can mount the horde directory to change the source code without rebuilding the image.
+      - ./horde:/app/horde
     networks:
       - aihorde_network
     depends_on:
       - postgres
       - redis
 
   postgres:
-    image: postgres:15.3-alpine
+    image: ghcr.io/haidra-org/ai-horde-postgres:latest
     container_name: postgres
     restart: always
     environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: admin
-      POSTGRES_DB: postgres
+        POSTGRES_PASSWORD: changeme
+    volumes:
+      # Use a named volume to persist the data even if the container is deleted.
+      - postgres_data:/var/lib/postgresql/data/
     ports:
       - "5432:5432"
     networks:
@@ -45,3 +55,6 @@ services:
 networks:
   aihorde_network:
     driver: bridge
+
+volumes:
+  postgres_data:

horde/redis_ctrl.py

@@ -24,7 +24,14 @@
 
 def is_redis_up() -> bool:
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        return s.connect_ex((redis_hostname, redis_port)) == 0
+        try:
+            return s.connect_ex((redis_hostname, redis_port)) == 0
+        except socket.gaierror as e:
+            # connect_ex suppresses exceptions from POSIX connect() call
+            # but can still raise gaierror if e.g. the hostname is invalid.
+            # This may be transient, so log the error and return False.
+            logger.error(f"Redis server at {redis_hostname}:{redis_port} is not reachable: {e}")
+            return False
 
 
 def is_local_redis_up() -> bool:

sql_statements/4.43.0.txt

@@ -10,4 +10,4 @@ CREATE INDEX idx_workers_allow_painting ON public.workers USING btree(allow_pain
 CREATE INDEX idx_workers_allow_post_processing ON public.workers USING btree(allow_post_processing);
 CREATE INDEX idx_workers_allow_controlnet ON public.workers USING btree(allow_controlnet);
 CREATE INDEX idx_workers_allow_sdxl_controlnet ON public.workers USING btree(allow_sdxl_controlnet);
-CREATE INDEX idx_workers_allow_lora ON public.workers USING btree(allow_lora);
+CREATE INDEX idx_workers_allow_lora ON public.workers USING btree(allow_lora);

sql_statements/README.md

@@ -22,4 +22,4 @@ SPDX-License-Identifier: AGPL-3.0-or-later
   - `compile_*gen_stats_*.sql`
     - These files defined stored procedures which populated the `compiled_*` tables and generally represent minute/hour/day/total statistics about generations.
   - `cron_jobs/`
-    - Schedules any stats compile jobs via `schedule_cron_job`. 
+    - Schedules any stats compile jobs via `schedule_cron_job`.