From 8561ddb0435e53e89146c778c24b2dccc8b7750c Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Mon, 5 Jun 2023 20:04:42 +0100
Subject: [PATCH 01/14] improved container image release workflow

---
 .github/workflows/container-image-release.yml | 121 +++++++++++++-----
 1 file changed, 91 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/container-image-release.yml b/.github/workflows/container-image-release.yml
index 271a1bf6..7e78ff6b 100644
--- a/.github/workflows/container-image-release.yml
+++ b/.github/workflows/container-image-release.yml
@@ -1,24 +1,34 @@
 name: Container Image Release
-concurrency: production
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+permissions:
+  packages: write
+  contents: read
 on:
   release:
     types: [published]
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: 'Image tag'
+        required: true
+        default: 'nightly-<put_date_here>'
+
+env:
+  REGISTRY_IMAGE: ghcr.io/jikan-me/jikan-rest
 
 jobs:
-  release-app-image:
+  build-app-image:
     runs-on: ubuntu-latest
-    name: Release App container image
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+    name: Build App container image
     steps:
-      - name: Check if base container image exists
-        id: baseImageExists
-        run: |
-          GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
-          curl --fail -H "Authorization: Bearer ${GHCR_TOKEN}" https://ghcr.io/v2/jikan-me/jikan-rest-php/tags/list | grep -q latest
-
-      - name: Base image existance check failed
-        if: ${{ always() && steps.baseImageExists.outcome == 'failure' }}
-        run: echo "Base image doesn't exist yet. Please run the base image creation workflow first."
-      
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -29,18 +39,75 @@ jobs:
 
       - name: Set up docker buildx
         uses: docker/setup-buildx-action@v3
-        with:
-          platforms: linux/amd64,linux/arm64
 
       - name: Read metadata
         id: meta
         uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          labels: |
+            org.opencontainers.image.title=Jikan REST API
+            org.opencontainers.image.description=REST API for Jikan
+            org.opencontainers.image.url=https://jikan.moe
+            org.opencontainers.image.source=https://github.com/jikan-me/jikan-rest
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          # let's use github action cache storage
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  publish-app-image:
+    runs-on: ubuntu-latest
+    name: Publish app container image
+    needs:
+      - build-app-image
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Read metadata
+        id: meta
+        uses: docker/metadata-action@v4
         with:
           images: |
-            ghcr.io/jikan-me/jikan-rest
+            ${{ env.REGISTRY_IMAGE }}
             jikanme/jikan-rest
           tags: |
-            type=raw,value=${{ github.ref_name }}
+            type=raw,value=${{ inputs.image_tag || github.ref_name }}
             type=sha
 
       - name: Login to GitHub Container Registry
@@ -56,16 +123,10 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          push: true
-          context: .
-          # let's use github action cache storage
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          build-args: |
-            GITHUB_PERSONAL_TOKEN=${{ secrets.GITHUB_TOKEN }}
-            BASE_IMAGE_VERSION=latest
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}

From 40812e814d6d48791bbd5e0521304fd4a80a3547 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Mon, 5 Jun 2023 20:06:11 +0100
Subject: [PATCH 02/14] improved Dockerfile - fixed cpu architecture - Removed
 unnecessary build args

---
 Dockerfile         | 10 +---------
 container_usage.md | 25 +++++++++++++++----------
 2 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index d00a7c1c..642d8312 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,6 @@ FROM spiralscout/roadrunner:2.12.2 as roadrunner
 FROM composer:2.5.1 as composer
 FROM mlocati/php-extension-installer:1.5.52 as php-ext-installer
 FROM php:8.1.16-bullseye
-ARG GITHUB_PERSONAL_TOKEN
 COPY --from=composer /usr/bin/composer /usr/bin/composer
 COPY --from=php-ext-installer /usr/bin/install-php-extensions /usr/local/bin/
 ENV COMPOSER_HOME="/tmp/composer"
@@ -14,8 +13,6 @@ RUN set -x \
 # install roadrunner
 COPY --from=roadrunner /usr/bin/rr /usr/bin/rr
 LABEL org.opencontainers.image.source=https://github.com/jikan-me/jikan-rest
-# used only for supercronic atm. Supported values are: amd64, arm64
-ARG TARGET_ARCH="amd64"
 RUN	set -ex \
     && apt-get update && apt-get install -y --no-install-recommends \
 	openssl \
@@ -24,7 +21,7 @@ RUN	set -ex \
 	unzip \
   wget \
   # install supercronic (for laravel task scheduling), project page: <https://github.com/aptible/supercronic>
-	&& wget -q "https://github.com/aptible/supercronic/releases/download/v0.1.12/supercronic-linux-${TARGET_ARCH}" \
+	&& wget -q "https://github.com/aptible/supercronic/releases/download/v0.1.12/supercronic-linux-$(dpkg --print-architecture)" \
 	   -O /usr/bin/supercronic \
 	&& chmod +x /usr/bin/supercronic \
 	&& mkdir /etc/supercronic \
@@ -51,11 +48,6 @@ WORKDIR /app
 # copy composer (json|lock) files for dependencies layer caching
 COPY --chown=jikanapi:jikanapi ./composer.* /app/
 
-# check if GITHUB_PERSONAL_TOKEN is set and configure it for composer
-# it is recommended to set this for the build, otherwise the build might fail because of github's rate limits
-RUN if [ -z "$GITHUB_PERSONAL_TOKEN" ]; then echo "** GITHUB_PERSONAL_TOKEN is not set. This build may fail due to github rate limits."; \
-    else composer config github-oauth.github.com "$GITHUB_PERSONAL_TOKEN"; fi
-
 # install composer dependencies (autoloader MUST be generated later!)
 RUN composer install -n --no-dev --no-cache --no-ansi --no-autoloader --no-scripts --prefer-dist
 
diff --git a/container_usage.md b/container_usage.md
index ffc64fd7..30f4bb4d 100644
--- a/container_usage.md
+++ b/container_usage.md
@@ -25,14 +25,6 @@ docker build -t jikan-rest:nightly .
 docker run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env jikan-rest:nightly
 ```
 
-If you need a different CPU architecture, set the `TARGET_ARCH` build argument:
-
-```bash
-docker build -t jikan-rest:nightly --build-arg TARGET_ARCH=arm64 .
-```
-
-`TARGET_ARCH` is `amd64` by default.
-
 ### Docker compose usage
 
 ```
@@ -42,10 +34,23 @@ docker-compose up
 Docker compose will use the `.env` file from the folder where you execute it from to load configurations for the
 services. If you don't have a `.env` file yet in the folder, copy the `.env.dist` file, and set the passwords.
 
+Docker compose will start up a production ready setup with redis, typesense and mongodb.
+Change the environment variables for the `jikan_rest` service in the `docker-compose.yml` file to customise the configuration of the Jikan API.
+By default docker-compose sets it to:
+- Not use [queuing](https://laravel.com/docs/9.x/scout#queueing) for search requests. (Not yet supported for  typesense.)
+- Use typesense search for more accurate search results
+- Use redis for microcaching and caching.
+- Add CORS headers to responses
+
+Of course these defaults can be overridden through the `.env` file or through setting environment variables in the command-line before  executing `docker-compose up`.
+
 > **Please note**: The syntax rules of docker compose for `.env` applies
 > here: https://docs.docker.com/compose/env-file/#syntax-rules
 
-#### Note for Podman
+> **Additional configuration**: You can change the mongodb memory usage via `MONGO_CACHE_SIZE_GB` environment variable. 
+> It sets how many gigabytes of memory is available for wired tiger. Default is `1`. This is useful for systems with low memory capacity.
+
+### Note for Podman
 
 If you build the container image yourself with podman, the resulting image format will be OCI by default.
 To make the health checks work in that situation you need to run the container the following way:
@@ -54,7 +59,7 @@ To make the health checks work in that situation you need to run the container t
 podman run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env --health-start-period=5s --health-cmd="curl --fail http://localhost:2114/health?plugin=http || exit 1" jikan-rest:nightly
 ```
 
-#### Configuration of the container
+### Configuration of the container
 
 You can change the settings of Jikan through setting environment variables via the `-e` command line argument option for
 the `docker run` command.

From 317906cdf90a8afe3b624800596b1bbdbee06183 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Mon, 5 Jun 2023 20:19:04 +0100
Subject: [PATCH 03/14] added support for container secrets

---
 docker-entrypoint.php | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/docker-entrypoint.php b/docker-entrypoint.php
index 3eccdcd7..97d41be0 100644
--- a/docker-entrypoint.php
+++ b/docker-entrypoint.php
@@ -19,6 +19,8 @@
     "DB_PASSWORD" => ""
 ];
 
+$current_env = $_ENV;
+
 if (!file_exists(".env")) {
     copy(".env.dist", ".env");
     $writer = new \MirazMac\DotEnv\Writer(__DIR__ . '/' . '.env');
@@ -29,10 +31,30 @@
     $writer->write();
 }
 
+// We'd like to support Container secrets. So we'll check if any of the env vars has a __FILE suffix
+// then we'll try to load the file and set the env var to the contents of the file.
+// https://docs.docker.com/engine/swarm/secrets/
+$envWriter = new \MirazMac\DotEnv\Writer(__DIR__ . '/' . '.env');
+$itemsWritten = 0;
+foreach (array_keys($current_env) as $env_key) {
+    if (!str_contains($env_key, "__FILE")) {
+        continue;
+    }
+    if (!file_exists($current_env[$env_key])) {
+        echo "Couldn't load secret: " . $_ENV[$env_key] . PHP_EOL;
+        continue;
+    }
+    $originalKey = str_replace("__FILE", "", $env_key);
+    $envWriter->set($originalKey, file_get_contents($current_env[$env_key]));
+    $itemsWritten++;
+}
+
+if ($itemsWritten > 0) {
+    $envWriter->write();
+}
 
 $dotenv = Dotenv::createImmutable(__DIR__);
 $dotenv->load();
-$current_env = $_ENV;
 
 if ($current_env["SCOUT_DRIVER"] === "typesense" && empty($current_env["TYPESENSE_API_KEY"])) {
     echo "Please set the TYPESENSE_API_KEY environment variable when setting SCOUT_DRIVER to typesense.";

From d4892264734baab516725a6a706628020299b15e Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Mon, 5 Jun 2023 20:19:20 +0100
Subject: [PATCH 04/14] wip docker-compose

---
 docker-compose.yml | 79 +++++++++++++++++++++++++++-------------------
 1 file changed, 47 insertions(+), 32 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index f7815e22..e6216b16 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,40 +1,51 @@
-# For now this is just for local development. This is not production ready.
 version: '3.8'
 volumes:
-  mongo-data: {}
-  redis-data: {}
-  tmp-data: {}
-  typesense-data: {}
+  mongo-data: { }
+  redis-data: { }
+  typesense-data: { }
+
+secrets:
+  db_password:
+    file: db_password.txt
+  redis_password:
+    file: redis_password.txt
+  typesense_api_key:
+    file: typesense_api_key.txt
 
 services:
   jikan_rest: &jikan_rest
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: jikanme/jikan-rest:latest
     user: "${APP_UID:-10001}:${APP_GID:-10001}"
+    secrets:
+      - db_password
+      - typesense_api_key
     environment:
       PS1: '\[\033[1;32m\]\[\033[1;36m\][\u@\h] \[\033[1;34m\]\w\[\033[0;35m\] \[\033[1;36m\]# \[\033[0m\]'
-      HOME: /tmp
-      APP_DEBUG: 'true'
-      APP_ENV: local
+      APP_DEBUG: 'false'
+      APP_ENV: production
+      CACHING: 'true'
+      CACHE_DRIVER: redis
       REDIS_HOST: redis
-      REDIS_PASSWORD: "${REDIS_PASSWORD:-null}"
+      REDIS_PASSWORD__FILE: /run/secrets/redis_password
       DB_CONNECTION: mongodb
       DB_HOST: mongodb
       DB_DATABASE: jikan
       DB_PORT: 27017
-      DB_ADMIN: jikan
-      DB_USERNAME: "${DB_USERNAME}"
-      DB_PASSWORD: "${DB_PASSWORD}"
-    volumes:
-      - /etc/passwd:/etc/passwd:ro
-      - /etc/group:/etc/group:ro
-      - tmp-data:/tmp:rw
-      - .:/app:rw
+      DB_ADMIN: "${DB_USERNAME:-root}"
+      DB_USERNAME: "${DB_USERNAME:-root}"
+      SCOUT_DRIVER: typesense
+      SCOUT_QUEUE: 'false'
+      DB_PASSWORD__FILE: /run/secrets/db_password
+      TYPESENSE_HOST: typesense
+      TYPESENSE_PORT: 8108
+      TYPESENSE_API_KEY__FILE: /run/secrets/typesense_api_key
+      CORS_MIDDLEWARE: 'true'
+      MICROCACHING: 'true'
+      MICROCACHING_EXPIRE: 60
     depends_on:
-      mongodb: {condition: service_healthy}
-      redis: {condition: service_healthy}
-      typesense: {condition: service_healthy}
+      mongodb: { condition: service_healthy }
+      redis: { condition: service_healthy }
+      typesense: { condition: service_healthy }
 
   web:
     <<: *jikan_rest
@@ -45,20 +56,18 @@ services:
       interval: 2s
       timeout: 2s
 
-  cron:
-    <<: *jikan_rest
-    command: supercronic /etc/supercronic/laravel # it runs artisan schedule:run
-
   mongodb:
     image: mongo:focal
     volumes:
       - mongo-data:/data/db
     ports:
       - '27017/tcp'
-    command: --wiredTigerCacheSizeGB 1
+    command: --wiredTigerCacheSizeGB ${MONGO_CACHE_SIZE_GB:1}
+    secrets:
+      - db_password
     environment:
       MONGO_INITDB_ROOT_USERNAME: "${DB_USERNAME:-root}"
-      MONGO_INITDB_ROOT_PASSWORD: "${DB_PASSWORD}"
+      MONGO_INITDB_ROOT_PASSWORD_FILE: /run/secrets/db_password
     healthcheck:
       test: echo 'db.runCommand("ping").ok' | mongo mongodb://localhost:27017 --quiet
       interval: 30s
@@ -67,10 +76,14 @@ services:
 
   redis:
     image: redis:7-alpine
+    secrets:
+      - redis_password
+    environment:
+      REDIS_PASSWORD: "${REDIS_PASSWORD:-null}"
     command:
       - /bin/sh
       - -c
-      - redis-server --requirepass "$${REDIS_PASSWORD:?REDIS_PASSWORD variable is not set}"
+      - redis-server --requirepass "$${cat /run/secrets/redis_password}"
     volumes:
       - redis-data:/data:rw
     ports:
@@ -81,11 +94,13 @@ services:
       timeout: 1s
 
   typesense:
-    image: typesense/typesense:0.23.1
+    image: typesense/typesense:0.24.1
     entrypoint: /bin/sh
+    secrets:
+      - typesense_api_key
     command:
       - -c
-      - /opt/typesense-server --data-dir /data --api-key "$${TYPESENSE_API_KEY:?TYPESENSE_API_KEY variable is not set}"
+      - TYPESENSE_API_KEY="$$(cat /run/secrets/typesense_api_key)" /opt/typesense-server --data-dir /data
     restart: no
     volumes:
       - typesense-data:/data

From 5f8fd79a4ce37fb88f9cb3b974acb38e4f264e7b Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Mon, 5 Jun 2023 20:32:30 +0100
Subject: [PATCH 05/14] wip docker-compose

---
 docker-compose.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docker-compose.yml b/docker-compose.yml
index e6216b16..8fdacb3d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -106,3 +106,7 @@ services:
       - typesense-data:/data
     ports:
       - "8108/tcp"
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl', '-s', '-f', 'http://localhost:8108/health' ]
+      interval: 5s
+      timeout: 2s

From 9b56d7b397fb7c52a5c33092dab43a48a9a85351 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Sun, 23 Jul 2023 12:33:51 +0100
Subject: [PATCH 06/14] wip setup cli

---
 .dockerignore              |  4 ++
 .gitignore                 |  4 ++
 .rr.yaml                   |  2 +-
 container-setup.sh         | 75 ++++++++++++++++++++++++++++++++++++++
 docker-compose.yml         | 53 ++++++++++++---------------
 docker/config/.env.compose | 21 +++++++++++
 6 files changed, 128 insertions(+), 31 deletions(-)
 create mode 100644 container-setup.sh
 create mode 100644 docker/config/.env.compose

diff --git a/.dockerignore b/.dockerignore
index 069a9248..4222f426 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -19,3 +19,7 @@ node_modules
 .vscode
 .github
 .git
+db_username.txt
+db_password.txt
+redis_password.txt
+typesense_api_key.txt
diff --git a/.gitignore b/.gitignore
index 4e303537..80bda74b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,7 @@ composer.phar
 /coverage
 .DS_Store
 *.cache
+db_username.txt
+db_password.txt
+redis_password.txt
+typesense_api_key.txt
diff --git a/.rr.yaml b/.rr.yaml
index b5d4e792..0f9a29ef 100644
--- a/.rr.yaml
+++ b/.rr.yaml
@@ -61,7 +61,7 @@ logs:
 
   # we want to use docker's log drivers, so push logs to stdout
   output: stdout
-  # we to use docker's log drivers, so push error logs to stdout
+  # we want to use docker's log drivers, so push error logs to stdout
   # this way it is possible for example to pipe logs to journald or to AWS Cloudwatch
   err_output: stdout
 
diff --git a/container-setup.sh b/container-setup.sh
new file mode 100644
index 00000000..17d23767
--- /dev/null
+++ b/container-setup.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+JIKAN_API_VERSION=v4.0.0-rc.11
+DOCKER_COMPOSE_PROJECT_NAME=jikan-api-$JIKAN_API_VERSION
+
+display_help() {
+  echo "============================================================"
+  echo "Jikan API Container Setup CLI"
+  echo "============================================================"
+  echo "Syntax: ./container-setup.sh [command]"
+  echo "Jikan API Version: $JIKAN_API_VERSION"
+  echo "---commands---"
+  echo "help                   Print CLI help"
+  echo "build-image            Build Image Locally"
+  echo "start                  Start Jikan API (mongodb, typesense, redis, jikan-api workers)"
+  echo "validate-prereqs       Validate pre-reqs installed (docker, docker-compose)"
+  echo "execute-indexers       Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days)"
+  echo ""
+}
+
+validate_prereqs() {
+   docker -v >/dev/null 2>&1
+   if [ $? -ne 0 ]; then
+      echo -e "'docker' is not installed or not runnable without sudo. \xE2\x9D\x8C"
+   else
+      echo -e "Docker is Installed. \xE2\x9C\x94"
+   fi
+
+   docker-compose -v >/dev/null 2>&1
+   if [ $? -ne 0 ]; then
+      echo -e "'docker-compose' is not installed. \xE2\x9D\x8C"
+   else
+      echo -e "Docker compose is Installed. \xE2\x9C\x94"
+   fi
+}
+
+build_image() {
+   docker build --rm --compress -t jikanme/jikan-rest:$JIKAN_API_VERSION .
+}
+
+start() {
+   docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME up -d
+}
+
+case "$1" in
+   "help")
+      display_help
+      ;;
+   "validate-prereqs")
+      validate_prereqs
+      ;;
+   "build-image")
+      build_image
+      ;;
+   "start")
+      start
+      ;;
+    "execute-indexers")
+      echo "Indexing anime..."
+      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:anime
+      echo "Indexing manga..."
+      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:manga
+      echo "Indexing characters and people..."
+      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:common
+      echo "Indexing genres..."
+      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:genres
+      echo "Indexing producers..."
+      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:producers
+      echo "Indexing done!"
+      ;;
+   *)
+      echo "No command specified, displaying help"
+      display_help
+      ;;
+esac
diff --git a/docker-compose.yml b/docker-compose.yml
index 8fdacb3d..8ebe53c9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,7 +4,12 @@ volumes:
   redis-data: { }
   typesense-data: { }
 
+networks:
+  jikan_network: { }
+
 secrets:
+  db_username:
+    file: db_username.txt
   db_password:
     file: db_password.txt
   redis_password:
@@ -13,48 +18,30 @@ secrets:
     file: typesense_api_key.txt
 
 services:
-  jikan_rest: &jikan_rest
-    image: jikanme/jikan-rest:latest
+  jikan_rest:
+    image: jikanme/jikan-rest:v4.0.0-rc.11
     user: "${APP_UID:-10001}:${APP_GID:-10001}"
+    networks:
+      - jikan_network
     secrets:
+      - db_username
       - db_password
       - typesense_api_key
+      - redis_password
     environment:
       PS1: '\[\033[1;32m\]\[\033[1;36m\][\u@\h] \[\033[1;34m\]\w\[\033[0;35m\] \[\033[1;36m\]# \[\033[0m\]'
-      APP_DEBUG: 'false'
-      APP_ENV: production
-      CACHING: 'true'
-      CACHE_DRIVER: redis
-      REDIS_HOST: redis
-      REDIS_PASSWORD__FILE: /run/secrets/redis_password
-      DB_CONNECTION: mongodb
-      DB_HOST: mongodb
-      DB_DATABASE: jikan
-      DB_PORT: 27017
-      DB_ADMIN: "${DB_USERNAME:-root}"
-      DB_USERNAME: "${DB_USERNAME:-root}"
-      SCOUT_DRIVER: typesense
-      SCOUT_QUEUE: 'false'
-      DB_PASSWORD__FILE: /run/secrets/db_password
-      TYPESENSE_HOST: typesense
-      TYPESENSE_PORT: 8108
-      TYPESENSE_API_KEY__FILE: /run/secrets/typesense_api_key
-      CORS_MIDDLEWARE: 'true'
-      MICROCACHING: 'true'
-      MICROCACHING_EXPIRE: 60
-    depends_on:
-      mongodb: { condition: service_healthy }
-      redis: { condition: service_healthy }
-      typesense: { condition: service_healthy }
-
-  web:
-    <<: *jikan_rest
+    env_file:
+      - ./docker/config/.env.compose
     ports:
       - '8080:8080/tcp'
     healthcheck:
       test: [ 'CMD-SHELL', 'wget --spider -q "http://127.0.0.1:2114/health?plugin=http"' ]
       interval: 2s
       timeout: 2s
+    depends_on:
+      mongodb: { condition: service_healthy }
+      redis: { condition: service_healthy }
+      typesense: { condition: service_healthy }
 
   mongodb:
     image: mongo:focal
@@ -63,6 +50,8 @@ services:
     ports:
       - '27017/tcp'
     command: --wiredTigerCacheSizeGB ${MONGO_CACHE_SIZE_GB:1}
+    networks:
+      - jikan_network
     secrets:
       - db_password
     environment:
@@ -80,6 +69,8 @@ services:
       - redis_password
     environment:
       REDIS_PASSWORD: "${REDIS_PASSWORD:-null}"
+    networks:
+      - jikan_network
     command:
       - /bin/sh
       - -c
@@ -98,6 +89,8 @@ services:
     entrypoint: /bin/sh
     secrets:
       - typesense_api_key
+    networks:
+      - jikan_network
     command:
       - -c
       - TYPESENSE_API_KEY="$$(cat /run/secrets/typesense_api_key)" /opt/typesense-server --data-dir /data
diff --git a/docker/config/.env.compose b/docker/config/.env.compose
new file mode 100644
index 00000000..193d4530
--- /dev/null
+++ b/docker/config/.env.compose
@@ -0,0 +1,21 @@
+APP_DEBUG=false
+LOG_LEVEL=info
+APP_ENV=production
+CACHING=true
+CACHE_DRIVER=redis
+REDIS_HOST=redis
+REDIS_PASSWORD__FILE=/run/secrets/redis_password
+DB_CONNECTION=mongodb
+DB_HOST=mongodb
+DB_DATABASE=jikan
+DB_USERNAME__FILE=/run/secrets/db_username
+DB_ADMIN__FILE=/run/secrets/db_username
+DB_PASSWORD__FILE=/run/secrets/db_password
+SCOUT_DRIVER=typesense
+SCOUT_QUEUE=false
+TYPESENSE_HOST=typesense
+TYPESENSE_PORT=8108
+TYPESENSE_API_KEY__FILE=/run/secrets/typesense_api_key
+CORS_MIDDLEWARE=true
+MICROCACHING=true
+MICROCACHING_EXPIRE=60

From 309e1d6402dc51c02464898fae65b978ab7ce9ed Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Sun, 20 Aug 2023 22:35:02 +0100
Subject: [PATCH 07/14] wip - container setup cli and fixed some issues with
 docker compose yml (still buggy)

---
 .dockerignore          |   4 ++
 Dockerfile             |   6 +--
 app/Console/Kernel.php |  32 ++++---------
 container-setup.sh     | 101 +++++++++++++++++++++++++++++++++--------
 docker-compose.yml     |  21 +++++----
 docker-entrypoint.php  |   2 +
 storage/app/.gitignore |   2 +
 7 files changed, 117 insertions(+), 51 deletions(-)
 mode change 100644 => 100755 container-setup.sh

diff --git a/.dockerignore b/.dockerignore
index 4222f426..2fb65560 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -23,3 +23,7 @@ db_username.txt
 db_password.txt
 redis_password.txt
 typesense_api_key.txt
+.phpunit.result.cache
+.env
+.env.dist
+docker-compose.yml
diff --git a/Dockerfile b/Dockerfile
index 642d8312..9ec08f33 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
-FROM spiralscout/roadrunner:2.12.2 as roadrunner
-FROM composer:2.5.1 as composer
-FROM mlocati/php-extension-installer:1.5.52 as php-ext-installer
+FROM docker.io/spiralscout/roadrunner:2.12.2 as roadrunner
+FROM docker.io/composer:2.5.1 as composer
+FROM docker.io/mlocati/php-extension-installer:1.5.52 as php-ext-installer
 FROM php:8.1.16-bullseye
 COPY --from=composer /usr/bin/composer /usr/bin/composer
 COPY --from=php-ext-installer /usr/bin/install-php-extensions /usr/local/bin/
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 59f915af..22c9b19c 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -2,20 +2,8 @@
 
 namespace App\Console;
 
-use App\Console\Commands\ClearQueuedJobs;
 use App\Console\Commands\CacheRemove;
-use App\Console\Commands\Indexer\AnimeIndexer;
-use App\Console\Commands\Indexer\AnimeSweepIndexer;
-use App\Console\Commands\Indexer\AnimeScheduleIndexer;
-use App\Console\Commands\Indexer\CommonIndexer;
-use App\Console\Commands\Indexer\CurrentSeasonIndexer;
-use App\Console\Commands\Indexer\GenreIndexer;
-use App\Console\Commands\Indexer\MangaIndexer;
-use App\Console\Commands\Indexer\MangaSweepIndexer;
-use App\Console\Commands\Indexer\ProducersIndexer;
-use App\Console\Commands\ManageMicrocaching;
-use App\Console\Commands\ModifyCacheDriver;
-use App\Console\Commands\ModifyCacheMethod;
+use App\Console\Commands\Indexer;
 use Illuminate\Console\Scheduling\Schedule;
 use Laravel\Lumen\Console\Kernel as ConsoleKernel;
 
@@ -28,15 +16,15 @@ class Kernel extends ConsoleKernel
      */
     protected $commands = [
         CacheRemove::class,
-        CommonIndexer::class,
-        AnimeScheduleIndexer::class,
-        CurrentSeasonIndexer::class,
-        AnimeIndexer::class,
-        MangaIndexer::class,
-        GenreIndexer::class,
-        ProducersIndexer::class,
-        AnimeSweepIndexer::class,
-        MangaSweepIndexer::class,
+        Indexer\CommonIndexer::class,
+        Indexer\AnimeScheduleIndexer::class,
+        Indexer\CurrentSeasonIndexer::class,
+        Indexer\AnimeIndexer::class,
+        Indexer\MangaIndexer::class,
+        Indexer\GenreIndexer::class,
+        Indexer\ProducersIndexer::class,
+        Indexer\AnimeSweepIndexer::class,
+        Indexer\MangaSweepIndexer::class
     ];
 
     /**
diff --git a/container-setup.sh b/container-setup.sh
old mode 100644
new mode 100755
index 17d23767..03f52ca3
--- a/container-setup.sh
+++ b/container-setup.sh
@@ -1,14 +1,22 @@
 #!/bin/bash
 
-JIKAN_API_VERSION=v4.0.0-rc.11
-DOCKER_COMPOSE_PROJECT_NAME=jikan-api-$JIKAN_API_VERSION
+_JIKAN_API_VERSION=v4.0.0
+SUBSTITUTE_VERSION=$_JIKAN_API_VERSION
+if [ -x "$(command -v git)" ]; then
+  SUBSTITUTE_VERSION=$(git describe --tags | sed -e "s/-[a-z0-9]\{8\}/-$(git rev-parse --short HEAD)/g")
+fi
+export _JIKAN_API_VERSION=${JIKAN_API_VERSION:-$SUBSTITUTE_VERSION}
+
+DOCKER_COMPOSE_PROJECT_NAME=jikan-api-$_JIKAN_API_VERSION
+DOCKER_CMD="docker"
+DOCKER_COMPOSE_CMD="docker-compose"
 
 display_help() {
   echo "============================================================"
   echo "Jikan API Container Setup CLI"
   echo "============================================================"
   echo "Syntax: ./container-setup.sh [command]"
-  echo "Jikan API Version: $JIKAN_API_VERSION"
+  echo "Jikan API Version: $_JIKAN_API_VERSION"
   echo "---commands---"
   echo "help                   Print CLI help"
   echo "build-image            Build Image Locally"
@@ -19,27 +27,80 @@ display_help() {
 }
 
 validate_prereqs() {
-   docker -v >/dev/null 2>&1
-   if [ $? -ne 0 ]; then
-      echo -e "'docker' is not installed or not runnable without sudo. \xE2\x9D\x8C"
+  docker_exists=$(command -v docker)
+  docker_compose_exists=$(command -v docker-compose)
+  podman_exists=$(command -v podman)
+  podman_compose_exists=$(command -v podman-compose)
+
+   if [ -x "$docker_exists" ] && [ -x "$podman_exists" ]; then
+      echo -e "'docker' is not installed. \xE2\x9D\x8C"
+      exit 1
    else
       echo -e "Docker is Installed. \xE2\x9C\x94"
    fi
 
-   docker-compose -v >/dev/null 2>&1
-   if [ $? -ne 0 ]; then
-      echo -e "'docker-compose' is not installed. \xE2\x9D\x8C"
-   else
-      echo -e "Docker compose is Installed. \xE2\x9C\x94"
+   if [ -x "$docker_exists" ]; then
+      DOCKER_CMD="docker"
+      docker -v >/dev/null 2>&1
+      if [ $? -ne 0 ]; then
+         echo -e "'docker' is not executable without sudo. \xE2\x9D\x8C"
+         exit 1
+      fi
+   elif [ -n "$podman_exists" ]; then
+      DOCKER_CMD="podman"
    fi
+
+   if [ -x "$docker_compose_exists" ] && [ -x "$docker_compose_exists" ]; then
+       echo -e "'docker-compose' is not installed. \xE2\x9D\x8C"
+       exit 1
+    else
+       echo -e "Docker compose is Installed. \xE2\x9C\x94"
+    fi
+
+    if [ -x "$docker_compose_exists" ]; then
+       DOCKER_COMPOSE_CMD="docker-compose"
+    elif [ -x "$podman_compose_exists" ]; then
+       DOCKER_COMPOSE_CMD="podman-compose"
+    else
+      echo "Error"
+      exit 1
+    fi
 }
 
 build_image() {
-   docker build --rm --compress -t jikanme/jikan-rest:$JIKAN_API_VERSION .
+  validate_prereqs
+  $DOCKER_CMD build --rm --compress -t jikanme/jikan-rest:"$_JIKAN_API_VERSION" .
+  $DOCKER_CMD tag jikanme/jikan-rest:"$_JIKAN_API_VERSION" jikanme/jikan-rest:latest
+}
+
+ensure_secrets() {
+  declare -a secrets=("db_password" "db_username" "redis_password" "typesense_api_key")
+
+  for secret_name in "${secrets[@]}"
+  do
+    if [ ! -f "$secret_name.txt" ]; then
+      if [ "$secret_name" == "db_username" ]; then
+        generated_secret="jikan"
+      else
+        generated_secret=$(LC_ALL=c tr -dc 'A-Za-z0-9!"#$%&'\''()*+,-./:;<=>?@[\]^_{|}~' </dev/urandom | head -c 16  ; echo)
+      fi
+      echo "$secret_name.txt not found, please provide a $secret_name [default is $generated_secret]:"
+      # prompt for secret and save it in file
+      read -r secret_value
+      if [ -z "$secret_value" ]; then
+        secret_value=$generated_secret
+      fi
+      echo "$secret_value" > "$secret_name.txt"
+    else
+      echo -e "$secret_name.txt found, using it's value. \xE2\x9C\x94"
+    fi
+  done
 }
 
 start() {
-   docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME up -d
+  validate_prereqs
+  ensure_secrets
+  exec $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" up -d
 }
 
 case "$1" in
@@ -55,17 +116,21 @@ case "$1" in
    "start")
       start
       ;;
+    "stop")
+      validate_prereqs
+      $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" down
+      ;;
     "execute-indexers")
       echo "Indexing anime..."
-      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:anime
+      $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:anime
       echo "Indexing manga..."
-      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:manga
+      $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:manga
       echo "Indexing characters and people..."
-      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:common
+      $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:common
       echo "Indexing genres..."
-      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:genres
+      $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:genres
       echo "Indexing producers..."
-      docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME exec jikan_rest php /app/artisan indexer:producers
+      $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:producers
       echo "Indexing done!"
       ;;
    *)
diff --git a/docker-compose.yml b/docker-compose.yml
index 8ebe53c9..96a2b9a2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -19,7 +19,7 @@ secrets:
 
 services:
   jikan_rest:
-    image: jikanme/jikan-rest:v4.0.0-rc.11
+    image: "jikanme/jikan-rest:${_JIKAN_API_VERSION:-latest}"
     user: "${APP_UID:-10001}:${APP_GID:-10001}"
     networks:
       - jikan_network
@@ -34,6 +34,7 @@ services:
       - ./docker/config/.env.compose
     ports:
       - '8080:8080/tcp'
+    hostname: jikan-rest-api
     healthcheck:
       test: [ 'CMD-SHELL', 'wget --spider -q "http://127.0.0.1:2114/health?plugin=http"' ]
       interval: 2s
@@ -44,18 +45,20 @@ services:
       typesense: { condition: service_healthy }
 
   mongodb:
-    image: mongo:focal
+    image: docker.io/mongo:focal
+    hostname: mongodb
     volumes:
       - mongo-data:/data/db
     ports:
       - '27017/tcp'
-    command: --wiredTigerCacheSizeGB ${MONGO_CACHE_SIZE_GB:1}
+    command: "--wiredTigerCacheSizeGB ${MONGO_CACHE_SIZE_GB:1}"
     networks:
       - jikan_network
     secrets:
+      - db_username
       - db_password
     environment:
-      MONGO_INITDB_ROOT_USERNAME: "${DB_USERNAME:-root}"
+      MONGO_INITDB_ROOT_USERNAME_FILE: /run/secrets/db_username
       MONGO_INITDB_ROOT_PASSWORD_FILE: /run/secrets/db_password
     healthcheck:
       test: echo 'db.runCommand("ping").ok' | mongo mongodb://localhost:27017 --quiet
@@ -64,7 +67,8 @@ services:
       retries: 5
 
   redis:
-    image: redis:7-alpine
+    image: docker.io/redis:7-alpine
+    hostname: redis
     secrets:
       - redis_password
     environment:
@@ -74,7 +78,7 @@ services:
     command:
       - /bin/sh
       - -c
-      - redis-server --requirepass "$${cat /run/secrets/redis_password}"
+      - redis-server --requirepass "$$(cat /run/secrets/redis_password)"
     volumes:
       - redis-data:/data:rw
     ports:
@@ -85,7 +89,8 @@ services:
       timeout: 1s
 
   typesense:
-    image: typesense/typesense:0.24.1
+    image: docker.io/typesense/typesense:0.24.1
+    hostname: typesense
     entrypoint: /bin/sh
     secrets:
       - typesense_api_key
@@ -100,6 +105,6 @@ services:
     ports:
       - "8108/tcp"
     healthcheck:
-      test: [ 'CMD-SHELL', 'curl', '-s', '-f', 'http://localhost:8108/health' ]
+      test: [ 'CMD-SHELL', '{ ! [ -f "curl_created" ] && apt -qq update -y && apt -qq install -y curl && touch curl_created && curl -s -f http://localhost:8108/health; } || { curl -s -f http://localhost:8108/health; }' ]
       interval: 5s
       timeout: 2s
diff --git a/docker-entrypoint.php b/docker-entrypoint.php
index 97d41be0..c80a0bfe 100644
--- a/docker-entrypoint.php
+++ b/docker-entrypoint.php
@@ -37,6 +37,7 @@
 $envWriter = new \MirazMac\DotEnv\Writer(__DIR__ . '/' . '.env');
 $itemsWritten = 0;
 foreach (array_keys($current_env) as $env_key) {
+    echo $env_key;
     if (!str_contains($env_key, "__FILE")) {
         continue;
     }
@@ -51,6 +52,7 @@
 
 if ($itemsWritten > 0) {
     $envWriter->write();
+    echo "Secrets loaded successfully.";
 }
 
 $dotenv = Dotenv::createImmutable(__DIR__);
diff --git a/storage/app/.gitignore b/storage/app/.gitignore
index ce9fcc1f..11df8699 100644
--- a/storage/app/.gitignore
+++ b/storage/app/.gitignore
@@ -2,3 +2,5 @@
 failovers.json
 source_failover.lock
 jikan_model_classes.json
+container_compose_runtime
+container_runtime

From 7db32987ffaef2f528eb180a830e2d96e1f35828 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Tue, 17 Oct 2023 23:29:51 +0100
Subject: [PATCH 08/14] Update Dockerfile

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 9ec08f33..45f2a139 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,7 +6,7 @@ COPY --from=composer /usr/bin/composer /usr/bin/composer
 COPY --from=php-ext-installer /usr/bin/install-php-extensions /usr/local/bin/
 ENV COMPOSER_HOME="/tmp/composer"
 RUN set -x \
-    && install-php-extensions gd exif intl bz2 gettext mongodb-stable redis opcache sockets pcntl \
+    && install-php-extensions intl bz2 gettext mongodb-stable redis opcache sockets pcntl \
     # install xdebug (for testing with code coverage), but do not enable it
     && IPE_DONT_ENABLE=1 install-php-extensions xdebug-3.2.0
 

From c615d652ac6679962b64431ce2d2a29a0f014a18 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Mon, 23 Oct 2023 22:10:31 +0100
Subject: [PATCH 09/14] fixed dockerignore

---
 .dockerignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index 2fb65560..2e6676cb 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -25,5 +25,4 @@ redis_password.txt
 typesense_api_key.txt
 .phpunit.result.cache
 .env
-.env.dist
 docker-compose.yml

From 201883d9874b582216a07f2e993e032dfd1f1e53 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Tue, 24 Oct 2023 00:41:28 +0100
Subject: [PATCH 10/14] wip fixed issues with container start ups

---
 Dockerfile            |  7 +++----
 container-setup.sh    |  6 ++++--
 docker-compose.yml    | 16 ++++++++++------
 docker-entrypoint.php |  5 +++--
 4 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 45f2a139..8e07b4cd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,12 @@
 FROM docker.io/spiralscout/roadrunner:2.12.2 as roadrunner
 FROM docker.io/composer:2.5.1 as composer
-FROM docker.io/mlocati/php-extension-installer:1.5.52 as php-ext-installer
+FROM docker.io/mlocati/php-extension-installer:2.1.58 as php-ext-installer
 FROM php:8.1.16-bullseye
 COPY --from=composer /usr/bin/composer /usr/bin/composer
 COPY --from=php-ext-installer /usr/bin/install-php-extensions /usr/local/bin/
 ENV COMPOSER_HOME="/tmp/composer"
 RUN set -x \
-    && install-php-extensions intl bz2 gettext mongodb-stable redis opcache sockets pcntl \
+    && install-php-extensions intl mbstring mongodb-stable redis opcache sockets pcntl \
     # install xdebug (for testing with code coverage), but do not enable it
     && IPE_DONT_ENABLE=1 install-php-extensions xdebug-3.2.0
 
@@ -17,9 +17,8 @@ RUN	set -ex \
     && apt-get update && apt-get install -y --no-install-recommends \
 	openssl \
 	git \
-	dos2unix \
+    wget \
 	unzip \
-  wget \
   # install supercronic (for laravel task scheduling), project page: <https://github.com/aptible/supercronic>
 	&& wget -q "https://github.com/aptible/supercronic/releases/download/v0.1.12/supercronic-linux-$(dpkg --print-architecture)" \
 	   -O /usr/bin/supercronic \
diff --git a/container-setup.sh b/container-setup.sh
index 03f52ca3..38140382 100755
--- a/container-setup.sh
+++ b/container-setup.sh
@@ -21,6 +21,7 @@ display_help() {
   echo "help                   Print CLI help"
   echo "build-image            Build Image Locally"
   echo "start                  Start Jikan API (mongodb, typesense, redis, jikan-api workers)"
+  echo "stop                   Stop Jikan API"
   echo "validate-prereqs       Validate pre-reqs installed (docker, docker-compose)"
   echo "execute-indexers       Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days)"
   echo ""
@@ -69,6 +70,7 @@ validate_prereqs() {
 
 build_image() {
   validate_prereqs
+  $DOCKER_CMD inspect jikanme/jikan-rest:"$_JIKAN_API_VERSION" &> /dev/null && $DOCKER_CMD rmi jikanme/jikan-rest:"$_JIKAN_API_VERSION"
   $DOCKER_CMD build --rm --compress -t jikanme/jikan-rest:"$_JIKAN_API_VERSION" .
   $DOCKER_CMD tag jikanme/jikan-rest:"$_JIKAN_API_VERSION" jikanme/jikan-rest:latest
 }
@@ -116,11 +118,11 @@ case "$1" in
    "start")
       start
       ;;
-    "stop")
+   "stop")
       validate_prereqs
       $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" down
       ;;
-    "execute-indexers")
+   "execute-indexers")
       echo "Indexing anime..."
       $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:anime
       echo "Indexing manga..."
diff --git a/docker-compose.yml b/docker-compose.yml
index 96a2b9a2..0eaa7176 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,6 +39,10 @@ services:
       test: [ 'CMD-SHELL', 'wget --spider -q "http://127.0.0.1:2114/health?plugin=http"' ]
       interval: 2s
       timeout: 2s
+    links:
+      - mongodb
+      - redis
+      - typesense
     depends_on:
       mongodb: { condition: service_healthy }
       redis: { condition: service_healthy }
@@ -51,7 +55,7 @@ services:
       - mongo-data:/data/db
     ports:
       - '27017/tcp'
-    command: "--wiredTigerCacheSizeGB ${MONGO_CACHE_SIZE_GB:1}"
+    command: "--wiredTigerCacheSizeGB ${MONGO_CACHE_SIZE_GB:-1.0}"
     networks:
       - jikan_network
     secrets:
@@ -61,18 +65,16 @@ services:
       MONGO_INITDB_ROOT_USERNAME_FILE: /run/secrets/db_username
       MONGO_INITDB_ROOT_PASSWORD_FILE: /run/secrets/db_password
     healthcheck:
-      test: echo 'db.runCommand("ping").ok' | mongo mongodb://localhost:27017 --quiet
+      test: echo 'db.runCommand("ping").ok' | mongosh mongodb://localhost:27017 --quiet
       interval: 30s
       timeout: 10s
       retries: 5
 
   redis:
-    image: docker.io/redis:7-alpine
+    image: docker.io/redis:6-alpine
     hostname: redis
     secrets:
       - redis_password
-    environment:
-      REDIS_PASSWORD: "${REDIS_PASSWORD:-null}"
     networks:
       - jikan_network
     command:
@@ -99,7 +101,9 @@ services:
     command:
       - -c
       - TYPESENSE_API_KEY="$$(cat /run/secrets/typesense_api_key)" /opt/typesense-server --data-dir /data
-    restart: no
+    deploy:
+      restart_policy:
+        condition: none
     volumes:
       - typesense-data:/data
     ports:
diff --git a/docker-entrypoint.php b/docker-entrypoint.php
index c80a0bfe..6d69d077 100644
--- a/docker-entrypoint.php
+++ b/docker-entrypoint.php
@@ -37,7 +37,6 @@
 $envWriter = new \MirazMac\DotEnv\Writer(__DIR__ . '/' . '.env');
 $itemsWritten = 0;
 foreach (array_keys($current_env) as $env_key) {
-    echo $env_key;
     if (!str_contains($env_key, "__FILE")) {
         continue;
     }
@@ -52,12 +51,14 @@
 
 if ($itemsWritten > 0) {
     $envWriter->write();
-    echo "Secrets loaded successfully.";
+    echo "Secrets loaded successfully.\n";
 }
 
 $dotenv = Dotenv::createImmutable(__DIR__);
 $dotenv->load();
 
+$current_env = $_ENV;
+
 if ($current_env["SCOUT_DRIVER"] === "typesense" && empty($current_env["TYPESENSE_API_KEY"])) {
     echo "Please set the TYPESENSE_API_KEY environment variable when setting SCOUT_DRIVER to typesense.";
     exit(1);

From 7b6a0c6d25e2c6b055a2136f6259e97e7652d6ac Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Tue, 24 Oct 2023 23:10:14 +0100
Subject: [PATCH 11/14] fixed mongodb setup

---
 .gitignore            |  2 ++
 container-setup.sh    | 35 +++++++++++++++++++++++++++++++----
 docker-compose.yml    | 20 ++++++++++++++------
 docker-entrypoint.php |  7 ++++++-
 docker/mongo-init.js  | 17 +++++++++++++++++
 5 files changed, 70 insertions(+), 11 deletions(-)
 create mode 100644 docker/mongo-init.js

diff --git a/.gitignore b/.gitignore
index 80bda74b..dfe0c39f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,8 @@ composer.phar
 .DS_Store
 *.cache
 db_username.txt
+db_admin_username.txt
 db_password.txt
+db_admin_password.txt
 redis_password.txt
 typesense_api_key.txt
diff --git a/container-setup.sh b/container-setup.sh
index 38140382..415a25a7 100755
--- a/container-setup.sh
+++ b/container-setup.sh
@@ -3,7 +3,12 @@
 _JIKAN_API_VERSION=v4.0.0
 SUBSTITUTE_VERSION=$_JIKAN_API_VERSION
 if [ -x "$(command -v git)" ]; then
-  SUBSTITUTE_VERSION=$(git describe --tags | sed -e "s/-[a-z0-9]\{8\}/-$(git rev-parse --short HEAD)/g")
+  git symbolic-ref HEAD &> /dev/null
+  if [ $? -ne 0 ]; then
+    SUBSTITUTE_VERSION=$(git describe --tags)
+  else
+    SUBSTITUTE_VERSION=$(git describe --tags | sed -e "s/-[a-z0-9]\{8\}/-$(git rev-parse --short HEAD)/g")
+  fi
 fi
 export _JIKAN_API_VERSION=${JIKAN_API_VERSION:-$SUBSTITUTE_VERSION}
 
@@ -76,7 +81,29 @@ build_image() {
 }
 
 ensure_secrets() {
-  declare -a secrets=("db_password" "db_username" "redis_password" "typesense_api_key")
+  declare -a secrets=("db_password" "db_admin_password" "redis_password" "typesense_api_key")
+
+  if [ ! -f "db_username.txt" ]; then
+    echo "db_username.txt not found, please provide a db_username [default is jikan]:"
+    read -r db_username
+    if [ -z "$db_username" ]; then
+      db_username="jikan"
+    fi
+    echo -n "$db_username" > "db_username.txt"
+  else
+    echo -e "db_username.txt found, using it's value. \xE2\x9C\x94"
+  fi
+
+  if [ ! -f "db_admin_username.txt" ]; then
+    echo "db_admin_username.txt not found, please provide a db_admin_username [default is jikan_admin]:"
+    read -r db_admin_username
+    if [ -z "$db_admin_username" ]; then
+      db_admin_username="jikan_admin"
+    fi
+    echo -n "$db_admin_username" > "db_admin_username.txt"
+  else
+    echo -e "db_admin_username.txt found, using it's value. \xE2\x9C\x94"
+  fi
 
   for secret_name in "${secrets[@]}"
   do
@@ -84,7 +111,7 @@ ensure_secrets() {
       if [ "$secret_name" == "db_username" ]; then
         generated_secret="jikan"
       else
-        generated_secret=$(LC_ALL=c tr -dc 'A-Za-z0-9!"#$%&'\''()*+,-./:;<=>?@[\]^_{|}~' </dev/urandom | head -c 16  ; echo)
+        generated_secret=$(LC_ALL=c tr -dc 'A-Za-z0-9!'\''()*+,-;<=>_' </dev/urandom | head -c 16  ; echo)
       fi
       echo "$secret_name.txt not found, please provide a $secret_name [default is $generated_secret]:"
       # prompt for secret and save it in file
@@ -92,7 +119,7 @@ ensure_secrets() {
       if [ -z "$secret_value" ]; then
         secret_value=$generated_secret
       fi
-      echo "$secret_value" > "$secret_name.txt"
+      echo -n "$secret_value" > "$secret_name.txt"
     else
       echo -e "$secret_name.txt found, using it's value. \xE2\x9C\x94"
     fi
diff --git a/docker-compose.yml b/docker-compose.yml
index 0eaa7176..e1c0843d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -12,6 +12,10 @@ secrets:
     file: db_username.txt
   db_password:
     file: db_password.txt
+  db_admin_username:
+    file: db_admin_username.txt
+  db_admin_password:
+    file: db_admin_password.txt
   redis_password:
     file: redis_password.txt
   typesense_api_key:
@@ -40,9 +44,9 @@ services:
       interval: 2s
       timeout: 2s
     links:
-      - mongodb
-      - redis
-      - typesense
+      - mongodb:mongodb
+      - redis:redis
+      - typesense:typesense
     depends_on:
       mongodb: { condition: service_healthy }
       redis: { condition: service_healthy }
@@ -53,17 +57,21 @@ services:
     hostname: mongodb
     volumes:
       - mongo-data:/data/db
+      - ./docker/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro
     ports:
-      - '27017/tcp'
+      - 27017/tcp
     command: "--wiredTigerCacheSizeGB ${MONGO_CACHE_SIZE_GB:-1.0}"
     networks:
       - jikan_network
     secrets:
       - db_username
       - db_password
+      - db_admin_username
+      - db_admin_password
     environment:
-      MONGO_INITDB_ROOT_USERNAME_FILE: /run/secrets/db_username
-      MONGO_INITDB_ROOT_PASSWORD_FILE: /run/secrets/db_password
+      MONGO_INITDB_ROOT_USERNAME_FILE: /run/secrets/db_admin_username
+      MONGO_INITDB_ROOT_PASSWORD_FILE: /run/secrets/db_admin_password
+      MONGO_INITDB_DATABASE: jikan_admin
     healthcheck:
       test: echo 'db.runCommand("ping").ok' | mongosh mongodb://localhost:27017 --quiet
       interval: 30s
diff --git a/docker-entrypoint.php b/docker-entrypoint.php
index 6d69d077..8fd3fe74 100644
--- a/docker-entrypoint.php
+++ b/docker-entrypoint.php
@@ -19,6 +19,8 @@
     "DB_PASSWORD" => ""
 ];
 
+// get a copy of the current env vars.
+// these are the ones that are set during the container creation
 $current_env = $_ENV;
 
 if (!file_exists(".env")) {
@@ -34,6 +36,8 @@
 // We'd like to support Container secrets. So we'll check if any of the env vars has a __FILE suffix
 // then we'll try to load the file and set the env var to the contents of the file.
 // https://docs.docker.com/engine/swarm/secrets/
+// Additionally we need to write the secrets to the .env file so the workers in roadrunner can access them.
+// (it might just pass down the global env vars, but haven't tested that yet)
 $envWriter = new \MirazMac\DotEnv\Writer(__DIR__ . '/' . '.env');
 $itemsWritten = 0;
 foreach (array_keys($current_env) as $env_key) {
@@ -45,7 +49,8 @@
         continue;
     }
     $originalKey = str_replace("__FILE", "", $env_key);
-    $envWriter->set($originalKey, file_get_contents($current_env[$env_key]));
+    $secretsFileContents = file_get_contents($current_env[$env_key]);
+    $envWriter->set($originalKey, str_replace(["\n", "\r"], "", $secretsFileContents));
     $itemsWritten++;
 }
 
diff --git a/docker/mongo-init.js b/docker/mongo-init.js
new file mode 100644
index 00000000..01274f0e
--- /dev/null
+++ b/docker/mongo-init.js
@@ -0,0 +1,17 @@
+const userToCreate = fs.readFileSync('/run/secrets/db_username', 'utf8');
+const userPassword = fs.readFileSync('/run/secrets/db_password', 'utf8');
+db = db.getSiblingDB("admin");
+
+db.createUser({
+    user: userToCreate,
+    pwd: userPassword,
+    roles: [{ role: "readWrite", db: "jikan" }],
+});
+
+db = db.getSiblingDB("jikan");
+
+db.createUser({
+    user: userToCreate,
+    pwd: userPassword,
+    roles: [{ role: "readWrite", db: "jikan" }],
+});

From 89ff3b2d606b745e51962ad76d8c47d488a17c12 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Wed, 25 Oct 2023 19:11:14 +0100
Subject: [PATCH 12/14] improved docs around container usage

---
 container_usage.md | 102 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 86 insertions(+), 16 deletions(-)

diff --git a/container_usage.md b/container_usage.md
index 30f4bb4d..bce6bd74 100644
--- a/container_usage.md
+++ b/container_usage.md
@@ -1,5 +1,79 @@
 # 🐳 Running Jikan API in a container
 
+The most easiest way to get started is to use our container setup cli script after checking out the repo with git (linux only):
+
+```bash
+./container-setup.sh start
+```
+
+This will:
+
+- Prompt you for the required passwords and usernames
+- Sets up a production ready setup with `redis`, `typesense` and `mongodb` (almost same as the public api at `api.jikan.moe`)
+- Sets mongodb to use max 1gb of memory
+- Configures jikan-api to add CORS headers to responses.
+
+> **Note**: The script supports both `docker` and `podman`. In case of `podman` please bare in mind that sometimes the container name resolution doesn't work on the container network. 
+> In those cases you might have to install `aardvark-dns` package. On `Arch Linux` podman uses `netavark` network by default (in 2023) so you will need to install the before mentioned package.
+
+The script has the following prerequisites and will notify you if these are not present:
+
+- git
+- `docker` or `podman`
+- `docker-compose` or `podman-compose`
+
+### Available commands in the cli script
+
+```
+============================================================
+Jikan API Container Setup CLI
+============================================================
+Syntax: ./container-setup.sh [command]
+---commands---
+help                   Print CLI help
+build-image            Build Image Locally
+start                  Start Jikan API (mongodb, typesense, redis, jikan-api workers)
+stop                   Stop Jikan API
+validate-prereqs       Validate pre-reqs installed (docker, docker-compose)
+execute-indexers       Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days)
+```
+
+### Running the indexer with the script
+
+When you first startup the app you will have an empty database. To fill it up you can execute the following command:
+
+```bash
+./container-setup.sh execute-indexers
+```
+
+Please note that this command can take 4-5 days to run. You can run it in the background with the `&` marker:
+
+```bash
+./container-setup.sh execute-indexers &
+```
+
+If interrupted then you will have to manually resume the indexing, otherwise the above command will just start again from the beginning.
+
+### Updating to a newer version
+
+You need to stop the app first:
+
+```bash
+./container-setup.sh stop
+```
+
+Then remove the jikan-api image from your local storage and pull the new one. Set the `JIKAN_API_VERSION` environment variable to the latest image tag. This can be either `latest` or the version `v4.0.0-11`.
+
+```bash
+JIKAN_API_VERSION=latest ./container-setup.sh start
+```
+
+## More customised setups
+
+Some of you might only want to run the `jikan-rest` app with only mongodb, without the more sophisticated search functionality. In those cases we don't have a `docker-compose` config for you. You need to start the `jikan-rest` container with atleast a `mongodb` instance.
+The `jikan-rest` container will require a `.env` file mounted where you configure the credentials for `mongodb`.
+
+
 ```bash
 docker run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env jikanme/jikan-rest:latest
 ```
@@ -14,8 +88,7 @@ docker run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env jikanme/jikan-r
 > address of these services.
 
 > **Tip**: If you run the container on a non-default network, you can use the container names in the configuration to
-> specify the address of services like MongoDB and TypeSense. However, this is not a concern if you
-> use `docker-compose`.
+> specify the address of services like MongoDB and TypeSense.
 
 There is also a `Dockerfile` in the repo which you can use to build the container image and startup the app in a
 container:
@@ -25,24 +98,24 @@ docker build -t jikan-rest:nightly .
 docker run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env jikan-rest:nightly
 ```
 
+> Most of the time it's enough to just use the image from [Docker Hub](https://hub.docker.com/r/jikanme/jikan-rest).
+
 ### Docker compose usage
 
 ```
 docker-compose up
 ```
 
-Docker compose will use the `.env` file from the folder where you execute it from to load configurations for the
-services. If you don't have a `.env` file yet in the folder, copy the `.env.dist` file, and set the passwords.
+This does the same thing as the `container-setup.sh` script mostly, but you will have to create the secret files yourself. The following secret files are required for credentials (put them next to the `docker-compose.yml` file):
 
-Docker compose will start up a production ready setup with redis, typesense and mongodb.
-Change the environment variables for the `jikan_rest` service in the `docker-compose.yml` file to customise the configuration of the Jikan API.
-By default docker-compose sets it to:
-- Not use [queuing](https://laravel.com/docs/9.x/scout#queueing) for search requests. (Not yet supported for  typesense.)
-- Use typesense search for more accurate search results
-- Use redis for microcaching and caching.
-- Add CORS headers to responses
+- db_admin_password.txt
+- db_admin_username.txt
+- db_password.txt
+- db_username.txt
+- redis_password.txt
+- typesense_api_key.txt
 
-Of course these defaults can be overridden through the `.env` file or through setting environment variables in the command-line before  executing `docker-compose up`.
+You can customise the Jikan API config through `./docker/config/.env.compose` file. (E.g. you don't want CORS headers)
 
 > **Please note**: The syntax rules of docker compose for `.env` applies
 > here: https://docs.docker.com/compose/env-file/#syntax-rules
@@ -61,7 +134,7 @@ podman run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env --health-start-
 
 ### Configuration of the container
 
-You can change the settings of Jikan through setting environment variables via the `-e` command line argument option for
+You can also change the settings of Jikan through setting environment variables via the `-e` command line argument option for
 the `docker run` command.
 These environment variables are the same as the options found in the `.env` file. We also provide a sample file
 called `.env.dist`.      
@@ -106,6 +179,3 @@ the [Configuration Wiki page](https://github.com/jikan-me/jikan-rest/wiki/Config
     - the php processes ingesting the http requests
     - [Supercronic](https://github.com/aptible/supercronic), which runs cron jobs.
     - Queue workers for populating the search index and other background jobs.
-- The container does all the scheduled background jobs automatically out-of-the-box:
-  - Importing new documents into the search index.
-  - Updating entries from upstream.

From eecbb06d00a9aafef4570abfcb683a9c38777d23 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Wed, 25 Oct 2023 19:11:39 +0100
Subject: [PATCH 13/14] improved container setup cli

---
 container-setup.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/container-setup.sh b/container-setup.sh
index 415a25a7..49800107 100755
--- a/container-setup.sh
+++ b/container-setup.sh
@@ -3,16 +3,21 @@
 _JIKAN_API_VERSION=v4.0.0
 SUBSTITUTE_VERSION=$_JIKAN_API_VERSION
 if [ -x "$(command -v git)" ]; then
+  # check if we have checked out a tag or not
   git symbolic-ref HEAD &> /dev/null
   if [ $? -ne 0 ]; then
+    # if a tag is checked out then use the tag name as the version
     SUBSTITUTE_VERSION=$(git describe --tags)
   else
+    # this is used when building locally
     SUBSTITUTE_VERSION=$(git describe --tags | sed -e "s/-[a-z0-9]\{8\}/-$(git rev-parse --short HEAD)/g")
   fi
 fi
+# set JIKAN_API_VERSION env var to "latest" or a tag which exists in the container registry to use the remote image
+# otherwise docker-compose will look for a locally builded image
 export _JIKAN_API_VERSION=${JIKAN_API_VERSION:-$SUBSTITUTE_VERSION}
 
-DOCKER_COMPOSE_PROJECT_NAME=jikan-api-$_JIKAN_API_VERSION
+DOCKER_COMPOSE_PROJECT_NAME=jikan-api
 DOCKER_CMD="docker"
 DOCKER_COMPOSE_CMD="docker-compose"
 
@@ -127,6 +132,7 @@ ensure_secrets() {
 }
 
 start() {
+   # todo: create a marker file for initial startup, and on initial startup ask the user whether they want a local image or the remote one
   validate_prereqs
   ensure_secrets
   exec $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" up -d

From 048727c16c8f023a3f08b7f7b1f6e07799e9a837 Mon Sep 17 00:00:00 2001
From: pushrbx <contact@pushrbx.net>
Date: Thu, 26 Oct 2023 10:40:10 +0100
Subject: [PATCH 14/14] added some improvements to the github action workflow

---
 .github/workflows/container-image-release.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/container-image-release.yml b/.github/workflows/container-image-release.yml
index 7e78ff6b..41c20a53 100644
--- a/.github/workflows/container-image-release.yml
+++ b/.github/workflows/container-image-release.yml
@@ -13,7 +13,7 @@ on:
       image_tag:
         description: 'Image tag'
         required: true
-        default: 'nightly-<put_date_here>'
+        default: 'v4.0.0-nightly-<put_date_here>'
 
 env:
   REGISTRY_IMAGE: ghcr.io/jikan-me/jikan-rest
@@ -50,6 +50,8 @@ jobs:
             org.opencontainers.image.description=REST API for Jikan
             org.opencontainers.image.url=https://jikan.moe
             org.opencontainers.image.source=https://github.com/jikan-me/jikan-rest
+            org.opencontainers.image.documentation=https://github.com/jikan-me/jikan-rest/blob/master/container_usage.md
+            org.opencontainers.image.revision=${{ github.sha }}
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -107,7 +109,8 @@ jobs:
             ${{ env.REGISTRY_IMAGE }}
             jikanme/jikan-rest
           tags: |
-            type=raw,value=${{ inputs.image_tag || github.ref_name }}
+            type=raw,value=${{ inputs.image_tag }}
+            type=raw,value=${{ github.ref_type == "tag" && 'latest' || 'latest-nightly' }}
             type=sha
 
       - name: Login to GitHub Container Registry