build docker image with cleanup and add basic test

.github/workflows/test_docker.yml

@@ -0,0 +1,32 @@
+name: test-with-docker-compose
+
+on: [ push, pull_request ]
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        name: setup Python
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          pip3 install -r requirements.txt
+        
+      - name: Docker Compose up -d --build
+        working-directory: docker_compose_test
+        run:
+          docker-compose.yml up -d  --build
+
+      - name: Check status code /list
+        run: |
+          status_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/list)
+          if [[ $status_code -ne 200 ]]; then
+            echo "Request failed with status code $status_code"
+            exit 1
+          fi

.github/workflows/update_ghcr.yml

@@ -0,0 +1,66 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: publish image to ghcr.io
+
+on:
+  push:
+    branches:
+      - main
+  release:
+    types: [published]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=pep440,pattern={{version}}
+
+      - name: Build and push
+        uses: docker/[email protected]
+        with:
+          context: ./
+          file: ./Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/arm64, linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -0,0 +1,52 @@
+# Use an official Python runtime as a parent image
+FROM python:3.9-slim
+
+EXPOSE 5000
+
+# define config variables
+ENV DOWNLOAD_DIR /app/downloads
+ENV RETENTION_PERIOD_HOURS 24
+ENV BROKER_URL "globalbroker.meteo.fr"
+ENV BROKER_PORT 443
+ENV BROKER_USERNAME "everyone"
+ENV BROKER_PASSWORD "everyone"
+ENV BROKER_PROTOCOL "websockets"
+ENV FLASK_HOST "0.0.0.0"
+ENV FLASK_PORT 5000
+ENV DOWNLOAD_WORKERS 8
+ENV SAVE_LOGS false
+ENV LOGS_DIR /app/logs
+
+# update pyopenssl and pin requests and urllib3 to avoid SSL error
+RUN pip install pyopenssl --upgrade && pip install requests==2.26.0 urllib3==1.26.0
+# install cron and envsubst
+RUN apt-get update && apt-get install -y cron gettext-base
+
+# copy all
+COPY . /app
+
+# install the latest version of the package
+RUN pip install wis2downloader
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the clean-script to the Docker image
+COPY ./docker/clean.py /app/clean.py
+
+# add wis2box.cron to crontab
+COPY ./docker/clean.cron /etc/cron.d/clean.cron
+
+# set permissions for the cron job and install it
+RUN chmod 0644 /etc/cron.d/clean.cron && crontab /etc/cron.d/clean.cron
+
+# Copy the entrypoint script to the Docker image
+COPY ./docker/entrypoint.sh /entrypoint.sh
+
+# set permissions for the entrypoint script
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]
+
+# Run wis2-downloader when the container launches
+CMD ["wis2downloader","--config","/app/config.json"]

config_template.json

@@ -0,0 +1,15 @@
+{
+    "broker_url": "${BROKER_URL}",
+    "broker_port": $BROKER_PORT,
+    "username": "${BROKER_USERNAME}",
+    "password":  "${BROKER_PASSWORD}",
+    "protocol": "${BROKER_PROTOCOL}",
+    "topics": {},
+    "download_dir": "${DOWNLOAD_DIR}",
+    "retention_period_hours": "${RETENTION_PERIOD_HOURS}",
+    "flask_host": "${FLASK_HOST}",
+    "flask_port": "${FLASK_PORT}",
+    "download_workers": $DOWNLOAD_WORKERS,
+    "save_logs": $SAVE_LOGS,
+    "log_dir": "${LOG_DIR}"
+}

docker/clean.cron

@@ -0,0 +1 @@
+*/2 * * * * /usr/local/bin/python /app/clean.py > /proc/1/fd/1 2>/proc/1/fd/2

docker/clean.py

@@ -0,0 +1,37 @@
+import os
+import time
+
+# get download dir from environment variable
+download_dir = os.environ.get('DOWNLOAD_DIR', '/app/downloads')
+# get retention period from environment variable
+retention_period_hours = int(os.environ.get('RETENTION_PERIOD_HOURS', 24))
+
+def clean_directory(directory):
+    # get the current time
+    current_time = time.time()
+
+    files_removed = 0
+    directories_removed = 0
+    # loop through the files in the directory, including subdirectories
+    for file in os.listdir(directory):
+        # get the full path of the file
+        file_path = os.path.join(directory, file)
+        # check if the path is a file or a directory
+        if os.path.isfile(file_path):
+            # get the time the file was last modified
+            file_time = os.path.getmtime(file_path)
+            # check if the file is older than the retention period
+            if current_time - file_time > retention_period_hours * 3600:
+                os.remove(file_path)
+                files_removed += 1
+        elif os.path.isdir(file_path):
+            # recursively clean the directory
+            clean_directory(file_path)
+            # if the directory is empty, remove it
+            if not os.listdir(file_path):
+                os.rmdir(file_path)
+                directories_removed += 1
+    print(f'CLEANER: removed {files_removed} old files and {directories_removed} empty directories')
+
+# start cleaning from the download directory
+clean_directory(download_dir)

docker/entrypoint.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+echo "START /entrypoint.sh"
+
+set -e
+
+# ensure cron is running
+service cron start
+service cron status
+
+echo "END /entrypoint.sh"
+
+# print the download_dir
+echo "Download directory in container: $DOWNLOAD_DIR"
+# print the retention period hours
+echo "Retention period in hours: $RETENTION_PERIOD_HOURS"
+
+# ensure DOWNLOAD_DIR exists
+if [ ! -d $DOWNLOAD_DIR ]; then
+    echo "Creating download directory: $DOWNLOAD_DIR"
+    mkdir -p $DOWNLOAD_DIR
+fi
+envsubst < config_template.json > config.json
+
+# print the config
+echo "Config:"
+cat /app/config.json
+
+exec "$@"

docker_compose_test/docker-compose.yml

@@ -0,0 +1,9 @@
+services:
+  wis2-downloader:
+    image: wis2-downloader
+    build:
+      context: ../
+      dockerfile: Dockerfile
+    container_name: wis2-downloader
+    ports: 
+      - "5000:5000"