feat(alert): add wrapper on cron command to send alert on mattermost …

…in case of failure

api/Dockerfile

@@ -62,10 +62,13 @@ RUN useradd --no-log-init -g gunicorn gunicorn
 # Copy venv with compiled dependencies
 COPY --chown=gunicorn:gunicorn --from=compile-image /srv/venv /srv/venv
 
-COPY --chown=gunicorn:gunicorn ["docker-entrypoint.sh", "pyproject.toml", "alembic.ini", "/srv/"]
+COPY --chown=gunicorn:gunicorn ["execute_and_notify.sh", "docker-entrypoint.sh", "pyproject.toml", "alembic.ini", "/srv/"]
 COPY --chown=gunicorn:gunicorn src /srv/src
 COPY --chown=gunicorn:gunicorn tests /srv/tests
+
+
 RUN chmod +x docker-entrypoint.sh
+RUN chmod +x execute_and_notify.sh
 
 USER gunicorn
 EXPOSE 8000

api/cron.json

@@ -1,15 +1,15 @@
 {
     "jobs": [
         {
-            "command": "30 * * * * TQDM_DISABLE=1 test $ENV = 'prod' && data-inclusion-api load_inclusion_data",
+            "command": "30 * * * * TQDM_DISABLE=1 test $ENV = 'prod' && ./execute_and_notify.sh data-inclusion-api load_inclusion_data",
             "size": "XL"
         },
         {
             "command": "30 * * * * TQDM_DISABLE=1 test $ENV != 'prod' && data-inclusion-api load_inclusion_data",
             "size": "S"
         },
         {
-            "command": "0 4 * * * vacuumdb --full --analyze --verbose --table api__structures --table api__services $DATABASE_URL",
+            "command": "0 4 * * * ./execute_and_notify.sh vacuumdb --full --analyze --verbose --table api__structures --table api__services $DATABASE_URL",
             "size": "S"
         }
     ]

api/execute_and_notify.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -u
+
+"$@" &
+command_pid=$! 
+wait $command_pid
+
+exit_code=$?  
+if [ $exit_code -ne 0 ] && [ "$ENV" = "prod" ]; then
+    echo "Error with exit code \`$exit_code\` in command \`$*\`, sending message to Mattermost"
+    curl -i -X POST -H "Content-Type: application/json" -d "{\"text\": \"Error with exit code \`$exit_code\` in command \`$*\`:  \nLogs: \`scalingo --region osc-secnum-fr1 --app data-inclusion-api-prod logs --lines 1000 -F $CONTAINER -f\`\"}" $MATTERMOST_HOOK
+fi
+
+exit $exit_code