From 1748f100b3fc04691045e4fa647d26754adea903 Mon Sep 17 00:00:00 2001
From: Hugo Lecuyer <hugo.lecuyer1@gmail.com>
Date: Wed, 28 Aug 2024 17:30:04 +0200
Subject: [PATCH] feat(alert): add wrapper on cron command to send alert on
 mattermost in case of failure

---
 api/Dockerfile            |  5 ++++-
 api/cron.json             |  4 ++--
 api/execute_and_notify.sh | 14 ++++++++++++++
 3 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100755 api/execute_and_notify.sh

diff --git a/api/Dockerfile b/api/Dockerfile
index 57ce329c..72f7fe8e 100644
--- a/api/Dockerfile
+++ b/api/Dockerfile
@@ -62,10 +62,13 @@ RUN useradd --no-log-init -g gunicorn gunicorn
 # Copy venv with compiled dependencies
 COPY --chown=gunicorn:gunicorn --from=compile-image /srv/venv /srv/venv
 
-COPY --chown=gunicorn:gunicorn ["docker-entrypoint.sh", "pyproject.toml", "alembic.ini", "/srv/"]
+COPY --chown=gunicorn:gunicorn ["execute_and_notify.sh", "docker-entrypoint.sh", "pyproject.toml", "alembic.ini", "/srv/"]
 COPY --chown=gunicorn:gunicorn src /srv/src
 COPY --chown=gunicorn:gunicorn tests /srv/tests
+
+
 RUN chmod +x docker-entrypoint.sh
+RUN chmod +x execute_and_notify.sh
 
 USER gunicorn
 EXPOSE 8000
diff --git a/api/cron.json b/api/cron.json
index b38e442c..f6ebc582 100644
--- a/api/cron.json
+++ b/api/cron.json
@@ -1,7 +1,7 @@
 {
     "jobs": [
         {
-            "command": "30 * * * * TQDM_DISABLE=1 test $ENV = 'prod' && data-inclusion-api load_inclusion_data",
+            "command": "30 * * * * TQDM_DISABLE=1 test $ENV = 'prod' && ./execute_and_notify.sh data-inclusion-api load_inclusion_data",
             "size": "XL"
         },
         {
@@ -9,7 +9,7 @@
             "size": "S"
         },
         {
-            "command": "0 4 * * * vacuumdb --full --analyze --verbose --table api__structures --table api__services $DATABASE_URL",
+            "command": "0 4 * * * ./execute_and_notify.sh vacuumdb --full --analyze --verbose --table api__structures --table api__services $DATABASE_URL",
             "size": "S"
         }
     ]
diff --git a/api/execute_and_notify.sh b/api/execute_and_notify.sh
new file mode 100755
index 00000000..fc683448
--- /dev/null
+++ b/api/execute_and_notify.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+set -u
+
+"$@" &
+command_pid=$! 
+wait $command_pid
+
+exit_code=$?  
+if [ $exit_code -ne 0 ] && [ "$ENV" = "prod" ]; then
+    echo "Error with exit code \`$exit_code\` in command \`$*\`, sending message to Mattermost"
+    curl -i -X POST -H "Content-Type: application/json" -d "{\"text\": \"Error with exit code \`$exit_code\` in command \`$*\`:  \nLogs: \`scalingo --region osc-secnum-fr1 --app data-inclusion-api-prod logs --lines 1000 -F $CONTAINER -f\`\"}" $MATTERMOST_HOOK
+fi
+
+exit $exit_code