Remote archive compression

archive_compression/Dockerfile

@@ -1,10 +1,11 @@
-FROM python:3.9-alpine3.16
+FROM python:3.9
 
-COPY . .
+COPY archive_compression.py archive_compression.py
+COPY crontab /etc/cron.d/crontab
 
-RUN apt-get -y update && apt-get -y install ffmpeg
-RUN pip install ffmpeg-python
+RUN apt-get update -y && apt-get install -y cron ffmpeg
+RUN pip install numpy requests python-dotenv ffmpeg-python
 
-RUN crontab crontab
+RUN crontab /etc/cron.d/crontab
 
 CMD ["crond", "-f"]

archive_compression/archive_compression.py

@@ -50,6 +50,7 @@ def check_ftp_directory(ftp, path):
 
         for yr in years:
             year_dir = os.path.join(st_dir, yr)
+
             check_ftp_directory(ftp, year_dir)
 
             # Months folders
@@ -58,11 +59,12 @@ def check_ftp_directory(ftp, path):
 
             if len(months) == 0:
                 print(f'{source_year} empty')
-                # ftp.rmd(source_year)
+                ftp.rmd(source_year)
 
             else:
                 for mth in months:
                     month_dir = os.path.join(year_dir, mth)
+
                     check_ftp_directory(ftp, month_dir)
 
                     # Days folders
@@ -71,11 +73,12 @@ def check_ftp_directory(ftp, path):
 
                     if len(days) == 0:
                         print(f'{source_month} empty')
-                        # ftp.rmd(source_month)
+                        ftp.rmd(source_month)
 
                     else:
                         for dy in days:
-                            day_dir = os.path.join(year_dir, dy)
+                            day_dir = os.path.join(month_dir, dy)
+
                             check_ftp_directory(ftp, day_dir)
 
                             source_day = os.path.join(source_month, dy)
@@ -84,12 +87,13 @@ def check_ftp_directory(ftp, path):
 
                             if len(day_clips) == 0:
                                 print(f'{source_day} empty')
-                                # ftp.rmd(source_day)
+                                ftp.rmd(source_day)
 
                             else:
                                 clips.append(day_clips)
 
-    clips = list(np.concatenate(clips). flat)           
+    clips = list(np.concatenate(clips). flat)
+    clips.sort()
 
 
     for ftp_clip in clips:
@@ -115,21 +119,22 @@ def check_ftp_directory(ftp, path):
         stream = ffmpeg.input(local_source_path).output(local_output_path, vcodec='libx264', crf=23)
 
         try:
-            ffmpeg.run(stream)
+            ffmpeg.run(stream, quiet=True)
         except:
             print(f'!! Invalid data found in {ftp_clip} !!')
             continue
 
         # Upload encoded video
-        print('uploading encoded video...')
+        print(f'uploading encoded video to {output_path}...')
         with open(local_output_path, 'rb') as f:
             ftp.storbinary('STOR ' + output_path, f)
 
-        # Remove local files
         print('removing files...')
-        os.remove(local_source_path)
-        os.remove(local_output_path)
-
         # Remove distant source file
-        #TODO: Test if output file exists before removing
+        #TODO: check if files are the same before deleting
         # ftp.delete(ftp_clip)
+
+        # Remove local files
+        os.remove(local_source_path)
+        os.remove(local_output_path)
+

archive_compression/crontab

@@ -1 +1 @@
-0 2 * * * python /archive_compression.py >> /logfile
+0 2 * * * python /archive_compression.py >> /logfile

docker-compose.aux.yaml

@@ -7,4 +7,4 @@ services:
     environment:
       TZ: $SCHEDULER_TIMEZONE
     volumes:
-      - ./.env:./.env
+      - ./.env:/.env

prometheus/stations_alerts.yml

@@ -17,5 +17,5 @@ groups:
       app: woodcam-rm
       severity: warning
     annotations:
-      description: 'Station {{ $labels.common_name }} - data usage exceeds 90% of monthly plan'
+      description: 'Station {{ $labels.common_name }} - data usage exceeds 95% of monthly plan'
       summary: 'The SNMP gateway reports a usage exceeding 95% of the monthly data plan over the last 31 rolling days. The recording will probably stop soon and a data recovery must be planned.'

woodcamrm/cli.py

@@ -48,7 +48,7 @@ def prometheus_update():
             'receiver': 'woodcamrm-users',
             'routes': [
                     {
-                        'match': {'alertname': 'ConnectionUnstable'},
+                        'match': {'app': 'woodcam-rm'},
                         'mute_time_intervals': ['gateways-reboot']
                     }
                 ]