Merge branch 'release-v0.4.1'

CHANGELOG

@@ -1,4 +1,8 @@
 
+ ** v0.4.1   08.01.2020
+    - Archive old vehicle locations
+
+
  ** v0.4.0   23.12.2019
     - Improve plotting data gaps
     - Remove old media files

MANAGEMENT/0-crontab

@@ -4,13 +4,21 @@ PYTHONIOENCODING=UTF-8
 DJANGO_PYTHON_PATH=...
 DJANGO_MANAGE_PATH=...
 MEDIA_DIR=...
+DB_ARCHIVE_DIR=...
+MPK_URL=...
 
 
 # Get vehicle locations
 * * * * * $DJANGO_PYTHON_PATH $DJANGO_MANAGE_PATH get_locations
 * * * * * sleep 20 && $DJANGO_PYTHON_PATH $DJANGO_MANAGE_PATH get_locations
 * * * * * sleep 40 && $DJANGO_PYTHON_PATH $DJANGO_MANAGE_PATH get_locations
 
+# Archive old vehicle locations
+10 2 * * * $DJANGO_PYTHON_PATH $DJANGO_MANAGE_PATH archive_old_locations --keep-days 60 --out-dir $DB_ARCHIVE_DIR
+
 # Clean old media
 10 1 * * * find $MEDIA_DIR -type d -maxdepth 1 -mtime +14 | xargs -L100 rm -r
 
+# Keep alive
+5 7,19 * * * curl -s $MPK_URL > /dev/null
+

mpk/apps/vehicle_locations/management/commands/archive_old_locations.py

@@ -0,0 +1,65 @@
+"""
+Dumps records older than keep-days to a file and deletes these records from the database.
+The data are dumped to daily files. The file for date d contains records from [d, d+1day) in *local* timezone.
+"""
+import gzip
+import os
+from datetime import datetime, timedelta
+
+import pytz
+from django.conf import settings
+from django.core.management import BaseCommand
+from django.db import connection
+
+from vehicle_locations.models import VehicleLocation
+
+LOCATION_TABLE = VehicleLocation._meta.db_table
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument('-d', '--keep-days', dest='keep_days', type=int, help='Number of days of data to keep', required=True)
+        parser.add_argument('-o', '--out-dir', dest='out_dir', help='Out dir', required=True)
+
+    def handle(self, *args, **kwargs):
+        # Parse arguments
+        keep_days = kwargs['keep_days']
+        out_dir = kwargs['out_dir']
+
+        if keep_days <= 0:
+            raise ValueError('keep-days must be positive')
+
+        # Current and oldest date to keep
+        current_date = datetime.now().date()
+        last_date = current_date - timedelta(days=keep_days)
+
+        # Earliest record
+        earliest_record = VehicleLocation.objects \
+                .order_by('date') \
+                .first()
+        if earliest_record is None:
+            return
+
+        this_date = earliest_record.date.astimezone(settings.LOCAL_TIMEZONE).date()
+        while this_date < last_date:
+            next_date = this_date + timedelta(days=1)
+            next_date_dt = settings.LOCAL_TIMEZONE.localize(datetime.combine(next_date, datetime.min.time()))
+            next_date_utc_dt = next_date_dt.astimezone(pytz.utc)
+
+            # Archive records
+            query = 'copy (select * from {} where date < \'{}\' order by date) to stdout'.format(LOCATION_TABLE, next_date_utc_dt.strftime('%Y-%m-%d %H:%M+00:00'))
+            out_filename = '{}/loc-{}.dump.gz'.format(out_dir, this_date.strftime('%Y%m%d'))
+
+            cursor = connection.cursor()
+            cursor.copy_expert(query, gzip.open(out_filename, 'w'))
+
+            # Make sure that the file has been created
+            if not os.path.isfile(out_filename):
+                raise RuntimeError('Dump for {} failed; {} wasn\'t been created'.format(this_date, out_filename))
+
+            # Delete records
+            VehicleLocation.objects.filter(date__lt=next_date_dt).order_by().delete()
+
+            this_date = next_date
+

mpk/script/create_plot/create_plot.py

@@ -178,7 +178,7 @@ def create_plot(line_no, date_from_local, date_to_local, out_filename):
         earliest_data = route.vehiclelocation_set \
                 .order_by('date') \
                 .first()
-        if earliest_data is None or date_from_local < earliest_data.date:
+        if earliest_data is None or date_to_local < earliest_data.date:
             is_request_too_early = True
 
     # Vehicle directions
@@ -233,7 +233,7 @@ def create_plot(line_no, date_from_local, date_to_local, out_filename):
         if not earliest_data:
             no_data_msg = 'No data collected so far for line {}'.format(line_no)
         else:
-            no_data_msg = 'The earliest data available for line {}\nis at {}'.format(line_no, earliest_data.date.astimezone(timezone_local).strftime('%Y-%m-%d %H:%M:%S'))
+            no_data_msg = 'The earliest data available for line {}\nis at {}'.format(line_no, earliest_data.date.astimezone(timezone_local).strftime('%Y-%m-%d %H:%M'))
         plt.text(.5, .5, no_data_msg, fontsize=params.no_data_fontsize, ha='center', va='center', transform=canvas_h.transAxes)
 
     # Title