diff --git a/CHANGELOG b/CHANGELOG index cd003da..f06d477 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,8 @@ + ** v0.4.1 08.01.2020 + - Archive old vehicle locations + + ** v0.4.0 23.12.2019 - Improve plotting data gaps - Remove old media files diff --git a/MANAGEMENT/0-crontab b/MANAGEMENT/0-crontab index aaedf39..ad40d58 100644 --- a/MANAGEMENT/0-crontab +++ b/MANAGEMENT/0-crontab @@ -4,6 +4,8 @@ PYTHONIOENCODING=UTF-8 DJANGO_PYTHON_PATH=... DJANGO_MANAGE_PATH=... MEDIA_DIR=... +DB_ARCHIVE_DIR=... +MPK_URL=... # Get vehicle locations @@ -11,6 +13,12 @@ MEDIA_DIR=... * * * * * sleep 20 && $DJANGO_PYTHON_PATH $DJANGO_MANAGE_PATH get_locations * * * * * sleep 40 && $DJANGO_PYTHON_PATH $DJANGO_MANAGE_PATH get_locations +# Archive old vehicle locations +10 2 * * * $DJANGO_PYTHON_PATH $DJANGO_MANAGE_PATH archive_old_locations --keep-days 60 --out-dir $DB_ARCHIVE_DIR + # Clean old media 10 1 * * * find $MEDIA_DIR -type d -maxdepth 1 -mtime +14 | xargs -L100 rm -r +# Keep alive +5 7,19 * * * curl -s $MPK_URL > /dev/null + diff --git a/mpk/apps/vehicle_locations/management/commands/archive_old_locations.py b/mpk/apps/vehicle_locations/management/commands/archive_old_locations.py new file mode 100644 index 0000000..7908901 --- /dev/null +++ b/mpk/apps/vehicle_locations/management/commands/archive_old_locations.py @@ -0,0 +1,65 @@ +""" +Dumps records older than keep-days to a file and deletes these records from the database. +The data are dumped to daily files. The file for date d contains records from [d, d+1day) in *local* timezone. +""" +import gzip +import os +from datetime import datetime, timedelta + +import pytz +from django.conf import settings +from django.core.management import BaseCommand +from django.db import connection + +from vehicle_locations.models import VehicleLocation + +LOCATION_TABLE = VehicleLocation._meta.db_table + + +class Command(BaseCommand): + + def add_arguments(self, parser): + parser.add_argument('-d', '--keep-days', dest='keep_days', type=int, help='Number of days of data to keep', required=True) + parser.add_argument('-o', '--out-dir', dest='out_dir', help='Out dir', required=True) + + def handle(self, *args, **kwargs): + # Parse arguments + keep_days = kwargs['keep_days'] + out_dir = kwargs['out_dir'] + + if keep_days <= 0: + raise ValueError('keep-days must be positive') + + # Current and oldest date to keep + current_date = datetime.now().date() + last_date = current_date - timedelta(days=keep_days) + + # Earliest record + earliest_record = VehicleLocation.objects \ + .order_by('date') \ + .first() + if earliest_record is None: + return + + this_date = earliest_record.date.astimezone(settings.LOCAL_TIMEZONE).date() + while this_date < last_date: + next_date = this_date + timedelta(days=1) + next_date_dt = settings.LOCAL_TIMEZONE.localize(datetime.combine(next_date, datetime.min.time())) + next_date_utc_dt = next_date_dt.astimezone(pytz.utc) + + # Archive records + query = 'copy (select * from {} where date < \'{}\' order by date) to stdout'.format(LOCATION_TABLE, next_date_utc_dt.strftime('%Y-%m-%d %H:%M+00:00')) + out_filename = '{}/loc-{}.dump.gz'.format(out_dir, this_date.strftime('%Y%m%d')) + + cursor = connection.cursor() + cursor.copy_expert(query, gzip.open(out_filename, 'w')) + + # Make sure that the file has been created + if not os.path.isfile(out_filename): + raise RuntimeError('Dump for {} failed; {} wasn\'t been created'.format(this_date, out_filename)) + + # Delete records + VehicleLocation.objects.filter(date__lt=next_date_dt).order_by().delete() + + this_date = next_date + diff --git a/mpk/script/create_plot/create_plot.py b/mpk/script/create_plot/create_plot.py index 417f1b1..6ad07fc 100644 --- a/mpk/script/create_plot/create_plot.py +++ b/mpk/script/create_plot/create_plot.py @@ -178,7 +178,7 @@ def create_plot(line_no, date_from_local, date_to_local, out_filename): earliest_data = route.vehiclelocation_set \ .order_by('date') \ .first() - if earliest_data is None or date_from_local < earliest_data.date: + if earliest_data is None or date_to_local < earliest_data.date: is_request_too_early = True # Vehicle directions @@ -233,7 +233,7 @@ def create_plot(line_no, date_from_local, date_to_local, out_filename): if not earliest_data: no_data_msg = 'No data collected so far for line {}'.format(line_no) else: - no_data_msg = 'The earliest data available for line {}\nis at {}'.format(line_no, earliest_data.date.astimezone(timezone_local).strftime('%Y-%m-%d %H:%M:%S')) + no_data_msg = 'The earliest data available for line {}\nis at {}'.format(line_no, earliest_data.date.astimezone(timezone_local).strftime('%Y-%m-%d %H:%M')) plt.text(.5, .5, no_data_msg, fontsize=params.no_data_fontsize, ha='center', va='center', transform=canvas_h.transAxes) # Title