move_data.py

# -*- coding: utf-8 -*-
"""
copies data from pyscada.models.RecordedDataOld to RecordedData
the script will terminate after 5 minutes, if you would like to copy more content change the timeout accordingly
"""

from __future__ import unicode_literals
import os

import gc
from datetime import datetime
from time import time
from pytz import UTC
import logging

logger = logging.getLogger('pyscada.core.move_data')


def queryset_iterator(queryset, chunk_size=100000):
    counter = 0
    count = chunk_size
    while count == chunk_size:
        offset = counter - counter % chunk_size
        count = 0
        for item in queryset.all()[offset:offset + chunk_size]:
            count += 1
            yield item
        counter += count
        gc.collect()


if __name__ == '__main__':
    os.chdir('/var/www/pyscada/PyScadaServer')
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PyScadaServer.settings")
    import django

    django.setup()
    from pyscada.models import RecordedData, RecordedDataOld
    min_pk = RecordedData.objects.first().pk
    recoded_data_set = queryset_iterator(RecordedDataOld.objects.filter(pk__lt=min_pk).order_by('-pk'))
    #recoded_data_set = queryset_iterator(RecordedDataOld.objects.all().order_by('-pk'))
    #logger.info('%d'%min_pk)
    #logger.info('%d'%RecordedDataOld.objects.filter(pk__lt=min_pk).count())
    count = 0
    count_all = 0
    timeout = time() + 60 * 10  # <-- set the timeout in minutes
    items = []
    for item in recoded_data_set:
        #if RecordedData.objects.filter(pk=item.id):
        #    continue
        items.append(RecordedData(id=item.id,
                                  variable_id=item.variable_id,
                                  value_boolean=item.value_boolean,
                                  value_int16=item.value_int16,
                                  value_int32=item.value_int32,
                                  value_int64=item.value_int64,
                                  value_float64=item.value_float64,
                                  date_saved=datetime.fromtimestamp((item.pk - item.variable.pk) / 2097152 / 1000.0,
                                                                    UTC)
                                  ))
        if time() > timeout:
            break

        count += 1
        if count >= 10000:
            RecordedData.objects.bulk_create(items)
            items = []
            count_all += count
            count = 0
            logger.info('wrote %d lines in total\n' % count_all)

    if len(items) > 0:
        RecordedData.objects.bulk_create(items)
        count_all += len(items)
        logger.info('wrote %d lines in total\n' % count_all)