Improve import of multiple files

Closes #198

comptages/comptages.py

@@ -9,12 +9,11 @@
     QgsProject)
 from qgis.utils import qgsfunction, plugins
 
-
 from comptages.core.settings import Settings, SettingsDialog
 from comptages.core.layers import Layers
 from comptages.core.filter_dialog import FilterDialog
 from comptages.core.yearly_report_dialog import YearlyReportDialog
-from comptages.core.utils import push_info
+from comptages.core.utils import push_info, connect_to_db
 from comptages.importer.data_importer import DataImporter
 from comptages.importer.data_importer_vbv1 import DataImporterVbv1
 from comptages.importer.data_importer_int2 import DataImporterInt2
@@ -214,8 +213,16 @@ def do_import_files_action(self):
             file_dialog, title, path,
             "Data file (*.A?? *.aV? *.I?? *.V?? *.txt)")[0]
 
+        self.tm.allTasksFinished.connect(self.task_finished)
+
+        self.db = connect_to_db()
+
+        tasks = []
         for file_path in files:
-            self.import_file(file_path)
+            tasks.append(self.import_file(file_path))
+
+        for t in tasks:
+            self.tm.addTask(t)
 
     def import_file(self, file_path, count_id=None):
         QgsMessageLog.logMessage(
@@ -267,19 +274,17 @@ def import_file(self, file_path, count_id=None):
         file_format = file_header['FORMAT']
 
         if file_format == 'VBV-1':
-            task = DataImporterVbv1(file_path, count_id)
+            task = DataImporterVbv1(file_path, count_id, self.db)
         elif file_format == 'INT-2':
-            task = DataImporterInt2(file_path, count_id)
+            task = DataImporterInt2(file_path, count_id, self.db)
         elif file_format == 'MC':
-            task = DataImporterMC(file_path, count_id)
+            task = DataImporterMC(file_path, count_id, self.db)
         else:
             push_info('Format {} of {} not supported'.format(
                 file_format,
                 os.path.basename(file_path)))
             return
 
-        self.tm.allTasksFinished.connect(self.task_finished)
-        self.tm.addTask(task)
         return task
 
     def task_finished(self):
@@ -292,6 +297,9 @@ def task_finished(self):
 
         self.chart_dock.show_next_quarantined_chart()
 
+        self.db.close()
+        del self.db
+
     def do_validate_imported_files_action(self):
         if self.tm.countActiveTasks() > 0:
             push_info(("Veuillez patienter jusqu'à ce que l'importation "
@@ -458,7 +466,11 @@ def do_import_single_file_action(self, count_id):
         if not file_path:
             return
 
-        self.import_file(file_path, count_id)
+        self.tm.allTasksFinished.connect(self.task_finished)
+
+        self.db = connect_to_db()
+
+        self.tm.addTask(self.import_file(file_path, count_id))
 
     def do_generate_report_action(self, count_id):
         QgsMessageLog.logMessage(

comptages/importer/data_importer.py

@@ -10,13 +10,13 @@
 
 class DataImporter(QgsTask):
 
-    def __init__(self, file_path, count_id):
+    def __init__(self, file_path, count_id, db):
         self.basename = os.path.basename(file_path)
         super().__init__(
             'Importation fichier {}'.format(self.basename))
         self.file_path = file_path
         self.count_id = count_id
-        self.db = connect_to_db()
+        self.db = db
         self.file_header = self.parse_file_header(self.file_path)
         self.lanes = dict()
         self.populate_lane_dict()
@@ -42,8 +42,6 @@ def finished(self, result):
                     datetime.now(), self.basename, self.exception),
                 'Comptages', Qgis.Info)
 
-        self.db.close()
-        del self.db
 
     def cancel(self):
         # TODO: Cancel needed?
@@ -65,6 +63,8 @@ def populate_lane_dict(self):
         while query.next():
             self.lanes[int(query.value(0))] = int(query.value(1))
 
+        # e.g. self.lanes = {1: 435, 2: 436}
+
     def populate_category_dict(self):
         if 'CLASS' not in self.file_header:
             return
@@ -87,6 +87,8 @@ def populate_category_dict(self):
         while query.next():
             self.categories[int(query.value(0))] = int(query.value(1))
 
+        # e.g. self.categories = {0: 922, 1: 22, 2: 23, 3: 24, 4: 25, 5: 26, 6: 27, 7: 28, 8: 29, 9: 30, 10: 31}
+
     @staticmethod
     def parse_file_header(file_path):
         file_header = dict()

comptages/importer/data_importer_int2.py

@@ -7,8 +7,8 @@
 
 
 class DataImporterInt2(DataImporter):
-    def __init__(self, file_path, count_id):
-        super().__init__(file_path, count_id)
+    def __init__(self, file_path, count_id, db):
+        super().__init__(file_path, count_id, db)
         self.intspec = self.get_intspec()
         self.number_of_lines = self.get_number_of_lines()
 

comptages/importer/data_importer_mc.py

@@ -11,8 +11,8 @@
 
 class DataImporterMC(DataImporter):
 
-    def __init__(self, file_path, count_id):
-        super().__init__(file_path, count_id)
+    def __init__(self, file_path, count_id, db):
+        super().__init__(file_path, count_id, db)
         self.numbering = 0
         self.bulk_mgr = BulkCreateManager(chunk_size=1000)
 

comptages/importer/data_importer_vbv1.py

@@ -8,8 +8,8 @@
 
 class DataImporterVbv1(DataImporter):
 
-    def __init__(self, file_path, count_id):
-        super().__init__(file_path, count_id)
+    def __init__(self, file_path, count_id, db):
+        super().__init__(file_path, count_id, db)
         self.instances = []
         self.bulk_mgr = BulkCreateManager(chunk_size=1000)