Fix CSV reader issues

Author: qb-dornak

bankreader/migrations/0004_defaults.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.23 on 2024-02-18 12:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bankreader", "0003_transaction_account_statement_cleanup"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="transaction",
+            name="recipient_description",
+            field=models.CharField(default="", max_length=256, verbose_name="description for recipient"),
+        ),
+        migrations.AlterField(
+            model_name="transaction",
+            name="remote_account_name",
+            field=models.CharField(default="", max_length=128, verbose_name="remote account name"),
+        ),
+        migrations.AlterField(
+            model_name="transaction",
+            name="remote_account_number",
+            field=models.CharField(default="", max_length=64, verbose_name="remote account number"),
+        ),
+        migrations.AlterField(
+            model_name="transaction",
+            name="sender_description",
+            field=models.CharField(default="", max_length=256, verbose_name="description for sender"),
+        ),
+    ]

bankreader/models.py

@@ -83,14 +83,14 @@ class Transaction(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, verbose_name=_("account"))
     entry_date = models.DateField(_("entry date"))
     accounted_date = models.DateField(_("accounted date"))
-    remote_account_number = models.CharField(_("remote account number"), max_length=64)
-    remote_account_name = models.CharField(_("remote account name"), max_length=128)
+    remote_account_number = models.CharField(_("remote account number"), default="", max_length=64)
+    remote_account_name = models.CharField(_("remote account name"), default="", max_length=128)
     amount = models.DecimalField(_("amount"), decimal_places=2, max_digits=20)
     variable_symbol = models.BigIntegerField(_("variable symbol"), default=0)
     constant_symbol = models.BigIntegerField(_("constant symbol"), default=0)
     specific_symbol = models.BigIntegerField(_("specific symbol"), default=0)
-    sender_description = models.CharField(_("description for sender"), max_length=256)
-    recipient_description = models.CharField(_("description for recipient"), max_length=256)
+    sender_description = models.CharField(_("description for sender"), default="", max_length=256)
+    recipient_description = models.CharField(_("description for recipient"), default="", max_length=256)
 
     class Meta:
         ordering = ("accounted_date",)
@@ -100,3 +100,10 @@ class Meta:
 
     def __str__(self) -> str:
         return f"{self.accounted_date} {self.amount} {self.remote_account_name} {self.sender_description}"
+
+    def save(self, *args, **kwargs) -> None:
+        if self.entry_date is None and self.accounted_date is not None:
+            self.entry_date = self.accounted_date
+        if self.accounted_date is None and self.entry_date is not None:
+            self.accounted_date = self.entry_date
+        return super().save(**kwargs)

bankreader/readers/csv.py

@@ -31,12 +31,17 @@ def read_transactions(self, statemen_file: IO) -> Iterable[Transaction]:
         column_mapping: Dict[str, int] = {}
         csv_reader = csv.reader(rows, delimiter=self.delimiter, quotechar=self.quotechar)
         for row in csv_reader:
+            # skip empty lines
+            if row == []:
+                continue
+            # skip header until we find the column mapping
             if not column_mapping:
                 try:
-                    column_mapping = {key: row.index(csv_key) for csv_key, key in self.column_mapping.items()}
+                    column_mapping = {key: row.index(csv_key) for key, csv_key in self.column_mapping.items()}
                 except ValueError:
                     pass
                 continue
+            # read individual transactions
             try:
                 data = {key: self.get_value(key, row[column_mapping[key]]) for key in column_mapping}
             except IndexError:

bankreader_demo/demoapp/readers.py

@@ -14,17 +14,17 @@ class KbCsvReader(CsvReader):
     encoding = "cp1250"
     delimiter = ";"
     column_mapping = {
-        "Datum splatnosti": "accounted_date",
-        "Datum odepsani JB": "entry_date",
-        "Protiucet/Kod banky": "remote_account_number",
-        "Nazev protiuctu": "remote_account_name",
-        "Castka": "amount",
-        "VS": "variable_symbol",
-        "KS": "constant_symbol",
-        "SS": "specific_symbol",
-        "Identifikace transakce": "transaction_id",
-        "Popis prikazce": "sender_description",
-        "Popis pro prijemce": "recipient_description",
+        "accounted_date": "Datum splatnosti",
+        "entry_date": "Datum odepsani JB",
+        "remote_account_number": "Protiucet/Kod banky",
+        "remote_account_name": "Nazev protiuctu",
+        "amount": "Castka",
+        "variable_symbol": "VS",
+        "constant_symbol": "KS",
+        "specific_symbol": "SS",
+        "transaction_id": "Identifikace transakce",
+        "sender_description": "Popis prikazce",
+        "recipient_description": "Popis pro prijemce",
     }
     date_format = "%Y-%m-%d"
     decimal_separator = ","