[IMP]account_statement_import_txt_xlsx: additional mapping controls

account_statement_import_sheet_file/README.rst

@@ -103,6 +103,8 @@ Contributors
 
   * Alexey Pelykh <[email protected]>
 
+* Sebastiano Picchi <[email protected]>
+
 Maintainers
 ~~~~~~~~~~~
 

account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py

@@ -66,6 +66,18 @@ class AccountStatementImportSheetMapping(models.Model):
         help="When this occurs please indicate the column number in the Columns section "
         "instead of the column name, considering that the first column is 0",
     )
+    skip_empty_lines = fields.Boolean(
+        default=False,
+        help="Allows to skip empty lines",
+    )
+    offset_column = fields.Integer(
+        default=0,
+        help="Horizontal spaces to ignore before starting to parse",
+    )
+    offset_row = fields.Integer(
+        default=0,
+        help="Vertical spaces to ignore before starting to parse",
+    )
     timestamp_column = fields.Char(required=True)
     currency_column = fields.Char(
         help=(
@@ -217,6 +229,12 @@ def onchange_decimal_separator(self):
         elif "comma" == self.float_thousands_sep == self.float_decimal_sep:
             self.float_thousands_sep = "dot"
 
+    @api.constrains("offset_column", "offset_row")
+    def _check_columns(self):
+        for mapping in self:
+            if mapping.offset_column < 0 or mapping.offset_row < 0:
+                raise ValidationError(_("Offsets cannot be negative"))
+
     def _get_float_separators(self):
         self.ensure_one()
         separators = {

account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py

@@ -182,12 +182,14 @@ def _parse_lines(self, mapping, data_file, currency_code):
             header_line = mapping.header_lines_skip_count - 1
             if isinstance(csv_or_xlsx, tuple):
                 header = [
-                    str(value).strip()
-                    for value in csv_or_xlsx[1].row_values(header_line)
+                    str(value)
+                    for value in csv_or_xlsx[1].row_values(mapping.offset_row)
                 ]
             else:
                 [next(csv_or_xlsx) for _i in range(header_line)]
                 header = [value.strip() for value in next(csv_or_xlsx)]
+        if mapping.offset_column:
+            header = header[mapping.offset_column :]
 
         # NOTE no seria necesario debit_column y credit_column ya que tenemos los
         # respectivos campos related
@@ -353,7 +355,7 @@ def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
         footer_line = numrows - mapping.footer_lines_skip_count
 
         if isinstance(csv_or_xlsx, tuple):
-            rows = range(mapping.header_lines_skip_count, footer_line)
+            rows = range(mapping.offset_row + 1, csv_or_xlsx[1].nrows)
         else:
             rows = csv_or_xlsx
 
@@ -363,7 +365,7 @@ def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
                 book = csv_or_xlsx[0]
                 sheet = csv_or_xlsx[1]
                 values = []
-                for col_index in range(0, sheet.row_len(row)):
+                for col_index in range(mapping.offset_column, sheet.row_len(row)):
                     cell_type = sheet.cell_type(row, col_index)
                     cell_value = sheet.cell_value(row, col_index)
                     if cell_type == xlrd.XL_CELL_DATE:
@@ -373,6 +375,8 @@ def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
                 if index >= footer_line:
                     continue
                 values = list(row)
+            if mapping.skip_empty_lines and not any(values):
+                continue
             line = self._parse_row(mapping, currency_code, values, columns)
             if line:
                 lines.append(line)

account_statement_import_sheet_file/readme/CONTRIBUTORS.rst

@@ -15,3 +15,5 @@
 * `CorporateHub <https://corporatehub.eu/>`__
 
   * Alexey Pelykh <[email protected]>
+
+* Sebastiano Picchi <[email protected]>

account_statement_import_sheet_file/static/description/index.html

@@ -8,10 +8,11 @@
 
 /*
 :Author: David Goodger ([email protected])
-:Id: $Id: html4css1.css 8954 2022-01-20 10:10:25Z milde $
+:Id: $Id: html4css1.css 9511 2024-01-13 09:50:07Z milde $
 :Copyright: This stylesheet has been placed in the public domain.
 
 Default cascading style sheet for the HTML output of Docutils.
+Despite the name, some widely supported CSS2 features are used.
 
 See https://docutils.sourceforge.io/docs/howto/html-stylesheets.html for how to
 customize this style sheet.
@@ -274,7 +275,7 @@
   margin-left: 2em ;
   margin-right: 2em }
 
-pre.code .ln { color: grey; } /* line numbers */
+pre.code .ln { color: gray; } /* line numbers */
 pre.code, code { background-color: #eeeeee }
 pre.code .comment, code .comment { color: #5C6576 }
 pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold }
@@ -300,7 +301,7 @@
 span.pre {
   white-space: pre }
 
-span.problematic {
+span.problematic, pre.problematic {
   color: red }
 
 span.section-subtitle {
@@ -455,12 +456,15 @@ <h2><a class="toc-backref" href="#toc-entry-8">Contributors</a></h2>
 <li>Alexey Pelykh &lt;<a class="reference external" href="mailto:alexey.pelykh&#64;corphub.eu">alexey.pelykh&#64;corphub.eu</a>&gt;</li>
 </ul>
 </li>
+<li>Sebastiano Picchi &lt;<a class="reference external" href="mailto:sebastiano.picchi&#64;pytech.it">sebastiano.picchi&#64;pytech.it</a>&gt;</li>
 </ul>
 </div>
 <div class="section" id="maintainers">
 <h2><a class="toc-backref" href="#toc-entry-9">Maintainers</a></h2>
 <p>This module is maintained by the OCA.</p>
-<a class="reference external image-reference" href="https://odoo-community.org"><img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" /></a>
+<a class="reference external image-reference" href="https://odoo-community.org">
+<img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" />
+</a>
 <p>OCA, or the Odoo Community Association, is a nonprofit organization whose
 mission is to support the collaborative development of Odoo features and
 promote its widespread use.</p>

account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv

@@ -0,0 +1,5 @@
+"Date","Label","Currency","Amount","Amount Currency","Partner Name","Bank Account"
+"02/25/2018","AAAOOO 1","EUR","-33.50","0.0","John Doe","123456789"
+"02/26/2018","AAAOOO 2","EUR","1,525.00","1,000.00","Azure Interior",""
+,,,,,,
+"02/27/2018","AAAOOO 3","EUR","800.00","800.00","Azure Interior","123456789"

account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py

@@ -1,7 +1,7 @@
 # Copyright 2019 ForgeFlow, S.L.
 # Copyright 2020 CorporateHub (https://corporatehub.eu)
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
+import decimal
 from base64 import b64encode
 from decimal import Decimal
 from os import path
@@ -678,3 +678,96 @@ def test_decimal_and_float_inputs(self):
             self.parser._parse_decimal(Decimal("1234.56"), self.mock_mapping_comma_dot),
             1234.56,
         )
+
+    def test_offsets(self):
+        journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_usd.id,
+                "suspense_account_id": self.suspense_account.id,
+            }
+        )
+        file_name = "fixtures/sample_statement_offsets.xlsx"
+        data = self._data_file(file_name)
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": self.sample_statement_map.id,
+            }
+        )
+        with self.assertRaises(ValueError):
+            wizard.with_context(
+                account_statement_import_txt_xlsx_test=True
+            ).import_file_button()
+        statement_map_offsets = self.sample_statement_map.copy(
+            {
+                "offset_column": 1,
+                "offset_row": 2,
+            }
+        )
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": statement_map_offsets.id,
+            }
+        )
+        wizard.with_context(
+            account_statement_import_txt_xlsx_test=True
+        ).import_file_button()
+        statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+        # import wdb; wdb.set_trace()
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 2)
+        self.assertEqual(statement.balance_start, 0.0)
+        self.assertEqual(statement.balance_end_real, 1491.5)
+        self.assertEqual(statement.balance_end, 1491.5)
+
+    def test_skip_empty_lines(self):
+        journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_usd.id,
+                "suspense_account_id": self.suspense_account.id,
+            }
+        )
+        file_name = "fixtures/empty_lines_statement.csv"
+        data = self._data_file(file_name, "utf-8")
+
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": self.sample_statement_map.id,
+            }
+        )
+        with self.assertRaises(decimal.InvalidOperation):
+            wizard.with_context(
+                account_statement_import_txt_xlsx_test=True
+            ).import_file_button()
+        statement_map_empty_line = self.sample_statement_map.copy(
+            {
+                "skip_empty_lines": True,
+            }
+        )
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": statement_map_empty_line.id,
+            }
+        )
+        wizard.with_context(
+            account_statement_import_txt_xlsx_test=True
+        ).import_file_button()
+        statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 3)
+        self.assertEqual(statement.balance_start, 0.0)
+        self.assertEqual(statement.balance_end_real, 2291.5)
+        self.assertEqual(statement.balance_end, 2291.5)

account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml

@@ -50,6 +50,15 @@
                                     class="fa fa-info-circle"
                                 /> indicate the column number in the Columns section. The first column is 0.
                             </div>
+                            <field name="skip_empty_lines" />
+                            <field
+                                name="offset_column"
+                                attrs="{'invisible': [('no_header', '=', True)]}"
+                            />
+                            <field
+                                name="offset_row"
+                                attrs="{'invisible': [('no_header', '=', True)]}"
+                            />
                         </group>
                         <group
                             attrs="{'invisible': [('debit_credit_column', '=', False)]}"