[IMP]account_statement_import_txt_xlsx: additional mapping controls

account_statement_import_sheet_file/README.rst

@@ -106,6 +106,8 @@ Contributors
 
    -  Alexey Pelykh <[email protected]>
 
+-  Sebastiano Picchi [email protected]
+
 Maintainers
 -----------
 

account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py

@@ -174,6 +174,14 @@ class AccountStatementImportSheetMapping(models.Model):
         help="Set the Header lines number.",
         default="0",
     )
+    skip_empty_lines = fields.Boolean(
+        default=True,
+        help="Allows to skip empty lines",
+    )
+    offset_column = fields.Integer(
+        default=0,
+        help="Columns to ignore before starting to parse",
+    )
 
     @api.constrains(
         "amount_type",
@@ -219,6 +227,12 @@ def onchange_decimal_separator(self):
         elif "comma" == self.float_thousands_sep == self.float_decimal_sep:
             self.float_thousands_sep = "dot"
 
+    @api.constrains("offset_column")
+    def _check_columns(self):
+        for mapping in self:
+            if mapping.offset_column < 0:
+                raise ValidationError(_("Offsets cannot be negative"))
+
     def _get_float_separators(self):
         self.ensure_one()
         separators = {

account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py

@@ -188,6 +188,8 @@ def _parse_lines(self, mapping, data_file, currency_code):
             else:
                 [next(csv_or_xlsx) for _i in range(header_line)]
                 header = [value.strip() for value in next(csv_or_xlsx)]
+            if mapping.offset_column:
+                header = header[mapping.offset_column :]
 
         # NOTE no seria necesario debit_column y credit_column ya que tenemos los
         # respectivos campos related
@@ -226,7 +228,7 @@ def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
         footer_line = numrows - mapping.footer_lines_skip_count
 
         if isinstance(csv_or_xlsx, tuple):
-            rows = range(mapping.header_lines_skip_count, footer_line)
+            rows = range(label_line, footer_line)
         else:
             rows = csv_or_xlsx
 
@@ -236,7 +238,7 @@ def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
                 book = csv_or_xlsx[0]
                 sheet = csv_or_xlsx[1]
                 values = []
-                for col_index in range(0, sheet.row_len(row)):
+                for col_index in range(mapping.offset_column, sheet.row_len(row)):
                     cell_type = sheet.cell_type(row, col_index)
                     cell_value = sheet.cell_value(row, col_index)
                     if cell_type == xlrd.XL_CELL_DATE:
@@ -246,6 +248,8 @@ def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
                 if index >= footer_line:
                     continue
                 values = list(row)
+            if mapping.skip_empty_lines and not any(values):
+                continue
 
             timestamp = self._get_values_from_column(
                 values, columns, "timestamp_column"
@@ -395,7 +399,9 @@ def _decimal(column_name, values):
                 line["bank_name"] = bank_name
             if bank_account is not None:
                 line["bank_account"] = bank_account
-            lines.append(line)
+
+            if line:
+                lines.append(line)
         return lines
 
     @api.model

account_statement_import_sheet_file/readme/CONTRIBUTORS.md

@@ -10,3 +10,4 @@
   - Miquel Raïch Regué \<<[email protected]>\>
 - [CorporateHub](https://corporatehub.eu/)
   - Alexey Pelykh \<<[email protected]>\>
+- Sebastiano Picchi <[email protected]>

account_statement_import_sheet_file/static/description/index.html

@@ -8,10 +8,11 @@
 
 /*
 :Author: David Goodger ([email protected])
-:Id: $Id: html4css1.css 8954 2022-01-20 10:10:25Z milde $
+:Id: $Id: html4css1.css 9511 2024-01-13 09:50:07Z milde $
 :Copyright: This stylesheet has been placed in the public domain.
 
 Default cascading style sheet for the HTML output of Docutils.
+Despite the name, some widely supported CSS2 features are used.
 
 See https://docutils.sourceforge.io/docs/howto/html-stylesheets.html for how to
 customize this style sheet.
@@ -274,7 +275,7 @@
   margin-left: 2em ;
   margin-right: 2em }
 
-pre.code .ln { color: grey; } /* line numbers */
+pre.code .ln { color: gray; } /* line numbers */
 pre.code, code { background-color: #eeeeee }
 pre.code .comment, code .comment { color: #5C6576 }
 pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold }
@@ -300,7 +301,7 @@
 span.pre {
   white-space: pre }
 
-span.problematic {
+span.problematic, pre.problematic {
   color: red }
 
 span.section-subtitle {
@@ -458,12 +459,15 @@ <h2><a class="toc-backref" href="#toc-entry-8">Contributors</a></h2>
 <li>Alexey Pelykh &lt;<a class="reference external" href="mailto:alexey.pelykh&#64;corphub.eu">alexey.pelykh&#64;corphub.eu</a>&gt;</li>
 </ul>
 </li>
+<li>Sebastiano Picchi <a class="reference external" href="mailto:sebastiano.picchi&#64;pytech.it">sebastiano.picchi&#64;pytech.it</a></li>
 </ul>
 </div>
 <div class="section" id="maintainers">
 <h2><a class="toc-backref" href="#toc-entry-9">Maintainers</a></h2>
 <p>This module is maintained by the OCA.</p>
-<a class="reference external image-reference" href="https://odoo-community.org"><img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" /></a>
+<a class="reference external image-reference" href="https://odoo-community.org">
+<img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" />
+</a>
 <p>OCA, or the Odoo Community Association, is a nonprofit organization whose
 mission is to support the collaborative development of Odoo features and
 promote its widespread use.</p>

account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv

@@ -0,0 +1,5 @@
+"Date","Label","Currency","Amount","Amount Currency","Partner Name","Bank Account"
+"02/25/2018","AAAOOO 1","EUR","-33.50","0.0","John Doe","123456789"
+"02/26/2018","AAAOOO 2","EUR","1,525.00","1,000.00","Azure Interior",""
+,,,,,,
+"02/27/2018","AAAOOO 3","EUR","800.00","800.00","Azure Interior","123456789"

account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py

@@ -678,3 +678,94 @@ def test_decimal_and_float_inputs(self):
             self.parser._parse_decimal(Decimal("1234.56"), self.mock_mapping_comma_dot),
             1234.56,
         )
+
+    def test_offsets(self):
+        journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_usd.id,
+                "suspense_account_id": self.suspense_account.id,
+            }
+        )
+        file_name = "fixtures/sample_statement_offsets.xlsx"
+        data = self._data_file(file_name)
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": self.sample_statement_map.id,
+            }
+        )
+        with self.assertRaises(UserError):
+            wizard.with_context(
+                account_statement_import_txt_xlsx_test=True
+            ).import_file_button()
+        statement_map_offsets = self.sample_statement_map.copy(
+            {
+                "offset_column": 1,
+                "header_lines_skip_count": 3,
+            }
+        )
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": statement_map_offsets.id,
+            }
+        )
+        wizard.with_context(
+            account_statement_import_txt_xlsx_test=True
+        ).import_file_button()
+        statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 2)
+        self.assertEqual(statement.balance_start, 0.0)
+        self.assertEqual(statement.balance_end_real, 1491.5)
+        self.assertEqual(statement.balance_end, 1491.5)
+
+    def test_skip_empty_lines(self):
+        journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_usd.id,
+                "suspense_account_id": self.suspense_account.id,
+            }
+        )
+        file_name = "fixtures/empty_lines_statement.csv"
+        data = self._data_file(file_name, "utf-8")
+        statement_map_empty_line = self.sample_statement_map.copy(
+            {
+                "skip_empty_lines": False,
+            }
+        )
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": statement_map_empty_line.id,
+            }
+        )
+        with self.assertRaises(UserError):
+            wizard.with_context(
+                account_statement_import_txt_xlsx_test=True
+            ).import_file_button()
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": self.sample_statement_map.id,
+            }
+        )
+        wizard.with_context(
+            account_statement_import_txt_xlsx_test=True
+        ).import_file_button()
+        statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 3)
+        self.assertEqual(statement.balance_start, 0.0)
+        self.assertEqual(statement.balance_end_real, 2291.5)
+        self.assertEqual(statement.balance_end, 2291.5)

account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml

@@ -56,8 +56,10 @@
                             <field name="credit_value" required="debit_credit_column" />
                         </group>
                         <group>
+                            <field name="skip_empty_lines" />
                             <field name="header_lines_skip_count" />
                             <field name="footer_lines_skip_count" />
+                            <field name="offset_column" invisible="no_header" />
                         </group>
                     </group>
                     <group string="Columns">