From f35935cba46800ad44ba09dc30b05ab4d6743849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Wed, 8 Dec 2021 23:02:23 +0100 Subject: [PATCH 01/10] [IMP] start extraction of chunk processing \o/ --- chunk_processing/__init__.py | 2 + chunk_processing/__manifest__.py | 28 ++++ chunk_processing/components/__init__.py | 3 + chunk_processing/components/processor.py | 14 ++ chunk_processing/components/splitter.py | 50 ++++++ chunk_processing/components/splitter_json.py | 18 +++ chunk_processing/models/__init__.py | 2 + chunk_processing/models/chunk_group.py | 69 ++++++++ .../models/chunk_item.py | 52 +++--- .../views/chunk_item_view.xml | 10 +- pattern_import_export/__init__.py | 1 + pattern_import_export/__manifest__.py | 4 +- pattern_import_export/components/__init__.py | 1 + pattern_import_export/components/processor.py | 47 ++++++ pattern_import_export/models/__init__.py | 2 +- pattern_import_export/models/chunk_group.py | 22 +++ pattern_import_export/models/pattern_file.py | 151 ++++++------------ .../security/ir.model.access.csv | 1 - .../tests/test_pattern_import.py | 21 ++- pattern_import_export/views/pattern_file.xml | 6 +- .../odoo/addons/chunk_processing | 1 + setup/chunk_processing/setup.py | 6 + 22 files changed, 361 insertions(+), 150 deletions(-) create mode 100644 chunk_processing/__init__.py create mode 100644 chunk_processing/__manifest__.py create mode 100644 chunk_processing/components/__init__.py create mode 100644 chunk_processing/components/processor.py create mode 100644 chunk_processing/components/splitter.py create mode 100644 chunk_processing/components/splitter_json.py create mode 100644 chunk_processing/models/__init__.py create mode 100644 chunk_processing/models/chunk_group.py rename pattern_import_export/models/pattern_chunk.py => chunk_processing/models/chunk_item.py (70%) rename pattern_import_export/views/pattern_chunk.xml => chunk_processing/views/chunk_item_view.xml (76%) create mode 100644 pattern_import_export/components/__init__.py create mode 100644 pattern_import_export/components/processor.py create mode 100644 pattern_import_export/models/chunk_group.py create mode 120000 setup/chunk_processing/odoo/addons/chunk_processing create mode 100644 setup/chunk_processing/setup.py diff --git a/chunk_processing/__init__.py b/chunk_processing/__init__.py new file mode 100644 index 00000000..0f00a673 --- /dev/null +++ b/chunk_processing/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import components diff --git a/chunk_processing/__manifest__.py b/chunk_processing/__manifest__.py new file mode 100644 index 00000000..934a070a --- /dev/null +++ b/chunk_processing/__manifest__.py @@ -0,0 +1,28 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + + +{ + "name": "Chunk Processing", + "summary": "Base module for processing chunk", + "version": "14.0.1.0.0", + "category": "Uncategorized", + "website": "https://github.com/shopinvader/pattern-import-export", + "author": " Akretion", + "license": "AGPL-3", + "application": False, + "installable": True, + "external_dependencies": { + "python": [], + "bin": [], + }, + "depends": [ + "queue_job", + "component", + ], + "data": [ + "views/chunk_item_view.xml", + ], + "demo": [], +} diff --git a/chunk_processing/components/__init__.py b/chunk_processing/components/__init__.py new file mode 100644 index 00000000..3242cabf --- /dev/null +++ b/chunk_processing/components/__init__.py @@ -0,0 +1,3 @@ +from . import processor +from . import splitter +from . import splitter_json diff --git a/chunk_processing/components/processor.py b/chunk_processing/components/processor.py new file mode 100644 index 00000000..65ac3361 --- /dev/null +++ b/chunk_processing/components/processor.py @@ -0,0 +1,14 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + + +from odoo.addons.component.core import AbstractComponent + + +class ChunkProcessor(AbstractComponent): + _name = "chunk.processor" + _collection = "chunk.item" + + def run(self): + raise NotImplementedError diff --git a/chunk_processing/components/splitter.py b/chunk_processing/components/splitter.py new file mode 100644 index 00000000..96813c88 --- /dev/null +++ b/chunk_processing/components/splitter.py @@ -0,0 +1,50 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import AbstractComponent + + +class ChunkSplitter(AbstractComponent): + _name = "chunk.splitter" + _collection = "chunk.group" + + def _parse_data(self, data): + raise NotImplementedError + + def _prepare_chunk(self, start_idx, stop_idx, data): + return { + "start_idx": start_idx, + "stop_idx": stop_idx, + "data": data, + "nbr_item": len(data), + "state": "pending", + "group_id": self.collection.id, + } + + def _should_create_chunk(self, items, next_item): + """Customise this code if you want to add some additionnal + item after reaching the limit""" + return len(items) > self.collection.chunk_size + + def _create_chunk(self, start_idx, stop_idx, data): + vals = self._prepare_chunk(start_idx, stop_idx, data) + chunk = self.env["chunk.item"].create(vals) + # we enqueue the chunk in case of multi process of if it's the first chunk + if self.collection.process_multi or len(self.collection.item_ids) == 1: + chunk.with_delay(priority=self.collection.job_priority).run() + return chunk + + def run(self, data): + items = [] + start_idx = 1 + previous_idx = None + for idx, item in self._parse_data(data): + if self._should_create_chunk(items, item): + self._create_chunk(start_idx, previous_idx, items) + items = [] + start_idx = idx + items.append((idx, item)) + previous_idx = idx + if items: + self._create_chunk(start_idx, idx, items) diff --git a/chunk_processing/components/splitter_json.py b/chunk_processing/components/splitter_json.py new file mode 100644 index 00000000..0220e6c1 --- /dev/null +++ b/chunk_processing/components/splitter_json.py @@ -0,0 +1,18 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import json + +from odoo.addons.component.core import Component + + +class ChunkSplitterJson(Component): + _inherit = "chunk.splitter" + _name = "chunk.splitter.json" + _usage = "json" + + def _parse_data(self, data): + items = json.loads(data.decode("utf-8")) + for idx, item in enumerate(items): + yield idx + 1, item diff --git a/chunk_processing/models/__init__.py b/chunk_processing/models/__init__.py new file mode 100644 index 00000000..23028c82 --- /dev/null +++ b/chunk_processing/models/__init__.py @@ -0,0 +1,2 @@ +from . import chunk_item +from . import chunk_group diff --git a/chunk_processing/models/chunk_group.py b/chunk_processing/models/chunk_group.py new file mode 100644 index 00000000..d2b64be7 --- /dev/null +++ b/chunk_processing/models/chunk_group.py @@ -0,0 +1,69 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + + +from odoo import _, api, fields, models + + +class ChunkGroup(models.Model): + _inherit = "collection.base" + _name = "chunk.group" + + item_ids = fields.One2many("chunk.item", "group_id", "Item") + process_multi = fields.Boolean() + job_priority = fields.Integer(default=20) + chunk_size = fields.Integer(default=500, help="Define the size of the chunk") + progress = fields.Float(compute="_compute_stat") + date_done = fields.Datetime() + data_format = fields.Selection( + [ + ("json", "Json"), + ("xml", "XML"), + ] + ) + state = fields.Selection( + [("pending", "Pending"), ("failed", "Failed"), ("done", "Done")], + default="pending", + ) + info = fields.Char() + nbr_error = fields.Integer(compute="_compute_stat") + nbr_success = fields.Integer(compute="_compute_stat") + apply_on_model = fields.Char() + usage = fields.Char() + + @api.depends("item_ids.nbr_error", "item_ids.nbr_success") + def _compute_stat(self): + for record in self: + record.nbr_error = sum(record.mapped("item_ids.nbr_error")) + record.nbr_success = sum(record.mapped("item_ids.nbr_success")) + todo = sum(record.mapped("item_ids.nbr_item")) + if todo: + record.progress = (record.nbr_error + record.nbr_success) * 100.0 / todo + else: + record.progress = 0 + + def _get_data(self): + raise NotImplementedError + + def split_in_chunk(self): + """Split Group into Chunk""" + # purge chunk in case of retring a job + self.item_ids.unlink() + try: + data = self._get_data() + with self.work_on(self._name) as work: + splitter = work.component(usage=self.data_format) + splitter.run(data) + except Exception as e: + self.state = "failed" + self.info = _("Failed to create the chunk: %s") % e + return True + + def set_done(self): + for record in self: + if record.nbr_error: + record.state = "failed" + else: + record.state = "done" + record.date_done = fields.Datetime.now() diff --git a/pattern_import_export/models/pattern_chunk.py b/chunk_processing/models/chunk_item.py similarity index 70% rename from pattern_import_export/models/pattern_chunk.py rename to chunk_processing/models/chunk_item.py index e4cf6ed4..562f2d1b 100644 --- a/pattern_import_export/models/pattern_chunk.py +++ b/chunk_processing/models/chunk_item.py @@ -5,14 +5,15 @@ from odoo import fields, models -class PatternChunk(models.Model): - _name = "pattern.chunk" - _description = "Pattern Chunk" +class ChunkItem(models.Model): + _inherit = "collection.base" + _name = "chunk.item" + _description = "Chunk Item" _order = "start_idx" _rec_name = "start_idx" - pattern_file_id = fields.Many2one( - "pattern.file", "Pattern File", required=True, ondelete="cascade" + group_id = fields.Many2one( + "chunk.group", "Chunk Group", required=True, ondelete="cascade" ) start_idx = fields.Integer() stop_idx = fields.Integer() @@ -32,33 +33,31 @@ class PatternChunk(models.Model): ] ) - def run_import(self): - model = self.pattern_file_id.pattern_config_id.model_id.model - res = ( - self.with_context(pattern_config={"model": model, "record_ids": []}) - .env[model] - .load([], self.data) - ) - self.write(self._prepare_chunk_result(res)) - config = self.pattern_file_id.pattern_config_id - priority = config.job_priority - if not config.process_multi: + def manual_run(self): + """ Run the import without try/except, easier for debug """ + return self._run() + + def _run(self): + with self.work_on(self.group_id.apply_on_model) as work: + processor = work.component(usage=self.group_id.usage) + processor.run() + if not self.group_id.process_multi: next_chunk = self.get_next_chunk() if next_chunk: - next_chunk.with_delay(priority=priority).run() + next_chunk.with_delay(priority=self.group_id.job_priority).run() else: self.with_delay(priority=5).check_last() else: self.with_delay(priority=5).check_last() def run(self): - """Process Import of Pattern Chunk""" + """Process Chunk Item in a savepoint""" cr = self.env.cr try: self.state = "started" cr.commit() # pylint: disable=invalid-commit with cr.savepoint(): - self.run_import() + self._run() except Exception as e: self.write( { @@ -70,6 +69,7 @@ def run(self): self.with_delay().check_last() return "OK" + # TODO move this in pattern-import def _prepare_chunk_result(self, res): # TODO rework this part and add specific test case nbr_error = len(res["messages"]) @@ -98,23 +98,19 @@ def _prepare_chunk_result(self, res): } def get_next_chunk(self): - return self.search( - [ - ("pattern_file_id", "=", self.pattern_file_id.id), - ("state", "=", "pending"), - ], - limit=1, + return fields.first( + self.group_id.item_ids.filtered(lambda s: s.state == "pending") ) def is_last_job(self): - return not self.pattern_file_id.chunk_ids.filtered( + return not self.group_id.item_ids.filtered( lambda s: s.state in ("pending", "started") ) def check_last(self): """Check if all chunk have been processed""" if self.is_last_job(): - self.pattern_file_id.set_import_done() - return "Pattern file is done" + self.group_id.set_done() + return "Chunk group is done" else: return "There is still some running chunk" diff --git a/pattern_import_export/views/pattern_chunk.xml b/chunk_processing/views/chunk_item_view.xml similarity index 76% rename from pattern_import_export/views/pattern_chunk.xml rename to chunk_processing/views/chunk_item_view.xml index d9dbbd30..e1334344 100644 --- a/pattern_import_export/views/pattern_chunk.xml +++ b/chunk_processing/views/chunk_item_view.xml @@ -1,8 +1,8 @@ - - pattern.chunk + + chunk.item @@ -14,12 +14,12 @@ - - pattern.chunk + + chunk.item
-
diff --git a/pattern_import_export/__init__.py b/pattern_import_export/__init__.py index 9b429614..d0c2ad41 100644 --- a/pattern_import_export/__init__.py +++ b/pattern_import_export/__init__.py @@ -1,2 +1,3 @@ from . import models from . import wizard +from . import components diff --git a/pattern_import_export/__manifest__.py b/pattern_import_export/__manifest__.py index 93a54145..7d50c134 100644 --- a/pattern_import_export/__manifest__.py +++ b/pattern_import_export/__manifest__.py @@ -15,6 +15,7 @@ "web_notify", "base_sparse_field_list_support", "base_sparse_field", + "chunk_processing", ], "data": [ "security/pattern_security.xml", @@ -23,11 +24,10 @@ "wizard/import_pattern_wizard.xml", "views/pattern_config.xml", "views/pattern_file.xml", - "views/pattern_chunk.xml", "views/menuitems.xml", "views/templates.xml", "data/queue_job_channel_data.xml", - "data/queue_job_function_data.xml", + # "data/queue_job_function_data.xml", ], "demo": ["demo/demo.xml"], "installable": True, diff --git a/pattern_import_export/components/__init__.py b/pattern_import_export/components/__init__.py new file mode 100644 index 00000000..ad6975d7 --- /dev/null +++ b/pattern_import_export/components/__init__.py @@ -0,0 +1 @@ +from . import processor diff --git a/pattern_import_export/components/processor.py b/pattern_import_export/components/processor.py new file mode 100644 index 00000000..5e9111d9 --- /dev/null +++ b/pattern_import_export/components/processor.py @@ -0,0 +1,47 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component + + +class ChunkProcessorPattern(Component): + _inherit = "chunk.processor" + _name = "chunk.processor.pattern" + _usage = "pattern.import" + + def run(self): + model = self.collection.group_id.apply_on_model + res = ( + self.env[model] + .with_context(pattern_config={"model": model, "record_ids": []}) + .load([], self.collection.data) + ) + self.collection.write(self._prepare_chunk_result(res)) + + def _prepare_chunk_result(self, res): + # TODO rework this part and add specific test case + nbr_error = len(res["messages"]) + nbr_success = max(self.collection.nbr_item - nbr_error, 0) + + # case where error are not return and record are not imported + nbr_imported = len(res.get("ids") or []) + if nbr_success > nbr_imported: + nbr_success = nbr_imported + nbr_error = self.collection.nbr_item - nbr_imported + + if nbr_error: + state = "failed" + else: + state = "done" + result = self.env["ir.qweb"]._render( + "pattern_import_export.format_message", res + ) + return { + "record_ids": res.get("ids"), + "messages": res.get("messages"), + "result_info": result, + "state": state, + "nbr_success": nbr_success, + "nbr_error": nbr_error, + } diff --git a/pattern_import_export/models/__init__.py b/pattern_import_export/models/__init__.py index 74322801..307fad64 100644 --- a/pattern_import_export/models/__init__.py +++ b/pattern_import_export/models/__init__.py @@ -1,9 +1,9 @@ from . import pattern_config +from . import chunk_group from . import ir_exports_line from . import ir_exports from . import ir_actions from . import base from . import ir_fields from . import pattern_file -from . import pattern_chunk from . import ir_attachment diff --git a/pattern_import_export/models/chunk_group.py b/pattern_import_export/models/chunk_group.py new file mode 100644 index 00000000..7bf0e521 --- /dev/null +++ b/pattern_import_export/models/chunk_group.py @@ -0,0 +1,22 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 + +from odoo import fields, models + + +class ChunkGroup(models.Model): + _inherit = "chunk.group" + + # We use a O2M but as there is a sql constraint we can have only + # one pattern file, this is why the fieldname end with "id" + pattern_file_id = fields.One2many("pattern.file", "chunk_group_id", "Pattern File") + + def _get_data(self): + self.ensure_one() + if self.pattern_file_id: + return base64.b64decode(self.pattern_file_id.datas.decode("utf-8")) + else: + return super()._get_data() diff --git a/pattern_import_export/models/pattern_file.py b/pattern_import_export/models/pattern_file.py index ccbe4879..ee085337 100644 --- a/pattern_import_export/models/pattern_file.py +++ b/pattern_import_export/models/pattern_file.py @@ -1,8 +1,6 @@ # Copyright (c) Akretion 2020 # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html) -import base64 -import json import urllib.parse from odoo import _, api, fields, models @@ -14,46 +12,61 @@ class PatternFile(models.Model): _description = "Attachment with pattern file metadata" attachment_id = fields.Many2one("ir.attachment", required=True, ondelete="cascade") - state = fields.Selection( - [("pending", "Pending"), ("failed", "Failed"), ("done", "Done")], - default="pending", - ) - info = fields.Char() kind = fields.Selection([("import", "import"), ("export", "export")], required=True) pattern_config_id = fields.Many2one( "pattern.config", required=True, string="Export pattern" ) - nbr_error = fields.Integer(compute="_compute_stat") - nbr_success = fields.Integer(compute="_compute_stat") - progress = fields.Float(compute="_compute_stat") - chunk_ids = fields.One2many("pattern.chunk", "pattern_file_id", "Chunk") - date_done = fields.Datetime() - - @api.depends("chunk_ids.nbr_error", "chunk_ids.nbr_success") - def _compute_stat(self): + chunk_group_id = fields.Many2one("chunk.group") + chunk_item_ids = fields.One2many("chunk.item", related="chunk_group_id.item_ids") + state = fields.Selection( + [("pending", "Pending"), ("failed", "Failed"), ("done", "Done")], + compute="_compute_state", + ) + date_done = fields.Date(compute="_compute_date_done", store=True) + progress = fields.Float(related="chunk_group_id.progress") + nbr_error = fields.Integer(related="chunk_group_id.nbr_error") + nbr_success = fields.Integer(related="chunk_group_id.nbr_success") + info = fields.Char(related="chunk_group_id.info") + + _sql_constraints = [ + ("uniq_group_id", "unique(group_id)", "The Group must be unique!") + ] + + def _add_chunk_group(self): for record in self: - record.nbr_error = sum(record.mapped("chunk_ids.nbr_error")) - record.nbr_success = sum(record.mapped("chunk_ids.nbr_success")) - todo = sum(record.mapped("chunk_ids.nbr_item")) - if todo: - record.progress = (record.nbr_error + record.nbr_success) * 100.0 / todo - else: - record.progress = 0 + config = record.pattern_config_id + record.chunk_group_id = self.env["chunk.group"].create( + { + "job_priority": config.job_priority, + "process_multi": config.process_multi, + "data_format": "json", + "apply_on_model": config.resource, + "usage": "pattern.import", + } + ) @api.model_create_multi - def create(self, vals): - result = super().create(vals) - for record in result: - if record.state != "pending": - record._notify_user() - return result - - def write(self, vals): - result = super().write(vals) - if "state" in vals.keys() and vals["state"] != "pending": - for rec in self: - rec._notify_user() - return result + def create(self, vals_list): + records = super().create(vals_list) + records._add_chunk_group() + return records + + @api.depends("kind", "chunk_group_id.date_done") + def _compute_date_done(self): + for record in self: + if record.kind == "export" and not record.date_done: + record.date_done = fields.Date.today() + elif record.kind == "import": + record.date_done = record.chunk_group_id.date_done + + @api.depends("kind", "chunk_group_id.state") + def _compute_state(self): + for record in self: + if record.kind == "export": + record.state = "done" + elif record.kind == "import": + record.state = record.chunk_group_id.state + record._notify_user() def _notify_user(self): import_or_export = _("Import") if self.kind == "import" else _("Export") @@ -114,74 +127,8 @@ def _helper_build_content_link(self): link += "" + _("Download") + "" return link - def _parse_data(self): - data = base64.b64decode(self.datas.decode("utf-8")) - target_function = "_parse_data_{format}".format( - format=self.pattern_config_id.export_format or "" - ) - if not hasattr(self, target_function): - raise NotImplementedError() - return getattr(self, target_function)(data) - - def _parse_data_json(self, data): - items = json.loads(data.decode("utf-8")) - for idx, item in enumerate(items): - yield idx + 1, item - - def _prepare_chunk(self, start_idx, stop_idx, data): - return { - "start_idx": start_idx, - "stop_idx": stop_idx, - "data": data, - "nbr_item": len(data), - "state": "pending", - "pattern_file_id": self.id, - } - - def _should_create_chunk(self, items, next_item): - """Customise this code if you want to add some additionnal - item after reaching the limit""" - return len(items) > self.pattern_config_id.chunk_size - - def _create_chunk(self, start_idx, stop_idx, data): - vals = self._prepare_chunk(start_idx, stop_idx, data) - chunk = self.env["pattern.chunk"].create(vals) - # we enqueue the chunk in case of multi process of if it's the first chunk - if self.pattern_config_id.process_multi or len(self.chunk_ids) == 1: - chunk.with_delay(priority=self.pattern_config_id.job_priority).run() - return chunk - def split_in_chunk(self): - """Split Pattern File into Pattern Chunk""" - # purge chunk in case of retring a job - self.chunk_ids.unlink() - try: - items = [] - start_idx = 1 - previous_idx = None - # idx is the index position in the original file - # we can have empty line that can be skipped - for idx, item in self._parse_data(): - if self._should_create_chunk(items, item): - self._create_chunk(start_idx, previous_idx, items) - items = [] - start_idx = idx - items.append((idx, item)) - previous_idx = idx - if items: - self._create_chunk(start_idx, idx, items) - except Exception as e: - self.state = "failed" - self.info = _("Failed to create the chunk: %s") % e - return True - - def set_import_done(self): - for record in self: - if record.nbr_error: - record.state = "failed" - else: - record.state = "done" - record.date_done = fields.Datetime.now() + return self.chunk_group_id.split_in_chunk() def refresh(self): """Empty function to refresh view""" diff --git a/pattern_import_export/security/ir.model.access.csv b/pattern_import_export/security/ir.model.access.csv index c97bba1b..02566180 100644 --- a/pattern_import_export/security/ir.model.access.csv +++ b/pattern_import_export/security/ir.model.access.csv @@ -2,6 +2,5 @@ id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink access_export_pattern_wizard_user,export.pattern.wizard.user,model_export_pattern_wizard,group_pattern_user,1,1,1,1 access_import_pattern_wizard_user,import.pattern.wizard.user,model_import_pattern_wizard,group_pattern_user,1,1,1,1 access_pattern_file_user,pattern.file.user,model_pattern_file,group_pattern_user,1,1,1,1 -access_pattern_chunk_user,pattern.chunk.user,model_pattern_chunk,group_pattern_user,1,1,1,1 access_pattern_config_user,pattern.config.user,model_pattern_config,group_pattern_user,1,0,0,0 access_pattern_config_manager,pattern.config.manager,model_pattern_config,group_pattern_manager,1,1,1,1 diff --git a/pattern_import_export/tests/test_pattern_import.py b/pattern_import_export/tests/test_pattern_import.py index fa326c29..9776deb4 100644 --- a/pattern_import_export/tests/test_pattern_import.py +++ b/pattern_import_export/tests/test_pattern_import.py @@ -2,13 +2,14 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). from uuid import uuid4 -from odoo.tests.common import SavepointCase from odoo.tools import mute_logger +from odoo.addons.component.tests.common import SavepointComponentCase + from .common import PatternCommon -class TestPatternImport(PatternCommon, SavepointCase): +class TestPatternImport(PatternCommon, SavepointComponentCase): @classmethod def setUpClass(cls): super().setUpClass() @@ -58,7 +59,7 @@ def test_update_with_external_id_bad_data_1(self): self.assertEqual(pattern_file.state, "failed") # TODO it will be better to retour a better exception # but it's not that easy - chunk = pattern_file.chunk_ids + chunk = pattern_file.chunk_item_ids self.assertEqual( chunk.result_info, "

Fail to process chunk 'int' object has no attribute 'split'

", @@ -76,7 +77,7 @@ def test_update_with_external_id_bad_data_2(self): records = self.run_pattern_file(pattern_file) self.assertFalse(records) self.assertEqual(pattern_file.state, "failed") - chunk = pattern_file.chunk_ids + chunk = pattern_file.chunk_item_ids self.assertIn("Invalid database identifier", chunk.result_info) def test_create_new_record(self): @@ -254,7 +255,7 @@ def test_wrong_import(self): self.run_pattern_file(pattern_file) self.assertEqual(pattern_file.state, "failed") self.assertEqual(pattern_file.nbr_error, 1) - self.assertIn("res_partner_check_name", pattern_file.chunk_ids.result_info) + self.assertIn("res_partner_check_name", pattern_file.chunk_item_ids.result_info) def test_m2m_with_empty_columns(self): unique_name = str(uuid4()) @@ -334,7 +335,7 @@ def test_missing_record(self): "No value found for model 'Country' with the field 'code' " "and the value 'Fake'" ), - pattern_file.chunk_ids.result_info, + pattern_file.chunk_item_ids.result_info, ) def test_import_m2o_key(self): @@ -415,5 +416,9 @@ def test_partial_import_too_many_error(self): self.assertEqual(len(records), 2) self.assertEqual(pattern_file.state, "failed") self.assertEqual(pattern_file.nbr_error, 16) - self.assertIn("Contacts require a name", pattern_file.chunk_ids.result_info) - self.assertIn("Found more than 10 errors", pattern_file.chunk_ids.result_info) + self.assertIn( + "Contacts require a name", pattern_file.chunk_item_ids.result_info + ) + self.assertIn( + "Found more than 10 errors", pattern_file.chunk_item_ids.result_info + ) diff --git a/pattern_import_export/views/pattern_file.xml b/pattern_import_export/views/pattern_file.xml index 001e3acf..eafca5f3 100644 --- a/pattern_import_export/views/pattern_file.xml +++ b/pattern_import_export/views/pattern_file.xml @@ -12,9 +12,9 @@ - + @@ -76,7 +76,7 @@ readonly="1" attrs="{'invisible': [('info', '=', False)]}" /> - + diff --git a/setup/chunk_processing/odoo/addons/chunk_processing b/setup/chunk_processing/odoo/addons/chunk_processing new file mode 120000 index 00000000..d865efb9 --- /dev/null +++ b/setup/chunk_processing/odoo/addons/chunk_processing @@ -0,0 +1 @@ +../../../../chunk_processing \ No newline at end of file diff --git a/setup/chunk_processing/setup.py b/setup/chunk_processing/setup.py new file mode 100644 index 00000000..28c57bb6 --- /dev/null +++ b/setup/chunk_processing/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +) From 791bfcbf32992f8a749fdc214038320f784bac06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Thu, 9 Dec 2021 18:12:02 +0100 Subject: [PATCH 02/10] [WIP] start adding xml support --- chunk_processing/__manifest__.py | 1 + chunk_processing/components/__init__.py | 2 + chunk_processing/components/processor_xml.py | 26 ++++++++ chunk_processing/components/splitter.py | 11 +++- chunk_processing/components/splitter_xml.py | 25 ++++++++ chunk_processing/models/chunk_group.py | 1 + chunk_processing/models/chunk_item.py | 2 +- chunk_processing/views/chunk_group_view.xml | 64 ++++++++++++++++++++ chunk_processing/views/chunk_item_view.xml | 1 + pattern_import_export/models/pattern_file.py | 4 +- 10 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 chunk_processing/components/processor_xml.py create mode 100644 chunk_processing/components/splitter_xml.py create mode 100644 chunk_processing/views/chunk_group_view.xml diff --git a/chunk_processing/__manifest__.py b/chunk_processing/__manifest__.py index 934a070a..0a17d355 100644 --- a/chunk_processing/__manifest__.py +++ b/chunk_processing/__manifest__.py @@ -23,6 +23,7 @@ ], "data": [ "views/chunk_item_view.xml", + "views/chunk_group_view.xml", ], "demo": [], } diff --git a/chunk_processing/components/__init__.py b/chunk_processing/components/__init__.py index 3242cabf..0466f74e 100644 --- a/chunk_processing/components/__init__.py +++ b/chunk_processing/components/__init__.py @@ -1,3 +1,5 @@ from . import processor +from . import processor_xml from . import splitter from . import splitter_json +from . import splitter_xml diff --git a/chunk_processing/components/processor_xml.py b/chunk_processing/components/processor_xml.py new file mode 100644 index 00000000..ff85cd0b --- /dev/null +++ b/chunk_processing/components/processor_xml.py @@ -0,0 +1,26 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 + +from lxml import objectify + +from odoo.addons.component.core import AbstractComponent + + +class ChunkProcessorXml(AbstractComponent): + _name = "chunk.importer.xml" + _collection = "chunk.item" + + def _parse_data(self): + return objectify.fromstring( + base64.b64decode(self.collection.data) + ).iterchildren() + + def _import_item(self): + raise NotImplementedError + + def run(self): + for item in self._parse_data(): + self._import_item(item) diff --git a/chunk_processing/components/splitter.py b/chunk_processing/components/splitter.py index 96813c88..c939d6a8 100644 --- a/chunk_processing/components/splitter.py +++ b/chunk_processing/components/splitter.py @@ -2,6 +2,8 @@ # @author Sébastien BEAU # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +import base64 + from odoo.addons.component.core import AbstractComponent @@ -12,12 +14,15 @@ class ChunkSplitter(AbstractComponent): def _parse_data(self, data): raise NotImplementedError - def _prepare_chunk(self, start_idx, stop_idx, data): + def _convert_items_to_data(self, items): + raise NotImplementedError + + def _prepare_chunk(self, start_idx, stop_idx, items): return { "start_idx": start_idx, "stop_idx": stop_idx, - "data": data, - "nbr_item": len(data), + "data": base64.b64encode(self._convert_items_to_data(items)), + "nbr_item": len(items), "state": "pending", "group_id": self.collection.id, } diff --git a/chunk_processing/components/splitter_xml.py b/chunk_processing/components/splitter_xml.py new file mode 100644 index 00000000..88df4890 --- /dev/null +++ b/chunk_processing/components/splitter_xml.py @@ -0,0 +1,25 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from lxml import etree + +from odoo.addons.component.core import Component + + +class ChunkSplitterXml(Component): + _inherit = "chunk.splitter" + _name = "chunk.splitter.xml" + _usage = "xml" + + def _parse_data(self, data): + tree = etree.fromstring(data) + items = tree.xpath(self.collection.xml_split_xpath) + for idx, item in enumerate(items): + yield idx + 1, item + + def _convert_items_to_data(self, items): + data = etree.Element("data") + for item in items: + data.append(item[1]) + return etree.tostring(data) diff --git a/chunk_processing/models/chunk_group.py b/chunk_processing/models/chunk_group.py index d2b64be7..df02093a 100644 --- a/chunk_processing/models/chunk_group.py +++ b/chunk_processing/models/chunk_group.py @@ -22,6 +22,7 @@ class ChunkGroup(models.Model): ("xml", "XML"), ] ) + xml_split_xpath = fields.Char() state = fields.Selection( [("pending", "Pending"), ("failed", "Failed"), ("done", "Done")], default="pending", diff --git a/chunk_processing/models/chunk_item.py b/chunk_processing/models/chunk_item.py index 562f2d1b..d80cca5f 100644 --- a/chunk_processing/models/chunk_item.py +++ b/chunk_processing/models/chunk_item.py @@ -17,7 +17,7 @@ class ChunkItem(models.Model): ) start_idx = fields.Integer() stop_idx = fields.Integer() - data = fields.Serialized() + data = fields.Binary() record_ids = fields.Serialized() messages = fields.Serialized() result_info = fields.Html() diff --git a/chunk_processing/views/chunk_group_view.xml b/chunk_processing/views/chunk_group_view.xml new file mode 100644 index 00000000..cc9fd600 --- /dev/null +++ b/chunk_processing/views/chunk_group_view.xml @@ -0,0 +1,64 @@ + + + + + chunk.group + + + + + + + + + + chunk.group + + +
+
+ + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + diff --git a/chunk_processing/views/chunk_item_view.xml b/chunk_processing/views/chunk_item_view.xml index e1334344..f1d30232 100644 --- a/chunk_processing/views/chunk_item_view.xml +++ b/chunk_processing/views/chunk_item_view.xml @@ -23,6 +23,7 @@ + diff --git a/pattern_import_export/models/pattern_file.py b/pattern_import_export/models/pattern_file.py index ee085337..8f6e4c2b 100644 --- a/pattern_import_export/models/pattern_file.py +++ b/pattern_import_export/models/pattern_file.py @@ -16,7 +16,7 @@ class PatternFile(models.Model): pattern_config_id = fields.Many2one( "pattern.config", required=True, string="Export pattern" ) - chunk_group_id = fields.Many2one("chunk.group") + chunk_group_id = fields.Many2one("chunk.group", string="Chunk Group") chunk_item_ids = fields.One2many("chunk.item", related="chunk_group_id.item_ids") state = fields.Selection( [("pending", "Pending"), ("failed", "Failed"), ("done", "Done")], @@ -29,7 +29,7 @@ class PatternFile(models.Model): info = fields.Char(related="chunk_group_id.info") _sql_constraints = [ - ("uniq_group_id", "unique(group_id)", "The Group must be unique!") + ("uniq_chunk_group_id", "unique(chunk_group_id)", "The Group must be unique!") ] def _add_chunk_group(self): From 1c67416c3a81e071a18d85d28c2447142eaced91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Mon, 20 Dec 2021 23:40:37 +0100 Subject: [PATCH 03/10] [IMP] automatically run split after create --- chunk_processing/models/chunk_group.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/chunk_processing/models/chunk_group.py b/chunk_processing/models/chunk_group.py index df02093a..1e3f02bf 100644 --- a/chunk_processing/models/chunk_group.py +++ b/chunk_processing/models/chunk_group.py @@ -68,3 +68,10 @@ def set_done(self): else: record.state = "done" record.date_done = fields.Datetime.now() + + @api.model_create_multi + def create(self, vals_list): + records = super().create(vals_list) + for record in records: + record.with_delay(priority=self.job_priority).split_in_chunk() + return records From 7ebe578da6bb2aa4cea9a8ef5882df3b4c7bc371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Wed, 22 Dec 2021 23:24:15 +0100 Subject: [PATCH 04/10] [IMP] continue to refactor, start to reimplement error management --- chunk_processing/__manifest__.py | 2 ++ chunk_processing/components/processor_xml.py | 16 ++++++++-- chunk_processing/models/chunk_group.py | 7 +++-- chunk_processing/models/chunk_item.py | 31 +++++++++++-------- .../views/templates.xml | 0 pattern_import_export/__manifest__.py | 1 - 6 files changed, 39 insertions(+), 18 deletions(-) rename {pattern_import_export => chunk_processing}/views/templates.xml (100%) diff --git a/chunk_processing/__manifest__.py b/chunk_processing/__manifest__.py index 0a17d355..f85d2855 100644 --- a/chunk_processing/__manifest__.py +++ b/chunk_processing/__manifest__.py @@ -20,10 +20,12 @@ "depends": [ "queue_job", "component", + "web_refresher", ], "data": [ "views/chunk_item_view.xml", "views/chunk_group_view.xml", + "views/templates.xml", ], "demo": [], } diff --git a/chunk_processing/components/processor_xml.py b/chunk_processing/components/processor_xml.py index ff85cd0b..7197601a 100644 --- a/chunk_processing/components/processor_xml.py +++ b/chunk_processing/components/processor_xml.py @@ -22,5 +22,17 @@ def _import_item(self): raise NotImplementedError def run(self): - for item in self._parse_data(): - self._import_item(item) + res = {"ids": [], "messages": []} + for idx, item in enumerate(self._parse_data()): + try: + with self.env.cr.savepoint(): + res["ids"] += self._import_item(item) + except Exception as e: + res["messages"].append( + { + "rows": {"from": idx, "to": idx}, + "type": type(e).__name__, + "message": str(e), + } + ) + return res diff --git a/chunk_processing/models/chunk_group.py b/chunk_processing/models/chunk_group.py index 1e3f02bf..968cfa7b 100644 --- a/chunk_processing/models/chunk_group.py +++ b/chunk_processing/models/chunk_group.py @@ -57,8 +57,11 @@ def split_in_chunk(self): splitter = work.component(usage=self.data_format) splitter.run(data) except Exception as e: - self.state = "failed" - self.info = _("Failed to create the chunk: %s") % e + if self._context.get("chunk_raise_if_exception"): + raise + else: + self.state = "failed" + self.info = _("Failed to create the chunk: %s") % e return True def set_done(self): diff --git a/chunk_processing/models/chunk_item.py b/chunk_processing/models/chunk_item.py index d80cca5f..92113825 100644 --- a/chunk_processing/models/chunk_item.py +++ b/chunk_processing/models/chunk_item.py @@ -40,7 +40,10 @@ def manual_run(self): def _run(self): with self.work_on(self.group_id.apply_on_model) as work: processor = work.component(usage=self.group_id.usage) - processor.run() + res = processor.run() + vals = self._prepare_chunk_result(res) + self.write(vals) + if not self.group_id.process_multi: next_chunk = self.get_next_chunk() if next_chunk: @@ -49,6 +52,7 @@ def _run(self): self.with_delay(priority=5).check_last() else: self.with_delay(priority=5).check_last() + return True def run(self): """Process Chunk Item in a savepoint""" @@ -59,22 +63,25 @@ def run(self): with cr.savepoint(): self._run() except Exception as e: - self.write( - { - "result_info": "Fail to process chunk %s" % e, - "nbr_error": self.nbr_item, - "state": "failed", - } - ) - self.with_delay().check_last() + if self._context.get("chunk_raise_if_exception"): + raise + else: + self.write( + { + "result_info": "Fail to process chunk %s" % e, + "nbr_error": self.nbr_item, + "state": "failed", + } + ) + self.with_delay().check_last() return "OK" - # TODO move this in pattern-import def _prepare_chunk_result(self, res): # TODO rework this part and add specific test case nbr_error = len(res["messages"]) nbr_success = max(self.nbr_item - nbr_error, 0) + # TODO move this in pattern-import # case where error are not return and record are not imported nbr_imported = len(res.get("ids") or []) if nbr_success > nbr_imported: @@ -85,9 +92,7 @@ def _prepare_chunk_result(self, res): state = "failed" else: state = "done" - result = self.env["ir.qweb"]._render( - "pattern_import_export.format_message", res - ) + result = self.env["ir.qweb"]._render("chunk_processing.format_message", res) return { "record_ids": res.get("ids"), "messages": res.get("messages"), diff --git a/pattern_import_export/views/templates.xml b/chunk_processing/views/templates.xml similarity index 100% rename from pattern_import_export/views/templates.xml rename to chunk_processing/views/templates.xml diff --git a/pattern_import_export/__manifest__.py b/pattern_import_export/__manifest__.py index 7d50c134..2b943ec6 100644 --- a/pattern_import_export/__manifest__.py +++ b/pattern_import_export/__manifest__.py @@ -25,7 +25,6 @@ "views/pattern_config.xml", "views/pattern_file.xml", "views/menuitems.xml", - "views/templates.xml", "data/queue_job_channel_data.xml", # "data/queue_job_function_data.xml", ], From 0b6bab2cbc65a063b0ec6d5f24b6c5fb5ff75a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Thu, 23 Dec 2021 09:40:39 +0100 Subject: [PATCH 05/10] [IMP] raise if needed --- chunk_processing/components/processor_xml.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chunk_processing/components/processor_xml.py b/chunk_processing/components/processor_xml.py index 7197601a..5d4cb246 100644 --- a/chunk_processing/components/processor_xml.py +++ b/chunk_processing/components/processor_xml.py @@ -28,6 +28,8 @@ def run(self): with self.env.cr.savepoint(): res["ids"] += self._import_item(item) except Exception as e: + if self.env.context.get("chunk_raise_if_exception"): + raise res["messages"].append( { "rows": {"from": idx, "to": idx}, From 6ad30d57a498abe9038bd0b12c25e3cd560b1272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Tue, 18 Jan 2022 15:56:52 +0100 Subject: [PATCH 06/10] [IMP] improve api: add possibility to customize error message --- chunk_processing/components/processor_xml.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/chunk_processing/components/processor_xml.py b/chunk_processing/components/processor_xml.py index 5d4cb246..0f7f8c9c 100644 --- a/chunk_processing/components/processor_xml.py +++ b/chunk_processing/components/processor_xml.py @@ -21,6 +21,13 @@ def _parse_data(self): def _import_item(self): raise NotImplementedError + def _prepare_error_message(self, idx, item, error): + return { + "rows": {"from": idx, "to": idx}, + "type": type(error).__name__, + "message": str(error), + } + def run(self): res = {"ids": [], "messages": []} for idx, item in enumerate(self._parse_data()): @@ -30,11 +37,5 @@ def run(self): except Exception as e: if self.env.context.get("chunk_raise_if_exception"): raise - res["messages"].append( - { - "rows": {"from": idx, "to": idx}, - "type": type(e).__name__, - "message": str(e), - } - ) + res["messages"].append(self._prepare_error_message(idx, item, e)) return res From 9f70400e5e1b86cad92fc20e5d783e84e0e1577b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Mon, 7 Feb 2022 12:33:06 +0100 Subject: [PATCH 07/10] [IMP] add security rule --- chunk_processing/security/ir.model.access.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 chunk_processing/security/ir.model.access.csv diff --git a/chunk_processing/security/ir.model.access.csv b/chunk_processing/security/ir.model.access.csv new file mode 100644 index 00000000..a32970c3 --- /dev/null +++ b/chunk_processing/security/ir.model.access.csv @@ -0,0 +1,3 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_chunk_group,Chunk group System,model_chunk_group,base.group_system,1,1,1,1 +access_chunk_item,Chunk item System,model_chunk_item,base.group_system,1,1,1,1 From 9d3a54fc9a4c55e7d0122305d668c35b586a7292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Fri, 22 Apr 2022 15:39:57 +0200 Subject: [PATCH 08/10] chunk_processing: add filename on chunk item and fix json splitter --- chunk_processing/components/splitter_json.py | 3 +++ chunk_processing/models/chunk_item.py | 7 +++++++ chunk_processing/views/chunk_item_view.xml | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/chunk_processing/components/splitter_json.py b/chunk_processing/components/splitter_json.py index 0220e6c1..40f16d31 100644 --- a/chunk_processing/components/splitter_json.py +++ b/chunk_processing/components/splitter_json.py @@ -12,6 +12,9 @@ class ChunkSplitterJson(Component): _name = "chunk.splitter.json" _usage = "json" + def _convert_items_to_data(self, items): + return json.dumps(items, indent=2).encode("utf-8") + def _parse_data(self, data): items = json.loads(data.decode("utf-8")) for idx, item in enumerate(items): diff --git a/chunk_processing/models/chunk_item.py b/chunk_processing/models/chunk_item.py index 92113825..1fcd4e94 100644 --- a/chunk_processing/models/chunk_item.py +++ b/chunk_processing/models/chunk_item.py @@ -32,6 +32,13 @@ class ChunkItem(models.Model): ("failed", "Failed"), ] ) + filename = fields.Char(compute="_compute_filename") + + def _compute_filename(self): + for record in self: + record.filename = ( + f"{record.start_idx}-{record.stop_idx}.{record.group_id.data_format}" + ) def manual_run(self): """ Run the import without try/except, easier for debug """ diff --git a/chunk_processing/views/chunk_item_view.xml b/chunk_processing/views/chunk_item_view.xml index f1d30232..3b69ab4a 100644 --- a/chunk_processing/views/chunk_item_view.xml +++ b/chunk_processing/views/chunk_item_view.xml @@ -23,7 +23,8 @@ - + + From 9354a87edb7740c7e0ca6fb7d8b92ca65ca34ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Fri, 22 Apr 2022 16:18:18 +0200 Subject: [PATCH 09/10] chunk_processing: add json processor --- chunk_processing/components/__init__.py | 1 + chunk_processing/components/processor.py | 21 ++++++++++++++++- chunk_processing/components/processor_json.py | 17 ++++++++++++++ chunk_processing/components/processor_xml.py | 23 +------------------ 4 files changed, 39 insertions(+), 23 deletions(-) create mode 100644 chunk_processing/components/processor_json.py diff --git a/chunk_processing/components/__init__.py b/chunk_processing/components/__init__.py index 0466f74e..faf2ed24 100644 --- a/chunk_processing/components/__init__.py +++ b/chunk_processing/components/__init__.py @@ -1,5 +1,6 @@ from . import processor from . import processor_xml +from . import processor_json from . import splitter from . import splitter_json from . import splitter_xml diff --git a/chunk_processing/components/processor.py b/chunk_processing/components/processor.py index 65ac3361..6d27b829 100644 --- a/chunk_processing/components/processor.py +++ b/chunk_processing/components/processor.py @@ -10,5 +10,24 @@ class ChunkProcessor(AbstractComponent): _name = "chunk.processor" _collection = "chunk.item" - def run(self): + def _import_item(self): raise NotImplementedError + + def _prepare_error_message(self, idx, item, error): + return { + "rows": {"from": idx, "to": idx}, + "type": type(error).__name__, + "message": str(error), + } + + def run(self): + res = {"ids": [], "messages": []} + for idx, item in enumerate(self._parse_data()): + try: + with self.env.cr.savepoint(): + res["ids"] += self._import_item(item) + except Exception as e: + if self.env.context.get("chunk_raise_if_exception"): + raise + res["messages"].append(self._prepare_error_message(idx, item, e)) + return res diff --git a/chunk_processing/components/processor_json.py b/chunk_processing/components/processor_json.py new file mode 100644 index 00000000..6fb8f1d5 --- /dev/null +++ b/chunk_processing/components/processor_json.py @@ -0,0 +1,17 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 +import json + +from odoo.addons.component.core import AbstractComponent + + +class ChunkProcessorJson(AbstractComponent): + _name = "chunk.importer.json" + _inherit = "chunk.processor" + _collection = "chunk.item" + + def _parse_data(self): + return json.loads(base64.b64decode(self.collection.data)) diff --git a/chunk_processing/components/processor_xml.py b/chunk_processing/components/processor_xml.py index 0f7f8c9c..95a25dd5 100644 --- a/chunk_processing/components/processor_xml.py +++ b/chunk_processing/components/processor_xml.py @@ -11,31 +11,10 @@ class ChunkProcessorXml(AbstractComponent): _name = "chunk.importer.xml" + _inherit = "chunk.processor" _collection = "chunk.item" def _parse_data(self): return objectify.fromstring( base64.b64decode(self.collection.data) ).iterchildren() - - def _import_item(self): - raise NotImplementedError - - def _prepare_error_message(self, idx, item, error): - return { - "rows": {"from": idx, "to": idx}, - "type": type(error).__name__, - "message": str(error), - } - - def run(self): - res = {"ids": [], "messages": []} - for idx, item in enumerate(self._parse_data()): - try: - with self.env.cr.savepoint(): - res["ids"] += self._import_item(item) - except Exception as e: - if self.env.context.get("chunk_raise_if_exception"): - raise - res["messages"].append(self._prepare_error_message(idx, item, e)) - return res From e95706739d17504532e857c682c22fc220c4bbeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20BEAU?= Date: Tue, 10 May 2022 16:25:26 +0200 Subject: [PATCH 10/10] chunk_processing: add txt splitter --- chunk_processing/components/__init__.py | 2 ++ chunk_processing/components/processor_txt.py | 17 +++++++++++++++++ chunk_processing/components/splitter_txt.py | 20 ++++++++++++++++++++ chunk_processing/models/chunk_group.py | 1 + 4 files changed, 40 insertions(+) create mode 100644 chunk_processing/components/processor_txt.py create mode 100644 chunk_processing/components/splitter_txt.py diff --git a/chunk_processing/components/__init__.py b/chunk_processing/components/__init__.py index faf2ed24..c6e3896d 100644 --- a/chunk_processing/components/__init__.py +++ b/chunk_processing/components/__init__.py @@ -1,6 +1,8 @@ from . import processor from . import processor_xml from . import processor_json +from . import processor_txt from . import splitter from . import splitter_json from . import splitter_xml +from . import splitter_txt diff --git a/chunk_processing/components/processor_txt.py b/chunk_processing/components/processor_txt.py new file mode 100644 index 00000000..a996e76c --- /dev/null +++ b/chunk_processing/components/processor_txt.py @@ -0,0 +1,17 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 + +from odoo.addons.component.core import AbstractComponent + + +class ChunkProcessorTxt(AbstractComponent): + _name = "chunk.importer.txt" + _inherit = "chunk.processor" + _collection = "chunk.item" + _end_of_line = b"\n" + + def _parse_data(self): + return base64.b64decode(self.collection.data).split(self._end_of_line) diff --git a/chunk_processing/components/splitter_txt.py b/chunk_processing/components/splitter_txt.py new file mode 100644 index 00000000..27ac179d --- /dev/null +++ b/chunk_processing/components/splitter_txt.py @@ -0,0 +1,20 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component + + +class ChunkSplitterTxt(Component): + _inherit = "chunk.splitter" + _name = "chunk.splitter.txt" + _usage = "txt" + _end_of_line = b"\n" + + def _parse_data(self, data): + for idx, item in enumerate(data.split(self._end_of_line)): + if item: + yield idx + 1, item + + def _convert_items_to_data(self, items): + return self._end_of_line.join([x[1] for x in items]) diff --git a/chunk_processing/models/chunk_group.py b/chunk_processing/models/chunk_group.py index 968cfa7b..db182261 100644 --- a/chunk_processing/models/chunk_group.py +++ b/chunk_processing/models/chunk_group.py @@ -20,6 +20,7 @@ class ChunkGroup(models.Model): [ ("json", "Json"), ("xml", "XML"), + ("txt", "Txt"), ] ) xml_split_xpath = fields.Char()