From 6a24dec96fc2d68cf008880af7e32bdaacb01ec5 Mon Sep 17 00:00:00 2001 From: Ronald Portier Date: Wed, 10 Jan 2024 22:56:47 +0100 Subject: [PATCH] [FIX] fetchmail_*: adapt to Odoo 16.0 and python 3.x --- .../match_algorithm/__init__.py | 1 - .../match_algorithm/base.py | 14 -- .../match_algorithm/email_domain.py | 10 +- .../match_algorithm/email_exact.py | 21 +- .../match_algorithm/odoo_standard.py | 23 -- .../models/__init__.py | 1 + .../models/fetchmail_server.py | 12 +- .../models/fetchmail_server_folder.py | 202 +++++++----------- .../models/mail_thread.py | 70 ++++++ .../security/ir.model.access.csv | 2 + .../tests/test_match_algorithms.py | 28 +-- .../views/fetchmail_server.xml | 16 +- .../wizard/attach_mail_manually.py | 6 + 13 files changed, 188 insertions(+), 218 deletions(-) delete mode 100644 fetchmail_attach_from_folder/match_algorithm/base.py delete mode 100644 fetchmail_attach_from_folder/match_algorithm/odoo_standard.py create mode 100644 fetchmail_attach_from_folder/models/mail_thread.py diff --git a/fetchmail_attach_from_folder/match_algorithm/__init__.py b/fetchmail_attach_from_folder/match_algorithm/__init__.py index b74ba409cbd..93da714acaf 100644 --- a/fetchmail_attach_from_folder/match_algorithm/__init__.py +++ b/fetchmail_attach_from_folder/match_algorithm/__init__.py @@ -2,4 +2,3 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from . import email_exact from . import email_domain -from . import odoo_standard diff --git a/fetchmail_attach_from_folder/match_algorithm/base.py b/fetchmail_attach_from_folder/match_algorithm/base.py deleted file mode 100644 index 850124a8eb2..00000000000 --- a/fetchmail_attach_from_folder/match_algorithm/base.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright - 2013-2024 Therp BV . -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). - - -class Base(object): - def search_matches(self, folder, mail_message): - """Returns recordset found for model with mail_message.""" - return [] - - def handle_match( - self, connection, match_object, folder, mail_message, mail_message_org, msgid - ): - """Do whatever it takes to handle a match""" - folder.attach_mail(match_object, mail_message) diff --git a/fetchmail_attach_from_folder/match_algorithm/email_domain.py b/fetchmail_attach_from_folder/match_algorithm/email_domain.py index 7ad6bfb7977..a256459499e 100644 --- a/fetchmail_attach_from_folder/match_algorithm/email_domain.py +++ b/fetchmail_attach_from_folder/match_algorithm/email_domain.py @@ -9,21 +9,21 @@ class EmailDomain(EmailExact): Beware of match_first here, this is most likely to get it wrong (gmail). """ - def search_matches(self, folder, mail_message): + def search_matches(self, folder, message_dict): """Returns recordset of matching objects.""" - matches = super().search_matches(folder, mail_message) + matches = super().search_matches(folder, message_dict) if not matches: object_model = folder.env[folder.model_id.model] domains = [] - for addr in self._get_mailaddresses(folder, mail_message): + for addr in self._get_mailaddresses(folder, message_dict): domains.append(addr.split("@")[-1]) matches = object_model.search( self._get_mailaddress_search_domain( folder, - mail_message, + message_dict, operator="like", values=["%@" + domain for domain in set(domains)], ), order=folder.model_order, ) - return matches + return matches.ids diff --git a/fetchmail_attach_from_folder/match_algorithm/email_exact.py b/fetchmail_attach_from_folder/match_algorithm/email_exact.py index ea4ed1e5efb..b454f8d0af9 100644 --- a/fetchmail_attach_from_folder/match_algorithm/email_exact.py +++ b/fetchmail_attach_from_folder/match_algorithm/email_exact.py @@ -3,24 +3,22 @@ from odoo.tools.mail import email_split from odoo.tools.safe_eval import safe_eval -from .base import Base - -class EmailExact(Base): +class EmailExact: """Search for exactly the mailadress as noted in the email""" - def _get_mailaddresses(self, folder, mail_message): + def _get_mailaddresses(self, folder, message_dict): mailaddresses = [] fields = folder.mail_field.split(",") for field in fields: - if field in mail_message: - mailaddresses += email_split(mail_message[field]) + if field in message_dict: + mailaddresses += email_split(message_dict[field]) return [addr.lower() for addr in mailaddresses] def _get_mailaddress_search_domain( - self, folder, mail_message, operator="=", values=None + self, folder, message_dict, operator="=", values=None ): - mailaddresses = values or self._get_mailaddresses(folder, mail_message) + mailaddresses = values or self._get_mailaddresses(folder, message_dict) if not mailaddresses: return [(0, "=", 1)] search_domain = ( @@ -30,8 +28,9 @@ def _get_mailaddress_search_domain( ) return search_domain - def search_matches(self, folder, mail_message): + def search_matches(self, folder, message_dict): """Returns recordset of matching objects.""" object_model = folder.env[folder.model_id.model] - search_domain = self._get_mailaddress_search_domain(folder, mail_message) - return object_model.search(search_domain, order=folder.model_order) + search_domain = self._get_mailaddress_search_domain(folder, message_dict) + matches = object_model.search(search_domain, order=folder.model_order) + return matches.ids diff --git a/fetchmail_attach_from_folder/match_algorithm/odoo_standard.py b/fetchmail_attach_from_folder/match_algorithm/odoo_standard.py deleted file mode 100644 index d2b77dde104..00000000000 --- a/fetchmail_attach_from_folder/match_algorithm/odoo_standard.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright - 2013-2024 Therp BV . -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -from .base import Base - - -class OdooStandard(Base): - """No search at all. Use Odoo's standard mechanism to attach mails to - mail.thread objects. Note that this algorithm always matches.""" - - def search_matches(self, folder, mail_message): - """Always match. Duplicates will be fished out by message_id""" - return [True] - - def handle_match( - self, connection, match_object, folder, mail_message, mail_message_org, msgid - ): - thread_model = folder.env["mail.thread"] - thread_model.message_process( - folder.model_id.model, - mail_message_org, - save_original=folder.server_id.original, - strip_attachments=(not folder.server_id.attach), - ) diff --git a/fetchmail_attach_from_folder/models/__init__.py b/fetchmail_attach_from_folder/models/__init__.py index 0340cfd077b..c5a9d317e31 100644 --- a/fetchmail_attach_from_folder/models/__init__.py +++ b/fetchmail_attach_from_folder/models/__init__.py @@ -2,3 +2,4 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from . import fetchmail_server from . import fetchmail_server_folder +from . import mail_thread diff --git a/fetchmail_attach_from_folder/models/fetchmail_server.py b/fetchmail_attach_from_folder/models/fetchmail_server.py index 2e0d9373959..7d344395ab4 100644 --- a/fetchmail_attach_from_folder/models/fetchmail_server.py +++ b/fetchmail_attach_from_folder/models/fetchmail_server.py @@ -19,7 +19,10 @@ def _compute_folders_available(self): """Retrieve available folders from IMAP server.""" def parse_list_response(line): - flags, delimiter, mailbox_name = list_response_pattern.match(line).groups() + string_line = line.decode("utf-8") + flags, delimiter, mailbox_name = list_response_pattern.match( + string_line + ).groups() mailbox_name = mailbox_name.strip('"') return (flags, delimiter, mailbox_name) @@ -34,7 +37,7 @@ def parse_list_response(line): continue folders_available = [] for folder_entry in list_result[1]: - folders_available.append(parse_list_response(str(folder_entry))[2]) + folders_available.append(parse_list_response(folder_entry)[2]) this.folders_available = "\n".join(folders_available) connection.logout() @@ -47,13 +50,14 @@ def parse_list_response(line): string="Folders", context={"active_test": False}, ) - object_id = fields.Many2one(required=False) # comodel_name='ir.model' - server_type = fields.Selection(default="imap") folders_only = fields.Boolean( string="Only folders, not inbox", help="Check this field to leave imap inbox alone" " and only retrieve mail from configured folders.", ) + # Below existing fields, that are modified by this module. + object_id = fields.Many2one(required=False) # comodel_name='ir.model' + server_type = fields.Selection(default="imap") @api.onchange("server_type", "is_ssl", "object_id") def onchange_server_type(self): diff --git a/fetchmail_attach_from_folder/models/fetchmail_server_folder.py b/fetchmail_attach_from_folder/models/fetchmail_server_folder.py index 207dabb10c4..a733596750c 100644 --- a/fetchmail_attach_from_folder/models/fetchmail_server_folder.py +++ b/fetchmail_attach_from_folder/models/fetchmail_server_folder.py @@ -1,18 +1,18 @@ # Copyright - 2013-2024 Therp BV . # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -import base64 import logging from odoo import _, fields, models from odoo.exceptions import UserError, ValidationError -from .. import match_algorithm - _logger = logging.getLogger(__name__) class FetchmailServerFolder(models.Model): + """Define folders (IMAP mailboxes) from which to fetch mail.""" + _name = "fetchmail.server.folder" + _description = __doc__ _rec_name = "path" _order = "sequence" @@ -84,22 +84,6 @@ class FetchmailServerFolder(models.Model): ) active = fields.Boolean(default=True) - def get_algorithm(self): - """Translate algorithm code to implementation class. - - We used to load this dynamically, but having it more or less hardcoded - allows to adapt the UI to the selected algorithm, withouth needing - the (deprecated) fields_view_get trickery we used in the past. - """ - self.ensure_one() - if self.match_algorithm == "odoo_standard": - return match_algorithm.odoo_standard.OdooStandard - if self.match_algorithm == "email_domain": - return match_algorithm.email_domain.EmailDomain - if self.match_algorithm == "email_exact": - return match_algorithm.email_exact.EmailExact - return None - def button_confirm_folder(self): self.write({"state": "draft"}) for this in self: @@ -127,6 +111,53 @@ def set_draft(self): self.write({"state": "draft"}) return True + def fetch_mail(self): + """Retrieve all mails for IMAP folders. + + We will use a separate connection for each folder. + """ + for this in self: + if not this.active or this.state != "done": + continue + connection = None + try: + # New connection per folder + connection = this.server_id.connect() + this.retrieve_imap_folder(connection) + connection.close() + except Exception: + _logger.error( + ( + "General failure when trying to connect to" + " %(server_type)s server %(server)s." + ), + { + "server_type": this.server_id.server_type, + "server": this.server_id.name, + }, + exc_info=True, + ) + finally: + if connection: + connection.logout() + + def retrieve_imap_folder(self, connection): + """Retrieve all mails for one IMAP folder.""" + self.ensure_one() + msgids = self.get_msgids(connection, "UNDELETED") + for msgid in msgids[0].split(): + # We will accept exceptions for single messages + try: + self.env.cr.execute("savepoint apply_matching") + self.apply_matching(connection, msgid) + self.env.cr.execute("release savepoint apply_matching") + except Exception: + self.env.cr.execute("rollback to savepoint apply_matching") + _logger.exception( + "Failed to fetch mail %(msgid)s from server %(server)s", + {"msgid": msgid, "server": self.server_id.name}, + ) + def get_msgids(self, connection, criteria): """Return imap ids of messages to process""" self.ensure_one() @@ -152,6 +183,28 @@ def get_msgids(self, connection, criteria): ) return msgids + def apply_matching(self, connection, msgid): + """Return ids of objects matched""" + self.ensure_one() + thread_model = self.env["mail.thread"] + message_org = self.fetch_msg(connection, msgid) + custom_values = ( + None + if self.match_algorithm == "odoo_standard" + else { + "folder": self, + } + ) + thread_id = thread_model.message_process( + self.model_id.model, + message_org, + custom_values=custom_values, + save_original=self.server_id.original, + strip_attachments=(not self.server_id.attach), + ) + matched = True if thread_id else False + self.update_msg(connection, msgid, matched=matched) + def fetch_msg(self, connection, msgid): """Select a single message from a folder.""" self.ensure_one() @@ -162,58 +215,7 @@ def fetch_msg(self, connection, msgid): % {"msgid": msgid, "folder": self.path, "server": self.server_id.name} ) message_org = msgdata[0][1] # rfc822 message source - mail_message = self.env["mail.thread"].message_parse( - message_org, save_original=self.server_id.original - ) - return (mail_message, message_org) - - def retrieve_imap_folder(self, connection): - """Retrieve all mails for one IMAP folder.""" - self.ensure_one() - msgids = self.get_msgids(connection, "UNDELETED") - match_algorithm = self.get_algorithm() - for msgid in msgids[0].split(): - # We will accept exceptions for single messages - try: - self.env.cr.execute("savepoint apply_matching") - self.apply_matching(connection, msgid, match_algorithm) - self.env.cr.execute("release savepoint apply_matching") - except Exception: - self.env.cr.execute("rollback to savepoint apply_matching") - _logger.exception( - "Failed to fetch mail %(msgid)s from server %(server)s", - {"msgid": msgid, "server": self.server_id.name}, - ) - - def fetch_mail(self): - """Retrieve all mails for IMAP folders. - - We will use a separate connection for each folder. - """ - for this in self: - if not this.active or this.state != "done": - continue - connection = None - try: - # New connection per folder - connection = this.server_id.connect() - this.retrieve_imap_folder(connection) - connection.close() - except Exception: - _logger.error( - ( - "General failure when trying to connect to" - " %(server_type)s server %(server)s." - ), - { - "server_type": this.server_id.server_type, - "server": this.server_id.name, - }, - exc_info=True, - ) - finally: - if connection: - connection.logout() + return message_org def update_msg(self, connection, msgid, matched=True, flagged=False): """Update msg in imap folder depending on match and settings.""" @@ -225,61 +227,3 @@ def update_msg(self, connection, msgid, matched=True, flagged=False): else: if self.flag_nonmatching: connection.store(msgid, "+FLAGS", "\\FLAGGED") - - def apply_matching(self, connection, msgid, match_algorithm): - """Return ids of objects matched""" - self.ensure_one() - mail_message, message_org = self.fetch_msg(connection, msgid) - if self.env["mail.message"].search( - [("message_id", "=", mail_message["message_id"])] - ): - # Ignore mails that have been handled already - return - matches = match_algorithm.search_matches(self, mail_message) - matched = matches and (len(matches) == 1 or self.match_first) - if matched: - match_algorithm.handle_match( - connection, matches[0], self, mail_message, message_org, msgid - ) - self.update_msg(connection, msgid, matched=matched) - - def attach_mail(self, match_object, mail_message): - """Attach mail to match_object.""" - self.ensure_one() - partner = False - model_name = self.model_id.model - if model_name == "res.partner": - partner = match_object - elif "partner_id" in self.env[model_name]._fields: - partner = match_object.partner_id - attachments = [] - if self.server_id.attach and mail_message.get("attachments"): - for attachment in mail_message["attachments"]: - # Attachment should at least have filename and data, but - # might have some extra element(s) - if len(attachment) < 2: - continue - fname, fcontent = attachment[:2] - data_attach = { - "name": fname, - "datas": base64.b64encode(fcontent), - "datas_fname": fname, - "description": _("Mail attachment"), - "res_model": model_name, - "res_id": match_object.id, - } - attachments.append(self.env["ir.attachment"].create(data_attach)) - self.env["mail.message"].create( - { - "author_id": partner and partner.id or False, - "model": model_name, - "res_id": match_object.id, - "message_type": "email", - "body": mail_message.get("body"), - "subject": mail_message.get("subject"), - "email_from": mail_message.get("from"), - "date": mail_message.get("date"), - "message_id": mail_message.get("message_id"), - "attachment_ids": [(6, 0, [a.id for a in attachments])], - } - ) diff --git a/fetchmail_attach_from_folder/models/mail_thread.py b/fetchmail_attach_from_folder/models/mail_thread.py new file mode 100644 index 00000000000..c39c3304667 --- /dev/null +++ b/fetchmail_attach_from_folder/models/mail_thread.py @@ -0,0 +1,70 @@ +# Copyright - 2024 Therp BV . +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import logging + +from odoo import api, models + +from .. import match_algorithm + +_logger = logging.getLogger(__name__) + + +class MailThread(models.AbstractModel): + _inherit = "mail.thread" + + @api.model + def message_route( + self, + message, + message_dict, + model=None, + thread_id=None, + custom_values=None, + ): + """Override to apply matching algorithm to determine thread_id if requested.""" + if not thread_id and custom_values and "folder" in custom_values: + thread_id = self._find_match(custom_values, message_dict) + if not thread_id: + return [] # This will ultimately return thread_id = False + return super().message_route( + message, + message_dict, + model=model, + thread_id=thread_id, + custom_values=custom_values, + ) + + @api.model + def _find_match(self, custom_values, message_dict): + """Try to find existing object to link mail to.""" + folder = custom_values.pop("folder") + matcher = self._get_algorithm(folder.match_algorithm) + if not matcher: + return None + matches = matcher.search_matches(folder, message_dict) + if not matches: + _logger.info( + "No match found for message %(subject)s with msgid %(msgid)s", + { + "subject": message_dict.get("subject", "no subject"), + "msgid": message_dict.get("message_id", "no msgid"), + }, + ) + return None + matched = len(matches) == 1 or folder.match_first + return matched and matches[0] or None + + @api.model + def _get_algorithm(self, algorithm): + """Translate algorithm code to implementation class. + + We used to load this dynamically, but having it more or less hardcoded + allows to adapt the UI to the selected algorithm, withouth needing + the (deprecated) fields_view_get trickery we used in the past. + """ + if algorithm == "email_domain": + return match_algorithm.email_domain.EmailDomain() + if algorithm == "email_exact": + return match_algorithm.email_exact.EmailExact() + _logger.error("Unknown algorithm %(algorithm)s", {"algorithm": algorithm}) + return None diff --git a/fetchmail_attach_from_folder/security/ir.model.access.csv b/fetchmail_attach_from_folder/security/ir.model.access.csv index c63f46bb835..50c676fb3b9 100644 --- a/fetchmail_attach_from_folder/security/ir.model.access.csv +++ b/fetchmail_attach_from_folder/security/ir.model.access.csv @@ -1,2 +1,4 @@ id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink access_model_fetchmail_server_folder,fetchmail.server.folder,model_fetchmail_server_folder,base.group_system,1,1,1,1 +access_fetchmail_attach_mail_manually,access_fetchmail_attach_mail_manually,model_fetchmail_attach_mail_manually,base.group_system,1,1,1,1 +access_fetchmail_attach_mail_manually_mail,access_fetchmail_attach_mail_manually_mail,model_fetchmail_attach_mail_manually_mail,base.group_system,1,1,1,1 diff --git a/fetchmail_attach_from_folder/tests/test_match_algorithms.py b/fetchmail_attach_from_folder/tests/test_match_algorithms.py index 89bf7598178..523e32839d9 100644 --- a/fetchmail_attach_from_folder/tests/test_match_algorithms.py +++ b/fetchmail_attach_from_folder/tests/test_match_algorithms.py @@ -2,7 +2,7 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from odoo.tests.common import TransactionCase -from ..match_algorithm import email_domain, email_exact, odoo_standard +from ..match_algorithm import email_domain, email_exact MSG_BODY = [ ( @@ -116,7 +116,7 @@ def test_email_domain(self): } folder = self.folder folder.match_algorithm = "email_domain" - folder.use_first_match = True + folder.match_first = True self.do_matching( email_domain.EmailDomain, "base.res_partner_address_31", @@ -128,32 +128,12 @@ def test_email_domain(self): mail_message["subject"], ) - def test_odoo_standard(self): - mail_message_org = ( - "To: demo@yourcompany.example.com\n" - "From: someone@else.com\n" - "Subject: testsubject\n" - "Message-Id: 42\n" - "Hello world" - ) - folder = self.folder - folder.match_algorithm = "odoo_standard" - matcher = odoo_standard.OdooStandard() - matches = matcher.search_matches(folder, None) - self.assertEqual(len(matches), 1) - matcher.handle_match(None, matches[0], folder, None, mail_message_org, None) - self.assertIn( - "Hello world", - self.env["mail.message"].search([("subject", "=", "testsubject")]).body, - ) - def test_apply_matching_exact(self): folder = self.folder - folder.match_algorithm = "email_domain" + folder.match_algorithm = "email_exact" connection = MockConnection() msgid = "<485a8041-d560-a981-5afc-d31c1f136748@acme.com>" - matcher = email_exact.EmailExact() - folder.apply_matching(connection, msgid, matcher) + folder.apply_matching(connection, msgid) def test_retrieve_imap_folder_domain(self): folder = self.folder diff --git a/fetchmail_attach_from_folder/views/fetchmail_server.xml b/fetchmail_attach_from_folder/views/fetchmail_server.xml index 24ee8fc5124..47a383e3ffd 100644 --- a/fetchmail_attach_from_folder/views/fetchmail_server.xml +++ b/fetchmail_attach_from_folder/views/fetchmail_server.xml @@ -11,14 +11,14 @@ name="attrs" >{'required': [('server_type', '!=', 'imap')]} - - - + + + @@ -58,10 +58,12 @@ states="done" /> - - - - + + + + + +