From 4dbc3b00e587f3d64cfd964a685f2bddd1b499ad Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Thu, 31 Aug 2023 10:03:37 +0200 Subject: [PATCH 1/3] [FIX] base: correctly parse utf8 html module descriptions Apparently `lxml.html.document_fromstring` (and possibly other `lxml.html` loaders) parses byte-strings as latin1 regardless of their actual encoding, maybe because python2, maybe because there's a super legacy html4 parser underlying it. Either way that means ever since loading `static/description/index.html` files was added 10 years ago (4bf6a7ea4c1703a02aa547db477ad7162ac1378c) `_get_desc` has been loading these files in latin1 rather than the utf8 most people would expect. Add an explicit decoding phase to try and load html description files in UTF8. Fall back to latin1 in case there are description files which are genuinely in latin1, or even just some random-ass broken stuff which very much isn't utf8 (the extended-ascii encodings -- of which latin1 is one -- will happily accept and mangle any input as every byte value is valid, utf8 is a lot more structured). Closes #127846 closes odoo/odoo#133708 Signed-off-by: Xavier Morel (xmo) --- odoo/addons/base/models/ir_module.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/odoo/addons/base/models/ir_module.py b/odoo/addons/base/models/ir_module.py index 285eb51b80a5..3952ac30ddf8 100644 --- a/odoo/addons/base/models/ir_module.py +++ b/odoo/addons/base/models/ir_module.py @@ -178,7 +178,11 @@ def _get_desc(self): if path: with tools.file_open(path, 'rb') as desc_file: doc = desc_file.read() - html = lxml.html.document_fromstring(doc) + try: + contents = doc.decode('utf-8') + except UnicodeDecodeError: + contents = doc + html = lxml.html.document_fromstring(contents) for element, attribute, link, pos in html.iterlinks(): if element.get('src') and not '//' in element.get('src') and not 'static/' in element.get('src'): element.set('src', "/%s/static/description/%s" % (module.name, element.get('src'))) From 51d375603d68ac65881fa5b0a096a8f3c1fa616d Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 1 Sep 2023 09:24:28 +0200 Subject: [PATCH 2/3] [FIX] base: encoding guessing of html module descriptions I missed a critical issue in #133708: various users had discovered they could already fix description issues by adding an XML declaration to their document which is very cool (though technically not really valid). What is a lot less cool is that lxml gets *extremely* unhappy when asked to parse *strings* with an encoding declaration, raising a ValueError, so the purported fix breaks on any module which does that, which seems to include a lot of OCA modules. Gate the encoding guessing by bailing if the document has an XML declaration, in which case we just assume the author knows what they're doing and we leave them alone. For extra safety, check the encoding declaration in ascii and utf16. Could also have checked for BOMs, but lxml seems to not care about them overly much (in fact it seems to prefer them decoded which is odd). closes odoo/odoo#133900 Reported-by: @rezak400 Signed-off-by: Xavier Morel (xmo) --- odoo/addons/base/models/ir_module.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/odoo/addons/base/models/ir_module.py b/odoo/addons/base/models/ir_module.py index 3952ac30ddf8..ac9a3056b8ac 100644 --- a/odoo/addons/base/models/ir_module.py +++ b/odoo/addons/base/models/ir_module.py @@ -148,6 +148,12 @@ def get_transforms(self): ('to install', 'To be installed'), ] +XML_DECLARATION = ( + ' Date: Thu, 25 May 2023 17:09:26 +0200 Subject: [PATCH 3/3] [IMP] core: ease testing upgrade scripts in custom modules Upgrade (aka migration) scripts are a core part of Odoo, allowing database manipulations for modules during version changes. Any module, including custom ones can run upgrade scripts, even if the `--upgrade-path` flag (and with it, the `odoo.upgrade` sub-module) is not present. Currently only the "standard" modules benefit of easy upgrade script testing. Any custom modules that want to run tests of their upgrades have to import the tests in the usual `tests` folder, which is not ideal. Therefore, to allow TDD and programmatic testing of upgrade scripts in custom modules, the test discovery is here modified to also parse the module's `migrations` and `upgrades` sub-modules for tests. closes odoo/odoo#122569 Signed-off-by: Christophe Simonis (chs) --- odoo/modules/module.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/odoo/modules/module.py b/odoo/modules/module.py index 354f796f495e..4ab2cbf8f241 100644 --- a/odoo/modules/module.py +++ b/odoo/modules/module.py @@ -467,13 +467,7 @@ def get_test_modules(module): feed unittest.TestLoader.loadTestsFromModule() """ # Try to import the module results = _get_tests_modules('odoo.addons', module) - - try: - importlib.import_module('odoo.upgrade.%s' % module) - except ImportError: - pass - else: - results += list(_get_upgrade_test_modules(module)) + results += list(_get_upgrade_test_modules(module)) return results @@ -501,16 +495,26 @@ def _get_tests_modules(path, module): return result def _get_upgrade_test_modules(module): - upg = importlib.import_module("odoo.upgrade") - for path in map(Path, upg.__path__): - for test in (path / module / "tests").glob("test_*.py"): - spec = importlib.util.spec_from_file_location(f"odoo.upgrade.{module}.tests.{test.stem}", test) - if not spec: - continue - pymod = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = pymod - spec.loader.exec_module(pymod) - yield pymod + upgrade_modules = ( + f"odoo.upgrade.{module}", + f"odoo.addons.{module}.migrations", + f"odoo.addons.{module}.upgrades", + ) + for module_name in upgrade_modules: + try: + upg = importlib.import_module(module_name) + except ImportError: + continue + + for path in map(Path, upg.__path__): + for test in path.glob("tests/test_*.py"): + spec = importlib.util.spec_from_file_location(f"{upg.__name__}.tests.{test.stem}", test) + if not spec: + continue + pymod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = pymod + spec.loader.exec_module(pymod) + yield pymod class OdooTestResult(unittest.result.TestResult):