From 30bda0eacb735f59f54558a659b6253228508332 Mon Sep 17 00:00:00 2001 From: Dan Ambrosio Date: Thu, 23 Jan 2025 13:44:42 -0700 Subject: [PATCH 1/8] feat: add ability to ignore parse exception console messages Refs: #55 --- sphinxcontrib/doxylink/__init__.py | 3 ++- sphinxcontrib/doxylink/doxylink.py | 29 +++++++++++++++++++++-------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/sphinxcontrib/doxylink/__init__.py b/sphinxcontrib/doxylink/__init__.py index a920715..d1061a4 100644 --- a/sphinxcontrib/doxylink/__init__.py +++ b/sphinxcontrib/doxylink/__init__.py @@ -1,10 +1,11 @@ __version__ = "1.12.4" - def setup(app): from .doxylink import setup_doxylink_roles app.add_config_value('doxylink', {}, 'env') app.add_config_value('doxylink_pdf_files', {}, 'env') + app.add_config_value('doxylink_parse_exception_ignore_regex_list', + default=[], types=[str], rebuild='env') app.connect('builder-inited', setup_doxylink_roles) return { diff --git a/sphinxcontrib/doxylink/doxylink.py b/sphinxcontrib/doxylink/doxylink.py index 9a5411b..b758662 100644 --- a/sphinxcontrib/doxylink/doxylink.py +++ b/sphinxcontrib/doxylink/doxylink.py @@ -134,8 +134,8 @@ def is_url(str_to_validate: str) -> bool: class SymbolMap: """A SymbolMap maps symbols to Entries.""" - def __init__(self, xml_doc: ET.ElementTree) -> None: - entries = parse_tag_file(xml_doc) + def __init__(self, xml_doc: ET.ElementTree, parse_exception_ignore_pattern: Union['re.Pattern', None]) -> None: + entries = parse_tag_file(xml_doc, parse_exception_ignore_pattern) # Sort the entry list for use with bisect self._entries = sorted(entries) @@ -225,7 +225,7 @@ def __getitem__(self, item: str) -> Entry: return self._disambiguate(symbol, candidates) -def parse_tag_file(doc: ET.ElementTree) -> List[Entry]: +def parse_tag_file(doc: ET.ElementTree, parse_exception_ignore_pattern: Union['re.Pattern', None]) -> List[Entry]: """ Takes in an XML tree from a Doxygen tag file and returns a list that looks something like: @@ -290,7 +290,14 @@ def parse_tag_file(doc: ET.ElementTree) -> List[Entry]: entries.append( Entry(name=member_symbol, kind=member_kind, file=member_file, arglist=normalised_arglist)) except ParseException as e: - print(f'Skipping {member_kind} {member_symbol}{arglist}. Error reported from parser was: {e}') + # Check if the parse exception message matches the ignore regular expression, if it does do not print the message + message = f'Skipping {member_kind} {member_symbol}{arglist}. Error reported from parser was: {e}' + matched = parse_exception_ignore_pattern.match(message) if parse_exception_ignore_pattern else False + + if not matched: + print(message) + + continue else: # Put the simple things directly into the list entries.append(Entry(name=member_symbol, kind=member_kind, file=member_file, arglist=None)) @@ -303,6 +310,12 @@ def join(*args): def create_role(app, tag_filename, rootdir, cache_name, pdf=""): + parse_exception_ignore_regex_list = getattr(app.config, 'doxylink_parse_exception_ignore_regex_list') + parse_exception_ignore_pattern = re.compile('|'.join(parse_exception_ignore_regex_list)) if parse_exception_ignore_regex_list else None + + if parse_exception_ignore_pattern: + report_info(app.env, f'Ignoring parsing exceptions using `{parse_exception_ignore_pattern}`') + # Tidy up the root directory path if not rootdir.endswith(('/', '\\')): rootdir = join(rootdir, os.sep) @@ -330,22 +343,22 @@ def _parse(): if not hasattr(app.env, 'doxylink_cache'): # no cache present at all, initialise it report_info(app.env, 'No cache at all, rebuilding...') - mapping = SymbolMap(_parse()) + mapping = SymbolMap(_parse(), parse_exception_ignore_pattern) app.env.doxylink_cache = {cache_name: {'mapping': mapping, 'mtime': modification_time, 'version': __version__}} elif not app.env.doxylink_cache.get(cache_name): # Main cache is there but the specific sub-cache for this tag file is not report_info(app.env, 'Sub cache is missing, rebuilding...') - mapping = SymbolMap(_parse()) + mapping = SymbolMap(_parse(), parse_exception_ignore_pattern) app.env.doxylink_cache[cache_name] = {'mapping': mapping, 'mtime': modification_time, 'version': __version__} elif app.env.doxylink_cache[cache_name]['mtime'] < modification_time: # tag file has been modified since sub-cache creation report_info(app.env, 'Sub-cache is out of date, rebuilding...') - mapping = SymbolMap(_parse()) + mapping = SymbolMap(_parse(), parse_exception_ignore_pattern) app.env.doxylink_cache[cache_name] = {'mapping': mapping, 'mtime': modification_time} elif not app.env.doxylink_cache[cache_name].get('version') or app.env.doxylink_cache[cache_name].get('version') != __version__: # sub-cache doesn't have a version or the version doesn't match report_info(app.env, 'Sub-cache schema version doesn\'t match, rebuilding...') - mapping = SymbolMap(_parse()) + mapping = SymbolMap(_parse(), parse_exception_ignore_pattern) app.env.doxylink_cache[cache_name] = {'mapping': mapping, 'mtime': modification_time, 'version': __version__} else: # The cache is up to date From 76830adcd79255d471848b384ca0528d10615677 Mon Sep 17 00:00:00 2001 From: Dan Ambrosio Date: Fri, 7 Feb 2025 11:03:08 -0700 Subject: [PATCH 2/8] docs: Added configuration documenation for doxylink_parse_exception_ignore_regex_list Refs: #55 --- doc/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index 88ea0f8..7f92618 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -180,6 +180,9 @@ Configuration values 'qtogre_doxygen.pdf': '/home/matt/qtogre/doxygen.pdf', } +.. confval:: doxylink_parse_exception_ignore_regex_list + + A list of regular expressions that can be used to ignore specific ``ParseException`` console messages. Default is ``[]``. Bug reports ----------- From 6a1d432ed11788249672b1d264809d46612fe9b5 Mon Sep 17 00:00:00 2001 From: Dan Ambrosio Date: Fri, 7 Feb 2025 11:08:57 -0700 Subject: [PATCH 3/8] test: ensure tests pass with a None parse_exception_ignore_pattern Refs: #55 --- sphinxcontrib/doxylink/doxylink.py | 2 +- tests/test_doxylink.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinxcontrib/doxylink/doxylink.py b/sphinxcontrib/doxylink/doxylink.py index b758662..e7fcb5e 100644 --- a/sphinxcontrib/doxylink/doxylink.py +++ b/sphinxcontrib/doxylink/doxylink.py @@ -134,7 +134,7 @@ def is_url(str_to_validate: str) -> bool: class SymbolMap: """A SymbolMap maps symbols to Entries.""" - def __init__(self, xml_doc: ET.ElementTree, parse_exception_ignore_pattern: Union['re.Pattern', None]) -> None: + def __init__(self, xml_doc: ET.ElementTree, parse_exception_ignore_pattern: Union['re.Pattern', None] = None) -> None: entries = parse_tag_file(xml_doc, parse_exception_ignore_pattern) # Sort the entry list for use with bisect diff --git a/tests/test_doxylink.py b/tests/test_doxylink.py index a0524e4..a8b3ef0 100644 --- a/tests/test_doxylink.py +++ b/tests/test_doxylink.py @@ -90,7 +90,7 @@ def test_file_different(examples_tag_file, symbol1, symbol2): def test_parse_tag_file(examples_tag_file): tag_file = ET.parse(examples_tag_file) - mapping = doxylink.parse_tag_file(tag_file) + mapping = doxylink.parse_tag_file(tag_file, None) def has_entry(name): """ From f396ba735e6643dbba049ca15b6afa166d8969d6 Mon Sep 17 00:00:00 2001 From: jce Date: Wed, 26 Feb 2025 14:57:38 +0100 Subject: [PATCH 4/8] Ignore tag file generated by test suite --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9e796c5..cafa79e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ parsing_profile sphinxcontrib_doxylink.egg-info/ dist/ build/ +examples/my_lib.tag From 07d98b2b674b824cd11a5a2e7d0b200d3e4e452c Mon Sep 17 00:00:00 2001 From: jce Date: Wed, 26 Feb 2025 15:01:39 +0100 Subject: [PATCH 5/8] Use report_warning instead of print; don't combine regexes into a single pattern for robustness; rename config variable --- doc/index.rst | 8 +++-- sphinxcontrib/doxylink/__init__.py | 2 +- sphinxcontrib/doxylink/doxylink.py | 47 ++++++++++++++---------- tests/test_doxylink.py | 57 ++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 21 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index 7f92618..a62ac0a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -180,9 +180,13 @@ Configuration values 'qtogre_doxygen.pdf': '/home/matt/qtogre/doxygen.pdf', } -.. confval:: doxylink_parse_exception_ignore_regex_list +.. confval:: doxylink_parse_error_ignore_regexes - A list of regular expressions that can be used to ignore specific ``ParseException`` console messages. Default is ``[]``. + A list of regular expressions that can be used to ignore specific errors reported from the parser. + Default is ``[]``. This is useful if you have a lot of errors that you know are not important. + For example, you may want to ignore errors related to a specific namespace. + The regular expression is matched against the error message using Python's + `re.search `_ function. Bug reports ----------- diff --git a/sphinxcontrib/doxylink/__init__.py b/sphinxcontrib/doxylink/__init__.py index d1061a4..5cae1bc 100644 --- a/sphinxcontrib/doxylink/__init__.py +++ b/sphinxcontrib/doxylink/__init__.py @@ -4,7 +4,7 @@ def setup(app): from .doxylink import setup_doxylink_roles app.add_config_value('doxylink', {}, 'env') app.add_config_value('doxylink_pdf_files', {}, 'env') - app.add_config_value('doxylink_parse_exception_ignore_regex_list', + app.add_config_value('doxylink_parse_error_ignore_regexes', default=[], types=[str], rebuild='env') app.connect('builder-inited', setup_doxylink_roles) diff --git a/sphinxcontrib/doxylink/doxylink.py b/sphinxcontrib/doxylink/doxylink.py index e7fcb5e..f328e1a 100644 --- a/sphinxcontrib/doxylink/doxylink.py +++ b/sphinxcontrib/doxylink/doxylink.py @@ -134,8 +134,8 @@ def is_url(str_to_validate: str) -> bool: class SymbolMap: """A SymbolMap maps symbols to Entries.""" - def __init__(self, xml_doc: ET.ElementTree, parse_exception_ignore_pattern: Union['re.Pattern', None] = None) -> None: - entries = parse_tag_file(xml_doc, parse_exception_ignore_pattern) + def __init__(self, xml_doc: ET.ElementTree, parse_error_ignore_regexes: Optional[List[str]] = None) -> None: + entries = parse_tag_file(xml_doc, parse_error_ignore_regexes) # Sort the entry list for use with bisect self._entries = sorted(entries) @@ -225,7 +225,7 @@ def __getitem__(self, item: str) -> Entry: return self._disambiguate(symbol, candidates) -def parse_tag_file(doc: ET.ElementTree, parse_exception_ignore_pattern: Union['re.Pattern', None]) -> List[Entry]: +def parse_tag_file(doc: ET.ElementTree, parse_error_ignore_regexes: Optional[List[str]]) -> List[Entry]: """ Takes in an XML tree from a Doxygen tag file and returns a list that looks something like: @@ -290,13 +290,21 @@ def parse_tag_file(doc: ET.ElementTree, parse_exception_ignore_pattern: Union['r entries.append( Entry(name=member_symbol, kind=member_kind, file=member_file, arglist=normalised_arglist)) except ParseException as e: - # Check if the parse exception message matches the ignore regular expression, if it does do not print the message message = f'Skipping {member_kind} {member_symbol}{arglist}. Error reported from parser was: {e}' - matched = parse_exception_ignore_pattern.match(message) if parse_exception_ignore_pattern else False - - if not matched: - print(message) - + should_report = True + + if parse_error_ignore_regexes: + for pattern in parse_error_ignore_regexes: + try: + if re.search(pattern, message): + should_report = False + break + except re.error: + # Invalid regex pattern - ignore it + continue + + if should_report: + report_warning(None, message) # Use None as env since we don't have access to it here continue else: # Put the simple things directly into the list @@ -309,12 +317,15 @@ def join(*args): return ''.join(args) -def create_role(app, tag_filename, rootdir, cache_name, pdf=""): - parse_exception_ignore_regex_list = getattr(app.config, 'doxylink_parse_exception_ignore_regex_list') - parse_exception_ignore_pattern = re.compile('|'.join(parse_exception_ignore_regex_list)) if parse_exception_ignore_regex_list else None +def create_role(app: 'sphinx.application.Sphinx', + tag_filename: str, + rootdir: str, + cache_name: str, + pdf: str = "") -> None: + parse_error_ignore_regexes = getattr(app.config, 'doxylink_parse_error_ignore_regexes', []) - if parse_exception_ignore_pattern: - report_info(app.env, f'Ignoring parsing exceptions using `{parse_exception_ignore_pattern}`') + if parse_error_ignore_regexes: + report_info(app.env, f'Using parse error ignore patterns: {", ".join(parse_error_ignore_regexes)}') # Tidy up the root directory path if not rootdir.endswith(('/', '\\')): @@ -343,22 +354,22 @@ def _parse(): if not hasattr(app.env, 'doxylink_cache'): # no cache present at all, initialise it report_info(app.env, 'No cache at all, rebuilding...') - mapping = SymbolMap(_parse(), parse_exception_ignore_pattern) + mapping = SymbolMap(_parse(), parse_error_ignore_regexes) app.env.doxylink_cache = {cache_name: {'mapping': mapping, 'mtime': modification_time, 'version': __version__}} elif not app.env.doxylink_cache.get(cache_name): # Main cache is there but the specific sub-cache for this tag file is not report_info(app.env, 'Sub cache is missing, rebuilding...') - mapping = SymbolMap(_parse(), parse_exception_ignore_pattern) + mapping = SymbolMap(_parse(), parse_error_ignore_regexes) app.env.doxylink_cache[cache_name] = {'mapping': mapping, 'mtime': modification_time, 'version': __version__} elif app.env.doxylink_cache[cache_name]['mtime'] < modification_time: # tag file has been modified since sub-cache creation report_info(app.env, 'Sub-cache is out of date, rebuilding...') - mapping = SymbolMap(_parse(), parse_exception_ignore_pattern) + mapping = SymbolMap(_parse(), parse_error_ignore_regexes) app.env.doxylink_cache[cache_name] = {'mapping': mapping, 'mtime': modification_time} elif not app.env.doxylink_cache[cache_name].get('version') or app.env.doxylink_cache[cache_name].get('version') != __version__: # sub-cache doesn't have a version or the version doesn't match report_info(app.env, 'Sub-cache schema version doesn\'t match, rebuilding...') - mapping = SymbolMap(_parse(), parse_exception_ignore_pattern) + mapping = SymbolMap(_parse(), parse_error_ignore_regexes) app.env.doxylink_cache[cache_name] = {'mapping': mapping, 'mtime': modification_time, 'version': __version__} else: # The cache is up to date diff --git a/tests/test_doxylink.py b/tests/test_doxylink.py index a8b3ef0..e386c9c 100644 --- a/tests/test_doxylink.py +++ b/tests/test_doxylink.py @@ -197,3 +197,60 @@ def test_process_configuration_warn(rootdir, pdf_filename, builder, msg): with LogCapture() as l: doxylink.process_configuration(app, 'doxygen/project.tag', rootdir, pdf_filename) l.check(('sphinx.sphinxcontrib.doxylink.doxylink', 'WARNING', msg)) + + +def test_parse_error_ignore_regexes(): + # Create a modified tag file content with problematic entries + problematic_xml = """ + + + test.h + test_8h + + foo + (transform_pb2.Rotation2f a) + 1234 + + + bad + (*int i) + 5678 + + + baz + (int i, float f) + 9012 + + + unexpected + (*int i) + 5678 + + +""" + + # Write temporary tag file + test_tag_file = 'test_temp.tag' + with open(test_tag_file, 'w') as f: + f.write(problematic_xml) + + try: + tag_file = ET.parse(test_tag_file) + patterns = [r'kipping function test\.h::foo', r'kipping.*bad'] + + with LogCapture() as log: + mapping = doxylink.parse_tag_file(tag_file, patterns) + + # Verify that the mapping still contains valid entries + assert any(entry.name.endswith('baz') for entry in mapping) + + # Verify that messages matching our patterns were not logged + assert not any('test.h::foo' in record.msg for record in log.records) + assert not any('bar' in record.msg for record in log.records) + + # Verify other error messages were logged + assert any('Skipping' in record.msg for record in log.records) + + finally: + if os.path.exists(test_tag_file): + os.unlink(test_tag_file) From 70db62b26e6663e9637809d99dc537d1acde5476 Mon Sep 17 00:00:00 2001 From: jce Date: Wed, 26 Feb 2025 15:26:08 +0100 Subject: [PATCH 6/8] revert type annotations for create_role as it's complex --- sphinxcontrib/doxylink/doxylink.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sphinxcontrib/doxylink/doxylink.py b/sphinxcontrib/doxylink/doxylink.py index f328e1a..3a73393 100644 --- a/sphinxcontrib/doxylink/doxylink.py +++ b/sphinxcontrib/doxylink/doxylink.py @@ -317,11 +317,7 @@ def join(*args): return ''.join(args) -def create_role(app: 'sphinx.application.Sphinx', - tag_filename: str, - rootdir: str, - cache_name: str, - pdf: str = "") -> None: +def create_role(app, tag_filename, rootdir, cache_name, pdf=""): parse_error_ignore_regexes = getattr(app.config, 'doxylink_parse_error_ignore_regexes', []) if parse_error_ignore_regexes: From e38632af8ca487a3a82fc935e24bd8186f17ca8f Mon Sep 17 00:00:00 2001 From: jce Date: Wed, 26 Feb 2025 16:13:04 +0100 Subject: [PATCH 7/8] Test that parser error, a proper Sphinx warning, is ignored due to doxylink_parse_error_ignore_regexes --- examples/conf.py | 1 + examples/my_lib.h | 3 +++ 2 files changed, 4 insertions(+) diff --git a/examples/conf.py b/examples/conf.py index 3127ffa..fffeb08 100644 --- a/examples/conf.py +++ b/examples/conf.py @@ -10,5 +10,6 @@ doxylink = { 'my_lib': (os.path.abspath('./my_lib.tag'), 'https://examples.com/'), } +doxylink_parse_error_ignore_regexes = [r"DEFINE.*"] master_doc = 'index' diff --git a/examples/my_lib.h b/examples/my_lib.h index 89c774d..fff1d8b 100644 --- a/examples/my_lib.h +++ b/examples/my_lib.h @@ -46,3 +46,6 @@ enum Color { red, green, blue }; // An enum class enum class Color_c { red, green, blue }; + +// A function that triggers a warning from the parser +void DEFINE_bool(show, false, "Enable visualization"); From 71e7de0c411b8ebe017c3f7b12920acd9aeed361 Mon Sep 17 00:00:00 2001 From: jce Date: Wed, 26 Feb 2025 16:13:51 +0100 Subject: [PATCH 8/8] Address deprecation warning about whitelist_externals --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 26319f8..9ddb266 100644 --- a/tox.ini +++ b/tox.ini @@ -3,14 +3,14 @@ envlist = benchmark, test, examples, doc isolated_build = True [testenv:benchmark] -whitelist_externals = poetry +allowlist_externals = poetry commands= poetry install python tests/test_parser.py [testenv:examples] changedir = examples -whitelist_externals = +allowlist_externals = doxygen poetry commands= @@ -19,13 +19,13 @@ commands= sphinx-build -W -b html . {envtmpdir}/examples/_build [testenv:test] -whitelist_externals = poetry +allowlist_externals = poetry commands= poetry install pytest [testenv:doc] -whitelist_externals = poetry +allowlist_externals = poetry commands= poetry install sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees doc {envtmpdir}/linkcheck