diff --git a/CHANGELOG.md b/CHANGELOG.md index 378d73e..d00e31e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v0.4.3 - 2020-05-31 +- Fixed regression in `[code_blocks]` functionality +- Fixed minor issues in syntax highlighter +- Added symbols from doxygen tagfiles to the syntax highlighter +- Minor style tweaks + ## v0.4.1 - 2020-05-30 - Fixed `.dirs` being glommed as source paths - Added config option `scripts` diff --git a/__main__.py b/__main__.py deleted file mode 100644 index 4a13336..0000000 --- a/__main__.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 -# This file is a part of marzer/poxy and is subject to the the terms of the MIT license. -# Copyright (c) Mark Gillard -# See https://github.com/marzer/poxy/blob/master/LICENSE for the full license text. -# SPDX-License-Identifier: MIT - -from poxy import main - -if __name__ == '__main__': - main() diff --git a/poxy/__init__.py b/poxy/__init__.py index 6c260c3..27fa2d4 100644 --- a/poxy/__init__.py +++ b/poxy/__init__.py @@ -4,7 +4,14 @@ # See https://github.com/marzer/poxy/blob/master/LICENSE for the full license text. # SPDX-License-Identifier: MIT -from .run import run, main +from .run import run from .utils import lib_version, Error, WarningTreatedAsError +__all__ = [ + r'run', + r'lib_version', + r'Error', + r'WarningTreatedAsError' +] + __version__ = r'.'.join(lib_version()) diff --git a/poxy/__main__.py b/poxy/__main__.py index f8840a1..e884dc0 100644 --- a/poxy/__main__.py +++ b/poxy/__main__.py @@ -5,9 +5,11 @@ # SPDX-License-Identifier: MIT try: - from poxy.run import main + from poxy.main import _run except: - from run import main + from main import _run + + if __name__ == '__main__': - main() + _run() diff --git a/poxy/data/poxy-light.css b/poxy/data/poxy-light.css index ed2a417..51bf493 100644 --- a/poxy/data/poxy-light.css +++ b/poxy/data/poxy-light.css @@ -144,6 +144,28 @@ section > ol li:last-child margin-top: 0.4rem; } +/* figures and top-level images */ +figure.m-figure +{ + margin-top: 1.5rem !important; +} +section > img +{ + margin-top: 2rem !important; + margin-bottom: 2rem !important; +} + +/* solid m-success boxes */ +.m-note.m-success a +{ + color: #ffffff !important; + font-weight: bold; +} +.m-note.m-success a:hover +{ + text-decoration: underline !important; +} + /* ==================================================== code blocks ==================================================== */ diff --git a/poxy/data/poxy.css b/poxy/data/poxy.css index 6b541fc..f07f9ac 100644 --- a/poxy/data/poxy.css +++ b/poxy/data/poxy.css @@ -222,11 +222,10 @@ pre > p.godbolt figure.m-figure > figcaption { font-weight: initial !important; - font-size: initial !important; + font-size: 1rem !important; text-align: center; margin-top: initial; } - figure.m-figure::before { border-style: none !important; diff --git a/poxy/data/version.txt b/poxy/data/version.txt index 267577d..17b2ccd 100644 --- a/poxy/data/version.txt +++ b/poxy/data/version.txt @@ -1 +1 @@ -0.4.1 +0.4.3 diff --git a/poxy/fixers.py b/poxy/fixers.py index fb20ff3..bbda1cd 100644 --- a/poxy/fixers.py +++ b/poxy/fixers.py @@ -444,10 +444,24 @@ def __colourize_compound_def(cls, tags, context): return False + @classmethod + def __adjacent_maybe_by_whitespace(cls, a, b): + mid = a.next_sibling + if mid is b: + return True + if not mid.next_sibling is b: + return False + if not isinstance(mid, soup.NavigableString): + return False + if len(mid.string.strip()) > 0: + return False + return True + def __call__(self, doc, context): + changed = False + # fix up syntax highlighting code_blocks = doc.body(('pre','code'), class_='m-code') - changed = False changed_this_pass = True while changed_this_pass: changed_this_pass = False @@ -490,10 +504,10 @@ def __call__(self, doc, context): for i in range(0, len(spans)): current = spans[i] - if current in compound_name_evaluated_tags: + if id(current) in compound_name_evaluated_tags: continue - compound_name_evaluated_tags.add(current) + compound_name_evaluated_tags.add(id(current)) tags = [ current ] while True: prev = current.previous_sibling @@ -506,7 +520,7 @@ def __call__(self, doc, context): break current = prev tags.insert(0, current) - compound_name_evaluated_tags.add(current) + compound_name_evaluated_tags.add(id(current)) current = spans[i] while True: @@ -520,7 +534,7 @@ def __call__(self, doc, context): break current = nxt tags.append(current) - compound_name_evaluated_tags.add(current) + compound_name_evaluated_tags.add(id(current)) full_str = ''.join([tag.get_text() for tag in tags]) if self.__ns_full_expr.fullmatch(full_str): @@ -555,33 +569,30 @@ def __call__(self, doc, context): spans = code_block('span', class_=('n', 'nl', 'kt', 'nc', 'nf'), string=True) for span in spans: if context.code_blocks.macros.fullmatch(span.get_text()): - soup.set_class(span, 'm') + soup.set_class(span, r'm') changed_this_block = True # misidentifed keywords spans = code_block('span', class_=('nf', 'nb', 'kt', 'ut', 'kr'), string=True) for span in spans: if (span.string in self.__keywords): - span['class'] = 'k' + soup.set_class(span, r'k') changed_this_block = True # 'using' statements - spans = code_block('span', class_=('k'), string=r'using') - for using in spans: - assign = using.find_next_sibling('span', class_='o', string='=') - if assign is None: - continue - next = using.next_sibling - while next != assign: - current = next - next = current.next_sibling - if isinstance(current, soup.NavigableString): - if len(current.string.strip()) > 0: - break + if 1: + spans = code_block(r'span', class_=r'k', string=r'using') + for using in spans: + next_identifier = using.find_next_sibling(r'span', class_=r'n', string=True) + if next_identifier is None: continue - if current.name != r'span' or r'class' not in current.attrs or r'n' not in current['class']: + next_assign = next_identifier.find_next_sibling(r'span', class_=r'o', string=r'=') + if next_assign is None: continue - soup.set_class(current, r'ut') + if not (self.__adjacent_maybe_by_whitespace(using, next_identifier) + and self.__adjacent_maybe_by_whitespace(next_identifier, next_assign)): + continue + soup.set_class(next_identifier, r'ut') changed_this_block = True if changed_this_block: @@ -591,7 +602,6 @@ def __call__(self, doc, context): # fix doxygen butchering code blocks as inline nonsense code_blocks = doc.body('code', class_=('m-code', 'm-console')) - changed = False changed_this_pass = True while changed_this_pass: changed_this_pass = False @@ -610,7 +620,6 @@ def __call__(self, doc, context): or (len(parent.contents) == 1 and parent.contents[0].string.strip() == '')): soup.destroy_node(parent) - changed = changed or changed_this_pass return changed diff --git a/poxy/main.py b/poxy/main.py new file mode 100644 index 0000000..ca5022e --- /dev/null +++ b/poxy/main.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# This file is a part of marzer/poxy and is subject to the the terms of the MIT license. +# Copyright (c) Mark Gillard +# See https://github.com/marzer/poxy/blob/master/LICENSE for the full license text. +# SPDX-License-Identifier: MIT + +try: + from poxy.utils import * + import poxy.run as run +except: + from utils import * + import run + +import re +import argparse +import datetime +from schema import SchemaError + + + +def _invoker(func, **kwargs): + try: + func(**kwargs) + except WarningTreatedAsError as err: + print(rf'Error: {err} (warning treated as error)', file=sys.stderr) + sys.exit(1) + except SchemaError as err: + print(err, file=sys.stderr) + sys.exit(1) + except Error as err: + print(rf'Error: {err}', file=sys.stderr) + sys.exit(1) + except Exception as err: + print_exception(err, include_type=True, include_traceback=True, skip_frames=1) + sys.exit(-1) + sys.exit(0) + + + +def _run(invoker=True): + """ + The entry point when the library is invoked as `poxy`. + """ + if invoker: + _invoker(_run, invoker=False) + return + + args = argparse.ArgumentParser( + description=r'Generate fancy C++ documentation.', + formatter_class=argparse.RawTextHelpFormatter + ) + args.add_argument( + r'config', + type=Path, + nargs='?', + default=Path('.'), + help=r'path to poxy.toml or a directory containing it (default: %(default)s)' + ) + args.add_argument( + r'-v', r'--verbose', + action=r'store_true', + help=r"enable very noisy diagnostic output" + ) + args.add_argument( + r'--dry', + action=r'store_true', + help=r"do a 'dry run' only, stopping after emitting the effective Doxyfile", + dest=r'dry_run' + ) + args.add_argument( + r'--threads', + type=int, + default=0, + metavar=r'', + help=r"set the number of threads to use (default: automatic)" + ) + args.add_argument( + r'--m.css', + type=Path, + default=None, + metavar=r'', + help=r"specify the version of m.css to use (default: uses the bundled one)", + dest=r'mcss' + ) + args.add_argument( + r'--doxygen', + type=Path, + default=None, + metavar=r'', + help=r"specify the Doxygen executable to use (default: finds Doxygen on system path)", + ) + args.add_argument( + r'--werror', + action=r'store_true', + help=r"always treat warnings as errors regardless of config file settings", + dest=r'treat_warnings_as_errors' + ) + args.add_argument( + r'--version', + action=r'store_true', + help=r"print the version and exit", + dest=r'print_version' + ) + args.add_argument(r'--nocleanup', action=r'store_true', help=argparse.SUPPRESS) + args = args.parse_args() + + if args.print_version: + print(r'.'.join(lib_version())) + return + + with ScopeTimer(r'All tasks', print_start=False, print_end=not args.dry_run) as timer: + run( + config_path = args.config, + output_dir = Path.cwd(), + threads = args.threads, + cleanup = not args.nocleanup, + verbose = args.verbose, + mcss_dir = args.mcss, + doxygen_path = args.doxygen, + logger=True, # stderr + stdout + dry_run=args.dry_run, + treat_warnings_as_errors=True if args.treat_warnings_as_errors else None + ) + + + +def _make_blog_post(invoker=True): + """ + The entry point when the library is invoked as `poxyblog`. + """ + if invoker: + _invoker(_make_blog_post, invoker=False) + return + + args = argparse.ArgumentParser( + description=r'Initializes a new blog post for Poxy sites.', + formatter_class=argparse.RawTextHelpFormatter + ) + args.add_argument( + r'title', + type=str, + help=r'the title of the new blog post' + ) + args.add_argument( + r'-v', r'--verbose', + action=r'store_true', + help=r"enable very noisy diagnostic output" + ) + args.add_argument( + r'--version', + action=r'store_true', + help=r"print the version and exit", + dest=r'print_version' + ) + args = args.parse_args() + + if args.print_version: + print(r'.'.join(lib_version())) + return + + date = datetime.datetime.now().date() + + title = args.title.strip() + if not title: + raise Error(r'title cannot be blank.') + if re.search(r''''[\n\v\f\r]''', title) is not None: + raise Error(r'title cannot contain newline characters.') + file = re.sub(r'''[!@#$%^&*;:'"<>?/\\\s|+]+''', '_', title) + file = rf'{date}_{file.lower()}.md' + + blog_dir = Path(r'blog') + if blog_dir.exists() and not blog_dir.is_dir(): + raise Error(rf'{blog_dir.resolve()} already exists and is not a directory') + blog_dir.mkdir(exist_ok=True) + + file = Path(blog_dir, file) + if file.exists(): + if not file.is_file(): + raise Error(rf'{file.resolve()} already exist and is not a file') + raise Error(rf'{file.resolve()} already exists') + + with open(file, r'w', encoding=r'utf-8', newline='\n') as f: + write = lambda s='', end='\n': print(s, end=end, file=f) + write(rf'# {title}') + write() + write() + write() + write() + write() + write(r'') + print(rf'Blog post file initialized: {file.resolve()}') + + + +if __name__ == '__main__': + _run() diff --git a/poxy/project.py b/poxy/project.py index 6dd8e93..d49531f 100644 --- a/poxy/project.py +++ b/poxy/project.py @@ -494,6 +494,7 @@ class _Defaults(object): r'detail' : r'@details', r'inline_subheading{1}' : r'[h4]\1[/h4]', r'conditional_return{1}' : r'\1: ', + r'inline_success' : r'[set_class m-note m-success]', r'inline_note' : r'[set_class m-note m-info]', r'inline_warning' : r'[set_class m-note m-danger]', r'inline_attention' : r'[set_class m-note m-warning]', @@ -563,26 +564,10 @@ class _Defaults(object): r'sv?' } cb_types = { - #------ standard/built-in types + #------ built-in types r'__(?:float|fp)[0-9]{1,3}', r'__m[0-9]{1,3}[di]?', r'_Float[0-9]{1,3}', - r'(?:std::)?(?:basic_)?ios(?:_base)?', - r'(?:std::)?(?:const_)?(?:reverse_)?iterator', - r'(?:std::)?(?:shared_|recursive_)?(?:timed_)?mutex', - r'(?:std::)?array', - r'(?:std::)?byte', - r'(?:std::)?exception', - r'(?:std::)?lock_guard', - r'(?:std::)?optional', - r'(?:std::)?pair', - r'(?:std::)?span', - r'(?:std::)?streamsize', - r'(?:std::)?string(?:_view)?', - r'(?:std::)?tuple', - r'(?:std::)?vector', - r'(?:std::)?(?:unique|shared|scoped)_(?:ptr|lock)', - r'(?:std::)?(?:unordered_)?(?:map|set)', r'[a-zA-Z_][a-zA-Z_0-9]*_t(?:ype(?:def)?|raits)?', r'bool', r'char', @@ -593,9 +578,8 @@ class _Defaults(object): r'short', r'signed', r'unsigned', - r'(?:std::)?w?(?:(?:(?:i|o)?(?:string|f))|i|o|io)stream', #------ documentation-only types - r'[T-V][0-9]', + r'[S-Z][0-9]?', r'Foo', r'Bar', r'[Vv]ec(?:tor)?[1-4][hifd]?', @@ -703,8 +687,8 @@ def __init__(self, config, macros): self.enums = copy.deepcopy(_Defaults.cb_enums) self.namespaces = copy.deepcopy(_Defaults.cb_namespaces) - if 'code' in config: - config = config['code'] + if 'code_blocks' in config: + config = config['code_blocks'] if 'types' in config: for t in coerce_collection(config['types']): @@ -758,34 +742,39 @@ class _Inputs(object): Optional(r'recursive_paths') : ValueOrArray(str, name=r'recursive_paths'), } - def __init__(self, config, key, input_dir): + def __init__(self, config, key, input_dir, additional_inputs=None, additional_recursive_inputs=None): self.paths = [] - if key not in config: - return - config = config[key] + if key in config: + config = config[key] + else: + config = None - paths = set() + all_paths = set() for recursive in (False, True): key = r'recursive_paths' if recursive else r'paths' - if key in config: - for v in coerce_collection(config[key]): - path = v.strip() - if not path: - continue - path = Path(path) - if not path.is_absolute(): - path = Path(input_dir, path) - path = path.resolve() - if not path.exists(): - raise Error(rf"{key}: '{path}' does not exist") - if not (path.is_file() or path.is_dir()): - raise Error(rf"{key}: '{path}' was not a directory or file") - paths.add(str(path)) - if recursive and path.is_dir(): - for subdir in enum_subdirs(path, filter=lambda p: not p.name.startswith(r'.')): - paths.add(str(subdir)) - self.paths = list(paths) + paths = [] + if not recursive and additional_inputs is not None: + paths = paths + [p for p in coerce_collection(additional_inputs)] + if recursive and additional_recursive_inputs is not None: + paths = paths + [p for p in coerce_collection(additional_recursive_inputs)] + if config is not None and key in config: + paths = paths + [p for p in coerce_collection(config[key])] + paths = [p for p in paths if p] + paths = [str(p).strip() for p in paths] + paths = [Path(p) for p in paths if p] + paths = [Path(input_dir, p) if not p.is_absolute() else p for p in paths] + paths = [p.resolve() for p in paths] + for path in paths: + if not path.exists(): + raise Error(rf"{key}: '{path}' does not exist") + if not (path.is_file() or path.is_dir()): + raise Error(rf"{key}: '{path}' was not a directory or file") + all_paths.add(path) + if recursive and path.is_dir(): + for subdir in enum_subdirs(path, filter=lambda p: not p.name.startswith(r'.'), recursive=True): + all_paths.add(subdir) + self.paths = list(all_paths) self.paths.sort() @@ -795,8 +784,14 @@ class _FilteredInputs(_Inputs): Optional(r'patterns') : ValueOrArray(str, name=r'patterns') }) - def __init__(self, config, key, input_dir): - super().__init__(config, key, input_dir) + def __init__(self, config, key, input_dir, additional_inputs=None, additional_recursive_inputs=None): + super().__init__( + config, + key, + input_dir, + additional_inputs=additional_inputs, + additional_recursive_inputs=additional_recursive_inputs + ) self.patterns = None if key not in config: @@ -819,8 +814,14 @@ class _Sources(_FilteredInputs): Optional(r'extract_all') : bool, }) - def __init__(self, config, key, input_dir): - super().__init__(config, key, input_dir) + def __init__(self, config, key, input_dir, additional_inputs=None, additional_recursive_inputs=None): + super().__init__( + config, + key, + input_dir, + additional_inputs=additional_inputs, + additional_recursive_inputs=additional_recursive_inputs + ) self.strip_paths = [] self.strip_includes = [] @@ -950,7 +951,7 @@ def verbose_value(self, name, val): if not first: print(f'\n{" ":<35}', file=buf, end='') first = False - print(rf'{k:<{rpad}} => {v}', file=buf, end='') + print(rf'{str(k):<{rpad}} => {v}', file=buf, end='') elif is_collection(val): if val: first = True @@ -966,8 +967,11 @@ def verbose_value(self, name, val): def verbose_object(self, name, obj): if not self.__verbose: return - for k, v in obj.__dict__.items(): - self.verbose_value(rf'{name}.{k}', v) + if isinstance(obj, (tuple, list, dict)): + self.verbose_value(name, obj) + else: + for k, v in obj.__dict__.items(): + self.verbose_value(rf'{name}.{k}', v) @classmethod def __init_data_files(cls, context): @@ -1032,8 +1036,7 @@ def __init__(self, config_path, output_dir, threads, cleanup, verbose, mcss_dir, if treat_warnings_as_errors: self.warnings.treat_as_errors = True - now = datetime.datetime.utcnow().replace(microsecond=0, tzinfo=datetime.timezone.utc) - self.now = now + self.now = datetime.datetime.utcnow().replace(microsecond=0, tzinfo=datetime.timezone.utc) # resolve paths if 1: @@ -1101,6 +1104,8 @@ def __init__(self, config_path, output_dir, threads, cleanup, verbose, mcss_dir, self.config_path = self.config_path.resolve() self.verbose_value(r'Context.config_path', self.config_path) self.verbose_value(r'Context.doxyfile_path', self.doxyfile_path) + self.blog_dir = Path(self.input_dir, r'blog') + self.verbose_value(r'Context.blog_dir', self.blog_dir) # temp dirs self.global_temp_dir = Path(tempfile.gettempdir(), r'poxy') @@ -1113,10 +1118,8 @@ def __init__(self, config_path, output_dir, threads, cleanup, verbose, mcss_dir, self.temp_dir = sha1(self.temp_dir) self.temp_dir = Path(self.global_temp_dir, self.temp_dir) self.verbose_value(r'Context.temp_dir', self.temp_dir) - self.blog_dir = Path(self.temp_dir, r'blog') - self.verbose_value(r'Context.blog_dir', self.blog_dir) - self.pages_dir = Path(self.temp_dir, r'pages') - self.verbose_value(r'Context.pages_dir', self.pages_dir) + self.temp_pages_dir = Path(self.temp_dir, r'pages') + self.verbose_value(r'Context.temp_pages_dir', self.temp_pages_dir) # output paths self.xml_dir = Path(self.temp_dir, 'xml') @@ -1157,6 +1160,21 @@ def __init__(self, config_path, output_dir, threads, cleanup, verbose, mcss_dir, self.mcss_dir = mcss_dir self.verbose_value(r'Context.mcss_dir', self.mcss_dir) + # misc + self.cppref_tagfile = coerce_path(self.data_dir, r'cppreference-doxygen-web.tag.xml') + self.verbose_value(r'Context.cppref_tagfile', self.cppref_tagfile) + assert_existing_file(self.cppref_tagfile) + + # initialize temp + output dirs + if not self.dry_run: + delete_directory(self.html_dir, logger=self.verbose_logger) + delete_directory(self.xml_dir, logger=self.verbose_logger) + if self.cleanup: + delete_directory(self.temp_dir, logger=self.verbose_logger) + self.global_temp_dir.mkdir(exist_ok=True) + self.temp_dir.mkdir(exist_ok=True) + self.temp_pages_dir.mkdir(exist_ok=True) + # read + check config if 1: extra_files = [] @@ -1231,7 +1249,7 @@ def __init__(self, config_path, output_dir, threads, cleanup, verbose, mcss_dir, # project C++ version # defaults to 'current' cpp year version based on (current year - 2) - self.cpp = max(int(now.year) - 2, 2011) + self.cpp = max(int(self.now.year) - 2, 2011) self.cpp = self.cpp - ((self.cpp - 2011) % 3) if 'cpp' in config: self.cpp = str(config['cpp']).lstrip('0 \t').rstrip() @@ -1310,24 +1328,57 @@ def __init__(self, config_path, output_dir, threads, cleanup, verbose, mcss_dir, extra_files.append(file) self.verbose_value(r'Context.scripts', self.scripts) + # enumerate blog files (need to add them to the doxygen sources) + self.blog_files = [] + if self.blog_dir.exists() and self.blog_dir.is_dir(): + self.blog_files = get_all_files(self.blog_dir, any=(r'*.md', r'*.markdown'), recursive=True) + sep = re.compile(r'[-֊‐‑‒–—―−_ ,;.]+') + expr = re.compile( + rf'^(?:blog{sep.pattern})?((?:[0-9]{{4}}){sep.pattern}(?:[0-9]{{2}}){sep.pattern}(?:[0-9]{{2}})){sep.pattern}[a-zA-Z0-9_ -]+$' + ) + for i in range(len(self.blog_files)): + f = self.blog_files[i] + m = expr.fullmatch(f.stem) + if not m: + raise Error(rf"blog post filename '{f.name}' was not formatted correctly; " + + r"it should be of the form 'YYYY-MM-DD_this_is_a_post.md'.") + try: + d = datetime.datetime.strptime(sep.sub('-', m[1]), r'%Y-%m-%d').date() + self.blog_files[i] = (f, d) + except Exception as exc: + raise Error(rf"failed to parse date from blog post filename '{f.name}': {str(exc)}") + self.verbose_value(r'Context.blog_files', self.blog_files) + # sources (INPUT, FILE_PATTERNS, STRIP_FROM_PATH, STRIP_FROM_INC_PATH, EXTRACT_ALL) - self.sources = _Sources(config, 'sources', self.input_dir) - self.sources.paths.append(str(self.blog_dir)) - self.sources.paths.append(str(self.pages_dir)) - self.sources.paths.sort() + self.sources = _Sources( + config, + r'sources', + self.input_dir, + additional_inputs=[ self.temp_pages_dir, *[f for f,d in self.blog_files] ] + ) self.verbose_object(r'Context.sources', self.sources) # images (IMAGE_PATH) - self.images = _Inputs(config, 'images', self.input_dir) + self.images = _Inputs( + config, + r'images', + self.input_dir, + additional_recursive_inputs=[ self.blog_dir if self.blog_files else None ] + ) self.verbose_object(r'Context.images', self.images) # examples (EXAMPLES_PATH, EXAMPLE_PATTERNS) - self.examples = _FilteredInputs(config, 'examples', self.input_dir) + self.examples = _FilteredInputs( + config, + r'examples', + self.input_dir, + additional_recursive_inputs=[ self.blog_dir if self.blog_files else None ] + ) self.verbose_object(r'Context.examples', self.examples) # tagfiles (TAGFILES) self.tagfiles = { - str(coerce_path(self.data_dir, r'cppreference-doxygen-web.tag.xml')) : r'http://en.cppreference.com/w/' + self.cppref_tagfile : (self.cppref_tagfile, r'http://en.cppreference.com/w/') } self.unresolved_tagfiles = False for k,v in _extract_kvps(config, 'tagfiles').items(): @@ -1335,17 +1386,17 @@ def __init__(self, config_path, output_dir, threads, cleanup, verbose, mcss_dir, dest = str(v) if source and dest: if is_uri(source): - file = str(Path(self.global_temp_dir, rf'tagfile_{sha1(source)}_{now.year}_{now.isocalendar().week}.xml')) + file = Path(self.global_temp_dir, rf'tagfile_{sha1(source)}_{self.now.year}_{self.now.isocalendar().week}.xml') self.tagfiles[source] = (file, dest) self.unresolved_tagfiles = True else: source = Path(source) if not source.is_absolute(): source = Path(self.input_dir, source) - source = str(source.resolve()) - self.tagfiles[source] = dest + source = source.resolve() + self.tagfiles[str(source)] = (source, dest) for k, v in self.tagfiles.items(): - if isinstance(v, str): + if isinstance(v, (Path, str)): assert_existing_file(k) self.verbose_value(r'Context.tagfiles', self.tagfiles) @@ -1528,17 +1579,7 @@ def __init__(self, config_path, output_dir, threads, cleanup, verbose, mcss_dir, self.emoji = self.__emoji self.emoji_codepoints = self.__emoji_codepoints - def __enter__(self): - if not self.dry_run: - if self.cleanup: - delete_directory(self.temp_dir, logger=self.verbose_logger) - delete_directory(self.html_dir, logger=self.verbose_logger) - delete_directory(self.xml_dir, logger=self.verbose_logger) - self.global_temp_dir.mkdir(exist_ok=True) - self.temp_dir.mkdir(exist_ok=True) - self.blog_dir.mkdir(exist_ok=True) - self.pages_dir.mkdir(exist_ok=True) return self def __exit__(self, type, value, traceback): diff --git a/poxy/run.py b/poxy/run.py index 4a57766..874200f 100644 --- a/poxy/run.py +++ b/poxy/run.py @@ -20,12 +20,11 @@ import os import subprocess import concurrent.futures as futures -import argparse + import tempfile import requests from lxml import etree from io import BytesIO, StringIO -from schema import SchemaError #======================================================================================================================= @@ -144,6 +143,8 @@ (r'XML_PROGRAMLISTING', False), ) + + def _preprocess_doxyfile(context): assert context is not None assert isinstance(context, project.Context) @@ -237,7 +238,7 @@ def _preprocess_doxyfile(context): home_md_path = p break if home_md_path is not None: - home_md_temp_path = Path(context.pages_dir, r'home.poxy_md') + home_md_temp_path = Path(context.temp_pages_dir, r'home.md') if not context.dry_run: copy_file(home_md_path, home_md_temp_path, logger=context.verbose_logger) df.set_value(r'USE_MDFILE_AS_MAINPAGE', home_md_temp_path) @@ -265,7 +266,6 @@ def _preprocess_doxyfile(context): df.add_value(r'INPUT', context.sources.paths) df.set_value(r'FILE_PATTERNS', context.sources.patterns) - df.add_value(r'FILE_PATTERNS', [ r'*.poxy_blog', r'*.poxy_md', r'*.poxy_cpp', r'*.poxy_h', r'*.poxy_dox' ]) df.add_value(r'EXCLUDE', context.html_dir) df.add_value(r'STRIP_FROM_PATH', context.sources.strip_paths) @@ -279,7 +279,6 @@ def _preprocess_doxyfile(context): df.add_value(r'EXAMPLE_PATH', context.examples.paths) df.set_value(r'EXAMPLE_PATTERNS', context.examples.patterns) - df.add_value(r'EXTENSION_MAPPING', [ r'poxy_blog=md', r'poxy_md=md', r'poxy_cpp=C++', r'poxy_h=C++', r'poxy_dox=C++' ]) if context.images.paths: # ---------------------------------------------------- df.append() @@ -289,7 +288,7 @@ def _preprocess_doxyfile(context): if context.tagfiles: # ---------------------------------------------------- df.append() df.append(r'# context.tagfiles', end='\n\n') - df.add_value(r'TAGFILES', [rf'{k if isinstance(v, str) else v[0]}={v if isinstance(v, str) else v[1]}' for k,v in context.tagfiles.items()]) + df.add_value(r'TAGFILES', [rf'{file}={dest}' for _,(file, dest) in context.tagfiles.items()]) if context.aliases: # ---------------------------------------------------- df.append() @@ -394,7 +393,6 @@ def _preprocess_doxyfile(context): - def _postprocess_xml(context): assert context is not None assert isinstance(context, project.Context) @@ -407,7 +405,7 @@ def _postprocess_xml(context): if not xml_files: return - with ScopeTimer(rf'Post-processing {len(xml_files)} XML files', print_start=True, print_end=context.verbose_logger): + with ScopeTimer(rf'Post-processing {len(xml_files) + len(context.tagfiles)} XML files', print_start=True, print_end=context.verbose_logger): pretty_print_xml = False xml_parser = etree.XMLParser( @@ -450,7 +448,7 @@ def _postprocess_xml(context): # pre-pass to delete file and dir entries where appropriate: if 1: - dox_files = (r'.dox', r'.md', r'.poxy_md', r'.poxy_blog', r'.poxy_dox') + dox_files = (r'.dox', r'.md') dox_files = [rf'*{doxygen.mangle_name(ext)}.xml' for ext in dox_files] dox_files.append(r'md_home.xml') for xml_file in get_all_files(context.xml_dir, any=dox_files): @@ -478,6 +476,8 @@ def _postprocess_xml(context): macros = set() cpp_tree = CppTree() xml_files = get_all_files(context.xml_dir, any=(r'*.xml')) + tagfiles = [f for _,(f,_) in context.tagfiles.items()] + xml_files = xml_files + tagfiles for xml_file in xml_files: context.verbose(rf'Pre-processing {xml_file}') xml = etree.parse(str(xml_file), parser=xml_parser) @@ -537,6 +537,32 @@ def _postprocess_xml(context): context.__dict__[r'compound_pages'] = pages context.verbose_value(r'Context.compound_pages', pages) + # a tag file + elif root.tag == r'tagfile': + for compound in [tag for tag in root.findall(r'compound') if tag.get(r'kind') in (r'namespace', r'class', r'struct', r'union')]: + + compound_name = compound.find(r'name').text + if compound_name.find(r'<') != -1: + continue + + compound_type = compound.get(r'kind') + if compound_type in (r'class', r'struct', r'union'): + cpp_tree.add_type(compound_name) + else: + cpp_tree.add_namespace(compound_name) + + for member in [tag for tag in compound.findall(r'member') if tag.get(r'kind') in (r'namespace', r'class', r'struct', r'union')]: + + member_name = member.find(r'name').text + if member_name.find(r'<') != -1: + continue + + member_type = member.get(r'kind') + if member_type in (r'class', r'struct', r'union'): + cpp_tree.add_type(compound_name) + else: + cpp_tree.add_namespace(compound_name) + # some other compound definition else: compounddef = root.find(r'compounddef') @@ -896,6 +922,7 @@ def _dump_output_streams(context, outputs, source=''): context.info(outputs[r'stderr'], indent=r' ') + _warnings_regexes = ( # doxygen re.compile(r'^(?P.+?):(?P[0-9]+): warning:\s*(?P.+?)\s*$', re.I), @@ -954,6 +981,7 @@ def _extract_warnings(outputs): return warnings + def run(config_path='.', output_dir='.', threads=-1, @@ -989,11 +1017,8 @@ def run(config_path='.', # resolve any uri tagfiles if context.unresolved_tagfiles: with ScopeTimer(r'Resolving remote tagfiles', print_start=True, print_end=context.verbose_logger) as t: - for source, v in context.tagfiles.items(): - if isinstance(v, str): - continue - file = Path(v[0]) - if file.exists(): + for source, (file, _) in context.tagfiles.items(): + if file.exists() or not is_uri(source): continue context.verbose(rf'Downloading {source} => {file}') response = requests.get( @@ -1100,106 +1125,3 @@ def run(config_path='.', # post-process html files if 1: _postprocess_html(context) - - - - -def main(): - verbose = False - try: - args = argparse.ArgumentParser( - description=r'Generate fancy C++ documentation.', - formatter_class=argparse.RawTextHelpFormatter - ) - args.add_argument( - r'config', - type=Path, - nargs='?', - default=Path('.'), - help=r'path to poxy.toml or a directory containing it (default: %(default)s)' - ) - args.add_argument( - r'-v', r'--verbose', - action=r'store_true', - help=r"enable very noisy diagnostic output" - ) - args.add_argument( - r'--dry', - action=r'store_true', - help=r"do a 'dry run' only, stopping after emitting the effective Doxyfile", - dest=r'dry_run' - ) - args.add_argument( - r'--threads', - type=int, - default=0, - metavar=r'', - help=r"set the number of threads to use (default: automatic)" - ) - args.add_argument( - r'--m.css', - type=Path, - default=None, - metavar=r'', - help=r"specify the version of m.css to use (default: uses the bundled one)", - dest=r'mcss' - ) - args.add_argument( - r'--doxygen', - type=Path, - default=None, - metavar=r'', - help=r"specify the Doxygen executable to use (default: finds Doxygen on system path)", - ) - args.add_argument( - r'--werror', - action=r'store_true', - help=r"always treat warnings as errors regardless of config file settings", - dest=r'treat_warnings_as_errors' - ) - args.add_argument( - r'--version', - action=r'store_true', - help=r"print the version and exit", - dest=r'print_version' - ) - args.add_argument(r'--nocleanup', action=r'store_true', help=argparse.SUPPRESS) - args = args.parse_args() - - if args.print_version: - print(r'.'.join(lib_version())) - sys.exit(0) - - verbose = args.verbose - with ScopeTimer(r'All tasks', print_start=False, print_end=not args.dry_run) as timer: - run( - config_path = args.config, - output_dir = Path.cwd(), - threads = args.threads, - cleanup = not args.nocleanup, - verbose = verbose, - mcss_dir = args.mcss, - doxygen_path = args.doxygen, - logger=True, # stderr + stdout - dry_run=args.dry_run, - treat_warnings_as_errors=True if args.treat_warnings_as_errors else None - ) - sys.exit(0) - - except WarningTreatedAsError as err: - print(rf'Error: {err} (warning treated as error)', file=sys.stderr) - sys.exit(1) - except SchemaError as err: - print(err, file=sys.stderr) - sys.exit(1) - except Error as err: - print(rf'Error: {err}', file=sys.stderr) - sys.exit(1) - except Exception as err: - print_exception(err, include_type=True, include_traceback=True, skip_frames=1) - sys.exit(-1) - - - -if __name__ == '__main__': - main() diff --git a/poxy/utils.py b/poxy/utils.py index 1ae0099..2b02e98 100644 --- a/poxy/utils.py +++ b/poxy/utils.py @@ -62,7 +62,7 @@ def log(logger, msg, level=logging.INFO): -def enum_subdirs(root, filter=None): +def enum_subdirs(root, filter=None, recursive=False): root = coerce_path(root) assert root.is_dir() subdirs = [] @@ -71,7 +71,8 @@ def enum_subdirs(root, filter=None): if filter is not None and not filter(p): continue subdirs.append(p) - subdirs = subdirs + enum_subdirs(p, filter=filter) + if recursive: + subdirs = subdirs + enum_subdirs(p, filter=filter, recursive=True) return subdirs diff --git a/setup.py b/setup.py index 26a896a..361e41a 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,10 @@ def enum_subdirs(root): ] }, entry_points = { - r'console_scripts' : [ r'poxy = poxy.__main__:main' ] + r'console_scripts' : [ + r'poxy = poxy.main:_run', + r'poxyblog = poxy.main:_make_blog_post' + ] } )