From 61308c141ca851aa5b606d018a5a941bce57ef6d Mon Sep 17 00:00:00 2001 From: Jonathan Cubides Date: Thu, 5 Dec 2024 12:50:12 +0100 Subject: [PATCH] Bump up version to 0.4.7 (#15) Minor fixes --- mkdocs_juvix/env.py | 6 +- mkdocs_juvix/links.py | 7 +- mkdocs_juvix/logger.py | 31 +----- mkdocs_juvix/main.py | 130 +++++++++++++----------- mkdocs_juvix/snippets.py | 14 ++- pyproject.toml | 2 +- src/fixtures/ci.yml | 4 +- src/fixtures/tutorial/juvix_markdown.md | 15 +++ 8 files changed, 108 insertions(+), 101 deletions(-) diff --git a/mkdocs_juvix/env.py b/mkdocs_juvix/env.py index 80d5a0d..4816240 100644 --- a/mkdocs_juvix/env.py +++ b/mkdocs_juvix/env.py @@ -50,7 +50,7 @@ class ENV: REMOVE_CACHE: bool = bool(getenv("REMOVE_CACHE", False)) - JUVIX_ENABLED: bool = bool(getenv("JUVIX_ENABLED", True)) + PROCESS_JUVIX: bool = bool(getenv("PROCESS_JUVIX", False)) JUVIX_FULL_VERSION: str JUVIX_BIN_NAME: str = getenv("JUVIX_BIN", "juvix") JUVIX_BIN_PATH: str = getenv("JUVIX_PATH", "") @@ -243,7 +243,7 @@ def __init__(self, config: Optional[MkDocsConfig] = None): log.debug( "Juvix version not found. Make sure Juvix is installed, for now support for Juvix Markdown is disabled." ) - self.JUVIX_ENABLED = False + self.PROCESS_JUVIX = False self.JUVIX_AVAILABLE = False return @@ -264,7 +264,7 @@ def __init__(self, config: Optional[MkDocsConfig] = None): @property def juvix_enabled(self) -> bool: - return self.JUVIX_ENABLED and self.JUVIX_AVAILABLE + return self.PROCESS_JUVIX and self.JUVIX_AVAILABLE @staticmethod def when_juvix_enabled(func): diff --git a/mkdocs_juvix/links.py b/mkdocs_juvix/links.py index 3410948..38b8884 100644 --- a/mkdocs_juvix/links.py +++ b/mkdocs_juvix/links.py @@ -20,7 +20,7 @@ from mkdocs_juvix.common.preprocesors.links import WLPreprocessor from mkdocs_juvix.common.utils import fix_site_url, get_page_title from mkdocs_juvix.env import ENV -from mkdocs_juvix.logger import clear_line, clear_screen, log +from mkdocs_juvix.logger import log files_relation: List[ResultEntry] = [] EXCLUDED_DIRS = [ @@ -117,7 +117,6 @@ def on_pre_build(self, config: MkDocsConfig) -> None: config["nodes"][url]["page"]["names"].append(page) node_index += 1 pbar.update(1) - clear_line() if self.NODES_JSON.exists(): self.NODES_JSON.unlink() @@ -168,14 +167,12 @@ def process_file(file: File) -> None: url = urljoin(config.get("site_url", ""), file.url) config["url_for"][_title] = [url] config["aliases_for"][url] = [_title] - clear_screen() + with sync_tqdm(total=len(files), desc="> processing files") as pbar: for file in files: if file.is_documentation_page(): process_file(file) pbar.update(1) - clear_line() - if self.LINKS_JSON.exists(): self.LINKS_JSON.unlink() diff --git a/mkdocs_juvix/logger.py b/mkdocs_juvix/logger.py index 5feb19e..3499713 100644 --- a/mkdocs_juvix/logger.py +++ b/mkdocs_juvix/logger.py @@ -1,36 +1,11 @@ import os import logging -from typing import Any, MutableMapping from colorama import Fore, Style # type: ignore -DEBUG = os.getenv("DEBUG", "false").lower() == "true" -print(f"{Fore.GREEN}DEBUG: {DEBUG}") - -class Logger(logging.Logger): - def __init__(self, logger: logging.Logger): - self.logger = logger - super().__init__(logger.name, logger.level) - - def info(self, msg, *args, **kwargs): - if DEBUG: - self.debug(msg, *args, **kwargs) - else: - super().info(msg, *args, **kwargs) - - def debug(self, msg, *args, **kwargs): - if DEBUG: - print("-"*100) - print(msg, *args, **kwargs) - clear_line(2) - else: - super().debug(msg, *args, **kwargs) +log = logging.getLogger("mkdocs.plugins.juvix") -def get_plugin_logger(name: str) -> Logger: - logger = logging.getLogger(f"mkdocs.plugins.{name}") - setattr(logger, "info", lambda msg: getattr(logger, "info")(msg)) - return Logger(logger) - -log = get_plugin_logger(f"{Fore.BLUE}juvix_mkdocs{Style.RESET_ALL}") +DEBUG = bool(os.getenv("DEBUG", "false").lower()) == "true" +print(f"{Fore.GREEN}DEBUG: {DEBUG}") def clear_screen(): if os.getenv("DEBUG", "false").lower() != "true": diff --git a/mkdocs_juvix/main.py b/mkdocs_juvix/main.py index cf56c33..8ab4d49 100644 --- a/mkdocs_juvix/main.py +++ b/mkdocs_juvix/main.py @@ -29,7 +29,7 @@ from mkdocs_juvix.env import ENV, FIXTURES_PATH from mkdocs_juvix.images import process_images from mkdocs_juvix.links import TOKEN_LIST_WIKILINKS, WikilinksPlugin -from mkdocs_juvix.logger import clear_line, clear_screen, log +from mkdocs_juvix.logger import log from mkdocs_juvix.snippets import RE_SNIPPET_SECTION, SnippetPreprocessor from mkdocs_juvix.utils import ( compute_sha_over_folder, @@ -952,7 +952,7 @@ def generate_juvix_markdown_output( ) return None - log.debug(f"> Generating Juvix markdown for {self.relative_filepath}") + log.debug(f"> Generating Juvix markdown for {self.relative_filepath}") if self.changed_since_last_run(): log.debug("> File has changed since last run, generating markdown") if ( @@ -964,7 +964,7 @@ def generate_juvix_markdown_output( f"> Skipping markdown generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output" ) return None - + log.debug( f"> Reading cached markdown for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}" ) @@ -1093,7 +1093,7 @@ def run_snippet_preprocessor( self.env.CACHE_PROCESSED_MARKDOWN_PATH.resolve().absolute(), ] snippet_preprocessor.check_paths = check_paths - + if content: try: @@ -1155,7 +1155,7 @@ def run_wikilinks_preprocessor( content: str, modify_markdown_output: bool = True, ) -> str: - + assert "url_for" in self.config, "url_for is not in the config" assert "nodes" in self.config, "nodes is not in the config" @@ -1248,6 +1248,7 @@ def __init__(self, config, env: ENV, docs: Optional[Path] = None): log.error(f"Error initializing JuvixMarkdownCollection: {e}") raise + @time_spent(message="> scanning originals") def scanning_originals(self) -> List[EnhancedMarkdownFile]: """ Cache the original Juvix Markdown files in the cache folder for faster @@ -1277,6 +1278,7 @@ def scanning_originals(self) -> List[EnhancedMarkdownFile]: log.debug(f"Creating EnhancedMarkdownFile for {file}") enhanced_file = EnhancedMarkdownFile(file, self.env, self.config) self.files.append(enhanced_file) + log.debug(f"Creating parent directory for {enhanced_file.original_in_cache_filepath}") enhanced_file.original_in_cache_filepath.parent.mkdir( parents=True, exist_ok=True @@ -1286,7 +1288,6 @@ def scanning_originals(self) -> List[EnhancedMarkdownFile]: ) shutil.copy(file, enhanced_file.original_in_cache_filepath) pbar.update(1) - clear_line() return self.files except Exception as e: @@ -1393,7 +1394,7 @@ def run_pipeline_on_collection( if self.files is None: log.debug(f"{Fore.YELLOW}> no files to process{Style.RESET_ALL}") return - + log.debug( f"> running pipeline on {Fore.GREEN}{len(self.files)}{Style.RESET_ALL} files" ) @@ -1426,11 +1427,13 @@ async def process_original_markdowns(): ) asyncio.run(process_original_markdowns()) - juvix_files = [ - file - for file in files_to_process - if is_juvix_markdown_file(file.absolute_filepath) - ] + juvix_files = [] + if (generate_juvix_markdown or generate_juvix_isabelle) and self.env.juvix_enabled: + juvix_files = [ + file + for file in files_to_process + if is_juvix_markdown_file(file.absolute_filepath) + ] if generate_juvix_markdown and self.env.juvix_enabled: with sync_tqdm( @@ -1467,23 +1470,22 @@ async def process_original_markdowns(): f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" ) pbar.update(1) - - @time_spent(message="> processing wikilinks") - def process_wikilinks(): - flist = self.files - with sync_tqdm( - total=len(flist), desc="> processing wikilinks" - ) as pbar: - for file in flist: - pbar.set_postfix_str( - f"{Fore.MAGENTA}{file.relative_filepath}{Style.RESET_ALL}" - ) - file.replaces_wikilinks_by_markdown_links() - pbar.update(1) - clear_line() - - process_wikilinks() - log.debug(f"{Fore.GREEN}finished wikilinks{Style.RESET_ALL}") + if generate_wikilinks: + @time_spent(message="> processing wikilinks") + def process_wikilinks(): + flist = self.files + with sync_tqdm( + total=len(flist), desc="> processing wikilinks" + ) as pbar: + for file in flist: + pbar.set_postfix_str( + f"{Fore.MAGENTA}{file.relative_filepath}{Style.RESET_ALL}" + ) + file.replaces_wikilinks_by_markdown_links() + pbar.update(1) + + process_wikilinks() + log.debug(f"{Fore.GREEN}finished wikilinks{Style.RESET_ALL}") if generate_snippets: async def process_snippets(): @@ -1495,7 +1497,6 @@ async def process_snippets(): asyncio.run(process_snippets()) log.debug(f"{Fore.GREEN}finished snippets{Style.RESET_ALL}") - clear_line() self.update_cached_hash() self.save_juvix_modules_json() @@ -1515,7 +1516,7 @@ def generate_html(self, force: bool = False) -> None: all the Juvix Markdown files. Otherwise, we generate the HTML output for every Juvix Markdown file individually (not recommended). """ - if not self.env.JUVIX_ENABLED: + if not self.env.juvix_enabled: log.info( f"{Fore.YELLOW}Juvix is not enabled, skipping HTML generation{Style.RESET_ALL}" ) @@ -1566,6 +1567,12 @@ def clean_juvix_dependencies(self) -> None: Clean the Juvix dependencies. This is necessary to avoid typechecking errors when the Juvix compiler version changes. """ + if not self.env.juvix_enabled: + log.info( + f"{Fore.YELLOW}Juvix is not enabled, skipping Juvix dependencies cleaning{Style.RESET_ALL}" + ) + return + if not self.env.CLEAN_DEPS: log.info( f"Skipping Juvix dependencies cleaning because " @@ -1594,6 +1601,12 @@ def update_juvix_dependencies(self) -> bool: """ Update the Juvix dependencies. """ + if not self.env.juvix_enabled: + log.info( + f"{Fore.YELLOW}Juvix is not enabled, skipping Juvix dependencies updating{Style.RESET_ALL}" + ) + return False + update_command = [self.env.JUVIX_BIN, "dependencies", "update"] res = subprocess.run( update_command, @@ -1619,25 +1632,22 @@ class JuvixPlugin(BasePlugin): first_run: bool = True def on_startup(self, *, command: str, dirty: bool) -> None: - log.debug(f"{Fore.GREEN}on_startup...{Style.RESET_ALL}") + log.info(f"{Fore.GREEN}Starting up...{Style.RESET_ALL}") def on_config(self, config: MkDocsConfig) -> MkDocsConfig: - clear_screen() + log.info(f"{Fore.GREEN}Configuring...{Style.RESET_ALL}") self.env = ENV(config) config = fix_site_url(config) self.env.SITE_DIR = config.get("site_dir", getenv("SITE_DIR", None)) - if os.getenv("SKIP_JUVIX"): - self.env.JUVIX_ENABLED = False - - if self.env.JUVIX_ENABLED and not self.env.JUVIX_AVAILABLE: - log.error( - "You have requested Juvix but it is not available. Check your configuration." - "\nEnvironment variables relevant to the build process:" - "\n- JUVIX_ENABLED" - "\n- JUVIX_BIN" - "\n- JUVIX_PATH" + if self.env.JUVIX_AVAILABLE and not self.env.PROCESS_JUVIX: + log.info( + f"The Juvix compiler is available but Juvix is not enabled by default. " + f"Therefore, the output of the Juvix Markdown processor will be " + f"the same as the original markdown. If you want to process " + f"Juvix Markdown, run: {Fore.GREEN}`PROCESS_JUVIX=true poetry " + f"run mkdocs build`{Style.RESET_ALL}." ) if self.first_run: @@ -1674,7 +1684,9 @@ def on_files(self, files: Files, *, config: MkDocsConfig) -> Optional[Files]: the site directory. """ log.debug(f"{Fore.GREEN}on_files...{Style.RESET_ALL}") - self.wikilinks_plugin.on_files(files, config) + if self.enhanced_collection.has_changes(): + self.wikilinks_plugin.on_files(files, config) + Files( [ file @@ -1688,7 +1700,7 @@ def on_files(self, files: Files, *, config: MkDocsConfig) -> Optional[Files]: mkdocs_config_filepath = self.env.ROOT_ABSPATH / "mkdocs.yml" if mkdocs_config_filepath in [file.abs_src_path for file in files]: self.enhanced_collection.force_wikilinks_generation = True - log.debug(f"{Fore.GREEN}finished on_files...{Style.RESET_ALL}") + log.info("> Now is the time for MkDocs to finish") return files def on_nav(self, nav, config: MkDocsConfig, files: Files): @@ -1705,7 +1717,7 @@ def on_page_read_source(self, page: Page, config: MkDocsConfig) -> Optional[str] return None abs_src_path: Path = Path(abs_src_str) - log.debug(f"{Fore.CYAN}Processing file: {abs_src_path}{Style.RESET_ALL}") + log.info(f"Mkdocs (on_page_read_source): {Fore.MAGENTA}{abs_src_path}{Style.RESET_ALL}") try: file: Optional[EnhancedMarkdownFile] = ( self.enhanced_collection.get_enhanced_file_entry(abs_src_path) @@ -1744,6 +1756,7 @@ def on_page_markdown( if not abs_src_str: return markdown + log.info(f"Mkdocs (on_page_markdown): {Fore.MAGENTA}{abs_src_str}{Style.RESET_ALL}") page.file.name = page.file.name.replace(".juvix", "") page.file.url = page.file.url.replace(".juvix", "") @@ -1756,31 +1769,32 @@ def on_page_markdown( def on_page_content( self, html: str, page: Page, config: MkDocsConfig, files: Files ) -> Optional[str]: - # html = self.wikilinks_plugin.on_page_content(html, page, config, files) + log.info(f"Mkdocs (on_page_content): {Fore.MAGENTA}{page.file.abs_src_path}{Style.RESET_ALL}") return html def on_post_page(self, output: str, page: Page, config: MkDocsConfig) -> str: soup = BeautifulSoup(output, "html.parser") for a in soup.find_all("a"): a["href"] = a["href"].replace(".juvix.html", ".html") + log.info(f"Mkdocs (on_post_page): {Fore.MAGENTA}{page.file.abs_src_path}{Style.RESET_ALL}") return str(soup) - + def get_context(self, context, page, config, nav): log.info(f"{Fore.GREEN}Processing context...{Style.RESET_ALL}") return context - + def get_template(self, template, context): log.info(f"{Fore.GREEN}Processing template...{Style.RESET_ALL}") return template, context - + def render(self, template, context): log.info(f"{Fore.GREEN}Rendering template...{Style.RESET_ALL}") return template, context - + def on_post_build(self, config: MkDocsConfig) -> None: - log.info(f"{Fore.GREEN}on_post_build...{Style.RESET_ALL}") - if self.env.JUVIX_ENABLED: + log.info("Mkdocss (on_post_build)") + if self.env.PROCESS_JUVIX: log.debug(f"{Fore.GREEN}generating HTML...{Style.RESET_ALL}") self.enhanced_collection.generate_html() log.debug(f"{Fore.GREEN}moving HTML cache to site directory...{Style.RESET_ALL}") @@ -1795,17 +1809,17 @@ def on_post_build(self, config: MkDocsConfig) -> None: ) for file in files_to_check: file.load_and_print_saved_error_messages() - + files = self.enhanced_collection.files if self.enhanced_collection.files else [] files_to_process: List[EnhancedMarkdownFile] = [ file for file in files if files and file.has_error_message() ] - log.info(f"{Fore.GREEN}Files with errors: {len(files_to_process)}{Style.RESET_ALL}") - log.info("Next time, we will process the following files:") + log.info(f"{Fore.YELLOW}Files with some processing errors or warnings: {len(files_to_process)}{Style.RESET_ALL}") + log.info("Based on the previous run, we are forced to process the following files, next time:") for file in files_to_process: - log.info(f"{Fore.GREEN}{file.relative_filepath}{Style.RESET_ALL}") + log.info(f"> {Fore.MAGENTA}{file.relative_filepath}{Style.RESET_ALL}") log.debug(f"{Fore.GREEN}finished on_post_build...{Style.RESET_ALL}") def move_html_cache_to_site_dir(self) -> None: @@ -1874,8 +1888,6 @@ def wrapper(event: FileSystemEvent) -> None: ): return - # clear the console - clear_screen() fpath = Path(fpathstr) if fpath.is_relative_to(self.env.DOCS_ABSPATH): log.info( diff --git a/mkdocs_juvix/snippets.py b/mkdocs_juvix/snippets.py index 4f4c2e6..1b4e5b1 100644 --- a/mkdocs_juvix/snippets.py +++ b/mkdocs_juvix/snippets.py @@ -338,6 +338,12 @@ def get_snippet_path(self, path: Path | str): search_for_juvix_isabelle_output = False if path and path.as_posix().endswith(".juvix.md!thy"): + if not self.env.PROCESS_JUVIX: + log.info( + f"{Fore.YELLOW}Juvix is not enabled, skipping Juvix snippets{Style.RESET_ALL}" + ) + return None + search_for_juvix_isabelle_output = True log.debug( f"Path ends with .juvix.md!thy: {Fore.MAGENTA}{path}{Style.RESET_ALL}" @@ -463,10 +469,10 @@ def parse_snippets( end = None start = None section = None - log.debug(f"{Fore.YELLOW}>>>>>> path: {path}{Style.RESET_ALL}") + log.debug(f"{Fore.YELLOW}> path: {path}{Style.RESET_ALL}") m = RE_SNIPPET_FILE.match(path) if m is None: - log.debug(f"{Fore.YELLOW}>>>>>> m is None{Style.RESET_ALL}") + log.debug(f"{Fore.YELLOW}> RE_SNIPPET_FILE match is None{Style.RESET_ALL}") continue path = m.group(1).strip() @@ -500,7 +506,9 @@ def parse_snippets( if found_snippet is None: if self.check_paths: - msg = f"Wrong snippet path: '{Fore.MAGENTA}{path}{Style.RESET_ALL}' could not be found. {Fore.YELLOW}Check the path, perhaps you forgot e.g., adding the prefix './' to the path or ./docs/{Style.RESET_ALL}" + msg = f"Wrong snippet path: '{Fore.MAGENTA}{path}{Style.RESET_ALL}' " + msg += f"could not be found. {Fore.YELLOW}Check the path, perhaps you" + msg += f"forgot e.g., adding the prefix './' to the path or ./docs/{Style.RESET_ALL}" return SnippetMissingError(msg) log.debug(f"{Fore.GREEN}Snippet found:{found_snippet}{Style.RESET_ALL}") diff --git a/pyproject.toml b/pyproject.toml index 5bcaa92..93ec2eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mkdocs-juvix-plugin" -version = "0.4.6" +version = "0.4.7" description = "MkDocs documentation with support for Juvix Markdown files" authors = ["Jonathan Prieto-Cubides, and GitHub contributors"] license = "MIT" diff --git a/src/fixtures/ci.yml b/src/fixtures/ci.yml index 4346553..56af4ac 100644 --- a/src/fixtures/ci.yml +++ b/src/fixtures/ci.yml @@ -9,7 +9,7 @@ env: PYTHON_VERSION: 3.x POETRY_VERSION: 1.8.3 JUVIX_VERSION: {juvix_version} - JUVIX_ENABLED: true + PROCESS_JUVIX: true GIT_COMMITTER_EMAIL: {site_author_email} GIT_COMMITTER_NAME: {site_author} BRANCH_NAME: ${{{{ github.head_ref || github.ref_name }}}} @@ -80,7 +80,7 @@ jobs: git-config-name: ${{{{ env.GIT_COMMITTER_NAME }}}} git-config-email: ${{{{ env.GIT_COMMITTER_EMAIL }}}} - - if: env.JUVIX_ENABLED == 'true' + - if: env.PROCESS_JUVIX == 'true' run: | cd docs juvix typecheck everything.juvix.md diff --git a/src/fixtures/tutorial/juvix_markdown.md b/src/fixtures/tutorial/juvix_markdown.md index febc95d..0e4a799 100644 --- a/src/fixtures/tutorial/juvix_markdown.md +++ b/src/fixtures/tutorial/juvix_markdown.md @@ -1,5 +1,20 @@ # Juvix Markdown Structure +The Juvix Markdown processor can turn Juvix code blocks into HTML. +The syntax highlighted is semantic, not lexical. This brings a more clear +understanding of the code. However, at the moment, processing Juvix +Markdown takes two seconds per file, on average. Therefore it is disabled by +default. + +To enable it, set the `PROCESS_JUVIX` environment variable to `true`. +So, next time you run `mkdocs build`, Juvix Markdown will be processed. For +example, you can run the command in the following way: + +```bash +PROCESS_JUVIX=true poetry run mkdocs serve +``` + + ## File Structure A Juvix Markdown file (`.juvix.md`) must follow these rules: