From 4bea8e6085c595a4c98c395247b526a56b9cf585 Mon Sep 17 00:00:00 2001
From: Jonathan Cubides <jonathan.cubides@uib.no>
Date: Mon, 2 Dec 2024 17:01:36 +0100
Subject: [PATCH] v0.4.6 (#14)

---
 mkdocs_juvix/common/preprocesors/links.py |  36 ++--
 mkdocs_juvix/env.py                       |   9 +-
 mkdocs_juvix/links.py                     |  75 ++++----
 mkdocs_juvix/logger.py                    |  93 ++++-----
 mkdocs_juvix/main.py                      | 222 ++++++++++++++++------
 mkdocs_juvix/snippets.py                  |   9 +-
 mkdocs_juvix/utils.py                     |  10 +-
 pyproject.toml                            |   2 +-
 8 files changed, 273 insertions(+), 183 deletions(-)

diff --git a/mkdocs_juvix/common/preprocesors/links.py b/mkdocs_juvix/common/preprocesors/links.py
index 7a6081d..be37c27 100644
--- a/mkdocs_juvix/common/preprocesors/links.py
+++ b/mkdocs_juvix/common/preprocesors/links.py
@@ -5,7 +5,7 @@
 from urllib.parse import urljoin
 
 import numpy as np  # type: ignore
-from colorama import Fore, Style
+from colorama import Fore, Style  # type: ignore
 from fuzzywuzzy import fuzz  # type: ignore
 from markdown.preprocessors import Preprocessor  # type: ignore
 from ncls import NCLS  # type: ignore
@@ -101,10 +101,9 @@ def process_wikilink(config, full_text, match, md_filepath) -> Optional[WikiLink
     )
 
     link_page = link.page
-    # print white space with "X"
 
-    if len(config["url_for"].get(link_page, [])) > 1 and link_page in config["url_for"]:
-        possible_pages = config["url_for"][link_page]
+    if len(config.get("url_for", {}).get(link_page, [])) > 1 and link_page in config.get("url_for", {}):
+        possible_pages = config.get("url_for", {}).get(link_page, [])
         hint = link.hint if link.hint else ""
         token = hint + link_page
         coefficients = {p: fuzz.WRatio(fun_normalise(p), token) for p in possible_pages}
@@ -121,8 +120,8 @@ def process_wikilink(config, full_text, match, md_filepath) -> Optional[WikiLink
                 Our choice: {link.html_path}"""
         )
 
-    elif link_page in config["url_for"]:
-        link.html_path = config["url_for"].get(link_page, [""])[0]
+    elif link_page in config.get("url_for", {}):
+        link.html_path = config.get("url_for", {}).get(link_page, [""])[0]
         log.debug(f"Single page found. html_path: {link.html_path}")
     else:
         log.debug("Link page not in config['url_for']")
@@ -135,11 +134,11 @@ def process_wikilink(config, full_text, match, md_filepath) -> Optional[WikiLink
 
         # Update links_found TODO: move this to the model
         try:
-            url_page = config["url_for"][link_page][0]
-            if url_page in config["nodes"]:
-                actuallink = config["nodes"][url_page]
+            url_page = config.get("url_for", {}).get(link_page, [""])[0]
+            if url_page in config.get("nodes", {}):
+                actuallink = config.get("nodes", {}).get(url_page, {})
                 if actuallink:
-                    pageName = actuallink["page"].get("names", [""])[0]
+                    pageName = actuallink.get("page", {}).get("names", [""])[0]
                     html_path: str = link.html_path if link.html_path else ""
                     config.get("links_found", []).append(
                         {
@@ -192,7 +191,8 @@ def _run(self, content: str) -> str:
 
         # Find all code blocks, HTML comments, and script tags in a single pass
         ignore_blocks = re.compile(
-            r"((`{1,3})(?:[\s\S]*?)(\2)|<!--[\s\S]*?-->|<script>[\s\S]*?</script>)",
+            # r"((`{1,3})(?:[\s\S]*?)(\2)|<!--[\s\S]*?-->|<script>[\s\S]*?</script>)",
+            r"((`{1,3})(?:[\s\S]*?)(\2))",
             re.DOTALL,
         )
         intervals = []
@@ -205,10 +205,12 @@ def _run(self, content: str) -> str:
         except Exception as e:
             log.error(f"Error occurred while processing ignore patterns: {str(e)}")
             return content
-        intervals_where_not_to_look = None
-        if intervals:
-            starts, ends, ids = map(np.array, zip(*intervals))
-            intervals_where_not_to_look = NCLS(starts, ends, ids)
+        
+        # Review this for later improvements
+        # intervals_where_not_to_look = None
+        # if intervals:
+        #     starts, ends, ids = map(np.array, zip(*intervals))
+        #     intervals_where_not_to_look = NCLS(starts, ends, ids)
 
         # Find all wikilinks
         str_wikilinks = list(WIKILINK_PATTERN.finditer(content))
@@ -218,9 +220,7 @@ def _run(self, content: str) -> str:
             start, end = m.start(), m.end()
 
             # TODO: review this
-            if intervals_where_not_to_look and not list(
-                intervals_where_not_to_look.find_overlap(start, end)
-            ):
+            if True:
                 log.debug(
                     f"{Fore.YELLOW}Processing wikilink: {m.group(0)}{Style.RESET_ALL}"
                 )
diff --git a/mkdocs_juvix/env.py b/mkdocs_juvix/env.py
index d531aef..80d5a0d 100644
--- a/mkdocs_juvix/env.py
+++ b/mkdocs_juvix/env.py
@@ -14,14 +14,13 @@
 
 from colorama import Fore, Style  # type: ignore
 from mkdocs.config.defaults import MkDocsConfig
-from mkdocs.plugins import get_plugin_logger
 from semver import Version
 
 import mkdocs_juvix.utils as utils
 from mkdocs_juvix.juvix_version import MIN_JUVIX_VERSION
 from mkdocs_juvix.utils import is_juvix_markdown_file
 
-log = get_plugin_logger(f"{Fore.BLUE}[juvix_mkdocs] (env) {Style.RESET_ALL}")
+from mkdocs_juvix.logger import log
 
 BASE_PATH = Path(__file__).parent
 FIXTURES_PATH = BASE_PATH / "fixtures"
@@ -182,6 +181,12 @@ def __init__(self, config: Optional[MkDocsConfig] = None):
             )
             exit(1)
 
+        if not self.CACHE_ABSPATH.exists():
+            log.info(
+                f"{Fore.YELLOW}Creating cache directory {self.CACHE_ABSPATH}{Style.RESET_ALL}"
+            )
+            self.CACHE_ABSPATH.mkdir(parents=True, exist_ok=True)
+
         if (
             self.CACHE_ABSPATH.exists()
             and self.REMOVE_CACHE
diff --git a/mkdocs_juvix/links.py b/mkdocs_juvix/links.py
index b9694e3..3410948 100644
--- a/mkdocs_juvix/links.py
+++ b/mkdocs_juvix/links.py
@@ -4,13 +4,12 @@
 
 import json
 import re
-from concurrent.futures import ThreadPoolExecutor
 from os import getenv
 from pathlib import Path
 from typing import Dict, List, Optional
 from urllib.parse import urljoin
 
-from colorama import Fore, Style  # type: ignore
+from tqdm import tqdm as sync_tqdm  # type: ignore
 from markdown.extensions import Extension  # type: ignore
 from mkdocs.config.defaults import MkDocsConfig
 from mkdocs.structure.files import File, Files
@@ -21,7 +20,7 @@
 from mkdocs_juvix.common.preprocesors.links import WLPreprocessor
 from mkdocs_juvix.common.utils import fix_site_url, get_page_title
 from mkdocs_juvix.env import ENV
-from mkdocs_juvix.logger import log
+from mkdocs_juvix.logger import clear_line, clear_screen, log
 
 files_relation: List[ResultEntry] = []
 EXCLUDED_DIRS = [
@@ -57,7 +56,6 @@ def extendMarkdown(self, md):  # noqa: N802
 
 TOKEN_LIST_WIKILINKS: str = "<!-- list_wikilinks -->"
 
-
 class WikilinksPlugin:
     env: Optional[ENV] = None
 
@@ -100,35 +98,40 @@ def on_pre_build(self, config: MkDocsConfig) -> None:
         config["wikilinks_issues"] = 0
         config["nodes"] = {}
         node_index = 0
-
-        for _url, page in _extract_aliases_from_nav(config["nav"]):
-            url = urljoin(config["site_url"], _url)
-
-            config["aliases_for"][url] = [page]
-            config["url_for"].setdefault(page, [])
-            config["url_for"][page].append(url)
-
-            # Create a new entry if the URL is not already present in config["nodes"]
-            if url not in config["nodes"]:
-                config["nodes"][url] = {
-                    "index": node_index,
-                    "page": {"names": [], "path": _url.replace("./", "")},
-                }
-            # Append the page to the "names" list
-            config["nodes"][url]["page"]["names"].append(page)
-            node_index += 1
+        nav_items = list(_extract_aliases_from_nav(config["nav"]))
+
+        with sync_tqdm(total=len(nav_items), desc="> processing nav items") as pbar:
+            for _url, page in nav_items:
+                url = urljoin(config["site_url"], _url)
+                config["aliases_for"][url] = [page]
+                config["url_for"].setdefault(page, [])
+                config["url_for"][page].append(url)
+
+                # Create a new entry if the URL is not already present in config["nodes"]
+                if url not in config["nodes"]:
+                    config["nodes"][url] = {
+                        "index": node_index,
+                        "page": {"names": [], "path": _url.replace("./", "")},
+                    }
+                # Append the page to the "names" list
+                config["nodes"][url]["page"]["names"].append(page)
+                node_index += 1
+                pbar.update(1)
+        clear_line()   
 
         if self.NODES_JSON.exists():
             self.NODES_JSON.unlink()
-
-        with open(self.NODES_JSON, "w") as f:
-            json.dump(
+        try:    
+            with open(self.NODES_JSON, "w") as f:
+                json.dump(
                 {
                     "nodes": config.get("nodes", {}),
                 },
                 f,
                 indent=2,
             )
+        except Exception as e:
+            log.error(f"Error writing nodes.json: {e}")
         config["current_page"] = None  # current page being processed
         return
 
@@ -161,18 +164,17 @@ def process_file(file: File) -> None:
                             _title = _title.strip()
                             _title = re.sub(r'^[\'"`]|["\'`]$', "", _title)
 
-                            if _title not in config["url_for"]:
-                                url = urljoin(config["site_url"], file.url)
+                            if _title not in config.get("url_for", {}):
+                                url = urljoin(config.get("site_url", ""), file.url)
                                 config["url_for"][_title] = [url]
                                 config["aliases_for"][url] = [_title]
-
-        with ThreadPoolExecutor() as executor:
-            list(
-                executor.map(
-                    process_file, filter(lambda f: f.is_documentation_page(), files)
-                )
-            )
-            executor.shutdown(wait=True)
+        clear_screen()
+        with sync_tqdm(total=len(files), desc="> processing files") as pbar:
+            for file in files:
+                if file.is_documentation_page():
+                    process_file(file)
+                    pbar.update(1)
+        clear_line()
 
         if self.LINKS_JSON.exists():
             self.LINKS_JSON.unlink()
@@ -201,16 +203,20 @@ def on_page_content(
         frontmatter has the `list_wikilinks` flag set to true.
         """
         if "current_page" not in config or "nodes" not in config:
+            log.debug("No current_page or nodes in config")
             return html
         current_page = config["current_page"]
         url = current_page.canonical_url.replace(".html", ".md")
         if url not in config["nodes"]:
+            log.debug(f"URL {url} not found in nodes. It's probably ignored because it's not in the mkdocs.yml file.")
             return html
 
         if url not in config["nodes"] or "index" not in config["nodes"][url]:
+            log.debug(f"URL {url} not found in nodes or no index for URL")
             return html
         links_number: List[Dict[str, int]] = config.get("links_number", [])
         if len(links_number) > 0:
+            log.debug(f"Processing {len(links_number)} links for {url}")
             actualindex = config["nodes"][url]["index"]
             result_entry = ResultEntry(
                 file=current_page.url,
@@ -222,6 +228,7 @@ def on_page_content(
             files_relation.append(result_entry)
 
             if page.meta.get("list_wikilinks", False):
+                log.debug(f"Generating wikilinks list for {url}")
                 # Creat a bullet list of links
                 wrapped_links = "<details class='quote'><summary>Relevant internal links on this page</summary><ul>"
                 unique_links = {
diff --git a/mkdocs_juvix/logger.py b/mkdocs_juvix/logger.py
index ed84c8c..5feb19e 100644
--- a/mkdocs_juvix/logger.py
+++ b/mkdocs_juvix/logger.py
@@ -1,68 +1,43 @@
+import os
 import logging
 from typing import Any, MutableMapping
-
 from colorama import Fore, Style  # type: ignore
 
-
-class PrefixedLogger(logging.LoggerAdapter):
-    """A logger adapter to prefix log messages."""
-
-    def __init__(self, prefix: str, logger: logging.Logger) -> None:
-        """
-        Initialize the logger adapter.
-
-        Arguments:
-            prefix: The string to insert in front of every message.
-            logger: The logger instance.
-        """
-        super().__init__(logger, {})
-        self.prefix = prefix
-
-    def process(self, msg: str, kwargs: MutableMapping[str, Any]) -> tuple[str, Any]:
-        """
-        Process the message.
-
-        Arguments:
-            msg: The message:
-            kwargs: Remaining arguments.
-
-        Returns:
-            The processed message.
-        """
-        return f"{self.prefix}: {msg}", kwargs
-
-
-def get_plugin_logger(name: str) -> PrefixedLogger:
-    """
-    Return a logger for plugins.
-
-    Arguments:
-        name: The name to use with `logging.getLogger`.
-
-    Returns:
-        A logger configured to work well in MkDocs,
-            prefixing each message with the plugin package name.
-
-    Example:
-        ```python
-        from mkdocs.plugins import get_plugin_logger
-
-        log = get_plugin_logger(__name__)
-        log.info("My plugin message")
-        ```
-    """
+DEBUG = os.getenv("DEBUG", "false").lower() == "true"
+print(f"{Fore.GREEN}DEBUG: {DEBUG}")
+
+class Logger(logging.Logger):
+    def __init__(self, logger: logging.Logger):
+        self.logger = logger
+        super().__init__(logger.name, logger.level)
+        
+    def info(self, msg, *args, **kwargs):
+        if DEBUG:
+            self.debug(msg, *args, **kwargs)
+        else:
+            super().info(msg, *args, **kwargs)
+
+    def debug(self, msg, *args, **kwargs):
+        if DEBUG:
+            print("-"*100)
+            print(msg, *args, **kwargs)
+            clear_line(2)
+        else:
+            super().debug(msg, *args, **kwargs)
+
+def get_plugin_logger(name: str) -> Logger:
     logger = logging.getLogger(f"mkdocs.plugins.{name}")
-    setattr(logger, "info", lambda msg: clear_screen() and getattr(logger, "info")(msg))
-    return PrefixedLogger(name.split(".", 1)[0], logger)
-
+    setattr(logger, "info", lambda msg: getattr(logger, "info")(msg))
+    return Logger(logger)
 
 log = get_plugin_logger(f"{Fore.BLUE}juvix_mkdocs{Style.RESET_ALL}")
 
-
 def clear_screen():
-    print("\033[H\033[J", end="", flush=True)
-
-
-def clear_line():
-    print("\033[A", end="", flush=True)
-    print("\033[K", end="\r", flush=True)
+    if os.getenv("DEBUG", "false").lower() != "true":
+        print("\033[H\033[J", end="", flush=True)
+
+def clear_line(n=1):
+    if os.getenv("DEBUG", "false").lower() != "true":
+        for _ in range(n):
+            print("\033[A", end="", flush=True)
+        print("\033[K", end="\r", flush=True)
diff --git a/mkdocs_juvix/main.py b/mkdocs_juvix/main.py
index 5ca97a5..cf56c33 100644
--- a/mkdocs_juvix/main.py
+++ b/mkdocs_juvix/main.py
@@ -11,7 +11,6 @@
 from urllib.parse import urljoin
 
 import pathspec
-import questionary
 import yaml  # type:ignore
 from bs4 import BeautifulSoup  # type:ignore
 from colorama import Back, Fore, Style  # type: ignore
@@ -21,7 +20,6 @@
 from mkdocs.structure.files import Files
 from mkdocs.structure.pages import Page
 from rich.console import Console  # type: ignore
-from rich.markdown import Markdown  # type: ignore
 from semver import Version
 from tqdm import tqdm as sync_tqdm  # type: ignore
 from tqdm.asyncio import tqdm as async_tqdm  # type: ignore
@@ -267,6 +265,10 @@ def changed_since_last_run(self) -> bool:
             )
             self.reset_processed_flags()
             return True
+        log.debug(
+            f"> Found cached output for {Fore.GREEN}{self}{Style.RESET_ALL}, "
+            f"checking if it has changed"
+        )
         try:
             cached_hash = self.cached_hash
             if cached_hash is None:
@@ -307,13 +309,14 @@ def markdown_output(self) -> Optional[str]:
                     f"> Because it was asked markdown and it was not cached, "
                     f"generating markdown output for {Fore.GREEN}{self}{Style.RESET_ALL}"
                 )
-                self.run_pipeline(save_markdown=True, force=True)
+                self.run_pipeline(save_markdown=True)
                 return self._markdown_output
             except Exception as e:
                 log.error(f"Error generating markdown output: {e}")
                 return None
         return self._markdown_output
 
+    @time_spent(message="> running pipeline per individual file")
     def run_pipeline(self, save_markdown: bool = True, force: bool = False) -> None:
         """
         Run the pipeline of tasks to generate the markdown output of the file.
@@ -338,7 +341,6 @@ def run_pipeline(self, save_markdown: bool = True, force: bool = False) -> None:
             )
             self.render_snippets(save_markdown=save_markdown, force=force)
 
-    @time_spent(message="> saving markdown output")
     def save_markdown_output(self, md_output: str) -> Optional[Path]:
         """
         Cache the input provided as the cached markdown. Update the hash of the
@@ -360,6 +362,7 @@ def save_markdown_output(self, md_output: str) -> Optional[Path]:
 
         return self.cache_filepath.absolute()
 
+    @time_spent(message="> copying original file to cache")
     def copy_original_file_to_cache(self) -> None:
         try:
             self.original_in_cache_filepath.parent.mkdir(parents=True, exist_ok=True)
@@ -367,6 +370,7 @@ def copy_original_file_to_cache(self) -> None:
         except Exception as e:
             log.error(f"Error copying original file to cache: {e}")
 
+    @time_spent(message="> reading original markdown from cache")
     def save_and_read_original_markdown_from_cache(self) -> Optional[str]:
         """
         Read the original markdown from the cache folder.
@@ -475,12 +479,27 @@ def add_errors_to_markdown(
         if all(value is None for value in self._cached_error_messages.values()):
             return content
 
+        title_error = {
+            "juvix_markdown": "Juvix Markdown",
+            "juvix_isabelle": "Isabelle",
+            "juvix_html": "Juvix HTML",
+            "juvix_root": "Juvix Root",
+            "images": "Images Preprocessor",
+            "snippets": "Snippets Preprocessor",
+            "wikilinks": "Wikilinks Preprocessor",
+        }
+
         def format_error_message(kind: str) -> str:
             error_message = self.get_error_message(kind)
             if error_message is None:
                 return ""
+            error_message = (
+                error_message.replace(self.env.DOCS_ABSPATH.as_posix(), "***")
+                .replace(self.env.CACHE_ABSPATH.as_posix(), "*(cache)*")
+            )
+
             formatted_error_message: str = (
-                f"<details class='failure'><summary>{kind} error</summary>\n\n"
+                f"<details class='failure'><summary>{title_error.get(kind, kind)} error</summary>\n\n"
                 f"<pre><code>\n"
                 f"    {textwrap.fill(error_message, width=110)}\n\n"
                 f"</code></pre></details>\n\n"
@@ -933,6 +952,9 @@ def generate_juvix_markdown_output(
             )
             return None
 
+        log.debug(f"> Generating Juvix markdown for {self.relative_filepath}")   
+        if self.changed_since_last_run():
+            log.debug("> File has changed since last run, generating markdown")
         if (
             self._processed_juvix_markdown
             and not force
@@ -942,17 +964,27 @@ def generate_juvix_markdown_output(
                 f"> Skipping markdown generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output"
             )
             return None
-
+        
+        log.debug(
+            f"> Reading cached markdown for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}"
+        )
         markdown_output: str = self.cache_filepath.read_text()
         metadata = parse_front_matter(markdown_output) or {}
 
         preprocess = metadata.get("preprocess", {})
         _output = None
-        if preprocess.get("juvix", True) and (
+        needs_juvix = bool(preprocess.get("juvix", True))
+        if needs_juvix and (
             not self._processed_juvix_markdown or force
         ):
+            log.debug(
+                f"> Processing Juvix markdown for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}"
+            )
             _output = self._run_command_juvix_markdown()
             if _output and save_markdown:
+                log.debug(
+                    f"> Saving processed markdown for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}"
+                )
                 self._processed_juvix_markdown = True
                 self.save_markdown_output(_output)
             else:
@@ -977,27 +1009,39 @@ def generate_isabelle_theories(
                 f"> Skipping isabelle generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output"
             )
             return None
-
         markdown_output: str = self.cache_filepath.read_text()
         metadata = parse_front_matter(markdown_output) or {}
         preprocess = metadata.get("preprocess", {})
-        self._needs_isabelle = preprocess.get("isabelle", False)
-        self._needs_isabelle_at_bottom = preprocess.get(
+        self._needs_isabelle = bool(preprocess.get("isabelle", False))
+        self._needs_isabelle_at_bottom = bool(preprocess.get(
             "isabelle_at_bottom",
             self._needs_isabelle,
+        ))
+        log.debug(
+            f"> Isabelle processing needed: {Fore.GREEN}{self._needs_isabelle}{Style.RESET_ALL}, "
+            f"at bottom: {Fore.GREEN}{self._needs_isabelle_at_bottom}{Style.RESET_ALL}"
         )
         _output = None
         if (self._needs_isabelle or self._needs_isabelle_at_bottom) and (
             not self._processed_juvix_isabelle or force
         ):
+            log.debug(
+                f"> Processing Isabelle translation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}"
+            )
             _output = self.process_isabelle_translation(
                 content=markdown_output,
                 modify_markdown_output=self._needs_isabelle_at_bottom,
             )
             if _output and save_markdown:
+                log.debug(
+                    f"> Saving processed Isabelle output for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}"
+                )
                 self._processed_juvix_isabelle = True
                 self.save_markdown_output(_output)
             else:
+                log.debug(
+                    f"{Fore.RED}> Failed to process Isabelle output for {self.relative_filepath}{Style.RESET_ALL}"
+                )
                 self._processed_juvix_isabelle = False
 
     @time_spent(message="> extracting snippets")
@@ -1006,33 +1050,41 @@ def render_snippets(self, save_markdown: bool = True, force: bool = False) -> No
         Modify the markdown output by adding the snippets found in the file.
         This requires the preprocess of Juvix and Isabelle to be ocurred before.
         """
-        if self._processed_snippets and not force and not self.changed_since_last_run():
+        if self._processed_snippets and (not force and not self.changed_since_last_run()):
             log.debug(
                 f"> Skipping snippets generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output"
             )
             return None
-
         log.debug(
             f"Processing snippets for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}"
         )
 
+        log.debug("> Reading markdown content")
         _markdown_output: str = self.cache_filepath.read_text()
+        log.debug("> Parsing front matter")
         metadata = parse_front_matter(_markdown_output) or {}
+        log.debug("> Getting preprocess metadata")
         preprocess = metadata.get("preprocess", {})
-        needs_snippets = preprocess.get("snippets", True)
+        log.debug("> Checking if snippets are needed")
+        needs_snippets = bool(preprocess.get("snippets", True))
+        check_paths = bool(preprocess.get("check_paths", False))
         _output = None
         if needs_snippets and (not self._processed_snippets or force):
-            _output = self.run_snippet_preprocessor(content=_markdown_output)
+            log.debug("> Running snippet preprocessor")
+            _output = self.run_snippet_preprocessor(content=_markdown_output, check_paths=check_paths)
             if _output and save_markdown:
+                log.debug("> Saving processed snippets")
                 self._processed_snippets = True
                 self.save_markdown_output(_output)
             else:
+                log.debug("> Failed to process snippets")
                 self._processed_snippets = False
 
     @time_spent(message="> processing snippets")
     def run_snippet_preprocessor(
         self,
         content: Optional[str] = None,
+        check_paths: bool = True,
     ) -> str:
         snippet_preprocessor = SnippetPreprocessor()
         snippet_preprocessor.enhanced_mdfile = self
@@ -1040,6 +1092,8 @@ def run_snippet_preprocessor(
             self.cache_filepath.parent.resolve().absolute(),
             self.env.CACHE_PROCESSED_MARKDOWN_PATH.resolve().absolute(),
         ]
+        snippet_preprocessor.check_paths = check_paths
+        
 
         if content:
             try:
@@ -1065,8 +1119,7 @@ def replaces_wikilinks_by_markdown_links(
 
         if (
             self._processed_wikilinks
-            and not force
-            and not self.changed_since_last_run()
+            and (not force or not self.changed_since_last_run())
         ):
             log.debug(
                 f"> Skipping wikilinks generation for {Fore.GREEN}{self}{Style.RESET_ALL} using cached output"
@@ -1076,25 +1129,25 @@ def replaces_wikilinks_by_markdown_links(
         log.debug(f"Processing wikilinks for {Fore.GREEN}{self}{Style.RESET_ALL}")
         _output = None
         _markdown_output = self.cache_filepath.read_text()
-        log.debug(f"Read markdown content from {self.cache_filepath}")
+        log.debug(f"Read markdown content from {Fore.GREEN}{self.cache_filepath}{Style.RESET_ALL}")
         metadata = parse_front_matter(_markdown_output) or {}
-        log.debug(f"Parsed front matter: {metadata}")
+        log.debug(f"Parsed front matter: {Fore.GREEN}{metadata}{Style.RESET_ALL}")
         preprocess = metadata.get("preprocess", {})
-        needs_wikilinks = preprocess.get("wikilinks", True)
-        log.debug(f"Needs wikilinks: {needs_wikilinks}")
+        needs_wikilinks = bool(preprocess.get("wikilinks", True))
+        log.debug(f"Needs wikilinks: {Fore.GREEN}{needs_wikilinks}{Style.RESET_ALL}")
         if needs_wikilinks and (not self._processed_wikilinks or force):
             _output = self.run_wikilinks_preprocessor(
                 content=_markdown_output,
                 modify_markdown_output=True,
             )
-            log.debug(f"Ran wikilinks preprocessor, output: {_output}")
+            log.debug("> Ran wikilinks preprocessor")
             if _output and save_markdown:
                 self._processed_wikilinks = True
                 self.save_markdown_output(_output)
-                log.debug(f"Saved markdown output for {self.relative_filepath}")
+                log.debug(f"Saved markdown output for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}")
             else:
                 self._processed_wikilinks = False
-                log.debug(f"Did not save markdown output for {self.relative_filepath}")
+                log.debug(f"Did not save markdown output for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}")
 
     @time_spent(message="> processing wikilinks")
     def run_wikilinks_preprocessor(
@@ -1102,6 +1155,7 @@ def run_wikilinks_preprocessor(
         content: str,
         modify_markdown_output: bool = True,
     ) -> str:
+        
         assert "url_for" in self.config, "url_for is not in the config"
         assert "nodes" in self.config, "nodes is not in the config"
 
@@ -1136,7 +1190,7 @@ def generate_images(
         _markdown_output = self.cache_filepath.read_text()
         metadata = parse_front_matter(_markdown_output) or {}
         preprocess = metadata.get("preprocess", {})
-        needs_images = preprocess.get("images", True)
+        needs_images = bool(preprocess.get("images", True))
         log.debug(f"Needs images: {needs_images}")
         if needs_images:
             _output = process_images(
@@ -1160,6 +1214,7 @@ class EnhancedMarkdownCollection:
     env: ENV
     config: MkDocsConfig
     force_wikilinks_generation: bool = False
+    cached_hash: Optional[str] = None
 
     # @time_spent(message="> initializing enhanced markdown collection")
     def __init__(self, config, env: ENV, docs: Optional[Path] = None):
@@ -1171,6 +1226,8 @@ def __init__(self, config, env: ENV, docs: Optional[Path] = None):
             # The list of markdown files to be processed, filled later
             self.files: Optional[List[EnhancedMarkdownFile]] = None
 
+            log.debug(f"Initializing EnhancedMarkdownCollection with docs path: {self.docs_path}")
+
             # The "everything" file is a special Juvix Markdown file that contains
             # links to all the Juvix Markdown files in the folder.
             self.everything_html_file: Optional[EnhancedMarkdownFile] = (
@@ -1181,17 +1238,16 @@ def __init__(self, config, env: ENV, docs: Optional[Path] = None):
                 )
             )
             if not self.everything_html_file.absolute_filepath.exists():
+                log.debug("No 'everything.juvix.md' file found")
                 self.everything_html_file = None
 
             # The hash is used to check if the markdown files have changed since the
             # last time the project was built.
-            self.cached_hash: Optional[str] = None
 
         except Exception as e:
             log.error(f"Error initializing JuvixMarkdownCollection: {e}")
             raise
 
-    @time_spent(message="> storing original files in cache for faster lookup")
     def scanning_originals(self) -> List[EnhancedMarkdownFile]:
         """
         Cache the original Juvix Markdown files in the cache folder for faster
@@ -1218,8 +1274,10 @@ def scanning_originals(self) -> List[EnhancedMarkdownFile]:
                     pbar.set_postfix_str(
                         f"{Fore.MAGENTA}{file.relative_to(self.docs_path)}{Style.RESET_ALL}"
                     )
+                    log.debug(f"Creating EnhancedMarkdownFile for {file}")
                     enhanced_file = EnhancedMarkdownFile(file, self.env, self.config)
                     self.files.append(enhanced_file)
+                    log.debug(f"Creating parent directory for {enhanced_file.original_in_cache_filepath}")
                     enhanced_file.original_in_cache_filepath.parent.mkdir(
                         parents=True, exist_ok=True
                     )
@@ -1228,7 +1286,7 @@ def scanning_originals(self) -> List[EnhancedMarkdownFile]:
                     )
                     shutil.copy(file, enhanced_file.original_in_cache_filepath)
                     pbar.update(1)
-
+            clear_line()  
             return self.files
 
         except Exception as e:
@@ -1285,6 +1343,7 @@ def update_cached_hash(self) -> Optional[str]:
             hash = self.hash
             self.env.CACHE_PROJECT_HASH_FILEPATH.write_text(hash)
             self.cached_hash = hash
+            log.debug(f"> Updated cached hash of the entire project: {hash}")
             return hash
         except Exception as e:
             log.error(f"Error updating cached hash of the entire project: {e}")
@@ -1302,12 +1361,15 @@ def save_juvix_modules_json(self) -> Optional[Path]:
         if self.files is None:
             return None
         try:
+            log.debug("> Saving Juvix modules JSON")
             json_path = self.env.CACHE_ABSPATH / "juvix_modules.json"
+            log.debug(f"> Writing Juvix modules JSON to {json_path}")
             json_content = json.dumps(
                 [file.to_dict() for file in self.files if file.absolute_filepath],
                 indent=2,
             )
             json_path.write_text(json_content)
+            log.debug(f"> Saved Juvix modules JSON to {json_path}")
             return json_path.relative_to(self.env.ROOT_ABSPATH)
         except Exception as e:
             log.error(f"Error saving juvix_modules.json: {e}")
@@ -1327,11 +1389,12 @@ def run_pipeline_on_collection(
         the Juvix Markdown files. Then, generate the HTML output for all the
         Juvix Markdown files.
         """
+        log.debug(f"{Fore.GREEN}run_pipeline_on_collection...{Style.RESET_ALL}")
         if self.files is None:
-            log.debug("> no files to process")
+            log.debug(f"{Fore.YELLOW}> no files to process{Style.RESET_ALL}")
             return
-        clear_screen()
-        log.info(
+        
+        log.debug(
             f"> running pipeline on {Fore.GREEN}{len(self.files)}{Style.RESET_ALL} files"
         )
 
@@ -1345,7 +1408,7 @@ def run_pipeline_on_collection(
             log.debug(f"{Fore.YELLOW}no files to process{Style.RESET_ALL}")
             return
 
-        log.info(
+        log.debug(
             f"> {Fore.GREEN}{len(files_to_process)}{Style.RESET_ALL} file"
             f"{'s' if len(files_to_process) > 1 else ''} need to be processed "
             "because it's the first time, their cached version is not up to date, "
@@ -1361,9 +1424,7 @@ async def process_original_markdowns():
                 ],
                 desc="> collecting original markdowns for caching",
             )
-
         asyncio.run(process_original_markdowns())
-        clear_line()
 
         juvix_files = [
             file
@@ -1382,7 +1443,6 @@ async def process_original_markdowns():
                     )
                     file.generate_juvix_markdown_output()
                     pbar.update(1)
-            clear_line()
 
         if generate_juvix_isabelle and self.env.juvix_enabled:
             with sync_tqdm(
@@ -1395,9 +1455,7 @@ async def process_original_markdowns():
                     )
                     file.generate_isabelle_theories()
                     pbar.update(1)
-            clear_line()
 
-        clear_line()
         if generate_images:
             with sync_tqdm(
                 total=len(files_to_process), desc="> processing images"
@@ -1410,29 +1468,24 @@ async def process_original_markdowns():
                     )
                     pbar.update(1)
 
-        if generate_wikilinks:
-
-            @time_spent(message="> processing wikilinks")
-            async def process_wikilinks():
-                flist = (
-                    files_to_process
-                    if not self.force_wikilinks_generation
-                    else self.files
-                )
-                await async_tqdm.gather(
-                    *[
-                        asyncio.to_thread(file.replaces_wikilinks_by_markdown_links)
-                        for file in flist
-                    ],
-                    total=len(flist),
-                    desc="> processing wikilinks",
-                )
-
-            asyncio.run(process_wikilinks())
+        @time_spent(message="> processing wikilinks")
+        def process_wikilinks():
+            flist = self.files
+            with sync_tqdm(
+                total=len(flist), desc="> processing wikilinks"
+            ) as pbar:
+                for file in flist:
+                    pbar.set_postfix_str(
+                        f"{Fore.MAGENTA}{file.relative_filepath}{Style.RESET_ALL}"
+                    )
+                    file.replaces_wikilinks_by_markdown_links()
+                    pbar.update(1)
             clear_line()
 
-        if generate_snippets:
+        process_wikilinks()
+        log.debug(f"{Fore.GREEN}finished wikilinks{Style.RESET_ALL}")
 
+        if generate_snippets:
             async def process_snippets():
                 await async_tqdm.gather(
                     *[asyncio.to_thread(file.render_snippets) for file in self.files],
@@ -1441,10 +1494,12 @@ async def process_snippets():
                 )
 
             asyncio.run(process_snippets())
+            log.debug(f"{Fore.GREEN}finished snippets{Style.RESET_ALL}")
             clear_line()
 
         self.update_cached_hash()
         self.save_juvix_modules_json()
+        log.debug(f"{Fore.GREEN}finished pipeline{Style.RESET_ALL}")
 
     @time_spent(message="> removing html cache")
     def remove_html_cache(self) -> None:
@@ -1564,9 +1619,10 @@ class JuvixPlugin(BasePlugin):
     first_run: bool = True
 
     def on_startup(self, *, command: str, dirty: bool) -> None:
-        clear_screen()
+        log.debug(f"{Fore.GREEN}on_startup...{Style.RESET_ALL}")
 
     def on_config(self, config: MkDocsConfig) -> MkDocsConfig:
+        clear_screen()
         self.env = ENV(config)
         config = fix_site_url(config)
 
@@ -1599,7 +1655,6 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig:
             if self.env.UPDATE_DEPS:
                 self.enhanced_collection.update_juvix_dependencies()
             self.first_run = False
-
         return config
 
     def on_pre_build(self, config: MkDocsConfig) -> None:
@@ -1608,6 +1663,8 @@ def on_pre_build(self, config: MkDocsConfig) -> None:
         typechecking as part of the markdown processing, we include the error
         message as part of the content of the page
         """
+        log.debug(f"{Fore.GREEN}on_pre_build...{Style.RESET_ALL}")
+        # if self.first_run or self.enhanced_collection.force_wikilinks_generation:
         self.wikilinks_plugin.on_pre_build(config)  # This needs to be run first
         self.enhanced_collection.run_pipeline_on_collection()
 
@@ -1616,6 +1673,7 @@ def on_files(self, files: Files, *, config: MkDocsConfig) -> Optional[Files]:
         List of the files to be included in the final build. These are copied to
         the site directory.
         """
+        log.debug(f"{Fore.GREEN}on_files...{Style.RESET_ALL}")
         self.wikilinks_plugin.on_files(files, config)
         Files(
             [
@@ -1630,6 +1688,7 @@ def on_files(self, files: Files, *, config: MkDocsConfig) -> Optional[Files]:
         mkdocs_config_filepath = self.env.ROOT_ABSPATH / "mkdocs.yml"
         if mkdocs_config_filepath in [file.abs_src_path for file in files]:
             self.enhanced_collection.force_wikilinks_generation = True
+        log.debug(f"{Fore.GREEN}finished on_files...{Style.RESET_ALL}")
         return files
 
     def on_nav(self, nav, config: MkDocsConfig, files: Files):
@@ -1639,18 +1698,28 @@ def on_pre_page(self, page: Page, config: MkDocsConfig, files: Files) -> Page:
         return page
 
     def on_page_read_source(self, page: Page, config: MkDocsConfig) -> Optional[str]:
+        log.debug(f"{Fore.GREEN}Reading source for page...{Style.RESET_ALL}")
         abs_src_str: Optional[str] = page.file.abs_src_path
         if not abs_src_str:
+            log.debug(f"{Fore.YELLOW}No source path found for page{Style.RESET_ALL}")
             return None
 
         abs_src_path: Path = Path(abs_src_str)
+        log.debug(f"{Fore.CYAN}Processing file: {abs_src_path}{Style.RESET_ALL}")
         try:
             file: Optional[EnhancedMarkdownFile] = (
                 self.enhanced_collection.get_enhanced_file_entry(abs_src_path)
             )
+        except Exception as e:
+            log.error(f"Error getting file from collection: {e}")
+        try:
             if file:
+                log.debug(f"{Fore.GREEN}Found file in collection, reading...{Style.RESET_ALL}")
                 output = file.cache_filepath.read_text()
-                return file.add_errors_to_markdown(output)
+                log.debug(f"{Fore.GREEN}Adding errors to markdown...{Style.RESET_ALL}")
+                with_errors = file.add_errors_to_markdown(output)
+                log.debug(f"{Fore.GREEN}Finished adding errors to markdown{Style.RESET_ALL}")
+                return with_errors
             else:
                 log.error(
                     f"{Fore.RED}File not found in collection: "
@@ -1659,6 +1728,7 @@ def on_page_read_source(self, page: Page, config: MkDocsConfig) -> Optional[str]
                 )
         except Exception as e:
             log.error(f"Error getting file from collection: {e}")
+        log.debug(f"{Fore.GREEN}Finished reading source for page{Style.RESET_ALL}")
         return None
 
     def on_page_markdown(
@@ -1686,20 +1756,38 @@ def on_page_markdown(
     def on_page_content(
         self, html: str, page: Page, config: MkDocsConfig, files: Files
     ) -> Optional[str]:
-        return self.wikilinks_plugin.on_page_content(html, page, config, files)
+        # html = self.wikilinks_plugin.on_page_content(html, page, config, files)
+        return html
 
     def on_post_page(self, output: str, page: Page, config: MkDocsConfig) -> str:
         soup = BeautifulSoup(output, "html.parser")
         for a in soup.find_all("a"):
             a["href"] = a["href"].replace(".juvix.html", ".html")
         return str(soup)
+    
+    def get_context(self, context, page, config, nav):
+        log.info(f"{Fore.GREEN}Processing context...{Style.RESET_ALL}")
+        return context
+    
+    def get_template(self, template, context):
+        log.info(f"{Fore.GREEN}Processing template...{Style.RESET_ALL}")
+        return template, context
+    
+    def render(self, template, context):
+        log.info(f"{Fore.GREEN}Rendering template...{Style.RESET_ALL}")
+        return template, context
+    
 
     def on_post_build(self, config: MkDocsConfig) -> None:
-        log.debug("> post build task: generating HTML for files")
+        log.info(f"{Fore.GREEN}on_post_build...{Style.RESET_ALL}")
         if self.env.JUVIX_ENABLED:
+            log.debug(f"{Fore.GREEN}generating HTML...{Style.RESET_ALL}")
             self.enhanced_collection.generate_html()
+            log.debug(f"{Fore.GREEN}moving HTML cache to site directory...{Style.RESET_ALL}")
             self.move_html_cache_to_site_dir()
+            log.debug(f"{Fore.GREEN}updating cached hash...{Style.RESET_ALL}")
             self.enhanced_collection.update_cached_hash()
+            log.debug(f"{Fore.GREEN}saving Juvix modules JSON...{Style.RESET_ALL}")
             self.enhanced_collection.save_juvix_modules_json()
 
         files_to_check = (
@@ -1707,6 +1795,18 @@ def on_post_build(self, config: MkDocsConfig) -> None:
         )
         for file in files_to_check:
             file.load_and_print_saved_error_messages()
+        
+        files = self.enhanced_collection.files if self.enhanced_collection.files else []
+        files_to_process: List[EnhancedMarkdownFile] = [
+            file
+            for file in files
+            if files and file.has_error_message()
+        ]
+        log.info(f"{Fore.GREEN}Files with errors: {len(files_to_process)}{Style.RESET_ALL}")
+        log.info("Next time, we will process the following files:")
+        for file in files_to_process:
+            log.info(f"{Fore.GREEN}{file.relative_filepath}{Style.RESET_ALL}")
+        log.debug(f"{Fore.GREEN}finished on_post_build...{Style.RESET_ALL}")
 
     def move_html_cache_to_site_dir(self) -> None:
         """
diff --git a/mkdocs_juvix/snippets.py b/mkdocs_juvix/snippets.py
index baa0b01..4f4c2e6 100644
--- a/mkdocs_juvix/snippets.py
+++ b/mkdocs_juvix/snippets.py
@@ -330,6 +330,7 @@ def _get_snippet_path(self, base_paths: List[Path], path: Path):
     def get_snippet_path(self, path: Path | str):
         """Get snippet path."""
         log.debug(f"{Fore.CYAN}> getting snippet path for {path}{Style.RESET_ALL}")
+
         if isinstance(path, str):
             path = Path(path)
         base_paths = self.base_path
@@ -367,6 +368,10 @@ def get_snippet_path(self, path: Path | str):
         if path.is_relative_to("./docs"):
             log.debug(f"Path is relative to ./docs: {path}")
             path = path.relative_to("./docs")
+            
+        if self.env.DOCS_ABSPATH not in base_paths:
+            base_paths.append(self.env.DOCS_ABSPATH)
+            base_paths.append(self.env.ROOT_ABSPATH)
 
         if path.is_relative_to(self.env.ISABELLE_OUTPUT_PATH):
             log.debug(f"Path is relative to Isabelle output path: {path}")
@@ -495,7 +500,7 @@ def parse_snippets(
 
                 if found_snippet is None:
                     if self.check_paths:
-                        msg = f"Error type 3. Snippet at path '{Fore.MAGENTA}{path}{Style.RESET_ALL}' could not be found"
+                        msg = f"Wrong snippet path: '{Fore.MAGENTA}{path}{Style.RESET_ALL}' could not be found. {Fore.YELLOW}Check the path, perhaps you forgot e.g., adding the prefix './' to the path or ./docs/{Style.RESET_ALL}"
                         return SnippetMissingError(msg)
 
                 log.debug(f"{Fore.GREEN}Snippet found:{found_snippet}{Style.RESET_ALL}")
@@ -568,7 +573,7 @@ def parse_snippets(
                         except SnippetMissingError:
                             if self.check_paths:
                                 return SnippetMissingError(
-                                    f"Error type 2 while processing {Fore.MAGENTA}{file_name}{Style.RESET_ALL} when trying to extract snippet: {snippet}"
+                                    f"Error while processing {Fore.MAGENTA}{file_name}{Style.RESET_ALL} when trying to extract snippet: {snippet}"
                                 )
                             s_lines = []
 
diff --git a/mkdocs_juvix/utils.py b/mkdocs_juvix/utils.py
index ba10ec2..4c30cb7 100644
--- a/mkdocs_juvix/utils.py
+++ b/mkdocs_juvix/utils.py
@@ -9,7 +9,7 @@
 from colorama import Fore, Style  # type: ignore
 from mkdocs.config.defaults import MkDocsConfig
 
-from mkdocs_juvix.logger import PrefixedLogger, log
+from mkdocs_juvix.logger import log
 
 EXCLUDED_DIRS = {
     ".git",
@@ -155,7 +155,7 @@ def get_filepath_for_cached_hash_for(
 
 
 def time_spent(
-    log: PrefixedLogger,
+    log: Any,
     message: Optional[Any] = None,
     print_result: bool = False,
 ):
@@ -167,8 +167,7 @@ def wrapper(*args, **kwargs):
             exception = None
             if message:
                 log_message = f"{Fore.YELLOW}{message} ...{Style.RESET_ALL}"
-                if os.getenv("DEBUG", False):
-                    log.info(log_message)
+                log.debug(log_message)
             try:
                 result = func(*args, **kwargs)
             except Exception as e:
@@ -183,8 +182,7 @@ def wrapper(*args, **kwargs):
                 log_message = f"`{func.__name__} =>` {result} - {log_message}"
             if not print_result:
                 log_message = f"`{func.__name__}` - {log_message}"
-            if os.getenv("DEBUG", False):
-                log.info(log_message)
+                log.debug(log_message)
             if exception:
                 raise exception
             return result
diff --git a/pyproject.toml b/pyproject.toml
index 6b5d4a3..5bcaa92 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mkdocs-juvix-plugin"
-version = "0.4.5"
+version = "0.4.6"
 description = "MkDocs documentation with support for Juvix Markdown files"
 authors = ["Jonathan Prieto-Cubides, and GitHub contributors"]
 license = "MIT"