Template improvements

CHANGES.txt

@@ -5,6 +5,9 @@ Features
 --------
 
 * Support passing ``--poll`` to ``nikola auto`` to better deal with symlink farms.
+* Trace template usage when an environment variable ``NIKOLA_TEMPLATES_TRACE``
+  is set to any non-empty value.
+* Allow configuration of Jinja2 extensions through the theme configuration.
 
 Bugfixes
 --------

docs/theming.rst

@@ -7,7 +7,7 @@
 .. author: The Nikola Team
 
 :Version: 8.3.0
-:Author: Roberto Alsina <[email protected]>
+:Author: Roberto Alsina <[email protected]> and others
 
 .. class:: alert alert-primary float-md-right
 
@@ -104,8 +104,8 @@ with the same name as your theme, and a ``.theme`` extension, eg.
 .. code:: ini
 
    [Theme]
-   engine = mako
-   parent = base
+   engine = jinja
+   parent = base-jinja
    author = The Nikola Contributors
    author_url = https://getnikola.com/
    based_on = Bootstrap 3 <https://getbootstrap.com/>
@@ -120,6 +120,10 @@ with the same name as your theme, and a ``.theme`` extension, eg.
    [Nikola]
    bootswatch = True
 
+   [jinja]
+   # Good for investigation, but not recommended to leave active in production:
+   extensions = jinja2.ext.debug
+
 The following keys are currently supported:
 
 * ``Theme`` — contains information about the theme.
@@ -130,8 +134,9 @@ The following keys are currently supported:
 
     The parent is so you don’t have to create a full theme each time: just
     create an empty theme, set the parent, and add the bits you want modified.
-    You **must** define a parent, otherwise many features won’t work due to
-    missing templates, messages, and assets.
+    It is strongly recommended you define a parent.  If you don't, many features
+    won’t work due to missing templates, messages, and assets until your home-grown
+    template is complete.
 
     The following settings are recommended:
 
@@ -163,6 +168,13 @@ The following keys are currently supported:
   * ``ignored_assets`` — comma-separated list of assets to ignore (relative to
     the ``assets/`` directory, eg. ``css/theme.css``)
 
+* ``jinja`` - This section is ignored unless your theme's engine is ``jinja``.
+
+  * ``extensions`` - comma-separated list of
+    `jinja2-extensions <https://jinja.palletsprojects.com/en/3.1.x/extensions/>`_
+    that you want to be available when rendering your templates.
+
+
 Templates
 ---------
 
@@ -184,8 +196,16 @@ so ``post.tmpl`` only define the content, and the layout is inherited from ``bas
 
 Another concept is theme inheritance. You do not need to duplicate all the
 default templates in your theme — you can just override the ones you want
-changed, and the rest will come from the parent theme. (Every theme needs a
-parent.)
+changed, and the rest will come from the parent theme.  If your theme does not
+define a parent, it needs to be complete.  It is generally a lot harder to
+come up with a complete theme, compared to only changing a few files and using
+the rest from a suitable parent theme.
+
+.. Tip::
+
+   If you set the environment variable ``NIKOLA_TEMPLATES_TRACE`` to any non-empty value
+   (``true`` is recommended), Nikola will log template usage, both on output and also
+   into a file ``templates_log.txt``.
 
 Apart from the `built-in templates`_ listed below, you can add other templates for specific
 pages, which the user can then use in his ``POSTS`` or ``PAGES`` option in
@@ -194,11 +214,11 @@ page via the ``template`` metadata, and custom templates can be added in the
 ``templates/`` folder of your site.
 
 If you want to modify (override) a built-in template, use ``nikola theme -c
-<name>.tmpl``.  This command will copy the specified template file to the
-``templates/`` directory of your currently used theme.
+<name>.tmpl``.  This command will copy the specified template file from the
+parent theme to the ``templates/`` directory of your currently used theme.
 
 Keep in mind that your theme is *yours*, so you can require whatever data you
-want (eg. you may depend on specific custom ``GLOBAL_CONTEXT`` variables, or
+want (e.g., you may depend on specific custom ``GLOBAL_CONTEXT`` variables, or
 post meta attributes). You don’t need to keep the same theme structure as the
 default themes do (although many of those names are hardcoded). Inheriting from
 at least ``base`` (or ``base-jinja``) is heavily recommended, but not strictly

nikola/__init__.py

@@ -29,8 +29,15 @@
 import os
 import sys
 
+# The current Nikola version:
 __version__ = '8.3.0'
+# A flag whether logging should emmit debug information:
 DEBUG = bool(os.getenv('NIKOLA_DEBUG'))
+# A flag whether special templates trace logging should be generated:
+TEMPLATES_TRACE = bool(os.getenv('NIKOLA_TEMPLATES_TRACE'))
+# When this flag is set, fewer exceptions are handled internally;
+# instead they are left unhandled for the run time system to deal with them,
+# which typically leads to the stack traces being exposed.
 SHOW_TRACEBACKS = bool(os.getenv('NIKOLA_SHOW_TRACEBACKS'))
 
 if sys.version_info[0] == 2:

nikola/log.py

@@ -30,7 +30,7 @@
 import logging
 import warnings
 
-from nikola import DEBUG
+from nikola import DEBUG, TEMPLATES_TRACE
 
 __all__ = (
     "get_logger",
@@ -86,6 +86,10 @@ class LoggingMode(enum.Enum):
     QUIET = 2
 
 
+_LOGGING_FMT = "[%(asctime)s] %(levelname)s: %(name)s: %(message)s"
+_LOGGING_DATEFMT = "%Y-%m-%d %H:%M:%S"
+
+
 def configure_logging(logging_mode: LoggingMode = LoggingMode.NORMAL) -> None:
     """Configure logging for Nikola.
 
@@ -101,12 +105,7 @@ def configure_logging(logging_mode: LoggingMode = LoggingMode.NORMAL) -> None:
         return
 
     handler = logging.StreamHandler()
-    handler.setFormatter(
-        ColorfulFormatter(
-            fmt="[%(asctime)s] %(levelname)s: %(name)s: %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-    )
+    handler.setFormatter(ColorfulFormatter(fmt=_LOGGING_FMT, datefmt=_LOGGING_DATEFMT))
 
     handlers = [handler]
     if logging_mode == LoggingMode.STRICT:
@@ -137,6 +136,39 @@ def get_logger(name: str, handlers=None) -> logging.Logger:
 
 
 LOGGER = get_logger("Nikola")
+TEMPLATES_LOGGER = get_logger("nikola.templates")
+
+
+def init_template_trace_logging(filename: str) -> None:
+    """Initialize the tracing of the template system.
+
+    This tells a theme designer which templates are being exercised
+    and for which output files, and, if applicable, input files.
+
+    As there is lots of other stuff happening on the normal output stream,
+    this info is also written to a log file.
+    """
+    TEMPLATES_LOGGER.level = logging.DEBUG
+    formatter = logging.Formatter(
+        fmt=_LOGGING_FMT,
+        datefmt=_LOGGING_DATEFMT,
+    )
+    shandler = logging.StreamHandler()
+    shandler.setFormatter(formatter)
+    shandler.setLevel(logging.DEBUG)
+
+    fhandler = logging.FileHandler(filename, encoding="UTF-8")
+    fhandler.setFormatter(formatter)
+    fhandler.setLevel(logging.DEBUG)
+
+    TEMPLATES_LOGGER.handlers = [shandler, fhandler]
+    TEMPLATES_LOGGER.propagate = False
+
+    TEMPLATES_LOGGER.info("Template usage being traced to file %s", filename)
+
+
+if TEMPLATES_TRACE:
+    init_template_trace_logging("templates_log.txt")
 
 
 # Push warnings to logging

nikola/nikola.py

@@ -36,6 +36,7 @@
 import pathlib
 import sys
 import typing
+from typing import Any, Dict, Iterable, List, Optional, Set
 import mimetypes
 from collections import defaultdict
 from copy import copy
@@ -373,7 +374,7 @@ class Nikola(object):
     plugin_manager: PluginManager
     _template_system: TemplateSystem
 
-    def __init__(self, **config):
+    def __init__(self, **config) -> None:
         """Initialize proper environment for running tasks."""
         # Register our own path handlers
         self.path_handlers = {
@@ -395,7 +396,7 @@ def __init__(self, **config):
         self.timeline = []
         self.pages = []
         self._scanned = False
-        self._template_system = None
+        self._template_system: Optional[TemplateSystem] = None
         self._THEMES = None
         self._MESSAGES = None
         self.filters = {}
@@ -996,13 +997,13 @@ def __init__(self, **config):
         # WebP files have no official MIME type yet, but we need to recognize them (Issue #3671)
         mimetypes.add_type('image/webp', '.webp')
 
-    def _filter_duplicate_plugins(self, plugin_list: typing.Iterable[PluginCandidate]):
+    def _filter_duplicate_plugins(self, plugin_list: Iterable[PluginCandidate]):
         """Find repeated plugins and discard the less local copy."""
         def plugin_position_in_places(plugin: PluginInfo):
             # plugin here is a tuple:
             # (path to the .plugin file, path to plugin module w/o .py, plugin metadata)
+            place: pathlib.Path
             for i, place in enumerate(self._plugin_places):
-                place: pathlib.Path
                 try:
                     # Path.is_relative_to backport
                     plugin.source_dir.relative_to(place)
@@ -1025,7 +1026,7 @@ def plugin_position_in_places(plugin: PluginInfo):
             result.append(plugins[-1])
         return result
 
-    def init_plugins(self, commands_only=False, load_all=False):
+    def init_plugins(self, commands_only=False, load_all=False) -> None:
         """Load plugins as needed."""
         extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
         self._loading_commands_only = commands_only
@@ -1086,9 +1087,9 @@ def init_plugins(self, commands_only=False, load_all=False):
         # Search for compiler plugins which we disabled but shouldn't have
         self._activate_plugins_of_category("PostScanner")
         if not load_all:
-            file_extensions = set()
+            file_extensions: Set[str] = set()
+            post_scanner: PostScanner
             for post_scanner in [p.plugin_object for p in self.plugin_manager.get_plugins_of_category('PostScanner')]:
-                post_scanner: PostScanner
                 exts = post_scanner.supported_extensions()
                 if exts is not None:
                     file_extensions.update(exts)
@@ -1126,8 +1127,8 @@ def init_plugins(self, commands_only=False, load_all=False):
 
         self._activate_plugins_of_category("Taxonomy")
         self.taxonomy_plugins = {}
+        taxonomy: Taxonomy
         for taxonomy in [p.plugin_object for p in self.plugin_manager.get_plugins_of_category('Taxonomy')]:
-            taxonomy: Taxonomy
             if not taxonomy.is_enabled():
                 continue
             if taxonomy.classification_name in self.taxonomy_plugins:
@@ -1322,7 +1323,7 @@ def _activate_plugin(self, plugin_info: PluginInfo) -> None:
             if candidate.exists() and candidate.is_dir():
                 self.template_system.inject_directory(str(candidate))
 
-    def _activate_plugins_of_category(self, category) -> typing.List[PluginInfo]:
+    def _activate_plugins_of_category(self, category) -> List[PluginInfo]:
         """Activate all the plugins of a given category and return them."""
         # this code duplicated in tests/base.py
         plugins = []
@@ -1390,13 +1391,15 @@ def _get_global_context(self):
     def _get_template_system(self):
         if self._template_system is None:
             # Load template plugin
-            template_sys_name = utils.get_template_engine(self.THEMES)
+            template_sys_name, template_sys_user_config = utils.get_template_engine(self.THEMES)
             pi = self.plugin_manager.get_plugin_by_name(template_sys_name, "TemplateSystem")
             if pi is None:
                 sys.stderr.write("Error loading {0} template system "
                                  "plugin\n".format(template_sys_name))
                 sys.exit(1)
             self._template_system = typing.cast(TemplateSystem, pi.plugin_object)
+            if template_sys_user_config is not None:
+                self._template_system.user_configuration(template_sys_user_config)
             lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates")
                                            for name in self.THEMES]
             self._template_system.set_directories(lookup_dirs,
@@ -1444,7 +1447,7 @@ def get_compiler(self, source_name):
 
         return compiler
 
-    def render_template(self, template_name, output_name, context, url_type=None, is_fragment=False):
+    def render_template(self, template_name: str, output_name: str, context, url_type=None, is_fragment=False):
         """Render a template with the global context.
 
         If ``output_name`` is None, will return a string and all URL
@@ -1459,7 +1462,11 @@ def render_template(self, template_name, output_name, context, url_type=None, is
         If ``is_fragment`` is set to ``True``, a HTML fragment will
         be rendered and not a whole HTML document.
         """
-        local_context = {}
+        if "post" in context and context["post"] is not None:
+            utils.TEMPLATES_LOGGER.debug("For %s, template %s builds %s", context["post"].source_path, template_name, output_name)
+        else:
+            utils.TEMPLATES_LOGGER.debug("Template %s builds %s", template_name, output_name)
+        local_context: Dict[str, Any] = {}
         local_context["template_name"] = template_name
         local_context.update(self.GLOBAL_CONTEXT)
         local_context.update(context)
@@ -1695,7 +1702,7 @@ def _register_templated_shortcodes(self):
 
         builtin_sc_dir = utils.pkg_resources_path(
             'nikola',
-            os.path.join('data', 'shortcodes', utils.get_template_engine(self.THEMES)))
+            os.path.join('data', 'shortcodes', utils.get_template_engine(self.THEMES)[0]))
 
         for sc_dir in [builtin_sc_dir, 'shortcodes']:
             if not os.path.isdir(sc_dir):