Merge branch 'frontmatter' into v0.9.0

Author: mDuo13

dactyl/common.py

@@ -69,3 +69,28 @@ def guess_title_from_md_file(filepath):
     #basically if the first line's not a markdown header, we give up and use
     # the filename instead
     return os.path.basename(filepath)
+
+def parse_frontmatter(text):
+    """Separate YAML frontmatter, if any, from a string, and return the
+    text separate from the parsed front-matter."""
+    if len(text) < 6:
+        logger.debug("...too short for frontmatter")
+        return text, {}
+
+    if text[:3] == "---" and text.find("---", 3) != -1:
+        logger.debug("...has front matter")
+        raw_frontmatter = text[3:text.find("---", 3)]
+        frontmatter = yaml.load(raw_frontmatter)
+        # Map some Jekyll-specific frontmatter variables to their Dactyl equivs
+        if "title" in frontmatter.keys():
+            # We don't care about the Jekyll "page.name" field so it's OK to
+            #   overwrite it.
+            frontmatter["name"] = frontmatter["title"]
+        if "categories" in frontmatter.keys() and len(frontmatter["categories"]):
+            frontmatter["category"] = frontmatter["categories"][0]
+        print("Loaded frontmatter:", frontmatter)#TODO: remove me
+
+        return text[text.find("---", 3)+4:], frontmatter
+    else:
+        logger.debug("...no front matter detected")
+        return text, {}

dactyl/dactyl_build.py

@@ -35,6 +35,7 @@
 from dactyl.config import DactylConfig
 from dactyl.cli import DactylCLIParser
 from dactyl.openapi import ApiDef
+from dactyl.jinja_loaders import FrontMatterRemoteLoader, FrontMatterFSLoader
 
 # These fields are special, and pages don't inherit them directly
 RESERVED_KEYS_TARGET = [
@@ -360,9 +361,11 @@ def get_categories(pages):
     logger.debug("categories: %s" % categories)
     return categories
 
+
 def preprocess_markdown(page, target=None, categories=[], page_filters=[],
                         mode="html", current_time="TIME_UNKNOWN",
-                        bypass_errors=False, skip_preprocessor="NOT SPECIFIED"):
+                        bypass_errors=False, skip_preprocessor="NOT SPECIFIED",
+                        read_frontmatter=True):
     """Read a markdown file, local or remote, and preprocess it, returning the
     preprocessed text."""
     target=get_target(target)
@@ -384,6 +387,15 @@ def preprocess_markdown(page, target=None, categories=[], page_filters=[],
             with open(page["md"], "r", encoding="utf-8") as f:
                 md = f.read()
 
+        if read_frontmatter:
+            try:
+                md, frontmatter = parse_frontmatter(md)
+                merge_dicts(frontmatter, page)
+            except Exception as e:
+                traceback.print_tb(e.__traceback__)
+                recoverable_error("Error reading frontmatter for page %s: %s" %
+                     (page, repr(e)), bypass_errors)
+
     else:
         if config["preprocessor_allow_undefined"] == False and not bypass_errors:
             strict_undefined = True
@@ -394,6 +406,13 @@ def preprocess_markdown(page, target=None, categories=[], page_filters=[],
             md_raw = pp_env.get_template("_")
         else:
             md_raw = pp_env.get_template(page["md"])
+
+        if "fm_map" in dir(pp_env.loader): #TODO: this is a hack
+            fm_vars = pp_env.loader.fm_map.get(page["md"], {})
+        else:
+            fm_vars = {}
+        merge_dicts(fm_vars, page)
+
         md = md_raw.render(
             currentpage=page,
             categories=categories,
@@ -404,7 +423,6 @@ def preprocess_markdown(page, target=None, categories=[], page_filters=[],
             config=config
         )
 
-
     # Apply markdown-based filters here
     for filter_name in page_filters:
         if "filter_markdown" in dir(config.filters[filter_name]):
@@ -525,7 +543,7 @@ def setup_pp_env(page=None, page_filters=[], no_loader=False, strict_undefined=F
     if how == HOW_FROM_URL:
         logger.debug("Using remote template loader for page %s" % page)
         pp_env = jinja2.Environment(undefined=preferred_undefined,
-                loader=jinja2.FunctionLoader(read_markdown_remote))
+                loader=FrontMatterRemoteLoader())
     elif how == HOW_FROM_GENERATOR:
         logger.debug("Using a generator-loader for page %s" % page)
         pp_env = jinja2.Environment(undefined=preferred_undefined,
@@ -536,7 +554,7 @@ def setup_pp_env(page=None, page_filters=[], no_loader=False, strict_undefined=F
     else:
         logger.debug("Using FileSystemLoader for page %s" % page)
         pp_env = jinja2.Environment(undefined=preferred_undefined,
-                loader=jinja2.FileSystemLoader(path))
+                loader=FrontMatterFSLoader(path))
 
     # Add custom "defined_and_" tests
     def defined_and_equalto(a,b):

dactyl/jinja_loaders.py

@@ -0,0 +1,52 @@
+import requests
+from jinja2 import BaseLoader, FileSystemLoader, TemplateNotFound, Template
+from urllib.parse import urlparse
+from dactyl.common import *
+
+class FrontMatterRemoteLoader(BaseLoader):
+    def __init__(self):
+        self.baseurl = None
+        self.fm_map = {} # Save frontmatter so it can be attached to the template
+
+    def get_source(self, environment, template):
+        """Fetch a remote markdown file and return its contents"""
+        parsed_url = urlparse(template)
+
+        # save this base URL if it's new; that way we can try to let remote
+        # templates inherit from other templates at related URLs
+        if parsed_url.scheme or (self.baseurl is None):
+            base_path = os.path.dirname(parsed_url.path)
+            self.baseurl = (
+                parsed_url[0],
+                parsed_url[1],
+                base_path,
+                parsed_url[3],
+                parsed_url[4],
+                parsed_url[5],
+            )
+        # if we're at read_markdown_remote without a scheme, it's probably a remote
+        # template trying to import another template, so let's assume it's from the
+        # base path of the imported template
+        if not parsed_url.scheme:
+            url = os.path.join(self.baseurl, template)
+        else:
+            url = template
+
+        response = requests.get(url)
+        if response.status_code == 200:
+            text, frontmatter = parse_frontmatter(response.text)
+            self.fm_map[template] = frontmatter
+            return text, url, None
+        else:
+            raise TemplateNotFound(template)
+
+class FrontMatterFSLoader(FileSystemLoader):
+    def __init__(self, searchpath, encoding='utf-8', followlinks=False):
+        super().__init__(searchpath, encoding, followlinks)
+        self.fm_map = {}
+
+    def get_source(self, environment, template):
+        text, filename, uptodate = super().get_source(environment, template)
+        text, frontmatter = parse_frontmatter(text)
+        self.fm_map[template] = frontmatter
+        return text, filename, uptodate

dactyl/version.py

@@ -1 +1 @@
-__version__ = '0.9.0ascanius94'
+__version__ = '0.9.0b2'

examples/content/with-frontmatter.md

@@ -0,0 +1,18 @@
+---
+desc: This file has Jekyll-style frontmatter
+categories: ["Tests", "Dactyl ignores categories beyond the first"]
+title: Page With Frontmatter
+---
+# Page That Has Frontmatter
+
+This page demonstrates a file that contains frontmatter.
+
+The frontmatter can be referenced by the preprocessor. For example:
+
+```
+Description: {{"{{"}}currentpage.desc{{"}}"}}
+```
+
+Results:
+
+> Description: {{currentpage.desc}}

examples/dactyl-config.yml

@@ -34,6 +34,12 @@ pages:
         targets:
             - everything
 
+    -   md: with-frontmatter.md
+        name: placeholder_with_frontmatter_local
+        html: with-frontmatter.html
+        targets:
+            - everything
+
     -   name: Conditionals Test
         category: Tests
         md: conditionals.md