From c46457f2346cbc7631aeeff684cd4e5edb06938a Mon Sep 17 00:00:00 2001
From: Omikhleia <didier.willis@gmail.com>
Date: Sun, 20 Aug 2023 15:24:29 +0200
Subject: [PATCH 1/7] fix: Horizontal rules are too thin for print and
 inconsistent

A fullrule has a default thickness (0.2pt IIRC), but the other rules
have no height, leading to a very thin representation. Enforce 0.4pt
everywhere.
---
 packages/markdown/commands.lua | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/markdown/commands.lua b/packages/markdown/commands.lua
index 0c1a326..7b8f813 100644
--- a/packages/markdown/commands.lua
+++ b/packages/markdown/commands.lua
@@ -242,18 +242,18 @@ function package:registerCommands ()
     elseif hasClass(options, "bigrule") then
       SILE.call("center", {}, function ()
         SILE.call("raise", { height = "0.5ex" }, function ()
-          SILE.call("hrule", { width = "33%lw" })
+          SILE.call("hrule", { width = "33%lw", height = "0.4pt" })
         end)
       end)
     elseif hasClass(options, "fullrule") and self:hasCouyards() then
-      SILE.call("fullrule")
+      SILE.call("fullrule", { thickness = "0.4pt" })
     elseif hasClass(options, "pendant") and self:hasCouyards() then
       SILE.call("smallskip")
       SILE.call("couyard", { type = 6, width = "default" })
     elseif not hasClass(options, "none") then
       SILE.call("center", {}, function ()
         SILE.call("raise", { height = "0.5ex" }, function ()
-          SILE.call("hrule", { width = "20%lw" })
+          SILE.call("hrule", { width = "20%lw", height = "0.4pt" })
         end)
       end)
     end
@@ -271,13 +271,13 @@ function package:registerCommands ()
     elseif options.separator == "---" then
         SILE.call("center", {}, function ()
           SILE.call("raise", { height = "0.5ex" }, function ()
-            SILE.call("hrule", { width = "20%lw" })
+            SILE.call("hrule", { width = "20%lw", height = "0.4pt" })
         end)
       end)
     elseif options.separator == "----" then
       SILE.call("center", {}, function ()
         SILE.call("raise", { height = "0.5ex" }, function ()
-          SILE.call("hrule", { width = "33%lw" })
+          SILE.call("hrule", { width = "33%lw", height = "0.4pt" })
         end)
       end)
     elseif options.separator == "- - - -" and self:hasCouyards() then
@@ -286,7 +286,7 @@ function package:registerCommands ()
     elseif options.separator == "--------------" then -- Page break
       SILE.call("eject")
     else
-      SILE.call("fullrule")
+      SILE.call("fullrule", { thickness = "0.4pt" })
     end
   end, "Horizontal rule in Markdown (internal)")
 

From ee37468f941400020e8ac3c309f3ac2b25b0bcb4 Mon Sep 17 00:00:00 2001
From: Omikhleia <didier.willis@gmail.com>
Date: Mon, 21 Aug 2023 01:59:20 +0200
Subject: [PATCH 2/7] fix: Propagate header shifting and metadata to embedded
 rendered blocks

I am not sure this is really required, but it makes seems more
expectable probably.
---
 inputters/djot.lua             | 58 ++++++++++++++++++++++++++--------
 inputters/markdown.lua         | 17 ++++++++++
 inputters/pandocast.lua        | 19 ++++++++++-
 packages/markdown/commands.lua |  9 +-----
 packages/markdown/utils.lua    |  9 ++++++
 5 files changed, 89 insertions(+), 23 deletions(-)

diff --git a/inputters/djot.lua b/inputters/djot.lua
index 370cfbf..509b3f4 100644
--- a/inputters/djot.lua
+++ b/inputters/djot.lua
@@ -6,6 +6,7 @@
 -- @copyright License: MIT (c) 2023 Omikhleia
 -- @module inputters.djot
 --
+local utils = require("packages.markdown.utils")
 local ast = require("silex.ast")
 local createCommand, createStructuredCommand
         = ast.createCommand, ast.createStructuredCommand
@@ -19,13 +20,15 @@ local Renderer = pl.class()
 function Renderer:_init(options)
   self.references = {}
   self.footnotes = {}
-  self.metadata = {}
   self.shift_headings = SU.cast("integer", options.shift_headings or 0)
+  self.metadata = {}
+  self.parentmetadata = {}
   for key, val in pairs(options) do
     local meta = key:match("^meta:(.*)")
     if meta then
       if meta:match("[%w_+-]+") then
         self.metadata[meta] = val
+        self.parentmetadata[key] = val
       else
         SU.warn("Invalid metadata key is skipped: "..meta)
       end
@@ -161,6 +164,20 @@ end
 function Renderer:code_block (node)
   local options = node.attr or {}
   options.class = node.lang and ((options.class and (options.class.." ") or "") .. node.lang) or options.class
+  if utils.hasClass(options, "djot") or utils.hasClass(options, "markdown") then
+    --options = pl.tablex.union(self.parentmetadata, options)
+    options.shift_headings = self.shift_headings
+    self:resolveAllUserDefinedSymbols()
+    -- Parent metadata just have a label xxx
+    -- User-defined memoized metadata have the symbol (:xxx:) as key
+    -- So we sort the keys to get the user-defined metadata first as overrides.
+    for key, val in SU.sortedpairs(self.metadata) do
+      key = key.gsub(key, ":", "")
+      if not options["meta:" .. key] then
+        options["meta:" .. key] = val
+      end
+    end
+  end
   return createCommand("markdown:internal:codeblock", options, node.s, self:render_pos(node))
 end
 
@@ -576,6 +593,31 @@ local predefinedSymbols = {
   },
 }
 
+function Renderer:getUserDefinedSymbol (label, node_fake_metadata)
+  local content
+  if self.metadata[label] then -- use memoized
+    content = self.metadata[label]
+  else
+    if #node_fake_metadata.c == 1 and node_fake_metadata.c[1].t == "para" then
+      -- Skip a single para node.
+      content = self:render_children(node_fake_metadata.c[1])
+    else
+      content = self:render_children(node_fake_metadata)
+    end
+    self.metadata[label] = content -- memoize
+  end
+  return content
+end
+
+function Renderer:resolveAllUserDefinedSymbols ()
+  -- Ensure all fake footnotes are rendered and memoized, even unused ones.
+  for label, node_fake_metadata in pairs(self.footnotes) do
+    if label:match("^:([%w_+-]+):$") then
+      self:getUserDefinedSymbol(label, node_fake_metadata)
+    end
+  end
+end
+
 function Renderer:symbol (node)
   -- Let's first look at fake footnotes to resolve the symbol.
   -- We just added unforeseen templating and recursive variable substitution to Djot.
@@ -586,19 +628,7 @@ function Renderer:symbol (node)
     if #node_fake_metadata.c > 1 and not node._standalone_ then
       SU.error("Cannot use multi-paragraph metatada "..label.." as inline content")
     end
-
-    local content
-    if self.metadata[label] then -- use memoized
-      content = self.metadata[label]
-    else
-      if #node_fake_metadata.c == 1 and node_fake_metadata.c[1].t == "para" then
-        -- Skip a single para node.
-        content = self:render_children(node_fake_metadata.c[1])
-      else
-        content = self:render_children(node_fake_metadata)
-      end
-      self.metadata[label] = content -- memoize
-    end
+    local content = self:getUserDefinedSymbol(label, node_fake_metadata)
     if node.attr then
       if not node._standalone_ then
         -- Add a span for attributes on the inline variant.
diff --git a/inputters/markdown.lua b/inputters/markdown.lua
index 3328bbd..4e59860 100644
--- a/inputters/markdown.lua
+++ b/inputters/markdown.lua
@@ -72,6 +72,19 @@ local function SileAstWriter (writerOps, renderOps)
   local generic = require("lunamark.writer.generic")
   local writer = generic.new(writerOps or {})
   local shift_headings = SU.cast("integer", renderOps.shift_headings or 0)
+  local parentmetadata = {}
+  for key, val in pairs(renderOps) do
+    local meta = key:match("^meta:(.*)")
+    if meta then
+      if meta:match("[%w_+-]+") then
+        -- We don't use them in this renderer, but we can pass them through
+        -- to embedded djot documents.
+        parentmetadata[key] = val
+      else
+        SU.warn("Invalid metadata key is skipped: "..meta)
+      end
+    end
+  end
 
   -- Simple one-to-one mappings between lunamark AST and SILE
 
@@ -148,6 +161,10 @@ local function SileAstWriter (writerOps, renderOps)
 
   writer.fenced_code = function (content, infostring, attr)
     local opts = attr or { class = infostring }
+    if utils.hasClass(opts, "djot") or utils.hasClass(opts, "markdown") then
+      opts = pl.tablex.union(parentmetadata, opts)
+      opts.shift_headings = shift_headings
+    end
     return createCommand("markdown:internal:codeblock", opts, content)
   end
 
diff --git a/inputters/pandocast.lua b/inputters/pandocast.lua
index c195887..b123b8f 100644
--- a/inputters/pandocast.lua
+++ b/inputters/pandocast.lua
@@ -48,6 +48,19 @@ local Renderer = pl.class()
 
 function Renderer:_init(options)
   self.shift_headings = SU.cast("integer", options.shift_headings or 0)
+  self.parentmetadata = {}
+  for key, val in pairs(options) do
+    local meta = key:match("^meta:(.*)")
+    if meta then
+      if meta:match("[%w_+-]+") then
+        -- We don't use them in this renderer, but we can pass them through
+        -- to embedded djot documents.
+        self.parentmetadata[key] = val
+      else
+        SU.warn("Invalid metadata key is skipped: "..meta)
+      end
+    end
+  end
 end
 
 -- Allows unpacking tables on some Pandoc AST nodes so as to map them to methods
@@ -202,8 +215,12 @@ end
 
 -- CodeBlock Attr Text
 -- Code block (literal) with attributes
-function Renderer.CodeBlock (_, attributes, text)
+function Renderer:CodeBlock (attributes, text)
   local options = pandocAttributes(attributes)
+  if utils.hasClass(options, "djot") or utils.hasClass(options, "markdown") then
+    options = pl.tablex.union(self.parentmetadata, options)
+    options.shift_headings = self.shift_headings
+  end
   return createCommand("markdown:internal:codeblock", options, text)
 end
 
diff --git a/packages/markdown/commands.lua b/packages/markdown/commands.lua
index 7b8f813..ca04e3d 100644
--- a/packages/markdown/commands.lua
+++ b/packages/markdown/commands.lua
@@ -9,6 +9,7 @@
 --
 require("silex.lang")
 local utils = require("packages.markdown.utils")
+local hasClass = utils.hasClass
 local ast = require("silex.ast")
 local createCommand, createStructuredCommand,
       extractFromTree, subContent
@@ -60,14 +61,6 @@ local function getSectioningCommand (level)
   return "markdown:fallback:header"
 end
 
-local function hasClass (options, classname)
-  -- N.B. we want a true boolean here
-  if options.class and string.match(' ' .. options.class .. ' ',' '..classname..' ') then
-    return true
-  end
-  return false
-end
-
 local function hasLinkContent(tree)
   if type(tree) == "table" then
     return #tree > 1 or hasLinkContent(tree[1])
diff --git a/packages/markdown/utils.lua b/packages/markdown/utils.lua
index 3f1ca61..0959cea 100644
--- a/packages/markdown/utils.lua
+++ b/packages/markdown/utils.lua
@@ -32,8 +32,17 @@ local function nbspFilter (str)
   return #t == 1 and t[1] or t
 end
 
+local function hasClass (options, classname)
+  -- N.B. we want a true boolean here
+  if options.class and string.match(' ' .. options.class .. ' ',' '..classname..' ') then
+    return true
+  end
+  return false
+end
+
 --- @export
 return {
   getFileExtension = getFileExtension,
   nbspFilter = nbspFilter,
+  hasClass = hasClass
 }

From 073bb3c3fb8c5bef07cad3bc461600c78f8f2cda Mon Sep 17 00:00:00 2001
From: Omikhleia <didier.willis@gmail.com>
Date: Mon, 21 Aug 2023 02:40:34 +0200
Subject: [PATCH 3/7] fix: Hard breaks do not work well in centered or ragged
 left blocks

---
 inputters/djot.lua             |  2 +-
 inputters/markdown.lua         |  2 +-
 inputters/pandocast.lua        |  2 +-
 packages/markdown/commands.lua | 22 +++++++++++++++++++++-
 4 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/inputters/djot.lua b/inputters/djot.lua
index 509b3f4..7eff9ed 100644
--- a/inputters/djot.lua
+++ b/inputters/djot.lua
@@ -368,7 +368,7 @@ function Renderer.softbreak (_)
 end
 
 function Renderer.hardbreak (_)
-  return createCommand("cr")
+  return createCommand("markdown:internal:hardbreak")
 end
 
 function Renderer:nbsp (node)
diff --git a/inputters/markdown.lua b/inputters/markdown.lua
index 4e59860..ad49e3b 100644
--- a/inputters/markdown.lua
+++ b/inputters/markdown.lua
@@ -99,7 +99,7 @@ local function SileAstWriter (writerOps, renderOps)
   writer.blockquote = simpleCommandWrapper("markdown:internal:blockquote")
   writer.verbatim = simpleCommandWrapper("verbatim")
   writer.listitem = simpleCommandWrapper("item")
-  writer.linebreak = simpleCommandWrapper("cr")
+  writer.linebreak = simpleCommandWrapper("markdown:internal:hardbreak")
   writer.singlequoted = simpleCommandWrapper("singlequoted")
   writer.doublequoted = simpleCommandWrapper("doublequoted")
 
diff --git a/inputters/pandocast.lua b/inputters/pandocast.lua
index b123b8f..6b50621 100644
--- a/inputters/pandocast.lua
+++ b/inputters/pandocast.lua
@@ -498,7 +498,7 @@ end
 -- LineBreak
 -- Hard line break
 function Renderer.LineBreak (_)
-  return createCommand("cr")
+  return createCommand("markdown:internal:hardbreak")
 end
 
 -- Math MathType Text
diff --git a/packages/markdown/commands.lua b/packages/markdown/commands.lua
index ca04e3d..e5b5be4 100644
--- a/packages/markdown/commands.lua
+++ b/packages/markdown/commands.lua
@@ -505,13 +505,33 @@ Please consider using a resilient-compatible class!]])
       SILE.processString(rawtext, "lua")
     elseif format == "html" then
       if rawtext:match("^<br[>/%s]") then
-        SILE.call("cr")
+        SILE.call("markdown:internal:hardbreak")
       elseif rawtext:match("^<wbr[>/%s]") then
         SILE.call("penalty", { penalty = 100 })
       end
     end
   end, "Raw native inline content in Markdown (internal)")
 
+  self:registerCommand("markdown:internal:hardbreak", function (_, _)
+    -- It's a bit tricky to decide whether we need a break or a cr (fill + break) here,
+    -- depending on the alignment of the paragraph.
+    --     justified = we must use a cr
+    --     ragged left = we must use a break
+    --     centered =  we must use a break
+    --     ragged right = we don't care, a break is sufficient and safer
+    local lskip = SILE.settings:get("document.lskip")
+    local rskip = SILE.settings:get("document.rskip")
+    -- Knowning the alignment is not obvious, we have to guess it from the skips.
+    -- A bit tricky and probably no a very clever HACK.
+    -- Maybe we should check whether the stretch is actually infinite?
+    if (lskip and lskip.width.stretch > 0)
+       or (rskip and rskip.width.stretch > 0) then
+      SILE.call("break")
+    else
+      SILE.call("cr")
+    end
+  end, "Hard break in Markdown (internal)")
+
   self:registerCommand("markdown:internal:rawblock", function (options, content)
     local format = SU.required(options, "format", "rawcontent")
     if format == "sile" or format == "sile-lua" then

From eaf50a92c6c946323c472c2757772cf50005b964 Mon Sep 17 00:00:00 2001
From: Omikhleia <didier.willis@gmail.com>
Date: Mon, 21 Aug 2023 18:54:35 +0200
Subject: [PATCH 4/7] chore: Better solution for hard breaks

---
 packages/markdown/commands.lua | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/packages/markdown/commands.lua b/packages/markdown/commands.lua
index e5b5be4..3c77213 100644
--- a/packages/markdown/commands.lua
+++ b/packages/markdown/commands.lua
@@ -513,23 +513,20 @@ Please consider using a resilient-compatible class!]])
   end, "Raw native inline content in Markdown (internal)")
 
   self:registerCommand("markdown:internal:hardbreak", function (_, _)
-    -- It's a bit tricky to decide whether we need a break or a cr (fill + break) here,
-    -- depending on the alignment of the paragraph.
-    --     justified = we must use a cr
-    --     ragged left = we must use a break
-    --     centered =  we must use a break
-    --     ragged right = we don't care, a break is sufficient and safer
-    local lskip = SILE.settings:get("document.lskip")
-    local rskip = SILE.settings:get("document.rskip")
-    -- Knowning the alignment is not obvious, we have to guess it from the skips.
-    -- A bit tricky and probably no a very clever HACK.
-    -- Maybe we should check whether the stretch is actually infinite?
-    if (lskip and lskip.width.stretch > 0)
-       or (rskip and rskip.width.stretch > 0) then
-      SILE.call("break")
-    else
-      SILE.call("cr")
-    end
+    -- We don't want to use a cr here, because it would affect parindents,
+    -- insert a parskip, and maybe other things.
+    -- It's a bit tricky to handle a hardbreak depending on depending on the
+    -- alignment of the paragraph:
+    --    justified = we can't use a break, a cr (hfill+break) would work
+    --    ragged left = we can't use a cr
+    --    centered = we can't use a cr
+    --    ragged right = we don't care, a break is sufficient
+    -- Knowning the alignment is not obvious, neither guessing it from the skips.
+    -- Using a parfillskip seems to do the trick, but it's maybe a bit hacky.
+    -- This is nevertheless what would have occurred with a par in the same
+    -- situation.
+    SILE.typesetter:pushGlue(SILE.settings:get("typesetter.parfillskip"))
+    SILE.call("break")
   end, "Hard break in Markdown (internal)")
 
   self:registerCommand("markdown:internal:rawblock", function (options, content)

From 16cedf84fb7e6f4c6f2a92cf504249aed8c5de03 Mon Sep 17 00:00:00 2001
From: Omikhleia <didier.willis@gmail.com>
Date: Mon, 21 Aug 2023 19:00:18 +0200
Subject: [PATCH 5/7] fix: Soft breaks may leave multiple spaces in the output

---
 inputters/djot.lua | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/inputters/djot.lua b/inputters/djot.lua
index 7eff9ed..58176af 100644
--- a/inputters/djot.lua
+++ b/inputters/djot.lua
@@ -63,7 +63,16 @@ function Renderer:render_children(node)
         self.tight = node.tight
       end
       for i=1,#node.c do
-        out[#out+1] = self[node.c[i].t](self, node.c[i])
+        local content = self[node.c[i].t](self, node.c[i])
+        -- Simplify outputs by collating strings
+        if type(content) == "string" and type(out[#out]) == "string" then
+          out[#out] = out[#out] .. content
+        else
+          -- Simplify out by removing empty elements
+          if type(content) ~= "table" or content.command or #content > -1 then
+            out[#out+1] = content
+          end
+        end
       end
       if node.tight ~= nil then
         self.tight = oldtight

From 8cf3b7bbab015e41d525ae610c379f3a09509954 Mon Sep 17 00:00:00 2001
From: Omikhleia <didier.willis@gmail.com>
Date: Mon, 21 Aug 2023 19:20:25 +0200
Subject: [PATCH 6/7] refactor: Code cleanup - I must have been tired

---
 inputters/djot.lua | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/inputters/djot.lua b/inputters/djot.lua
index 58176af..8d1a58d 100644
--- a/inputters/djot.lua
+++ b/inputters/djot.lua
@@ -22,13 +22,11 @@ function Renderer:_init(options)
   self.footnotes = {}
   self.shift_headings = SU.cast("integer", options.shift_headings or 0)
   self.metadata = {}
-  self.parentmetadata = {}
   for key, val in pairs(options) do
     local meta = key:match("^meta:(.*)")
     if meta then
       if meta:match("[%w_+-]+") then
         self.metadata[meta] = val
-        self.parentmetadata[key] = val
       else
         SU.warn("Invalid metadata key is skipped: "..meta)
       end
@@ -174,16 +172,15 @@ function Renderer:code_block (node)
   local options = node.attr or {}
   options.class = node.lang and ((options.class and (options.class.." ") or "") .. node.lang) or options.class
   if utils.hasClass(options, "djot") or utils.hasClass(options, "markdown") then
-    --options = pl.tablex.union(self.parentmetadata, options)
     options.shift_headings = self.shift_headings
     self:resolveAllUserDefinedSymbols()
     -- Parent metadata just have a label xxx
     -- User-defined memoized metadata have the symbol (:xxx:) as key
     -- So we sort the keys to get the user-defined metadata first as overrides.
     for key, val in SU.sortedpairs(self.metadata) do
-      key = key.gsub(key, ":", "")
-      if not options["meta:" .. key] then
-        options["meta:" .. key] = val
+      local name = key:match("^:([%w_+-]+):$") or key -- remove leading and trailing colons
+      if not options["meta:" .. name] then
+        options["meta:" .. name] = val
       end
     end
   end

From d9a3ff5d4f366b8a903b1d59c829d2a930181609 Mon Sep 17 00:00:00 2001
From: Omikhleia <didier.willis@gmail.com>
Date: Sat, 2 Sep 2023 18:37:38 +0200
Subject: [PATCH 7/7] refactor: Improve AST tree simplification

---
 inputters/djot.lua      | 5 ++++-
 inputters/pandocast.lua | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/inputters/djot.lua b/inputters/djot.lua
index 8d1a58d..de6e8b1 100644
--- a/inputters/djot.lua
+++ b/inputters/djot.lua
@@ -67,8 +67,11 @@ function Renderer:render_children(node)
           out[#out] = out[#out] .. content
         else
           -- Simplify out by removing empty elements
-          if type(content) ~= "table" or content.command or #content > -1 then
+          if type(content) ~= "table" or content.command or #content > 1 then
             out[#out+1] = content
+          elseif #content == 1 then
+            -- Simplify out by removing useless grouping
+            out[#out+1] = content[1]
           end
         end
       end
diff --git a/inputters/pandocast.lua b/inputters/pandocast.lua
index 6b50621..9d26b76 100644
--- a/inputters/pandocast.lua
+++ b/inputters/pandocast.lua
@@ -90,8 +90,8 @@ local function addNodes(out, elements)
     out[#out] = out[#out] .. elements
   else
     -- Simplify out by removing empty elements
-    if type(elements) ~= "table" or elements.command or #elements > -1 then
-      out [#out+1] = elements
+    if type(elements) ~= "table" or elements.command or #elements > 0 then
+      out[#out+1] = elements
     end
   end
 end