From 5b26d7977c07c5552ba595a724ddc0bcf6d0c69f Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Thu, 16 Jan 2025 20:49:17 +0200 Subject: [PATCH 1/9] feat: add language.toml entry for djot --- languages.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/languages.toml b/languages.toml index 31deb99070ca..e86595b478bf 100644 --- a/languages.toml +++ b/languages.toml @@ -1665,6 +1665,18 @@ grammar = "markdown_inline" name = "markdown_inline" source = { git = "https://github.com/tree-sitter-grammars/tree-sitter-markdown", rev = "62516e8c78380e3b51d5b55727995d2c511436d8", subpath = "tree-sitter-markdown-inline" } +[[language]] +name = "djot" +scope = "source.djot" +injection-regex = "dj|djot" +file-types = ["dj", "djot"] +indent = { tab-width = 2, unit = " " } +block-comment-tokens = { start = "{%", end = "%}" } + +[[grammar]] +name = "djot" +source = { git = "https://github.com/treeman/tree-sitter-djot", rev = "886601b67d1f4690173a4925c214343c30704d32" } + [[language]] name = "dart" scope = "source.dart" From 3287530902930f76a8445e8be35495d84c86fbaf Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Thu, 16 Jan 2025 20:49:28 +0200 Subject: [PATCH 2/9] feat: add highlights for djot --- runtime/queries/djot/highlights.scm | 311 ++++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 runtime/queries/djot/highlights.scm diff --git a/runtime/queries/djot/highlights.scm b/runtime/queries/djot/highlights.scm new file mode 100644 index 000000000000..dde43dc48eeb --- /dev/null +++ b/runtime/queries/djot/highlights.scm @@ -0,0 +1,311 @@ +[ + (footnote_marker_begin) + (footnote_marker_end) +] @punctuation.bracket + +(footnote_reference (reference_label) @markup.link.label) + +(footnote (reference_label) @markup.link.label) + +[ + (autolink) + (inline_link_destination) + (link_destination) +] @markup.link.url + +(link_reference_definition (link_label) @markup.link.label) + +(link_reference_definition + [ + "[" + "]" + ] @punctuation.bracket) + +(image_description + [ + "[" + "]" + ] @punctuation.bracket) + +(image_description) @label + +(inline_image + [ + "![" + "]" + ] @punctuation.bracket) + +(collapsed_reference_image + [ + "![" + "]" + ] @punctuation.bracket) + +(collapsed_reference_image "[]" @punctuation.bracket) + +(full_reference_image + [ + "![" + "[" + "]" + ] @punctuation.bracket) + +(full_reference_image (link_label) @markup.link.label) + +(inline_link (link_text) @markup.link.text) + +(collapsed_reference_link (link_text) @markup.link.text) + +(full_reference_link + [ + "[" + "]" + ] @punctuation.bracket) + +(collapsed_reference_link "[]" @punctuation.bracket) + +(full_reference_link (link_label) @markup.link.label) + +(full_reference_link (link_text) @markup.link.text) + +(link_reference_definition ":" @punctuation.delimiter) + +(inline_link_destination + [ + "(" + ")" + ] @punctuation.bracket) + +(autolink + [ + "<" + ">" + ] @punctuation.bracket) + +(link_text + [ + "[" + "]" + ] @punctuation.bracket) + +(key_value (value) @string) + +(key_value (key) @attribute) + +(key_value "=" @operator) + +; NOTE: Not perfectly semantically accurate, but a fair approximation. +(identifier) @string.special.symbol + +[ + (class) + (class_name) +] @type + +(block_attribute + [ + "{" + "}" + ] @punctuation.bracket) + +(inline_attribute + [ + "{" + "}" + ] @punctuation.bracket) + +(span + [ + "[" + "]" + ] @punctuation.bracket) + +(comment + [ + "{" + "}" + "%" + ] @comment.block) + +; INFO: djot only has block-scoped comments. +(comment) @comment.block + +(raw_inline) @markup.raw + +(verbatim) @markup.raw + +(math) @markup.raw + +([ + (emphasis_begin) + (emphasis_end) + (strong_begin) + (strong_end) + (verbatim_marker_begin) + (verbatim_marker_end) + (math_marker) + (math_marker_begin) + (math_marker_end) + (raw_inline_attribute) + (raw_inline_marker_begin) + (raw_inline_marker_end) +] @punctuation.bracket) + +(subscript + [ + "~" + "{~" + "~}" + ] @punctuation.bracket) + +(superscript + [ + "^" + "{^" + "^}" + ] @punctuation.bracket) + +(delete + [ + "{-" + "-}" + ] @punctuation.bracket) + +(insert + [ + "{+" + "+}" + ] @punctuation.bracket) + +; NOTE: We need to target tokens specifically because `{=` etc can exist as fallback symbols in +; regular text, which we don't want to highlight or conceal. +(highlighted + [ + "{=" + "=}" + ] @punctuation.bracket) + +; TEMP: Scope not available, with no appropriate alternative. +(subscript) @markup.subscript + +; TEMP: Scope not available, with no appropriate alternative. +(superscript) @markup.superscript + +; TEMP: Scope not available, with no appropriate alternative. +(highlighted) @markup.highlighted + +; TEMP: Scope not available, with no appropriate alternative. +(insert) @markup.insert + +(delete) @markup.strikethrough + +; INFO: This applies to emojis. +(symbol) @string.special.symbol + +(strong) @markup.bold + +(emphasis) @markup.italic + +(backslash_escape) @constant.character.escape + +(hard_line_break) @constant.character.escape + +((quotation_marks) @constant.character.escape + (#any-of? @constant.character.escape "\\\"" "\\'")) + +; NOTE: Opting for this over the @punctuation.bracket used for things like +; {-delete-} to emphasise the semantic significance of specified opening/closing +; quotation marks. +(quotation_marks) @markup.quote + +; INFO: `term` only matches on definition list items -- other list items won't be highlighted. +(list_item (term) @type) + +[ + (ellipsis) + (en_dash) + (em_dash) +] @punctuation.special + +(checked + [ + "x" + "X" + ] @constant.builtin.boolean) @markup.list.checked + +(list_marker_task (checked)) @markup.list.checked + +(list_marker_task (unchecked)) @markup.list.unchecked + +[ + (list_marker_decimal_period) + (list_marker_decimal_paren) + (list_marker_decimal_parens) + (list_marker_lower_alpha_period) + (list_marker_lower_alpha_paren) + (list_marker_lower_alpha_parens) + (list_marker_upper_alpha_period) + (list_marker_upper_alpha_paren) + (list_marker_upper_alpha_parens) + (list_marker_lower_roman_period) + (list_marker_lower_roman_paren) + (list_marker_lower_roman_parens) + (list_marker_upper_roman_period) + (list_marker_upper_roman_paren) + (list_marker_upper_roman_parens) +] @markup.list.numbered + +[ + (list_marker_dash) + (list_marker_plus) + (list_marker_star) + (list_marker_definition) +] @markup.list.unnumbered + +(table_caption) @label + +(table_caption (marker) @punctuation.special) + +(table_separator) @punctuation.special + +(table_row "|" @punctuation.special) + +(table_header "|" @punctuation.special) + +(table_header) @markup.heading + +(block_quote) @markup.quote + +(block_quote_marker) @markup.quote + +(language_marker) @punctuation.delimiter + +(inline_attribute _ @attribute) + +(language) @type.enum.variant + +[ + (code_block_marker_begin) + (code_block_marker_end) + (raw_block_marker_begin) + (raw_block_marker_end) +] @punctuation.bracket + +[ + (code_block) + (raw_block) +] @markup.raw.block + +[ + (div_marker_begin) + (div_marker_end) +] @tag + +(thematic_break) @special + +(heading1 (marker) @markup.heading.marker) @markup.heading.1 +(heading2 (marker) @markup.heading.marker) @markup.heading.2 +(heading3 (marker) @markup.heading.marker) @markup.heading.3 +(heading4 (marker) @markup.heading.marker) @markup.heading.4 +(heading5 (marker) @markup.heading.marker) @markup.heading.5 +(heading6 (marker) @markup.heading.marker) @markup.heading.6 From 6e2f42f221dcc2513340ae936118a461e81d21d7 Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Thu, 16 Jan 2025 20:49:35 +0200 Subject: [PATCH 3/9] feat: add injections for djot --- runtime/queries/djot/injections.scm | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 runtime/queries/djot/injections.scm diff --git a/runtime/queries/djot/injections.scm b/runtime/queries/djot/injections.scm new file mode 100644 index 000000000000..59bd773dc3bb --- /dev/null +++ b/runtime/queries/djot/injections.scm @@ -0,0 +1,16 @@ +((content) @injection.content + (#set! injection.language "comment")) + +(code_block + (language) @injection.language + (code) @injection.content (#set! injection.include-unnamed-children)) + +(raw_block + (raw_block_info + (language) @injection.language) + (content) @injection.content (#set! injection.include-unnamed-children)) + +(raw_inline + (content) @injection.content + (raw_inline_attribute + (language) @injection.language) (#set! injection.include-unnamed-children)) From db95764acead3c2a6cae97d858ee7bd21c04cf69 Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Thu, 16 Jan 2025 20:49:54 +0200 Subject: [PATCH 4/9] docs: run cargo xtask docgen for djot entry --- book/src/generated/lang-support.md | 1 + 1 file changed, 1 insertion(+) diff --git a/book/src/generated/lang-support.md b/book/src/generated/lang-support.md index 376080b56948..6bcac24e065c 100644 --- a/book/src/generated/lang-support.md +++ b/book/src/generated/lang-support.md @@ -38,6 +38,7 @@ | devicetree | ✓ | | | | | dhall | ✓ | ✓ | | `dhall-lsp-server` | | diff | ✓ | | | | +| djot | ✓ | | | | | docker-compose | ✓ | ✓ | ✓ | `docker-compose-langserver`, `yaml-language-server` | | dockerfile | ✓ | ✓ | | `docker-langserver` | | dot | ✓ | | | `dot-language-server` | From a4722b521e5389a5799ba4b2a136b740d2fd07f2 Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Fri, 17 Jan 2025 18:37:25 +0200 Subject: [PATCH 5/9] feat: add latex highlighting for math and fix inline code highlighting --- runtime/queries/djot/injections.scm | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/runtime/queries/djot/injections.scm b/runtime/queries/djot/injections.scm index 59bd773dc3bb..d102e74f14b4 100644 --- a/runtime/queries/djot/injections.scm +++ b/runtime/queries/djot/injections.scm @@ -1,6 +1,9 @@ -((content) @injection.content +(comment (content) @injection.content (#set! injection.language "comment")) +(math (content) @injection.content + (#set! injection.language "latex") (#set! injection.include-unnamed-children)) + (code_block (language) @injection.language (code) @injection.content (#set! injection.include-unnamed-children)) @@ -11,6 +14,6 @@ (content) @injection.content (#set! injection.include-unnamed-children)) (raw_inline - (content) @injection.content + (content) @injection.content (#set! injection.include-unnamed-children) (raw_inline_attribute - (language) @injection.language) (#set! injection.include-unnamed-children)) + (language) @injection.language)) From d8a87af4fb7e597210c16509ba166ee95729e5f8 Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Sat, 18 Jan 2025 19:50:39 +0200 Subject: [PATCH 6/9] refactor: adjust term highlighting to use the constructor scope --- runtime/queries/djot/highlights.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/queries/djot/highlights.scm b/runtime/queries/djot/highlights.scm index dde43dc48eeb..9d4865533038 100644 --- a/runtime/queries/djot/highlights.scm +++ b/runtime/queries/djot/highlights.scm @@ -219,7 +219,7 @@ (quotation_marks) @markup.quote ; INFO: `term` only matches on definition list items -- other list items won't be highlighted. -(list_item (term) @type) +(list_item (term) @constructor) [ (ellipsis) From 0e71c22b5344bc5d43edd00b6a4e5f9021ca4e6b Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Fri, 31 Jan 2025 18:06:51 +0200 Subject: [PATCH 7/9] chore: update highlights queries --- runtime/queries/djot/highlights.scm | 151 ++++++++++++---------------- 1 file changed, 65 insertions(+), 86 deletions(-) diff --git a/runtime/queries/djot/highlights.scm b/runtime/queries/djot/highlights.scm index 9d4865533038..7bec214a8690 100644 --- a/runtime/queries/djot/highlights.scm +++ b/runtime/queries/djot/highlights.scm @@ -13,15 +13,15 @@ (link_destination) ] @markup.link.url -(link_reference_definition (link_label) @markup.link.label) - -(link_reference_definition +(inline_link_destination [ - "[" - "]" + "(" + ")" ] @punctuation.bracket) -(image_description +(link_reference_definition (link_label) @markup.link.label) + +(link_reference_definition [ "[" "]" @@ -29,13 +29,7 @@ (image_description) @label -(inline_image - [ - "![" - "]" - ] @punctuation.bracket) - -(collapsed_reference_image +(image_description [ "![" "]" @@ -45,7 +39,6 @@ (full_reference_image [ - "![" "[" "]" ] @punctuation.bracket) @@ -70,11 +63,7 @@ (link_reference_definition ":" @punctuation.delimiter) -(inline_link_destination - [ - "(" - ")" - ] @punctuation.bracket) +(inline_link (inline_link_destination) @markup.link.url) (autolink [ @@ -120,27 +109,31 @@ "]" ] @punctuation.bracket) -(comment - [ - "{" - "}" - "%" - ] @comment.block) +(inline_comment) @comment.line -; INFO: djot only has block-scoped comments. (comment) @comment.block -(raw_inline) @markup.raw - (verbatim) @markup.raw +(raw_inline) @markup.raw + (math) @markup.raw -([ +[ (emphasis_begin) (emphasis_end) (strong_begin) (strong_end) + (superscript_begin) + (superscript_end) + (subscript_begin) + (subscript_end) + (highlighted_begin) + (highlighted_end) + (insert_begin) + (insert_end) + (delete_begin) + (delete_end) (verbatim_marker_begin) (verbatim_marker_end) (math_marker) @@ -149,41 +142,7 @@ (raw_inline_attribute) (raw_inline_marker_begin) (raw_inline_marker_end) -] @punctuation.bracket) - -(subscript - [ - "~" - "{~" - "~}" - ] @punctuation.bracket) - -(superscript - [ - "^" - "{^" - "^}" - ] @punctuation.bracket) - -(delete - [ - "{-" - "-}" - ] @punctuation.bracket) - -(insert - [ - "{+" - "+}" - ] @punctuation.bracket) - -; NOTE: We need to target tokens specifically because `{=` etc can exist as fallback symbols in -; regular text, which we don't want to highlight or conceal. -(highlighted - [ - "{=" - "=}" - ] @punctuation.bracket) +] @punctuation.bracket ; TEMP: Scope not available, with no appropriate alternative. (subscript) @markup.subscript @@ -199,43 +158,42 @@ (delete) @markup.strikethrough -; INFO: This applies to emojis. (symbol) @string.special.symbol (strong) @markup.bold (emphasis) @markup.italic -(backslash_escape) @constant.character.escape - -(hard_line_break) @constant.character.escape +[ + (hard_line_break) + (backslash_escape) +] @constant.character.escape ((quotation_marks) @constant.character.escape (#any-of? @constant.character.escape "\\\"" "\\'")) -; NOTE: Opting for this over the @punctuation.bracket used for things like -; {-delete-} to emphasise the semantic significance of specified opening/closing -; quotation marks. (quotation_marks) @markup.quote -; INFO: `term` only matches on definition list items -- other list items won't be highlighted. (list_item (term) @constructor) [ (ellipsis) (en_dash) (em_dash) + (quotation_marks) ] @punctuation.special -(checked +(checked [ - "x" - "X" + "x" + "X" ] @constant.builtin.boolean) @markup.list.checked -(list_marker_task (checked)) @markup.list.checked +(list_marker_task + (checked)) @markup.list.checked -(list_marker_task (unchecked)) @markup.list.unchecked +(list_marker_task + (unchecked)) @markup.list.unchecked [ (list_marker_decimal_period) @@ -274,9 +232,10 @@ (table_header) @markup.heading -(block_quote) @markup.quote - -(block_quote_marker) @markup.quote +[ + (block_quote) + (block_quote_marker) +] @markup.quote (language_marker) @punctuation.delimiter @@ -294,6 +253,7 @@ [ (code_block) (raw_block) + (frontmatter) ] @markup.raw.block [ @@ -303,9 +263,28 @@ (thematic_break) @special -(heading1 (marker) @markup.heading.marker) @markup.heading.1 -(heading2 (marker) @markup.heading.marker) @markup.heading.2 -(heading3 (marker) @markup.heading.marker) @markup.heading.3 -(heading4 (marker) @markup.heading.marker) @markup.heading.4 -(heading5 (marker) @markup.heading.marker) @markup.heading.5 -(heading6 (marker) @markup.heading.marker) @markup.heading.6 +((heading + (marker) @markup.heading.marker) @markup.heading.1 + (#eq? @markup.heading.marker "# ")) + +((heading + (marker) @markup.heading.marker) @markup.heading.2 + (#eq? @markup.heading.marker "## ")) + +((heading + (marker) @markup.heading.marker) @markup.heading.3 + (#eq? @markup.heading.marker "### ")) + +((heading + (marker) @markup.heading.marker) @markup.heading.4 + (#eq? @markup.heading.marker "##### ")) + +((heading + (marker) @markup.heading.marker) @markup.heading.5 + (#eq? @markup.heading.marker "###### ")) + +((heading + (marker) @markup.heading.marker) @markup.heading.6 + (#eq? @markup.heading.marker "####### ")) + +(heading) @markup.heading From 3a0f5b0bd14379608d15d52638e75fe9f30f86d4 Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Fri, 31 Jan 2025 18:07:11 +0200 Subject: [PATCH 8/9] style: correct bad indentation --- runtime/queries/djot/injections.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/queries/djot/injections.scm b/runtime/queries/djot/injections.scm index d102e74f14b4..63d9b0a7fb95 100644 --- a/runtime/queries/djot/injections.scm +++ b/runtime/queries/djot/injections.scm @@ -1,5 +1,5 @@ (comment (content) @injection.content - (#set! injection.language "comment")) + (#set! injection.language "comment")) (math (content) @injection.content (#set! injection.language "latex") (#set! injection.include-unnamed-children)) From 5f3f68a8bc695f28a1fff0ca040dd71f8663fa74 Mon Sep 17 00:00:00 2001 From: Matthew Mark Ibbetson Date: Fri, 31 Jan 2025 18:07:21 +0200 Subject: [PATCH 9/9] chore: bump djot grammar revision --- languages.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/languages.toml b/languages.toml index e86595b478bf..b847c5169abb 100644 --- a/languages.toml +++ b/languages.toml @@ -1675,7 +1675,7 @@ block-comment-tokens = { start = "{%", end = "%}" } [[grammar]] name = "djot" -source = { git = "https://github.com/treeman/tree-sitter-djot", rev = "886601b67d1f4690173a4925c214343c30704d32" } +source = { git = "https://github.com/treeman/tree-sitter-djot", rev = "67e6e23ba7be81a4373e0f49e21207bdc32d12a5" } [[language]] name = "dart"