add fuzz tests and fix several bugs found by them

djot_parser/djot_ast.go

@@ -907,7 +907,7 @@ func buildDjotAst(
 					Children:   []TreeNode[DjotNode]{{Type: TextNode, Text: []byte("↩︎︎")}},
 					Attributes: attributes,
 				}
-				if children[len(children)-1].Type == ParagraphNode {
+				if len(children) > 0 && children[len(children)-1].Type == ParagraphNode {
 					children[len(children)-1].Children = append(children[len(children)-1].Children, backrefLinkNode)
 				} else {
 					children = append(children, TreeNode[DjotNode]{Type: ParagraphNode, Children: []TreeNode[DjotNode]{backrefLinkNode}})

djot_parser/djot_fuzz_test.go

@@ -0,0 +1,42 @@
+package djot_parser
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/sivukhin/godjot/djot_tokenizer"
+)
+
+func seedFuzz(f *testing.F) {
+	dir, err := os.ReadDir(examplesDir)
+	require.Nil(f, err)
+	for _, entry := range dir {
+		name := entry.Name()
+		example, ok := strings.CutSuffix(name, ".html")
+		if !ok {
+			continue
+		}
+		djotExample, err := os.ReadFile(path.Join(examplesDir, fmt.Sprintf("%v.djot", example)))
+		require.Nil(f, err)
+		f.Add(string(djotExample))
+	}
+}
+
+func FuzzDjotE2E(f *testing.F) {
+	seedFuzz(f)
+	f.Fuzz(func(t *testing.T, input string) { _ = printDjot(input) })
+}
+
+func FuzzDjotTokenizer(f *testing.F) {
+	seedFuzz(f)
+	f.Fuzz(func(t *testing.T, input string) { _ = djot_tokenizer.BuildDjotTokens([]byte(input)) })
+}
+
+func TestName(t *testing.T) {
+	t.Log(djot_tokenizer.BuildDjotTokens([]byte("[^]:")))
+}

djot_parser/examples/empty-footnote.djot

@@ -0,0 +1,3 @@
+[^]
+
+[^]:

djot_parser/examples/empty-footnote.html

@@ -0,0 +1,9 @@
+<p><a id="fnref1" href="#fn1" role="doc-noteref"><sup>1</sup></a></p>
+<section role="doc-endnotes">
+<hr>
+<ol>
+<li id="fn1">
+<p><a href="#fnref1" role="doc-backlink">↩︎︎</a></p>
+</li>
+</ol>
+</section>

djot_parser/testdata/fuzz/FuzzDjotE2E/505457fa1e917292

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("[^]:")

djot_parser/testdata/fuzz/FuzzDjotE2E/55c570d46dd4cf42

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("```\r0")

djot_tokenizer/djot_block_token.go

@@ -7,9 +7,9 @@ import (
 )
 
 var (
-	NotSpaceByteMask      = tokenizer.SpaceNewLineByteMask.Negate()
-	NotBracketByteMask    = tokenizer.NewByteMask([]byte("]")).Negate()
-	ThematicBreakByteMask = tokenizer.NewByteMask([]byte(" \t\n*-"))
+	NotSpaceNewLineByteMask = tokenizer.SpaceNewLineByteMask.Negate()
+	NotBracketByteMask      = tokenizer.NewByteMask([]byte("]")).Negate()
+	ThematicBreakByteMask   = tokenizer.NewByteMask([]byte(" \t\n*-"))
 
 	DigitByteMask      = tokenizer.NewByteMask([]byte("0123456789"))
 	LowerAlphaByteMask = tokenizer.NewByteMask([]byte("abcdefghijklmnopqrstuvwxyz"))
@@ -79,8 +79,11 @@ func MatchBlockToken(
 		}
 
 		metaStart := next
-		next, ok = r.MaskRepeat(next, NotSpaceByteMask, 1)
-		tokenizer.Assertf(ok, "MaskRepeat must match because !r.IsEmpty(next) and next symbol is not in SpaceByteMask")
+		next, ok = r.MaskRepeat(next, NotSpaceNewLineByteMask, 1)
+		// usually MaskRepeat must match because !r.IsEmpty(next) and next symbol is not in SpaceByteMask but in some broken cases this can fail (see panic1 test)
+		if !ok {
+			return fail()
+		}
 		metaEnd := next
 
 		if next, ok = r.EmptyOrWhiteSpace(next); !ok {