feature: tweaking the search to work better

go.mod

@@ -6,10 +6,11 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/yuin/goldmark v1.5.4
 	github.com/yuin/goldmark-meta v1.1.0
+	golang.org/x/net v0.20.0
 	golang.org/x/text v0.14.0
 )
 
 require (
-	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect
 )

go.sum

@@ -4,8 +4,10 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
 github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
 github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

internal/document.go

@@ -4,6 +4,9 @@ import (
 	"fmt"
 	"html/template"
 	"path/filepath"
+	"strings"
+
+	"golang.org/x/net/html"
 )
 
 // document represents a single markdown document in the site
@@ -22,6 +25,33 @@ func (doc document) String() string {
 	return fmt.Sprintf("Document: %v", doc.title)
 }
 
+func (doc document) Tokens() string {
+	var s string
+
+	domDocTest := html.NewTokenizer(strings.NewReader(string(doc.html)))
+	previousStartTokenTest := domDocTest.Token()
+loopDomTest:
+	for {
+		tt := domDocTest.Next()
+		switch {
+		case tt == html.ErrorToken:
+			break loopDomTest // End of the document,  done
+		case tt == html.StartTagToken:
+			previousStartTokenTest = domDocTest.Token()
+		case tt == html.TextToken:
+			if previousStartTokenTest.Data == "script" {
+				continue
+			}
+			TxtContent := strings.TrimSpace(html.UnescapeString(string(domDocTest.Text())))
+			if len(TxtContent) > 0 {
+				s += TxtContent + " "
+			}
+		}
+	}
+
+	return s
+}
+
 // NewDocument takes the path of a document and its content
 // and returns a parsed document with the metadata applied.
 func NewDocument(path string, content []byte) (document, error) {

internal/generate.go

@@ -34,9 +34,10 @@ type generatedPage struct {
 
 	Title       string        `json:"title"`
 	SectionName string        `json:"section_name"`
-	Content     template.HTML `json:"content"`
+	Content     template.HTML `json:"-"`
 	Link        string        `json:"link"`
 	filePath    string        `json:"-"`
+	Tokens      string        `json:"content"`
 
 	Prev       navlink       `json:"-"`
 	Next       navlink       `json:"-"`
@@ -94,6 +95,7 @@ func Generate(srcFolder, dstFolder string, site *site) error {
 
 				Content: doc.html,
 				Style:   template.CSS(style),
+				Tokens:  doc.Tokens(),
 
 				Navigation: desktopNavigation(site, doc),
 			}

internal/search.js

@@ -27,15 +27,13 @@ document.addEventListener('DOMContentLoaded', () => {
     .then(response => response.json())
     .then(data => {
         window.searchIndex = new Fuse(data, {
-            shouldSort: true,
+            threshold: 0.8,
+            tokenize: false,
             includeMatches: true,
-            threshold: 0.5,
-            tokenize: true,
-            location: 0,
-            distance: 100,
             maxPatternLength: 32,
-            minMatchCharLength: 2,
-            keys: ["title", "section_name", "content"],
+            minMatchCharLength: 1,
+            location: 80_000,
+            keys: ["content","title", "section_name"],
         });
 
     }).catch(error => console.error('Error:', error));
@@ -82,7 +80,7 @@ function search(searchQuery) {
 
 
         populateResults(result);
-    }, 200);
+    }, 400);
 }
 
 function populateResults(result) {
@@ -107,7 +105,7 @@ function populateResults(result) {
 function render(templateString, data) {
     var conditionalMatches, conditionalPattern, copy;
     conditionalPattern = /\$\{\s*isset ([a-zA-Z]*) \s*\}(.*)\$\{\s*end\s*}/g;
-    //since loop below depends on re.lastInxdex, we use a copy to capture any manipulations whilst inside the loop
+    //since loop below depends on re.lastIndex, we use a copy to capture any manipulations whilst inside the loop
     copy = templateString;
     while ((conditionalMatches = conditionalPattern.exec(templateString)) !== null) {
         if (data[conditionalMatches[1]]) {

internal/template.html

@@ -244,7 +244,7 @@ <h2 class="sr-only">Quick actions</h2>
     </script>
     <script id="search-result-template" type="text/x-js-template">
         <li class="text-gray-700 group cursor-default select-none items-center rounded-md px-3 py-2 hover:text-blue-500">
-            <a class="flex items-center" href="${link}">
+            <a class="flex items-center" href="/${link}">
                 <svg class="h-6 w-6 flex-none" fill="none" stroke="currentColor" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M7 21h10a2 2 0 002-2V9.414a1 1 0 00-.293-.707l-5.414-5.414A1 1 0 0012.586 3H7a2 2 0 00-2 2v14a2 2 0 002 2z"></path></svg>
                 <span class="ml-3 flex-grow flex-auto truncate">${title}</span>
                 <svg class="w-6 h-6" fill="none" stroke="currentColor" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M17 8l4 4m0 0l-4 4m4-4H3"></path></svg>