brain: unexport tokenizer function

brain/learn.go

@@ -49,7 +49,7 @@ var tuplesPool tpool.Pool[[]Tuple]
 
 // Learn records a message into a Learner.
 func Learn(ctx context.Context, l Learner, tag string, msg *Message) error {
-	toks := Tokens(tokensPool.Get(), msg.Text)
+	toks := tokens(tokensPool.Get(), msg.Text)
 	defer func() { tokensPool.Put(toks[:0]) }()
 	if len(toks) == 0 {
 		return nil

brain/speak.go

@@ -27,7 +27,7 @@ var (
 // regardless of the prompt, with no error.
 func Speak(ctx context.Context, s Speaker, tag, prompt string) (string, []string, error) {
 	w := builderPool.Get()
-	toks := Tokens(tokensPool.Get(), prompt)
+	toks := tokens(tokensPool.Get(), prompt)
 	defer func() {
 		w.Reset()
 		builderPool.Put(w)

brain/words.go

@@ -14,8 +14,8 @@ var (
 	syms = rangetable.Merge(unicode.M, unicode.P, unicode.S)
 )
 
-// Tokens converts a message into a list of its words appended to dst.
-func Tokens(dst []string, msg string) []string {
+// tokens converts a message into a list of its words appended to dst.
+func tokens(dst []string, msg string) []string {
 	start := len(dst)
 	for len(msg) > 0 {
 		// The general procedure is to find which of several sets of runes

brain/words_test.go

@@ -1,12 +1,11 @@
-package brain_test
+package brain
 
 import (
 	"fmt"
 	"math/rand/v2"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/zephyrtronium/robot/brain"
 )
 
 func TestWords(t *testing.T) {
@@ -61,7 +60,7 @@ func TestWords(t *testing.T) {
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			got := brain.Tokens(c.in, c.msg)
+			got := tokens(c.in, c.msg)
 			if diff := cmp.Diff(c.want, got); diff != "" {
 				t.Errorf("wrong result (+got/-want):\n%s", diff)
 			}
@@ -82,6 +81,6 @@ func BenchmarkWords(b *testing.B) {
 	dst := make([]string, 0, 16)
 	b.ResetTimer()
 	for range b.N {
-		dst = brain.Tokens(dst[:0], msgs[rand.Uint32()%uint32(len(msgs))])
+		dst = tokens(dst[:0], msgs[rand.Uint32()%uint32(len(msgs))])
 	}
 }