Tokens by value (#187)

This results in about a 8% improvement in speed.

cmd/chroma/main.go

@@ -259,7 +259,7 @@ func format(w io.Writer, style *chroma.Style, it chroma.Iterator) {
 
 func check(filename string, it chroma.Iterator) {
 	line, col := 1, 0
-	for token := it(); token != nil; token = it() {
+	for token := it(); token != chroma.EOF; token = it() {
 		if token.Type == chroma.Error {
 			fmt.Printf("%s:%d:%d %q\n", filename, line, col, token.String())
 		}

coalesce.go

@@ -6,17 +6,17 @@ func Coalesce(lexer Lexer) Lexer { return &coalescer{lexer} }
 type coalescer struct{ Lexer }
 
 func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
-	var prev *Token
+	var prev Token
 	it, err := d.Lexer.Tokenise(options, text)
 	if err != nil {
 		return nil, err
 	}
-	return func() *Token {
-		for token := it(); token != nil; token = it() {
+	return func() Token {
+		for token := it(); token != (EOF); token = it() {
 			if len(token.Value) == 0 {
 				continue
 			}
-			if prev == nil {
+			if prev == EOF {
 				prev = token
 			} else {
 				if prev.Type == token.Type && len(prev.Value) < 8192 {
@@ -29,7 +29,7 @@ func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (Iterator, e
 			}
 		}
 		out := prev
-		prev = nil
+		prev = EOF
 		return out
 	}, nil
 }

coalesce_test.go

@@ -14,6 +14,6 @@ func TestCoalesce(t *testing.T) {
 	}))
 	actual, err := Tokenise(lexer, nil, "!@#$")
 	assert.NoError(t, err)
-	expected := []*Token{{Punctuation, "!@#$"}}
+	expected := []Token{{Punctuation, "!@#$"}}
 	assert.Equal(t, expected, actual)
 }

delegate.go

@@ -31,7 +31,7 @@ func (d *delegatingLexer) Config() *Config {
 // An insertion is the character range where language tokens should be inserted.
 type insertion struct {
 	start, end int
-	tokens     []*Token
+	tokens     []Token
 }
 
 func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
@@ -44,15 +44,15 @@ func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Itera
 	insertions := []*insertion{}
 	var insert *insertion
 	offset := 0
-	var last *Token
+	var last Token
 	for _, t := range tokens {
 		if t.Type == Other {
-			if last != nil && insert != nil && last.Type != Other {
+			if last != EOF && insert != nil && last.Type != Other {
 				insert.end = offset
 			}
 			others.WriteString(t.Value)
 		} else {
-			if last == nil || last.Type == Other {
+			if last == EOF || last.Type == Other {
 				insert = &insertion{start: offset}
 				insertions = append(insertions, insert)
 			}
@@ -73,12 +73,12 @@ func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Itera
 	}
 
 	// Interleave the two sets of tokens.
-	out := []*Token{}
+	var out []Token
 	offset = 0 // Offset into text.
 	tokenIndex := 0
-	nextToken := func() *Token {
+	nextToken := func() Token {
 		if tokenIndex >= len(rootTokens) {
-			return nil
+			return EOF
 		}
 		t := rootTokens[tokenIndex]
 		tokenIndex++
@@ -95,18 +95,18 @@ func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Itera
 	}
 	t := nextToken()
 	i := nextInsertion()
-	for t != nil || i != nil {
+	for t != EOF || i != nil {
 		// fmt.Printf("%d->%d:%q   %d->%d:%q\n", offset, offset+len(t.Value), t.Value, i.start, i.end, Stringify(i.tokens...))
-		if t == nil || (i != nil && i.start < offset+len(t.Value)) {
-			var l *Token
+		if t == EOF || (i != nil && i.start < offset+len(t.Value)) {
+			var l Token
 			l, t = splitToken(t, i.start-offset)
-			if l != nil {
+			if l != EOF {
 				out = append(out, l)
 				offset += len(l.Value)
 			}
 			out = append(out, i.tokens...)
 			offset += i.end - i.start
-			if t == nil {
+			if t == EOF {
 				t = nextToken()
 			}
 			i = nextInsertion()
@@ -119,15 +119,15 @@ func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Itera
 	return Literator(out...), nil
 }
 
-func splitToken(t *Token, offset int) (l *Token, r *Token) {
-	if t == nil {
-		return nil, nil
+func splitToken(t Token, offset int) (l Token, r Token) {
+	if t == EOF {
+		return EOF, EOF
 	}
 	if offset == 0 {
-		return nil, t
+		return EOF, t
 	}
 	if offset == len(t.Value) {
-		return t, nil
+		return t, EOF
 	}
 	l = t.Clone()
 	r = t.Clone()

delegate_test.go

@@ -1,6 +1,7 @@
 package chroma
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/alecthomas/assert"
@@ -31,9 +32,9 @@ func TestDelegate(t *testing.T) {
 	testdata := []struct {
 		name     string
 		source   string
-		expected []*Token
+		expected []Token
 	}{
-		{"SourceInMiddle", `hello world <? what ?> there`, []*Token{
+		{"SourceInMiddle", `hello world <? what ?> there`, []Token{
 			{Keyword, "hello"},
 			{TextWhitespace, " "},
 			{Name, "world"},
@@ -48,7 +49,7 @@ func TestDelegate(t *testing.T) {
 			{TextWhitespace, " "},
 			{Name, "there"},
 		}},
-		{"SourceBeginning", `<? what ?> hello world there`, []*Token{
+		{"SourceBeginning", `<? what ?> hello world there`, []Token{
 			{CommentPreproc, "<?"},
 			{TextWhitespace, " "},
 			{Keyword, "what"},
@@ -61,7 +62,7 @@ func TestDelegate(t *testing.T) {
 			{TextWhitespace, " "},
 			{Name, "there"},
 		}},
-		{"SourceEnd", `hello world <? what there`, []*Token{
+		{"SourceEnd", `hello world <? what there`, []Token{
 			{Keyword, "hello"},
 			{TextWhitespace, " "},
 			{Name, "world"},
@@ -73,7 +74,7 @@ func TestDelegate(t *testing.T) {
 			{TextWhitespace, " "},
 			{Error, "there"},
 		}},
-		{"SourceMultiple", "hello world <? what ?> hello there <? what ?> hello", []*Token{
+		{"SourceMultiple", "hello world <? what ?> hello there <? what ?> hello", []Token{
 			{Keyword, "hello"},
 			{TextWhitespace, " "},
 			{Name, "world"},
@@ -104,6 +105,7 @@ func TestDelegate(t *testing.T) {
 			it, err := delegate.Tokenise(nil, test.source)
 			assert.NoError(t, err)
 			actual := it.Tokens()
+			fmt.Println(actual)
 			assert.Equal(t, test.expected, actual)
 		})
 	}

formatters/api.go

@@ -11,7 +11,7 @@ import (
 var (
 	// NoOp formatter.
 	NoOp = Register("noop", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, iterator chroma.Iterator) error {
-		for t := iterator(); t != nil; t = iterator() {
+		for t := iterator(); t != chroma.EOF; t = iterator() {
 			if _, err := io.WriteString(w, t.Value); err != nil {
 				return err
 			}

formatters/html/html.go

@@ -129,7 +129,7 @@ func (f *Formatter) restyle(style *chroma.Style) (*chroma.Style, error) {
 // We deliberately don't use html/template here because it is two orders of magnitude slower (benchmarked).
 //
 // OTOH we need to be super careful about correct escaping...
-func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []*chroma.Token) (err error) { // nolint: gocyclo
+func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.Token) (err error) { // nolint: gocyclo
 	style, err = f.restyle(style)
 	if err != nil {
 		return err
@@ -391,8 +391,8 @@ func compressStyle(s string) string {
 	return strings.Join(out, ";")
 }
 
-func splitTokensIntoLines(tokens []*chroma.Token) (out [][]*chroma.Token) {
-	line := []*chroma.Token{}
+func splitTokensIntoLines(tokens []chroma.Token) (out [][]chroma.Token) {
+	var line []chroma.Token
 	for _, token := range tokens {
 		for strings.Contains(token.Value, "\n") {
 			parts := strings.SplitAfterN(token.Value, "\n", 2)

formatters/html/html_test.go

@@ -32,11 +32,11 @@ func BenchmarkHTMLFormatter(b *testing.B) {
 }
 
 func TestSplitTokensIntoLines(t *testing.T) {
-	in := []*chroma.Token{
+	in := []chroma.Token{
 		{Value: "hello", Type: chroma.NameKeyword},
 		{Value: " world\nwhat?\n", Type: chroma.NameKeyword},
 	}
-	expected := [][]*chroma.Token{
+	expected := [][]chroma.Token{
 		{
 			{Type: chroma.NameKeyword, Value: "hello"},
 			{Type: chroma.NameKeyword, Value: " world\n"},
@@ -53,7 +53,7 @@ func TestSplitTokensIntoLines(t *testing.T) {
 }
 
 func TestIteratorPanicRecovery(t *testing.T) {
-	it := func() *chroma.Token {
+	it := func() chroma.Token {
 		panic(errors.New("bad"))
 	}
 	err := New().Format(ioutil.Discard, styles.Fallback, it)

formatters/json.go

@@ -12,7 +12,7 @@ import (
 var JSON = Register("json", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, it chroma.Iterator) error {
 	fmt.Fprintln(w, "[")
 	i := 0
-	for t := it(); t != nil; t = it() {
+	for t := it(); t != chroma.EOF; t = it() {
 		if i > 0 {
 			fmt.Fprintln(w, ",")
 		}

formatters/tokens.go

@@ -9,7 +9,7 @@ import (
 
 // Tokens formatter outputs the raw token structures.
 var Tokens = Register("tokens", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, it chroma.Iterator) error {
-	for t := it(); t != nil; t = it() {
+	for t := it(); t != chroma.EOF; t = it() {
 		if _, err := fmt.Fprintln(w, t.GoString()); err != nil {
 			return err
 		}

formatters/tty_indexed.go

@@ -216,7 +216,7 @@ func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style, it chroma
 		}
 	}()
 	theme := styleToEscapeSequence(c.table, style)
-	for token := it(); token != nil; token = it() {
+	for token := it(); token != chroma.EOF; token = it() {
 		// TODO: Cache token lookups?
 		clr, ok := theme[token.Type]
 		if !ok {

formatters/tty_truecolour.go

@@ -11,7 +11,7 @@ import (
 var TTY16m = Register("terminal16m", chroma.FormatterFunc(trueColourFormatter))
 
 func trueColourFormatter(w io.Writer, style *chroma.Style, it chroma.Iterator) error {
-	for token := it(); token != nil; token = it() {
+	for token := it(); token != chroma.EOF; token = it() {
 		entry := style.Get(token.Type)
 		if !entry.IsZero() {
 			out := ""

iterator.go

@@ -5,36 +5,36 @@ package chroma
 // nil will be returned at the end of the Token stream.
 //
 // If an error occurs within an Iterator, it may propagate this in a panic. Formatters should recover.
-type Iterator func() *Token
+type Iterator func() Token
 
 // Tokens consumes all tokens from the iterator and returns them as a slice.
-func (i Iterator) Tokens() []*Token {
-	out := []*Token{}
-	for t := i(); t != nil; t = i() {
+func (i Iterator) Tokens() []Token {
+	var out []Token
+	for t := i(); t != EOF; t = i() {
 		out = append(out, t)
 	}
 	return out
 }
 
 // Concaterator concatenates tokens from a series of iterators.
 func Concaterator(iterators ...Iterator) Iterator {
-	return func() *Token {
+	return func() Token {
 		for len(iterators) > 0 {
 			t := iterators[0]()
-			if t != nil {
+			if t != EOF {
 				return t
 			}
 			iterators = iterators[1:]
 		}
-		return nil
+		return EOF
 	}
 }
 
 // Literator converts a sequence of literal Tokens into an Iterator.
-func Literator(tokens ...*Token) Iterator {
-	return func() (out *Token) {
+func Literator(tokens ...Token) Iterator {
+	return func() Token {
 		if len(tokens) == 0 {
-			return nil
+			return EOF
 		}
 		token := tokens[0]
 		tokens = tokens[1:]

lexer.go

@@ -66,12 +66,12 @@ type Token struct {
 func (t *Token) String() string   { return t.Value }
 func (t *Token) GoString() string { return fmt.Sprintf("&Token{%s, %q}", t.Type, t.Value) }
 
-func (t *Token) Clone() *Token {
-	clone := &Token{}
-	*clone = *t
-	return clone
+func (t *Token) Clone() Token {
+	return *t
 }
 
+var EOF Token
+
 type TokeniseOptions struct {
 	// State to start tokenisation in. Defaults to "root".
 	State string

lexer_test.go

@@ -35,7 +35,7 @@ func TestSimpleLexer(t *testing.T) {
 	a = 10
 `)
 	assert.NoError(t, err)
-	expected := []*Token{
+	expected := []Token{
 		{Whitespace, "\n\t"},
 		{Comment, "; this is a comment"},
 		{Whitespace, "\n\t"},