From 3aaf3e542f6703b0811a248571839856d4a6de03 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Tue, 5 Nov 2019 13:54:35 +1100 Subject: [PATCH] Fix Java lexer (synced with latest Pygments). Fixes #297. --- go.mod | 3 +- lexers/c/cpp_test.go | 4 +- lexers/j/java.go | 7 +- lexers/testdata/cql.expected | 12 +- lexers/testdata/java.actual | 26 ++++ lexers/testdata/java.expected | 195 ++++++++++++++++++++++++++++++ lexers/testdata/markdown.expected | 12 +- regexp_test.go | 3 +- 8 files changed, 240 insertions(+), 22 deletions(-) create mode 100644 lexers/testdata/java.actual create mode 100644 lexers/testdata/java.expected diff --git a/go.mod b/go.mod index fdacf0782..2add0a3f3 100644 --- a/go.mod +++ b/go.mod @@ -14,9 +14,8 @@ require ( github.com/gorilla/mux v1.7.3 github.com/mattn/go-colorable v0.0.9 github.com/mattn/go-isatty v0.0.4 - github.com/pkg/errors v0.8.1 github.com/sergi/go-diff v1.0.0 // indirect - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.3.0 // indirect golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect ) diff --git a/lexers/c/cpp_test.go b/lexers/c/cpp_test.go index 54ace67c4..8f1b0b3bb 100644 --- a/lexers/c/cpp_test.go +++ b/lexers/c/cpp_test.go @@ -3,7 +3,7 @@ package c_test import ( "testing" - "github.com/stretchr/testify/require" + "github.com/alecthomas/assert" "github.com/alecthomas/chroma" "github.com/alecthomas/chroma/lexers/c" @@ -17,7 +17,7 @@ double c = 0011111111010011001100110011001100110011001100110011001100110011; double a + b = 0011111111010011001100110011001100110011001100110011001100110100; // Note that this is not quite equal to the "canonical" 0.3!a ` it, err := c.CPP.Tokenise(nil, input) - require.NoError(t, err) + assert.NoError(t, err) for { token := it() if token == chroma.EOF { diff --git a/lexers/j/java.go b/lexers/j/java.go index d9beb46ec..c6b9a762e 100644 --- a/lexers/j/java.go +++ b/lexers/j/java.go @@ -20,7 +20,7 @@ var Java = internal.Register(MustNewLexer( {`//.*?\n`, CommentSingle, nil}, {`/\*.*?\*/`, CommentMultiline, nil}, {`(assert|break|case|catch|continue|default|do|else|finally|for|if|goto|instanceof|new|return|switch|this|throw|try|while)\b`, Keyword, nil}, - {`((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)((?:[^\W\d]|\$)[\w$]*)(\s*)`, ByGroups(UsingSelf("root"), NameFunction, Text, Operator), nil}, + {`((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)((?:[^\W\d]|\$)[\w$]*)(\s*)(\()`, ByGroups(UsingSelf("root"), NameFunction, Text, Operator), nil}, {`@[^\W\d][\w.]*`, NameDecorator, nil}, {`(abstract|const|enum|extends|final|implements|native|private|protected|public|static|strictfp|super|synchronized|throws|transient|volatile)\b`, KeywordDeclaration, nil}, {`(boolean|byte|char|double|float|int|long|short|void)\b`, KeywordType, nil}, @@ -30,7 +30,7 @@ var Java = internal.Register(MustNewLexer( {`(import(?:\s+static)?)(\s+)`, ByGroups(KeywordNamespace, Text), Push("import")}, {`"(\\\\|\\"|[^"])*"`, LiteralString, nil}, {`'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'`, LiteralStringChar, nil}, - {`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Punctuation, NameAttribute), nil}, + {`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Operator, NameAttribute), nil}, {`^\s*([^\W\d]|\$)[\w$]*:`, NameLabel, nil}, {`([^\W\d]|\$)[\w$]*`, Name, nil}, {`([0-9][0-9_]*\.([0-9][0-9_]*)?|\.[0-9][0-9_]*)([eE][+\-]?[0-9][0-9_]*)?[fFdD]?|[0-9][eE][+\-]?[0-9][0-9_]*[fFdD]?|[0-9]([eE][+\-]?[0-9][0-9_]*)?[fFdD]|0[xX]([0-9a-fA-F][0-9a-fA-F_]*\.?|([0-9a-fA-F][0-9a-fA-F_]*)?\.[0-9a-fA-F][0-9a-fA-F_]*)[pP][+\-]?[0-9][0-9_]*[fFdD]?`, LiteralNumberFloat, nil}, @@ -38,8 +38,7 @@ var Java = internal.Register(MustNewLexer( {`0[bB][01][01_]*[lL]?`, LiteralNumberBin, nil}, {`0[0-7_]+[lL]?`, LiteralNumberOct, nil}, {`0|[1-9][0-9_]*[lL]?`, LiteralNumberInteger, nil}, - {`[~^*!%&<>|+=:/?-]`, Operator, nil}, - {`[\[\](){};,.]`, Punctuation, nil}, + {`[~^*!%&\[\](){}<>|+=:;,./?-]`, Operator, nil}, {`\n`, Text, nil}, }, "class": { diff --git a/lexers/testdata/cql.expected b/lexers/testdata/cql.expected index 3ad730de5..bc5f84984 100644 --- a/lexers/testdata/cql.expected +++ b/lexers/testdata/cql.expected @@ -1068,17 +1068,17 @@ {"type":"Keyword","value":"return"}, {"type":"Text","value":" "}, {"type":"Name","value":"Double"}, - {"type":"Punctuation","value":"."}, + {"type":"Operator","value":"."}, {"type":"NameAttribute","value":"valueOf"}, - {"type":"Punctuation","value":"("}, + {"type":"Operator","value":"("}, {"type":"Name","value":"Math"}, - {"type":"Punctuation","value":"."}, + {"type":"Operator","value":"."}, {"type":"NameAttribute","value":"log"}, - {"type":"Punctuation","value":"("}, + {"type":"Operator","value":"("}, {"type":"Name","value":"input"}, - {"type":"Punctuation","value":"."}, + {"type":"Operator","value":"."}, {"type":"NameAttribute","value":"doubleValue"}, - {"type":"Punctuation","value":"()));"}, + {"type":"Operator","value":"()));"}, {"type":"LiteralStringHeredoc","value":"'"}, {"type":"Punctuation","value":";"}, {"type":"TextWhitespace","value":"\n"} diff --git a/lexers/testdata/java.actual b/lexers/testdata/java.actual new file mode 100644 index 000000000..aed93a8c4 --- /dev/null +++ b/lexers/testdata/java.actual @@ -0,0 +1,26 @@ +package example; + +import com.oracle.svm.core.annotate.Alias; +import com.oracle.svm.core.annotate.RecomputeFieldValue; +import com.oracle.svm.core.annotate.Targetclass; + +@TargetClass(className = "io.netty.util.internal.cleanerJava6") +final class TargetCleanerJava6 { + @Alias + @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Fieldoffset, declClassName = "java.nio.DirectBy") + private static long CLEANER_FIELD_OFFSET; +} + +@TargetClass(className = "io.netty.util.internal.PlatformDependent0") +final class TargetPlatformDependent0 { + @Alias + @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Fieldoffset, declClassName = "java.nio.Buffer", rest = "IDK") + private static long ADDRESS_FIELD_OFFSET; +} + +@TargetClass(io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess.class) +final class TargetUnsafeRefArrayAccess { + @Alias + @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayIndexShift, declClass = Object[].class) + public static int REF_ELEMENT_SHIFT; +} diff --git a/lexers/testdata/java.expected b/lexers/testdata/java.expected new file mode 100644 index 000000000..162983fb0 --- /dev/null +++ b/lexers/testdata/java.expected @@ -0,0 +1,195 @@ +[ + {"type":"KeywordNamespace","value":"package"}, + {"type":"Text","value":" "}, + {"type":"NameNamespace","value":"example"}, + {"type":"Operator","value":";"}, + {"type":"Text","value":"\n\n"}, + {"type":"KeywordNamespace","value":"import"}, + {"type":"Text","value":" "}, + {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Alias"}, + {"type":"Operator","value":";"}, + {"type":"Text","value":"\n"}, + {"type":"KeywordNamespace","value":"import"}, + {"type":"Text","value":" "}, + {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.RecomputeFieldValue"}, + {"type":"Operator","value":";"}, + {"type":"Text","value":"\n"}, + {"type":"KeywordNamespace","value":"import"}, + {"type":"Text","value":" "}, + {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Targetclass"}, + {"type":"Operator","value":";"}, + {"type":"Text","value":"\n\n"}, + {"type":"NameDecorator","value":"@TargetClass"}, + {"type":"Operator","value":"("}, + {"type":"Name","value":"className"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"\"io.netty.util.internal.cleanerJava6\""}, + {"type":"Operator","value":")"}, + {"type":"Text","value":"\n"}, + {"type":"KeywordDeclaration","value":"final"}, + {"type":"Text","value":" "}, + {"type":"KeywordDeclaration","value":"class"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"TargetCleanerJava6"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"NameDecorator","value":"@Alias"}, + {"type":"Text","value":"\n "}, + {"type":"NameDecorator","value":"@RecomputeFieldValue"}, + {"type":"Operator","value":"("}, + {"type":"Name","value":"kind"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"RecomputeFieldValue"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"Kind"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"Fieldoffset"}, + {"type":"Operator","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"declClassName"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"\"java.nio.DirectBy\""}, + {"type":"Operator","value":")"}, + {"type":"Text","value":"\n "}, + {"type":"KeywordDeclaration","value":"private"}, + {"type":"Text","value":" "}, + {"type":"KeywordDeclaration","value":"static"}, + {"type":"Text","value":" "}, + {"type":"KeywordType","value":"long"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"CLEANER_FIELD_OFFSET"}, + {"type":"Operator","value":";"}, + {"type":"Text","value":"\n"}, + {"type":"Operator","value":"}"}, + {"type":"Text","value":"\n\n"}, + {"type":"NameDecorator","value":"@TargetClass"}, + {"type":"Operator","value":"("}, + {"type":"Name","value":"className"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"\"io.netty.util.internal.PlatformDependent0\""}, + {"type":"Operator","value":")"}, + {"type":"Text","value":"\n"}, + {"type":"KeywordDeclaration","value":"final"}, + {"type":"Text","value":" "}, + {"type":"KeywordDeclaration","value":"class"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"TargetPlatformDependent0"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"NameDecorator","value":"@Alias"}, + {"type":"Text","value":"\n "}, + {"type":"NameDecorator","value":"@RecomputeFieldValue"}, + {"type":"Operator","value":"("}, + {"type":"Name","value":"kind"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"RecomputeFieldValue"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"Kind"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"Fieldoffset"}, + {"type":"Operator","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"declClassName"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"\"java.nio.Buffer\""}, + {"type":"Operator","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"rest"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"\"IDK\""}, + {"type":"Operator","value":")"}, + {"type":"Text","value":"\n "}, + {"type":"KeywordDeclaration","value":"private"}, + {"type":"Text","value":" "}, + {"type":"KeywordDeclaration","value":"static"}, + {"type":"Text","value":" "}, + {"type":"KeywordType","value":"long"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"ADDRESS_FIELD_OFFSET"}, + {"type":"Operator","value":";"}, + {"type":"Text","value":"\n"}, + {"type":"Operator","value":"}"}, + {"type":"Text","value":"\n\n"}, + {"type":"NameDecorator","value":"@TargetClass"}, + {"type":"Operator","value":"("}, + {"type":"Name","value":"io"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"netty"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"util"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"internal"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"shaded"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"org"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"jctools"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"util"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"UnsafeRefArrayAccess"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"class"}, + {"type":"Operator","value":")"}, + {"type":"Text","value":"\n"}, + {"type":"KeywordDeclaration","value":"final"}, + {"type":"Text","value":" "}, + {"type":"KeywordDeclaration","value":"class"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"TargetUnsafeRefArrayAccess"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"NameDecorator","value":"@Alias"}, + {"type":"Text","value":"\n "}, + {"type":"NameDecorator","value":"@RecomputeFieldValue"}, + {"type":"Operator","value":"("}, + {"type":"Name","value":"kind"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"RecomputeFieldValue"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"Kind"}, + {"type":"Operator","value":"."}, + {"type":"NameAttribute","value":"ArrayIndexShift"}, + {"type":"Operator","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"declClass"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Object"}, + {"type":"Operator","value":"[]."}, + {"type":"NameAttribute","value":"class"}, + {"type":"Operator","value":")"}, + {"type":"Text","value":"\n "}, + {"type":"KeywordDeclaration","value":"public"}, + {"type":"Text","value":" "}, + {"type":"KeywordDeclaration","value":"static"}, + {"type":"Text","value":" "}, + {"type":"KeywordType","value":"int"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"REF_ELEMENT_SHIFT"}, + {"type":"Operator","value":";"}, + {"type":"Text","value":"\n"}, + {"type":"Operator","value":"}"}, + {"type":"Text","value":"\n"} +] diff --git a/lexers/testdata/markdown.expected b/lexers/testdata/markdown.expected index 9c62d3c52..1d39de5fc 100644 --- a/lexers/testdata/markdown.expected +++ b/lexers/testdata/markdown.expected @@ -119,17 +119,17 @@ {"type":"Keyword","value":"return"}, {"type":"Text","value":" "}, {"type":"Name","value":"Double"}, - {"type":"Punctuation","value":"."}, + {"type":"Operator","value":"."}, {"type":"NameAttribute","value":"valueOf"}, - {"type":"Punctuation","value":"("}, + {"type":"Operator","value":"("}, {"type":"Name","value":"Math"}, - {"type":"Punctuation","value":"."}, + {"type":"Operator","value":"."}, {"type":"NameAttribute","value":"log"}, - {"type":"Punctuation","value":"("}, + {"type":"Operator","value":"("}, {"type":"Name","value":"input"}, - {"type":"Punctuation","value":"."}, + {"type":"Operator","value":"."}, {"type":"NameAttribute","value":"doubleValue"}, - {"type":"Punctuation","value":"()));"}, + {"type":"Operator","value":"()));"}, {"type":"LiteralStringHeredoc","value":"'"}, {"type":"Punctuation","value":";"}, {"type":"TextWhitespace","value":"\n"}, diff --git a/regexp_test.go b/regexp_test.go index 463457a22..0ac7715e3 100644 --- a/regexp_test.go +++ b/regexp_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/alecthomas/assert" - "github.com/stretchr/testify/require" ) func TestNewlineAtEndOfFile(t *testing.T) { @@ -40,7 +39,7 @@ func TestMatchingAtStart(t *testing.T) { })) it, err := l.Tokenise(nil, `-module ->`) assert.NoError(t, err) - require.Equal(t, + assert.Equal(t, []Token{{Punctuation, "-"}, {NameEntity, "module"}, {Whitespace, " "}, {Operator, "->"}}, it.Tokens()) }