parser: allow no space between some tokens

src/lang/clojure-language.kt

@@ -114,7 +114,8 @@ object ClojureTokens {
   @JvmField val WHITESPACES = TokenSet.create(C_COMMA, TokenType.WHITE_SPACE)
   @JvmField val COMMENTS = TokenSet.create(LINE_COMMENT)
   @JvmField val STRINGS = TokenSet.create(C_STRING)
-  @JvmField val LITERALS = TokenSet.create(C_BOOL, C_CHAR, C_HEXNUM, C_NIL, C_NUMBER, C_RATIO, C_RDXNUM, C_STRING, C_SYM)
+  @JvmField val SYM_ALIKE = TokenSet.create(C_BOOL, C_NIL, C_SYM)
+  @JvmField val LITERALS = TokenSet.create(C_BOOL, C_CHAR, C_HEXNUM, C_NIL, C_NUMBER, C_RATIO, C_RDXNUM, C_STRING)
 
   @JvmField val SHARPS = TokenSet.create(C_SHARP, C_SHARP_COMMENT, C_SHARP_QMARK, C_SHARP_QMARK_AT, C_SHARP_EQ, C_SHARP_HAT,
       C_SHARP_QUOTE, C_SHARP_NS, C_SHARP_SYM)

src/lang/clojure-parser.kt

@@ -43,8 +43,9 @@ class ClojureLexer(language: Language) : LookAheadLexer(FlexAdapter(_ClojureLexe
     when (tokenType0) {
       in ClojureTokens.LITERALS -> {
         baseLexer.advance()
-        if (baseLexer.tokenType === C_SYM ||
-            baseLexer.tokenType in ClojureTokens.LITERALS) {
+        val tokenType = baseLexer.tokenType
+        if (tokenType0 == C_NUMBER && ClojureTokens.SYM_ALIKE.contains(tokenType) ||
+            tokenType0 == C_CHAR && (tokenType == C_SYM || ClojureTokens.LITERALS.contains(tokenType))) {
           advanceAs(baseLexer, TokenType.BAD_CHARACTER)
         }
         else {
@@ -105,36 +106,42 @@ abstract class ClojureParserDefinitionBase : ParserDefinition {
 class ClojureParserUtil {
   @Suppress("UNUSED_PARAMETER")
   companion object {
-    @JvmStatic fun adapt_builder_(root: IElementType, builder: PsiBuilder, parser: PsiParser, extendsSets: Array<TokenSet>?): PsiBuilder =
+    @JvmStatic
+    fun adapt_builder_(root: IElementType, builder: PsiBuilder, parser: PsiParser, extendsSets: Array<TokenSet>?): PsiBuilder =
         GeneratedParserUtilBase.adapt_builder_(root, builder, parser, extendsSets).apply {
           (this as? GeneratedParserUtilBase.Builder)?.state?.braces = null
         }
 
-    @JvmStatic fun parseTree(b: PsiBuilder, l: Int, p: GeneratedParserUtilBase.Parser) =
+    @JvmStatic
+    fun parseTree(b: PsiBuilder, l: Int, p: GeneratedParserUtilBase.Parser) =
         GeneratedParserUtilBase.parseAsTree(GeneratedParserUtilBase.ErrorState.get(b), b, l,
             GeneratedParserUtilBase.DUMMY_BLOCK, false, p, GeneratedParserUtilBase.TRUE_CONDITION)
 
-    @JvmStatic fun nospace(b: PsiBuilder, l: Int): Boolean {
+    @JvmStatic
+    fun nospace(b: PsiBuilder, l: Int): Boolean {
       if (space(b, l)) {
         b.mark().apply { b.tokenType; error("no <whitespace> allowed") }
             .setCustomEdgeTokenBinders(WhitespacesBinders.GREEDY_LEFT_BINDER, WhitespacesBinders.GREEDY_RIGHT_BINDER)
       }
       return true
     }
 
-    @JvmStatic fun space(b: PsiBuilder, l: Int): Boolean {
+    @JvmStatic
+    fun space(b: PsiBuilder, l: Int): Boolean {
       return b.rawLookup(0).wsOrComment() || b.rawLookup(-1).wsOrComment()
     }
 
     private val RECOVER_SET = TokenSet.orSet(
         ClojureTokens.SHARPS, ClojureTokens.MACROS, ClojureTokens.PAREN_ALIKE, ClojureTokens.LITERALS,
-        TokenSet.create(C_DOT, C_DOTDASH))
+        TokenSet.create(C_DOT, C_DOTDASH, C_SYM))
 
-    @JvmStatic fun formRecover(b: PsiBuilder, l: Int): Boolean {
+    @JvmStatic
+    fun formRecover(b: PsiBuilder, l: Int): Boolean {
       return !RECOVER_SET.contains(b.tokenType)
     }
 
-    @JvmStatic fun rootFormRecover(b: PsiBuilder, l: Int): Boolean {
+    @JvmStatic
+    fun rootFormRecover(b: PsiBuilder, l: Int): Boolean {
       val type = b.tokenType
       return ClojureTokens.PAREN2_ALIKE.contains(type) || !RECOVER_SET.contains(type)
     }

testData/highlighting/ClojureLang.txt

@@ -13,13 +13,13 @@
       81360: dynamic 'deref'
       103977: dynamic 'nth'
       122220: dynamic 'close'
-      176293: dynamic 'offer'
-      179500: dynamic 'visitParameterAnnotation'
-      179656: dynamic 'visitAnnotation'
-      179826: dynamic 'visitEnd'
-      192141: dynamic 'hasRoot'
-      235450: dynamic 'getProperty'
-      235982: dynamic 'contains'
+      176311: dynamic 'offer'
+      179518: dynamic 'visitParameterAnnotation'
+      179674: dynamic 'visitAnnotation'
+      179844: dynamic 'visitEnd'
+      192159: dynamic 'hasRoot'
+      235552: dynamic 'getProperty'
+      236084: dynamic 'contains'
 /clojure/core/protocols.clj              0 errors, 0 warnings, 0 dynamic
 /clojure/core/reducers.clj               0 errors, 0 warnings, 0 dynamic
 /clojure/core/server.clj                 0 errors, 0 warnings, 2 dynamic
@@ -167,8 +167,8 @@
       960: dynamic 'instanceMember'
       2986: dynamic 'instanceFieldName-symbol'
       3061: dynamic 'staticFieldName-symbol'
-      10970: dynamic 'getClassName'
-      11628: dynamic 'stop'
+      10949: dynamic 'getClassName'
+      11607: dynamic 'stop'
 /clojure/set.clj                         0 errors, 0 warnings, 0 dynamic
 /clojure/stacktrace.clj                  0 errors, 0 warnings, 0 dynamic
 /clojure/string.clj                      0 errors, 0 warnings, 0 dynamic
@@ -184,4 +184,4 @@
 /clojure/walk.clj                        0 errors, 0 warnings, 0 dynamic
 /clojure/xml.clj                         0 errors, 0 warnings, 0 dynamic
 /clojure/zip.clj                         0 errors, 0 warnings, 0 dynamic
-Total: 66 warnings, 77 dynamic in 43 files (755.59 kB)
+Total: 66 warnings, 77 dynamic in 43 files (757.47 kB)

testData/highlighting/ClojureScript.txt

@@ -36,19 +36,18 @@
 /cljs/build/api.clj                      0 errors, 0 warnings, 0 dynamic
 /cljs/cli.clj                            0 errors, 0 warnings, 3 dynamic
       3124: dynamic 'length'
-      11805: dynamic 'exists'
-      12778: dynamic 'exists'
-/cljs/closure.clj                        1 errors, 9 warnings, 43 dynamic
+      11602: dynamic 'exists'
+      12525: dynamic 'exists'
+/cljs/closure.clj                        0 errors, 9 warnings, 43 dynamic
       874: unable to resolve 'clojure.reflect'
       62592: unable to resolve 'read-str'
-      68287: '"\""dep' unexpected
       69334: unable to resolve 'write-str'
-      75702: unable to resolve 'read-str'
-      77038: unable to resolve 'write-str'
-      109943: unable to resolve 'read-str'
-      112336: unable to resolve 'read-str'
-      120874: unable to resolve 'input-stream-push-back-reader'
-      120994: unable to resolve 'read'
+      75708: unable to resolve 'read-str'
+      77044: unable to resolve 'write-str'
+      110034: unable to resolve 'read-str'
+      112427: unable to resolve 'read-str'
+      120965: unable to resolve 'input-stream-push-back-reader'
+      121085: unable to resolve 'read'
       2873: dynamic 'getCanonicalPath'
       16606: dynamic 'setOptionsForCompilationLevel'
       18132: dynamic 'getFile'
@@ -58,40 +57,40 @@
       58229: dynamic 'toBytes'
       58459: dynamic 'toBytes'
       62755: dynamic 'split'
-      76111: dynamic 'split'
-      76387: dynamic 'split'
-      77343: dynamic 'endsWith'
-      80625: dynamic 'getSymbol'
-      80928: dynamic 'toSource'
-      82838: dynamic 'setDependencySorting'
-      83303: dynamic 'setProcessCommonJSModules'
-      83372: dynamic 'setLanguageIn'
-      83459: dynamic 'setLanguageOut'
-      83568: dynamic 'setDependencyOptions'
-      83656: dynamic 'setPackageJsonEntryNames'
-      83976: dynamic 'getName'
-      84808: dynamic 'toURI'
-      85096: dynamic 'getPath'
-      85293: dynamic 'setGoogModule'
-      85314: dynamic 'build'
-      85750: dynamic 'indexOf'
-      85831: dynamic 'indexOf'
-      85966: dynamic 'compile'
-      92545: dynamic 'getResources'
-      105613: dynamic 'exitValue'
-      112186: dynamic 'getAbsolutePath'
-      112649: dynamic 'getAbsolutePath'
-      114553: dynamic 'lastIndexOf'
-      122168: dynamic 'getResources'
-      135662: dynamic 'split'
-      138863: dynamic 'toURI'
-      138938: dynamic 'getFileSystem'
-      138976: dynamic 'newWatchService'
-      141045: dynamic 'poll'
-      141129: dynamic 'pollEvents'
-      142367: dynamic 'watchable'
-      146577: dynamic 'isFile'
-      146596: dynamic 'delete'
+      76117: dynamic 'split'
+      76393: dynamic 'split'
+      77349: dynamic 'endsWith'
+      80631: dynamic 'getSymbol'
+      80934: dynamic 'toSource'
+      82844: dynamic 'setDependencySorting'
+      83309: dynamic 'setProcessCommonJSModules'
+      83378: dynamic 'setLanguageIn'
+      83465: dynamic 'setLanguageOut'
+      83574: dynamic 'setDependencyOptions'
+      83662: dynamic 'setPackageJsonEntryNames'
+      83982: dynamic 'getName'
+      84814: dynamic 'toURI'
+      85102: dynamic 'getPath'
+      85299: dynamic 'setGoogModule'
+      85320: dynamic 'build'
+      85756: dynamic 'indexOf'
+      85837: dynamic 'indexOf'
+      85972: dynamic 'compile'
+      92551: dynamic 'getResources'
+      105704: dynamic 'exitValue'
+      112277: dynamic 'getAbsolutePath'
+      112740: dynamic 'getAbsolutePath'
+      114644: dynamic 'lastIndexOf'
+      122259: dynamic 'getResources'
+      135753: dynamic 'split'
+      138954: dynamic 'toURI'
+      139029: dynamic 'getFileSystem'
+      139067: dynamic 'newWatchService'
+      141136: dynamic 'poll'
+      141220: dynamic 'pollEvents'
+      142458: dynamic 'watchable'
+      146668: dynamic 'isFile'
+      146687: dynamic 'delete'
 /cljs/compiler.cljc                      0 errors, 5 warnings, 28 dynamic
       604: unable to resolve 'ensure'
       53451: unable to resolve 'write-str'
@@ -1051,11 +1050,11 @@
       45083: unable to resolve 'resolve-symbol'
       45163: unable to resolve '*data-readers*'
       45249: unable to resolve '*alias-map*'
-      62088: unable to resolve 'source-logging-push-back-reader'
-      62178: unable to resolve 'read-line'
-      62225: unable to resolve '*alias-map*'
-      62280: unable to resolve '*data-readers*'
-      62349: unable to resolve 'read'
+      62288: unable to resolve 'source-logging-push-back-reader'
+      62378: unable to resolve 'read-line'
+      62425: unable to resolve '*alias-map*'
+      62480: unable to resolve '*data-readers*'
+      62549: unable to resolve 'read'
       12888: dynamic 'startsWith'
       13506: dynamic 'getCanonicalFile'
       15814: dynamic 'startsWith'
@@ -1064,9 +1063,9 @@
       20802: dynamic 'getBytes'
       25331: dynamic 'substring'
       30362: dynamic 'getAbsolutePath'
-      52407: dynamic 'instanceMethod'
-      52456: dynamic 'instanceField'
-      64788: dynamic 'stack'
+      52607: dynamic 'instanceMethod'
+      52656: dynamic 'instanceField'
+      64988: dynamic 'stack'
 /cljs/repl.cljs                          0 errors, 1 warnings, 3 dynamic
       591: unable to resolve 'goog.string.format'
       2861: dynamic 'name'
@@ -1090,11 +1089,11 @@
       9289: dynamic 'clear'
       15609: dynamic 'isShutdown'
       15644: dynamic 'shutdownNow'
-      16663: dynamic 'stack'
+      16926: dynamic 'stack'
 /cljs/repl/node.clj                      0 errors, 3 warnings, 11 dynamic
       2211: unable to resolve 'write-str'
       3248: unable to resolve 'read-str'
-      8221: unable to resolve 'write-str'
+      8289: unable to resolve 'write-str'
       1573: dynamic 'close'
       1592: dynamic 'close'
       1612: dynamic 'close'
@@ -1104,8 +1103,8 @@
       3467: dynamic 'write'
       3531: dynamic 'flush'
       5042: dynamic 'start'
-      7984: dynamic 'require'
-      8907: dynamic 'isClosed'
+      8052: dynamic 'require'
+      8975: dynamic 'isClosed'
 /cljs/repl/reflect.clj                   0 errors, 0 warnings, 1 dynamic
       2763: dynamic 'startsWith'
 /cljs/repl/server.clj                    0 errors, 0 warnings, 19 dynamic
@@ -1338,9 +1337,9 @@
       1444: dynamic '-message'
       2005: dynamic '-message'
 /cljs/test.cljc                          0 errors, 0 warnings, 5 dynamic
-      4463: dynamic 'message'
-      7606: dynamic 'cljs$lang$var'
-      7646: dynamic 'cljs$lang$var'
+      5828: dynamic 'message'
+      9330: dynamic 'cljs$lang$var'
+      9370: dynamic 'cljs$lang$var'
 /cljs/test.cljs                          0 errors, 0 warnings, 10 dynamic
       13468: dynamic 'split'
       13881: dynamic 'split'
@@ -1401,39 +1400,39 @@
       1341: unable to resolve 'cljs.repl'
       1368: unable to resolve 'HOST'
       1399: unable to resolve 'PORT'
-      1630: dynamic 'json/serialize'
-      1759: dynamic 'push'
-      1985: dynamic 'product/SAFARI'
-      2012: dynamic 'product/CHROME'
-      2039: dynamic 'product/FIREFOX'
-      2068: dynamic 'product/IE'
-      2845: dynamic 'log'
-      4375: dynamic 'getResponseText'
-      4394: dynamic 'currentTarget'
-      4527: dynamic 'json/parse'
-      4831: dynamic 'json/parse'
-      5356: dynamic 'require__'
-      5475: dynamic 'isProvided_'
-      5596: dynamic 'writeScriptTag__'
-      6232: dynamic 'apply'
-      6269: dynamic 'shift'
-      6304: dynamic 'appendChild'
-      6353: dynamic 'createElement'
-      6778: dynamic 'writeScriptTag_'
-      6869: dynamic 'push'
-      7388: dynamic 'writeScriptTag_'
-      7597: dynamic 'require'
-      7694: dynamic 'cljsReloadAll_'
-      7761: dynamic 'cljsReloadAll__'
-      7881: dynamic 'getPathFromDeps_'
-      7954: dynamic 'written_'
-      8020: dynamic 'written_'
-      8431: dynamic 'require__'
-      8520: dynamic 'cljsReloadAll_'
-      9491: dynamic 'json/parse'
-      9681: dynamic 'json/serialize'
-      9960: dynamic 'display'
-      9971: dynamic 'style'
+      1772: dynamic 'json/serialize'
+      1901: dynamic 'push'
+      2127: dynamic 'product/SAFARI'
+      2154: dynamic 'product/CHROME'
+      2181: dynamic 'product/FIREFOX'
+      2210: dynamic 'product/IE'
+      2987: dynamic 'log'
+      4475: dynamic 'getResponseText'
+      4494: dynamic 'currentTarget'
+      4627: dynamic 'json/parse'
+      4931: dynamic 'json/parse'
+      5456: dynamic 'require__'
+      5575: dynamic 'isProvided_'
+      5696: dynamic 'writeScriptTag__'
+      6332: dynamic 'apply'
+      6369: dynamic 'shift'
+      6404: dynamic 'appendChild'
+      6453: dynamic 'createElement'
+      6878: dynamic 'writeScriptTag_'
+      6969: dynamic 'push'
+      7488: dynamic 'writeScriptTag_'
+      7697: dynamic 'require'
+      7794: dynamic 'cljsReloadAll_'
+      7861: dynamic 'cljsReloadAll__'
+      7981: dynamic 'getPathFromDeps_'
+      8054: dynamic 'written_'
+      8120: dynamic 'written_'
+      8531: dynamic 'require__'
+      8620: dynamic 'cljsReloadAll_'
+      9675: dynamic 'json/parse'
+      9865: dynamic 'json/serialize'
+      10144: dynamic 'display'
+      10155: dynamic 'style'
 /clojure/browser/repl/preload.cljs       0 errors, 2 warnings, 0 dynamic
       594: unable to resolve 'HOST'
       608: unable to resolve 'PORT'
@@ -1485,4 +1484,4 @@
 /clojure/zip.cljs                        0 errors, 0 warnings, 0 dynamic
 /process/env.cljs                        0 errors, 1 warnings, 0 dynamic
       541: unable to resolve 'NODE_ENV'
-Total: 348 warnings, 1651 dynamic in 74 files (1.63 MB)
+Total: 348 warnings, 1651 dynamic in 74 files (1.64 MB)

testData/lexer/Literals.txt

@@ -100,5 +100,26 @@ WHITE_SPACE (' ')
 BAD_CHARACTER ('\uFFFFuuu')
 WHITE_SPACE (' ')
 BAD_CHARACTER ('\o1234')
+WHITE_SPACE ('\n\n')
+sym ('a')
+string ('""')
+sym ('+')
+WHITE_SPACE (' ')
+string ('""')
+sym ('a+')
+WHITE_SPACE (' ')
+string ('""')
+sym ('+a')
+WHITE_SPACE (' ')
+sym ('+')
+string ('""')
+sym ('a')
+WHITE_SPACE (' \n')
+number ('1')
+char ('\b')
 WHITE_SPACE ('\n')
+BAD_CHARACTER ('1true')
+WHITE_SPACE (' ')
+BAD_CHARACTER ('1+')
+WHITE_SPACE ('\n\n')
 BAD_CHARACTER ('"unclosed')