fix bug where an identifier starts with a keyword

musiKk · Jul 1, 2024 · 6a71583 · 6a71583
1 parent 6219c3a
commit 6a71583
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/src/main/java/com/github/musiKk/Tokenizer.java b/src/main/java/com/github/musiKk/Tokenizer.java
@@ -76,15 +76,20 @@ interface Pattern {
     static class StaticPattern implements Pattern {
         String pattern;
         TokenType tokenType;
+        boolean isWord;
 
         public StaticPattern(String pattern, TokenType tokenType) {
             this.pattern = pattern;
             this.tokenType = tokenType;
+            this.isWord = pattern.matches("\\w+");
         }
 
         @Override
         public Optional<Token> match(String programString, int index) {
             if (programString.startsWith(pattern, index)) {
+                if (isWord && index + pattern.length() < programString.length() && Character.isJavaIdentifierPart(programString.charAt(index + pattern.length()))) {
+                    return Optional.empty();
+                }
                 return Optional.of(new Token(tokenType, pattern, index, index + pattern.length()));
             } else {
                 return Optional.empty();