diff --git a/src/main/java/com/github/musiKk/Tokenizer.java b/src/main/java/com/github/musiKk/Tokenizer.java index 9302bfe..851eef5 100644 --- a/src/main/java/com/github/musiKk/Tokenizer.java +++ b/src/main/java/com/github/musiKk/Tokenizer.java @@ -76,15 +76,20 @@ interface Pattern { static class StaticPattern implements Pattern { String pattern; TokenType tokenType; + boolean isWord; public StaticPattern(String pattern, TokenType tokenType) { this.pattern = pattern; this.tokenType = tokenType; + this.isWord = pattern.matches("\\w+"); } @Override public Optional match(String programString, int index) { if (programString.startsWith(pattern, index)) { + if (isWord && index + pattern.length() < programString.length() && Character.isJavaIdentifierPart(programString.charAt(index + pattern.length()))) { + return Optional.empty(); + } return Optional.of(new Token(tokenType, pattern, index, index + pattern.length())); } else { return Optional.empty();