Fix incorrect token edges with extra whitespace

Author: mgod

library/build.gradle

@@ -30,6 +30,8 @@ task jar(type: Jar) {
 }
 
 dependencies {
+    testCompile 'junit:junit:4.12'
+
     compile 'com.android.support:support-annotations:25.3.1'
 }
 

library/src/main/java/com/tokenautocomplete/CharacterTokenizer.java

@@ -28,7 +28,7 @@ public int findTokenStart(CharSequence text, int cursor) {
         while (i > 0 && !splitChar.contains(text.charAt(i - 1))) {
             i--;
         }
-        while (i < cursor && text.charAt(i) == ' ') {
+        while (i > 0 && text.charAt(i) == ' ') {
             i++;
         }
 

library/src/test/java/com/tokenautocomplete/CharacterTokenizerTest.java

@@ -0,0 +1,71 @@
+package com.tokenautocomplete;
+
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+/**
+ * Make sure the tokenizer finds the right boundaries
+ *
+ * Created by mgod on 8/24/17.
+ */
+
+public class CharacterTokenizerTest {
+
+    @Test
+    public void handleWhiteSpaceWithCommaTokens() {
+        char[] splits = {','};
+        CharacterTokenizer tokenizer = new CharacterTokenizer(splits);
+        String text = "bears, ponies";
+        assertEquals(0, tokenizer.findTokenStart(text, 0));
+        assertEquals(0, tokenizer.findTokenStart(text, 1));
+        assertEquals(0, tokenizer.findTokenStart(text, 5));
+        assertEquals(7, tokenizer.findTokenStart(text, 6));
+        assertEquals(7, tokenizer.findTokenStart(text, 7));
+        assertEquals(7, tokenizer.findTokenStart(text, 11));
+
+        assertEquals(5, tokenizer.findTokenEnd(text, 0));
+        assertEquals(5, tokenizer.findTokenEnd(text, 4));
+        assertEquals(5, tokenizer.findTokenEnd(text, 5));
+        assertEquals(13, tokenizer.findTokenEnd(text, 6));
+        assertEquals(13, tokenizer.findTokenEnd(text, 10));
+        assertEquals(13, tokenizer.findTokenEnd(text, 11));
+    }
+
+    @Test
+    public void handleWhiteSpaceWithWhitespaceTokens() {
+        char[] splits = {' '};
+        CharacterTokenizer tokenizer = new CharacterTokenizer(splits);
+        String text = "bears ponies";
+        assertEquals(0, tokenizer.findTokenStart(text, 0));
+        assertEquals(0, tokenizer.findTokenStart(text, 1));
+        assertEquals(0, tokenizer.findTokenStart(text, 5));
+        assertEquals(6, tokenizer.findTokenStart(text, 6));
+        assertEquals(6, tokenizer.findTokenStart(text, 7));
+        assertEquals(6, tokenizer.findTokenStart(text, 11));
+
+        assertEquals(5, tokenizer.findTokenEnd(text, 0));
+        assertEquals(5, tokenizer.findTokenEnd(text, 4));
+        assertEquals(5, tokenizer.findTokenEnd(text, 5));
+        assertEquals(12, tokenizer.findTokenEnd(text, 6));
+        assertEquals(12, tokenizer.findTokenEnd(text, 10));
+        assertEquals(12, tokenizer.findTokenEnd(text, 11));
+    }
+
+    @Test
+    public void handleLotsOfWhitespace() {
+        char[] splits = {','};
+        CharacterTokenizer tokenizer = new CharacterTokenizer(splits);
+        String text = "bears,      ponies     ,another";
+        assertEquals(12, tokenizer.findTokenStart(text, 6));
+        assertEquals(12, tokenizer.findTokenStart(text, 7));
+        assertEquals(12, tokenizer.findTokenStart(text, 18));
+        assertEquals(12, tokenizer.findTokenStart(text, 23));
+        assertEquals(24, tokenizer.findTokenStart(text, 24));
+
+        assertEquals(23, tokenizer.findTokenEnd(text, 6));
+        assertEquals(23, tokenizer.findTokenEnd(text, 7));
+        assertEquals(23, tokenizer.findTokenEnd(text, 18));
+        assertEquals(23, tokenizer.findTokenEnd(text, 23));
+        assertEquals(31, tokenizer.findTokenEnd(text, 24));
+    }
+}