Skip stemming strings with non-ascii chars

wltsmrz · Nov 16, 2020 · d6e39c0 · d6e39c0
1 parent d4ff3ad
commit d6e39c0
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/src/stemmer.carp b/src/stemmer.carp
@@ -5,7 +5,7 @@
   (register to-lower (Fn [&String] String) "String_to_lower_")
 
   (defn trim-from [x j]
-     (String.slice x 0 (+ (utf8-length x) j)))
+     (String.slice x 0 (+ (String.length x) j)))
 
   (defn replace-from [x j r]
     (String.append &(trim-from x j) r))
@@ -219,7 +219,7 @@
     x))
 
   (defn stem [x]
-    (cond (< (String.length x) 3)
+    (cond (or (< (String.length x) 3) (not (= (String.length x) (utf8-length x))))
       @x
       (=> x
         (to-lower)
@@ -234,5 +234,5 @@
   )))
 
   (defn stem-cstr [x]
-    (cstr &(stem &(String.from-cstr x))))
+    (String.cstr &(stem &(String.from-cstr x))))
 )
diff --git a/src/string_add.h b/src/string_add.h
@@ -9,4 +9,3 @@ String String_to_lower_(const String* p) {
   while (i--) *(r + i) = tolower(*(r + i));
   return r;
 }
-
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,4 +9,3 @@ String String_to_lower_(const String* p) {
		while (i--) (r + i) = tolower((r + i));
		return r;
		}