Merge pull request #912 from S2P2/fix-join-broken-num

Fix empty string ('') added (in some cases) when using word_tokenize with join_broken_num=True
PyThaiNLP · May 11, 2024 · fd4175e · fd4175e
2 parents a38fd5e + dcd2b47
commit fd4175e
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py
@@ -61,8 +61,8 @@ def rejoin_formatted_num(segments: List[str]) -> List[str]:
                 connected_token += segments[segment_idx]
                 pos += len(segments[segment_idx])
                 segment_idx += 1
-
-            tokens_joined.append(connected_token)
+            if connected_token:
+                tokens_joined.append(connected_token)
             match = next(matching_results, None)
         else:
             tokens_joined.append(token)