feat: [sc-47267] Add LowerCaser to kapiche_tokenizer (quickwit-oss#193)

Sidhant29 · Mar 12, 2024 · a671ba8 · a671ba8
1 parent a096a83
commit a671ba8
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/src/index.rs b/src/index.rs
@@ -302,6 +302,7 @@ pub(crate) struct Index {
 // and a PossessiveContractionFilter.
 fn get_kapiche_tokenizer() -> TextAnalyzer {
     TextAnalyzer::builder(WhitespaceTokenizer::default())
+        .filter(LowerCaser)
         .filter(OuterPunctuationFilter::new(vec!['#', '@']))
         .filter(PossessiveContractionFilter)
         .build()

diff --git a/tests/test_stat_collector.py b/tests/test_stat_collector.py
@@ -154,5 +154,5 @@ def test_stat_searcher_memory():
 
     result = index.stat_searcher().search(query)
     items = sorted(result.unique_docs_frames)
-    assert len(items) == 439
-    assert items[:4] == [(0, 0), (2, 2), (11, 11), (18, 18)]
+    assert len(items) == 441
+    assert items[:4] == [(0, 0), (2, 2), (8, 8), (11, 11)]