diff --git a/examples/aggregation.rs b/examples/aggregation.rs
index eb03e7815e..7397f9aa9b 100644
--- a/examples/aggregation.rs
+++ b/examples/aggregation.rs
@@ -37,7 +37,7 @@ fn main() -> tantivy::Result<()> {
.set_index_option(IndexRecordOption::WithFreqs)
.set_tokenizer("raw"),
)
- .set_fast(None)
+ .set_fast("default")
.set_stored();
schema_builder.add_text_field("category", text_fieldtype);
schema_builder.add_f64_field("stock", FAST);
diff --git a/src/aggregation/bucket/term_agg.rs b/src/aggregation/bucket/term_agg.rs
index adfa01673e..5a9ae5e7aa 100644
--- a/src/aggregation/bucket/term_agg.rs
+++ b/src/aggregation/bucket/term_agg.rs
@@ -1293,13 +1293,13 @@ mod tests {
// searching for terma, but min_doc_count will return all terms
let res = exec_request_with_query(agg_req, &index, Some(("string2", "hit")))?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "A");
+ assert_eq!(res["my_texts"]["buckets"][0]["key"], "a");
assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);
assert_eq!(
res["my_texts"]["buckets"][0]["elhistogram"]["buckets"],
json!([{ "doc_count": 1, "key": 1.0 }, { "doc_count": 1, "key": 2.0 } ])
);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "B");
+ assert_eq!(res["my_texts"]["buckets"][1]["key"], "b");
assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 1);
assert_eq!(
res["my_texts"]["buckets"][1]["elhistogram"]["buckets"],
@@ -1421,10 +1421,10 @@ mod tests {
let res = exec_request_with_query(agg_req, &index, None).unwrap();
println!("{}", serde_json::to_string_pretty(&res).unwrap());
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Hallo Hallo");
+ assert_eq!(res["my_texts"]["buckets"][0]["key"], "hallo hallo");
assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 1);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "Hello Hello");
+ assert_eq!(res["my_texts"]["buckets"][1]["key"], "hello hello");
assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 1);
Ok(())
diff --git a/src/aggregation/mod.rs b/src/aggregation/mod.rs
index 93116e0d9c..fc09e15bb8 100644
--- a/src/aggregation/mod.rs
+++ b/src/aggregation/mod.rs
@@ -411,7 +411,7 @@ mod tests {
.set_index_option(IndexRecordOption::Basic)
.set_fieldnorms(false),
)
- .set_fast(None)
+ .set_fast("default")
.set_stored();
let text_field = schema_builder.add_text_field("text", text_fieldtype.clone());
let text_field_id = schema_builder.add_text_field("text_id", text_fieldtype);
@@ -466,7 +466,7 @@ mod tests {
.set_indexing_options(
TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),
)
- .set_fast(None)
+ .set_fast("default")
.set_stored();
let text_field = schema_builder.add_text_field("text", text_fieldtype);
let date_field = schema_builder.add_date_field("date", FAST);
diff --git a/src/core/index.rs b/src/core/index.rs
index dcc49b9022..bd41bbf3b3 100644
--- a/src/core/index.rs
+++ b/src/core/index.rs
@@ -120,8 +120,8 @@ impl IndexBuilder {
Self {
schema: None,
index_settings: IndexSettings::default(),
- tokenizer_manager: TokenizerManager::default(),
- fast_field_tokenizer_manager: TokenizerManager::default(),
+ tokenizer_manager: TokenizerManager::default_for_indexing(),
+ fast_field_tokenizer_manager: TokenizerManager::default_for_fast_fields(),
}
}
@@ -400,8 +400,8 @@ impl Index {
settings: metas.index_settings.clone(),
directory,
schema,
- tokenizers: TokenizerManager::default(),
- fast_field_tokenizers: TokenizerManager::default(),
+ tokenizers: TokenizerManager::default_for_indexing(),
+ fast_field_tokenizers: TokenizerManager::default_for_fast_fields(),
executor: Arc::new(Executor::single_thread()),
inventory,
}
diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs
index d450e3e593..beb0da1a6f 100644
--- a/src/fastfield/mod.rs
+++ b/src/fastfield/mod.rs
@@ -446,7 +446,8 @@ mod tests {
#[test]
fn test_text_fastfield() {
let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("text", TEXT | FAST);
+ let text_options: TextOptions = TextOptions::from(TEXT).set_fast("raw");
+ let text_field = schema_builder.add_text_field("text", text_options);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
@@ -1082,7 +1083,7 @@ mod tests {
#[test]
fn test_fast_field_in_json_field_expand_dots_disabled() {
let mut schema_builder = Schema::builder();
- let json_option = JsonObjectOptions::default().set_fast(None);
+ let json_option = JsonObjectOptions::default().set_fast("default");
let json = schema_builder.add_json_field("json", json_option);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
@@ -1108,7 +1109,7 @@ mod tests {
#[test]
fn test_fast_field_in_json_field_with_tokenizer() {
let mut schema_builder = Schema::builder();
- let json_option = JsonObjectOptions::default().set_fast(Some("default"));
+ let json_option = JsonObjectOptions::default().set_fast("default");
let json = schema_builder.add_json_field("json", json_option);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
@@ -1134,7 +1135,7 @@ mod tests {
fn test_fast_field_in_json_field_expand_dots_enabled() {
let mut schema_builder = Schema::builder();
let json_option = JsonObjectOptions::default()
- .set_fast(None)
+ .set_fast("default")
.set_expand_dots_enabled();
let json = schema_builder.add_json_field("json", json_option);
let schema = schema_builder.build();
@@ -1202,10 +1203,10 @@ mod tests {
#[test]
fn test_fast_field_tokenizer() {
let mut schema_builder = Schema::builder();
- let opt = TextOptions::default().set_fast(Some("custom_lowercase"));
+ let opt = TextOptions::default().set_fast("custom_lowercase");
let text_field = schema_builder.add_text_field("text", opt);
let schema = schema_builder.build();
- let ff_tokenizer_manager = TokenizerManager::default();
+ let ff_tokenizer_manager = TokenizerManager::default_for_fast_fields();
ff_tokenizer_manager.register(
"custom_lowercase",
TextAnalyzer::builder(RawTokenizer::default())
@@ -1238,7 +1239,7 @@ mod tests {
.set_index_option(crate::schema::IndexRecordOption::WithFreqs)
.set_tokenizer("raw"),
)
- .set_fast(Some("default"))
+ .set_fast("default")
.set_stored();
let log_field = schema_builder.add_text_field("log_level", text_fieldtype);
@@ -1271,7 +1272,7 @@ mod tests {
fn test_shadowing_fast_field_with_expand_dots() {
let mut schema_builder = Schema::builder();
let json_option = JsonObjectOptions::default()
- .set_fast(None)
+ .set_fast("default")
.set_expand_dots_enabled();
let json_field = schema_builder.add_json_field("jsonfield", json_option.clone());
let shadowing_json_field = schema_builder.add_json_field("jsonfield.attr", json_option);
diff --git a/src/fastfield/readers.rs b/src/fastfield/readers.rs
index c8e8b1d4a1..6495e39de3 100644
--- a/src/fastfield/readers.rs
+++ b/src/fastfield/readers.rs
@@ -349,7 +349,7 @@ mod tests {
schema_builder.add_json_field(
"json_expand_dots_enabled",
JsonObjectOptions::default()
- .set_fast(None)
+ .set_fast("default")
.set_expand_dots_enabled(),
);
let dynamic_field = schema_builder.add_json_field("_dyna", FAST);
diff --git a/src/fastfield/writer.rs b/src/fastfield/writer.rs
index 6389ce8a2c..98bd087589 100644
--- a/src/fastfield/writer.rs
+++ b/src/fastfield/writer.rs
@@ -18,6 +18,8 @@ const JSON_DEPTH_LIMIT: usize = 20;
pub struct FastFieldsWriter {
columnar_writer: ColumnarWriter,
fast_field_names: Vec