Settings customizing tokenization

dump/src/lib.rs

@@ -262,6 +262,9 @@ pub(crate) mod test {
             sortable_attributes: Setting::Set(btreeset! { S("age") }),
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
+            non_separator_tokens: Setting::NotSet,
+            separator_tokens: Setting::NotSet,
+            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,

dump/src/reader/compat/v5_to_v6.rs

@@ -340,6 +340,9 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 }
             },
             stop_words: settings.stop_words.into(),
+            non_separator_tokens: v6::Setting::NotSet,
+            separator_tokens: v6::Setting::NotSet,
+            dictionary: v6::Setting::NotSet,
             synonyms: settings.synonyms.into(),
             distinct_attribute: settings.distinct_attribute.into(),
             typo_tolerance: match settings.typo_tolerance {

meilisearch-types/src/error.rs

@@ -259,6 +259,9 @@ InvalidSettingsRankingRules           , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSearchableAttributes   , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSortableAttributes     , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsStopWords              , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsNonSeparatorTokens     , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSeparatorTokens        , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsDictionary             , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSynonyms               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsTypoTolerance          , InvalidRequest       , BAD_REQUEST ;
 InvalidState                          , Internal             , INTERNAL_SERVER_ERROR ;

meilisearch-types/src/settings.rs

@@ -171,6 +171,15 @@ pub struct Settings<T> {
     #[deserr(default, error = DeserrJsonError<InvalidSettingsStopWords>)]
     pub stop_words: Setting<BTreeSet<String>>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default, error = DeserrJsonError<InvalidSettingsNonSeparatorTokens>)]
+    pub non_separator_tokens: Setting<BTreeSet<String>>,
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default, error = DeserrJsonError<InvalidSettingsSeparatorTokens>)]
+    pub separator_tokens: Setting<BTreeSet<String>>,
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default, error = DeserrJsonError<InvalidSettingsDictionary>)]
+    pub dictionary: Setting<BTreeSet<String>>,
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSynonyms>)]
     pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
@@ -201,6 +210,9 @@ impl Settings<Checked> {
             ranking_rules: Setting::Reset,
             stop_words: Setting::Reset,
             synonyms: Setting::Reset,
+            non_separator_tokens: Setting::Reset,
+            separator_tokens: Setting::Reset,
+            dictionary: Setting::Reset,
             distinct_attribute: Setting::Reset,
             typo_tolerance: Setting::Reset,
             faceting: Setting::Reset,
@@ -217,6 +229,9 @@ impl Settings<Checked> {
             sortable_attributes,
             ranking_rules,
             stop_words,
+            non_separator_tokens,
+            separator_tokens,
+            dictionary,
             synonyms,
             distinct_attribute,
             typo_tolerance,
@@ -232,6 +247,9 @@ impl Settings<Checked> {
             sortable_attributes,
             ranking_rules,
             stop_words,
+            non_separator_tokens,
+            separator_tokens,
+            dictionary,
             synonyms,
             distinct_attribute,
             typo_tolerance,
@@ -274,6 +292,9 @@ impl Settings<Unchecked> {
             ranking_rules: self.ranking_rules,
             stop_words: self.stop_words,
             synonyms: self.synonyms,
+            non_separator_tokens: self.non_separator_tokens,
+            separator_tokens: self.separator_tokens,
+            dictionary: self.dictionary,
             distinct_attribute: self.distinct_attribute,
             typo_tolerance: self.typo_tolerance,
             faceting: self.faceting,
@@ -335,6 +356,28 @@ pub fn apply_settings_to_builder(
         Setting::NotSet => (),
     }
 
+    match settings.non_separator_tokens {
+        Setting::Set(ref non_separator_tokens) => {
+            builder.set_non_separator_tokens(non_separator_tokens.clone())
+        }
+        Setting::Reset => builder.reset_non_separator_tokens(),
+        Setting::NotSet => (),
+    }
+
+    match settings.separator_tokens {
+        Setting::Set(ref separator_tokens) => {
+            builder.set_separator_tokens(separator_tokens.clone())
+        }
+        Setting::Reset => builder.reset_separator_tokens(),
+        Setting::NotSet => (),
+    }
+
+    match settings.dictionary {
+        Setting::Set(ref dictionary) => builder.set_dictionary(dictionary.clone()),
+        Setting::Reset => builder.reset_dictionary(),
+        Setting::NotSet => (),
+    }
+
     match settings.synonyms {
         Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
         Setting::Reset => builder.reset_synonyms(),
@@ -459,15 +502,14 @@ pub fn settings(
         })
         .transpose()?
         .unwrap_or_default();
+
+    let non_separator_tokens = index.non_separator_tokens(rtxn)?.unwrap_or_default();
+    let separator_tokens = index.separator_tokens(rtxn)?.unwrap_or_default();
+    let dictionary = index.dictionary(rtxn)?.unwrap_or_default();
+
     let distinct_field = index.distinct_field(rtxn)?.map(String::from);
 
-    // in milli each word in the synonyms map were split on their separator. Since we lost
-    // this information we are going to put space between words.
-    let synonyms = index
-        .synonyms(rtxn)?
-        .iter()
-        .map(|(key, values)| (key.join(" "), values.iter().map(|value| value.join(" ")).collect()))
-        .collect();
+    let synonyms = index.user_defined_synonyms(rtxn)?;
 
     let min_typo_word_len = MinWordSizeTyposSetting {
         one_typo: Setting::Set(index.min_word_len_one_typo(rtxn)?),
@@ -520,6 +562,9 @@ pub fn settings(
         sortable_attributes: Setting::Set(sortable_attributes),
         ranking_rules: Setting::Set(criteria.iter().map(|c| c.clone().into()).collect()),
         stop_words: Setting::Set(stop_words),
+        non_separator_tokens: Setting::Set(non_separator_tokens),
+        separator_tokens: Setting::Set(separator_tokens),
+        dictionary: Setting::Set(dictionary),
         distinct_attribute: match distinct_field {
             Some(field) => Setting::Set(field),
             None => Setting::Reset,
@@ -642,6 +687,9 @@ pub(crate) mod test {
             sortable_attributes: Setting::NotSet,
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
+            non_separator_tokens: Setting::NotSet,
+            separator_tokens: Setting::NotSet,
+            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,
@@ -663,6 +711,9 @@ pub(crate) mod test {
             sortable_attributes: Setting::NotSet,
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
+            non_separator_tokens: Setting::NotSet,
+            separator_tokens: Setting::NotSet,
+            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,

meilisearch/src/analytics/mock_analytics.rs

@@ -20,7 +20,7 @@ pub struct SearchAggregator;
 #[allow(dead_code)]
 impl SearchAggregator {
     pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
-        Self::default()
+        Self
     }
 
     pub fn succeed(&mut self, _: &dyn Any) {}
@@ -32,7 +32,7 @@ pub struct MultiSearchAggregator;
 #[allow(dead_code)]
 impl MultiSearchAggregator {
     pub fn from_queries(_: &dyn Any, _: &dyn Any) -> Self {
-        Self::default()
+        Self
     }
 
     pub fn succeed(&mut self) {}
@@ -44,7 +44,7 @@ pub struct FacetSearchAggregator;
 #[allow(dead_code)]
 impl FacetSearchAggregator {
     pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
-        Self::default()
+        Self
     }
 
     pub fn succeed(&mut self, _: &dyn Any) {}

meilisearch/src/routes/indexes/settings.rs

@@ -310,6 +310,81 @@ make_setting_route!(
     }
 );
 
+make_setting_route!(
+    "/non-separator-tokens",
+    put,
+    std::collections::BTreeSet<String>,
+    meilisearch_types::deserr::DeserrJsonError<
+        meilisearch_types::error::deserr_codes::InvalidSettingsNonSeparatorTokens,
+    >,
+    non_separator_tokens,
+    "nonSeparatorTokens",
+    analytics,
+    |non_separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "nonSeparatorTokens Updated".to_string(),
+            json!({
+                "non_separator_tokens": {
+                    "total": non_separator_tokens.as_ref().map(|non_separator_tokens| non_separator_tokens.len()),
+                },
+            }),
+            Some(req),
+        );
+    }
+);
+
+make_setting_route!(
+    "/separator-tokens",
+    put,
+    std::collections::BTreeSet<String>,
+    meilisearch_types::deserr::DeserrJsonError<
+        meilisearch_types::error::deserr_codes::InvalidSettingsSeparatorTokens,
+    >,
+    separator_tokens,
+    "separatorTokens",
+    analytics,
+    |separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "separatorTokens Updated".to_string(),
+            json!({
+                "separator_tokens": {
+                    "total": separator_tokens.as_ref().map(|separator_tokens| separator_tokens.len()),
+                },
+            }),
+            Some(req),
+        );
+    }
+);
+
+make_setting_route!(
+    "/dictionary",
+    put,
+    std::collections::BTreeSet<String>,
+    meilisearch_types::deserr::DeserrJsonError<
+        meilisearch_types::error::deserr_codes::InvalidSettingsDictionary,
+    >,
+    dictionary,
+    "dictionary",
+    analytics,
+    |dictionary: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "dictionary Updated".to_string(),
+            json!({
+                "dictionary": {
+                    "total": dictionary.as_ref().map(|dictionary| dictionary.len()),
+                },
+            }),
+            Some(req),
+        );
+    }
+);
+
 make_setting_route!(
     "/synonyms",
     put,
@@ -466,6 +541,9 @@ generate_configure!(
     searchable_attributes,
     distinct_attribute,
     stop_words,
+    separator_tokens,
+    non_separator_tokens,
+    dictionary,
     synonyms,
     ranking_rules,
     typo_tolerance,

meilisearch/src/search.rs

@@ -491,6 +491,20 @@ pub fn perform_search(
         tokenizer_builder.allow_list(&script_lang_map);
     }
 
+    let separators = index.allowed_separators(&rtxn)?;
+    let separators: Option<Vec<_>> =
+        separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
+    if let Some(ref separators) = separators {
+        tokenizer_builder.separators(separators);
+    }
+
+    let dictionary = index.dictionary(&rtxn)?;
+    let dictionary: Option<Vec<_>> =
+        dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
+    if let Some(ref dictionary) = dictionary {
+        tokenizer_builder.words_dict(dictionary);
+    }
+
     let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_builder.build());
     formatter_builder.crop_marker(query.crop_marker);
     formatter_builder.highlight_prefix(query.highlight_pre_tag);