Further work on dictionary locator

Work to improve test framework

Correct affix test pathing

Fixed clippy errors

crates/zspell/Cargo.toml

@@ -27,25 +27,33 @@ name = "zspell"
 path = "src/bin/bin.rs"
 
 [dependencies]
+# Base dependencies
+atty = "0.2"
+cfg-if = "1.0"
 futures = "0.3"
+hashbrown = { version = "0.12", features = ["rayon"] }
+home = "0.5"
 lazy_static = "1.4"
+rayon = "1.5"
 regex = "1"
+stringmetrics = "2"
 strum = { version = "0.24", features = ["derive"] }
 strum_macros = "0.24"
+sys-locale = "0.2"
+thiserror = "1.0"
 unicode-segmentation = "1.9.0"
+
+# CLI dependencies
+# At some point, we may wish to split these crates off
 clap = { version = "3.2", features = ["derive"] }
-stringmetrics = "2"
-thiserror = "1.0"
-rayon = "1.5"
-hashbrown = { version = "0.12", features = ["rayon"] }
-home = "0.5"
-cfg-if = "1.0"
+termcolor = "1.1"
 
 [dev-dependencies]
 criterion = "0.3"
 assert_cmd = "2.0"
 predicates = "2.1"
 tempfile = "3.3"
+util = { path = "util" }
 
 [[bench]]
 name = "datastructure"

crates/zspell/src/dictionary.rs

@@ -251,7 +251,8 @@ impl Dictionary {
                 || self.wordlist_nosuggest.contains(lower))
     }
 
-    /// Create a sorted vector of all items in the word list
+    /// Create a iterator over all items in the dictionary's wordlist. That is,
+    /// words that will always be accepted and succested.
     ///
     /// Note that this is relatively slow. Prefer [`Dictionary::check`] for
     /// validating a word exists.
@@ -267,6 +268,41 @@ impl Dictionary {
         Ok(self.wordlist.iter())
     }
 
+    /// Create a iterator over all items in the dictionary's nonsuggesting
+    /// wordlist. These are words that will always be accepted but never be
+    /// suggested.
+    ///
+    /// Note that this is relatively slow. Prefer [`Dictionary::check`] for
+    /// validating a word exists.
+    ///
+    /// # Errors
+    ///
+    /// Returns [`DictError::NotCompiled`] if the dictionary has not yet been
+    /// compiled
+    #[inline]
+    pub fn iter_wordlist_nosuggest_items(&self) -> Result<HashSetIter<String>, DictError> {
+        self.break_if_not_compiled()?;
+
+        Ok(self.wordlist_nosuggest.iter())
+    }
+
+    /// Create a iterator over all items in the dictionary's forbidden wordlist.
+    /// These are words that are never accepted as correct.
+    ///
+    /// Note that this is relatively slow. Prefer [`Dictionary::check`] for
+    /// validating a word exists.
+    ///
+    /// # Errors
+    ///
+    /// Returns [`DictError::NotCompiled`] if the dictionary has not yet been
+    /// compiled
+    #[inline]
+    pub fn iter_wordlist_forbidden_items(&self) -> Result<HashSetIter<String>, DictError> {
+        self.break_if_not_compiled()?;
+
+        Ok(self.wordlist_forbidden.iter())
+    }
+
     /// Helper function to error if we haven't compiled when we needed to
     #[inline]
     const fn break_if_not_compiled(&self) -> Result<(), DictError> {

crates/zspell/src/system.rs

@@ -8,6 +8,7 @@ use std::{
     env,
     path::{Component, Path, PathBuf},
 };
+// use sys_locale::get_locale;
 
 use crate::errors::DictError;
 // use crate::errors::FileError;
@@ -230,6 +231,10 @@ pub fn expand_dir_wildcards(paths: &mut Vec<PathBuf>) -> HashSet<PathBuf> {
     ret
 }
 
+// pub fn find_dict_from_path() {
+//     let locale = get_locale().unwrap_or_else(|| String::from("en-US"));
+// }
+
 /// Take in a path and load the dictionary
 ///
 /// # Errors
@@ -272,9 +277,6 @@ pub fn create_dict_from_path(basepath: &str) -> Result<Dictionary, DictError> {
     Ok(dic)
 }
 
-// Need function to expand wildcard paths. Will need to look through the parent
-// directory and see if anything is a RE match
-
 #[cfg(test)]
 mod tests {
     use super::*;

crates/zspell/tests/affix_integration.rs

@@ -1,11 +1,11 @@
-use std::fs;
+use util::TestCollection;
 use zspell::Config;
 
 #[test]
 fn affix_create_words() {
     let mut afx = Config::new();
 
-    let content = fs::read_to_string("tests/files/1_pfxsfx.aff").unwrap();
+    let content = TestCollection::load("1_pfxsfx.test").afx_str;
 
     afx.load_from_str(content.as_str()).unwrap();
 

crates/zspell/tests/dictionary_integration.rs

@@ -1,4 +1,6 @@
 use std::fs;
+
+use util::TestCollection;
 use zspell::Dictionary;
 
 /// Run integration tests on a file located in tests/files
@@ -22,40 +24,22 @@ fn create_dic_from_file(fname: &str) -> Dictionary {
     dic
 }
 
-/// Validate a dictionary's wordlist is correct
-fn test_dic_wordlist(dic: &Dictionary, fname: &str) {
-    let out_name = format!("tests/files/{}.words", fname);
-
-    let out_content = fs::read_to_string(out_name.clone())
-        .expect(format!("error reading file {}", out_name).as_str());
-    let mut correct: Vec<_> = out_content.lines().collect();
-    correct.sort_unstable();
-    let mut result: Vec<_> = dic
-        .iter_wordlist_items()
-        .expect("Error getting wordlist")
-        .collect();
-    result.sort_unstable();
-
-    assert_eq!(result, correct);
-}
-
-/// Test compiling the dictionary from our short test file
-#[test]
-fn test_short_compile() {
-    let dic = create_dic_from_file("1_pfxsfx");
-    test_dic_wordlist(&dic, "1_pfxsfx");
-}
-
-/// Test check functionality on our short file
+/// Test check functionality on a real file
 #[test]
 fn test_short_check() {
-    let dic = create_dic_from_file("1_pfxsfx");
+    let dic = create_dic_from_file("w1_eng_short");
 
     // Test all ownership methods
-    assert_eq!(dic.check("xxx"), Ok(true));
-    assert_eq!(dic.check("yybb".to_string()), Ok(true));
-    assert_eq!(dic.check("aazzzcc".to_owned()), Ok(true));
-    assert_eq!(dic.check(&"zzz".to_string()), Ok(true));
+    assert_eq!(dic.check("bananas"), Ok(true));
+    assert_eq!(dic.check("pines".to_string()), Ok(true));
+    assert_eq!(dic.check("pillowing".to_owned()), Ok(true));
+    assert_eq!(dic.check(&"pined".to_string()), Ok(true));
 
     assert_eq!(dic.check("not contained"), Ok(false));
 }
+
+#[test]
+fn test_prefixes() {
+    let coll = TestCollection::load("1_pfxsfx.test");
+    coll.validate();
+}

crates/zspell/tests/files/1_pfxsfx.test

@@ -0,0 +1,31 @@
+==== afx_str ====
+SET UTF-8
+
+PFX A Y 1
+PFX A   0     aa         .
+
+SFX B Y 2
+SFX B   y     bb         y
+SFX B   0     cc         [^y]
+
+
+==== dic_str =====
+3
+xxx/A
+yyy/B
+zzz/AB
+
+
+==== check_valid ====
+xxx
+yyy
+zzz
+aaxxx
+yybb
+aazzz
+zzzcc
+aazzzcc
+
+
+==== check_invalid ====
+not contained

crates/zspell/tests/files/example.test

@@ -0,0 +1,29 @@
+# Example test file
+
+==== afx_str ====
+# This section contains contents of the .aff file
+
+==== dic_str ====
+# This section contains contents of the .dic file
+
+==== check_valid ====
+# Words that should come up as "valid" when checked
+
+==== check_valid ====
+# These words should appear as valid
+
+==== check_invalid ====
+# These words should not be valid
+
+==== wordlist ====
+# The dictionary's internal word list contents
+
+==== wordlist_nosuggest ====
+# Contents of the never suggested wordlist
+
+==== wordlist_forbidden ====
+# Contents of the non-accepted wordlist
+
+==== suggestions ====
+# Something like the following
+appl > apple | Apfel | app

crates/zspell/util/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "util"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+zspell = { path = "../" }