Add common wordlist

Author: blopker

codebook/src/lib.rs

@@ -3,7 +3,10 @@ mod queries;
 mod splitter;
 use lru::LruCache;
 
-use crate::queries::{get_language_name_from_filename, get_language_setting, LanguageSetting};
+use crate::queries::{
+    get_language_name_from_filename, get_language_setting, LanguageSetting, LanguageType,
+    COMMON_DICTIONARY,
+};
 use std::{
     collections::{HashMap, HashSet},
     num::NonZeroUsize,
@@ -39,9 +42,9 @@ pub struct TextRange {
 
 #[derive(Debug)]
 pub struct CodeDictionary {
-    custom_dictionary: HashSet<String>,
+    custom_dictionary: Arc<RwLock<HashSet<String>>>,
     dictionary: spellbook::Dictionary,
-    dictionary_lookup_cache: Arc<RwLock<LruCache<String, Vec<String>>>>,
+    suggestion_cache: Arc<RwLock<LruCache<String, Vec<String>>>>,
 }
 
 impl CodeDictionary {
@@ -50,29 +53,41 @@ impl CodeDictionary {
         let dic = std::fs::read_to_string(dic_path)?;
         let dict = spellbook::Dictionary::new(&aff, &dic)
             .map_err(|e| format!("Dictionary parse error: {}", e))?;
-
+        let mut custom_dictionary: HashSet<String> = HashSet::new();
+        for word in COMMON_DICTIONARY.lines() {
+            custom_dictionary.insert(word.to_string());
+        }
         Ok(CodeDictionary {
-            custom_dictionary: HashSet::new(),
+            custom_dictionary: Arc::new(RwLock::new(custom_dictionary)),
             dictionary: dict,
-            dictionary_lookup_cache: Arc::new(RwLock::new(LruCache::new(
+            suggestion_cache: Arc::new(RwLock::new(LruCache::new(
                 NonZeroUsize::new(10000).unwrap(),
             ))),
         })
     }
 
     pub fn check(&self, word: &str) -> bool {
-        self.custom_dictionary.contains(word) || self.dictionary.check(word)
-        // self.dictionary_lookup_cache.read().unwrap().contains(word) || self.dictionary.check(word)
+        self.custom_dictionary
+            .read()
+            .unwrap()
+            .contains(word.to_lowercase().as_str())
+            || self.dictionary.check(word)
+        // self.lookup_cache.read().unwrap().contains(word) || self.dictionary.check(word)
     }
 
-    pub fn add_to_dictionary(&mut self, word: String) {
-        self.custom_dictionary.insert(word);
+    pub fn add_to_dictionary(&self, strings: &str) {
+        for line in strings.lines() {
+            self.custom_dictionary
+                .write()
+                .unwrap()
+                .insert(line.to_string());
+        }
     }
 
     pub fn suggest(&self, word: &str) -> Vec<String> {
         println!("Checking Cache: {:?}", word);
         // First try to get from cache with write lock since get() needs to modify LRU order
-        if let Some(suggestions) = self.dictionary_lookup_cache.write().unwrap().get_mut(word) {
+        if let Some(suggestions) = self.suggestion_cache.write().unwrap().get_mut(word) {
             println!("Cache hit for {:?}", word);
             return suggestions.clone();
         }
@@ -81,7 +96,7 @@ impl CodeDictionary {
         let mut suggestions = Vec::new();
         self.dictionary.suggest(word, &mut suggestions);
         if !suggestions.is_empty() {
-            self.dictionary_lookup_cache
+            self.suggestion_cache
                 .write()
                 .unwrap()
                 .put(word.to_string(), suggestions.clone());
@@ -90,27 +105,41 @@ impl CodeDictionary {
     }
 
     pub fn spell_check(&self, text: &str, language: &str) -> Vec<SpellCheckResult> {
-        // print!("language: {:?}", language);
-        let lang = get_language_setting(language);
-        match lang {
-            None => {
-                return self.spell_check_text(text);
-            }
-            Some(lang) => {
-                return self.spell_check_code(text, lang);
-            }
-        }
+        let lang_type = LanguageType::from_str(language);
+        return self.spell_check_enum(text, lang_type);
     }
 
     pub fn spell_check_file(&self, path: &str) -> Vec<SpellCheckResult> {
-        let lang_name = get_language_name_from_filename(path);
+        let lang_type = get_language_name_from_filename(path);
         let file_text = std::fs::read_to_string(path).unwrap();
-        return self.spell_check(&file_text, &lang_name);
+        return self.spell_check_enum(&file_text, lang_type);
     }
 
     pub fn spell_check_file_memory(&self, path: &str, contents: &str) -> Vec<SpellCheckResult> {
-        let lang_name = get_language_name_from_filename(path);
-        return self.spell_check(&contents, &lang_name);
+        let lang_type = get_language_name_from_filename(path);
+        return self.spell_check_enum(&contents, lang_type);
+    }
+
+    fn spell_check_enum(
+        &self,
+        text: &str,
+        language_type: Option<LanguageType>,
+    ) -> Vec<SpellCheckResult> {
+        let language = match language_type {
+            None => None,
+            Some(lang) => get_language_setting(lang),
+        };
+        match language {
+            None => {
+                return self.spell_check_text(text);
+            }
+            Some(lang) => {
+                // if let Some(dictionary) = lang.language_dictionary {
+                //     self.add_to_dictionary(dictionary);
+                // }
+                return self.spell_check_code(text, lang);
+            }
+        }
     }
 
     fn spell_check_text(&self, text: &str) -> Vec<SpellCheckResult> {
@@ -301,7 +330,7 @@ mod lib_tests {
         let mut cdict =
             CodeDictionary::new("./tests/en_index.aff", "./tests/en_index.dic").unwrap();
         for word in EXTRA_WORDS {
-            cdict.add_to_dictionary(word.to_string());
+            cdict.add_to_dictionary(word);
         }
         cdict
     }
@@ -311,7 +340,7 @@ mod lib_tests {
         let processor = get_processor();
 
         let text = "HelloWorld calc_wrld";
-        let misspelled = processor.spell_check(text, "text");
+        let misspelled = processor.spell_check_enum(text, None);
         println!("{:?}", misspelled);
         assert!(misspelled.iter().any(|r| r.word == "wrld"));
     }

codebook/src/main.rs

@@ -1,15 +1,13 @@
 mod downloader;
-
 use codebook::CodeDictionary;
 use downloader::DictionaryDownloader;
 use std::env;
 use std::path::Path;
 
 fn main() {
     let args: Vec<String> = env::args().collect();
-    let downloader =
-        DictionaryDownloader::new(downloader::DEFAULT_BASE_URL, "../.cache/dictionaries");
-    let files = downloader.get("en").unwrap();
+    let loader = DictionaryDownloader::new(downloader::DEFAULT_BASE_URL, "../.cache/dictionaries");
+    let files = loader.get("en").unwrap();
     let processor = CodeDictionary::new(&files.aff_local_path, &files.dic_local_path).unwrap();
 
     // println!("My path is {:?}", args);

codebook/src/queries.rs

@@ -1,8 +1,38 @@
 use tree_sitter::Language;
 
+#[derive(Debug, Clone, PartialEq, Copy)]
+pub enum LanguageType {
+    Rust,
+    Python,
+    Javascript,
+    Typescript,
+    Html,
+    Css,
+    Go,
+    Text,
+}
+
+impl LanguageType {
+    pub fn from_str(s: &str) -> Option<LanguageType> {
+        match s {
+            "rust" => Some(LanguageType::Rust),
+            "python" => Some(LanguageType::Python),
+            "javascript" => Some(LanguageType::Javascript),
+            "typescript" => Some(LanguageType::Typescript),
+            "html" => Some(LanguageType::Html),
+            "css" => Some(LanguageType::Css),
+            "go" => Some(LanguageType::Go),
+            "text" => Some(LanguageType::Text),
+            _ => None,
+        }
+    }
+}
+
+pub static COMMON_DICTIONARY: &str = include_str!("../../wordlists/common.txt");
 // Use https://intmainreturn0.com/ts-visualizer/ to help with writing grammar queries
 pub static LANGUAGE_SETTINGS: [LanguageSetting; 7] = [
     LanguageSetting {
+        type_: LanguageType::Rust,
         name: "rust",
         query: r#"
                 (function_item
@@ -18,6 +48,7 @@ pub static LANGUAGE_SETTINGS: [LanguageSetting; 7] = [
         extensions: &["rs"],
     },
     LanguageSetting {
+        type_: LanguageType::Python,
         name: "python",
         query: r#"
             (identifier) @identifier
@@ -32,6 +63,7 @@ pub static LANGUAGE_SETTINGS: [LanguageSetting; 7] = [
         extensions: &["py"],
     },
     LanguageSetting {
+        type_: LanguageType::Javascript,
         name: "javascript",
         query: r#"
             (identifier) @identifier
@@ -49,6 +81,7 @@ pub static LANGUAGE_SETTINGS: [LanguageSetting; 7] = [
         extensions: &["js"],
     },
     LanguageSetting {
+        type_: LanguageType::Typescript,
         name: "typescript",
         query: r#"
             (identifier) @identifier
@@ -65,6 +98,7 @@ pub static LANGUAGE_SETTINGS: [LanguageSetting; 7] = [
         extensions: &["ts"],
     },
     LanguageSetting {
+        type_: LanguageType::Html,
         name: "html",
         query: r#"
             (text) @string
@@ -74,6 +108,7 @@ pub static LANGUAGE_SETTINGS: [LanguageSetting; 7] = [
         extensions: &["html", "htm"],
     },
     LanguageSetting {
+        type_: LanguageType::Css,
         name: "css",
         query: r#"
             (class_name) @identifier
@@ -86,6 +121,7 @@ pub static LANGUAGE_SETTINGS: [LanguageSetting; 7] = [
         extensions: &["css"],
     },
     LanguageSetting {
+        type_: LanguageType::Go,
         name: "go",
         query: r#"
                 (comment) @comment
@@ -99,28 +135,30 @@ pub static LANGUAGE_SETTINGS: [LanguageSetting; 7] = [
 
 #[derive(Debug)]
 pub struct LanguageSetting {
+    pub type_: LanguageType,
     pub query: &'static str,
     pub name: &'static str,
     pub extensions: &'static [&'static str],
 }
 
 impl LanguageSetting {
     pub fn language(&self) -> Option<Language> {
-        match self.name {
-            "rust" => Some(tree_sitter_rust::LANGUAGE.into()),
-            "python" => Some(tree_sitter_python::LANGUAGE.into()),
-            "javascript" => Some(tree_sitter_javascript::LANGUAGE.into()),
-            "typescript" => Some(tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()),
-            "html" => Some(tree_sitter_html::LANGUAGE.into()),
-            "go" => Some(tree_sitter_go::LANGUAGE.into()),
-            _ => None,
+        match self.type_ {
+            LanguageType::Rust => Some(tree_sitter_rust::LANGUAGE.into()),
+            LanguageType::Python => Some(tree_sitter_python::LANGUAGE.into()),
+            LanguageType::Javascript => Some(tree_sitter_javascript::LANGUAGE.into()),
+            LanguageType::Typescript => Some(tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()),
+            LanguageType::Html => Some(tree_sitter_html::LANGUAGE.into()),
+            LanguageType::Css => None,
+            LanguageType::Go => Some(tree_sitter_go::LANGUAGE.into()),
+            LanguageType::Text => None,
         }
     }
 }
 
-pub fn get_language_setting(language_name: &str) -> Option<&LanguageSetting> {
+pub fn get_language_setting(language_type: LanguageType) -> Option<&'static LanguageSetting> {
     for setting in LANGUAGE_SETTINGS.iter() {
-        if setting.name == language_name {
+        if setting.type_ == language_type {
             if setting.language().is_some() {
                 return Some(setting);
             }
@@ -129,14 +167,14 @@ pub fn get_language_setting(language_name: &str) -> Option<&LanguageSetting> {
     None
 }
 
-pub fn get_language_name_from_filename(filename: &str) -> String {
+pub fn get_language_name_from_filename(filename: &str) -> Option<LanguageType> {
     let extension = filename.split('.').last().unwrap();
     for setting in LANGUAGE_SETTINGS.iter() {
         for ext in setting.extensions.iter() {
             if ext == &extension {
-                return setting.name.to_string();
+                return Some(setting.type_);
             }
         }
     }
-    "text".to_string()
+    None
 }

codebook/tests/test_files.rs

@@ -114,7 +114,7 @@ fn test_example_files() {
             vec!["Wolrd", "bvd", "regulr", "splellin", "wolrd"],
         ),
         ("example.txt", vec!["Splellin", "bd"]),
-        ("example.rs", vec!["birt", "curent", "jalopin", "usr"]),
+        ("example.rs", vec!["birt", "calclate", "curent", "jalopin"]),
         (
             "example.go",
             vec!["speling", "Wolrd", "mispeled", "Funcion"],
@@ -146,7 +146,7 @@ fn test_example_files() {
     ];
     for mut file in files {
         let path = example_file_path(file.0);
-        println!("Checking file: {path:?}");
+        println!("---------- Checking file: {path:?} ----------");
         let processor = utils::get_processor();
         let results = processor.spell_check_file(&path);
         let mut misspelled = results

codebook/tests/test_python.rs

@@ -11,7 +11,7 @@ fn test_python_simple() {
             usrAge = get_curent_date() - bithDate
             userAge
     "#;
-    let expected = vec!["bith", "calculat", "curent", "examle", "usr"];
+    let expected = vec!["bith", "calculat", "curent", "examle"];
     let binding = processor.spell_check(sample_text, "python").to_vec();
     let mut misspelled = binding
         .iter()

codebook/tests/test_rust.rs

@@ -12,7 +12,7 @@ fn test_rust_simple() {
             userAge
         }
     "#;
-    let expected = vec!["bith", "calculat", "examle", "usr"];
+    let expected = vec!["bith", "calculat", "examle"];
     let binding = processor.spell_check(sample_text, "rust").to_vec();
     let mut misspelled = binding
         .iter()

codebook/tests/utils/mod.rs

@@ -4,7 +4,7 @@ static EXTRA_WORDS: &'static [&'static str] = &["http", "https", "www", "viewpor
 pub fn get_processor() -> CodeDictionary {
     let mut cdict = CodeDictionary::new("./tests/en_index.aff", "./tests/en_index.dic").unwrap();
     for word in EXTRA_WORDS {
-        cdict.add_to_dictionary(word.to_string());
+        cdict.add_to_dictionary(word);
     }
     cdict
 }