continuation marker for incompletion?

Author: flammie

accuracy/src/main.rs

@@ -21,6 +21,7 @@ static CFG: SpellerConfig = SpellerConfig {
     beam: None,
     case_handling: Some(CaseHandlingConfig::default()),
     node_pool_size: 128,
+    completion_marker: None,
 };
 
 fn load_words(

divvunspell-bin/src/main.rs

@@ -42,7 +42,9 @@ impl OutputWriter for StdoutWriter {
 
     fn write_suggestions(&mut self, _word: &str, suggestions: &[Suggestion]) {
         for sugg in suggestions {
-            println!("{}\t\t{}", sugg.value, sugg.weight);
+
+            println!("{}\t\t{} (is complete {})", sugg.value, sugg.weight,
+                     sugg.completed);
         }
         println!();
     }
@@ -183,6 +185,9 @@ struct SuggestArgs {
     #[options(help = "maximum number of results")]
     nbest: Option<usize>,
 
+    #[options(help = "Character for incomplete predictions")]
+    continuation_marker: Option<String>,
+
     #[options(
         no_short,
         long = "no-case-handling",
@@ -310,7 +315,7 @@ fn suggest(args: SuggestArgs) -> anyhow::Result<()> {
     if args.disable_case_handling {
         suggest_cfg.case_handling = None;
     }
-
+    suggest_cfg.completion_marker = args.continuation_marker;
     if let Some(v) = args.nbest {
         if v == 0 {
             suggest_cfg.n_best = None;

divvunspell/src/speller/mod.rs

@@ -29,6 +29,7 @@ pub struct SpellerConfig {
     pub beam: Option<Weight>,
     pub case_handling: Option<CaseHandlingConfig>,
     pub node_pool_size: usize,
+    pub completion_marker: Option<String>,
 }
 
 impl SpellerConfig {
@@ -39,6 +40,7 @@ impl SpellerConfig {
             beam: None,
             case_handling: Some(CaseHandlingConfig::default()),
             node_pool_size: 128,
+            completion_marker: None,
         }
     }
 }
@@ -236,6 +238,7 @@ where
     fn suggest_single(self: Arc<Self>, word: &str, config: &SpellerConfig) -> Vec<Suggestion> {
         let worker = SpellerWorker::new(self.clone(), self.to_input_vec(word), config.clone());
 
+        log::trace!("suggesting single {}", word);
         worker.suggest()
     }
 
@@ -247,6 +250,7 @@ where
     ) -> Vec<Suggestion> {
         use crate::tokenizer::case_handling::*;
 
+        log::trace!("suggesting cases...");
         let CaseHandler {
             original_input,
             mutation,
@@ -256,6 +260,7 @@ where
         let mut best: HashMap<SmolStr, f32> = HashMap::new();
 
         for word in std::iter::once(&original_input).chain(words.iter()) {
+            log::trace!("suggesting for word {}", word);
             let worker = SpellerWorker::new(self.clone(), self.to_input_vec(&word), config.clone());
             let mut suggestions = worker.suggest();
 
@@ -275,7 +280,9 @@ where
 
             match mode {
                 CaseMode::MergeAll => {
+                    log::trace!("Case merge all");
                     for sugg in suggestions.into_iter() {
+                        log::trace!("for {}", sugg.value);
                         let penalty_start =
                             if !sugg.value().starts_with(word.chars().next().unwrap()) {
                                 case_handling.start_penalty
@@ -316,14 +323,27 @@ where
         if best.is_empty() {
             return vec![];
         }
-
-        let mut out = best
-            .into_iter()
-            .map(|(k, v)| Suggestion {
-                value: k,
-                weight: v,
-            })
+        let mut out: Vec<Suggestion>;
+        if let Some(s) = &config.completion_marker {
+            out = best
+                .into_iter()
+                .map(|(k, v)| Suggestion {
+                    value: k.clone(),
+                    weight: v,
+                    completed: !k.ends_with(s),
+                })
             .collect::<Vec<_>>();
+        }
+        else {
+            out = best
+                .into_iter()
+                .map(|(k, v)| Suggestion {
+                    value: k,
+                    weight: v,
+                    completed: true,
+                })
+            .collect::<Vec<_>>();
+        }
         out.sort();
         if let Some(n_best) = config.n_best {
             out.truncate(n_best);

divvunspell/src/speller/suggestion.rs

@@ -8,11 +8,12 @@ use std::cmp::Ordering::Equal;
 pub struct Suggestion {
     pub value: SmolStr,
     pub weight: Weight,
+    pub completed: bool,
 }
 
 impl Suggestion {
-    pub fn new(value: SmolStr, weight: Weight) -> Suggestion {
-        Suggestion { value, weight }
+    pub fn new(value: SmolStr, weight: Weight, completed: bool) -> Suggestion {
+        Suggestion { value, weight, completed }
     }
 
     pub fn value(&self) -> &str {
@@ -22,6 +23,10 @@ impl Suggestion {
     pub fn weight(&self) -> Weight {
         self.weight
     }
+    
+    pub fn completed(&self) -> bool {
+        self.completed
+    }
 }
 
 impl PartialOrd for Suggestion {

divvunspell/src/speller/worker.rs

@@ -611,7 +611,7 @@ where
                 .lexicon()
                 .alphabet()
                 .string_from_symbols(&next_node.string);
-            log::trace!("suggesting? {}::{}", string, weight);
+            // log::trace!("suggesting? {}::{}", string, weight);
             if weight < best_weight {
                 best_weight = weight;
             }
@@ -633,17 +633,24 @@ where
         corrections: &HashMap<SmolStr, Weight>,
     ) -> Vec<Suggestion> {
         log::trace!("Generating sorted suggestions");
-        let mut c: Vec<Suggestion> = corrections
-            .into_iter()
-            .map(|x| Suggestion::new(x.0.clone(), *x.1))
-            .collect();
-
+        let mut c: Vec<Suggestion>;
+        if let Some(s) = &self.config.completion_marker {
+            c = corrections
+                .into_iter()
+                .map(|x| Suggestion::new(x.0.clone(), *x.1, x.0.ends_with(s)))
+                .collect();
+        }
+        else {
+            c = corrections
+                .into_iter()
+                .map(|x| Suggestion::new(x.0.clone(), *x.1, true))
+                .collect();
+        }
         c.sort();
 
         if let Some(n) = self.config.n_best {
             c.truncate(n);
         }
-
         c
     }
 }