From 78334571e1fbf26511ea627b7e3293b9e6dfae59 Mon Sep 17 00:00:00 2001 From: teners Date: Wed, 10 Aug 2016 20:29:58 +0300 Subject: [PATCH 01/21] Initial commit --- backend/spellchecker/README.md | 16 ++++++ backend/spellchecker/huntojson.sh | 81 +++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 backend/spellchecker/README.md create mode 100755 backend/spellchecker/huntojson.sh diff --git a/backend/spellchecker/README.md b/backend/spellchecker/README.md new file mode 100644 index 0000000..e99a7a0 --- /dev/null +++ b/backend/spellchecker/README.md @@ -0,0 +1,16 @@ +### huntojson.sh +Convert hunspell output to JSON for spellchecker. + +Launch hunspell on input file, convert output to spellchecker JSON format and write to standard output (if used without `-o`). +Usage: `./huntojson.sh [options...] [file]` +Options: +`-h` display this help and exit +`-o output` file where to write JSON +`-d dict` use custom dictionaries + +Example: +```bash +./huntojson.sh demo.tex # launch hunspell on 'demo.tex' +./huntojson.sh -o json.txt summer-2016.tex # write output to 'json.txt' +./huntojson.sh -d ru_RU ru_demo.tex # use russian dictionary +``` diff --git a/backend/spellchecker/huntojson.sh b/backend/spellchecker/huntojson.sh new file mode 100755 index 0000000..efab16c --- /dev/null +++ b/backend/spellchecker/huntojson.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# usage info +show_help() { +cat << EOF +Usage: ${0##*/} [options...] [file] +Launch hunspell on file, convert output to JSON format +and write to standard output if used without '-o'. + + -h display this help and exit + -o output file where to write JSON + -d dict use custom dictionaries + +Example: ${0##*/} demo.tex # launch hunspell on 'demo.tex' + ${0##*/} -o json.txt summer-2016.tex # write output to 'json.txt' + ${0##*/} -d ru_RU ru_demo.tex # use russian dictionary +EOF +} + +OUTFILE= +DICT= + +# command line arguments processing +OPTIND=1 +while getopts ":hd:o:" opt; do + case "$opt" in + h) + show_help + exit 0 + ;; + d) + DICT="-p $OPTARG" + ;; + o) + OUTFILE="$OPTARG" + ;; + ?) + echo -e "Invalid option: -$OPTARG\n" >&2 + show_help >&2 + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ "$#" != 1 ] +then + echo -e "Missing input file\n" >&2 + show_help >&2 + exit 2 +fi + +INFILE=$1 + +JSON=` + cat $INFILE | + hunspell -at | + grep "^&.*" | + sed "s/,//g" | + awk $DICT ' + BEGIN { print "{" } + { + print "\t\""$2"\": [" ; + + for (i = 5; i <= NF; i++) + if (i != NF) + {print "\t\t\""$i"\","} + else + {print "\t\t\""$i"\""} ; + + print "\t]," } + END { print "}" } + '` + +if [ -n "$OUTFILE" ] +then + echo $JSON > $OUTFILE +else + echo $JSON +fi + From a197b6e1f82925c14798e018e0060fd01779370f Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 11 Aug 2016 15:02:17 +0300 Subject: [PATCH 02/21] Removed output file option, added corrections with spaces support, fixed hunspell options to interpret input as latex file --- backend/spellchecker/README.md | 4 +-- backend/spellchecker/huntojson.sh | 43 ++++++++++++------------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/backend/spellchecker/README.md b/backend/spellchecker/README.md index e99a7a0..9819dd4 100644 --- a/backend/spellchecker/README.md +++ b/backend/spellchecker/README.md @@ -1,16 +1,14 @@ ### huntojson.sh Convert hunspell output to JSON for spellchecker. -Launch hunspell on input file, convert output to spellchecker JSON format and write to standard output (if used without `-o`). +Launch hunspell on input file, convert output to spellchecker JSON format and write to standard output. Usage: `./huntojson.sh [options...] [file]` Options: `-h` display this help and exit -`-o output` file where to write JSON `-d dict` use custom dictionaries Example: ```bash ./huntojson.sh demo.tex # launch hunspell on 'demo.tex' -./huntojson.sh -o json.txt summer-2016.tex # write output to 'json.txt' ./huntojson.sh -d ru_RU ru_demo.tex # use russian dictionary ``` diff --git a/backend/spellchecker/huntojson.sh b/backend/spellchecker/huntojson.sh index efab16c..a998be5 100755 --- a/backend/spellchecker/huntojson.sh +++ b/backend/spellchecker/huntojson.sh @@ -5,24 +5,21 @@ show_help() { cat << EOF Usage: ${0##*/} [options...] [file] Launch hunspell on file, convert output to JSON format -and write to standard output if used without '-o'. +and write to standard output. -h display this help and exit - -o output file where to write JSON -d dict use custom dictionaries Example: ${0##*/} demo.tex # launch hunspell on 'demo.tex' - ${0##*/} -o json.txt summer-2016.tex # write output to 'json.txt' ${0##*/} -d ru_RU ru_demo.tex # use russian dictionary EOF } -OUTFILE= DICT= +OPTIND=1 # command line arguments processing -OPTIND=1 -while getopts ":hd:o:" opt; do +while getopts ":hd:" opt; do case "$opt" in h) show_help @@ -31,9 +28,6 @@ while getopts ":hd:o:" opt; do d) DICT="-p $OPTARG" ;; - o) - OUTFILE="$OPTARG" - ;; ?) echo -e "Invalid option: -$OPTARG\n" >&2 show_help >&2 @@ -43,7 +37,7 @@ while getopts ":hd:o:" opt; do done shift $((OPTIND-1)) -if [ "$#" != 1 ] +if [ "$#" != "1" ] then echo -e "Missing input file\n" >&2 show_help >&2 @@ -52,30 +46,25 @@ fi INFILE=$1 -JSON=` - cat $INFILE | - hunspell -at | - grep "^&.*" | - sed "s/,//g" | +JSON=$( + cat $INFILE | + hunspell -a -t | + grep "^&.*" | awk $DICT ' BEGIN { print "{" } { print "\t\""$2"\": [" ; - for (i = 5; i <= NF; i++) - if (i != NF) - {print "\t\t\""$i"\","} - else - {print "\t\t\""$i"\""} ; + split($0, split_string, ": "); + options_number=split(split_string[2], options, ", "); + + for (i = 1; i <= options_number; i++) + {print "\t\t\""options[i]"\","} print "\t]," } END { print "}" } - '` + ' +) -if [ -n "$OUTFILE" ] -then - echo $JSON > $OUTFILE -else - echo $JSON -fi +echo $JSON From ccada06eb38027b3b502893d1219ce8b003a53f4 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 11 Aug 2016 16:30:45 -0400 Subject: [PATCH 03/21] Fixed duplicates issue, comma-after-last-element issue, removed "pretty-printing efforts" --- backend/spellchecker/huntojson.sh | 32 +++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/backend/spellchecker/huntojson.sh b/backend/spellchecker/huntojson.sh index a998be5..59a033b 100755 --- a/backend/spellchecker/huntojson.sh +++ b/backend/spellchecker/huntojson.sh @@ -10,8 +10,8 @@ and write to standard output. -h display this help and exit -d dict use custom dictionaries -Example: ${0##*/} demo.tex # launch hunspell on 'demo.tex' - ${0##*/} -d ru_RU ru_demo.tex # use russian dictionary +Example: ${0##*/} demo.tex # launch hunspell on 'demo.tex' + ${0##*/} -d ru_RU ru_demo.tex # use russian dictionary EOF } @@ -26,7 +26,7 @@ while getopts ":hd:" opt; do exit 0 ;; d) - DICT="-p $OPTARG" + DICT="-p \"$OPTARG\"" ;; ?) echo -e "Invalid option: -$OPTARG\n" >&2 @@ -47,24 +47,32 @@ fi INFILE=$1 JSON=$( - cat $INFILE | - hunspell -a -t | - grep "^&.*" | - awk $DICT ' + cat "$INFILE" | + hunspell -a -t $DICT | + grep '^&.*' | + sort | uniq | + awk ' BEGIN { print "{" } { - print "\t\""$2"\": [" ; + print "\""$2"\": [" ; split($0, split_string, ": "); options_number=split(split_string[2], options, ", "); for (i = 1; i <= options_number; i++) - {print "\t\t\""options[i]"\","} + { + printf "\""options[i]"\"" + if (i < options_number) + { print "," } + } - print "\t]," } + print "]," } END { print "}" } - ' + ' ) -echo $JSON +echo "$JSON" | +sed ':a;N;$!ba;s/,\n}/\n}/g' +# some kind of sed magic here to remove +# the last comma before the last closing brace From d225ac5c90188e67dd8784be50f496c973a07c67 Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 11 Aug 2016 16:55:24 -0400 Subject: [PATCH 04/21] Removed dummy sed solution for the last comma issue, fixed it with an additional variable in awk --- backend/spellchecker/huntojson.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/spellchecker/huntojson.sh b/backend/spellchecker/huntojson.sh index 59a033b..1893249 100755 --- a/backend/spellchecker/huntojson.sh +++ b/backend/spellchecker/huntojson.sh @@ -52,9 +52,10 @@ JSON=$( grep '^&.*' | sort | uniq | awk ' - BEGIN { print "{" } + BEGIN { printf "{"; comma=""; } { - print "\""$2"\": [" ; + printf comma"\""$2"\": [" ; + comma="," split($0, split_string, ": "); options_number=split(split_string[2], options, ", "); @@ -63,16 +64,16 @@ JSON=$( { printf "\""options[i]"\"" if (i < options_number) - { print "," } + { + printf "," + } } - print "]," } - END { print "}" } + printf "]" + } + END { printf "}" } ' ) -echo "$JSON" | -sed ':a;N;$!ba;s/,\n}/\n}/g' -# some kind of sed magic here to remove -# the last comma before the last closing brace +echo "$JSON" From d1a31ddabfefe4d092fdf225de23668d152a5821 Mon Sep 17 00:00:00 2001 From: teners Date: Fri, 12 Aug 2016 22:39:48 +0300 Subject: [PATCH 05/21] Deleted shell script and readme, added gitignore for .idea stuff --- backend/spellchecker/.gitignore | 1 + backend/spellchecker/README.md | 14 ------- backend/spellchecker/huntojson.sh | 70 ------------------------------- 3 files changed, 1 insertion(+), 84 deletions(-) create mode 100644 backend/spellchecker/.gitignore delete mode 100644 backend/spellchecker/README.md delete mode 100755 backend/spellchecker/huntojson.sh diff --git a/backend/spellchecker/.gitignore b/backend/spellchecker/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/backend/spellchecker/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/backend/spellchecker/README.md b/backend/spellchecker/README.md deleted file mode 100644 index 9819dd4..0000000 --- a/backend/spellchecker/README.md +++ /dev/null @@ -1,14 +0,0 @@ -### huntojson.sh -Convert hunspell output to JSON for spellchecker. - -Launch hunspell on input file, convert output to spellchecker JSON format and write to standard output. -Usage: `./huntojson.sh [options...] [file]` -Options: -`-h` display this help and exit -`-d dict` use custom dictionaries - -Example: -```bash -./huntojson.sh demo.tex # launch hunspell on 'demo.tex' -./huntojson.sh -d ru_RU ru_demo.tex # use russian dictionary -``` diff --git a/backend/spellchecker/huntojson.sh b/backend/spellchecker/huntojson.sh deleted file mode 100755 index a998be5..0000000 --- a/backend/spellchecker/huntojson.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -# usage info -show_help() { -cat << EOF -Usage: ${0##*/} [options...] [file] -Launch hunspell on file, convert output to JSON format -and write to standard output. - - -h display this help and exit - -d dict use custom dictionaries - -Example: ${0##*/} demo.tex # launch hunspell on 'demo.tex' - ${0##*/} -d ru_RU ru_demo.tex # use russian dictionary -EOF -} - -DICT= -OPTIND=1 - -# command line arguments processing -while getopts ":hd:" opt; do - case "$opt" in - h) - show_help - exit 0 - ;; - d) - DICT="-p $OPTARG" - ;; - ?) - echo -e "Invalid option: -$OPTARG\n" >&2 - show_help >&2 - exit 1 - ;; - esac -done -shift $((OPTIND-1)) - -if [ "$#" != "1" ] -then - echo -e "Missing input file\n" >&2 - show_help >&2 - exit 2 -fi - -INFILE=$1 - -JSON=$( - cat $INFILE | - hunspell -a -t | - grep "^&.*" | - awk $DICT ' - BEGIN { print "{" } - { - print "\t\""$2"\": [" ; - - split($0, split_string, ": "); - options_number=split(split_string[2], options, ", "); - - for (i = 1; i <= options_number; i++) - {print "\t\t\""options[i]"\","} - - print "\t]," } - END { print "}" } - ' -) - -echo $JSON - From 768d0e25a8c38cb4c12a5ee40a09b6b808e04916 Mon Sep 17 00:00:00 2001 From: teners Date: Sun, 14 Aug 2016 04:17:43 +0300 Subject: [PATCH 06/21] Iniital commit for Java implementation --- backend/spellchecker/.gitignore | 3 + .../src/com/papeeria/SpellChecker.java | 121 ++++++++++++++++++ .../src/com/papeeria/SpellCheckerTest.java | 35 +++++ 3 files changed, 159 insertions(+) create mode 100644 backend/spellchecker/src/com/papeeria/SpellChecker.java create mode 100644 backend/spellchecker/src/com/papeeria/SpellCheckerTest.java diff --git a/backend/spellchecker/.gitignore b/backend/spellchecker/.gitignore index 485dee6..386b7a5 100644 --- a/backend/spellchecker/.gitignore +++ b/backend/spellchecker/.gitignore @@ -1 +1,4 @@ .idea +spellchecker.iml +lib +out \ No newline at end of file diff --git a/backend/spellchecker/src/com/papeeria/SpellChecker.java b/backend/spellchecker/src/com/papeeria/SpellChecker.java new file mode 100644 index 0000000..90ac7e6 --- /dev/null +++ b/backend/spellchecker/src/com/papeeria/SpellChecker.java @@ -0,0 +1,121 @@ +package com.papeeria; + +import dk.dren.hunspell.Hunspell; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Scanner; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + + +/** + * SpellChecker is designed to check texts spelling with hunspell (via JNAHunspell, hunspell Java API). + * + */ +class SpellChecker { + private HashMap LanguageMap; + + public SpellChecker() { + LanguageMap = new HashMap<>(); + } + + public SpellChecker(String path, String language) { + this(); + addDictionary(path, language); + } + + public void addDictionary(String path, String language) { + try { + Hunspell.Dictionary dictionary = Hunspell.getInstance().getDictionary(path + "/" + language); + LanguageMap.put(language, dictionary); + } catch (FileNotFoundException e) { + System.err.println("File not found: " + path + "/" + language); + } catch (UnsupportedEncodingException e) { + System.err.println("Unsupported encoding: " + path + "/" + language); + } + + } + + /** + * Check word spelling and return list of suggestions. + * Use every dictionary that corresponds the given languages if + * the current SpellChecker instance has such dictionary. + * + * @param word Word to check. + * @param languages Languages chosen by user. + * @return List of suggestions for the word. + */ + // TODO fix punctiation issue: a word followed with a comma considered as misspelled + public List checkWord(String word, String... languages) { + if (LanguageMap.isEmpty()) { + return null; + } + + List suggestionsList = new ArrayList<>(); + + for (String language: languages) { + if (LanguageMap.containsKey(language) && LanguageMap.get(language).misspelled(word)) { + List suggestions = LanguageMap.get(language).suggest(word); + + if (!suggestions.isEmpty()) { + suggestionsList.addAll(suggestions); + } + } + } + + return suggestionsList.isEmpty() ? null : suggestionsList; + } + + /** + * Check each word in the file with checkWord function, put result in + * JSON structure like {"word": ["suggestions", "list", ...], ...}. + * + * @param path Path to file to check. + * @param languages Languages chosen by user. + * @return String that represents JSON of words and it's suggestions list. + */ + public String checkFile(String path, String... languages) { + Scanner scanner; + try { + scanner = new Scanner(new File(path)); + } catch (FileNotFoundException e) { + System.err.println("File not found: " + path); + return "{}"; + } + + JsonFactory jsonFactory = new JsonFactory(); + jsonFactory.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); + + JsonNodeFactory nodeFactory = new JsonNodeFactory(false); + ObjectNode correctionsJson = nodeFactory.objectNode(); + + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + + String[] split = line.split(" "); + for (String word : split) { + List corrections = checkWord(word, languages); + + if (corrections != null) { + ArrayNode node = nodeFactory.arrayNode(); + + for (String correction : corrections) { + node.add(correction); + } + + correctionsJson.set(word, node); + } + } + } + return correctionsJson.toString(); + } +} \ No newline at end of file diff --git a/backend/spellchecker/src/com/papeeria/SpellCheckerTest.java b/backend/spellchecker/src/com/papeeria/SpellCheckerTest.java new file mode 100644 index 0000000..f17bee1 --- /dev/null +++ b/backend/spellchecker/src/com/papeeria/SpellCheckerTest.java @@ -0,0 +1,35 @@ +package com.papeeria; + +import org.junit.Before; + +import java.util.List; + +import static org.junit.Assert.*; + +public class SpellCheckerTest { + private SpellChecker checker; + + @Before + public void initChecker() { + checker = new SpellChecker("/usr/share/hunspell", "en_US"); + } + + @org.junit.Test + public void checkMisspelledWord() { + List suggestions = checker.checkWord("teest", "en_US"); + assertNotEquals(suggestions, null); + } + + @org.junit.Test + public void checkCorrectWord() { + List suggestions = checker.checkWord("test", "en_US"); + assertEquals(suggestions, null); + } + + @org.junit.Test + public void checkWordWithPunctuation() { + List suggestions = checker.checkWord("test,", "en_US"); + assertEquals(suggestions, null); + } + +} \ No newline at end of file From f3f1597b4733b2b2fa8e5a898914c36e9e48a27d Mon Sep 17 00:00:00 2001 From: teners Date: Mon, 22 Aug 2016 23:46:00 +0300 Subject: [PATCH 07/21] Removed Java spellchecker implementation --- .../src/com/papeeria/SpellChecker.java | 121 ------------------ .../src/com/papeeria/SpellCheckerTest.java | 35 ----- 2 files changed, 156 deletions(-) delete mode 100644 backend/spellchecker/src/com/papeeria/SpellChecker.java delete mode 100644 backend/spellchecker/src/com/papeeria/SpellCheckerTest.java diff --git a/backend/spellchecker/src/com/papeeria/SpellChecker.java b/backend/spellchecker/src/com/papeeria/SpellChecker.java deleted file mode 100644 index 90ac7e6..0000000 --- a/backend/spellchecker/src/com/papeeria/SpellChecker.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.papeeria; - -import dk.dren.hunspell.Hunspell; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Scanner; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; - - -/** - * SpellChecker is designed to check texts spelling with hunspell (via JNAHunspell, hunspell Java API). - * - */ -class SpellChecker { - private HashMap LanguageMap; - - public SpellChecker() { - LanguageMap = new HashMap<>(); - } - - public SpellChecker(String path, String language) { - this(); - addDictionary(path, language); - } - - public void addDictionary(String path, String language) { - try { - Hunspell.Dictionary dictionary = Hunspell.getInstance().getDictionary(path + "/" + language); - LanguageMap.put(language, dictionary); - } catch (FileNotFoundException e) { - System.err.println("File not found: " + path + "/" + language); - } catch (UnsupportedEncodingException e) { - System.err.println("Unsupported encoding: " + path + "/" + language); - } - - } - - /** - * Check word spelling and return list of suggestions. - * Use every dictionary that corresponds the given languages if - * the current SpellChecker instance has such dictionary. - * - * @param word Word to check. - * @param languages Languages chosen by user. - * @return List of suggestions for the word. - */ - // TODO fix punctiation issue: a word followed with a comma considered as misspelled - public List checkWord(String word, String... languages) { - if (LanguageMap.isEmpty()) { - return null; - } - - List suggestionsList = new ArrayList<>(); - - for (String language: languages) { - if (LanguageMap.containsKey(language) && LanguageMap.get(language).misspelled(word)) { - List suggestions = LanguageMap.get(language).suggest(word); - - if (!suggestions.isEmpty()) { - suggestionsList.addAll(suggestions); - } - } - } - - return suggestionsList.isEmpty() ? null : suggestionsList; - } - - /** - * Check each word in the file with checkWord function, put result in - * JSON structure like {"word": ["suggestions", "list", ...], ...}. - * - * @param path Path to file to check. - * @param languages Languages chosen by user. - * @return String that represents JSON of words and it's suggestions list. - */ - public String checkFile(String path, String... languages) { - Scanner scanner; - try { - scanner = new Scanner(new File(path)); - } catch (FileNotFoundException e) { - System.err.println("File not found: " + path); - return "{}"; - } - - JsonFactory jsonFactory = new JsonFactory(); - jsonFactory.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); - - JsonNodeFactory nodeFactory = new JsonNodeFactory(false); - ObjectNode correctionsJson = nodeFactory.objectNode(); - - while (scanner.hasNextLine()) { - String line = scanner.nextLine(); - - String[] split = line.split(" "); - for (String word : split) { - List corrections = checkWord(word, languages); - - if (corrections != null) { - ArrayNode node = nodeFactory.arrayNode(); - - for (String correction : corrections) { - node.add(correction); - } - - correctionsJson.set(word, node); - } - } - } - return correctionsJson.toString(); - } -} \ No newline at end of file diff --git a/backend/spellchecker/src/com/papeeria/SpellCheckerTest.java b/backend/spellchecker/src/com/papeeria/SpellCheckerTest.java deleted file mode 100644 index f17bee1..0000000 --- a/backend/spellchecker/src/com/papeeria/SpellCheckerTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.papeeria; - -import org.junit.Before; - -import java.util.List; - -import static org.junit.Assert.*; - -public class SpellCheckerTest { - private SpellChecker checker; - - @Before - public void initChecker() { - checker = new SpellChecker("/usr/share/hunspell", "en_US"); - } - - @org.junit.Test - public void checkMisspelledWord() { - List suggestions = checker.checkWord("teest", "en_US"); - assertNotEquals(suggestions, null); - } - - @org.junit.Test - public void checkCorrectWord() { - List suggestions = checker.checkWord("test", "en_US"); - assertEquals(suggestions, null); - } - - @org.junit.Test - public void checkWordWithPunctuation() { - List suggestions = checker.checkWord("test,", "en_US"); - assertEquals(suggestions, null); - } - -} \ No newline at end of file From 1a2ee88d42bd43cc9f4e720ccc4d94c80e42bacc Mon Sep 17 00:00:00 2001 From: teners Date: Wed, 31 Aug 2016 17:04:08 +0300 Subject: [PATCH 08/21] Initial commit for python grpc version --- backend/spellchecker/.gitignore | 5 +- backend/spellchecker/libparser/CMakeLists.txt | 15 + backend/spellchecker/libparser/build/Makefile | 238 ++++++++++++++ backend/spellchecker/libparser/parser.cpp | 47 +++ .../libparser/parsers/latexparser.cxx | 264 +++++++++++++++ .../libparser/parsers/latexparser.hxx | 76 +++++ .../libparser/parsers/textparser.cxx | 308 ++++++++++++++++++ .../libparser/parsers/textparser.hxx | 109 +++++++ backend/spellchecker/proto/spellchecker.proto | 15 + backend/spellchecker/server/client.py | 12 + backend/spellchecker/server/parser.py | 70 ++++ backend/spellchecker/server/parser.pyc | Bin 0 -> 2155 bytes backend/spellchecker/server/run_codegen.py | 12 + backend/spellchecker/server/server.py | 44 +++ backend/spellchecker/server/spellchecker.py | 62 ++++ .../spellchecker/server/spellchecker_pb2.py | 197 +++++++++++ backend/spellchecker/server/test.py | 13 + 17 files changed, 1484 insertions(+), 3 deletions(-) create mode 100644 backend/spellchecker/libparser/CMakeLists.txt create mode 100644 backend/spellchecker/libparser/build/Makefile create mode 100644 backend/spellchecker/libparser/parser.cpp create mode 100644 backend/spellchecker/libparser/parsers/latexparser.cxx create mode 100644 backend/spellchecker/libparser/parsers/latexparser.hxx create mode 100644 backend/spellchecker/libparser/parsers/textparser.cxx create mode 100644 backend/spellchecker/libparser/parsers/textparser.hxx create mode 100644 backend/spellchecker/proto/spellchecker.proto create mode 100644 backend/spellchecker/server/client.py create mode 100644 backend/spellchecker/server/parser.py create mode 100644 backend/spellchecker/server/parser.pyc create mode 100644 backend/spellchecker/server/run_codegen.py create mode 100644 backend/spellchecker/server/server.py create mode 100644 backend/spellchecker/server/spellchecker.py create mode 100644 backend/spellchecker/server/spellchecker_pb2.py create mode 100644 backend/spellchecker/server/test.py diff --git a/backend/spellchecker/.gitignore b/backend/spellchecker/.gitignore index 386b7a5..91e6b2c 100644 --- a/backend/spellchecker/.gitignore +++ b/backend/spellchecker/.gitignore @@ -1,4 +1,3 @@ .idea -spellchecker.iml -lib -out \ No newline at end of file +libparser/build/* +!libparser/build/Makefile diff --git a/backend/spellchecker/libparser/CMakeLists.txt b/backend/spellchecker/libparser/CMakeLists.txt new file mode 100644 index 0000000..0ec6e49 --- /dev/null +++ b/backend/spellchecker/libparser/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.2) +project(parsers) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lhunspell") + +set(SOURCE_FILES + parsers/latexparser.cxx + parsers/latexparser.hxx + parsers/textparser.cxx + parsers/textparser.hxx + parser.cpp) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/parsers) + +add_library(parser SHARED ${SOURCE_FILES}) diff --git a/backend/spellchecker/libparser/build/Makefile b/backend/spellchecker/libparser/build/Makefile new file mode 100644 index 0000000..08a7e55 --- /dev/null +++ b/backend/spellchecker/libparser/build/Makefile @@ -0,0 +1,238 @@ +# CMAKE generated file: DO NOT EDIT! +# Generated by "Unix Makefiles" Generator, CMake Version 3.6 + +# Default target executed when no arguments are given to make. +default_target: all + +.PHONY : default_target + +# Allow only one "make -f Makefile2" at a time, but pass parallelism. +.NOTPARALLEL: + + +#============================================================================= +# Special targets provided by cmake. + +# Disable implicit rules so canonical targets will work. +.SUFFIXES: + + +# Remove some rules from gmake that .SUFFIXES does not remove. +SUFFIXES = + +.SUFFIXES: .hpux_make_needs_suffix_list + + +# Suppress display of executed commands. +$(VERBOSE).SILENT: + + +# A target that is always out of date. +cmake_force: + +.PHONY : cmake_force + +#============================================================================= +# Set environment variables for the build. + +# The shell in which to execute make rules. +SHELL = /bin/sh + +# The CMake executable. +CMAKE_COMMAND = /usr/bin/cmake + +# The command to remove a file. +RM = /usr/bin/cmake -E remove -f + +# Escaping for special characters. +EQUALS = = + +# The top-level source directory on which CMake was run. +CMAKE_SOURCE_DIR = /home/tener/git/papeeria/backend/spellchecker/libparser + +# The top-level build directory on which CMake was run. +CMAKE_BINARY_DIR = /home/tener/git/papeeria/backend/spellchecker/libparser/build + +#============================================================================= +# Targets provided globally by CMake. + +# Special rule for the target edit_cache +edit_cache: + @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake cache editor..." + /usr/bin/ccmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) +.PHONY : edit_cache + +# Special rule for the target edit_cache +edit_cache/fast: edit_cache + +.PHONY : edit_cache/fast + +# Special rule for the target rebuild_cache +rebuild_cache: + @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake to regenerate build system..." + /usr/bin/cmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) +.PHONY : rebuild_cache + +# Special rule for the target rebuild_cache +rebuild_cache/fast: rebuild_cache + +.PHONY : rebuild_cache/fast + +# The main all target +all: cmake_check_build_system + $(CMAKE_COMMAND) -E cmake_progress_start /home/tener/git/papeeria/backend/spellchecker/libparser/build/CMakeFiles /home/tener/git/papeeria/backend/spellchecker/libparser/build/CMakeFiles/progress.marks + $(MAKE) -f CMakeFiles/Makefile2 all + $(CMAKE_COMMAND) -E cmake_progress_start /home/tener/git/papeeria/backend/spellchecker/libparser/build/CMakeFiles 0 +.PHONY : all + +# The main clean target +clean: + $(MAKE) -f CMakeFiles/Makefile2 clean +.PHONY : clean + +# The main clean target +clean/fast: clean + +.PHONY : clean/fast + +# Prepare targets for installation. +preinstall: all + $(MAKE) -f CMakeFiles/Makefile2 preinstall +.PHONY : preinstall + +# Prepare targets for installation. +preinstall/fast: + $(MAKE) -f CMakeFiles/Makefile2 preinstall +.PHONY : preinstall/fast + +# clear depends +depend: + $(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 1 +.PHONY : depend + +#============================================================================= +# Target rules for targets named parser + +# Build rule for target. +parser: cmake_check_build_system + $(MAKE) -f CMakeFiles/Makefile2 parser +.PHONY : parser + +# fast build rule for target. +parser/fast: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/build +.PHONY : parser/fast + +parser.o: parser.cpp.o + +.PHONY : parser.o + +# target to build an object file +parser.cpp.o: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parser.cpp.o +.PHONY : parser.cpp.o + +parser.i: parser.cpp.i + +.PHONY : parser.i + +# target to preprocess a source file +parser.cpp.i: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parser.cpp.i +.PHONY : parser.cpp.i + +parser.s: parser.cpp.s + +.PHONY : parser.s + +# target to generate assembly for a file +parser.cpp.s: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parser.cpp.s +.PHONY : parser.cpp.s + +parsers/latexparser.o: parsers/latexparser.cxx.o + +.PHONY : parsers/latexparser.o + +# target to build an object file +parsers/latexparser.cxx.o: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parsers/latexparser.cxx.o +.PHONY : parsers/latexparser.cxx.o + +parsers/latexparser.i: parsers/latexparser.cxx.i + +.PHONY : parsers/latexparser.i + +# target to preprocess a source file +parsers/latexparser.cxx.i: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parsers/latexparser.cxx.i +.PHONY : parsers/latexparser.cxx.i + +parsers/latexparser.s: parsers/latexparser.cxx.s + +.PHONY : parsers/latexparser.s + +# target to generate assembly for a file +parsers/latexparser.cxx.s: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parsers/latexparser.cxx.s +.PHONY : parsers/latexparser.cxx.s + +parsers/textparser.o: parsers/textparser.cxx.o + +.PHONY : parsers/textparser.o + +# target to build an object file +parsers/textparser.cxx.o: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parsers/textparser.cxx.o +.PHONY : parsers/textparser.cxx.o + +parsers/textparser.i: parsers/textparser.cxx.i + +.PHONY : parsers/textparser.i + +# target to preprocess a source file +parsers/textparser.cxx.i: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parsers/textparser.cxx.i +.PHONY : parsers/textparser.cxx.i + +parsers/textparser.s: parsers/textparser.cxx.s + +.PHONY : parsers/textparser.s + +# target to generate assembly for a file +parsers/textparser.cxx.s: + $(MAKE) -f CMakeFiles/parser.dir/build.make CMakeFiles/parser.dir/parsers/textparser.cxx.s +.PHONY : parsers/textparser.cxx.s + +# Help Target +help: + @echo "The following are some of the valid targets for this Makefile:" + @echo "... all (the default if no target is provided)" + @echo "... clean" + @echo "... depend" + @echo "... edit_cache" + @echo "... rebuild_cache" + @echo "... parser" + @echo "... parser.o" + @echo "... parser.i" + @echo "... parser.s" + @echo "... parsers/latexparser.o" + @echo "... parsers/latexparser.i" + @echo "... parsers/latexparser.s" + @echo "... parsers/textparser.o" + @echo "... parsers/textparser.i" + @echo "... parsers/textparser.s" +.PHONY : help + + + +#============================================================================= +# Special targets to cleanup operation of make. + +# Special rule to run CMake to check the build system integrity. +# No rule that depends on this can have commands that come from listfiles +# because they might be regenerated. +cmake_check_build_system: + $(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0 +.PHONY : cmake_check_build_system + diff --git a/backend/spellchecker/libparser/parser.cpp b/backend/spellchecker/libparser/parser.cpp new file mode 100644 index 0000000..6a46879 --- /dev/null +++ b/backend/spellchecker/libparser/parser.cpp @@ -0,0 +1,47 @@ +#include "parsers/latexparser.hxx" +#include +#include +#include + +extern "C" { + + struct Token { + char* token; + }; + + std::string alphabet("qwertzuiopasdfghjklyxcvbnmQWERTZUIOPASDFGHJKLYXCVBNM"); + + LaTeXParser* latex_parser() { + return new LaTeXParser(alphabet.c_str()); + } + + void delete_parser(LaTeXParser *parser) { + delete parser; + } + + void put_line(LaTeXParser* parser, char* line) { + parser->put_line(line); + } + + int next_token(LaTeXParser *parser, Token *token) { + parser->set_url_checking(true); + std::string next; + if (parser->next_token(next)) { + token->token = new char[next.size()+1]; + std::copy(next.begin(), next.end(), token->token); + token->token[next.size()] = '\0'; + return 1; + } + else { + token->token = NULL; + return 0; + } + } + + void free_token(Token *token) { + if (token->token) { + delete [] token->token; + } + } +} + diff --git a/backend/spellchecker/libparser/parsers/latexparser.cxx b/backend/spellchecker/libparser/parsers/latexparser.cxx new file mode 100644 index 0000000..ece6405 --- /dev/null +++ b/backend/spellchecker/libparser/parsers/latexparser.cxx @@ -0,0 +1,264 @@ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Hunspell, based on MySpell. + * + * The Initial Developers of the Original Code are + * Kevin Hendricks (MySpell) and Németh László (Hunspell). + * Portions created by the Initial Developers are Copyright (C) 2002-2005 + * the Initial Developers. All Rights Reserved. + * + * Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno, + * Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád, + * Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter, + * Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls, + * Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#include +#include +#include +#include + +#include +#include "latexparser.hxx" + +#ifndef W32 +using namespace std; +#endif + +static struct { + const char* pat[2]; + int arg; +} PATTERN[] = {{{"\\(", "\\)"}, 0}, + {{"$$", "$$"}, 0}, + {{"$", "$"}, 0}, + {{"\\begin{math}", "\\end{math}"}, 0}, + {{"\\[", "\\]"}, 0}, + {{"\\begin{displaymath}", "\\end{displaymath}"}, 0}, + {{"\\begin{equation}", "\\end{equation}"}, 0}, + {{"\\begin{equation*}", "\\end{equation*}"}, 0}, + {{"\\cite", NULL}, 1}, + {{"\\nocite", NULL}, 1}, + {{"\\index", NULL}, 1}, + {{"\\label", NULL}, 1}, + {{"\\ref", NULL}, 1}, + {{"\\pageref", NULL}, 1}, + {{"\\autoref", NULL}, 1}, + {{"\\parbox", NULL}, 1}, + {{"\\begin{verbatim}", "\\end{verbatim}"}, 0}, + {{"\\verb+", "+"}, 0}, + {{"\\verb|", "|"}, 0}, + {{"\\verb#", "#"}, 0}, + {{"\\verb*", "*"}, 0}, + {{"\\documentstyle", "\\begin{document}"}, 0}, + {{"\\documentclass", "\\begin{document}"}, 0}, + // { { "\\documentclass", NULL } , 1 }, + {{"\\usepackage", NULL}, 1}, + {{"\\includeonly", NULL}, 1}, + {{"\\include", NULL}, 1}, + {{"\\input", NULL}, 1}, + {{"\\vspace", NULL}, 1}, + {{"\\setlength", NULL}, 2}, + {{"\\addtolength", NULL}, 2}, + {{"\\settowidth", NULL}, 2}, + {{"\\rule", NULL}, 2}, + {{"\\hspace", NULL}, 1}, + {{"\\vspace", NULL}, 1}, + {{"\\\\[", "]"}, 0}, + {{"\\pagebreak[", "]"}, 0}, + {{"\\nopagebreak[", "]"}, 0}, + {{"\\enlargethispage", NULL}, 1}, + {{"\\begin{tabular}", NULL}, 1}, + {{"\\addcontentsline", NULL}, 2}, + {{"\\begin{thebibliography}", NULL}, 1}, + {{"\\bibliography", NULL}, 1}, + {{"\\bibliographystyle", NULL}, 1}, + {{"\\bibitem", NULL}, 1}, + {{"\\begin", NULL}, 1}, + {{"\\end", NULL}, 1}, + {{"\\pagestyle", NULL}, 1}, + {{"\\pagenumbering", NULL}, 1}, + {{"\\thispagestyle", NULL}, 1}, + {{"\\newtheorem", NULL}, 2}, + {{"\\newcommand", NULL}, 2}, + {{"\\renewcommand", NULL}, 2}, + {{"\\setcounter", NULL}, 2}, + {{"\\addtocounter", NULL}, 1}, + {{"\\stepcounter", NULL}, 1}, + {{"\\selectlanguage", NULL}, 1}, + {{"\\inputencoding", NULL}, 1}, + {{"\\hyphenation", NULL}, 1}, + {{"\\definecolor", NULL}, 3}, + {{"\\color", NULL}, 1}, + {{"\\textcolor", NULL}, 1}, + {{"\\pagecolor", NULL}, 1}, + {{"\\colorbox", NULL}, 2}, + {{"\\fcolorbox", NULL}, 2}, + {{"\\declaregraphicsextensions", NULL}, 1}, + {{"\\psfig", NULL}, 1}, + {{"\\url", NULL}, 1}, + {{"\\eqref", NULL}, 1}, + {{"\\vskip", NULL}, 1}, + {{"\\vglue", NULL}, 1}, + {{"\'\'", NULL}, 1}}; + +#define PATTERN_LEN (sizeof(PATTERN) / sizeof(PATTERN[0])) + +LaTeXParser::LaTeXParser(const char* wordchars) + : pattern_num(0), depth(0), arg(0), opt(0) { + init(wordchars); +} + +LaTeXParser::LaTeXParser(const w_char* wordchars, int len) + : pattern_num(0), depth(0), arg(0), opt(0) { + init(wordchars, len); +} + +LaTeXParser::~LaTeXParser() {} + +int LaTeXParser::look_pattern(int col) { + for (unsigned int i = 0; i < PATTERN_LEN; i++) { + const char* j = line[actual].c_str() + head; + const char* k = PATTERN[i].pat[col]; + if (!k) + continue; + while ((*k != '\0') && (tolower(*j) == *k)) { + j++; + k++; + } + if (*k == '\0') + return i; + } + return -1; +} + +/* + * LaTeXParser + * + * state 0: not wordchar + * state 1: wordchar + * state 2: comments + * state 3: commands + * state 4: commands with arguments + * state 5: % comment + * + */ + +bool LaTeXParser::next_token(std::string& t) { + t.clear(); + int i; + int slash = 0; + int apostrophe; + for (;;) { + // fprintf(stderr,"depth: %d, state: %d, , arg: %d, token: + // %s\n",depth,state,arg,line[actual]+head); + + switch (state) { + case 0: // non word chars + if ((pattern_num = look_pattern(0)) != -1) { + if (PATTERN[pattern_num].pat[1]) { + state = 2; + } else { + state = 4; + depth = 0; + arg = 0; + opt = 1; + } + head += strlen(PATTERN[pattern_num].pat[0]) - 1; +// } else if (line[actual][head] == '%') { +// state = 5; + } else if (is_wordchar(line[actual].c_str() + head)) { + state = 1; + token = head; + } else if (line[actual][head] == '\\') { + if (line[actual][head + 1] == '\\' || // \\ (linebreak) + (line[actual][head + 1] == '$') || // \$ (dollar sign) + (line[actual][head + 1] == '%')) { // \% (percent) + head++; + break; + } + state = 3; + } + break; + case 1: // wordchar + apostrophe = 0; + if (!is_wordchar(line[actual].c_str() + head) || + (line[actual][head] == '\'' && line[actual][head + 1] == '\'' && + ++apostrophe)) { + state = 0; + bool ok = alloc_token(token, &head, t); + if (apostrophe) + head += 2; + if (ok) + return true; + } + break; + case 2: // comment, labels, etc + if (((i = look_pattern(1)) != -1) && + (strcmp(PATTERN[i].pat[1], PATTERN[pattern_num].pat[1]) == 0)) { + state = 0; + head += strlen(PATTERN[pattern_num].pat[1]) - 1; + } + break; + case 3: // command + if ((tolower(line[actual][head]) < 'a') || + (tolower(line[actual][head]) > 'z')) { + state = 0; + head--; + } + break; + case 4: // command with arguments + if (slash && (line[actual][head] != '\0')) { + slash = 0; + head++; + break; + } else if (line[actual][head] == '\\') { + slash = 1; + } else if ((line[actual][head] == '{') || + ((opt) && (line[actual][head] == '['))) { + depth++; + opt = 0; + } else if (line[actual][head] == '}') { + depth--; + if (depth == 0) { + opt = 1; + arg++; + } + if (((depth == 0) && (arg == PATTERN[pattern_num].arg)) || + (depth < 0)) { + state = 0; // XXX not handles the last optional arg. + } + } else if (line[actual][head] == ']') + depth--; + } // case + if (next_char(line[actual].c_str(), &head)) { + if (state == 5) + state = 0; + return false; + } + } +} diff --git a/backend/spellchecker/libparser/parsers/latexparser.hxx b/backend/spellchecker/libparser/parsers/latexparser.hxx new file mode 100644 index 0000000..3289da0 --- /dev/null +++ b/backend/spellchecker/libparser/parsers/latexparser.hxx @@ -0,0 +1,76 @@ +/* + * parser classes for MySpell + * + * implemented: text, HTML, TeX + * + * Copyright (C) 2002, Laszlo Nemeth + * + */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Hunspell, based on MySpell. + * + * The Initial Developers of the Original Code are + * Kevin Hendricks (MySpell) and Németh László (Hunspell). + * Portions created by the Initial Developers are Copyright (C) 2002-2005 + * the Initial Developers. All Rights Reserved. + * + * Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno, + * Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád, + * Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter, + * Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls, + * Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#ifndef _LATEXPARSER_HXX_ +#define _LATEXPARSER_HXX_ + +#include "textparser.hxx" + +/* + * HTML Parser + * + */ + +class LaTeXParser : public TextParser { + int pattern_num; // number of comment + int depth; // depth of blocks + int arg; // arguments's number + int opt; // optional argument attrib. + + public: + explicit LaTeXParser(const char* wc); + LaTeXParser(const w_char* wordchars, int len); + virtual ~LaTeXParser(); + + virtual bool next_token(std::string&); + + private: + int look_pattern(int col); +}; + +#endif diff --git a/backend/spellchecker/libparser/parsers/textparser.cxx b/backend/spellchecker/libparser/parsers/textparser.cxx new file mode 100644 index 0000000..44a0d63 --- /dev/null +++ b/backend/spellchecker/libparser/parsers/textparser.cxx @@ -0,0 +1,308 @@ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Hunspell, based on MySpell. + * + * The Initial Developers of the Original Code are + * Kevin Hendricks (MySpell) and Németh László (Hunspell). + * Portions created by the Initial Developers are Copyright (C) 2002-2005 + * the Initial Developers. All Rights Reserved. + * + * Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno, + * Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád, + * Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter, + * Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls, + * Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#include +#include +#include +#include + +#include +#include "textparser.hxx" + +#include + +#ifndef W32 +using namespace std; +#endif + +// ISO-8859-1 HTML character entities + +static const char* LATIN1[] = { + "À", "Ã", "Å", "Æ", "È", "Ê", + "Ì", "Ï", "Ð", "Ñ", "Ò", "Ø", + "Ù", "Þ", "à", "ã", "å", "æ", + "è", "ê", "ì", "ï", "ð", "ñ", + "ò", "ø", "ù", "þ", "ÿ"}; + +#define LATIN1_LEN (sizeof(LATIN1) / sizeof(char*)) + +#define ENTITY_APOS "'" +#define UTF8_APOS "\xe2\x80\x99" +#define APOSTROPHE "'" + +TextParser::TextParser() { + init((char*)NULL); +} + +TextParser::TextParser(const char* wordchars) { + init(wordchars); +} + +TextParser::TextParser(const w_char* wordchars, int len) { + init(wordchars, len); +} + +TextParser::~TextParser() {} + +int TextParser::is_wordchar(const char* w) { + if (*w == '\0') + return 0; + if (utf8) { + std::vector wc; + unsigned short idx; + u8_u16(wc, w); + if (wc.empty()) + return 0; + idx = (wc[0].h << 8) + wc[0].l; + return (unicodeisalpha(idx) || + (wordchars_utf16 && + std::binary_search(wordchars_utf16, wordchars_utf16 + wclen, wc[0]))); + } else { + return wordcharacters[(*w + 256) % 256]; + } +} + +const char* TextParser::get_latin1(const char* s) { + if (s[0] == '&') { + unsigned int i = 0; + while ((i < LATIN1_LEN) && strncmp(LATIN1[i], s, strlen(LATIN1[i]))) + i++; + if (i != LATIN1_LEN) + return LATIN1[i]; + } + return NULL; +} + +void TextParser::init(const char* wordchars) { + actual = 0; + head = 0; + token = 0; + state = 0; + utf8 = 0; + checkurl = 0; + wordchars_utf16 = NULL; + wclen = 0; + unsigned int j; + for (j = 0; j < 256; j++) { + wordcharacters[j] = 0; + } + if (!wordchars) + wordchars = "qwertzuiopasdfghjklyxcvbnmQWERTZUIOPASDFGHJKLYXCVBNM"; + for (j = 0; j < strlen(wordchars); j++) { + wordcharacters[(wordchars[j] + 256) % 256] = 1; + } +} + +void TextParser::init(const w_char* wc, int len) { + actual = 0; + head = 0; + token = 0; + state = 0; + utf8 = 1; + checkurl = 0; + wordchars_utf16 = wc; + wclen = len; +} + +int TextParser::next_char(const char* ln, size_t* pos) { + if (*(ln + *pos) == '\0') + return 1; + if (utf8) { + if (*(ln + *pos) >> 7) { + // jump to next UTF-8 character + for ((*pos)++; (*(ln + *pos) & 0xc0) == 0x80; (*pos)++) + ; + } else { + (*pos)++; + } + } else + (*pos)++; + return 0; +} + +void TextParser::put_line(const char* word) { + actual = (actual + 1) % MAXPREVLINE; + line[actual].assign(word); + token = 0; + head = 0; + check_urls(); +} + +std::string TextParser::get_prevline(int n) const { + return line[(actual + MAXPREVLINE - n) % MAXPREVLINE]; +} + +std::string TextParser::get_line() const { + return get_prevline(0); +} + +bool TextParser::next_token(std::string &t) { + const char* latin1; + + for (;;) { + switch (state) { + case 0: // non word chars + if (is_wordchar(line[actual].c_str() + head)) { + state = 1; + token = head; + } else if ((latin1 = get_latin1(line[actual].c_str() + head))) { + state = 1; + token = head; + head += strlen(latin1); + } + break; + case 1: // wordchar + if ((latin1 = get_latin1(line[actual].c_str() + head))) { + head += strlen(latin1); + } else if ((is_wordchar((char*)APOSTROPHE) || + (is_utf8() && is_wordchar((char*)UTF8_APOS))) && + !line[actual].empty() && line[actual][head] == '\'' && + is_wordchar(line[actual].c_str() + head + 1)) { + head++; + } else if (is_utf8() && + is_wordchar((char*)APOSTROPHE) && // add Unicode apostrophe + // to the WORDCHARS, if + // needed + strncmp(line[actual].c_str() + head, UTF8_APOS, strlen(UTF8_APOS)) == + 0 && + is_wordchar(line[actual].c_str() + head + strlen(UTF8_APOS))) { + head += strlen(UTF8_APOS) - 1; + } else if (!is_wordchar(line[actual].c_str() + head)) { + state = 0; + if (alloc_token(token, &head, t)) + return true; + } + break; + } + if (next_char(line[actual].c_str(), &head)) + return false; + } +} + +size_t TextParser::get_tokenpos() { + return token; +} + +int TextParser::change_token(const char* word) { + if (word) { + std::string remainder(line[actual].substr(head)); + line[actual].resize(token); + line[actual].append(word); + line[actual].append(remainder); + head = token; + return 1; + } + return 0; +} + +void TextParser::check_urls() { + urlline.resize(line[actual].size() + 1); + int url_state = 0; + size_t url_head = 0; + size_t url_token = 0; + int url = 0; + for (;;) { + switch (url_state) { + case 0: // non word chars + if (is_wordchar(line[actual].c_str() + url_head)) { + url_state = 1; + url_token = url_head; + // Unix path + } else if (line[actual][url_head] == '/') { + url_state = 1; + url_token = url_head; + url = 1; + } + break; + case 1: // wordchar + char ch = line[actual][url_head]; + // e-mail address + if ((ch == '@') || + // MS-DOS, Windows path + (strncmp(line[actual].c_str() + url_head, ":\\", 2) == 0) || + // URL + (strncmp(line[actual].c_str() + url_head, "://", 3) == 0)) { + url = 1; + } else if (!(is_wordchar(line[actual].c_str() + url_head) || (ch == '-') || + (ch == '_') || (ch == '\\') || (ch == '.') || + (ch == ':') || (ch == '/') || (ch == '~') || (ch == '%') || + (ch == '*') || (ch == '$') || (ch == '[') || (ch == ']') || + (ch == '?') || (ch == '!') || + ((ch >= '0') && (ch <= '9')))) { + url_state = 0; + if (url == 1) { + for (size_t i = url_token; i < url_head; ++i) { + urlline[i] = true; + } + } + url = 0; + } + break; + } + urlline[url_head] = false; + if (next_char(line[actual].c_str(), &url_head)) + return; + } +} + +int TextParser::get_url(size_t token_pos, size_t* hd) { + for (size_t i = *hd; i < line[actual].size() && urlline[i]; i++, (*hd)++) + ; + return checkurl ? 0 : urlline[token_pos]; +} + +void TextParser::set_url_checking(int check) { + checkurl = check; +} + +bool TextParser::alloc_token(size_t tokn, size_t* hd, std::string& t) { + size_t url_head = *hd; + if (get_url(tokn, &url_head)) + return false; + t = line[actual].substr(tokn, *hd - tokn); + // remove colon for Finnish and Swedish language + if (!t.empty() && t[t.size() - 1] == ':') { + t.resize(t.size() - 1); + if (t.empty()) { + return false; + } + } + return true; +} diff --git a/backend/spellchecker/libparser/parsers/textparser.hxx b/backend/spellchecker/libparser/parsers/textparser.hxx new file mode 100644 index 0000000..6110f28 --- /dev/null +++ b/backend/spellchecker/libparser/parsers/textparser.hxx @@ -0,0 +1,109 @@ +/* + * parser classes for MySpell + * + * implemented: text, HTML, TeX + * + * Copyright (C) 2002, Laszlo Nemeth + * + */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Hunspell, based on MySpell. + * + * The Initial Developers of the Original Code are + * Kevin Hendricks (MySpell) and Németh László (Hunspell). + * Portions created by the Initial Developers are Copyright (C) 2002-2005 + * the Initial Developers. All Rights Reserved. + * + * Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno, + * Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád, + * Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter, + * Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls, + * Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#ifndef _TEXTPARSER_HXX_ +#define _TEXTPARSER_HXX_ + +// set sum of actual and previous lines +#define MAXPREVLINE 4 + +#ifndef MAXLNLEN +#define MAXLNLEN 8192 +#endif + +#include + +#include + +/* + * Base Text Parser + * + */ + +class TextParser { + protected: + int wordcharacters[256]; // for detection of the word boundaries + std::string line[MAXPREVLINE]; // parsed and previous lines + std::vector urlline; // mask for url detection + int checkurl; + int actual; // actual line + size_t head; // head position + size_t token;// begin of token + int state; // state of automata + int utf8; // UTF-8 character encoding + int next_char(const char* line, size_t* pos); + const w_char* wordchars_utf16; + int wclen; + + public: + TextParser(); + TextParser(const w_char* wordchars, int len); + explicit TextParser(const char* wc); + void init(const char*); + void init(const w_char* wordchars, int len); + virtual ~TextParser(); + + void put_line(const char* line); + std::string get_line() const; + std::string get_prevline(int n) const; + virtual bool next_token(std::string&); + virtual int change_token(const char* word); + void set_url_checking(int check); + + size_t get_tokenpos(); + int is_wordchar(const char* w); + inline int is_utf8() { return utf8; } + const char* get_latin1(const char* s); + char* next_char(); + int tokenize_urls(); + void check_urls(); + int get_url(size_t token_pos, size_t* head); + bool alloc_token(size_t token, size_t* head, std::string& out); +}; + +#endif diff --git a/backend/spellchecker/proto/spellchecker.proto b/backend/spellchecker/proto/spellchecker.proto new file mode 100644 index 0000000..b69d3cd --- /dev/null +++ b/backend/spellchecker/proto/spellchecker.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; +option java_package = "com.papeeria"; + +service Spellcheck { + rpc CheckText(Text) returns (Suggestions) {} +} + +message Text { + bytes text = 1; + repeated string language = 2; +} + +message Suggestions { + string json = 1; +} diff --git a/backend/spellchecker/server/client.py b/backend/spellchecker/server/client.py new file mode 100644 index 0000000..3a0dcf4 --- /dev/null +++ b/backend/spellchecker/server/client.py @@ -0,0 +1,12 @@ +import grpc + +import spellchecker_pb2 + + +channel = grpc.insecure_channel('localhost:50051') +stub = spellchecker_pb2.SpellcheckStub(channel) +Text = spellchecker_pb2.Text() +Text.text = b"test line wth errorz" +Text.language.extend(["en_US", "ru_RU"]) +response = stub.CheckText(Text) +print(response.json) diff --git a/backend/spellchecker/server/parser.py b/backend/spellchecker/server/parser.py new file mode 100644 index 0000000..502281c --- /dev/null +++ b/backend/spellchecker/server/parser.py @@ -0,0 +1,70 @@ +from ctypes import cdll, c_char_p, pointer, Structure +from weakref import proxy + + +class Parser: + """ + Class wraps CFFI for LaTeX parser from Hunspell source code. + """ + def __init__(self): + # note: these fields' names doesn't start with "_" since deleting of this + # object isn't so straightforward -- for some reason interpreter destroys + # these "private" fields before __del__ is invoked. + self.parser_lib_ = cdll.LoadLibrary("../libparser/build/libparser.so") + self.parser_ = self.parser_lib_.latex_parser() + + def __del__(self): + self.parser_lib_.delete_parser(self.parser_) + + def put_line(self, line: str): + """ + Put the line into the parser instance so to tokenize it. + + :param line: A line from text to tokenize. + """ + self.parser_lib_.put_line(self.parser_, bytes(line, encoding="UTF-8")) + + def next_token(self): + """ + Get next token from the line. + + :return: Token from the line that was inserted into the parser. + """ + # actually, we have to put a pointer to pointer to char so + # to allocate new memory for the token + if self.parser_lib_.next_token(self.parser_, self._token_ptr_): + word = str(self._token_ptr_.contents.token, encoding="UTF-8") + # also we have to free the allocated memory manually + self.parser_lib_.free_token(self._token_ptr_) + return word + else: + return None + + def tokenize(self, line: str): + """ + Tokenize line and return as generator. + Generator function simpifies the interface and makes parser rather + lightweight and easy to use. + + :param line: A line to tokenize. + :return: generator object that extracts and yields tokens from the line. + """ + self.put_line(line) + word = True + while word: + if self.parser_lib_.next_token(self.parser_, self._token_ptr_): + word = str(self._token_ptr_.contents.token, encoding="UTF-8") + # also we have to free the allocated memory manually + self.parser_lib_.free_token(self._token_ptr_) + yield word + else: + return + + class Token(Structure): + """ + We need this class just to point to pointer to char. + It could be just char**, but this way seemed more elegant. + """ + _fields_ = [("token", c_char_p)] + + _token_ptr_ = pointer(Token()) diff --git a/backend/spellchecker/server/parser.pyc b/backend/spellchecker/server/parser.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7ff4966d6326d31dfa5725a216505c0d6981c76 GIT binary patch literal 2155 zcmaJ?ZBHac5U!bBhFugTo}MxI()z({F1i=L8Dfkc2}*+G9Jp||Uplihy)fLoY`S{` zYt%2?7k`sK%O9XmRS)ce;Rd?5db+y0>UpYadjD*0y!YzwlbpJzhuX;@m)jgW@sP5CGPZ4XxRIO2+&?LdSM@2$KpI#AN5xtnKQ?W)zqjgv-|HU)v zL-IW3MOFC$3pzi`te$&Tnzv=+jrBvwN8T=Szp!Q$Vdd9&+qRcWpFnz=S!e7mzn|c> zJHcd#-Gy&(7XiQ*cnm*T=o5ToVXbo$Zo@^hPBR=yZX6$w8Il=r6jz9TgVGR%`xO^# zC_KAW=9~(Cs0*9To!Z^o+gG#Js)Ovj!MbWv8tYCU%ChzUwD$iIc@pv{Vda%wZX6t=W3Or^Dd8+$q!(J-fl zp}7&Vc(2O5x$G2#^K0d_kKI5oOU_iY5q?P~$4L;Zj_1p8PM1yTwdNhizESc)jKApr z0Dgi^4qq??be$}qao!%uwTF_BDW_h__WUL$1ZBMGd`vlT(M>s&``i-~524@~T#9o3 z;lar&jD88>#-Tur2XjI&lJhExu41X6Bk{@o34KWpPPV5Dul$)&RoNJY5@}`CC0N-w zpEXD*hu8f7(lq67(DUPs8w)#7&T28RqkasABr`z`Y_E=mjO0rNa?>D9%jVQ^{XIM0 z`}*4sGl7ez`ND(s#t2eUrqiW2PO?lelX_>3d%@d+nU636c!{vd;Bit2c+Vi*0gsnp zGba1Xbx}wOoFmBrsv!DDC>&^pS_td|^CNTt83F4{n7={lC*xHUKmm}Ue6O*89Z+k~ zoSGeV%y{2c13Q^l7nuWBXj9%4cS&2JoW6_8Uq_ddJ0#-GXd z)82+w+M;&ZO0Fyek_+r z>^Z;ZhmAhLa9n(My0GcdET&WR(mwZ6R&{@2x`F+~0CfQ6&q}9qX|CrBC#fl41^a*DSfJ^D&brEp;JB3mkFOpP1r)*6KxKg}9c2xmG!>4)%U z#97|8nkK6anj1&0b=?+=ikI9f(D&M0>r}o=TmpJlnyPSmg`r5dJ6O66&2KOqr*+a# zq7jpse#{dimVU|;cc9RaxGGXcKV_2Tcq+WTkL8=M;xMG6z5{`;8g34I!@+Rt;rd}X xIlzYW>D({p#z~WxfWj4gqr`e)vX|D(q*JGC_Kg<{bl(NAzGrwO)-R&1_&;}SrK11< literal 0 HcmV?d00001 diff --git a/backend/spellchecker/server/run_codegen.py b/backend/spellchecker/server/run_codegen.py new file mode 100644 index 0000000..0fc4f5d --- /dev/null +++ b/backend/spellchecker/server/run_codegen.py @@ -0,0 +1,12 @@ +from grpc.tools import protoc + + +protoc.main ( + ( + '', + '-I../proto', + '--python_out=.', + '--grpc_python_out=.', + '../proto/spellchecker.proto' + ) +) diff --git a/backend/spellchecker/server/server.py b/backend/spellchecker/server/server.py new file mode 100644 index 0000000..39d09e8 --- /dev/null +++ b/backend/spellchecker/server/server.py @@ -0,0 +1,44 @@ +import time + +import grpc +from concurrent import futures + +from spellchecker import Spellchecker +import spellchecker_pb2 + + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class SpellcheckerServicer(spellchecker_pb2.SpellcheckServicer): + """ + gRPC service to check text that comes from stubs. + """ + def __init__(self): + self._checker_ = Spellchecker("/usr/share/hunspell", "en_US") + + def CheckText(self, request, context): + text = str(request.text, encoding="UTF-8") + languages = [lang for lang in request.language] + json = self._checker_.check_text(text, languages) + return spellchecker_pb2.Suggestions(json=json) + + +def serve(max_workers: int = 2): + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=max_workers) + ) + spellchecker_pb2.add_SpellcheckServicer_to_server( + SpellcheckerServicer(), + server + ) + server.add_insecure_port('[::]:50051') + server.start() + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + +if __name__ == '__main__': + serve() diff --git a/backend/spellchecker/server/spellchecker.py b/backend/spellchecker/server/spellchecker.py new file mode 100644 index 0000000..347862a --- /dev/null +++ b/backend/spellchecker/server/spellchecker.py @@ -0,0 +1,62 @@ +import os.path + +import hunspell +import ujson + +from parser import Parser + + +class Spellchecker: + """ + Class contains multiple hunspell instances (one per language) and provides + text spellchecking function using all chosen dictionaries. + """ + def __init__(self, path: str = '', language: str ='', log: object = None): + self._hunspell_instances = {} + self._parser = Parser() + self._log = log + + if path is not '': + self.add_dictionary(path, language) + + def add_dictionary(self, path: str, language: str) -> None: + """ + Create new hunspell instance and put into instances dictionary. + + :param path: Path to .dic and .aff files. + :param language: Language that corresponds .dic and .aff files. + """ + if language in self._hunspell_instances: + return + + dictionary = "{}/{}.dic".format(path, language) + if not os.path.isfile(dictionary): + if self._log: + self._log.write("File not found: {}".format(dictionary)) + return + affix = "{}/{}.aff".format(path, language) + if not os.path.isfile(affix): + if self._log: + self._log.write("File not found: {}".format(affix)) + return + + self._hunspell_instances[language] = hunspell.HunSpell(dictionary, affix) + + def check_text(self, text: str, languages) -> str: + """ + Check text using chosen languages and return JSON with suggestions. + + :param text: Text to check. + :param languages: Languages list. + :return: JSON with suggestions for misspelled words. + """ + suggestions = {} + hunspells = (self._hunspell_instances[lang] for lang in languages if lang in self._hunspell_instances) + tokens = self._parser.tokenize(text) + + for h in hunspells: + for token in tokens: + if not h.spell(token): + suggestions[token] = h.suggest(token) + + return ujson.dumps(suggestions) diff --git a/backend/spellchecker/server/spellchecker_pb2.py b/backend/spellchecker/server/spellchecker_pb2.py new file mode 100644 index 0000000..be0d735 --- /dev/null +++ b/backend/spellchecker/server/spellchecker_pb2.py @@ -0,0 +1,197 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: spellchecker.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='spellchecker.proto', + package='', + syntax='proto3', + serialized_pb=_b('\n\x12spellchecker.proto\"&\n\x04Text\x12\x0c\n\x04text\x18\x01 \x01(\x0c\x12\x10\n\x08language\x18\x02 \x03(\t\"\x1b\n\x0bSuggestions\x12\x0c\n\x04json\x18\x01 \x01(\t20\n\nSpellcheck\x12\"\n\tCheckText\x12\x05.Text\x1a\x0c.Suggestions\"\x00\x42\x15\n\x13\x63om.papeeria.clientb\x06proto3') +) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + + + +_TEXT = _descriptor.Descriptor( + name='Text', + full_name='Text', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='text', full_name='Text.text', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='language', full_name='Text.language', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=22, + serialized_end=60, +) + + +_SUGGESTIONS = _descriptor.Descriptor( + name='Suggestions', + full_name='Suggestions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='json', full_name='Suggestions.json', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=62, + serialized_end=89, +) + +DESCRIPTOR.message_types_by_name['Text'] = _TEXT +DESCRIPTOR.message_types_by_name['Suggestions'] = _SUGGESTIONS + +Text = _reflection.GeneratedProtocolMessageType('Text', (_message.Message,), dict( + DESCRIPTOR = _TEXT, + __module__ = 'spellchecker_pb2' + # @@protoc_insertion_point(class_scope:Text) + )) +_sym_db.RegisterMessage(Text) + +Suggestions = _reflection.GeneratedProtocolMessageType('Suggestions', (_message.Message,), dict( + DESCRIPTOR = _SUGGESTIONS, + __module__ = 'spellchecker_pb2' + # @@protoc_insertion_point(class_scope:Suggestions) + )) +_sym_db.RegisterMessage(Suggestions) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.papeeria.client')) +import grpc +from grpc.beta import implementations as beta_implementations +from grpc.beta import interfaces as beta_interfaces +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities + + +class SpellcheckStub(object): + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.CheckText = channel.unary_unary( + '/Spellcheck/CheckText', + request_serializer=Text.SerializeToString, + response_deserializer=Suggestions.FromString, + ) + + +class SpellcheckServicer(object): + + def CheckText(self, request, context): + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_SpellcheckServicer_to_server(servicer, server): + rpc_method_handlers = { + 'CheckText': grpc.unary_unary_rpc_method_handler( + servicer.CheckText, + request_deserializer=Text.FromString, + response_serializer=Suggestions.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'Spellcheck', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + +class BetaSpellcheckServicer(object): + def CheckText(self, request, context): + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + + +class BetaSpellcheckStub(object): + def CheckText(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + raise NotImplementedError() + CheckText.future = None + + +def beta_create_Spellcheck_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + request_deserializers = { + ('Spellcheck', 'CheckText'): Text.FromString, + } + response_serializers = { + ('Spellcheck', 'CheckText'): Suggestions.SerializeToString, + } + method_implementations = { + ('Spellcheck', 'CheckText'): face_utilities.unary_unary_inline(servicer.CheckText), + } + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + + +def beta_create_Spellcheck_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + request_serializers = { + ('Spellcheck', 'CheckText'): Text.SerializeToString, + } + response_deserializers = { + ('Spellcheck', 'CheckText'): Suggestions.FromString, + } + cardinalities = { + 'CheckText': cardinality.Cardinality.UNARY_UNARY, + } + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'Spellcheck', cardinalities, options=stub_options) +# @@protoc_insertion_point(module_scope) diff --git a/backend/spellchecker/server/test.py b/backend/spellchecker/server/test.py new file mode 100644 index 0000000..e2c6612 --- /dev/null +++ b/backend/spellchecker/server/test.py @@ -0,0 +1,13 @@ +from spellchecker import Spellchecker +import time +import hunspell + +with open("/home/tener/sandbox/papeeria-hunspell/testcase/smalltest") as tests: + t = "" + for l in tests: + t += l + "\n" + s = Spellchecker("/usr/share/hunspell", "en_US") + t1 = time.time() + x = s.check_text(t, ["en_US"]) + print(time.time() - t1) + print(x) From 039864a717761ec8467daf3bc685b615c2bb674d Mon Sep 17 00:00:00 2001 From: teners Date: Wed, 31 Aug 2016 17:06:39 +0300 Subject: [PATCH 09/21] Deleted .pyc file and added that extension to gitignore --- backend/spellchecker/.gitignore | 2 ++ backend/spellchecker/server/parser.pyc | Bin 2155 -> 0 bytes 2 files changed, 2 insertions(+) delete mode 100644 backend/spellchecker/server/parser.pyc diff --git a/backend/spellchecker/.gitignore b/backend/spellchecker/.gitignore index 91e6b2c..a4167ad 100644 --- a/backend/spellchecker/.gitignore +++ b/backend/spellchecker/.gitignore @@ -1,3 +1,5 @@ +*.pyc +__pycache__ .idea libparser/build/* !libparser/build/Makefile diff --git a/backend/spellchecker/server/parser.pyc b/backend/spellchecker/server/parser.pyc deleted file mode 100644 index b7ff4966d6326d31dfa5725a216505c0d6981c76..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2155 zcmaJ?ZBHac5U!bBhFugTo}MxI()z({F1i=L8Dfkc2}*+G9Jp||Uplihy)fLoY`S{` zYt%2?7k`sK%O9XmRS)ce;Rd?5db+y0>UpYadjD*0y!YzwlbpJzhuX;@m)jgW@sP5CGPZ4XxRIO2+&?LdSM@2$KpI#AN5xtnKQ?W)zqjgv-|HU)v zL-IW3MOFC$3pzi`te$&Tnzv=+jrBvwN8T=Szp!Q$Vdd9&+qRcWpFnz=S!e7mzn|c> zJHcd#-Gy&(7XiQ*cnm*T=o5ToVXbo$Zo@^hPBR=yZX6$w8Il=r6jz9TgVGR%`xO^# zC_KAW=9~(Cs0*9To!Z^o+gG#Js)Ovj!MbWv8tYCU%ChzUwD$iIc@pv{Vda%wZX6t=W3Or^Dd8+$q!(J-fl zp}7&Vc(2O5x$G2#^K0d_kKI5oOU_iY5q?P~$4L;Zj_1p8PM1yTwdNhizESc)jKApr z0Dgi^4qq??be$}qao!%uwTF_BDW_h__WUL$1ZBMGd`vlT(M>s&``i-~524@~T#9o3 z;lar&jD88>#-Tur2XjI&lJhExu41X6Bk{@o34KWpPPV5Dul$)&RoNJY5@}`CC0N-w zpEXD*hu8f7(lq67(DUPs8w)#7&T28RqkasABr`z`Y_E=mjO0rNa?>D9%jVQ^{XIM0 z`}*4sGl7ez`ND(s#t2eUrqiW2PO?lelX_>3d%@d+nU636c!{vd;Bit2c+Vi*0gsnp zGba1Xbx}wOoFmBrsv!DDC>&^pS_td|^CNTt83F4{n7={lC*xHUKmm}Ue6O*89Z+k~ zoSGeV%y{2c13Q^l7nuWBXj9%4cS&2JoW6_8Uq_ddJ0#-GXd z)82+w+M;&ZO0Fyek_+r z>^Z;ZhmAhLa9n(My0GcdET&WR(mwZ6R&{@2x`F+~0CfQ6&q}9qX|CrBC#fl41^a*DSfJ^D&brEp;JB3mkFOpP1r)*6KxKg}9c2xmG!>4)%U z#97|8nkK6anj1&0b=?+=ikI9f(D&M0>r}o=TmpJlnyPSmg`r5dJ6O66&2KOqr*+a# zq7jpse#{dimVU|;cc9RaxGGXcKV_2Tcq+WTkL8=M;xMG6z5{`;8g34I!@+Rt;rd}X xIlzYW>D({p#z~WxfWj4gqr`e)vX|D(q*JGC_Kg<{bl(NAzGrwO)-R&1_&;}SrK11< From 54a60d240a9514564cd73a7a997226e74533e83a Mon Sep 17 00:00:00 2001 From: teners Date: Wed, 31 Aug 2016 17:26:13 +0300 Subject: [PATCH 10/21] Added author's name, removed unneeded test.py --- backend/spellchecker/server/client.py | 1 + backend/spellchecker/server/parser.py | 2 +- backend/spellchecker/server/run_codegen.py | 2 +- backend/spellchecker/server/server.py | 1 + backend/spellchecker/server/spellchecker.py | 1 + backend/spellchecker/server/test.py | 13 ------------- 6 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 backend/spellchecker/server/test.py diff --git a/backend/spellchecker/server/client.py b/backend/spellchecker/server/client.py index 3a0dcf4..b1905af 100644 --- a/backend/spellchecker/server/client.py +++ b/backend/spellchecker/server/client.py @@ -1,3 +1,4 @@ +# Author: Sergey Sokolov import grpc import spellchecker_pb2 diff --git a/backend/spellchecker/server/parser.py b/backend/spellchecker/server/parser.py index 502281c..f217382 100644 --- a/backend/spellchecker/server/parser.py +++ b/backend/spellchecker/server/parser.py @@ -1,5 +1,5 @@ +# Author: Sergey Sokolov from ctypes import cdll, c_char_p, pointer, Structure -from weakref import proxy class Parser: diff --git a/backend/spellchecker/server/run_codegen.py b/backend/spellchecker/server/run_codegen.py index 0fc4f5d..6c812fe 100644 --- a/backend/spellchecker/server/run_codegen.py +++ b/backend/spellchecker/server/run_codegen.py @@ -1,7 +1,7 @@ from grpc.tools import protoc -protoc.main ( +protoc.main( ( '', '-I../proto', diff --git a/backend/spellchecker/server/server.py b/backend/spellchecker/server/server.py index 39d09e8..d47e60c 100644 --- a/backend/spellchecker/server/server.py +++ b/backend/spellchecker/server/server.py @@ -1,3 +1,4 @@ +# Author: Sergey Sokolov import time import grpc diff --git a/backend/spellchecker/server/spellchecker.py b/backend/spellchecker/server/spellchecker.py index 347862a..5eb258b 100644 --- a/backend/spellchecker/server/spellchecker.py +++ b/backend/spellchecker/server/spellchecker.py @@ -1,3 +1,4 @@ +# Author: Sergey Sokolov import os.path import hunspell diff --git a/backend/spellchecker/server/test.py b/backend/spellchecker/server/test.py deleted file mode 100644 index e2c6612..0000000 --- a/backend/spellchecker/server/test.py +++ /dev/null @@ -1,13 +0,0 @@ -from spellchecker import Spellchecker -import time -import hunspell - -with open("/home/tener/sandbox/papeeria-hunspell/testcase/smalltest") as tests: - t = "" - for l in tests: - t += l + "\n" - s = Spellchecker("/usr/share/hunspell", "en_US") - t1 = time.time() - x = s.check_text(t, ["en_US"]) - print(time.time() - t1) - print(x) From 81adc926acc090827f6421aa4cca2979f91829c6 Mon Sep 17 00:00:00 2001 From: teners Date: Wed, 31 Aug 2016 18:10:25 +0300 Subject: [PATCH 11/21] Replaced json-string with typed interface in .proto file --- backend/spellchecker/proto/spellchecker.proto | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/spellchecker/proto/spellchecker.proto b/backend/spellchecker/proto/spellchecker.proto index b69d3cd..07f57ae 100644 --- a/backend/spellchecker/proto/spellchecker.proto +++ b/backend/spellchecker/proto/spellchecker.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -option java_package = "com.papeeria"; +option java_package = "com.bardsoftware.papeeria.backend.spellchecker"; service Spellcheck { rpc CheckText(Text) returns (Suggestions) {} @@ -11,5 +11,9 @@ message Text { } message Suggestions { - string json = 1; + message Suggestion { + string key = 1; + repeated string values = 2; + } + map suggestions = 1; } From 77b1f9f668806e6d3dc98838bd97e2e9fd7cce4f Mon Sep 17 00:00:00 2001 From: teners Date: Wed, 31 Aug 2016 18:13:51 +0300 Subject: [PATCH 12/21] Added path to shared library as constructor argument in Parser --- backend/spellchecker/server/parser.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/spellchecker/server/parser.py b/backend/spellchecker/server/parser.py index f217382..1d9a4a3 100644 --- a/backend/spellchecker/server/parser.py +++ b/backend/spellchecker/server/parser.py @@ -6,15 +6,15 @@ class Parser: """ Class wraps CFFI for LaTeX parser from Hunspell source code. """ - def __init__(self): + def __init__(self, libparser: str): # note: these fields' names doesn't start with "_" since deleting of this # object isn't so straightforward -- for some reason interpreter destroys # these "private" fields before __del__ is invoked. - self.parser_lib_ = cdll.LoadLibrary("../libparser/build/libparser.so") - self.parser_ = self.parser_lib_.latex_parser() + self.parser_lib = cdll.LoadLibrary(libparser) + self.parser = self.parser_lib.latex_parser() def __del__(self): - self.parser_lib_.delete_parser(self.parser_) + self.parser_lib.delete_parser(self.parser) def put_line(self, line: str): """ @@ -22,7 +22,7 @@ def put_line(self, line: str): :param line: A line from text to tokenize. """ - self.parser_lib_.put_line(self.parser_, bytes(line, encoding="UTF-8")) + self.parser_lib.put_line(self.parser, bytes(line, encoding="UTF-8")) def next_token(self): """ @@ -32,10 +32,10 @@ def next_token(self): """ # actually, we have to put a pointer to pointer to char so # to allocate new memory for the token - if self.parser_lib_.next_token(self.parser_, self._token_ptr_): + if self.parser_lib.next_token(self.parser, self._token_ptr_): word = str(self._token_ptr_.contents.token, encoding="UTF-8") # also we have to free the allocated memory manually - self.parser_lib_.free_token(self._token_ptr_) + self.parser_lib.free_token(self._token_ptr_) return word else: return None @@ -52,10 +52,10 @@ def tokenize(self, line: str): self.put_line(line) word = True while word: - if self.parser_lib_.next_token(self.parser_, self._token_ptr_): + if self.parser_lib.next_token(self.parser, self._token_ptr_): word = str(self._token_ptr_.contents.token, encoding="UTF-8") # also we have to free the allocated memory manually - self.parser_lib_.free_token(self._token_ptr_) + self.parser_lib.free_token(self._token_ptr_) yield word else: return From 893fca6aaf690159e6e04d76a0decf3935746aa2 Mon Sep 17 00:00:00 2001 From: teners Date: Wed, 31 Aug 2016 23:51:17 +0300 Subject: [PATCH 13/21] Replaced json with protobuf interface, fixed small flaws --- backend/spellchecker/server/client.py | 2 +- backend/spellchecker/server/server.py | 31 ++++-- backend/spellchecker/server/spellchecker.py | 10 +- .../spellchecker/server/spellchecker_pb2.py | 105 ++++++++++++++++-- 4 files changed, 128 insertions(+), 20 deletions(-) diff --git a/backend/spellchecker/server/client.py b/backend/spellchecker/server/client.py index b1905af..7c935b4 100644 --- a/backend/spellchecker/server/client.py +++ b/backend/spellchecker/server/client.py @@ -10,4 +10,4 @@ Text.text = b"test line wth errorz" Text.language.extend(["en_US", "ru_RU"]) response = stub.CheckText(Text) -print(response.json) +print(response.suggestions) diff --git a/backend/spellchecker/server/server.py b/backend/spellchecker/server/server.py index d47e60c..0141634 100644 --- a/backend/spellchecker/server/server.py +++ b/backend/spellchecker/server/server.py @@ -13,28 +13,45 @@ class SpellcheckerServicer(spellchecker_pb2.SpellcheckServicer): """ - gRPC service to check text that comes from stubs. + gRPC service to check text that comes from clients. """ def __init__(self): - self._checker_ = Spellchecker("/usr/share/hunspell", "en_US") + self.checker = Spellchecker("/usr/share/hunspell", "en_US") def CheckText(self, request, context): + """ + Method gets request from client that contains text to check and + languages list and returns Suggestions message (JSON-like). + + :param request: Object generated by gRPC, contains text and languages inside. + :param context: gRPC service object, never mind this one. + :return: Suggestions message. + """ text = str(request.text, encoding="UTF-8") languages = [lang for lang in request.language] - json = self._checker_.check_text(text, languages) - return spellchecker_pb2.Suggestions(json=json) + return self.checker.check_text(text, languages) -def serve(max_workers: int = 2): + +def serve(port: str, max_workers: int = 2): + """ + Initialize and run the gRPC server. + + :param port: Port on which the server would be listening. + :param max_workers: Size of thread pool to serve clients. + """ server = grpc.server( futures.ThreadPoolExecutor(max_workers=max_workers) ) + spellchecker_pb2.add_SpellcheckServicer_to_server( SpellcheckerServicer(), server ) - server.add_insecure_port('[::]:50051') + + server.add_insecure_port('[::]:{}'.format(port)) server.start() + try: while True: time.sleep(_ONE_DAY_IN_SECONDS) @@ -42,4 +59,4 @@ def serve(max_workers: int = 2): server.stop(0) if __name__ == '__main__': - serve() + serve(50051) diff --git a/backend/spellchecker/server/spellchecker.py b/backend/spellchecker/server/spellchecker.py index 5eb258b..e0a3c9c 100644 --- a/backend/spellchecker/server/spellchecker.py +++ b/backend/spellchecker/server/spellchecker.py @@ -3,6 +3,7 @@ import hunspell import ujson +from spellchecker_pb2 import Suggestions from parser import Parser @@ -14,7 +15,7 @@ class Spellchecker: """ def __init__(self, path: str = '', language: str ='', log: object = None): self._hunspell_instances = {} - self._parser = Parser() + self._parser = Parser("../libparser/build/libparser.so") self._log = log if path is not '': @@ -51,13 +52,14 @@ def check_text(self, text: str, languages) -> str: :param languages: Languages list. :return: JSON with suggestions for misspelled words. """ - suggestions = {} + suggestions_map = Suggestions() hunspells = (self._hunspell_instances[lang] for lang in languages if lang in self._hunspell_instances) tokens = self._parser.tokenize(text) for h in hunspells: for token in tokens: if not h.spell(token): - suggestions[token] = h.suggest(token) + suggestions_map.suggestions[token].values.extend(h.suggest(token)) - return ujson.dumps(suggestions) + # return ujson.dumps(suggestions) + return suggestions_map diff --git a/backend/spellchecker/server/spellchecker_pb2.py b/backend/spellchecker/server/spellchecker_pb2.py index be0d735..63dd235 100644 --- a/backend/spellchecker/server/spellchecker_pb2.py +++ b/backend/spellchecker/server/spellchecker_pb2.py @@ -19,7 +19,7 @@ name='spellchecker.proto', package='', syntax='proto3', - serialized_pb=_b('\n\x12spellchecker.proto\"&\n\x04Text\x12\x0c\n\x04text\x18\x01 \x01(\x0c\x12\x10\n\x08language\x18\x02 \x03(\t\"\x1b\n\x0bSuggestions\x12\x0c\n\x04json\x18\x01 \x01(\t20\n\nSpellcheck\x12\"\n\tCheckText\x12\x05.Text\x1a\x0c.Suggestions\"\x00\x42\x15\n\x13\x63om.papeeria.clientb\x06proto3') + serialized_pb=_b('\n\x12spellchecker.proto\"&\n\x04Text\x12\x0c\n\x04text\x18\x01 \x01(\x0c\x12\x10\n\x08language\x18\x02 \x03(\t\"\xac\x01\n\x0bSuggestions\x12\x32\n\x0bsuggestions\x18\x01 \x03(\x0b\x32\x1d.Suggestions.SuggestionsEntry\x1a\x1c\n\nSuggestion\x12\x0e\n\x06values\x18\x01 \x03(\t\x1aK\n\x10SuggestionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.Suggestions.Suggestion:\x02\x38\x01\x32\x30\n\nSpellcheck\x12\"\n\tCheckText\x12\x05.Text\x1a\x0c.Suggestions\"\x00\x42\x30\n.com.bardsoftware.papeeria.backend.spellcheckerb\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -64,36 +64,107 @@ ) -_SUGGESTIONS = _descriptor.Descriptor( - name='Suggestions', - full_name='Suggestions', +_SUGGESTIONS_SUGGESTION = _descriptor.Descriptor( + name='Suggestion', + full_name='Suggestions.Suggestion', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='values', full_name='Suggestions.Suggestion.values', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=130, + serialized_end=158, +) + +_SUGGESTIONS_SUGGESTIONSENTRY = _descriptor.Descriptor( + name='SuggestionsEntry', + full_name='Suggestions.SuggestionsEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='json', full_name='Suggestions.json', index=0, + name='key', full_name='Suggestions.SuggestionsEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + _descriptor.FieldDescriptor( + name='value', full_name='Suggestions.SuggestionsEntry.value', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=160, + serialized_end=235, +) + +_SUGGESTIONS = _descriptor.Descriptor( + name='Suggestions', + full_name='Suggestions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='suggestions', full_name='Suggestions.suggestions', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_SUGGESTIONS_SUGGESTION, _SUGGESTIONS_SUGGESTIONSENTRY, ], + enum_types=[ + ], options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], - serialized_start=62, - serialized_end=89, + serialized_start=63, + serialized_end=235, ) +_SUGGESTIONS_SUGGESTION.containing_type = _SUGGESTIONS +_SUGGESTIONS_SUGGESTIONSENTRY.fields_by_name['value'].message_type = _SUGGESTIONS_SUGGESTION +_SUGGESTIONS_SUGGESTIONSENTRY.containing_type = _SUGGESTIONS +_SUGGESTIONS.fields_by_name['suggestions'].message_type = _SUGGESTIONS_SUGGESTIONSENTRY DESCRIPTOR.message_types_by_name['Text'] = _TEXT DESCRIPTOR.message_types_by_name['Suggestions'] = _SUGGESTIONS @@ -105,15 +176,33 @@ _sym_db.RegisterMessage(Text) Suggestions = _reflection.GeneratedProtocolMessageType('Suggestions', (_message.Message,), dict( + + Suggestion = _reflection.GeneratedProtocolMessageType('Suggestion', (_message.Message,), dict( + DESCRIPTOR = _SUGGESTIONS_SUGGESTION, + __module__ = 'spellchecker_pb2' + # @@protoc_insertion_point(class_scope:Suggestions.Suggestion) + )) + , + + SuggestionsEntry = _reflection.GeneratedProtocolMessageType('SuggestionsEntry', (_message.Message,), dict( + DESCRIPTOR = _SUGGESTIONS_SUGGESTIONSENTRY, + __module__ = 'spellchecker_pb2' + # @@protoc_insertion_point(class_scope:Suggestions.SuggestionsEntry) + )) + , DESCRIPTOR = _SUGGESTIONS, __module__ = 'spellchecker_pb2' # @@protoc_insertion_point(class_scope:Suggestions) )) _sym_db.RegisterMessage(Suggestions) +_sym_db.RegisterMessage(Suggestions.Suggestion) +_sym_db.RegisterMessage(Suggestions.SuggestionsEntry) DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.papeeria.client')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n.com.bardsoftware.papeeria.backend.spellchecker')) +_SUGGESTIONS_SUGGESTIONSENTRY.has_options = True +_SUGGESTIONS_SUGGESTIONSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) import grpc from grpc.beta import implementations as beta_implementations from grpc.beta import interfaces as beta_interfaces From 8c0e6fd496ab01b9fa0e95e036ea8ecefe3ca46f Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 1 Sep 2016 02:48:26 +0300 Subject: [PATCH 14/21] Fixed Suggestions interface in proto file --- backend/spellchecker/proto/spellchecker.proto | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/spellchecker/proto/spellchecker.proto b/backend/spellchecker/proto/spellchecker.proto index 07f57ae..744c66d 100644 --- a/backend/spellchecker/proto/spellchecker.proto +++ b/backend/spellchecker/proto/spellchecker.proto @@ -12,8 +12,7 @@ message Text { message Suggestions { message Suggestion { - string key = 1; - repeated string values = 2; + repeated string values = 1; } map suggestions = 1; } From 9e4af1a2c98575cf6221be260f8edfb92689303d Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 1 Sep 2016 03:51:13 +0300 Subject: [PATCH 15/21] Replaced all string literals with command line arguments, minor fixes --- backend/spellchecker/server/server.py | 54 ++++++++++++++++++--- backend/spellchecker/server/spellchecker.py | 25 +++++----- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/backend/spellchecker/server/server.py b/backend/spellchecker/server/server.py index 0141634..20e9b85 100644 --- a/backend/spellchecker/server/server.py +++ b/backend/spellchecker/server/server.py @@ -1,5 +1,6 @@ # Author: Sergey Sokolov import time +import argparse import grpc from concurrent import futures @@ -15,8 +16,8 @@ class SpellcheckerServicer(spellchecker_pb2.SpellcheckServicer): """ gRPC service to check text that comes from clients. """ - def __init__(self): - self.checker = Spellchecker("/usr/share/hunspell", "en_US") + def __init__(self, libparser, dict_path, language): + self.checker = Spellchecker(libparser, dict_path, language) def CheckText(self, request, context): """ @@ -33,23 +34,26 @@ def CheckText(self, request, context): return self.checker.check_text(text, languages) -def serve(port: str, max_workers: int = 2): +def serve(port: str, max_workers: int, libparser: str, dict_path: str, language: str): """ Initialize and run the gRPC server. :param port: Port on which the server would be listening. :param max_workers: Size of thread pool to serve clients. + :param libparser: Path to C parser shared library. + :param dict_path: Path to .dic and .aff files + :param language: Language to add to spellchecker. """ server = grpc.server( futures.ThreadPoolExecutor(max_workers=max_workers) ) spellchecker_pb2.add_SpellcheckServicer_to_server( - SpellcheckerServicer(), + SpellcheckerServicer(libparser, dict_path, language), server ) - server.add_insecure_port('[::]:{}'.format(port)) + server.add_insecure_port("[::]:{}".format(port)) server.start() try: @@ -58,5 +62,41 @@ def serve(port: str, max_workers: int = 2): except KeyboardInterrupt: server.stop(0) -if __name__ == '__main__': - serve(50051) +if __name__ == "__main__": + args_parser = argparse.ArgumentParser(description="Spellchecker gRPC server.") + args_parser.add_argument("-w", "--workers", + action="store", + type=int, + default=2, + metavar="NUM", + help="size of thread pool") + + required = args_parser.add_argument_group("required") + required.add_argument("-p", "--port", + required=True, + action="store", + type=int, + metavar="NUM", + help="port which server would be listening") + required.add_argument("-l", "--libparser", + required=True, + action="store", + type=str, + metavar="PATH", + help="path to libparser shared library") + required.add_argument("-d", "--dir", + required=True, + action="store", + type=str, + metavar="PATH", + help="path to .dic and .aff files") + required.add_argument("-L", "--language", + required=True, + action="store", + type=str, + metavar="LANG", + help="dictionary language") + + args = args_parser.parse_args() + + serve(args.port, args.workers, args.libparser, args.dir, args.language) diff --git a/backend/spellchecker/server/spellchecker.py b/backend/spellchecker/server/spellchecker.py index e0a3c9c..90fe3fc 100644 --- a/backend/spellchecker/server/spellchecker.py +++ b/backend/spellchecker/server/spellchecker.py @@ -13,10 +13,10 @@ class Spellchecker: Class contains multiple hunspell instances (one per language) and provides text spellchecking function using all chosen dictionaries. """ - def __init__(self, path: str = '', language: str ='', log: object = None): - self._hunspell_instances = {} - self._parser = Parser("../libparser/build/libparser.so") - self._log = log + def __init__(self, libparser: str, path: str = '', language: str ='', log: object = None): + self.hunspell_instances = {} + self.parser = Parser(libparser) + self.log = log if path is not '': self.add_dictionary(path, language) @@ -28,21 +28,21 @@ def add_dictionary(self, path: str, language: str) -> None: :param path: Path to .dic and .aff files. :param language: Language that corresponds .dic and .aff files. """ - if language in self._hunspell_instances: + if language in self.hunspell_instances: return dictionary = "{}/{}.dic".format(path, language) if not os.path.isfile(dictionary): - if self._log: - self._log.write("File not found: {}".format(dictionary)) + if self.log: + self.log.write("File not found: {}".format(dictionary)) return affix = "{}/{}.aff".format(path, language) if not os.path.isfile(affix): - if self._log: - self._log.write("File not found: {}".format(affix)) + if self.log: + self.log.write("File not found: {}".format(affix)) return - self._hunspell_instances[language] = hunspell.HunSpell(dictionary, affix) + self.hunspell_instances[language] = hunspell.HunSpell(dictionary, affix) def check_text(self, text: str, languages) -> str: """ @@ -53,13 +53,12 @@ def check_text(self, text: str, languages) -> str: :return: JSON with suggestions for misspelled words. """ suggestions_map = Suggestions() - hunspells = (self._hunspell_instances[lang] for lang in languages if lang in self._hunspell_instances) - tokens = self._parser.tokenize(text) + hunspells = (self.hunspell_instances[lang] for lang in languages if lang in self.hunspell_instances) + tokens = self.parser.tokenize(text) for h in hunspells: for token in tokens: if not h.spell(token): suggestions_map.suggestions[token].values.extend(h.suggest(token)) - # return ujson.dumps(suggestions) return suggestions_map From 01d8e7a860e6b6acb324ce4410fad1efeabcb221 Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 1 Sep 2016 03:54:24 +0300 Subject: [PATCH 16/21] Fixed docstring --- backend/spellchecker/server/spellchecker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/spellchecker/server/spellchecker.py b/backend/spellchecker/server/spellchecker.py index 90fe3fc..c83b33e 100644 --- a/backend/spellchecker/server/spellchecker.py +++ b/backend/spellchecker/server/spellchecker.py @@ -46,11 +46,11 @@ def add_dictionary(self, path: str, language: str) -> None: def check_text(self, text: str, languages) -> str: """ - Check text using chosen languages and return JSON with suggestions. + Check text using chosen languages and return Suggestions message object. :param text: Text to check. :param languages: Languages list. - :return: JSON with suggestions for misspelled words. + :return: Suggestions message (dictionary-like) for misspelled words. """ suggestions_map = Suggestions() hunspells = (self.hunspell_instances[lang] for lang in languages if lang in self.hunspell_instances) From 9e79d4da137ab24a57a48bcd130fb227fd4d471f Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 1 Sep 2016 03:56:44 +0300 Subject: [PATCH 17/21] Reformatted parser.cpp --- backend/spellchecker/libparser/parser.cpp | 55 ++++++++++++----------- backend/spellchecker/server/test.py | 0 2 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 backend/spellchecker/server/test.py diff --git a/backend/spellchecker/libparser/parser.cpp b/backend/spellchecker/libparser/parser.cpp index 6a46879..bf9f7e7 100644 --- a/backend/spellchecker/libparser/parser.cpp +++ b/backend/spellchecker/libparser/parser.cpp @@ -1,3 +1,4 @@ +// Author: Sergey Sokolov #include "parsers/latexparser.hxx" #include #include @@ -5,43 +6,43 @@ extern "C" { - struct Token { - char* token; - }; + struct Token { + char* token; + }; - std::string alphabet("qwertzuiopasdfghjklyxcvbnmQWERTZUIOPASDFGHJKLYXCVBNM"); + std::string alphabet("qwertzuiopasdfghjklyxcvbnmQWERTZUIOPASDFGHJKLYXCVBNM"); LaTeXParser* latex_parser() { return new LaTeXParser(alphabet.c_str()); } - void delete_parser(LaTeXParser *parser) { - delete parser; - } + void delete_parser(LaTeXParser *parser) { + delete parser; + } void put_line(LaTeXParser* parser, char* line) { parser->put_line(line); } - int next_token(LaTeXParser *parser, Token *token) { - parser->set_url_checking(true); - std::string next; - if (parser->next_token(next)) { - token->token = new char[next.size()+1]; - std::copy(next.begin(), next.end(), token->token); - token->token[next.size()] = '\0'; - return 1; - } - else { - token->token = NULL; - return 0; - } - } - - void free_token(Token *token) { - if (token->token) { - delete [] token->token; - } - } + int next_token(LaTeXParser *parser, Token *token) { + parser->set_url_checking(true); + std::string next; + if (parser->next_token(next)) { + token->token = new char[next.size()+1]; + std::copy(next.begin(), next.end(), token->token); + token->token[next.size()] = '\0'; + return 1; + } + else { + token->token = NULL; + return 0; + } + } + + void free_token(Token *token) { + if (token->token) { + delete [] token->token; + } + } } diff --git a/backend/spellchecker/server/test.py b/backend/spellchecker/server/test.py new file mode 100644 index 0000000..e69de29 From eb1d37e321f235c842e99f83b7540f68b200b13b Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 1 Sep 2016 04:00:22 +0300 Subject: [PATCH 18/21] Removed unused ujson import --- backend/spellchecker/server/spellchecker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/spellchecker/server/spellchecker.py b/backend/spellchecker/server/spellchecker.py index c83b33e..af20b4a 100644 --- a/backend/spellchecker/server/spellchecker.py +++ b/backend/spellchecker/server/spellchecker.py @@ -2,7 +2,6 @@ import os.path import hunspell -import ujson from spellchecker_pb2 import Suggestions from parser import Parser From d8aaff20ecb9f159c0f46546844d7f8e0a800e5b Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 1 Sep 2016 04:08:40 +0300 Subject: [PATCH 19/21] Added requirements.txt --- backend/spellchecker/server/requirements.txt | 3 +++ backend/spellchecker/server/server.py | 2 +- backend/spellchecker/server/spellchecker.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 backend/spellchecker/server/requirements.txt diff --git a/backend/spellchecker/server/requirements.txt b/backend/spellchecker/server/requirements.txt new file mode 100644 index 0000000..4258098 --- /dev/null +++ b/backend/spellchecker/server/requirements.txt @@ -0,0 +1,3 @@ +grpcio>=1.0.0 +grpcio-tools>=1.0.0 +hunspell>=0.4.1 \ No newline at end of file diff --git a/backend/spellchecker/server/server.py b/backend/spellchecker/server/server.py index 20e9b85..b1d78e5 100644 --- a/backend/spellchecker/server/server.py +++ b/backend/spellchecker/server/server.py @@ -1,9 +1,9 @@ # Author: Sergey Sokolov import time import argparse +from concurrent import futures import grpc -from concurrent import futures from spellchecker import Spellchecker import spellchecker_pb2 diff --git a/backend/spellchecker/server/spellchecker.py b/backend/spellchecker/server/spellchecker.py index af20b4a..928f790 100644 --- a/backend/spellchecker/server/spellchecker.py +++ b/backend/spellchecker/server/spellchecker.py @@ -2,8 +2,8 @@ import os.path import hunspell -from spellchecker_pb2 import Suggestions +from spellchecker_pb2 import Suggestions from parser import Parser From 9ae2d84635e99b101bc66f2755f1cc991a06ddf7 Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 1 Sep 2016 04:31:10 +0300 Subject: [PATCH 20/21] Moved new-hunspell-instance-initialization on request from check_text --- backend/spellchecker/server/spellchecker.py | 27 +++++++++++---------- backend/spellchecker/server/test.py | 0 2 files changed, 14 insertions(+), 13 deletions(-) delete mode 100644 backend/spellchecker/server/test.py diff --git a/backend/spellchecker/server/spellchecker.py b/backend/spellchecker/server/spellchecker.py index 928f790..76c80e7 100644 --- a/backend/spellchecker/server/spellchecker.py +++ b/backend/spellchecker/server/spellchecker.py @@ -12,30 +12,27 @@ class Spellchecker: Class contains multiple hunspell instances (one per language) and provides text spellchecking function using all chosen dictionaries. """ - def __init__(self, libparser: str, path: str = '', language: str ='', log: object = None): + def __init__(self, libparser: str, path: str = '', log: object = None): self.hunspell_instances = {} self.parser = Parser(libparser) self.log = log + self.path = path - if path is not '': - self.add_dictionary(path, language) - - def add_dictionary(self, path: str, language: str) -> None: + def add_dictionary(self, language: str) -> None: """ Create new hunspell instance and put into instances dictionary. - :param path: Path to .dic and .aff files. :param language: Language that corresponds .dic and .aff files. """ if language in self.hunspell_instances: return - dictionary = "{}/{}.dic".format(path, language) + dictionary = "{}/{}.dic".format(self.path, language) if not os.path.isfile(dictionary): if self.log: self.log.write("File not found: {}".format(dictionary)) return - affix = "{}/{}.aff".format(path, language) + affix = "{}/{}.aff".format(self.path, language) if not os.path.isfile(affix): if self.log: self.log.write("File not found: {}".format(affix)) @@ -51,13 +48,17 @@ def check_text(self, text: str, languages) -> str: :param languages: Languages list. :return: Suggestions message (dictionary-like) for misspelled words. """ - suggestions_map = Suggestions() - hunspells = (self.hunspell_instances[lang] for lang in languages if lang in self.hunspell_instances) + for lang in languages: + if lang not in self.hunspell_instances: + self.add_dictionary(lang) + + dictionaries = (self.hunspell_instances[lang] for lang in languages if lang in self.hunspell_instances) tokens = self.parser.tokenize(text) + suggestions_map = Suggestions() - for h in hunspells: + for dictionary in dictionaries: for token in tokens: - if not h.spell(token): - suggestions_map.suggestions[token].values.extend(h.suggest(token)) + if not dictionary.spell(token): + suggestions_map.suggestions[token].values.extend(dictionary.suggest(token)) return suggestions_map diff --git a/backend/spellchecker/server/test.py b/backend/spellchecker/server/test.py deleted file mode 100644 index e69de29..0000000 From a7e2db58d27f67a8743845f3b82bccb418595f2f Mon Sep 17 00:00:00 2001 From: teners Date: Thu, 1 Sep 2016 04:32:10 +0300 Subject: [PATCH 21/21] Removed "language" command line argument --- backend/spellchecker/server/server.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/backend/spellchecker/server/server.py b/backend/spellchecker/server/server.py index b1d78e5..dc0139f 100644 --- a/backend/spellchecker/server/server.py +++ b/backend/spellchecker/server/server.py @@ -16,8 +16,8 @@ class SpellcheckerServicer(spellchecker_pb2.SpellcheckServicer): """ gRPC service to check text that comes from clients. """ - def __init__(self, libparser, dict_path, language): - self.checker = Spellchecker(libparser, dict_path, language) + def __init__(self, libparser, dict_path): + self.checker = Spellchecker(libparser, dict_path) def CheckText(self, request, context): """ @@ -34,7 +34,7 @@ def CheckText(self, request, context): return self.checker.check_text(text, languages) -def serve(port: str, max_workers: int, libparser: str, dict_path: str, language: str): +def serve(port: str, max_workers: int, libparser: str, dict_path: str): """ Initialize and run the gRPC server. @@ -42,14 +42,13 @@ def serve(port: str, max_workers: int, libparser: str, dict_path: str, language: :param max_workers: Size of thread pool to serve clients. :param libparser: Path to C parser shared library. :param dict_path: Path to .dic and .aff files - :param language: Language to add to spellchecker. """ server = grpc.server( futures.ThreadPoolExecutor(max_workers=max_workers) ) spellchecker_pb2.add_SpellcheckServicer_to_server( - SpellcheckerServicer(libparser, dict_path, language), + SpellcheckerServicer(libparser, dict_path), server ) @@ -90,13 +89,7 @@ def serve(port: str, max_workers: int, libparser: str, dict_path: str, language: type=str, metavar="PATH", help="path to .dic and .aff files") - required.add_argument("-L", "--language", - required=True, - action="store", - type=str, - metavar="LANG", - help="dictionary language") args = args_parser.parse_args() - serve(args.port, args.workers, args.libparser, args.dir, args.language) + serve(args.port, args.workers, args.libparser, args.dir)