OPENNLP-1930: End a quoted lemma at the first matching quote at a field boundary

The leaf scans searched backward from the end of the line, so a lemma
extended to the last quote whose rest still parsed, the greedy behavior
of the replaced patterns: =H:n("x" M S)<tab>a') b gave lemma
'x" M S)<tab>a', no morphology, and lexeme 'b'. Now the first matching
quote followed by whitespace, a secondary tag, or a closing bracket ends
the field, giving lemma 'x', morphology 'M S', and lexeme 'a') b'.
Embedded word quotes stay literal, and the square bracket after some
Floresta time and measurement lemmas is accepted.

Red: seven assertion failures over 247 parser cases. Green: 500 focused
AD cases. No parsed field changed for the 920,203 quoted tree-line
candidates in Bosque and FlorestaVirgem. Adds
evalPortugueseTokenizerDefaultNaiveBayes, which cross-validates the
registered AD token converter with the default preserved compounds and
ADDetokenizer (F1 0.9951936406770416, separate reference boundaries from
the split path). The manual documents the lemma boundary rule.

Author: krickert

opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java

@@ -74,6 +74,7 @@ public static class SentenceParser {
     private static final char GROUP_CLOSE = ')';
     private static final char BRACKET_OPEN = '<';
     private static final char BRACKET_CLOSE = '>';
+    private static final char SQUARE_BRACKET_CLOSE = ']';
     private static final char DOUBLE_QUOTE = '"';
     private static final char SINGLE_QUOTE = '\'';
     private static final char CLOSING_GUILLEMET = '\u00BB';
@@ -478,26 +479,47 @@ private boolean isLemmaStart(String line, int from) {
      */
     private Leaf parseLeafAfterTag(String line, Tag tag, LineScan scan) {
       int lemmaStart = tag.end() + 2;
-      // the longest lemma after which the rest of the line still parses is used
-      for (int lemmaEnd = line.length() - 1; lemmaEnd > lemmaStart; lemmaEnd--) {
-        if (!isQuote(line.charAt(lemmaEnd))) {
-          continue;
-        }
-        int tagsStart = skipWhitespace(line, lemmaEnd + 1);
-        LeafRest rest = scanSecondaryTags(line, tagsStart, scan);
-        if (rest != null) {
-          Leaf leaf = new Leaf();
-          leaf.setLevel(tag.start() + 1);
-          leaf.setSyntacticTag(line.substring(tag.start(), tag.separator()));
-          leaf.setFunctionalTag(line.substring(tag.separator() + 1, tag.end()));
-          leaf.setLemma(line.substring(lemmaStart, lemmaEnd));
-          leaf.setSecondaryTag(line.substring(tagsStart, rest.tagsEnd()));
-          leaf.setMorphologicalTag(morphology(line, rest.end()));
-          leaf.setLexeme(line.substring(rest.end().lexemeStart()));
-          return leaf;
+      int lemmaEnd = lemmaEnd(line, lemmaStart - 1);
+      if (lemmaEnd <= lemmaStart) {
+        return null;
+      }
+      int tagsStart = skipWhitespace(line, lemmaEnd + 1);
+      LeafRest rest = scanSecondaryTags(line, tagsStart, scan);
+      if (rest == null) {
+        return null;
+      }
+      Leaf leaf = new Leaf();
+      leaf.setLevel(tag.start() + 1);
+      leaf.setSyntacticTag(line.substring(tag.start(), tag.separator()));
+      leaf.setFunctionalTag(line.substring(tag.separator() + 1, tag.end()));
+      leaf.setLemma(line.substring(lemmaStart, lemmaEnd));
+      leaf.setSecondaryTag(line.substring(tagsStart, rest.tagsEnd()));
+      leaf.setMorphologicalTag(morphology(line, rest.end()));
+      leaf.setLexeme(line.substring(rest.end().lexemeStart()));
+      return leaf;
+    }
+
+    /**
+     * Finds the matching quote that ends a lemma field. A quote followed by whitespace, a
+     * secondary tag, or a closing parenthesis or square bracket ends the field; quotes
+     * embedded in a word remain literal. Later quotes in the morphology or lexeme cannot
+     * extend the lemma.
+     *
+     * @param line The tree line.
+     * @param open The index of the opening quote.
+     * @return The closing quote index, or -1 if the field has no closing quote.
+     */
+    private int lemmaEnd(String line, int open) {
+      char quote = line.charAt(open);
+      for (int i = open + 1; i < line.length(); i++) {
+        if (line.charAt(i) == quote && (i + 1 == line.length()
+            || StringUtil.isUnicodeWhitespace(line.charAt(i + 1))
+            || line.charAt(i + 1) == BRACKET_OPEN || line.charAt(i + 1) == GROUP_CLOSE
+            || line.charAt(i + 1) == SQUARE_BRACKET_CLOSE)) {
+          return i;
         }
       }
-      return null;
+      return -1;
     }
 
     /**
@@ -540,12 +562,11 @@ private Leaf parseBizarreLeafAfterTag(String line, Tag tag, LineScan scan) {
       String lemma = null;
       LeafEnd end = null;
       if (open < line.length() && isQuote(line.charAt(open))) {
-        for (int lemmaEnd = line.length() - 1; lemmaEnd > open + 1 && end == null; lemmaEnd--) {
-          if (isQuote(line.charAt(lemmaEnd))) {
-            end = scanLeafEnd(line, lemmaEnd + 1, scan);
-            if (end != null) {
-              lemma = line.substring(open + 1, lemmaEnd);
-            }
+        int lemmaEnd = lemmaEnd(line, open);
+        if (lemmaEnd > open + 1) {
+          end = scanLeafEnd(line, lemmaEnd + 1, scan);
+          if (end != null) {
+            lemma = line.substring(open + 1, lemmaEnd);
           }
         }
       }

opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java

@@ -202,9 +202,21 @@ private static Stream<Arguments> leafLines() {
         // quotes inside lemma and lexeme
         Arguments.of("=H:n(\"d'água\" <x> M S)\td'água",
             2, "H", "n", "d'água", "<x>", "M S", "d'água"),
-        // the lemma extends to the last quote after which the rest of the line still parses
-        Arguments.of("=H:n(\"a\" <b> \"c\")\tw", 2, "H", "n", "a\" <b> \"c", "", null, "w"),
-        Arguments.of("=H:n(\"x\" M S)\ta') b", 2, "H", "n", "x\" M S)\ta", "", null, "b"),
+        // A closing lemma quote ends the field, even if later fields contain quotes.
+        Arguments.of("=H:n(\"a\" <b> \"c\")\tw", 2, "H", "n", "a", "<b>", "\"c\"", "w"),
+        Arguments.of("=H:n(\"x\" M S)\ta') b", 2, "H", "n", "x", "", "M S", "a') b"),
+        Arguments.of("=H:n(\"x\" M S)\ta\") b", 2, "H", "n", "x", "", "M S", "a\") b"),
+        Arguments.of("=H:n('x' M S)\ta') b", 2, "H", "n", "x", "", "M S", "a') b"),
+        // Embedded quotes without a field boundary remain part of the lemma.
+        Arguments.of("=H:n('d'água' M S)\td'água",
+            2, "H", "n", "d'água", "", "M S", "d'água"),
+        Arguments.of("=H:prop(\"Eduardo_2º_\"=,=de=Marlwe\" M S)\tEduardo",
+            2, "H", "prop", "Eduardo_2º_\"=,=de=Marlwe", "", "M S", "Eduardo"),
+        Arguments.of("=H:n(\"x\"\u00A0M S)\ta\") b",
+            2, "H", "n", "x", "", "M S", "a\") b"),
+        // Floresta has an extra square bracket after some time/measurement lemmas.
+        Arguments.of("=H:n(\"8h\"] <temp> F P)\t8h\"",
+            2, "H", "n", "8h", "", "] <temp> F P", "8h\""),
         // the secondary tags extend to the last closing angle bracket
         Arguments.of("=H:n(\"casa\" <a>b<c> M S)\tcasa",
             2, "H", "n", "casa", "<a>b<c>", "M S", "casa"),
@@ -246,6 +258,8 @@ private static Stream<Arguments> bizarreLeafLines() {
         Arguments.of("=x=y() b", 2, "x=y", null, null, "b"),
         // a quoted part without a closing quote is the morphological tag
         Arguments.of("=x=y(\"q) b", 2, "x=y", null, "\"q", "b"),
+        Arguments.of("=x=y(\"q\" M S)\ta') b", 2, "x=y", "q", "M S", "a') b"),
+        Arguments.of("=x=y('q' M S)\ta') b", 2, "x=y", "q", "M S", "a') b"),
         // the level prefix gives up hyphens so that the tag can start
         Arguments.of("==-=x(a) b", 3, "-=x", null, "a", "b"),
         Arguments.of("=-=x=y(a) b", 4, "x=y", null, "a", "b"),

opennlp-docs/src/docbkx/corpora.xml

@@ -467,6 +467,12 @@ Execution time: 1,955 seconds]]>
 				detokenizer dictionary's hyphen joining rule to reconstruct <code>ofereceu-me</code>
 				without adding a space or changing those annotated token boundaries.
 			</para>
+			<para>
+				A quoted lemma ends at a matching quote followed by a field boundary. Embedded
+				apostrophes and quotes within a word remain part of the lemma. Quotes in subsequent
+				annotation fields or in the surface text do not extend the lemma or remove part of
+				the token text.
+			</para>
 			<para>
 				The AD reader separates the parts of a line on whitespace as the Unicode White_Space
 				property defines it, independent of the toolkit whitespace mode: tabs, no-break spaces,

opennlp-eval-tests/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java

@@ -28,9 +28,12 @@
 
 import opennlp.tools.chunker.ChunkerCrossValidator;
 import opennlp.tools.chunker.ChunkerFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
 import opennlp.tools.formats.ad.ADChunkSampleStream;
 import opennlp.tools.formats.ad.ADNameSampleStream;
+import opennlp.tools.formats.ad.ADNameSampleStreamFactory;
 import opennlp.tools.formats.ad.ADSentenceSampleStream;
+import opennlp.tools.formats.ad.ADTokenSampleStreamFactory;
 import opennlp.tools.formats.convert.NameToTokenSampleStream;
 import opennlp.tools.namefind.NameSample;
 import opennlp.tools.sentdetect.SDCrossValidator;
@@ -114,14 +117,20 @@ private void tokenizerCrossEval(TrainingParameters params,
     ObjectStream<TokenSample> samples = new NameToTokenSampleStream(
         detokenizer, nameSamples);
 
-    TokenizerCrossValidator validator;
+    tokenizerCrossEval(params, expectedScore, samples);
+  }
 
+  /** Evaluates the supplied conversion path with identical training and held-out settings. */
+  private void tokenizerCrossEval(TrainingParameters params, double expectedScore,
+                                  ObjectStream<TokenSample> samples) throws IOException {
     TokenizerFactory tokFactory = TokenizerFactory.create(null, LANG, null,
         true, null);
-    validator = new opennlp.tools.tokenize.TokenizerCrossValidator(params,
+    TokenizerCrossValidator validator = new TokenizerCrossValidator(params,
         tokFactory);
 
-    validator.evaluate(samples, 10);
+    try (samples) {
+      validator.evaluate(samples, 10);
+    }
 
     Assertions.assertEquals(expectedScore, validator.getFMeasure().getFMeasure(), 0.0001d);
   }
@@ -178,6 +187,19 @@ void evalPortugueseTokenizerNaiveBayes() throws IOException {
     tokenizerCrossEval(createNaiveBayesParams(), 0.9962358244502717d);
   }
 
+  /** Exercises the CLI default, including preserved compounds and AD clitic detokenization. */
+  @Test
+  void evalPortugueseTokenizerDefaultNaiveBayes() throws IOException {
+    ADNameSampleStreamFactory.registerFactory();
+    ADTokenSampleStreamFactory.registerFactory();
+    ObjectStream<TokenSample> samples = StreamFactoryRegistry.getFactory(TokenSample.class, "ad")
+        .create(new String[] {"-data", new File(getOpennlpDataDir(), FLORESTA_VIRGEM).getPath(),
+            "-lang", LANG, "-encoding", StandardCharsets.ISO_8859_1.name(),
+            "-detokenizer", "lang/pt/tokenizer/pt-detokenizer.xml"});
+    // Preserved compounds have different reference boundaries from the split-token baseline.
+    tokenizerCrossEval(createNaiveBayesParams(), 0.9951936406770416d, samples);
+  }
+
   @Test
   void evalPortugueseTokenizerMaxentQnMultipleThreads() throws IOException {
     TrainingParameters params = createMaxentQnParams();