fix unconsumed tags/lemq in lt-proc -db (closes #182) (#195)

* fix unconsumed tags/lemq in `lt-proc -db` (#182)

* new mode gm_bilgen for biltrans generation, giving @ on unconsumed tags

---------

Co-authored-by: Kevin Brubeck Unhammer <[email protected]>

Author: mr-martian

lttoolbox/fst_processor.cc

@@ -1282,6 +1282,7 @@ FSTProcessor::generation(InputFile& input, UFILE *output, GenerationMode mode)
         case gm_tagged:
           u_fputc(rd.mark, output);
           [[fallthrough]];
+        case gm_bilgen:
         case gm_clean:
           writeEscaped(removeTags(rd.content), output);
           break;
@@ -1343,6 +1344,7 @@ FSTProcessor::generation(InputFile& input, UFILE *output, GenerationMode mode)
           case gm_tagged:
             if (!rd.content.empty()) u_fputc('#', output);
             [[fallthrough]];
+          case gm_bilgen:
           case gm_clean:
             writeEscaped(removeTags(rd.content), output);
             break;
@@ -1775,7 +1777,9 @@ FSTProcessor::bilingual(InputFile& input, UFILE *output, GenerationMode mode)
       }
     }
     // if there are no tags, we only return complete matches
-    if (!seenTags && queue_start + 1 < symbols.size()) result.clear();
+    if ((!seenTags || mode == gm_all || mode == gm_bilgen) && queue_start + 1 < symbols.size()) {
+      result.clear();
+    }
 
     UString source;
     size_t queue_pos = 0;

lttoolbox/fst_processor.h

@@ -43,6 +43,7 @@ enum GenerationMode
   gm_clean,      // clear all
   gm_unknown,    // display unknown words, clear transfer and generation tags
   gm_all,        // display all
+  gm_bilgen,     // generation using biltrans (don't allow unconsumed tags)
   gm_tagged,     // tagged generation
   gm_tagged_nm,  // clean tagged generation
   gm_carefulcase // try lowercase iff no uppercase

lttoolbox/lt_proc.cc

@@ -89,6 +89,7 @@ int main(int argc, char *argv[])
   if (args["generation"]) {
     if (cmd && cmd != 'b') cli.print_usage();
     if (!cmd) cmd = 'g';
+    else if(cmd == 'b') bilmode = gm_bilgen;
   }
   if (args["decompose-nouns"]) {
     if (cmd) cli.print_usage();

tests/lt_proc/__init__.py

@@ -516,6 +516,21 @@ class BiltransAnyCharEscapes(ProcTest):
     inputs = [r'^«\[\[tf:i:a\]\]s\\\^å\[\[\/\]\]»<MERGED>$']
     expectedOutputs = [r'^«\[\[tf:i:a\]\]s\\\^å\[\[\/\]\]»<MERGED>/«\[\[tf:i:a\]\]s\\\^å\[\[\/\]\]»<MERGED>$']
 
+class BiltransGenDebugSymbols(ProcTest):
+    procdix = 'data/minimal-mono.dix'
+    procdir = 'rl'
+    procflags = ['-d', '-b']
+    inputs = [
+        '^ab<n><def>$',
+        '^ab<n><def><potato>$',
+        '^ab<n><def>#c$',
+    ]
+    expectedOutputs = [
+        '^ab<n><def>/abc$',
+        '^ab<n><def><potato>/#ab<n><def><potato>$',
+        '^ab<n><def>#c/#ab<n><def>#c$',
+    ]
+
 
 # These fail on some systems:
 #from null_flush_invalid_stream_format import *