fix(tokenizers): discard citation from nominative reporter on overlap

Solves #221 and #174

Uses a list of problematic nominative reporters to resolve overlpas

Due to the way we tokenize, an overlap was always resolved in favor
of the first token. In the case of nominative reporters, this caused
 a CitationToken to be found when a party name matched the
reporter's name, discarding the actual citation

This could be solved in a cleaner way by being consistent on
tagging nominative reporters on reporters-db

CHANGES.md

@@ -11,7 +11,8 @@ Changes:
 - None
 
 Fixes:
-- None
+- Prefer the other full citation on overlap with nominative reporter 
+  citations #237
 
 
 ## Current

eyecite/tokenizers.py

@@ -55,6 +55,46 @@
 EXTRACTORS = []
 EDITIONS_LOOKUP = defaultdict(list)
 
+NOMINATIVE_REPORTER_NAMES = {
+    "Thompson",
+    "Cooke",
+    "Holmes",
+    "Olcott",
+    "Chase",
+    "Gilmer",
+    "Bee",
+    "Deady",
+    "Taney",
+}
+
+
+def token_is_from_nominative_reporter(token: Token) -> bool:
+    """Returns true if the token is a citation from a nominative reporter
+
+    Cleaner way to do this would be via an attribute or named group from
+    reporters-db. However; this tagging is currently not complete, so we can
+    use a list of the most problematic names `NOMINATIVE_REPORTER_NAMES`
+
+    ```
+    volume_nominative = token.groups.get("volume_nominative", False)
+    reporter_nominative = token.groups.get("reporter_nominative", False)
+    token.exact_editions[0].reporter.name
+    return volume_nominative is None or volume_nominative
+        or reporter_nominative is None or reporter_nominative
+    ```
+
+    :param token: the token
+    :return: True if the token has a `volume_nominative` group, even if there
+        was no match; False if it didn't have the group
+    """
+    if not isinstance(token, CitationToken):
+        return False
+    if token.exact_editions:
+        name = token.exact_editions[0].reporter.short_name
+    else:
+        name = token.variation_editions[0].reporter.short_name
+    return name in NOMINATIVE_REPORTER_NAMES
+
 
 def _populate_reporter_extractors():
     """Populate EXTRACTORS and EDITIONS_LOOKUP."""
@@ -313,8 +353,19 @@ def tokenize(self, text: str) -> Tuple[Tokens, List[Tuple[int, Token]]]:
                 if merged:
                     continue
             if offset > token.start:
-                # skip overlaps
-                continue
+                if (
+                    last_token
+                    and isinstance(token, CitationToken)
+                    and token_is_from_nominative_reporter(last_token)
+                ):
+                    # if a token has overlapping matches between a nominative
+                    # reporter and another type of case citation, prefer the
+                    # other case citation. See #221 and #174
+                    citation_tokens.pop(-1)
+                    all_tokens.pop(-1)
+                else:
+                    # skip overlaps
+                    continue
             if offset < token.start:
                 # capture plain text before each match
                 self.append_text(all_tokens, text[offset : token.start])
@@ -326,6 +377,7 @@ def tokenize(self, text: str) -> Tuple[Tokens, List[Tuple[int, Token]]]:
         # capture plain text after final match
         if offset < len(text):
             self.append_text(all_tokens, text[offset:])
+
         return all_tokens, citation_tokens
 
     def get_extractors(self, text: str):

tests/test_FindTest.py

@@ -878,6 +878,55 @@ def test_disambiguate_citations(self):
         ]
         self.run_test_pairs(test_pairs, "Disambiguation")
 
+    def test_nominative_reporter_overlaps(self):
+        """Can we parse a full citation where a name looks like a nominative
+        reporter?"""
+        pairs = [
+            (
+                "In re Cooke, 93 Wn. App. 526, 529",
+                case_citation(volume="93", reporter="Wn. App.", page="526"),
+            ),
+            (
+                "Shapiro v. Thompson, 394 U. S. 618",
+                case_citation(volume="394", reporter="U. S.", page="618"),
+            ),
+            (
+                "MacArdell v. Olcott, 82 N.E. 161",
+                case_citation(volume="82", reporter="N.E.", page="161"),
+            ),
+            (
+                "Connecticut v. Holmes, 221 A.3d 407",
+                case_citation(volume="221", reporter="A.3d", page="407"),
+            ),
+            (
+                "Kern v Taney, 11 Pa. D. & C.5th 558 [2010])",
+                case_citation(
+                    volume="11", reporter="Pa. D. & C.5th", page="558"
+                ),
+            ),
+            (
+                "Ellenburg v. Chase, 2004 MT 66",
+                case_citation(volume="2004", reporter="MT", page="66"),
+            ),
+            (
+                "Gilmer, 500 U.S. at 25;",
+                case_citation(
+                    volume="500", reporter="U. S.", page="25", short=True
+                ),
+            ),
+            (
+                "Bison Bee, 778 F. 13 App’x at 73.",
+                case_citation(volume="778", reporter="F.", page="13"),
+            ),
+        ]
+        for cite_string, cite_object in pairs:
+            parsed_cite = get_citations(cite_string)[0]
+            self.assertEqual(
+                parsed_cite,
+                cite_object,
+                f"Nominative reporters getting in the way of parsing: {parsed_cite}",
+            )
+
     def test_custom_tokenizer(self):
         extractors = []
         for e in EXTRACTORS: