feat(eyecite): Improve name extraction

Add check inside method for plaintiff
Use that to identify the full plaintiff
name

Also improve regex building to avoid
unnecessary escapes

Author: flooie

eyecite/find.py

@@ -86,7 +86,7 @@ def get_citations(
             if citation_token.short:
                 citation = _extract_shortform_citation(document.words, i)
             else:
-                citation = _extract_full_citation(document.words, i)
+                citation = _extract_full_citation(document, i)
                 if (
                     citations
                     and isinstance(citation, FullCaseCitation)
@@ -215,7 +215,7 @@ def extract_pincited_reference_citations(
 
 
 def _extract_full_citation(
-    words: Tokens,
+    document: Document,
     index: int,
 ) -> FullCitation:
     """Given a list of words and the index of a citation, return
@@ -225,7 +225,7 @@ def _extract_full_citation(
     # one or more of the sources in reporters_db (e.g. reporters, laws,
     # journals). Get the set of all sources that matched, preferring exact
     # matches to variations:
-    token = cast(CitationToken, words[index])
+    token = cast(CitationToken, document.words[index])
     cite_sources = set(
         e.reporter.source
         for e in (token.exact_editions or token.variation_editions)
@@ -249,7 +249,7 @@ def _extract_full_citation(
         exact_editions=token.exact_editions,
         variation_editions=token.variation_editions,
     )
-    citation.add_metadata(words)
+    citation.add_metadata(document)
 
     return citation
 
@@ -413,10 +413,10 @@ def find_reference_citations_from_markup(
                 continue
             if not is_valid_name(value):
                 continue
-            value = re.sub(r"\s+", re.escape(" "), re.escape(value.strip()))
-            regexes.append(
-                r"(?P<{}>{})".format(key, value.replace(" ", r"\s+"))
+            regex_value = r"\s+".join(
+                re.escape(token) for token in value.strip().split()
             )
+            regexes.append(r"(?P<{}>{})".format(key, regex_value))
         if not regexes:
             continue
 

eyecite/helpers.py

@@ -1,14 +1,17 @@
 import logging
+from bisect import bisect_right
 from datetime import date
 from typing import List, Optional, Tuple, cast
 
 import regex as re
+from bs4 import BeautifulSoup
 from courts_db import courts
 
 from eyecite.models import (
     CaseCitation,
     CitationBase,
     CitationToken,
+    Document,
     FullCaseCitation,
     FullJournalCitation,
     FullLawCitation,
@@ -129,13 +132,50 @@ def add_post_citation(citation: CaseCitation, words: Tokens) -> None:
         citation.metadata.court = get_court_by_paren(m["court"])
 
 
-def add_defendant(citation: CaseCitation, words: Tokens) -> None:
+def update_plaintiff_from_markup(document, citation, offset) -> None:
+    """Update plaintiff if in Markup
+
+    Check if the plaintiff is inside a markup tag and complete the tag text
+    Args:
+        document: An object containing the document's plain text; markup text
+        citation: An object representing a citation
+        offset: An integer offset used to adjust the citation's span
+
+    Returns: None
+
+    """
+    if document.plain_to_markup is None:
+        raise ValueError("document.plain_to_markup must not be None")
+
+    start = citation.span()[0] - offset
+    end = start + len(citation.metadata.plaintiff)
+
+    m_start = document.plain_to_markup.update(start, bisect_right)
+    m_end = document.plain_to_markup.update(end, bisect_right)
+
+    soup = BeautifulSoup(document.markup_text, "html.parser")
+    for tag in soup.find_all(["em", "i"]):
+        # Convert the tag back to a string.
+        tag_html = str(tag)
+        tag_start = document.markup_text.find(tag_html)
+        if tag_start == -1:
+            continue
+        tag_end = tag_start + len(tag_html)
+        # If the target text is entirely within the tag boundaries, return it.
+        if tag_start <= m_start and m_end <= tag_end:
+            t_start = document.markup_to_plain.update(tag_start, bisect_right)
+            citation.metadata.plaintiff = document.plain_text[t_start:end]
+            break
+
+
+def add_defendant(citation: CaseCitation, document: Document) -> None:
     """Scan backwards from reporter until you find v., in re,
     etc. If no known stop-token is found, no defendant name is stored.  In the
     future, this could be improved.
     """
     # To turn word indexing into char indexing,
     # useful for span, account for shift
+    words = document.words
     offset = 0
     start_index = None
     back_seek = citation.index - BACKWARD_SEEK
@@ -151,6 +191,10 @@ def add_defendant(citation: CaseCitation, words: Tokens) -> None:
                     str(w) for w in words[max(index - 2, 0) : index]
                 ).strip("( ")
                 offset += len(citation.metadata.plaintiff) + 1
+
+                if document.markup_text:
+                    update_plaintiff_from_markup(document, citation, offset)
+
             else:
                 # We don't want to include stop words such as
                 # 'citing' in the span

eyecite/models.py

@@ -291,7 +291,7 @@ class Metadata(CitationBase.Metadata):
 
         year: Optional[str] = None
 
-    def add_metadata(self, words: "Tokens"):
+    def add_metadata(self, document: "Document"):
         """Extract metadata from text before and after citation."""
         self.guess_edition()
 
@@ -375,13 +375,13 @@ class Metadata(FullCitation.Metadata):
         day: Optional[str] = None
         month: Optional[str] = None
 
-    def add_metadata(self, words: "Tokens"):
+    def add_metadata(self, document: "Document"):
         """Extract metadata from text before and after citation."""
         # pylint: disable=import-outside-toplevel
         from eyecite.helpers import add_law_metadata
 
-        add_law_metadata(self, words)
-        super().add_metadata(words)
+        add_law_metadata(self, document.words)
+        super().add_metadata(document)
 
     def corrected_citation_full(self):
         """Return citation with any variations normalized, including extracted
@@ -404,13 +404,13 @@ def corrected_citation_full(self):
 class FullJournalCitation(FullCitation):
     """Citation to a source from `reporters_db/journals.json`."""
 
-    def add_metadata(self, words: "Tokens"):
+    def add_metadata(self, document: "Document"):
         """Extract metadata from text before and after citation."""
         # pylint: disable=import-outside-toplevel
         from eyecite.helpers import add_journal_metadata
 
-        add_journal_metadata(self, words)
-        super().add_metadata(words)
+        add_journal_metadata(self, document.words)
+        super().add_metadata(document)
 
     def corrected_citation_full(self):
         """Return citation with any variations normalized, including extracted
@@ -521,7 +521,7 @@ class Metadata(CaseCitation.Metadata):
         resolved_case_name_short: Optional[str] = None
         resolved_case_name: Optional[str] = None
 
-    def add_metadata(self, words: "Tokens"):
+    def add_metadata(self, document: "Document"):
         """Extract metadata from text before and after citation."""
         # pylint: disable=import-outside-toplevel
         from eyecite.helpers import (
@@ -530,12 +530,12 @@ def add_metadata(self, words: "Tokens"):
             add_pre_citation,
         )
 
-        add_post_citation(self, words)
-        add_defendant(self, words)
-        add_pre_citation(self, words)
+        add_post_citation(self, document.words)
+        add_defendant(self, document)
+        add_pre_citation(self, document.words)
 
         self.guess_court()
-        super().add_metadata(words)
+        super().add_metadata(document)
 
     def corrected_citation_full(self):
         """Return formatted version of extracted cite."""

tests/test_FindTest.py

@@ -1121,3 +1121,194 @@ def test_reference_filtering(self):
                     [isinstance(cite, ReferenceCitation) for cite in citations]
                 )
             )
+
+    def test_unbalanced_case_names(self) -> None:
+        # Can we identify full case names in markup text
+        test_pairs = (
+            # Case Name unbalanced across two tags
+            (
+                (
+                    "and more and more <em>Jin Fuey Moy</em><em>v. United States,</em>\n"
+                    "            254 U.S. 189. Petitioner contends"
+                ),
+                [
+                    case_citation(
+                        volume="254",
+                        reporter="U.S.",
+                        page="189",
+                        metadata={
+                            "plaintiff": "Jin Fuey Moy",
+                            "defendant": "United States",
+                        },
+                    )
+                ],
+                {"clean_steps": ["html", "all_whitespace"]},
+            ),
+            # Extract from one tag and ignore the other
+            (
+                (
+                    "<em>Overruled</em> and so on <em>Jin Fuey Moy v. United States,</em> "
+                    "254 U.S. 189. Petitioner contends"
+                ),
+                [
+                    case_citation(
+                        volume="254",
+                        reporter="U.S.",
+                        page="189",
+                        metadata={
+                            "plaintiff": "Jin Fuey Moy",
+                            "defendant": "United States",
+                        },
+                    )
+                ],
+                {"clean_steps": ["html", "all_whitespace"]},
+            ),
+            # corporation name
+            (
+                "<em>Bell Atlantic Corp. </em>v. <em>Twombly, </em>550 U. S. 544 (2007),",
+                [
+                    case_citation(
+                        volume="550",
+                        reporter="U. S.",
+                        page="544",
+                        year=2007,
+                        metadata={
+                            "plaintiff": "Bell Atlantic Corp.",
+                            "defendant": "Twombly",
+                            "year": "2007",
+                            "court": "scotus",
+                        },
+                    )
+                ],
+                {"clean_steps": ["html", "all_whitespace"]},
+            ),
+            # two word plaintiff
+            (
+                "con-firmable. <em>United States v. Am. Sav. Bank, </em> 508 U.S. 324 (1993). That plan "
+                "proposed to bifurcate the claim and",
+                [
+                    case_citation(
+                        volume="508",
+                        reporter="U.S.",
+                        page="324",
+                        year=1993,
+                        metadata={
+                            "plaintiff": "United States",
+                            "defendant": "Am. Sav. Bank",
+                            "year": "1993",
+                            "court": "scotus",
+                        },
+                    )
+                ],
+                {"clean_steps": ["html", "all_whitespace"]},
+            ),
+            # Extract reference citation full name
+            (
+                (
+                    ". <em>Jin Fuey Moy</em> <em>v. United States,</em> 254 U.S. 189. Petitioner contends.  "
+                    "Regardless in <em>Jin Fuey Moy</em> the court ruled"
+                ),
+                [
+                    case_citation(
+                        volume="254",
+                        reporter="U.S.",
+                        page="189",
+                        metadata={
+                            "plaintiff": "Jin Fuey Moy",
+                            "defendant": "United States",
+                        },
+                    ),
+                    reference_citation(
+                        "Jin Fuey Moy", metadata={"plaintiff": "Jin Fuey Moy"}
+                    ),
+                ],
+                {"clean_steps": ["html", "all_whitespace"]},
+            ),
+            # Extract out with whitespace across two tags
+            (
+                (
+                    '<p id="b453-6">\n'
+                    "  The supreme court of Connecticut, in\n"
+                    "  <em>\n"
+                    "   Beardsley\n"
+                    "  </em>\n"
+                    "  v.\n"
+                    "  <em>\n"
+                    "   Hartford,\n"
+                    "  </em>\n"
+                    "  50 Conn. 529, 541-542, after quoting the maxim of the common law;\n"
+                    "  <em>\n"
+                    "   cessante ratione legis-, cessat ipsa lex,\n"
+                    "  </em>"
+                ),
+                [
+                    case_citation(
+                        volume="50",
+                        reporter="Conn.",
+                        page="529",
+                        metadata={
+                            "plaintiff": "Beardsley",
+                            "defendant": "Hartford",
+                            "pin_cite": "541-542",
+                        },
+                    )
+                ],
+                {"clean_steps": ["html", "all_whitespace"]},
+            ),
+            # identify reference
+            (
+                (
+                    " partially secured by a debtor’s principal residence was not "
+                    "con-firmable. <em>Smart Nobelman v. Am. Sav. Bank, </em>"
+                    "508 U.S. 324 (1993). That plan proposed to bifurcate the claim and... pay the unsecured"
+                    "... only by a lien on the debtor’s principal residence.” "
+                    "codifies the <em>Smart Nobelman </em>decision in individual debtor chapter 11 cases."
+                ),
+                [
+                    case_citation(
+                        volume="508",
+                        reporter="U.S.",
+                        page="324",
+                        metadata={
+                            "plaintiff": "Smart Nobelman",
+                            "defendant": "Am. Sav. Bank",
+                            "year": "1993",
+                        },
+                    ),
+                    reference_citation(
+                        "Smart Nobelman",
+                        metadata={"plaintiff": "Smart Nobelman"},
+                    ),
+                ],
+                {"clean_steps": ["html", "all_whitespace"]},
+            ),
+            # Identify pincite reference
+            (
+                (
+                    " partially secured by a debtor’s principal residence was not "
+                    "con-firmable. <em>Nobelman v. Am. Sav. Bank, </em>"
+                    "508 U.S. 324 (1993). That plan proposed to bifurcate the claim and... pay the unsecured"
+                    "... only by a lien on the debtor’s principal residence.” "
+                    "codifies the  a lien on the debtor’s principal residence.” "
+                    "<em>Nobelman </em>at 332, decision in individual debtor chapter 11 cases."
+                ),
+                [
+                    case_citation(
+                        volume="508",
+                        reporter="U.S.",
+                        page="324",
+                        metadata={
+                            "plaintiff": "Nobelman",
+                            "defendant": "Am. Sav. Bank",
+                            "year": "1993",
+                        },
+                    ),
+                    reference_citation(
+                        "Nobelman",
+                        metadata={"plaintiff": "Nobelman", "pin_cite": "332"},
+                    ),
+                ],
+                {"clean_steps": ["html", "all_whitespace"]},
+            ),
+        )
+        self.run_test_pairs(test_pairs, "Citation extraction")