Working transcript and interest models with basic tests

Author: ajparsons

data/debates2023-03-28d.xml

@@ -2,9 +2,10 @@
 mysoc democracy validation models
 """
 
+from .models.interests import Register
 from .models.popolo import Popolo
 from .models.transcripts import Transcript
 
-__all__ = ["Popolo", "Transcript"]
+__all__ = ["Popolo", "Transcript", "Register"]
 
 __version__ = "0.1.0"

data/regmem2024-05-28.xml

@@ -5,6 +5,7 @@
 import rich
 import typer
 
+from .models.interests import Register
 from .models.popolo import Popolo
 from .models.transcripts import Transcript
 
@@ -14,6 +15,7 @@
 class ValidateOptions(str, Enum):
     POPOLO = "popolo"
     TRANSCRIPT = "transcript"
+    INTERESTS = "interests"
 
 
 @app.command()
@@ -44,9 +46,14 @@ def validate(
             validate_popolo_url_file(url)
     elif type == ValidateOptions.TRANSCRIPT:
         if not file:
-            typer.echo("Must provide a file for a transcript.")
+            typer.echo("Must provide a local file for a transcript.")
             raise typer.Exit(code=1)
         validate_transcript(file)
+    elif type == ValidateOptions.INTERESTS:
+        if not file:
+            typer.echo("Must provide a local file for interests.")
+            raise typer.Exit(code=1)
+        validate_interests(file)
 
 
 def validate_popolo_file(file: Path):
@@ -89,13 +96,27 @@ def validate_transcript(file: Path):
     Returns Exit 1 if a validation error.
     """
     try:
-        Transcript.from_path(file)
+        Transcript.from_xml_path(file)
     except Exception as e:
         typer.echo(f"Error: {e}")
         rich.print("[red]Invalid Transcript file[/red]")
         raise typer.Exit(code=1)
     rich.print("[green]Valid Transcript file[/green]")
 
 
+def validate_interests(file: Path):
+    """
+    Validate a mysoc style Popolo file.
+    Returns Exit 1 if a validation error.
+    """
+    try:
+        Register.from_xml_path(file)
+    except Exception as e:
+        typer.echo(f"Error: {e}")
+        rich.print("[red]Invalid Interests file[/red]")
+        raise typer.Exit(code=1)
+    rich.print("[green]Valid Interests file[/green]")
+
+
 if __name__ == "__main__":
     app()

src/mysoc_validator/__init__.py

@@ -0,0 +1,50 @@
+"""
+Structure for handling a register of interests
+"""
+
+from __future__ import annotations
+
+from datetime import date
+from typing import Literal as Literal
+from typing import Optional
+
+from pydantic import AliasChoices, Field
+
+from .xml_base import AsAttrSingle, Items, MixedContent, StrictBaseXMLModel
+
+
+class Item(StrictBaseXMLModel, tags=["item"]):
+    item_class: str = Field(validation_alias="class", serialization_alias="class")
+    contents: MixedContent
+
+
+class Record(StrictBaseXMLModel, tags=["record"]):
+    item_class: Optional[str] = Field(
+        validation_alias="class", serialization_alias="class", default=None
+    )
+    items: Items[Item]
+
+
+class Category(StrictBaseXMLModel, tags=["category"]):
+    type: str
+    name: str
+    records: Items[Record]
+
+
+class PersonEntry(StrictBaseXMLModel, tags=["regmem"]):
+    person_id: str = Field(
+        validation_alias=AliasChoices("person_id", "personid"),
+        serialization_alias="personid",
+    )
+    membername: str
+    date: date
+    record: AsAttrSingle[Optional[Record]]
+    categories: Items[Category] = Field(
+        default_factory=list,
+        validation_alias=AliasChoices("categories", "@children"),
+        serialization_alias="@children",
+    )
+
+
+class Register(StrictBaseXMLModel, tags=["publicwhip"]):
+    person_entires: Items[PersonEntry]

src/mysoc_validator/__main__.py

@@ -1,12 +1,18 @@
+"""
+Structure for handling the main Parlparse people.json file
+Following the general shape of the popolo standard.
+
+"""
+
 from __future__ import annotations
 
+import json
 import re
 from bisect import bisect_left
 from datetime import date
 from enum import Enum
 from itertools import groupby
 from pathlib import Path
-from types import TracebackType
 from typing import (
     Annotated,
     Any,
@@ -1204,12 +1210,20 @@ def model_post_init(self, __context: Any) -> None:
         self.posts.set_parent(self)
 
     @classmethod
-    def from_path(cls, json_path: Path) -> Popolo:
-        return cls.model_validate_json(json_path.read_text())
+    def from_json_str(cls, json_str: str, *, validate: bool = True) -> Popolo:
+        if validate:
+            return cls.model_validate_json(json_str)
+        else:
+            data = json.loads(json_str)
+            return cls.model_construct(data)
+
+    @classmethod
+    def from_path(cls, json_path: Path, validate: bool = True) -> Popolo:
+        return cls.from_json_str(json_path.read_text(), validate=validate)
 
     @classmethod
-    def from_url(cls, url: str) -> Popolo:
-        return cls.model_validate_json(requests.get(url).text)
+    def from_url(cls, url: str, validate: bool = True) -> Popolo:
+        return cls.from_json_str(requests.get(url).text, validate=validate)
 
     @classmethod
     def from_parlparse(cls, branch: str = "master") -> Popolo:
@@ -1230,15 +1244,3 @@ def to_json_str(self) -> str:
     def to_path(self, json_path: Path) -> None:
         data = self.to_json_str()
         json_path.write_text(data)
-
-    def __enter__(self, json_path: Path) -> Popolo:
-        self._json_path = json_path
-        return self.from_path(json_path)
-
-    def __exit__(
-        self,
-        exc_type: type[BaseException] | None,
-        exc_val: BaseException | None,
-        exc_tb: TracebackType | None,
-    ) -> None:
-        self.to_path(self._json_path)

src/mysoc_validator/models/interests.py

@@ -1,3 +1,7 @@
+"""
+Structure for handling a transcript file.
+"""
+
 from __future__ import annotations
 
 from typing import (
@@ -16,7 +20,7 @@
 from pydantic import AliasChoices, Discriminator, Field, Tag
 
 from .xml_base import (
-    AsAttr,
+    AsAttrSingle,
     BaseXMLModel,
     Items,
     MixedContent,
@@ -142,6 +146,8 @@ class RepList(
     StrictBaseXMLModel,
     tags=["replist", "mplist", "msplist", "mslist", "mlalist", "lordlist"],
 ):
+    # this duplication is in the sources - twfy internally converts to
+    # aye, no, both, absent
     vote: Literal[
         "aye",
         "no",
@@ -177,9 +183,9 @@ class Division(StrictBaseXMLModel, tags=["division"]):
     divnumber: int
     colnum: Optional[int] = None
     time: Optional[str] = None
-    count: AsAttr[Optional[DivisionCount]]
+    count: AsAttrSingle[Optional[DivisionCount]]
     motion: Optional[Motion] = None
-    items: Items[RepList]
+    representatives: Items[RepList]
 
 
 def extract_tag(v: Any) -> str:
@@ -188,9 +194,11 @@ def extract_tag(v: Any) -> str:
 
 class Transcript(StrictBaseXMLModel, tags=["publicwhip"]):
     scraper_version: Optional[str] = Field(
-        default=None, validation_alias="scraperversion"
+        default=None,
+        validation_alias=AliasChoices("scraper_version", "scraperversion"),
+        serialization_alias="scraperversion",
     )
-    latest: Optional[str] = Field(default=None, validation_alias="latest")
+    latest: Optional[str] = Field(default=None)
     items: Items[
         Annotated[
             Union[

src/mysoc_validator/models/popolo.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+
+from mysoc_validator.models.interests import Register
+
+
+def test_interests_load():
+    Register.from_xml_path(Path("data", "regmem2024-05-28.xml"))
+
+
+def test_interests_round_trip():
+    t = Register.from_xml_path(Path("data", "regmem2024-05-28.xml"))
+
+    dumped_xml = t.model_dump_xml()
+
+    t2 = Register.model_validate_xml(dumped_xml)
+
+    dumped_xml_2 = t2.model_dump_xml()
+
+    assert dumped_xml == dumped_xml_2

src/mysoc_validator/models/transcripts.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+
+from mysoc_validator.models.transcripts import Transcript
+
+
+def test_transcript_load():
+    Transcript.from_xml_path(Path("data", "debates2023-03-28d.xml"))
+
+
+def test_transcript_round_trip():
+    t = Transcript.from_xml_path(Path("data", "debates2023-03-28d.xml"))
+
+    dumped_xml = t.model_dump_xml()
+
+    t2 = Transcript.model_validate_xml(dumped_xml)
+
+    dumped_xml_2 = t2.model_dump_xml()
+
+    assert dumped_xml == dumped_xml_2