🏗️(project) migrate to pydantic v2 and switch tests to polyfactory

Migrating to `pydantic` v2 should speed up processing and allow
interoperability with projects such as `warren`. This migration makes the
hypothesis package used in tests obsolete, which is why we introduce
`polyfactory`.

Author: wilbrdt

CHANGELOG.md

@@ -8,6 +8,11 @@ and this project adheres to
 
 ## [Unreleased]
 
+### Changed
+
+- Upgrade `pydantic` to `2.7.0`
+- Migrate model tests from hypothesis strategies to polyfactory
+
 ## [4.2.0] - 2024-04-08
 
 ### Added

pyproject.toml

@@ -32,7 +32,8 @@ dependencies = [
     # library (mostly models).
     "importlib-metadata>=7.0.1, <8.0",
     "langcodes>=3.2.0",
-    "pydantic[dotenv,email]>=1.10.0, <2.0",
+    "pydantic[email]>=2.5.3,<3.0",
+    "pydantic_settings>=2.1.0,<3.0",
     "rfc3987>=1.3.0",
 ]
 dynamic = ["version"]
@@ -92,7 +93,6 @@ dev = [
     "black==24.3.0",
     "cryptography==42.0.5",
     "factory-boy==3.3.0",
-    "hypothesis<6.92.0", # pin as hypothesis 6.92.0 observability feature seems broken
     "logging-gelf==0.0.32",
     "mike==2.0.0",
     "mkdocs==1.5.3",
@@ -102,6 +102,7 @@ dev = [
     "moto==5.0.5",
     "mypy==1.9.0",
     "neoteroi-mkdocs==1.0.5",
+    "polyfactory==2.15.0",
     "pyfakefs==5.4.0",
     "pymdown-extensions==10.7.1",
     "pytest<8.0.0", # pin as pytest-httpx<0.23.0 is not compatible with pytest 8.0.0 

src/ralph/api/__init__.py

@@ -51,4 +51,7 @@ async def whoami(
     user: AuthenticatedUser = Depends(get_authenticated_user),
 ) -> Dict[str, Any]:
     """Return the current user's username along with their scopes."""
-    return {"agent": user.agent, "scopes": user.scopes}
+    return {
+        "agent": user.agent.model_dump(mode="json", exclude_none=True),
+        "scopes": user.scopes,
+    }

src/ralph/api/auth/basic.py

@@ -1,6 +1,7 @@
 """Basic authentication & authorization related tools for the Ralph API."""
 
 import logging
+import os
 from functools import lru_cache
 from pathlib import Path
 from threading import Lock
@@ -10,7 +11,7 @@
 from cachetools import TTLCache, cached
 from fastapi import Depends, HTTPException, status
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
-from pydantic import BaseModel, root_validator
+from pydantic import RootModel, model_validator
 from starlette.authentication import AuthenticationError
 
 from ralph.api.auth.user import AuthenticatedUser
@@ -40,45 +41,42 @@ class UserCredentials(AuthenticatedUser):
     username: str
 
 
-class ServerUsersCredentials(BaseModel):
+class ServerUsersCredentials(RootModel[List[UserCredentials]]):
     """Custom root pydantic model.
 
     Describe expected list of all server users credentials as stored in
     the credentials file.
 
     Attributes:
-        __root__ (List): Custom root consisting of the
+        root (List): Custom root consisting of the
                         list of all server users credentials.
     """
 
-    __root__: List[UserCredentials]
-
     def __add__(self, other) -> Any:  # noqa: D105
-        return ServerUsersCredentials.parse_obj(self.__root__ + other.__root__)
+        return ServerUsersCredentials.model_validate(self.root + other.root)
 
     def __getitem__(self, item: int) -> UserCredentials:  # noqa: D105
-        return self.__root__[item]
+        return self.root[item]
 
     def __len__(self) -> int:  # noqa: D105
-        return len(self.__root__)
+        return len(self.root)
 
     def __iter__(self) -> Iterator[UserCredentials]:  # noqa: D105
-        return iter(self.__root__)
+        return iter(self.root)
 
-    @root_validator
-    @classmethod
-    def ensure_unique_username(cls, values: Any) -> Any:
+    @model_validator(mode="after")
+    def ensure_unique_username(self) -> Any:
         """Every username should be unique among registered users."""
-        usernames = [entry.username for entry in values.get("__root__")]
+        usernames = [entry.username for entry in self.root]
         if len(usernames) != len(set(usernames)):
             raise ValueError(
                 "You cannot create multiple credentials with the same username"
             )
-        return values
+        return self
 
 
 @lru_cache()
-def get_stored_credentials(auth_file: Path) -> ServerUsersCredentials:
+def get_stored_credentials(auth_file: os.PathLike) -> ServerUsersCredentials:
     """Helper to read the credentials/scopes file.
 
     Read credentials from JSON file and stored them to avoid reloading them with every
@@ -96,7 +94,9 @@ def get_stored_credentials(auth_file: Path) -> ServerUsersCredentials:
         msg = "Credentials file <%s> not found."
         logger.warning(msg, auth_file)
         raise AuthenticationError(msg.format(auth_file))
-    return ServerUsersCredentials.parse_file(auth_file)
+
+    with open(auth_file, encoding=settings.LOCALE_ENCODING) as f:
+        return ServerUsersCredentials.model_validate_json(f.read())
 
 
 @cached(

src/ralph/api/auth/oidc.py

@@ -9,7 +9,7 @@
 from fastapi.security import HTTPBearer, OpenIdConnect
 from jose import ExpiredSignatureError, JWTError, jwt
 from jose.exceptions import JWTClaimsError
-from pydantic import AnyUrl, BaseModel, Extra
+from pydantic import AnyUrl, BaseModel, ConfigDict
 from typing_extensions import Annotated
 
 from ralph.api.auth.user import AuthenticatedUser, UserScopes
@@ -45,14 +45,13 @@ class IDToken(BaseModel):
 
     iss: str
     sub: str
-    aud: Optional[str]
+    aud: Optional[str] = None
     exp: int
     iat: int
-    scope: Optional[str]
-    target: Optional[str]
+    scope: Optional[str] = None
+    target: Optional[str] = None
 
-    class Config:  # noqa: D106
-        extra = Extra.ignore
+    model_config = ConfigDict(extra="ignore")
 
 
 @lru_cache()
@@ -144,7 +143,7 @@ def get_oidc_user(
             headers={"WWW-Authenticate": "Bearer"},
         ) from exc
 
-    id_token = IDToken.parse_obj(decoded_token)
+    id_token = IDToken.model_validate(decoded_token)
 
     user = AuthenticatedUser(
         agent={"openid": f"{id_token.iss}/{id_token.sub}"},

src/ralph/api/auth/user.py

@@ -1,8 +1,10 @@
 """Authenticated user for the Ralph API."""
 
-from typing import Dict, FrozenSet, Literal, Optional
+from typing import FrozenSet, Literal, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, RootModel
+
+from ralph.models.xapi.base.agents import BaseXapiAgent
 
 Scope = Literal[
     "statements/write",
@@ -18,7 +20,7 @@
 ]
 
 
-class UserScopes(FrozenSet[Scope]):
+class UserScopes(RootModel[FrozenSet[Scope]]):
     """Scopes available to users."""
 
     def is_authorized(self, requested_scope: Scope):
@@ -47,19 +49,11 @@ def is_authorized(self, requested_scope: Scope):
         }
 
         expanded_user_scopes = set()
-        for scope in self:
+        for scope in self.root:
             expanded_user_scopes.update(expanded_scopes.get(scope, {scope}))
 
         return requested_scope in expanded_user_scopes
 
-    @classmethod
-    def __get_validators__(cls):  # noqa: D105
-        def validate(value: FrozenSet[Scope]):
-            """Transform value to an instance of UserScopes."""
-            return cls(value)
-
-        yield validate
-
 
 class AuthenticatedUser(BaseModel):
     """Pydantic model for user authentication.
@@ -70,6 +64,6 @@ class AuthenticatedUser(BaseModel):
         target (str or None): The target index or database to store statements into.
     """
 
-    agent: Dict
+    agent: BaseXapiAgent
     scopes: UserScopes
-    target: Optional[str]
+    target: Optional[str] = None

src/ralph/api/forwarding.py

@@ -42,16 +42,16 @@ async def forward_xapi_statements(
             try:
                 # NB: post or put
                 req = await getattr(client, method)(
-                    forwarding.url,
+                    str(forwarding.url),
                     json=statements,
                     auth=(forwarding.basic_username, forwarding.basic_password),
                     timeout=forwarding.timeout,
                 )
                 req.raise_for_status()
                 msg = "Forwarded %s statements to %s with success."
                 if isinstance(statements, list):
-                    logger.debug(msg, len(statements), forwarding.url)
+                    logger.debug(msg, len(statements), str(forwarding.url))
                 else:
-                    logger.debug(msg, 1, forwarding.url)
+                    logger.debug(msg, 1, str(forwarding.url))
             except (RequestError, HTTPStatusError) as error:
                 logger.error("Failed to forward xAPI statements. %s", error)

src/ralph/api/models.py

@@ -7,7 +7,7 @@
 from typing import Optional, Union
 from uuid import UUID
 
-from pydantic import AnyUrl, BaseModel, Extra
+from pydantic import AnyUrl, BaseModel, ConfigDict
 
 from ..models.xapi.base.agents import BaseXapiAgent
 from ..models.xapi.base.groups import BaseXapiGroup
@@ -30,13 +30,7 @@ class BaseModelWithLaxConfig(BaseModel):
     we receive statements through the API.
     """
 
-    class Config:
-        """Enable extra properties.
-
-        Useful for not having to perform comprehensive validation.
-        """
-
-        extra = Extra.allow
+    model_config = ConfigDict(extra="allow", coerce_numbers_to_str=True)
 
 
 class LaxObjectField(BaseModelWithLaxConfig):
@@ -65,6 +59,6 @@ class LaxStatement(BaseModelWithLaxConfig):
     """
 
     actor: Union[BaseXapiAgent, BaseXapiGroup]
-    id: Optional[UUID]
+    id: Optional[UUID] = None
     object: LaxObjectField
     verb: LaxVerbField

src/ralph/api/routers/health.py

@@ -47,7 +47,7 @@ async def heartbeat(response: Response) -> Heartbeat:
 
     Return a 200 if all checks are successful.
     """
-    statuses = Heartbeat.construct(
+    statuses = Heartbeat.model_construct(
         database=await await_if_coroutine(BACKEND_CLIENT.status())
     )
     if not statuses.is_alive: