(DS-2816)[API] feat: create Pydantic v2 models for Titelive products provided by Data

Author: jmontagnat-pass

api/src/pcapi/connectors/big_query/queries/base.py

@@ -1,17 +1,20 @@
 import typing
 from collections.abc import Mapping
 
+import pydantic as pydantic_v2
 import pydantic.v1 as pydantic_v1
 
 import pcapi.connectors.big_query as big_query_connector
 
 
-Row = typing.TypeVar("Row", bound=pydantic_v1.BaseModel)
+AnyBaseModel = pydantic_v1.BaseModel | pydantic_v2.BaseModel
+
+Row = typing.TypeVar("Row", bound=AnyBaseModel)
 RowIterator = typing.Generator[Row, None, None]
 
 
 class MalformedRow(Exception):
-    def __init__(self, msg: str, index: int, model: type[pydantic_v1.BaseModel], raw_query: str):
+    def __init__(self, msg: str, index: int, model: type[AnyBaseModel], raw_query: str):
         self.index = index
         self.model = model
         self.raw_query = raw_query
@@ -30,13 +33,13 @@ def execute(self, page_size: int = 1_000, **parameters: typing.Any) -> RowIterat
         for index, row in enumerate(rows):
             try:
                 yield self.model(**typing.cast(Mapping, row))
-            except (pydantic_v1.ValidationError, TypeError) as err:
+            except (pydantic_v1.ValidationError, pydantic_v2.ValidationError, TypeError) as err:
                 raise MalformedRow(msg=str(row), index=index, model=self.model, raw_query=self.raw_query) from err
 
     @property
     def raw_query(self) -> str:
         raise NotImplementedError()
 
     @property
-    def model(self) -> type[pydantic_v1.BaseModel]:
+    def model(self) -> type[AnyBaseModel]:
         raise NotImplementedError()

api/src/pcapi/connectors/big_query/queries/product.py

@@ -1,26 +1,123 @@
+import datetime
 import logging
 import typing
+from typing import Any
 
-from pydantic.v1.class_validators import validator
+import pydantic as pydantic_v2
 
 from pcapi import settings
 from pcapi.connectors.big_query.queries.base import BaseQuery
-from pcapi.connectors.serialization.titelive_serializers import TiteLiveBookArticle
-from pcapi.connectors.serialization.titelive_serializers import TiteliveMusicArticle
 from pcapi.connectors.titelive import TiteliveBase
+from pcapi.utils import date as date_utils
 
 
 logger = logging.getLogger(__name__)
 
 
-class BigQueryTiteliveBookProductModel(TiteLiveBookArticle):
+def _format_gtl_code(code: str) -> str:
+    # A GTL id is 8 characters long.
+    # Each pair represents a GTL level.
+    # The first 2 characters are level 1, the next 2 are level 2, etc.
+    #  - example: 05030000 corresponds to a level 1 GTL of 05 and a level 2 of 03. So "Tourism & Travel World".
+
+    # We receive gtl_ids without leading zeros, and sometimes without trailing ones.
+    # We must add them to have an 8-character code.
+
+    # We start by adding the missing zeros to the left.
+    # If we receive a code with an odd number of characters, we must add a zero to the left.
+    # '5030000'  -> '05030000'
+    # Otherwise, we don't add anything.
+    # '110400' -> '110400'
+
+    if len(code) % 2 == 1:
+        code = "0" + code
+
+    # Then we add the missing zeros to the right to have 8 characters.
+    # '050300' -> '05030000'
+    # '110400' -> '11040000'
+    code = code.ljust(8, "0")
+    return code
+
+
+class GenreTitelive(pydantic_v2.BaseModel):
+    code: str = pydantic_v2.Field(min_length=8, max_length=8)
+    libelle: str
+
+    @pydantic_v2.field_validator("code", mode="before")
+    @classmethod
+    def validate_code(cls, code: str) -> str:
+        return _format_gtl_code(code)
+
+
+class TiteliveGtl(pydantic_v2.BaseModel):
+    first: dict[str, GenreTitelive] | None = None
+
+
+class BigQueryTiteliveProductBaseModel(pydantic_v2.BaseModel):
+    model_config = pydantic_v2.ConfigDict(populate_by_name=True)
+
     ean: str
     titre: str
-    recto_uuid: str | None
-    verso_uuid: str | None
+    recto_uuid: str | None = None
+    verso_uuid: str | None = None
+    has_image: bool = pydantic_v2.Field(alias="image", default=False)
+    has_verso_image: bool = pydantic_v2.Field(alias="image_4", default=False)
+
+    resume: str | None = None
+    codesupport: str | None = None
+    gtl: TiteliveGtl | None = None
+    dateparution: datetime.date | None = None
+    editeur: str | None = None
+    prix: float | None = None
+
+    gencod: str = pydantic_v2.Field(min_length=13, max_length=13)
+
+    @pydantic_v2.model_validator(mode="before")
+    @classmethod
+    def parse_empty_strings_as_none(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            return {k: (None if v == "" else v) for k, v in data.items()}
+        return data
+
+    @pydantic_v2.field_validator("dateparution", mode="before")
+    @classmethod
+    def parse_dates(cls, v: Any) -> Any:
+        return date_utils.parse_french_date(v)
+
+    @pydantic_v2.field_validator("gtl", mode="before")
+    @classmethod
+    def validate_gtl(cls, gtl: TiteliveGtl | list) -> TiteliveGtl | None:
+        if isinstance(gtl, list):
+            return None
+        return gtl
+
+    @pydantic_v2.field_validator("has_image", "has_verso_image", mode="before")
+    @classmethod
+    def validate_image(cls, image: str | int | None) -> bool:
+        # The API currently sends 0 (int) if no image is available, and "1" (str) if an image is available.
+        # Because it has been famously flaky in the past, we are being defensive here and consider:
+        # - all forms of 0 and None as False.
+        # - all forms of "1" as True.
+        if image is not None and int(image) not in (0, 1):
+            raise ValueError(f"unhandled image value {image}")
+        return bool(image and int(image) == 1)
+
+
+class BigQueryTiteliveBookProductModel(BigQueryTiteliveProductBaseModel):
     auteurs_multi: list[str]
-
-    @validator("auteurs_multi", pre=True)
+    langueiso: str | None = None
+    taux_tva: str | None = None
+    id_lectorat: str | None = None
+
+    @pydantic_v2.field_validator("taux_tva", mode="before")
+    @classmethod
+    def validate_code_tva(cls, value: typing.Literal[0] | str | None) -> str | None:
+        if value == 0:
+            return None
+        return value
+
+    @pydantic_v2.field_validator("auteurs_multi", mode="before")
+    @classmethod
     def validate_auteurs_multi(cls, auteurs_multi: typing.Any) -> list:
         if isinstance(auteurs_multi, list):
             return auteurs_multi
@@ -62,11 +159,16 @@ class BigQueryTiteliveBookProductDeltaQuery(BaseQuery):
     model = BigQueryTiteliveBookProductModel
 
 
-class BigQueryTiteliveMusicProductModel(TiteliveMusicArticle):
-    ean: str
-    titre: str
-    recto_uuid: str | None
-    verso_uuid: str | None
+class BigQueryTiteliveMusicProductModel(BigQueryTiteliveProductBaseModel):
+    label: str | None = None
+    compositeur: str | None = None
+    interprete: str | None = None
+    nb_galettes: str | None = None
+    artiste: str | None = None
+    commentaire: str | None = None
+    contenu_explicite: int | None = None
+    dispo: int | None = None
+    distributeur: str | None = None
 
 
 class BigQueryTiteliveMusicProductDeltaQuery(BaseQuery):
@@ -87,7 +189,10 @@ class BigQueryTiteliveMusicProductDeltaQuery(BaseQuery):
             performer as interprete,
             nb_discs as nb_galettes,
             artist as artiste,
-
+            comment as commentaire,
+            explicit_content as contenu_explicite,
+            availability as dispo,
+            distributor as distributeur,
             recto_uuid,
             verso_uuid,
             image,

api/src/pcapi/core/providers/titelive_book_search.py

@@ -1,9 +1,13 @@
 import logging
 import textwrap
+from typing import Protocol
+from typing import cast
 
 import pydantic.v1 as pydantic
 
 import pcapi.core.fraud.models as fraud_models
+from pcapi.connectors.big_query.queries.product import BigQueryTiteliveBookProductModel
+from pcapi.connectors.big_query.queries.product import BigQueryTiteliveProductBaseModel
 from pcapi.connectors.serialization.titelive_serializers import GenreTitelive
 from pcapi.connectors.serialization.titelive_serializers import TiteLiveBookArticle
 from pcapi.connectors.serialization.titelive_serializers import TiteLiveBookWork
@@ -156,15 +160,20 @@ def extract_eans_from_titelive_response(json_response: list[dict]) -> set[str]:
 EMPTY_GTL = GenreTitelive(code="".zfill(8), libelle="Empty GTL")
 
 
-def get_gtl_id(article: TiteliveArticle) -> str:
+class HasCode(Protocol):
+    code: str
+
+
+def get_gtl_id(article: TiteliveArticle | BigQueryTiteliveProductBaseModel) -> str:
     if not article.gtl or not article.gtl.first:
         return EMPTY_GTL.code
-    most_precise_genre = max(article.gtl.first.values(), key=lambda gtl: gtl.code)
-    gtl_id = most_precise_genre
-    return gtl_id.code
+    most_precise_genre = max(article.gtl.first.values(), key=lambda gtl: cast(HasCode, gtl).code)
+    return cast(HasCode, most_precise_genre).code
 
 
-def get_ineligibility_reasons(article: TiteLiveBookArticle, title: str) -> list[str] | None:
+def get_ineligibility_reasons(
+    article: TiteLiveBookArticle | BigQueryTiteliveBookProductModel, title: str
+) -> list[str] | None:
     # Ouvrage avec pierres ou encens, jeux de société ou escape game en coffrets,
     # marchandisage : jouets, goodies, peluches, posters, papeterie, etc...
     reasons = []

api/src/pcapi/core/providers/titelive_bq_music_search.py

@@ -59,7 +59,7 @@ def fill_product_specifics(
             artist=bq_product.artiste,
             author=bq_product.compositeur,
             comment=bq_product.commentaire,
-            contenu_explicite=bq_product.contenu_explicite,
+            contenu_explicite=str(bq_product.contenu_explicite),
             date_parution=str(bq_product.dateparution) if bq_product.dateparution else None,
             dispo=bq_product.dispo,
             distributeur=bq_product.distributeur,

api/tests/providers/test_titelive_music_product_synchro_with_bigquery.py

@@ -168,7 +168,7 @@ def test_titelive_sync_failure_event(self, mock_gcp_data, mock_gcp_backend):
     def test_sync_skips_unallowed_format(self, mock_execute, mock_gcp_data, mock_gcp_backend):
         providers_factories.ProviderFactory.create(name=providers_constants.TITELIVE_ENRICHED_BY_DATA)
         fixture = copy.deepcopy(fixtures.MUSIC_SEARCH_FIXTURE)
-        fixture["result"][1]["article"]["1"]["codesupport"] = 35
+        fixture["result"][1]["article"]["1"]["codesupport"] = "35"
         bq_products = self._prepare_iterator_from_fixture(fixture)
         mock_execute.return_value = iter(bq_products)