0.13.0a1 Pydantic v2 support

.github/workflows/ci.yml

@@ -4,13 +4,13 @@ on: [push, pull_request]
 
 jobs:
   test:
-    name: "Test Python ${{ matrix.python-version }} (${{ (matrix.pin-versions == '' && 'Latest Dependencies') || 'Compatibility Check' }})"
+    name: "Test Python ${{ matrix.python-version }} (${{ (matrix.pin-versions == '' && 'Latest Dependencies') || (contains(matrix.pin-versions, 'pydantic<2') && 'Pydantic v1') || 'Compatibility Check' }})"
     timeout-minutes: 5
     strategy:
       fail-fast: false
       matrix:
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
-        pin-versions: ["", "boto3==1.10.0 pydantic==1.7.1 \"\\\"importlib_metadata==1.0.0; python_version<'3.8'\\\"\""]
+        pin-versions: ["", '"\"pydantic<2\""', 'boto3==1.10.0 pydantic==1.7.1 "\"importlib_metadata==1.0.0; python_version<''3.8''\""']
 
     env:
       PYTHON: ${{ matrix.python-version }}

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.13.0a1 2023-11-03
+
+- Add support for pydantic v2
+
 ## 0.12.0 2023-09-22
 
 - Support KEYS_ONLY and INCLUDE DynamoDB indexes

README.md

@@ -61,10 +61,10 @@ class Event(Dyntastic):
 p = Product(name="bread", price=3.49)
 # Product(product_id='d2e91c30-e701-422f-b71b-465b02749f18', name='bread', description=None, price=3.49, tax=None)
 
-p.dict()
+p.model_dump()
 # {'product_id': 'd2e91c30-e701-422f-b71b-465b02749f18', 'name': 'bread', 'description': None, 'price': 3.49, 'tax': None}
 
-p.json()
+p.model_dump_json()
 # '{"product_id": "d2e91c30-e701-422f-b71b-465b02749f18", "name": "bread", "description": null, "price": 3.49, "tax": null}'
 
 ```

dyntastic/attr.py

@@ -1,4 +1,3 @@
-import json
 from collections import defaultdict
 from decimal import Decimal
 from typing import Optional, Union
@@ -7,16 +6,17 @@
 from boto3.dynamodb.conditions import Key as _DynamoKey
 from boto3.dynamodb.types import TypeDeserializer
 from pydantic import BaseModel
-from pydantic.json import pydantic_encoder
+
+from . import pydantic_compat
 
 # serialization helpers
 
 
-# Except for sets and Decimal, json.loads(pydantic_model.json()) would work.
+# Except for sets and Decimal, pydantic_core.as_jsonable_python would work.
 # To properly support these cases, however, we need to walk through the data.
 def serialize(data):
     if isinstance(data, BaseModel):
-        return serialize(data.dict())
+        return serialize(pydantic_compat.model_dump(data))
     elif isinstance(data, dict):
         # TODO: May not actually want to filter out None. Without the filter,
         # all None fields in the pydantic model appear as Null instead of
@@ -30,7 +30,7 @@ def serialize(data):
         return data
     else:
         # handle types like datetime
-        return json.loads(json.dumps(data, default=pydantic_encoder))
+        return pydantic_compat.to_jsonable_python(data)
 
 
 # Hacky way to avoid boto3's annoying Binary type

dyntastic/main.py

@@ -15,9 +15,9 @@
 from contextvars import ContextVar
 
 from boto3.dynamodb.conditions import ConditionBase
-from pydantic import BaseModel, PrivateAttr, validate_model
+from pydantic import BaseModel, PrivateAttr
 
-from . import attr, transact
+from . import attr, pydantic_compat, transact
 from .attr import Attr, _UpdateAction, translate_updates
 from .batch import BatchWriter, invoke_with_backoff
 from .exceptions import DoesNotExist
@@ -75,7 +75,7 @@ def __init__(
         self.index_name = index_name
 
 
-class Dyntastic(_TableMetadata, BaseModel):
+class Dyntastic(_TableMetadata, pydantic_compat.BaseModel):
     _dyntastic_unrefreshed: bool = PrivateAttr(default=False)
     _dyntastic_missing_attributes_from_index: bool = PrivateAttr(default=False)
 
@@ -93,14 +93,10 @@ def get_model(cls, item: dict):
     def _dyntastic_load_model(cls, item: dict, load_full_item: bool = False):
         model = cls.get_model(item)
 
-        validated, fields_set, errors = validate_model(model, item)
-        if errors:
+        data, had_validation_errors = pydantic_compat.try_model_construct(model, item)
+        if had_validation_errors:
             # assume KEYS_ONLY or INCLUDE index
-            fields_in_dynamo = {key: value for key, value in validated.items() if key in fields_set}
-            data = model.construct(**fields_in_dynamo)
             data._dyntastic_missing_attributes_from_index = True
-        else:
-            data = model(**item)
 
         if load_full_item:
             data.refresh()
@@ -140,7 +136,7 @@ def batch_get(
         keys: Union[List[str], List[Tuple[str, str]]],
         consistent_read: bool = False,
     ) -> List[_T]:
-        hash_key_type = cls.__fields__[cls.__hash_key__].type_
+        hash_key_type = pydantic_compat.field_type(cls, cls.__hash_key__)
 
         if cls.__range_key__ and not all(isinstance(key, (list, tuple)) and len(key) == 2 for key in keys):
             raise ValueError(f"Must provide (hash_key, range_key) tuples as `keys` to {cls.__name__}.batch_get()")
@@ -304,7 +300,7 @@ def scan_page(
         return ResultPage(items, last_evaluated_key)
 
     def save(self, *, condition: Optional[ConditionBase] = None):
-        data = self.dict(by_alias=True)
+        data = pydantic_compat.model_dump(self, by_alias=True)
         dynamo_serialized = attr.serialize(data)
         return self._dyntastic_call("put_item", Item=dynamo_serialized, ConditionExpression=condition)
 
@@ -449,7 +445,11 @@ def _resolve_table_name(cls) -> str:
     def _dynamodb_type(cls, key: str) -> str:
         # Note: pragma nocover on the following line as coverage marks the ->exit branch as
         # being missed (since we can always find a field matching the key passed in)
-        python_type = next(field.type_ for field in cls.__fields__.values() if field.alias == key)  # pragma: nocover
+        python_type = next(
+            pydantic_compat.annotation(field)
+            for field_name, field in pydantic_compat.model_fields(cls).items()
+            if pydantic_compat.alias(field_name, field) == key
+        )  # pragma: nocover
         if python_type == bytes:
             return "B"
         elif python_type in (int, Decimal, float):
@@ -606,15 +606,40 @@ def _dyntastic_call(cls, operation: str, **kwargs):
     def ignore_unrefreshed(self):
         self._dyntastic_unrefreshed = False
 
+    def _get_private_field(self, attr: str):
+        try:
+            return getattr(self, attr)
+        except AttributeError:  # pragma: nocover
+            # Note: Without this remapping AttributeError -> Exception, it is
+            # particularly difficult to debug the issues that arise. For
+            # example, without "model_post_init" in the __getattribute__
+            # function below, the error appears as all model fields raising
+            # AttributeError on access due to private fields like
+            # _dyntastic_unrefreshed triggering that during pydantic's
+            # __getattr__.
+            #
+            # Long story short, this should catch bugs in a much more easy-to-debug way.
+
+            raise Exception(f"{attr} could not be accessed, dyntastic<->pydantic bug")
+
     def __getattribute__(self, attr: str):
+        # breakpoint()
         # All of the code in this function works to "disable" an instance
         # that has been updated with refresh=False, to avoid accidentally
         # working with stale data
 
-        if attr.startswith("_") or attr in {"refresh", "ignore_unrefreshed", "ConditionException"}:
+        if attr.startswith("_") or attr in {
+            "refresh",
+            "ignore_unrefreshed",
+            "ConditionException",
+            # Note: Without model_post_init here, _dyntastic_unrefreshed will
+            # be accessed below before pydantic v2 is fully initialized,
+            # which causes a bad state (for example, no field attribute can be accessed on the class)
+            "model_post_init",
+        }:
             return super().__getattribute__(attr)
 
-        if object.__getattribute__(self, "_dyntastic_unrefreshed"):
+        if self._get_private_field("_dyntastic_unrefreshed"):
             raise ValueError(
                 "Dyntastic instance was not refreshed after update. "
                 "Call refresh(), or use ignore_unrefreshed() to ignore safety checks"
@@ -623,18 +648,24 @@ def __getattribute__(self, attr: str):
         try:
             return super().__getattribute__(attr)
         except AttributeError:
-            if object.__getattribute__(self, "_dyntastic_missing_attributes_from_index"):
+            if self._get_private_field("_dyntastic_missing_attributes_from_index"):
                 raise ValueError(
                     "Dyntastic instance was loaded from a KEYS_ONLY or INCLUDE index. "
                     "Call refresh() to load the full item, or pass load_full_item=True to query() or scan()"
                 )
             raise
 
-    class Config:
-        allow_population_by_field_name = True
-
     def __init_subclass__(cls, **kwargs):
-        super().__init_subclass__(**kwargs)
+        # Note: in pydantic v2, our private attributes like __hash_key__ are
+        # not exposed on the model until the class is fully initialized, at
+        # which point __pydantic_init_subclass__ is called.
+        if pydantic_compat.IS_VERSION_1:  # pragma: nocover
+            cls.__pydantic_init_subclass__(**kwargs)
+
+    @classmethod
+    def __pydantic_init_subclass__(cls, **kwargs):
+        if not pydantic_compat.IS_VERSION_1:  # pragma: nocover
+            super().__pydantic_init_subclass__(**kwargs)  # type: ignore[unused-ignore,misc]
 
         cls._clear_boto3_state()
 
@@ -654,8 +685,8 @@ def __init_subclass__(cls, **kwargs):
 
 
 def _has_alias(model: Type[BaseModel], name: str) -> bool:
-    for field in model.__fields__.values():
-        if field.alias == name:
+    for field_name, field in pydantic_compat.model_fields(model).items():
+        if pydantic_compat.alias(field_name, field) == name:
             return True
 
     return False

dyntastic/pydantic_compat.py

@@ -0,0 +1,156 @@
+# pragma: nocover
+from typing import TYPE_CHECKING, Any, Dict, Tuple, Type, TypeVar
+
+import pydantic
+
+_pydantic_major_version = int(pydantic.VERSION.split(".")[0])
+IS_VERSION_1 = _pydantic_major_version == 1
+
+
+BaseModelT = TypeVar("BaseModelT", bound=pydantic.BaseModel)
+
+if TYPE_CHECKING:
+    try:
+        from pydantic.fields import ModelField as FieldInfo  # type: ignore[unused-ignore, attr-defined]
+    except ImportError:
+        from pydantic.fields import FieldInfo  # type: ignore[unused-ignore, assignment]
+
+    def model_fields(model: Type[pydantic.BaseModel]) -> Dict[str, FieldInfo]:
+        ...
+
+    def model_dump(instance: pydantic.BaseModel, **kwargs) -> Dict[str, Any]:
+        ...
+
+    def annotation(field: FieldInfo) -> Any:
+        ...
+
+    def alias(field_name, field: Any) -> str:
+        ...
+
+    def to_jsonable_python(value: Any) -> Any:
+        ...
+
+    def try_model_construct(model: Type[BaseModelT], item: dict) -> Tuple[BaseModelT, bool]:
+        ...
+
+    class BaseModel(pydantic.BaseModel):
+        ...
+
+elif IS_VERSION_1:
+    FieldInfo = pydantic.fields.ModelField
+
+    def model_fields(model: Type[pydantic.BaseModel]) -> Dict[str, pydantic.fields.ModelField]:
+        return model.__fields__
+
+    def model_dump(instance: pydantic.BaseModel, **kwargs) -> Dict[str, Any]:
+        return instance.dict(**kwargs)
+
+    def annotation(field: FieldInfo) -> Any:
+        return field.type_
+
+    def alias(field_name, field: FieldInfo) -> str:
+        return field.alias
+
+    def to_jsonable_python(value: Any) -> Any:
+        import json
+
+        from pydantic.json import pydantic_encoder
+
+        return json.loads(json.dumps(value, default=pydantic_encoder))
+
+    def try_model_construct(model: Type[BaseModelT], item: dict) -> Tuple[BaseModelT, bool]:
+        validated, fields_set, errors = pydantic.validate_model(model, item)
+        if errors:
+            # assume KEYS_ONLY or INCLUDE index
+            fields_in_dynamo = {key: value for key, value in validated.items() if key in fields_set}
+            data = model.construct(**fields_in_dynamo)
+            return data, True
+        else:
+            data = model(**item)
+            return data, False
+
+    class BaseModel(pydantic.BaseModel):
+        class Config:
+            allow_population_by_field_name = True
+
+else:
+    from typing import Union
+
+    try:
+        # Python >= 3.8
+        from typing import get_args, get_origin
+    except ImportError:
+        # Python 3.7
+        def get_args(t):
+            return getattr(t, "__args__", ())
+
+        def get_origin(t):
+            return getattr(t, "__origin__", None)
+
+    FieldInfo = pydantic.fields.FieldInfo
+
+    def model_fields(model: Type[pydantic.BaseModel]) -> Dict[str, pydantic.fields.FieldInfo]:
+        return model.model_fields
+
+    def model_dump(instance: pydantic.BaseModel, **kwargs) -> Dict[str, Any]:
+        return instance.model_dump(**kwargs)
+
+    def annotation(field: FieldInfo) -> Any:
+        # Get rid of Optional[...] type if present (this is only used for
+        # creating tables, where we want to know the actual inner type)
+        type_ = field.annotation
+        if get_origin(type_) is Union:
+            args = get_args(type_)
+            if len(args) == 2 and args[1] is type(None):  # noqa: E721
+                return args[0]
+
+        return type_
+
+    def alias(field_name: str, field: FieldInfo) -> str:
+        return field.alias or field_name
+
+    def to_jsonable_python(value: Any) -> Any:
+        import pydantic_core
+
+        return pydantic_core.to_jsonable_python(value)
+
+    class _FieldCollector(dict):
+        pass
+
+    def try_model_construct(model: Type[BaseModelT], item: dict) -> Tuple[BaseModelT, bool]:
+        # Note: Hopefully there will be a better way to do this in the future
+        # Related issue https://github.com/pydantic/pydantic/issues/7586
+
+        collector = _FieldCollector()
+        try:
+            data = model.model_validate(item, context=collector)
+            return data, False
+        except pydantic.ValidationError:
+            return model.model_construct(**collector), True
+
+    class BaseModel(pydantic.BaseModel):
+        model_config = pydantic.ConfigDict(populate_by_name=True)
+
+        # TODO: eliminate the case where nested Dyntastic models overwrite the
+        # top level _FieldCollector, potential to cause annoying bugs
+        @pydantic.field_validator("*", mode="after")
+        def collect_valid_fields(cls, v, info):
+            if isinstance(info.context, _FieldCollector):
+                info.context[info.field_name] = v
+            return v
+
+
+def field_type(model: Type[pydantic.BaseModel], field: str) -> Type:
+    return annotation(model_fields(model)[field])
+
+
+__all__ = [
+    "BaseModel",
+    "model_fields",
+    "model_dump",
+    "annotation",
+    "alias",
+    "to_jsonable_python",
+    "try_model_construct",
+    "field_type",
+]

pyproject.toml

@@ -12,3 +12,8 @@ warn_unused_ignores = true
 check_untyped_defs = true
 warn_redundant_casts = true
 ignore_missing_imports = true
+
+[tool.coverage.run]
+omit = [
+    "dyntastic/pydantic_compat.py",
+]