I am using Pydantic with FastApi to output ORM data into JSON. I would like to flatten and remap the ORM model to eliminate an unnecessary level in the JSON.
Here's a simplified example to illustrate the problem.
original output: {"id": 1, "billing":
[
{"id": 1, "order_id": 1, "first_name": "foo"},
{"id": 2, "order_id": 1, "first_name": "bar"}
]
}
desired output: {"id": 1, "name": ["foo", "bar"]}
How to map values from nested dict to Pydantic Model? provides a solution that works for dictionaries by using the init function in the Pydantic model class. This example shows how that works with dictionaries:
from pydantic import BaseModel
# The following approach works with a dictionary as the input
order_dict = {"id": 1, "billing": {"first_name": "foo"}}
# desired output: {"id": 1, "name": "foo"}
class Order_Model_For_Dict(BaseModel):
id: int
name: str = None
class Config:
orm_mode = True
def __init__(self, **kwargs):
print(
"kwargs for dictionary:", kwargs
) # kwargs for dictionary: {'id': 1, 'billing': {'first_name': 'foo'}}
kwargs["name"] = kwargs["billing"]["first_name"]
super().__init__(**kwargs)
print(Order_Model_For_Dict.parse_obj(order_dict)) # id=1 name='foo'
(This script is complete, it should run "as is")
However, when working with ORM objects, this approach does not work. It appears that the init function is not called. Here's an example which will not provide the desired output.
from pydantic import BaseModel, root_validator
from typing import List
from sqlalchemy.orm import relationship
from sqlalchemy import Column, Integer, String, ForeignKey
from sqlalchemy.dialects.postgresql import ARRAY
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
from pydantic.utils import GetterDict
class BillingOrm(Base):
__tablename__ = "billing"
id = Column(Integer, primary_key=True, nullable=False)
order_id = Column(ForeignKey("orders.id", ondelete="CASCADE"), nullable=False)
first_name = Column(String(20))
class OrderOrm(Base):
__tablename__ = "orders"
id = Column(Integer, primary_key=True, nullable=False)
billing = relationship("BillingOrm")
class Billing(BaseModel):
id: int
order_id: int
first_name: str
class Config:
orm_mode = True
class Order(BaseModel):
id: int
name: List[str] = None
# billing: List[Billing] # uncomment to verify the relationship is working
class Config:
orm_mode = True
def __init__(self, **kwargs):
# This __init__ function does not run when using from_orm to parse ORM object
print("kwargs for orm:", kwargs)
kwargs["name"] = kwargs["billing"]["first_name"]
super().__init__(**kwargs)
billing_orm_1 = BillingOrm(id=1, order_id=1, first_name="foo")
billing_orm_2 = BillingOrm(id=2, order_id=1, first_name="bar")
order_orm = OrderOrm(id=1)
order_orm.billing.append(billing_orm_1)
order_orm.billing.append(billing_orm_2)
order_model = Order.from_orm(order_orm)
# Output returns 'None' for name instead of ['foo','bar']
print(order_model) # id=1 name=None
(This script is complete, it should run "as is")
The output returns name=None instead of the desired list of names.
In the above example, I am using Order.from_orm to create the Pydantic model. This approach seems to be the same that is used by FastApi when specifying a response model. The desired solution should support use in the FastApi response model as shown in this example:
#router.get("/orders", response_model=List[schemas.Order])
async def list_orders(db: Session = Depends(get_db)):
return get_orders(db)
Update:
Regarding MatsLindh comment to try validators, I replaced the init function with a root validator, however, I'm unable to mutate the return values to include a new attribute. I suspect this issue is because it is a ORM object and not a true dictionary. The following code will extract the names and print them in the desired list. However, I can't see how to include this updated result in the model response:
#root_validator(pre=True)
def flatten(cls, values):
if isinstance(values, GetterDict):
names = [
billing_entry.first_name for billing_entry in values.get("billing")
]
print(names)
# values["name"] = names # error: 'GetterDict' object does not support item assignment
return values
I also found a couple other discussions on this problem that led me to try this approach:
https://github.com/samuelcolvin/pydantic/issues/717
https://gitmemory.com/issue/samuelcolvin/pydantic/821/744047672
What if you override the from_orm class method?
class Order(BaseModel):
id: int
name: List[str] = None
billing: List[Billing]
class Config:
orm_mode = True
#classmethod
def from_orm(cls, obj: Any) -> 'Order':
# `obj` is the orm model instance
if hasattr(obj, 'billing'):
obj.name = obj.billing.first_name
return super().from_orm(obj)
I really missed the handy Django REST Framework serializers while working with the FastAPI + Pydantic stack... So I wrangled with GetterDict to allow defining field getter function in the Pydantic model like this:
class User(FromORM):
fullname: str
class Config(FromORM.Config):
getter_dict = FieldGetter.bind(lambda: User)
#staticmethod
def get_fullname(obj: User) -> str:
return f'{obj.firstname} {obj.lastname}'
where the magic part FieldGetter is implemented as
from typing import Any, Callable, Optional, Type
from types import new_class
from pydantic import BaseModel
from pydantic.utils import GetterDict
class FieldGetter(GetterDict):
model_class_forward_ref: Optional[Callable] = None
model_class: Optional[Type[BaseModel]] = None
def __new__(cls, *args, **kwargs):
inst = super().__new__(cls)
if cls.model_class_forward_ref:
inst.model_class = cls.model_class_forward_ref()
return inst
#classmethod
def bind(cls, model_class_forward_ref: Callable):
sub_class = new_class(f'{cls.__name__}FieldGetter', (cls,))
sub_class.model_class_forward_ref = model_class_forward_ref
return sub_class
def get(self, key: str, default):
if hasattr(self._obj, key):
return super().get(key, default)
getter_fun_name = f'get_{key}'
if not (getter := getattr(self.model_class, getter_fun_name, None)):
raise AttributeError(f'no field getter function found for {key}')
return getter(self._obj)
class FromORM(BaseModel):
class Config:
orm_mode = True
getter_dict = FieldGetter
Related
Pretty new to using Pydantic, but I'm currently passing in the json returned from the API to the Pydantic class and it nicely decodes the json into the classes without me having to do anything.
However, I now want to pass an extra value from a parent class into the child class upon initialization, but I can't figure out how.
In the example below, I want to pass the id of the parent to the child class.
json data example returned from api
{
"id": "162172481",
"filed_at": "2022-11-12",
"child": {
"items": ["item1", "item2", "item3"]
}
}
pydantic class
class ExampleA(BaseModel):
class ChildA(BaseModel):
parent_id: str # how do I pass this in
items: list[str]
id: str
filed_at: date = Field(alias="filedAt")
child: ChildA
class Config:
allow_population_by_field_name = True
initializing the data
data = API.get_example_data()
example_class = ExampleA(**data)
I don't think that there is a way around doing it yourself. But I think the most elegant way is to use a validator.
class ExampleA(BaseModel):
class ChildA(BaseModel):
parent_id: str # how do I pass this in
items: list[str]
id: str
filed_at: date = Field(alias="filedAt")
child: ChildA
#validator('child', pre=True)
def inject_id(cls, v, values):
v['parent_id'] = values['id']
return v
class Config:
allow_population_by_field_name = True
The important part is pre=True so that validator is run before the child gets initialized.
The way PynamoDB is implemented is that it looks to a specific single DynamoDB table:
class UserModel(Model):
class Meta:
# Specific table.
table_name = 'dynamodb-user'
region = 'us-west-1'
The way my infrastructure works is that it has as many dynamodb tables as I have clients, so a single Lambda function has to deal with any amount of separate tables that are identical in structure e.g. represent "UserModel". I can't specify a concrete one.
How would I make this model definition dynamic?
Thanks!
Possible solution:
def create_user_model(table_name: str, region: str):
return type("UserModel", (Model,), {
"key" : UnicodeAttribute(hash_key=True),
"range_key" : UnicodeAttribute(range_key=True),
# Place for other keys
"Meta": type("Meta", (object,), {
"table_name": table_name,
"region": region,
"host": None,
"billing_mode": 'PAY_PER_REQUEST',
})
})
UserModel_dev = create_user_model("user_model_dev", "us-west-1")
UserModel_prod = create_user_model("user_model_prod", "us-west-1")
Update:
A cleaner version:
class UserModel(Model):
key = UnicodeAttribute(hash_key=True)
range_key = UnicodeAttribute(range_key=True)
#staticmethod
def create(table_name: str, region: str):
return type("UserModelDynamic", (UserModel,), {
"Meta": type("Meta", (object,), {
"table_name": table_name,
"region": region,
"host": None,
"billing_mode": 'PAY_PER_REQUEST',
})
})
Open-sourced a solution that is tested and works.
https://github.com/Biomapas/B.DynamoDbCommon/blob/master/b_dynamodb_common/models/model_type_factory.py
Read README.md for more details.
Code:
from typing import TypeVar, Generic, Type
from pynamodb.models import Model
T = TypeVar('T')
class ModelTypeFactory(Generic[T]):
def __init__(self, model_type: Type[T]):
self.__model_type = model_type
# Ensure that given generic belongs to pynamodb.Model class.
if not issubclass(model_type, Model):
raise TypeError('Given model type must inherit from pynamodb.Model class!')
def create(self, custom_table_name: str, custom_region: str) -> Type[T]:
parent_class = self.__model_type
class InnerModel(parent_class):
class Meta:
table_name = custom_table_name
region = custom_region
return InnerModel
How to validate input to get the following Dict passed!
d = dict()
d['en'] = 'English content'
d['it'] = 'Italian content'
d['es'] = 'Spanish content'
print(d)
# {'en': 'English content', 'it': 'Italian content', 'es': 'Spanish content'}
In this example, keys are ISO 639-1 codes using pycountry python package.
code = 'en'
pycountry.languages.get(alpha_2=code.upper()).alpha_2 # = 'en'
The point is how to validate keys using pydantic reusable validator or any other methods?
And validate values either to be str or int?
Pydantic model schema should be similar to this sample :
# products/model.py
from sqlalchemy import Column, String, Integer
from sqlalchemy.ext.declarative import declarative_base
from custom_field import Translatable
Base = declarative_base()
class Model(Base):
__tablename__ = "products"
id = Column(Integer, unique=True, index=True)
name = Column(Translatable())
price = Column(Integer)
# products/pydantic.py
from pydantic import BaseModel
import custom_pydantic_field
class BaseSchema(BaseModel):
id: int
class CreateSchema(BaseSchema):
name: custom_pydantic_field.translatable
price: int
Keep in mind reusability in other models/schemas.
Create pydantic custom class
# validators/translated_field.py
from typing import Dict
from pydantic import ValidationError
from pydantic.error_wrappers import ErrorWrapper
import pycountry
class Translatable(Dict):
"""
Validate Translation Dict Field (Json) where Language is Key and Translation as Value
Languages : ISO 639-1 code
Translation : Int, str, None
ref:
- https://pydantic-docs.helpmanual.io/usage/types/#classes-with-__get_validators__
By: Khalid Murad
"""
#property
def __translation_interface__(self):
return self.dict()
#classmethod
def __get_validators__(cls):
yield cls.validate
#classmethod
def validate(cls, base_dictionary):
result = dict()
dictionary = dict()
errors = []
dictionary = base_dictionary
for key in dictionary:
try:
parsed_language = pycountry.languages.get(alpha_2=key.upper())
except ValueError as exc:
errors.append(ErrorWrapper(Exception(f"Invalid language: {key}."), loc="language"))
if not parsed_language:
errors.append(ErrorWrapper(Exception(f"Invalid language: {key}."), loc="language"))
if isinstance(dictionary[key], int | str | None):
result[key] = dictionary[key]
else:
errors.append(ErrorWrapper(Exception(f"Invalid content for language: {key}."), loc=("language","content")))
if errors:
raise ValidationError(
errors,
cls,
)
return cls(result)
Then use it in you schema/pydantic model like:
# products/pydantic.py
from pydantic import BaseModel
from validators.translated_field import Translatable
class BaseSchema(BaseModel):
id: int
class CreateSchema(BaseSchema):
name: Translatable
...your code
And use normal JSON field in SQLALchemy model!
# products/model.py
...
from sqlalchemy import Column, JSON
...
class Model(Base):
name = Column(JSON, nullable=True)
...
In pydantic is there a cleaner way to exclude multiple fields from the model, something like:
class User(UserBase):
class Config:
exclude = ['user_id', 'some_other_field']
I am aware that following works, but I was looking for something cleaner like django.
class User(UserBase):
class Config:
fields = {'user_id': {'exclude':True},
'some_other_field': {'exclude':True}
}
Pydantic will exclude the class variables which begin with an underscore.
so if it fits your use case, you can rename your attribues.
class User(UserBase):
_user_id=str
some_other_field=str
....
I wrote something like this for my json :
from pydantic import BaseModel
class CustomBase(BaseModel):
def json(self, **kwargs):
include = getattr(self.Config, "include", set())
if len(include) == 0:
include = None
exclude = getattr(self.Config, "exclude", set())
if len(exclude) == 0:
exclude = None
return super().json(include=include, exclude=exclude, **kwargs)
class User(CustomBase):
name :str = ...
family :str = ...
class Config:
exclude = {"family"}
u = User(**{"name":"milad","family":"vayani"})
print(u.json())
you can overriding dict and other method like.
A possible solution is creating a new class based in the baseclass using create_model:
from pydantic import BaseModel, create_model
def exclude_id(baseclass, to_exclude: list):
# Here we just extract the fields and validators from the baseclass
fields = baseclass.__fields__
validators = {'__validators__': baseclass.__validators__}
new_fields = {key: (item.type_, ... if item.required else None)
for key, item in fields.items() if key not in to_exclude}
return create_model(f'{baseclass.__name__}Excluded', **new_fields, __validators__=validators)
class User(BaseModel):
ID: str
some_other: str
list_to_exclude = ['ID']
UserExcluded = exclude_id(User, list_to_exclude)
UserExcluded(some_other='hola')
Which will return:
> UserExcluded(some_other='hola')
Which is a copy of the baseclass but with no parameter 'ID'.
If you have the id in the validators you may want also to exclude those validators.
I'm following this guide from the Fastapi documentation and I have a question what if want to add a custom field when I return an object from DB. In Django I can use serializers.
My case:
I want to save an image name into DB, but before that I need save an actual file in a static folder. When I call GET /items/1 I want to return not just an image name from DB, but full URL, so I need to execute some logic on every request in order to build the URL. The question is how can I achieve that? The only thing I can think of is to add an additional DTO layer that coverts input data to Pydantic classes, so it's gonna be:
DTO class -> Pydantic -> DB
Is there more fancy way of doing that?
Code example:
schemas.py
from typing import List, Literal, Optional
from enum import Enum, IntEnum
from pydantic import BaseModel, constr, validator
class Ingredient(BaseModel):
quantity: int
quantityUnit: QuantityUnitEnum
name: constr(max_length=50)
class RecipeBase(BaseModel):
id: int = None
title: constr(max_length=50)
# image_name: str
#validator('ingredients')
def ingredients_must_have_unique_name(cls, values):
names = []
for item in values:
names.append(item.name)
if len(names) > len(set(names)):
raise ValueError('must contain unique names')
return values
class RecipeCreate(RecipeBase):
pass
class Recipe(RecipeBase):
id: int
class Config:
orm_mode = True
model.py
class Recipe(Base):
__tablename__ = "recipes"
id = Column(Integer, primary_key=True, index=True)
title = Column(String(50), index=True, nullable=False)
image_name = Column(String(50), index=True, nullable=False)
main.py
#app.post("/recipes", response_model=schemas.Recipe)
def create_recipe(recipe: schemas.RecipeCreate, db: Session = Depends(get_db)):
return repository.create_recipe(db=db, recipe=recipe)
#app.get("/recipes/{recipe_id}", response_model=schemas.Recipe)
def get_recipe(recipe_id, db: Session = Depends(get_db)):
return repository.get_recipe(db, recipe_id=recipe_id)
repository.py
def get_recipe(db: Session, recipe_id: int):
return db.query(models.Recipe).get(recipe_id)