How can I turn this sync sqlalchemy query logic to async sqlalchemy.
def update_job_by_id(id:int, job: JobCreate,db: Session,owner_id):
existing_job = db.query(Job).filter(Job.id == id)
if not existing_job.first():
return 0
job.__dict__.update(owner_id=owner_id) #update dictionary with new key value of owner_id
existing_job.update(job.__dict__)
db.commit()
return 1
I tried to convert this logic into async sqlalchemy but no luck.Here is the async version of the above code:
async def update_job_by_id(self, id: int, job: PydanticValidationModel, owner_id):
job_query = await self.db_session.get(db_table, id)
if not job_query:
return 0
updated_job = update(db_table).where(db_table.id == id).values(job.__dict__.update(owner_id = owner_id)).execution_options(synchronize_session="fetch")
await self.db_session.execute(updated_job)
return 1
it produces this error:
AttributeError: 'NoneType' object has no attribute 'items'
Scope:
Making a simple Job Board API using FastAPI and Async SQLAlchemy and I am struggling through update feature of the API in function async def update_job_by_id which I mention above first checking the ID of the job if ID is True then it will export the PydanticModel to dict object to update the PydanticModel with owner_id and finally updating the Model and update the job, but unfortunately when I try to update the PydanticModel update(Job).values(**job_dict.update({"owner_id": owner_id})).where(Job.id == id).execution_options(synchronize_session="fetch") I get None instead of updated dict.
models/jobs.py
from sqlalchemy import Column, Integer, String, Boolean, Date, ForeignKey
from sqlalchemy.orm import relationship
from db.base_class import Base
class Job(Base):
id = Column(Integer, primary_key=True, index=True)
title = Column(String, nullable=False)
company_name = Column(String, nullable=False)
company_url = Column(String)
location = Column(String, nullable=False)
description = Column(String)
date_posted = Column(Date)
is_active = Column(Boolean, default=True)
owner_id = Column(Integer, ForeignKey('user.id'))
owner = relationship("User", back_populates="jobs")
schemas/jobs.py
from typing import Optional
from pydantic import BaseModel
from datetime import date, datetime
class JobBase(BaseModel):
title: Optional[str] = None
company_name: Optional[str] = None
company_url: Optional[str] = None
location: Optional[str] = "remote"
description: Optional[str] = None
date_posted: Optional[date] = datetime.now().date()
class JobCreate(JobBase):
title: str
company_name: str
location: str
description: str
class ShowJob(JobBase):
title: str
company_name: str
company_url: Optional[str]
location: str
date_posted: date
description: str
class Config():
orm_mode = True
routes/route_jobs.py
from typing import List
from fastapi import APIRouter, HTTPException, status
from fastapi import Depends
from db.repository.job_board_dal import job_board
from schemas.jobs import JobCreate, ShowJob
from db.repository.job_board_dal import Job
from depends import get_db
router = APIRouter()
#router.post("/create-job",response_model=ShowJob)
async def create_user(Job: JobCreate, jobs: Job = Depends(get_db)):
owner_id = 1
return await jobs.create_new_job(Job, owner_id)
#router.get("/get/{id}", response_model=ShowJob)
async def retrieve_job_by_id(id:int, id_job: job_board = Depends(get_db)):
job_id = await job_board.retrieve_job(id_job, id=id)
if not job_id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"Job with id {id} does not exist")
return job_id
#router.get("/all", response_model=List[ShowJob])
async def retrieve_all_jobs(all_jobs: job_board = Depends(get_db)):
return await all_jobs.get_all_jobs()
#router.put("/update/{id}")
async def update_job(id: int, job: JobCreate, job_update: job_board = Depends(get_db)):
current_user = 1
response = await job_update.update_job_by_id(id = id, job = job, owner_id = current_user)
if not response:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"Job with id {id} does not exist")
return {"response": "Successfully updated the Job."}
db/repository/job_board_dal.py
from typing import List
from sqlalchemy import update
from sqlalchemy.engine import result
from sqlalchemy.orm import Session
from sqlalchemy.future import select
from schemas.users import UserCreate
from schemas.jobs import JobCreate
from db.models.users import User
from db.models.jobs import Job
from core.hashing import Hasher
class job_board():
def __init__(self, db_session: Session):
self.db_session = db_session
async def register_user(self, user: UserCreate):
new_user = User(username=user.username,
email=user.email,
hashed_password=Hasher.get_password_hash(user.password),
is_active = False,
is_superuser=False
)
self.db_session.add(new_user)
await self.db_session.flush()
return new_user
async def create_new_job(self, job: JobCreate, owner_id: int):
new_job = Job(**job.dict(), owner_id = owner_id)
self.db_session.add(new_job)
await self.db_session.flush()
return new_job
async def retrieve_job(self, id:int):
item = await self.db_session.get(Job, id)
return item
async def get_all_jobs(self) -> List[Job]:
query = await self.db_session.execute(select(Job).order_by(Job.is_active == True))
return query.scalars().all()
async def update_job_by_id(self, id: int, job: JobCreate, owner_id):
_job = await self.db_session.execute(select(Job).where(Job.id==id))
(result, ) = _job.one()
job_dict = job.dict()
print(job_dict.update({"owner_id": owner_id}))
#print(job_update)
if not result:
return 0
job_update = update(Job).values(job_dict.update({"owner_id": owner_id})).where(Job.id == id).execution_options(synchronize_session="fetch")
return await self.db_session.execute(job_update)
If anyone can point out what am I missing here it would be much appreciated.
Sqlalchemy didn't support async operations until version 1.4, see this.
Related
Been trying to get SQLAlchemy setup Async to work with a Postgres Database -- I've been following multiple tutorials but the main one I was using was this one SQLAlchemy Async ORM IS Finally Here
In it he uses a models.py file with Class Methods for create, update, and get, and I really like that so I was trying to follow along, but I'm getting a circular import between my database.py file and my models.py file and I'm unsure how to fix it as the models.py relies on the Async DB sessions + Base from database.py and database.py relies on the class methods to update, get, create data. I'm probably being lazy or silly and missing something but I would appreciate any help
Realistically my end goal is just to get a nicely setup database for creating, updating, getting, that's really all I need and I'm trying to navigate this but not many people seem to know about SQLAlchemy and Async
models.py
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy import Column, Integer, BigInteger, String, Float, DateTime, func, ForeignKey, Boolean
from sqlalchemy import update as sqlalchemy_update
from sqlalchemy.future import select
from helpers.database import async_db, Base
class ModelAdmin:
#classmethod
async def create(cls, **kwargs):
session = await async_db.get_session()
session.add(cls(**kwargs))
await session.commit()
#classmethod
async def update(cls, id, **kwargs):
query = (
sqlalchemy_update(cls)
.where(cls.id == id)
.values(**kwargs)
.execution_options(synchronize_session="fetch")
)
session = await async_db.get_session()
await session.execute(query)
await session.commit()
#classmethod
async def get(cls, id):
query = select(cls).where(cls.id == id)
session = await async_db.get_session()
results = await session.execute(query)
(result,) = results.one()
return result
class Employee(Base, ModelAdmin):
__tablename__ = "employees"
id = Column(BigInteger, primary_key=True)
username = Column(String(100))
displayname = Column(String(100))
email = Column(String(100), default="")
emp_type = Column(String(30), default="")
team = Column(String(100), default="")
redline = Column(Float, default=-1)
tier = Column(Integer, default=-1)
cur_kw = Column(Float, default=0.0)
emp_over = Column(BigInteger, default=-1)
emp_under = Column(String, default="")
deals = relationship("Deal")
__mapper_args__ = {"eager_defaults": True}
def get_tier(self) -> int:
return .2
def set_tier(self, tier: int) -> None:
if self.emp_type == "canvasser":
self.tier = tier
def add_emp_over(self, emp):
'''Adds an employee OVER another, AKA makes them a parent of the employee'''
if emp.id not in self.emp_over.keys():
self.emp_over[emp.id] = emp
return f"Added {emp.username} as parent of {self.username}"
def add_emp_under(self, emp):
'''Adds an employee UNDER another, AKA makes them a child of the employee'''
if emp.id not in self.emp_under.keys():
self.emp_under[emp.id] = emp
class Deal(Base, ModelAdmin):
__tablename__ = "deals"
id = Column(Integer, primary_key=True)
closer_id = Column(ForeignKey("employees.id"))
deal_size = Column(Float)
ppw = Column(Float)
dealer_fee = Column(Float)
lead_owner = relationship("Employee")
installed = Column(Boolean)
create_date = Column(DateTime, server_default=func.now())
def get_commission(self):
'''
Gets the commission from this deal for the closer and the canvasser if the canvasser is set (otherwise it's a self-gen)
'''
commission = 0
if not self.canvasser:
commission = (self.deal_size * 1000) * \
((self.ppw * self.dealer_fee) - self.closer.blueline)
return {commission}
else:
commission = {(self.deal_size * 1000) *
((self.ppw * self.dealer_fee) - self.closer.blueline)}
commission[1] = commission[0] * self.canvasser.getTier()
commission[0] = commission[0] - commission[1]
return commission
database.py
import discord
import asyncpg
import os
import json
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import declarative_base, sessionmaker
from dotenv import load_dotenv
from helpers.configuration import Config
load_dotenv()
db_name = os.environ.get('DB_NAME')
db_user = os.environ.get('DB_USER')
db_pass = os.environ.get('DB_PASS')
db_host = os.environ.get('DB_URL')
Base = declarative_base()
class AsyncDatabaseSession:
'''Handles database connections, updates the employee list, receives member role updates, that good stuff'''
from helpers.models import Employee
def __init__(self):
# here we should connect to and setup the database (if not setup) as well as
# scan and index all members into Employee's then keep their database counterpart updated as well
# any role updates should also ping here, let's get it
self.employee_list = {}
self._session = None
self._engine = None
async def init(self):
self._engine = create_async_engine(
f'postgresql+asyncpg://{db_user}:{db_pass}#{db_host}/{db_name}',
echo=True,
future=True
)
self._session = sessionmaker(
self._engine, expire_on_commit=False, class_=AsyncSession
)
async def create_all(self):
async with self._engine.begin() as conn:
await conn.run_sync(Base.metadata.drop_all)
await conn.run_sync(Base.metadata.create_all)
async def async_session_generator(self):
return sessionmaker(
self._engine, expire_on_commit=False, class_1=AsyncSession
)
async def get_session(self):
try:
async_session = await self.async_session_generator()
async with async_session() as session:
yield session
except:
await session.rollback()
raise
finally:
await session.close()
async def update_employee(self, user: Employee):
"""Updates an employee"""
from helpers.models import Employee
if not len(user.emp_under) > 0:
await Employee.update(
id=user.user_id, username=user.username, displayname=user.displayname, email=user.email,
emp_type=user.emp_type, team=user.team, redline=user.redline, tier=user.tier, cur_kw=user.cur_kw,
emp_over=user.emp_over
)
else:
await Employee.update(
id=user.user_id, username=user.username, displayname=user.displayname, email=user.email,
emp_type=user.emp_type, redline=user.redline, tier=user.tier, cur_kw=user.cur_kw,
emp_over=user.emp_over, emp_under=json.dumps(user.emp_under)
)
async_db = AsyncDatabaseSession()
I'm trying to use SQLAlchemy attribute event "set" in asynchronous mode in the same way, as it was for synchronous.
async_session = sessionmaker(
engine, class_=AsyncSession, expire_on_commit=False
)
Base = declarative_base()
async def init_db():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
class Event(Base):
__tablename__ = "events"
id = Column(Integer, primary_key=True)
name = Column(String)
updated_at = Column(TIMESTAMP, default=datetime.now, onupdate=datetime.now)
parent_id = Column(Integer, ForeignKey('events.id', ondelete='CASCADE'))
children = relationship(
"Event",
join_depth=2,
backref=backref('parents', remote_side=[id])
)
#event.listens_for(Event.name, "set")
#event.listens_for(Event.updated_at, "set")
def set_event(target, value, oldvalue, initiator): # noqa
if target.parents:
target.parents.updated_at = datetime.now()
async def run_main():
await init_db()
async with async_session() as session:
parent = Event(name="Event1")
subparent = Event(name="Event2")
child = Event(name="Event3")
parent.children.append(subparent)
subparent.children.append(child)
session.add(parent)
session.add(subparent)
session.add(child)
await session.commit()
await session.refresh(child)
print(parent.name, parent.updated_at)
print("Sleep for 3sec")
await asyncio.sleep(3)
async with async_session() as session:
result = await session.execute(select(Event).options(selectinload(Event.children)).where(Event.id == 3))
child = result.scalar_one_or_none()
result = await session.execute(select(Event).options(selectinload(Event.children)).where(Event.id == 1))
parent = result.scalar_one_or_none()
setattr(child, "name", "Event3a")
session.add(child)
await session.commit()
await session.refresh(child)
print(parent.name, parent.updated_at)
So, we have three instances, parent, subparent and child, where the parent includes subparent and subparent includes child. The idea is to update parent, when the child's name updated.
But I'm getting an error:
sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: https://sqlalche.me/e/14/xd2s)
sys:1: RuntimeWarning: coroutine 'Connection.cursor' was never awaited
It looks like SQLAlchemy tries to lazy_load child.parents.
How how to use SQLAlchemy attribute events in async mode correctly?
I'm trying to receive image file for my FastAPI file, upload it to a server and then Save url in database.
My create_product and create_category endpoint works as expected without the file: UploadFile = File(...) passed to the router function
I could successfully upload files with other routes and get their filename but not with the above mentioned routes
Routes:
#router.post('/category/create', response_model=schemas.CategoryOut)
def create_category(*, request: schemas.CategoryIn, file: UploadFile = File(...), db: Session = Depends(get_db), current_user: schemas.UserOut = Depends(get_current_active_user)):
category = storeview.get_category_by_slug(db, request.slug)
if category:
raise HTTPException(status.HTTP_400_BAD_REQUEST,
detail=f"Category with slug {request.slug} already exists!")
request.image = storeview.save_file(file)
return storeview.create_category(db, request)
#router.post('/product/create', response_model=schemas.Product)
async def create_product(*, file: UploadFile = File(...), request: schemas.ProductIn, category_id: int, db: Session = Depends(get_db), current_user: schemas.UserOut = Depends(get_current_active_user)):
product = storeview.get_category_by_slug(db, request.slug)
if product:
raise HTTPException(status.HTTP_400_BAD_REQUEST,
detail=f"Product with slug {request.slug} already exists!")
request.image = storeview.save_file(file)
return storeview.create_product(db, request, category_id)
Views:
def create_category(db: Session, request: schemas.CategoryIn):
new_category = models.Category(
name=request.name, slug=request.slug, image=request.image)
db.add(new_category)
db.commit()
db.refresh(new_category)
return new_category
def create_product(db: Session, request: schemas.ProductIn, category_id: int):
new_product = models.Product(**request.dict(), category_id=category_id)
db.add(new_product)
db.commit()
db.refresh(new_product)
return new_product
def save_file(file: UploadFile = File(...)):
result = cloudinary.uploader.upload(file.file)
url = result.get("url")
return url
Schemas:
class CategoryBase(BaseModel):
name: str
slug: str
image: str
class Category(CategoryBase):
id: int
class Config:
orm_mode = True
class CategoryIn(CategoryBase):
pass
class ProductBase(BaseModel):
name: str
description: Optional[str] = None
price: float
image: str
slug: str
class Product(ProductBase):
id: int
category_id: int
category: Category
class Config:
orm_mode = True
class ProductIn(ProductBase):
pass
class CategoryOut(CategoryBase):
products: List[Product] = []
id: int
class Config:
orm_mode = True
I'm trying to build a simple job board using Python, FastAPI and Async sqlalchemy by following the official FastAPI documentation. I'm having some issues regarding retrieving the data from database by ID.
The Following is hopefully a minimum reproducible code segment:
schemas/jobs.py
from typing import Optional
from pydantic import BaseModel
from datetime import date, datetime
class JobBase(BaseModel):
title: Optional[str] = None
company_name: Optional[str] = None
company_url: Optional[str] = None
location: Optional[str] = "remote"
description: Optional[str] = None
date_posted: Optional[date] = datetime.now().date()
class JobCreate(JobBase):
title: str
company_name: str
location: str
description: str
class ShowJob(JobBase):
title: str
company_name: str
company_url: Optional[str]
location: str
date_posted: date
description: str
class Config():
orm_mode = True
routes/route_jobs.py
from fastapi import APIRouter, HTTPException, status
from fastapi import Depends
from db.repository.job_board_dal import job_board
from schemas.jobs import JobCreate, ShowJob
from db.repository.job_board_dal import Job
from depends import get_db
router = APIRouter()
#router.post("/create-job",response_model=ShowJob)
async def create_user(Job: JobCreate, jobs: Job = Depends(get_db)):
owner_id = 1
return await jobs.create_new_job(Job, owner_id)
#router.get("/get/{id}")
def retrieve_job_by_id(id:int, job_board = Depends(get_db)):
#print(type(session))
job_id = job_board.retrieve_job(job_board, id=id)
if not job_id:
HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"Job with id {id} does not exist")
return job_id
db/repository/job_board_dal.py
from sqlalchemy.orm import Session, query
from schemas.users import UserCreate
from schemas.jobs import JobCreate
from db.models.users import User
from db.models.jobs import Job
from core.hashing import Hasher
class job_board():
def __init__(self, db_session: Session):
self.db_session = db_session
async def register_user(self, user: UserCreate):
new_user = User(username=user.username,
email=user.email,
hashed_password=Hasher.get_password_hash(user.password),
is_active = False,
is_superuser=False
)
self.db_session.add(new_user)
await self.db_session.flush()
return new_user
async def create_new_job(self, job: JobCreate, owner_id: int):
new_job = Job(**job.dict(), owner_id = owner_id)
self.db_session.add(new_job)
await self.db_session.flush()
return new_job
def retrieve_job(self, id:int):
item = self.db_session.query(Job).filter(Job.id == id).first()
return item
depends.py
from db.session import async_session
from db.repository.job_board_dal import job_board
async def get_db():
async with async_session() as session:
async with session.begin():
yield job_board(session)
I tried to change the dependency
#router.get("/get/{id}")
def retrieve_job_by_id(id:int, id_job: job_board = Depends(get_db)):
#print(type(session))
job_id = job_board.retrieve_job(id_job, id=id)
if not job_id:
HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"Job with id {id} does not exist")
return job_id
it gives me this error AttributeError: 'AsyncSession' object has no attribute 'query'.
Any help would be much appreciated.
AsyncSession class doesn't declare a query property.
You can use AsyncSession.get the fetch the Job record matching a given id. This method return None if no record is found matching the said id.
For example,
async def retrieve_job(self, id:int):
item = await self.db_session.get(Job, id)
return item
I'm trying to get a todo_ID using GET method. I'm still new at using sqlalchemy and most of the things i have tried are telling me that sqlalchemy doesn't use them. Also, can someone tell me how to use HEAD, i want my methods to return http statuses, i kinda tried using them and imported the render template but when i try to use them it says it has no idea what they are.
this is my attempt at looking at a tutorial and making changes
from flask import Flask, jsonify,json, request, render_template, abort
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config.from_pyfile('Config.py')
db = SQLAlchemy(app)
class JsonModel(object): # Class for making objects JSON serializable
def as_dict(self):
return {c.name: getattr(self, c.name) for c in self.__table__.columns}
class User(db.Model, JsonModel): # Class which is a model for the User table in the database
User_ID = db.Column(db.Integer, primary_key = True)
FirstName = db.Column(db.String(20))
LastName = db.Column(db.String(20))
def __init__(self,User_ID,FirstName, LastName):
self.User_ID = User_ID
self.FirstName = FirstName
self.LastName = LastName
class Todo(db.Model, JsonModel): # Class which is a model for the Todo table in the database
todo_ID = db.Column(db.Integer, primary_key = True)
UserID = db.Column(db.Integer, db.ForeignKey("user.User_ID"))
details = db.Column(db.String(30))
def __init__(self, UserID, details):
self.UserID = UserID
self.details = details
#app.route('/todo', methods = ['GET']) # Uses GET method to return all information in the database.
def index():
return json.dumps([u.as_dict() for u in User.query.all()+Todo.query.all()]), 201
#app.route('/todo/<int:todo_ID>', methods = ['GET'])
def get(todo_ID):
query = Todo.query.get()
return {'todo': [dict(zip(tuple(query.keys()), i)) for i in query.cursor if i[1] == todo_ID]}
#app.before_first_request #Creates everything before the first request.
def startup():
db.create_all()
if __name__ == '__main__':
app.run()
My most recent attempt was:
#app.route('/todo/<int:todo_ID>', methods = ['GET'])
def get(todo_ID):
query = Todo.query("select * from Todo")
return {'todo': [dict(zip(tuple(query.keys()), i)) for i in query.cursor if i[1] == todo_ID]}
And the error that I get is this.
query = Todo.query("select * from Todo")
TypeError: 'BaseQuery' object is not callable
127.0.0.1 - - [30/Nov/2016 21:15:28] "GET /todo/1 HTTP/1.1" 500 -
If you want to query Todo by primary key and only return one record you can use:
from flask import jsonify
#app.route('/todo/<int:todo_ID>', methods = ['GET'])
def get(todo_ID):
response = {}
todo = Todo.query.get(todo_ID)
response['id'] = todo.id
response['user_id'] = todo.UserID
response['details'] = todo.details
response['status_code'] = 201
return jsonify(response)
Or you can use Marshmallow to have a serializer for each of your models so it can serialize them for you automatically.
Not sure I understand your problem, if your intention is to return json output as response and you want to control the status code, you could
use jsonify
from flask import jsonify
#app.route('/todo/<int:todo_ID>', methods = ['GET'])
def get(todo_ID):
query = Todo.query("select * from Todo")
response = {'todo': [dict(zip(tuple(query.keys()), i)) for i in query.cursor if i[1] == todo_ID]}
response = jsonify(response)
response.status_code = 201
return response