I'm trying to use SQLAlchemy attribute event "set" in asynchronous mode in the same way, as it was for synchronous.
async_session = sessionmaker(
engine, class_=AsyncSession, expire_on_commit=False
)
Base = declarative_base()
async def init_db():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
class Event(Base):
__tablename__ = "events"
id = Column(Integer, primary_key=True)
name = Column(String)
updated_at = Column(TIMESTAMP, default=datetime.now, onupdate=datetime.now)
parent_id = Column(Integer, ForeignKey('events.id', ondelete='CASCADE'))
children = relationship(
"Event",
join_depth=2,
backref=backref('parents', remote_side=[id])
)
#event.listens_for(Event.name, "set")
#event.listens_for(Event.updated_at, "set")
def set_event(target, value, oldvalue, initiator): # noqa
if target.parents:
target.parents.updated_at = datetime.now()
async def run_main():
await init_db()
async with async_session() as session:
parent = Event(name="Event1")
subparent = Event(name="Event2")
child = Event(name="Event3")
parent.children.append(subparent)
subparent.children.append(child)
session.add(parent)
session.add(subparent)
session.add(child)
await session.commit()
await session.refresh(child)
print(parent.name, parent.updated_at)
print("Sleep for 3sec")
await asyncio.sleep(3)
async with async_session() as session:
result = await session.execute(select(Event).options(selectinload(Event.children)).where(Event.id == 3))
child = result.scalar_one_or_none()
result = await session.execute(select(Event).options(selectinload(Event.children)).where(Event.id == 1))
parent = result.scalar_one_or_none()
setattr(child, "name", "Event3a")
session.add(child)
await session.commit()
await session.refresh(child)
print(parent.name, parent.updated_at)
So, we have three instances, parent, subparent and child, where the parent includes subparent and subparent includes child. The idea is to update parent, when the child's name updated.
But I'm getting an error:
sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: https://sqlalche.me/e/14/xd2s)
sys:1: RuntimeWarning: coroutine 'Connection.cursor' was never awaited
It looks like SQLAlchemy tries to lazy_load child.parents.
How how to use SQLAlchemy attribute events in async mode correctly?
Related
Been trying to get SQLAlchemy setup Async to work with a Postgres Database -- I've been following multiple tutorials but the main one I was using was this one SQLAlchemy Async ORM IS Finally Here
In it he uses a models.py file with Class Methods for create, update, and get, and I really like that so I was trying to follow along, but I'm getting a circular import between my database.py file and my models.py file and I'm unsure how to fix it as the models.py relies on the Async DB sessions + Base from database.py and database.py relies on the class methods to update, get, create data. I'm probably being lazy or silly and missing something but I would appreciate any help
Realistically my end goal is just to get a nicely setup database for creating, updating, getting, that's really all I need and I'm trying to navigate this but not many people seem to know about SQLAlchemy and Async
models.py
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy import Column, Integer, BigInteger, String, Float, DateTime, func, ForeignKey, Boolean
from sqlalchemy import update as sqlalchemy_update
from sqlalchemy.future import select
from helpers.database import async_db, Base
class ModelAdmin:
#classmethod
async def create(cls, **kwargs):
session = await async_db.get_session()
session.add(cls(**kwargs))
await session.commit()
#classmethod
async def update(cls, id, **kwargs):
query = (
sqlalchemy_update(cls)
.where(cls.id == id)
.values(**kwargs)
.execution_options(synchronize_session="fetch")
)
session = await async_db.get_session()
await session.execute(query)
await session.commit()
#classmethod
async def get(cls, id):
query = select(cls).where(cls.id == id)
session = await async_db.get_session()
results = await session.execute(query)
(result,) = results.one()
return result
class Employee(Base, ModelAdmin):
__tablename__ = "employees"
id = Column(BigInteger, primary_key=True)
username = Column(String(100))
displayname = Column(String(100))
email = Column(String(100), default="")
emp_type = Column(String(30), default="")
team = Column(String(100), default="")
redline = Column(Float, default=-1)
tier = Column(Integer, default=-1)
cur_kw = Column(Float, default=0.0)
emp_over = Column(BigInteger, default=-1)
emp_under = Column(String, default="")
deals = relationship("Deal")
__mapper_args__ = {"eager_defaults": True}
def get_tier(self) -> int:
return .2
def set_tier(self, tier: int) -> None:
if self.emp_type == "canvasser":
self.tier = tier
def add_emp_over(self, emp):
'''Adds an employee OVER another, AKA makes them a parent of the employee'''
if emp.id not in self.emp_over.keys():
self.emp_over[emp.id] = emp
return f"Added {emp.username} as parent of {self.username}"
def add_emp_under(self, emp):
'''Adds an employee UNDER another, AKA makes them a child of the employee'''
if emp.id not in self.emp_under.keys():
self.emp_under[emp.id] = emp
class Deal(Base, ModelAdmin):
__tablename__ = "deals"
id = Column(Integer, primary_key=True)
closer_id = Column(ForeignKey("employees.id"))
deal_size = Column(Float)
ppw = Column(Float)
dealer_fee = Column(Float)
lead_owner = relationship("Employee")
installed = Column(Boolean)
create_date = Column(DateTime, server_default=func.now())
def get_commission(self):
'''
Gets the commission from this deal for the closer and the canvasser if the canvasser is set (otherwise it's a self-gen)
'''
commission = 0
if not self.canvasser:
commission = (self.deal_size * 1000) * \
((self.ppw * self.dealer_fee) - self.closer.blueline)
return {commission}
else:
commission = {(self.deal_size * 1000) *
((self.ppw * self.dealer_fee) - self.closer.blueline)}
commission[1] = commission[0] * self.canvasser.getTier()
commission[0] = commission[0] - commission[1]
return commission
database.py
import discord
import asyncpg
import os
import json
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import declarative_base, sessionmaker
from dotenv import load_dotenv
from helpers.configuration import Config
load_dotenv()
db_name = os.environ.get('DB_NAME')
db_user = os.environ.get('DB_USER')
db_pass = os.environ.get('DB_PASS')
db_host = os.environ.get('DB_URL')
Base = declarative_base()
class AsyncDatabaseSession:
'''Handles database connections, updates the employee list, receives member role updates, that good stuff'''
from helpers.models import Employee
def __init__(self):
# here we should connect to and setup the database (if not setup) as well as
# scan and index all members into Employee's then keep their database counterpart updated as well
# any role updates should also ping here, let's get it
self.employee_list = {}
self._session = None
self._engine = None
async def init(self):
self._engine = create_async_engine(
f'postgresql+asyncpg://{db_user}:{db_pass}#{db_host}/{db_name}',
echo=True,
future=True
)
self._session = sessionmaker(
self._engine, expire_on_commit=False, class_=AsyncSession
)
async def create_all(self):
async with self._engine.begin() as conn:
await conn.run_sync(Base.metadata.drop_all)
await conn.run_sync(Base.metadata.create_all)
async def async_session_generator(self):
return sessionmaker(
self._engine, expire_on_commit=False, class_1=AsyncSession
)
async def get_session(self):
try:
async_session = await self.async_session_generator()
async with async_session() as session:
yield session
except:
await session.rollback()
raise
finally:
await session.close()
async def update_employee(self, user: Employee):
"""Updates an employee"""
from helpers.models import Employee
if not len(user.emp_under) > 0:
await Employee.update(
id=user.user_id, username=user.username, displayname=user.displayname, email=user.email,
emp_type=user.emp_type, team=user.team, redline=user.redline, tier=user.tier, cur_kw=user.cur_kw,
emp_over=user.emp_over
)
else:
await Employee.update(
id=user.user_id, username=user.username, displayname=user.displayname, email=user.email,
emp_type=user.emp_type, redline=user.redline, tier=user.tier, cur_kw=user.cur_kw,
emp_over=user.emp_over, emp_under=json.dumps(user.emp_under)
)
async_db = AsyncDatabaseSession()
I'm trying to show the current users that are connected to same websocket (in my case chat room) but I have a little problem. I want to create variable that stores the users and send it trough websockets to my frontend, I've achieved that, but here is the problem.
Consumers.py - approach 1
class ChatRoomConsumer(AsyncWebsocketConsumer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.users: list = []
#database_sync_to_async
def create_msg(self, user_id=None, message=None, room=None):
if user_id is not None:
sender = User.objects.get(id=user_id)
msg = Message.objects.create(
author=sender, message=message, room_name=room)
msg.save()
return msg
else:
get_msgs = Message.objects.filter(room_name__in=[room])
serializer = MessageSerializer(get_msgs, many=True)
return serializer.data
async def connect(self):
self.room_name = self.scope['url_route']['kwargs']['room_name']
self.room_group_name = f'chat_{self.room_name}'
self.messages = await self.create_msg(room=self.room_name)
await self.channel_layer.group_add(
self.room_group_name,
self.channel_name
)
await self.accept()
await self.send(text_data=json.dumps({
'db_messages': self.messages,
}))
async def disconnect(self, close_code):
print(close_code)
await self.channel_layer.group_discard(
self.room_group_name,
self.channel_name
)
async def receive(self, text_data):
text_data_json = json.loads(text_data)
type = text_data_json['type']
message = text_data_json['message']
username = text_data_json['username']
user_id = text_data_json['user_id']
self.user_id = text_data_json['user_id']
if type == 'chatroom_message':
self.msg = await self.create_msg(user_id, message, self.room_name)
await self.channel_layer.group_send(
self.room_group_name, {
'type': type,
'message': message,
'username': username,
'user_id': user_id
}
)
async def chatroom_message(self, event):
message = event['message']
username = event['username']
await self.send(text_data=json.dumps({
'message': message,
'username': username,
}))
# nefunkcni ukazatel momentalnich uzivatelu v mistnosti
async def get_user(self, event):
print('get_user called')
if event['message'] == 'disconnect':
print('remove', event['username'])
try:
self.users.remove(event['username'])
except ValueError:
print('user already removed')
else:
if event['username'] not in self.users:
self.users.append(event['username'])
print(self.users)
await self.send(text_data=json.dumps({
'users': self.users
}))
In this approach, it corretly shows the current logged in users in the view only from the 1st user that has entered the chat, other users dont see, that the user that has joined before them is there.
In this approach I define the variable users in the constructor.
Consumers.py - approach 2
from channels.generic.websocket import AsyncWebsocketConsumer
from channels.db import database_sync_to_async
from .models import Message
from .serializers import MessageSerializer
from django.contrib.auth.models import User
from django.db.models import Prefetch
from django.core.serializers.json import DjangoJSONEncoder
from asgiref.sync import sync_to_async, async_to_sync
import channels
import json
users: list = []
class ChatRoomConsumer(AsyncWebsocketConsumer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
#self.users: list = []
#database_sync_to_async
def create_msg(self, user_id=None, message=None, room=None):
if user_id is not None:
sender = User.objects.get(id=user_id)
msg = Message.objects.create(
author=sender, message=message, room_name=room)
msg.save()
return msg
else:
get_msgs = Message.objects.filter(room_name__in=[room])
serializer = MessageSerializer(get_msgs, many=True)
return serializer.data
async def connect(self):
self.room_name = self.scope['url_route']['kwargs']['room_name']
self.room_group_name = f'chat_{self.room_name}'
self.messages = await self.create_msg(room=self.room_name)
await self.channel_layer.group_add(
self.room_group_name,
self.channel_name
)
await self.accept()
await self.send(text_data=json.dumps({
'db_messages': self.messages,
}))
async def disconnect(self, close_code):
print(close_code)
await self.channel_layer.group_discard(
self.room_group_name,
self.channel_name
)
async def receive(self, text_data):
text_data_json = json.loads(text_data)
type = text_data_json['type']
message = text_data_json['message']
username = text_data_json['username']
user_id = text_data_json['user_id']
self.user_id = text_data_json['user_id']
if type == 'chatroom_message':
self.msg = await self.create_msg(user_id, message, self.room_name)
await self.channel_layer.group_send(
self.room_group_name, {
'type': type,
'message': message,
'username': username,
'user_id': user_id
}
)
async def chatroom_message(self, event):
message = event['message']
username = event['username']
await self.send(text_data=json.dumps({
'message': message,
'username': username,
}))
# nefunkcni ukazatel momentalnich uzivatelu v mistnosti
async def get_user(self, event):
print('get_user called')
if event['message'] == 'disconnect':
print('remove', event['username'])
try:
users.remove(event['username'])
except ValueError:
print('user already removed')
else:
if event['username'] not in users:
users.append(event['username'])
print(users)
await self.send(text_data=json.dumps({
'users': users
}))
In this approach, the users show correctly, but not from the current room (it shows users from all the other rooms). It's kinda logical because the declaration of the variable is at the top level.
But my question is where should I declare it then? When it's in the constructor, it always overwrites the users that were in the room before the current user and when it's in the top level, it takes all the users from all the rooms.
In case you haven't found an answer yet, here's my 2 cents:
Consumers are singletons, meaning that there is one instance for every channel (websocket connection). So probably that is the problem of having users inside the consumer class.
An alternative would be to make user a dictionary, not an array. So you can have a key for every room, like so:
users = { "room_1": [], "room_2": [] }
I know there is no code in my answer, but I hope it serves as a guide for your solution!
How can I turn this sync sqlalchemy query logic to async sqlalchemy.
def update_job_by_id(id:int, job: JobCreate,db: Session,owner_id):
existing_job = db.query(Job).filter(Job.id == id)
if not existing_job.first():
return 0
job.__dict__.update(owner_id=owner_id) #update dictionary with new key value of owner_id
existing_job.update(job.__dict__)
db.commit()
return 1
I tried to convert this logic into async sqlalchemy but no luck.Here is the async version of the above code:
async def update_job_by_id(self, id: int, job: PydanticValidationModel, owner_id):
job_query = await self.db_session.get(db_table, id)
if not job_query:
return 0
updated_job = update(db_table).where(db_table.id == id).values(job.__dict__.update(owner_id = owner_id)).execution_options(synchronize_session="fetch")
await self.db_session.execute(updated_job)
return 1
it produces this error:
AttributeError: 'NoneType' object has no attribute 'items'
Scope:
Making a simple Job Board API using FastAPI and Async SQLAlchemy and I am struggling through update feature of the API in function async def update_job_by_id which I mention above first checking the ID of the job if ID is True then it will export the PydanticModel to dict object to update the PydanticModel with owner_id and finally updating the Model and update the job, but unfortunately when I try to update the PydanticModel update(Job).values(**job_dict.update({"owner_id": owner_id})).where(Job.id == id).execution_options(synchronize_session="fetch") I get None instead of updated dict.
models/jobs.py
from sqlalchemy import Column, Integer, String, Boolean, Date, ForeignKey
from sqlalchemy.orm import relationship
from db.base_class import Base
class Job(Base):
id = Column(Integer, primary_key=True, index=True)
title = Column(String, nullable=False)
company_name = Column(String, nullable=False)
company_url = Column(String)
location = Column(String, nullable=False)
description = Column(String)
date_posted = Column(Date)
is_active = Column(Boolean, default=True)
owner_id = Column(Integer, ForeignKey('user.id'))
owner = relationship("User", back_populates="jobs")
schemas/jobs.py
from typing import Optional
from pydantic import BaseModel
from datetime import date, datetime
class JobBase(BaseModel):
title: Optional[str] = None
company_name: Optional[str] = None
company_url: Optional[str] = None
location: Optional[str] = "remote"
description: Optional[str] = None
date_posted: Optional[date] = datetime.now().date()
class JobCreate(JobBase):
title: str
company_name: str
location: str
description: str
class ShowJob(JobBase):
title: str
company_name: str
company_url: Optional[str]
location: str
date_posted: date
description: str
class Config():
orm_mode = True
routes/route_jobs.py
from typing import List
from fastapi import APIRouter, HTTPException, status
from fastapi import Depends
from db.repository.job_board_dal import job_board
from schemas.jobs import JobCreate, ShowJob
from db.repository.job_board_dal import Job
from depends import get_db
router = APIRouter()
#router.post("/create-job",response_model=ShowJob)
async def create_user(Job: JobCreate, jobs: Job = Depends(get_db)):
owner_id = 1
return await jobs.create_new_job(Job, owner_id)
#router.get("/get/{id}", response_model=ShowJob)
async def retrieve_job_by_id(id:int, id_job: job_board = Depends(get_db)):
job_id = await job_board.retrieve_job(id_job, id=id)
if not job_id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"Job with id {id} does not exist")
return job_id
#router.get("/all", response_model=List[ShowJob])
async def retrieve_all_jobs(all_jobs: job_board = Depends(get_db)):
return await all_jobs.get_all_jobs()
#router.put("/update/{id}")
async def update_job(id: int, job: JobCreate, job_update: job_board = Depends(get_db)):
current_user = 1
response = await job_update.update_job_by_id(id = id, job = job, owner_id = current_user)
if not response:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"Job with id {id} does not exist")
return {"response": "Successfully updated the Job."}
db/repository/job_board_dal.py
from typing import List
from sqlalchemy import update
from sqlalchemy.engine import result
from sqlalchemy.orm import Session
from sqlalchemy.future import select
from schemas.users import UserCreate
from schemas.jobs import JobCreate
from db.models.users import User
from db.models.jobs import Job
from core.hashing import Hasher
class job_board():
def __init__(self, db_session: Session):
self.db_session = db_session
async def register_user(self, user: UserCreate):
new_user = User(username=user.username,
email=user.email,
hashed_password=Hasher.get_password_hash(user.password),
is_active = False,
is_superuser=False
)
self.db_session.add(new_user)
await self.db_session.flush()
return new_user
async def create_new_job(self, job: JobCreate, owner_id: int):
new_job = Job(**job.dict(), owner_id = owner_id)
self.db_session.add(new_job)
await self.db_session.flush()
return new_job
async def retrieve_job(self, id:int):
item = await self.db_session.get(Job, id)
return item
async def get_all_jobs(self) -> List[Job]:
query = await self.db_session.execute(select(Job).order_by(Job.is_active == True))
return query.scalars().all()
async def update_job_by_id(self, id: int, job: JobCreate, owner_id):
_job = await self.db_session.execute(select(Job).where(Job.id==id))
(result, ) = _job.one()
job_dict = job.dict()
print(job_dict.update({"owner_id": owner_id}))
#print(job_update)
if not result:
return 0
job_update = update(Job).values(job_dict.update({"owner_id": owner_id})).where(Job.id == id).execution_options(synchronize_session="fetch")
return await self.db_session.execute(job_update)
If anyone can point out what am I missing here it would be much appreciated.
Sqlalchemy didn't support async operations until version 1.4, see this.
I'm trying to build a simple job board using Python, FastAPI and Async sqlalchemy by following the official FastAPI documentation. I'm having some issues regarding retrieving the data from database by ID.
The Following is hopefully a minimum reproducible code segment:
schemas/jobs.py
from typing import Optional
from pydantic import BaseModel
from datetime import date, datetime
class JobBase(BaseModel):
title: Optional[str] = None
company_name: Optional[str] = None
company_url: Optional[str] = None
location: Optional[str] = "remote"
description: Optional[str] = None
date_posted: Optional[date] = datetime.now().date()
class JobCreate(JobBase):
title: str
company_name: str
location: str
description: str
class ShowJob(JobBase):
title: str
company_name: str
company_url: Optional[str]
location: str
date_posted: date
description: str
class Config():
orm_mode = True
routes/route_jobs.py
from fastapi import APIRouter, HTTPException, status
from fastapi import Depends
from db.repository.job_board_dal import job_board
from schemas.jobs import JobCreate, ShowJob
from db.repository.job_board_dal import Job
from depends import get_db
router = APIRouter()
#router.post("/create-job",response_model=ShowJob)
async def create_user(Job: JobCreate, jobs: Job = Depends(get_db)):
owner_id = 1
return await jobs.create_new_job(Job, owner_id)
#router.get("/get/{id}")
def retrieve_job_by_id(id:int, job_board = Depends(get_db)):
#print(type(session))
job_id = job_board.retrieve_job(job_board, id=id)
if not job_id:
HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"Job with id {id} does not exist")
return job_id
db/repository/job_board_dal.py
from sqlalchemy.orm import Session, query
from schemas.users import UserCreate
from schemas.jobs import JobCreate
from db.models.users import User
from db.models.jobs import Job
from core.hashing import Hasher
class job_board():
def __init__(self, db_session: Session):
self.db_session = db_session
async def register_user(self, user: UserCreate):
new_user = User(username=user.username,
email=user.email,
hashed_password=Hasher.get_password_hash(user.password),
is_active = False,
is_superuser=False
)
self.db_session.add(new_user)
await self.db_session.flush()
return new_user
async def create_new_job(self, job: JobCreate, owner_id: int):
new_job = Job(**job.dict(), owner_id = owner_id)
self.db_session.add(new_job)
await self.db_session.flush()
return new_job
def retrieve_job(self, id:int):
item = self.db_session.query(Job).filter(Job.id == id).first()
return item
depends.py
from db.session import async_session
from db.repository.job_board_dal import job_board
async def get_db():
async with async_session() as session:
async with session.begin():
yield job_board(session)
I tried to change the dependency
#router.get("/get/{id}")
def retrieve_job_by_id(id:int, id_job: job_board = Depends(get_db)):
#print(type(session))
job_id = job_board.retrieve_job(id_job, id=id)
if not job_id:
HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"Job with id {id} does not exist")
return job_id
it gives me this error AttributeError: 'AsyncSession' object has no attribute 'query'.
Any help would be much appreciated.
AsyncSession class doesn't declare a query property.
You can use AsyncSession.get the fetch the Job record matching a given id. This method return None if no record is found matching the said id.
For example,
async def retrieve_job(self, id:int):
item = await self.db_session.get(Job, id)
return item
I have a time-consuming operation to modify the database which is triggered by request the route，So I want to perform the operation asynchronously without waiting for it to end，and return the state directly. I tried to use threading module in the route,but got this error:
The following is the code：
define a model:
class Plan(db.Model):
__tablename__ = 'opt_version_plans'
planid = db.Column(db.Integer,primary_key=True)
planname = db.Column(db.String(255))
jobname = db.Column(db.String(255))
branch = db.Column(db.String(32))
define function:
def test():
time.sleep(20)
p = Plan.query.get(1)
print p.planname
route:
#app.route('/')
def index():
t = threading.Thread(target=test)
t.start()
return "helloworld"
How can I achieve my needs in this way？