Im hitting an issue after configuring sqlalchemy ORM with sqlite in multithreaded environment. Here is what I'm trying to achieve. Im trying to subscribe to mqtt topic using Paho mqtt python library. The mqttsubscribe class is running in separate thread. The MqttSubscribe class callbacks the function I registered while initializing the class. Whenever there is message im trying to write to database using models.py defined. Here is the sample code of models.py.
import os
import datetime
from sqlalchemy.orm import sessionmaker, relationship
from sqlalchemy.pool import StaticPool, NullPool
from sqlalchemy.engine import Engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine, event, Column, Integer, String, DateTime, Boolean, ForeignKey, and_, desc
Base = declarative_base()
Session = sessionmaker()
session = ''
# Sqlite doesnt support foreign key by default
# Below code is logic is taken from
# https://stackoverflow.com/questions/31794195/
# how-to-correctly-add-foreign-key-constraints-to-sqlite-db-using-sqlalchemy
#event.listens_for(Engine, "connect")
def set_sqlite_pragma(connection, connection_record):
cursor = connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
def init_db_connection(engine, echo=True):
global session
engine = create_engine(engine, echo=echo)
Session.configure(bind=engine, connect_args={'check_same_thread':False},
poolclass=NullPool)
session = Session()
Base.metadata.create_all(engine)
return session
class Person(Base):
__tablename__ = 'Person'
id = Column(Integer, primary_key=True)
first_name = Column(String(50))
last_name = Column(String(50))
created_on = Column(DateTime, nullable=False, default=datetime.datetime.utcnow())
updated_on = Column(DateTime, nullable=False, default=datetime.datetime.utcnow(),\
onupdate=datetime.datetime.utcnow())
def __init__(self, first_name, last_name):
self.last_name = last_name
self.first_name = first_name
#classmethod
def create(cls, first_name, last_name):
result = cls(first_name, last_name)
session.add(result)
session.commit()
return result
def __repr__(self):
return "<Person (name=%s last_name=%s)>" % (self.first_name, self.last_name)
Im trying to initialize the db_connnection using this method in another library by importing model.py and init_db_connection
init_db_connection('sqlite:////tmp/foo.db', echo=False)
current_time = datetime.datetime.utcnow()
time.sleep(1)
print('Current_time: {}'.format(current_time))
person = Person.create('Andy','Boss')
print('Person name: {}'.format(person.first_name))
The callback function is able to write to the database from different thread. Im facing problem while reading the database. Im hitting this error
self = <sqlalchemy.orm.session.SessionTransaction object at 0x756550b0>, prepared_ok = False, rollback_ok = False, deactive_ok = False
closed_msg = 'This transaction is closed'
def _assert_active(self, prepared_ok=False,
rollback_ok=False,
deactive_ok=False,
closed_msg="This transaction is closed"):
if self._state is COMMITTED:
raise sa_exc.InvalidRequestError(
"This session is in 'committed' state; no further "
"SQL can be emitted within this transaction."
)
elif self._state is PREPARED:
if not prepared_ok:
raise sa_exc.InvalidRequestError(
> "This session is in 'prepared' state; no further "
"SQL can be emitted within this transaction."
E sqlalchemy.exc.InvalidRequestError: This session is in 'prepared' state; no further SQL can be emitted within this transaction.
sanity/lib/python3.5/site-packages/sqlalchemy/orm/session.py:264: InvalidRequestError
Should I be using scoped_session instead of session to overcome this issue?. I dont whether sqlite is right database for such environment.
Related
I have noticed the listens_for decorator for sqlalchemy starts before the table is fully created, at least in my code. What should I be doing in order to make sure it starts after. Let me know if you need further code.
The error is
Exception has occurred: ProgrammingError
(psycopg2.errors.UndefinedTable) relation "organization" does not exist
My database session is created in database.py
engine = sqlalchemy.create_engine(current_app.config['DATABASE_URL'])
db_session = scoped_session(sessionmaker(
autocommit=False,
autoflush=False,
bind=engine))
Base = declarative_base(bind=engine)
Base.query = db_session.query_property()
def init_db():
from . import models
Base.metadata.drop_all(bind=engine)
Base.metadata.create_all(bind=engine)
#current_app.teardown_appcontext
def shutdown_session(exception=None):
db_session.remove()
My model is created in models.py
class BaseModel(Base):
__abstract__ = True
created_on = Column(DateTime, default=func.now())
updated_on = Column(DateTime, default=func.now(), onupdate=func.now())
class BaseModelId(BaseModel):
__abstract__ = True
id = Column(Integer, primary_key=True)
class Organization(BaseModelId):
__tablename__ = 'organization'
name = Column(String(128))
The issue comes from the decorator listens_for
#event.listens_for(Organization.__table__, 'after_create')
def insert_organizations(*args, **kwargs):
db_session.add(Organization(
name='organization',
))
db_session.commit()
If I put some logging in, the event is called before Base.metadata.create_all(bind=engine) finishes
The error message goes away if I allow a second or two to pass by using Timer
def insert_organizations(*args, **kwargs):
Timer(2, _insert_organizations).start()
def _insert_organizations():
db_session.add(Organization(
name='organization',
))
db_session.commit()
The problem here is that Base.metadata.create_all is using a different connection than db_session, so when commit() is called in the event listener create_all's transaction has yet to commit.
The event listener receives both the table object and the connection being used by create_all, so we can use those objects to insert the record instead of the session.
#sa.event.listens_for(Organization.__table__, 'after_create')
def insert_organizations(tbl, conn, **kw):
conn.execute(tbl.insert().values(name='organisation'))
i was reading and testing the tutorial of sqlalchemy, all was going well until i tried to get values i previusly added to the db. when i do this, i get a error message:
OperationalError: (raised as a result of Query-invoked autoflush; consider using a session.no_autoflush block if this flush is occurring prematurely)
(sqlite3.OperationalError) no such table: userss
if i use close function to close my session, i get this:
OperationalError: (sqlite3.OperationalError) no such table: userss
[SQL: SELECT userss.id AS userss_id, userss.name AS userss_name, userss.fullname AS userss_fullname, userss.nickname AS userss_nickname
this is my code
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Column, Integer, String
engine=create_engine('sqlite:///prueba.db')
Base = declarative_base()
Session=sessionmaker(bind=engine)
#defining a extension of declarative base
class User(Base):
__tablename__ ='userss'
id = Column(Integer, primary_key=True)
name = Column(String)
fullname = Column(String)
nickname=Column(String)
def __repr__(self):
return "<User(name='%s', fullname='%s', nickname='%s')>" % (
self.name, self.fullname, self.nickname)
#makin a session
session = Session()
ed_user = User(name='ed', fullname='Ed Jones', nickname='edsnickname')
session.add(ed_user)
#when line below is uncommented, the error changes
#session.close()
our_user = session.query(User).filter_by(name='ed').first()
i get error when i apply the first() to the session.query(User).filter_by(name='ed'), but when i don't, i get a query.filter object. i read some similar problems but i have not typo errors in the definition of User class. how can i solve this?
thank you so much!
Seems that the table was not created.
Base.metadata.create_all(bind=engine)
With the following change it works:
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Column, Integer, String
engine=create_engine('sqlite:///prueba.db')
Base = declarative_base()
Session=sessionmaker(bind=engine)
#defining a extension of declarative base
class User(Base):
__tablename__ ='userss'
id = Column(Integer, primary_key=True)
name = Column(String)
fullname = Column(String)
nickname=Column(String)
def __repr__(self):
return "<User(name='%s', fullname='%s', nickname='%s')>" % (
self.name, self.fullname, self.nickname)
# initialize tables
Base.metadata.create_all(bind=engine)
#makin a session
session = Session()
ed_user = User(name='ed', fullname='Ed Jones', nickname='edsnickname')
session.add(ed_user)
#when line below is uncommented, the error changes
#session.close()
our_user = session.query(User).filter_by(name='ed').first()
print(our_user)
With Parent and Child tables:
from sqlalchemy import Column, ForeignKey, String, create_engine, desc, asc
from sqlalchemy.ext.declarative import declarative_base
import uuid
Base = declarative_base()
class Parent(Base):
__tablename__ = 'parents'
uuid = Column(String(64), primary_key=True, unique=True)
def __init__(self):
self.uuid = uuid.uuid4()
class Child(Base):
__tablename__ = 'children'
uuid = Column(String(64), primary_key=True, unique=True)
parent_uuid = Column(String(64), ForeignKey('parents.uuid'))
def __init__(self, parent_uuid=None):
self.uuid = uuid.uuid4()
self.parent_uuid = parent_uuid
I can go ahead and create a Parent entity:
engine = create_engine('mysql://root:pass#localhost/dbname', echo=False)
session = scoped_session(sessionmaker())
session.remove()
session.configure(bind=engine, autoflush=False, expire_on_commit=False)
parent = Parent()
session.add(parent)
session.commit()
session.close()
The resulting parent variable is a regular Python ORM object.
If I would query a database instead of creating one the result of query would be a list of ORM objects:
result = session.query(Parent).order_by(desc(Parent.uuid)).all()
But there are times when we need to query database using a raw Sql command.
Is there a way to run a raw SQL command using session object so to ensure that the resulting query return is a ORM object or a list of objects?
You can use the execute() method of Session:
session.execute('select * from table')
The execute method's documentation can be found here:
http://docs.sqlalchemy.org/en/latest/orm/session_api.html#sqlalchemy.orm.session.Session.execute
Please note this does not protect against SQL Injection.
With SQLAlchemey 1.4/2.0, you need to wrap the SQL string in an Executable.
from sqlalchemy import text
session.execute(text("select * from table"))
I took a look at the tutorial here: http://www.pythoncentral.io/introductory-tutorial-python-sqlalchemy/
The decisive parts for my question are first:
import os
import sys
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy import create_engine
Base = declarative_base()
class Person(Base):
__tablename__ = 'person'
# Here we define columns for the table person
# Notice that each column is also a normal Python instance attribute.
id = Column(Integer, primary_key=True)
name = Column(String(250), nullable=False)
class Address(Base):
__tablename__ = 'address'
# Here we define columns for the table address.
# Notice that each column is also a normal Python instance attribute.
id = Column(Integer, primary_key=True)
street_name = Column(String(250))
street_number = Column(String(250))
post_code = Column(String(250), nullable=False)
person_id = Column(Integer, ForeignKey('person.id'))
person = relationship(Person)
# Create an engine that stores data in the local directory's
# sqlalchemy_example.db file.
engine = create_engine('sqlite:///sqlalchemy_example.db')
# Create all tables in the engine. This is equivalent to "Create Table"
# statements in raw SQL.
Base.metadata.create_all(engine)
and second:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy_declarative import Address, Base, Person
engine = create_engine('sqlite:///sqlalchemy_example.db')
# Bind the engine to the metadata of the Base class so that the
# declaratives can be accessed through a DBSession instance
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
# A DBSession() instance establishes all conversations with the database
# and represents a "staging zone" for all the objects loaded into the
# database session object. Any change made against the objects in the
# session won't be persisted into the database until you call
# session.commit(). If you're not happy about the changes, you can
# revert all of them back to the last commit by calling
# session.rollback()
session = DBSession()
# Insert a Person in the person table
new_person = Person(name='new person')
session.add(new_person)
session.commit()
# Insert an Address in the address table
new_address = Address(post_code='00000', person=new_person)
session.add(new_address)
session.commit()
If the second part is run (e.g. "python second_part"), won't the import statement
from sqlalchemy_declarative import Address, Base, Person
cause all executable code in the first part to run? (Which would trigger the code which creates tables in a db each time the first part is run)
Best regards
Yes, every time, the first file is imported, the tables are created again.
To prevent this, use the following statements:
if __name__ == '__main__':
Base.metadata.create_all(engine)
I am trying to follow this tutorial from SQLAlchemy on how to create entries in and query a MYSQL database in python. When I try and query the database for the first time following along in their adding new objects section to test whether an object has been added to the database (see large code block below), I get the following error: AttributeError: 'Connection' object has no attribute 'contextual_connect'
I can query the database. For example, if I change the final line of code to our_user = session.query(User).filter_by(name='ed') it successfully returns a query object, but I cannot figure out how to get the object I entered into the database out of this query result.
Similarly, if I try to loop over the results as they suggest in their querying
section:
for instance in session.query(User).order_by(User.id):
print instance.name, instance.fullname
I get the same error. How can I fix this particular error and are there any other tutorials on using MYSQL in Python with SQLAlchemy that you could point me to?
My code:
import MySQLdb
from sqlalchemy import create_engine
db1 = MySQLdb.connect(host="127.0.0.1",
user="root",
passwd="****",
db="mydata")
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
from sqlalchemy import Column, Integer, String
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String)
fullname = Column(String)
password = Column(String)
def __init__(self, name, fullname, password):
self.name = name
self.fullname = fullname
self.password = password
def __repr__(self):
return "<User('%s','%s', '%s')>" % (self.name, self.fullname, self.password)
ed_user = User('ed', 'Ed Jones', 'edspassword')
from sqlalchemy.orm import sessionmaker
Session = sessionmaker()
Session.configure(bind=db1)
session = Session()
session.add(ed_user)
our_user = session.query(User).filter_by(name='ed').first()
Update/Working Code:
(1) Change to SQLAlchemy engine as discussed by codeape below.
(2) Remember to create the table: Base.metadata.create_all(engine)
(3) Use the "foolproof" version of the User class from SQLAlchemy's tutorial. Note to SQLAlchemy, we (at least I) feel like a fool and would like you to use to always use the foolproof version in the main body of your tutorial and not as an aside that a busy reader might skip over.
All that yields working code:
import MySQLdb
from sqlalchemy import create_engine
engine = create_engine("mysql://user:password#host/database")
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
from sqlalchemy import Column, Integer, String, Sequence
class User(Base):
__tablename__ = 'users'
id = Column(Integer, Sequence('user_id_seq'), primary_key=True)
name = Column(String(50))
fullname = Column(String(50))
password = Column(String(12))
def __init__(self, name, fullname, password):
self.name = name
self.fullname = fullname
self.password = password
def __repr__(self):
return "<User('%s','%s', '%s')>" % (self.name, self.fullname, self.password)
Base.metadata.create_all(engine)
ed_user = User('ed', 'Ed Jones', 'edspassword')
from sqlalchemy.orm import sessionmaker
Session = sessionmaker()
Session.configure(bind=engine)
session = Session()
session.add(ed_user)
our_user = session.query(User).filter_by(name='ed').first()
print(our_user is ed_user)
You must bind the session to a SQLAlchemy engine, not directly to a MySQLDb connection object.
engine = create_engine("mysql://user:password#host/dbname")
Session.configure(bind=engine)
(You can remove your db1 variable.)
From the tutorial:
The return value of create_engine() is an instance of Engine, and it represents the core interface to the database, adapted through a dialect that handles the details of the database and DBAPI in use.
See also https://docs.sqlalchemy.org/en/latest/orm/tutorial.html
from sqlalchemy.orm import sessionmaker
engine = create_engine("mysql://user:password#host/dbname")
Session = sessionmaker(bind=engine)
session = Session()