Im trying to make a multiple database connection. Right now im able to generate a single engine connection for multiple databases. This works by having for my main database (Shared database) defined in mainschema.py (alias dbm):
mymetadata = MetaData()
Base = declarative_base(metadata=mymetadata)
class newbase(DeferredReflection, Base):
__abstract__ = True
table_args = {
'mysql_engine': dict_constants['sql_mainDB_engine'],
'mysql_charset': dict_constants['sql_mainDB_collation'],
'schema' : 'smv',
}
class Company(newbase):
__tablename__ = 'companies'
__table_args__ = table_args
id = Column(INTEGER(unsigned=True), primary_key=True)
company = Column(String(255))
db = Column(String(255))
enabled = Column(BOOLEAN, nullable=False)
Then for each client i will have a specific database defined in dbschema.py (alias dbc)
mymetadata = MetaData()
Base = declarative_base()
table_args = {
'mysql_engine': dict_constants['sql_mainDB_engine']
}
# __abstract__ causes declarative to skip the production of a table or mapper for the class entirely
class newbase(DeferredReflection, Base):
__abstract__ = True
#Creating Base Classes
class ObjectGroup(newbase):
__tablename__ = 'objectgroups'
__table_args__ = table_args
id = Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True)
name = Column(String(30), unique=True, nullable=False)
parentobjectgroup_id = Column(INTEGER(unsigned=True), ForeignKey(id,onupdate="CASCADE", ondelete="RESTRICT"), nullable=False)
created_at = Column(DATETIME, nullable=False)
updated_at = Column(DATETIME, nullable=False)
enabled = Column(BOOLEAN, nullable=False)
After that i create the engine:
self.clientconnection = '%s://%s:%s#%s:%s/%s?charset=%s&use_unicode=0' % (self.dict_constants['sql_clientDB_driver'],
self.dict_constants['sql_clientDB_user'], self.dict_constants['sql_clientDB_passwd'],
host, port, db, self.dict_constants['sql_mainDB_collation'])
self.engine = create_engine(self.clientconnection, echo=False, pool_size=1, pool_timeout=30)
self.base = dbc.newbase
#opening several threads on the same time produces error in prepare, only one at a time
with sema_clientengine:
try:
self.base.prepare(self.engine)
example of a query:
position_table = dbc.Position.__table__
position = alias(position_table, name='position', flat=True)
filter = and_(position.c.object_id == objectid, position.c.validpos == 1)
query = select([position]).where(filter).order_by(position.c.moment_id.desc()).limit(1)
This is how i read:
with sema_reads:
debug_starttime = time()
try:
self.engine.dispose()
with self.engine.connect() as connection:
proxyobject = connection.execute(query)
Okay so basically i have in a same instance one main database called 'smv' and for each client a indepent DB.
So when i create an engine and ran a connect depending on the class it will go to the defined schema. This works great, i can even make joins between databases. Now i need to put the main database 'smv' on a different DB instance so i can scale up and here is when everything stops working because the shared engine only connects to the clients instance and now the smv schema doesnt exist.
Any ideas?
Added on 2019-02-05:
Okay so after some testing i could make it work with sessions, notice that Company table is not just in a different database than Objects, its on a different database instance, different IP and Port:
config_1 = {
"sqlalchemy.url": "%s://%s:%s#%s:%s/%s" % (dict_constants['sql_mainDB_driver'], dict_constants['sql_mainDB_user'],
dict_constants['sql_mainDB_passwd'], dict_constants['sql_mainDB_host'],
dict_constants['sql_mainDB_port'], dict_constants['sql_mainDB_name']),
"sqlalchemy.echo": False,
}
config_2 = {
"sqlalchemy.url": "%s://%s:%s#%s:3307/clientdb" % (dict_constants['sql_mainDB_driver'], dict_constants['sql_mainDB_user'],
dict_constants['sql_mainDB_passwd'], dict_constants['sql_mainDB_host']),
"sqlalchemy.echo": False,
}
engine_1 = engine_from_config(config_1)
engine_2 = engine_from_config(config_2)
Base = declarative_base()
Base2 = declarative_base()
class Company(Base):
__tablename__ = 'companies'
id = Column(INTEGER(unsigned=True), primary_key=True)
company = Column(String(255))
db = Column(String(255))
enabled = Column(BOOLEAN, nullable=False)
class Object(Base2):
__tablename__ = 'objects'
id = Column(INTEGER(unsigned=True), primary_key=True)
name = Column(String(30), unique=True, nullable=False)
objectgroup_id = Column(INTEGER(unsigned=True), nullable=False)
objectaction_id = Column(SMALLINT(unsigned=True), nullable=False)
created_at = Column(DATETIME, nullable=False)
updated_at = Column(DATETIME, nullable=False)
enabled = Column(BOOLEAN, nullable=False)
#session.configure(bind={Company:engine_1})
binds = {Base:engine_1, Base2:engine_2}
session = sessionmaker(binds=binds)
session = session()
print session.query(Company).all()
print session.query(Object).all()
So this works, but i need to ran more complex queries and for that for me ts better this way but it only works when i specify the engine. Any way that session detects it automatically depending on the table class?. Also is it possible to ran a join with tables from different database instance?
company = alias(Company.__table__, name='company', flat=True)
query = select([Company]).order_by(company.c.company)
proxyobject = session.execute(query, bind=engine_1)
rows = proxyobject.fetchall()
print rows
Related
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE+URI'] = \
"mariadb+mariadbconnector://harold:password#localhost:3306/movie?charset=utf8mb4"
app.config['SQLALCHEMY_BINDS'] = \
{'two' : "mariadb+mariadbconnector://harold:password#localhost:3306 /tvshow?charset=utf8mb4"}
db = SQLAlchemy(app)
class Movie(db.Model):
__tablename__ = "movie"
movie_id = Column(VARCHAR(length=25), primary_key=True, nullable=False)
title = Column(VARCHAR(length=255), nullable=False)
series_id = Column(VARCHAR(length=25), nullable=False)
rel_date = Column(VARCHAR(length=25), nullable=False)
class TVShow(db.Model):
__bind_key__ = 'two'
__tablename__ = "tvshow"
tv_id = Column(VARCHAR(length=25), primary_key=True, nullable=False)
name = Column(VARCHAR(length=255), nullable=False)
seasons = Column(INTEGER, nullable=False)
episodes = Column(INTEGER, nullable=False)
def set_df():
main1_df = pd.read_sql_table('movie', engine1)
main2_df = pd.read_sql_table('tvshow', engine2)
So, how do I specify which database for the con/engine? I need to make dataframe from both tables.
I have tried using complete sqlalchemy_database_uri, did not work.
Tried using two for second database, did not work. Do I have to make engines? How?
With MySQL/MariaDB, "database" and "schema" mean the same thing, so if both databases are on the same server we can just use schema= to specify the database name.
import pandas as pd
import sqlalchemy as sa
# make default database some other one (just for illustration)
engine = sa.create_engine("mysql://scott:tiger#localhost:3307/mydb", echo=True)
main1_df = pd.read_sql_table("movie", engine, schema="movie")
"""
SELECT movie.movie.id, … FROM movie.movie
"""
main2_df = pd.read_sql_table("tvshow", engine, schema="tvshow")
"""
SELECT tvshow.tvshow.id, … FROM tvshow.tvshow
"""
I have two tables, Users and ChatSessions. ChatSessions has two fields, user_id and friend_id, both foreign keys to the Users table.
user_id always contains the user that initiated the chat session, friend_id is the other user. As a certain user can have chat sessions initiated by him, or his friends, he can have his id either as user_id or as friend_id, in various sessions.
Is it possible to define a relationship in the Users table, where i have access to all the chat_sessions of that user, no matter whether his id is in user_id or friend_id?
Something like this:
chat_sessions = db.relationship('chat_sessions',
primaryjoin="or_(User.id==ChatSession.user_id, User.id==ChatSession.friend_id)",
backref="user")
I receive the following error when I try to commit an entry to the Users table:
ERROR main.py:76 [10.0.2.2] Unhandled Exception [93e3f515-7dd6-4e8d-b096-8239313433f2]: relationship 'chat_sessions' expects a class or a mapper argument (received: <class 'sqlalchemy.sql.schema.Table'>)
The models:
class User(db.Model):
__tablename__ = 'users'
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(60), index=True, unique=True)
password = db.Column(db.String(255))
name = db.Column(db.String(100))
active = db.Column(db.Boolean(), nullable=False)
chat_sessions = db.relationship('chat_sessions',
primaryjoin="or_(User.id==ChatSession.user_id, User.id==ChatSession.friend_id)")
class ChatSession(db.Model):
__tablename__ = 'chat_sessions'
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
friend_id = db.Column(db.Integer, db.ForeignKey('users.id'))
status = db.Column(db.String(50))
user = db.relationship('User', foreign_keys=[user_id])
friend = db.relationship('User', foreign_keys=[friend_id])
It's difficult to be certain without seeing the tables' code, but it might be sufficient to remove the backref argument.
Here's a pure SQLAlchemy implementation that seems to do what you want:
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import orm
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
id = sa.Column(sa.Integer, primary_key=True)
name = sa.Column(sa.String)
all_chats = orm.relationship('Chat',
primaryjoin="or_(User.id==Chat.user_id, User.id==Chat.friend_id)")
def __repr__(self):
return f'User(name={self.name})'
class Chat(Base):
__tablename__ = 'chats'
id = sa.Column(sa.Integer, primary_key=True)
user_id = sa.Column(sa.Integer, sa.ForeignKey('users.id'))
friend_id = sa.Column(sa.Integer, sa.ForeignKey('users.id'))
user = orm.relationship('User', foreign_keys=[user_id])
friend = orm.relationship('User', foreign_keys=[friend_id])
def __repr__(self):
return f'Chat(user={self.user.name}, friend={self.friend.name})'
engine = sa.create_engine('sqlite:///')
Base.metadata.create_all(bind=engine)
Session = orm.sessionmaker(bind=engine)
usernames = ['Alice', 'Bob', 'Carol']
session = Session()
users = [User(name=name) for name in usernames]
session.add_all(users)
session.flush()
a, b, c = users
session.add(Chat(user_id=a.id, friend_id=b.id))
session.add(Chat(user_id=a.id, friend_id=c.id))
session.add(Chat(user_id=c.id, friend_id=a.id))
session.commit()
session.close()
session = Session()
users = session.query(User)
for user in users:
for chat in user.all_chats:
print(user, chat)
print()
session.close()
This is the output:
User(name=Alice) Chat(user=Alice, friend=Bob)
User(name=Alice) Chat(user=Alice, friend=Carol)
User(name=Alice) Chat(user=Carol, friend=Alice)
User(name=Bob) Chat(user=Alice, friend=Bob)
User(name=Carol) Chat(user=Alice, friend=Carol)
User(name=Carol) Chat(user=Carol, friend=Alice)
How to add objects in the constructor with relationship? The id is not yet ready when constructor is evaluated. In simpler cases it is possible to just provide a list, calculated beforehand. In the example below I tried to say there is a complex_cls_method, in a way it is more like black box.
from sqlalchemy import create_engine, MetaData, Column, Integer, String, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
DB_URL = "mysql://user:password#localhost/exampledb?charset=utf8"
engine = create_engine(DB_URL, encoding='utf-8', convert_unicode=True, pool_recycle=3600, pool_size=10)
session = sessionmaker(autocommit=False, autoflush=False, bind=engine)()
Model = declarative_base()
class User(Model):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
simple = Column(String(255))
main_address = Column(String(255))
addresses = relationship("Address",
cascade="all, delete-orphan")
def __init__(self, addresses, simple):
self.simple = simple
self.main_address = addresses[0]
return # because the following does not work
self.addresses = Address.complex_cls_method(
user_id_=self.id, # <-- this does not work of course
key_="address",
value_=addresses
)
class Address(Model):
__tablename__ = 'address'
id = Column(Integer, primary_key=True)
keyword = Column(String(255))
value = Column(String(255))
user_id = Column(Integer, ForeignKey('user.id'), nullable=False)
parent_id = Column(Integer, ForeignKey('address.id'), nullable=True)
#classmethod
def complex_cls_method(cls, user_id_, key_, value_):
main = Address(keyword=key_, value="", user_id=user_id_, parent_id=None)
session.add_all([main])
session.flush()
addrs = [Address(keyword=key_, value=item, user_id=user_id_, parent_id=main.id) for item in value_]
session.add_all(addrs)
return [main] + addrs
if __name__ == "__main__":
# Model.metadata.create_all(engine)
user = User([u"address1", u"address2"], "simple")
session.add(user)
session.flush()
# as it can't be done in constructor, these additional statements needed
user.addresses = Address.complex_cls_method(
user_id_=user.id,
key_="address",
value_=[u"address1", u"address2"]
)
session.commit()
The question is, is there syntactically elegant (and technically sound) way to do this with User's constructor, or is it safer to just call a separate method of User class after session.flush to add desired objects to relationships (as in the example code)?
Giving up on constructor altogether is still possible, but less desirable option as resulting signature change would require significant refactorings.
Instead of manually flushing and setting ids etc. you could let SQLAlchemy handle persisting your object graph. You'll just need one more adjacency list relationship in Address and you're all set:
class User(Model):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
simple = Column(String(255))
main_address = Column(String(255))
addresses = relationship("Address",
cascade="all, delete-orphan")
def __init__(self, addresses, simple):
self.simple = simple
self.main_address = addresses[0]
self.addresses = Address.complex_cls_method(
key="address",
values=addresses
)
class Address(Model):
__tablename__ = 'address'
id = Column(Integer, primary_key=True)
keyword = Column(String(255))
value = Column(String(255))
user_id = Column(Integer, ForeignKey('user.id'), nullable=False)
parent_id = Column(Integer, ForeignKey('address.id'), nullable=True)
# For handling parent/child relationships in factory method
parent = relationship("Address", remote_side=[id])
#classmethod
def complex_cls_method(cls, key, values):
main = cls(keyword=key, value="")
addrs = [cls(keyword=key, value=item, parent=main) for item in values]
return [main] + addrs
if __name__ == "__main__":
user = User([u"address1", u"address2"], "simple")
session.add(user)
session.commit()
print(user.addresses)
Note the absence of manual flushes etc. SQLAlchemy automatically figures out the required order of insertions based on the object relationships, so that dependencies between rows can be honoured. This is a part of the Unit of Work pattern.
I'm trying to take an existing string in a retrieved SQLAlchemy object and concatenate it with a second string, however no value is being written.
print(messages)
testsuite = session.query(Testsuite).get(test_id)
testsuite.console += messages
session.commit()
Inspecting the database, the record has kept its original empty value - messages was never added.
My Testsuite model is as follows:
# Represents schema - used by engine to create tables etc.
Base = declarative_base()
# These SQL fields are horrendously inefficient - need replacing ASAP!
class Testsuite(Base):
"""Testsuite model to map testsuite in progress to SQL DB."""
__tablename__ = 'testsuites'
id = Column(Integer, primary_key=True)
name = Column(String)
timestamp = Column(DateTime, default=datetime.datetime.utcnow)
product_name = Column(String)
serial_number = Column(String)
total_tests = Column(Integer)
completed_tests = Column(Integer)
console = Column(Text)
report_id = Column(Integer)
testcases = relationship('Testcase', backref='testsuite')
result = Column(String)
def __init__(self, testsuite_name, product_name, serial_number, total_tests=0):
self.name = testsuite_name
self.product_name = product_name
self.serial_number = serial_number
self.total_tests = total_tests
self.completed_tests = 0
self.result = 'pending'
I've read that the way I am modifying my objects can lead to race conditions, though I am unsure of a suitable alternative. Can anyone point out the issues with what I'm doing and why my messages string isn't being added?
Thanks :)
So after a bit of experimentation, it seems that the code was failing because Testsuite.console never had an initial value.
The code now works with the following change to the mode:
class Testsuite(Base):
"""Testsuite model to map testsuite in progress to SQL DB."""
__tablename__ = 'testsuites'
id = Column(Integer, primary_key=True)
name = Column(String)
timestamp = Column(DateTime, default=datetime.datetime.utcnow)
product_name = Column(String)
serial_number = Column(String)
total_tests = Column(Integer)
completed_tests = Column(Integer, default=0)
console = Column(String, default="Waiting for incoming log data...\n")
report_id = Column(Integer)
testcases = relationship('Testcase', backref='testsuite')
result = Column(String, default='pending')
def __init__(self, testsuite_name, product_name, serial_number, total_tests=0):
self.name = testsuite_name
self.product_name = product_name
self.serial_number = serial_number
self.total_tests = total_tests
Hi im having some trouble with foreign key in sqlalchemy not auto incrementing on a primary key ID
Im using: python 2.7, pyramid 1.3 and sqlalchemy 0.7
Here is my models
class Page(Base):
__tablename__ = 'page'
id = Column(Integer, ForeignKey('mapper.object_id'), autoincrement=True, primary_key=True)
title = Column(String(30), unique=True)
title_slug = Column(String(75), unique=True)
text = Column(Text)
date_added = Column(DateTime)
class User(Base):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
name = Column(String(100), unique=True)
email = Column(String(100), unique=True)
password = Column(String(100))
class Group(Base):
__tablename__ = 'groups'
id = Column(Integer, primary_key=True)
name = Column(String(100), unique=True)
class Member(Base):
__tablename__ = 'members'
user_id = Column(Integer, ForeignKey('user.id'), primary_key=True)
group_id = Column(Integer, ForeignKey('groups.id'), primary_key=True)
class Resource(Base):
__tablename__ = 'resource'
id = Column(Integer, primary_key=True)
tablename = Column(Text)
action = Column(Text)
class Mapper(Base):
__tablename__ = 'mapper'
resource_id = Column(Integer, ForeignKey('resource.id'), primary_key=True)
group_id = Column(Integer, ForeignKey('groups.id'), primary_key=True)
object_id = Column(Integer, primary_key=True)
and here is my RAW SQL query which i've written in SQLAlchemys ORM
'''
SELECT g.name, r.action
FROM groups AS g
INNER JOIN resource AS r
ON m.resource_id = r.id
INNER JOIN page AS p
ON p.id = m.object_id
INNER JOIN mapper AS m
ON m.group_id = g.id
WHERE p.id = ? AND
r.tablename = ?;
'''
obj = Page
query = DBSession().query(Group.name, Resource.action)\
.join(Mapper)\
.join(obj)\
.join(Resource)\
.filter(obj.id == obj_id, Resource.tablename == obj.__tablename__).all()
the raw SQL Query works fine without any relations between Page and Mapper, but SQLAlchemys ORM seem to require a ForeignKey link to be able to join them. So i decided to put the ForeignKey at Page.id since Mapper.object_id will link to several different tables.
This makes the SQL ORM query with the joins work as expected but adding new data to the Page table results in a exception.
FlushError: Instance <Page at 0x3377c90> has a NULL identity key.
If this is an auto- generated value, check that the database
table allows generation of new primary key values, and that the mapped
Column object is configured to expect these generated values.
Ensure also that this flush() is not occurring at an inappropriate time,
such as within a load() event.
here is my view code:
try:
session = DBSession()
with transaction.manager:
page = Page(title, text)
session.add(page)
return HTTPFound(location=request.route_url('home'))
except Exception as e:
print e
pass
finally:
session.close()
I really don't know why, but i'd rather have the solution in SQLalchemy than doing the RAW SQL since im making this project for learning purposes :)
I do not think autoincrement=True and ForeignKey(...) play together well.
In any case, for join to work without any ForeignKey, you can just specify the join condition in the second parameter of the join(...):
obj = Page
query = DBSession().query(Group.name, Resource.action)\
.join(Mapper)\
.join(Resource)\
.join(obj, Resource.tablename == obj.__tablename__)\
.filter(obj.id == obj_id)\
.all()