I'm trying to understand if there is a better way to save only part of a list (only the objects I don't have in the database). With my current solution I'm doing it in O(n^2) complexity and I'm also holding a lot of hashes of the database objects in memory.
my class :
class Product(Base):
__tablename__ = 'products'
id = Column('id',BIGINT, primary_key=True)
barcode = Column('barcode' ,BIGINT)
productName = Column('name', TEXT,nullable=False)
objectHash=Column('objectHash',TEXT,unique=True,nullable=False)
def __init__(self, productData,picture=None):
self.barcode = productData[ProductTagsEnum.barcode.value]
self.productName = productData[ProductTagsEnum.productName.value]
self.objectHash = md5((str(self.barcode)+self.produtName).encode('utf-8')).hexdigest()
my solution :
def saveNewProducts(self,products):
Session = sessionmaker()
session=Session()
productsHashes=[ product.objectHash for product in products]
query = session.query(Product.objectHash).filter(Product.objectHash.in_(productsHashes))
existedHashes=query.all()
newProducts = [ product for product in products if product.objectHash not in productsHashes]
session.bulk_save_objects(newProducts)
Related
#hybrid_method
# #paginate
def investors(self, **kwargs):
"""All investors for a given Custodian"""
ind_inv_type_id = InvestorType.where(description="Individual").first().id
inv_query = Investor.with_joined(InvestorAddress, InvestmentAddress, CustodianAddress) \
.filter_by(custodians_id=self.id) \
.with_joined(Investment) \
.filter_by(investor_types_id=ind_inv_type_id)
investors = Investor.where(None, False, inv_query, **kwargs)
temp_inv_query = Investor.with_joined(CustodianInvestor, Custodian)\
.filter_by(Custodian.id==self.id)
temp_investors = Investor.where(None, False, temp_inv_query, **kwargs)
return list(set(investors + temp_investors))
# end def investors
# #auth.access_controlled
class InvestorAddress(db.Model, EntityAddressMixin):
# Metadata
__tablename__ = 'investor_addresses'
# Database Columns
investors_id = db.Column(db.ForeignKey("investors.investors_id"),
nullable=False)
investor = db.relationship("Investor", foreign_keys=[investors_id],
backref=db.backref("InvestorAddress"))
# end class InvestorAddress
class InvestmentAddress(db.Model):
"""This model differs from other EntityAddress Models because it links to either an investor_address or an custodian_address."""
# Metadata
__tablename__ = 'investment_addresses'
# Database Columns
address_types_id = db.Column(
db.ForeignKey("address_types.address_types_id"),
nullable=False)
address_type = db.relationship("AddressType",
foreign_keys=[address_types_id],
backref=db.backref("InvestmentAddress"))
investments_id = db.Column(db.ForeignKey("investments.investments_id"),
nullable=False)
investment = db.relationship("Investment",
foreign_keys=[investments_id],
backref=db.backref("InvestmentAddress"))
investor_addresses_id = db.Column(db.ForeignKey(
"investor_addresses.investor_addresses_id"))
investor_address = db.relationship("InvestorAddress",
foreign_keys=[investor_addresses_id],
backref=db.backref("InvestmentAddress"))
custodian_addresses_id = db.Column(db.ForeignKey(
"custodian_addresses.custodian_addresses_id"))
custodian_address = db.relationship("CustodianAddress",
foreign_keys=[custodian_addresses_id],
backref=db.backref("InvestmentAddress")
)
# end class InvestmentAddress
class CustodianAddress(db.Model, EntityAddressMixin):
"""Defines the relationship between a Custodian and their addresses."""
# Metadata
__tablename__ = 'custodian_addresses'
# Database Columns
custodians_id = db.Column(db.ForeignKey(
"custodians.custodians_id"), nullable=False)
custodian = db.relationship("Custodian", foreign_keys=[custodians_id],
backref=db.backref("CustodianAddress"))
# end CustodianAddress
i have an application and this function is supposed to return a list of 'investors' for a given 'Custodian'. Now when it executes i get an error: "sqlalchemy.exc.ArgumentError: mapper option expects string key or list of attributes". The error comes from the 'join' in the 'inv_query'.
I have included my 3 models that im using for the Join.
As described in the documentation provided by you. here
You should provide string arguments(table names) in with_joined. Given you have defined the relationship
Investor.with_joined('investorAddressTable', 'investmentAddressTable, 'custodianAddressTable')
In case you can use session then you can query the ORM classes directly like
session.query(Investor).join(InvestorAddress).join(InvestmentAddress).join(CustodianAddress).all() # will assume you have set the foreign key properly
:)
I develop a Android Cook-App with a Meal price calculation.
I nearly finished my Api but now I get a TypeError: 'RelationshipProperty' object is not iterable.
I struggle with my sum(mealprice) I have my json but I like to query my mealprice with
#classmethod
def find_by_mealprice(cls, mealprice):
return cls.query.filter_by(mealprice=mealprice).first()
But I can only build my sum in the json method.
class MealModel(db.Model):
__tablename__ = 'meal'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(80))
description = db.Column(db.String)
usp = db.Column(db.String)
workTime = db.Column(db.TIME)
mainIngredients =db.Column(db.String)
img = db.Column(db.String)
difficulty_id = db.Column(db.Integer, db.ForeignKey('difficulty.id'))
difficulty = db.relationship('DifficultyModel')
diet_id = db.Column(db.Integer, db.ForeignKey('diet.id'))
diet = db.relationship('DietModel')
category_id = db.Column(db.Integer, db.ForeignKey('category.id'))
category = db.relationship('CategoryModel')
recipes = db.relationship('RecipeModel', lazy="dynamic")
mealprice = (sum([(recipe.ingredients.price/recipe.ingredients.minamount * recipe.quantity) for recipe in recipes])) <-- TypeError?
def __repr__(self):
return (self.name)
def json(self):
mealprice = (sum([(recipe.ingredients.price/recipe.ingredients.minamount * recipe.quantity) for recipe in self.recipes]))
return { 'id': self.id,
'name': self.name,
'mIng': self.mainIngredients,
'usp': self.usp,
'difficulty': self.difficulty.name,
'workTime': self.workTime.isoformat(),
'diet': self.diet.name,
'description': self.description,
'mealImage': self.img,
'category': self.category.name,
'recipes': [recipe.json() for recipe in self.recipes],
'meal_price': mealprice
}
I hope this Question is not to stupid, I am newish in Flask Api and Python, I startet programming a few months ago with Android Studio.
I hope you can help! :) How can I query with mealprice??
Short answer:
You did not include the column you want to query into your table.
Long answer:
SQLAlchemy helps you convert models that reside in a database to move them into classes in your python code so both can interact pretty magically. You can use python to extend the models generated with SQLAlchemy, just like you did when declaring mealprice and json. You can also overload existing attributes, just like you did with repr.
To be able to query an attribute, you have to make it an attribute in the SQLAlchemy sense of the word, and that is a column. When you do:
category_id = db.Column(db.Integer, db.ForeignKey('category.id'))
You create a category_id attribute in python, based on a db.Column, which is an SQLAlchemy attribute, therefore you can put queries to it.
In other words, if you do:
mealprice = db.Column(db.Numeric, default=calc_mealprice, onupdate=calc_mealprice)
And you define your function however you like, http://docs.sqlalchemy.org/en/latest/core/defaults.html
That would make your mealprice queryable, since it is now a column. However, you will have problems making your function in your current database state, because your mealprice is the sum of colums inside the results of a query.
The error you recieve, TypeError, is because you're mixing queries with numbers, and they are evaluated after the object has been constructed. So recipe.ingredients.price is a query when you define mealprice (during construction), but becomes a list during json.
EDIT:
Personally, I would keep concerns separate and avoid hybrid attributes for now, because they may blur the limits between the database and python for you. If your ingredients may change, I would do something like:
def calc_price_by_minamount(context):
price = context.get_current_parameters()['price']
minamount = context.get_current_parameters()['minamount']
return price / ingredient.minamount
class IngredientModel(db.Model):
...
price_by_minamount = db.Column(db.Numeric,
default=calc_price_by_minamount,
onupdate=calc_price_by_minamount)
def calc_price(context):
ingredients = context.get_current_parameters()['ingredients']
quantity = context.get_current_parameters()['quantity']
return sum([ingredient.price_by_minamount
for ingredient
in ingredients]) * quantity
class RecipeModel(db.Model):
...
price = db.Column(db.Numeric,
default=calc_price,
onupdate=calc_price)
def calc_mealprice(context):
recipes = context.get_current_parameters()['recipes']
return sum([recipe.price for recipe in recipes])
class MealModel(db.Model):
...
mealprice = db.Column(db.Numeric,
default=calc_mealprice,
onupdate=calc_mealprice)
You could start from there to implement hybrid attributes if you want.
I've created models for my database:
class Album(db.Model):
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(128))
year = db.Column(db.String(4))
tracklist = db.relationship('Track', secondary=tracklist,
backref=db.backref('albums',
lazy='dynamic'), lazy='dynamic')
class Track(db.Model):
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(128))
class Artist(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(128))
releases = db.relationship('Track', secondary=releases,
backref=db.backref('artists',
lazy='dynamic'), lazy='dynamic')
They are many-to-many related Album <--> Track <--> Artist
Next, I have this form:
class SearchForm(FlaskForm):
search_by_album = StringField('Album', validators=[Optional()])
search_by_artist = StringField('Artist', validators=[Optional()])
search_track = StringField('Track', validators=[Optional()])
year = StringField('Year', validators=[Optional(), Length(max=4)])
My idea is to give the user freedom in filling desired combination of forms (but at least one is required), so I've got this function, which recieves SearchForm().data (an immutable dict 'field_name': 'data'):
def construct_query(form):
query = db.session.query(*[field.label.text for field in form if field.data and field.name != 'csrf_token'])
if form.search_by_album.data:
query = query.filter(Album.title == form.search_by_album.data)
if form.search_by_artist.data:
query = query.filter(Artist.name == form.search_by_artist.data)
if form.search_track.data:
query = query.filter(Track.title == form.search_track.data)
if form.year.data:
query = query.filter(Album.year == form.year.data)
result = query.all()
return result
My question is if there is a more abstract way of adding filters in the function above? If one day I decide to add more columns to my tables (or even create new tables), I will have to add more monstrous ifs to constrcut_query(), which will eventually grow enormous. Or such an abstractions is not a pythonic way because "Explicit is better than implicit"?
PS
I know about forms from models, but I don't think that they are my case
One way would be associating the filter-attribute with the fields at some place, e.g. as a class attribute on the form itself:
class SearchForm(FlaskForm):
search_by_album = StringField('Album', validators=[Optional()])
search_by_artist = StringField('Artist', validators=[Optional()])
search_track = StringField('Track', validators=[Optional()])
year = StringField('Year', validators=[Optional(), Length(max=4)])
# map form fields to database fields/attributes
field_to_attr = {search_by_album: Album.title,
search_by_artist: Artist.name,
search_track: Track.title,
year: Album.year}
When building the query, you could then build the where clause in a pretty comfortable way:
def construct_query(form):
query = db.session.query(*[field.label.text for field in form if field.data and field.name != 'csrf_token'])
for field in form:
if field.data:
query = query.filter(form.field_to_attr[field] == field.data)
# or:
# for field, attr in form.field_to_attr.items():
# if field.data:
# query = query.filter(attr == field.data)
result = query.all()
return result
Adding new fields and attributes to filter on would then only translate to the creating the field and its mapping to an attribute.
I have an association object in SQLAlchemy that has some extra information (actually a single field) for 2 other objects.
The first object is a Photo model, the second object is a PhotoSet and the association object is called PhotoInSet which holds the position attribute which tells us in what position is the Photo in the current PhotoSet.
class Photo(Base):
__tablename__ = 'photos'
id = Column(Integer, primary_key=True)
filename = Column(String(128), index=True)
title = Column(String(256))
description = Column(Text)
pub_date = Column(SADateTime)
class PhotoInSet(Base):
__tablename__ = 'set_order'
photo_id = Column(Integer, ForeignKey('photos.id'), primary_key=True)
photoset_id = Column(Integer, ForeignKey('photo_set.id'), primary_key=True)
position = Column(Integer)
photo = relationship('Photo', backref='sets')
def __repr__(self):
return '<PhotoInSet %r>' % self.position
class PhotoSet(Base):
__tablename__ = 'photo_set'
id = Column(Integer, primary_key=True)
name = Column(String(256))
description = Column(Text)
timestamp = Column(SADateTime)
user_id = Column(Integer, ForeignKey('users.id'))
user = relationship('User', backref=backref('sets', lazy='dynamic'))
photo_id = Column(Integer, ForeignKey('photos.id'))
photos = relationship('PhotoInSet', backref=backref('set', lazy='select'))
I have no problems creating a new PhotoSet saving the position and creating the relationship, which is (roughly) done like this:
# Create the Set
new_set = PhotoSet(name, user)
# Add the photos with positions applied in the order they came
new_set.photos.extend(
[
PhotoInSet(position=pos, photo=photo)
for pos, photo in
enumerate(photo_selection)
]
)
But I am having a lot of trouble attempting to figure out how to update the position when the order changes.
If I had, say, 3 Photo objects with ids: 1, 2, and 3, and positions 1, 2, and 3 respectively, would look like this after creation:
>>> _set = PhotoSet.get(1)
>>> _set.photos
[<PhotoInSet 1>, <PhotoInSet 2>, <PhotoInSet 3>]
If the order changes, (lets invert the order for this example), is there anyway SQLAlchemy can help me update the position value? So far I am not happy with any of the approaches I can come up with.
What would be the most concise way to do this?
Take a look at the Ordering List extension:
orderinglist is a helper for mutable ordered relationships. It will
intercept list operations performed on a relationship()-managed
collection and automatically synchronize changes in list position onto
a target scalar attribute.
I believe you could change your schema to look like:
from sqlalchemy.ext.orderinglist import ordering_list
# Photo and PhotoInSet stay the same...
class PhotoSet(Base):
__tablename__ = 'photo_set'
id = Column(Integer, primary_key=True)
name = Column(String(256))
description = Column(Text)
photo_id = Column(Integer, ForeignKey('photos.id'))
photos = relationship('PhotoInSet',
order_by="PhotoInSet.position",
collection_class=ordering_list('position'),
backref=backref('set', lazy='select'))
# Sample usage...
session = Session()
# Create two photos, add them to the set...
p_set = PhotoSet(name=u'TestSet')
p = Photo(title=u'Test')
p2 = Photo(title='uTest2')
p_set.photos.append(PhotoInSet(photo=p))
p_set.photos.append(PhotoInSet(photo=p2))
session.add(p_set)
session.commit()
print 'Original list of titles...'
print [x.photo.title for x in p_set.photos]
print ''
# Change the order...
p_set.photos.reverse()
# Any time you change the order of the list in a way that the existing
# items are in a different place, you need to call "reorder". It will not
# automatically try change the position value for you unless you are appending
# an object with a null position value.
p_set.photos.reorder()
session.commit()
p_set = session.query(PhotoSet).first()
print 'List after reordering...'
print [x.photo.title for x in p_set.photos]
The results of this script...
Original list of titles...
[u'Test', u'uTest2']
List after reordering...
[u'uTest2', u'Test']
In your comment, you said...
So this would mean that if I assign a new list to _set.photos I get the positioning for free?
I doubt this is the case.
I'm building a basic CMS in flask for an iPhone oriented site and I'm having a little trouble with something. I have a very small database with just 1 table (pages). Here's the model:
class Page(db.Model):
__tablename__ = 'pages'
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(100), nullable=False)
content = db.Column(db.Text, nullable=False)
parent_id = db.Column(db.Integer, db.ForeignKey("pages.id"), nullable=True)
As you can see, for sub pages, they just reference another page object in the parent_id field. What I'm trying to do in the admin panel is have a nested unordered list with all the pages nested in their parent pages. I have very little idea on how to do this. All i can think of is the following (which will only work (maybe—I haven't tested it) 2 levels down):
pages = Page.query.filter_by(parent_id=None)
for page in pages:
if Page.query.filter_by(parent_id=page.id):
page.sub_pages = Page.query.filter_by(parent_id=page.id)
I would then just format it into a list in the template. How would I make this work with potentially over 10 nested pages?
Thanks heaps in advance!
EDIT: I've looked around a bit and found http://www.sqlalchemy.org/docs/orm/relationships.html#adjacency-list-relationships, so I added
children = db.relationship("Page", backref=db.backref("parent", remote_side=id))
to the bottom of my Page model. and I'm looking at recursively going through everything and adding it to a tree of objects. I've probably made no sense, but that's the best way I can describe it
EDIT 2: I had a go at making a recursive function to run through all the pages and generate a big nested dictionary with all the pages and their children, but it keeps crashing python so i think it's just an infinite loop... here's the function
def get_tree(base_page, dest_dict):
dest_dict = { 'title': base_page.title, 'content': base_page.content }
children = base_page.children
if children:
dest_dict['children'] = {}
for child in children:
get_tree(base_page, dest_dict)
else:
return
and the page i'm testing it with:
#app.route('/test/')
def test():
pages = Page.query.filter_by(parent_id=None)
pages_dict = {}
for page in pages:
get_tree(page, pages_dict)
return str(pages_dict)
anyone got any ideas?
Look at http://sqlamp.angri.ru/index.html
or http://www.sqlalchemy.org/trac/browser/examples/adjacency_list/adjacency_list.py
UPD: For adjacency_list.py declarative example
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base(metadata=metadata)
class TreeNode(Base):
__tablename__ = 'tree'
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey('tree.id'))
name = Column(String(50), nullable=False)
children = relationship('TreeNode',
# cascade deletions
cascade="all",
# many to one + adjacency list - remote_side
# is required to reference the 'remote'
# column in the join condition.
backref=backref("parent", remote_side='TreeNode.id'),
# children will be represented as a dictionary
# on the "name" attribute.
collection_class=attribute_mapped_collection('name'),
)
def __init__(self, name, parent=None):
self.name = name
self.parent = parent
def append(self, nodename):
self.children[nodename] = TreeNode(nodename, parent=self)
def __repr__(self):
return "TreeNode(name=%r, id=%r, parent_id=%r)" % (
self.name,
self.id,
self.parent_id
)
Fix recursion
def get_tree(base_page, dest_dict):
dest_dict = { 'title': base_page.title, 'content': base_page.content }
children = base_page.children
if children:
dest_dict['children'] = {}
for child in children:
get_tree(child, dest_dict)
else:
return
Use query in example for recursive fetch data from db:
# 4 level deep
node = session.query(TreeNode).\
options(joinedload_all("children", "children",
"children", "children")).\
filter(TreeNode.name=="rootnode").\
first()