Is there a better way to do the following:
if column_sort == 'size':
if sort == 'desc':
results = results.order_by('-size')
else:
results = results.order_by('size')
elif column_sort == 'modified':
if sort == 'desc':
results = results.order_by('-last_modified')
else:
results = results.order_by('last_modified')
else:
if sort == 'desc':
results = results.order_by('-path')
else:
results = results.order_by('path')
results = results[:100]
For example, a way in which to reverse the query order?
How about you just write the code once?
valid_column_sorts = ['size', 'last_modified', 'path']
if column_sort in valid_column_sorts:
if sort == 'desc':
column_sort = '-' + column_sort
results = results.order_by(column_sort)
Oh yeah, I see that there's some difference between sort fields and values, in which case you need a simple map, like results.order_by(col_map[column_sort])
A better way would be to use a dict to map the values of column_sort:
d = {'size': 'size',
'modified': 'last_modified',
#... add more items as needed
}
Now you just need to call get on the dict, with the second argument specifying a default if the value of column_sort isn't in the dictionary:
orderby = d.get(column_sort, "path")
For the sort variable, just add the '-' as neccessary:
orderby = '-' + orderby if sort == "desc" else orderby
And now just execute it:
results = results.order_by(orderby)
This should word:
columns = dict(size='size', modified='last_modified')
column_sort = columns.get(column_sort, 'path')
order = lambda v: '-' if v == 'desc' else ''
results.order_by(order(sort) + column_sort)
It will translate user-specified column name to your schema and assume that the default is 'path'.
Related
I've been asked to do some filters before passing the data to the website.
I have four(4) models that are linked with many2many fields. Let me add an image of the four models.
In order to print the model.a, we need to check if it has model.b linked to it, then check if some model.c is linked to model.b and finally, check if some model.d is linked to model.c. After all of that. The result is the same as this image
To do that, I wrote this code :
#http.route(['/agenda'], auth="public", website=True)
def agenda(self):
months = DATES_SELECT
# all dictionary used in the implementation
model_c_dict = {}
model_b_dict = {}
model_a_dict = {}
model_a_key = []
# filter the model.d according to certain condition
# should I set registrations_left field as store=True for performance when using .search()
model_d_ids = request.env['model.d'].search([('date_start', '>', dt.now().date()), ('state', '=', 'opened')], order="date_start").filtered(lambda k: k.registrations_left != 0)
for session in model_d_ids:
course_id = session.course_id_many[:1]
if not course_id.state == 'validated':
continue
model_c_dict.setdefault(course_id.id, {'object': course_id, 'sessions': []})
model_c_dict[course_id.id]['sessions'].append(session)
for k, v in model_c_dict.items():
category_id = v['object'].category_ids[:1]
if not category_id:
continue
model_b_dict.setdefault(category_id.id, {'object': category_id, 'course': {}})
model_b_dict[category_id.id]['course'].setdefault(k, v)
for k, v in model_b_dict.items():
catalogue_id = v['object'].catalogue_ids[:1]
if not catalogue_id:
continue
model_a_dict.setdefault(catalogue_id.id, {'object': catalogue_id, 'category': {}})
model_a_dict[catalogue_id.id]['category'].setdefault(k, v)
if catalogue_id.id in model_a_dict:
model_a_key.append(catalogue_id)
# sort the model_a with model_a.sequence as key
model_a_key = sorted(list(set(model_a_key)), key=lambda k: k.sequence)
# pack key
dict_key = {'model_a_key': model_a_key}
values = {
'months': months,
'categs': model_a_dict,
'dict_key': dict_key,
}
return request.render('website_custom.agenda', values)
It works as intended, but I don't know if It has performance issues, if it's bad coding, ...
So I'm asking your opinion.
PS: I didn't design the models and its relations.
I loved the slice technique to avoid index out of range error, and can be very usefull to check if the record is connected
all the way up to A (catalogue model) in filtered function k.course_id_many[:1].category_ids[:1].catalogue_ids[:1] but I prefer doing this in the domain:
#http.route(['/agenda'], auth="public", website=True)
def agenda(self):
courses_dict = {}
category_dict = {}
catalogue_dict = {}
# extract all record of Model D connected all the way up to A model
sessions = request.env['model.d'].search([('date_start', '>', dt.now().date()),
('state', '=', 'opened'),
# this will make sure that the record retrieved will be connected to catalogue model (A)
('course_id_many.category_ids.catalogue_ids', '!=', False)], order="date_start") \
.filtered(lambda k: k.registrations_left != 0)
for session in sessions:
# if you want to treat olny the first record you can add the slice on the many2many [:1]
# but I think you will skip the rest of the record in the many2many field
# and if this what you want the loop are not needed at all just do `course = session.course_id_many[0]`
# and do the same for all loops. because you don't have to check if the record are connected we all ready did that in search method
course = session.course_id_many[0]
if not course.state == 'validated': continue # skip validated courses
# add course to dict, and add the session to it's list of sessions
course_obj = courses_dict.setdefault(course.id, {'object': course, 'sessions': []})
course_obj['sessions'].append(session)
category = course.category_ids[0]
# store category, and add course to it's list of courses
category_obj = category_dict.setdefault(category.id, {'object': category, 'course': {}})
category_obj = category_dict[category.id]['course'][course.id] = course_obj
catalogue = category.catalogue_ids[0]
# sotre catalog, and add category to it's categories list
catalogue_dict.setdefault(catalogue.id, {'object': catalogue, 'category': {}})['category'][category.id] = category_obj
# sort catalogue
catalogue_keys = sorted(catalogue_dict.keys(), key=lambda k: catalogue_dict[k]['object'].sequence)
values = {
'months': DATES_SELECT,
'categs': catalogue_dict,
'dict_key': catalogue_keys,
}
return request.render('website_custom.agenda', values)
I hope this work I did the best to check for syntax errors, It should work.
I have the following code in Python that takes in several filters. Then I have a loop that calls .filter on each of them. However when you call multiple filters these are ANDed. How can I change it so that the multiple calls to filter are ORed. The problem is in the "else" part of my code.
for filter_ in filters:
field_models = {'type': ProductType, 'category': ProductCategory}
if filter_['field'] in field_models:
model = field_models[filter_['field']]
organizations_products = organizations_products.join(model).filter(or_(
model.code.ilike('%{}%'.format(escape_like(filter_['value']))),
model.description.ilike('%{}%'.format(escape_like(filter_['value'])))
))
else:
field = getattr(Product, filter_['field'])
organizations_products = organizations_products.filter(
field.ilike('%{}%'.format(escape_like(filter_['value']))))
There are two parts to the solution. First, we must construct the from clause, and then the where clause.
def get_joined_stmt(filters, stmt):
if 'type' in filters.keys():
stmt = stmt.join(ProductType)
if 'category' in filters.keys():
stmt = stmt.join(ProductCategory)
return stmt
def get_exprs(field, value):
def _ilike_expr(x): return '%{}%'.format(escape_like(x))
model_dict = {'type': ProductType, 'category': ProductCategory}
model = model_dict[field]
stmt = organizations_products.join(model)
try:
return [model.code.ilike(_ilike_expr(value)),
model.description.ilike(_ilike_expr(value))]
except KeyError:
return [getattr(Product, field).ilike(_ilike_expr(value))]
organizations_products = get_joined_stmt(filters, organizations_products)
where_exprs = []
for filter_ in filters.items():
where_exprs.extend(get_exprs(**filter_))
organizations_products = organizations_products.filter(or_(*where_exprs))
Just build up a list of filters and or_ them at the end:
exprs = []
for filter_ in filters:
exprs.append(field.ilike(...))
organizations_products = organizations_products.filter(or_(*exprs))
By the way, implementing search like this is a terrible way to go performance-wise (unless you're on PostgreSQL and have a trigram index, in which case ignore this). You're much better served by using the fulltext search features of your DB.
I've got an Opportunity table and an Owner table. The Opportunity table has a many to one relationship with Owner. I'm using Python, SQLAlchemy, FLask-Alchemy and SQLite on this project. I need to do an inner join on the Opportunity table against the Owner table and return a single result set (with fields from both tables).
I need to build a dictionary out of the result set so that I can easily generate Json. The following code all works... but... It took a ton of debugging to figure out how to transform the result set returned (which, in this case is a generator object) into a dictionary.
Did I make this harder than I needed to? Is there a better way to do this using SQLAlchemy? (such as using the expression language instead of the ORM)
owner = db.session.query(Owner).filter(Owner.id == owner_id).first()
if owner is None:
return None
result = {'data': []}
report_date = datetime.datetime.strptime('2016-05-01', '%Y-%m-%d')
rows = db.session.query(Opportunity)\
.filter(Opportunity.status == 'Won',
Opportunity.owner == owner,
or_(Opportunity.actual_close_date_iso_year_week == '2016/16',\
Opportunity.actual_close_date_iso_year_week == '2016/17'),
Opportunity.report_date == report_date)\
.order_by(Opportunity.account)\
.join(Owner, Owner.id == Opportunity.owner_id)\
.values(Opportunity.account,
Opportunity.report_date_iso_year_week,
Opportunity.report_date,
Owner.first_name)
for row in rows:
result_row = {}
fields = getattr(row, '_fields')
for field in fields:
result_row[field] = getattr(row, field)
result['data'].append(result_row)
I think that basically this is the way to get what you need, however I would suggest few minor changes:
You don't really need to join, as after .filter(Opportunity.owner==owner) all the rows that you get from the Opportunity table have the same owner_id.
I think it's better to define the list of needed fields once, instead of trying to get it from each row tuple.
So, the code may be like this:
required_fields = ('account','report_date_iso_year_week','report_date')
owner = db.session.query(Owner).filter(Owner.id == owner_id).first()
if owner is None:
return None
result = {'data': []}
report_date = datetime.datetime.strptime('2016-05-01', '%Y-%m-%d')
rows = db.session.query(Opportunity)\
.filter(Opportunity.status == 'Won',
Opportunity.owner == owner,
or_(Opportunity.actual_close_date_iso_year_week == '2016/16',\
Opportunity.actual_close_date_iso_year_week == '2016/17'),
Opportunity.report_date == report_date)\
.order_by(Opportunity.account)\
.values(*[getattr(Opportunity,f) for f in required_fields])
for row in rows:
result_row = {'first_name':owner.first_name}
for field in required_fields:
result_row[field] = getattr(row, field)
result['data'].append(result_row)
Here is a class that analyses data:
class TopFive:
def __init__(self, catalog_data, sales_data, query, **kwargs):
self.catalog_data = catalog_data
self.sales_data = sales_data
self.query = query
def analyse(self):
CATALOG_DATA = self.catalog_data
SALES_DATA = self.sales_data
query = self.query
products = {}
# Creating a dict with ID, city or hour ( depending on query ) as keys and their income as values.
for row in SALES_DATA:
QUERIES = {
'category': row[0],
'city': row[2],
'hour': row[3]
}
if QUERIES[query] in products:
products[QUERIES[query]] += float(row[4])
products[QUERIES[query]] = round(products[QUERIES[query]], 2)
else:
products[QUERIES[query]] = float(row[4])
if query == 'category':
top_five = {}
top_five_items = sorted(products, key=products.get, reverse=True)[:5] # Getting top 5 categories.
for key in top_five_items:
for row in CATALOG_DATA:
if key == row[0]:
key_string = row[5] + ', ' + row[4]
top_five[key_string] = products[key]
return top_five
else:
return products
It is being called like so:
holder = TopFive(catalog_data=catalog_data, sales_data=sales_data, query='hour')
top_hour = holder.analyse()
What I want to do now is work with the dates. They come in from an input csv file looking like this:
2015-12-11T17:14:05+01:00
Now I need to change to UTC time zone. I thought of using:
.astimezone(pytz.utc)
And now to my question: Can I somehow do so in my QUERIES dictionary, so that when the 'hour' argument is passed to the class I can then execute the program, without changing the following code's structure:
if QUERIES[query] in products:
products[QUERIES[query]] += float(row[4])
products[QUERIES[query]] = round(products[QUERIES[query]], 2)
else:
products[QUERIES[query]] = float(row[4])
and without adding more conditions.
I am thinking of something like:
'hour': row[3].astimezone(pytz.utc)
But this is not working. I can understand why, I am just wondering if there is a similar approach that works. Otherwise I would have to add yet another condition with separate return value and work there.
Got it! The answer to my question is yes: you can use methods in dictionary, just as I tried:
QUERIES = {
'category': row[0],
'city': row[2],
'hour': hour.astimezone(pytz.utc)
}
What I just realized was that I forgot to parse the csv input into datetime format. So obviously when I try to use .astimezone on string it raises error. Sorry for the long useless post, but I'm still very new to OOP and its quite difficult keeping track of all files, instances and so on ;D Thanks
I'm reading data from a SELECT statement of SQLite. Date comes in the following form:
ID|Phone|Email|Status|Role
Multiple rows may be returned for the same ID, Phone, or Email. And for a given row, either Phone or Email can be empty/NULL. However, for the same ID, it's always the same value for Status and the same for Role. for example:
1|1234567892|a#email.com| active |typeA
2|3434567893|b#email.com| active |typeB
2|3434567893|c#email.com| active |typeB
3|5664567891|d#email.com|inactive|typeC
3|7942367891|d#email.com|inactive|typeC
4|5342234233| NULL | active |typeD
5| NULL |e#email.com| active |typeD
These data are returned as a list by Sqlite3, let's call it results. I need to go through them and reorganize the data to construct another list structure in Python. The final list basically consolidates the data for each ID, such that:
Each item of the final list is a dict, one for each unique ID in results. In other words, multiple rows for the same ID will be merged.
Each dict contains these keys: 'id', 'phones', 'emails', 'types', 'role', 'status'.
'phones' and 'emails' are lists, and contains zero or more items, but no duplicates.
'types' is also a list, and contains either 'phone' or 'email' or both, but no duplicates.
The order of dicts in the final list does not matter.
So far I have come up this:
processed = {}
for r in results:
if r['ID'] in processed:
p_data = processed[r['ID']]
if r['Phone']:
p_data['phones'].add(r['Phone'])
p_data['types'].add('phone')
if r['Email']:
p_data['emails'].add(r['Email'])
p_data['types'].add('email')
else:
p_data = {'id': r['ID'], 'status': r['Status'], 'role': r['Role']}
if r['Phone']:
p_data['phones'] = set([r['Phone']])
p_data.setdefault('types', set).add('phone')
if r['Email']:
p_data['emails'] = set([r['Email']])
p_data.setdefault('types', set).add('email')
processed[r['ID']] = p_data
consolidated = list(processed.values())
I wonder if there is a faster and/or more concise way to do this.
EDIT:
A final detail: I would prefer to have 'phones', 'emails', and 'types' in each dict as list instead of set. The reason is that I need to dump consolidated into JSON, and JSON does not allow set.
When faced with something like this I usually use:
processed = collections.defaultdict(lambda:{'phone':set(),'email':set(),'status':None,'type':set()})
and then something like:
for r in results:
for field in ['Phone','Email']:
if r[field]:
processed[r['ID']][field.lower()].add(r[field])
processed[r['ID']]['type'].add(field.lower())
Finally, you can dump it into a dictionary or a list:
a_list = processed.items()
a_dict = dict(a_list)
Regarding the JSON problem with sets, you can either convert the sets to lists right before serializing or write a custom encoder (very useful!). Here is an example of one I have for dates extended to handle sets:
class JSONDateTimeEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
return int(time.mktime(obj.timetuple()))
elif isinstance(ojb, set):
return list(obj)
try:
return json.JSONEncoder.default(self, obj)
except:
return str(obj)
and to use it:
json.dumps(a_list,sort_keys=True, indent=2, cls =JSONDateTimeEncoder)
I assume results is a 2d list:
print results
#[['1', '1234567892', 'a#email.com', ' active ', 'typeA'],
#['2', '3434567893', 'b#email.com', ' active ', 'typeB'],
#['2', '3434567893', 'c#email.com', ' active ', 'typeB'],
#['3', '5664567891', 'd#email.com', 'inactive', 'typeC'],
#['3', '7942367891', 'd#email.com', 'inactive', 'typeC'],
#['4', '5342234233', ' NULL ', ' active ', 'typeD'],
#['5', ' NULL ', 'e#email.com', ' active ', 'typeD']]
Now we group this list by id:
from itertools import groupby
data_grouped = [ (k,list(v)) for k,v in groupby( sorted(results, key=lambda x:x[0]) , lambda x : x[0] )]
# make list of column names (should correspond to results). These will be dict keys
names = [ 'id', 'phone','email', 'status', 'roll' ]
ID_info = { g[0]: {names[i]: list(list( map( set, zip(*g[1] )))[i]) for i in range( len(names))} for g in data_grouped }
Now for the types:
for k in ID_info:
email = [ i for i in ID_info[k]['email'] if i.strip() != 'NULL' and i != '']
phone = [ i for i in ID_info[k]['phone'] if i.strip() != 'NULL' and i != '']
if email and phone:
ID_info[k]['types'] = [ 'phone', 'email' ]
elif email and not phone:
ID_info[k]['types'] = ['email']
elif phone and not email:
ID_info[k]['types'] = ['phone']
else:
ID_info[k]['types'] = []
# project
ID_info[k]['id'] = ID_info[k]['id'][0]
ID_info[k]['roll'] = ID_info[k]['roll'][0]
ID_info[k]['status'] = ID_info[k]['status'][0]
And what you asked for (a list of dicts) is returned by ID_info.values()