Page loading is very slow - how to properly optimize? - python

I'm working on a big page with different product statistics. I use there multiple charts, tables etc. It takes for example 5 seconds to load the page.
This is a very simplified models.py
EDIT
For example, OccurencesTable contains this row:
last_scan_time = tables.columns.TemplateColumn("""{{ record.get_last_scan.datetime }}""",accessor='get_last_scan.datetime', verbose_name='Last scan time')
So for each row, there must be executed query to the database which calls last scan. Better approach would be to preload all Scan objects.
PRODUCT
class Product(models.Model):
user = models.ForeignKey(User, null=False, blank=False, related_name='products')
name = models.CharField(max_length=200)
def get_occurences(self):
return self.occurences.prefetch_related("scans__price__currency")
def get_occurences_count(self):
return self.occurences.all().count()
def get_all_scans(self):
return [item for sublist in [x.scans.all() for x in self.get_occurences()] for item in sublist]
def get_all_valid_scans(self):
return [item for sublist in [x.get_valid_scans() for x in self.get_occurences()] for item in sublist]
def get_last_scans(self):
scans = []
for occ in self.get_occurences():
scan = occ.get_last_scan()
if scan:
scans.append(scan)
return scans
# #property
#def last_scan_time(self):
#scans = sorted([x for x in self.get_last_scans() if x.datetime],key = lambda x: x.datetime, reverse=True)
#Scan.objects.filter(occurence__product=self,price__amount__isnull=False)
#return str(scans[0].datetime)
def get_last_min_scan(self):
sorted_last_scans = [x for x in self.get_last_scans() if x.valid]
sorted_last_scans.sort(key=lambda x: x.price.eur_price)
return sorted_last_scans[0] if sorted_last_scans else None
def get_last_min_price(self):
last_scan = self.get_last_min_scan()
if last_scan:
return last_scan.price.eur_price
return None
def get_active_occurences(self):
return self.get_occurences().filter(active=True)
OCCURENCE
class Occurence(models.Model):
product = models.ForeignKey(Product, related_name='occurences', on_delete=models.CASCADE)
_last_scan = models.OneToOneField('Scan',null=True,blank=True,related_name='+')
currency = models.ForeignKey('Currency',related_name='occurences')
def get_last_scan(self):
try:
last = self.scans.select_related("price__amount").order_by('datetime').last()
except:
last = None
return last
def get_last_valid_scan(self):
try:
last = self.scans.exclude(price__isnull=True).order_by('-datetime').first()
except:
last = None
return last
def get_second_last_valid_scan(self):
scans = self.scans.exclude(price__isnull=True).order_by('-datetime').select_related("price")
if scans.count()>=2:
return scans[1]
return None
def get_valid_scans(self):
return self.scans.all().exclude(price__isnull=True)
def get_min_scan(self):
scan = self.get_valid_scans().order_by('price__amount').first()
if scan:
return scan
return None
""" STATS METHODS """
def stats_get_difference_for_two_last_scans(self):
second_last_valid_scan = self.get_second_last_valid_scan()
if second_last_valid_scan:
difference_in_percent = math_ops.round_eur(decimal.Decimal(-1 * (100 - self.get_last_valid_scan().price.eur_price / second_last_valid_scan.price.eur_price * 100), 2))
else:
difference_in_percent = decimal.Decimal(0)
return {'percent':difference_in_percent,
'xml_tag':'<two_last_scans_difference_in_percent>',
'xml_close_tag':'</two_last_scans_difference_in_percent>',
'label':'Last scans diff'}
def stats_get_min_price(self):
scan = self.get_min_scan()
if scan:
price = scan.price.eur_price
else:
price = None
return {'price': price,
'xml_tag': '<min_price>',
'xml_close_tag': '</min_price>',
'label': 'Min'}
def stats_get_avg_price(self):
prices = [x.price for x in self.scans.all() if x.price]
if prices:
price = math_ops.round_eur(decimal.Decimal(sum([x.eur_price for x in prices]) / len(prices), 2))
else:
price = None
preferred_currency = self.product.user.userprofile.preferred_currency
if preferred_currency:
if preferred_currency.shortcut == 'czk':
amount = Exchange.eur_to_czk(price)
pref_currency_string = '{} CZK'.format(amount)
pref_currency_amount = amount
else:
amount = price
pref_currency_string = u'{} €'.format(amount)
pref_currency_amount = amount
else:
if self.currency.shortcut == 'czk':
amount = Exchange.eur_to_czk(price)
pref_currency_string = '{} CZK'.format(amount)
pref_currency_amount = amount
else:
amount = price
pref_currency_string = u'{} €'.format(amount)
pref_currency_amount = amount
return {'price': price,
'pref_currency_string':pref_currency_string,
'pref_currency_amount':pref_currency_amount,
'xml_tag': '<average_price>',
'xml_close_tag': '</average_price>',
'label': 'AVG'}
PRICE
class Price(models.Model):
currency = models.ForeignKey('Currency',related_name='prices')
amount = models.DecimalField(max_digits=10,decimal_places=2)
def __unicode__(self):
return u'{} {}'.format(self.amount,self.currency)
def to_eur(self):
if self.currency.shortcut=='eur':
return self.amount
elif self.currency.shortcut=='czk':
return Exchange.objects.first().czk_to_eur(self.amount)
def to_czk(self):
if self.currency.shortcut == 'czk':
return self.amount
elif self.currency.shortcut == 'eur':
return Exchange.objects.first().eur_to_czk(self.amount)
#property
def eur_price(self):
if self.currency.shortcut=='eur':
return self.amount
elif self.currency.shortcut=='czk':
return self.to_eur()
#property
def czk_price(self):
cents = decimal.Decimal('01')
if self.currency.shortcut == 'czk':
return (self.amount).quantize(cents, decimal.ROUND_HALF_UP)
elif self.currency.shortcut == 'eur':
return self.to_czk()
#property
def pref_currency_amount(self):
pref_currency = self.scan.occurence.product.user.userprofile.preferred_currency
if pref_currency:
if pref_currency.shortcut == 'czk':
return self.czk_price
else: return self.eur_price
return self.amount
#property
def pref_currency_string(self):
pref_currency = self.scan.occurence.product.user.userprofile.preferred_currency
# return pref_currency.shortcut
if pref_currency:
if pref_currency.shortcut.lower() == 'czk':
return u'{} {}'.format(self.czk_price, pref_currency.shortcut)
else:
return u'{} {}'.format(self.eur_price, pref_currency.special_sign)
return u'{} {}'.format(self.amount,self.currency.special_sign)
def get_price(self,currency):
if currency=='eur':
return self.eur_price
elif currency=='czk':
return self.czk_price
def get_exchanged_price_string(self):
if self.currency.shortcut=='czk':
return u'{} {}'.format(Exchange.czk_to_eur(self.amount),u'€')
else:
return u'{} {}'.format(Exchange.eur_to_czk(self.amount),'CZK')
def get_original_price_string(self):
if self.currency.shortcut=='czk':
return u'{} {}'.format(self.amount,u'€')
else:
return u'{} {}'.format(Exchange.eur_to_czk(self.amount),'CZK')
For example rendering table occurences takes almost 2 seconds according to django-debug-toolbar. I'm trying to optimize it using select_related and prefetch_related but its still slow.
It's because of different methods I call has the same queries and those queries are called multiple times.
class OccurencesTable(tables.Table):
site = tables.columns.TemplateColumn("""{{ record.site.name }}""",accessor='site.name', verbose_name=u'Site')
avg_price = tables.columns.TemplateColumn("""{{ record.stats_get_avg_price.pref_currency_string }}""",accessor='stats_get_avg_price.price', verbose_name='AVG price')
last_scan_price = tables.columns.TemplateColumn("""{{ record.get_last_scan.price.pref_currency_string }} """,accessor='get_last_scan.price.amount', verbose_name='Last scan price')
last_scan_time = tables.columns.TemplateColumn("""{{ record.get_last_scan.datetime }}""",accessor='get_last_scan.datetime', verbose_name='Last scan time')
difference = tables.columns.TemplateColumn("""{% load static %}{% with diff=record.stats_get_difference_for_two_last_scans.percent %}
{% if diff > 0 %}+{% endif %}{{ diff }} % <img style="height: 15px" src="{% if diff < 0 %}{% static "img/icons/arrow-trend-minus.png" %}{% elif diff == 0 %}{% static "img/icons/arrow-trend-normal.png" %}{% else %}{% static "img/icons/arrow-trend-plus.png" %}{% endif %}">
{% endwith %}""",verbose_name='Difference')
class Meta:
model = Occurence
fields = ('id', 'site', 'last_scan_time','last_scan_price', 'difference', 'avg_price')
attrs = {'id': 'id_occurences_table',
'class': 'table', }
Can't figure out how to optimize methods of models Occurence and Product. Do you have any ideas?

With code like this, you should be thrilled with the timing that you get
#property
def last_scan_time(self):
scans = sorted([x for x in self.get_last_scans() if x.datetime],key = lambda x: x.datetime, reverse=True)
Scan.objects.filter(occurence__product=self,price__amount__isnull=False)
return str(scans[0].datetime)
This code is retrieving an entire table by the call to get_last_scans() then you are sorting that result inside python code! Databases have very fast built in sort functionality. Please use it.
There are plenty of other functions like this in this code. You will have to fix each of them. Do the filtering and sorting in the database. Not in your python code.

Related

How to get model number?

The first class contains the car details and in the second Service class using the get_car_by_year() method. I need to return car based on the date of manufacturing using get_* function of the first class, but I am getting None no matter which year I enter.
Code:
class Car:
def __init__(self,model,year,registration_number):
self.__model=model
self.__year=year
self.__registration_number=registration_number
def get_model(self):
return self.__model
def get_year(self):
return self.__year
def get_registration_number(self):
return self.__registration_number
def __str__(self):
return(self.__model+" "+self.__registration_number+" "+(str)(self.__year))
class Service:
def __init__(self,car_list):
self.__car_list = car_list
def get_car_by_year(self,year):
result_list = []
for car in self.__car_list:
if(car.get_year() == year):
result_list.append(car.get_model())
if(len(result_list) == 0):
return None
return result_list
car1=Car("WagonR",2010,"KA09 3056")
car2=Car("Beat", 2011, "MH10 6776")
car3=Car("Ritz", 2013,"KA12 9098")
car4=Car("Polo",2013,"GJ01 7854")
car5=Car("Amaze",2014,"KL07 4332")
car_list=[car1, car2, car3, car4,car5]
a1 = Service(car_list)
print(a1.get_car_by_year(2013))
Your return statements are inside loop which will always return null, because the first car in your car list is WagonR which is 2010. If you would call your current function get_car_by_year with a 2010 year you will get your desired output.
Placing both return statements outside for loop will do the trick
def get_car_by_year(self,year):
result_list = []
for car in self.__car_list:
if(car.get_year() == year):
result_list.append(car.get_model())
if (len(result_list) == 0):
return None
return result_list

How to assign subtotal from one function to another function?

I have to set the total_cost variable to be equal to the subtotal variable or the from the product class. However, when I try to derive the value from the class. It gives me an attribute error of AttributeError: 'Product' object has no attribute '_Product__total_price'. I am implementing where you apply the promo code then it will deduct the total price from the cart.
This is init.py and functions used.
#app.route('/applyPromo/<string:id>/', methods=['GET','POST'])
def apply_promo(id):
promotions_dict = {}
db = shelve.open('promotions.db','w')
promotions_dict = db['Promotions']
total_cost = 100
hidden = True
applied = True
click = True
get_from_class = False
print(promotions_dict)
promotions = promotions_dict.get(UUID(id))
db = shelve.open('storage.db', 'r')
product_dict = db['products']
for key in product_dict:
product = product_dict[key]
total_cost = product.get_total_price()
print(total_cost)
#when user apply promo, promo will get deleted from the list/dictionary of promos they have.
if promotions["type"] == 1:
total_cost = total_cost - 10
hidden = False
print(f"Total Cost : {total_cost}")
promotions_dict.pop(UUID(id))
elif promotions["type"] == 2:
total_cost = total_cost - 5
hidden = False
print(f"Total Cost : {total_cost}")
promotions_dict.pop(UUID(id))
elif promotions["type"] == 3:
total_cost = (70/100)*total_cost
hidden = False
print(f"Total Cost : {total_cost}")
promotions_dict.pop(UUID(id))
elif promotions["type"] == 4:
total_cost = (80/100)*total_cost
hidden = False
print(f"Total Cost : {total_cost}")
promotions_dict.pop(UUID(id))
elif promotions["type"] == 5:
total_cost = (85/100)*total_cost
hidden = False
print(f"Total Cost : {total_cost}")
promotions_dict.pop(UUID(id))
else:
total_cost = (90/100)*total_cost
hidden = False
print(f"Total Cost : {total_cost}")
promotions_dict.pop(UUID(id))
db['Promotions'] = promotions_dict
db.close()
db.close()
print(promotions_dict)
session['promotion_applied'] = promotions["id"]
return render_template("generatePromo.html", total_cost=total_cost,applied=applied, promotions_dict=promotions_dict,hidden=hidden,promotions=promotions,get_from_class=get_from_class, click=click)
#app.route('/shopping_cart')
def shopping_cart():
# session.clear()
error = None
cart_items = []
quantity_list = []
subtotal = 0
db = shelve.open('storage.db', 'r')
product_dict = db['products']
db.close()
for products in session:
item = product_dict.get(products)
cart_items.append(item)
if None in cart_items:
cart_items.remove(None)
quantity_list.append(session[products])
if products in quantity_list:
quantity_list.remove(products)
for i in range(len(cart_items)):
cart_items[i].set_purchased_quantity(quantity_list[i])
# set total price for single item
item_total = int(cart_items[i].get_price()) * int(cart_items[i].get_purchased_quantity())
cart_items[i].set_total_price(item_total)
# set total price of all items in cart
subtotal += item_total
print('QTY LIST', quantity_list)
print('CART', cart_items)
if not cart_items:
error = "Cart Is Empty"
return render_template('shoppingcart.html', products_list=cart_items, error=error, subtotal=subtotal)
This is the product class.
from uuid import uuid4
class Product:
def __init__(self, name, price, quantity, color, vase, remarks):
self.__product__id = str(uuid4())
self.__name = name
self.__price = price
self.__quantity = quantity
self.__color = color
self.__vase = vase
self.__remarks = remarks
self.__purchased_quantity = 0
self.__total_price = 0
def get_product_id(self):
return self.__product__id
def get_name(self):
return self.__name
def get_price(self):
return self.__price
def get_quantity(self):
return self.__quantity
def get_color(self):
return self.__color
def get_vase(self):
return self.__vase
def get_remarks(self):
return self.__remarks
def get_image(self):
return self.__image
def get_purchased_quantity(self):
return self.__purchased_quantity
def get_total_price(self):
return self.__total_price
def set_name(self, name):
self.__name = name
def set_price(self, price):
self.__price = price
def set_quantity(self, quantity):
self.__quantity = quantity
def set_color(self, color):
self.__color = color
def set_vase(self, vase):
self.__vase = vase
def set_remarks(self, remarks):
self.__remarks = remarks
def set_image(self, image):
self.__image = image
def set_purchased_quantity(self, purchased_quantity):
self.__purchased_quantity = purchased_quantity
def set_total_price(self, total_price):
self.__total_price = total_price
This is the traceback for the error.
Traceback
Leading double underscores should not be used as you are using them to define attributes of your class - single underscores is conventional. The problem with using leading double underscores is that the interpreter "mangles" the names of those attributes to avoid a subtle problem with inheritance, but leading to the problem, I believe, you are seeing. Here is a good read on the subject.

How to receive real time data through Bitmex Websocket Api on python?

I understand that I can use "while true" and call the 'get_ticker' method to get the last price of the product, but this drives from python instead of the market itself. I was wondering if there is a way to get the last price as BitMEX's website changes. Thanks
Check my bitmex project, there is solution of your problem: bitmex-supervisor
Essential code fragments:
In __init__():
self.last_price = 0
self._min_price = float('inf')
self._max_price = -1
self.initial_price = float('nan')
self.tracking = False
Methods:
#property
def min_price(self):
return self._min_price
#min_price.setter
def min_price(self, value):
if value < self.initial_price:
self.callback_price_decreased() # here you can do some stuff
self._min_price = value
#property
def max_price(self):
return self._max_price
#max_price.setter
def max_price(self, value):
if value > self.initial_price:
self.callback_price_increased() # here you can do some stuff
self._max_price = value
def stop_trailing(self):
self.tracking = False
def start_trailing(self, initial_price: float):
"""
:param initial_price: the price after reaching which order will be moving
"""
self._max_price = -1
self._min_price = float('inf')
self.initial_price = initial_price
self.tracking = True
In __on_message():
instrument = self.get_instrument(symbol=self.order.symbol)
if instrument is not None:
self.last_price = instrument['lastPrice']
if self.tracking:
if self.last_price > self.max_price and self.order.side == 'Sell':
self.max_price = self.last_price
elif self.last_price < self.min_price and self.order.side == 'Buy':
self.min_price = self.last_price

Product Inventory program that takes products with an ID, quantity, and price and uses an Inventory class to keep track of the products

The Product class seems to work fine but I'm trying to figure out how to get the Inventory class to separate each product into there specific categories. I feel like I'm close but whenever I try and print out the inventory it just shows where it's stored in memory and doesn't actually print anything out. The output i receive when running is at the bottom. I want it to print out the actual products and data, not the instance of it stored in memory.
class Product:
def __init__(self, pid, price, quantity):
self.pid = pid
self.price = price
self.quantity = quantity
def __str__(self):
#Return the strinf representing the product
return "Product ID: {}\t Price: {}\t Quantity: {}\n".format(self.pid, self.price, self.quantity)
def get_id(self):
#returns id
return self.pid
def get_price(self):
#returns price
return self.price
def get_quantity(self):
#returns quantity
return self.quantity
def increase_quantity(self):
self.quantity += 1
def decrease_quantity(self):
self.quantity -= 1
def get_value(self):
value = self.quantity * self.price
return 'value is {}'.format(value)
product_1 = Product('fishing', 20, 10)
product_2 = Product('apparel', 35, 20)
class Inventory:
def __init__(self, products):
self.products = products
self.fishing_list = []
self.apparel_list = []
self.value = 0
def __repr__(self):
return "Inventory(products: {}, fishing_list: {}, apparel_list: {}, value: {})".format(self.products, self.fishing_list, self.apparel_list, self.value)
def add_fishing(self):
for product in self.products:
if product.get_id() == 'fishing':
self.fishing_list.append(product)
return '{} is in the fishing section'.format(self.fishing_list)
def add_apparel(self):
for product in self.products:
if product.get_id() == 'apparel':
self.apparel_list.append(product)
return '{} is in the apparel section'.format(self.apparel_list)
inventory_1 = Inventory([product_1, product_2])
inventory_1.add_fishing()
print(inventory_1)
OUTPUT = Inventory(products: [<main.Product instance at 0x10dbc8248>, <main.Product instance at 0x10dbc8290>], fishing_list: [<main.Product instance at 0x10dbc8248>], apparel_list: [], value: 0)
You need to specify how an object of the class Inventory should be printed.
To do this you need to implement at least one of the following functions in your class.
__repr__
__str__
This answer helps, which of both you should use: https://stackoverflow.com/a/2626364/8411228
An implementation could look something like this:
class Inventory:
# your code ...
def __repr__(self):
return str(self.products) + str(self.fishing_list) + str(self.apparel_list) + str(self.value)
# or even better with formatting
def __repr__(self):
return f"Inventory(products: {self.products}, fishing_list: {self.fishing_list}, apparel_list: {self.apparel_list}, value: {self.value})
Note that I used in the second example f strings, to format the output string.

Python shopping cart add to cart, get total get num items

I am studying for my final and this was a quiz question I missed. I need most of the help on the getTotal method. I need to loop through the list, find the price of each item, add the price to the total and return the total. I struggle with loops and I am not sure how to pull the second item out of a list.. [1] ?? I have tried many ways and am getting frustrated.
If there is anyone up that is willing to help me that would be great. I am still learning and am new at this so go easy on me, but I really want to learn it. It's probably not as hard as I make it out to be, but Ill be waiting for some input. Thank you!
class Item:
def __init__(self, name, price):
self.name = name
self.price = price
def getPrice(self):
return self.price
def getName(self):
return self.name
class Cart:
def __init__(self, list):
self.list = []
def addItem(self, item):
self.list.append(self.list)
def getTotal(self):
total = 0
for item in self.list:
name, price = item # or price = item[1]
total = total + price
def getNumItems(self):
count = 0
for c in range(self.list):
count = self.list + 1
return count
def removeItem(self, item)
#removes the item from the cart's item list
def main():
item1 = Item("Banana", .69)
item2 = Item("Eggs", 2.39)
item3 = Item("Donut", .99)
c = Cart()
c.addItem(item1)
c.addItem(item2)
c.addItem(item3)
print "You have %i items in your cart for a total of $%.02f" %(c.getNumItems(), c.getTotal())
c.removeItem(item3)
print "You have %i items in your cart for a total of $%.02f" % (c.getNumItems(), c.getTotal())
main()
Here gives the time and I changed the code and now it is fully functional shopping cart
class Item(object):
def __init__(self, unq_id, name, price, qty):
self.unq_id = unq_id
self.product_name = name
self.price = price
self.qty = qty
class Cart(object):
def __init__(self):
self.content = dict()
def update(self, item):
if item.unq_id not in self.content:
self.content.update({item.unq_id: item})
return
for k, v in self.content.get(item.unq_id).iteritems():
if k == 'unq_id':
continue
elif k == 'qty':
total_qty = v.qty + item.qty
if total_qty:
v.qty = total_qty
continue
self.remove_item(k)
else:
v[k] = item[k]
def get_total(self):
return sum([v.price * v.qty for _, v in self.content.iteritems()])
def get_num_items(self):
return sum([v.qty for _, v in self.content.iteritems()])
def remove_item(self, key):
self.content.pop(key)
if __name__ == '__main__':
item1 = Item(1, "Banana", 1., 1)
item2 = Item(2, "Eggs", 1., 2)
item3 = Item(3, "Donut", 1., 5)
cart = Cart()
cart.update(item1)
cart.update(item2)
cart.update(item3)
print "You have %i items in your cart for a total of $%.02f" % (cart.get_num_items(), cart.get_total())
cart.remove_item(1)
print "You have %i items in your cart for a total of $%.02f" % (cart.get_num_items(), cart.get_total())
And a output is:
You have 8 items in your cart for a total of $8.00
You have 7 items in your cart for a total of $7.00
for getTotal:
def getTotal(self):
total = 0
for item in self.list:
name, price = item # or price = item[1]
total = total + price
BTW, Your addItem and getNumItems method are also wrong. Since it is final, you should try to understand what you are working on.

Categories

Resources