Develop Reference

Django import export edit queryset before export

I'm trying to calculate the pending amount via models and export result in the csv. But the csv shows an empty column for amountpending
class FinancePendingResource(resources.ModelResource):
invoiceNumber = Field(attribute='invoiceNumber', column_name='Invoice Number')
student = Field(attribute='student', column_name='Student')
Schedule = Field(attribute='Schedule', column_name='Schedule')
TotalAmount = Field(attribute='TotalAmount', column_name='Total Value(PKR ₨)')
issueDate = Field(attribute='issueDate', column_name='Issue Date')
dueDate = Field(attribute='dueDate', column_name='Due Date')
amountPaid = Field(attribute='amountPaid', column_name='Amount Paid (PKR ₨)')
class Meta:
model = FinancePending
import_id_fields = ('invoiceNumber',)
fields = ('invoiceNumber', 'student', 'amountPaid', 'issueDate', 'dueDate', 'Schedule', 'TotalAmount',
'AmountPending',)
exclude = ('id',)
skip_unchanged = True
report_skipped = True
def before_export(self, queryset, *args, **kwargs):
amount_paid = FinancePending.objects.values_list('amountPaid', flat=True)
amount_paid = list(amount_paid)
total_amount = FinancePending.objects.values_list('TotalAmount', flat=True)
total_amount = list(total_amount)
# total - paid
TotalFee = [float(s.replace(',', '')) for s in total_amount]
AmountPaid = [float(s.replace(',', '')) for s in amount_paid]
def Diff(li1, li2):
return (list(set(li1) - set(li2)))
amount_pending = Diff(TotalFee, AmountPaid)
finance_pending = FinancePending()
i = 1
while i <= len(amount_pending):
FinancePending.objects.filter(invoiceNumber=i).update(AmountPending=str(amount_pending[i]))
i = i + 1
queryset.refresh_from_db()

Assuming that you have the data to compute amountPending already in the dataset, perhaps you don't need to read from the DB: you could calculate the amount by processing the dataset in memory. This could be done in after_export(). Then you can added the computed column to the dataset.
Perhaps tablib's dynamic columns can assist in adding the amountPending column:
import decimal
import tablib
headers = ('invoiceNumber', 'amountPaid', 'totalAmount')
rows = [
('inv100', '100.00', "500.00"),
('inv101', '200.00', "250.00")
]
def amount_pending(row):
return decimal.Decimal(row[2]) - decimal.Decimal(row[1])
data = tablib.Dataset(*rows, headers=headers)
data.append_col(amount_pending, header="amountPending")
print(data)
This will produce the following:
invoiceNumber|amountPaid|totalAmount|amountPending
-------------|----------|-----------|-------------
inv100 |100.00 |500.00 |400.00
inv101 |200.00 |250.00 |50.00

How to fill through model instances using forms or formsets

I have the following models :
class criterion(models.Model):
quantity = '0'
quality = '1'
ascending = '0'
descending = '1'
three = '3'
five = '5'
seven = '7'
ten = '10'
type_choices = (
(quantity,'Ποσοτικό'),
(quality,'Ποιοτικό'),
)
monotonicity_choices = (
(ascending,'Αύξον'),
(descending,'Φθίνον'),
)
a_choices = (
(three,'Τριβάθμια'),
(five,'Πενταβάθμια'),
(seven,'Εφταβάθμια'),
(ten,'Δεκαβάθμια'),
)
criterion_research = models.ForeignKey('research', on_delete=models.CASCADE,null=True)
criterion_name = models.CharField(max_length = 200,verbose_name='Όνομα Κριτηρίου')
criterion_type = models.CharField(max_length = 15,choices = type_choices , default = quantity,verbose_name='Τύπος')
criterion_monotonicity = models.CharField(max_length = 15,choices = monotonicity_choices , default = ascending,verbose_name='Μονοτονία')
criterion_a = models.CharField(max_length = 30,choices = a_choices , default = five,verbose_name='Κλίμακα',help_text='Επιλέξτε μία από τις 4 κλίμακες που θα μετρηθεί το κριτήριο.')
criterion_worst = models.FloatField(default=' ',verbose_name='Χειρότερη Τιμή',help_text='Επιλέξτε τη χειρότερη τιμή την οποία μπορεί να πάρει το κριτήριο.',blank=True)
criterion_best = models.FloatField(default=' ',verbose_name='Καλύτερη Τιμή',help_text='Επιλέξτε τη καλύτερη τιμή την οποία μπορεί να πάρει το κριτήριο.',blank=True)
def __str__(self):
return self.criterion_name
class alternative(models.Model):
name = models.CharField(max_length = 200,verbose_name='Όνομα Εναλλακτικής')
ranking = models.IntegerField(default='1')
alternative_research = models.ForeignKey('research', on_delete=models.CASCADE)
criteria = models.ManyToManyField('criterion', through='criterionvalue',blank=True)
def __str__(self):
return self.name
class criterionvalue(models.Model):
value = models.IntegerField(default='0')
alt = models.ForeignKey('alternative',on_delete=models.CASCADE)
crit = models.ForeignKey('criterion',on_delete=models.CASCADE)
I have made a view that populates the criterion instances and the alternative instances(except the manytomany field).
I want to query for all the possibles criterion and alternatives and then based on that populate the criterionvaue instances.After I complete these I want then to specify the manytomany relatioship of the criterionvalue instances with the alternatives instances .I thought of making a formset of criterionvalue model then associate them(the forms made by the formset) with the alternatives and criteria using the length of the alternatives and criteria queryset(In order to render the relatioship matrix using forms) .
But I think such an idea is farfetched.
Any ideas on how i could achieve such thing ?

Create error message datefield

I want to create an error message for following form:
class ExaminationCreateForm(forms.ModelForm):
class Meta:
model = Examination
fields = ['patient', 'number_of_examination', 'date_of_examination']
Models:
class Patient(models.Model):
patientID = models.CharField(max_length=200, unique=True, help_text='Insert PatientID')
birth_date = models.DateField(auto_now=False, auto_now_add=False, help_text='YYYY-MM-DD')
gender = models.CharField(max_length=200,choices=Gender_Choice, default='UNDEFINED')
class Examination(models.Model):
number_of_examination = models.IntegerField(choices=EXA_Choices)
patient = models.ForeignKey(Patient, on_delete=models.CASCADE)
date_of_examination = models.DateField(auto_now=False, auto_now_add=False, help_text='YYYY-MM-DD')
Every Patient has 2 Examinations (number of examination = Choices 1 or 2) and the error message should be activated when the date of the second examination < date of the first examination. Something like this:
Solution: `
def clean_date_of_examination(self):
new_exam = self.cleaned_data.get('date_of_examination')
try:
old_exam = Examination.objects.get(patient=self.cleaned_data.get('patient'))
except Examination.DoesNotExist:
return new_exam
if old_exam:
if old_exam.date_of_examination > new_exam:
raise forms.ValidationError("Second examination should take place after first examination")
return new_exam`

def clean_date_of_examination(self):
new_exam = self.cleaned_data.get('date_of_examination')
old_exam = Examination.objects.get(patient = self.cleaned_data.get('Patient'))
if old_exam:
if old_exam.date_of_examination > new_exam.date_of_examination:
raise forms.ValidationError("Second examination should take place after first examination")
return data

def clean_date_of_examination(self):
# Where 'data' is used?
date_of_exam = self.cleaned_data['date_of_examination']
try:
pat1 = Patient.object.get(examination__number_of_examination=1, date_of_examination=date_of_exam)
except Patiens.DoesNotExist:
# Patient 1 with given query doesn't exist. Handle it!
try:
pat2 = Patient.object.get(examination__number_of_examination=2, date_of_examination=date_of_exam)
except Patiens.DoesNotExist:
# Patient 2 with given query doesn't exist. Handle it!
if pat2.date_of_examination < pat1.date_of_examination:
raise forms.ValidationError("Second examination should take place after first examination")`
return data`

Django advanced search

I am currently learning Django by making a web app that sell used bikes and I'm having problems with on site search.
I would like to have a search field for every model field and I just can't figure out how to do it.
What would be the best way to do this?
Any help is more than welcome!
Here is my model:
class UsedBike(models.Model):
manufacturers = (
('Aprilia', 'Aprilia'),
('Benelli', 'Benelli'),
('BMW', 'BMW'),
('Cagiva', 'Cagiva'),
('Gilera', 'Gilera'),
('Harley-Davidson', 'Harley-Davidson'),
('Husaberg', 'Husaberg'),
('Husquarna', 'Husquarna'),
('Hyosung', 'Hyosung'),
('Kawasaki', 'Kawasaki'),
('KTM', 'KTM'),
('Kymco', 'Kymco'),
('Moto Guzzi', 'Moto Guzzi'),
('MV Agusta', 'MV Agusta'),
('Suzuki', 'Suzuki'),
('Tomos', 'Tomos'),
('Triumph', 'Triumph'),
('Yamaha', 'Yamaha'),
)
manufacturer = models.CharField(help_text = 'Manufacturer: ',
max_length = 20,
choices = manufacturers)
model = models.CharField(max_length = 20, help_text = 'Model: ')

I solved this but I forgot to post the answer so that anyone who have the similar problem can use it. It is not perfect but it worked for me, if someone have a better solution feel free to answer.
In my models I have:
class UsedBike(models.Model):
manufacturer = models.CharField(max_length = 20)
model = models.CharField(max_length = 20)
year = models.PositiveIntegerField(default = 0)
bike_type = models.CharField(max_length = 20)
price = models.PositiveIntegerField(default = 0)
engine_size = models.PositiveIntegerField(default = 0)
And in views:
def searchbike(request):
man = request.GET.get('manufacturer')
mod = request.GET.get('model')
year_f = request.GET.get('year_from')
year_t = request.GET.get('year_to')
price_f = request.GET.get('price_from')
price_t = request.GET.get('price_to')
bike_t = request.GET.get('bike_type')
capacity_f = request.GET.get('cubic_capacity_from')
capacity_t = request.GET.get('cubic_capacity_to')
question_set = UsedBike.objects.filter()
if request.GET.get('manufacturer'):
question_set = question_set.filter(manufacturer__exact = man)
if request.GET.get('model'):
question_set = question_set.filter(model__icontains = mod)
if request.GET.get('year_from'):
question_set = question_set.filter(year__gte = year_f)
if request.GET.get('year_to'):
question_set = question_set.filter(year__lte = year_t)
if request.GET.get('price_from'):
question_set = question_set.filter(price__gte = price_f)
if request.GET.get('price_to'):
question_set = question_set.filter(price__lte = price_t)
if request.GET.get('bike_type'):
question_set = question_set.filter(bike_type__exact = bike_t)
if request.GET.get('cubic_capacity_from'):
question_set = question_set.filter(engine_size__gte = capacity_f)
if request.GET.get('cubic_capacity_to'):
question_set = question_set.filter(engine_size__lte = capacity_t)
return render(request, 'shop/search.html', { 'searched': question_set})

You can use django-smart-selects:
If you have the following model:
class Location(models.Model)
continent = models.ForeignKey(Continent)
country = models.ForeignKey(Country)
area = models.ForeignKey(Area)
city = models.CharField(max_length=50)
street = models.CharField(max_length=100)
And you want that if you select a continent only the countries are available that are located on this continent and the same for areas you can do the following:
from smart_selects.db_fields import ChainedForeignKey
class Location(models.Model)
continent = models.ForeignKey(Continent)
country = ChainedForeignKey(
Country,
chained_field="continent",
chained_model_field="continent",
show_all=False,
auto_choose=True
)
area = ChainedForeignKey(Area, chained_field="country", chained_model_field="country")
city = models.CharField(max_length=50)
street = models.CharField(max_length=100)

Django - Getting/Saving large objects takes a lot of time

I'm trying to get a few million of items from a model, and parsing them. However, somehow it spends a lot of time trying to get the data saved.
These are the current models that I have:
class mapt(models.Model):
s = models.IntegerField(primary_key=True)
name = models.CharField(max_length=2000)
def __unicode__(self):
return str(self.s)
class datt(models.Model):
s = models.IntegerField(primary_key=True)
setid = models.IntegerField()
var = models.IntegerField()
val = models.IntegerField()
def __unicode(self):
return str(self.s)
class sett(models.Model):
setid = models.IntegerField(primary_key=True)
block = models.IntegerField()
username = models.IntegerField()
ts = models.IntegerField()
def __unicode__(self):
return str(self.setid)
class data_parsed(models.Model):
setid = models.IntegerField(max_length=2000, primary_key=True)
block = models.CharField(max_length=2000)
username = models.CharField(max_length=2000)
data = models.CharField(max_length=200000)
time = models.IntegerField()
def __unicode__(self):
return str(self.setid)
The s parameter for the datt model should actually act as a foreign key to mapt's s parameter. Furthermore, sett's setid field should act as a foreign key to setid's setid.
Lastly, data_parsed's setid is a foreign key to sett's models.
The algorithm is currently written this way:
def database_rebuild(start_data_parsed):
listSetID = []
#Part 1
for items in sett.objects.filter(setid__gte=start_data_parsed):
listSetID.append(items.setid)
uniqueSetID = listSetID
#Part 2
for items in uniqueSetID:
try:
SetID = items
settObject = sett.objects.get(setid=SetID)
UserName = mapt.objects.get(pk=settObject.username).name
TS = pk=settObject.ts
BlockName = mapt.objects.get(pk=settObject.block).name
DataPairs_1 = []
DataPairs_2 = []
DataPairs_1_Data = []
DataPairs_2_Data = []
for data in datt.objects.filter(setid__exact=SetID):
DataPairs_1.append(data.var)
DataPairs_2.append(data.val)
for Data in DataPairs_1:
DataPairs_1_Data.append(mapt.objects.get(pk=Data).name)
for Data in DataPairs_2:
DataPairs_2_Data.append(mapt.objects.get(pk=Data).name)
assert (len(DataPairs_1) == len(DataPairs_2)), "Length not equal"
#Part 3
Serialize = []
for idx, val in enumerate(DataPairs_1_Data):
Serialize.append(str(DataPairs_1_Data[idx]) + ":PARSEABLE:" + str(DataPairs_2_Data[idx]) + ":PARSEABLENEXT:")
Serialize_Text = ""
for Data in Serialize:
Serialize_Text += Data
Data = Serialize_Text
p = data_parsed(SetID, BlockName, UserName, Data, TS)
p.save()
except AssertionError, e:
print "Error:" + str(e.args)
print "Possibly DataPairs does not have equal length"
except Exception as e:
print "Error:" + str(sys.exc_info()[0])
print "Stack:" + str(e.args)
Basically, what it does is that:
Finds all sett objects that is greater than a number
Gets the UserName, TS, and BlockName, then get all the fields in datt field that correspond to a var and val field maps to the mapt 's' field. Var and Val is basically NAME_OF_FIELD:VALUE type of relationship.
Serialize all the var and val parameters so that I could get all the parameters from var and val that is spread across the mapt table in a row in data_parsed.
The current solution does everything I would like to, however, on a Intel Core i5-4300U CPU # 1.90Ghz, it parses around 15000 rows of data daily on a celery periodic worker. I have around 3355566 rows of data at my sett table, and it will take around ~23 days to parse them all.
Is there a way to speed up the process?
============================Updated============================
New Models:
class mapt(models.Model):
s = models.IntegerField(primary_key=True)
name = models.CharField(max_length=2000)
def __unicode__(self):
return str(self.s)
class sett(models.Model):
setid = models.IntegerField(primary_key=True)
block = models.ForeignKey(mapt, related_name='sett_block')
username = models.ForeignKey(mapt, related_name='sett_username')
ts = models.IntegerField()
def __unicode__(self):
return str(self.setid)
# class sett(models.Model):
# setid = models.IntegerField(primary_key=True)
# block = models.IntegerField()
# username = models.IntegerField()
# ts = models.IntegerField()
# def __unicode__(self):
# return str(self.setid)
class datt(models.Model):
s = models.IntegerField(primary_key=True)
setid = models.ForeignKey(sett, related_name='datt_setid')
var = models.ForeignKey(mapt, related_name='datt_var')
val = models.ForeignKey(mapt, related_name='datt_val')
def __unicode(self):
return str(self.s)
# class datt(models.Model):
# s = models.IntegerField(primary_key=True)
# setid = models.IntegerField()
# var = models.IntegerField()
# val = models.IntegerField()
# def __unicode(self):
# return str(self.s)
class data_parsed(models.Model):
setid = models.ForeignKey(sett, related_name='data_parsed_setid', primary_key=True)
block = models.CharField(max_length=2000)
username = models.CharField(max_length=2000)
data = models.CharField(max_length=2000000)
time = models.IntegerField()
def __unicode__(self):
return str(self.setid)
New Parsing:
def database_rebuild(start_data_parsed, end_data_parsed):
for items in sett.objects.filter(setid__gte=start_data_parsed, setid__lte=end_data_parsed):
try:
UserName = mapt.objects.get(pk=items.username_id).name
TS = pk=items.ts
BlockName = mapt.objects.get(pk=items.block_id).name
DataPairs_1 = []
DataPairs_2 = []
DataPairs_1_Data = []
DataPairs_2_Data = []
for data in datt.objects.filter(setid_id__exact=items.setid):
DataPairs_1.append(data.var_id)
DataPairs_2.append(data.val_id)
for Data in DataPairs_1:
DataPairs_1_Data.append(mapt.objects.get(pk=Data).name)
for Data in DataPairs_2:
DataPairs_2_Data.append(mapt.objects.get(pk=Data).name)
assert (len(DataPairs_1) == len(DataPairs_2)), "Length not equal"
Serialize = []
for idx, val in enumerate(DataPairs_1_Data):
Serialize.append(str(DataPairs_1_Data[idx]) + ":PARSEABLE:" + str(DataPairs_2_Data[idx]))
Data = ":PARSEABLENEXT:".join(Serialize)
p = data_parsed(items.setid, BlockName, UserName, Data, TS)
p.save()
except AssertionError, e:
print "Error:" + str(e.args)
print "Possibly DataPairs does not have equal length"
except Exception as e:
print "Error:" + str(sys.exc_info()[0])
print "Stack:" + str(e.args)

Defining lists by appending repeadedly is very slow. Use list comprehensions or even just the list() constructor.
In python you should not join a list of strings using for loops and +=, you should use join().
But that is not the primary bottleneck here. You have a lot of objects.get()s which each takes a database roundtrip. If you didn't have milions of rows in the mapt table, you should probably just make a dictionary mapping mapt primary keys to mapt objects.
Had you defined your foreign keys as foreign keys the django orm could help you do much of this in like five queries in total. That is, instead of SomeModel.objects.get(id=some_instance.some_fk_id) you can do some_instance.some_fk (which will only hit the databse the first time you do it for each instance). You can then even get rid of the foreign key query if some_instance had been initialized as some_instance = SomeOtherModel.objects.select_related('some_fk').get(id=id_of_some_instance).
Perhaps changing the models without changing the database will work.