Develop Reference

Django : Best way to Query a M2M Field , and count occurences

class Edge(BaseInfo):
source = models.ForeignKey('Node', on_delete=models.CASCADE,related_name="is_source")
target = models.ForeignKey('Node', on_delete=models.CASCADE,related_name="is_target")
def __str__(self):
return '%s' % (self.label)
class Meta:
unique_together = ('source','target','label','notes')
class Node(BaseInfo):
item_type_list = [('profile','Profile'),
('page','Page'),
('group','Group'),
('post','Post'),
('phone','Phone'),
('website','Website'),
('email','Email'),
('varia','Varia')
]
item_type = models.CharField(max_length=200,choices=item_type_list,blank = True,null=True)
firstname = models.CharField(max_length=200,blank = True, null=True)
lastname = models.CharField(max_length=200,blank = True,null=True)
identified = models.BooleanField(blank=True,null=True,default=False)
username = models.CharField(max_length=200, blank=True, null=True)
uid = models.CharField(max_length=200,blank=True,null=True)
url = models.CharField(max_length=2000,blank=True,null=True)
edges = models.ManyToManyField('self', through='Edge',blank = True)
I have a Model Node (in this case a soc media profile - item_type) that has relations with other nodes (in this case a post). A profile can be the author of a post. An other profile can like or comment that post.
Question : what is the most efficient way to get all the distinct profiles that liked or commented on anothes profile's post + the count of these likes /comments.
print(Edge.objects.filter(Q(label="Liked")|Q(label="Commented"),q).values("source").annotate(c=Count('source')))
Gets me somewhere but i have the values then (id) and i want to pass the objects to my template rather then .get() all the profiles again...
Result :
Thanks in advance

I ended up with iterating over the queryset and adding the objects that i wanted in a dictionary , if the object was already in dictionary , i would count +1 and add the relation in a nested list.
This doesnt feel right but works for now.
posts = Edge.objects.filter(source = self,target__item_type='post',label='Author')
if posts:
q = Q()
for post in posts:
q = q | Q(target=post.target)
contributors = Edge.objects.filter(Q(label="Liked")|Q(label="Commented"),q)
if contributors:
for i in contributors:
if i.source.uid in results:
if i.label in results[i.source.uid]['relation']:
pass
else:
results[i.source.uid]["relation"].append(i.label)
if 'post' in results[i.source.uid]:
results[i.source.uid]['post'].append(i.target)
else:
results[i.source.uid]['post']=[i.target]
else:
results[i.source.uid] = {'profile' : i.source , 'relation':[i.label],'post':[i.target]}

How to make a python object json-serialized?

I want to serialize a python object, after saved it into mysql(based on Django ORM) I want to get it and pass this object to a function which need this kind of object as a param.
Following two parts are my main logic code:
1 save param part :
class Param(object):
def __init__(self, name=None, targeting=None, start_time=None, end_time=None):
self.name = name
self.targeting = targeting
self.start_time = start_time
self.end_time = end_time
#...
param = Param()
param.name = "name1"
param.targeting= "targeting1"
task_param = {
"task_id":task_id, # string
"user_name":user_name, # string
"param":param, # Param object
"save_param":save_param_dict, # dictionary
"access_token":access_token, # string
"account_id": account_id, # string
"page_id": page_id, # string
"task_name":"sync_create_ad" # string
}
class SyncTaskList(models.Model):
task_id = models.CharField(max_length=128, blank=True, null=True)
ad_name = models.CharField(max_length=128, blank=True, null=True)
user_name = models.CharField(max_length=128, blank=True, null=True)
task_status = models.SmallIntegerField(blank=True, null=True)
task_fail_reason = models.CharField(max_length=255, blank=True, null=True)
task_name = models.CharField(max_length=128, blank=True, null=True)
start_time = models.DateTimeField()
end_time = models.DateTimeField(blank=True, null=True)
task_param = models.TextField(blank=True, null=True)
class Meta:
managed = False
db_table = 'sync_task_list'
SyncTaskList(
task_id=task_id,
ad_name=param.name,
user_name=user_name,
task_status=0,
task_param = task_param,
).save()
2 use param part
def add_param(param, access_token):
pass
task_list = SyncTaskList.objects.filter(task_status=0)
for task in task_list:
task_param = json.loads(task.task_param)
add_param(task_param["param"], task_param["access_token"]) # pass param object to function add_param
If I directly use Django ORM to save task_param into mysql, I get error,
json.decoder.JSONDecodeError: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
for after ORM operation, I get string who's property name enclosed in single quotes like :
# in mysql it saved as
task_param: " task_param: {'task_id': 'e4b8b240cefaf58fa9fa5a591221c90a',
'user_name': 'jimmy',
'param': Param(name='name1',
targeting='geo_locations',
),
'save_param': {}}"
I am now confused with serializing an python object, then how to load this original object and pass it to a function?
Any commentary is very welcome. great thanks.
update my solution so far
task_param = {
# ...
"param":vars(param), # turn Param object to dictionary
# ...
}
SyncTaskList(
#...
task_param = json.dumps(task_param),
#...
).save()
#task_list = SyncTaskList.objects.filter(task_status=0)
#for task in task_list:
task_param = json.loads(task.task_param)
add_param(Param(**task_param["param"]), task_param["access_token"])
update based on #AJS's answer
directly pickle dumps and saved it as an binary field, then pickle loadsit also works
Any better solution for this?

Try looking into msgpack
https://msgpack.org/index.html
unlike pickle, which is python-specific, msgpack is supported by many languages (so the language you use to write to mysql can be different than the language used to read).
There are also some projects out there that integrate these serializer-libraries into Django model fields:
Pickle: https://pypi.org/project/django-picklefield/
MsgPack: https://github.com/vakorol/django-msgpackfield/blob/master/msgpackfield/msgpackfield.py

You can use pickle basically you are serializing your python object and save it as bytes in your MySQL db using BinaryField as your model field type in Django, as i don't think JSON serialization would work in your case as you have a python object as a value as well in your dict, when you fetch your data from db simpily unpickle it syntax is similar to json library see below.
import pickle
#to pickle
data = pickle.dumps({'name':'testname'})
# to unpickle just do
pickle.loads(data)
so in your case when you unpickle your object you should get your data in same form as it was before you did pickle.
Hope this helps.

Create error message datefield

I want to create an error message for following form:
class ExaminationCreateForm(forms.ModelForm):
class Meta:
model = Examination
fields = ['patient', 'number_of_examination', 'date_of_examination']
Models:
class Patient(models.Model):
patientID = models.CharField(max_length=200, unique=True, help_text='Insert PatientID')
birth_date = models.DateField(auto_now=False, auto_now_add=False, help_text='YYYY-MM-DD')
gender = models.CharField(max_length=200,choices=Gender_Choice, default='UNDEFINED')
class Examination(models.Model):
number_of_examination = models.IntegerField(choices=EXA_Choices)
patient = models.ForeignKey(Patient, on_delete=models.CASCADE)
date_of_examination = models.DateField(auto_now=False, auto_now_add=False, help_text='YYYY-MM-DD')
Every Patient has 2 Examinations (number of examination = Choices 1 or 2) and the error message should be activated when the date of the second examination < date of the first examination. Something like this:
Solution: `
def clean_date_of_examination(self):
new_exam = self.cleaned_data.get('date_of_examination')
try:
old_exam = Examination.objects.get(patient=self.cleaned_data.get('patient'))
except Examination.DoesNotExist:
return new_exam
if old_exam:
if old_exam.date_of_examination > new_exam:
raise forms.ValidationError("Second examination should take place after first examination")
return new_exam`

def clean_date_of_examination(self):
new_exam = self.cleaned_data.get('date_of_examination')
old_exam = Examination.objects.get(patient = self.cleaned_data.get('Patient'))
if old_exam:
if old_exam.date_of_examination > new_exam.date_of_examination:
raise forms.ValidationError("Second examination should take place after first examination")
return data

def clean_date_of_examination(self):
# Where 'data' is used?
date_of_exam = self.cleaned_data['date_of_examination']
try:
pat1 = Patient.object.get(examination__number_of_examination=1, date_of_examination=date_of_exam)
except Patiens.DoesNotExist:
# Patient 1 with given query doesn't exist. Handle it!
try:
pat2 = Patient.object.get(examination__number_of_examination=2, date_of_examination=date_of_exam)
except Patiens.DoesNotExist:
# Patient 2 with given query doesn't exist. Handle it!
if pat2.date_of_examination < pat1.date_of_examination:
raise forms.ValidationError("Second examination should take place after first examination")`
return data`

Iterating with Django ORM through large datasets is slow

I'm using Django ORM to get data out of a database with a few million items. However, computation takes a while (40 minutes+), and I'm not sure how to pin point where the issue is located.
Models I've used:
class user_chartConfigurationData(models.Model):
username_chartNum = models.ForeignKey(user_chartConfiguration, related_name='user_chartConfigurationData_username_chartNum')
openedConfig = models.ForeignKey(user_chartConfigurationChartID, related_name='user_chartConfigurationData_user_chartConfigurationChartID')
username_selects = models.CharField(max_length=200)
blockName = models.CharField(max_length=200)
stage = models.CharField(max_length=200)
variable = models.CharField(max_length=200)
condition = models.CharField(max_length=200)
value = models.CharField(max_length=200)
type = models.CharField(max_length=200)
order = models.IntegerField()
def __unicode__(self):
return str(self.username_chartNum)
order = models.IntegerField()
class data_parsed(models.Model):
setid = models.ForeignKey(sett, related_name='data_parsed_setid', primary_key=True)
setid_hash = models.CharField(max_length=100, db_index = True)
block = models.CharField(max_length=2000, db_index = True)
username = models.CharField(max_length=2000, db_index = True)
time = models.IntegerField(db_index = True)
time_string = models.CharField(max_length=200, db_index = True)
def __unicode__(self):
return str(self.setid)
class unique_variables(models.Model):
setid = models.ForeignKey(sett, related_name='unique_variables_setid')
setid_hash = models.CharField(max_length=100, db_index = True)
block = models.CharField(max_length=200, db_index = True)
stage = models.CharField(max_length=200, db_index = True)
variable = models.CharField(max_length=200, db_index = True)
value = models.CharField(max_length=2000, db_index = True)
class Meta:
unique_together = (("setid", "block", "variable", "stage", "value"),)
The code I'm running is looping through data_parsed, with relevant data that matches between user_chartConfigurationData and unique_variables.
#After we get the tab, we will get the configuration data from the config button. We will need the tab ID, which is chartNum, and the actual chart
#That is opened, which is the chartID.
chartIDKey = user_chartConfigurationChartID.objects.get(chartID = chartID)
for i in user_chartConfigurationData.objects.filter(username_chartNum = chartNum, openedConfig = chartIDKey).order_by('order').iterator():
iterator = data_parsed.objects.all().iterator()
#We will loop through parsed objects, and at the same time using the setid (unique for all blocks), which contains multiple
#variables. Using the condition, we can set the variable gte (greater than equal), or lte (less than equal), so that the condition match
#the setid for the data_parsed object, and variable condition
for contents in iterator:
#These are two flags, found is when we already have an entry inside a dictionary that already
#matches the same setid. Meaning they are the same blocks. For example FlowBranch and FlowPure can belong
#to the same block. Hence when we find an entry that matches the same id, we will put it in the same dictionary.
#Added is used when the current item does not map to a previous setid entry in the dictionary. Then we will need
#to add this new entry to the array of dictionary (set_of_pk_values). Otherwise, we will be adding a lot
#of entries that doesn't have any values for variables (because the value was added to another entry inside a dictionary)
found = False
added = False
storeItem = {}
#Initial information for the row
storeItem['block'] = contents.block
storeItem['username'] = contents.username
storeItem['setid'] = contents.setid
storeItem['setid_hash'] = contents.setid_hash
if (i.variable != ""):
for findPrevious in set_of_pk_values:
if(str(contents.setid) == str(findPrevious['setid'])):
try:
items = unique_variables.objects.get(setid = contents.setid, variable = i.variable)
findPrevious[variableName] = items.value
found = True
break
except:
pass
if(found == False):
try:
items = unique_variables.objects.get(setid = contents.setid, variable = i.variable)
storeItem[variableName] = items.value
added = True
except:
pass
if(found == False and added == True):
storeItem['time_string'] = contents.time_string
set_of_pk_values.append(storeItem)
I've tried to use select_related() or prefetch_related(), since it needs to go to unique_variables object and get some data, however, it still takes a long time.
Is there a better way to approach this problem?

Definitely, have a look at django_debug_toolbar. It will tell you how many queries you execute, and how long they last. Can't really live without this package when I have to optimize something =).
PS: Execution will be even slower.
edit: You may also want to enable db_index for the fields you use to filter with or index_together for more than one field. Ofc, measure the times between your changes so you make sure which option is better.

django model - queryset issue

I am trying to search for an item in db. there are two items in db, but i cannot get the second one somehow. with my code below the result is only the first row of Bewertung but not the second one.
my code is simple:
locations = Location.objects.all()[:5]
bewertungs = Bewertung.objects.filter(von_location__in=locations)
what can be the reason of why i cannot find the second entry in db? i am getting the first record where bewertung is 4, but the second isnot coming up in the result.
EDIT:
this is the Bewertung Model.
class Bewertung(models.Model):
von_location= models.ForeignKey(Location,related_name="locations_bewertung",default="")
von_user = models.ForeignKey(User,related_name="users_bewertung",default="")
price_leistung = models.IntegerField(max_length=5,default=00)
romantic = models.IntegerField(max_length=3,default=00)
bewertung = models.IntegerField(max_length=3,default=00)
def __unicode__(self):
return self.bewertung
these are the records:

class Bewertung(models.Model):
//you don't have to put default="" because this is already required
von_location= models.ForeignKey(Location,related_name="locations_bewertung")
von_user = models.ForeignKey(User,related_name="users_bewertung")
//use DecimalField instead of IntergerField
//use max_digits not max_length because it is for string
price_leistung = models.DecimalField(max_digits=3, decimal_place=2, default=0)
romantic = models.DecimalField(max_digits=3, decimal_place=2, default=0)
bewertung = models.DecimalField(max_digits=3, decimal_place=2, default=0)
//you return your unicode with an int field which result to error
//so you must do it this way
def __unicode__(self):
return "{0}".format(self.bewertung)