Efficient Count of Related Models in Django - python

In my Django app, when I add the host_count field to my serializer to get the number of hosts for each domain, the performance of the API response suffers dramatically.
Without host_count: 300ms
With host_count: 15s
I tried adding 'host_set' to the prefetch_related method but it did not help.
Would an annotation using Count help me here? How can I optimize the fetching of this value?
serializers.py
class DomainSerializer(serializers.Serializer):
name = serializers.CharField(read_only=True)
org_name = serializers.CharField(source='org.name', read_only=True)
created = serializers.DateTimeField(read_only=True)
last_host_search = serializers.DateTimeField(read_only=True)
host_count = serializers.SerializerMethodField()
def get_host_count(self, obj):
return Host.objects.filter(domain=obj).count()
views.py
class DomainList(generics.ListAPIView):
def get(self, request, format=None):
domains = Domain.objects.prefetch_related('org').all()
serializer = DomainSerializer(domains, many=True)
return Response(serializer.data)
models.py
class Domain(models.Model):
created = models.DateTimeField(auto_now_add=True)
last_host_search = models.DateTimeField(auto_now=True)
name = models.CharField(unique=True, max_length=settings.MAX_CHAR_COUNT, blank=False, null=False)
org = models.ForeignKey(Org, on_delete=models.CASCADE, blank=True, null=True)

You can work with .annotate(…) [Django-doc] to count the related objects in the same query:
from django.db.models import Count
class DomainList(generics.ListAPIView):
def get(self, request, format=None):
domains = Domain.objects.prefetch_related('org').annotate(
host_count=Count('host')
)
serializer = DomainSerializer(domains, many=True)
return Response(serializer.data)
In the serializer, you then simply retrieve the corresponding attribute:
class DomainSerializer(serializers.Serializer):
name = serializers.CharField(read_only=True)
org_name = serializers.CharField(source='org.name', read_only=True)
created = serializers.DateTimeField(read_only=True)
last_host_search = serializers.DateTimeField(read_only=True)
host_count = serializers.IntegerField(read_only=True)
This will make a query that looks like:
SELECT domain.*, org.*, COUNT(host.id)
FROM domain
LEFT OUTER JOIN org ON domain.org_id = org.id
LEFT OUTER JOIN host ON host.domain_id = domain.id
GROUP BY domain.id, org.id

Related

How to update the value of a column in a foreign table in django

I have 2 tables, one of the tables is a foreign table to the other (foreign key). I want to decrease the value of the quantity column of items in the foreign table based on value of the quantity column of the parent table.
Below is my code
model.py
from django.db import models
# Create your models here.
PAYMENT_METHOD = (
('CASH', 'cash'),
('TRANSFER', 'transfer'),
)
class ItemSold(models.Model):
item_name = models.CharField(max_length=80)
quantity = models.IntegerField()
def __str__(self):
return self.item_name
class DailySales(models.Model):
customername = models.CharField(max_length=100)
itemsold = models.ForeignKey(ItemSold, related_name="soldItem", on_delete=models.CASCADE)
quantity = models.IntegerField()
rate = models.IntegerField()
totalprice = models.IntegerField(default=0)
datesold = models.DateTimeField(auto_now_add=True, auto_now=False)
paymentmethod = models.CharField(max_length=40, choices=PAYMENT_METHOD, default=PAYMENT_METHOD[0][0])
havepaid = models.BooleanField(default=False)
datepaid = models.DateTimeField(auto_now_add=False, auto_now=True)
class Meta:
verbose_name_plural = 'Daily Sales'
def __str__(self):
return self.customername
def save(self, *args, **kwargs):
self.totalprice = self.rate * self.quantity
super(DailySales, self).save(*args, **kwargs)
views.py
class DailySalesListView(generics.GenericAPIView):
serializer_class = DailySalesSerializer
queryset = DailySales.objects.all()
name = 'Daily Sales List'
filter_backends = (DjangoFilterBackend,)
filterset_fields = ('customername','havepaid', 'datesold', 'itemsold', 'datepaid')
def get(self, request):
sales = self.filter_queryset(self.get_queryset())
serializer = self.serializer_class(instance=sales, many=True)
return Response(data=serializer.data, status=status.HTTP_200_OK)
def post(self, request):
data = request.data
serializer = self.serializer_class(data=data)
if serializer.is_valid():
serializer.save()
return Response(data=serializer.data, status=status.HTTP_201_CREATED)
return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST)
I haven't tried anything yet, I don't know how to go about it
If I understood you correctly, the two model fields quantity are identical, in that case, you don't need them both. You can do that logic in your view with the help of get_query_context.
Example code:
class DailySales(ListView):
model = ItemSold
def get_context_data(self, **kwargs):
context = super(DailySales, self).get_context_data(**kwargs)
# now you need the count for each quantity model field
dailysales_quantity = DailySales.objects.filter(field= field) # check this is correct I'm not sure
# now you need to update the ItemsSold quantity so first get the current quantity count from ItemSold
itemsold_quantity = ItemSold.objects.quantity.count() # make sure the last part is okay I did not have time to test this
# now subtract the ItemSold model field (above as itemsold_quantity) by the DailySales quantity field ( above as dailysales_quantity
context['live_quantity'] = itemsold_quantity - dailysales_quantity # you might have to convert them into int since I didn't test this myself
# finally return the data
return context
I hope this helps, it's not a complete solution but gets a step closer.

Posting to multiply related tables Django

I would like to create my own endpoint for POST request to two related tables. I have two tables User and Userattribute.
models.py
class User(models.Model):
email = models.CharField(unique=True, max_length=180)
roles = models.JSONField(default=dict)
password = models.CharField(max_length=255, blank=True, null=True)
name = models.CharField(max_length=255, blank=True, null=True)
firebase_id = models.CharField(max_length=255, blank=True, null=True)
created_at = models.DateTimeField(default=now)
progress_sub_step = models.IntegerField(blank=True, null=True)
step_available_date = models.DateTimeField(blank=True, null=True)
progress_step = models.IntegerField(blank=True, null=True)
active = models.IntegerField(default=1)
last_login_at = models.DateTimeField(blank=True, null=True)
class Meta:
managed = False
db_table = 'user'
class Userattribute(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True, related_name = 'attribute')
attribute = models.ForeignKey(Attribute, on_delete=models.CASCADE)
The table Userattribute contains the field user which is OnetoOne to Id primary key from User table.
I tried to implement POST to two tables in serializers.py In the commented section there is a create definition which works perfectly for me. However, I wouldlike to move it to views.py as register_in_course endpoint
serializers.py
class FilmSerializer(serializers.ModelSerializer):
class Meta:
model = Film
fields = ['tytul', 'opis', 'po_premierze']
class UserattributeSerializer(serializers.ModelSerializer):
class Meta:
model = Userattribute
fields = ['user', 'attribute']
class UASerializer(serializers.ModelSerializer):
class Meta:
model = Userattribute
fields = ['attribute']
class UserSerializer(serializers.ModelSerializer):
attribute = UASerializer(many = False)
class Meta:
model = User
fields = ['email', 'name', 'firebase_id', 'attribute']
# This is what workks perfectly for me, and I want to move it to views.py
# VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV
# def create(self, validated_data):
# attribute_data = validated_data.pop('attribute')
# user = User.objects.create(**validated_data)
# Userattribute.objects.create(user=user, **attribute_data)
# return user
Current views.py:
class UserViewSet(viewsets.ModelViewSet):
queryset = User.objects.all()
serializer_class = UserSerializer
#action(detail = False, methods = ['post'])
def register_in_course(self, request, **kwargs):
data = self.get_object()
user = User.objects.create(email=request.data['email'],
name=request.data['name'],
firebase_id=request.data['firebase_id'])
user_id = User.objects.filter(firebase_id = request.data['firebase_id'])['id']
attribute = Userattribute.objects.create(user = user_id, attribute = request.data['attribute']['attribute'])
user = user.attribute.add(attribute)
serializer = UserSerializer(user, many = false)
return Response(serializer.data)
Using endpoint register_in_course to POST I get following error:
Expected view UserViewSet to be called with a URL keyword argument named "pk". Fix your URL conf, or set the .lookup_field attribute on the view correctly.
urls.py
from django.urls import include, path
from django.conf.urls import url
from rest_framework import routers
from api import views
router = routers.DefaultRouter()
router.register(r'users', views.UserViewSet)
router.register(r'userattribute', views.UserattributeViewSet)
urlpatterns = [
url('', include(router.urls))
]
i removed one line user_id variable and changed attribute variable. please check, maybe it should solve your problem, because you have already have Assigned variable as a User object..
class UserViewSet(viewsets.ModelViewSet):
queryset = User.objects.all()
serializer_class = UserSerializer
#action(detail = False, methods = ['post'])
def register_in_course(self, request, **kwargs):
data = self.get_object()
user = User.objects.create(email=request.data['email'],
name=request.data['name'],
firebase_id=request.data['firebase_id'])
attribute = Userattribute.objects.create(user = user, attribute = request.data['attribute']['attribute']) # changed this line
user = user.attribute.add(attribute)
serializer = UserSerializer(user, many = false)
return Response(serializer.data)
This issue is caused by calling get_object in a view that is defined with detail=False:
#action(detail = False, methods = ['post'])
def register_in_course(self, request, **kwargs):
data = self.get_object() # The problem is caused by this line
It seems you don't need this data, as you are using request.data.
So you can define your view like this:
#action(detail = False, methods = ['post'])
def register_in_course(self, request, **kwargs):
user = User.objects.create(
email=request.data['email'],
name=request.data['name'],
firebase_id=request.data['firebase_id']
)
Userattribute.objects.create(
user=user,
attribute = request.data.get('attribute', {}).get('attribute', {})
)
return Response(UserSerializer(user).data)

DRF select_related and prefetch_related doesn't work

I'm try to decrease query counts for using prefetch_related and select_related. However, it seems doesn't work.
in Match Model have 5 ForeignKey fields, so when i get the query counts it will return 5. Also when i delete def get_queryset method in MatchDetailAPIView. The Api still work. ( e.g 127.0.0.1:8000/game/match/match-1 is working whether or not the get_queryset method.
I can't find where I'm doing wrong.
Models.py
class Game(models.Model):
name = models.CharField(max_length=255)
...
class Match(models.Model):
name = models.TextField(blank=False, null=False)
game = models.ForeignKey(Game, on_delete=models.SET_NULL, null=True)
tournament = models.ForeignKey(Tournament, on_delete=models.SET_NULL, null=True, blank=True)
....
serializers.py
class MatchSerializer(serializers.ModelSerializer):
class Meta:
model = Match
fields = '__all__'
#exclude = ['participant', ]
views.py
class MatchDetailAPIView(RetrieveAPIView):
serializer_class = MatchSerializer
def get_queryset(self):
queryset =Match.objects.all().prefetch_related('game_id')
return queryset
def get_object(self):
gameslug = self.kwargs.get('gameslug')
slug = self.kwargs.get('slug')
# find the user
game = Game.objects.get(slug=gameslug)
return Match.objects.get(slug=slug, game__slug=game.slug)
def get_serilizer_context(self, *args, **kwargs):
return {'request': self.request}
You should use select_related for ForeignKey fields, since prefetch_related does the joining in Python, and select_related creates an SQL join.
You should also refer to the relationship by it's name, not the id of the ForeignKey ('game_id' vs. 'game'):
def get_queryset(self):
queryset =Match.objects.all().select_related('game')
return queryset
As mentioned in django docs, select_related should be used for foeign-key relationships. Also for RetrieveAPIView override the get method.
def get(self, request, *args, **kwargs):
gameslug = self.kwargs.get('gameslug')
slug = self.kwargs.get('slug')
game = Game.objects.get(slug=gameslug)
return Match.objects.filter(slug=slug, game__slug=gameslug).select_related('game').first()

JSONField serializes as json for POST, but string for GET

There is likely a very simple problem with my code, but I've been slamming my head against this problem for a couple days and can't make any headway.
Important Packages:
Django==1.11.3
django-cors-headers==2.1.0
djangorestframework==3.7.0
drf-nested-routers==0.90.0
psycopg2==2.7.3
pycparser==2.18
Here is what is happening:
I create a model via an AJAX call
My server correctly serializes the brainstorm_data field as a json object.
Now I navigate my user to the next page and fetch the current model
For some reason, brainstorm_data is now be returned as a string. Anytime I call a GET request on this resource I always get a string representation of the JSON object.
Here is the code associated:
models.py
from django.contrib.postgres.fields import JSONField
class Adventure(TimeStampedModel,
models.Model):
name = models.CharField(max_length=200)
user = models.ForeignKey(User)
world = models.ForeignKey(World)
theme = models.ForeignKey(Theme, default=1)
brainstorm_data = JSONField()
image_src = models.CharField(max_length=400, null=True, blank=True)
sentence_summary = models.TextField(null=True, blank=True)
paragraph_summary = models.TextField(null=True, blank=True)
page_summary = models.TextField(null=True, blank=True)
outline_complete = models.BooleanField(default=False)
brainstorm_complete = models.BooleanField(default=False)
private = models.BooleanField(default=False)
def __str__(self):
return self.name
views.py
class MyAdventuresViewSet(viewsets.ModelViewSet):
queryset = Adventure.objects.all()
serializer_class = AdventureSerializer
permission_classes = (permissions.IsAuthenticated,)
def get_queryset(self):
return Adventure.objects.filter(user=self.request.user)
def create(self, request, *args, **kwargs):
user = self.request.user
world = World.objects.filter(user=user).first()
if not world:
world = World.objects.create(name='My World', user=user,
description="This is a default world we created for your adventures",
image_src=static('worlds/images/world_placeholder.png'))
data = request.data.copy()
data['user'] = user.pk
data['world'] = world.pk
data['theme'] = 1 # default theme
data['brainstorm_data'] = default_brainstorm
serializer = self.get_serializer(data=data)
serializer.is_valid(raise_exception=True)
adventure = serializer.save()
Storyboard.objects.create(adventure=adventure, raw=default_storyboard['raw'], html=default_storyboard['html'])
return JsonResponse(serializer.data)
#detail_route(methods=['post'])
def complete_outline(self, request, pk):
adventure = Adventure.objects.get(pk=self.kwargs['pk'])
complete_adventure_outline(adventure)
serializer = self.get_serializer(data=adventure)
serializer.is_valid(raise_exception=True)
return JsonResponse(serializer.data, status=200)
#detail_route(methods=['post'])
def genres(self, request, pk):
genre_names = request.data
genre_models = Genre.objects.filter(name__in=genre_names)
adventure = self.get_object()
adventure.genre_set.set(genre_models)
adventure.save()
serializer = AdventureSerializer(adventure)
return JsonResponse(serializer.data)
serializers.py
class AdventureSerializer(serializers.ModelSerializer):
genre_set = serializers.StringRelatedField(many=True, read_only=True)
character_set = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
location_set = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
storyboard = serializers.PrimaryKeyRelatedField(read_only=True)
theme = serializers.PrimaryKeyRelatedField(queryset=Theme.objects.all())
class Meta:
model = Adventure
fields = '__all__'
mixins
# this is a dictionary used to default brainstorm data each time an adventure is created
default_brainstorm = {
"nodes": [...],
"edges": [...]
}
You can override the to_internal_value and to_representation in a new serializer field to handle the return data for JSON field.
class JSONSerializerField(serializers.Field):
"""Serializer for JSONField -- required to make field writable"""
def to_internal_value(self, data):
return data
def to_representation(self, value):
return value
And in turn, you would use this Field in a serializer:
class SomeSerializer(serializers.ModelSerializer):
json_field = JSONSerializerField()
class Meta:
model = SomeModelClass
fields = ('json_field', )
This should solve your problem :)
When I originally created the columns I did it with a different json field package. The base DB columns was actually text instead of json or jsonb. Creating new columns (django json fields), migrating the data, and then shifting the data back got my database back in a consistent order.

Django-rest serializer returning code instead of list of objects

I have an endpoint in my Django-rest application in which I expect to receive the following get response:
{
"my_objects": [
{
"my_object_order": 1,
"related_topics": [{"title": "my_title", "subtitle": "my_subtitle"}, {"title": "my_title2", "subtitle": "my_subtitle2"}],
"collected_at": "2016-05-02T20:52:38.989Z",
}]
}
In order to achieve that, below you can observe my serializers.py
class TopicSerializer(serializers.ModelSerializer):
class Meta:
model = MyTopic
fields = ["title", "subtitle"]
class MyObjectSerializer(serializers.ModelSerializer):
related_topics = TopicSerializer(many=True)
class Meta:
model = MyObject
fields = ("my_object_order",
"related_topics")
def create(self, validated_data):
"""
Saving serialized data
"""
related_topics_list = validated_data.pop("related_topics", [])
obj = MyObject.objects.create(**validated_data)
for topics_data in related_topics_list:
MyTopic.objects.create(trend=trend, **topics_data)
return obj
As suggested, here you can see my models.py
class MyObject(models.Model):
my_object_order = models.IntegerField()
collected_at = models.DateTimeField(auto_now=True)
def __unicode__(self):
return self.story_title
class MyTopic(models.Model):
my_obj = models.ForeignKey(MyObject, related_name="related_topics")
title = models.CharField(max_length=50, blank=False, null=True)
subtitle = models.CharField(max_length=50, blank=True, null=True)
def __unicode__(self):
return self.title
Below you have the excerpt from my views.py
def get(self, request):
params = request.QUERY_PARAMS
# Filtering data
obj_list = my_fun(MyObject, params)
response = {"my_objects": obj_list.values("my_object_order",
"collected_at",
"related_topics")}
return Response(response)
I have looked on the documentation, however I am confused/not understanding fundamentally what I should do.
Your problem is in views.py, you are not using actually the serializer at all. You are just filter some data and return whatever values you get from database (hence the ids only).
I suggest you to check Generic Class Based Views
from myapp.models import MyObject
from myapp.serializers import MyObjectSerializer
from rest_framework import generics
class MyObjectListAPIView(generics.ListAPIView):
queryset = MyObject.objects.all()
serializer_class = MyObjectSerializer
Also if you need any filtering check documentation here. Basically you can filter by fields from model with this snippet
filter_backends = (filters.DjangoFilterBackend,)
filter_fields = ('field1', 'field2')
PS: You can do the view as normal function, but you have to handle yourself filtering/serialization part, the code may not look as cleaner as you get with class based views.

Categories

Resources