Develop Reference

Django Rest Framework ordering by count of filtered pre-fetched related objects

I'm hoping someone might be able to help me. I am working with Django Rest Framework, and attempting to create an API that allows users to search for Providers that provide specific Procedures in particular Regions, and only return the relevant details.
Set up
I have these models (heavily simplified):
# models.py
from django.db import models
from django.core.validators import MinValueValidator, MaxValueValidator
class Region(models.Model):
name = models.CharField(max_length=100)
class Meta:
ordering = ["name"]
class Procedure(models.Model):
description = models.CharField(max_length=100)
class Meta:
ordering = ["description"]
class Provider(models.Model):
provider_name = models.CharField(max_length=200)
class Meta:
ordering = ["provider_name"]
class Provision(models.Model):
fk_procedure = models.ForeignKey(
Procedure,
related_name="providers",
on_delete=models.RESTRICT,
)
fk_provider = models.ForeignKey(
Provider,
related_name="services",
on_delete=models.CASCADE,
)
discount = models.FloatField(
validators=[MaxValueValidator(100), MinValueValidator(0)],
default=0,
)
class Meta:
ordering = ["-discount"]
unique_together = ["fk_procedure", "fk_provider"]
class ProvisionLinkRegion(models.Model):
fk_provision = models.ForeignKey(
Provision,
related_name="regions",
on_delete=models.CASCADE,
)
fk_region = models.ForeignKey(
Region,
related_name="services",
on_delete=models.RESTRICT,
)
location = models.BooleanField(default=False)
As you can see, there is a ManyToMany link between Provision and Region via ProvisionLinkRegion. I haven't defined this as a ManyToMany field though, as I need to store additional details (location) about the pairing.
I have defined the following serializers on these models:
# serializers.py
from rest_framework import serializers
from models import (
Provider,
Region,
Procedure,
ProvisionLinkRegion,
Provision,
)
class ProvisionLinkRegionSerializer(serializers.ModelSerializer):
class Meta:
model = ProvisionLinkRegion
fields = ["fk_region", "location"]
class ProvisionDetailsSerializer(serializers.ModelSerializer):
regions = ProvisionLinkRegionSerializer(many=True)
class Meta:
model = Provision
fields = ["fk_procedure", "discount", "mff_opt_out", "regions"]
class ProviderProvisionSerializer(serializers.ModelSerializer):
services = ProvisionDetailsSerializer(many=True)
number_services = serializers.IntegerField()
class Meta:
model = Provider
fields = [
"provider_name",
"services",
"number_services",
]
And have defined my API like this:
# api.py
from django.db.models import Prefetch, Count
from rest_framework import generics, pagination, permissions, status
from rest_framework.response import Response
from serializers import ProviderProvisionSerializer
from models import (
Provider,
ProvisionLinkRegion,
Provision,
)
class CustomPagination(pagination.PageNumberPagination):
page_size_query_param = "limit"
def get_paginated_response(self, data):
return Response(
{
"pagination": {
"previous": self.get_previous_link(),
"next": self.get_next_link(),
"count": self.page.paginator.count,
"current_page": self.page.number,
"total_pages": self.page.paginator.num_pages,
"items_on_page": len(data),
},
"results": data,
}
)
class ProvisionListAPI(generics.ListAPIView):
permission_classes = [permissions.IsAuthenticated]
serializer_class = ProviderProvisionSerializer
pagination_class = CustomPagination
def get_queryset(self):
queryset = Provider.objects.distinct()
# Extract the query parameters
params = self.request.query_params
region_list = params["region"].split(",")
procedure_list = param["procedure"].split(",")
# Build up the prefetch Provision table filtering on the regions
# and services
services_prefetch_qs = (
Provision.objects.distinct()
.filter(regions__fk_region__in=region_list)
.filter(fk_procedure__in=procedure_list)
.prefetch_related(
Prefetch(
"regions",
queryset=ProvisionLinkRegion.objects.filter(
fk_region__in=region_list
),
)
)
)
# Apply the filters and prefetch required tables
queryset = queryset.filter(
services__regions__fk_region__in=region_list
).prefetch_related(
Prefetch("services", queryset=services_prefetch_qs),
)
# Add the ordering parameters
queryset = (
queryset.annotate(
number_services=Count("services", distinct=True) # FIXME
)
.filter(number_services__gt=0)
.order_by("-number_services")
)
return queryset.all()
def list(self, response):
# Check it has the right headers
params = self.request.query_params
if "procedure" not in params:
return Response(
{"detail": "procedure not provided as a query parameter"},
status.HTTP_400_BAD_REQUEST,
)
if "region" not in params:
return Response(
{"detail": "region not provided as a query parameter"},
status.HTTP_400_BAD_REQUEST,
)
# Paginate and filter queryset
queryset = self.get_queryset()
page = self.paginate_queryset(queryset)
if page is not None:
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data, status=status.HTTP_200_OK)
My problem is, the number_services value is not correct, as it's not doing the count on the full filtered results. It's only doing it on the pre-filtered ones (although the region filter does work). I also don't want any providers to appear when they don't have any services (hence the .filter(number_services__gt=0)).
I think that it's related to not filtering on the main Provider queryset like I do with the region i.e. to include:
queryset = queryset.filter(services__fk_procedure__in=procedure_list)
But when I include this, it doesn't actually remove the services, but just the providers that don't provide ANY of those services, so the count is still off.
Example
If my data with no filtering or prefetching looks like this:
"results": [
{
"provider_name": "Provider 2.0",
"services": [
{
"fk_procedure": 3,
"discount": 0.05,
"regions": [
{
"fk_region": 1,
"location": true
},
{
"fk_region": 2,
"location": false
}
{
"fk_region": 3,
"location": true
}
]
},
{
"fk_procedure": 5,
"discount": 0.05,
"regions": [
{
"fk_region": 1,
"location": true
}
]
}
]
},
{
"provider_name": "Test Provider",
"services": [
{
"fk_procedure": 2,
"discount": 0.00,
"regions": [
{
"fk_region": 1,
"location": true
}
]
}
]
}
]
If I then run this on it:
GET /api/v1/provision?page=1&limit=10&region=1,3&services=3`
I want to show all the providers and the relevant details where they relate to either region 1 or 3, and procedure 3.
Actual Result
{
"pagination": {
"previous": null,
"next": null,
"count": 2,
"current_page": 1,
"total_pages": 1,
"items_on_page": 2
},
"results": [
{
"provider_name": "Provider 2.0",
"services": [
{
"fk_procedure": 3,
"discount": 0.05,
"regions": [
{
"fk_region": 1,
"location": true
},
{
"fk_region": 3,
"location": true
}
]
}
],
"number_services": 2
},
{
"provider_name": "Test Provider",
"services": [],
"number_services": 1
}
]
}
Desired Result
{
"pagination": {
"previous": null,
"next": null,
"count": 2,
"current_page": 1,
"total_pages": 1,
"items_on_page": 2
},
"results": [
{
"provider_name": "Provider 2.0",
"services": [
{
"fk_procedure": 3,
"discount": 0.05,
"regions": [
{
"fk_region": 1,
"location": true
},
{
"fk_region": 3,
"location": true
}
]
}
],
"number_services": 1,
}
]
}
Things I've tried
SerializerMethodField
I've been able to get number_services to work using a SerializerMethodField by including:
# serializers.py
class ProviderProvisionSerializer(serializers.ModelSerializer):
...
number_services = serializers.SerializerMethodField()
...
class Meta:
...
def get_number_services(self, obj):
return obj.services.count()
Unfortunately though, I can't use this for ordering, or filtering within the API, and I also can't use it with pagination either, so is pretty useless for what I need it for.
Subquery
In the get_queryset method for the API, I've also tried using what I currently have as a subquery, and using the main queryset where the ID is in the other one, but then I lose all the prefetch from the first subquery, and have regions and services that don't relate to my filter.
## TLDR
How do I filter a queryset in a get_queryset method for a ListAPIView on properties of the children of the main model, and be able to return a count of the remaining children after the filter has taken place?

How to get list of related objects in Django-rest-Framework

I am trying to implement Nested serializer for the first time in django-rest-framework
My requirement is i need the response data in below form :-
{
id: 0,
title: "Fresh",
data: [
[{
image: ".../assets/images/banana.jpeg",
name: "Fruits",
image_scaling: 2,
API: "http://database_api"
},
{
image: "../assets/images/banana.jpeg",
name: "Vegetables",
image_scaling: 2,
API: "http://database_api"
},
{
image: "../assets/images/banana.jpeg",
name: "Exotic Vegetables",
image_scaling: 2,
API: "http://database_api"
}
]
]
}
I am validating the token for the get api and listing the objects form DB.
My current implementation of views.py is :-
class home(APIView):
def get(self,request,format=None):
header_token = request.META.get('HTTP_AUTHORIZATION', None)
if header_token is not None:
token = header_token
access_token=token.split(' ')[1]
print(access_token)
if(validate_token(access_token) == None):
raise NotFound('user does exist')
queryset=Fresh.objects.all().select_related('data')
print(queryset)
serializer = FreshSerializer(queryset,many=True)
return Response(serializer.data)
Serializer.py is :-
class DataSerializer(serializers.ModelSerializer):
class Meta:
model=data
fields = ('name','image_scaling','image')
class FreshSerializer(serializers.ModelSerializer):
data=DataSerializer(read_only=True, many=False)
class Meta:
model = Fresh
fields = ('title','data')
models.py is :-
class data(models.Model):
name=models.CharField(max_length=9,null=True)
image_scaling=models.IntegerField(default=0,null=True)
image=models.URLField(max_length = 200)
def __str__(self):
return str(self.name)
class Fresh(models.Model):
title=models.CharField(max_length=9)
data=models.ForeignKey(data,on_delete=models.CASCADE,default=None, null=True)
def __str__(self):
return str(self.title)+" " +str(self.data)
My current output is :-
[
{
"title": "vegetable",
"data": {
"name": "vegetable",
"image_scaling": 2,
"image": "https......jpg"
}
},
{
"title": "exoticveg",
"data": {
"name": "exoticveg",
"image_scaling": 2,
"image": "https://.......jpg"
}
} ]
I was trying to do by listapiview and modelviewset, but i am not able to get the desired output.I have seen similar queries in stack-overflow, but no solution seems to work for me. I am not getting list of objects of foreign key , am i doing something wrong in my model or running wrong query? I am really stuck at it.
How shall i go about it.
Please help!!!

Django serialize POST & PUT request in a nested object

I'm using the Writable Nested Serializer to serialize my request. I had no problem serializing doing PUT/POST when the data is nested in 1 layer.
(i.e. {name:'personA', project:{ name:'projA', client:'clientA'}})
However, I ran into a problem when it is nested in 2 layers - I couldn't figure out on how to modify the update() function. Please help!
data sample
{
"id": 6106,
"name": {
"id": 213,
"name": "personA"
},
"project": {
"id": 1663,
"project": "ProjectA",
"client": {
"id": 72,
"name": "ClientA"
},
"project_manager": {
"id": 32,
"name": "personB"
}
},
"booking": 100,
"date": "2017-12-01"
}
serializers.py
class projectSerializer(serializers.ModelSerializer):
client = clientSerializer()
project_manager = userSerializer()
class Meta:
model = project
fields = ('id', 'project', 'client', 'project_manager')
class bookingListSerializer(serializers.ModelSerializer):
project = projectSerializer()
name = userSerializer()
class Meta:
model = bookingList
fields = ('id', 'name', 'project', 'booking', 'date')
def update(self, instance, validated_data):
project_data = validated_data.pop('project')
name_data = validated_data.pop('name')
try:
project_instance = project.objects.filter(**project_data)[0]
name_instance = user.objects.filter(**name_data)[0]
except IndexError:
raise serializers.ValidationError
# update the project if request is valid
instance.project = project_instance
instance.name = name_instance
instance.save()
return instance
views.py
# other viewsets...
class bookingListViewSet(viewsets.ModelViewSet):
queryset = bookingList.objects.all()
serializer_class = bookingListSerializer

Django - How to map object from another API and send in GET response

I would like to map object from another API and send in GET response. I'm going to change only id of received object. Let's assume I get data from another API in such format:
{
"id": "31242",
"name": "sth1",
"price": "44",
"data": "2017-06-07",
}
In my database I have table object1 with values:
{
"id": "123",
"name": "sth1",
},
{
"id": "124",
"name": "sth2",
},
{
"id": "125",
"name": "sth3",
}
Field name is unique both in data from API and in data from database. I receive an object named sth1. So now I would like to find it in my database and get his id, replace with id from API and send GET response. In this case my response would look in this way:
{
"id": "123",
"name": "sth1",
"price": "44",
"data": "2017-06-07",
}
At this moment this is my URL - url(r'^data/(?P<name>\w+)$', views.DataList),
but I would like to have such URL - localhost:8000/data?name=sth
Myview.py:
#api_view(['GET'])
def DataList(request, name=None):
if request.method == 'GET':
quote = getDataFromAPI().get(name)
return Response(quote)
serializers.py:
class Object1Serializer(serializers.ModelSerializer):
class Meta:
model = Object1
depth = 1
fields = '__all__'
models.py:
class Object1(models.Model):
name = models.CharField(max_length=200)
I have done it in this way:
#api_view(['GET'])
def DataList(request):
t = request.GET.get("t","")
quote = getDataFromAPI().get(t)
id = Object1.objects.get(t=t)
quote["id"] = id
return Response(quote)
But I get error:
TypeError: Object of type 'Object1' is not JSON serializable

I suppose, your view should look somewhat like this,
#api_view(['GET'])
def DataList(request):
t = request.GET.get("t","")
quote = getDataFromAPI().get(t)
id = Object1.objects.get(t=t).id #put the id of the object in the variable.
#not the object itself.
quote["id"] = id
return Response(quote)

If you want to change the url from
url(r'^data/(?P<name>\w+)$', views.DataList) to localhost:8000/data?name=sth you'd need to change your api endpoint from
#api_view(['GET'])
def DataList(request, name=None):
to
#api_view(['GET'])
def DataList(request):
name = request.GET.get("name","")
and then take the id of object from your database by querying
id = Object1.objects.get(name=name)
and then updating id in response to be sent
quote["id"] = id

auto field values in ndb.StructuredProperty

I want to store locations in google's datastore. Each entry shall have got 'sys'-fields, which shall contain information set by the datastore.
I've got the class model below and the WebService JSON request/response looks ok, but I have to set the values manually. It looks like auto_current_user_add, auto_now_add, auto_current_user and auto_now does not trigger.
from google.appengine.ext import ndb
from endpoints_proto_datastore.ndb import EndpointsModel
class Created(EndpointsModel):
by = ndb.UserProperty(auto_current_user_add=True)
on = ndb.DateTimeProperty(auto_now_add=True)
class Updated(EndpointsModel):
by = ndb.UserProperty(auto_current_user=True)
on = ndb.DateTimeProperty(auto_now=True)
class Sys(EndpointsModel):
created = ndb.StructuredProperty(Created)
updated = ndb.StructuredProperty(Updated)
class Location(EndpointsModel):
name = ndb.StringProperty(required=True)
description = ndb.TextProperty()
address = ndb.StringProperty()
sys = ndb.StructuredProperty(Sys)
When I submit a create request (location.put()) I get the following response:
{
"id": "4020001",
"name": "asdf"
}
When I set it manually using:
location.sys = Sys(created=Created(on=datetime.datetime.now(),
by=current_user),
updated=Updated(on=datetime.datetime.now(),
by=current_user))
location.put()
I get the expected result:
{
"id": "4020002",
"name": "asdf",
"sys": {
"created": {
"by": {
"auth_domain": "gmail.com",
"email": "decurgia#XYZ"
},
"on": "2015-01-27T16:05:41.465497"
},
"updated": {
"by": {
"auth_domain": "gmail.com",
"email": "decurgia#XYZ"
},
"on": "2015-01-27T16:05:41.465577"
}
}
}
How can I get those fields (sys.created.on, sys.created.by, sys.updated.on, sys.updated.by) automatically set?

In my limited work with StructuredProperty, I found it to be slower and more difficult to use than simply inserting the properties directly into the model. NDB seems to store those properties separately and perform a "join" when retrieving them. My recommendation is to use a "flat" model:
class Location(EndpointsModel):
name = ndb.StringProperty(required=True)
description = ndb.TextProperty()
address = ndb.StringProperty()
created_by = ndb.UserProperty(auto_current_user_add=True)
created_on = ndb.DateTimeProperty(auto_now_add=True)
updated_by = ndb.UserProperty(auto_current_user=True)
updated_on = ndb.DateTimeProperty(auto_now=True)
This should cause the auto_ properties to be triggered automatically.