How can i use django pagination on elasticsearch dsl.
My code:
query = MultiMatch(query=q, fields=['title', 'body'], fuzziness='AUTO')
s = Search(using=elastic_client, index='post').query(query).sort('-created_at')
response = s.execute()
// this always returns page count 1
paginator = Paginator(response, 100)
page = request.GET.get('page')
try:
posts = paginator.page(page)
except PageNotAnInteger:
posts = paginator.page(1)
except EmptyPage:
posts = paginator.page(paginator.num_pages)
Any solution for this?
I found this paginator on this link:
from django.core.paginator import Paginator, Page
class DSEPaginator(Paginator):
"""
Override Django's built-in Paginator class to take in a count/total number of items;
Elasticsearch provides the total as a part of the query results, so we can minimize hits.
"""
def __init__(self, *args, **kwargs):
super(DSEPaginator, self).__init__(*args, **kwargs)
self._count = self.object_list.hits.total
def page(self, number):
# this is overridden to prevent any slicing of the object_list - Elasticsearch has
# returned the sliced data already.
number = self.validate_number(number)
return Page(self.object_list, number, self)
and then in view i use:
q = request.GET.get('q', None)
page = int(request.GET.get('page', '1'))
start = (page-1) * 10
end = start + 10
query = MultiMatch(query=q, fields=['title', 'body'], fuzziness='AUTO')
s = Search(using=elastic_client, index='post').query(query)[start:end]
response = s.execute()
paginator = DSEPaginator(response, settings.POSTS_PER_PAGE)
try:
posts = paginator.page(page)
except PageNotAnInteger:
posts = paginator.page(1)
except EmptyPage:
posts = paginator.page(paginator.num_pages)
this way it works perfectly..
Following the advice from Danielle Madeley, I also created a proxy to search results which works well with the latest version of django-elasticsearch-dsl==0.4.4.
from django.utils.functional import LazyObject
class SearchResults(LazyObject):
def __init__(self, search_object):
self._wrapped = search_object
def __len__(self):
return self._wrapped.count()
def __getitem__(self, index):
search_results = self._wrapped[index]
if isinstance(index, slice):
search_results = list(search_results)
return search_results
Then you can use it in your search view like this:
paginate_by = 20
search = MyModelDocument.search()
# ... do some filtering ...
search_results = SearchResults(search)
paginator = Paginator(search_results, paginate_by)
page_number = request.GET.get("page")
try:
page = paginator.page(page_number)
except PageNotAnInteger:
# If page parameter is not an integer, show first page.
page = paginator.page(1)
except EmptyPage:
# If page parameter is out of range, show last existing page.
page = paginator.page(paginator.num_pages)
Django's LazyObject proxies all attributes and methods from the object assigned to the _wrapped attribute. I am overriding a couple of methods that are required by Django's paginator, but don't work out of the box with the Search() instances.
A very simple solution is to use MultipleObjectMixin and extract your Elastic results in get_queryset() by overriding it. In this case Django will take care of the pagination itself if you add the paginate_by attribute.
It should look like that:
class MyView(MultipleObjectMixin, ListView):
paginate_by = 10
def get_queryset(self):
object_list = []
""" Query Elastic here and return the response data in `object_list`.
If you wish to add filters when querying Elastic,
you can use self.request.GET params here. """
return object_list
Note: The code above is broad and different from my own case so I can not guarantee it works. I used similar solution by inheriting other Mixins, overriding get_queryset() and taking advantage of Django's built in pagination - it worked great for me. As it was an easy fix I decided to post it here with a similar example.
Another way forward is to create a proxy between the Paginator and the Elasticsearch query. Paginator requires two things, __len__ (or count) and __getitem__ (that takes a slice). A rough version of the proxy works like this:
class ResultsProxy(object):
"""
A proxy object for returning Elasticsearch results that is able to be
passed to a Paginator.
"""
def __init__(self, es, index=None, body=None):
self.es = es
self.index = index
self.body = body
def __len__(self):
result = self.es.count(index=self.index,
body=self.body)
return result['count']
def __getitem__(self, item):
assert isinstance(item, slice)
results = self.es.search(
index=self.index,
body=self.body,
from_=item.start,
size=item.stop - item.start,
)
return results['hits']['hits']
A proxy instance can be passed to Paginator and will make requests to ES as needed.
Related
I'm building a webpage using wagtail CMS, Django, Postgresql and at the bottom of the page, I'm building a section where I would display the videos using pagination.
I'm trying to retrieve the data from Django Models using all_posts = MultiBlogPage.objects.values("all_blogs_content_pages")
and I'm getting the output as
<PageQuerySet [{'all_blogs_content_pages':
[
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ee2d21d0>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ed8fa2e8>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35f617e6a0>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ed90aac8>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ed90af60>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ed90a978>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ed90ae48>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ee2b9320>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ee2b9630>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ee339f28>,
<wagtail.core.blocks.stream_block.StreamValue.StreamChild object at 0x7f35ee339470>
]}]>
Could someone please look at my code below and let me know how to get the exact values from Django Models ?
def get_context(self, request, *args, **kwargs):
context = super(MultiBlogPage, self).get_context(request, *args, **kwargs)
context['multiblog_page'] = self
// The name of the stream field panel is "all_blogs_content_pages"
all_posts = MultiBlogPage.objects.values("all_blogs_content_pages")
print("all_posts...",all_posts)
paginator = Paginator(all_posts, 3)
print("paginator", paginator)
page = request.GET.get("page")
try:
posts = paginator.page(page)
except PageNotAnInteger:
posts = paginator.page(1)
except EmptyPage:
posts = paginator.page(paginator.num_pages)
context["posts"] = posts
return context
Wagtail models can be used like standard django models, so converting the models to a list of dicts does not seem necessary.
live(), filters only published versions, and specific() gets the actual model from your models.py rather than the base Page class.
def get_context(self, request, *args, **kwargs):
context = super(MultiBlogPage, self).get_context(request, *args, **kwargs)
context['multiblog_page'] = self
// The name of the stream field panel is "all_blogs_content_pages"
all_posts = MultiBlogPage.objects.live().specific()
print("all_posts...",all_posts)
paginator = Paginator(all_posts, 3)
print("paginator", paginator)
page = request.GET.get("page")
try:
posts = paginator.page(page)
except PageNotAnInteger:
posts = paginator.page(1)
except EmptyPage:
posts = paginator.page(paginator.num_pages)
context["posts"] = posts
return context
template.html
<li>
{{ post.all_blogs_content_pages }}
</li>
I'm a beginner at Django. Recently, I started writing a web app for inventory management and I realised that when i was writing the views, there were a lot of them with similar structures. For instance:
def invoices(request):
"""The page for displaying invoices."""
invoice_list = Document.objects.filter(type_name__name='Invoice')
page = request.GET.get('page', 1)
paginator = Paginator(invoice_list, 10)
try:
invoices = paginator.page(page)
except PageNotAnInteger:
invoices = paginator.page(1)
except EmptyPage:
invoices = paginator.page(paginator.num_pages)
context = {'invoices':invoices}
return render(request, 'imsapp/invoices.html', context)
and this one:
def credit_notes(request):
"""The page for displaying credit notes."""
credit_notes_list = Document.objects.filter(type_name__name='Credit Note')
page = request.GET.get('page', 1)
paginator = Paginator(credit_notes_list, 10)
try:
credit_notes = paginator.page(page)
except PageNotAnInteger:
credit_notes = paginator.page(1)
except EmptyPage:
credit_notes = paginator.page(paginator.num_pages)
context = {'credit_notes':credit_notes}
return render(request, 'imsapp/credit_notes.html', context)
So, I'm just thinking if there is a more elegant way to represent the above function definitions. Is Class-based view what I'm looking for?
You could refactor the logic such that a function handles the common logic or you can use a class based view.
def paginate(request, queryset):
page_index = request.GET.get('page', 1)
paginator = Paginator(queryset, 10)
try:
page = paginator.page(page_index)
except PageNotAnInteger:
page = paginator.page(1)
except EmptyPage:
page = paginator.page(paginator.num_pages)
return page
def credit_notes(request):
"""The page for displaying credit notes."""
credit_notes_list = Document.objects.filter(type_name__name='Credit Note')
credit_notes = paginate(request, credit_notes_list)
context = {'credit_notes':credit_notes}
return render(request, 'imsapp/credit_notes.html', context)
Or you can use ListView
from django.views.generic.list import ListView
class CreditNoteListView(ListView):
queryset = Document.objects.filter(type_name__name='Credit Note')
paginate_by = 10
template_name = 'imsapp/credit_notes.html'
You'll need to change your template in that case though since the template context will be slightly different.
Views are just Python functions - and as such, tehy can call further functions.
In the above code, it looks like the only things that change are the type object name and the template.
Simply create a new function that takes these two as parameters
def paged_view(request, type_name, template, name_in_context):
"""The page for displaying various items in a paginated way."""
item_list = Document.objects.filter(type_name__name=time_name)
page = request.GET.get('page', 1)
paginator = Paginator(item_list, 10)
try:
items = paginator.page(page)
except PageNotAnInteger:
items = paginator.page(1)
except EmptyPage:
items = paginator.page(paginator.num_pages)
context = {name_in_context:items}
return render(request, template, context)
def invoices(request):
"""The page for displaying invoices."""
return paged_view(requests, 'imsapp/invoices.html', 'Invoice', 'invoices')
def credit_notes(request):
"""The page for displaying credit notes."""
return paged_view(requests, 'imsapp/credit_notes.html', 'Credit Note', 'credit_notes')
views.py
from django.core.paginator import Paginator
def index(request):
posts_list = Post.objects.all().order_by('-id')
paginator = Paginator(posts_list, 5)
try:
page = int(request.GET.get('page', '1'))
except:
page = 1
try:
posts = paginator.page(page)
except(EmptyPage, InvalidPage):
posts = paginator.page(paginator.num_pages)
return render_to_response('home/index.html',
{ 'posts' : posts },
context_instance=RequestContext(request))
Well this is a mix between Python's get method features and GET of Django,
Basically, since GET is a A dictionary-like object containing all given HTTP GET parameters, what you are trying to achieve here is find the value for the given key 'page'. If it doesn't exist it will fallback to the default value 1 which is what get intends to do.
In simple way... you are using get() method that will check if the element that you want exists if not return None (null), so you are searching for GET (HTTP) Paramter if "page" exists, if not exists it return 1
mypage.com/?page=2
request.GET['page'] # That will force get page param, and you will if not found
request.GET.get('page', '1') # Tha will check if param exists, and return 1 if not found
Using GET.get() is a bad pratice, since your erros will silent fail, better your GET['page'] and handle the errors using try/except
try:
page = request.GET['page']
...
except Exception as e:
print(e) # handle your errors
page = 1 # The Default value when erros comes
...
I need to manually set count of page for pagination in view.py
I need to get number of current page for processing it in function.
I next view.py:
class VideoListView(ListView):
template_name = "video/list.html"
context_object_name = 'videos'
paginate_by = 12
request = requests.get(settings.YOUTUBE_VIDEO_COUNT_URL)
count = simplejson.loads(request.text)['data']['totalItems']
def get_queryset(self, **kwargs):
request = requests.get(settings.YOUTUBE_VIDEO_URL)
data_about = simplejson.loads(request.text)
video_list = []
for item in data_about['data']['items']:
video_list.append(item)
return video_list
Count of pages must be: count/paginate_by, and on every page request json will be different.
The Django pagination is stored via GET, so in your ListView you need to access:
# This will assume, if no page selected, it is on the first page
actual_page = request.GET.get('page', 1)
if actual_page:
print actual_page
so in your list view code, depends on where you need it, but if you need it in your get_queryset function, you can access the request using self:
class VideoListView(ListView):
# ..... your fields
def get_queryset(self, **kwargs):
# ... Your code ...
actual_page = self.request.GET.get('page', 1)
if actual_page:
print actual_page
# ... Your code ...
Custom pagination object using Django Pagination:
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
def CreatePagination(request, obj_list):
# Create the pagination
RESULTS_PER_PAGE = 10
paginator = Paginator(obj_list, RESULTS_PER_PAGE)
page = request.GET.get('page') # Actual page
try:
page_list = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
page_list = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
page_list = paginator.page(paginator.num_pages)
return page_list
To use this CreatePagination function you need to pass it the request and the object list. The request is used to get the actual page, and the object list is used to generate the pagination.
This function will return a pagination that you can manage in the template the same way you're managing the automatic generated pagination from the ListView
I practice how to use django to write a website
I finished one with model name = Traveltime
But I still have many to do (like: aaaaa,bbbbb,ccccc below)
They are do similar job ,just the model name is different
I feel it duplicate and don't know how to do.
how can I edit my urls.py?? Please help me thank you!
urls.py:
urlpatterns = patterns('',
url(r'^travel/$', views.object_list, {'model': models.Traveltime}),
url(r'^travel/result/$', views.object_result, {'model': models.Traveltime}),
url(r'^travel/update/$', views.update),
#I have many urls to set (below)
url(r'^aaaaa/$', views.object_list, {'model': models.aaaaa}),
url(r'^aaaaa/result/$', views.object_result, {'model': models.aaaaa}),
url(r'^aaaaa/update/$', views.update),
url(r'^bbbbb/$', views.object_list, {'model': models.bbbbb}),
url(r'^bbbbb/result/$', views.object_result, {'model': models.bbbbb}),
url(r'^bbbbb/update/$', views.update),
url(r'^ccccc/$', views.object_list, {'model': models.ccccc}),
url(r'^ccccc/result/$', views.object_result, {'model': models.ccccc}),
url(r'^ccccc/update/$', views.ccccc),
views.py
def object_list(request, model):
obj_list = model.objects.filter(image_elect='')
paginator = Paginator(obj_list, 10)
page = request.GET.get('page')
try:
contacts = paginator.page(page)
except PageNotAnInteger:
contacts = paginator.page(1)
except EmptyPage:
contacts = paginator.page(paginator.num_pages)
template_name = 'filterimgs/%s_list.html' % model.__name__.lower()
return render_to_response(template_name, {'object_list': obj_list,"contacts": contacts},
context_instance=RequestContext(request))
def update(request):
travel = Traveltime.objects.filter(title=request.POST['title'])
# travel.update(image_elect='asd')
return redirect(object_result)
def object_result(request, model):
obj_list = model.objects.all()
paginator = Paginator(obj_list, 10)
page = request.GET.get('page')
try:
contacts = paginator.page(page)
except PageNotAnInteger:
contacts = paginator.page(1)
except EmptyPage:
contacts = paginator.page(paginator.num_pages)
template_name = 'filterimgs/%s_result.html' % model.__name__.lower()
return render_to_response(template_name, {'object_list': obj_list,"contacts": contacts},
context_instance=RequestContext(request))
Django url patterns are regular expressions, any grouped expression will be passed into the appropriate view as an additional parameter.
urlpatterns = patterns('',
url(r'^([a-zA-Z]+)/$', views.object_list),
url(r'^([a-zA-Z]+)/result/$', views.object_result),
url(r'^([a-zA-Z]+)/update/$', views.update),
)
Then in your request you can
import importlib
def object_list(request, model_type):
# use the model_type which is passed in from
# the URL to actually grab the proper model
found = importlib.import_module('models.{0}'.format(model_type))
Python is gracious enough to allow you to import modules using a string .
Also, found is a lame name, so you should name it accordingly depending on what it's purpose is.