Develop Reference

django - prefetch only the newest record?

I am trying to prefetch only the latest record against the parent record.
my models are as such
class LinkTargets(models.Model):
device_circuit_subnet = models.ForeignKey(DeviceCircuitSubnets, verbose_name="Device", on_delete=models.PROTECT)
interface_index = models.CharField(max_length=100, verbose_name='Interface index (SNMP)', blank=True, null=True)
get_bgp = models.BooleanField(default=False, verbose_name="get BGP Data?")
dashboard = models.BooleanField(default=False, verbose_name="Display on monitoring dashboard?")
class LinkData(models.Model):
link_target = models.ForeignKey(LinkTargets, verbose_name="Link Target", on_delete=models.PROTECT)
interface_description = models.CharField(max_length=200, verbose_name='Interface Description', blank=True, null=True)
...
The below query fails with the error
AttributeError: 'LinkData' object has no attribute '_iterable_class'
Query:
link_data = LinkTargets.objects.filter(dashboard=True) \
.prefetch_related(
Prefetch(
'linkdata_set',
queryset=LinkData.objects.all().order_by('-id')[0]
)
)
I thought about getting LinkData instead and doing a select related but ive no idea how to get only 1 record for each link_target_id
link_data = LinkData.objects.filter(link_target__dashboard=True) \
.select_related('link_target')..?
EDIT:
using rtindru's solution, the pre fetched seems to be empty. there is 6 records in there currently, atest 1 record for each of the 3 LinkTargets
>>> link_data[0]
<LinkTargets: LinkTargets object>
>>> link_data[0].linkdata_set.all()
<QuerySet []>
>>>

The reason is that Prefetch expects a Django Queryset as the queryset parameter and you are giving an instance of an object.
Change your query as follows:
link_data = LinkTargets.objects.filter(dashboard=True) \
.prefetch_related(
Prefetch(
'linkdata_set',
queryset=LinkData.objects.filter(pk=LinkData.objects.latest('id').pk)
)
)
This does have the unfortunate effect of undoing the purpose of Prefetch to a large degree.
Update
This prefetches exactly one record globally; not the latest LinkData record per LinkTarget.
To prefetch the max LinkData for each LinkTarget you should start at LinkData: you can achieve this as follows:
LinkData.objects.filter(link_target__dashboard=True).values('link_target').annotate(max_id=Max('id'))
This will return a dictionary of {link_target: 12, max_id: 3223}
You can then use this to return the right set of objects; perhaps filter LinkData based on the values of max_id.
That will look something like this:
latest_link_data_pks = LinkData.objects.filter(link_target__dashboard=True).values('link_target').annotate(max_id=Max('id')).values_list('max_id', flat=True)
link_data = LinkTargets.objects.filter(dashboard=True) \
.prefetch_related(
Prefetch(
'linkdata_set',
queryset=LinkData.objects.filter(pk__in=latest_link_data_pks)
)
)

The following works on PostgreSQL. I understand it won't help OP, but it might be useful to somebody else.
from django.db.models import Count, Prefetch
from .models import LinkTargets, LinkData
link_data_qs = LinkData.objects.order_by(
'link_target__id',
'-id',
).distinct(
'link_target__id',
)
qs = LinkTargets.objects.prefetch_related(
Prefetch(
'linkdata_set',
queryset=link_data_qs,
)
).all()

LinkData.objects.all().order_by('-id')[0] is not a queryset, it is an model object, hence your error.
You could try LinkData.objects.all().order_by('-id')[0:1] which is indeed a QuerySet, but it's not going to work. Given how prefetch_related works, the queryset argument must return a queryset that contains all the LinkData records you need (this is then further filtered, and the items in it joined up with the LinkTarget objects). This queryset only contains one item, so that's no good. (And Django will complain "Cannot filter a query once a slice has been taken" and raise an exception, as it should).
Let's back up. Essentially you are asking an aggregation/annotation question - for each LinkTarget, you want to know the most recent LinkData object, or the 'max' of an 'id' column. The easiest way is to just annotate with the id, and then do a separate query to get all the objects.
So, it would look like this (I've checked with a similar model in my project, so it should work, but the code below may have some typos):
linktargets = (LinkTargets.objects
.filter(dashboard=True)
.annotate(most_recent_linkdata_id=Max('linkdata_set__id'))
# Now, if we need them, lets collect and get the actual objects
linkdata_ids = [t.most_recent_linkdata_id for t in linktargets]
linkdata_objects = LinkData.objects.filter(id__in=linkdata_ids)
# And we can decorate the LinkTarget objects as well if we want:
linkdata_d = {l.id: l for l in linkdata_objects}
for t in linktargets:
if t.most_recent_linkdata_id is not None:
t.most_recent_linkdata = linkdata_d[t.most_recent_linkdata_id]
I have deliberately not made this into a prefetch that masks linkdata_set, because the result is that you have objects that lie to you - the linkdata_set attribute is now missing results. Do you really want to be bitten by that somewhere down the line? Best to make a new attribute that has just the thing you want.

Tricky, but it seems to work:
class ForeignKeyAsOneToOneField(models.OneToOneField):
def __init__(self, to, on_delete, to_field=None, **kwargs):
super().__init__(to, on_delete, to_field=to_field, **kwargs)
self._unique = False
class LinkData(models.Model):
# link_target = models.ForeignKey(LinkTargets, verbose_name="Link Target", on_delete=models.PROTECT)
link_target = ForeignKeyAsOneToOneField(LinkTargets, verbose_name="Link Target", on_delete=models.PROTECT, related_name='linkdata_helper')
interface_description = models.CharField(max_length=200, verbose_name='Interface Description', blank=True, null=True)
link_data = LinkTargets.objects.filter(dashboard=True) \
.prefetch_related(
Prefetch(
'linkdata_helper',
queryset=LinkData.objects.all().order_by('-id'),
'linkdata'
)
)
# Now you can access linkdata:
link_data[0].linkdata
Ofcourse with this approach you can't use linkdata_helper to get related objects.

This is not a direct answer to you question, but solves the same problem. It is possible annotate newest object with a subquery, which I think is more clear. You also don't have to do stuff like Max("id") to limit the prefetch query.
It makes use of django.db.models.functions.JSONObject (added in Django 3.2) to combine multiple fields:
MainModel.objects.annotate(
last_object=RelatedModel.objects.filter(mainmodel=OuterRef("pk"))
.order_by("-date_created")
.values(
data=JSONObject(
id="id", body="body", date_created="date_created"
)
)[:1]
)

Django restrict data that can be given to model field

I have the following model in django:
class Cast(TimeStampedModel):
user = models.ForeignKey(User, unique=True)
count = models.PositiveIntegerField(default=1)
kind = models.CharField(max_length = 7)
def __str__(self):
return(f"{self.kind} || {self.count} || {self.modified.strftime('%x')}")
But I want the 'kind' field to only take one of the following values: up, down, strange, charm, top, or bottom. How can I enforce this in the database or can this only be enforced in the views when taking in data?

I think choices should do?
class Cast(TimeStampedModel):
user = models.ForeignKey(User, unique=True)
count = models.PositiveIntegerField(default=1)
kind = models.CharField(
max_length=7,
choices=(
("up", "Up"),
("down", "Down"),
("strange", "Strange"),
("charm", "Charm"),
("top", "Top"),
("bottom", "Bottom")
)
)
Although in many occasions I've seen it used as a SmallInteger to save space in the database: In the DB you store a number, and in your Admin area you'll see a drop down with the "human friendly" choices.
kind = models.PositiveSmallIntegerField(
choices=(
(1, "Up"),
(2, "Down"),
(3, "Strange"),
(4, "Charm"),
(5, "Top"),
(6, "Bottom")
)
)
See:
This is not enforced at the DB level (see this ticket and this SO question) which means you still can do:
>>> c = Cast.objects.first()
>>> c.kind = 70
>>> c.save()
but it is enforced in the Admin. If you need it to be enforced in a lower level, I suggest you go with Noah Lc's answer.
As far as I understand, that is (still) not 100% enforced: You can still do bulk updates that don't go through the .save() method of the model; meaning: doing Cast.objects.all().update(kind=70) would still set an invalid value (70) in the kind field, but his solution is, indeed, one step "lower" than the Admin choices. You won't be able to do model updates that go through the .save() method of the instance. Meaning, you won't be allowed to do this:
>>> c=Cast.objects.first()
>>> c.kind=70
>>> c.save()
If you do need REAL database enforcement, you will need to actually check your databases's possibilities and add a constraint on the cast.kind column.
For instance, for Postgres (and probably for most of other SQL flavors) you could create a new migration that did this:
from django.db import migrations
def add_kind_constraint(apps, schema_editor):
table = apps.get_model('stackoverflow', 'Cast')._meta.db_table
schema_editor.execute("ALTER TABLE %s ADD CONSTRAINT check_cast_kind"
" CHECK (kind IN (1, 2, 3, 4, 5, 6) )" % table)
def remove_kind_constraint(apps, schema_editor):
table = apps.get_model('stackoverflow', 'Cast')._meta.db_table
schema_editor.execute("ALTER TABLE %s DROP CONSTRAINT check_cast_kind" % table)
class Migration(migrations.Migration):
dependencies = [
('stackoverflow', '0003_auto_20171231_0526'),
]
operations = [
migrations.RunPython(add_kind_constraint, reverse_code=remove_kind_constraint)
]
And then yeah... You'd be 100% secured (the check doesn't depend on Django: now is in the hands of your database engine):
>>> c = Cast.objects.all().update(kind=70)
django.db.utils.IntegrityError: new row for relation "stackoverflow_cast" violates check constraint "check_cast_kind"
DETAIL: Failing row contains (2, 1, 70, 1).

Do it inside the save method of your model:
def save(self, *args, **kwargs):
mylist = ['up', 'down', 'strange', 'charm',....]
if self.kind in mylist:
super(Foo, self).save(*args, **kwargs)
else:
raise Exception, "kind take only one of the following values: up, down, strange, charm,...."

Django model: Conversation between two users

I'm trying to create a Model which represents a conversation between two users (only two).
Can't figure out how to create two fields because users are equivalent.
class Conversation(models.Model):
user_one = ForeignKey('auth.User')
user_two = ForeignKey('auth.User')
class Meta:
unique_together = ('user_one','user_two')
Is this the best way I can design a model?
And then manager method:
def get_conversation(user_one,user_two):
c = Conversation.objects.filter(Q(user_one=user_one,user_two=user_two)|Q(user_one=user_one,user_two=user_two))
return c
Or is there a more comfortable way to handle such model? For example using ManyToManyField and check if there are two and only two users?:
users = ManyToManyField('auth.User')

Use the related_name field when you have more than 1 foreign key to the same model. Because you often don't care who specifically is user_one and user_two, you can simply make sure that user_one and user_two are consistent. In this case, I'm using the user's id field to say which user will be user_one and which will be user_two. This makes querying simpler because you don't need to do a query for the two pairs (A, B) and (B, A)
class Conversation(models.Model):
user_one = ForeignKey('auth.User', related_name="user_one")
user_two = ForeignKey('auth.User', related_name="user_two")
class Meta:
unique_together = ('user_one','user_two')
def clean(self):
# Ensure that user_one's id is always less than user_two's
if self.user_one and self.user_two and self.user_one.id > self.user_two.id:
(self.user_one, self.user_two) = (self.user_two, self.user_one)
#classmethod
def get(cls, userA, userB):
""" Gets all conversations between userA and userB
"""
if userA.id > userB.id:
(userA, userB) = (userB, userA)
return cls.objects.filter(user_one=userA, user_two=userB)

If you are using postgres you could use an ArrayField:
class Conversation(models.Model):
users = ArrayField(
ForeignKey('auth.User'),
size=2,
)
That would help with lookups. However note what the documentation currently says about the size parameter:
This is an optional argument. If passed, the array will have a maximum size as specified. This will be passed to the database, although PostgreSQL at present does not enforce the restriction.

Can I inspect a sqlalchemy query object to find the already joined tables?

I'm trying to programmatically build a search query, and to do so, I'm joining a table.
class User(db.Model):
id = db.Column(db.Integer(), primary_key=True)
class Tag(db.Model):
id = db.Column(db.Integer(), primary_key=True)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
title = db.Column(db.String(128))
description = db.Column(db.String(128))
This is a bit of a contrived example - I hope it makes sense.
Say my search function looks something like:
def search(title_arg, desc_arg):
query = User.query
if title_arg:
query = query.join(Tag)
query = query.filter(Tag.title.contains(title_arg))
if desc_arg:
query = query.join(Tag)
query = query.filter(Tag.description.contains(desc_arg))
return query
Previously, I’ve kept track of what tables that have already been joined in a list, and if the table is in the list, assume it’s already joined, and just add the filter.
It would be cool if I could look at the query object, see that Tag is already joined, and skip it if so. I have some more complex query building that would really benefit from this.
If there’s a completely different strategy for query building for searches that I’ve missed, that would be great too. Or, if the above code is fine if I join the table twice, that's great info as well. Any help is incredibly appreciated!!!

You can find joined tables in query._join_entities
joined_tables = [mapper.class_ for mapper in query._join_entities]

Since SQLAlchemy 1.4, the earlier proposed solutions including _join_entities don't work anymore.
SQLAlchemy 1.4
I tried to solve this in SQLAlchemy 1.4, but there is a caveat:
This approach includes all entities in the query, so not only joined entities
from sqlalchemy.sql import visitors
from contextlib import suppress
def _has_entity(self, model) -> bool:
for visitor in visitors.iterate(self.statement):
# Checking for `.join(Parent.child)` clauses
if visitor.__visit_name__ == 'binary':
for vis in visitors.iterate(visitor):
# Visitor might not have table attribute
with suppress(AttributeError):
# Verify if already present based on table name
if model.__table__.fullname == vis.table.fullname:
return True
# Checking for `.join(Child)` clauses
if visitor.__visit_name__ == 'table':
# Visitor might be of ColumnCollection or so,
# which cannot be compared to model
with suppress(TypeError):
if model == visitor.entity_namespace:
return True
# Checking for `Model.column` clauses
if visitor.__visit_name__ == "column":
with suppress(AttributeError):
if model.__table__.fullname == visitor.table.fullname:
return True
return False
def unique_join(self, model, *args, **kwargs):
"""Join if given model not yet in query"""
if not self._has_entity(model):
self = self.join(model, *args, **kwargs)
return self
Query._has_entity = _has_entity
Query.unique_join = unique_join
SQLAlchemy <= 1.3
For SQLAlchemy 1.3 and before, #mtoloo and #r-m-n had perfect answers, I've included them for the sake of completeness.
Some where in your initialization of your project, add a unique_join method to the sqlalchemy.orm.Query object like this:
def unique_join(self, *props, **kwargs):
if props[0] in [c.entity for c in self._join_entities]:
return self
return self.join(*props, **kwargs)
Now use query.unique_join instead of query.join:
Query.unique_join = unique_join

According to the r-m-n answer:
Some where in your initialization of your project, add a unique_join method to the sqlalchemy.orm.Query object like this:
def unique_join(self, *props, **kwargs):
if props[0] in [c.entity for c in self._join_entities]:
return self
return self.join(*props, **kwargs)
Query.unique_join = unique_join
Now use query.unique_join instead of query.join:
query = query.unique_join(Tag)

django query based on dynamic property()

I was wondering if there was a way to use Django's filter() on query sets using a dynamically generated python property using property(). I have first_name and last_name of every user, and I want to filter based on their concatenated name first_name last_name. (The reason behind this is that when I do autocomplete I search to see if the query matches first name, last name, or part of the concatenation. I want John S to match John Smith, for example.
I created a property of name:
def _get_name(self):
return self.first_name + " " + self.last_name
name = property(_get_name)
This way I can call user.name to get the concatenated name.
However, if I try to do User.objects.filter(name__istartswith=query) I get the error Cannot resolve keyword 'name' into field.
Any ideas on how to do this? Do I have to create another field in the database to store the full name?

The accepted answer is not entirely true.
For many cases, you can override get() in the model manager to pop dynamic properties from the keyword arguments, then add the actual attributes you want to query against into the kwargs keyword arguments dictionary. Be sure to return a super so any regular get() calls return the expected result.
I'm only pasting my own solution, but for the __startswith and other conditional queries you could add some logic to split the double-underscore and handle appropriately.
Here was my work-around to allow querying by a dynamic property:
class BorrowerManager(models.Manager):
def get(self, *args, **kwargs):
full_name = kwargs.pop('full_name', None)
# Override #1) Query by dynamic property 'full_name'
if full_name:
names = full_name_to_dict(full_name)
kwargs = dict(kwargs.items() + names.items())
return super(BorrowerManager, self).get(*args, **kwargs)
In models.py:
class Borrower(models.Model):
objects = BorrowerManager()
first_name = models.CharField(null=False, max_length=30)
middle_name = models.CharField(null=True, max_length=30)
last_name = models.CharField(null=False, max_length=30)
created = models.DateField(auto_now_add=True)
In utils.py (for the sake of context):
def full_name_to_dict(full_name):
ret = dict()
values = full_name.split(' ')
if len(values) == 1:
raise NotImplementedError("Not enough names to unpack from full_name")
elif len(values) == 2:
ret['first_name'] = values[0]
ret['middle_name'] = None
ret['last_name'] = values[1]
return ret
elif len(values) >= 3:
ret['first_name'] = values[0]
ret['middle_name'] = values[1:len(values)-1]
ret['last_name'] = values[len(values)-1]
return ret
raise NotImplementedError("Error unpacking full_name to first, middle, last names")

filter() operates on the database level (it actually writes SQL), so it won't be possible to use it for any queries based on your python code (dynamic property in your question).
This is an answer put together from many other answers in this department : )

I had a similar problem and was looking for solution. Taking for granted that a search engine would be the best option (e.g. django-haystack with Elasticsearch), that's how I would implement some code for your needs using only the Django ORM (you can replace icontains with istartswith):
from django.db.models import Value
from django.db.models.functions import Concat
queryset = User.objects.annotate(full_name=Concat('first_name', Value(' '), 'last_name')
return queryset.filter(full_name__icontains=value)
In my case I didn't know whether the user would insert 'first_name last_name' or viceversa, so I used the following code.
from django.db.models import Q, Value
from django.db.models.functions import Concat
queryset = User.objects.annotate(first_last=Concat('first_name', Value(' '), 'last_name'), last_first=Concat('last_name', Value(' '), 'first_name'))
return queryset.filter(Q(first_last__icontains=value) | Q(last_first__icontains=value))
With Django <1.8, you would probably need to resort to extra with the SQL CONCAT function, something like the following:
queryset.extra(where=['UPPER(CONCAT("auth_user"."last_name", \' \', "auth_user"."first_name")) LIKE UPPER(%s) OR UPPER(CONCAT("auth_user"."first_name", \' \', "auth_user"."last_name")) LIKE UPPER(%s)'], params=['%'+value+'%', '%'+value+'%'])

Think it's not possible in django to filter on properties that does not present as a database filed, but what you can do to make cool autocomplete search is something like this:
if ' ' in query:
query = query.split()
search_results = list(chain(User.objects.filter(first_name__icontains=query[0],last_name__icontains=query[1]),
User.objects.filter(first_name__icontains=query[1],last_name__icontains=query[0])))
else:
search_results = User.objects.filter(Q(first_name__icontains=query)| Q(last_name__icontains=query))
This code gives the user of your system a flexibility to start typing either first name or last name and the user will be thankful to you for allowing this.