Develop Reference

Django - Can I add a calculated field that only exists for a particular sub-set or occurences of my model?

Imagine that you have a model with some date-time fields that can be categorized depending on the date. You make an annotation for the model with different cases that assign a different 'status' depending on the calculation for the date-time fields:
#Models.py
class Status(models.TextChoices):
status_1 = 'status_1'
status_2 = 'status_2'
status_3 = 'status_3'
special_status = 'special_status'
class MyModel(models.Model):
important_date_1 = models.DateField(null=True)
important_date_2 = models.DateField(null=True)
calculated_status = models.CharField(max_length=32, choices=Status.choices, default=None, null=True, blank=False,)
objects = MyModelCustomManager()
And the manager with which to do the calculation as annotations:
# managers.py
class MyModelCustomManager(models.Manager):
def get_queryset(self):
queryset = super().get_queryset().annotate(**{
'status': Case(
When(**{'important_date_1' is foo, 'then':
Value(Status.status_1)}),
When(**{'important_date_2' is fii, 'then':
Value(Status.status_2)}),
When(**{'important_date_1' is foo AND 'importante_date_2' is whatever, 'then':
Value(Status.status_3)}),
# And so on and so on
)
}
)
return queryset
Now, here's where it gets tricky. Only one of these sub-sets of occurrences on the model requires an ADDITIONAL CALCULATED FIELD that literally only exists for it, that looks something like this:
special_calculated_field = F('important_date_1') - F('importante_date_2') #Only for special_status
So, basically I want to make a calculated field with the condition that the model instance must belong to this specific status. I don't want to make it an annotation, because other instances of the model would always have this value set to Null or empty if it were a field or annotation and I feel like it would be a waste of a row in the database.
Is there way, for example to do this kind of query:
>>> my_model_instance = MyModel.objects.filter(status='special_status')
>>> my_model_instance.special_calculated_field
Thanks a lot in advance if anyone can chime in with some help.

Update field with Django ORM based on computed value

I have a basic model:
class MyModel(Model):
id = models.IntegerField()
is_admin = models.BooleanField()
What I want to achieve is to update the value of the is_admin field for the entire table at once, based on whether or not the id value is in a certain list of values.
Basically, in raw SQL, this is the query I want:
UPDATE my_model
SET is_admin = (id IN (1, 2, 3, 4))
How can I achieve this with Django's ORM?
This is what I tried so far:
from django.db.models import F, Value
admin_ids = (1, 2, 3, 4)
MyModel.objects.update(is_admin=F("id") in admin_ids)
# Resulting query is:
# UPDATE my_model SET is_admin = false
MyModel.objects.update(is_admin=F("id") in Value(admin_ids))
# TypeError: argument of type 'Value' is not iterable
MyModel.objects.filter(id__in=admin_ids).update(admin=True)
MyModel.objects.exclude(id__in=admin_ids).update(admin=False)
# it works... but can I do this in a single query instead of two?
I'm using Django 3.2 and PostgreSQL.

You can use a CASE / WHEN construction.
https://docs.djangoproject.com/en/3.2/ref/models/conditional-expressions/
MyModel.objects.update(
is_admin=Case(
When(id__in=admin_ids, then=Value(True)),
default=Value(False)
)
)
P.S. If you need this kind of queries a lot, you can use the following custom expression (also useful for annotations). Though be careful, I've had it break once for one query, maybe due to the old version of Django used. On other projects, I am using this in production without any issues.
class BooleanQ(ExpressionWrapper):
output_field = BooleanField()
def __init__(self, *args, **kwargs):
expression = models.Q(*args, **kwargs)
super().__init__(expression, output_field=None)
def as_sql(self, compiler, connection):
try:
return super().as_sql(compiler, connection)
except EmptyResultSet:
return compiler.compile(Value(False))
MyModel.objects.update(is_admin=BooleanQ(id__in=admin_ids))

django - prefetch only the newest record?

I am trying to prefetch only the latest record against the parent record.
my models are as such
class LinkTargets(models.Model):
device_circuit_subnet = models.ForeignKey(DeviceCircuitSubnets, verbose_name="Device", on_delete=models.PROTECT)
interface_index = models.CharField(max_length=100, verbose_name='Interface index (SNMP)', blank=True, null=True)
get_bgp = models.BooleanField(default=False, verbose_name="get BGP Data?")
dashboard = models.BooleanField(default=False, verbose_name="Display on monitoring dashboard?")
class LinkData(models.Model):
link_target = models.ForeignKey(LinkTargets, verbose_name="Link Target", on_delete=models.PROTECT)
interface_description = models.CharField(max_length=200, verbose_name='Interface Description', blank=True, null=True)
...
The below query fails with the error
AttributeError: 'LinkData' object has no attribute '_iterable_class'
Query:
link_data = LinkTargets.objects.filter(dashboard=True) \
.prefetch_related(
Prefetch(
'linkdata_set',
queryset=LinkData.objects.all().order_by('-id')[0]
)
)
I thought about getting LinkData instead and doing a select related but ive no idea how to get only 1 record for each link_target_id
link_data = LinkData.objects.filter(link_target__dashboard=True) \
.select_related('link_target')..?
EDIT:
using rtindru's solution, the pre fetched seems to be empty. there is 6 records in there currently, atest 1 record for each of the 3 LinkTargets
>>> link_data[0]
<LinkTargets: LinkTargets object>
>>> link_data[0].linkdata_set.all()
<QuerySet []>
>>>

The reason is that Prefetch expects a Django Queryset as the queryset parameter and you are giving an instance of an object.
Change your query as follows:
link_data = LinkTargets.objects.filter(dashboard=True) \
.prefetch_related(
Prefetch(
'linkdata_set',
queryset=LinkData.objects.filter(pk=LinkData.objects.latest('id').pk)
)
)
This does have the unfortunate effect of undoing the purpose of Prefetch to a large degree.
Update
This prefetches exactly one record globally; not the latest LinkData record per LinkTarget.
To prefetch the max LinkData for each LinkTarget you should start at LinkData: you can achieve this as follows:
LinkData.objects.filter(link_target__dashboard=True).values('link_target').annotate(max_id=Max('id'))
This will return a dictionary of {link_target: 12, max_id: 3223}
You can then use this to return the right set of objects; perhaps filter LinkData based on the values of max_id.
That will look something like this:
latest_link_data_pks = LinkData.objects.filter(link_target__dashboard=True).values('link_target').annotate(max_id=Max('id')).values_list('max_id', flat=True)
link_data = LinkTargets.objects.filter(dashboard=True) \
.prefetch_related(
Prefetch(
'linkdata_set',
queryset=LinkData.objects.filter(pk__in=latest_link_data_pks)
)
)

The following works on PostgreSQL. I understand it won't help OP, but it might be useful to somebody else.
from django.db.models import Count, Prefetch
from .models import LinkTargets, LinkData
link_data_qs = LinkData.objects.order_by(
'link_target__id',
'-id',
).distinct(
'link_target__id',
)
qs = LinkTargets.objects.prefetch_related(
Prefetch(
'linkdata_set',
queryset=link_data_qs,
)
).all()

LinkData.objects.all().order_by('-id')[0] is not a queryset, it is an model object, hence your error.
You could try LinkData.objects.all().order_by('-id')[0:1] which is indeed a QuerySet, but it's not going to work. Given how prefetch_related works, the queryset argument must return a queryset that contains all the LinkData records you need (this is then further filtered, and the items in it joined up with the LinkTarget objects). This queryset only contains one item, so that's no good. (And Django will complain "Cannot filter a query once a slice has been taken" and raise an exception, as it should).
Let's back up. Essentially you are asking an aggregation/annotation question - for each LinkTarget, you want to know the most recent LinkData object, or the 'max' of an 'id' column. The easiest way is to just annotate with the id, and then do a separate query to get all the objects.
So, it would look like this (I've checked with a similar model in my project, so it should work, but the code below may have some typos):
linktargets = (LinkTargets.objects
.filter(dashboard=True)
.annotate(most_recent_linkdata_id=Max('linkdata_set__id'))
# Now, if we need them, lets collect and get the actual objects
linkdata_ids = [t.most_recent_linkdata_id for t in linktargets]
linkdata_objects = LinkData.objects.filter(id__in=linkdata_ids)
# And we can decorate the LinkTarget objects as well if we want:
linkdata_d = {l.id: l for l in linkdata_objects}
for t in linktargets:
if t.most_recent_linkdata_id is not None:
t.most_recent_linkdata = linkdata_d[t.most_recent_linkdata_id]
I have deliberately not made this into a prefetch that masks linkdata_set, because the result is that you have objects that lie to you - the linkdata_set attribute is now missing results. Do you really want to be bitten by that somewhere down the line? Best to make a new attribute that has just the thing you want.

Tricky, but it seems to work:
class ForeignKeyAsOneToOneField(models.OneToOneField):
def __init__(self, to, on_delete, to_field=None, **kwargs):
super().__init__(to, on_delete, to_field=to_field, **kwargs)
self._unique = False
class LinkData(models.Model):
# link_target = models.ForeignKey(LinkTargets, verbose_name="Link Target", on_delete=models.PROTECT)
link_target = ForeignKeyAsOneToOneField(LinkTargets, verbose_name="Link Target", on_delete=models.PROTECT, related_name='linkdata_helper')
interface_description = models.CharField(max_length=200, verbose_name='Interface Description', blank=True, null=True)
link_data = LinkTargets.objects.filter(dashboard=True) \
.prefetch_related(
Prefetch(
'linkdata_helper',
queryset=LinkData.objects.all().order_by('-id'),
'linkdata'
)
)
# Now you can access linkdata:
link_data[0].linkdata
Ofcourse with this approach you can't use linkdata_helper to get related objects.

This is not a direct answer to you question, but solves the same problem. It is possible annotate newest object with a subquery, which I think is more clear. You also don't have to do stuff like Max("id") to limit the prefetch query.
It makes use of django.db.models.functions.JSONObject (added in Django 3.2) to combine multiple fields:
MainModel.objects.annotate(
last_object=RelatedModel.objects.filter(mainmodel=OuterRef("pk"))
.order_by("-date_created")
.values(
data=JSONObject(
id="id", body="body", date_created="date_created"
)
)[:1]
)

Django object locking

I'm using Django 1.8 and I have a model
class ModelA(models.Model):
some_field = models.PositiveIntegerField()
Now, in my view I want to add a new ModelA object, but only if there are fewer than x entries for that value already.
def my_view(request):
# Using the value of 4 here just as an example
c = ModelA.objects.filter(some_field=4).count()
# Check if fewer than (x=20) objects with this field already
if c < 20:
# Fewer, so create one
new_model = ModelA(4)
new_model.save()
else:
# Return a message saying "too many"
From my understanding, there could be more than one thread running this method and so thread 1 may perform the count and there are fewer than 20 and then the other thread saves a new object, then thread 1 would save its object and there be 20 or more.
Is there some way to have the view be
def my_view(request):
get_a_lock_on_model(ModelA)
c = ModelA.objects....
# Rest of the code the same
release_lock_on_model(ModelA)
Or is there some other way I should be thinking about doing this? There are only ever inserts, never updates or deletes.
Thanks!

In order to do this you need to lock the entire table and how to do that depends on the RDBMS that you are using. It will involve the use of raw sql. An alternative approach is to do the count after you have saved your record
def my_view(request):
new_model = ModelA(4)
new_model.save()
try :
c = ModelA.objects.filter(some_field=4)[20]
if c.pk == new_model.pk:
c.delete()
# Return a message saying "too many"
except IndexError:
pass
This approach does not get in each others way, each thread is responsible for deleting the extra item that it added. Instead of deleting you can use atomic and rollback if the count is greater than 20

Tested on Django 1.10.x and postgres:
models.py:
class ModelA(models.Model):
some_field = models.PositiveIntegerField()
active = models.BooleanField()
And:
from django.db.models.expressions import RawSQL
n = 42
maximum = 3
raw_sql = RawSQL('select (select count(*) from fooapp_modela where some_field=%s) < %s', (n, maximum))
while True:
o = ModelA.objects.create(some_field=n, active=raw_sql)
o.refresh_from_db()
print(o.id, o.active)
if not o.active:
# o.delete()
break
Caveat: By default, while one transaction is active, other transactions on other connections could not "see" inserted rows until the transactions are committed. Try to avoid creating rows in a complex transactions. I believe that this means that this method is not completely bullet proof :-( More info: https://www.postgresql.org/docs/9.6/static/transaction-iso.html .
A more robust solution might include a db constraint (probably unique_together):
class ModelA(models.Model):
some_field = models.PositiveIntegerField()
ordinal = models.IntegerField()
class Meta:
unique_together = (
('some_field', 'ordinal'),
)
#...
raw_sql = RawSQL('select count(*) + 1 from fooapp_modela where some_field=%s', (n,))
o = ModelA.objects.create(some_field=n, ordinal=raw_sql) # retry a few times on IntegrityError
o.refresh_from_db()
print(o.id, o.ordinal)

How to group the choices in a Django Select widget?

Is it possible to created named choice groups in a Django select (dropdown) widget, when that widget is on a form that is auto-generated from a data Model? Can I create the widget on the left-side picture below?
My first experiment in creating a form with named groups, was done manually, like this:
class GroupMenuOrderForm(forms.Form):
food_list = [(1, 'burger'), (2, 'pizza'), (3, 'taco'),]
drink_list = [(4, 'coke'), (5, 'pepsi'), (6, 'root beer'),]
item_list = ( ('food', tuple(food_list)), ('drinks', tuple(drink_list)),)
itemsField = forms.ChoiceField(choices = tuple(item_list))
def GroupMenuOrder(request):
theForm = GroupMenuOrderForm()
return render_to_response(menu_template, {'form': theForm,})
# generates the widget in left-side picture
And it worked nicely, creating the dropdown widget on the left, with named groups.
I then created a data Model that had basically the same structure, and used Django's ability to auto-generate forms from Models. It worked - in the sense that it showed all of the options. But the options were not in named groups, and so far, I haven't figured out how to do so - if it's even possible.
I have found several questions, where the answer was, “create a form constructor and do any special processing there”. But It seems like the forms.ChoiceField requires a tuple for named groups, and I’m not sure how to convert a tuple to a QuerySet (which is probably impossible anyway, if I understand QuerySets correctly as being pointer to the data, not the actual data).
The code I used for the data Model is:
class ItemDesc(models.Model):
''' one of "food", "drink", where ID of “food” = 1, “drink” = 2 '''
desc = models.CharField(max_length=10, unique=True)
def __unicode__(self):
return self.desc
class MenuItem(models.Model):
''' one of ("burger", 1), ("pizza", 1), ("taco", 1),
("coke", 2), ("pepsi", 2), ("root beer", 2) '''
name = models.CharField(max_length=50, unique=True)
itemDesc = models.ForeignKey(ItemDesc)
def __unicode__(self):
return self.name
class PatronOrder(models.Model):
itemWanted = models.ForeignKey(MenuItem)
class ListMenuOrderForm(forms.ModelForm):
class Meta:
model = PatronOrder
def ListMenuOrder(request):
theForm = ListMenuOrderForm()
return render_to_response(menu_template, {'form': theForm,})
# generates the widget in right-side picture
I'll change the data model, if need be, but this seemed like a straightforward structure. Maybe too many ForeignKeys? Collapse the data and accept denormalization? :) Or is there some way to convert a tuple to a QuerySet, or something acceptable to a ModelChoiceField?
Update: final code, based on meshantz' answer:
class FooIterator(forms.models.ModelChoiceIterator):
def __init__(self, *args, **kwargs):
super(forms.models.ModelChoiceIterator, self).__init__(*args, **kwargs)
def __iter__(self):
yield ('food', [(1L, u'burger'), (2L, u'pizza')])
yield ('drinks', [(3L, u'coke'), (4L, u'pepsi')])
class ListMenuOrderForm(forms.ModelForm):
def __init__(self, *args, **kwargs):
super(ListMenuOrderForm, self).__init__(*args, **kwargs)
self.fields['itemWanted'].choices = FooIterator()
class Meta:
model = PatronOrder
(Of course the actual code, I'll have something pull the item data from the database.)
The biggest change from the djangosnippet he linked, appears to be that Django has incorporated some of the code, making it possible to directly assign an Iterator to choices, rather than having to override the entire class. Which is very nice.

After a quick look at the ModelChoiceField code in django.forms.models, I'd say try extending that class and override its choice property.
Set up the property to return a custom iterator, based on the orignial ModelChoiceIterator in the same module (which returns the tuple you're having trouble with) - a new GroupedModelChoiceIterator or some such.
I'm going to have to leave the figuring out of exactly how to write that iterator to you, but my guess is you just need to get it returning a tuple of tuples in a custom manner, instead of the default setup.
Happy to reply to comments, as I'm pretty sure this answer needs a little fine tuning :)
EDIT BELOW
Just had a thought and checked djangosnippets, turns out someone's done just this:
ModelChoiceField with optiongroups. It's a year old, so it might need some tweaks to work with the latest django, but it's exactly what I was thinking.

Here's what worked for me, not extending any of the current django classes:
I have a list of types of organism, given the different Kingdoms as the optgroup. In a form OrganismForm, you can select the organism from a drop-down select box, and they are ordered by the optgroup of the Kingdom, and then all of the organisms from that kingdom. Like so:
[----------------|V]
|Plantae |
| Angiosperm |
| Conifer |
|Animalia |
| Mammal |
| Amphibian |
| Marsupial |
|Fungi |
| Zygomycota |
| Ascomycota |
| Basidiomycota |
| Deuteromycota |
|... |
|________________|
models.py
from django.models import Model
class Kingdom(Model):
name = models.CharField(max_length=16)
class Organism(Model):
kingdom = models.ForeignKeyField(Kingdom)
name = models.CharField(max_length=64)
forms.py:
from models import Kingdom, Organism
class OrganismForm(forms.ModelForm):
organism = forms.ModelChoiceField(
queryset=Organism.objects.all().order_by('kingdom__name', 'name')
)
class Meta:
model = Organism
views.py:
from models import Organism, Kingdom
from forms import OrganismForm
form = OrganismForm()
form.fields['organism'].choices = list()
# Now loop the kingdoms, to get all organisms in each.
for k in Kingdom.objects.all():
# Append the tuple of OptGroup Name, Organism.
form.fields['organism'].choices = form.fields['organism'].choices.append(
(
k.name, # First tuple part is the optgroup name/label
list( # Second tuple part is a list of tuples for each option.
(o.id, o.name) for o in Organism.objects.filter(kingdom=k).order_by('name')
# Each option itself is a tuple of id and name for the label.
)
)
)

You don't need custom iterators. You're gonna need to support that code. Just pass the right choices:
from django import forms
from django.db.models import Prefetch
class ProductForm(forms.ModelForm):
class Meta:
model = Product
fields = [...]
def __init__(self, *args, **kwargs):
super(ProductForm, self).__init__(*args, **kwargs)
cats = Category.objects \
.filter(category__isnull=True) \
.order_by('order') \
.prefetch_related(Prefetch('subcategories',
queryset=Category.objects.order_by('order')))
self.fields['subcategory'].choices = \
[("", self.fields['subcategory'].empty_label)] \
+ [(c.name, [
(self.fields['subcategory'].prepare_value(sc),
self.fields['subcategory'].label_from_instance(sc))
for sc in c.subcategories.all()
]) for c in cats]
Here,
class Category(models.Model):
category = models.ForeignKey('self', null=True, on_delete=models.CASCADE,
related_name='subcategories', related_query_name='subcategory')
class Product(models.Model):
subcategory = models.ForeignKey(Category, on_delete=models.CASCADE,
related_name='products', related_query_name='product')
This same technique can be used to customize a Django admin form. Although, Meta class is not needed in this case.