I have a problem where deleting an object form the admin won't delete the file associated with it. after some research I decided to implement a post_delete in the model.
For some reason I am not able to make the s3 delete the file, even after searching numerous guides and snippets, maybe someone here knows.
I use django 1.5 and boto.
Heres my code for the model:
from django.db import models
from django.contrib.auth.models import User
from fileservice.formatChecker import ContentTypeRestrictedFileField
from south.modelsinspector import add_introspection_rules
import os
from django.dispatch import receiver
from django.utils.translation import ugettext_lazy as _
from django.core.files.storage import default_storage as storage
add_introspection_rules([
(
[ContentTypeRestrictedFileField], # Class(es) these apply to
[], # Positional arguments (not used)
{ # Keyword argument
"content_types": ["content_types", {}],
"max_upload_size": ["max_upload_size", {}]
},
),
], ["^fileservice\.formatChecker\.ContentTypeRestrictedFileField"])
class Contentfile(models.Model):
content = ContentTypeRestrictedFileField(upload_to='uploads/', content_types=['video/mp4', 'application/pdf', 'image/gif', 'image/jpeg', 'image/png'],max_upload_size=5242880,blank=True, null=True, help_text='Upload a file to add it to the content the app displayes')
created_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
title = models.CharField(max_length=255, unique=True)
file_type = models.CharField(max_length=5)
published = models.BooleanField(default=True)
file_owner = models.ForeignKey(User, related_name='Contentfiles')
class Meta:
ordering = ["title"]
def __unicode__(self):
return self.title
def save(self, *args, **kwargs):
file_name = os.path.basename(self.content.name)
self.file_type = file_name.split('.')[-1]
self.title = file_name.split('.')[0]
self.published = True
super(Contentfile, self).save(*args, **kwargs)
#receiver(models.signals.post_delete, sender=Contentfile)
def auto_delete_file_on_delete(sender, instance, **kwargs):
"""Deletes file from filesystem
when corresponding `MediaFile` object is deleted.
"""
if instance.content:
if os.path.isfile(storage.open(instance.content.path)):
os.remove(storage.open(instance.content.path))
#receiver(models.signals.pre_save, sender=Contentfile)
def auto_delete_file_on_change(sender, instance, **kwargs):
"""Deletes file from filesystem
when corresponding `MediaFile` object is changed.
"""
if not instance.pk:
return False
try:
old_file = Contentfile.objects.get(pk=instance.pk).content
except Conentfile.DoesNotExist:
return False
new_file = instance.content
if not old_file == new_file:
if os.path.isfile(storage.open(old_file.path)):
os.remove(storage.open(old_file.path))
It is MUCH safer to do post_delete. If something goes wrong you will start missing S3 files and you wont notice it because your DB records are intact. post_delete will be safer since it is less likely that S3 delete operation would fail after you have deleted your db record. Furthermore even if file delete fails you will be left with a bunch of unreferenced S3 file which are harmless and can be easily cleaned up.
#receiver(models.signals.post_delete, sender=Picture)
def remove_file_from_s3(sender, instance, using, **kwargs):
instance.img.delete(save=False)
You need to call FieldFile's delete() method to remove the file in S3. In your case, add a pre_delete signal where you call it:
#receiver(models.signals.pre_delete, sender=ContentFile)
def remove_file_from_s3(sender, instance, using):
instance.content.delete(save=False)
Try django-cleanup, it automatically invokes delete method on FileField when you remove model.
This worked for me by deleting files both in DB and in AWS S3.
from django.db import models
from django.dispatch import receiver
from django.views import generic
from project.models import ContentFile
from django.contrib.auth.mixins import LoginRequiredMixin,UserPassesTestMixin
class DeleteFileView(LoginRequiredMixin,UserPassesTestMixin,generic.DeleteView):
model = ContentFile
template_name = 'file-delete.html'
success_url = 'redirect-to'
#Check if the owner of the file is the one trying to delete a file
def test_func(self):
obj = self.get_object()
if obj.user == self.request.user:
return True
return False
#This code does magic for S3 file deletion
#receiver(models.signals.pre_delete, sender=ContentFile)
def remove_file_from_s3(sender, instance, using, **kwargs):
instance.image_file.delete(save=False)
I am using pre_delete you can check the django documentation.File reference deletion in DB is done by DeleteView, I hope this helps someone
Related
In my django project I am trying to migrate my database from sqlite to postgres. I've made a dump file of the database, changed my settings file to reflect the new database and now I want to upload this data to postgres. So initially I was running the following in the terminal:
python manage.py loaddata data.json
But this took very long - it was still running for more than a day, with no errors, and I eventually I stopped it.
So then I found this answer on SO, and therefore I created a load_data.py file, like so:
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
import django
django.setup()
from django.core.management import call_command
from django.db import transaction
DJANGO_SETTINGS_MODULE='mysite.settings'
with transaction.atomic():
call_command('loaddata', 'data.json')
And then ran it via python load_data.py. But it still takes incredibly long and the data has not loaded. My data file is 20.8MB. Is there anything that I can do to speed things up?
For reference, the model that has the most data looks like this:
class WordManager(models.Manager):
def search(self, query=None):
qs = self.get_queryset()
if query is not None:
or_lookup = (Q(target_word__icontains=query) |
Q(source_word__icontains=query)|
Q(example_sentence__icontains=query)
)
qs = qs.filter(or_lookup).distinct() # distinct() is often necessary with Q lookups
return qs
class Word(models.Model):
objects = WordManager()
target_word = models.CharField('Word in Russian', max_length=35,help_text="The word you want to add, in Russian")
source_word = models.CharField('What does it mean?',max_length=35, help_text="Write down the translation in any language", blank=False, null=True)
add_to_word_list = models.BooleanField('Would you like to create a flashcard?', default=True)
deck_name = models.CharField('Deck name', max_length=25, default="My Words")
category = models.CharField(max_length=25,blank=True,help_text="Create a custom category for this word, e.g. Food")
example_sentence = models.CharField(max_length=150,blank=True,help_text="It's important to learn words in context!")
user = models.ForeignKey(User, related_name="words",on_delete=models.CASCADE)
created_at = models.DateTimeField(auto_now=True)
def __str__(self):
return self.target_word
def get_absolute_url(self):
return reverse("vocab:detail",
kwargs={"username": self.user.username, "pk": self.pk})
class Meta:
ordering = ["target_word"]
## constrain user to add only one instance of target word
constraints = [
models.UniqueConstraint(fields=['user','target_word', 'source_word'],name='unique_word')]
from django.db.models.signals import post_save
from django.dispatch import receiver
from api.models import Flashcard, Deck
#receiver(post_save, sender=Word)
def create_deck(sender, created, instance, **kwargs):
if created:
instance._meta.model.objects.all()
deck_name = instance.deck_name
user = instance.user
flash_wanted = Word.objects.filter(add_to_word_list=True)
if flash_wanted:
deck_name, created = Deck.objects.get_or_create(owner=user, name=deck_name)
post_save.connect(create_deck, sender=Word)
#receiver(post_save, sender=Word)
def flash_from_word(sender, created, instance, **kwargs):
if created:
flash_wanted = Word.objects.filter(add_to_word_list=True)
instance._meta.model.objects.all()
target_word = instance.target_word
source_word = instance.source_word
deck_name = instance.deck_name
user = instance.user
if flash_wanted:
flash = Flashcard.objects.create(owner=user, question=target_word,answer=source_word,deck=Deck.objects.get(name=deck_name,owner=user))
post_save.connect(flash_from_word, sender=Word)
So I want to create a folder automatically for each user, Here is my models.py:
models.py
#Creating a folder automatically for each user
def folder_path(instance, filename):
return "user_{0}/MyFolder/{1}".format(instance.user.id, filename)
# Create your models here.
class SLRModel(models.Model):
user = models.ForeignKey(User, on_delete= models.CASCADE, default= None)
my_folder = models.FileField(upload_to= folder_path, default= None)
Answer 1:
Option 1
from_the_docs: you can go for Custom User Model. It goes something like this:
class MyUser(AbstractUser):
USERNAME_FIELD = 'username'
weights_dir = models.CharField('Weights Directory')
def save(self, *args, **kwargs):
if self.weights_dir is None:
self.weights_dir = "user_{0}/MyFolder/{1}".format(self.id, self.username)
# add creating directory code here
super(AbstractUser, self).save(*args, **kwargs)
and in settings.py set
AUTH_USER_MODEL = 'yourappname.MyUser'
you will also need to updated the admin.py file:
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from .models import User
admin.site.register(User, UserAdmin)
Don't forget to migrate after these changes.
Option 2
If you are not comfortable impleminting a Custom user model and want to go with default one you can using post_save signal, from_the_docs. Something like:
from django.contrib.auth.models import User
from django.db.models.signals import post_save
from django.dispatch import receiver
#receiver(post_save, sender=User)
def create_directory(sender, instance, created, **kwargs):
if created:
pass # add create directory code here
Answer 2:
In views.py you can do something like this:
from django.http import HttpResponse
def home(request):
# add wigths save code here
return HttpResponse("Weights Created.")
#Creating a folder automatically for each user
def folder_path(instance, filename):
user = instance.user.username
basename, file_extension = filename.split(".")
new_filename = "%s-%s.%s" %(user, instance.id, file_extension)
return "MyFolder/%s/%s" %(user, new_filename)
# Create your models here.
class SLRModel(models.Model):
user = models.ForeignKey(User, on_delete= models.CASCADE, default= None)
my_folder = models.FileField(upload_to= folder_path, default= None)
instance.id is returning None when upload images through the admin page. The idea was to upload all the images of each Residence to a different folder. Here's my code:
models.py
from django.db import models
import os
def get_image_path(instance, filename):
return os.path.join('photos', "residence_%s" % instance.id, filename)
# Create your models here.
class Residence(models.Model):
big_image = models.ImageField("Main Image",upload_to=get_image_path)
small_images = models.ImageField("Small Images",upload_to=get_image_path, blank=True, null=True)
settings.py
MEDIA_URL = '/media/'
EDIT: It works if I modify the image after the model is already added.
You can't do it in that way unless you implement your custom dynamic file upload field. Because you try to access instance.id, but instance isn't saved yet and doesn't have an id.
Here you are some resources that will help you achieve what you want:
dynamic file upload path with current instance id
Django admin file upload with current model id
Another nice way to solve this problem that requires much less code is to have your model use a UUID for the primary key rather then the database generated id. This means at the point the model is saved for the first time the UUID is already known and can be used with any upload_to callbacks.
So for the original example you would do something like this
from django.db import models
import uuid
import os
def get_image_path(instance, filename):
return os.path.join('photos', "residence_%s" % str(instance.id), filename)
# Create your models here.
class Residence(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
big_image = models.ImageField("Main Image",upload_to=get_image_path)
small_images = models.ImageField("Small Images",upload_to=get_image_path, blank=True, null=True)
See Django's UUIDField reference for more info
If you don't specify an update_to on the ImageField you could upload to your media root then change the path using a post_save signal.
#receiver(post_save, sender=Product)
def update_file_path(instance, created, **kwargs):
if created:
initial_path = instance.image.path
new_path = settings.MEDIA_ROOT + f'/product_{instance.id}/{instance.image.name}'
os.makedirs(os.path.dirname(new_path), exist_ok=True)
os.rename(initial_path, new_path)
instance.image = new_path
instance.save()
you can create a model instance by passing cleaned data from your form as **kwargs to django model i did it that way & its much easier than anything else
in your views post method add this (this code is from my project not adapted to this question)
pk = request.session['_auth_user_id']
user_obj = User.objects.get(pk=pk)
lab_form_instance = lab_form(request.POST,request.FILES)
lab_form_instance.save(commit=False)
# here you can put the form.is_valid() statement
lab_form_instance.cleaned_data['owner'] =user_obj # here iam adding additional needed data for the model
obj = lab(**lab_form_instance.cleaned_data)
obj.save()
I have two apps in Django where one app's model (ScopeItem) on its instance creation must create an instance of the other app's model as well (Workflow); i.e. the ScopeItem contains it's workflow.
This works nicely when tried from the shell. Creating a new ScopeItem creates a Workflow and stores it in the ScopeItem. In admin I get an error, that the workflow attribute is required. The attribute is not filled in and the model definition requires it to be set. The overwritten save method though does this. Hence my question is, how to call save before the check in admin happens?
If I pick an existing Workflow instance in admin and save (successfully then), then I can see that my save method is called later and a new Workflow is created and attached to the ScopeItem instance. It is just called too late.
I am aware that I could allow empty workflow attributes in a ScopeItem or merge the ScopeItem and the Workflow class to avoid the issue with admin. Both would cause trouble later though and I like to avoid such hacks.
Also I do not want to duplicate code in save_item. Just calling save from there apparently does not cut it.
Here is the code from scopeitems/models.py:
class ScopeItem(models.Model):
title = models.CharField(max_length=64)
description = models.CharField(max_length=4000, null=True)
workflow = models.ForeignKey(Workflow)
def save(self, *args, **kwargs):
if not self.id:
workflow = Workflow(
description='ScopeItem %s workflow' % self.title,
status=Workflow.PENDING)
workflow.save()
self.workflow = workflow
super(ScopeItem, self).save(*args, **kwargs)
And workflow/models.py:
from django.utils.timezone import now
class Workflow(models.Model):
PENDING = 0
APPROVED = 1
CANCELLED = 2
STATUS_CHOICES = (
(PENDING, 'Pending'),
(APPROVED, 'Done'),
(CANCELLED, 'Cancelled'),
)
description = models.CharField(max_length=4000)
status = models.IntegerField(choices=STATUS_CHOICES)
approval_date = models.DateTimeField('date approved', null=True)
creation_date = models.DateTimeField('date created')
update_date = models.DateTimeField('date updated')
def save(self, *args, **kwargs):
if not self.id:
self.creation_date = now()
self.update_date = now()
super(Workflow, self).save(*args, **kwargs)
In scopeitems/admin.py I have:
from django.contrib import admin
from .models import ScopeItem
from workflow.models import Workflow
class ScopeItemAdmin(admin.ModelAdmin):
list_display = ('title', 'description', 'status')
list_filter = ('workflow__status', )
search_fields = ['title', 'description']
def save_model(self, request, obj, form, change):
obj.save()
def status(self, obj):
return Workflow.STATUS_CHOICES[obj.workflow.status][1]
admin.site.register(ScopeItem, ScopeItemAdmin)
You could set the field blank=True on workflow.
You said you don't want to allow "empty workflow attributes in a ScopeItem." Setting blank=True is purely validation-related. Thus, on the backend workflow will still be NOT NULL. From the Django docs:
If a field has blank=True, form validation will allow entry of an empty value.
Referring to your example you should be able to use:
workflow = models.ForeignKey(Workflow, blank=True)
You need to exclude the field from the form used in the admin, so that it won't be validated.
class ScopeItemForm(forms.ModelForm):
class Meta:
exclude = ('workflow',)
model = ScopeItem
class ScopeItemAdmin(admin.ModelAdmin):
form = ScopeItemForm
...
admin.site.register(ScopeItem, ScopeItemAdmin)
#Daniel Roseman's answer is correct as long as you don't need to edit the workflow field in admin at any time. If you do need to edit it then you'll need to write a custom clean() method on the admin form.
forms.py
class ScopeItemAdminForm(forms.ModelForm):
class Meta:
model = ScopeItem
def clean(self):
cleaned_data = super(ScopeItemAdminForm, self).clean()
if 'pk' not in self.instance:
workflow = Workflow(
description='ScopeItem %s workflow' % self.title,
status=Workflow.PENDING)
workflow.save()
self.workflow = workflow
return cleaned_data
admin.py
class ScopeItemAdmin(admin.ModelAdmin):
form = ScopeItemAdminForm
...
admin.site.register(ScopeItem, ScopeItemAdmin)
Answering my own question:
As #pcoronel suggested, the workflow attribute in ScopeItem must have blank=True set to get out of the form in the first place.
Overwriting the form's clean method as suggested by #hellsgate was also needed to create and store the new Workflow.
To prevent code duplication I added a function to workflow/models.py:
def create_workflow(title="N/A"):
workflow = Workflow(
description='ScopeItem %s workflow' % title,
status=Workflow.PENDING)
workflow.save()
return workflow
This makes the ScopeItemAdminForm look like this:
class ScopeItemAdminForm(forms.ModelForm):
class Meta:
model = ScopeItem
def clean(self):
cleaned_data = super(ScopeItemAdminForm, self).clean()
cleaned_data['workflow'] = create_workflow(cleaned_data['title'])
return cleaned_data
Additionally I changed the save method in scopeitems/models.py to:
def save(self, *args, **kwargs):
if not self.id:
if not self.workflow:
self.workflow = create_workflow(self.title)
super(ScopeItem, self).save(*args, **kwargs)
I'm trying to upload files using Django-filetransfer and I was wondering how to import the username to use as the name of the file the uploads are saved in. Right now I'm using User.username, but that is invalid. I guess the big question is how to access a user's attributes from models.py. Below is my models.py:
from django.db import models
from django.contrib.auth.models import User, UserManager
class UploadModel(User,models.Model):
title = models.CharField(max_length=64, blank=True)
user = models.ForeignKey(User)
file = models.FileField(upload_to='./uploads/'+str(User.username))
#property
def filename(self):
return self.file.name.rsplit('/', 1)[-1]
UPDATED:
app/upload/models.py
from django.db import models
from django.contrib.auth.models import User, UserManager
def uploadmodel_file_upload_to(instance, filename):
return 'uploads/%s/%s' % (instance.user.username, filename)
class UploadModel(models.Model):
user = models.ForeignKey('auth.user')
file = models.FileField(upload_to=uploadmodel_file_upload_to)
TemplateSyntaxError at /upload
Caught DatabaseError while rendering: column upload_uploadmodel.user_id does not exist
LINE 1: SELECT "upload_uploadmodel"."id", "upload_uploadmodel"."user...
Model attributes are officially defined when Python parses models.py code. You should use a callback for upload_to, to be generate the upload file path using the user related instance, for example:
def uploadmodel_file_upload_to(instance, filename):
return 'uploads/%s/%s' % (instance.user.username, filename)
class UploadModel(models.Model):
user = models.ForeignKey('auth.user')
file = models.FileField(upload_to=uploadmodel_file_upload_to)