Django Rest Framework - Downloading files on server - python

I am writing an API that allows the client to post a link to a file, and then I want to download the file and store it as a FileField on one of my models. Here is the code I have so far:
Serializer:
from rest_framework import serializers
from django.core.files import File
from rest_framework.serializers import ValidationError
class MySerializer(serializers.ModelSerializer):
class Meta:
model = MyModel
fields = ('url')
def to_internal_value(self, data):
validated = dict()
url = data.get('url')
# do url validation here
# download the file to local disk
validated['file'] = File(open('path/to/downloaded/file'))
validated['url'] = url
return validated
Model:
from django.db import models
class MyModel(models.Model):
url = models.CharField(max_length=255)
file = models.FileField(upload_to='files/')
For the view, I use a generic ListCreateAPIView.
There are two main problems, first is that with my current implementation, I end up with two copies of the file because I download it to some location on disk first, and then later when the FileField is stored into the database it copies the file again to the files/ folder. Is there some way to avoid this? Secondly, how can I download the file asynchronously yet still be able to add the FileField attribute on the model after the download has completed?

A better way of doing the same thing will be to create Your own serializer field
from django.core.validators import URLValidator
from django.core.files.base import ContentFile
from rest_framework import serializers
from urllib.request import urlretrieve
class FileUrlField(serializers.FileField):
def to_internal_value(self, data):
try:
URLValidator()(data)
except ValidationError as e:
raise ValidationError('Invalid Url')
# download the contents from the URL
file, http_message = urlretrieve(data)
file = File(open(file, 'rb'))
return super(FileUrlField, self).to_internal_value(ContentFile(file.read(), name=file.name))
and then Use it in your serializer
class MySerializer(serializers.ModelSerializer):
file = FileUrlField()
class Meta:
model = MyModel
However i am Not tested it but should work.

Related

Django: have admin take image file but store it as a base64 string

I was tasked with creating an admin view such that a user could input an image file, which however would be stored as a base64 string as a model field rather than exist in a static files dir on our server.
I'm unclear on how exactly this process would be done, should I be intercepting the POST request from the admin view and pre-processing it to be stored in the field? Should I be overwriting the save method of the base form? I'm a little confused by the different methods as I'm new to Django and have been unable to produce a working result.
Here's my setup:
models.py
from django.db import models
class Product(models.Model):
organization = models.ForeignKey(Organization)
name = models.CharField(max_length=50)
logo = models.TextField()
admin.py
from django.contrib import admin
from .models import Product
class ProductAdmin(admin.ModelAdmin):
exclude = ('logo',)
admin.site.register(Product, ProductAdmin)
misc.py
#how i'd process an image?
from PIL import Image
from base64 import b64encode
def image_to_b64(image_file):
imgdata = Image(image_file)
encoded = b64encode(open(imgdata, 'rb'))
return encoded
from django.db import models
class Product(models.Model):
organization = models.ForeignKey(Organization)
name = models.CharField(max_length=50)
logo = models.TextField()
logo_image = models.ImageFiled(null=True, blank=True, upload_to='logo')
def image_to_b64(image_file):
import base64
with open(image_file.path, "rb") as f:
encoded_string = base64.b64encode(f.read())
return encoded_string
from django.dispatch import receiver
from django.db.models.signals import post_save, m2m_changed
#receiver(post_save, sender=Product)
def create_base64_str(sender, instance=None, created=False, **kwargs):
if created:
instance.logo = image_to_b64(instance.logo_image)
instance.logo_image.delete()
instance.save()

Validation in django rest framework

I am trying to validate existing record in django rest framework and following the link
In my serializer class I have my class like .
from django.forms import widgets
from rest_framework import serializers
from models import Part
class PartSerializer(serializers.Serializer):
part_id = serializers.CharField(required=True, validators=[UniqueValidator(queryset=Part.objects.all())] )
capacity = serializers.IntegerField(required=True)
price = serializers.IntegerField(required=True)
def create(self, validated_data):
"""
Create and return a new `Part` instance, given the validated data.
"""
# try:part_exist = Part.objects.get(part_id = validated_data['part_id'])
# except:part_exist = None
# if part_exist:
# raise serializers.ValidationError('Part name already exist.')
# else:
return Part.objects.create(**validated_data)
But I am always getting error name 'UniqueValidator' is not defined
I don't know how to import this as it is not mentioned in doc .Please help me out how to do this .And if it is not possible should I write own validation logic under views ?
Thanks
You should import the UniqueValidator from the rest_framework.validators module:
from rest_framework.validators import UniqueValidator

Django - Delete file from amazon S3

I have a problem where deleting an object form the admin won't delete the file associated with it. after some research I decided to implement a post_delete in the model.
For some reason I am not able to make the s3 delete the file, even after searching numerous guides and snippets, maybe someone here knows.
I use django 1.5 and boto.
Heres my code for the model:
from django.db import models
from django.contrib.auth.models import User
from fileservice.formatChecker import ContentTypeRestrictedFileField
from south.modelsinspector import add_introspection_rules
import os
from django.dispatch import receiver
from django.utils.translation import ugettext_lazy as _
from django.core.files.storage import default_storage as storage
add_introspection_rules([
(
[ContentTypeRestrictedFileField], # Class(es) these apply to
[], # Positional arguments (not used)
{ # Keyword argument
"content_types": ["content_types", {}],
"max_upload_size": ["max_upload_size", {}]
},
),
], ["^fileservice\.formatChecker\.ContentTypeRestrictedFileField"])
class Contentfile(models.Model):
content = ContentTypeRestrictedFileField(upload_to='uploads/', content_types=['video/mp4', 'application/pdf', 'image/gif', 'image/jpeg', 'image/png'],max_upload_size=5242880,blank=True, null=True, help_text='Upload a file to add it to the content the app displayes')
created_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
title = models.CharField(max_length=255, unique=True)
file_type = models.CharField(max_length=5)
published = models.BooleanField(default=True)
file_owner = models.ForeignKey(User, related_name='Contentfiles')
class Meta:
ordering = ["title"]
def __unicode__(self):
return self.title
def save(self, *args, **kwargs):
file_name = os.path.basename(self.content.name)
self.file_type = file_name.split('.')[-1]
self.title = file_name.split('.')[0]
self.published = True
super(Contentfile, self).save(*args, **kwargs)
#receiver(models.signals.post_delete, sender=Contentfile)
def auto_delete_file_on_delete(sender, instance, **kwargs):
"""Deletes file from filesystem
when corresponding `MediaFile` object is deleted.
"""
if instance.content:
if os.path.isfile(storage.open(instance.content.path)):
os.remove(storage.open(instance.content.path))
#receiver(models.signals.pre_save, sender=Contentfile)
def auto_delete_file_on_change(sender, instance, **kwargs):
"""Deletes file from filesystem
when corresponding `MediaFile` object is changed.
"""
if not instance.pk:
return False
try:
old_file = Contentfile.objects.get(pk=instance.pk).content
except Conentfile.DoesNotExist:
return False
new_file = instance.content
if not old_file == new_file:
if os.path.isfile(storage.open(old_file.path)):
os.remove(storage.open(old_file.path))
It is MUCH safer to do post_delete. If something goes wrong you will start missing S3 files and you wont notice it because your DB records are intact. post_delete will be safer since it is less likely that S3 delete operation would fail after you have deleted your db record. Furthermore even if file delete fails you will be left with a bunch of unreferenced S3 file which are harmless and can be easily cleaned up.
#receiver(models.signals.post_delete, sender=Picture)
def remove_file_from_s3(sender, instance, using, **kwargs):
instance.img.delete(save=False)
You need to call FieldFile's delete() method to remove the file in S3. In your case, add a pre_delete signal where you call it:
#receiver(models.signals.pre_delete, sender=ContentFile)
def remove_file_from_s3(sender, instance, using):
instance.content.delete(save=False)
Try django-cleanup, it automatically invokes delete method on FileField when you remove model.
This worked for me by deleting files both in DB and in AWS S3.
from django.db import models
from django.dispatch import receiver
from django.views import generic
from project.models import ContentFile
from django.contrib.auth.mixins import LoginRequiredMixin,UserPassesTestMixin
class DeleteFileView(LoginRequiredMixin,UserPassesTestMixin,generic.DeleteView):
model = ContentFile
template_name = 'file-delete.html'
success_url = 'redirect-to'
#Check if the owner of the file is the one trying to delete a file
def test_func(self):
obj = self.get_object()
if obj.user == self.request.user:
return True
return False
#This code does magic for S3 file deletion
#receiver(models.signals.pre_delete, sender=ContentFile)
def remove_file_from_s3(sender, instance, using, **kwargs):
instance.image_file.delete(save=False)
I am using pre_delete you can check the django documentation.File reference deletion in DB is done by DeleteView, I hope this helps someone

How to call data from Django database to template (html)?

how to retrieve data from the database django. I have models.py like this :
from cms.models.pluginmodel import CMSPlugin
from django.db import models
class Category(CMSPlugin):
name = models.CharField(max_length=50)
def __unicode__(self):
return self.name
and file cmsplugins.py like this :
from cms.plugin_base import CMSPluginBase
from cms.plugin_pool import plugin_pool
from django.utils.translation import ugettext_lazy as _
from models import Category
class CategoryPlugin(CMSPluginBase):
model = Category
name = _("Category Plugin")
render_template = "about.html"
def render(self, context, instance, placeholder):
context['instance'] = instance
return context
plugin_pool.register_plugin(CategoryPlugin)
how to retrieve a list of name to be displayed in html?
Thanks before ^^
In Django, you should divide work in 3 parts.
Models: All your db models and logic resides under it.
Templates: Where you place your markup by passing in the least possible only relevant data for rendering.
Routes (views): Your URL's are mapped to these methods in urls.py They Make a call to models and further pass in the processed data to templates so as to get rendered.
Follow the link #jdotjdot gave in comments. Thats the way its done in Django.

Django File-Transfers Save to New File for Each User

I'm trying to upload files using Django-filetransfer and I was wondering how to import the username to use as the name of the file the uploads are saved in. Right now I'm using User.username, but that is invalid. I guess the big question is how to access a user's attributes from models.py. Below is my models.py:
from django.db import models
from django.contrib.auth.models import User, UserManager
class UploadModel(User,models.Model):
title = models.CharField(max_length=64, blank=True)
user = models.ForeignKey(User)
file = models.FileField(upload_to='./uploads/'+str(User.username))
#property
def filename(self):
return self.file.name.rsplit('/', 1)[-1]
UPDATED:
app/upload/models.py
from django.db import models
from django.contrib.auth.models import User, UserManager
def uploadmodel_file_upload_to(instance, filename):
return 'uploads/%s/%s' % (instance.user.username, filename)
class UploadModel(models.Model):
user = models.ForeignKey('auth.user')
file = models.FileField(upload_to=uploadmodel_file_upload_to)
TemplateSyntaxError at /upload
Caught DatabaseError while rendering: column upload_uploadmodel.user_id does not exist
LINE 1: SELECT "upload_uploadmodel"."id", "upload_uploadmodel"."user...
Model attributes are officially defined when Python parses models.py code. You should use a callback for upload_to, to be generate the upload file path using the user related instance, for example:
def uploadmodel_file_upload_to(instance, filename):
return 'uploads/%s/%s' % (instance.user.username, filename)
class UploadModel(models.Model):
user = models.ForeignKey('auth.user')
file = models.FileField(upload_to=uploadmodel_file_upload_to)

Categories

Resources