I have a fully functional Django web form that will upload and store images on Amazon S3.
How can I also create an additional re-sized copies of the same image that is being uploaded and upload them at the same time to Amazon S3?
Ideally, I will have two additional sizes (120x90, 240x180) saved to /small/ and /medium/ folders on S3.
For the upload to S3, I'm using django-storages and boto.
I have looked around and tried different ways, but it seems that all the thumb generating codes floating around are dealing with locally stored files or images that are already stored somewhere online.
My Code:
models.py
def content_file_name(instance, filename):
ext = filename.split('.')[-1]
name = uuid.uuid4().hex
filename = "%s_%s.%s" % (instance.business_id, name, ext)
path = 'originals'
return os.path.join(path, filename)
class Photo(models.Model):
filename = models.ImageField(upload_to=content_file_name)
phototitle = models.CharField(max_length=50, null=True)
class Meta:
db_table = 'photo'
views.py
def create(request):
if request.method == 'POST':
form = UploadForm(request.POST, request.FILES)
if form.is_valid():
form.save()
return ...
else:
form = UploadForm()
return render(request, 'upload.html', {
'form': form,
})
form.py
class UplaodForm(ModelForm):
class Meta:
model = Photo
fields = "__all__"
settings.py
AWS_STORAGE_BUCKET_NAME = '...'
AWS_ACCESS_KEY_ID = '...'
AWS_SECRET_ACCESS_KEY = '...'
...
INSTALLED_APPS = (
...
'storages',
My solution works with any storage, but I tested it for AWS via django-storage and boto3. I created a Storage named ThumbnailStorage inheriting from the default one (whatever it is) and added the creation of the thumbnails in method _save, so that I have the name and the content. As a last step I declared this ThumbnailStorage as the default Storage for the field:
the storage
import os
import io
from django.core.files.storage import get_storage_class
from PIL import Image
default_storage = get_storage_class()
class ThumbnailStorage(default_storage):
def _save(self, name, content):
if hasattr(content, 'content_type'):
if content.content_type.startswith('image/'):
self.generate_thumbnails(name, content)
return super()._save(name, content)
def generate_thumbnails(self, name, content):
name1, ext1 = os.path.splitext(name)
fname, ext = os.path.splitext(content.name)
format = (ext1 if ext1 != '.jpg' else '.jpeg').strip('.')
im = Image.open(content)
w, h = im.size
im.thumbnail((120, 90)) # <<< choose your dims
dest_filename = f'/small/{name}' # set your name
fh = default_storage().open(dest_filename, 'wb')
sfile = io.BytesIO()
im.save(sfile, format=format)
fh.write(sfile.getvalue())
fh.close()
the model
class Photo(models.Model):
filename = models.ImageField(
upload_to=content_file_name,
storage=TumbnailStorage() # <<< this is where the work is done
)
...
Note that jpg is not accepted by Pillow, so you need to translate to jpeg and that I was not able to im.save(fh) when the format is jpeg (while it works on png) so I had to add io.StringIO trick
Related
I have uploaded directory using django and JavaScript. Now I am trying to download this directory from django admin. For this purpose I have followed this link click here. When I have uploaded directory in django, the admin created the following table. File name indicates the uploaded folder name.
If I click on the file name, it shows me following error. I want to download this directory as a zip file from django admin. Can anyone please help me with this?
Here is my model.py:
class Prearchive(models.Model):
file = models.FileField(upload_to='media/', null=True)
file_name = models.CharField(max_length=500,primary_key=True)
file_id = models.ForeignKey(CreateScan, on_delete=models.CASCADE,default=1)
def _str_(self):
return self.file_name
admin.py:
from django.contrib import admin
from .models import CreateScan, Prearchive
# Register your models here.
class PrearchiveAdmin(admin.ModelAdmin):
model=Prearchive
list_display = ('file_name','file_id','file_link')
def file_link(self, obj):
if obj.file:
return "<a href='%s' download>Download</a>" % (obj.file.url,)
else:
return "No attachment"
file_link.allow_tags = True
file_link.short_description = 'File Download'
admin.site.register(Prearchive , PrearchiveAdmin)
view.py:
#login_required
def downloadScanView(request,url):
print('url',url)
response = HttpResponse(content_type='application/zip')
file_path = os.path.join(settings.MEDIA_ROOT, url)
zip_download = ZipFile(response, 'w',ZIP_DEFLATED)
if os.path.exists(file_path):
for root, dir, files in os.walk(file_path):
print('r = ',root)
for file in files:
zip_download.write(os.path.join(root,file))
response['Content-Disposition'] = 'attachment; filename={}'.format(os.path.basename(url)+'.zip')
return response
Create a view to download the file, and add urls.py entry for /media to this view.
views.py
def filedownload(request, filename):
zf = zipfile.ZipFile('download.zip', 'w', zipfile.ZIP_DEFLATED)
zf.write("media/" + filename)
response = HttpResponse(zf, content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename="download.zip"'
return response
Main urls.py
path('media/', include('your_app.urls')
App urls.py
path("<filename>/", views.filezip, name="filedownload")
Suppose I had a Dog Object and it had fields like name, weight, and breed. Serializing these and returning them was easy as they were strings. I then added a field called image_url, which was the s3 path (I am storing their images in my s3 bucket). In my DogView (which returns all the dogs a user owns), I would have something like this:
class DogsView(ListAPIView):
def get(self, request):
user = request.user
dogs = Dog.objects.filter(user=user)
dog_serializer = DogSerializer(dogs, many=True)
s3 = boto3.resource("s3")
for dog in dog_serializer.data:
if len(dog["image_url"]) > 0:
s3_object = s3.Object('my-dog-photos', dog["image_url"])
file_stream = io.StringIO()
s3_object.download_fileobj(file_stream)
img = img.imread(file_stream)
dog["image"] = img
return Response(product_serializer.data)
However, when I do this, I get
TypeError: string argument expected, got 'bytes'
How would I be able to append my bytes/file to my serializer.data (or just return an array of images via HttpResponse) ?
DRF Response class expects a json serializable input so if you really have to pass bytes data in Response, you can just convert it to string by changing this line:
dog["image"] = str(img)
You will still have to convert it back to bytes on your client application.
However, this is NOT how json should ever be used. You should just pass urls of your images and let client application handle the downloads. Here is a simple setup for s3 media and static file storage for django. This way serializer will return direct links to s3 bucket and you can also upload your files directly to s3 with django admin (if you're using it).
in your_project/settings.py
AWS_STORAGE_BUCKET_NAME = 'your bucket name'
AWS_S3_REGION_NAME = 'us-east-1'
AWS_ACCESS_KEY_ID = 'access_key_id'
AWS_SECRET_ACCESS_KEY = 'secret_access_key'
AWS_S3_CUSTOM_DOMAIN = '%s.s3.amazonaws.com' % AWS_STORAGE_BUCKET_NAME
STATICFILES_LOCATION = 'static'
STATICFILES_STORAGE = 'your_project.custom_storages.StaticStorage'
DEFAULT_FILE_STORAGE = 'your_project.custom_storages.MediaStorage'
MEDIAFILES_LOCATION = 'media'
MEDIA_ROOT = '/%s/' % MEDIAFILES_LOCATION
MEDIA_URL = '//%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, MEDIAFILES_LOCATION)
create your_project/custom_storages.py
from storages.backends.s3boto3 import S3Boto3Storage
class StaticStorage(S3Boto3Storage):
location = 'static'
class MediaStorage(S3Boto3Storage):
location = 'media'
you will need django-storages and boto3 libraries
I'm trying to create an endpoint to upload images(using postman) to a specific folder using django rest framework. This is my settings for the folder,
MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
This is my model,
class UserMedia(models.Model):
user = models.OneToOneField(User, related_name='medias', on_delete=models.CASCADE, )
profile_image_web = models.FileField(null=True)
profile_image_android = models.FileField(null=True)
profile_image_ios = models.FileField(null=True)
thumbnail = models.FileField(null=True)
This is the Serializer,
class UserMediaSerializer(serializers.ModelSerializer):
class Meta:
model = models.UserMedia
fields = (
'profile_image_web', 'profile_image_ios', 'profile_image_android', 'thumbnail',
)
This is the api,
class CreateUpdateUserMedia(views.APIView):
parser_classes = (MultiPartParser, FormParser)
def post(self, request, **kwargs):
serializer = UserMediaSerializer(request.data)
if serializer.is_valid():
serializer.save()
return Response(serializer.data, status=status.HTTP_201_CREATED)
else:
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
Now when I try to upload ONE image corresponding to one of the fields using POSTMAN, this is the error I get.
'Cannot call `.is_valid()` as no `data=` keyword argument was '
AssertionError: Cannot call `.is_valid()` as no `data=` keyword argument was passed when instantiating the serializer instance.
Which is perfectly understandable, but I dont know how to fix it.
Here are my questions,
How do I correctly upload images using django rest framework.
I don't expect this api to be called with 4 images together, but 4 times using one image at a time, how do I pass the relevant name and
modify the serializer accordingly.
How do I provide a subpath to the root media directory.
Finally I want the serializer to display the full image url, how do I do that?
You are using serializer = UserMediaSerializer(request.data) but you should call it by serializer = UserMediaSerializer(data=request.data)
For uploading images in Django rest framework you should either upload images on S3 and pass s3 url in DRF API or use base64 field in serializer and send base64 encoded value of image in API
import uuid
import base64
import imghdr
from django.utils.translation import ugettext_lazy as _
from django.core.files.base import ContentFile
from rest_framework import serializers
ALLOWED_IMAGE_TYPES = (
"jpeg",
"jpg",
"png",
"gif"
)
class Base64ImageField(serializers.ImageField):
"""
A django-rest-framework field for handling image-uploads through raw post data.
It uses base64 for en-/decoding the contents of the file.
"""
def to_internal_value(self, base64_data):
# Check if this is a base64 string
if not base64_data:
return None
if isinstance(base64_data, basestring):
# Try to decode the file. Return validation error if it fails.
try:
decoded_file = base64.b64decode(base64_data)
except TypeError:
raise serializers.ValidationError(_("Please upload a valid image."))
# Generate file name:
file_name = str(uuid.uuid4())[:12] # 12 characters are more than enough.
# Get the file name extension:
file_extension = self.get_file_extension(file_name, decoded_file)
if file_extension not in ALLOWED_IMAGE_TYPES:
raise serializers.ValidationError(_("The type of the image couldn't been determined."))
complete_file_name = file_name + "." + file_extension
data = ContentFile(decoded_file, name=complete_file_name)
return super(Base64ImageField, self).to_internal_value(data)
raise serializers.ValidationError('This is not an base64 string')
def to_representation(self, value):
# Return url including domain name.
return value.name
def get_file_extension(self, filename, decoded_file):
extension = imghdr.what(filename, decoded_file)
extension = "jpg" if extension == "jpeg" else extension
return extension
Updated
You should use ImageField (not FileField) for images.
You can use above field directly in serializer like any other field.
class UserMediaSerializer(serializers.ModelSerializer):
profile_image_web = Base64ImageField(required=False)
class Meta:
model = models.UserMedia
fields = ('profile_image_web', 'profile_image_ios', 'profile_image_android', 'thumbnail')
I'm trying to save a file to disk, not in a class or anything just straight to disk but I can't for the life of me figure out how.
So far I've got:
View: `
def uploadview(request):
uploadtestvar='Uploadpage!'
request.session['test']='Is this working?'
if request.method == 'POST':
form=UploadForm(request.POST, request.FILES)
if form.is_valid():
forcetitle='test'
try:
pass
except:
pass
return HttpResponseRedirect('/test/')
else:
form=UploadForm()
return render(request, 'test/uploads.html',{'uploadtestvar':uploadtestvar, 'form':form})`
Form stolen directly from Django:
from django import forms
class UploadForm(forms.Form):
title=forms.CharField(max_length=50)
file=forms.FileField()
I've searched countless threads and found none that gives an example simple enough for me to understand how to get that request.FILES['file'] saved to disk somewhere. The possible filetypes are png, jpg, and pdf.
Pass FileSystemStorage to FileField in models where FileSystemStorage has the storage directory
from django.core.files.storage import FileSystemStorage
fs = FileSystemStorage(location = 'media/files/')
class UploadModel(models.Model):
title=forms.CharField(max_length = 50)
file=forms.FileField(storage = fs)
I'm sooo close... but I don't quite see the connection from the upload view to the model. When I use the callback in the model's FileField the upload works, but I'm not sure where the actual file copy is taking place. The goal is to make sure that chunking is happening, but the file copy action seems to be hidden somewhere?
Here's what I have:
Model:
def get_media_upload_dir(instance, filename):
user_id = instance.user.id
upload_dir = "%s/%d/%s" % (settings.MEDIA_ROOT, user_id, filename)
print "Upload dir set to: %s" % upload_dir
return upload_dir
class MediaFile(models.Model):
media_file = models.FileField(upload_to=get_media_upload_dir)
download_count = models.PositiveIntegerField(default=0)
View:
def file_upload(request, course_id):
if request.method == 'POST':
form = FileUploadForm(request.POST, request.FILES)
if form.is_valid():
uploaded = form.cleaned_data['file_upload']
mediaFile = MediaFile(media_file=uploaded,
owner=request.user.profile,
creator=request.user.profile)
mediaFile.save()
return HttpResponseRedirect('/course/%s/' % course_id)
else:
form = FileUploadForm()
return render_to_response('course/file_upload.html', {'form':form,'course':course}, context_instance=RequestContext(request))
The storing happens here: http://code.djangoproject.com/browser/django/trunk/django/db/models/fields/files.py#L90. Django uses it's own API for accessing the file storage: http://docs.djangoproject.com/en/dev/ref/files/storage/. But if chunking is what you need you can go with Bartek's proposal!