I have set up a form and view to upload multiple *.gpx files to my website at once. These files are validated using a clean() method on the form and then once validated passed to a function for processing.
When I upload some invalid files the clean() method catches them and informs the user as expected.
When I upload some valid files the processing function crashes with an error saying the files are empty.
If I comment out the clean() method then the valid files are uploaded fine.
What can be happening to the form during the clean() method than means the files are being blanked?
here is my form:
class UploadGpxForm(forms.Form):
gpx_file = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True}))
here is my view:
class UploadGpxView(FormView):
form_class = UploadGpxForm
template_name = 'dashboard/upload.html' # Replace with your template.
success_url = reverse_lazy('dashboard:index') # Replace with your URL or reverse().
def post(self, request, *args, **kwargs):
form_class = self.get_form_class()
form = self.get_form(form_class)
files = request.FILES.getlist('gpx_file')
if form.is_valid():
for f in files:
SaveGPXtoPostGIS(f)
return self.form_valid(form)
else:
return self.form_invalid(form)
Here is my clean method for the UploadGpxForm:
def clean(self):
file_errors=[]
files = list(self.files.getlist('gpx_file'))
for f in list(files):
#check file has only one full stop in it.
if len(f.name.split('.')) != 2:
file_errors.append(ValidationError(
_('%(file_name)s has not been uploaded:'\
'File type is not supported')
, params = { 'file_name': f.name }
, code = 'file_type')
)
#check file doesn't breach the file size listed in settings
if f.content_type in settings.DASHBOARD_UPLOAD_FILE_TYPES:
if f._size > settings.DASHBOARD_UPLOAD_FILE_MAX_SIZE:
file_errors.append(ValidationError(
_('%(file_name)s has not been uploaded: File too big.'\
'Please keep filesize under %(setting_size)s.'\
'Current filesize %(file_size)s') ,
params = {
'file_name': f.name,
'setting_size': filesizeformat(
settings.DASHBOARD_UPLOAD_FILE_MAX_SIZE),
'file_size': filesizeformat(f._size)
},
code = 'file_size'
)
)
#check it is one of our allowed file types
else:
file_errors.append(ValidationError(
_('%(file_name)s has not been uploaded:'\
'File type is not supported')
, params = { 'file_name' : f.name }
, code = 'file_type'
)
)
#next check the file hasn't been uploaded before
#generate MD5
md5hash = md5()
for chunk in f.chunks():
md5hash.update(chunk)
file_hash = md5hash.hexdigest()
if gpxTrack.objects.filter(file_hash=file_hash).exists():
file_errors.append(ValidationError(
_('%(file_name)s has not been uploaded as a identical file'\
'has already been uploaded previously'),
params = { 'file_name' : f.name },
code = 'file_hash'))
#finally raise errors if there are any
if len(file_errors) > 0:
raise ValidationError(file_errors)
else:
return files
When you read the file content (for calculating md5 hash) you need to move the file object’s position to the beginning (0th byte) using file.seek:
md5hash = md5()
for chunk in f.chunks():
md5hash.update(chunk)
file_hash = md5hash.hexdigest()
f.seek(0) #<-- add this line
Related
I have a scenario where a user uploads some data, Django does some processing in pandas and returns a potentially large txt file. I've got this working but I'm unsure about the scalability of the approach and want to know if there's a better way.
Adapted the Outputting to CSV section of the Django doc I have the following:
class MyClass(LoginRequiredMixin,FormView):
template_name = 'myapp/mytemplate.html'
form_class = MyForm
success_url = '/' # Replace with your URL or reverse().
def post(self, request, *args, **kwargs):
if request.method == 'POST':
form = MyForm(request.POST, request.FILES)
#print("filename",files[0].name)
if form.is_valid() :
filename = "my-file.txt"
content = 'any string generated by django'
response = HttpResponse(content, content_type='text/plain')
response['Content-Disposition'] = 'attachment; filename={0}'.format(filename)
return response
else:
print("i am invalid")
return self.form_invalid(form)
In practice I will need to output a text file of perhaps 1000 lines, built by looping over numerous dataframes, should I just build an extremely long text string (content), or is there a better way? In pure python I am more used to creating txt file output using:
f = open( 'some_file.txt', 'w+')
f.write("text")
f.write("text")
f.close()
Which seems more intuitive.
As requested by comments, updated to show exactly the code I was trying in Django which was returning an empty text file:
class MyClass(LoginRequiredMixin,FormView):
template_name = 'myapp/mytemplate.html'
form_class = MyForm
success_url = '/' # Replace with your URL or reverse().
def post(self, request, *args, **kwargs):
if request.method == 'POST':
form = MyForm(request.POST, request.FILES)
if form.is_valid() :
f = open( 'some_file.txt', 'w+')
f.write("text")
return FileResponse(f, as_attachment=True, filename='some_file.txt')
It's very simple:
response = HttpResponse(content_type='text/plain')
response['Content-Disposition'] = 'attachment; filename="filename.txt"'
response.write('Hello')
return response
https://docs.djangoproject.com/en/3.2/howto/outputting-csv/
It's same as CSV, just change extension to .txt
Example:
response = HttpResponse(content_type='text/plain')
response['Content-Disposition'] = 'attachment; filename="filename.txt"'
writer = csv.writer(response)
writer.writerow(['Hello'])
I am submitting csv file from form using POST method and want to save its content into model
forms.py
class SaveCsvForm(forms.Form):
upload_file = forms.FileField(label="Uplaod CSV File", help_text="Uploa file in CSV format")
Views.py
def save_csv(request):
if request.method == 'POST':
form = SaveCsvForm(request.POST, request.FILES)
if form.is_valid():
print("All ok I am check One")
# instance = CsvStorage(file_field=request.FILES['file'])
file = request.FILES['upload_file'].read()
print(file)
csv_to_db(file)
ack = {
'status': 'Success',
'message': 'Data is successfuly submited into database'
}
return JsonResponse(ack)
else:
ack = {
'status': 'Error',
'message': 'Server Returned '+form.errors.as_json(),
}
return JsonResponse(ack)
else:
form = SaveCsvForm()
return render(request, 'upload_csv.html', {'form': form})
file handler method for storing data in Model
def csv_to_db(filename):
with open(filename, 'r') as file:
''' reading csv file in dictionary format '''
reader = csv.DictReader(file)
for row in reader:
instance = CsvStorage(username=row['username'],
first_name=row['user'],
city=row['city'],
mobile=row['mobile']
)
instance.save()
if instance:
return({'status': 'successfuly'})
else:
return({'status': 'error'})
Note:
1.The main problem is while reading the file
2.I don't want to store file in models fileField
I ended up using FileSystemStorage class in Django as described in docs.
I have a method to read a Newick file and return a String in Django framework which is the following:
def handle_uploaded_file(f):
output = " "
for chunk in f.chunks():
output += chunk.decode('ascii')
return output.replace("\n", "").replace("\r", "")
def post(self, request):
form = HomeForm(request.POST, request.FILES)
if form.is_valid():
input = handle_uploaded_file(request.FILES['file'])
treeGelezen = Tree(input, format=1)
script, div = mainmain(treeGelezen)
form = HomeForm()
args = {'form': form, 'script': script, 'div': div}
return render(request, self.template_name, args)
Which works fine for normal Newick files but i also have some files which have a string at the beginning of the file. I'm trying to make another method which checks if the file has the following String before it (which is the case in some files): "newick;" and removes the string if found. It works locally but i can't seem to merge them. This is how it locally looks like:
def removeNewick(tree_with_newick):
for x in tree_with_newick:
if x.startswith('newick;'):
print('')
return x
filepath = "C:\\Users\\msi00\\Desktop\\ncbi-taxanomy.tre"
tree_with_newick = open(filepath)
tree = Tree(newick=removeNewick(tree_with_newick), format=1)
which works perfectly when i specify the path just in python so i tried combining them in Django like this:
def handle_uploaded_file(f):
tree_with_newick = open(f)
for x in tree_with_newick:
if x.startswith('newick;'):
print('')
return cutFile(x)
def cutFile(f):
output = " "
for chunk in f.chunks():
output += chunk.decode('ascii')
return output.replace("\n", "").replace("\r", "")
def post(self, request):
form = HomeForm(request.POST, request.FILES)
if form.is_valid():
input = handle_uploaded_file(request.FILES['file'])
treeGelezen = Tree(input, format=1)
script, div = mainmain(treeGelezen)
form = HomeForm()
args = {'form': form, 'script': script, 'div': div}
return render(request, self.template_name, args)
Which doesn't work and it gives the following error:
expected str, bytes or os.PathLike object, not InMemoryUploadedFile
I've been working on it for two days already and couldn't figure out why the error is popping up.
The error is happening because the function handle_uploaded_file(f) is trying to open an already opened file.
The value of request.FILES['file'] is a InMemoryUploadedFile and can be used like a normal file. You don't need to open it again.
To fix, just remove the line that tries to open the file:
def handle_uploaded_file(f):
for x in f:
if x.startswith('newick;'):
print('')
return cutFile(x)
In my setting.py I had
MEDIA_ROOT = os.path.join(BASE_DIR, 'media'),
when it should've been
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
which solved this error for me.
the solution for me;
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
instead of :
MEDIA_ROOT = [os.path.join(BASE_DIR, 'media')]
in settigs.py
I want to get the value of a CharField based on the value of an ImageField. My form and view are defined as:
#Form
class GpsImForm(forms.Form):
image = forms.ImageField(required=True)
gps_data = forms.CharField(required=True)
#View
def gpsim_gen_view(request):
if request.method == 'POST':
form = GpsImForm(request.POST, request.FILES)
if 'image' in request.FILES:
im = request.FILES['image']
i = Image.open(im)
... # functions to extract exif data from i
request.POST.update({ 'gps_data': ...}) # set gps_data based on exif data from i
if form.is_valid():
obj = form.save()
return ... #returns the gpsim
else:
form = GpsImForm()
return direct_to_template(request, 'gpsim_generate.html', {'form': form,})
The gps_data is updated, but, as soon as I use Image.open(), I get the following error message:
Upload a valid image. The file you uploaded was either not an image or a corrupted image.
If I comment the lines concerning i and modify the gps_data to whatever, the form (with the image) is saved without any error...
# i = Image.open(im)
# ...
# functions to extract exif data from i
request.POST.update({ 'gps_data': 'some text'}) # set gps_data to 'test'
first of all, make sure that your form has the enctype tag
<form enctype="multipart/form-data" ... >
Try to write the img (all the chunks) on disk:
import Image
from random import choice
from django.conf import settings
random_file_name = getattr(settings, 'FILE_UPLOAD_TEMP_DIR', '/tmp')
random_file_name += '/' + ''.join([choice('abcdefg') for i in range(12)]) + '.jpg'
destination = open(random_file_name, 'wb+')
for chunk in request.FILES['image'].chunks():
destination.write(chunk)
destination.close()
Then, you can open it from disk:
image = Image.open(random_file_name)
if image.mode != "RGB":
image = image.convert("RGB")
...
This URL may help you:
https://docs.djangoproject.com/en/dev/topics/http/file-uploads/
I finally found a cleaner solution:
I removed the "required=True" from my models and defined a clean() method which does the job in my form models:
#Form
class GpsImForm(forms.Form):
image = forms.ImageField(required=True)
gps_data = forms.CharField()
def clean(self):
super(forms.Form, self)).clean()
if not self.cleaned_data['gps_data']: # the user can provide the gps_data manually
if self.cleaned_data['image']: # if he provided no gps_data but an image
i = Image.open(self.cleaned_data['image'])
... # functions to extract exif data from i
else:
msg = _("You have to provide an image or a gps_data.")
self._errors['gps_data'] = self.error_class([msg])
return self.cleaned_data
#View
def gpsim_gen_view(request):
if request.method == 'POST':
form = GpsImForm(request.POST, request.FILES)
if form.is_valid():
obj = form.save()
return ... #returns the gpsim
else:
form = GpsImForm()
return direct_to_template(request, 'gpsim_generate.html', {'form': form,})
I've written an app that uses forms to collect information that is then sent in an email. Many of these forms have a filefield used to attach files to the email. I'd like to validate two things, the size of the file (to ensure the emails are accepted by our mail server. I'd also like to check the file extension, to discourage attaching file types not useable for our users.
(This is the python class I'm trying to extend)
class FileField(Field):
widget = FileInput
default_error_messages = {
'invalid': _(u"No file was submitted. Check the encoding type on the form."),
'missing': _(u"No file was submitted."),
'empty': _(u"The submitted file is empty."),
'max_length': _(u'Ensure this filename has at most %(max)d characters (it has %(length)d).'),
}
def __init__(self, *args, **kwargs):
self.max_length = kwargs.pop('max_length', None)
super(FileField, self).__init__(*args, **kwargs)
def clean(self, data, initial=None):
super(FileField, self).clean(initial or data)
if not self.required and data in EMPTY_VALUES:
return None
elif not data and initial:
return initial
# UploadedFile objects should have name and size attributes.
try:
file_name = data.name
file_size = data.size
except AttributeError:
raise ValidationError(self.error_messages['invalid'])
if self.max_length is not None and len(file_name) > self.max_length:
error_values = {'max': self.max_length, 'length': len(file_name)}
raise ValidationError(self.error_messages['max_length'] % error_values)
if not file_name:
raise ValidationError(self.error_messages['invalid'])
if not file_size:
raise ValidationError(self.error_messages['empty'])
return data
Just overload the "clean" method:
def clean(self, data, initial=None):
try:
if data.size > somesize:
raise ValidationError('File is too big')
(junk, ext) = os.path.splitext(data.name)
if not ext in ('.jpg', '.gif', '.png'):
raise ValidationError('Invalid file type')
except AttributeError:
raise ValidationError(self.error_messages['invalid'])
return FileField.clean(self, data, initial)
in my opinion subclassing the actual field class is way to much effort. It should be easier to simply extend your form class. You could add a method which "cleans" the file field.
For example:
class MyForm(forms.Form):
attachment = forms.FileField(...)
def clean_attachment(self):
data = self.cleaned_data['attachment'] // UploadedFile object
exts = ['jpg', 'png'] // allowed extensions
// 1. check file size
if data.size > x:
raise forms.ValidationError("file to big")
// 2. check file extension
file_extension = data.name.split('.')[1] // simple method
if file_extension not in exts:
raise forms.ValidationError("Wrong file type")
return data
This is only a basic example and there are some rough edges. But you can use this example and improve it until you have a version that works for you.
Recommended readings:
Django Doc - Cleaning a specific field
Django Doc - UploadedFile class
Django Doc - File class
This is what I ended up doing:
In my app's setting file:
exts = ['doc', 'docx', 'pdf', 'jpg', 'png', 'xls', 'xlsx', '.xlsm', '.xlsb']
max_email_attach_size = 10485760 #10MB written in bytes
In a new file I called formfunctions:
from django import forms
from django.forms.util import ErrorList, ValidationError
from app.settings import exts, max_email_attach_size
class SizedFileField(forms.FileField):
def clean(self, data, initial=None):
if not data in (None, ''):
try:
if data.size > max_email_attach_size:
raise ValidationError("The file is too big")
file_extension = data.name.split('.')[1]
if file_extension not in exts:
raise ValidationError("Invalid File Type")
except AttributeError:
raise ValidationError(self.error_messages['invalid'])
return forms.FileField.clean(self, data, initial)
and in my forms file:
from formfunctions import SizedFileField
An example class from the forms file:
class ExampleClass(forms.Form):
Email_Body = forms.CharField(widget=forms.Textarea, required=False)
Todays_Date = forms.CharField()
Attachment = SizedFileField(required=False)