I'm having a problem when I'm trying to convert the the first column of the uploaded csv file in django into a list. Originally, my code was like this without using django:
with open("export.csv") as f:
my_list = [row["BASE_NAME"] for row in DictReader(f)]
But when developing the user interface using django, I'm not having the required my_list like above, what am I doing wrong? I tried many methods that you can see in the commented codes:
Here is my view.py:
def handle_csv_data(csv_file):
logging.warning('Watch out!') # will print a message to the console
#print(csv_file)
# with open(csv_file) as f:
# my_list = [row["BASE_NAME"] for row in DictReader(f)]
users = []
for row in csv_file:
users.append(row)
return (users)
def home(request):
if request.method=="POST":
img = UploadForm(request.POST, request.FILES)
if img.is_valid():
logging.warning('Watch out!') # will print a message to the console
# paramFile = request.FILES['pic']
paramFile =TextIOWrapper(request.FILES['pic'].file).read()
portfolio1 = csv.DictReader(paramFile)
# portfolio = csv.DictReader(request.FILES['pic'].file)
# csv_file = request.FILES['pic'].file
# with open(default_storage.path('images/' + "500 clusters.csv"), 'wb+') as destination:
# for chunk in csv_file.chunks():
# destination.write(chunk)
# ifile = open("500 clusters.csv", "r")
# data = [row for row in DictReader(csv_file.read().splitlines())]
# print(users)
# paramFile = csv_file.read()
# portfolio1 = csv.DictReader(paramFile)
#ifile = open('sample.csv', "r")
#read = csv.reader(ifile)
#for row in read:
# print(row)
#data = [row for row in DictReader(csv_file.read().splitlines())]
# for row in portfolio:
# my_list = [row["BASE_NAME"]]
#print(my_list)
portfolio= handle_csv_data(portfolio1)
print(portfolio)
# my_list = portfolio
# return HttpResponseRedirect(reverse('portfolio'))
return render(request, 'home.html', {'portfolio': portfolio})
else:
img=UploadForm()
images=Upload.objects.all()
return render(request,'home.html',{'form':img,'images':images})
Here is my model.py:
from django.db import models
from django.forms import ModelForm
class Upload(models.Model):
pic = models.FileField("pic", upload_to="images/")
upload_date=models.DateTimeField(auto_now_add =True)
# FileUpload form class.
class UploadForm(ModelForm):
class Meta:
model = Upload
fields = '__all__'
I appreciate the help.
The problem with your code is here:
paramFile =TextIOWrapper(request.FILES['pic'].file).read()
portfolio1 = csv.DictReader(paramFile)
DictReader expects an iterable of strings (usually a file object), where each string represents a line. Because you called .read(), paramFileis a string, which is an iterable of strings but each yielded string is a character, not a line. Remove the .read().
paramFile =io.TextIOWrapper(request.FILES['pic'].file)
portfolio1 = csv.DictReader(paramFile)
print(type(paramFile))
users = []
users = [row["BASE_NAME"] for row in portfolio1]
print(len(users))
Somehow I was able to fix it with the above, by looking at the local variables in the local host page.
Related
in my Django application, i created a form which permit user to upload csv file.
What i want is when user upload the csv file, the contained data is read and database is filled in with them.
It works but not correctly. data are saved as tuples.
Here's my code
forms.py
class SupplierCSVForm(forms.ModelForm):
class Meta:
model = SuplierCsv
fields = '__all__'
exclude = ('slug',)
views.py
#login_required
def l_supplier(request):
suppliers_list = Supplier.objects.all()
paginator = Paginator(suppliers_list, 3, 2)
page = request.GET.get('page')
suppliers = paginator.get_page(page)
# Supplier csv form
if request.method == 'POST':
form = SupplierCSVForm(request.POST, request.FILES)
if form.is_valid():
uploaded_file = request.FILES['csvfile']
with open('f.csv', 'wb') as destination:
for chunk in uploaded_file.chunks():
destination.write(chunk)
destination.close()
#csvfile = io.TextIOWrapper(open('f.csv', 'rb'))
with open('f.csv', 'r') as the_source:
source_reader = csv.reader(sthe_source)
next(source_reader)
for Name, Email, Contact, City, Website, Activity, Cc, slug in source_reader:
new_supplier = Supplier()
new_supplier.name=Name,
new_supplier.email=Email,
new_supplier.contact=Contact,
new_supplier.city=City,
new_supplier.website=Website,
new_supplier.activity=Activity,
new_supplier.cc=Cc,
new_supplier.slug=slug,
new_supplier.save()
return redirect('good:l_good')
else:
form = SupplierCSVForm()
context = {
'suppliers': suppliers,
'form': form,
}
return render(request, 'supplier/l_supplier.html', context)
Remove the commas where you are assigning the new_supplier objects. Python converts your string objects into tuples if there are any trailing commas.
You have unnecessary commas at the end of your lines:
new_supplier.name=Name,
should be
new_supplier.name=Name
Python thinks that you are creating a tuple
i.e. x, == (x,)
I am trying to download some user data into a csv file. I am able to generate the User fields just fine but when I try to access the onetoone relation field I am running into issues!
Tried numerous different way to get here. Just can't seem to figure out how to get the correct related data.
def export_to_csv (modeladmin, request, queryset):
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
from wsgiref.util import FileWrapper
cols = ['username','email','first_name','last_name','my_profile.dealer_num']
# get qs values
data = list( queryset.values_list(*cols) )
if not data:
messages.error(request, 'No data to export')
return HttpResponseRedirect( request.get_full_path() )
# create empty csv
csv_file = StringIO()
csv_writer = csv.writer(csv_file, quoting = csv.QUOTE_ALL)
# add headers
csv_writer.writerow( cols )
# add qs values
for row in data:
csv_writer.writerow( [s.encode('utf-8') for s in row] )
csv_file.flush()
csv_file.seek(0)
response = HttpResponse(FileWrapper( csv_file ), content_type='text/csv')
response['Content-Disposition'] = "attachment; filename=user-csv-export.csv"
return response
export_to_csv.short_description = "Export to CSV"
Models.py
class MyProfile(UserenaBaseProfile):
user = models.OneToOneField(User,
unique=True,
verbose_name=_('user'),
related_name='my_profile')
dealer_num = models.CharField(blank=True,
max_length=15,
verbose_name="Dealer Number")
Should return everything including a 5 digit dealer number in a csv
You are not accessing the field correctly, you need to use __ as in a query.
Change cols to:
cols = ['username','email','first_name','last_name','my_profile__dealer_num']
I making a email delivery WebApp, I have an Email Class with uploadcsvfile to upload the .csv contact file from the user's computer to server at location CONTACT_DIR, what I'm unable to figure out is how to get the saved contacts_some_char.csv name in my views.py, the purpouse of getting it's name is so that I can parse the email list and send email to them.
I have my model.py as follows :
class Email(models.Model):
fromemail = models.EmailField(blank = False, max_length = 254)
uploadcsvfile = models.FileField(upload_to = 'Contacts', blank = False)
subject = models.CharField(blank=False,max_length = 254)
bodyHeading = models.CharField(blank = False,max_length = 254)
bodytext = models.TextField(blank = False)
name = models.CharField(blank = False,max_length = 254)
Here is my views.py :
def index(request):
if request.method == 'POST':
email_form = EmailForm(request.POST, request.FILES)
if email_form.is_valid():
email_form.save(commit = True)
print(Email.uploadcsvfile)
contacts = Contacts()
#path = settings.CONTACT_DIR
f = open(settings.CONTACT_DIR+"/"+str(uploaded_file_name))
csv_f = csv.reader(f)
Emails = []
Names= []
L_Names = []
for x in csv_f:
Emails.append(x[0])
Names.append(x[1])
L_Names.append(x[2])
f.close()
contacts.ListOfEmails = json.dumps(Emails)
contacts.ListOfFirstNames = json.dumps(Names)
contacts.ListOfLastNames = json.dumps(L_Names)
contacts.save()
Here is forms.py for the same :
from django import forms
from dcsvreader.models import Email
class EmailForm(forms.ModelForm):
class Meta():
model = Email
fields = "__all__"
A.K.A How do i get uploaded_file_name in views.py, I'm reading in various QnA at S.O that this is related to FileDescriptor but can't find how exactly also, print(Email.uploadcsvfile) gives <django.db.models.fields.files.FileDescriptor object at 0x03F7DF90> as output.
request.FILES is a dictionary-like object containing all uploaded files and the uploadcsvfile is the key in which you are making a POST request. Hence, request.FILES['uploadcsvfile'] will hold the particular file as InMemoryUploadedFile and it has a variable .name which holds the filename
Try this,
file_name = request.FILES['uploadcsvfile'].name
Reference
uploadfile.name
request.FILES
I'm pretty new to django myself but it looks like here:
print(Email.uploadcsvfile)
You're trying to print a class attribute rather than an attribute of an instance.
Have you tried the following?
print(email_form.uploadcsvfile.name)
I have to write a function which on input has a name of a table and conditionals/filters for that query, on output it returns a link which should be automatically be download by the client (browser).
How to implement this task using python/django?
E.g. I've written a small piece of code, but I'm not sure that it works correctly and there is no implementation of query conditionals parsing (I don't know how to implement it):
direct_db.py:
from django.db import connection
class DirectSQL:
def __init__(self,in_sql):
self.sql=in_sql
self.cursor = connection.cursor()
self.cursor.execute(in_sql)
def getDescription(self):
columns = [desc[0] for desc in self.cursor.description]
return columns
def getResult(self):
row = self.cursor.fetchall()
return row
def getResultAsDict(self):
desc = self.cursor.description
return [dict(zip([col[0].lower() for col in desc], row)) for row in self.cursor.fetchall()]
excel.py:
from ecc.direct_db import DirectSQL
import pandas as ps
class Excel:
def __init__(self, table_name):
self.table_name = table_name
def convert(in_args):
q = DirectSQL("select * from self.table_name" ) # where... order by... like...
columns = [desc[0] for desc in q.getDescription()]
data = q.getResults()
df = ps.DataFrame(list(data), columns)
writer = ps.ExcelWriter('converted.xlsx')
df.to_excel(writer, sheet_name='converted')
writer.save()
I've worked in something like this before, I used xlsxwriter, you can check its docs to find out how to create a xlsx and how set data into it. Then you should need some view:
from django.views.generic import View
from django.http import HttpResponse
class CreateReport(View):
def get_data(self):
# Query your data here, probably using self.request to get query string
...
return data
def generate_report(self):
# Here you will create xlsx doc and populate with data according to docs linked before
...
return workbook
def get(self, request, *args, **kwargs):
document = self.generate_report()
_file = open(document.filename, 'r')
response(HttpResponse(_file, content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'))
response['Content-Disposition'] = 'attachment; filename=%s' % document.filename.split('/')[-1] # Here will return a full path, that's why probably you will need a split to get only the filename
add_never_cache_headers(response=response) # To avoid download the same file with out of date data.
return response
Then you will need an url
from myapp.views import CreateReport
url(r'^create_report/(?P<some_param_if_needed>[-\w]+)',
CreateReport.as_view(),
name='create_report'),
and finally in template
Download Report
EDIT
Here's a more complete example for get_data() method.
get_data(self):
# Let's supose you have a `MyElements` model
elements = MyElements.objects.all()
# And let's supose you want to filter data with some GET parameter
filter = self.request.GET.get('filter_name', None)
if filter is not None:
elements = elements.filter(filter_field=filter)
return elements
I'm fairly new to django and Python and want to be able to export a list of items in my model i.e products. I'm looking at the documentation here - https://docs.djangoproject.com/en/dev/howto/outputting-csv/
I'm persuming I need will need to create a variable that stores all the data that I want. But not sure where it would within the snippet of code on the link above.
Apologies as this is a very noobish question but would really Any help at all.
Here is the code to my script so far:
import csv
from products.models import Product
from django.http import HttpResponse
def export_to_csv(request):
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="mytest.csv"'
Have a look at the python csv module.
You'll probably want to get the models fields with
def get_model_fields(model):
return model._meta.fields
Then use
getattr(instance, field.name)
to get the field values (as in this question).
Then you'll want something like
with open('your.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
# write your header first
for obj in YourModel.objects.all():
row = ""
for field in fields:
row += getattr(obj, field.name) + ","
writer.writerow(row)
It's a bit verbose (and untested), but it should give you an idea. (Oh and don't forget to close your file)
Depending on the scenario - you may want to have a CSV of your model. If you have access to the Django Admin site, you can plug in a generic action for any model displayed as a list (google: django admin actions)
http://djangosnippets.org/snippets/790/
If you're operating with a console (python manage.py ...), you can use such a script, which I just used:
(place it in: yourapp/management/commands/model2csv.py)
"""
Prints CSV of all fields of a model.
"""
from django.core.management.base import BaseCommand, CommandError
import csv
import sys
class Command(BaseCommand):
help = ("Output the specified model as CSV")
args = '[appname.ModelName]'
def handle(self, *app_labels, **options):
from django.db.models import get_model
app_name, model_name = app_labels[0].split('.')
model = get_model(app_name, model_name)
field_names = [f.name for f in model._meta.fields]
writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)
writer.writerow(field_names)
for instance in model.objects.all():
writer.writerow([unicode(getattr(instance, f)).encode('utf-8') for f in field_names])
This does not catch any exceptions etc., but as an Admin you won't cause them to be raised, right?
Use it like:
./manage.py model2csv my_ecommerce.Product > products.csv
You can also make a template to assist in formatting!
The template is a common Django template
from django.template import loader
def export_to_csv(request):
response = HttpResponse(mimetype='text/csv')
response['Content-Disposition'] = 'attachment; filename="products-list.csv"'
template = loader.get_template('templates/products_template.csb')
response.write(template.render(Context({'products': Products.objects.all()})))
return response
Using django.db.models.query.QuerySet.values results in more optimised queries for my use case.
import csv
from datetime import datetime
from django.http import HttpResponse
# Populate this list with your model's fields
# Replace MODEL with your model
fields = [f.name for f in MODEL._meta.fields]
# The following code will live inside your view
timestamp = datetime.now().isoformat()
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"
] = f"attachment; filename={timestamp}.csv"
writer = csv.writer(response)
# Write the header row
writer.writerow(fields)
# Replace MODEL with your model
for row in MODEL.objects.values(*fields):
writer.writerow([row[field] for field in fields])
return response
I use this on my code. A function called from view.
It automatically get model fields to make columns.
You can also customize the field list you want to export.
Function
import csv
from django.http import HttpResponse
from .models import Books
def export_qs_to_csv(model_class = None, qs = None, field_names = None):
if model_class and not qs:
qs = model_class.objects.all()
if qs and not model_class:
model_class = qs.model
meta = model_class._meta
if not field_names:
field_names = [field.name for field in meta.fields]
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename={}.csv'.format(meta)
writer = csv.writer(response)
writer.writerow(field_names)
for obj in qs:
row = writer.writerow([getattr(obj, field) for field in field_names])
return response
Usage
#user_passes_test(lambda u: u.is_superuser)
def export_books(request):
return export_qs_to_csv(model_class = Books)
# or
return export_qs_to_csv(qs = Books.objects.filter(published = True))
# or
return export_qs_to_csv(
qs = Books.objects.filter(published = True),
field_names = [
"title",
"price",
"publishing_date",
]
)
Original answer
It works, and it needs only to define model class in model_class variable.
This Django view let use downloads CSV. CSV name is Django_app.model_name.csv.
import csv
from django.http import HttpResponse
from .models import Trade
def export_to_csv(request):
# The only line to customize
model_class = Trade
meta = model_class._meta
field_names = [field.name for field in meta.fields]
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename={}.csv'.format(meta)
writer = csv.writer(response)
writer.writerow(field_names)
for obj in model_class.objects.all():
row = writer.writerow([getattr(obj, field) for field in field_names])
return response
Here is a potential solution, based on #tomasz-gandor 's answer, but updated to 2020:
"""
Prints CSV of all fields of a model.
"""
import csv
from django.core.management.base import BaseCommand, CommandError
class Command(BaseCommand):
help = ("Output the specified model as CSV")
def add_arguments(self, parser):
parser.add_argument('model',
nargs=1,
type=str,
help='Model name to export, like <app.model> or "members.Member"')
parser.add_argument('outfile',
nargs=1,
type=str,
help='Save path, like </path/to/outfile.csv> or "/data/members.csv"')
def handle(self, *app_labels, **options):
from django.apps import apps
app_name, model_name = options['model'][0].split('.')
model = apps.get_model(app_name, model_name)
field_names = [f.name for f in model._meta.fields]
writer = csv.writer(open(options['outfile'][0], 'w'), quoting=csv.QUOTE_ALL, delimiter=',')
writer.writerow(field_names)
for instance in model.objects.all():
writer.writerow([str(getattr(instance, f)) for f in field_names])
Can easily be used with:
python manage.py model2csv members.Member /data/members_export.csv
If you don't care about fieldnames and want all the fields, just do this.
with open('file_name.csv', 'w') as csvfile:
writer = csv.writer(csvfile)
for obj in YourModel.objects.values_list():
row = list(obj)
writer.writerow(row)
I combined some of the previous answers, because I needed to import some data from production and change some of it along the way. So here is my solution, which you can use to override some field values while writing the CSV file.
Export some queryset data into CSV file:
import csv
from myapp.models import MyModel
from user.models import User
# Make some queryset manually using Django shell:
user = User.objects.get(username='peterhil')
queryset = MyModel.objects.filter(user=user)
def query_to_csv(queryset, filename='items.csv', **override):
field_names = [field.name for field in queryset.model._meta.fields]
def field_value(row, field_name):
if field_name in override.keys():
return override[field_name]
else:
return row[field_name]
with open(filename, 'w') as csvfile:
writer = csv.writer(csvfile, quoting=csv.QUOTE_ALL, delimiter=',')
writer.writerow(field_names) # write the header
for row in queryset.values(*field_names):
writer.writerow([field_value(row, field) for field in field_names])
# Example usage:
query_to_csv(queryset, filename='data.csv', user=1, group=1)
Use this solution for model csv file.might being helpful
# Create the HttpResponse object with the appropriate CSV header.
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment;
filename="somefilename.csv"'
writer = csv.writer(response);
writer.writerow(["username","Email"]);
for i in User.objects.all():
writer.writerow([i.username,i.email])
return response
I used the django-queryset-csv package.
Follow these steps:
pip install django-queryset-csv
Your views.py:
import djqscsv
from products.models import Product
def get_csv(request):
qs = Product.objects.all()
return djqscsv.render_to_csv_response(qs)