I have to write a function which on input has a name of a table and conditionals/filters for that query, on output it returns a link which should be automatically be download by the client (browser).
How to implement this task using python/django?
E.g. I've written a small piece of code, but I'm not sure that it works correctly and there is no implementation of query conditionals parsing (I don't know how to implement it):
direct_db.py:
from django.db import connection
class DirectSQL:
def __init__(self,in_sql):
self.sql=in_sql
self.cursor = connection.cursor()
self.cursor.execute(in_sql)
def getDescription(self):
columns = [desc[0] for desc in self.cursor.description]
return columns
def getResult(self):
row = self.cursor.fetchall()
return row
def getResultAsDict(self):
desc = self.cursor.description
return [dict(zip([col[0].lower() for col in desc], row)) for row in self.cursor.fetchall()]
excel.py:
from ecc.direct_db import DirectSQL
import pandas as ps
class Excel:
def __init__(self, table_name):
self.table_name = table_name
def convert(in_args):
q = DirectSQL("select * from self.table_name" ) # where... order by... like...
columns = [desc[0] for desc in q.getDescription()]
data = q.getResults()
df = ps.DataFrame(list(data), columns)
writer = ps.ExcelWriter('converted.xlsx')
df.to_excel(writer, sheet_name='converted')
writer.save()
I've worked in something like this before, I used xlsxwriter, you can check its docs to find out how to create a xlsx and how set data into it. Then you should need some view:
from django.views.generic import View
from django.http import HttpResponse
class CreateReport(View):
def get_data(self):
# Query your data here, probably using self.request to get query string
...
return data
def generate_report(self):
# Here you will create xlsx doc and populate with data according to docs linked before
...
return workbook
def get(self, request, *args, **kwargs):
document = self.generate_report()
_file = open(document.filename, 'r')
response(HttpResponse(_file, content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'))
response['Content-Disposition'] = 'attachment; filename=%s' % document.filename.split('/')[-1] # Here will return a full path, that's why probably you will need a split to get only the filename
add_never_cache_headers(response=response) # To avoid download the same file with out of date data.
return response
Then you will need an url
from myapp.views import CreateReport
url(r'^create_report/(?P<some_param_if_needed>[-\w]+)',
CreateReport.as_view(),
name='create_report'),
and finally in template
Download Report
EDIT
Here's a more complete example for get_data() method.
get_data(self):
# Let's supose you have a `MyElements` model
elements = MyElements.objects.all()
# And let's supose you want to filter data with some GET parameter
filter = self.request.GET.get('filter_name', None)
if filter is not None:
elements = elements.filter(filter_field=filter)
return elements
Related
Currently, my application is to select the stock data, then analyzing it by using another python script and output the result(in JSON format).
Now I would like to add a button to output the result(w_df_split) to CSV, then when the user clicks the button and download the csv file.
But I am stuck in how to return the output CSV function in this view.
views.py:
def efficient_frontier_select(request):
user_holding = Position.objects.filter(created_by = request.user)
selected_stock = None
w_df_split = None
if request.method == 'POST':
selected_stock = request.POST.getlist('stock_symbol')
ta, tb, tc = ef.combination_frontier(selected_stock)
fy_df, fx_df, w_df, fxfy_df, ef_df = ef.transform_frontier_todf(ta,tb,tc, selected_stock)
w_df_split = json.loads(ef.json_format_split(w_df))
context = {
'w_df_split' : w_df_split,
}
return render(request, 'portfolio/efficient_frontier.html', context)
Asumming df is a dataframe, you you could use pandas's df.to_csv(). Like that: csv_data = w_df.to_csv(). You could do the same to json with w_df.to_json()
Just an update!
Django view design - export CSV from user-generated content
This works by setting the hidden input field to store the JSON data.
Someone helped me write a script that will take URLs from one csv file, compare it to another, and output a return value based on the match.
new.csv --> 1 column: urls
scrapers.csv --> 2 columns: scraper_dom, scraper_id
That script looks like:
import csv
from urllib.parse import urlparse
from typing import List
def fnetloc(any) -> str:
try:
p = urlparse(any)
return p.netloc
except IndexError:
return 'Error'
class Scraper:
def __init__(self, scraper_dom: str, scraper_id: str):
self.scraper_dom = scraper_dom
self.scraper_id = scraper_id
def matches(self, fnetloc: str) -> bool:
return fnetloc.endswith(self.scraper_dom)
class Site:
def __init__(self, url: str):
self.url = url
self.fnetloc = fnetloc(url)
def get_scraperid(self, scrapers: List[Scraper]) -> str:
try:
return next(x.scraper_id for x in scrapers if x.matches(self.fnetloc))
except:
return "[no scraper id found]"
sites = [Site(row[0]) for row in csv.reader(open("new.csv"))]
scrapers = [Scraper(row[0], row[1]) for row in csv.reader(open("scrapers.csv"))]
for site in sites:
print(site.get_scraperid(scrapers), site.url, sep="\t")
This works great, but I'm working on putting this into a webapp, and am trying to work out the same logic utilizing PythonAnywhere's MySql database and SQLAlchemy.
Here's what I've got so far (besides the HTML templates the views render):
from flask import Flask,render_template, request
from flask_sqlalchemy import SQLAlchemy
from urllib.parse import urlparse
from sqlalchemy.ext.declarative import declarative_base
app = Flask(__name__)
app.config["DEBUG"] = True
app.config["SECRET_KEY"] = "secret_key"
db = SQLAlchemy(app)
SQLALCHEMY_DATABASE_URI = db.create_engine("mysql+mysqlconnector://username:password#username.mysql.pythonanywhere-services.com/username$scrapers?charset=utf8mb4".format(
username="username",
password="password",
hostname="username.mysql.pythonanywhere-services.com",
databasename="username$scrapers",
))
app.config[SQLALCHEMY_DATABASE_URI] = SQLALCHEMY_DATABASE_URI
app.config["SQLALCHEMY_POOL_RECYCLE"] = 299
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
db.Model = declarative_base()
class Scrapers(db.Model):
__tablename__ = "Scrapers"
id = db.Column(db.Integer, primary_key = True)
scraper_dom = db.Column(db.String(255))
scraper_id = db.Column(db.String(128))
db.Model.metadata.create_all(SQLALCHEMY_DATABASE_URI)
Session = db.sessionmaker()
Session.configure(bind=SQLALCHEMY_DATABASE_URI)
session = Session()
scrapers = session.query(Scrapers.scraper_dom, Scrapers.scraper_id)
#app.route("/", methods=["GET","POST"])
def index():
if request.method == "Get":
return render_template("url_page.html")
else:
return render_template("url_page.html")
#app.route("/submit", methods=["GET","POST"])
def submit():
que = [urlparse(item).netloc for item in request.form["urls"].split('\n')]
ids = scrapers.filter(Scrapers.scraper_dom.in_(que))
results = (ids, '\t', request.form["urls"])
return render_template("submit.html", results=results)
The trouble I'm having is in this bit:
#app.route("/submit", methods=["GET","POST"])
def submit():
que = [urlparse(item).netloc for item in request.form["urls"].split('\n')]
ids = scrapers.filter(Scrapers.scraper_dom.in_(que))
results = (ids, '\t', request.form["urls"])
return render_template("submit.html", results=results)
I need que to iterate through my multiline input (users will be pasting large lists of urls from excel), which it does great. I then need to take each .netloc value and compare it to scraper_dom to return the corresponding scraper_id value. Finally, outputting the original url and scraper_id value in columns, similar to this (though I don't need column names in the output):
Problem is, as it's written, it is just returning the SQL from the line:
ids = scrapers.filter(Scrapers.scraper_dom.in_(que))
Something like (Given an input of www.1688.com):
What am I missing? I've tried writing ids a few different ways and I get errors:
ids = scrapers.filter(scrapers.scraper_dom.in_(que))
Error:
AttributeError: 'Query' object has no attribute 'scraper_dom'
ids = Scrapers.query.filter(Scrapers.scraper_dom.in_((que)))
Error:
AttributeError: type object 'Scrapers' has no attribute 'query'
I'm using the idea of matching based off of:
[x for x in a if x in b]
I am trying to download some user data into a csv file. I am able to generate the User fields just fine but when I try to access the onetoone relation field I am running into issues!
Tried numerous different way to get here. Just can't seem to figure out how to get the correct related data.
def export_to_csv (modeladmin, request, queryset):
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
from wsgiref.util import FileWrapper
cols = ['username','email','first_name','last_name','my_profile.dealer_num']
# get qs values
data = list( queryset.values_list(*cols) )
if not data:
messages.error(request, 'No data to export')
return HttpResponseRedirect( request.get_full_path() )
# create empty csv
csv_file = StringIO()
csv_writer = csv.writer(csv_file, quoting = csv.QUOTE_ALL)
# add headers
csv_writer.writerow( cols )
# add qs values
for row in data:
csv_writer.writerow( [s.encode('utf-8') for s in row] )
csv_file.flush()
csv_file.seek(0)
response = HttpResponse(FileWrapper( csv_file ), content_type='text/csv')
response['Content-Disposition'] = "attachment; filename=user-csv-export.csv"
return response
export_to_csv.short_description = "Export to CSV"
Models.py
class MyProfile(UserenaBaseProfile):
user = models.OneToOneField(User,
unique=True,
verbose_name=_('user'),
related_name='my_profile')
dealer_num = models.CharField(blank=True,
max_length=15,
verbose_name="Dealer Number")
Should return everything including a 5 digit dealer number in a csv
You are not accessing the field correctly, you need to use __ as in a query.
Change cols to:
cols = ['username','email','first_name','last_name','my_profile__dealer_num']
I'm trying to generate a custom HTML and I have a value I want to pass into xml.startElement (or root if you're thinking in generic terms). How do I go about doing this?
I'm currently using django rest framework a class view and a custom renderer -
This is the beginning of the renderer -
class TESTRenderer(renderers.BaseRenderer):
media_type = 'application/xml'
format = 'xml'
charset = 'utf-8'
def render(self, data, accepted_media_type=None, renderer_context=None):
"""
Renders *obj* into serialized XML.
"""
if data is None:
return ''
stream = StringIO()
xml = SimplerXMLGenerator(stream, self.charset)
xml.startDocument()
xml.startElement(header.data, {})
So as you can see I'm trying to pass a variable called header into the xml.startElement
Here's the view where that data lies -
class TestDetail(APIView):
permission_classes = (AllowAny,)
"""
Retrieve, update or delete a snippet instance.
"""
def get(self, request, pk, format=None):
jobmst_name = queryset1
nodmst_alias = queryset2
sysval_integer = queryset3
mst = queryset4
dtl = queryset5
dep = queryset6
trg = queryset7
name = str(jobmst_name)
master = str(nodmst_alias)
dbversion = str(sysval_integer)
header = 'job id="%s" name="%s" master="%s" dbversion="%s" xmlversion="1"' % (pk, name, master, dbversion)
jobmststring = JobmstSerializer(mst)
jobdtlstring = JobdtlSerializer(dtl)
jobdepstring = JobdepSerializer(dep, many=True)
trgjobstring = TrgjobSerializer(trg, many=True)
jobmst_serialized = {'jobmst': jobmststring.data}
jobdtl_serialized = {'jobdtl': jobdtlstring.data}
jobdep_serialized = [{'jobdep':item} for item in jobdepstring.data]
trgjob_serialized = [{'trgjob':item} for item in trgjobstring.data]
jobgroup = header, jobmst_serialized, jobdtl_serialized, jobdep_serialized, trgjob_serialized
return TestResponse(jobgroup)
The response it's using is here -
class TestResponse(HttpResponse):
"""
An HttpResponse that renders its content into XML.
"""
def __init__(self, data, **kwargs):
content = TESTRenderer().render(data)
kwargs['content_type'] = 'application/xml'
super(TestResponse, self).__init__(content, **kwargs)
Is there something I'm missing with the TestDetail where I should separate the header from the data?
maybe like this?
return TestResponse (header, jobgroup)
and then alter TestResponse to include?
def __init__(self, header, data, **kwargs):
I don't know python/django. but it seems the "Value" you are talking about are actually attributes you want to assign to the element node. I posted the same on your /r/django thread about this.
I'm fairly new to django and Python and want to be able to export a list of items in my model i.e products. I'm looking at the documentation here - https://docs.djangoproject.com/en/dev/howto/outputting-csv/
I'm persuming I need will need to create a variable that stores all the data that I want. But not sure where it would within the snippet of code on the link above.
Apologies as this is a very noobish question but would really Any help at all.
Here is the code to my script so far:
import csv
from products.models import Product
from django.http import HttpResponse
def export_to_csv(request):
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="mytest.csv"'
Have a look at the python csv module.
You'll probably want to get the models fields with
def get_model_fields(model):
return model._meta.fields
Then use
getattr(instance, field.name)
to get the field values (as in this question).
Then you'll want something like
with open('your.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
# write your header first
for obj in YourModel.objects.all():
row = ""
for field in fields:
row += getattr(obj, field.name) + ","
writer.writerow(row)
It's a bit verbose (and untested), but it should give you an idea. (Oh and don't forget to close your file)
Depending on the scenario - you may want to have a CSV of your model. If you have access to the Django Admin site, you can plug in a generic action for any model displayed as a list (google: django admin actions)
http://djangosnippets.org/snippets/790/
If you're operating with a console (python manage.py ...), you can use such a script, which I just used:
(place it in: yourapp/management/commands/model2csv.py)
"""
Prints CSV of all fields of a model.
"""
from django.core.management.base import BaseCommand, CommandError
import csv
import sys
class Command(BaseCommand):
help = ("Output the specified model as CSV")
args = '[appname.ModelName]'
def handle(self, *app_labels, **options):
from django.db.models import get_model
app_name, model_name = app_labels[0].split('.')
model = get_model(app_name, model_name)
field_names = [f.name for f in model._meta.fields]
writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)
writer.writerow(field_names)
for instance in model.objects.all():
writer.writerow([unicode(getattr(instance, f)).encode('utf-8') for f in field_names])
This does not catch any exceptions etc., but as an Admin you won't cause them to be raised, right?
Use it like:
./manage.py model2csv my_ecommerce.Product > products.csv
You can also make a template to assist in formatting!
The template is a common Django template
from django.template import loader
def export_to_csv(request):
response = HttpResponse(mimetype='text/csv')
response['Content-Disposition'] = 'attachment; filename="products-list.csv"'
template = loader.get_template('templates/products_template.csb')
response.write(template.render(Context({'products': Products.objects.all()})))
return response
Using django.db.models.query.QuerySet.values results in more optimised queries for my use case.
import csv
from datetime import datetime
from django.http import HttpResponse
# Populate this list with your model's fields
# Replace MODEL with your model
fields = [f.name for f in MODEL._meta.fields]
# The following code will live inside your view
timestamp = datetime.now().isoformat()
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"
] = f"attachment; filename={timestamp}.csv"
writer = csv.writer(response)
# Write the header row
writer.writerow(fields)
# Replace MODEL with your model
for row in MODEL.objects.values(*fields):
writer.writerow([row[field] for field in fields])
return response
I use this on my code. A function called from view.
It automatically get model fields to make columns.
You can also customize the field list you want to export.
Function
import csv
from django.http import HttpResponse
from .models import Books
def export_qs_to_csv(model_class = None, qs = None, field_names = None):
if model_class and not qs:
qs = model_class.objects.all()
if qs and not model_class:
model_class = qs.model
meta = model_class._meta
if not field_names:
field_names = [field.name for field in meta.fields]
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename={}.csv'.format(meta)
writer = csv.writer(response)
writer.writerow(field_names)
for obj in qs:
row = writer.writerow([getattr(obj, field) for field in field_names])
return response
Usage
#user_passes_test(lambda u: u.is_superuser)
def export_books(request):
return export_qs_to_csv(model_class = Books)
# or
return export_qs_to_csv(qs = Books.objects.filter(published = True))
# or
return export_qs_to_csv(
qs = Books.objects.filter(published = True),
field_names = [
"title",
"price",
"publishing_date",
]
)
Original answer
It works, and it needs only to define model class in model_class variable.
This Django view let use downloads CSV. CSV name is Django_app.model_name.csv.
import csv
from django.http import HttpResponse
from .models import Trade
def export_to_csv(request):
# The only line to customize
model_class = Trade
meta = model_class._meta
field_names = [field.name for field in meta.fields]
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename={}.csv'.format(meta)
writer = csv.writer(response)
writer.writerow(field_names)
for obj in model_class.objects.all():
row = writer.writerow([getattr(obj, field) for field in field_names])
return response
Here is a potential solution, based on #tomasz-gandor 's answer, but updated to 2020:
"""
Prints CSV of all fields of a model.
"""
import csv
from django.core.management.base import BaseCommand, CommandError
class Command(BaseCommand):
help = ("Output the specified model as CSV")
def add_arguments(self, parser):
parser.add_argument('model',
nargs=1,
type=str,
help='Model name to export, like <app.model> or "members.Member"')
parser.add_argument('outfile',
nargs=1,
type=str,
help='Save path, like </path/to/outfile.csv> or "/data/members.csv"')
def handle(self, *app_labels, **options):
from django.apps import apps
app_name, model_name = options['model'][0].split('.')
model = apps.get_model(app_name, model_name)
field_names = [f.name for f in model._meta.fields]
writer = csv.writer(open(options['outfile'][0], 'w'), quoting=csv.QUOTE_ALL, delimiter=',')
writer.writerow(field_names)
for instance in model.objects.all():
writer.writerow([str(getattr(instance, f)) for f in field_names])
Can easily be used with:
python manage.py model2csv members.Member /data/members_export.csv
If you don't care about fieldnames and want all the fields, just do this.
with open('file_name.csv', 'w') as csvfile:
writer = csv.writer(csvfile)
for obj in YourModel.objects.values_list():
row = list(obj)
writer.writerow(row)
I combined some of the previous answers, because I needed to import some data from production and change some of it along the way. So here is my solution, which you can use to override some field values while writing the CSV file.
Export some queryset data into CSV file:
import csv
from myapp.models import MyModel
from user.models import User
# Make some queryset manually using Django shell:
user = User.objects.get(username='peterhil')
queryset = MyModel.objects.filter(user=user)
def query_to_csv(queryset, filename='items.csv', **override):
field_names = [field.name for field in queryset.model._meta.fields]
def field_value(row, field_name):
if field_name in override.keys():
return override[field_name]
else:
return row[field_name]
with open(filename, 'w') as csvfile:
writer = csv.writer(csvfile, quoting=csv.QUOTE_ALL, delimiter=',')
writer.writerow(field_names) # write the header
for row in queryset.values(*field_names):
writer.writerow([field_value(row, field) for field in field_names])
# Example usage:
query_to_csv(queryset, filename='data.csv', user=1, group=1)
Use this solution for model csv file.might being helpful
# Create the HttpResponse object with the appropriate CSV header.
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment;
filename="somefilename.csv"'
writer = csv.writer(response);
writer.writerow(["username","Email"]);
for i in User.objects.all():
writer.writerow([i.username,i.email])
return response
I used the django-queryset-csv package.
Follow these steps:
pip install django-queryset-csv
Your views.py:
import djqscsv
from products.models import Product
def get_csv(request):
qs = Product.objects.all()
return djqscsv.render_to_csv_response(qs)