I built an app which allows the user to upload a file.
On the other hand,
I have some python script which takes a text file, convert it to CSV and do pandas. This script works perfectly when running in the terminal.
Now I want to apply that python script to the file upload in Django and show that file in httpResponse and make available to download it.
python script
import csv
import pandas as pd
df = pd.read_csv('raw_df.txt', delimiter = '\t' , usecols=['Sample','Cohort','Metabolite Name','Intensity'])
df = df[df['Cohort'].str.contains("std")]
df = df.groupby(['Cohort', 'Metabolite Name'])['Intensity'].sum().reset_index()
df = df[['Cohort','Intensity']]
c = 'Cohort'
s = df.set_index([c, df.groupby(c).cumcount() + 2]).Intensity
df = s.unstack().add_prefix('Intensity').reset_index()
df.to_csv()
print df;
views.py
from django.shortcuts import render, redirect
from django.conf import settings
from django.core.files.storage import FileSystemStorage
from uploads.core.models import Document
from uploads.core.forms import DocumentForm
def home(request):
documents = Document.objects.all()
return render(request, 'core/home.html', { 'documents': documents })
def model_form_upload(request):
if request.method == 'POST':
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
form.save()
return redirect('home')
else:
form = DocumentForm()
return render(request, 'core/model_form_upload.html', {
'form': form
})
Models.py
class Document(models.Model):
document = models.FileField(upload_to='documents/')
template-upload page
{% extends 'base.html' %}
{% block content %}
<form method="post" enctype="multipart/form-data">
{% csrf_token %}
{{ form.as_p }}
<button type="submit">Upload</button>
</form>
<p>Return to home</p>
{% endblock %}
Wrap your file processing in a method:
import csv
import pandas as pd
def process_file(file_handle):
df = pd.read_csv(file_handle, delimiter = '\t' , usecols['Sample','Cohort','Metabolite Name','Intensity'])
df = df[df['Cohort'].str.contains("std")]
df = df.groupby(['Cohort', 'Metabolite Name'])['Intensity'].sum().reset_index()
df = df[['Cohort','Intensity']]
c = 'Cohort'
s = df.set_index([c, df.groupby(c).cumcount() + 2]).Intensity
df = s.unstack().add_prefix('Intensity').reset_index()
return df.to_csv()
in your view:
...
if form.is_valid():
document = form.save()
# call to the new method
csv = process_file(document.document)
response = HttpResponse(csv, content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=result.csv'
return response
...
Related
I have an Altair chart which I want to render in a Django Template. The chart is converted into a json object in the views. Here is the code
views.py
def home(request):
if request.method=='POST':
year = request.POST['year']
df, cus_dict = generate_df(year)
bar_obj = barGraphAltair(year, df)
bar_obj = json.loads(bar_obj.to_json())
print(bar_obj)
context = {'bar': bar_obj}
return render(request, 'index.html', context=context)
return render(request, 'index.html')
template
<div id='altair-viz'>
{% if bar %}
{{ bar|safe }}
{% endif %}
</div>
This just prints the json in the template. I know I have to use Vega to render the graph but I am not sure how to do that in jinja syntax
A temp solution
One way I got this to work, is by creating a different view and calling that view in the template as follows
views.py
def renderAltair(request):
df, cus_dict = generate_df('2017')
bar_obj = barGraphAltair('2017', df)
bar_obj = json.loads(bar_obj.to_json())
return JsonResponse(bar_obj)
template
<script>
vegaEmbed('#altair-viz', "{% url 'altair' %}")
</script>
This works, but as you can see from the original code, I get the year by submitting a form and passing that to the function for generating the graph. So I need the graph to be created in the home view
You can try this way.
def home(request):
if request.method=='POST':
year = request.POST['year']
context = {'year': year}
return render(request, 'index.html', context=context)
return render(request, 'index.html', {})
Not passing data in home view, will get that using json view.
template
<div id='altair-viz' data-url="{% url 'altair' %}?year={{year}}"></div>
<script>
var data_url = $('#altair-viz').attr('data-url');
vegaEmbed('#altair-viz', data_url)
</script>
and get data function
def renderAltair(request):
year = request.GET.get('year')
df, cus_dict = generate_df(year)
bar_obj = barGraphAltair(year, df)
bar_obj = json.loads(bar_obj.to_json())
return JsonResponse(bar_obj)
I spent a few hours trying to display the image in Django, I am trying to display a wordcloud in Django.
Here's my views.py:
import tweepy
from tweepy.auth import OAuthHandler
from .models import Tweet
from .models import Dates
from django.core.paginator import Paginator, EmptyPage, InvalidPage
from django.shortcuts import render
from django.db import models, transaction
from django.db.models import Q
import os
import tweepy as tw
import pandas as pd
import nltk
from .forms import TweetIDForm
from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
import pandas as pd
from io import StringIO
from django import template
import urllib, base64
import io
import requests
consumer_key = 'consumer_key'
consumer_secret = 'consumer_secret'
access_token = 'access_token'
access_token_secret = 'access_token_secret'
def clean_tweet_id(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = TweetIDForm(request.POST)
# check whether it's valid:
if form.is_valid():
# process the data in form.cleaned_data as required
tweet_id = form.cleaned_data.get("tweet_id")
auth = tw.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tw.API(auth, wait_on_rate_limit=True)
twtjson = requests.get('https://publish.twitter.com/oembed?url=' + tweet_id + '&omit_script=true')
tweets = twtjson.json()
tweet = tweets.get('html')
ida = tweet_id.split('/')[-1].split('?')[0]
identification = int(ida)
status = api.get_status(identification)
user_name = status.user.screen_name
words = ""
for twt in tweepy.Cursor(api.search,q='to:'+user_name, result_type='recent', timeout=999999).items(100):
if hasattr(twt, 'in_reply_to_status_id_str'):
if twt.in_reply_to_status_id_str==ida:
tokens = nltk.word_tokenize(twt.text)
txt = nltk.Text(tokens)
fd_words = nltk.FreqDist(txt)
mostcommonwordsoftxt = fd_words.most_common(100)
wordc = str(mostcommonwordsoftxt)
print(wordc)
wc = WordCloud(width = 800, height = 400, background_color ='white', stopwords = STOPWORDS, max_font_size = 12)
wcloud = wc.generate(wordc)
cloudtofile = wcloud.to_file('Wordcloud.png')
return render(request, 'tweet/tweet-list.html', {'form': form, 'tweet_id':tweet_id, 'tweets': tweet,'image':cloudtofile})
else:
form = TweetIDForm()
return render(request, 'tweet/tweet-list.html', {'form': form})
def home(request):
context = {}
context['word'] = clean_tweet_id(request)
return render(request, 'tweet/dashboard.html', context)
My HTML template:
<html lang="en">
<h1>Tweets</h1>
<form action="" method="post">
{% csrf_token %}
{{ form }}
<button type="submit">Submit</button>
</form>
<body>
{% if tweets %}
{{tweets|safe}}<script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
{% load static %}
<img src= "{% static 'Wordcloud.png' %}" />
{% endif %}
</body>
</html>
I am trying to display the wordcloud that is generated through Django. However, for some reason, the image does not display, despite the fact that the image does get generated, its just the user cannot see the image in the webserver.
I am using Flask, Pandas, WTForms.
I want to populate a selectfield with data from a csv file.
I have
routes.py
forms.py
tabular.html
I create form in forms.py then via routes.py display it into tabular.html.
Forms.py
class QueryTableForm(FlaskForm):
state_select = []
district_select = []
country = SelectField(u'Country', choices=state_select)
state = SelectField(u'State', choices=district_select)
submit = SubmitField(u'Search')
Routes.py
#csv imports
country_wise_data = list(pd.read_csv('country_data.csv').values)
state_wise_data = list(pd.read_csv('state_data.csv').values)
district_wise_data = list(pd.read_csv('district_data.csv').values)
#app.route("/tabular", methods=['GET', 'POST'])
#login_required
def tabular():
form = QueryTableForm()
if form.validate_on_submit():
post = Post(title=form.country.data, content=form.state.data)
flash('Your Selection is Registered!', 'success')
return render_template('tabular.html', state_wise_data = state_wise_data,
country_wise_data=country_wise_data, district_wise_data=district_wise_data,
title='Tabular Data', form=form, legend='Tabular',
max=17000, date=date)
tabular.html
<form method="POST" action="">
...
{{ form.country.label }} {{ form.country }}
{{ form.state.label }} {{ form.state}}
...
</form>
csv file format
State,Confirmed,Recovered,Deaths,Active,Last_Updated_Time,,,,,,
USA,3458996,2834473,58245,564746,12-04-2021 22.57,,,,,,
India,1180398,1123133,4815,52128,13-04-2021 19.02,,,,,,
Now I want to pull only the values "USA","India" etc and populate in the selectfield "form.country". I have tried to do a for loop on routes.py. But how do I pass it on to the HTML file ? I also tried to do a for loop via Jinja on the HTML file.
How do I dynamically populate the "choices" attribute of the WTF selectfield from the html file ?
#app.route("/tabular", methods=['GET', 'POST'])
#login_required
def tabular():
form = QueryTableForm()
form.country.choices = country_wise_data
if form.validate_on_submit():
post = Post(title=form.country.data, content=form.state.data)
flash('Your Selection is Registered!', 'success')
return render_template('tabular.html', state_wise_data = state_wise_data,
country_wise_data=country_wise_data, district_wise_data=district_wise_data,
title='Tabular Data', form=form, legend='Tabular',
max=17000, date=date)
I am trying to upload a CSV file into my Django model. Although the upload of the data works fine (all the rows get copied into the database), at the end Django returns a ValidationError ["'' value must be a decimal number."] error message.
From the local vars section of the error message I kind of get the reason - when the iteration reaches the end of the rows containing data, there is obviously no decimal number. So Django throws an error. However, I do not understand why, as there is always a last row after which there is no more data. I fiddled around a bit to try to find the problem. A method that worked is so copy the whole data from the original CSV into a new CSV - there was no error message any more. I would love to accomplish the whole process with the original CSV file and no error message! Would appreciate any help.
My CSV files are CSV UTF-8 and they are saved in Excel
models.py
from django.db import models
class Testdata3(models.Model):
key = models.CharField(max_length=100, primary_key=True)
assetclass = models.CharField(max_length=25)
value = models.DecimalField(max_digits=25,decimal_places=10)
performance = models.DecimalField(max_digits=25,decimal_places=10)
def __str__(self):
return self.key
views.py
from django.shortcuts import render
from .models import Testdata3
import csv, io
from django.contrib import messages
def file_upload(request):
template = "upload.html"
prompt = {
'order': 'Order of the CSV should be "placeholder_1", "placeholder_2", "placeholder_3" '
}
if request.method == "GET":
return render(request, template, prompt)
csv_file = request.FILES['file']
if not csv_file.name.endswith('.csv'):
messages.error(request, 'This is not a csv file')
data_set = csv_file.read().decode('UTF-8')
io_string = io.StringIO(data_set)
next(io_string)
for column in csv.reader(io_string, delimiter=';', quotechar="|"):
_, created = Testdata3.objects.update_or_create(
key = column[0],
defaults = {
'key' : column[0],
'assetclass' : column[10],
'value' : column[16],
'performance' : column[18],
}
)
context = {}
return render(request, template, context)
upload.html
{% if messages %}
{% for message in messages %}
<div>
<strong>{{message|safe}}</strong>
</div>
{% endfor %}
{%else %}
{{ order }}
<form method="post" enctype="multipart/form-data">
{% csrf_token %}
<label>Upload a file</label>
<input type="file" name="file">
<p>Only accepts CSV files</p>
<button type="submit">Upload</button>
</form>
{% endif %}
I've found a solution! I included an if-statement, which checks if the current iteration has only empty values, and if yes it jumps to the next iteration
# ...
views.py
for column in csv.reader(io_string, delimiter=';', quotechar="|"):
# Check for empty rows in CVS and jump to next iteration if empty
if all(elem == "" for elem in column):
next
else:
_, created = Testdata3.objects.update_or_create(
key = column[0],
defaults = {
'key' : column[0],
'assetclass' : column[10],
'value' : column[16],
'performance' : column[18],
}
)
# ...
I would like to upload a file, parse it to .txt and then store it in the disk via a django application. I've tried quiet a few things that didn't work. First I want to make the storage work. Here's my code. I would appreciate any help.
Index.html
<form name='F' action='AjoutTelechargement' method='POST'
enctype='multipart/form-data'>
{% csrf_token %}
<input type="file" name="filedir" style="color:white" size="100" >
<input type="submit" class="btn .btn-ajouter btn-xl page-scroll"
value="Ajouter" />
<script type="text/javascript" src="/media/js/jquery-1.10.2.min.js">
</script>
<script>
function myFun() {
$.get('AjoutTelechargement/'
, "value":"get_the_value_from_web"}
, function(ret) {
return ret;
});
}
models.py
from django.db import models
from django.core.files.storage import FileSystemStorage
key_store = FileSystemStorage(location='myFiles/')
class FileUpload(models.Model):
file = models.FileField(storage=key_store, upload_to='myFiles/%Y-%m-%d',
blank=False, null=False)
urls.py
from django.conf.urls import url
from django.contrib import admin
from omicCV import views as omicCV_views
from blog import views as blog_views
urlpatterns = [
url(r'^$', omicCV_views.index),
url(r'^admin', admin.site.urls),
url(r'^AjoutTelechargement$', blog_views.AjoutTelechargement),
]
views.py
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse
from .models import FileUpload
#csrf_exempt
def AjoutTelechargement(request):
if request.method == 'POST':
form = FileUpload(request.POST, request.FILES)
form.save()
return HttpResponse("done!")
I am trying to parse the uploaded pdf files to txt:
def AjoutTelechargement(request):
if request.method == 'POST' and request.FILES['filedir']:
filedir = request.FILES['filedir']
fs = FileSystemStorage(location='myFiles/')
for filename in filedir:
path=filename
if b"pdf" in filename:
print("repère2")
head, tail = os.path.split(path)
var = b"\\"
tail = tail.replace(b".pdf", b".txt")
name = head + var + tail
content = ""
pdf = PyPDF2.PdfFileReader(path, bool('rb'))
for i in range(0, pdf.getNumPages()):
content += pdf.getPage(i).extractText()
print(strftime("%H:%M:%S"), " pdf -> txt ")
with (name, 'a') as f:
text_file_doc = {"file_name": "test_file_name.txt",
"contents": f.write(content.encode('ascii', 'replace').decode('UTF-8'))}
fs.save(filedir.name, text_file_doc)
return HttpResponse("done!")
return HttpResponse("undone")
But got this error in this line: pdf = PyPDF2.PdfFileReader(path, bool('rb'))
Exception Value:
'bytes' object has no attribute 'seek'
Exception Location: C:\Users\RAHMA\omicroneCV\lib\site-packages\PyPDF2\pdf.py in read, line 1689