I would like to use Python's Dictreader to import a csv file based on my Django model into the database.
My post model is:
class Post(models.Model):
STATUS_CHOICES = (
("draft", "Draft"),
("published", "Published")
);
slug = models.SlugField(max_length=250);
wine_id = models.IntegerField();
country = models.CharField(max_length=100);
description = models.TextField();
designation = models.TextField();
price = models.FloatField();
province = models.CharField(max_length=100);
region_1 = models.CharField(max_length=100);
region_2 = models.CharField(max_length=100);
variety = models.CharField(max_length=100);
winery = models.CharField(max_length=100);
objects = models.Manager();
class Meta:
ordering = ("id",);
def __str__(self):
return self.variety;
def get_absolute_url(self):
return reverse("post_detail", args=[self.id,
self.province,
self.slug]);
My script to read the csv data is:
class Command(BaseCommand):
# Shows this when the user types help:
help = "Loads data from wine_data.csv into our Post model.";
def handle(self, *args, **kwargs):
if Post.objects.exists():
print("Wine data already loaded... exiting...");
print(ALREADY_LOADED_ERROR_MESSAGE);
return;
print("Loading wine data for WCC.");
for row in DictReader(open("wine_data.csv")):
post = Post();
post.wine_id = row["wine_id"];
post.country = row["country"];
post.description = row["description"];
post.designation = row["designation"];
post.price = row["price"];
post.province = row["province"];
post.region_1 = row["region_1"];
post.region_2 = row["region_2"];
post.variety = row["variety"];
post.winery = row["winery"];
post.save();
However, when I use "python manage.py load_wine_data", the cmd says it is an unknown command. What am I doing wrong and how can I solve it?
Related
I export products in excel format using xlwt.But foreign key fields are exported as id.
How can I export foreign key fields with their actual values?
I want to export brand_id and author fields with their actual values.
Here is my product model :
class Product(models.Model):
id = models.AutoField(primary_key=True)
author = models.ForeignKey(User,on_delete= models.CASCADE, verbose_name='Product Author', null=True)
brand_id = models.ForeignKey(Brand,on_delete=models.CASCADE, verbose_name="Brand Names")
name = models.CharField(max_length=255, verbose_name="Product Name")
barcode = models.CharField(max_length=255, verbose_name="Barcode")
unit = models.CharField(max_length=255,verbose_name="Product Unit")
def __str__(self):
return self.name
Here is my export view:
def export_excel(request):
response = HttpResponse(content_type='application/ms-excel')
response['Content-Disposition'] = "attachment; filename=Products-" + str(datetime.datetime.now().date())+".xls"
wb = xlwt.Workbook(encoding="utf-8")
ws = wb.add_sheet('Products')
row_num = 0
font_style = xlwt.XFStyle()
font_style.font.bold = True
columns = ["Product Id","Product Author","Product Brand","Product Name","Product Barcode","Product Unit"]
for col_num in range(len(columns)):
ws.write(row_num,col_num,columns[col_num],font_style)
font_style = xlwt.XFStyle()
rows = Product.objects.filter(author = request.user).values_list("id","author","brand_id","name","barcode","unit")
for row in rows:
row_num +=1
for col_num in range(len(row)):
ws.write(row_num,col_num,str(row[col_num]), font_style)
wb.save(response)
Thanks for your help. Kind regards
You could use django-import-export to export the data from a model to an excel file. This library also supports other data types in case you need them in the future.
As described in the documentation of django-import-export you can create a resource, which can then be used to both import and export data into a model. Start by creating a resource:
from import_export import resources
from import_export.fields import Field
from .models import Product
class ProductResource(resources.ModelResource):
author = Field() # for field with foreignkeys you need to add them here
brand_id = Field() # for field with foreignkeys you need to add them here
fields = ["id", "author", "brand_id", "name", "barcode", "unit"]
export_order = ["id", "author", "brand_id", "name", "barcode", "unit"]
def dehydrate_author(self, product: Product) -> str:
return f"{product.author.name}" # probably need to adapt the name of the field
def dehydrate_brand_id(self, product: Product) -> str:
return f"{product.brand_id.brand}" # probably need to adapt the name of the field
This is also documented here: django-import-export advanced manipulation
Now you can use this ModelResource to export your data to any supported format, in your case an Excel file. Import your resource you've created earlier all you need to do to return this in your view is the following:
from django.http import HttpResponse
from .resource import ProductRes
#... other code in your view
project_resource = ProjectResource()
dataset = project_resource.export()
response = HttpResponse(dataset.xlsx, ontent_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
response["Content-Disposition"] = 'attachment; filename="projects_export.xlsx"'
I am trying to populate database from data in the csv file using a python script. I did some research but couldn't find a relevant example rather I found python packages that would load csv. And there some articles guiding on how to upload csv file which isn't what I wanted.
Following is a glimpse of load.csv file. There are 11 columns as you can see.
# models.py
from django.db import models
class Category(models.Model):
category = models.CharField(max_length=128)
def __str__(self):
return self.name
class State(models.Model):
state = models.CharField(max_length=25)
def __str__(self):
return self.name
class Region(models.Model):
region = models.CharField(max_length=25)
def __str__(self):
return self.region
class Iso(models.Model):
iso = models.CharField(max_length=5)
def __str__(self):
return self.iso
class Site(models.Model):
name = models.CharField(max_length=128)
year = models.IntegerField(null=True)
area = models.FloatField(null=True)
describe = models.TextField(max_length=500)
justify = models.TextField(max_length=500, null=True)
longitude = models.TextField(max_length=25, null=True)
latitude = models.TextField(max_length=25, null=True)
#one to many field
category = models.ForeignKey(Category, on_delete=models.CASCADE)
state = models.ForeignKey(State, on_delete=models.CASCADE)
region = models.ForeignKey(Region, on_delete=models.CASCADE)
iso = models.ForeignKey(Iso, on_delete=models.CASCADE)
def __str__(self):
return self.name
I'm not being able to make the relationship among the models while inserting data. Earlier, I had used get_or_create() method but I was recommended to not to use it as I had no defaultvalue to be given and used create() method. Somewhere I had found that, until a data is saved Primary key isn't generated so, I tried saving data to the field each time after reading data but that also didn't work. . Following is the code in many_load.py script. There's no problem in reading data.
import csv
#python3 manage.py runscript many_load
from unesco.models import Site, Category, Iso, Region, State
def run():
fhand = open('unesco/load.csv')
reader = csv.reader(fhand)
next(reader)
# Category.objects.all().delete()
# Iso.objects.all().delete()
# Region.objects.all().delete()
# State.objects.all().delete()
# Site.objects.all().delete()
for row in reader:
# print(row)
# print (len(row))
nm= Site.objects.create(name=row[0])
dsc = Site.objects.create(describe=row[1])
jst = Site.objects.create(justify=row[2])
yr = Site.objects.create(year=row[3])
lng = Site.objects.create(longitude=row[4])
lat = Site.objects.create(latitude=row[5])
area = Site.objects.create(area=row[6])
st = Site(category=row[7], state=row[8], region=row[9], iso=row[10],
name=nm, area=area, describe=dsc, justify=jst, longitude=lng, latitude=lat)
st.save()
But when I try running the script python3 manage.py runscript many_load it gives me the following error. Help please. You could also refer me some good articles for understanding more on this.
Traceback (most recent call last):
File "/home/bu113t/envs/djpro/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/home/bu113t/envs/djpro/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 413, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.IntegrityError: NOT NULL constraint failed: unesco_site.category_id
This error is occurring because you are passing a string value to the category field of Site but it is expecting a Category object as it is a ForeignKey field.
Not only that you have other problems in many_load.py
The following code has the correction of all of the problems. Hope it will work for you.
import csv
# python3 manage.py runscript many_load
from unesco.models import Site, Category, Iso, Region, State
def run():
fhand = open('unesco/load.csv')
reader = csv.reader(fhand)
next(reader)
# Category.objects.all().delete()
# Iso.objects.all().delete()
# Region.objects.all().delete()
# State.objects.all().delete()
# Site.objects.all().delete()
for row in reader:
# print(row)
# print (len(row))
nm = row[0]
dsc = row[1]
jst = row[2]
yr = row[3]
lng = row[4]
lat = row[5]
area = row[6]
category = Category.objects.create(category=row[7])
state = State.objects.create(state=row[8])
region = Region.objects.create(region=row[9])
iso = Iso.objects.create(iso=row[10])
st = Site(category=category, state=state, region=region, iso=iso,
name=nm, area=area, describe=dsc, justify=jst, longitude=lng, latitude=lat)
st.save()
I have data that has been scraped from a website, parsed, and cleaned into the pieces I need. The data is stored in a two dimensional list as such [[address1, name1, ip1], [address2, name2, ip2]]. The scraping and storing of the aforementioned data is done through a django command, and I would like to update my model with the same command as it validates the data. I also have a model with the following fields and attributes:
class MonitoredData(models.Model):
external_id = models.UUIDField(
primary_key = True,
default = uuid.uuid4,
editable = False)
mac_address = models.CharField(max_length=12)
ipv4_address = models.CharField(max_length=200)
interface_name = models.CharField(max_length=200)
created_at = models.DateTimeField(auto_now_add=True)
update_at = models.DateTimeField(auto_now=True)
address1 needs to go in the mac_address field, name1 needs to go into the interface_name field, and ip1 needs to go into ipv4_address field. The other fields need to auto-fill according to their attributes.
The django command that grabs and parses the data is:
class Command(BaseCommand):
def handle(self, *args, **options):
url1 = 'https://next.json-generator.com/api/json/get/41wV8bj_O'
url2 = 'https://next.json-generator.com/api/json/get/Nk48cbjdO'
res1 = requests.get(url1)
data1 = str(res1.content)
res2 = requests.get(url2)
data2 = str(res2.content)
parsedData1 = parse1(data1)
goodMac1 = []
badMac1 = []
for k in parsedData1:
if len(k[0]) == 12:
if match(k[0]):
goodMac1.append(k)
else:
badMac1.append(k)
parsedData2 = parse2(data2)
goodMac2 = []
badMac2 = []
for j in parsedData2:
if len(j[0]) == 12:
if match(j[0]):
goodMac2.append(j)
else:
badMac2.append(j)
I'd like to store the data into the database instead of appending to the goodMac list in the nested if statement.
Any help with this would be greatly appreciated, I am using Python 3.7.5 and Django 3.0.5
I figured it out! I hope this will save someone all the time and trouble I went through solving this, the solution, as I suspected, was fairly trivial once I found it. You import your model, instantiate an object of it, then update the fields and use the save() function. Here is the fixed code.
import requests
from django.core.management.base import BaseCommand, CommandError
from Monitor.models import *
from Monitor.parse1 import parse1
from Monitor.parse2 import parse2
from Monitor.matcher import match
class Command(BaseCommand):
def handle(self, *args, **options):
url1 = 'https://next.json-generator.com/api/json/get/41wV8bj_O'
url2 = 'https://next.json-generator.com/api/json/get/Nk48cbjdO'
res1 = requests.get(url1)
data1 = str(res1.content)
res2 = requests.get(url2)
data2 = str(res2.content)
parsedData1 = parse1(data1)
goodMac1 = []
badMac1 = []
for k in parsedData1:
if len(k[0]) == 12:
if match(k[0]):
monInter = MonitoredData()
monInter.mac_address = k[0]
monInter.interface_name = k[1]
monInter.ipv4_address = k[2]
monInter.save()
goodMac1.append(k)
else:
badMac1.append(k)
parsedData2 = parse2(data2)
goodMac2 = []
badMac2 = []
for j in parsedData2:
if len(j[0]) == 12:
if match(j[0]):
goodMac2.append(j)
else:
badMac2.append(j)
Here are links to the documentation I ultimately ended up using:
https://docs.djangoproject.com/en/3.0/ref/models/instances/#django.db.models.Model.save
https://docs.djangoproject.com/en/3.0/topics/db/models/
I'm trying to calculate the pending amount via models and export result in the csv. But the csv shows an empty column for amountpending
class FinancePendingResource(resources.ModelResource):
invoiceNumber = Field(attribute='invoiceNumber', column_name='Invoice Number')
student = Field(attribute='student', column_name='Student')
Schedule = Field(attribute='Schedule', column_name='Schedule')
TotalAmount = Field(attribute='TotalAmount', column_name='Total Value(PKR ₨)')
issueDate = Field(attribute='issueDate', column_name='Issue Date')
dueDate = Field(attribute='dueDate', column_name='Due Date')
amountPaid = Field(attribute='amountPaid', column_name='Amount Paid (PKR ₨)')
class Meta:
model = FinancePending
import_id_fields = ('invoiceNumber',)
fields = ('invoiceNumber', 'student', 'amountPaid', 'issueDate', 'dueDate', 'Schedule', 'TotalAmount',
'AmountPending',)
exclude = ('id',)
skip_unchanged = True
report_skipped = True
def before_export(self, queryset, *args, **kwargs):
amount_paid = FinancePending.objects.values_list('amountPaid', flat=True)
amount_paid = list(amount_paid)
total_amount = FinancePending.objects.values_list('TotalAmount', flat=True)
total_amount = list(total_amount)
# total - paid
TotalFee = [float(s.replace(',', '')) for s in total_amount]
AmountPaid = [float(s.replace(',', '')) for s in amount_paid]
def Diff(li1, li2):
return (list(set(li1) - set(li2)))
amount_pending = Diff(TotalFee, AmountPaid)
finance_pending = FinancePending()
i = 1
while i <= len(amount_pending):
FinancePending.objects.filter(invoiceNumber=i).update(AmountPending=str(amount_pending[i]))
i = i + 1
queryset.refresh_from_db()
Assuming that you have the data to compute amountPending already in the dataset, perhaps you don't need to read from the DB: you could calculate the amount by processing the dataset in memory. This could be done in after_export(). Then you can added the computed column to the dataset.
Perhaps tablib's dynamic columns can assist in adding the amountPending column:
import decimal
import tablib
headers = ('invoiceNumber', 'amountPaid', 'totalAmount')
rows = [
('inv100', '100.00', "500.00"),
('inv101', '200.00', "250.00")
]
def amount_pending(row):
return decimal.Decimal(row[2]) - decimal.Decimal(row[1])
data = tablib.Dataset(*rows, headers=headers)
data.append_col(amount_pending, header="amountPending")
print(data)
This will produce the following:
invoiceNumber|amountPaid|totalAmount|amountPending
-------------|----------|-----------|-------------
inv100 |100.00 |500.00 |400.00
inv101 |200.00 |250.00 |50.00
I tried to create a table using a class that is not related to my database in django and this class is stored in models.py as shown below (InfoServer is the class). What I wanted to do is to use this class to populate my table using django_tables2. Add models.Model as a parameter is not an option because I don't want to save this class in the database.
Whenever I define the model = InfoServer in tables.py I got this error and I suppose it's because InfoServer did not take models.Model as a parameter.
TypeError: descriptor 'repr' of 'object' object needs an argument
Any help is appreciated.
models.py
class TestServeur(models.Model):
nom = models.CharField(max_length=200)
pid = models.CharField(max_length=200)
memoire = models.IntegerField(null=True)
class InfoServer:
# "This is a class to test my knowledge of python"
def __init__(self,p = '',c = 0,m = 0):
self.pid = p
self.cpu = c
self.memoire = m
def getData(self):
return ("A server with %s memory and %s cpu" % (self.cpu,self.memoire))
views.py
def index(request):
return HttpResponse("Hello, world. You're at the index.")
def cpu_view(request):
liste = []
proc1 = Popen(['ps','-eo','pid,%cpu,%mem,comm'], stdout=PIPE, stderr=PIPE)
proc2 = Popen(['grep','java'], stdin=proc1.stdout, stdout=PIPE)
proc1.stdout.close()
for line in iter(proc2.stdout.readlines()):
clean_line = line.decode("utf-8")
info_utiles = clean_line.split()
pid,cpu,mem,*rest = info_utiles
i1 = InfoServer(pid,cpu,mem)
liste.append(i1)
table = TestServeur(liste)
RequestConfig(request).configure(table)
return render(request, 'server/cpu.html', {'output': table})
tables.py
class TableServeur(tables.Table):
class Meta:
# model = InfoServer
fields = ['pid', 'memory', 'cpu']
template_name = 'django_tables2/bootstrap4.html'
As I can see, InfoServer class is not a Django Model. Also I don't think you need to use that directly anyway. So, you can simply provide a list with dictionary, and render it in template with table.
First, we need to update Table class and remove Meta class from it, as we are not going to use any django models.
class TableServeur(tables.Table):
pid = tables.Column()
memory = tables.Column()
cpu = tables.Column()
Now, adding a new object method to return dictionary from InfoServer class:
class InfoServer:
# "This is a class to test my knowledge of python"
def __init__(self,p = '',c = 0,m = 0):
self.pid = p
self.cpu = c
self.memoire = m
def getData(self):
return ("A server with %s memory and %s cpu" % (self.cpu,self.memoire))
def get_dict_data(self):
return {'pid': self.pid, 'cpu': self.cpu, 'memory': self.memoire}
Finally, update the view:
for line in iter(proc2.stdout.readlines()):
clean_line = line.decode("utf-8")
info_utiles = clean_line.split()
pid,cpu,mem,*rest = info_utiles
i1 = InfoServer(pid,cpu,mem)
liste.append(i1.get_dict_data())
table = TestServeur(liste)
return render(request, 'server/cpu.html', {'output': table})
More info can be found in documentation on how you can populate table with data.