I am reading a json file from a website and if the record is not in my Customers queryset I want to create a new Customer for that record. What is happening is when I iterate over the queryset, Django is trying to create a new Customer even when it is already in the queryset.
Please see my code below:
from rest_framework import generics
from customer.models import Customers
from .serializers import CustomersSerializer
import json
import urllib.request
class CustomerAPIView(generics.ListAPIView):
j = urllib.request.urlopen("https://web.njit.edu/~jsd38/json/customer.json")
customer_data = json.load(j)
queryset1 = Customers.objects.values_list('CustomerId', flat=True)
for customer in customer_data:
if customer["#Id"] not in queryset1.iterator():
CustomerId = customer["#Id"]
Name = customer["Name"]
PhoneNumber = customer["PhoneNumber"]
EmailAddress = customer["EmailAddress"]
StreetLine = customer["Address"]["StreetLine1"]
City = customer["Address"]["City"]
StateCode = customer["Address"]["StateCode"]
PostalCode = customer["Address"]["PostalCode"]
cus = Customers()
cus.CustomerId = CustomerId
cus.Name = Name
cus.PhoneNumber = PhoneNumber
cus.EmailAddress = EmailAddress
cus.StreetLine = StreetLine
cus.City = City
cus.StateCode = StateCode
cus.PostalCode = PostalCode
cus.save()
queryset = Customers.objects.all()
serializer_class = CustomersSerializer
Your JSON is returning strings for the "#Id" key, I'm assuming your model Customers has integers as CustomerId field.
You should convert them to str or int:
if int(customer["#Id"]) not in queryset1:
...
Related
I export products in excel format using xlwt.But foreign key fields are exported as id.
How can I export foreign key fields with their actual values?
I want to export brand_id and author fields with their actual values.
Here is my product model :
class Product(models.Model):
id = models.AutoField(primary_key=True)
author = models.ForeignKey(User,on_delete= models.CASCADE, verbose_name='Product Author', null=True)
brand_id = models.ForeignKey(Brand,on_delete=models.CASCADE, verbose_name="Brand Names")
name = models.CharField(max_length=255, verbose_name="Product Name")
barcode = models.CharField(max_length=255, verbose_name="Barcode")
unit = models.CharField(max_length=255,verbose_name="Product Unit")
def __str__(self):
return self.name
Here is my export view:
def export_excel(request):
response = HttpResponse(content_type='application/ms-excel')
response['Content-Disposition'] = "attachment; filename=Products-" + str(datetime.datetime.now().date())+".xls"
wb = xlwt.Workbook(encoding="utf-8")
ws = wb.add_sheet('Products')
row_num = 0
font_style = xlwt.XFStyle()
font_style.font.bold = True
columns = ["Product Id","Product Author","Product Brand","Product Name","Product Barcode","Product Unit"]
for col_num in range(len(columns)):
ws.write(row_num,col_num,columns[col_num],font_style)
font_style = xlwt.XFStyle()
rows = Product.objects.filter(author = request.user).values_list("id","author","brand_id","name","barcode","unit")
for row in rows:
row_num +=1
for col_num in range(len(row)):
ws.write(row_num,col_num,str(row[col_num]), font_style)
wb.save(response)
Thanks for your help. Kind regards
You could use django-import-export to export the data from a model to an excel file. This library also supports other data types in case you need them in the future.
As described in the documentation of django-import-export you can create a resource, which can then be used to both import and export data into a model. Start by creating a resource:
from import_export import resources
from import_export.fields import Field
from .models import Product
class ProductResource(resources.ModelResource):
author = Field() # for field with foreignkeys you need to add them here
brand_id = Field() # for field with foreignkeys you need to add them here
fields = ["id", "author", "brand_id", "name", "barcode", "unit"]
export_order = ["id", "author", "brand_id", "name", "barcode", "unit"]
def dehydrate_author(self, product: Product) -> str:
return f"{product.author.name}" # probably need to adapt the name of the field
def dehydrate_brand_id(self, product: Product) -> str:
return f"{product.brand_id.brand}" # probably need to adapt the name of the field
This is also documented here: django-import-export advanced manipulation
Now you can use this ModelResource to export your data to any supported format, in your case an Excel file. Import your resource you've created earlier all you need to do to return this in your view is the following:
from django.http import HttpResponse
from .resource import ProductRes
#... other code in your view
project_resource = ProjectResource()
dataset = project_resource.export()
response = HttpResponse(dataset.xlsx, ontent_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
response["Content-Disposition"] = 'attachment; filename="projects_export.xlsx"'
I created a script to avoid creating duplicate objects but it still created the same objects when I run the command 3 times it creates them 3 times over and over again. I would like you to help me and know what is wrong with my code.
from django.core.management.base import BaseCommand
from jobs.models import Job
import json
from datetime import datetime
import dateparser
class Command(BaseCommand):
help = 'Set up the database'
def handle(self, *args: str, **options: str):
with open('static/newdata.json', 'r') as handle:
big_json = json.loads(handle.read())
for item in big_json:
if len(item['description']) == 0:
print('Not created. Description empty')
continue
dt = dateparser.parse(item['publication_date'])
existing_job = Job.objects.filter(
job_title = item['job_title'],
company = item['company'],
company_url = item['company_url'],
description = item['description'],
publication_date = dt,
salary = item['salary'],
city = item['city'],
district = item['district'],
job_url = item['job_url'],
job_type = item['job_type'],
)
if existing_job.exists() is True:
print('This Job already exist')
else:
Job.objects.create(
job_title = item['job_title'],
company = item['company'],
company_url = item['company_url'],
description = item['description'],
publication_date = dt,
salary = item['salary'],
city = item['city'],
district = item['district'],
job_url = item['job_url'],
job_type = item['job_type'],
)
self.stdout.write(self.style.SUCCESS('added jobs!'))
Have you tried using built-in field validation unique=True?
https://docs.djangoproject.com/en/3.1/ref/models/fields/#unique
try
if existing_job.exists():
instead of
if existing_job.exists() is True:
because .exists() returns boolean itself
Have you tried using unique_together without the publication_date field? Docs
# models.py
class Job(models.Model):
# Your fields here...
class Meta:
unique_together = [[
'job_title',
'company',
'company_url',
'description',
'salary',
'city',
'district',
'job_url',
'job_type'
]]
dt = dateparser.parse(item['publication_date'])
new_date = date(dt.year, dt.month, dt.day)
here was the problem. I am scraping the publication date in this format ('1 Week Ago') and then I was changing to a date and time format. and when I run the script again the time of the conversion is a different time. so that's why the job is created again. because is not the same because the time creation
I have data that has been scraped from a website, parsed, and cleaned into the pieces I need. The data is stored in a two dimensional list as such [[address1, name1, ip1], [address2, name2, ip2]]. The scraping and storing of the aforementioned data is done through a django command, and I would like to update my model with the same command as it validates the data. I also have a model with the following fields and attributes:
class MonitoredData(models.Model):
external_id = models.UUIDField(
primary_key = True,
default = uuid.uuid4,
editable = False)
mac_address = models.CharField(max_length=12)
ipv4_address = models.CharField(max_length=200)
interface_name = models.CharField(max_length=200)
created_at = models.DateTimeField(auto_now_add=True)
update_at = models.DateTimeField(auto_now=True)
address1 needs to go in the mac_address field, name1 needs to go into the interface_name field, and ip1 needs to go into ipv4_address field. The other fields need to auto-fill according to their attributes.
The django command that grabs and parses the data is:
class Command(BaseCommand):
def handle(self, *args, **options):
url1 = 'https://next.json-generator.com/api/json/get/41wV8bj_O'
url2 = 'https://next.json-generator.com/api/json/get/Nk48cbjdO'
res1 = requests.get(url1)
data1 = str(res1.content)
res2 = requests.get(url2)
data2 = str(res2.content)
parsedData1 = parse1(data1)
goodMac1 = []
badMac1 = []
for k in parsedData1:
if len(k[0]) == 12:
if match(k[0]):
goodMac1.append(k)
else:
badMac1.append(k)
parsedData2 = parse2(data2)
goodMac2 = []
badMac2 = []
for j in parsedData2:
if len(j[0]) == 12:
if match(j[0]):
goodMac2.append(j)
else:
badMac2.append(j)
I'd like to store the data into the database instead of appending to the goodMac list in the nested if statement.
Any help with this would be greatly appreciated, I am using Python 3.7.5 and Django 3.0.5
I figured it out! I hope this will save someone all the time and trouble I went through solving this, the solution, as I suspected, was fairly trivial once I found it. You import your model, instantiate an object of it, then update the fields and use the save() function. Here is the fixed code.
import requests
from django.core.management.base import BaseCommand, CommandError
from Monitor.models import *
from Monitor.parse1 import parse1
from Monitor.parse2 import parse2
from Monitor.matcher import match
class Command(BaseCommand):
def handle(self, *args, **options):
url1 = 'https://next.json-generator.com/api/json/get/41wV8bj_O'
url2 = 'https://next.json-generator.com/api/json/get/Nk48cbjdO'
res1 = requests.get(url1)
data1 = str(res1.content)
res2 = requests.get(url2)
data2 = str(res2.content)
parsedData1 = parse1(data1)
goodMac1 = []
badMac1 = []
for k in parsedData1:
if len(k[0]) == 12:
if match(k[0]):
monInter = MonitoredData()
monInter.mac_address = k[0]
monInter.interface_name = k[1]
monInter.ipv4_address = k[2]
monInter.save()
goodMac1.append(k)
else:
badMac1.append(k)
parsedData2 = parse2(data2)
goodMac2 = []
badMac2 = []
for j in parsedData2:
if len(j[0]) == 12:
if match(j[0]):
goodMac2.append(j)
else:
badMac2.append(j)
Here are links to the documentation I ultimately ended up using:
https://docs.djangoproject.com/en/3.0/ref/models/instances/#django.db.models.Model.save
https://docs.djangoproject.com/en/3.0/topics/db/models/
I'm trying to calculate the pending amount via models and export result in the csv. But the csv shows an empty column for amountpending
class FinancePendingResource(resources.ModelResource):
invoiceNumber = Field(attribute='invoiceNumber', column_name='Invoice Number')
student = Field(attribute='student', column_name='Student')
Schedule = Field(attribute='Schedule', column_name='Schedule')
TotalAmount = Field(attribute='TotalAmount', column_name='Total Value(PKR ₨)')
issueDate = Field(attribute='issueDate', column_name='Issue Date')
dueDate = Field(attribute='dueDate', column_name='Due Date')
amountPaid = Field(attribute='amountPaid', column_name='Amount Paid (PKR ₨)')
class Meta:
model = FinancePending
import_id_fields = ('invoiceNumber',)
fields = ('invoiceNumber', 'student', 'amountPaid', 'issueDate', 'dueDate', 'Schedule', 'TotalAmount',
'AmountPending',)
exclude = ('id',)
skip_unchanged = True
report_skipped = True
def before_export(self, queryset, *args, **kwargs):
amount_paid = FinancePending.objects.values_list('amountPaid', flat=True)
amount_paid = list(amount_paid)
total_amount = FinancePending.objects.values_list('TotalAmount', flat=True)
total_amount = list(total_amount)
# total - paid
TotalFee = [float(s.replace(',', '')) for s in total_amount]
AmountPaid = [float(s.replace(',', '')) for s in amount_paid]
def Diff(li1, li2):
return (list(set(li1) - set(li2)))
amount_pending = Diff(TotalFee, AmountPaid)
finance_pending = FinancePending()
i = 1
while i <= len(amount_pending):
FinancePending.objects.filter(invoiceNumber=i).update(AmountPending=str(amount_pending[i]))
i = i + 1
queryset.refresh_from_db()
Assuming that you have the data to compute amountPending already in the dataset, perhaps you don't need to read from the DB: you could calculate the amount by processing the dataset in memory. This could be done in after_export(). Then you can added the computed column to the dataset.
Perhaps tablib's dynamic columns can assist in adding the amountPending column:
import decimal
import tablib
headers = ('invoiceNumber', 'amountPaid', 'totalAmount')
rows = [
('inv100', '100.00', "500.00"),
('inv101', '200.00', "250.00")
]
def amount_pending(row):
return decimal.Decimal(row[2]) - decimal.Decimal(row[1])
data = tablib.Dataset(*rows, headers=headers)
data.append_col(amount_pending, header="amountPending")
print(data)
This will produce the following:
invoiceNumber|amountPaid|totalAmount|amountPending
-------------|----------|-----------|-------------
inv100 |100.00 |500.00 |400.00
inv101 |200.00 |250.00 |50.00
I'm running a django app and I've implemented MySQL. When I run the server and try to load the page I get the error "badly formed hexadecimal UUID string." This occurs in the uuid.py file in init at line 140 and from what I can tell, this occurs because the value hex is not of length 32 but I don't know how to address this issue.
This is the model that the entries in the database follow.
from django.db import models
import uuid
from django.db.models import (
UUIDField,
CharField,
TextField,
IntegerField,
DecimalField,
ImageField
)
# Create your models here.
class AnimeCatalog(models.Model):
anime_id = UUIDField(primary_key = True, default=uuid.uuid4, editable=False)
name = CharField(max_length=300)
genre = CharField(max_length=300)
typeanime = CharField(max_length = 10)
episodes = IntegerField(default=0)
rating = DecimalField(max_digits = 4, decimal_places = 2, null = True)
members = IntegerField()
anime_cover = ImageField(blank = True, null = True, upload_to = "img/animeCover", verbose_name = "Profile Photo")
Then I try to call the objects.all() method on this model to retrieve it from mysql in the views.py file which is below
from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect
from django.views import View
from .forms import AnimeCatalogForm
from .models import AnimeCatalog
from .WebScraping import findAnimePic
import csv
from django.core.paginator import Paginator
# Create your views here.
def home(request):
temp = []
tempID = []
pics = []
fiveNames = []
names = []
new = AnimeCatalog.objects.all()
for line in new:
temp.append(line['name'].replace(''', ''))
tempID.append(line['anime_id'])
for x in range(0,5):
pics.append(findAnimePic(tempID[x],temp[x]))
for x in range(0, len(temp), 5):
for y in range (5):
if not(x+y >= len(temp)):
fiveNames.append({'name':temp[x+y],'img':pics[y]})
names.append(fiveNames)
fiveNames = []
items = {'names': names, 'imgs': pics}
html = render(request, 'home/home.html', {'names':names})
return html
Also this is a picture of the first few entries in the mysql which i imported a csv file.
Image of mysql database with imported csv files
As is clear from that screenshot, anime_id contains integers, not UUIDs. I don't know why you set it to be a UUIDField but you should use AutoField instead and remove the default.