I'm trying to produce a JSON format for a given entity and I'm having an issue getting the dictionary to NOT overwrite itself or become empty. This is pulling rows from a table in a MySQL database and attempting to produce JSON result from the query.
Here is my function:
def detail():
student = 'John Doe'
conn = get_db_connection()
cur = conn.cursor()
sql = ("""
select
a.student_name,
a.student_id,
a.student_homeroom_name,
a.test_id,
a.datetaken,
a.datecertified,
b.request_number
FROM student_information a
INNER JOIN homeroom b ON a.homeroom_id = b.homeroom_id
WHERE a.student_name = '""" + student + """'
ORDER BY datecertified DESC
""")
cur.execute(sql)
details=cur.fetchall()
dataset = defaultdict(dict)
case_dataset = defaultdict(dict)
case_dataset = dict(case_dataset)
for student_name, student_id, student_homeroom_name, test_id, datetaken, datecertified, request_number in details:
dataset[student_name]['student_id'] = student_id
dataset[student_name]['student_homeroom_name'] = student_homeroom_name
case_dataset['test_id'] = test_id
case_dataset['datetaken'] = datetaken
case_dataset['datecertified'] = datecertified
case_dataset['request_number'] = request_number
dataset[student_name]['additional_information'] = case_dataset
case_dataset.clear()
dataset= dict(dataset)
print(dataset)
cur.close()
conn.close()
I tried a few different ways but nothing seems to work. What I'm getting is nothing in the additonal_information key. What I'm getting is this:
{
"John Doe": {
"student_id": "1234",
"student_homeroom_name": "HR1",
"additional_information": []
}
}
What I'm expecting is something similar to the below JSON. However, I'm torn if this is even correct. Each student will have one to many test_id and I will need to iterate through them in my application.
{
"John Doe": {
"student_id": "1234",
"student_homeroom_name": "HR1",
"additional_information": [
{
"test_id": "0987",
"datetaken": "1-1-1970",
"datecertified": "1-2-1970",
"request_number": "5643"
},
{
"test_id": "12343",
"datetaken": "1-1-1980",
"datecertified": "1-2-1980",
"request_number": "39807"
}
]
}
}
Removing the clear() from the function produces this JSON:
{
"John Doe": {
"student_id": "1234",
"student_homeroom_name": "HR1",
"additional_information": [
{
"test_id": "0987",
"datetaken": "1-1-1970",
"datecertified": "1-2-1970",
"request_number": "5643"
},
{
"test_id": "0987",
"datetaken": "1-1-1970",
"datecertified": "1-2-1970",
"request_number": "5643"
}
]
}
}
lists are mutable objects. Which means that list's are passed by reference.
when you set
dataset[student]['additional_information'] = case_dataset
case_dataset.clear()
you're setting the list and then clearing it. So the list inside additional_information is also cleared.
Copy the list when setting it:
dataset[student]['additional_information'] = case_dataset[:]
case_dataset.clear()
Thanks everyone for the guidance and pointing me in the right direction.
I have what I'm looking for now. Based on some of the comments and troubleshooting, I updated my code. Here is what I did:
I added back additional_dataset as a list
Removed case_dataset = defaultdict(dict) and case_dataset = dict(case_dataset) and replaced it with case_dataset = {}.
Updated dataset[student_name]['additional_information'] = case_dataset with dataset[student_name]['additional_information'] = additional_dataset
Replaced case_dataset.clear() with case_dataset = {}
Here is my new code now
def detail():
student = 'John Doe'
conn = get_db_connection()
cur = conn.cursor()
sql = ("""
select
a.student_name,
a.student_id,
a.student_homeroom_name,
a.test_id,
a.datetaken,
a.datecertified,
b.request_number
FROM student_information a
INNER JOIN homeroom b ON a.homeroom_id = b.homeroom_id
WHERE a.student_name = '""" + student + """'
ORDER BY datecertified DESC
""")
cur.execute(sql)
details=cur.fetchall()
dataset = defaultdict(dict)
case_dataset = {} #2 - Updated to just dict
additional_dataset = [] #1 - added back additional_dataset as a list
for student_name, student_id, student_homeroom_name, test_id, datetaken, datecertified, request_number in details:
dataset[student_name]['student_id'] = student_id
dataset[student_name]['student_homeroom_name'] = student_homeroom_name
case_dataset['test_id'] = test_id
case_dataset['datetaken'] = datetaken
case_dataset['datecertified'] = datecertified
case_dataset['request_number'] = request_number
dataset[student_name]['additional_information'] = additional_dataset #3 - updated to additional_dataset
case_dataset = {} #4 - updated to clear with new dict
dataset= dict(dataset)
print(dataset)
cur.close()
conn.close()
This is what it produces now. This is a much better structure then what I was previously expecting.
{
"John Doe": {
"student_id": "1234",
"student_homeroom_name": "HR1",
"additional_information": [
{
"test_id": "0987",
"datetaken": "1-1-1970",
"datecertified": "1-2-1970",
"request_number": "5643"
},
{
"test_id": "12343",
"datetaken": "1-1-1980",
"datecertified": "1-2-1980",
"request_number": "39807"
}
]
}
}
I have to do the yelp API of a Django web app. I created db() to enter data into database but how to address the error? I'm trying to do it without Pandas:
Message=string indices must be integers
Source=C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py
StackTrace:
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 104, in <genexpr>
keys = (entry[c] for c in columns)
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 115, in db
cur.executemany(sql, keys)
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 153, in main
db()
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 157, in <module> (Current frame)
main()
# -*- coding: utf-8 -*-
from __future__ import print_function
import argparse
import json
import csv
import pprint
import requests
import sys
import sqlite3
#import pandas as pd
from urllib.error import HTTPError
from urllib.parse import quote
API_KEY = 'secret'
# API constants, you shouldn't have to change these.
API_HOST = 'https://api.yelp.com'
SEARCH_PATH = '/v3/businesses/search'
BUSINESS_PATH = '/v3/businesses/' # Business ID will come after slash.
# Defaults
DEFAULT_TERM = 'dinner'
DEFAULT_LOCATION = 'Glassboro, NJ'
SEARCH_LIMIT = 3
OFFSET = 0
def request(host, path, api_key, url_params=None):
url_params = url_params or {}
url = '{0}{1}'.format(host, quote(path.encode('utf8')))
headers = {
'Authorization': 'Bearer %s' % api_key,
}
print(u'Querying {0} ...'.format(url))
response = requests.request('GET', url, headers=headers, params=url_params)
return response.json()
def search(api_key, term, location):
url_params = {
'term': term.replace(' ', '+'),
'location': location.replace(' ', '+'),
'limit': SEARCH_LIMIT,
'offset': OFFSET
}
return request(API_HOST, SEARCH_PATH, api_key, url_params=url_params)
def get_business(api_key, business_id):
business_path = BUSINESS_PATH + business_id
return request(API_HOST, business_path, api_key)
def query_api(term, location):
response = search(API_KEY, term, location)
businesses = response.get('businesses')
if not businesses:
print(u'No businesses for {0} in {1} found.'.format(term, location))
return
business_id = businesses[0]['id']
print(u'{0} businesses found, querying business info ' \
'for the top result "{1}" ...'.format(
len(businesses), business_id))
response = get_business(API_KEY, business_id)
print(u'Result for business "{0}" found:'.format(business_id))
pprint.pprint(response, indent=2)
str_to_write_to_file = json.dumps(response, skipkeys=True, allow_nan=True, indent=4)
with open('yelp.json', 'w') as f:
f.write(str_to_write_to_file)
def db():
with open('yelp.json', 'r') as f:
data = f.readlines()
conn = sqlite3.connect('yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
location varchar(100),
display_phone varchar(15)
);"""
)
for entry in data:
columns = ["id" "alias", "location", "display_phone"]
keys = (entry[c] for c in columns)
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = """INSERT INTO yelp (id, alias, location, display_phone) VALUES(
?,
?,
?,
?
);"""
cur.executemany(sql, keys)
print(f'{entry["alias"]} data inserted Succefully')
conn.commit()
conn.close()
with sqlite3.connect("yelp.db") as conn:
cmd = """SELECT * FROM yelp;"""
cur = conn.execute(cmd)
res = cur.fetchall()
for r in res:
print(r)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-q', '--term', dest='term', default=DEFAULT_TERM,
type=str, help='Search term (default: %(default)s)')
parser.add_argument('-l', '--location', dest='location',
default=DEFAULT_LOCATION, type=str,
help='Search location (default: %(default)s)')
input_values = parser.parse_args()
try:
query_api(input_values.term, input_values.location)
except HTTPError as error:
sys.exit(
'Encountered HTTP error {0} on {1}:\n {2}\nAbort program.'.format(
error.code,
error.url,
error.read(),
)
)
db()
if __name__ == '__main__':
main()
JSON file :
{
"id": "umC69pkiPyk3qY7IB49ZYw",
"alias": "bosphorus-mediterranean-cuisine-glassboro",
"name": "Bosphorus Mediterranean Cuisine",
"image_url": "https://s3-media4.fl.yelpcdn.com/bphoto/G7VCO3tvx8NGPz5g0fSpMw/o.jpg",
"is_claimed": true,
"is_closed": false,
"url": "https://www.yelp.com/biz/bosphorus-mediterranean-cuisine-glassboro?adjust_creative=9aYQmmK21ApZ7TfokeTk1A&utm_campaign=yelp_api_v3&utm_medium=api_v3_business_lookup&utm_source=9aYQmmK21ApZ7TfokeTk1A",
"phone": "+18562432015",
"display_phone": "(856) 243-2015",
"review_count": 14,
"categories": [
{
"alias": "turkish",
"title": "Turkish"
},
{
"alias": "halal",
"title": "Halal"
},
{
"alias": "kebab",
"title": "Kebab"
}
],
"rating": 5.0,
"location": {
"address1": "524 Delsea Drive N",
"address2": null,
"address3": null,
"city": "Glassboro",
"zip_code": "08028",
"country": "US",
"state": "NJ",
"display_address": [
"524 Delsea Drive N",
"Glassboro, NJ 08028"
],
"cross_streets": ""
},
"coordinates": {
"latitude": 39.7150351328115,
"longitude": -75.1118882
},
"photos": [
"https://s3-media4.fl.yelpcdn.com/bphoto/G7VCO3tvx8NGPz5g0fSpMw/o.jpg",
"https://s3-media2.fl.yelpcdn.com/bphoto/HvhYRZO2rOYUBX0DagVE3w/o.jpg",
"https://s3-media2.fl.yelpcdn.com/bphoto/PQHr3upfVULUjwz1M-ILcw/o.jpg"
],
"hours": [
{
"open": [
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 0
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 1
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 2
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 3
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 4
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 5
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 6
}
],
"hours_type": "REGULAR",
"is_open_now": true
}
],
"transactions": [
"pickup",
"delivery"
],
"messaging": {
"url": "https://www.yelp.com/raq/umC69pkiPyk3qY7IB49ZYw?adjust_creative=9aYQmmK21ApZ7TfokeTk1A&utm_campaign=yelp_api_v3&utm_medium=api_v3_business_lookup&utm_source=9aYQmmK21ApZ7TfokeTk1A#popup%3Araq",
"use_case_text": "Message the Business"
}
}
You shouldn't use f.readlines() to read a JSON file, use json.load(f).
There's only one set of values in the JSON, so you don't need a loop or executemany().
def db():
with open('yelp.json', 'r') as f:
data = json.load(f)
conn = sqlite3.connect('yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
location varchar(100),
display_phone varchar(15)
);"""
)
columns = ["id" "alias", "location", "display_phone"]
keys = [entry[c] for c in columns]
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = """INSERT INTO yelp (id, alias, location, display_phone) VALUES(
?,
?,
?,
?
);"""
cur.execute(sql, keys)
print(f'{entry["alias"]} data inserted Succefully')
conn.commit()
conn.close()
with sqlite3.connect("yelp.db") as conn:
cmd = """SELECT * FROM yelp;"""
cur = conn.execute(cmd)
res = cur.fetchall()
for r in res:
print(r)
So ultimately I figured it out... pretty much. I used what #Bramar said but the solution was making the json file an array and then I started getting this error sqlite3.ProgrammingError: Incorrect number of bindings supplied. The current statement uses 4, and there are 1 supplied. which turned out to be one of the entries that I had was stored in json as a dict so I eliminated it temporarily to see if I can make it work and it works, this is the code -
print(u'Result for business "{0}" found:'.format(business_id))
str_to_write_to_file = json.dumps([response], indent=4)
with open('yelp.json', 'w') as f:
f.write(str_to_write_to_file)
def db():
with open('yelp.json', 'r') as f:
data = json.load(f)
conn = sqlite3.connect('data/yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
display_phone varchar(15),
location dictionary
);"""
)
columns = ["alias", "display_phone"]
keys = [data[0][c] for c in columns]
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = '''INSERT INTO yelp (alias, display_phone) VALUES(
?,
?
);'''
cur.execute(sql, keys)
conn.commit()
conn.close()
Hopefully this helps someone, this can very confusing.
I'm trying to upload a test JSON string to SQL Server
json_string = """ {
"orderID": 42,
"customerName": "John Smith",
"customerPhoneN": "555-1234",
"orderContents": [
{
"productID": 23,
"productName": "keyboard",
"quantity": 1
},
{
"productID": 13,
"productName": "mouse",
"quantity": 1
}
],
"orderCompleted": true
} """
parsed_string = json.loads(json_string)
cursor.execute("update Table set Status = ? where Name like ? ",(json.dumps(parsed_string), "Blabla"))
cnxn.commit()
How to return and work with this JSON from the database?
cursor.execute("""select Status from Table where Name like ?""", "Blabla")
rows = cursor.fetchall()
How can I print the value of the JSON?
Use the JSON data type that is supported in MySQL. You can find more about it here:
https://dev.mysql.com/doc/refman/5.7/en/json.html
s = json.dumps(DATA)
cursor.execute("update Table set Status = ? where Name like ? ",(s, "Blabla"))
cnxn.commit()
and
cursor.execute("""select Status from Table where Name like ?""", "Blabla")
res = cursor.fetchall()
DATA = json.loads(res[0][X])
I'm new to python and trying to work out how to insert some JSON into MySQL table.
How can I insert my JSON Object into MySQL using Python?
Here is the code that I am using
import requests
import urllib.request
import json
import pymysql
con = pymysql.connect(host = 'localhost',user = 'root',passwd = 'root',db = 'micro')
cursor = con.cursor()
url = 'https://api.amazon.com/v1/products(onlineAvailability=true)?pageSize=100&show=upc,sku,salePrice&page=45&callback=JSON_CALLBACK&format=json'
urllib.request.urlopen(url).read()
response = urllib.request.urlopen(url).read()
json_obj = str(response, 'utf-8')
cursor.execute("INSERT INTO bestb (sku, upc, salePrice) VALUES (%s,%s,%s)", (sku, upc, salePrice))
con.commit()
con.close()
print (json_obj)
here is the JSON that i'm trying to parse.
"products": [
{
"upc": "715187763623",
"sku": 1833591,
"salePrice": 13.99
},
{
"upc": "8809269504036",
"sku": 26220187,
"salePrice": 16.99
}
]
})
Thanks in advance.
Use json.loads(string) to convert json string to Python object. And then you can use it as normal dictionary and list
BTW: you have incorrect JSON in your example
response = b'''{"products": [
{
"upc": "715187763623",
"sku": 1833591,
"salePrice": 13.99
},
{
"upc": "8809269504036",
"sku": 26220187,
"salePrice": 16.99
}
]
}'''
json_obj = json.loads(response.decode('utf-8'))
#print(json_obj["products"][0]["upc"])
for product in json_obj["products"]:
print("upc:", product["upc"])
print("sku:", product["sku"])
print("salePrice:", product["salePrice"])
print('---')
cursor.execute("INSERT INTO bestb (sku, upc, salePrice) VALUES (%s,%s,%s)", (product["sku"], product["upc"], product["salePrice"]))