I have to do the yelp API of a Django web app. I created db() to enter data into database but how to address the error? I'm trying to do it without Pandas:
Message=string indices must be integers
Source=C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py
StackTrace:
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 104, in <genexpr>
keys = (entry[c] for c in columns)
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 115, in db
cur.executemany(sql, keys)
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 153, in main
db()
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 157, in <module> (Current frame)
main()
# -*- coding: utf-8 -*-
from __future__ import print_function
import argparse
import json
import csv
import pprint
import requests
import sys
import sqlite3
#import pandas as pd
from urllib.error import HTTPError
from urllib.parse import quote
API_KEY = 'secret'
# API constants, you shouldn't have to change these.
API_HOST = 'https://api.yelp.com'
SEARCH_PATH = '/v3/businesses/search'
BUSINESS_PATH = '/v3/businesses/' # Business ID will come after slash.
# Defaults
DEFAULT_TERM = 'dinner'
DEFAULT_LOCATION = 'Glassboro, NJ'
SEARCH_LIMIT = 3
OFFSET = 0
def request(host, path, api_key, url_params=None):
url_params = url_params or {}
url = '{0}{1}'.format(host, quote(path.encode('utf8')))
headers = {
'Authorization': 'Bearer %s' % api_key,
}
print(u'Querying {0} ...'.format(url))
response = requests.request('GET', url, headers=headers, params=url_params)
return response.json()
def search(api_key, term, location):
url_params = {
'term': term.replace(' ', '+'),
'location': location.replace(' ', '+'),
'limit': SEARCH_LIMIT,
'offset': OFFSET
}
return request(API_HOST, SEARCH_PATH, api_key, url_params=url_params)
def get_business(api_key, business_id):
business_path = BUSINESS_PATH + business_id
return request(API_HOST, business_path, api_key)
def query_api(term, location):
response = search(API_KEY, term, location)
businesses = response.get('businesses')
if not businesses:
print(u'No businesses for {0} in {1} found.'.format(term, location))
return
business_id = businesses[0]['id']
print(u'{0} businesses found, querying business info ' \
'for the top result "{1}" ...'.format(
len(businesses), business_id))
response = get_business(API_KEY, business_id)
print(u'Result for business "{0}" found:'.format(business_id))
pprint.pprint(response, indent=2)
str_to_write_to_file = json.dumps(response, skipkeys=True, allow_nan=True, indent=4)
with open('yelp.json', 'w') as f:
f.write(str_to_write_to_file)
def db():
with open('yelp.json', 'r') as f:
data = f.readlines()
conn = sqlite3.connect('yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
location varchar(100),
display_phone varchar(15)
);"""
)
for entry in data:
columns = ["id" "alias", "location", "display_phone"]
keys = (entry[c] for c in columns)
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = """INSERT INTO yelp (id, alias, location, display_phone) VALUES(
?,
?,
?,
?
);"""
cur.executemany(sql, keys)
print(f'{entry["alias"]} data inserted Succefully')
conn.commit()
conn.close()
with sqlite3.connect("yelp.db") as conn:
cmd = """SELECT * FROM yelp;"""
cur = conn.execute(cmd)
res = cur.fetchall()
for r in res:
print(r)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-q', '--term', dest='term', default=DEFAULT_TERM,
type=str, help='Search term (default: %(default)s)')
parser.add_argument('-l', '--location', dest='location',
default=DEFAULT_LOCATION, type=str,
help='Search location (default: %(default)s)')
input_values = parser.parse_args()
try:
query_api(input_values.term, input_values.location)
except HTTPError as error:
sys.exit(
'Encountered HTTP error {0} on {1}:\n {2}\nAbort program.'.format(
error.code,
error.url,
error.read(),
)
)
db()
if __name__ == '__main__':
main()
JSON file :
{
"id": "umC69pkiPyk3qY7IB49ZYw",
"alias": "bosphorus-mediterranean-cuisine-glassboro",
"name": "Bosphorus Mediterranean Cuisine",
"image_url": "https://s3-media4.fl.yelpcdn.com/bphoto/G7VCO3tvx8NGPz5g0fSpMw/o.jpg",
"is_claimed": true,
"is_closed": false,
"url": "https://www.yelp.com/biz/bosphorus-mediterranean-cuisine-glassboro?adjust_creative=9aYQmmK21ApZ7TfokeTk1A&utm_campaign=yelp_api_v3&utm_medium=api_v3_business_lookup&utm_source=9aYQmmK21ApZ7TfokeTk1A",
"phone": "+18562432015",
"display_phone": "(856) 243-2015",
"review_count": 14,
"categories": [
{
"alias": "turkish",
"title": "Turkish"
},
{
"alias": "halal",
"title": "Halal"
},
{
"alias": "kebab",
"title": "Kebab"
}
],
"rating": 5.0,
"location": {
"address1": "524 Delsea Drive N",
"address2": null,
"address3": null,
"city": "Glassboro",
"zip_code": "08028",
"country": "US",
"state": "NJ",
"display_address": [
"524 Delsea Drive N",
"Glassboro, NJ 08028"
],
"cross_streets": ""
},
"coordinates": {
"latitude": 39.7150351328115,
"longitude": -75.1118882
},
"photos": [
"https://s3-media4.fl.yelpcdn.com/bphoto/G7VCO3tvx8NGPz5g0fSpMw/o.jpg",
"https://s3-media2.fl.yelpcdn.com/bphoto/HvhYRZO2rOYUBX0DagVE3w/o.jpg",
"https://s3-media2.fl.yelpcdn.com/bphoto/PQHr3upfVULUjwz1M-ILcw/o.jpg"
],
"hours": [
{
"open": [
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 0
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 1
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 2
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 3
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 4
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 5
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 6
}
],
"hours_type": "REGULAR",
"is_open_now": true
}
],
"transactions": [
"pickup",
"delivery"
],
"messaging": {
"url": "https://www.yelp.com/raq/umC69pkiPyk3qY7IB49ZYw?adjust_creative=9aYQmmK21ApZ7TfokeTk1A&utm_campaign=yelp_api_v3&utm_medium=api_v3_business_lookup&utm_source=9aYQmmK21ApZ7TfokeTk1A#popup%3Araq",
"use_case_text": "Message the Business"
}
}
You shouldn't use f.readlines() to read a JSON file, use json.load(f).
There's only one set of values in the JSON, so you don't need a loop or executemany().
def db():
with open('yelp.json', 'r') as f:
data = json.load(f)
conn = sqlite3.connect('yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
location varchar(100),
display_phone varchar(15)
);"""
)
columns = ["id" "alias", "location", "display_phone"]
keys = [entry[c] for c in columns]
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = """INSERT INTO yelp (id, alias, location, display_phone) VALUES(
?,
?,
?,
?
);"""
cur.execute(sql, keys)
print(f'{entry["alias"]} data inserted Succefully')
conn.commit()
conn.close()
with sqlite3.connect("yelp.db") as conn:
cmd = """SELECT * FROM yelp;"""
cur = conn.execute(cmd)
res = cur.fetchall()
for r in res:
print(r)
So ultimately I figured it out... pretty much. I used what #Bramar said but the solution was making the json file an array and then I started getting this error sqlite3.ProgrammingError: Incorrect number of bindings supplied. The current statement uses 4, and there are 1 supplied. which turned out to be one of the entries that I had was stored in json as a dict so I eliminated it temporarily to see if I can make it work and it works, this is the code -
print(u'Result for business "{0}" found:'.format(business_id))
str_to_write_to_file = json.dumps([response], indent=4)
with open('yelp.json', 'w') as f:
f.write(str_to_write_to_file)
def db():
with open('yelp.json', 'r') as f:
data = json.load(f)
conn = sqlite3.connect('data/yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
display_phone varchar(15),
location dictionary
);"""
)
columns = ["alias", "display_phone"]
keys = [data[0][c] for c in columns]
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = '''INSERT INTO yelp (alias, display_phone) VALUES(
?,
?
);'''
cur.execute(sql, keys)
conn.commit()
conn.close()
Hopefully this helps someone, this can very confusing.
I have created a Python script that creates a table in MySQL and another one that populates it with data from a JSON file.
Sample JSON file:
{
"ansible_facts":{
"ansible_network_resources":{
"l3_interfaces":[
{
"name":"GigabitEthernet0/0"
},
{
"name":"GigabitEthernet0/0.100",
"ipv4":[
{
"address":"172.1.1.1 255.255.255.252"
}
]
},
{
"name":"GigabitEthernet0/0.101",
"ipv4":[
{
"address":"172.1.1.1 255.255.255.252"
}
]
},
{
"name":"GigabitEthernet0/1",
"ipv4":[
{
"address":"56.2.1.1 255.255.255.252"
}
]
},
{
"name":"GigabitEthernet0/2"
}
]
},
"ansible_net_python_version":"3.6.9",
"ansible_net_hostname":"host02342-mpls",
"ansible_net_model":"CISCO-CHA",
"ansible_net_serialnum":"F1539AM",
"ansible_net_gather_subset":[
"default"
],
"ansible_net_gather_network_resources":[
"l3_interfaces"
],
"ansible_net_version":"15.3(2)T",
"ansible_net_api":"cliconf",
"ansible_net_system":"ios",
"ansible_net_image":"flash0:/c3900-universalk9-mz.spa.153-2.t.bin",
"ansible_net_iostype":"IOS"
}
}
Table creation script
import mysql.connector
mydb = mysql.connector.connect(host="IPaddress", user="user", password="pw", database="db")
mycursor = mydb.cursor()
mycursor.execute("CREATE TABLE Routers (ansible_net_hostname NVARCHAR(255), ansible_net_model NVARCHAR(255), ansible_network_resources NVARCHAR(255))")
The script to import JSON data into MySQL
import json, pymysql
json_data = open("L3_out.json").read()
json_obj = json.loads(json_data)
con = pymysql.connect(host="IPaddress", user="user", password="pw", database="db")
cursor = con.cursor()
for item in json_obj:
ansible_net_hostname = item.get("ansible_net_hostname")
ansible_net_model = item.get("ansible_net_model")
ansible_network_resources = item.get("ansible_network_resources")
cursor.execute(
"insert into Routers(ansible_net_hostname, ansible_net_model, ansible_network_resources) value(%s, %s, %s)",
(ansible_net_hostname, ansible_net_model, ansible_network_resources)
con.commit()
con.close()
I'm having issues importing ansible_network_resources field object into the Routers table. The other columns (ansible_net_hostname, ansible_net_model) get inserted perfectly. What am I doing wrong?
First of all, it's not clear how does
for item in json_obj:
ansible_net_hostname=item.get("ansible_net_hostname")
work.
Since 'item' in your case is a key from the dictionary. In the file you shown there is only one root key "ansible_facts". So you are trying to call get() on the string.
To get the data of "ansible_network_resources" do the following:
for key in json_obj:
ansible_network_resources=json_obj[key].get("ansible_network_resources")
I am trying to search for a variable in a JSON file.
Current JSON file (devices.json) is:
{
"NYC": {
"Floor1": [
{
"name": "server1",
"ip": "1.1.1.1"
},
{
"name": "server2",
"ip": "1.1.1.2"
}
],
"Floor2": [
...
],
"sitenum": 1
},
"Boston": {
...
"sitenum": 2
}
...
}
Two questions:
I am new to JSON, so is this formatted correctly for
lists/dictionaries?
I'd like to preform a python query to display Floor(s){name, ip} for sitenum (x)
Current Python file is
import json
with open('devices.json') as jsonfile:
data = json.load(jsonfile)
Thanks!
this python script will return the floor details as a list of json values for a given sitenum and floor.
import json
def get_floor_details(sitenum, floor, data):
for k,v in data.items():
if v['sitenum'] == sitenum:
return(v[floor])
with open('sample.json') as json_file:
data = json.load(json_file)
sitenum = 1
floor = 'Floor1'
floor_details = get_floor_details(sitenum, floor, data)
print(floor_details)
Output:
[{'name': 'server1', 'ip': '1.1.1.1'}, {'name': 'server2', 'ip': '1.1.1.2'}]
def findFloor(site_num_val, data):
return_val = {}
for each_loc in data:
if site_num_val == each_loc["sitenum"]:
return_val = data[each_loc].copy()
del return_val["sitenum"]
break
else:
print "sitenum not found"
return return_val
I hope this solves your problem in trying to get the information.
I'm trying to upload a test JSON string to SQL Server
json_string = """ {
"orderID": 42,
"customerName": "John Smith",
"customerPhoneN": "555-1234",
"orderContents": [
{
"productID": 23,
"productName": "keyboard",
"quantity": 1
},
{
"productID": 13,
"productName": "mouse",
"quantity": 1
}
],
"orderCompleted": true
} """
parsed_string = json.loads(json_string)
cursor.execute("update Table set Status = ? where Name like ? ",(json.dumps(parsed_string), "Blabla"))
cnxn.commit()
How to return and work with this JSON from the database?
cursor.execute("""select Status from Table where Name like ?""", "Blabla")
rows = cursor.fetchall()
How can I print the value of the JSON?
Use the JSON data type that is supported in MySQL. You can find more about it here:
https://dev.mysql.com/doc/refman/5.7/en/json.html
s = json.dumps(DATA)
cursor.execute("update Table set Status = ? where Name like ? ",(s, "Blabla"))
cnxn.commit()
and
cursor.execute("""select Status from Table where Name like ?""", "Blabla")
res = cursor.fetchall()
DATA = json.loads(res[0][X])
I have a script which takes data in a SQL Server database and parses it into a key:value pair JSON. I would like to give the three items in the JSON dictionary; one key, such as "ServiceRequest" at the highest level. So that my output would read as:
{
"ServiceRequest": [
{
"SRNUMBER": "1-3580171",
"FirstName": "Myla",
"LastName": "Threeoneone"
}
]
}
Program:
import pyodbc
import json
import collections
import requests
import urllib
connstr = 'DRIVER={SQL Server};SERVER=ServerName;DATABASE=DataBase; UID=UID;PWD=PWD'
conn = pyodbc.connect(connstr)
cursor = conn.cursor()
cursor.execute("""
SELECT SRNUMBER, FirstName, LastName
FROM MYLA311 """)
rows = cursor.fetchall()
# Convert query to row arrays
rowarray_list = []
for row in rows:
t = (row.SRNUMBER)
rowarray_list.append(t)
j = json.dumps(rowarray_list)
rowarrays_file = 'student_rowarrays.js'
f = open(rowarrays_file,'w')
# Convert query to objects of key-value pairs
objects_list = []
for row in rows:
d = collections.OrderedDict()
d['SRNUMBER']= row.SRNUMBER
d['FirstName']= row.FirstName
d['LastName']= row.LastName
objects_list.append(d)
j = json.dumps(objects_list)
objects_file = 'C:\Users\Administrator\Desktop\JSONOutput.txt'
f = open(objects_file,'w')
print >> f, j
print j
conn.close()
Actual Output:
[
{
"SRNUMBER": "1-3580171",
"FirstName": "Myla",
"LastName": "Threeoneone"
}
]
JSON and Python dictionaries are very similar.
Your desired output is already valid Python:
{
"ServiceRequest": [
{
"SRNUMBER": "1-3580171",
"FirstName": "Myla",
"LastName": "Threeoneone"
}
]
}
So all you need to do, is wrap your objects list in a Python dictionary:
output = {
'ServiceRequest': object_list
}
# then dump it
json.dumps(output)
# ...