Django - how can i insert '.json' file to SQLite DB?

Django - how can i insert '.json' file to SQLite DB? - python

my '.json file' like
{
"users": [
{
"userId": 1,
"firstName": "AAAAA",
"lastName": "as23",
"phoneNumber": "123456",
"emailAddress": "AAAAA#test.com",
"homepage": "https://amogg.tistory.com/1"
},
{
"userId": 2,
"firstName": "BBBB",
"lastName": "h5jdd",
"phoneNumber": "123456",
"homepage": "https://amogg.tistory.com/2"
},
{
"userId": 3,
...
i was search that to google, and try to this problem.. but unresolved.
so i use pandas and sqlite3
import sqlite3 as db
import pandas as pd
df = pd.read_json('test.json')
con = db.connect('./test.db')
df.to_sql('test', con=con)
so DB is created, but .json file data dont save in DB
how can solve this problem...?

You will have to create the table 'test' beforehand, iterate over the pandas dataframe df and insert the records into the table one by one:
import sqlite3 as db
import pandas as pd
df = pd.read_json('test.json', orient='index')
con = db.connect('./test.db')
cursor = con.cursor()
cursor.execute('''create table test (userId int primary key,
firstName text,
lastName text,
phoneNumber text,
emailAddress text,
homePage text)''')
for index, row in df.iterrows():
for element in row.iteritems():
try:
firstName = element[1]['firstName']
except:
firstName = ''
try:
lastName = element[1]['lastName']
except:
lastName = ''
try:
phoneNumber = element[1]['phoneNumber']
except:
phoneNumber = ''
try:
emailAddress = element[1]['emailAddress']
except:
emailAddress = ''
try:
homepage = element[1]['homepage']
except:
homepage = ''
cursor.execute("INSERT INTO test VALUES (?,?,?,?,?,?)", (element[1]['userId'],
firstName,
lastName,
phoneNumber,
emailAddress,
homepage))
con.commit()
con.close()
Since not all the records have the same valid values for all the columns, you will need to validate the existance of the column with a try/except and store an empty string if the column does not exist in the row.

Related

Trying to insert from JSON file to database

I have to do the yelp API of a Django web app. I created db() to enter data into database but how to address the error? I'm trying to do it without Pandas:
Message=string indices must be integers
Source=C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py
StackTrace:
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 104, in <genexpr>
keys = (entry[c] for c in columns)
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 115, in db
cur.executemany(sql, keys)
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 153, in main
db()
File "C:\Users\diggt\OneDrive\College\Rowan\Fall22\10430_computing_and_informatics_capstone\yelp_VSCode\yelp.py", line 157, in <module> (Current frame)
main()
# -*- coding: utf-8 -*-
from __future__ import print_function
import argparse
import json
import csv
import pprint
import requests
import sys
import sqlite3
#import pandas as pd
from urllib.error import HTTPError
from urllib.parse import quote
API_KEY = 'secret'
# API constants, you shouldn't have to change these.
API_HOST = 'https://api.yelp.com'
SEARCH_PATH = '/v3/businesses/search'
BUSINESS_PATH = '/v3/businesses/' # Business ID will come after slash.
# Defaults
DEFAULT_TERM = 'dinner'
DEFAULT_LOCATION = 'Glassboro, NJ'
SEARCH_LIMIT = 3
OFFSET = 0
def request(host, path, api_key, url_params=None):
url_params = url_params or {}
url = '{0}{1}'.format(host, quote(path.encode('utf8')))
headers = {
'Authorization': 'Bearer %s' % api_key,
}
print(u'Querying {0} ...'.format(url))
response = requests.request('GET', url, headers=headers, params=url_params)
return response.json()
def search(api_key, term, location):
url_params = {
'term': term.replace(' ', '+'),
'location': location.replace(' ', '+'),
'limit': SEARCH_LIMIT,
'offset': OFFSET
}
return request(API_HOST, SEARCH_PATH, api_key, url_params=url_params)
def get_business(api_key, business_id):
business_path = BUSINESS_PATH + business_id
return request(API_HOST, business_path, api_key)
def query_api(term, location):
response = search(API_KEY, term, location)
businesses = response.get('businesses')
if not businesses:
print(u'No businesses for {0} in {1} found.'.format(term, location))
return
business_id = businesses[0]['id']
print(u'{0} businesses found, querying business info ' \
'for the top result "{1}" ...'.format(
len(businesses), business_id))
response = get_business(API_KEY, business_id)
print(u'Result for business "{0}" found:'.format(business_id))
pprint.pprint(response, indent=2)
str_to_write_to_file = json.dumps(response, skipkeys=True, allow_nan=True, indent=4)
with open('yelp.json', 'w') as f:
f.write(str_to_write_to_file)
def db():
with open('yelp.json', 'r') as f:
data = f.readlines()
conn = sqlite3.connect('yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
location varchar(100),
display_phone varchar(15)
);"""
)
for entry in data:
columns = ["id" "alias", "location", "display_phone"]
keys = (entry[c] for c in columns)
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = """INSERT INTO yelp (id, alias, location, display_phone) VALUES(
?,
?,
?,
?
);"""
cur.executemany(sql, keys)
print(f'{entry["alias"]} data inserted Succefully')
conn.commit()
conn.close()
with sqlite3.connect("yelp.db") as conn:
cmd = """SELECT * FROM yelp;"""
cur = conn.execute(cmd)
res = cur.fetchall()
for r in res:
print(r)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-q', '--term', dest='term', default=DEFAULT_TERM,
type=str, help='Search term (default: %(default)s)')
parser.add_argument('-l', '--location', dest='location',
default=DEFAULT_LOCATION, type=str,
help='Search location (default: %(default)s)')
input_values = parser.parse_args()
try:
query_api(input_values.term, input_values.location)
except HTTPError as error:
sys.exit(
'Encountered HTTP error {0} on {1}:\n {2}\nAbort program.'.format(
error.code,
error.url,
error.read(),
)
)
db()
if __name__ == '__main__':
main()
JSON file :
{
"id": "umC69pkiPyk3qY7IB49ZYw",
"alias": "bosphorus-mediterranean-cuisine-glassboro",
"name": "Bosphorus Mediterranean Cuisine",
"image_url": "https://s3-media4.fl.yelpcdn.com/bphoto/G7VCO3tvx8NGPz5g0fSpMw/o.jpg",
"is_claimed": true,
"is_closed": false,
"url": "https://www.yelp.com/biz/bosphorus-mediterranean-cuisine-glassboro?adjust_creative=9aYQmmK21ApZ7TfokeTk1A&utm_campaign=yelp_api_v3&utm_medium=api_v3_business_lookup&utm_source=9aYQmmK21ApZ7TfokeTk1A",
"phone": "+18562432015",
"display_phone": "(856) 243-2015",
"review_count": 14,
"categories": [
{
"alias": "turkish",
"title": "Turkish"
},
{
"alias": "halal",
"title": "Halal"
},
{
"alias": "kebab",
"title": "Kebab"
}
],
"rating": 5.0,
"location": {
"address1": "524 Delsea Drive N",
"address2": null,
"address3": null,
"city": "Glassboro",
"zip_code": "08028",
"country": "US",
"state": "NJ",
"display_address": [
"524 Delsea Drive N",
"Glassboro, NJ 08028"
],
"cross_streets": ""
},
"coordinates": {
"latitude": 39.7150351328115,
"longitude": -75.1118882
},
"photos": [
"https://s3-media4.fl.yelpcdn.com/bphoto/G7VCO3tvx8NGPz5g0fSpMw/o.jpg",
"https://s3-media2.fl.yelpcdn.com/bphoto/HvhYRZO2rOYUBX0DagVE3w/o.jpg",
"https://s3-media2.fl.yelpcdn.com/bphoto/PQHr3upfVULUjwz1M-ILcw/o.jpg"
],
"hours": [
{
"open": [
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 0
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 1
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 2
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 3
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 4
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 5
},
{
"is_overnight": false,
"start": "1100",
"end": "2200",
"day": 6
}
],
"hours_type": "REGULAR",
"is_open_now": true
}
],
"transactions": [
"pickup",
"delivery"
],
"messaging": {
"url": "https://www.yelp.com/raq/umC69pkiPyk3qY7IB49ZYw?adjust_creative=9aYQmmK21ApZ7TfokeTk1A&utm_campaign=yelp_api_v3&utm_medium=api_v3_business_lookup&utm_source=9aYQmmK21ApZ7TfokeTk1A#popup%3Araq",
"use_case_text": "Message the Business"
}
}

You shouldn't use f.readlines() to read a JSON file, use json.load(f).
There's only one set of values in the JSON, so you don't need a loop or executemany().
def db():
with open('yelp.json', 'r') as f:
data = json.load(f)
conn = sqlite3.connect('yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
location varchar(100),
display_phone varchar(15)
);"""
)
columns = ["id" "alias", "location", "display_phone"]
keys = [entry[c] for c in columns]
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = """INSERT INTO yelp (id, alias, location, display_phone) VALUES(
?,
?,
?,
?
);"""
cur.execute(sql, keys)
print(f'{entry["alias"]} data inserted Succefully')
conn.commit()
conn.close()
with sqlite3.connect("yelp.db") as conn:
cmd = """SELECT * FROM yelp;"""
cur = conn.execute(cmd)
res = cur.fetchall()
for r in res:
print(r)

So ultimately I figured it out... pretty much. I used what #Bramar said but the solution was making the json file an array and then I started getting this error sqlite3.ProgrammingError: Incorrect number of bindings supplied. The current statement uses 4, and there are 1 supplied. which turned out to be one of the entries that I had was stored in json as a dict so I eliminated it temporarily to see if I can make it work and it works, this is the code -
print(u'Result for business "{0}" found:'.format(business_id))
str_to_write_to_file = json.dumps([response], indent=4)
with open('yelp.json', 'w') as f:
f.write(str_to_write_to_file)
def db():
with open('yelp.json', 'r') as f:
data = json.load(f)
conn = sqlite3.connect('data/yelp.db')
cur = conn.cursor()
# Create the table if it doesn't exist.
cur.execute(
"""CREATE TABLE IF NOT EXISTS yelp(
id INTEGER PRIMARY KEY,
alias varchar(100),
display_phone varchar(15),
location dictionary
);"""
)
columns = ["alias", "display_phone"]
keys = [data[0][c] for c in columns]
# Execute the command and replace '?' with the each value
# in 'values'. DO NOT build a string and replace manually.
# the sqlite3 library will handle non safe strings by doing this.
sql = '''INSERT INTO yelp (alias, display_phone) VALUES(
?,
?
);'''
cur.execute(sql, keys)
conn.commit()
conn.close()
Hopefully this helps someone, this can very confusing.

How do I convert a JSON file to a SQL lite database if the JSON file has multiple values?

I'm new to using Python sqlite and parsing json files. I'm trying to create a (3 table) database using python sqlite. I am able to create these tables using the code below, but now I need to populate the tables using a json file. How do I add the json file to the tables when there are multiple values (as seen below)? For example, there are multiple items and I want to create a counter variable (itemid) for each item in the order.
import sqlite3
from sqlite3 import Error
def create_connection(db_file):
""" create a database connection to the SQLite database
specified by db_file
:param db_file: database file
:return: Connection object or None
"""
conn = None
try:
conn = sqlite3.connect(db_file)
return conn
except Error as e:
print(e)
return conn
def create_table(conn, create_table_sql):
""" create a table from the create_table_sql statement
:param conn: Connection object
:param create_table_sql: a CREATE TABLE statement
:return:
"""
try:
c = conn.cursor()
c.execute(create_table_sql)
except Error as e:
print(e)
def main():
database = r"C:\sqlite\db\pythonsqlite.db"
sql_create_items_table = """ CREATE TABLE IF NOT EXISTS items (
orderid integer PRIMARY KEY,
itemid integer PRIMARY KEY,
name text,
price numeric); """
sql_create_charges_table = """CREATE TABLE IF NOT EXISTS charges (
FOREIGN KEY (items_orderid) REFERENCES items (orderid),
date datetime,
subtotal numeric,
taxes numeric,
total numeric);"""
sql_create_payment_table = """CREATE TABLE IF NOT EXISTS payment (
FOREIGN KEY (items_orderid) REFERENCES items (orderid),
card_type text,
card_number integer,
zip text,
cardholder text,
method text);"""
# create a database connection
conn = create_connection(database)
# create tables
if conn is not None:
# create items table
create_table(conn, sql_create_items_table)
# create charges table
create_table(conn, sql_create_charges_table)
# create payment table
create_table(conn, sql_create_payment_table)
else:
print("Error! cannot create the database connection.")
if __name__ == '__main__':
main()
This is an example of the json file entries:
{
"orders": [
{
"items": [
{
"name": "coffee",
"price": 2.75
},
{
"name": "espresso",
"price": 1.25
}
],
"charges": {
"date": "04/01/21 11:10",
"subtotal": 4.0,
"taxes": 0.28,
"total": 4.28
},
"payment": {
"card_type": "visa",
"last_4_card_number": "6072",
"zip": "21213",
"cardholder": "Andrew Luna",
"method": "credit_card"
}
}
}

Issue importing JSON object into a MySQL column

I have created a Python script that creates a table in MySQL and another one that populates it with data from a JSON file.
Sample JSON file:
{
"ansible_facts":{
"ansible_network_resources":{
"l3_interfaces":[
{
"name":"GigabitEthernet0/0"
},
{
"name":"GigabitEthernet0/0.100",
"ipv4":[
{
"address":"172.1.1.1 255.255.255.252"
}
]
},
{
"name":"GigabitEthernet0/0.101",
"ipv4":[
{
"address":"172.1.1.1 255.255.255.252"
}
]
},
{
"name":"GigabitEthernet0/1",
"ipv4":[
{
"address":"56.2.1.1 255.255.255.252"
}
]
},
{
"name":"GigabitEthernet0/2"
}
]
},
"ansible_net_python_version":"3.6.9",
"ansible_net_hostname":"host02342-mpls",
"ansible_net_model":"CISCO-CHA",
"ansible_net_serialnum":"F1539AM",
"ansible_net_gather_subset":[
"default"
],
"ansible_net_gather_network_resources":[
"l3_interfaces"
],
"ansible_net_version":"15.3(2)T",
"ansible_net_api":"cliconf",
"ansible_net_system":"ios",
"ansible_net_image":"flash0:/c3900-universalk9-mz.spa.153-2.t.bin",
"ansible_net_iostype":"IOS"
}
}
Table creation script
import mysql.connector
mydb = mysql.connector.connect(host="IPaddress", user="user", password="pw", database="db")
mycursor = mydb.cursor()
mycursor.execute("CREATE TABLE Routers (ansible_net_hostname NVARCHAR(255), ansible_net_model NVARCHAR(255), ansible_network_resources NVARCHAR(255))")
The script to import JSON data into MySQL
import json, pymysql
json_data = open("L3_out.json").read()
json_obj = json.loads(json_data)
con = pymysql.connect(host="IPaddress", user="user", password="pw", database="db")
cursor = con.cursor()
for item in json_obj:
ansible_net_hostname = item.get("ansible_net_hostname")
ansible_net_model = item.get("ansible_net_model")
ansible_network_resources = item.get("ansible_network_resources")
cursor.execute(
"insert into Routers(ansible_net_hostname, ansible_net_model, ansible_network_resources) value(%s, %s, %s)",
(ansible_net_hostname, ansible_net_model, ansible_network_resources)
con.commit()
con.close()
I'm having issues importing ansible_network_resources field object into the Routers table. The other columns (ansible_net_hostname, ansible_net_model) get inserted perfectly. What am I doing wrong?

First of all, it's not clear how does
for item in json_obj:
ansible_net_hostname=item.get("ansible_net_hostname")
work.
Since 'item' in your case is a key from the dictionary. In the file you shown there is only one root key "ansible_facts". So you are trying to call get() on the string.
To get the data of "ansible_network_resources" do the following:
for key in json_obj:
ansible_network_resources=json_obj[key].get("ansible_network_resources")

Column name with "." to be replaced with "_" using Python

Below code reads a JSON file and stores it into DB Table.
But few column names are generated with ".".
Only these particular column names has to be replaced with underscore "_". If at all "." is encountered in any column names before pushing to table. Data shouldn't be changed, but only column names with "." to "_" in my python code.
Below is what I tried and not sure how to replace the column names and then push to DB:
import pandas as pd
import json
import sys
import psycopg2
data = sys.argv[1]
user= sys.argv[2]
password = sys.argv[3]
host = sys.argv[4]
port = sys.argv[5]
db = sys.argv[6]
documenttype = sys.argv[7]
schema_name = sys.argv[8]
from sqlalchemy import create_engine
engine = create_engine('postgresql+psycopg2://'+user+':'+password+'#'+host+':'+port+'/'+db)
print("Database is connected")
df = pd.read_json(data)
df['RecordsNew'] = df['Records'].astype('|S80')
df_1 = pd.json_normalize(df['Records'])
df_1.columns = map(str.lower, df_1.columns)
table_name =documenttype.lower()
df_1.to_sql(table_name,schema=schema_name,con=engine, if_exists = 'append',index=False)
JSON code example:
{
"Records": [
{
"CommodityId": "3470",
"SourceSystem": "SSP-generic-CHILD4",
"CommodityName": "ANCHOR BOLTS - BILL OF MATERIALS",
"CommodityType": ""
},
{
"CommodityId": "468657",
"SourceSystem": "SSP-generic-CHILD4",
"CommodityName": "COOLING INSERT",
"CommodityType": ""
},
{
"CommodityId": "836519",
"SourceSystem": "SSP-generic-CHILD4",
"CommodityName": "DIAPHRAGM 2ND STAGE PGT25",
"CommodityType": ""
},
{
"CommodityId": "807525",
"SourceSystem": "SSP-generic-CHILD4",
"CommodityName": "MOBILE NOZZLE MACHINING 2ST MS5002C(S2N)",
"CommodityType": ""
"ReconciledBy":{"SourceSystem":"SSP-sap-CHILD1","UserId":"","PasswordAdapter":""}
},
...
],
"PageToken": "TlhXQ0FVcTlzNE8rQQ"
}
Example of column names generated with "."
"reconciledby.passwordadapter"
"procurementunit.sourcesystem"
"procurementunit.uniquename"
"sourcesystem.sourcesystemid"

Store and use json in MSSQL with python

I'm trying to upload a test JSON string to SQL Server
json_string = """ {
"orderID": 42,
"customerName": "John Smith",
"customerPhoneN": "555-1234",
"orderContents": [
{
"productID": 23,
"productName": "keyboard",
"quantity": 1
},
{
"productID": 13,
"productName": "mouse",
"quantity": 1
}
],
"orderCompleted": true
} """
parsed_string = json.loads(json_string)
cursor.execute("update Table set Status = ? where Name like ? ",(json.dumps(parsed_string), "Blabla"))
cnxn.commit()
How to return and work with this JSON from the database?
cursor.execute("""select Status from Table where Name like ?""", "Blabla")
rows = cursor.fetchall()
How can I print the value of the JSON?

Use the JSON data type that is supported in MySQL. You can find more about it here:
https://dev.mysql.com/doc/refman/5.7/en/json.html

s = json.dumps(DATA)
cursor.execute("update Table set Status = ? where Name like ? ",(s, "Blabla"))
cnxn.commit()
and
cursor.execute("""select Status from Table where Name like ?""", "Blabla")
res = cursor.fetchall()
DATA = json.loads(res[0][X])

Develop Reference

Python is a programming language that lets you work quickly and integrate systems more effectively.

Django - how can i insert '.json' file to SQLite DB? - python

Related

Trying to insert from JSON file to database

How do I convert a JSON file to a SQL lite database if the JSON file has multiple values?

Issue importing JSON object into a MySQL column

Column name with "." to be replaced with "_" using Python

Store and use json in MSSQL with python

Categories

Resources