I'm trying to produce a JSON format for a given entity and I'm having an issue getting the dictionary to NOT overwrite itself or become empty. This is pulling rows from a table in a MySQL database and attempting to produce JSON result from the query.
Here is my function:
def detail():
student = 'John Doe'
conn = get_db_connection()
cur = conn.cursor()
sql = ("""
select
a.student_name,
a.student_id,
a.student_homeroom_name,
a.test_id,
a.datetaken,
a.datecertified,
b.request_number
FROM student_information a
INNER JOIN homeroom b ON a.homeroom_id = b.homeroom_id
WHERE a.student_name = '""" + student + """'
ORDER BY datecertified DESC
""")
cur.execute(sql)
details=cur.fetchall()
dataset = defaultdict(dict)
case_dataset = defaultdict(dict)
case_dataset = dict(case_dataset)
for student_name, student_id, student_homeroom_name, test_id, datetaken, datecertified, request_number in details:
dataset[student_name]['student_id'] = student_id
dataset[student_name]['student_homeroom_name'] = student_homeroom_name
case_dataset['test_id'] = test_id
case_dataset['datetaken'] = datetaken
case_dataset['datecertified'] = datecertified
case_dataset['request_number'] = request_number
dataset[student_name]['additional_information'] = case_dataset
case_dataset.clear()
dataset= dict(dataset)
print(dataset)
cur.close()
conn.close()
I tried a few different ways but nothing seems to work. What I'm getting is nothing in the additonal_information key. What I'm getting is this:
{
"John Doe": {
"student_id": "1234",
"student_homeroom_name": "HR1",
"additional_information": []
}
}
What I'm expecting is something similar to the below JSON. However, I'm torn if this is even correct. Each student will have one to many test_id and I will need to iterate through them in my application.
{
"John Doe": {
"student_id": "1234",
"student_homeroom_name": "HR1",
"additional_information": [
{
"test_id": "0987",
"datetaken": "1-1-1970",
"datecertified": "1-2-1970",
"request_number": "5643"
},
{
"test_id": "12343",
"datetaken": "1-1-1980",
"datecertified": "1-2-1980",
"request_number": "39807"
}
]
}
}
Removing the clear() from the function produces this JSON:
{
"John Doe": {
"student_id": "1234",
"student_homeroom_name": "HR1",
"additional_information": [
{
"test_id": "0987",
"datetaken": "1-1-1970",
"datecertified": "1-2-1970",
"request_number": "5643"
},
{
"test_id": "0987",
"datetaken": "1-1-1970",
"datecertified": "1-2-1970",
"request_number": "5643"
}
]
}
}
lists are mutable objects. Which means that list's are passed by reference.
when you set
dataset[student]['additional_information'] = case_dataset
case_dataset.clear()
you're setting the list and then clearing it. So the list inside additional_information is also cleared.
Copy the list when setting it:
dataset[student]['additional_information'] = case_dataset[:]
case_dataset.clear()
Thanks everyone for the guidance and pointing me in the right direction.
I have what I'm looking for now. Based on some of the comments and troubleshooting, I updated my code. Here is what I did:
I added back additional_dataset as a list
Removed case_dataset = defaultdict(dict) and case_dataset = dict(case_dataset) and replaced it with case_dataset = {}.
Updated dataset[student_name]['additional_information'] = case_dataset with dataset[student_name]['additional_information'] = additional_dataset
Replaced case_dataset.clear() with case_dataset = {}
Here is my new code now
def detail():
student = 'John Doe'
conn = get_db_connection()
cur = conn.cursor()
sql = ("""
select
a.student_name,
a.student_id,
a.student_homeroom_name,
a.test_id,
a.datetaken,
a.datecertified,
b.request_number
FROM student_information a
INNER JOIN homeroom b ON a.homeroom_id = b.homeroom_id
WHERE a.student_name = '""" + student + """'
ORDER BY datecertified DESC
""")
cur.execute(sql)
details=cur.fetchall()
dataset = defaultdict(dict)
case_dataset = {} #2 - Updated to just dict
additional_dataset = [] #1 - added back additional_dataset as a list
for student_name, student_id, student_homeroom_name, test_id, datetaken, datecertified, request_number in details:
dataset[student_name]['student_id'] = student_id
dataset[student_name]['student_homeroom_name'] = student_homeroom_name
case_dataset['test_id'] = test_id
case_dataset['datetaken'] = datetaken
case_dataset['datecertified'] = datecertified
case_dataset['request_number'] = request_number
dataset[student_name]['additional_information'] = additional_dataset #3 - updated to additional_dataset
case_dataset = {} #4 - updated to clear with new dict
dataset= dict(dataset)
print(dataset)
cur.close()
conn.close()
This is what it produces now. This is a much better structure then what I was previously expecting.
{
"John Doe": {
"student_id": "1234",
"student_homeroom_name": "HR1",
"additional_information": [
{
"test_id": "0987",
"datetaken": "1-1-1970",
"datecertified": "1-2-1970",
"request_number": "5643"
},
{
"test_id": "12343",
"datetaken": "1-1-1980",
"datecertified": "1-2-1980",
"request_number": "39807"
}
]
}
}
Related
I have a csv file with the following structure:
team,tournament,player
Team 1,spring tournament,Rebbecca Cardone
Team 1,spring tournament,Salina Youngblood
Team 1,spring tournament,Catarina Corbell
Team 1,summer tournament,Cara Mejias
Team 1,summer tournament,Catarina Corbell
...
Team 10, spring tournament,Jessi Ravelo
I want to create a nested dictionary (team, tournament) with a list of player dictionary. The desired outcome would be something like:
{'data':
{Team 1:
{'spring tournament':
{'players': [
{name: Rebecca Cardone},
{name: Salina Youngblood},
{name: Catarina Corbell}]
},
{'summer tournament':
{'players': [
{name: Cara Mejias},
{name: Catarina Corbell}]
}
}
},
...
{Team 10:
{'spring tournament':
{'players': [
{name: Jessi Ravelo}]
}
}
}
}
I've been struggling to format it like this. I have been able to successfully nest the first level (team # --> tournament) but I cannot get the second level to nest. Currently, my code looks like this:
d = {}
header = True
with open("input.csv") as f:
for line in f.readlines():
if header:
header = False
continue
team, tournament, player = line.strip().split(",")
d_team = d.get(team,{})
d_tournament = d_team.get(tournament, {})
d_player = d_tournament.get('player',['name'])
d_player.append(player)
d_tournament['player'] = d_tournament
d_team[tournament] = d_tournament
d[team] = d_team
print(d)
What would be the next step in fixing my code so I can create the nested dictionary?
Some problems with your implementation:
You do d_player = d_tournament.get('player',['name']). But you actually want to get the key named players, and this should be a list of dictionaries. Each of these dictionaries must have the form {"name": "Player's Name"}. So you want
l_player = d_tournament.get('players',[]) (default to an empty list), and then do l_player.append({"name": player}) (I renamed it to l_player because it's a list, not a dict).
You do d_tournament['player'] = d_tournament. I suspect you meant d_tournament['player'] = d_player
Strip the whitespace off the elements in the rows. Do team, tournament, player = (word.strip() for word in line.split(","))
Your code works fine after you make these changes
I strongly suggest you use the csv.reader class to read your CSV file instead of manually splitting the line by commas.
Also, since python's containers (lists and dictionaries) hold references to their contents, you can just add the container once and then modify it using mydict["key"] = value or mylist.append(), and these changes will be reflected in parent containers too. Because of this behavior, you don't need to repeatedly assign these things in the loop like you do with d_team[tournament] = d_tournament
allteams = dict()
hasHeader = True
with open("input.csv") as f:
csvreader = csv.reader(f)
if hasHeader: next(csvreader) # Consume one line if a header exists
# Iterate over the rows, and unpack each row into three variables
for team_name, tournament_name, player_name in csvreader:
# If the team hasn't been processed yet, create a new dict for it
if team_name not in allteams:
allteams[team_name] = dict()
# Get the dict object that holds this team's information
team = allteams[team_name]
# If the tournament hasn't been processed already for this team, create a new dict for it in the team's dict
if tournament_name not in team:
team[tournament_name] = {"players": []}
# Get the tournament dict object
tournament = team[tournament_name]
# Add this player's information to the tournament dict's "player" list
tournament["players"].append({"name": player_name})
# Add all teams' data to the "data" key in our result dict
result = {"data": allteams}
print(result)
Which gives us what we want (prettified output):
{
'data': {
'Team 1': {
'spring tournament': {
'players': [
{ 'name': 'Rebbecca Cardone' },
{ 'name': 'Salina Youngblood' },
{ 'name': 'Catarina Corbell' }
]
},
'summer tournament': {
'players': [
{ 'name': 'Cara Mejias' },
{ 'name': 'Catarina Corbell' }
]
}
},
'Team 10': {
' spring tournament': {
'players': [
{ 'name': 'Jessi Ravelo' }
]
}
}
}
}
The example dictionary you describe is not possible (if you want multiple dictionaries under the key "Team 1", put them in a list), but this snippet:
if __name__ == '__main__':
your_dict = {}
with open("yourfile.csv") as file:
all_lines = file.readlines()
data_lines = all_lines[1:] # Skipping "team,tournament,player" line
for line in data_lines:
line = line.strip() # Remove \n
team, tournament_type, player_name = line.split(",")
team_dict = your_dict.get(team, {}) # e.g. "Team 1"
tournaments_of_team_dict = team_dict.get(tournament_type, {'players': []}) # e.g. "spring_tournament"
tournaments_of_team_dict["players"].append({'name': player_name})
team_dict[tournament_type] = tournaments_of_team_dict
your_dict[team] = team_dict
your_dict = {'data': your_dict}
For this example yourfile.csv:
team,tournament,player
Team 1,spring tournament,Rebbecca Cardone
Team 1,spring tournament,Salina Youngblood
Team 2,spring tournament,Catarina Corbell
Team 1,summer tournament,Cara Mejias
Team 2,summer tournament,Catarina Corbell
Gives the following:
{
"data": {
"Team 1": {
"spring tournament": {
"players": [
{
"name": "Rebbecca Cardone"
},
{
"name": "Salina Youngblood"
}
]
},
"summer tournament": {
"players": [
{
"name": "Cara Mejias"
}
]
}
},
"Team 2": {
"spring tournament": {
"players": [
{
"name": "Catarina Corbell"
}
]
},
"summer tournament": {
"players": [
{
"name": "Catarina Corbell"
}
]
}
}
}
}
Process finished with exit code 0
Maybe I overlook somethign but couldn't you use:
df.groupby(['team','tournament'])['player'].apply(list).reset_index().to_json(orient='records')
You might approach it this way:
from collections import defaultdict
import csv
from pprint import pprint
d = defaultdict(dict)
with open('f00.txt', 'r') as f:
reader = csv.DictReader(f)
for row in reader:
d[ row['team'] ].setdefault(row['tournament'], []
).append(row['player'])
pprint(dict(d))
Prints:
{'Team 1': {'spring tournament': ['Rebbecca Cardone',
'Salina Youngblood',
'Catarina Corbell'],
'summer tournament': ['Cara Mejias', 'Catarina Corbell']},
'Team 10': {' spring tournament': ['Jessi Ravelo']}}
I currently have this method in python code :
#app.route('/getData', methods = ['GET'])
def get_Data():
c.execute("SELECT abstract,category,date,url from Data")
data = c.fetchall()
resp = jsonify(data)
resp.status_code = 200
return resp
The output I get from this is:
[
[
"2020-04-23 15:32:13",
"Space",
"https://www.bisnow.com/new-jersey",
"temp"
],
[
"2020-04-23 15:32:13",
"Space",
"https://www.bisnow.com/events/new-york",
"temp"
]
]
However, I want the output to look like this:
[
{
"abstract": "test",
"category": "journal",
"date": "12-02-2020",
"link": "www.google.com"
},
{
"abstract": "test",
"category": "journal",
"date": "12-02-2020",
"link": "www.google.com"
}
]
How do I convert my output into an expected format?
As #jonrsharpe indicates, you simply cannot expect the tuple coming from this database query to turn into a dictionary in the JSON output. Your data variable does not contain the information necessary to construct the response you desire.
It will depend on your database but my recommendation would be to find a way to retrieve dicts from your database query instead of tuples, in which case the rest of your code should work as is. For instance, for sqlite, you could define your cursor c like this:
import sqlite3
connection = sqlite3.connect('dbname.db') # database connection details here...
connection.row_factory = sqlite3.Row
c = connection.cursor()
Now, if your database for some reason cannot support a dictionary cursor, you need to roll your own dictionary after retrieving the database query results. For your example, something like this:
fieldnames = ('abstract', 'category', 'date', 'link')
numfields = len(fieldnames)
data = []
for row in c.fetchall():
for idx in range(0, numfields - 1):
dictrow[fields[idx]] = row[idx]
data.append(dictrow)
I iterate over a list of field labels, which do not have to match your database columns but do have to be in the same order, and creating a dict by pairing the label with the datum from the db tuple in the same position. This passage would replace the single line data = c.fetchall() in OP.
I have a sample json file named a.json
The json data in a.json is as:
{
"a cappella": {
"word": "a cappella",
"wordset_id": "5feb6f679a",
"meanings": [
{
"id": "492099d426",
"def": "without musical accompaniment",
"example": "they performed a cappella",
"speech_part": "adverb"
},
{
"id": "0bf8d49e2e",
"def": "sung without instrumental accompaniment",
"example": "they sang an a cappella Mass",
"speech_part": "adjective"
}
]
},
"A.D.": {
"word": "A.D.",
"wordset_id": "b7e9d406a0",
"meanings": [
{
"id": "a7482f3e30",
"def": "in the Christian era",
"speech_part": "adverb",
"synonyms": [
"AD"
]
}
]
},.........
}
As suggested in my previous question I am looking on how to insert this data in to tables
Word: [word, wordset_id]
Meaning: [word, meaning_id, def, example, speech_part
Synonym: [word, synonym_word]
I tried reading file as:
import json
with open('a.json') as f:
d = json.load(f)
when I tried printing all words as:
for word in d:
print(word)
I got all words, but failed to get wordset_id for the same.
How can I insert the word and wordset_id in to the table word for the json format as above?
DBconnection as:
from flask import Flask
from flaskext.mysql import MySQL
app = Flask(__name__)
mysql = MySQL()
app.config['MYSQL_DATABASE_USER'] = 'root'
app.config['MYSQL_DATABASE_PASSWORD'] = 'root'
app.config['MYSQL_DATABASE_DB'] = 'wordstoday'
app.config['MYSQL_DATABASE_HOST'] = 'localhost'
mysql.init_app(app)
conn = mysql.connect()
cursor =conn.cursor()
When you try to execute code:
for word in d:
print(word)
It will only print the keys of the json object, not complete value. Instead, you can try doing something like this,
for word in d:
word_obj = d[word]
wordset_id = word_obj['wordset_id']
sql = "INSERT INTO Word (word, wordset_id) VALUES (%s, %s)"
values = (word, wordset_id)
cursor.execute(sql, values)
meaning_obj_list = d[word]['meanings']
for meaning_obj in meaning_obj_list:
meaning_id = meaning_obj['id']
definition = meaning_obj['def']
example = meaning_obj.get('example', None) # since it is not guaranteed that "example" key will be present in the data, it is safer to extract the value this way
speech_part = meaning_obj['speech_part']
sql = "INSERT INTO Meaning (word, meaning_id, def, example, speech_part) VALUES (%s, %s, %s, %s, %s)"
values = (word, meaning_id, definition, example, speech_part)
cursor.execute(sql, values)
db.commit()
Also, refrain from using the keys names such as def as this is a keyword in python.
I am designing a web page named display.html to display course content data in grid view.
Following link shows the http://www.prepbootstrap.com/bootstrap-theme/dark-admin/preview/bootstrap-grid.html type of display I needed.
First row will contain course name, second will contain sub section name and so on.
I'm using django python. I've written a function in views.py for retrieving data from mongodb.
The function returns a dictionary contains mapping of course name with list of chapter name, chapter name with list of sub-section name, and so on and renders it on web page display.html but I'm not able to display the data properly on html page.
My views.py functions are:
def get_course_structure(course_lst):
final_data = dict()
connection = MongoClient("server_ip", port)
database_name = 'edxapp'
database = connection[database_name]
cursor = database.modulestore.aggregate([{ "$match": {"_id.course":"CS000", "_id.category":"chapter"} },
{ "$project": { "metadata": 1, "definition": 1 } } ])
chapter = []
lst_chapter = []
for data in cursor:
chapter.append(data)
lst_chapter.append(data['metadata']['display_name'])
#print "CS000", lst_chapter
final_data= {}
final_data["CS000"] = lst_chapter
#for data in chp:
# print data['metadata']['display_name']
sequential = []
for data in chapter:
lst_children = data['definition']['children']
#print lst_children
lst_seq = []
for child in lst_children:
child = child.split('/')
#print child[5]
cur = database.modulestore.aggregate([{ "$match": {"_id.name":child[5]} },
{ "$project": { "metadata": 1, "definition": 1 } } ])
for rec in cur:
#print rec
sequential.append(rec)
lst_seq.append(rec['metadata']['display_name'])
#print data['metadata']['display_name'], lst_seq
final_data[data['metadata']['display_name']] = lst_seq
#for data in sequential:
# print data, "\n"
vertical = []
for data in sequential:
lst_children = data['definition']['children']
lst_ver = []
for child in lst_children:
child = child.split('/')
cur = database.modulestore.aggregate([{ "$match": {"_id.name":child[5]} },
{ "$project": { "metadata": 1, "definition": 1 } } ])
for rec in cur:
vertical.append(rec)
lst_ver.append(rec['metadata']['display_name'])
#print data['metadata']['display_name'], lst_ver
final_data[data['metadata']['display_name']] = lst_ver
#for data in vertical:
# print data, "\n"
for key in final_data:
print key, final_data[key]
return final_data
and
def get_dashboard(request):
courses_lst = get_student_enrolled_courses(request.GET['student_id'])
course_data = get_course_structure(courses_lst)
return render(request,'mis/display.html', {'course_data':course_data, 'student_id':request.GET['student_id'] })
Any help for doing this.
I have a script which takes data in a SQL Server database and parses it into a key:value pair JSON. I would like to give the three items in the JSON dictionary; one key, such as "ServiceRequest" at the highest level. So that my output would read as:
{
"ServiceRequest": [
{
"SRNUMBER": "1-3580171",
"FirstName": "Myla",
"LastName": "Threeoneone"
}
]
}
Program:
import pyodbc
import json
import collections
import requests
import urllib
connstr = 'DRIVER={SQL Server};SERVER=ServerName;DATABASE=DataBase; UID=UID;PWD=PWD'
conn = pyodbc.connect(connstr)
cursor = conn.cursor()
cursor.execute("""
SELECT SRNUMBER, FirstName, LastName
FROM MYLA311 """)
rows = cursor.fetchall()
# Convert query to row arrays
rowarray_list = []
for row in rows:
t = (row.SRNUMBER)
rowarray_list.append(t)
j = json.dumps(rowarray_list)
rowarrays_file = 'student_rowarrays.js'
f = open(rowarrays_file,'w')
# Convert query to objects of key-value pairs
objects_list = []
for row in rows:
d = collections.OrderedDict()
d['SRNUMBER']= row.SRNUMBER
d['FirstName']= row.FirstName
d['LastName']= row.LastName
objects_list.append(d)
j = json.dumps(objects_list)
objects_file = 'C:\Users\Administrator\Desktop\JSONOutput.txt'
f = open(objects_file,'w')
print >> f, j
print j
conn.close()
Actual Output:
[
{
"SRNUMBER": "1-3580171",
"FirstName": "Myla",
"LastName": "Threeoneone"
}
]
JSON and Python dictionaries are very similar.
Your desired output is already valid Python:
{
"ServiceRequest": [
{
"SRNUMBER": "1-3580171",
"FirstName": "Myla",
"LastName": "Threeoneone"
}
]
}
So all you need to do, is wrap your objects list in a Python dictionary:
output = {
'ServiceRequest': object_list
}
# then dump it
json.dumps(output)
# ...