Develop Reference

Convert JSON to CSV with Python 3

I need to get some data from the Meetup API, convert the JSON I get into a CSV, all of that in Python 3. I've never worked with JSON or Python, so I've run into some issues. Getting the data is not a problem, but I can't seem to parse the JSON object into a proper CSV file. My code, anonymized:
import json
import requests
import csv
def main():
# initialize variables
output_file = 'result_meetup.csv'
api_key = "YOUR_MEETUP_API_KEY"
params = {'key': api_key}
url_path = 'http://api.meetup.com/pro/:URLNAME/groups'
# get data from API, parse to JSON
data = requests.get(url_path, params=params)
data_parsed = data.json()
length_data = len(data_parsed) - 1
data_to_file = open(output_file, 'w')
csv_writer = csv.writer(data_to_file)
for i in range(0, length_data):
meetup = data_parsed[i]
csv_writer.writerow([meetup])
data_to_file.close()
if __name__ == "__main__":
main()
Now, as you can see, I write into the CSV as a list. This leads to a really ugly output, looking like this (again, the values in caps are anonymized):
{u'rsvps_per_event': 0.0, u'organizers': [{u'permission': u'organizer', u'name': u'NAME', u'member_id': ID}], u'repeat_rsvpers': 0, u'topics': [{u'lang': u'en_US', u'id': ID, u'urlkey': u'socialnetwork', u'name': u'Social Networking'}, {u'lang': u'en_US', u'id': ID, u'urlkey': u'education', u'name': u'Education'}, {u'lang': u'en_US', u'id': ID, u'urlkey': u'newtech', u'name': u'New Technology'}, {u'lang': u'en_US', u'id': ID, u'urlkey': u'business-referral-networking', u'name': u'Business Referral Networking'}], u'upcoming_events': 0, u'gender_female': 0.3499999940395355, u'pro_join_date': DATE, u'id': ID, u'category': [{u'shortname': u'career-business', u'sort_name': u'Career & Business', u'id': 2, u'name': u'Career & Business'}], u'city': u'CITY', u'member_count': 73, u'lon': LON, u'organizer_photo': {u'thumb_link': u'LINK.JPEG', u'base_url': u'URL', u'id': ID, u'type': u'member', u'photo_link': u'LINK.JPEG', u'highres_link': u'LINK.JPEG'}, u'average_age': 35.555599212646484, u'status': u'Active', u'description': u'DESCRIPTION' u'founded_date': DATE, lat': LAT, u'urlname': u'NAME', u'gender_male': 0.6000000238418579, u'name': u'NAME', u'country': u'Portugal', u'gender_unknown': 0.05000000074505806, u'past_events': 0, u'gender_other': 0.0, u'past_rsvps': 0}
So basically, the whole JSON object in a single CSV field, with weird 'u's, in lists and so on. However, if I don't write it as a list, I only get the fields of the JSON object, without the data, so it would just be 'Organizers', 'Name' etc, without the actual name.
As I said, I am a python beginner and haven't found any libraries to help me with it, but I'm sure they exist. Any help is really appreciated, and it would be great if it was Python3 compatible.
Edit: What I would like it to look like in the end:
I get multiple Meetup groups in the reply, all having the same structure as shown above. Therefore, the description of the value should be listed just once, as a header, and the values listed beneath (new lines and pipes indicating a new field of a csv sheet):
RSVPs_per_event | Organizer | ID
5 | Tom | 1
20 | Jack | 2
35 | Anne | 3

To convert the json data to csv you need to extract keys and write them in header and then work on the values. This might help you:
data_parsed = json.loads(Data)
header = data_parsed[0].keys()
csv_writer.writerow(header)
for i in range(0,length_data)
meetup = data_parsed[i].values()
csv_writer.writerow([meetup])

If anyone else has the same problem, I solved it, not very elegantly, but I needed the data. The JSON was too nestled and complex to parse, so now I just read the fields I need from the data.
import json
import requests
import csv
def main():
# initialize variables
output_file = 'result_meetup.csv'
api_key = "YOUR_API_KEY"
params = {'key': api_key}
url_path = 'http://api.meetup.com/pro/:URLNAME/groups'
# get data from API, parse to JSON
data = requests.get(url_path, params=params)
data_parsed = data.json()
length_data = len(data_parsed) - 1
data_to_file = open(output_file, 'w', newline='')
csv_writer = csv.writer(data_to_file, delimiter=";")
csv_writer.writerow(["id","name","city","country","member count","average age","founded_date","past_rsvps","rsvps_per_event","repeat_rsvpers","gender_unknown","gender_female","gender_male","gender_other"])
for i in range(0, length_data):
meetup = data_parsed[i]
id = meetup['id']
name = meetup['name']
city = meetup['city']
country = meetup['country']
member_count = meetup['member_count']
average_age = meetup['average_age']
founded_date = meetup['founded_date']
past_rsvps = meetup['past_rsvps']
rsvps_per_event = meetup['rsvps_per_event']
repeat_rsvpers = meetup['repeat_rsvpers']
gender_unknown = meetup['gender_unknown']
gender_female = meetup['gender_female']
gender_male = meetup['gender_male']
gender_other = meetup['gender_other']
csv_writer.writerow([id,name,city,country,member_count,average_age,founded_date,past_rsvps,rsvps_per_event,repeat_rsvpers,gender_unknown,gender_female,gender_male,gender_other])
data_to_file.close()
if __name__ == "__main__":
main()

"TypeError: string indices must be integers not str" seting a var from a dictionary key

I grab info from an API call that comes in the form of a dictionary. I need to grab a specific key and set a variable to it. Easy enough but observe. The code below is only to the processes involved.
totalEntries = 0 # global var
api_data = requests.get(apiEndpoint).json()
def populateVars():
global totalEntries
totalEntries = api_data['total_entries']
output of the api_data just for giggles:
{u'per_page': 100, u'total_entries': 1, u'current_page': 1, `u'restaurants': [{u'city': u'Abilene', u'reserve_url': u'http://www.opentable.com/single.aspx?rid=152869', u'name': u'Copper Creek Fine Texas Dining', u'area': u'Dallas - Fort Worth', u'country': u'US', u'price': 2, u'phone': u'3256924424', u'state': u'TX', u'postal_code': u'79602', u'address': u'4401 Loop 322', u'lat': 32.397913, u'image_url': u'https://www.opentable.com/img/restimages/152869.jpg', u'lng': -99.716776, u'mobile_reserve_url': u'http://mobile.opentable.com/opentable/?restId=152869', u'id': 152869}]}`
and again the error:
totalEntries = api_data['total_entries']
TypeError: string indices must be integers, not str
Why is this happening? What is the fix?

As jonsharpe already pointed out, your api_data is a string, which can only be indexed by integers.
Instead of using:
api_data = requests.get(apiEndpoint).json()
consider using json.load:
resp = requests.get(apiEndpoint)
api_data = json.loads(resp.text)

Can't get dictionary value from key in unicode dictionary

The following code stores a unicode dictionary in the variable webproperties_list:
webproperties_list = service.management().webproperties().list(
accountId='~all').execute()
profile_id = webproperties_list.get(u'defaultProfileId')
print profile_id
For some reason the .get() on the key of u'defaultProfileId' is giving me None, even though I know it is in the response. I also tried the get without the u and I still get a None response:
profile_id = webproperties_list.get('defaultProfileId')
Do I need to do something to the dict before I get the value from the key, or am I doing something else wrong entirely?
UPDATE:
Here is the response I get:
{u'username': u'removed', u'kind': u'analytics#webproperties', u'items': [{u'profileCount': 1, u'kind': u'analytics#webproperty', u'name': u'removed', u'level': u'STANDARD', u'defaultProfileId': u'removed'.....
I need to retrieve the value of u'defaultProfileId'
Not really sure how to get a value from a key that is in a dict within a list within a dict...

To figure out how to access it, it sometimes helps to go step by step:
>>> d
{u'username': u'removed', u'items': [{u'profileCount': 1, u'defaultProfileId': u'removed', u'kind': u'analytics#webproperty', u'name': u'removed', u'level': u'STANDARD'}], u'kind': u'analytics#webproperties'}
>>> d['items']
[{u'profileCount': 1, u'defaultProfileId': u'removed', u'kind': u'analytics#webproperty', u'name': u'removed', u'level': u'STANDARD'}]
>>> d['items'][0]
{u'profileCount': 1, u'defaultProfileId': u'removed', u'kind': u'analytics#webproperty', u'name': u'removed', u'level': u'STANDARD'}
>>> d['items'][0]['defaultProfileId']
u'removed'

Using the data for 'requests' module in python

I am able to load in some weather data from the requests module for python with the following code:
from pprint import pprint
import requests
r = requests.get('http://api.openweathermap.org/data/2.5/weather?q=London')
pprint(r.json())
But how do I actually use the data it produces? I cannot for the life of me find the relevant documentation or tutorial on how to do this. this is the output of the pprint:
{u'base': u'cmc stations',
u'clouds': {u'all': 0},
u'cod': 200,
u'coord': {u'lat': 42.98, u'lon': -81.23},
u'dt': 1397676977,
u'id': 6058560,
u'main': {u'humidity': 25,
u'pressure': 1024,
u'temp': 278,
u'temp_max': 283.15,
u'temp_min': 275.37},
u'name': u'London',
u'rain': {u'1h': 0.25},
u'snow': {u'3h': 0},
u'sys': {u'country': u'CA',
u'message': 0.0467,
u'sunrise': 1397644810,
u'sunset': 1397693338},
u'weather': [{u'description': u'light rain'
u'icon': u'10d',
u'id': 500,
u'main': u'Rain'}],
u'wind': {u'deg': 168.001, u'speed': 3.52}}
How could I address an item within the list? For example to print just the temp on it's own and maybe to use it as a variable. E.g.:
temp = *not sure what to put here*
print temp

Now that you have the results:
results = r.json()
just access it like any other Python dict:
main = results['main'] # Get the 'main' key's value out of results
temp = main['temp'] # Get the 'temp' key's value out of main
print temp
or more tersely (and the way you'd almost always write this in real life):
print results['main']['temp']

Python issue with getting data out of json.loads

I have the following JSON data which I pass through json.loads:
{
"meta":{
"limit":20,
"next":null,
"offset":0,
"previous":null,
"total_count":2
},
"objects":[
{
"attributes":"{u'_code': u'[ON CODE]/[OFF CODE]', u'_type': u'pick actuator or sensor', u'code': u'AF126E/AF1266', u'type': u'actuator'}",
"id":1,
"module":"/api/v1/module/1/",
"moduleName":"rfm_ninjablock (ninjablock)",
"name":"HallwayLight"
},
{
"attributes":"{u'_code': u'[ON CODE]/[OFF CODE]', u'_type': u'pick actuator or sensor', u'code': u'0x53df5c', u'type': u'sensor'}",
"id":2,
"module":"/api/v1/module/1/",
"moduleName":"rfm_ninjablock (ninjablock)",
"name":"ToiletDoor"
}
]
}
I'm trying to get all the data out of it, but I'm having trouble referencing the data.
My code is as follows:
for object in r['objects']:
for attributes in object.iteritems():
print attributes
Which gives me:
(u'attributes', u"{u'_code': u'[ON CODE]/[OFF CODE]', u'_type': u'pick actuator or sensor', u'code': u'AF126E/AF1266', u'type': u'actuator'}")
(u'moduleName', u'rfm_ninjablock (ninjablock)')
(u'id', 1)
(u'module', u'/api/v1/module/1/')
(u'name', u'HallwayLight')
(u'attributes', u"{u'_code': u'[ON CODE]/[OFF CODE]', u'_type': u'pick actuator or sensor', u'code': u'0x53df5c', u'type': u'sensor'}")
(u'moduleName', u'rfm_ninjablock (ninjablock)')
(u'id', 2)
(u'module', u'/api/v1/module/1/')
(u'name', u'ToiletDoor')
I'm not really sure of referencing these or if indeed I'm doing it right.
Attributes contains JSON already, as that is how it is stored in the database.

You have a problem with the way the original data was serialized. The attributes dict of each element was not serialized as a set of nested dicts, but as an outer dict containing Python string representations of the inner dicts.
You should post the code that did the original serialization.

As the dict inside attributes ends up as a string representation I used the following code to convert it to the dict:
for object in r['objects']:
attrib = []
attrib = ast.literal_eval(object['attributes'])
print attrib['code']