Develop Reference

How to retrieve the "Initiator" field from the devtools Network panel with Selenium?

I'm trying to extract all the URL network requests from a website and establish a relationship of hierarchy between them i.e. if one URL request is generating another request. Something like a chain of requests.
As you know, in the Network panel, there is a field called "Initiator" in the Requests table which tells you the origin or parent of a specific request (if there's any). Manually, I can use the browser, go to the Network panel in the developer tools, load the website and download the resulting HAR file. For example:
{
"startedDateTime": "2019-11-05T17:38:46.775Z",
"time": 15.676000155508518,
"request": {
"method": "POST",
"url": "https://www.google.com/gen_204?oq=&gs_l=psy-ab.22...0.0..847450...0.0..0.0.0.......0......gws-wiz.",
"httpVersion": "http/2.0",
"headers": [
{
"name": ":path",
"value": "/gen_204?oq=&gs_l=psy-ab.22...0.0..847450...0.0..0.0.0.......0......gws-wiz."
},
{
"name": "sec-fetch-mode",
"value": "no-cors"
},
{
"name": "origin",
"value": "https://www.google.com"
},
{
"name": "accept-encoding",
"value": "gzip, deflate, br"
},
{
"name": "accept-language",
"value": "en-GB,en;q=0.9,en-US;q=0.8,es-US;q=0.7,es;q=0.6"
},
{
"name": "user-agent",
"value": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/76.0.3809.100 Chrome/76.0.3809.100 Safari/537.36"
},
{
"name": "content-type",
"value": "text/plain;charset=UTF-8"
},
{
"name": "accept",
"value": "*/*"
},
{
"name": "referer",
"value": "https://www.google.com/"
},
{
"name": ":authority",
"value": "www.google.com"
},
{
"name": "cookie",
"value": "CONSENT=YES+GB.en+20160414-00-0; SEARCH_SAMESITE=CgQIg44B; ANID=AHWqTUlE3OPRfM5R1dtW0XvyIu2NOdLWoSHEgFemsFslXQTIzFKFCL-7kTtDZAr_; NID=190=Ezp7tXRaU_Rs2BS9RprlsS9QN9-PcwpYNSLwaOVGVFFp6pWepIjDqsYlgyLqb2eATn6HwUNs-SmgzAmtEm63fgX-YWVgbOyX7GU1esPamrN-GWXfwmXyrsqsTBOOQTzsHB3Q89tATDNQE_OKGd0YgCxMp9m9QXke2BJANdKdBYujl-g5tS8ZXcq0pw; 1P_JAR=2019-11-05-17; DV=o32RqCcqMlgsAJonGalrPPWlv0DK4xZ24gV5ztaaewMAAAA"
},
{
"name": ":scheme",
"value": "https"
},
{
"name": "sec-fetch-site",
"value": "same-origin"
},
{
"name": "content-length",
"value": "0"
},
{
"name": ":method",
"value": "POST"
}
],
"queryString": [
{
"name": "oq",
"value": ""
},
{
"name": "gs_l",
"value": "psy-ab.22...0.0..847450...0.0..0.0.0.......0......gws-wiz."
}
],
"cookies": [
{
"name": "CONSENT",
"value": "YES+GB.en+20160414-00-0",
"expires": null,
"httpOnly": false,
"secure": false
},
{
"name": "SEARCH_SAMESITE",
"value": "CgQIg44B",
"expires": null,
"httpOnly": false,
"secure": false
},
{
"name": "ANID",
"value": "AHWqTUlE3OPRfM5R1dtW0XvyIu2NOdLWoSHEgFemsFslXQTIzFKFCL-7kTtDZAr_",
"expires": null,
"httpOnly": false,
"secure": false
},
{
"name": "NID",
"value": "190=Ezp7tXRaU_Rs2BS9RprlsS9QN9-PcwpYNSLwaOVGVFFp6pWepIjDqsYlgyLqb2eATn6HwUNs-SmgzAmtEm63fgX-YWVgbOyX7GU1esPamrN-GWXfwmXyrsqsTBOOQTzsHB3Q89tATDNQE_OKGd0YgCxMp9m9QXke2BJANdKdBYujl-g5tS8ZXcq0pw",
"expires": null,
"httpOnly": false,
"secure": false
},
{
"name": "1P_JAR",
"value": "2019-11-05-17",
"expires": null,
"httpOnly": false,
"secure": false
},
{
"name": "DV",
"value": "o32RqCcqMlgsAJonGalrPPWlv0DK4xZ24gV5ztaaewMAAAA",
"expires": null,
"httpOnly": false,
"secure": false
}
],
"headersSize": -1,
"bodySize": 0
},
"response": {
"status": 204,
"statusText": "",
"httpVersion": "http/2.0",
"headers": [
{
"name": "date",
"value": "Tue, 05 Nov 2019 17:38:46 GMT"
},
{
"name": "server",
"value": "gws"
},
{
"name": "x-frame-options",
"value": "SAMEORIGIN"
},
{
"name": "content-type",
"value": "text/html; charset=UTF-8"
},
{
"name": "status",
"value": "204"
},
{
"name": "alt-svc",
"value": "quic=\":443\"; ma=2592000; v=\"46,43\",h3-Q050=\":443\"; ma=2592000,h3-Q049=\":443\"; ma=2592000,h3-Q048=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000"
},
{
"name": "content-length",
"value": "0"
},
{
"name": "x-xss-protection",
"value": "0"
}
],
"cookies": [],
"content": {
"size": 0,
"mimeType": "text/html"
},
"redirectURL": "",
"headersSize": -1,
"bodySize": -1,
"_transferSize": 54
},
"cache": {},
"timings": {
"blocked": 1.1320006029605865,
"dns": -1,
"ssl": -1,
"connect": -1,
"send": 0.16199999999999992,
"wait": 14.122000366747379,
"receive": 0.25999918580055237,
"_blocked_queueing": 0.5990006029605865
},
"serverIPAddress": "216.58.204.68",
"_initiator": {
"type": "script",
"stack": {
"callFrames": [
{
"functionName": "s_1pb",
"scriptId": "129",
"url": "https://www.google.com/xjs/_/js/k=xjs.s.en_GB.UISl_YucLj8.O/ck=xjs.s.or8k_ixGu54.L.W.O/m=Fkg7bd,HcFEGb,IvlUe,MC8mtf,OF7gzc,RMhBfe,T4BAC,TJw5qb,TbaHGc,Y33vzc,cdos,hsm,iDPoPb,jsa,mvYTse,tg8oTe,uz938c,vWNDde,ws9Tlc,yQ43ff,d,csi/am=BAAAsAjYuwOC_L8VAAQAfAYAAAFuwQYLhCGhYqwOEAE/d=1/dg=2/br=1/ct=zgms/rs=ACT90oGdwE1ooFdbHyz-Vk2BhYjwAv-QDQ",
"lineNumber": 2323,
"columnNumber": 376
},
In this case, the URL https://www.google.com/gen_204?oq=&gs_l=psy-ab.22... is initiated by the URL https://www.google.com/xjs/_/js/k=xjs.s.en_GB.UISl.... You can see this information in the key "Initiator" -> "callFrames" -> "url".
The idea is to obtain this information (URLs that are calling others) or download the HAR file automatically using Selenium. I have tried this:
1. Selenium + browsermob proxy
Problem: The resulting HAR files doesn't have the "Initiator" field and there's no way to connect initiator requests and their dependencies.
2. Selenium Performance Logs
I am using this code to get the performance logs from Selenium:
caps = DesiredCapabilities.CHROME
caps['loggingPrefs'] = {'performance': 'ALL'}
chrome_options = webdriver.ChromeOptions()
driver = webdriver.Chrome(driver_path,desired_capabilities=caps)
driver.get("http://google.com")
browser_log = driver.get_log('performance')
Here, the only method where the "initiator" field can be found is in the "Network.requestWillBeSent" method which has a URL but it's not related to another one. Seems that each "message" field is independent and again, there's no way to connect initiator requests and their dependencies.
driver.execute_script
I saw this code in this question: How to access Network panel on google chrome developer tools with selenium?
driver = webdriver.Chrome('/path/to/chromedriver)
driver.get('https://www.google.com');
log = driver.execute_script("return window.performance.getEntries();")
#ANOTHER WAY
#log = driver.execute_script("var performance = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {}; var network = performance.getEntries() || {}; return network;")
The resulting log is completely different to a HAR file or a performance log and it doesn't have any information that can be used to relate requests. Example:
{
"startTime": 0,
"initiatorType": "navigation",
"unloadEventStart": 0,
"fetchStart": 69.29999962449074,
"duration": 1311.3000001758337,
"responseStart": 172.89999965578318,
"nextHopProtocol": "h2",
"transferSize": 68052,
"connectStart": 70.19999995827675,
"domainLookupStart": 70.19999995827675,
"redirectStart": 0,
"domContentLoadedEventEnd": 504.90000005811453,
"responseEnd": 190.7999999821186,
"requestStart": 100.49999970942736,
"type": "navigate",
"secureConnectionStart": 80.40000032633543,
"connectEnd": 99.99999962747097,
"redirectCount": 0,
"workerStart": 0,
"decodedBodySize": 233300,
"loadEventStart": 1304.8000000417233,
"encodedBodySize": 67329,
"serverTiming": [],
"entryType": "navigation",
"domInteractive": 487.699999473989,
"domContentLoadedEventStart": 487.80000023543835,
"redirectEnd": 0,
"name": "https://www.google.com/?gws_rd=ssl",
"domainLookupEnd": 70.19999995827675,
"unloadEventEnd": 0,
"loadEventEnd": 1311.3000001758337,
"domComplete": 1303.3999996259809,
"toJSON": {}
},
{
"initiatorType": "img",
"fetchStart": 298.20000007748604,
"duration": 14.100000262260437,
"responseStart": 310.70000026375055,
"responseEnd": 312.3000003397465,
"transferSize": 6146,
"connectStart": 298.20000007748604,
"domainLookupStart": 298.20000007748604,
"redirectStart": 0,
"toJSON": {},
"requestStart": 299.80000015348196,
"secureConnectionStart": 0,
"connectEnd": 298.20000007748604,
"workerStart": 0,
"decodedBodySize": 5969,
"startTime": 298.20000007748604,
"encodedBodySize": 5969,
"serverTiming": [],
"entryType": "resource",
"redirectEnd": 0,
"name": "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png",
"domainLookupEnd": 298.20000007748604,
"nextHopProtocol": "h2"
},
Therefore, I think the HAR file downloaded from the developer tools Network panel is the only source of information that allows to establish relationships between URL requests. Since I'm crawling many websites, I need to automatise the process but I can't find a way to do it.
Any ideas about downloading the HAR files automatically of extracting the info I need using Selenium directly would be appreciated.

You can fetch the _initiator data from the page's raw_entry.
with open('at.har', 'r') as f:
har_parser = HarParser(json.loads(f.read()))
pages_root = har_parser.pages[0]
initiator_dict = {}
for page in pages_root:
if "url" in initiator:
initiator_dict[page.request.url] = initiator['url']
res = defaultdict(list)
for key, val in sorted(initiator_dict.items()):
res[val].append(key)

Json decoder error when loading a json file. While the File is validated as Json. in Python

so I am opening a Json file and when I try to load the file to a variable I get an errr because it can't read the file. While I have validated (online) that the Json file is valid. I am using this code:
with open("messagesTest2.json") as json_file:
data = json.load(json_file) <----- ERROR
for p in data['commits']:
print(p['message'])
And I get this error. While I have another json file that is also validated and this code works. But this file doesn't work. My guess is that somewhere in the file there is something that it cannot translate as json? Is the decoder's fault?
in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Do you have any idea how to fix it? Keep in mind that the json file is valid else I'll have to show the file but I'll have to hide some data :D
The Json file (The urls/passwards/logins/etc have been replaced but the format remains the same) :
{
"commits": [{
"sha": "asjdaskldjkalsk",
"node_id": "sakldjaskldjaskldjklas",
"commit": {
"author": {
"name": "korki",
"email": "korki#kth.se",
"date": "2015-09-07T22:06:51Z"
},
"committer": {
"name": "korki",
"email": "korki#kth.se",
"date": "2015-09-07T22:06:51Z"
},
"message": "Added LaTex template and instructions",
"tree": {
"sha": "askdljaskdlajsklda",
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915"
},
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"comment_count": 0,
"verification": {
"verified": "False",
"reason": "unsigned",
"signature": "None",
"payload": "None"
}
},
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"html_url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"comments_url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"author": {
"login": "korki",
"id": 999,
"node_id": "askljdklas==",
"type": "User",
"site_admin": "None"
},
"committer": {
"login": "korki",
"id": 999,
"node_id": "askljdklas==",
"type": "User",
"site_admin": "None"
},
"parents": [{
"sha": "asdaskldjasdklsjl",
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"html_url": "https://gits-15.ds.sd.se/04dd5b226dda1915"
}]
}, {
"sha": "kasdjklasdjklas",
"node_id": "sdklasjdklasjkl",
"commit": {
"author": {
"name": "korki",
"email": "korki#kth.se",
"date": "2015-08-31T10:45:24Z"
},
"committer": {
"name": "korki",
"email": "korki#kth.se",
"date": "2015-08-31T10:45:24Z"
},
"message": "Update README.md",
"tree": {
"sha": "askldjkasldjklas",
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915"
},
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"comment_count": 0,
"verification": {
"verified": "None",
"reason": "unsigned",
"signature": "None",
"payload": "None"
}
},
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"html_url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"comments_url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"author": {
"login": "korki",
"id": 999,
"node_id": "dkasdasdnas==",
"type": "User",
"site_admin": "None"
},
"committer": {
"login": "korki",
"id": 999,
"node_id": "askldaskldja==",
"type": "User",
"site_admin": "None"
},
"parents": [{
"sha": "dlkasdjklas;dlkjas;",
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"html_url": "https://gits-15.ds.sd.se/04dd5b226dda1915"
}]
}, {
"sha": "dsagadsgsgdsa",
"node_id": "sdagfsdgsd",
"commit": {
"author": {
"name": "korki",
"email": "korki#kth.se",
"date": "2015-08-31T10:44:42Z"
},
"committer": {
"name": "korki",
"email": "korki#kth.se",
"date": "2015-08-31T10:44:42Z"
},
"message": "Initial commit",
"tree": {
"sha": "asdasddasdas",
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915"
},
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"comment_count": 0,
"verification": {
"verified": "None",
"reason": "unsigned",
"signature": "None",
"payload": "None"
}
},
"url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"html_url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"comments_url": "https://gits-15.ds.sd.se/04dd5b226dda1915",
"author": {
"login": "korki",
"id": 999,
"node_id": "kjklklj==",
"type": "User",
"site_admin": "None"
},
"committer": {
"login": "korki",
"id": 999,
"node_id": "jhkjkj==",
"gravatar_id": "",
"type": "User",
"site_admin": "None"
},
"parents": []
}]
}

That error means it is reading a blank file. Make sure you are reading the file you think you are reading.
EDIT: Another possibility is that you have already read through all the lines of the file. If you read through all the lines and try to read the file, it will appear as a blank file.

I had the exact same issue. I used a powershell script to create a json file and I tried to read the file from another python script but I kept getting the same error as you, even though the JSON file was properly formatted. The issue was I was using a powershell command, "Out-File" Instead, I used Set-Content and it fixed the issue. I believe it was an encoding difference between the commands. Maybe look at how you created the JSON file and the encoding used. I know this is late but I'll share anyways just in case anyone else is having the same issue.

Getting a KeyError when trying to iterate a Python dict obtained using OpenCalais

I am trying to tag content using OpenCalais. The following is my code that I'm using the communicate with the API:
import httplib2
import json
import ast
# Some local values needed for the call
LOCAL_API_KEY = '***********************' # Aquire this by registering at the Calais site
CALAIS_TAG_API = 'https://api.thomsonreuters.com/permid/calais'
# Some sample text from a news story to pass to Calais for analysis
test_body = 'Samsung is closing its Milk Music streaming service'
# header information need by Calais.
# For more info see http://www.opencalais.com/documentation/calais-web-service-api/api-invocation/rest
headers = {
'X-AG-Access-Token': LOCAL_API_KEY,
'Content-Type': 'text/raw',
'outputFormat': 'application/json',
}
# Create your http object
http = httplib2.Http()
# Make the http post request, passing the body and headers as needed.
response, content = http.request(CALAIS_TAG_API, 'POST', headers=headers, body=test_body)
jcontent = json.loads(content) # Parse the json return into a python dict
output = json.dumps(jcontent, indent=4) # Pretty print the resulting dictionary returned.
print output
Anyway, this works nicely, as I am able to get the following output (print output).
{
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/cat/1": {
"score": 1,
"forenduserdisplay": "false",
"name": "Business_Finance",
"_typeGroup": "topics"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/cat/2": {
"score": 1,
"forenduserdisplay": "false",
"name": "Entertainment_Culture",
"_typeGroup": "topics"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/lid/DefaultLangId": {
"forenduserdisplay": "false",
"language": "http://d.opencalais.com/lid/DefaultLangId/InputTextTooShort",
"_typeGroup": "language"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/Industry/1": {
"name": "Phones & Handheld Devices - NEC",
"permid": "4294951233",
"forenduserdisplay": "false",
"_typeGroup": "industry",
"relevance": 0.8,
"rcscode": "B:1768"
},
"http://d.opencalais.com/comphash-1/c021d644-16e9-3060-96fe-b3be0cd4ae1e": {
"_typeReference": "http://s.opencalais.com/1/type/em/e/Company",
"_type": "Company",
"name": "Samsung",
"confidence": {
"aggregate": "0.905",
"resolution": "1.0",
"statisticalfeature": "0.876",
"dblookup": "0.0"
},
"_typeGroup": "entities",
"instances": [
{
"detection": "[]Samsung[ is closing its Milk Music streaming]",
"length": 7,
"exact": "Samsung",
"suffix": " is closing its Milk Music streaming",
"offset": 0
}
],
"confidencelevel": "0.905",
"relevance": 0.8,
"nationality": "N/A",
"resolutions": [
{
"name": "SAMSUNG ELECTRONICS CO,.LTD",
"permid": "4295882451",
"commonname": "Samsung Elec",
"primaryric": "005930.KS",
"score": 1,
"ticker": "005930",
"id": "https://permid.org/1-4295882451"
}
],
"forenduserdisplay": "false"
},
"doc": {
"info": {
"document": "Samsung is closing its Milk Music streaming service",
"docId": "http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1",
"docDate": "2016-08-22 13:02:01.814",
"docTitle": "",
"ontology": "http://mdaas-virtual-onecalais.int.thomsonreuters.com/owlschema/9.8/onecalais.owl.allmetadata.xml",
"calaisRequestID": "eef490a6-2e3e-7cac-156b-257ebcf3beba",
"id": "http://id.opencalais.com/b*RzenPxfvWZmjCvQqpzNA"
},
"meta": {
"stagsVer": "OneCalais_9.8-RELEASE-b6-2016-07-18_14:00:15",
"contentType": "text/raw",
"language": "InputTextTooShort",
"serverVersion": "OneCalais_9.8-RELEASE:109",
"submissionDate": "2016-08-22 13:02:01.679",
"processingVer": "AllMetadata",
"submitterCode": "0ca6a864-5659-789d-5f32-f365f695e757",
"signature": "digestalg-1|BovyytInhxJhSerNjEFvOZNAHJQ=|Q5g9GCOSi7+FnERjgY9y4B9oJukYPjYeTl6v+Zu81BJLwOBcIZZ/eA=="
}
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/ComponentVersions": {
"version": [
"Deals Index:201608221149:201608221149",
"index-refineries:201608202306:201608202306",
"config-physicalAssets-powerStations:480:480",
"OA Index:201608212349:201608212349",
"NextTags:OneCalais_9.8-RELEASE:109",
"config-sca-DataPackage:38:38",
"com.clearforest.infoext.dial4j.plugins-basistechconfig:OneCalais_9.8-RELEASE:109",
"People Index:201608221124:201608221124",
"config-negativeSignature:480:480",
"Dial4J:OneCalais_8.6-RELEASE:209",
"OA Override:507:507",
"People Override:480:480",
"index-vessels:201608201644:201608201644",
"config-refineries:480:480",
"config-cse:507:507",
"config-vessels:480:480",
"OneCalais:OneCalais_9.8-RELEASE:109",
"config-physicalAssets-mines:480:480",
"SocialTags Index:201608212334:201608212334",
"BlackList:504:504",
"index-ports:201608202256:201608202256",
"config-physicalAssets-ports:480:480",
"config-drugs:480:480"
],
"_typeGroup": "versions"
},
"http://d.opencalais.com/comphash-1/e89d0187-8b46-3f8d-9f6b-4995a709c85e": {
"_typeReference": "http://s.opencalais.com/1/type/em/e/Company",
"_type": "Company",
"name": "Milk Music",
"confidence": {
"aggregate": "0.499",
"resolution": "0.0",
"statisticalfeature": "0.775",
"dblookup": "0.0"
},
"_typeGroup": "entities",
"instances": [
{
"suffix": " streaming service",
"prefix": "Samsung is closing its ",
"detection": "[Samsung is closing its ]Milk Music[ streaming service]",
"length": 10,
"offset": 23,
"exact": "Milk Music"
}
],
"confidencelevel": "0.499",
"relevance": 0.8,
"nationality": "N/A",
"forenduserdisplay": "false"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/cat/3": {
"score": 1,
"forenduserdisplay": "false",
"name": "Technology_Internet",
"_typeGroup": "topics"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/1": {
"name": "Streaming music services",
"importance": "1",
"_typeGroup": "socialTag",
"originalValue": "Streaming music services",
"socialTag": "http://d.opencalais.com/genericHasher-1/d1447a37-4c52-3b2f-a9d2-40984014685b",
"forenduserdisplay": "true",
"id": "http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/1"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/3": {
"name": "Milk Music",
"importance": "1",
"_typeGroup": "socialTag",
"originalValue": "Milk Music (streaming service)",
"socialTag": "http://d.opencalais.com/genericHasher-1/471ee9b8-9f72-3a81-aa13-2a7d44658521",
"forenduserdisplay": "true",
"id": "http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/3"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/2": {
"name": "Digital audio",
"importance": "1",
"_typeGroup": "socialTag",
"originalValue": "Digital audio",
"socialTag": "http://d.opencalais.com/genericHasher-1/64447afb-045b-34db-9a52-dae5bed0254e",
"forenduserdisplay": "true",
"id": "http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/2"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/5": {
"name": "Smartphones",
"importance": "2",
"_typeGroup": "socialTag",
"originalValue": "Smartphones",
"socialTag": "http://d.opencalais.com/genericHasher-1/e42d9d7b-150b-3c30-974f-87a1fba000ef",
"forenduserdisplay": "true",
"id": "http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/5"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/4": {
"name": "Samsung",
"importance": "2",
"_typeGroup": "socialTag",
"originalValue": "Samsung",
"socialTag": "http://d.opencalais.com/genericHasher-1/97370f53-c2f8-31b8-bbcf-aa685e504714",
"forenduserdisplay": "true",
"id": "http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/4"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/7": {
"name": "Samsung Galaxy",
"importance": "2",
"_typeGroup": "socialTag",
"originalValue": "Samsung Galaxy",
"socialTag": "http://d.opencalais.com/genericHasher-1/64b8e664-bbdc-3731-b712-eb30990eab6f",
"forenduserdisplay": "true",
"id": "http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/7"
},
"http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/6": {
"name": "Samsung Music Hub",
"importance": "2",
"_typeGroup": "socialTag",
"originalValue": "Samsung Music Hub",
"socialTag": "http://d.opencalais.com/genericHasher-1/a3310a01-ef6f-314e-90b2-a303822b965c",
"forenduserdisplay": "true",
"id": "http://d.opencalais.com/dochash-1/8a7adab6-d07e-38e6-b0c9-5db5220336c1/SocialTag/6"
}
}
Something I noticed is that the keys are all hyperlinks. Anyway, I want to print all the socialTags from the output. To do that, I wrote this following code:
# Print all the social tags
for key, value in ast.literal_eval(output).items():
if value["_typeGroup"] == 'socialTag':
print value["name"]
However, I get this error:
Traceback (most recent call last):
File "opencal.py", line 30, in <module>
if value["_typeGroup"] == 'socialTag':
KeyError: '_typeGroup'
What is this error? Or to be more precise, what is the correct way to get the socialTags? Thanks.

Convert deeply nested json from facebook to dataframe in python

I am trying to get user details of persons who has put likes, comments on Facebook posts. I am using python facebook-sdk package. Code is as follows.
import facebook as fi
import json
graph = fi.GraphAPI('Access Token')
data = json.dumps(graph.get_object('DSIfootcandy/posts'))
From the above, I am getting a highly nested json. Here I will put only a json string for one post in the fb.
{
"paging": {
"next": "https://graph.facebook.com/v2.0/425073257683630/posts?access_token=&limit=25&until=1449201121&__paging_token=enc_AdD0DL6sN3aDZCwfYY25rJLW9IZBZCLM1QfX0venal6rpjUNvAWZBOoxTjbOYZAaFiBImzMqiv149HPH5FBJFo0nSVOPqUy78S0YvwZDZD",
"previous": "https://graph.facebook.com/v2.0/425073257683630/posts?since=1450843741&access_token=&limit=25&__paging_token=enc_AdCYobFJpcNavx6STzfPFyFe6eQQxRhkObwl2EdulwL7mjbnIETve7sJZCPMwVm7lu7yZA5FoY5Q4sprlQezF4AlGfZCWALClAZDZD&__previous=1"
},
"data": [
{
"picture": "https://fbcdn-photos-e-a.akamaihd.net/hphotos-ak-xfa1/v/t1.0-0/p130x130/1285_5066979392443_n.png?oh=b37a42ee58654f08af5abbd4f52b1ace&oe=570898E7&__gda__=1461440649_aa94b9ec60f22004675c4a527e8893f",
"is_hidden": false,
"likes": {
"paging": {
"cursors": {
"after": "MTU3NzQxODMzNTg0NDcwNQ==",
"before": "MTU5Mzc1MjA3NDE4ODgwMA=="
}
},
"data": [
{
"id": "1593752074188800",
"name": "Maduri Priyadarshani"
},
{
"id": "427605680763414",
"name": "Darshi Mashika"
},
{
"id": "599793563453832",
"name": "Shakeer Nimeshani Shashikala"
},
{
"id": "1577418335844705",
"name": "Däzlling Jalali Muishu"
}
]
},
"from": {
"category": "Retail and Consumer Merchandise",
"name": "Footcandy",
"category_list": [
{
"id": "2239",
"name": "Retail and Consumer Merchandise"
}
],
"id": "425073257683630"
},
"name": "Timeline Photos",
"privacy": {
"allow": "",
"deny": "",
"friends": "",
"description": "",
"value": ""
},
"is_expired": false,
"comments": {
"paging": {
"cursors": {
"after": "WTI5dGJXVnVkRjlqZFhKemIzSUVXdNVFExTURRd09qRTBOVEE0TkRRNE5EVT0=",
"before": "WTI5dGJXVnVkRjlqZFhKemIzNE16Y3dNVFExTVRFNE9qRTBOVEE0TkRRME5UVT0="
}
},
"data": [
{
"from": {
"name": "NiFû Shafrà",
"id": "1025030640553"
},
"like_count": 0,
"can_remove": false,
"created_time": "2015-12-23T04:20:55+0000",
"message": "wow lovely one",
"id": "50018692683829_500458145118",
"user_likes": false
},
{
"from": {
"name": "Shamnaz Lukmanjee",
"id": "160625809961884"
},
"like_count": 0,
"can_remove": false,
"created_time": "2015-12-23T04:27:25+0000",
"message": "Nice",
"id": "500186926838929_500450145040",
"user_likes": false
}
]
},
"actions": [
{
"link": "https://www.facebook.com/425073257683630/posts/5001866838929",
"name": "Comment"
},
{
"link": "https://www.facebook.com/42507683630/posts/500186926838929",
"name": "Like"
}
],
"updated_time": "2015-12-23T04:27:25+0000",
"link": "https://www.facebook.com/DSIFootcandy/photos/a.438926536298302.1073741827.4250732576630/50086926838929/?type=3",
"object_id": "50018692838929",
"shares": {
"count": 3
},
"created_time": "2015-12-23T04:09:01+0000",
"message": "Reach new heights in the cute and extremely comfortable \"Silviar\" www.focandy.lk",
"type": "photo",
"id": "425077683630_50018926838929",
"status_type": "added_photos",
"icon": "https://www.facebook.com/images/icons/photo1.gif"
}
]
}
Now I need to get this data into a dataframe as follows(no need to get all).
item | Like_id |Like_username | comments_userid |comments_username|comment(msg)|
-----+---------+--------------+-----------------+-----------------+------------+
Bag | 45546 | noel | 641 | James | nice work |
-----+---------+--------------+-----------------+-----------------+------------+
Any Help will be Highly Appreciated.

Not exactly like your intended format, but here is the making of a solution :
import pandas
DictionaryObject_as_List = str(mydict).replace("{","").replace("}","").replace("[","").replace("]","").split(",")
newlist = []
for row in DictionaryObject_as_List :
row = row.replace('https://',' ').split(":")
exec('newlist.append ( ' + "[" + " , ".join(row)+"]" + ')')
DataFrame_Object = pandas.DataFrame(newlist)
print DataFrame_Object

Handling multiple JSON data sets for POST request in Python

I have 3 JSON methods which are two query methods and an update method. I would like to parse through this information with and execute a POST request and pass this data into a database using arcpy for GIS use for all three methods. I have a script which works and gets a response, however the problem is locating the keys and values for each object so that I can successfully send my values to to a database.
Additionally, I handle this task with three different methods, all of which I need data from.
For instance,
query 1 would allow me to parse and find the address, lat/lng, etc.
query 2 would allow me to parse and find customer info, type of request, etc.
query 3 would allow me to update a request.
My first question is how do I successfully extract only the data that I want from each output; I have tested Key/Values in POSTman to no avail, the server is expecting an entire JSON file.
My second question is how do I handle 3 different requests; I am assuming 3 different post methods and selecting the data that I want.
Example of JSON one query passed to server
{
"RequestSpecificDetail": {
"ParentSRNumberForLink": ""
},
"MetaData": {
"appVersion": "1.34",
"deviceModel": "x86_64",
"dateAndTime": "01/15/2015 12:46:36",
"deviceToken": "A2C1DD9D-D17D-4031-BA3E-977C250BFD58",
"osVersion": "8.1"
},
"SRData": {
"LoginUser": "User89",
"NewContactEmail": "abc#gmail.com",
"UpdatedDate": "02/05/2015"
}
}
Example of Query 1 Output
{
"status": {
"code": 311,
"message": "Service Request Successfully Queried.",
"cause": ""
},
"Response": {
"NumOutputObjects": "2",
"ListOfServiceRequest": {
"ServiceRequest": [
{
"SRAddress": "1200 W TEMPLE ST, 90026",
"SRNumber": "1-5099871",
"SRType": "Feedback",
"CreatedDate": "02/05/2015 22:55:58",
"UpdatedDate": "02/05/2015 22:55:58",
"Status": "Open",
"imageURL": ""
},
{
"SRAddress": "1200 W TEMPLE ST, 90026",
"SRNumber": "1-5133051",
"SRType": "Feedback",
"CreatedDate": "02/05/2015 23:03:54",
"UpdatedDate": "02/05/2015 23:03:54",
"Status": "Open",
"imageURL": "https://SERVER_END_POINT/portal/docview?id=fe083ae14b52b1af0945b4d756c296a5"
}
]
},
"LastUpdateDate": "02/05/2015"
}
}
Example of Query 2 passed to server
{
"RequestSpecificDetail": {
"ParentSRNumberForLink": ""
},
"MetaData": {
"appVersion": "1.34",
"deviceModel": "x86_64",
"dateAndTime": "01/15/2015 12:46:36",
"deviceToken": "A2C1DD9D-D17D-4031-BA3E-977C250BFD58",
"osVersion": "8.1"
},
"SRData": {
"SRNumber": "1-1080871"
}
}
Query two output
{
"status": {
"code": 311,
"message": "Service Request Successfully Queried.",
"cause": ""
},
"Response": {
"NumOutputObjects": "1",
"ListOfServiceRequest": {
"ServiceRequest": [
{
"AddressVerified": "Y",
"SRNumber": "1-1080871",
"SRType": "Homeless Encampment",
"CreatedDate": "12/31/2014 13:49:23",
"UpdatedDate": "12/31/2014 13:49:23",
"IntegrationId": "1420033765921",
"Status": "Open",
"CreatedByUserLogin": "User89",
"UpdatedByUserLogin": "User89",
"Anonymous": "N",
"Zipcode": "90026",
"Latitude": "34.064937",
"Longitude": "-118.252968",
"CustomerAccessNumber": "",
"LADWPAccountNo": "",
"NewContactFirstName": "",
"NewContactLastName": "",
"NewContactPhone": "",
"NewContactEmail": "",
"ParentSRNumber": "",
"Priority": "Normal",
"Language": "English",
"ReasonCode": "",
"ServiceDate": "12/31/2014 00:00:00",
"Source": "311",
"Email": "user#email.com",
"FirstName": "User",
"HomePhone": "3123123123",
"LastName": "Pp",
"LoginUser": "",
"ResolutionCode": "",
"SRUnitNumber": "",
"MobilOS": "iOS",
"SRAddress": "1200 W TEMPLE ST, 90026",
"SRAddressName": "",
"SRAreaPlanningCommission": "Central APC",
"SRCommunityPoliceStation": "",
"SRCouncilDistrictMember": "Gilbert Cedillo",
"SRCouncilDistrictNo": "1",
"SRDirection": "W",
"SRNeighborhoodCouncilId": "44",
"SRNeighborhoodCouncilName": "GREATER ECHO PARK ELYSIAN NC",
"SRStreetName": "TEMPLE",
"SRSuffix": "ST",
"SRTBColumn": "E",
"SRTBMapGridPage": "634",
"SRTBRow": "2",
"SRXCoordinate": "6485064",
"SRYCoordinate": "1846114",
"AssignTo": "North Central - 104 - IED",
"Assignee": "Siebel Administrator",
"Owner": "BSS",
"ParentSRStatus": "",
"ParentSRType": "",
"ParentSRLinkDate": "",
"ParentSRLinkUser": "",
"SRAreaPlanningCommissionId": "4",
"SRCommunityPoliceStationAPREC": "RAMPART",
"SRCommunityPoliceStationPREC": "2",
"SRCrossStreet": "",
"ActionTaken": "",
"SRCity": "",
"RescheduleCounter": "",
"SRHouseNumber": "",
"ListOfDataBarricadeRemoval": {},
"ListOfDataBulkyItem": {},
"ListOfDataDeadAnimalRemoval": {},
"ListOfDataGraffitiRemoval": {},
"ListOfDataInformationOnly": {},
"ListOfDataMultipleStreetlightIssue": {},
"ListOfDataSingleStreetlightIssue": {},
"ListOfDataSrPhotoId": {
"DataSrPhotoId": []
},
"ListOfDataBusPadLanding": {},
"ListOfDataCurbRepair": {},
"ListOfDataFlooding": {},
"ListOfDataGeneralStreetInspection": {},
"ListOfDataGuardWarningRailMaintenance": {},
"ListOfDataGutterRepair": {},
"ListOfDataLandMudSlide": {},
"ListOfDataPothole": {},
"ListOfDataResurfacing": {},
"ListOfDataSidewalkRepair": {},
"ListOfDataStreetSweeping": {},
"ListOfDataBeesOrBeehive": {},
"ListOfDataMedianIslandMaintenance": {},
"ListOfDataOvergrownVegetationPlants": {},
"ListOfDataPalmFrondsDown": {},
"ListOfDataStreetTreeInspection": {},
"ListOfDataStreetTreeViolations": {},
"ListOfDataTreeEmergency": {},
"ListOfDataTreeObstruction": {},
"ListOfDataTreePermits": {},
"ListOfDataBrushItemsPickup": {},
"ListOfDataContainers": {},
"ListOfDataElectronicWaste": {},
"ListOfDataIllegalDumpingPickup": {},
"ListOfDataManualPickup": {},
"ListOfDataMetalHouseholdAppliancesPickup": {},
"ListOfDataMoveInMoveOut": {},
"ListOfDataHomelessEncampment": {
"DataHomelessEncampment": [
{
"ApprovedBy": "",
"AssignedTo": "",
"CompletedBy": "",
"Contact": "",
"ContactDate": "",
"Crew": "",
"DateCompleted": "12/31/2014 00:00:00",
"InspectedBy": "",
"InspectionDate": "",
"Location": "Alley",
"Type": "Homeless Encampment",
"LastUpdatedBy": "",
"OptionalTrackingCode": "",
"Name": "a5b5b2b9-d2e7-400a-bf75-1138ff013caa"
}
]
},
"ListOfDataIllegalAutoRepair": {},
"ListOfDataIllegalConstruction": {},
"ListOfDataIllegalConstructionFence": {},
"ListOfDataIllegalDischargeOfWater": {},
"ListOfDataIllegalDumpingInProgress": {},
"ListOfDataIllegalExcavation": {},
"ListOfDataIllegalSignRemoval": {},
"ListOfDataIllegalVending": {},
"ListOfDataLeafBlowerViolation": {},
"ListOfDataNewsRackViolation": {},
"ListOfDataObstructions": {},
"ListOfDataTablesAndChairsObstructing": {},
"ListOfDataGisLayer": {
"DataGisLayer": [
{
"A_Call_No": "",
"Area": "",
"Day": "",
"DirectionSuffix": "",
"DistrictAbbr": "",
"DistrictName": "Central",
"DistrictNumber": "104",
"DistrictOffice": "North Central",
"Fraction": "",
"R_Call_No": "",
"SectionId": "5279800",
"ShortDay": "",
"StreetFrom": "BOYLSTON ST",
"StreetTo": "FIRMIN ST",
"StreetLightId": "",
"StreetLightStatus": "",
"Type": "GIS",
"Y_Call_No": "",
"Name": "41572025-3803-49c4-8561-6e7ef41775df",
"CommunityPlanningArea": "Westlake",
"LastUpdatedBy": "",
"BOSRadioHolderName": ""
}
]
},
"ListOfDataServiceRequestNotes": {
"DataServiceRequestNotes": [
{
"CreatedDate": "12/31/2014 13:49:23",
"Comment": "",
"CreatedByUser": "User89",
"IsSrNoAvailable": "N",
"CommentType": "External",
"Notification": "N",
"FeedbackSRType": "",
"IntegrationId": "1420033765921",
"Date1": "",
"Date2": "",
"Date3": "",
"Text1": "",
"ListOfDataSrNotesAuditTrail": {}
}
]
},
"ListOfDataSubscribeDuplicateSr": {
"DataSubscribeDuplicateSr": [
{
"Activeflag": "Y",
"EmailId": "pratik.desai#yoopmail.com",
"Name": "010420150405",
"Type": "Subscription",
"LastUpdatedBy": ""
}
]
},
"ListOfChildServiceRequest": {},
"ListOfDataBillingCsscAdjustment": {},
"ListOfDataBillingEccAdjustment": {},
"ListOfDataBillingRsscAdjustment": {},
"ListOfDataBillingRsscExemption": {},
"ListOfDataSanitationBillingBif": {},
"ListOfDataSanitationBillingCssc": {},
"ListOfDataSanitationBillingEcc": {},
"ListOfDataSanitationBillingInquiry": {},
"ListOfDataSanitationBillingLifeline": {},
"ListOfDataSanitationBillingRssc": {},
"ListOfDataSanitationBillingSrf": {},
"ListOfDataDocumentLog": {},
"ListOfAuditTrailItem2": {},
"ListOfDataGenericBc": {
"DataGenericBc": [
{
"ATTRIB_08": "",
"NAME": "41572025-3803-49c4-8561-6e7ef41775df",
"PAR_ROW_ID": "1-N607",
"ROW_ID": "1-N60A",
"TYPE": "GIS",
"ListOfDataGenericbcAuditTrail": {}
},
{
"ATTRIB_08": "",
"NAME": "a5b5b2b9-d2e7-400a-bf75-1138ff013caa",
"PAR_ROW_ID": "1-N607",
"ROW_ID": "1-N609",
"TYPE": "Homeless Encampment",
"ListOfDataGenericbcAuditTrail": {}
},
{
"ATTRIB_08": "",
"NAME": "010420150405",
"PAR_ROW_ID": "1-N607",
"ROW_ID": "1-RN2D",
"TYPE": "Subscription",
"ListOfDataGenericbcAuditTrail": {}
}
]
},
"ListOfDataServiceNotComplete": {},
"ListOfDataOther": {},
"ListOfDataWeedAbatementForPrivateParcels": {}
}
]
}
}
}
Query 3 input
{
"MetaData": {},
"RequestSpecificDetail": {
"ParentSRNumberForLink": ""
},
"SRData": {
"SRNumber":"1-5968841",
"Anonymous": "N",
"Assignee": "",
"CreatedByUserLogin": "KAHUNA30DEC",
"CustomerAccessNumber": "",
"LADWPAccountNo": "",
"Language": "English",
"ListOfDataGisLayer": {},
"ListOfDataServiceRequestNotes": {
"DataServiceRequestNotes": [
{
"Comment": "description 1245",
"CommentType": "Feedback",
"CreatedByUser": "KAHUNA30DEC",
"FeedbackSRType": "Illegal Dumping in Progress",
"IsSrNoAvailable": "N"
},
{
"Comment": "comments 123568",
"CommentType": "External",
"CreatedByUser": "",
"IsSrNoAvailable": "N"
}
]
},
"LoginUser": "KAHUNA30DEC",
"MobilOS": "Android",
"NewContactEmail": "",
"NewContactFirstName": "",
"NewContactLastName": "",
"NewContactPhone": "",
"Owner": "Other",
"ParentSRNumber": "",
"Priority": "Normal",
"SRCommunityPoliceStation": "RAMPART",
"UpdatedByUserLogin": "KAHUNA30DEC",
"Status": "Open",
"SRType": "Feedback",
"ServiceDate": "02/11/2015",
"Source": "Mobile App"
}
}
Query 3 output
{
"status": {
"code": 311,
"message": "Service Request Successfully Submited",
"cause": ""
},
"Response": {
"PrimaryRowId": "1-3JXL5",
"ListOfServiceRequest": {
"ServiceRequest": [
{
"SRNumber": "1-5968841"
}
]
}
}
}
Python Script responds with query two output
import json
import jsonpickle
import arcpy
import json
import numpy
import requests
f = open('C:\Users\Administrator\Desktop\myData.json', 'r')
data = jsonpickle.encode( jsonpickle.decode(f.read()) )
url = "https://myDatatest.lacity.org/myDatarouter/srbe/1/QuerySR"
headers = {'Content-type': 'text/plain', 'Accept': '/'}
r = requests.post(url, data=data, headers=headers)
sr = arcpy.SpatialReference(4326)
decoded = json.loads(r.text)
# pretty printing of json-formatted string
print json.dumps(decoded, sort_keys=True, indent=4)
f.close()

decoded is a dictionary containing the data you're looking for.
try:
print decoded
print decoded.keys()
print decoded.items()