Develop Reference

parsing JSON with missing fields

I have json array with very dynamic field and some of the array doesn't have all the fields.
Example :
[
{
"Name": "AFG LIMITED",
"Vendor ID": "008343",
"EGID": "67888",
"FID": "83748374"
},
{
"Name": "ABC LIMITED",
"Vendor ID": "008333",
"EGID": "67888",
"AID": "0000292"
"FID": "98979"
},
]
I need to extract particular key with header & pipe delimiter like :Name|Vendor ID|EGID|AID(only present in second array).if any key is not present then it should have null value
I was try to parse this with below code but it's breaking in the second line itself as AID is missing.
import json
with open("sample.json", "r") as rf:
decoded_data = json.load(rf)
# Check is the json object was loaded correctly
try:
for i in decoded_data:
print i["Name"],"|",i["Vendor ID"]"|",i["EGID"],"|",i["AId"]
except KeyError:
print(null)
output from above code:
AFG LIMITED|008343|67888|null

Format a text file in python when it finds a specific work?

I am looking to format a text file from an api request output. So far my code looks like such:
import requests
url = 'http://URLhere.com'
headers = {'tokenname': 'tokenhash'}
response = requests.get(url, headers=headers,)
with open('newfile.txt', 'w') as outf:
outf.write(response.text)
and this creates a text file but the output is on one line.
What I am trying to do is:
Have it start a new line every time the code reaches a certain word like "id","status", or "closed_at" but unfortunately I have not been able to figure this out.
Also I am trying to get a count of how many "id" there are in the file but I think due to the formatting, the script does not like it.
The output is as follows:
{
[
{
"id": 12345,
"status": "open or close",
"closed_at": null,
"created_at": "yyyy-mm-ddTHH:MM:SSZ",
"due_date": "yyyy-mm-dd",
"notes": null,
"port": [pnumber
],
"priority": 1,
"identifiers": [
"12345"
],
"last_seen_time": "yyyy-mm-ddThh:mm:ss.sssZ",
"scanner_score": 1.0,
"fix_id": 12345,
"scanner_vulnerabilities": [
{
"port": null,
"external_unique_id": "12345",
"open": false
}
],
"asset_id": 12345
This continues on one line with the same names but for different assets.

This code :
with open ('text.txt') as text_file :
data = text_file.read ()
print ('\n'.join (data.split (',')))
Gives this output :
"{[{"id":12345
"status":"open or close"
"closed_at":null
"created_at":"yyyy-mm-ddTHH:MM:SSZ"
"due_date":"yyyy-mm-dd"
"notes":null
"port":[pnumber]
"priority":1
"identifiers":["12345"]
"last_seen_time":"yyyy-mm-ddThh:mm:ss.msmsmsZ"
"scanner_score":1.0
"fix_id":12345
"scanner_vulnerabilities":[{"port":null
"external_unique_id":"12345"
"open":false}]
"asset_id":12345"
And then to write it to a new file :
output = data.split (',')
with open ('new.txt', 'w') as write_file :
for line in output :
write_file.write (line + '\n')

Changing multiple keys on a JSON FILe using Python

I am trying to update a new key into my JSON file if the conditions are met. The following is my python code attempting to make multiple updates in a JSON file.
#!/usr/bin/env python
# Usage: update json file
import json
import os
json_dir="/opt/rdm/adggeth/ADGG-ETH-02/20181008/"
json_dir_processed="/opt/rdm/adggeth/ADGG-ETH-02/20181008updated/"
for json_file in os.listdir(json_dir):
if json_file.endswith(".json"):
processed_json = "%s%s" % (json_dir_processed, json_file)
json_file = json_dir + json_file
print "Processing %s -> %s" % (json_file, processed_json)
with open(json_file, 'r') as f:
json_data = json.load(f)
# replacement mapping
update_map = {"grp_farmerreg/farmerdetails/farmermobile":"grp_farmerdts/hh_id",
"grp_farmerdts/hh_region":"grp_farmerdts/region",
"grp_farmerdts/hh_district":"grp_farmerdts/district",
"grp_farmerdts/hh_ward":"grp_farmerdts/ward",
"grp_farmerdts/hh_village":"grp_farmerdts/village"}
diff_keys = update_map.keys() - json_data.keys()
if not diff_keys:
print("No Update to JSON keys")
else:
for k in diff_keys:
json_data[k] = json_data[update_map[k]]
with open(processed_json, 'w') as f:
f.write(json.dumps(json_data, indent=4))
else:
print "%s not a JSON file" % json_file
The JSON file i am trying to make update to is as follows:
{
....
"farmerregistrd": "1",
"grp_farmerdts/region": "5",
"datacollid": "0923678275",
"_status": "submitted_via_web",
"enumtype": "2",
"deviceid": "352948096845916",
"start_time": "2019-04-03T10:57:23.620+03",
"_uuid": "f1069eae-33f8-4850-a549-49fcde27f077",
"grp_farmerdts/village": "2852",
"_submitted_by": null,
"formhub/uuid": "42cb3fc351a74fd89702078160f849ca",
"grp_farmerdts/hh_id": "623",
"grp_farmerdts/ward": "136",
...
"_userform_id": "adggeth_ADGG-ETH-REG02-20181008",
"_id": 711097,
"grp_farmerdts/district": "31"
}
My expected output from running the following python file is as follows
{
....
"farmerregistrd": "1",
"grp_farmerdts/hh_region": "5",
"datacollid": "0923678275",
"_status": "submitted_via_web",
"enumtype": "2",
"deviceid": "352948096845916",
"start_time": "2019-04-03T10:57:23.620+03",
"_uuid": "f1069eae-33f8-4850-a549-49fcde27f077",
"grp_farmerdts/hh_village": "2852",
"_submitted_by": null,
"formhub/uuid": "42cb3fc351a74fd89702078160f849ca",
"grp_farmerdts/hh_id": "623",
"grp_farmerdts/hh_ward": "136",
...
"_userform_id": "adggeth_ADGG-ETH-REG02-20181008",
"_id": 711097,
"grp_farmerdts/hh_district": "31"
}

Using re module and json.loads() with object_hook= parameter (doc). This script will add hh_ prefix to every grp_farmerdts/* key where isn't:
json_str = '''{
"farmerregistrd": "1",
"grp_farmerdts/region": "5",
"datacollid": "0923678275",
"_status": "submitted_via_web",
"enumtype": "2",
"deviceid": "352948096845916",
"start_time": "2019-04-03T10:57:23.620+03",
"_uuid": "f1069eae-33f8-4850-a549-49fcde27f077",
"grp_farmerdts/village": "2852",
"_submitted_by": null,
"formhub/uuid": "42cb3fc351a74fd89702078160f849ca",
"grp_farmerdts/hh_id": "623",
"grp_farmerdts/ward": "136",
"_userform_id": "adggeth_ADGG-ETH-REG02-20181008",
"_id": 711097,
"grp_farmerdts/district": "31"
}'''
import re
import json
def change_keys(d):
return {re.sub(r'grp_farmerdts/((?!hh_)(\w+))', r'grp_farmerdts/hh_\1', k): v for k, v in d.items()}
print(json.dumps(json.loads(json_str, object_hook=change_keys), indent=4))
Prints:
{
"farmerregistrd": "1",
"grp_farmerdts/hh_region": "5",
"datacollid": "0923678275",
"_status": "submitted_via_web",
"enumtype": "2",
"deviceid": "352948096845916",
"start_time": "2019-04-03T10:57:23.620+03",
"_uuid": "f1069eae-33f8-4850-a549-49fcde27f077",
"grp_farmerdts/hh_village": "2852",
"_submitted_by": null,
"formhub/uuid": "42cb3fc351a74fd89702078160f849ca",
"grp_farmerdts/hh_id": "623",
"grp_farmerdts/hh_ward": "136",
"_userform_id": "adggeth_ADGG-ETH-REG02-20181008",
"_id": 711097,
"grp_farmerdts/hh_district": "31"
}

According to your expected output all particular keys need to be checked (not one of them). Change your logic as shown below:
...
json_data = json.load(f)
# replacement mapping
update_map = {"grp_farmerreg/farmerdetails/farmermobile":"grp_farmerdts/hh_id",
"grp_farmerdts/hh_region":"grp_farmerdts/region",
"grp_farmerdts/hh_district":"grp_farmerdts/district",
"grp_farmerdts/hh_ward":"grp_farmerdts/ward", "grp_farmerdts/hh_village":"grp_farmerdts/village"}
diff_keys = update_map.keys() - json_data.keys()
if not diff_keys:
print("No Update to JSON keys")
else:
for k in diff_keys:
if update_map[k] in json_data:
json_data[k] = json_data[update_map[k]]

json2html, python: json data not converted to html

I'm trying to format json data to html using json2html.
The json data look like this:
json_obj = [{"Agent Status": "status1", "Last backup": "", "hostId": 1234567, "hostfqdn": "test1.example.com", "username": "user1"}, {"Agent Status": "status2", "Last backup": "", "hostId": 2345678, "hostfqdn": "test2.example.com", "username": "user2"}]
As already reported in post "json2html not a valid json list python", to make the code works, the json parameter must be a dictionary and not a list, so I'm calling it that way:
json_obj_in_html = json2html.convert(json = { "data" : json_obj })
However it does not format the json data to html only the first level of dictionary { "data" : json_obj }:
print json_obj_in_html
<table border="1"><tr><th>data</th><td>[{"Agent Status": "status1", "Last backup": "", "hostId": 1234567, "hostfqdn": "test1.example.com", "username": "user1"}, {"Agent Status": "status2", "Last backup": "", "hostId": 2345678, "hostfqdn": "test2.example.com", "username": "user2"}]</td></tr></table>
Note that the online convert tool provides the right output: http://json2html.varunmalhotra.xyz/
<table border="1"><tr><th>data</th><td><ul><table border="1"><tr><th>Agent Status</th><td>status1</td></tr><tr><th>Last backup</th><td></td></tr><tr><th>hostId</th><td>1234567</td></tr><tr><th>hostfqdn</th><td>test1.example.com</td></tr><tr><th>username</th><td>user1</td></tr></table><table border="1"><tr><th>Agent Status</th><td>status2</td></tr><tr><th>Last backup</th><td></td></tr><tr><th>hostId</th><td>2345678</td></tr><tr><th>hostfqdn</th><td>test2.example.com</td></tr><tr><th>username</th><td>user2</td></tr></table></ul></td></tr></table>
Any help would be very welcome.

Make sure that json_obj is an array of objects and not a string (str).
I put your code to a complete sample:
from json2html import *
json_obj = [{"Agent Status": "status1", "Last backup": "", "hostId": 1234567, "hostfqdn": "test1.example.com", "username": "user1"}, {"Agent Status": "status2", "Last backup": "", "hostId": 2345678, "hostfqdn": "test2.example.com", "username": "user2"}]
json_obj_in_html = json2html.convert(json = { "data" : json_obj })
print json_obj_in_html
With Python 2.7 and json2html 1.0.1 this leads to this result:
If you receive a result like
<table border="1"><tr><th>data</th><td>[{"Agent Status": "sta...
it is very likely that json_obj is a str and not an array of objects. You can check this by inserting a statement like
print type(json_obj)
before jsonhtml.convert. I assume that type(json_obj) returns a <type 'str'> in your case and that is why the JSON like string appears in your html. To get it right you have to modify your code in that way that type(json_obj) returns <type 'list'>.

My list of dictionaries anomaly_list was already in a json format, so trying to convert it using json.dumps(anomaly_list, sort_keys=True) was turning into a string, which was not what I wanted.
I solved the issue by leaving my list of dictionaries as it is and this code now works:
json_obj_in_html = ''
for j in anomalies_list:
json_obj_in_html += json2html.convert(json = j)
It outputs what I wanted.
#gus42: thanks, your feedback made me understand where the real pb was.

Equivalent of Python "json.dumps()" in R?

I'm a very beginner student of R (still coursing the "R Programming" course on Coursera) and I'm trying to practice R porting some easy code from Python to R.
Currently I'm trying to make API calls for a KairosDB database. In order to make the query, I need to encode the Python object with json.dumps() (from the json native library), but I've searched a lot and I don't get how I can do that with R and it's jsonlite library. I don't even know if I'm creating the JSON object corretly, but that's what I've found in some searches.
My code written in Python 3 (from this repo):
import requests
import json
kairosdb_server = "http://localhost:8080"
# Simple test
query = {
"start_relative": {
"value": "4",
"unit": "years"
},
"metrics": [
{
"name": "test",
"limit": 10000
}
]
}
response = requests.post(kairosdb_server + "/api/v1/datapoints/query", data=json.dumps(query))
print("Status code: %d" % response.status_code)
print("JSON response:")
print(response.json())
My current code written in R 3.2.3:
library(httr)
library(jsonlite)
kairosdb_server <- 'http://localhost:8080'
query <- serializeJSON(toJSON('
"start_relative": {
"value": "4",
"unit": "years"
},
"metrics": [
{
"name": "test",
"limit": 1000
}
]
'))
url <- paste(kairosdb_server, '/api/v1/datapoints/query')
response <- POST(url, body = query, encode = 'json')
print(paste("Query status code: ", response$status_code))
print(paste("JSON response: \n", content(response, type = 'application/json')))
If I run that I got the following error:
print(paste("Query status code: ", response$status_code))
# [1] "Query status code: 400"
print(paste("JSON response: \n", content(response, type = 'application/json')))
# [1] "JSON response: \n list(\"query.metric[] must have a size of at least 1\")"
What I'm doing wrong?

Normally one would pass a named list into body but trying to get R to preserve the array in "metrics" is tricky. Since you kinda already have JSON with the original Python structure, why not just add brackets and pass it in as a character vector? i.e.
query <- '{"start_relative": {
"value": "4",
"unit": "years"
},
"metrics": [
{
"name": "test",
"limit": 10000
}
]}'
(then just use that query in the POST). It's equivalent JSON to what json.dumps() spits out:
# get rid of newlines and spaces just to show they are the same,
# the server won't (shouldn't) care if there are newlines/spaces
cat(gsub(" \\]", "]", gsub("\\[ ", "[", gsub(" \\}", "}", gsub("\\{ ", "{", gsub("\ +", " ", gsub("\\n", "", query)))))))
{"start_relative": {"value": "4", "unit": "years"}, "metrics": [{"name": "test", "limit": 10000}]}
# python
json.dumps(query)
'{"metrics": [{"limit": 10000, "name": "test"}], "start_relative": {"unit": "years", "value": "4"}}'
If you do need an R data structure to work with, you're going to end up manipulating the output of toJSON.