I have following JSON, returned from a REST service, where I want to generate a unique names for each value by combining parent keys. For example. name+phone+address+city+name , name+phone+address+city+population+skilled+male and so on.
{
"name": "name",
"phone": "343444444",
"address": {
"lat": 23.444,
"lng": 34.3322,
"city":{
"name": "city name",
"population": {
"skilled": {
"male": 2,
"female": 4
},
"uneducated": {
"male": 20,
"femail": 4
}
}
}
},
"email": "email",
"education": "phd"
}
I want to combine all key names starting from the parent of the JSON tree.
Here is what I am doing
class TestJson
def walk_through(self, json_object):
for k, v in json_object.items():
self.x_path = self.x_path + k
if type(v) is dict:
self.walk_through(v)
else:
print(self.x_path)
self.x_path = ""
This code is printing keys but only starting from the current parent node. I want to combine all keys up to root of the json.
If you ignore the name and phone keys, since they are not ancestors of city name or skilled male and the order of keys is not guaranteed, you can recursively build a flattened dict.
def walk_through(json_object):
d = {}
for k, v in json_object.items():
if isinstance(v, dict):
v = walk_through(v)
for vk, vv in v.items():
d["%s+%s" % (k, vk)] = vv
else:
d[k] = v
return d
print(json.dumps(walk_through(json_object), indent=2))
This prints:
{
"address+city+population+skilled+male": 2,
"name": "name",
"address+lng": 34.3322,
"address+city+name": "city name",
"address+lat": 23.444,
"address+city+population+uneducated+male": 20,
"phone": "343444444",
"address+city+population+uneducated+femail": 4,
"education": "phd",
"email": "email",
"address+city+population+skilled+female": 4
}
Note: this ignores lists an will not find dicts inside them.
If you want to print all keys of your python dict you can do the following:
def print_keys(d):
for key, value in d.iteritems():
print key,
if isinstance(value, dict):
print_keys(value)
Related
I have created a var that is equal to t.json. The JSON file is a follows:
{
"groups": {
"customerduy": {
"nonprod": {
"name": "customerduynonprod",
"id": "529646781943",
"owner": "cloudops#coerce.com",
"manager_email": ""
},
"prod": {
"name": "phishing_duyaccountprod",
"id": "241683454720",
"owner": "cloudops#coerce.com",
"manager_email": ""
}
},
"customerduyprod": {
"nonprod": {
"name": "phishing_duyaccountnonprod",
"id": "638968214142",
"owner": "cloudops#coerce.com",
"manager_email": ""
}
},
"ciasuppliergenius": {
"prod": {
"name": "ciasuppliergeniusprod",
"id": "220753788760",
"owner": "cia_developers#coerce.com",
"manager_email": "jarks#coerce.com"
}
}
}
}
my goal was to pars this JSON file and get value for "owner" and output it to a new var. Example below:
t.json = group_map
group_id_aws = group(
group.upper(),
"accounts",
template,
owner = group_map['groups']['prod'],
manager_description = "Groups for teams to access their product accounts.",
The error I keep getting is: KeyError: 'prod'
Owner occurs 4 times, so here is how to get all of them.
import json
# read the json
with open("C:\\test\\test.json") as f:
data = json.load(f)
# get all 4 occurances
owner_1 = data['groups']['customerduy']['nonprod']['owner']
owner_2 = data['groups']['customerduy']['prod']['owner']
owner_3 = data['groups']['customerduyprod']['nonprod']['owner']
owner_4 = data['groups']['ciasuppliergenius']['prod']['owner']
# print results
print(owner_1)
print(owner_2)
print(owner_3)
print(owner_4)
the result:
cloudops#coerce.com
cloudops#coerce.com
cloudops#coerce.com
cia_developers#coerce.com
You get a key error since the key 'prod' is not in 'groups'
What you have is
group_map['groups']['customerduy']['prod']
group_map['groups']['ciasuppliergenius']['prod']
So you will have to extract the 'owner' from each element in the tree:
def s(d,t):
for k,v in d.items():
if t == k:
yield v
try:
for i in s(v,t):
yield i
except:
pass
print(','.join(s(j,'owner')))
If your JSON is loaded in variable data, you can use a recursive function
that deals with the two containers types (dict and list) that can occur
in a JSON file, recursively:
def find_all_values_for_key(d, key, result):
if isinstance(d, dict):
if key in d:
result.append(d[key])
return
for k, v in d.items():
find_all_values_for_key(v, key, result)
elif isinstance(d, list):
for elem in d:
find_all_values_for_key(elem, key, result)
owners = []
find_all_values_for_key(data, 'owner', owners)
print(f'{owners=}')
which gives:
owners=['cloudops#coerce.com', 'cloudops#coerce.com', 'cloudops#coerce.com', 'cia_developers#coerce.com']
This way you don't have to bother with the names of intermediate keys, or in general the structure of your JSON file.
You don't have any lists in your example, but it is trivial to recurse through
them to any dict with an owner key that might "lurk" somewhere nested
under a a list element, so it is better to deal with potential future changes
to the JSON.
I have a json file for which I want to remove the $oid and $date and replace the keys like in the example below:
import json
def key_replacer(dictionary):
new_dictionary = {}
for k,v in dictionary.items():
if k in ('$oid', '$date'):
return v
if isinstance(v, dict):
v = key_replacer(v)
new_dictionary[k] = v
return new_dictionary
data = """{
"_id": {
"$oid": "5e7511c45cb29ef48b8cfcff"
},
"description": "some text",
"startDate": {
"$date": "5e7511c45cb29ef48b8cfcff"
},
"completionDate": {
"$date": "2021-01-05T14:59:58.046Z"
},
"videos":[{"$oid":"5ecf6cc19ad2a4dfea993fed"}]
}"""
info = json.loads(data)
refined = key_replacer(info)
new_data = json.dumps(refined)
print(new_data)
Output: {"_id": "5e7511c45cb29ef48b8cfcff", "description": "some text", "startDate": "5e7511c45cb29ef48b8cfcff", "completionDate": "2021-01-05T14:59:58.046Z", "videos": [{"$oid": "5ecf6cc19ad2a4dfea993fed"}]}. It works the way I want until "videos". How could I remove the $ sign for the "videos" part and replace the key like it happens in the other cases? It doesn't get into the contents of the list and I assume this is the cause.
Your original function doesn't account for the case where the dictionary value is a list. By accommodating for that, you will get the desired result
def key_replacer(dictionary):
new_dictionary = {}
for k, v in dictionary.items():
if k in ('$oid', '$date'):
return v
elif isinstance(v, dict):
v = key_replacer(v)
elif isinstance(v, list):
tmp = []
for itm in v:
tmp.append(key_replacer(itm))
v = tmp
new_dictionary[k] = v
return new_dictionary
NB: If the items in the list is not dictionaries the code will break, so be mindful of that
Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 1 year ago.
Improve this question
I want to replace the values inside a json for dictionary's key it could be present as dictionay directly in json or as a dictionary inside another list as below:
{
"appType": "popper",
"createdAt": "1970-01-01T00:00:00.000Z",
"updatedAt": "1970-01-01T00:00:00.000Z",
"people": [{
"name": "Vol1",
"label": "Vol1",
"peopleInfo": [{
"name": "ram",
"age": "2407653459860",
"id": "1b738651-da9f-4c85-88c1-70dbfe1976681"
}],
"itemInfo": {
"id": "ee763970-51e2-57a5-955c-d72fc3e28a3f",
"name": "xyz",
"type": "any",
"managed": False
}
}],
"itemInfo": [{
"managed": False,
"vendorName": "any",
"serialNumber": "AF-10124"
}],
}
Desired Output:
{
"appType": "popper",
"createdAt": "1970-01-01T00:00:00.000Z",
"updatedAt": "1970-01-01T00:00:00.000Z",
"peopleInfo": [{
"name": "Vol1",
"label": "Vol1",
"people": [{
"name": "ram",
"age": "2407653459860",
"id": "1b738651-da9f-4c85-88c1-70dbfe1976681"
}],
"itemInfo": {
"id": "ee763970-51e2-57a5-955c-d72fc3e28a3f",
"name": "xyz",
"type": "any",
"managed": True
}
}],
"itemInfo": [{
"managed": True,
"vendorName": "any",
"serialNumber": "AF-10124"
}],
}
so as in desired output i want to update/replace the managed flag as True from false for itemInfo directly in json and as well as itemInfo in peopleInfo List using python. The iteminfo dictionary can be present in entire json in some different list as well. Thankyou for the help.
I have written below code but not able to make it a general one:
i["details"]["storageSystemsInfo"][0]["managed"] = True
Update 2:
Change only "managed" fields within "itemInfo" dictionary like you asked.
(This is not pretty or clean code)
# Python 3
def change_this_key_to_true(key, var):
if hasattr(var,'items'):
for k, v in var.items():
if k == key:
var[k] = True
if isinstance(v, dict):
change_this_key_to_true(key, v)
elif isinstance(v, list):
for d in v:
change_this_key_to_true(key, d)
else:
if isinstance(var, list):
for element in var:
change_this_key_to_true(key, element)
all_item_info_dics = []
def get_all_values_with_key(key, var):
if hasattr(var, 'items'):
for k, v in var.items():
if k == key:
all_item_info_dics.append(var[k])
if isinstance(v, dict):
get_all_values_with_key(key, v)
elif isinstance(v, list):
for d in v:
get_all_values_with_key(key, d)
get_all_values_with_key("itemInfo", myJson)
print(all_item_info_dics)
print("\n")
change_this_key_to_true("managed", all_item_info_dics)
print(myJson)
Update:
(That's my first answer, took me a while, hope it's good :) )
# Python 3
def change_this_key_to_true(key, var):
if hasattr(var,'items'):
for k, v in var.items():
if k == key:
var[k] = True
if isinstance(v, dict):
change_this_key_to_true(key, v)
elif isinstance(v, list):
for d in v:
change_this_key_to_true(key, d)
change_this_key_to_true("managed", myJson)
print(myJson)
Go through all the fields in a complex dict/list/"weird combination of the two" and looking for a field named "managed" and change it to True.
Based my answer on this:
Find all occurrences of a key in nested dictionaries and lists
Original answer:
x["people"][0]["itemInfo"]["managed"] = True
x["itemInfo"][0]["managed"] = True
I am trying to update transaction ID from the following json:
{
"locationId": "5115",
"transactions": [
{
"transactionId": "1603804404-5650",
"source": "WEB"
} ]
I have done following code for the same, but it does not update the transaction id, but it inserts the transaction id to the end of block:-
try:
session = requests.Session()
with open(
"sales.json",
"r") as read_file:
payload = json.load(read_file)
payload["transactionId"] = random.randint(0, 5)
with open(
"sales.json",
"w") as read_file:
json.dump(payload, read_file)
Output:-
{
"locationId": "5115",
"transactions": [
{
"transactionId": "1603804404-5650",
"source": "WEB"
} ]
}
'transactionId': 1
}
Expected Outut:-
{
"locationId": "5115",
"transactions": [
{
"transactionId": "1",
"source": "WEB"
} ]
This would do it, but only in your specific case:
payload["transactions"][0]["transactionId"] = xxx
There should be error handling for cases like "transactions" key is not int the dict, or there are no records or there are more than one
also, you will need to assign =str(your_random_number) not the int if you wish to have the record of type string as the desired output suggests
If you just want to find the transactionId key and you don't know exactly where it may exist. You can do-
from collections.abc import Mapping
def update_key(key, new_value, jsondict):
new_dict = {}
for k, v in jsondict.items():
if isinstance(v, Mapping):
# Recursive traverse if value is a dict
new_dict[k] = update_key(key, new_value, v)
elif isinstance(v, list):
# Traverse through all values of list
# Recursively traverse if an element is a dict
new_dict[k] = [update_key(key, new_value, innerv) if isinstance(innerv, Mapping) else innerv for innerv in v]
elif k == key:
# This is the key to replace with new value
new_dict[k] = new_value
else:
# Just a regular value, assign to new dict
new_dict[k] = v
return new_dict
Given a dict-
{
"locationId": "5115",
"transactions": [
{
"transactionId": "1603804404-5650",
"source": "WEB"
} ]
}
You can do-
>>> update_key('transactionId', 5, d)
{'locationId': '5115', 'transactions': [{'transactionId': 5, 'source': 'WEB'}]}
Yes because transactionId is inside transactions node. So your code should be like:
payload["transactions"][0].transactionId = random.randint(0, 5)
or
payload["transactions"][0]["transactionId"] = random.randint(0, 5)
I'm looking to flatten a JSON hierarchy of unknown structure to a dictionary, capturing the full key hierarchy in the dictionary result to uniquely identify it.
So far I am able to print the key:value pair for the all the parent/child nodes recursively but I am having trouble:
(1) figuring out how to pass the parent hierarchy keys for a recursive (child) execution and then reset it when it exits the child key.
(2) writing to a single dictionary result - when I define the dictionary within the recursive function, I end up creating multiple dictionaries ... Do I need to wrap this function in a master function to avoid this?
Thanks!
# flatten/enumerate example I'm using
with open('_json\\file.json') as f:
data = json.load(f)
def parse_json_response(content):
if len (content.keys()) > 1 :
for key, value in content.items():
if type(value) is dict:
parse_json_response(value)
else:
print('{}:{}'.format(key,value))
else:
print(value)
if __name__ == '__main__':
parse_json_response(data)
# current result as print
id = 12345
firstName = John
lastName = Smith
DOB = 1980-01-01
phone = 123
line1 = Unit 4
line2 = 3 Main st
# desired result to dictionary {}
id = 12345
fields.firstName = John
fields.lastName = Smith
fields.DOB = 1980-01-01
fields.phone = 123
fields.address.residential.line1 = Unit 4
fields.address.residential.line2 = 3 Main st
You can create the flattened dictionary (rather than just print values), by keeping track of the parent and recursing in the correct spot. That might look something like:
d = {
"ID": "12345",
"fields": {
"firstName": "John",
"lastName": "Smith",
"DOB": "1980-01-01",
"phoneLand": "610292659333",
"address": {
"residential": {
"line1": "Unit 4",
"line2": "3 Main st"
}
}
}
}
def flattenDict(d, parent=None):
ret = {}
for k, v in d.items():
if parent:
k = f'{parent}.{k}'
if isinstance(v, dict):
ret.update(flattenDict(v, k))
else:
ret[k] = v
return ret
flat = flattenDict(d)
flat will be:
{'ID': '12345',
'fields.firstName': 'John',
'fields.lastName': 'Smith',
'fields.DOB': '1980-01-01',
'fields.phoneLand': '610292659333',
'fields.address.residential.line1': 'Unit 4',
'fields.address.residential.line2': '3 Main st'}
You can also arrange the output to be a generator that yields tuples. You can then pass this to dict() for the same result:
def flattenDict(d):
for k, v in d.items():
if isinstance(v, dict):
yield from ((f'{k}.{kk}', v) for kk, v in flattenDict(v))
else:
yield (k, v)
dict(flattenDict(d))
Try this below:
test = {
"ID": "12345",
"fields": {
"firstName": "John",
"lastName": "Smith",
"DOB": "1980-01-01",
"phoneLand": "610292659333",
"address": {
"residential": {
"line1": "Unit 4",
"line2": "3 Main st"
}
}
}
}
def func(d, parent=""):
for key, value in d.items():
if isinstance(value, dict):
func(value, parent=parent+key+".")
else:
print(f"{parent+key} = {value}")
func(test)
Result:
ID = 12345
fields.firstName = John
fields.lastName = Smith
fields.DOB = 1980-01-01
fields.phoneLand = 610292659333
fields.address.residential.line1 = Unit 4
fields.address.residential.line2 = 3 Main st