I am trying to update the keys of a JSON object which looks like this:
results =
{
'Game':12345,
'stats':[
{
'detail':[
{
'goals':4,
'refs':{
'number':0
I am currently manually updating each key as follow
##update Game to newValue
results['newValue'] = results['Game']
del results['Game']
## update nested key "goals" to "goals_against"
results['stats'][0]['detail'][0]['goals_against'] = results['stats'][0]['detail'][0]['goals']
del results['stats'][0]['detail'][0]['goals']
there has to be a better way to do as I am finding myself having to update multiple keys on results. For example, I also want to update the "number" key to "assis_ref".
I know how to update a key if is the json file is "simple": ie if i could do this:
result['stats']['details']['refs']
however, 'stats' and 'details' require [0] next to it which i assume is the index of the element i am trying to go next.
I wrote a recursive function to handle transforming keys in json objects. Especially useful for deeply nested json!
def walk_json(obj, key_transform):
"""
Recurse over a json object modifying the keys according to the `key_transform` function.
Returns a new json object with the modified keys.
"""
assert isinstance(obj, dict), "walk_json expects obj to be of type dict"
def _walk_json(obj, new):
if isinstance(obj, dict):
if isinstance(new, dict):
for key, value in obj.items():
new_key = key_transform(key)
if isinstance(value, dict):
new[new_key] = {}
_walk_json(value, new=new[new_key])
elif isinstance(value, list):
new[new_key] = []
for item in value:
_walk_json(item, new=new[new_key])
else: # take value as is
new[new_key] = value
elif isinstance(new, list):
new.append(_walk_json(obj, new={}))
else: # take object as is
new.append(obj)
return new
return _walk_json(obj, new={})
Here's how it's used for an overly simple json object:
def my_key_transform(x):
return x.upper()
my_obj = {"a": 1, "b": 2, "c": 3}
result = walk_json(my_obj, key_transform=my_key_transform)
result
{"A": 1, "B": 2, "C": 3}
It can be painful navigating and modify deeply nested objects derived from JSON objects. In Functions that help to understand json(dict) structure I posted code that allows you to navigate such objects. Please read the explanation in that answer. In this answer, I'll show how you can use that code to modify the dictionary keys in such objects.
Briefly, find_key is a recursive generator that will find all the keys with a given name. You can use the next function to get the first (or only) matching name. Or call find_key in a for loop if you need to work with multiple keys that have the same name.
Each value yielded by find_key is a list of the dict keys and list indices need to reach the desired key.
from json import dumps
def find_key(obj, key):
if isinstance(obj, dict):
yield from iter_dict(obj, key, [])
elif isinstance(obj, list):
yield from iter_list(obj, key, [])
def iter_dict(d, key, indices):
for k, v in d.items():
if k == key:
yield indices + [k], v
if isinstance(v, dict):
yield from iter_dict(v, key, indices + [k])
elif isinstance(v, list):
yield from iter_list(v, key, indices + [k])
def iter_list(seq, key, indices):
for k, v in enumerate(seq):
if isinstance(v, dict):
yield from iter_dict(v, key, indices + [k])
elif isinstance(v, list):
yield from iter_list(v, key, indices + [k])
results = {
"Game": 12345,
"stats": [
{
"detail": [
{
"goals": 4,
"refs": {
"number": 0
}
}
]
}
]
}
# Change oldkey to newkey
oldkey, newkey = 'goals', 'goals_against'
# Find the first occurrence of the oldkey
seq, val = next(find_key(results, oldkey))
print('seq:', seq, 'val:', val)
# Get the object that contains the oldkey
obj = results
for k in seq[:-1]:
obj = obj[k]
# Change the key
obj[newkey] = obj.pop(oldkey)
print(dumps(results, indent=4))
output
seq: ['stats', 0, 'detail', 0, 'goals'] val: 4
{
"Game": 12345,
"stats": [
{
"detail": [
{
"refs": {
"number": 0
},
"goals_against": 4
}
]
}
]
}
Related
i've got a dot delimited string which I need to convert to Json. This is an example with different types of strings:
my.dictionary.value -> value
my.dictionary.list[0].value -> value
my.dictionary.list[1].value.list[0].value -> value
I have no problems converting the first type of string using a recursive approach:
def put(d, keys, item):
if "." in keys:
key, rest = keys.split(".", 1)
if key not in d:
d[key] = {}
put(d[key], rest, item)
else:
d[keys] = item
But i'm struggling to find a solution for the lists. Is there a library that provides out of the box string to json conversion? Thank you for your time.
AFAIK, there isn't any modules that would do this
Here is a sample code to converted a series of dotted strings into json format. You just have create a new list when you see the pattern [n] in the string that would be used as a key.
import re
import json
def parse_dotted_strlines(strlines):
res= {}
for line in strlines.splitlines():
parse_dotted_str(line, res)
return res
def parse_dotted_str(s, res):
if '.' in s:
key, rest = s.split('.', 1)
# Check if key represents a list
match = re.search(r'(.*)\[(\d)\]$', key)
if match:
# List
key, index = match.groups()
val = res.get(key, {}) or []
assert type(val) == list, f'Cannot set key {key} as of type list as i
t was earlier marked as {type(val)}'
while len(val) <= int(index):
val.append({})
val[index] = parse_dotted_str(rest, {})
res[key] = val
else:
# Dict
res[key] = parse_dotted_str(rest, res.get(key, {}))
elif '->' in s:
key, val = s.split('->')
res[key.strip()] = val.strip()
return res
Sample input and output
lines = """
my.dictionary.value -> value
my.dictionary.list[0].value -> value
my.dictionary.list[1].value.list[0].value -> value
"""
res = parse_dotted_strlines(lines)
print (json.dumps(res, indent=4))
{
"my": {
"dictionary": {
"value": "value",
"list": [
{
"value": "value"
},
{
"value": {
"list": [
{
"value": "value"
}
]
}
}
]
}
}
}
the json package is what you need
import json
mydict = """{
"str1": "str",
"list1": ["list1_str1", "list1_str2"],
"list2": ["list2_str1", "list2_str2", ["list2_str11", "list_str12"]]
}"""
json.loads(mydict)
>> {'str1': 'str',
'list1': ['list1_str1', 'list1_str2'],
'list2': ['list2_str1', 'list2_str2', ['list2_str11', 'list_str12']]}
I have one json file and i need to list all "selftext" elements of all data.
Any example of it ?
data example
{ "data": [
{
"selftext": "hello there",
"textex": true,
},
If you want to be able to find a key from an arbitrary json at an arbitrary level, you should use recursion:
def findkey(data, key, resul = None):
if resul is None: resul=[] # initialize an empty list for the results
if isinstance(data, list): # walk down into lists
for d in data:
findkey(d, key, resul)
elif isinstance(data, dict): # dict processing
for k,v in data.items():
if (k == key) and isinstance(v, str): # the expected key and a string value?
resul.append(v)
elif isinstance(v, list) or isinstance(v, dict):
findkey(v, key, resul) # recurse if value is a list or a dict
return resul
Example:
>>> data = { "data": [
{
"selftext": "hello there",
"textex": True,
},
]}
>>> findkey(data, 'selftext')
['hello there']
I have a list of data like this
data = [
{
"name": "Box 0",
"type": "Box",
"vals": {
"corner1": "0,0,0",
"corner2": "0,0,5",
"rotate": "0",
"scale": "1",
"translate": "0,0,0"
}
},
{
"name": "Ovus 1",
"type": "Ovus",
"vals": {
"radiusb": "1",
"radiust": "0.5",
"rotate": "0",
"scale": "1",
"translate": "0,0,0"
}
}
]
I would like to create a function of form
fetch_attributes(name, key)
where I can give the key attribute values type,vals or scale.
My problem is that the scale attribute is within the vals dictionary. Can I make the function such that it gives me type when I give key=type and when I ask for key=scale it automatically goes inside the vals dictionary.
My Approach
I am using this code as example for name = 'Box 0'
obj = [i for i in data if i['name'] == name][0]
key_1 = "['vals']" # If I want complete dict of vals
key_2 = "['vals']['scale']" #If I want only one val out of vals
key_3 = "['type']" # If I want type
result_1 = eval('obj' + key_1)
result_2 = eval('obj' + key_2)
result_3 = eval('obj' + key_3)
I would like to know if there is a better or more pythonic way of doing this task exists
Edit
I would like a approach which will also help or guide to create an edit_attributes function as
edit_attributes(name, key, value)
I think this (now) does what you want:
def fetch_attributes(data, name, key):
for dct in data:
if dct['name'] == name:
obj = dct
break
else:
raise KeyError(f'No object with name {name!r} in data')
if isinstance(obj, dict):
if key in obj:
return obj[key]
for value in obj.values():
if isinstance(value, dict): # Nested dict?
if key in value:
return value[key]
raise KeyError(f'{key!r} not in data')
print(fetch_attributes(data, 'Box 0', 'vals')) # -> {'corner1': '0,0,0', 'corner2': '0,0,5', 'rotate': '0', 'scale': '1', 'translate': '0,0,0'}
print(fetch_attributes(data, 'Box 0', 'scale')) # -> 1
print(fetch_attributes(data, 'Box 0', 'type')) # -> Box
This should work for any levels of dictionary nesting.
def find(obj, key):
if key not in obj:
for k in obj.keys():
if type(obj[k]) == type({}):
return find(obj[k], key)
else:
return obj[key]
return None
def fetch_attributes(name, key):
obj = [i for i in data if i['name'] == name][0]
return find(obj, key)
However, if obj = [i for i in data if i['name'] == name][0] returns more than one value, then you will need a for loop there as well.
You can leverage the second parameter of the dictionary's get() function to cascade into the sub-dictionary when the property name is not found:
def getProp(d,key): return d.get(key,d["vals"].get(key,None))
ouput:
for d in data:
print([ getProp(d,"name"),getProp(d,"type"),getProp(d,"scale") ])
# ['Box 0', 'Box', '1']
# ['Ovus 1', 'Ovus', '1']
The assignment would be a little more convoluted but is also possible with setdefault():
def setProp(d,key,value):
[d,d.setdefault("vals",dict())][key=="vals"][key] = value
I have a large nested dictionary with an unknown depth and i would like to know how i can find the keys which led to the value. For example...
{'furniture':{'chair':{'sofa':{'cushion':{}}}}}
Ideally what i am looking for is a function to determine the path to the value that i have entered. I have tried researching online and this is what i tried...
def route(d,key):
if key in d: return d[key]
for k,v in d.items():
if isinstance(v,dict):
item = route(v, key)
if item is not None:
return item
This returns the items inside the key. I am looking to be able to extract the path which leads to that item. For example, route(dictionary,'sofa') then i would be able to get an expected output as such or something similar...
{'sofa':{'chair':'furniture'}}
What are some of the ways that i can achieve this ? Thanks for your help
You can do this recursively and return a list of keys that lead you to your target key:
def route(d, key):
if key in d: return [key]
for k, v in d.items():
if type(v) == dict:
found = route(v, key)
if found: return [k] + found
return []
If we run this on the following dictionary:
data = {
'furniture': {
'chair': {
'sofa': {
'cushion': {}
}
}
},
'electronics': {
'tv': {
'samsung43': 800,
'tcl54': 200
}
}
}
print(route(data, 'cushion'))
print(route(data, 'tcl54'))
print(route(data, 'hello'))
we get the following output:
['furniture', 'chair', 'sofa', 'cushion']
['electronics', 'tv', 'tcl54']
[]
The structure of the json file has to be specified in the python program. the program should parse any input json file only if it matches to that structure. I'm new to python please help me!
sample json file may be like this
"person":{
"sex":{
"name": "XXX",
"address":[
{
"street": "abc",
"area" : "xyz",
},
],
}
the json files having this structure only has to be parsed, if there is one extra field other than these shouldn't be parsed.
Please help me out Thanks in advance
With these two functions you can check all keys in your dictionaries
if the keys are equal it will return True
def get_keys(obj, keys=[]):
''' creates a list with all keys from a nested, json valid dictionary '''
if isinstance(obj, dict):
for k, v in obj.items():
if isinstance(v, dict) or isinstance(v, list) and all(isinstance(item,dict) for item in v):
keys.append(k)
return get_keys(v, keys)
else:
keys.append(k)
#calls get_keys with the list of keys as object this will trigger the else statement and return the object
else: return get_keys(keys)
elif isinstance(obj, list) and all(isinstance(item, dict) for item in obj):
#checks if obj is a list and if all obj in a list are dictionaries
for dct in obj:
for k, v in dct.items():
if isinstance(v, dict) or isinstance(v, list) and isinstance(v[0], dict):
keys.append(k)
return get_keys(v, keys)
else:
keys.append(k)
#same as above
else: return get_keys(keys)
else:
return obj
def has_same_structure(dct_1,dct_2):
''' checks if two (nested) dictionaries have the same keys '''
if get_keys(dct_1) == get_keys(dct_2, keys=[]):
return True
else:
return False
I hope this helps