I want to be able to obtain all the various paths to the keys in a JSON file. I often obtain large JSONs and I'm not exactly sure where a various data element might be. Or I need to query various elements of the data. Visualizing a tree of the JSON can be inconvient.
Basically I want to get a list of all the different paths to make various future tasks easier.
For example:
myjson = {'transportation':'car',
'address': {'driveway':'yes','home_address':{'state':'TX',
'city':'Houston'}},
'work_address':{
'state':'TX',
'city':'Sugarland',
'location':'office-tower',
'salary':30000}}
It would be great if I could run some type of loop to get a list back in this format below or in a format....
myjson['address']['driveway']
myjson.address
myjson.address.driveway
myjson.address.home_address
myjson.address.home_address.city
myjson.address.home_address.state
myjson.transportation
myjson.work_address
myjson.work_address.city
myjson.work_address.location
myjson.work_address.salary
myjson.work_address.state
For example I've started with
mylist = []
for key, value in myjson.items():
mylist.append(key)
if type(value) is dict:
for key2, value2 in myjson[key].items():
mylist.append(key+'.'+key2)
print(mylist)
I guess this kinda works, but I don't know how to make this iterate indefinitely. For example, how would I build this up to being 3-10+ layers deep?
Great snippet !
Here is a version which manage list:
def get_keys(some_dictionary, parent=None):
if isinstance(some_dictionary, str):
return
for key, value in some_dictionary.items():
if '{}.{}'.format(parent, key) not in my_list:
my_list.append('{}.{}'.format(parent, key))
if isinstance(value, dict):
get_keys(value, parent='{}.{}'.format(parent, key))
if isinstance(value, list):
for v in value:
get_keys(v, parent='{}.{}'.format(parent, key))
else:
pass
I think this should do what you're asking:
myjson = {
'transportation': 'car',
'address': {
'driveway': 'yes',
'home_address': {
'state': 'TX',
'city': 'Houston'}
},
'work_address': {
'state': 'TX',
'city': 'Sugarland',
'location': 'office-tower',
'salary': 30000}
}
def get_keys(some_dictionary, parent=None):
for key, value in some_dictionary.items():
if '{}.{}'.format(parent, key) not in my_list:
my_list.append('{}.{}'.format(parent, key))
if isinstance(value, dict):
get_keys(value, parent='{}.{}'.format(parent, key))
else:
pass
my_list = []
get_keys(myjson, parent='myjson')
print(my_list)
Outputs:
['myjson.transportation',
'myjson.work_address',
'myjson.work_address.city',
'myjson.work_address.state',
'myjson.work_address.location',
'myjson.work_address.salary',
'myjson.address',
'myjson.address.driveway',
'myjson.address.home_address',
'myjson.address.home_address.city',
'myjson.address.home_address.state']
The key is to just keep calling get_keys() recursively from within the function!
An implementation handling paths of lists in json also.
import json
def get_json_key_path(jsonStr, enable_index):
json_keys = []
jsonObj = json.loads(jsonStr)
def get_key_path(jsonObj, parent=None):
if not isinstance(json_obj, dict):
return
for key, value in jsonObj.items():
if not isinstance(value, list) and '{}.{}'.format(parent, key) not in json_keys:
json_keys.append('{}.{}'.format(parent, key))
if isinstance(value, dict):
get_key_path(value, parent='{}.{}'.format(parent, key))
elif isinstance(value, list):
i = 0
for obj in value:
if enable_index:
get_key_path(obj, parent='{}.{}.{}'.format(parent, key, i))
else:
get_key_path(obj, parent='{}.{}'.format(parent, key))
i = i + 1
else:
pass
get_key_path(jsonObj, "")
return [ s[1:] for s in json_keys]
I'm trying to write a very simple function to recursively search through a possibly nested (in the most extreme cases ten levels deep) Python dictionary and return the first value it finds from the given key.
I cannot understand why my code doesn't work for nested dictionaries.
def _finditem(obj, key):
if key in obj: return obj[key]
for k, v in obj.items():
if isinstance(v,dict):
_finditem(v, key)
print _finditem({"B":{"A":2}},"A")
It returns None.
It does work, however, for _finditem({"B":1,"A":2},"A"), returning 2.
I'm sure it's a simple mistake but I cannot find it. I feel like there already might be something for this in the standard library or collections, but I can't find that either.
If you are looking for a general explanation of what is wrong with code like this, the canonical is Why does my recursive function return None?. The answers here are mostly specific to the task of searching in a nested dictionary.
when you recurse, you need to return the result of _finditem
def _finditem(obj, key):
if key in obj: return obj[key]
for k, v in obj.items():
if isinstance(v,dict):
return _finditem(v, key) #added return statement
To fix the actual algorithm, you need to realize that _finditem returns None if it didn't find anything, so you need to check that explicitly to prevent an early return:
def _finditem(obj, key):
if key in obj: return obj[key]
for k, v in obj.items():
if isinstance(v,dict):
item = _finditem(v, key)
if item is not None:
return item
Of course, that will fail if you have None values in any of your dictionaries. In that case, you could set up a sentinel object() for this function and return that in the case that you don't find anything -- Then you can check against the sentinel to know if you found something or not.
Here's a function that searches a dictionary that contains both nested dictionaries and lists. It creates a list of the values of the results.
def get_recursively(search_dict, field):
"""
Takes a dict with nested lists and dicts,
and searches all dicts for a key of the field
provided.
"""
fields_found = []
for key, value in search_dict.iteritems():
if key == field:
fields_found.append(value)
elif isinstance(value, dict):
results = get_recursively(value, field)
for result in results:
fields_found.append(result)
elif isinstance(value, list):
for item in value:
if isinstance(item, dict):
more_results = get_recursively(item, field)
for another_result in more_results:
fields_found.append(another_result)
return fields_found
Here is a way to do this using a "stack" and the "stack of iterators" pattern (credits to Gareth Rees):
def search(d, key, default=None):
"""Return a value corresponding to the specified key in the (possibly
nested) dictionary d. If there is no item with that key, return
default.
"""
stack = [iter(d.items())]
while stack:
for k, v in stack[-1]:
if isinstance(v, dict):
stack.append(iter(v.items()))
break
elif k == key:
return v
else:
stack.pop()
return default
The print(search({"B": {"A": 2}}, "A")) would print 2.
Just trying to make it shorter:
def get_recursively(search_dict, field):
if isinstance(search_dict, dict):
if field in search_dict:
return search_dict[field]
for key in search_dict:
item = get_recursively(search_dict[key], field)
if item is not None:
return item
elif isinstance(search_dict, list):
for element in search_dict:
item = get_recursively(element, field)
if item is not None:
return item
return None
Here's a Python 3.3+ solution which can handle lists of lists of dicts.
It also uses duck typing, so it can handle any iterable, or object implementing the 'items' method.
from typing import Iterator
def deep_key_search(obj, key: str) -> Iterator:
""" Do a deep search of {obj} and return the values of all {key} attributes found.
:param obj: Either a dict type object or an iterator.
:return: Iterator of all {key} values found"""
if isinstance(obj, str):
# When duck-typing iterators recursively, we must exclude strings
return
try:
# Assume obj is a like a dict and look for the key
for k, v in obj.items():
if k == key:
yield v
else:
yield from deep_key_search(v, key)
except AttributeError:
# Not a dict type object. Is it iterable like a list?
try:
for v in obj:
yield from deep_key_search(v, key)
except TypeError:
pass # Not iterable either.
Pytest:
#pytest.mark.parametrize(
"data, expected, dscr", [
({}, [], "Empty dict"),
({'Foo': 1, 'Bar': 2}, [1], "Plain dict"),
([{}, {'Foo': 1, 'Bar': 2}], [1], "List[dict]"),
([[[{'Baz': 3, 'Foo': 'a'}]], {'Foo': 1, 'Bar': 2}], ['a', 1], "Deep list"),
({'Foo': 1, 'Bar': {'Foo': 'c'}}, [1, 'c'], "Dict of Dict"),
(
{'Foo': 1, 'Bar': {'Foo': 'c', 'Bar': 'abcdef'}},
[1, 'c'], "Contains a non-selected string value"
),
])
def test_deep_key_search(data, expected, dscr):
assert list(deep_key_search(data, 'Foo')) == expected
I couldn't add a comment to the accepted solution proposed by #mgilston because of lack of reputation. The solution doesn't work if the key being searched for is inside a list.
Looping through the elements of the lists and calling the recursive function should extend the functionality to find elements inside nested lists:
def _finditem(obj, key):
if key in obj: return obj[key]
for k, v in obj.items():
if isinstance(v,dict):
item = _finditem(v, key)
if item is not None:
return item
elif isinstance(v,list):
for list_item in v:
item = _finditem(list_item, key)
if item is not None:
return item
print(_finditem({"C": {"B": [{"A":2}]}}, "A"))
I had to create a general-case version that finds a uniquely-specified key (a minimal dictionary that specifies the path to the desired value) in a dictionary that contains multiple nested dictionaries and lists.
For the example below, a target dictionary is created to search, and the key is created with the wildcard "???". When run, it returns the value "D"
def lfind(query_list:List, target_list:List, targ_str:str = "???"):
for tval in target_list:
#print("lfind: tval = {}, query_list[0] = {}".format(tval, query_list[0]))
if isinstance(tval, dict):
val = dfind(query_list[0], tval, targ_str)
if val:
return val
elif tval == query_list[0]:
return tval
def dfind(query_dict:Dict, target_dict:Dict, targ_str:str = "???"):
for key, qval in query_dict.items():
tval = target_dict[key]
#print("dfind: key = {}, qval = {}, tval = {}".format(key, qval, tval))
if isinstance(qval, dict):
val = dfind(qval, tval, targ_str)
if val:
return val
elif isinstance(qval, list):
return lfind(qval, tval, targ_str)
else:
if qval == targ_str:
return tval
if qval != tval:
break
def find(target_dict:Dict, query_dict:Dict):
result = dfind(query_dict, target_dict)
return result
target_dict = {"A":[
{"key1":"A", "key2":{"key3": "B"}},
{"key1":"C", "key2":{"key3": "D"}}]
}
query_dict = {"A":[{"key1":"C", "key2":{"key3": "???"}}]}
result = find(target_dict, query_dict)
print("result = {}".format(result))
Thought I'd throw my hat in the ring, this will allow for recursive requests on anything that implements a __getitem__ method.
def _get_recursive(obj, args, default=None):
"""Apply successive requests to an obj that implements __getitem__ and
return result if something is found, else return default"""
if not args:
return obj
try:
key, *args = args
_obj = object.__getitem__(obj, key)
return _get_recursive(_obj, args, default=default)
except (KeyError, IndexError, AttributeError):
return default
I'm looking for a way to update dict dictionary1 with the contents of dict update wihout overwriting levelA
dictionary1 = {
"level1": {
"level2": {"levelA": 0, "levelB": 1}
}
}
update = {
"level1": {
"level2": {"levelB": 10}
}
}
dictionary1.update(update)
print(dictionary1)
{
"level1": {
"level2": {"levelB": 10}
}
}
I know that update deletes the values in level2 because it's updating the lowest key level1.
How could I tackle this, given that dictionary1 and update can have any length?
#FM's answer has the right general idea, i.e. a recursive solution, but somewhat peculiar coding and at least one bug. I'd recommend, instead:
Python 2:
import collections
def update(d, u):
for k, v in u.iteritems():
if isinstance(v, collections.Mapping):
d[k] = update(d.get(k, {}), v)
else:
d[k] = v
return d
Python 3:
import collections.abc
def update(d, u):
for k, v in u.items():
if isinstance(v, collections.abc.Mapping):
d[k] = update(d.get(k, {}), v)
else:
d[k] = v
return d
The bug shows up when the "update" has a k, v item where v is a dict and k is not originally a key in the dictionary being updated -- #FM's code "skips" this part of the update (because it performs it on an empty new dict which isn't saved or returned anywhere, just lost when the recursive call returns).
My other changes are minor: there is no reason for the if/else construct when .get does the same job faster and cleaner, and isinstance is best applied to abstract base classes (not concrete ones) for generality.
If you happen to be using pydantic (great lib, BTW), you can use one of its utility methods:
from pydantic.utils import deep_update
dictionary1 = deep_update(dictionary1, update)
UPDATE: reference to code, as pointed by #Jorgu. If installing pydantic is not desired, the code is short enough to be copied, provided adequate licenses compatibilities.
Took me a little bit on this one, but thanks to #Alex's post, he filled in the gap I was missing. However, I came across an issue if a value within the recursive dict happens to be a list, so I thought I'd share, and extend his answer.
import collections
def update(orig_dict, new_dict):
for key, val in new_dict.iteritems():
if isinstance(val, collections.Mapping):
tmp = update(orig_dict.get(key, { }), val)
orig_dict[key] = tmp
elif isinstance(val, list):
orig_dict[key] = (orig_dict.get(key, []) + val)
else:
orig_dict[key] = new_dict[key]
return orig_dict
Same solution as the accepted one, but clearer variable naming, docstring, and fixed a bug where {} as a value would not override.
import collections
def deep_update(source, overrides):
"""
Update a nested dictionary or similar mapping.
Modify ``source`` in place.
"""
for key, value in overrides.iteritems():
if isinstance(value, collections.Mapping) and value:
returned = deep_update(source.get(key, {}), value)
source[key] = returned
else:
source[key] = overrides[key]
return source
Here are a few test cases:
def test_deep_update():
source = {'hello1': 1}
overrides = {'hello2': 2}
deep_update(source, overrides)
assert source == {'hello1': 1, 'hello2': 2}
source = {'hello': 'to_override'}
overrides = {'hello': 'over'}
deep_update(source, overrides)
assert source == {'hello': 'over'}
source = {'hello': {'value': 'to_override', 'no_change': 1}}
overrides = {'hello': {'value': 'over'}}
deep_update(source, overrides)
assert source == {'hello': {'value': 'over', 'no_change': 1}}
source = {'hello': {'value': 'to_override', 'no_change': 1}}
overrides = {'hello': {'value': {}}}
deep_update(source, overrides)
assert source == {'hello': {'value': {}, 'no_change': 1}}
source = {'hello': {'value': {}, 'no_change': 1}}
overrides = {'hello': {'value': 2}}
deep_update(source, overrides)
assert source == {'hello': {'value': 2, 'no_change': 1}}
This functions is available in the charlatan package, in charlatan.utils.
#Alex's answer is good, but doesn't work when replacing an element such as an integer with a dictionary, such as update({'foo':0},{'foo':{'bar':1}}). This update addresses it:
import collections
def update(d, u):
for k, v in u.iteritems():
if isinstance(d, collections.Mapping):
if isinstance(v, collections.Mapping):
r = update(d.get(k, {}), v)
d[k] = r
else:
d[k] = u[k]
else:
d = {k: u[k]}
return d
update({'k1': 1}, {'k1': {'k2': {'k3': 3}}})
Here's an Immutable version of recursive dictionary merge in case anybody needs it.
Based upon #Alex Martelli's answer.
Python 3.x:
import collections
from copy import deepcopy
def merge(dict1, dict2):
''' Return a new dictionary by merging two dictionaries recursively. '''
result = deepcopy(dict1)
for key, value in dict2.items():
if isinstance(value, collections.Mapping):
result[key] = merge(result.get(key, {}), value)
else:
result[key] = deepcopy(dict2[key])
return result
Python 2.x:
import collections
from copy import deepcopy
def merge(dict1, dict2):
''' Return a new dictionary by merging two dictionaries recursively. '''
result = deepcopy(dict1)
for key, value in dict2.iteritems():
if isinstance(value, collections.Mapping):
result[key] = merge(result.get(key, {}), value)
else:
result[key] = deepcopy(dict2[key])
return result
Just use python-benedict (I did it), it has a merge (deepupdate) utility method and many others. It works with python 2 / python 3 and it is well tested.
from benedict import benedict
dictionary1=benedict({'level1':{'level2':{'levelA':0,'levelB':1}}})
update={'level1':{'level2':{'levelB':10}}}
dictionary1.merge(update)
print(dictionary1)
# >> {'level1':{'level2':{'levelA':0,'levelB':10}}}
Installation: pip install python-benedict
Documentation: https://github.com/fabiocaccamo/python-benedict
Note: I am the author of this project
This question is old, but I landed here when searching for a "deep merge" solution. The answers above inspired what follows. I ended up writing my own because there were bugs in all the versions I tested. The critical point missed was, at some arbitrary depth of the two input dicts, for some key, k, the decision tree when d[k] or u[k] is not a dict was faulty.
Also, this solution does not require recursion, which is more symmetric with how dict.update() works, and returns None.
import collections
def deep_merge(d, u):
"""Do a deep merge of one dict into another.
This will update d with values in u, but will not delete keys in d
not found in u at some arbitrary depth of d. That is, u is deeply
merged into d.
Args -
d, u: dicts
Note: this is destructive to d, but not u.
Returns: None
"""
stack = [(d,u)]
while stack:
d,u = stack.pop(0)
for k,v in u.items():
if not isinstance(v, collections.Mapping):
# u[k] is not a dict, nothing to merge, so just set it,
# regardless if d[k] *was* a dict
d[k] = v
else:
# note: u[k] is a dict
if k not in d:
# add new key into d
d[k] = v
elif not isinstance(d[k], collections.Mapping):
# d[k] is not a dict, so just set it to u[k],
# overriding whatever it was
d[k] = v
else:
# both d[k] and u[k] are dicts, push them on the stack
# to merge
stack.append((d[k], v))
Minor improvements to #Alex's answer that enables updating of dictionaries of differing depths as well as limiting the depth that the update dives into the original nested dictionary (but the updating dictionary depth is not limited). Only a few cases have been tested:
def update(d, u, depth=-1):
"""
Recursively merge or update dict-like objects.
>>> update({'k1': {'k2': 2}}, {'k1': {'k2': {'k3': 3}}, 'k4': 4})
{'k1': {'k2': {'k3': 3}}, 'k4': 4}
"""
for k, v in u.iteritems():
if isinstance(v, Mapping) and not depth == 0:
r = update(d.get(k, {}), v, depth=max(depth - 1, -1))
d[k] = r
elif isinstance(d, Mapping):
d[k] = u[k]
else:
d = {k: u[k]}
return d
The code below should solve the update({'k1': 1}, {'k1': {'k2': 2}}) issue in #Alex Martelli's answer the right way.
def deepupdate(original, update):
"""Recursively update a dict.
Subdict's won't be overwritten but also updated.
"""
if not isinstance(original, abc.Mapping):
return update
for key, value in update.items():
if isinstance(value, abc.Mapping):
original[key] = deepupdate(original.get(key, {}), value)
else:
original[key] = value
return original
I used the solution #Alex Martelli suggests, but it fails
TypeError 'bool' object does not support item assignment
when the two dictionaries differ in data type at some level.
In case at the same level the element of dictionary d is just a scalar (ie. Bool) while the element of dictionary u is still dictionary the reassignment fails as no dictionary assignment is possible into scalar (like True[k]).
One added condition fixes that:
from collections import Mapping
def update_deep(d, u):
for k, v in u.items():
# this condition handles the problem
if not isinstance(d, Mapping):
d = u
elif isinstance(v, Mapping):
r = update_deep(d.get(k, {}), v)
d[k] = r
else:
d[k] = u[k]
return d
In neither of these answers the authors seem to understand the concept of updating an object stored in a dictionary nor even of iterating over dictionary items (as opposed to keys). So I had to write one which doesn't make pointless tautological dictionary stores and retrievals.
The dicts are assumed to store other dicts or simple types.
def update_nested_dict(d, other):
for k, v in other.items():
if isinstance(v, collections.Mapping):
d_v = d.get(k)
if isinstance(d_v, collections.Mapping):
update_nested_dict(d_v, v)
else:
d[k] = v.copy()
else:
d[k] = v
Or even simpler one working with any type:
def update_nested_dict(d, other):
for k, v in other.items():
d_v = d.get(k)
if isinstance(v, collections.Mapping) and isinstance(d_v, collections.Mapping):
update_nested_dict(d_v, v)
else:
d[k] = deepcopy(v) # or d[k] = v if you know what you're doing
Update to #Alex Martelli's answer to fix a bug in his code to make the solution more robust:
def update_dict(d, u):
for k, v in u.items():
if isinstance(v, collections.Mapping):
default = v.copy()
default.clear()
r = update_dict(d.get(k, default), v)
d[k] = r
else:
d[k] = v
return d
The key is that we often want to create the same type at recursion, so here we use v.copy().clear() but not {}. And this is especially useful if the dict here is of type collections.defaultdict which can have different kinds of default_factorys.
Also notice that the u.iteritems() has been changed to u.items() in Python3.
def update(value, nvalue):
if not isinstance(value, dict) or not isinstance(nvalue, dict):
return nvalue
for k, v in nvalue.items():
value.setdefault(k, dict())
if isinstance(v, dict):
v = update(value[k], v)
value[k] = v
return value
use dict or collections.Mapping
I recommend to replace {} by type(v)() in order to propagate object type of any dict subclass stored in u but absent from d. For example, this would preserve types such as collections.OrderedDict:
Python 2:
import collections
def update(d, u):
for k, v in u.iteritems():
if isinstance(v, collections.Mapping):
d[k] = update(d.get(k, type(v)()), v)
else:
d[k] = v
return d
Python 3:
import collections.abc
def update(d, u):
for k, v in u.items():
if isinstance(v, collections.abc.Mapping):
d[k] = update(d.get(k, type(v)()), v)
else:
d[k] = v
return d
It could be that you stumble over a non-standard-dictionary, like me today, which has no iteritems-Attribute.
In this case it's easy to interpret this type of dictionary as a standard-dictionary. E.g.:
Python 2.7:
import collections
def update(orig_dict, new_dict):
for key, val in dict(new_dict).iteritems():
if isinstance(val, collections.Mapping):
tmp = update(orig_dict.get(key, { }), val)
orig_dict[key] = tmp
elif isinstance(val, list):
orig_dict[key] = (orig_dict[key] + val)
else:
orig_dict[key] = new_dict[key]
return orig_dict
import multiprocessing
d=multiprocessing.Manager().dict({'sample':'data'})
u={'other': 1234}
x=update(d, u)
x.items()
Python 3.8:
def update(orig_dict, new_dict):
orig_dict=dict(orig_dict)
for key, val in dict(new_dict).items():
if isinstance(val, collections.abc.Mapping):
tmp = update(orig_dict.get(key, { }), val)
orig_dict[key] = tmp
elif isinstance(val, list):
orig_dict[key] = (orig_dict[key] + val)
else:
orig_dict[key] = new_dict[key]
return orig_dict
import collections
import multiprocessing
d=multiprocessing.Manager().dict({'sample':'data'})
u={'other': 1234, "deeper": {'very': 'deep'}}
x=update(d, u)
x.items()
Thanks to hobs for his comment on Alex's answer. Indeed update({'k1': 1}, {'k1': {'k2': 2}}) will cause TypeError: 'int' object does not support item assignment.
We should check the types of the input values at the beginning of the function. So, I suggest the following function, which should solve this (and other) problem.
Python 3:
from collections.abc import Mapping
def deep_update(d1, d2):
if all((isinstance(d, Mapping) for d in (d1, d2))):
for k, v in d2.items():
d1[k] = deep_update(d1.get(k), v)
return d1
return d2
I know this question is pretty old, but still posting what I do when I have to update a nested dictionary. We can use the fact that dicts are passed by reference in python
Assuming that the path of the key is known and is dot separated. Forex if we have a dict named data:
{
"log_config_worker": {
"version": 1,
"root": {
"handlers": [
"queue"
],
"level": "DEBUG"
},
"disable_existing_loggers": true,
"handlers": {
"queue": {
"queue": null,
"class": "myclass1.QueueHandler"
}
}
},
"number_of_archived_logs": 15,
"log_max_size": "300M",
"cron_job_dir": "/etc/cron.hourly/",
"logs_dir": "/var/log/patternex/",
"log_rotate_dir": "/etc/logrotate.d/"
}
And we want to update the queue class, the path of the key would be - log_config_worker.handlers.queue.class
We can use the following function to update the value:
def get_updated_dict(obj, path, value):
key_list = path.split(".")
for k in key_list[:-1]:
obj = obj[k]
obj[key_list[-1]] = value
get_updated_dict(data, "log_config_worker.handlers.queue.class", "myclass2.QueueHandler")
This would update the dictionary correctly.
I made a simple function, in which you give the key, the new value and the dictionary as input, and it recursively updates it with the value:
def update(key,value,dictionary):
if key in dictionary.keys():
dictionary[key] = value
return
dic_aux = []
for val_aux in dictionary.values():
if isinstance(val_aux,dict):
dic_aux.append(val_aux)
for i in dic_aux:
update(key,value,i)
for [key2,val_aux2] in dictionary.items():
if isinstance(val_aux2,dict):
dictionary[key2] = val_aux2
dictionary1={'level1':{'level2':{'levelA':0,'levelB':1}}}
update('levelB',10,dictionary1)
print(dictionary1)
#output: {'level1': {'level2': {'levelA': 0, 'levelB': 10}}}
Hope it answers.
Yes! And another solution. My solution differs in the keys that are being checked.
In all other solutions we only look at the keys in dict_b. But here we look in the union of both dictionaries.
Do with it as you please
def update_nested(dict_a, dict_b):
set_keys = set(dict_a.keys()).union(set(dict_b.keys()))
for k in set_keys:
v = dict_a.get(k)
if isinstance(v, dict):
new_dict = dict_b.get(k, None)
if new_dict:
update_nested(v, new_dict)
else:
new_value = dict_b.get(k, None)
if new_value:
dict_a[k] = new_value
If you want to replace a "full nested dictionary with arrays" you can use this snippet :
It will replace any "old_value" by "new_value". It's roughly doing a depth-first rebuilding of the dictionary. It can even work with List or Str/int given as input parameter of first level.
def update_values_dict(original_dict, future_dict, old_value, new_value):
# Recursively updates values of a nested dict by performing recursive calls
if isinstance(original_dict, Dict):
# It's a dict
tmp_dict = {}
for key, value in original_dict.items():
tmp_dict[key] = update_values_dict(value, future_dict, old_value, new_value)
return tmp_dict
elif isinstance(original_dict, List):
# It's a List
tmp_list = []
for i in original_dict:
tmp_list.append(update_values_dict(i, future_dict, old_value, new_value))
return tmp_list
else:
# It's not a dict, maybe a int, a string, etc.
return original_dict if original_dict != old_value else new_value
Another way of using recursion:
def updateDict(dict1,dict2):
keys1 = list(dict1.keys())
keys2= list(dict2.keys())
keys2 = [x for x in keys2 if x in keys1]
for x in keys2:
if (x in keys1) & (type(dict1[x]) is dict) & (type(dict2[x]) is dict):
updateDict(dict1[x],dict2[x])
else:
dict1.update({x:dict2[x]})
return(dict1)
Credit to: #Gustavo Alves Casqueiro for original answer
I honestly would have preferred using a lib that could do the heavy lifting for me, but I just couldn't find something that did what I needed.
I have only added a couple of additional checks to this function.
I have included a check for lists within a dict and added a parameter for the name of a nested dict to correctly update the nested dict KEY when there may be another KEY within the OUTER dict with the same name.
Updated function:
def update(dictionary: dict[str, any], key: str, value: any, nested_dict_name: str = None) -> dict[str, any]:
if not nested_dict_name: # if current (outermost) dict should be updated
if key in dictionary.keys(): # check if key exists in current dict
dictionary[key] = value
return dictionary
else: # if nested dict should be updated
if nested_dict_name in dictionary.keys(): # check if dict is in next layer
if isinstance(dictionary[nested_dict_name], dict):
if key in dictionary[nested_dict_name].keys(): # check if key exists in current dict
dictionary[nested_dict_name][key] = value
return dictionary
if isinstance(dictionary[nested_dict_name], list):
list_index = random.choice(range(len(dictionary[nested_dict_name]))) # pick a random dict from the list
if key in dictionary[nested_dict_name][list_index].keys(): # check if key exists in current dict
dictionary[nested_dict_name][list_index][key] = value
return dictionary
dic_aux = []
# this would only run IF the above if-statement was not able to identity and update a dict
for val_aux in dictionary.values():
if isinstance(val_aux, dict):
dic_aux.append(val_aux)
# call the update function again for recursion
for i in dic_aux:
return update(dictionary=i, key=key, value=value, nested_dict_name=nested_dict_name)
Original dict:
{
"level1": {
"level2": {
"myBool": "Original",
"myInt": "Original"
},
"myInt": "Original",
"myBool": "Original"
},
"myStr": "Original",
"level3": [
{
"myList": "Original",
"myInt": "Original",
"myBool": "Original"
}
],
"level4": [
{
"myList": "Original",
"myInt": "UPDATED",
"myBool": "Original"
}
],
"level5": {
"level6": {
"myBool": "Original",
"myInt": "Original"
},
"myInt": "Original",
"myBool": "Original"
}
}
Data for updating (using pytest):
#pytest.fixture(params=[(None, 'myStr', 'UPDATED'),
('level1', 'myInt', 'UPDATED'),
('level2', 'myBool', 'UPDATED'),
('level3', 'myList', 'UPDATED'),
('level4', 'myInt', 'UPDATED'),
('level5', 'myBool', 'UPDATED')])
def sample_data(request):
return request.param
The 'UPDATED' parameter doesn't make sense in this smaller use case (since I could just hard-code it), but for simplicity when reading the logs, I didn't want to see multiple data-types and just made it show me an 'UPDATED' string.
Test:
#pytest.mark.usefixtures('sample_data')
def test_this(sample_data):
nested_dict, param, update_value = sample_data
if nested_dict is None:
print(f'\nDict Value: Level0\nParam: {param}\nUpdate Value: {update_value}')
else:
print(f'\nDict Value: {nested_dict}\nParam: {param}\nUpdate Value: {update_value}')
# initialise data dict
data_object = # insert data here (see example dict above)
# first print as is
print(f'\nOriginal Dict:\n{data_object}')
update(dictionary=data_object,
key=param,
value=update_value,
nested_dict_name=nested_dict)
# print updated
print(f'\nUpdated Dict:\n{data_object}')
There is one caveat, when you have a dict like this:
{
"level1": {
"level2": {
"myBool": "Original"
},
"myBool": "Original"
},
"level3": {
"level2": {
"myBool": "Original"
},
"myInt": "Original"
}
}
Where level2 is under level1 AND level3. This would require making using of a list or something with the nested_dict_name and passing in the name of the outer dict AND inner dict (['level5', 'level2']) and then somehow looping through the values to find that dict.
However, since I haven't yet ran into this issue for the data objects I use, I haven't spent the time trying to solve this "issue".
Convert your dictionaries into NestedDict
from ndicts.ndicts import NestedDict
dictionary1 = {'level1': {'level2': {'levelA': 0, 'levelB': 1}}}
update = {'level1': {'level2': {'levelB': 10}}}
nd, nd_update = NestedDict(dictionary1), NestedDict(update)
Then just use update
>>> nd.update(nd_update)
>>> nd
NestedDict({'level1': {'level2': {'levelA': 0, 'levelB': 10}}})
If you need the result as a dictionary nd.to_dict()
To install ndicts pip install ndicts
d is dict to update, u is dict-updater.
def recursively_update_dict(d: dict, u: dict):
for k, v in u.items():
if isinstance(v, dict):
d.setdefault(k, {})
recursively_update_dict(d[k], v)
else:
d[k] = v
Or for defaultdict
from collections import defaultdict
def recursively_update_defaultdict(d: defaultdict[dict], u: dict):
for k, v in u.items():
if isinstance(v, dict):
recursively_update_dict(d[k], v)
else:
d[k] = v
a new Q
how to By a keys chain
dictionary1={'level1':{'level2':{'levelA':0,'levelB':1}},'anotherLevel1':{'anotherLevel2':{'anotherLevelA':0,'anotherLevelB':1}}}
update={'anotherLevel1':{'anotherLevel2':1014}}
dictionary1.update(update)
print dictionary1
{'level1':{'level2':{'levelA':0,'levelB':1}},'anotherLevel1':{'anotherLevel2':1014}}
you could try this, it works with lists and is pure:
def update_keys(newd, dic, mapping):
def upsingle(d,k,v):
if k in mapping:
d[mapping[k]] = v
else:
d[k] = v
for ekey, evalue in dic.items():
upsingle(newd, ekey, evalue)
if type(evalue) is dict:
update_keys(newd, evalue, mapping)
if type(evalue) is list:
upsingle(newd, ekey, [update_keys({}, i, mapping) for i in evalue])
return newd
That's a bit to the side but do you really need nested dictionaries? Depending on the problem, sometimes flat dictionary may suffice... and look good at it:
>>> dict1 = {('level1','level2','levelA'): 0}
>>> dict1['level1','level2','levelB'] = 1
>>> update = {('level1','level2','levelB'): 10}
>>> dict1.update(update)
>>> print dict1
{('level1', 'level2', 'levelB'): 10, ('level1', 'level2', 'levelA'): 0}