I have a deeply nested dictionary (arbitrary keys and values) like:
data = {
'a': {
'path': '/a/a.txt'
},
'b': {
'b1': {
'path': '/b/b1/b1.txt'
},
'b2': {
'path': '/b/b2/b2.txt'
}
}
'c': {
'c1': {
'c12': {
'path': '/c/c1/c12/c12.txt'
}
},
'c2': {
'c22': {
'path': '/c/c1/c22/c22.txt'
}
},
'c3': {
'c32': {
'path': '/c/c1/c32/c32.txt'
}
}
}
.
.
.
}
My goal is to prepend every value in the dictionary with a specific path. So basically take in the data above, operate on it:
def prepend(value, data):
return magic
data = prepend('predir/z', data)
and have the resulting dict look like:
data = {
'a': {
'path': 'predir/z/a/a.txt'
},
'b': {
'b1': {
'path': 'predir/z/b/b1/b1.txt'
},
'b2': {
'path': 'predir/z/b/b2/b2.txt'
}
}
'c': {
'c1': {
'c12': {
'path': 'predir/z/c/c1/c12/c12.txt'
}
},
'c2': {
'c22': {
'path': 'predir/z/c/c1/c22/c22.txt'
}
},
'c3': {
'c32': {
'path': 'predir/z/c/c1/c32/c32.txt'
}
}
}
.
.
.
}
I know I can use recursion to loop through the dict like so:
def prepend(directory, config):
for k, v in config.items():
if isinstance(v, dict):
prepend(directory, v)
else:
# do something
But, I'm not able to change the values during iteration. Any and all help is greatly appreciated! Thank you!
In the else branch, just add the prefix to the existing value, and store it in the key :
def prepend(directory, config):
for k, v in config.items():
if isinstance(v, dict):
prepend(directory, v)
else:
config[k] = directory + v
return config
You have to use
else:
config[k] = directory + config[k]
or
else:
config[k] = directory + v
and it will change value directly in original data so you don't need return config.
BTW: If you want to keep original data then you would have to create new_config in function.
def prepend(directory, config):
new_config = {}
for k, v in config.items():
if isinstance(v, dict):
new_config[k] = prepend(directory, v)
else:
new_config[k] = directory + v
return new_config
new_data = prepend('predir/z', data)
print(data)
print(new_data)
Related
I have two nested dictionaries (loaded from json) and need to check if the keys in one already exist in the other, and if not add them.
Example json:
eg_dict = {
"A":
{
"A1":
{
"A1a": "bird",
"A1b": true,
"A1c": false
},
"A2":
{
"A2a":
{
"A2a1": "parrot",
"A2a2":
{
"enabled": true
}
}
}
},
"B":
{
"B1":
{
"B1a": "Reptile"
},
"A2":
{
"A2a":
{
"A2a2":
{
"enabled": true
}
}
}
}
}
I need to add A1 and A2a1 to B.
I've tried Check if a nested dictionary is a subset of another nested dictionary but it's not doing quite what I need.
I started trying to pass the parent key through a recursive function, but as I don't know how deep the nesting goes, this seems like a dead end?
def get_all_values(pkey, nested_dictionary):
#I don't think passing p(arent)key to the function gets me anywhere
for key, value in nested_dictionary.items():
if type(value) is dict:
print(key, ":", value)
get_all_values(pkey, value)
else:
print(key, ":", value)
def get_json(file, chartname):
#print(chartname)
with open(file) as file:
file= json.load(file)
b = file['B']
#Can do it if I know the key I want to copy
if 'A1' in file['A'].keys():
b['A1'] = file['A']['A1']
#Trying a function to get all missing keys from A
get_all_values(key=None, file['B'])
b = json.dumps(b)
return b
First time posting on stackoverflow, so help on improving my question welcome too!
A recursive solution to build a dictionary from A and B, where values of A have precedence over values of B:
a_dict = eg_dict['A']
b_dict = eg_dict['B']
print(a_dict)
# {'A1': {'A1a': 'bird', 'A1b': True, 'A1c': False}, 'A2': {'A2a': {'A2a1': 'parrot', 'A2a2': {'enabled': True}}}}
print(b_dict)
# {'B1': {'B1a': 'Reptile'}, 'A2': {'A2a': {'A2a2': {'enabled': True}}}}
def extend_dict(primary_dict, secondary_dict):
result_dict = {}
for k in set(primary_dict.keys()).union(set(secondary_dict.keys())):
if (k in primary_dict.keys() and k in secondary_dict.keys()) and (isinstance(primary_dict[k], dict) and isinstance(secondary_dict[k], dict)):
result_dict.update({k: extend_dict(primary_dict[k], secondary_dict[k])})
elif k in primary_dict.keys():
result_dict.update({k: primary_dict[k]})
elif k in secondary_dict.keys():
result_dict.update({k: secondary_dict[k]})
return result_dict
extended = extend_dict(a_dict, b_dict)
print(extended)
# {'B1': {'B1a': 'Reptile'}, 'A2': {'A2a': {'A2a2': {'enabled': True}, 'A2a1': 'parrot'}}, 'A1': {'A1a': 'bird', 'A1b': True, 'A1c': False}}
If you want to switch the precedence, just switch A and B, such that extended = extend_dict(b_dict, a_dict).
Let me know if this is what you are looking for.
I receive a response I have no control over from an API. Using requests response.json() will filter out duplicate keys. So I would need to turn this response into a list where each key is an element in that list: What I get now:
{
"user": {
//...
},
"user": {
//...
},
//...
}
What I need:
{
"users": [
{
"user": {
//...
}
},
{
"user": {
//...
}
},
//...
]
}
This way JSON won't filter out any of the results, and I can loop through users.
Okay, let me have a try by method used in Python json parser allow duplicate keys
All we should do is handle the pairs_list by ourself.
from json import JSONDecoder
def parse_object_pairs(pairs):
return pairs
data = """
{"foo": {"key": 2, "key": 3}, "foo": 4, "foo": 23}
"""
decoder = JSONDecoder(object_pairs_hook=parse_object_pairs)
pairs_list = decoder.decode(data)
# the pairs_list is the real thing which we can use
aggre_key = 's'
def recusive_handle(pairs_list):
dct = {}
for k, v in pairs_list:
if v and isinstance(v, list) and isinstance(v[0], tuple):
v = recusive_handle(v)
if k + aggre_key in dct:
dct[k + aggre_key].append({k: v})
elif k in dct:
first_dict = {k: dct.pop(k)}
dct[k + aggre_key] = [first_dict, {k: v}]
else:
dct[k] = v
return dct
print(recusive_handle(pairs_list))
output:
{'foos': [{'foo': {'keys': [{'key': 2}, {'key': 3}]}}, {'foo': {'bar': 4}}, {'foo': 23}]}
I have a dictionary that looks like:
normalDictionary = {'a' : {'b': {}},
'a1': {'b1': {'c1' : {},
'd1' : {}}}}
and I want to invert it so that it looks like:
invertedDictionary = {'d1': {'b1': {'a1': {}}},
'c1': {'b1': {'a1': {}}},
'b': {'a': {}}}
What would that python function look like?
I cannot seem to get much past:
def invert_dictionary( node, leafName, indent ):
keys = list( node.keys() )
keys.sort()
constructed = { leafName: {} }
for key in keys:
inverted = invert_dictionary( node[ key ], key, indent + 4 )
return constructed
invertedDictionary = {}
for key in normalDictionary
inverted = invert_dictionary( normalDictionary[ key ], key, indent = 0 )
This may not be the optimal algorithm but you can start with this. The idea is
we convert the dictionary into all "walk paths" from the "root" to all "leaves"
then we build another dictionary from such paths, in reversed order
Here is the code:
def getpaths(dictionary, pathhead):
if not dictionary:
return [pathhead]
paths = []
for key in dictionary:
paths.extend(getpaths(dictionary[key], pathhead+[key]))
return paths
def invert(dictionary):
paths = getpaths(dictionary, [])
inverted = {}
for path in paths:
head = inverted
for node in path[::-1]:
if node not in head:
head[node] = {}
head = head[node]
return inverted
and this is how it works:
>>> normalDictionary
{'a': {'b': {}}, 'a1': {'b1': {'c1': {}, 'd1': {}}}}
>>> invert(normalDictionary)
{'b': {'a': {}}, 'c1': {'b1': {'a1': {}}}, 'd1': {'b1': {'a1': {}}}}
A recursive implementation:
def asdict(xs: list) -> dict:
return {} if len(xs) == 0 else {xs[0]: asdict(xs[1:])}
def inverted_dict_as_tuples(d: dict, stack: list):
for k, v in d.items():
if len(v) == 0:
yield (k, *reversed(stack))
else:
yield from inverted_dict_as_tuples(v, [*stack, k])
def inverted_dict(d: dict) -> dict:
return {x: asdict(xs) for x, *xs in inverted_dict_as_tuples(d, [])}
Usage:
>>> import json
>>> d = {"a": {"b": {}}, "a1": {"b1": {"c1": {}, "d1": {}}}}
>>> print(json.dumps(d, indent=2))
{
"a": {
"b": {}
},
"a1": {
"b1": {
"c1": {},
"d1": {}
}
}
}
>>> d_inv = inverted_dict(d)
>>> print(json.dumps(d_inv, indent=2))
{
"b": {
"a": {}
},
"c1": {
"b1": {
"a1": {}
}
},
"d1": {
"b1": {
"a1": {}
}
}
}
This is a working solution:
def add_path(d, path):
while path:
k = path.pop()
if k not in d:
d[k] = {}
d = d[k]
def invert_dict(d, target=None, path=None):
if target is None:
target = {}
if path is None:
path = []
if not d:
add_path(target, path)
else:
for k, v in d.items():
invert_dict(v, target, path + [k])
return target
print(invert_dict(normalDictionary))
This assumes your dictionary only contains dictionaries like your example, though. Not sure what the actual use case is, where you may have more mixed data types.
Result:
{'b': {'a': {}}, 'c1': {'b1': {'a1': {}}}, 'd1': {'b1': {'a1': {}}}}
In technical terms, you have an n-ary tree and it sounds like you want to build root-to-leaf paths (using depth-first search), then create another n-ary tree by expanding each of those paths in reverse. Here's one way (all_paths yields root-to-leaf paths, reverse each path, then iterate over each path plugging values into a dict in paths_to_dict):
import json
def all_paths(d, path=[]):
if not d:
yield path[:]
for k, v in d.items():
path.append(k)
yield from all_paths(v, path)
path.pop()
def paths_to_dict(paths):
d = {}
for path in paths:
curr = d
for node in path:
curr[node] = curr = {}
return d
if __name__ == "__main__":
d = {
'a': {
'b': {}
},
'a1': {
'b1': {
'c1': {},
'd1': {}
}
}
}
inverted_d = paths_to_dict([list(reversed(x)) for x in all_paths(d)])
print(json.dumps(inverted_d, indent=2))
Output:
{
"b": {
"a": {}
},
"c1": {
"b1": {
"a1": {}
}
},
"d1": {
"b1": {
"a1": {}
}
}
}
I think you want a single recursive function like this.
def reverse_dict(final_result, middle_result, normal_dictionary):
for key, value in normal_dictionary.items():
if len(value.keys()) == 0:
final_result[key] = value
middle_result.append(final_result[key])
else:
reverse_dict(final_result, middle_result, value)
for item in middle_result:
item[key] = {}
middle_result = []
for item in middle_result:
middle_result.append(item[key])
Example:
test_normal_dictionary = {
'a': {
'b': {}
},
'a1': {
'b1': {
'c1': {},
'd1': {}
}
}
}
result_dictionary = {}
print(f"Origin dict: {test_normal_dictionary}")
reverse_dict(result_dictionary, [], test_normal_dictionary)
print(f"Reversed dict: {result_dictionary}")
Output:
Origin dict: {'a': {'b': {}}, 'a1': {'b1': {'c1': {}, 'd1': {}}}}
Reversed dict: {'b': {'a': {}}, 'c1': {'b1': {}, 'a1': {}}, 'd1': {'b1': {}, 'a1': {}}}
I have class with a nested dictionary data object. I need to get all the key values from it. What's the best efficient way to do this?
I'm stuck with following:
for k,v in data.items():
print v.keys()
This is the data:
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"OFFLINE": {"online": None, "offline_few": "ALERT"},
}
An elegant way to concatenate lists (your value.keys() lists) into one is using a double-loop list comprehenstion, like this:
nested_keys = [
key
for val in data.values()
for key in val.keys()]
Using a generator:
def all_keys(d):
for k, v in d.items():
yield k
# assume that anything with `items` property will be a mapping
# this can be replaced with: `isinstance(v, dict)` or `isinstance(v, collections.Mapping)`
if hasattr(v, 'items'):
yield from all_keys(v)
On your input this produces:
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"OFFLINE": {"online": None, "offline_few": "ALERT"},
}
print(list(all_keys(data)))
# ['BANK', 'no_data', 'SHOCK', 'drop', 'rise', 'high_risk', 'OFFLINE', 'online', 'offline_few']
If all your "actual" key-value pairs are at a certain depth, for example for depth 1, you can go like:
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"OFFLINE": {"online": None, "offline_few": "ALERT"},
}
dic = {k:v for val in data.values() for k,v in val.items()}
But if you dont know that:
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"online": None,
"offline_few": "ALERT"
}
In this case you need to use recursion:
def unnest(dic, final=dict()):
for key, val in dic.items():
if not isinstance(val, dict):
final[key] = val
else:
dic2 = dict()
for k, v in val.items():
dic2[k] = v
unnest(dic2, final)
return final
dic = unnest(data, {}) #every use of the function should have {} to solve issue pointed by #Ch3steR
In any case, once you have the "un-nested" dictionary it is trivial to print out the keys:
print(dic.keys())
Recursive Function
Gets all keys at all levels of nested dictionary
def get_keys(d, result = None):
# use default of None to fix issue noted by #Ch3steR
# namely: http://effbot.org/zone/default-values.htm
if result is None:
result = []
for k, v in d.items():
if isinstance(v, dict):
result.append(k)
get_keys(v, result)
else:
result.append(k)
return result
Test
print(get_keys(data))
Output
['BANK', 'no_data', 'SHOCK', 'drop', 'rise', 'high_risk', 'OFFLINE', 'online', 'offline_few']
You could use a NestedDict. First install ndicts
pip install ndicts
Then
from ndicts.ndicts import NestedDict
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"OFFLINE": {"online": None, "offline_few": "ALERT"},
}
nd = NestedDict(data)
The result
>>> list(nd.keys())
[('BANK', 'no_data'), ('SHOCK', 'drop'), ('SHOCK', 'rise'), ('SHOCK', 'high_risk'), ('OFFLINE', 'online'), ('OFFLINE', 'offline_few')]
I have below dictionary (generated from a report, so the structure can change).
I need to go to the depth of the dictionary, find the id which in this case is 'id': u'ef3c8cf1-0987-4e56-a6d5-763c42be1f75', (there can be more than 1), delete that id and then move to one level up and repeat the same till I get to the top id which I finally delete. Since there is a dependency, I need to delete the orphan id first and then move to the top.
Any help is appreciable. If any other file/information is needed, please let me know.
{
'id': u'4c31d813-a989-47dd-b01b-9a27b8db2dfc',
'snapshots':
[
{
'id': u'3ddc7ddd-02ca-4669-a0cb-fb0d56a4a6f5',
'volumes':
[
{
'id': u'5488de90-50dc-4d72-a6aa-c995422fa179',
'snapshots': [],
'snapshot_id': u'3ddc7ddd-02ca-4669-a0cb-fb0d56a4a6f5'
},
{
'id': u'e566645f-4fb3-4778-be67-447a5bdd678d',
'snapshots':
[
{
'id': u'd637f6ea-4a41-448c-874f-ffe624ddc597',
'volumes':
[
{
'id': u'ef3c8cf1-0987-4e56-a6d5-763c42be1f75',
'snapshots': [],
'snapshot_id': u'd637f6ea-4a41-448c-874f-ffe624ddc597'
}
]
}
],
'snapshot_id': u'3ddc7ddd-02ca-4669-a0cb-fb0d56a4a6f5'},
{
'id': u'196483ee-4f21-4d83-8e15-8caea532b2ab',
'snapshots': [],
'snapshot_id': u'3ddc7ddd-02ca-4669-a0cb-fb0d56a4a6f5'
}
]
}
],
'snapshot_id': None
}
Python code
oh=openstack_helper.OpenstackHelper()
def get_objects(item):
items=None
if item == 'stacks':
items=oh.get_stacks()
if item == 'volumes':
items=oh.get_volumes()
if item == 'snapshots':
items=oh.get_snapshots()
return items
def dep_graph(volumes,snapshots,snapshot_id=None):
vol_list=[]
for volume in volumes:
if volume.snapshot_id == snapshot_id:
info={'id':volume.id,'snapshot_id':volume.snapshot_id,'snapshots':[],
}
vol_list.append(info)
for snapshot in snapshots:
for volume in vol_list:
snap_list=[]
if snapshot.volume_id == volume['id']:
info={'id':snapshot.id, 'volumes':[]}
info['volumes'].extend(dep_graph(volumes,snapshots,snapshot.id))
volume['snapshots'].append(info)
return vol_list
if __name__ == '__main__':
volumes = get_objects('volumes')
snapshots = get_objects('snapshots')
output = dep_graph(volumes, snapshots)
print output
Here is the solution I propose:
def remove_id(data):
if isinstance(data, List):
return [remove_id(sub_data) for sub_data in data]
if isinstance(data, Dict):
return {key: remove_id(value) for key, value in data.items()
if key != 'id'}
return data
And the result:
{'snapshot_id': None,
'snapshots': [{'volumes': [{'snapshot_id': '3ddc7ddd-02ca-4669-a0cb-fb0d56a4a6f5',
'snapshots': []},
{'snapshot_id': '3ddc7ddd-02ca-4669-a0cb-fb0d56a4a6f5',
'snapshots': [{'volumes': [{'snapshot_id': 'd637f6ea-4a41-448c-874f-ffe624ddc597',
'snapshots': []}]}]},
{'snapshot_id': '3ddc7ddd-02ca-4669-a0cb-fb0d56a4a6f5',
'snapshots': []}]}]}