For some post-processing, I need to flatten a structure like this
{'foo': {
'cat': {'name': 'Hodor', 'age': 7},
'dog': {'name': 'Mordor', 'age': 5}},
'bar': { 'rat': {'name': 'Izidor', 'age': 3}}
}
into this dataset:
[{'foobar': 'foo', 'animal': 'dog', 'name': 'Mordor', 'age': 5},
{'foobar': 'foo', 'animal': 'cat', 'name': 'Hodor', 'age': 7},
{'foobar': 'bar', 'animal': 'rat', 'name': 'Izidor', 'age': 3}]
So I wrote this function:
def flatten(data, primary_keys):
out = []
keys = copy.copy(primary_keys)
keys.reverse()
def visit(node, primary_values, prim):
if len(prim):
p = prim.pop()
for key, child in node.iteritems():
primary_values[p] = key
visit(child, primary_values, copy.copy(prim))
else:
new = copy.copy(node)
new.update(primary_values)
out.append(new)
visit(data, { }, keys)
return out
out = flatten(a, ['foo', 'bar'])
I was not really satisfied because I have to use copy.copy to protect my inputs. Obviously, when using flatten one does not want the inputs be altered.
Then I thought about one alternative that uses more global variables (at least global to flatten) and uses an index instead of directly passing primary_keys to visit. However, this does not really help me to get rid of the ugly initial copy:
keys = copy.copy(primary_keys)
keys.reverse()
So here is my final version:
def flatten(data, keys):
data = copy.copy(data)
keys = copy.copy(keys)
keys.reverse()
out = []
values = {}
def visit(node, id):
if id:
id -= 1
for key, child in node.iteritems():
values[keys[id]] = key
visit(child, id)
else:
node.update(values)
out.append(node)
visit(data, len(keys))
return out
Is there a better implementation (that can avoid the use of copy.copy)?
Edit: modified to account for variable dictionary depth.
By using the merge function from my previous answer (below), you can avoid calling update which modifies the caller. There is then no need to copy the dictionary first.
def flatten(data, keys):
out = []
values = {}
def visit(node, id):
if id:
id -= 1
for key, child in node.items():
values[keys[id]] = key
visit(child, id)
else:
out.append(merge(node, values)) # use merge instead of update
visit(data, len(keys))
return out
One thing I don't understand is why you need to protect the keys input. I don't see them being modified anywhere.
Previous answer
How about list comprehension?
def merge(d1, d2):
return dict(list(d1.items()) + list(d2.items()))
[[merge({'foobar': key, 'animal': sub_key}, sub_sub_dict)
for sub_key, sub_sub_dict in sub_dict.items()]
for key, sub_dict in a.items()]
The tricky part was merging the dictionaries without using update (which returns None).
Related
I'm scraping a website, which returns a dictionary:
person = {'name0':{'first0': 'John', 'last0':'Smith'},
'age0':'10',
'location0':{'city0':'Dublin'}
}
I'm trying to write a function that will return a dictionary {'name':'John', 'age':'10'} when passed the above dictionary.
I want to ideally put a try:... except KeyError around each item since sometimes keys will be missing.
def func(person):
filters = [('age', 'age0'), ('name', ['name0', 'first0'])]
result = {'name': None, 'age': None}
for i in filters:
try:
result[i[0]] = person[i[1]]
except KeyError:
pass
return result
The problem is result[i[0]] = person[i[1]] doesn't work for 'name' since there's two keys that need to be followed sequentially and I don't know how to do that.
I want some way of telling it (in the loop) to go to person['name0']['first0'] (and so on to whatever depth the thing I want is).
I have lots of things to extract, so I'd rather do it in a loop instead of a try..except statement for each variable individually.
In order to follow several key sequentially, you can use get and set the default value to {} (empty dictionary) for the upper levels. Set the default value to None (or whatever suits you) for the last level:
def func(person):
return {'name': person.get('name0', {}).get('first0', None),
'age': person.get('age0', None)}
Best I could manage was using a for loop to iterate through the keys:
person = {'name0':{'first0': 'John', 'last0':'Smith'},
'age0':'10',
'location0':{'city0':'Dublin'}
}
Additionally I used .get(key) rather than try..except as suggested by #wiwi
def func(person):
filters = [('age', ['age0']), ('name', ['name0', 'first0'])]
result = {'name': None, 'age': None}
for filter in filters:
temp = person.copy()
for key in filter[1]:
temp = temp.get(key)
if not temp: # NoneType doesn't have .get method
break
result[filter[0]] = temp
return result
func(person) then returns {'name': 'John', 'age': '10'}.
It handles missing input too:
person2 = {'age0':'10',
'location0':{'city0':'Dublin'}}
func(person2) returns {'name': None, 'age': '10'}
You can put the try...except in another loop, if there's a list of keys instead of a single key:
def getNestedVal(obj, kPath:list, defaultVal=None):
if isinstance(kPath, str) or not hasattr(kPath, '__iter__'):
kPath = [kPath] ## if not iterable, wrap as list
for k in kPath:
try: obj = obj[k]
except: return defaultVal
return obj
def func(person):
filters = [('age', 'age0'), ('name', ['name0', 'first0']),#]
('gender', ['gender0'], 'N/A')] # includes default value
return {k[0]: getNestedVal(person, *k[1:3]) for k in filters}
[I added gender just to demonstrate how defaults can also be specified for missing values.]
With this, func(person) should return
{'age': '10', 'name': 'John', 'gender': 'N/A'}
I also have a flattenObj function, a version of which is defined below:
def flattenDict(orig:dict, kList=[], kSep='_', stripNum=True):
if not isinstance(orig, dict): return [(kList, orig)]
tList = []
for k, v in orig.items():
if isinstance(k, str) and stripNum: k = k.strip('0123456789')
tList += flattenDict(v, kList+[str(k)], None)
if not isinstance(kSep, str): return tList
return {kSep.join(kl): v for kl,v in tList}
[I added stripNum just to get rid of the 0s in your keys...]
flattenDict(person) should return
{'name_first': 'John', 'name_last': 'Smith', 'age': '10', 'location_city': 'Dublin'}
I am trying to serialize a list of nested objects as scalar values by taking only one field from the nested item. Instead of [{key: value}, ...] I want to receive [value1, value2, ...].
Code:
from marshmallow import *
class MySchema(Schema):
key = fields.String(required=True)
class ParentSchema(Schema):
items = fields.Nested(MySchema, only='key', many=True)
Given the above schemas, I want to serialize some data:
>>> data = {'items': [{'key': 1}, {'key': 2}, {'key': 3}]}
>>> result, errors = ParentSchema().dump(data)
>>> result
{'items': ['1', '2', '3']}
This works as expected, giving me the list of scalar values. However, when trying to deserialize the data using the models above, the data is suddenly invalid:
>>> data, errors = ParentSchema().load(result)
>>> data
{'items': [{}, {}, {}]}
>>> errors
{'items': {0: {}, '_schema': ['Invalid input type.', 'Invalid input type.', 'Invalid input type.'], 1: {}, 2: {}}}
Is there any configuration option I am missing or is this simply not possible?
For anyone stumbling across the same issue, this is the workaround I am using currently:
class MySchema(Schema):
key = fields.String(required=True)
def load(self, data, *args):
data = [
{'key': item} if isinstance(item, str) else item
for item in data
]
return super().load(data, *args)
class ParentSchema(Schema):
items = fields.Nested(MySchema, only='key', many=True)
I have a data set which follows the structure of the following example:
exampleset = {
'body' : {
'abdomen' : [{
'arms' : {
'value' : 2,
}
},{
'legs': {
'value' : 2,
}
}],
'hands' : {
'fingers' : {
'value' : 5,
}
},
}
}
I am trying to reverse this so I get something like:
{'value': {'value1': {5: {'fingers': {'hands': {'body': {}}}}},
'value2': {2: {'legs': {'abdomen': {'body': {}}}}},
'value3': {2: {'arms': {'abdomen': {'body': {}}}}}},
}
(I hope I got the bracket matching right, but you get the idea.)
I am using a couple of recursion functions to do this, like so:
def recurse_find(data, values, count):
global conf
for key in data:
for v in conf['value_names']:
if key == v:
values[v+str(count)] = {}
values[v+str(count)][data[key]] = {}
count += 1
# originally just using this line:
# values[data[key]] = {}
if type(data[key]) is list:
for i in data[key]:
if type(i) is dict:
values = recurse_find(i, values, count)
values = add_new_level(values, key)
elif type(data[key]) is dict:
values = recurse_find(data[key], values, count)
values = add_new_level(values, key)
return values
def add_new_level(data, new_key):
for key in data:
if data[key] == {}:
data[key][new_key] = {}
else:
data[key] = add_new_level(data[key], new_key)
return data
conf = { "value_names": ["value"] }
for value in conf['value_names']:
values[value] = recurse_find(exampleset, {}, 1)
print(values)
At the moment I only get one value returned correctly, obviously I would like them all. Originally I didn't label the values (value1, value2 etc), but when doing this example set I realised that of course if the values are the same I'll only get one! If I remove the value name keys it finds all the values (unless duplicate) but still doesn't return the correct levels as it includes some of the others while it loops round. I don't care about the order of the values, just that they are labelled differently so I don't miss out any.
Current result:
{'value': {'value1': {5: {'fingers': {'hands': {'body': {}}}}}}}
I think that the solution is the inclusion of a pretty simple step, but I can't see it at the moment and I've already spent too long looking at this.
Any help appreciated.
EDIT:
I've gotten a little further by changing my recursive function to make count a global variable and having count=1 outside the function which has sorted out the getting all the values problem.
I have narrowed down the addition of extra keys to the add_new_level function, but haven't yet figured out how to change it.
Output:
{'value': {'value1': {2: {'arms': {'abdomen': {'legs': {'abdomen': {'fingers': {'hands': {'body': {}}}}}}}}},
'value2': {2: {'legs': {'abdomen': {'fingers': {'hands': {'body': {}}}}}}},
'value3': {5: {'fingers': {'hands': {'body': {}}}}}}}
I have adjusted your output type slightly to make the dictionary containing 'value1' 'value2' etc... to an array. I believe this is better because the order of these will be lost anyway unless an OrderedDict (from collections package) is used and in any case an array will translate quite easily from index 0,1,2,3.. to val1, val2, val3, etc...
res = {'value': []}
def revnest(inp, keys=[]):
res2 = res['value']
if type(inp) == list:
inp = {i:j[i] for j in inp for i in j}
for x in inp:
if x == 'value':
res2.append({inp[x]:{}})
res2 = res2[-1][inp[x]]
for y in keys[::-1]:
res2[y] = {}
res2 = res2[y]
else:
revnest(inp[x], keys+[x])
revnest(exampleset)
print res
which given your exampleset, prints:
{'value': [{2: {'legs': {'abdomen': {'body': {}}}}}, {2: {'arms': {'abdomen': {'body': {}}}}}, {5: {'fingers': {'hands': {'body': {}}}}}]}
I have a bit of a complex question that I can't seem to get to the bottom of. I have a list of keys corresponding to a position in a Python dictionary. I would like to be able to dynamically change the value at the position (found by the keys in the list).
For example:
listOfKeys = ['car', 'ford', 'mustang']
I also have a dictionary:
DictOfVehiclePrices = {'car':
{'ford':
{'mustang': 'expensive',
'other': 'cheap'},
'toyota':
{'big': 'moderate',
'small': 'cheap'}
},
'truck':
{'big': 'expensive',
'small': 'moderate'}
}
Via my list, how could I dynamically change the value of DictOfVehiclePrices['car']['ford']['mustang']?
In my actual problem, I need to follow the list of keys through the dictionary and change the value at the end position. How can this be done dynamically (with loops, etc.)?
Thank you for your help! :)
Use reduce and operator.getitem:
>>> from operator import getitem
>>> lis = ['car', 'ford', 'mustang']
Update value:
>>> reduce(getitem, lis[:-1], DictOfVehiclePrices)[lis[-1]] = 'cheap'
Fetch value:
>>> reduce(getitem, lis, DictOfVehiclePrices)
'cheap'
Note that in Python 3 reduce has been moved to functools module.
A very simple approach would be:
DictOfVehiclePrices[listOfKeys[0]][listOfKeys[1]][listOfKeys[2]] = 'new value'
print reduce(lambda x, y: x[y], listOfKeys, dictOfVehiclePrices)
Output
expensive
In order to change the values,
result = dictOfVehiclePrices
for key in listOfKeys[:-1]:
result = result[key]
result[listOfKeys[-1]] = "cheap"
print dictOfVehiclePrices
Output
{'car': {'toyota': {'small': 'cheap', 'big': 'moderate'},
'ford': {'mustang': 'cheap', 'other': 'cheap'}},
'truck': {'small': 'moderate', 'big': 'expensive'}}
You have a great solution here by #Joel Cornett.
based on Joel method you can use it like this:
def set_value(dict_nested, address_list):
cur = dict_nested
for path_item in address_list[:-2]:
try:
cur = cur[path_item]
except KeyError:
cur = cur[path_item] = {}
cur[address_list[-2]] = address_list[-1]
DictOfVehiclePrices = {'car':
{'ford':
{'mustang': 'expensive',
'other': 'cheap'},
'toyota':
{'big': 'moderate',
'small': 'cheap'}
},
'truck':
{'big': 'expensive',
'small': 'moderate'}
}
set_value(DictOfVehiclePrices,['car', 'ford', 'mustang', 'a'])
print DictOfVehiclePrices
STDOUT:
{'car': {'toyota': {'small': 'cheap', 'big': 'moderate'}, 'ford':
{'mustang': 'a', 'other': 'cheap'}}, 'truck': {'small': 'moderate',
'big': 'expensive'}}
def update_dict(parent, data, value):
'''
To update the value in the data if the data
is a nested dictionary
:param parent: list of parents
:param data: data dict in which value to be updated
:param value: Value to be updated in data dict
:return:
'''
if parent:
if isinstance(data[parent[0]], dict):
update_dict(parent[1:], data[parent[0]], value)
else:
data[parent[0]] = value
parent = ["test", "address", "area", "street", "locality", "country"]
data = {
"first_name": "ttcLoReSaa",
"test": {
"address": {
"area": {
"street": {
"locality": {
"country": "india"
}
}
}
}
}
}
update_dict(parent, data, "IN")
Here is a recursive function to update a nested dict based on a list of keys:
1.Trigger the update dict function with the required params
2.The function will iterate the list of keys, and retrieves the value from the dict.
3.If the retrieved value is dict, it pops the key from the list and also it updates the dict with the value of the key.
4.Sends the updated dict and list of keys to the same function recursively.
5.When the list gets empty, it means that we have reached the desired the key, where we need to apply our replacement. So if the list is empty, the funtion replaces the dict[key] with the value
Is there a way in Python to serialize a dictionary that is using a tuple as key?
e.g.
a = {(1, 2): 'a'}
simply using json.dumps(a) raises this error:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python2.6/json/__init__.py", line 230, in dumps
return _default_encoder.encode(obj)
File "/usr/lib/python2.6/json/encoder.py", line 367, in encode
chunks = list(self.iterencode(o))
File "/usr/lib/python2.6/json/encoder.py", line 309, in _iterencode
for chunk in self._iterencode_dict(o, markers):
File "/usr/lib/python2.6/json/encoder.py", line 268, in _iterencode_dict
raise TypeError("key {0!r} is not a string".format(key))
TypeError: key (1, 2) is not a string
You can't serialize that as json, json has a much less flexible idea about what counts as a dict key than python.
You could transform the mapping into a sequence of key, value pairs, something like this:
import json
def remap_keys(mapping):
return [{'key':k, 'value': v} for k, v in mapping.iteritems()]
...
json.dumps(remap_keys({(1, 2): 'foo'}))
>>> '[{"value": "foo", "key": [1, 2]}]'
from json import loads, dumps
from ast import literal_eval
x = {(0, 1): 'la-la la', (0, 2): 'extricate'}
# save: convert each tuple key to a string before saving as json object
s = dumps({str(k): v for k, v in x.items()})
# load in two stages:
# (i) load json object
obj = loads(s)
# (ii) convert loaded keys from string back to tuple
d = {literal_eval(k): v for k, v in obj.items()}
See https://stackoverflow.com/a/12337657/2455413.
JSON only supports strings as keys. You'll need to choose a way to represent those tuples as strings.
You could just use str((1,2)) as key because json only expects the keys as strings but if you use this you'll have to use a[str((1,2))] to get the value.
json can only accept strings as keys for dict,
what you can do, is to replace the tuple keys with string like so
with open("file", "w") as f:
k = dic.keys()
v = dic.values()
k1 = [str(i) for i in k]
json.dump(json.dumps(dict(zip(*[k1,v]))),f)
And than when you want to read it, you can change the keys back to tuples using
with open("file", r) as f:
data = json.load(f)
dic = json.loads(data)
k = dic.keys()
v = dic.values()
k1 = [eval(i) for i in k]
return dict(zip(*[k1,v]))
This solution:
Avoids the security risk of eval().
Is short.
Is copy-pastable as save and load functions.
Keeps the structure of tuple as the key, in case you are editing the JSON by hand.
Adds ugly \" to the tuple representation, which is worse than the other str()/eval() methods here.
Can only handle tuples as keys at the first level for nested dicts (as of this writing no other solution here can do better)
def json_dumps_tuple_keys(mapping):
string_keys = {json.dumps(k): v for k, v in mapping.items()}
return json.dumps(string_keys)
def json_loads_tuple_keys(string):
mapping = json.loads(string)
return {tuple(json.loads(k)): v for k, v in mapping.items()}
m = {(0,"a"): "first", (1, "b"): [9, 8, 7]}
print(m) # {(0, 'a'): 'first', (1, 'b'): [9, 8, 7]}
s = json_dumps_tuple_keys(m)
print(s) # {"[0, \"a\"]": "first", "[1, \"b\"]": [9, 8, 7]}
m2 = json_loads_tuple_keys(s)
print(m2) # {(0, 'a'): 'first', (1, 'b'): [9, 8, 7]}
print(m==m2) # True
Here is one way to do it. It will require the key to be json decoded after the main dictionary is decoded and the whole dictionary re-sequenced, but it is doable:
import json
def jsonEncodeTupleKeyDict(data):
ndict = dict()
# creates new dictionary with the original tuple converted to json string
for key,value in data.iteritems():
nkey = json.dumps(key)
ndict[nkey] = value
# now encode the new dictionary and return that
return json.dumps(ndict)
def main():
tdict = dict()
for i in range(10):
key = (i,"data",5*i)
tdict[key] = i*i
try:
print json.dumps(tdict)
except TypeError,e:
print "JSON Encode Failed!",e
print jsonEncodeTupleKeyDict(tdict)
if __name__ == '__main__':
main()
I make no claim to any efficiency of this method. I needed this for saving some joystick mapping data to a file. I wanted to use something that would create a semi-human readable format so it could be edited if needed.
You can actually not serialize tuples as key to json, but you can convert the tuple to a string and recover it, after you have deserialized the file.
with_tuple = {(0.1, 0.1): 3.14} ## this will work in python but is not serializable in json
{(0.1, 0.1): 3.14}
But you cannot serialize it with json. However, you can use
with_string = {str((0.1, 0.1))[1:-1]: 3.14} ## the expression [1,-1] removes the parenthesis surrounding the tuples in python.
{'0.1, 0.1': 3.14} # This is serializable
With a bit of cheating, you will recover the original tuple (after having deserialized the whole file) by treating each key (as str) separately
tuple(json.loads("["+'0.1, 0.1'+"]")) ## will recover the tuple from string
(0.1, 0.1)
It is a bit of overload to convert a string to a tuple using json.loads, but it will work. Encapsulate it and you are done.
Peace out and happy coding!
Nicolas
Here are two functions you could use to convert a dict_having_tuple_as_key into a json_array_having_key_and_value_as_keys and then de-convert it the way back
import json
def json_dumps_dict_having_tuple_as_key(dict_having_tuple_as_key):
if not isinstance(dict_having_tuple_as_key, dict):
raise Exception('Error using json_dumps_dict_having_tuple_as_key: The input variable is not a dictionary.')
list_of_dicts_having_key_and_value_as_keys = [{'key': k, 'value': v} for k, v in dict_having_tuple_as_key.items()]
json_array_having_key_and_value_as_keys = json.dumps(list_of_dicts_having_key_and_value_as_keys)
return json_array_having_key_and_value_as_keys
def json_loads_dictionary_split_into_key_and_value_as_keys_and_underwent_json_dumps(json_array_having_key_and_value_as_keys):
list_of_dicts_having_key_and_value_as_keys = json.loads(json_array_having_key_and_value_as_keys)
if not all(['key' in diz for diz in list_of_dicts_having_key_and_value_as_keys]) and all(['value' in diz for diz in list_of_dicts_having_key_and_value_as_keys]):
raise Exception('Error using json_loads_dictionary_split_into_key_and_value_as_keys_and_underwent_json_dumps: at least one dictionary in list_of_dicts_having_key_and_value_as_keys ismissing key "key" or key "value".')
dict_having_tuple_as_key = {}
for dict_having_key_and_value_as_keys in list_of_dicts_having_key_and_value_as_keys:
dict_having_tuple_as_key[ tuple(dict_having_key_and_value_as_keys['key']) ] = dict_having_key_and_value_as_keys['value']
return dict_having_tuple_as_key
usage example:
my_dict = {
('1', '1001', '2021-12-21', '1', '484'): {"name": "Carl", "surname": "Black", "score": 0},
('1', '1001', '2021-12-22', '1', '485'): {"name": "Joe", "id_number": 134, "percentage": 11}
}
my_json = json_dumps_dict_having_tuple_as_key(my_dict)
print(my_json)
[{'key': ['1', '1001', '2021-12-21', '1', '484'], 'value': {'name': 'Carl', 'surname': 'Black', 'score': 0}},
{'key': ['1', '1001', '2021-12-22', '1', '485'], 'value': {'name': 'Joe', 'id_number': 134, 'percentage': 11}}]
my_dict_reconverted = json_loads_dictionary_split_into_key_and_value_as_keys_and_underwent_json_dumps(my_json)
print(my_dict_reconverted)
{('1', '1001', '2021-12-21', '1', '484'): {'name': 'Carl', 'surname': 'Black', 'score': 0},
('1', '1001', '2021-12-22', '1', '485'): {'name': 'Joe', 'id_number': 134, 'percentage': 11}}
# proof of working 1
my_dict == my_dict_reconverted
True
# proof of working 2
my_dict == json_loads_dictionary_split_into_key_and_value_as_keys_and_underwent_json_dumps(
json_dumps_dict_having_tuple_as_key(my_dict)
)
True
(Using concepts expressed by #SingleNegationElimination to answer #Kvothe comment)
Here's a complete example to encode/decode nested dictionaries with tuple keys and values into/from json. tuple key will be a string in JSON.
values of types tuple or set will be converted to list
def JSdecoded(item:dict, dict_key=False):
if isinstance(item, list):
return [ JSdecoded(e) for e in item ]
elif isinstance(item, dict):
return { literal_eval(key) : value for key, value in item.items() }
return item
def JSencoded(item, dict_key=False):
if isinstance(item, tuple):
if dict_key:
return str(item)
else:
return list(item)
elif isinstance(item, list):
return [JSencoded(e) for e in item]
elif isinstance(item, dict):
return { JSencoded(key, True) : JSencoded(value) for key, value in item.items() }
elif isinstance(item, set):
return list(item)
return item
usage
import json
pydata = [
{ ('Apple','Green') : "Tree",
('Orange','Yellow'):"Orchard",
('John Doe', 1945) : "New York" }
]
jsstr= json.dumps(JSencoded(pydata), indent='\t')
print(jsstr)
#[
# {
# "('Apple', 'Green')": "Tree",
# "('Orange', 'Yellow')": "Orchard",
# "('John Doe', 1945)": "New York"
# }
#]
data = json.loads(jsstr) #string keys
newdata = JSdecoded(data) #tuple keys
print(newdata)
#[{('Apple', 'Green'): 'Tree', ('Orange', 'Yellow'): 'Orchard', ('John Doe', 1945): 'New York'}]
def stringify_keys(d):
if isinstance(d, dict):
return {str(k): stringify_keys(v) for k, v in d.items()}
if isinstance(d, (list, tuple)):
return type(d)(stringify_keys(v) for v in d)
return d
json.dumps(stringify_keys(mydict))