Develop Reference

Turn dict with duplicate keys into list containing these keys

I receive a response I have no control over from an API. Using requests response.json() will filter out duplicate keys. So I would need to turn this response into a list where each key is an element in that list: What I get now:
{
"user": {
//...
},
"user": {
//...
},
//...
}
What I need:
{
"users": [
{
"user": {
//...
}
},
{
"user": {
//...
}
},
//...
]
}
This way JSON won't filter out any of the results, and I can loop through users.

Okay, let me have a try by method used in Python json parser allow duplicate keys
All we should do is handle the pairs_list by ourself.
from json import JSONDecoder
def parse_object_pairs(pairs):
return pairs
data = """
{"foo": {"key": 2, "key": 3}, "foo": 4, "foo": 23}
"""
decoder = JSONDecoder(object_pairs_hook=parse_object_pairs)
pairs_list = decoder.decode(data)
# the pairs_list is the real thing which we can use
aggre_key = 's'
def recusive_handle(pairs_list):
dct = {}
for k, v in pairs_list:
if v and isinstance(v, list) and isinstance(v[0], tuple):
v = recusive_handle(v)
if k + aggre_key in dct:
dct[k + aggre_key].append({k: v})
elif k in dct:
first_dict = {k: dct.pop(k)}
dct[k + aggre_key] = [first_dict, {k: v}]
else:
dct[k] = v
return dct
print(recusive_handle(pairs_list))
output:
{'foos': [{'foo': {'keys': [{'key': 2}, {'key': 3}]}}, {'foo': {'bar': 4}}, {'foo': 23}]}

Get key value from nested dict python

I have class with a nested dictionary data object. I need to get all the key values from it. What's the best efficient way to do this?
I'm stuck with following:
for k,v in data.items():
print v.keys()
This is the data:
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"OFFLINE": {"online": None, "offline_few": "ALERT"},
}

An elegant way to concatenate lists (your value.keys() lists) into one is using a double-loop list comprehenstion, like this:
nested_keys = [
key
for val in data.values()
for key in val.keys()]

Using a generator:
def all_keys(d):
for k, v in d.items():
yield k
# assume that anything with `items` property will be a mapping
# this can be replaced with: `isinstance(v, dict)` or `isinstance(v, collections.Mapping)`
if hasattr(v, 'items'):
yield from all_keys(v)
On your input this produces:
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"OFFLINE": {"online": None, "offline_few": "ALERT"},
}
print(list(all_keys(data)))
# ['BANK', 'no_data', 'SHOCK', 'drop', 'rise', 'high_risk', 'OFFLINE', 'online', 'offline_few']

If all your "actual" key-value pairs are at a certain depth, for example for depth 1, you can go like:
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"OFFLINE": {"online": None, "offline_few": "ALERT"},
}
dic = {k:v for val in data.values() for k,v in val.items()}
But if you dont know that:
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"online": None,
"offline_few": "ALERT"
}
In this case you need to use recursion:
def unnest(dic, final=dict()):
for key, val in dic.items():
if not isinstance(val, dict):
final[key] = val
else:
dic2 = dict()
for k, v in val.items():
dic2[k] = v
unnest(dic2, final)
return final
dic = unnest(data, {}) #every use of the function should have {} to solve issue pointed by #Ch3steR
In any case, once you have the "un-nested" dictionary it is trivial to print out the keys:
print(dic.keys())

Recursive Function
Gets all keys at all levels of nested dictionary
def get_keys(d, result = None):
# use default of None to fix issue noted by #Ch3steR
# namely: http://effbot.org/zone/default-values.htm
if result is None:
result = []
for k, v in d.items():
if isinstance(v, dict):
result.append(k)
get_keys(v, result)
else:
result.append(k)
return result
Test
print(get_keys(data))
Output
['BANK', 'no_data', 'SHOCK', 'drop', 'rise', 'high_risk', 'OFFLINE', 'online', 'offline_few']

You could use a NestedDict. First install ndicts
pip install ndicts
Then
from ndicts.ndicts import NestedDict
data = {
"BANK": {
"no_data": "INT",
},
"SHOCK": {
"drop": "NOTI",
"rise": "NOTI",
"high_risk": "ALERT",
},
"OFFLINE": {"online": None, "offline_few": "ALERT"},
}
nd = NestedDict(data)
The result
>>> list(nd.keys())
[('BANK', 'no_data'), ('SHOCK', 'drop'), ('SHOCK', 'rise'), ('SHOCK', 'high_risk'), ('OFFLINE', 'online'), ('OFFLINE', 'offline_few')]

Update key without affecting the key's values within a nested dictionary

I am trying to update a key while retaining its values within a nested dictionaries.
While I have found a method to do so, I had to create new dictionaries in order to cater for it. As such, wondering if there anyone could provide me with a better insight on the approach I have taken?
init_dict = {
'pageA' : {
0 : {
'menuA' : [
'a01',
'a02'
]
}
},
'pageB' : {
1 : {
'menuB' : [
'b10'
]
}
}
}
changed = {'pageB' : 0, 'pageA' : 1}
condense_dict = {}
for k, v in init_dict.items():
for i in v.keys():
condense_dict[k] = init_dict[k][i]
new_dict = {}
for i in condense_dict:
new_dict[i] = {}
new_dict[i][changed.get(i)] = condense_dict.get(i)
My expected output is as follows:
{
'pageA' : {
1 : {
'menuA' : [
'a01',
'a02'
]
}
},
'pageB' : {
0 : {
'menuB' : [
'b10'
]
}
}
}

You can pop the presumably only key from the sub-dict and assign it to the new key for each entry in changed:
for k, v in changed.items():
init_dict[k][v] = init_dict[k].pop(next(iter(init_dict[k])))
init_dict becomes:
{'pageA': {1: {'menuA': ['a01', 'a02']}}, 'pageB': {0: {'menuB': ['b10']}}}

Using the .pop() method this can be done similar to this (although I'm sure you could rewrite it better)
init_dict = {
'pageA': {
0: {
'menuA' : [
'a01',
'a02'
]
}
},
'pageB': {
1: {
'menuB': [
'b10'
]
}
}
}
print(init_dict)
thing = init_dict.pop('pageA')
sub_thing = thing.pop(0)
redone = {1: sub_thing}
init_dict.update({'pageA': redone})
print(init_dict)
{'pageA': {0: {'menuA': ['a01', 'a02']}}, 'pageB': {1: {'menuB': ['b10']}}}
{'pageA': {1: {'menuA': ['a01', 'a02']}}, 'pageB': {1: {'menuB': ['b10']}}}
You can see it's the same data as we start with, but we changed 0 to 1

Here I use .pop() and change it inplace. With the same init_dict as you:
change_to = {1: 0, 0: 1}
for k, v in init_dict.items():
for old_key in v.keys():
if old_key in change_to:
v[change_to[old_key]] = v.pop(old_key)

json.load agony to create dictionary instead of list

I have a python script which grabs data from a site in json:
channels_json = json.loads(url)
the site returns data as follows:
[ { '1': 'http://ht.co/bbda24210d7bgfbbbbcdfc2a023f' },
{ '2': 'http://ht.co/bbd10d7937932965369c248f7ccdfc2a023f' },
{ '3': 'http://ht.co/d3a01f6e5e74eb2cb5840556d80a52adf2871d' },
{ '4': 'http://ht.co/56d3a01f6e5e72cb5840556d80a52adf2871d' },
{ '5': 'http://ht.co/9ed0bb4cc447b99c9ce609916ccf931f16a' },
{ '6': 'http://ht.co/9ed0bb4cc44bb99c9ce609916ccf931f16a' },
....]
The problem is that Python is making this into a list rather than a dictionary. So I can't reference '4' like this:
print (channels_json["4"])
and get the response:
http://ht.co/56d3a01f6e5e72cb5840556d80a52adf2871d
Instead Python spits out:
TypeError: list indices must be integers, not str
If I run this code:
for c in channels_json:
print c
Python prints out each set of coupled data like this:
{u'1': u'http://ht.co/bbda24210d7bgfbbbbcdfc2a023f' },
{ u'2': u'http://ht.co/bbd10d7937932965369c248f7ccdfc2a023f' },
{ u'3': u'http://ht.co/d3a01f6e5e74eb2cb5840556d80a52adf2871d' },
{ u'4': u'http://ht.co/56d3a01f6e5e72cb5840556d80a52adf2871d' },
{ u'5': u'http://ht.co/9ed0bb4cc447b99c9ce609916ccf931f16a' },
{ u'6': u'http://ht.co/9ed0bb4cc44bb99c9ce609916ccf931f16a' },
How can I get the above into a dictionary so I can reference value '6' as a string and get back
http://ht.co/9ed0bb4cc44bb99c9ce609916ccf931f16a

You can create the dictionary you want by iterating over the dictionaries in the list returned from json.load() as shown below:
#json_data = json.loads(url)
json_data = [{ '1': 'http://ht.co/bbda24210d7bgfbbbbcdfc2a023f' },
{ '2': 'http://ht.co/bbd10d7937932965369c248f7ccdfc2a023f' },
{ '3': 'http://ht.co/d3a01f6e5e74eb2cb5840556d80a52adf2871d' },
{ '4': 'http://ht.co/56d3a01f6e5e72cb5840556d80a52adf2871d' },
{ '5': 'http://ht.co/9ed0bb4cc447b99c9ce609916ccf931f16a' },
{ '6': 'http://ht.co/9ed0bb4cc44bb99c9ce609916ccf931f16a' },]
# Convert to single dictionary.
channels_json = dict(d.popitem() for d in json_data)
print(json.dumps(channels_json, indent=4)) # Pretty-print result.
Output:
{
"1": "http://ht.co/bbda24210d7bgfbbbbcdfc2a023f",
"2": "http://ht.co/bbd10d7937932965369c248f7ccdfc2a023f",
"3": "http://ht.co/d3a01f6e5e74eb2cb5840556d80a52adf2871d",
"4": "http://ht.co/56d3a01f6e5e72cb5840556d80a52adf2871d",
"5": "http://ht.co/9ed0bb4cc447b99c9ce609916ccf931f16a",
"6": "http://ht.co/9ed0bb4cc44bb99c9ce609916ccf931f16a"
}

dd = {}
for d in c:
for key, value in d.items():
dd[key] = value

You can just iterate through the array and build your dictionary
channels_json = {}
channels_array = json.loads(url)
for d in channels_array:
key = list(d.keys())[0]
val = d[key]
channels_json[key] = val
Now you should be able to reference your dictionary channels_json

More pythonic way of doing this.
channels = {}
for i in json.loads(url): channels.update(i)
or
channels = {}
[channels.update(i) for i in json.loads(url)]
In both cases the dictionary gets updated with your list of separate dictionaries.

aggregating values in dictionary

I have a deep dictionary like this:
myDict = { '123456': {
'348adbd39r' : {
'LONDON': {
'c_name': 'abc',
'acct': '84720'
},
'PARIS': {
'c_name': 'xyz',
'acct': '73642'
}
},
'2862aef3' : {
'NYC': {
'c_name': 'hhdls3',
'acct': '92742'
}
},
'82gfg24' : {
'NYC': {
'c_name': 'hquer',
'acct': '34567'
},
'PARIS': {
'c_name': 'ljad',
'acct': '93742'
}
}
}
I want to 'aggregate' it based on the city names. The output should look like below:
outDict = {
'LONDON': {
'c_name': ['abc'],
'acct': ['84720']
},
'PARIS': {
'c_name': ['xyz', 'ljad'],
'acct': ['73642', '93742']
},
'NYC': {
'c_name': ['hhdls3', 'hquer'],
'acct': ['73642', '34567']
}
}
This is what I did:
cust_fields = ['c_name', 'acct']
field_dict = {field: [] for field in cust_fields}
aggregated_dict = {}
city_names = ['LONDON', 'PARIS', 'NYC']
for city in city_names:
aggregated_dict[city] = field_dict
for id, an_dict in myDict.iteritems():
for alphaNum, city_dict in an_dict.iteritems():
for city, acct_dict in city_dict.iteritems():
for field, val in acct_dict.iteritems():
aggregated_dict[city][field].append(val)
But, the above is updating the field-values for all the cities...rather than just the particular city it is working on. Not sure where the logic is wrong. Any help is appreciated (either correct where my mistake is or any new logic...).
Thanks!

The problem you have is you are assigning field_dict to the value of aggregated_dict[city] in your loop over city_names, which is simply assigning the same dictionary to each city. And when you update any reference (for any city) all references are updated.
An easy fix for this is to change
for city in city_names:
aggregated_dict[city] = field_dict
To:
for city in city_names:
aggregated_dict[city] = {field: [] for field in cust_fields}
I would also look at collections.defaultdict for this type of aggregation.
from collections import defaultdict
collected = defaultdict(lambda: defaultdict(list))
for _, city_records in myDict['123456'].items():
for city_name, records in city_records.items():
for record_name, record_value in records.items():
collected[city_name][record_name].append(record_value)
for city_name, records in collected.items():
print city_name
print dict(records)

for key,val in myDict.items():
for key1,val1 in val.items():
for key2,val2 in val1.items():
d = final_dict[key2] if key2 in final_dict else defaultdict(list)
for k,v in val2.items():
d[k].append(v)
final_dict[key2] = d

You can use recursion:
from collections import defaultdict
d1 = defaultdict(dict)
def aggregate(d):
for a, b in d.items():
if a in ['LONDON', 'PARIS', 'NYC']:
global d1
if a not in d1:
d1[a] = {}
d1[a]['c_name'] = [b['c_name']]
d1[a]['acct'] = [b['acct']]
else:
d1[a]['c_name'].append([b['c_name']])
d1[a]['acct'].append(b['acct'])
else:
aggregate(b)
aggregate(myDict)
print(dict(d1))
Output:
{'PARIS': {'acct': ['73642', '93742'], 'c_name': ['xyz', ['ljad']]}, 'NYC': {'acct': ['92742', '34567'], 'c_name': ['hhdls3', ['hquer']]}, 'LONDON': {'acct': ['84720'], 'c_name': ['abc']}}