Develop Reference

Easy way to map values from list of list to dictionary

Inputs
I have two lists of lists.
rule_seq =
[['#1', '#2', '#3'],
['#1', '#2', '#3']]
KG_seq =
[['nationality', 'placeOfBirth', 'locatedIn'],
['nationality', 'hasFather', 'nationality']]
I have to map the values in the same index to the dictionary with the value of rule_seq as the key in the list of above.
My desired output is
Output
unify_dict =
{'#1': ['nationality'],
'#2': ['placeOfBirth', 'hasFather'],
'#3': ['locatedIn', 'nationality']}
I made a dictionary as follows by flattening and zipping both lists of lists to check whether keys and values are in the dictionary.
My code is as follows.
def create_unify_dict(rule_seq, KG_seq):
unify_dict = collections.defaultdict(list)
flat_aug_rule_list = list(itertools.chain.from_iterable(rule_seq))
flat_aug_KG_list = list(itertools.chain.from_iterable(KG_seq))
[unify_dict[key].append(val) for key, val in zip(flat_aug_rule_list, flat_aug_KG_list)
if key not in unify_dict.keys() or val not in unify_dict[key]]
return unify_dict
unify_dict = create_unify_dict(rule_seq, KG_seq)
Is there a simpler way to get the result I want?

You can just call append using the same defualtdict with second level of nesting.
from collections import defaultdict
result = defaultdict(list)
for keyList,valueList in zip(rule_seq, KG_seq):
for key,item in zip(keyList, valueList):
if item not in result[key]: result[key].append(item)
OUTPUT:
defaultdict(<class 'list'>,
{'#1': ['nationality'],
'#2': ['placeOfBirth', 'hasFather'],
'#3': ['locatedIn', 'nationality']})

You can use collections:
import collections
# Create a defaultdict with list as value type
result = collections.defaultdict(list)
for s0, s1 in zip(rule_seq, KG_seq):
for v0, v1 in zip(s0, s1):
if v1 not in result[v0]:
result[v0].append(v1)
print({k: v for k, v in result.items()})
# {
# '#1': ['nationality'],
# '#2': ['placeOfBirth', 'hasFather'],
# '#3': ['locatedIn', 'nationality'],
# }

This would be my homemade approach using no modules. Vanilla Python.
combine = [list(set(l)) for l in [[lst[i] for lst in KG_seq] for i in range(len(KG_seq[0]))]]
dct = {place:st for place,st in zip(rule_seq[0],combine)}
output
{'#1': ['nationality'], '#2': ['hasFather', 'placeOfBirth'], '#3': ['nationality', 'locatedIn']}
oversimplified version
combine = []
for i in range(len(KG_seq[0])):
group = []
for lst in KG_seq:
group.append(lst[i])
combine.append(group)
newComb = []
for simp in combine:
newComb.append(list(set(simp)))
dct = {}
for place,st in zip(rule_seq[0],combine):
dct[place] = st
print(dct)
undersimplified
dct = {place:st for place,st in zip(rule_seq[0],[list(set(l)) for l in [[lst[i] for lst in KG_seq] for i in range(len(KG_seq[0]))]])}

Based on the following assumptions there could be several forms to what your method look like
rule_seq and kg_seq are equal in length
rule_seq and kg_seq items are also equal in length
One liner
def one_liner(rule_seq, kg_seq):
ret = {}
[ret.update({idx: ret.get(idx, set()) | {val}}) for arr_idx, arr_val in zip(rule_seq, kg_seq) for idx, val in zip(arr_idx, arr_val)]
return ret
Single loop + one liner
def one_loop(rule_seq, kg_seq):
ret = {}
for arr_idx, arr_val in zip(rule_seq, kg_seq):
[ret.update({idx: ret.get(idx, set()) | {val}}) for idx, val in zip(arr_idx, arr_val)]
return ret
Nested loops
def nested_loop(rule_seq, kg_seq):
ret = {}
for arr_idx, arr_val in zip(rule_seq, kg_seq):
for idx, val in zip(arr_idx, arr_val):
ret[idx] = ret.get(idx, set()) | {val}
return ret
Testing these out
one_liner(rule_seq, KG_seq)
{'#1': {'nationality'},
'#2': {'hasFather', 'placeOfBirth'},
'#3': {'locatedIn', 'nationality'}}
one_loop(rule_seq, KG_seq)
{'#1': {'nationality'},
'#2': {'hasFather', 'placeOfBirth'},
'#3': {'locatedIn', 'nationality'}}
nested_loop(rule_seq, KG_seq)
{'#1': {'nationality'},
'#2': {'hasFather', 'placeOfBirth'},
'#3': {'locatedIn', 'nationality'}}

d = {}
for i in range(len(rule_seq)):
for j in range(len(rule_seq[i])):
rule, kg = rule_seq[i][j], KG_seq[i][j]
if (rule not in d.keys()):
d[rule] = [kg]
elif kg not in d[rule]:
d[rule].append(kg)
result:
{'#1': ['nationality'], '#2': ['placeOfBirth', 'hasFather'], '#3': ['locatedIn', 'nationality']}

Nested and escaped JSON payload to flattened dictionary - python

I'm looking for any suggestions to resolve an issue I'm facing. It might seem as a simple problem, but after a few days trying to find an answer - I think it is not anymore.
I'm receiving data (StringType) in a following JSON-like format, and there is a requirement to turn it into flat key-value pair dictionary. Here is a payload sample:
s = """{"status": "active", "name": "{\"first\": \"John\", \"last\": \"Smith\"}", "street_address": "100 \"Y\" Street"}"""
and the desired output should look like this:
{'status': 'active', 'name_first': 'John', 'name_last': 'Smith', 'street_address': '100 "Y" Street'}
The issue is I can't find a way to turn original string (s) into a dictionary. If I can achieve that the flattening part is working perfectly fine.
import json
import collections
import ast
#############################################################
# Flatten complex structure into a flat dictionary
#############################################################
def flatten_dictionary(dictionary, parent_key=False, separator='_', value_to_str=True):
"""
Turn a nested complex json into a flattened dictionary
:param dictionary: The dictionary to flatten
:param parent_key: The string to prepend to dictionary's keys
:param separator: The string used to separate flattened keys
:param value_to_str: Force all returned values to string type
:return: A flattened dictionary
"""
items = []
for key, value in dictionary.items():
new_key = str(parent_key) + separator + key if parent_key else key
try:
value = json.loads(value)
except BaseException:
value = value
if isinstance(value, collections.MutableMapping):
if not value.items():
items.append((new_key,None))
else:
items.extend(flatten_dictionary(value, new_key, separator).items())
elif isinstance(value, list):
if len(value):
for k, v in enumerate(value):
items.extend(flatten_dictionary({str(k): (str(v) if value_to_str else v)}, new_key).items())
else:
items.append((new_key,None))
else:
items.append((new_key, (str(value) if value_to_str else value)))
return dict(items)
# Data sample; sting and dictionary
s = """{"status": "active", "name": "{\"first\": \"John\", \"last\": \"Smith\"}", "street_address": "100 \"Y\" Street"}"""
d = {"status": "active", "name": "{\"first\": \"John\", \"last\": \"Smith\"}", "street_address": "100 \"Y\" Street"}
# Works for dictionary type
print(flatten_dictionary(d))
# Doesn't work for string type, for any of the below methods
e = eval(s)
# a = ast.literal_eval(s)
# j = json.loads(s)

Try:
import json
import re
def jsonify(s):
s = s.replace('"{','{').replace('}"','}')
s = re.sub(r'street_address":\s+"(.+)"(.+)"(.+)"', r'street_address": "\1\2\3"',s)
return json.loads(s)
If you must keep the quotes around Y, try:
def jsonify(s):
s = s.replace('"{','{').replace('}"','}')
search = re.search(r'street_address":\s+"(.+)"(.+)"(.+)"',s)
if search:
s = re.sub(r'street_address":\s+"(.+)"(.+)"(.+)"', r'street_address": "\1\2\3"',s)
dict_version = json.loads(s)
dict_version['street_address'] = dict_version['street_address'].replace(search.group(2),'"'+search.group(2)+'"')
return dict_version
A more generalized attempt:
def jsonify(s):
pattern = r'(?<=[,}])\s*"(.[^\{\}:,]+?)":\s+"([^\{\}:,]+?)"([^\{\}:,]+?)"([^\{\}:,]+?)"([,\}])'
s = s.replace('"{','{').replace('}"','}')
search = re.search(pattern,s)
matches = []
if search:
matches = re.findall(pattern,s)
s = re.sub(pattern, r'"\1": "\2\3\4"\5',s)
dict_version = json.loads(s)
for match in matches:
dict_version[match[0]] = dict_version[match[0]].replace(match[2],'"'+match[2]+'"')
return dict_version

parse empty string using json

I was wondering if there was a way to use json.loads in order to automatically convert an empty string in something else, such as None.
For example, given:
data = json.loads('{"foo":"5", "bar":""}')
I would like to have:
data = {"foo":"5", "bar":None}
Instead of:
data = {"foo":"5", "bar":""}

You can use a dictionary comprehension:
data = json.loads('{"foo":"5", "bar":""}')
res = {k: v if v != '' else None for k, v in data.items()}
{'foo': '5', 'bar': None}
This will only deal with the first level of a nested dictionary. You can use a recursive function to deal with the more generalised nested dictionary case:
def updater(d, inval, outval):
for k, v in d.items():
if isinstance(v, dict):
updater(d[k], inval, outval)
else:
if v == '':
d[k] = None
return d
data = json.loads('{"foo":"5", "bar":"", "nested": {"test": "", "test2": "5"}}')
res = updater(data, '', None)
{'foo': '5', 'bar': None,
'nested': {'test': None, 'test2': '5'}}

You can also accomplish this with the json.loads object_hook parameter. For example:
import json
import six
def empty_string2none(obj):
for k, v in six.iteritems(obj):
if v == '':
obj[k] = None
return obj
print(json.loads('{"foo":"5", "bar":"", "hello": {"world": ""}}',
object_hook=empty_string2none))
This will print
{'foo': '5', 'bar': None, 'hello': {'world': None}}
This way, you don't need additional recursion.

I did some trial and error and it is impossible to parse None into a String using json.loads() you will have to use json.loads() with json.dumps() like I do in this example:
import json
data = json.loads('{"foo":"5", "bar":"%r"}' %(None))
data2 = json.loads(json.dumps({'foo': 5, 'bar': None}))
if data2['bar'] is None:
print('worked')
print(data['bar'])
else:
print('did not work')

Overriding a value in a iteritems loop

I've created a function for iterating through a multiple level dictionary, and execute a second function ssocr that needs four arguments: coord, background, foreground, type (they are the value of my keys). This is my dictionary which is taken from a json file.
document json
def parse_image(self, d):
bg = d['background']
fg = d['foreground']
results = {}
for k, v in d['boxes'].iteritems():
if 'foreground' in d['boxes']:
myfg = d['boxes']['foreground']
else:
myfg = fg
if k != 'players_home' and k != 'players_opponent':
results[k] = MyAgonism.ssocr(v['coord'], bg, myfg, v['type'])
results['players_home'] = {}
for k, v in d['boxes']['players_home'].iteritems():
if 'foreground' in d['boxes']['players_home']:
myfg = d['boxes']['players_home']['foreground']
else:
myfg = fg
if k != 'background' and k != 'foreground':
for k2, v2 in d['boxes']['players_home'][k].iteritems():
if k2 != 'fouls':
results['players_home'][k] = {}
results['players_home'][k][k2] = MyAgonism.ssocr(v2['coord'], bg, myfg, v2['type'])
return results
In the last iteritems I get the correct value just for the name key. The score key doesn't appear. It looks like name override score in my results['players_home'] dictionary
output: ... "player4": {"name": 9}, "player5": {"name": 24} ...
I would like something like ... "player4": {"name": 9, "score": value}, "player5": {"name": 24, "score": value} ...
What am I doing wrong? Here is the full code just in case: Full Code

This might be the/a problem:
if k2 != 'fouls':
results['players_home'][k] = {}
In your loop, each time that k2 is not 'fouls', you create a new empty dict and store that in results['players_home']. That means that any entry previously stored there is no longer accessible.

Parsing json and searching through it

I have this code
import json
from pprint import pprint
json_data=open('bookmarks.json')
jdata = json.load(json_data)
pprint (jdata)
json_data.close()
How can I search through it for u'uri': u'http:?

ObjectPath is a library that provides ability to query JSON and nested structures of dicts and lists. For example, you can search for all attributes called "foo" regardless how deep they are by using $..foo.
While the documentation focuses on the command line interface, you can perform the queries programmatically by using the package's Python internals. The example below assumes you've already loaded the data into Python data structures (dicts & lists). If you're starting with a JSON file or string you just need to use load or loads from the json module first.
import objectpath
data = [
{'foo': 1, 'bar': 'a'},
{'foo': 2, 'bar': 'b'},
{'NoFooHere': 2, 'bar': 'c'},
{'foo': 3, 'bar': 'd'},
]
tree_obj = objectpath.Tree(data)
tuple(tree_obj.execute('$..foo'))
# returns: (1, 2, 3)
Notice that it just skipped elements that lacked a "foo" attribute, such as the third item in the list. You can also do much more complex queries, which makes ObjectPath handy for deeply nested structures (e.g. finding where x has y that has z: $.x.y.z). I refer you to the documentation and tutorial for more information.

As json.loads simply returns a dict, you can use the operators that apply to dicts:
>>> jdata = json.load('{"uri": "http:", "foo", "bar"}')
>>> 'uri' in jdata # Check if 'uri' is in jdata's keys
True
>>> jdata['uri'] # Will return the value belonging to the key 'uri'
u'http:'
Edit: to give an idea regarding how to loop through the data, consider the following example:
>>> import json
>>> jdata = json.loads(open ('bookmarks.json').read())
>>> for c in jdata['children'][0]['children']:
... print 'Title: {}, URI: {}'.format(c.get('title', 'No title'),
c.get('uri', 'No uri'))
...
Title: Recently Bookmarked, URI: place:folder=BOOKMARKS_MENU(...)
Title: Recent Tags, URI: place:sort=14&type=6&maxResults=10&queryType=1
Title: , URI: No uri
Title: Mozilla Firefox, URI: No uri
Inspecting the jdata data structure will allow you to navigate it as you wish. The pprint call you already have is a good starting point for this.
Edit2: Another attempt. This gets the file you mentioned in a list of dictionaries. With this, I think you should be able to adapt it to your needs.
>>> def build_structure(data, d=[]):
... if 'children' in data:
... for c in data['children']:
... d.append({'title': c.get('title', 'No title'),
... 'uri': c.get('uri', None)})
... build_structure(c, d)
... return d
...
>>> pprint.pprint(build_structure(jdata))
[{'title': u'Bookmarks Menu', 'uri': None},
{'title': u'Recently Bookmarked',
'uri': u'place:folder=BOOKMARKS_MENU&folder=UNFILED_BOOKMARKS&(...)'},
{'title': u'Recent Tags',
'uri': u'place:sort=14&type=6&maxResults=10&queryType=1'},
{'title': u'', 'uri': None},
{'title': u'Mozilla Firefox', 'uri': None},
{'title': u'Help and Tutorials',
'uri': u'http://www.mozilla.com/en-US/firefox/help/'},
(...)
}]
To then "search through it for u'uri': u'http:'", do something like this:
for c in build_structure(jdata):
if c['uri'].startswith('http:'):
print 'Started with http'

Seems there's a typo (missing colon) in the JSON dict provided by jro.
The correct syntax would be:
jdata = json.load('{"uri": "http:", "foo": "bar"}')
This cleared it up for me when playing with the code.

Functions to search through and print dicts, like JSON.
*made in python 3
Search:
def pretty_search(dict_or_list, key_to_search, search_for_first_only=False):
"""
Give it a dict or a list of dicts and a dict key (to get values of),
it will search through it and all containing dicts and arrays
for all values of dict key you gave, and will return you set of them
unless you wont specify search_for_first_only=True
:param dict_or_list:
:param key_to_search:
:param search_for_first_only:
:return:
"""
search_result = set()
if isinstance(dict_or_list, dict):
for key in dict_or_list:
key_value = dict_or_list[key]
if key == key_to_search:
if search_for_first_only:
return key_value
else:
search_result.add(key_value)
if isinstance(key_value, dict) or isinstance(key_value, list) or isinstance(key_value, set):
_search_result = pretty_search(key_value, key_to_search, search_for_first_only)
if _search_result and search_for_first_only:
return _search_result
elif _search_result:
for result in _search_result:
search_result.add(result)
elif isinstance(dict_or_list, list) or isinstance(dict_or_list, set):
for element in dict_or_list:
if isinstance(element, list) or isinstance(element, set) or isinstance(element, dict):
_search_result = pretty_search(element, key_to_search, search_result)
if _search_result and search_for_first_only:
return _search_result
elif _search_result:
for result in _search_result:
search_result.add(result)
return search_result if search_result else None
Print:
def pretty_print(dict_or_list, print_spaces=0):
"""
Give it a dict key (to get values of),
it will return you a pretty for print version
of a dict or a list of dicts you gave.
:param dict_or_list:
:param print_spaces:
:return:
"""
pretty_text = ""
if isinstance(dict_or_list, dict):
for key in dict_or_list:
key_value = dict_or_list[key]
if isinstance(key_value, dict):
key_value = pretty_print(key_value, print_spaces + 1)
pretty_text += "\t" * print_spaces + "{}:\n{}\n".format(key, key_value)
elif isinstance(key_value, list) or isinstance(key_value, set):
pretty_text += "\t" * print_spaces + "{}:\n".format(key)
for element in key_value:
if isinstance(element, dict) or isinstance(element, list) or isinstance(element, set):
pretty_text += pretty_print(element, print_spaces + 1)
else:
pretty_text += "\t" * (print_spaces + 1) + "{}\n".format(element)
else:
pretty_text += "\t" * print_spaces + "{}: {}\n".format(key, key_value)
elif isinstance(dict_or_list, list) or isinstance(dict_or_list, set):
for element in dict_or_list:
if isinstance(element, dict) or isinstance(element, list) or isinstance(element, set):
pretty_text += pretty_print(element, print_spaces + 1)
else:
pretty_text += "\t" * print_spaces + "{}\n".format(element)
else:
pretty_text += str(dict_or_list)
if print_spaces == 0:
print(pretty_text)
return pretty_text

You can use jsonpipe if you just need the output (and more comfortable with command line):
cat bookmarks.json | jsonpipe |grep uri