Related
I'm looking for any suggestions to resolve an issue I'm facing. It might seem as a simple problem, but after a few days trying to find an answer - I think it is not anymore.
I'm receiving data (StringType) in a following JSON-like format, and there is a requirement to turn it into flat key-value pair dictionary. Here is a payload sample:
s = """{"status": "active", "name": "{\"first\": \"John\", \"last\": \"Smith\"}", "street_address": "100 \"Y\" Street"}"""
and the desired output should look like this:
{'status': 'active', 'name_first': 'John', 'name_last': 'Smith', 'street_address': '100 "Y" Street'}
The issue is I can't find a way to turn original string (s) into a dictionary. If I can achieve that the flattening part is working perfectly fine.
import json
import collections
import ast
#############################################################
# Flatten complex structure into a flat dictionary
#############################################################
def flatten_dictionary(dictionary, parent_key=False, separator='_', value_to_str=True):
"""
Turn a nested complex json into a flattened dictionary
:param dictionary: The dictionary to flatten
:param parent_key: The string to prepend to dictionary's keys
:param separator: The string used to separate flattened keys
:param value_to_str: Force all returned values to string type
:return: A flattened dictionary
"""
items = []
for key, value in dictionary.items():
new_key = str(parent_key) + separator + key if parent_key else key
try:
value = json.loads(value)
except BaseException:
value = value
if isinstance(value, collections.MutableMapping):
if not value.items():
items.append((new_key,None))
else:
items.extend(flatten_dictionary(value, new_key, separator).items())
elif isinstance(value, list):
if len(value):
for k, v in enumerate(value):
items.extend(flatten_dictionary({str(k): (str(v) if value_to_str else v)}, new_key).items())
else:
items.append((new_key,None))
else:
items.append((new_key, (str(value) if value_to_str else value)))
return dict(items)
# Data sample; sting and dictionary
s = """{"status": "active", "name": "{\"first\": \"John\", \"last\": \"Smith\"}", "street_address": "100 \"Y\" Street"}"""
d = {"status": "active", "name": "{\"first\": \"John\", \"last\": \"Smith\"}", "street_address": "100 \"Y\" Street"}
# Works for dictionary type
print(flatten_dictionary(d))
# Doesn't work for string type, for any of the below methods
e = eval(s)
# a = ast.literal_eval(s)
# j = json.loads(s)
Try:
import json
import re
def jsonify(s):
s = s.replace('"{','{').replace('}"','}')
s = re.sub(r'street_address":\s+"(.+)"(.+)"(.+)"', r'street_address": "\1\2\3"',s)
return json.loads(s)
If you must keep the quotes around Y, try:
def jsonify(s):
s = s.replace('"{','{').replace('}"','}')
search = re.search(r'street_address":\s+"(.+)"(.+)"(.+)"',s)
if search:
s = re.sub(r'street_address":\s+"(.+)"(.+)"(.+)"', r'street_address": "\1\2\3"',s)
dict_version = json.loads(s)
dict_version['street_address'] = dict_version['street_address'].replace(search.group(2),'"'+search.group(2)+'"')
return dict_version
A more generalized attempt:
def jsonify(s):
pattern = r'(?<=[,}])\s*"(.[^\{\}:,]+?)":\s+"([^\{\}:,]+?)"([^\{\}:,]+?)"([^\{\}:,]+?)"([,\}])'
s = s.replace('"{','{').replace('}"','}')
search = re.search(pattern,s)
matches = []
if search:
matches = re.findall(pattern,s)
s = re.sub(pattern, r'"\1": "\2\3\4"\5',s)
dict_version = json.loads(s)
for match in matches:
dict_version[match[0]] = dict_version[match[0]].replace(match[2],'"'+match[2]+'"')
return dict_version
I have a json in below format:
{"MainName":[{"col1":"12345","col2":"False","col3":"190809","SubName1":{"col4":30.00,"SubName2":{"col5":"19703","col6":"USD"}},"col7":"7372267","SubName3":[{"col8":"345337","col9":"PC"}],"col10":"10265","col11":"29889004","col12":"calculated","col13":"9218","SubName4":{"col14":1,"SubName5":{"col15":"1970324","col16":"integer"}},"col17":"434628","col18":"2020-02-06T13:47:40.000-0800","col19":"754878037","SubName6":{"col20":30.00,"SubName7":{"col21":"19703248","col22":"USD"}}},{"col1":"12345","col2":"False","col3":"190809","SubName1":{"col4":30.00,"SubName2":{"col5":"19703","col6":"USD"}},"col7":"7372267","SubName3":[{"col8":"345337","col9":"PC"}],"col10":"10265","col11":"29889004","col12":"calculated","col13":"9218","SubName4":{"col14":1,"SubName5":{"col15":"1970324","col16":"integer"}},"col17":"434628","col18":"2020-02-06T13:47:40.000-0800","col19":"754878037","SubName6":{"col20":30.00,"SubName7":{"col21":"19703248","col22":"USD"}}}],"skip":0,"top":2,"next":"/v1/APIName?skip=2&top=2"}
I want to convert it into csv with below format:
MainName_col1,MainName_col2,MainName_col3,MainName_SubName1_col4,MainName_SubName1_SubName2_col5,MainName_SubName1_SubName2_col6,MainName_col7,MainName_SubName3_col8,MainName_SubName3_col9,MainName_col10,MainName_col11,MainName_col12,MainName_col13,MainName_SubName4_col14,MainName_SubName4_SubName5_col15,MainName_SubName4_SubName5_col16,MainName_col17,MainName_col18,MainName_col19,MainName_SubName6_col20,MainName_SubName6_SubName7_col21,MainName_SubName6_SubName7_col22
12345,False,190809,30.0,19703,USD,7372267,345337,PC,10265,29889004,calculated,9218,1,1970324,integer,434628,2020-02-06T13:47:40.000-0800,754878037,30.0,19703248,USD
12345,False,190809,30.0,19703,USD,7372267,345337,PC,10265,29889004,calculated,9218,2,123453,integer,434628,2020-02-06T13:47:40.000-0800,754878037,30.0,19703248,USD
Kindly help me out in this.
Use below function to flatten your JSON data.
dc = {"MainName":[{"col1":"12345","col2":False,"col3":"190809","SubName1":{"col4":30.00,"SubName2":{"col5":"19703","col6":"USD"}},"col7":"7372267","SubName3":[{"col8":"345337","col9":"PC"}],"col10":"10265","col11":"29889004","col12":"calculated","col13":"9218","SubName4":{"col14":1,"SubName5":{"col15":"1970324","col16":"integer"}},"col17":"434628","col18":"2020-02-06T13:47:40.000-0800","col19":"754878037","SubName6":{"col20":30.00,"SubName7":{"col21":"19703248","col22":"USD"}}}],"skip":0,"top":1,"next":"/v1/APIName?skip=1&top=1"}
def flatten(root: str, dict_obj: dict):
flat = {}
for i in dict_obj.keys():
val = dict_obj[i]
if not isinstance(val, dict) and not isinstance(val, list):
flat[f'{root}_{i}'] = val
else:
if isinstance(val, list):
val = val[-1]
flat.update(flatten(f'{root}_{i}', val))
return flat
flatten('MainName', dc['MainName'][0])
It will give you expected output. Then use it the way you want.
{'MainName_col1': '12345',
'MainName_col2': False,
'MainName_col3': '190809',
'MainName_SubName1_col4': 30.0,
'MainName_SubName1_SubName2_col5': '19703',
'MainName_SubName1_SubName2_col6': 'USD',
'MainName_col7': '7372267',
'MainName_SubName3_col8': '345337',
'MainName_SubName3_col9': 'PC',
'MainName_col10': '10265',
'MainName_col11': '29889004',
'MainName_col12': 'calculated',
'MainName_col13': '9218',
'MainName_SubName4_col14': 1,
'MainName_SubName4_SubName5_col15': '1970324',
'MainName_SubName4_SubName5_col16': 'integer',
'MainName_col17': '434628',
'MainName_col18': '2020-02-06T13:47:40.000-0800',
'MainName_col19': '754878037',
'MainName_SubName6_col20': 30.0,
'MainName_SubName6_SubName7_col21': '19703248',
'MainName_SubName6_SubName7_col22': 'USD'}
As of my understanding, your dc will look like below
dc = {"MainName":[{"col1":"12345","col2":"False","col3":"190809","SubName1":{"col4":30.00,"SubName2":{"col5":"19703","col6":"USD"}},"col7":"7372267","SubName3":[{"col8":"345337","col9":"PC"}],"col10":"10265","col11":"29889004","col12":"calculated","col13":"9218","SubName4":{"col14":1,"SubName5":{"col15":"1970324","col16":"integer"}},"col17":"434628","col18":"2020-02-06T13:47:40.000-0800","col19":"754878037","SubName6":{"col20":30.00,"SubName7":{"col21":"19703248","col22":"USD"}}},{"col1_a":"12345XX","col2_b":"False","col3_c":"190809","SubName1":{"col4_d":30.00,"SubName2":{"col5_e":"19703","col6_f":"USD"}},"col7_g":"7372267","SubName3":[{"col8_h":"345337","col9":"PC"}],"col10_i":"10265","col11_j":"29889004","col12_k":"calculated","col13_l":"9218","SubName4":{"col14_m":1,"SubName5":{"col15_n":"1970324","col16_o":"integer"}},"col17_p":"434628","col18_q":"2020-02-06T13:47:40.000-0800","col19_r":"754878037","SubName6":{"col20_s":30.00,"SubName7":{"col21_t":"19703248","col22_u":"USDZZ"}}}],"skip":0,"top":2,"next":"/v1/APIName?skip=2&top=2"}
I used the above answer to flatten everything into single object
def flatten(root: str, dict_obj: dict):
flat = {}
for i in dict_obj.keys():
val = dict_obj[i]
if not isinstance(val, dict) and not isinstance(val, list):
flat[f'{root}_{i}'] = val
else:
if isinstance(val, list):
val = val[-1]
flat.update(flatten(f'{root}_{i}', val))
return flat
keys_list = []
values_list = []
for i in range(len(dc['MainName'])):
result = flatten('MainName', dc['MainName'][i])
keys_list.append(list(result.keys()))
values_list.append(list(result.values()))
for k in keys_list:
for res in k:
guestFile = open("sample.csv","a")
guestFile.write(res)
guestFile.write(",")
guestFile.close()
for v in values_list:
for res in v:
guestFile = open("sample.csv","a")
guestFile.write(str(res))
guestFile.write(",")
guestFile.close()
Checkout my code at https://repl.it/#TamilselvanLaks/jsontocsvmul
Note: Use the 'run' button to run the program, left side you can see sample.csv
there you can see all keys as like you want
Please let me know my answer meets your expectation
I need '-40' to be -40 on the output, how i can do this?
In [2]: foo
Out[2]: '{\n\t"rssiIntentRange":"-40"\n}'
In [3]: import json
In [4]: json.loads(foo)
Out[4]: {u'rssiIntentRange': u'-40'}
Im searching for something clean and generic. Doing treatments after the json.loads is what im already doing and its very dirty because of all types of data and indexes. If json.loads is not the best option im open to other approaches as well that treats JSON
If you want to influence what happens when you call json.loads(), you need to write an object hook:
import json
def int_please_object_hook(obj):
"""If a value in obj is a string, try to convert it to an int"""
rv = {}
for k, v in obj.items():
if isinstance(v, basestring):
try:
rv[k] = int(v)
except ValueError:
rv[k] = v
else:
rv[k] = v
return rv
j = '{"key1": "value1", "key2": "10", "key3": "-10"}'
print(json.loads(j))
# {'key1': 'value1', 'key2': '10', 'key3': '-10'}
print(json.loads(j, object_hook=int_please_object_hook))
# {'key1': 'value1', 'key2': 10, 'key3': -10}
One way to do this would be to convert the -40 string to int after loading the json.
Example -
>>> foo = '{\n\t"rssiIntentRange":"-40"\n}'
>>> import json
>>> d = json.loads(foo)
>>> d['rssiIntentRange'] = int(d['rssiIntentRange'])
>>> d
{'rssiIntentRange': -40}
For changing all such values inside the loaded dictionary, you can do -
d = json.loads(foo)
for k,v in d.items():
try:
d[k] = int(v)
except (ValueError, TypeError):
pass
Example/Demo -
>>> foo = '{\n\t"rssiIntentRange":"-40",\n\t"blah":"abcd",\n\t"anothernumber":"-10"\n}'
>>> d = json.loads(foo)
>>> for k,v in d.items():
... try:
... d[k] = int(v)
... except (ValueError, TypeError):
... pass
...
>>> d
{'blah': 'abcd', 'anothernumber': -10, 'rssiIntentRange': -40}
There is a way to initialize structure with dictionary:
fooData= {'y': 1, 'x': 2}
fooStruct = ffi.new("foo_t*", fooData)
fooBuffer = ffi.buffer(fooStruct)
Is there some ready function to do the conversion?
fooStruct = ffi.new("foo_t*")
(ffi.buffer(fooStruct))[:] = fooBuffer
fooData= convert_to_python( fooStruct[0] )
Do I have to use ffi.typeof("foo_t").fields by myself?
I come up with this code so far:
def __convert_struct_field( s, fields ):
for field,fieldtype in fields:
if fieldtype.type.kind == 'primitive':
yield (field,getattr( s, field ))
else:
yield (field, convert_to_python( getattr( s, field ) ))
def convert_to_python(s):
type=ffi.typeof(s)
if type.kind == 'struct':
return dict(__convert_struct_field( s, type.fields ) )
elif type.kind == 'array':
if type.item.kind == 'primitive':
return [ s[i] for i in range(type.length) ]
else:
return [ convert_to_python(s[i]) for i in range(type.length) ]
elif type.kind == 'primitive':
return int(s)
Is there a faster way?
Arpegius' solution works fine for me, and is quite elegant. I implemented a solution based on Selso's suggestion to use inspect. dir() can substitute inspect.
from inspect import getmembers
from cffi import FFI
ffi = FFI()
from pprint import pprint
def cdata_dict(cd):
if isinstance(cd, ffi.CData):
try:
return ffi.string(cd)
except TypeError:
try:
return [cdata_dict(x) for x in cd]
except TypeError:
return {k: cdata_dict(v) for k, v in getmembers(cd)}
else:
return cd
foo = ffi.new("""
struct Foo {
char name[6];
struct {
int a, b[3];
} item;
} *""",{
'name': b"Foo",
'item': {'a': 3, 'b': [1, 2, 3]}
})
pprint(cdata_dict(foo))
Output:
{'item': {'a': 3, 'b': [1, 2, 3]}, 'name': b'Foo'}
This code infortunately does not work for me, as some struct members are "pointer" types, it leads to storing "none" in the dict.
I am a Python noob, but maybe the inspect module would be another starting point, and a shorter way to print "simple" data. Then we would iterate over the result in order to unroll data structure.
For example with the following example :
struct foo {
int a;
char b[10];
};
Using inspect.getmembers( obj ) I have the following result :
[('a', 10), ('b', <cdata 'char[10]' 0x7f0be10e2824>)]
Your code is fine.
Even if there was a built-in way in CFFI, it would not be what you need here. Indeed, you can say ffi.new("foo_t*", {'p': p1}) where p1 is another cdata, but you cannot recursively pass a dictionary containing more dictionaries. The same would be true in the opposite direction: you would get a dictionary that maps field names to "values", but the values themselves would be more cdata objects anyway, and not recursively more dictionaries.
I have this code
import json
from pprint import pprint
json_data=open('bookmarks.json')
jdata = json.load(json_data)
pprint (jdata)
json_data.close()
How can I search through it for u'uri': u'http:?
ObjectPath is a library that provides ability to query JSON and nested structures of dicts and lists. For example, you can search for all attributes called "foo" regardless how deep they are by using $..foo.
While the documentation focuses on the command line interface, you can perform the queries programmatically by using the package's Python internals. The example below assumes you've already loaded the data into Python data structures (dicts & lists). If you're starting with a JSON file or string you just need to use load or loads from the json module first.
import objectpath
data = [
{'foo': 1, 'bar': 'a'},
{'foo': 2, 'bar': 'b'},
{'NoFooHere': 2, 'bar': 'c'},
{'foo': 3, 'bar': 'd'},
]
tree_obj = objectpath.Tree(data)
tuple(tree_obj.execute('$..foo'))
# returns: (1, 2, 3)
Notice that it just skipped elements that lacked a "foo" attribute, such as the third item in the list. You can also do much more complex queries, which makes ObjectPath handy for deeply nested structures (e.g. finding where x has y that has z: $.x.y.z). I refer you to the documentation and tutorial for more information.
As json.loads simply returns a dict, you can use the operators that apply to dicts:
>>> jdata = json.load('{"uri": "http:", "foo", "bar"}')
>>> 'uri' in jdata # Check if 'uri' is in jdata's keys
True
>>> jdata['uri'] # Will return the value belonging to the key 'uri'
u'http:'
Edit: to give an idea regarding how to loop through the data, consider the following example:
>>> import json
>>> jdata = json.loads(open ('bookmarks.json').read())
>>> for c in jdata['children'][0]['children']:
... print 'Title: {}, URI: {}'.format(c.get('title', 'No title'),
c.get('uri', 'No uri'))
...
Title: Recently Bookmarked, URI: place:folder=BOOKMARKS_MENU(...)
Title: Recent Tags, URI: place:sort=14&type=6&maxResults=10&queryType=1
Title: , URI: No uri
Title: Mozilla Firefox, URI: No uri
Inspecting the jdata data structure will allow you to navigate it as you wish. The pprint call you already have is a good starting point for this.
Edit2: Another attempt. This gets the file you mentioned in a list of dictionaries. With this, I think you should be able to adapt it to your needs.
>>> def build_structure(data, d=[]):
... if 'children' in data:
... for c in data['children']:
... d.append({'title': c.get('title', 'No title'),
... 'uri': c.get('uri', None)})
... build_structure(c, d)
... return d
...
>>> pprint.pprint(build_structure(jdata))
[{'title': u'Bookmarks Menu', 'uri': None},
{'title': u'Recently Bookmarked',
'uri': u'place:folder=BOOKMARKS_MENU&folder=UNFILED_BOOKMARKS&(...)'},
{'title': u'Recent Tags',
'uri': u'place:sort=14&type=6&maxResults=10&queryType=1'},
{'title': u'', 'uri': None},
{'title': u'Mozilla Firefox', 'uri': None},
{'title': u'Help and Tutorials',
'uri': u'http://www.mozilla.com/en-US/firefox/help/'},
(...)
}]
To then "search through it for u'uri': u'http:'", do something like this:
for c in build_structure(jdata):
if c['uri'].startswith('http:'):
print 'Started with http'
Seems there's a typo (missing colon) in the JSON dict provided by jro.
The correct syntax would be:
jdata = json.load('{"uri": "http:", "foo": "bar"}')
This cleared it up for me when playing with the code.
Functions to search through and print dicts, like JSON.
*made in python 3
Search:
def pretty_search(dict_or_list, key_to_search, search_for_first_only=False):
"""
Give it a dict or a list of dicts and a dict key (to get values of),
it will search through it and all containing dicts and arrays
for all values of dict key you gave, and will return you set of them
unless you wont specify search_for_first_only=True
:param dict_or_list:
:param key_to_search:
:param search_for_first_only:
:return:
"""
search_result = set()
if isinstance(dict_or_list, dict):
for key in dict_or_list:
key_value = dict_or_list[key]
if key == key_to_search:
if search_for_first_only:
return key_value
else:
search_result.add(key_value)
if isinstance(key_value, dict) or isinstance(key_value, list) or isinstance(key_value, set):
_search_result = pretty_search(key_value, key_to_search, search_for_first_only)
if _search_result and search_for_first_only:
return _search_result
elif _search_result:
for result in _search_result:
search_result.add(result)
elif isinstance(dict_or_list, list) or isinstance(dict_or_list, set):
for element in dict_or_list:
if isinstance(element, list) or isinstance(element, set) or isinstance(element, dict):
_search_result = pretty_search(element, key_to_search, search_result)
if _search_result and search_for_first_only:
return _search_result
elif _search_result:
for result in _search_result:
search_result.add(result)
return search_result if search_result else None
Print:
def pretty_print(dict_or_list, print_spaces=0):
"""
Give it a dict key (to get values of),
it will return you a pretty for print version
of a dict or a list of dicts you gave.
:param dict_or_list:
:param print_spaces:
:return:
"""
pretty_text = ""
if isinstance(dict_or_list, dict):
for key in dict_or_list:
key_value = dict_or_list[key]
if isinstance(key_value, dict):
key_value = pretty_print(key_value, print_spaces + 1)
pretty_text += "\t" * print_spaces + "{}:\n{}\n".format(key, key_value)
elif isinstance(key_value, list) or isinstance(key_value, set):
pretty_text += "\t" * print_spaces + "{}:\n".format(key)
for element in key_value:
if isinstance(element, dict) or isinstance(element, list) or isinstance(element, set):
pretty_text += pretty_print(element, print_spaces + 1)
else:
pretty_text += "\t" * (print_spaces + 1) + "{}\n".format(element)
else:
pretty_text += "\t" * print_spaces + "{}: {}\n".format(key, key_value)
elif isinstance(dict_or_list, list) or isinstance(dict_or_list, set):
for element in dict_or_list:
if isinstance(element, dict) or isinstance(element, list) or isinstance(element, set):
pretty_text += pretty_print(element, print_spaces + 1)
else:
pretty_text += "\t" * print_spaces + "{}\n".format(element)
else:
pretty_text += str(dict_or_list)
if print_spaces == 0:
print(pretty_text)
return pretty_text
You can use jsonpipe if you just need the output (and more comfortable with command line):
cat bookmarks.json | jsonpipe |grep uri