Related
I have a python dictionary, where I don't exactly know, how deeply nested it is, but here is an example of such:
{
"name":"a_struct",
"type":"int",
"data":{
"type":"struct",
"elements":[
{
"data":[
{
"name":"test1",
"data_id":0,
"type":"uint8",
"wire_type":0,
"data":0
},
{
"name":"test2",
"data_id":2,
"type":"uint32",
"wire_type":2,
"data":0
},
{
"name":"test3",
"data_id":3,
"type":"int",
"wire_type":4,
"data":{
"type":"uint32",
"elements":[
]
}
},
{
"name":"test4",
"data_id":4,
"type":"uint32",
"wire_type":2,
"data":0
},
{
"name":"test5",
"data_id":5,
"type":"int",
"wire_type":4,
"data":{
"type":"uint32",
"elements":[
]
}
}
]
}
]
}
}
My goal is to filter out each dictionary that does not contains values ["test1", "test3", "test5"] by the name key. This shall be applicable to various deeply nested dictionaries.
So in that case, the result shall be a filtered dictionary:
{
"name":"a_struct",
"type":"int",
"data":{
"type":"struct",
"elements":[
{
"data":[
{
"name":"test1",
"data_id":0,
"type":"uint8",
"wire_type":0,
"data":0
},
{
"name":"test3",
"data_id":3,
"type":"int",
"wire_type":4,
"data":{
"type":"uint32",
"elements":[
]
}
},
{
"name":"test5",
"data_id":5,
"type":"int",
"wire_type":4,
"data":{
"type":"uint32",
"elements":[
]
}
}
]
}
]
}
}
I tried to use the dpath lib (https://pypi.org/project/dpath/), by providing a filter criteria like so:
def afilter(x):
if isinstance(x, dict):
if "name" in x:
if x["name"] in ["test1", "test3", "test5"]:
return True
else:
return False
else:
return False
result = dpath.util.search(my_dict, "**", afilter=afilter)
But I get a wrong result, so every other key, has been filtered out, which is not what I want:
{
"data":{
"elements":[
{
"data":[
{
"name":"test1",
"data_id":0,
"type":"uint8",
"wire_type":0,
"data":0
},
null,
{
"name":"test3",
"data_id":3,
"type":"int",
"wire_type":4,
"data":{
"type":"uint32",
"elements":[
]
}
},
null,
{
"name":"test5",
"data_id":5,
"type":"int",
"wire_type":4,
"data":{
"type":"uint32",
"elements":[
]
}
}
]
}
]
}
}
How to get this right?
PS: I'm not forced to use the dpath lib. So, the solution might be written in pure python.
You can recursively process your dictionary while filtering unneeded records:
def delete_keys(data, keys_to_keep):
res = {}
for k, v in data.items():
if isinstance(v, dict):
res[k] = delete_keys(v, keys_to_keep)
elif isinstance(v, list):
if k == "data":
res[k] = [delete_keys(obj, keys_to_keep) for obj in v if obj.get('name') in keys_to_keep]
else:
res[k] = [delete_keys(obj, keys_to_keep) for obj in v]
else:
res[k] = v
return res
keys_to_keep = {'test1', 'test3', 'test5'}
print(delete_keys(data, keys_to_keep))
For your input, it gives:
{
"name": "a_struct",
"type": "int",
"data": {
"type": "struct",
"elements": [
{
"data": [
{
"name": "test1",
"data_id": 0,
"type": "uint8",
"wire_type": 0,
"data": 0,
},
{
"name": "test3",
"data_id": 3,
"type": "int",
"wire_type": 4,
"data": {"type": "uint32", "elements": []},
},
{
"name": "test5",
"data_id": 5,
"type": "int",
"wire_type": 4,
"data": {"type": "uint32", "elements": []},
},
]
}
],
},
}
I'm trying to move data from SQL to Mongo. Here is a challenge I'm facing, if any child object is empty I want to remove parent element. I want till insurance field to be removed.
Here is what I tried:
def remove_empty_elements(jsonData):
if(isinstance(jsonData, list) or isinstance(jsonData,dict)):
for elem in list(jsonData):
if not isinstance(elem, dict) and isinstance(jsonData[elem], list) and elem:
jsonData[elem] = [x for x in jsonData[elem] if x]
if(len(jsonData[elem])==0):
del jsonData[elem]
elif not isinstance(elem, dict) and isinstance(jsonData[elem], dict) and not jsonData[elem]:
del jsonData[elem]
else:
pass
return jsonData
sample data
{
"_id": "30546c62-8ea0-4f1a-a239-cc7508041a7b",
"IsActive": "True",
"name": "Pixel 3",
"phone": [
{
"Bill": 145,
"phonetype": "xyz",
"insurance": [
{
"year_one_claims": [
{
"2020": 200
},
{
},
{
},
{
},
{
}
]
},
{
"year_two_claims": [
{
},
{
},
{
},
{
},
{
}
]
},
]
}
],
"Provider": {
"agent": "aaadd",
}
}
Results should look like that
{
"_id": "30546c62-8ea0-4f1a-a239-cc7508041a7b",
"IsActive": "True",
"name": "Pixel 3",
"phone": [
{
"Bill": 145,
"phonetype": "xyz",
"insurance": [
{
"year_one_claims": [
{
"2020": 200
},
]
},
]
}
],
"Provider": {
"agent": "aaadd",
}
}
Your if statements are kind of confusing. I think you are looking for a recursion:
import json
# define which elements you want to remove:
to_be_deleted = [[], {}, "", None]
def remove_empty_elements(jsonData):
if isinstance(jsonData, list):
jsonData = [new_elem for elem in jsonData
if (new_elem := remove_empty_elements(elem)) not in to_be_deleted]
elif isinstance(jsonData,dict):
jsonData = {key: new_value for key, value in jsonData.items()
if (new_value := remove_empty_elements(value)) not in to_be_deleted}
return jsonData
print(json.dumps(remove_empty_elements(jsonData), indent=4))
Edit/Note: from Python3.8 you can use assignements (:=) in comprehensions
Output:
{
"_id": "30546c62-8ea0-4f1a-a239-cc7508041a7b",
"IsActive": "True",
"name": "Pixel 3",
"phone": [
{
"Bill": 145,
"phonetype": "xyz",
"insurance": [
{
"year_one_claims": [
{
"2020": 200
}
]
}
]
}
],
"Provider": {
"agent": "aaadd"
}
}
Try out this:
data = {
"_id": "30546c62-8ea0-4f1a-a239-cc7508041a7b",
"IsActive": "True",
"name": "Pixel 3",
"phone": [
{
"Bill": 145,
"phonetype": "xyz",
"insurance": [
{
"year_one_claims": [
{
"2020": 200
},
{
},
{
},
{
},
{
}
]
},
{
"year_two_claims": [
{
},
{
},
{
},
{
},
{
}
]
},
]
}
],
"Provider": {
"agent": "aaadd",
}
}
for phn_data in data['phone']:
for ins in phn_data['insurance']:
for key, val in list(ins.items()):
for ins_data in list(val):
if not ins_data:
val.remove(ins_data)
if not val:
del ins[key]
phn_data['insurance'].remove(ins)
print (data)
Output:
{
'_id': '30546c62-8ea0-4f1a-a239-cc7508041a7b',
'IsActive': 'True',
'name': 'Pixel 3',
'phone': [{
'Bill': 145,
'phonetype': 'xyz',
'insurance': [{
'year_one_claims': [{
'2020': 200
}]
}]
}],
'Provider': {
'agent': 'aaadd'
}
}
from collections import defaultdict
import pandas as pd
df = pd.DataFrame(data)
d = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(list))))
for row in df.itertuples():
d[row[1]][row[2]][row[3]][row[4]].append(row[5])
d = json.dumps(d)
d = json.loads(d)
I have this code that reads some csv data as a dataframe. I found a way to create a tree-like structure using defaultdict, and then looping through the df and populating it. The data will always have the same depth
Analyzer: Mode: SubMode: Filename: Signal
The result looks like this:
{
"Analyzer1": {
"Mode1": { "SubMode1": { "filename2": ["Signal1"] } },
"Mode2": {
"SubMode2": {
"filename1": [
"Signal2",
"Signal3"
]
}
},
"Mode3": {
"SubMode1": {
"filename1": ["Signal2"]
},
"SubMode3": {
"filename1": ["Signal3"]
}
}
},
"Analyzer2": {
"Mode1": {
"SubMode4": {
"filename1": ["Signal2"]
}
}
}
}
This is great, but I'm obligated to do some changes to the final result, as this tree will be used with a react package for displaying collapsible menus (react-checkbox-tree: https://www.npmjs.com/package/react-checkbox-tree)
The package uses a similar structure, the difference is that each level has additional properties, and it's children (if any) are in a list. This is the output that I am trying to achieve.
[
{
"label": "Analyzer1",
"value": "analyzer1",
"children": [
{
"label": "Mode1",
"value": "analyzer1/mode1",
"children": [
{
"label": "SubMode1",
"value": "analyzer1/mode1/submode1",
"children": [
{
"label": "Filename2",
"value": "analyzer1/mode1/submode1/filename2",
"children": [
{
"label": "Signal1",
"value": "analyzer1/mode1/submode1/filename2/signal1"
}
]
}
]
}
]
},
{
"label": "Mode2",
"value": "analyzer1/mode2",
"children": [
{
"label": "SubMode2",
"value": "analyzer1/mode2/submode2",
"children": [
{
"label": "Filename1",
"value": "analyzer1/mode2/submode2/filename1",
"children": [
{
"label": "Signal2",
"value": "analyzer1/mode2/submode2/filename1/signal2"
},
{
"label": "Signal3",
"value": "analyzer1/mode2/submode2/filename1/signal3"
}
]
}
]
}
]
},
{
"label": "Mode3",
"value": "analyzer1/mode3",
"children": [
{
"label": "SubMode1",
"value": "analyzer1/mode3/submode1",
"children": [
{
"label": "Filename1",
"value": "analyzer1/mode3/submode1/filename1",
"children": [
{
"label": "Signal2",
"value": "analyzer1/mode3/submode1/filename1/signal2"
}
]
}
]
},
{
"label": "SubMode3",
"value": "analyzer1/mode3/submode3",
"children": [
{
"label": "Filename1",
"value": "analyzer1/mode3/submode3/filename1",
"children": [
{
"label": "Signal3",
"value": "analyzer1/mode3/submode3/filename1/signal3"
}
]
}
]
}
]
}
]
},
{
"label": "Analyzer2",
"value": "analyzer2",
"children": [
{
"label": "Mode1",
"value": "analyzer2/mode1",
"children": [
{
"label": "SubMode4",
"value": "analyzer2/mode1/submode4",
"children": [
{
"label": "Filename1",
"value": "analyzer2/mode1/submode4/filename1",
"children": [
{
"label": "Signal2",
"value": "analyzer2/mode1/submode4/filename1/signal2"
}
]
}
]
}
]
}
]
}
]
I've tried the following, but it is incomplete, I can't find a way to add the children to the parent node.
def adjust(d, res, parent, children, path):
for k, v in d.items():
if(not isinstance(v, list)):
path = path + k.lower() + '/'
parent['value'] = k.lower()
parent['label'] = k
adjust(v, res, parent['children'][0], path)
else:
parent['children'] = []
res.append(parent)
adjust(d, [], {}, [], '')
Any suggestions or pointers would be appreciated, I'm not very skilled at doing recursion.
Your recursive function should not have that many arguments. Let it build its subtree autonomously, only needing the path as extra information from the caller. And make it functional, so that it returns the result for the subtree it deals with. That way the caller can inject the result in its own children attributes.
Here is how that could work:
def maketree(d, path=""):
if isinstance(d, list):
return [{
"label": k,
"value": path + k.lower()
} for k in d]
else:
return [{
"label": k,
"value": path + k.lower(),
"children": maketree(v, path + k.lower() + "/")
} for k, v in d.items()]
Call it as:
tree = maketree(d)
I have a scenario where I have below JSON data, which I want to parse and store the results in a dict under these conditions :
Condt --> Parse through json and find under data if groupProperty is equal to Tests then return that groupValue and value in a dict.
{
"dateFrom": "2020-03-26 07:35:00",
"dateTo": "2020-03-26 07:40:00",
"groupLabels": [
{
"groupProperty": "Tests",
"groupLabels": [
{
"groupId": "1053777",
"groupLabel": "testappzxco"
},
{
"groupId": "570009",
"groupLabel": "testappzkbo"
}
]
}
],
"binSize": 300,
"data": {
"points": [
{
"timestamp": 1585208100,
"numberOfDataPoints": 24,
"value": 0,
"groups": [
{
"groupProperty": "Tests",
"groupValue": "1053777"
},
{
"groupProperty": "Test Labels",
"groupValue": "61776"
}
]
},
{
"timestamp": 1585208100,
"numberOfDataPoints": 5,
"value": 4.888970,
"groups": [
{
"groupProperty": "Tests",
"groupValue": "1241460"
},
{
"groupProperty": "Test Labels",
"groupValue": "61710"
}
]
},
{
"timestamp": 1585208100,
"numberOfDataPoints": 96,
"value": 0,
"groups": [
{
"groupProperty": "Test Labels",
"groupValue": "61770"
}
]
},
{
"timestamp": 1585208100,
"numberOfDataPoints": 101,
"value": 0.01980198019801982,
"groups": [
{
"groupProperty": "Test Labels",
"groupValue": "61773"
}
]
},
{
"timestamp": 1585208100,
"numberOfDataPoints": 104,
"value": 0,
"groups": [
{
"groupProperty": "Test Labels",
"groupValue": "61776"
}
]
}
]
}
}
What I have tried and it doesn't even get the right details :
dat = json.loads(original_data)
testl=[]
for key in dat:
temp=key['data']['points']
for key1 in temp:
if key1['groups']['groupProperty'] == "Tests":
testl.append({key1['groupValue'], key['value']
})
Since the json is very complex I am not sure how to get the desired output.
Below is the desired O/P :
[{"tname":1241460, "tvalue":4.888970},{"tname":1053777, "tvalue":0}]
Any help would be great !
There is no special problem here: you just have to be very cautious in writing the comprehension:
[{"tname": g['groupValue'], 'tvalue': da['value']}
for da in d['data']['points'] for g in da['groups'] if g['groupProperty'] == 'Tests']
with the provided sample, it gives as expected:
[{'tname': '1053777', 'tvalue': 0}, {'tname': '1241460', 'tvalue': 4.88897}]
You have not noticed that groups is also an array, this way it should work:
points=dat['data']['points']
for key1 in points:
groups = key1['groups']
value = key1['value']
for group in groups:
if group['groupProperty'] =='Tests':
testl.append({'tname':group['groupValue'], 'value':value})
print (testl)
A recursive search making fewer assumptions on the structure of dat:
testl = []
def search(obj, seen_data, value):
if isinstance(obj, list):
for x in obj:
search(x, seen_data, value)
elif isinstance(obj, dict):
if not seen_data:
if 'data' in obj:
seen_data = True
else:
if value is not None:
if 'groupProperty' in obj and obj['groupProperty'] == 'Tests':
if 'groupValue' in obj:
tests = obj['groupValue']
testl.append({'tname': tests, 'tvalue': value})
value = None
elif 'value' in obj:
value = obj['value']
for x in obj.values():
search(x, seen_data, value)
search(dat, False, None)
print(testl)
Prints:
[{'tname': '1053777', 'tvalue': 0}, {'tname': '1241460', 'tvalue': 4.88897}]
See Python Demo
I have below Json.
My requirement is to search this Json and get 'id' value if 'name' is equal to 'Latisha Chase'
[
{
"_id": "5d3121cd001453772160a791",
"friends": [
{
"id": 6,
"name": "Mcknight Tran"
},
{
"id": 7,
"name": "Helena Bowers"
},
{
"id": 8,
"name": "Dorsey Ayala"
}
]
},
{
"_id": "5d3121cd838efa513e7dda96",
"friends": [ {
"friends": [
{
"id": 90,
"name": "w Stark"
},
{
"id": 91,
"name": "w Jacobs"
},
{
"id": 93,
"name": "w Garner"
}
]},
{
"id": 10,
"name": "Amalia Stark"
},
{
"id": 11,
"name": "Myra Jacobs"
},
{
"id": 12,
"name": "Norton Garner"
}
]
}
]
This is sample code that I have. Could anyone help me with this.?
I tried recursive codes online but didn't work with my example here.
Update:
Its not necessary that 'friends' will have single depth. it can have friends inside friends. ex: friends [{ friends[ {}]}]
A more general approach using recursion:
def recursive_function(name, l):
if isinstance(l,list):
for i in l:
recursive_function(name, i)
elif isinstance(l,dict):
if l.get("name") == name:
print (l.get("id"))
for v in l.values():
if isinstance(v, list) or isinstance(v, dict):
recursive_function(name, v)
recursive_function("Latisha Chase",json_obj)
Result:
3
Try this
j = [{
"_id": "5d3121cd001453772160a791",
"friends": [{
"id": 6,
"name": "Mcknight Tran"
},
{
"id": 7,
"name": "Helena Bowers"
},
{
"id": 8,
"name": "Dorsey Ayala"
}
]
},
{
"_id": "5d3121cded44d8ba6ad96b78",
"friends": [{
"id": 2,
"name": "June Gilbert"
},
{
"id": 3,
"name": "Latisha Chase"
},
{
"id": 4,
"name": "Franco Carlson"
}
]
},
{
"_id": "5d3121cd838efa513e7dda96",
"friends": [{
"id": 10,
"name": "Amalia Stark"
},
{
"id": 11,
"name": "Myra Jacobs"
},
{
"id": 12,
"name": "Norton Garner"
}
]
}
]
for x in j:
for y in x.get('friends'):
if y.get('name') == 'Latisha Chase':
print y.get('id')