Develop Reference

Remove key but keep value in JSON [Python]

I need to "Semi-Flatten" a JSON object where i have a JSON object with nested items. I have tried to use the flat_json with pandas and other "flatten_json" and json_normalize code in stackoverflow but i end up with completely flatten JSON (something i do not need).
Here is the JSON structure:
[{
"Stat": {
"event": "03458188-abf9-431c-8144-ad49c1d069ed",
"id": "102e1bb1f28ca44b70d02d33380b13",
"number": "1121",
"source": "",
"datetime": "2023-01-13T00:00:00Z",
"status": "ok"
},
"Goal": {
"name": "goalname"
},
"Fordel": {
"company": "companyname"
},
"Land": {
"name": "landname"
}
}, {
"Stat": {
"event": "22222",
"id": "44444",
"number": "5555",
"source": "",
"datetime": "2023-01-13T00:00:00Z",
"status": "ok"
},
"Goal": {
"name": "goalname2"
},
"Fordel": {
"company": "companyname2"
},
"Land": {
"name_land": "landname2"
}
}]
The result i need is this:
[{
"event": "03458188-abf9-431c-8144-ad49c1d069ed",
"id": "102e1bb1f28ca44b70d02d33380b13",
"number": "1121",
"source": "",
"datetime": "2023-01-13T00:00:00Z",
"status": "ok",
"name": "goalname",
"company": "companyname",
"name_land": "landname"
}, {
"event": "22222",
"id": "44444",
"number": "5555",
"source": "",
"datetime": "2023-01-13T00:00:00Z",
"status": "ok",
"name": "goalname2",
"company": "companyname2",
"name_land": "landname2"
}]
If this can be used with pandas or other json packages it would be great.
Coded i have tried: (copy/paste from another question/answer)
def flatten_data(y):
out = {}
def flatten(x, name=''):
if type(x) is dict:
for a in x:
flatten(x[a], name + a + '_')
elif type(x) is list:
i = 0
for a in x:
flatten(a, name + str(i) + '_')
i += 1
else:
out[name[:-1]] = x
flatten(y)
return out
That gives me:
{
"0_event": "03458188-abf9-431c-8144-ad49c1d069ed",
"0_id": "102e1bb1f28ca44b70d02d33380b13",
......
"1_event": "102e1bb1f28ca44b70d02d33380b13",
"1_id": "102e1bb1f28ca44b70d02d33380b13",
etc...
}

Map the flatten_data function over the list instead of flattening the entire list.
result = list(map(flatten_data, source)))

Writing resilient recursive code that will return results from a big json file

I have written a recursive code. I want more experienced people to tell me how resillient and fail-safe is my code:
I have a json file (Json file can be as big as 300MB):
[
{
"modules": {
"webpages": []
},
"webpages": {
"ip_addr": {
"value": "127.0.0.1",
"tags": []
},
"http": {
"status": {
"value": "Unavailable",
"tags": []
},
"title": {
"value": "403 Forbidden",
"tags": [
{
"category": "Server Code",
"match": "403"
},
{
"category": "Interesting Words",
"match": "Forbidden"
}
]
},
"server": {
"value": "Apache",
"tags": [
{
"category": "Apache Server",
"match": "Apache"
}
]
}
},
"redirects": [],
"robottxt": null
}
},
{
"modules": {
"webpages": []
}
}
]
I want to return value keys where tags are populated.
So I want to ignore:
"status": {
"value": "Unavailable",
"tags": []
},
But I want to return the title and server values. I also want to return ip_addr.value
I have written this code:
def getAllValues(nestedDictionary, firstArray, firstObj, firstUseful):
returnedArray = firstArray
tempValue = firstObj
useful = firstUseful
for key, value in nestedDictionary.items():
ipString = nestedDictionary.get("ip_addr")
if ipString is not None:
ipValue = ipString.get("value")
useful = {"ip_add": ipValue}
if isinstance(value, dict):
temp = {
"Key": key,
"useful": useful,
}
getAllValues(value, returnedArray, temp, useful)
else:
if key == "value":
tempValue["value"] = value
if key == "tags" and isinstance(value, list) and len(value) > 0:
tempValue["tags"] = value
returnedArray.append(tempValue)
return returnedArray
The above code should return:
[
{
"Key": "title",
"value": "403 Forbidden",
"useful": { "ip_addr": "127.0.0.1" },
"tags": [
{
"category": "Server Code",
"match": "403"
},
{
"category": "Interesting Words",
"match": "Forbidden"
}
]
},
{
"Key": "server",
"value": "Apache",
"useful": { "ip_addr": "127.0.0.1" },
"tags": [
{
"category": "Apache Server",
"match": "Apache"
}
]
}
]
Its a long post, but hopefully, someone can give me some assurance :)

JSON ID extraction from Array in Python

I wrote a script to pull data from Verizon's connectivity management API with the following. Below is the section that requests the line information based on the search item, in this case, the SIM or iccid. I did not include the previous parts because they are just to connect to the API and get credentials.
header = {
'accept': 'application/json',
'VZ-M2M-Token': session_token,
'Authorization': 'Bearer' + bearer_token,
'Content-Type': 'application/json',
}
data = '{ "deviceId": { "id": ' + SIM +', "kind": "ICCID" }}'
response = requests.post('https://thingspace.verizon.com/api/m2m/v1/devices/actions/list', headers=header, data=data)
And the response I get is a JSON Array which looks like
{
"hasMoreData": false,
"devices": [
{
"accountName": "123456789-00001",
"billingCycleEndDate": "2020-10-31T20:00:00-04:00",
"carrierInformations": [
{
"carrierName": "Verizon Wireless",
"servicePlan": "3rrrrx48wwwwrjgjtyjtyjtyjtyj",
"state": "active"
}
],
"connected": true,
"createdAt": "2016-11-04T11:06:28-04:00",
"deviceIds": [
{
"id": "5256694405",
"kind": "mdn"
},
{
"id": "3114949302094150",
"kind": "imsi"
},
{
"id": "35922505468230",
"kind": "imei"
},
{
"id": "891480000054957290575",
"kind": "iccId"
},
{
"id": "15256694405",
"kind": "msisdn"
},
{
"id": "5256694405",
"kind": "min"
}
],
"extendedAttributes": [
{
"key": "PrimaryPlaceOfUseTitle"
},
{
"key": "PrimaryPlaceOfUseFirstName",
"value": "5256694405",
},
{
"key": "PrimaryPlaceOfUseMiddleName"
},
{
"key": "PrimaryPlaceOfUseLastName",
"value": "ESN"
},
{
"key": "PrimaryPlaceOfUseSuffix"
},
{
"key": "PrimaryPlaceOfUseAddressLine1"
},
{
"key": "PrimaryPlaceOfUseAddressLine2"
},
{
"key": "PrimaryPlaceOfUseCity"
},
{
"key": "PrimaryPlaceOfUseState"
},
{
"key": "PrimaryPlaceOfUseCountry"
},
{
"key": "PrimaryPlaceOfUseZipCode"
},
{
"key": "PrimaryPlaceOfUseZipCode4"
},
{
"key": "PrimaryPlaceOfUseCBRPhone"
},
{
"key": "PrimaryPlaceOfUseCBRPhoneType"
},
{
"key": "PrimaryPlaceOfUseEmailAddress"
},
{
"key": "AccountNumber",
"value": "5256694405-00001"
},
{
"key": "SmsrOid"
},
{
"key": "ProfileStatus"
},
{
"key": "PromoCodes",
"value": ""
},
{
"key": "PromotionStartDate",
"value": ""
},
{
"key": "PromotionScheduledEndDate",
"value": ""
},
{
"key": "LeadId",
"value": ""
},
{
"key": "CustomerName",
"value": ""
},
{
"key": "CustomerAddressLine1",
"value": ""
},
{
"key": "CustomerAddressLine2",
"value": ""
},
{
"key": "CustomerAddressCity",
"value": ""
},
{
"key": "CustomerAddressState",
"value": ""
},
{
"key": "CustomerAddressZipCode",
"value": ""
},
{
"key": "ServiceZipCode",
"value": ""
},
{
"key": "SkuNumber",
"value": "VZW080000460053"
},
{
"key": "CostCenterCode"
},
{
"key": "PreIMEI",
"value": "3592254564568445"
},
{
"key": "PreSKU",
"value": "VZW080000100037"
},
{
"key": "SIMOTADate",
"value": "4/30/2020 1:22:18 PM"
},
{
"key": "RoamingStatus",
"value": "NotRoaming"
},
{
"key": "LastRoamingStatusUpdate",
"value": "9/24/2020 5:40:26 PM"
}
],
"groupNames": [
"Default: 0220433754-00001"
],
"ipAddress": "100.100.100.100",
"lastActivationBy": "User Verizon",
"lastActivationDate": "2016-11-04T11:06:28-04:00",
"lastConnectionDate": "2020-09-24T13:40:26-04:00"
}
]
}
I added a part to my script to pull the mdn, iccid and the imei from the array with the code that is below.
def puller(line_json):
line_data = json.loads(line_json)
mdn = (line_data['devices'][0]['deviceIds'][0]['id'])
iccid = (line_data['devices'][0]['deviceIds'][3]['id'])
imei = (line_data['devices'][0]['deviceIds'][2]['id'])
print('phone = ' ,mdn)
print('SIM = ' , iccid)
print('IMEI = ' , imei)
I tested this code and it works the way it should with one test ID. I then proceeded to test with another test ID and I learned that the array structure is not always the same. That second JSON array is below. I am wondering is there a better way to find the specific values that I want, but in the way that I am not specifically telling the script where in the structure the item will be as I did above.
{
"hasMoreData": false,
"devices": [
{
"accountName": "02234234234-00001",
"billingCycleEndDate": "2020-10-31T20:00:00-04:00",
"carrierInformations": [
{
"carrierName": "Verizon Wireless",
"servicePlan": "37776xdsfewsfwe576193",
"state": "active"
}
],
"connected": true,
"createdAt": "2016-05-24T15:55:06-04:00",
"deviceIds": [
{
"id": "0945437676404",
"kind": "esn"
},
{
"id": "1234565799",
"kind": "mdn"
},
{
"id": "31148454545458767",
"kind": "imsi"
},
{
"id": "01426786678211",
"kind": "imei"
},
{
"id": "89148000006456456454",
"kind": "iccId"
},
{
"id": "1234565799",
"kind": "min"
}
],
"extendedAttributes": [
{
"key": "PrimaryPlaceOfUseTitle"
},
{
"key": "PrimaryPlaceOfUseFirstName",
"value": "096114564506772"
},
{
"key": "PrimaryPlaceOfUseMiddleName"
},
{
"key": "PrimaryPlaceOfUseLastName",
"value": "096546454806772"
},
{
"key": "PrimaryPlaceOfUseSuffix"
},
{
"key": "PrimaryPlaceOfUseAddressLine1"
},
{
"key": "PrimaryPlaceOfUseAddressLine2"
},
{
"key": "PrimaryPlaceOfUseCity"
},
{
"key": "PrimaryPlaceOfUseState"
},
{
"key": "PrimaryPlaceOfUseCountry"
},
{
"key": "PrimaryPlaceOfUseZipCode"
},
{
"key": "PrimaryPlaceOfUseZipCode4"
},
{
"key": "PrimaryPlaceOfUseCBRPhone"
},
{
"key": "PrimaryPlaceOfUseCBRPhoneType"
},
{
"key": "PrimaryPlaceOfUseEmailAddress"
},
{
"key": "AccountNumber",
"value": "02242342354-00001"
},
{
"key": "SmsrOid"
},
{
"key": "ProfileStatus"
},
{
"key": "PromoCodes",
"value": ""
},
{
"key": "PromotionStartDate",
"value": ""
},
{
"key": "PromotionScheduledEndDate",
"value": ""
},
{
"key": "LeadId",
"value": ""
},
{
"key": "CustomerName",
"value": ""
},
{
"key": "CustomerAddressLine1",
"value": ""
},
{
"key": "CustomerAddressLine2",
"value": ""
},
{
"key": "CustomerAddressCity",
"value": ""
},
{
"key": "CustomerAddressState",
"value": ""
},
{
"key": "CustomerAddressZipCode",
"value": ""
},
{
"key": "ServiceZipCode",
"value": ""
},
{
"key": "SkuNumber",
"value": "VZW12000364343005"
},
{
"key": "CostCenterCode"
},
{
"key": "PreIMEI"
},
{
"key": "PreSKU",
"value": "VZW12000334340005"
},
{
"key": "SIMOTADate",
"value": "3/13/2020 10:52:07 AM"
},
{
"key": "RoamingStatus",
"value": "NotRoaming"
},
{
"key": "LastRoamingStatusUpdate",
"value": "10/20/2020 6:14:20 PM"
}
],
"groupNames": [
"Default: 02342343754-00001"
],
"ipAddress": "101.101.101.101",
"lastActivationBy": "User Verizon",
"lastActivationDate": "2016-05-24T15:55:16-04:00",
"lastConnectionDate": "2020-10-20T14:14:20-04:00"
}
]
}
I tried to use this block of code from some research I did to find the value that I was looking for; in this case, the mdn. Problem I have is that the response returns a blank set of brackets with no information, so I know there is something I probably did wrong.
def json_extract(obj, kind):
"""Recursively fetch values from nested JSON."""
arr = []
def extract(obj, arr, kind):
"""Recursively search for values of key in JSON tree."""
if isinstance(obj, dict):
for k, v in obj.items():
if isinstance(v, (dict, list)):
extract(v, arr, kind)
elif k == kind:
arr.append(v)
elif isinstance(obj, list):
for item in obj:
extract(item, arr, kind)
return arr
values = extract(obj, arr, kind)
return values
names = json_extract(response , 'mdn')
print(names)

I understood that you are trying to find, mdn, iccid and imei'id from the json object above,so, instead of recursion and the complicated coding that you have done there, it is easier to use python's inbuilt libraries to help you out:
You can use the next function for your purpose:
# load your json data
line_data = json.loads(data)
# narrow your focus on the array in question
device_ids = line_data['devices'][0]['deviceIds']
# This gets the first item's id attribute from the list that matches the condition, and returns None if no item matches.
mdn = next((x['id'] for x in device_ids if x['kind'] == "mdn"), None)
iccid = next((x['id'] for x in device_ids if x['kind'] == "iccid"), None)
imei = next((x['id'] for x in device_ids if x['kind'] == "imei"), None)
You will need to handle the None if it was unable to find such element in the array.
Reference : Find object in list that has attribute equal to some value (that meets any condition)

Recursive list inside dict? - python

I have started learning Python not long ago, and I'm trying to do recursive function for my list. This is the function:
def get_employee(j, field, value):
res = j
for x in res:
if x['Name'] == field and x['Value'] == value:
return res
elif "Properties" not in x:
if x is not None:
continue
elif "Properties" in x:
return get_employee(x['Properties'], field, value)
And this is my JSON:
[{
"Value": "Sales",
"Name": "Department",
"Properties": [{
"Value": "US",
"Name": "Country",
"Properties": [{
"Value": "Employee",
"Name": "EmployeeType",
"Properties": [{
"Value": "Manya Bishter",
"Name": "EmployeeFullName",
"Properties": [{
"Value": 1111,
"Name": "EmployeeID"
},
{
"Value": "Manya",
"Name": "EmployeeFirstName"
},
{
"Value": "Bishter",
"Name": "EmployeeLastName"
}
]
},
{
"Value": "Michael Ort",
"Name": "EmployeeFullName",
"Properties": [{
"Value": 1112,
"Name": "EmployeeID"
},
{
"Value": "Michael",
"Name": "EmployeeFirstName"
},
{
"Value": "Ort",
"Name": "EmployeeLastName"
}
]
}
]
},
{
"Value": "Manager",
"Name": "EmployeeType",
"Properties": [{
"Value": "Nick Fair",
"Name": "EmployeeFullName",
"Properties": [{
"Value": 1113,
"Name": "EmployeeID"
},
{
"Value": "Nick",
"Name": "EmployeeFirstName"
},
{
"Value": "Fair",
"Name": "EmployeeLastName"
}
]
}]
}
]
}]
},
{
"Value": "Marketing",
"Name": "Department",
"Properties": [{
"Value": "US",
"Name": "Country",
"Properties": [{
"Value": "Employee",
"Name": "EmployeeType",
"Properties": [{
"Value": "Tamta Hiresh",
"Name": "EmployeeFullName",
"Properties": [{
"Value": 1121,
"Name": "EmployeeID"
},
{
"Value": "Tamta",
"Name": "EmployeeFirstName"
},
{
"Value": "Hiresh",
"Name": "EmployeeLastName"
}
]
}]
}]
}]
}
]
The function work only on Manya, but nowhere else.
For example, if I do this:
print(get_employee(myjson, "EmployeeFirstName", "Nick"))
It will print:
[{'Value': 1111, 'Name': 'EmployeeID'}, {'Value': 'Manya', 'Name': 'EmployeeFirstName'}, {'Value': 'Bishter', 'Name': 'EmployeeLastName'}]
But for others (like Nick), it will return None.
Can you please help?
Thanks!

Here is the code:
def get_employee(data, field, value):
result = []
def recursor(j, field, value):
res = j
for x in res:
if x['Name'] == field and x['Value'] == value:
result.append(res)
elif "Properties" not in x:
if x is not None:
continue
elif "Properties" in x:
recursor(x['Properties'], field, value)
recursor(data,field,value)
return result
The problem with your recursive function is that, once it hits return get_employee(x['Properties'], field, value); it will stop the outer for loop for x in res: . Thus it will never run on the next item in your list.

Python dictionary from schema

I need to check if all the keys declared in a schema file are present in a dictionary and if they are not, I need to fill them with a default value, of a given type. I need to do this dynamically because the structure can be even more complicated than the one below.
{
"type": "object",
"properties": {
"vid": {
"type": ["null", "integer"]
},
"merged-vids": {
"type": ["null", "array"],
"items": {
"type": ["null", "integer"]
}
},
"portal-id": {
"type": ["null", "integer"]
},
"is-contact": {
"type": ["null", "boolean"]
}
"form-submissions": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"conversion-id": {
"type": ["null", "string"]
},
"timestamp": {
"type": ["null", "string"],
"format": "date-time"
},
"form-id": {
"type": ["null", "string"]
},
"portal-id": {
"type": ["null", "integer"]
},
"page-url": {
"type": ["null", "string"]
},
"title": {
"type": ["null", "string"]
}
}
}
}
}
}
This is an example of dictionary:
{
"vid": 1000,
"portal-id": 2512,
"is-contact": true,
"profile-token": "dummy_profile_token",
"profile-url": "dummy_profile_url",
"form-submissions": [
{
"conversion-id": "127-798",
"timestamp": 1484080167266,
"form-id": "70fd-4b98-14796-777",
"page-url": "https://example.com/landing-page-url",
"title": "A new test form",
"meta-data": []
}
]
}
I am also new to python and this is a bit too much.
This is what I tried, but I cannot figure out what to do.
def get_default(type_object):
if type_object == 'object':
new_dict = {}
elif type_object == 'array':
return []
else:
return ''
def fill_fields_with_empty_str(record, schema):
if isinstance(schema['type'], list):
type_obj = schema['type'][len(schema['type'])-1]
elif isinstance(schema['type'], str):
type_obj = schema['type']
if type_obj == 'object':
new_dict = {}
for key in schema['properties'].keys():
if not record.get(key):
record[key] = get_default(schema['properties'][key]['type'])
new_dict[key] = fill_fields_with_empty_str(record[key], schema['properties'][key])
return new_dict
elif type_obj == 'array':
new_list = []
type_obj = schema["items"]['type']
if len(record) == 0:
record = get_default(type_obj)
for element in schema["items"]:
new_list.append(fill_fields_with_empty_str(record, schema['items']))
return new_list
else:
return ''