I have the results of an SQL query in JSON format
value = [
{"Machine": "Mach 1", "Device": "Dev a", "Identifier": "HMI 1"},
{"Machine": "Mach 1", "Device": "Dev a", "Identifier": "HMI 2"},
{"Machine": "Mach 1", "Device": "Dev b", "Identifier": "HMI 3"},
{"Machine": "Mach 1", "Device": "Dev c", "Identifier": "HMI 5"},
{"Machine": "Mach 2", "Device": "Dev c", "Identifier": "HMI 6"},
{"Machine": "Mach 2", "Device": "Dev d", "Identifier": "HMI 7"},
{"Machine": "Mach 3", "Device": "Dev e", "Identifier": "HMI 8"}
]
I'm trying to generate a tree of the form:
Tree to be generated
[ ]- Mach 1
+[ ]- Dev a
| +-- HMI 2
| +-- HMI 3
+[ ]- Dev c
+-- HMI 5
[ ]- Mach 2
+[ ]- Dev c
| +-- HMI 6
+[ ]- Dev d
| +-- HMI 7
+[ ]- Dev e
+-- HMI 8
The output of the function is to be used by Inductive Automation's Perspective Tree component which expects it in the format:
items = [
{
"label": "Mach 1",
"expanded": true,
"data": "",
"items": [
{
"label": "Dev a",
"expanded": true,
"data": "",
"items": [
{
"label": "HMI 1",
"expanded": true,
"data": {
"Identifier": "HMI1",
"Device": "Dev a",
"Machine": "Mach 1"
},
"items": []
},
{
"label": "HMI 2",
"expanded": true,
"data": {
"Identifier": "HMI2",
"Device": "Dev a",
"Machine": "Mach 1"
},
"items": []
}
]
},
{
"label": "Dev b",
"expanded": true,
"data": "",
"items": [
{
"label": "HMI 3",
"expanded": true,
"data": {
"Identifier": "HMI3",
"Device": "Dev b",
"Machine": "Mach 1"
},
"items": []
}
]
}
]
},
…
I have created some linear Python code for a tree depth of three but I'd like to modify it to work automatically with tree depth from 1 to 6 (or so) returned by the SQL query. (The sample input and output above is three-level.) Unfortunately I can't figure out how to modify this to work with recursion for a variable number of columns.
Figure 1. The results of my lazy code (available on request).
Can anyone suggest an approach using Python - the script language of the Ignition application I'm using?
Many thanks.
You would need to provide the order in which the keys should be used to drill down in the hierarchy. This is good practice, as the order of the keys in a dictionary might not represent the desired order.
Once you have these keys as a list, you could use it to iteratively dig deeper into the hierarchy.
def makeForest(values, levels):
items = [] # The top level result array
paths = {} # Objects keyed by path
root = { "items": items } # Dummy: super root of the forest
for data in values:
parent = root
path = ""
for key in levels:
label = data[key]
path += repr([label])
node = paths.get(path, None)
if not node:
node = {
"label": data[key],
"expanded": True,
"data": "",
"items": []
}
paths[path] = node
parent["items"].append(node)
parent = node
parent["data"] = data
return items
# Example use:
value = [{"Machine": "Mach 1", "Device": "Dev a", "Identifier": "HMI 1"},{"Machine": "Mach 1", "Device": "Dev a", "Identifier": "HMI 2"},{"Machine": "Mach 1", "Device": "Dev b", "Identifier": "HMI 3"},{"Machine": "Mach 1", "Device": "Dev c", "Identifier": "HMI 5"},{"Machine": "Mach 2", "Device": "Dev c", "Identifier": "HMI 6"},{"Machine": "Mach 2", "Device": "Dev d", "Identifier": "HMI 7"},{"Machine": "Mach 3", "Device": "Dev e", "Identifier": "HMI 8"}]
forest = makeForest(value, ["Machine", "Device", "Identifier"])
print(forest)
Related
this reponse I am getting:
{
"value": [
{
"id": "/providers/Microsoft.Billing/Departments/1234/providers/Microsoft.Billing/billingPeriods/201903/providers/Microsoft.Consumption/usageDetails/usageDetails_Id1",
"name": "usageDetails_Id1",
"type": "Microsoft.Consumption/usageDetails",
"kind": "legacy",
"tags": {
"env": "newcrp",
"dev": "tools"
},
"properties": {
"billingAccountId": "xxxxxxxx",
"billingAccountName": "Account Name 1",
"billingPeriodStartDate": "2019-03-01T00:00:00.0000000Z",
"billingPeriodEndDate": "2019-03-31T00:00:00.0000000Z",
"billingProfileId": "xxxxxxxx",
"billingProfileName": "Account Name 1",
"accountName": "Account Name 1",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"subscriptionName": "Subscription Name 1",
"date": "2019-03-30T00:00:00.0000000Z",
"product": "Product Name 1",
"partNumber": "Part Number 1",
"meterId": "00000000-0000-0000-0000-000000000000",
"meterDetails": null,
"quantity": 0.7329,
"effectivePrice": 0.000402776395232,
"cost": 0.000295194820065,
"unitPrice": 4.38,
"billingCurrency": "CAD",
"resourceLocation": "USEast",
"consumedService": "Microsoft.Storage",
"resourceId": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/Resource Group 1/providers/Microsoft.Storage/storageAccounts/Resource Name 1",
"resourceName": "Resource Name 1",
"invoiceSection": "Invoice Section 1",
"costCenter": "DEV",
"resourceGroup": "Resource Group 1",
"offerId": "Offer Id 1",
"isAzureCreditEligible": false,
"chargeType": "Usage",
"benefitId": "00000000-0000-0000-0000-000000000000",
"benefitName": "Reservation_purchase_03-09-2018_10-59"
}
},
{
"id": "/providers/Microsoft.Billing/Departments/1234/providers/Microsoft.Billing/billingPeriods/201903/providers/Microsoft.Consumption/usageDetails/usageDetails_Id1",
"name": "usageDetails_Id1",
"type": "Microsoft.Consumption/usageDetails",
"kind": "legacy",
"tags": {
"env": "newcrp",
"dev": "tools"
},
"properties": {
"billingAccountId": "xxxxxxxx",
"billingAccountName": "Account Name 1",
"billingPeriodStartDate": "2019-03-01T00:00:00.0000000Z",
"billingPeriodEndDate": "2019-03-31T00:00:00.0000000Z",
"billingProfileId": "xxxxxxxx",
"billingProfileName": "Account Name 1",
"accountName": "Account Name 1",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"subscriptionName": "Subscription Name 1",
"date": "2019-03-30T00:00:00.0000000Z",
"product": "Product Name 1",
"partNumber": "Part Number 1",
"meterId": "00000000-0000-0000-0000-000000000000",
"meterDetails": null,
"quantity": 0.7329,
"effectivePrice": 0.000402776395232,
"cost": 0.000295194820065,
"unitPrice": 4.38,
"billingCurrency": "CAD",
"resourceLocation": "USEast",
"consumedService": "Microsoft.Storage",
"resourceId": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/Resource Group 1/providers/Microsoft.Storage/storageAccounts/Resource Name 1",
"resourceName": "Resource Name 1",
"invoiceSection": "Invoice Section 1",
"costCenter": "DEV",
"resourceGroup": "Resource Group 1",
"offerId": "Offer Id 1",
"isAzureCreditEligible": false,
"chargeType": "Usage",
"benefitId": "00000000-0000-0000-0000-000000000000",
"benefitName": "Reservation_purchase_03-09-2018_10-59"
}
}
]
}
code:
import pandas as pd
frame=pd.DataFrame()
for i in range (len(json_output['value'])):
df1= pd.DataFrame(data={'kind':json_output['value'][i]['kind'],
'id': json_output['value'][i]['id'],
'tags': json_output['value'][i]['tags'],
'name':json_output['value'][i]['name'],
'type':json_output['value'][i]['type'],
'billingAccountid':json_output['value'][i]['properties']['billingAccountId']},index=[i])
print(df1)
frame=frame.append(df1)
frame.to_csv('datt.csv')
Can you please help me to convert this data in to csv.
I am looking for
id,name,type,kind,tags,billingAccountId,resourceName etc into all column
I tried to convert into DataFrame it didn't work.
At last I am trying above python but its giving tags into null.
Note : I want to keep tags in dict format (for now)
I tried your code and stored json file into an output first:
-TAGS is a dictionary you access it without any keys so it will be NONE
If not comfortable by splitting TAGS use:
'tags':json_output['value'][i]['tags']['env']+json_output['value'][i]['tags']['dev']
Need to perform a python data transformation from the left format to the right format for a excel file.
This is a very common business problem in the finance world, matching debit vs credits to get even. I guess we might need a for loop, but after googling without success.
any suggestions will be highly appreciated. the original data set is in json format below. thanks
Transformation requirement
{
"from": [
{
"scenario": "case 1",
"amount": "55.65",
"debit/credit": "debit",
"uid": "S001"
},
{
"scenario": "case 1",
"amount": "43.98",
"debit/credit": "debit",
"uid": "S002"
},
{
"scenario": "case 1",
"amount": "21.52",
"debit/credit": "credit",
"uid": "S003"
},
{
"scenario": "case 1",
"amount": "4.5",
"debit/credit": "credit",
"uid": "S004"
},
{
"scenario": "case 1",
"amount": "23.78",
"debit/credit": "credit",
"uid": "S005"
},
{
"scenario": "case 1",
"amount": "0.99",
"debit/credit": "credit",
"uid": "S006"
},
{
"scenario": "case 1",
"amount": "48.84",
"debit/credit": "credit",
"uid": "S007"
},
{
"scenario": "case 2",
"amount": "88.38",
"debit/credit": "debit",
"uid": "S008"
},
{
"scenario": "case 2",
"amount": "9.95",
"debit/credit": "debit",
"uid": "S009"
},
{
"scenario": "case 2",
"amount": "4.23",
"debit/credit": "credit",
"uid": "S010"
},
{
"scenario": "case 2",
"amount": "94.1",
"debit/credit": "credit",
"uid": "S011"
}
]
}
You can read in your data as a json file. Then use pandas.read_json
method to convert to a pandas data frame. The following will do what
you want.
import pandas as pd
data = pd.read_json("./debit_credit.json")
# boolean mask: whether debit or credit
debits = data['debit/credit'] == 'debit'
credits = data['debit/credit'] == 'credit'
# desired output dataframes
debits_df = data.loc[debits]
credits_df = data.loc[credits]
print(debits_df)
print(credits_df)
# whether debits and credits match
is_match = debits_df.amount.sum() == credits_df.amount.sum()
print(f'credit and debit match: {is_match}')
Sample
template = {
"Table": [
{
"level": 2,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"flag": "flag",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": [
{
"level": 3,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": []
}
]
}
]
}
Considering the above dictionary, I want to append to the last "children" and every time loop runs it should append to the children created in previous iteration.
Loop 1:
"children":{
"level": 4,
"value": {"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"},
"RHS": {},
"children":
[]
}
Loop 2:
iteration 2
"children":{
"level": 5,
"value": {"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"},
"RHS": {},
"children":
[]
}
and so on.
My code is:
Python code for loop
for _ in range(sublevels):
number = number + 1
child = {"level": sublevels + 2,
"value": {"element Name": "", "add Row": False,
"cost Type": "", "cost": "",
"actions": True}, "RHS": {}, "children": []}
template['Table'][0]['children'].append(child)
Output:
After iteration, the JSON should look like below
{
"Table": [
{
"level": 2,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"flag": "flag",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": [
{
"level": 3,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": [
[
{
"level": 4,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": [
[
{
"level": 5,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": []
}
]
]
}
]
]
}
]
}
]
}
Iteration 1: template['Table'][0]['children']
Iteration 2: template['Table'][0]['children'][0]['children']
Iteration 3: template['Table'][0]['children'][0]['children'][0]['children']
import json
template = {"Table": []}
sublevels = 5
for _ in range(sublevels):
#number = number + 1
child = {"level": _ + 2,
"value": {"element Name": "", "add Row": False,
"cost Type": "", "cost": "",
"actions": True}, "RHS": {}, "children": []}
cur_path = "[0]['children']"*_
if _ == 0:
template['Table'].append(child)
else:
exec(f"template['Table']{cur_path}.append(child)")
print(json.dumps(template, indent = 2))
Not the prettiest way, you should avoid using exec, but I was trying to call a JSON path from a dict, and it wasn't working so I used exec.
This works well and nests it tho..
Output I got from running this code:
{
"Table": [
{
"level": 2,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": [
{
"level": 3,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": [
{
"level": 4,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": [
{
"level": 5,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": [
{
"level": 6,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": []
}
]
}
]
}
]
}
]
}
]
}
I am learning Python (teaching myself through articles and tutorials) and have come across something that I need assistance with. I have JSON data that represents TV listings, and I want to read it, edit it (remove outdated listings), and rewrite it back out. The sticking points for me are the nested data and how to iterate through/reference it to skip over the objects I don't want when writing out. Thanks. Here is some sample data:
{
"1": {
"channel_id": "1",
"img": "https://guide.tv/assets/images/channels/1.png",
"items": [
{
"category": "Comedy",
"channel": "1",
"description": "Latest episode of show A",
"end_time": "2017-09-11 20:30:00",
"language": "us",
"name": "Show A",
"quality": "720p",
"runtime": "30",
"time": "2017-09-11 20:00:00",
"version": "None"
},
{
"category": "Comedy",
"channel": "1",
"description": "Latest episode of show B",
"end_time": "2017-09-12 21:00:00",
"language": "us",
"name": "Show B",
"quality": "720p",
"runtime": "30",
"time": "2017-09-12 20:30:00",
"version": "None"
},
],
"name": "01 - NBC"
},
"2": {
"channel_id": "2",
"img": "https://guide.tv/assets/images/channels/2.png",
"items": [
{
"category": "Drama",
"channel": "2",
"description": "Latest episode of show C",
"end_time": "2017-09-10 23:00:00",
"language": "us",
"name": "Show C",
"quality": "720p",
"runtime": "180",
"time": "2017-09-10 20:00:00",
"version": "None"
},
{
"category": "Drama",
"channel": "2",
"description": "Latest episode of show D",
"end_time": "2017-09-11 23:00:00",
"language": "us",
"name": "Show D",
"quality": "720p",
"runtime": "60",
"time": "2017-09-11 22:00:00",
"version": "None"
},
{
"category": "Action",
"channel": "2",
"description": "Latest episode of Show E",
"end_time": "2017-09-11 22:00:00",
"language": "us",
"name": "Show E",
"quality": "720p",
"runtime": "180",
"time": "2017-09-11 19:00:00",
"version": "None"
},
{
"category": "Fiction",
"channel": "2",
"description": "Latest episode of show F",
"end_time": "2017-09-10 19:00:00",
"language": "us",
"name": "Show F",
"quality": "720p",
"runtime": "180",
"time": "2017-09-10 16:00:00",
"version": "None"
},
],
"name": "02 - CBS"
},
"3": {
"channel_id": "3",
"img": "https://guide.tv/assets/images/channels/3.png",
"items": [
{
"category": "Comedy",
"channel": "3",
"description": "Latest episode of show G",
"end_time": "2017-09-18 12:00:00",
"language": "us",
"name": "Show G",
"quality": "hqlq",
"runtime": "120",
"time": "2017-09-18 10:00:00",
"version": "None"
},
{
"category": "Action",
"channel": "3",
"description": "Latest episode of show H",
"end_time": "2017-09-19 12:00:00",
"language": "us",
"name": "Show H",
"quality": "hqlq",
"runtime": "120",
"time": "2017-09-19 10:00:00",
"version": "None"
},
],
"name": "03 - ABC"
}
}
This is the code I have tried:
import json
with open('file.json') as data_file:
data = json.load(data_file)
for element in data.values():
if 'items' in element:
for e2 in element['items']:
if '2017-09-10' in e2['time']:
del e2
print json.dumps(data, indent=4, sort_keys=True)
Given how del works, that's not too surprising. You want to remove an item from the list of items, not "delete a declared variable from the current execution scope". So instead of del e2, remove e2 from element['items'], using remove and friends, e.g.:
elements = data.values()
for element in elements:
if 'items' in element:
listing = element['items']
for entry in listing:
if 'time' in entry and '2017-09-10' in entry['time']:
listing.remove(entry)
I'm working in an application in Python. I have to do a GET request to get specific information. My code is somethings like that:
...
conn = httplib.HTTPConnection(self.url)
header = {"Authorization":"Bearer "+self.token}
conn.request("GET","/data",headers=header)
...
and the JSON that I obtain is similar to this (you can observe that there are 2 big different part... this is just an example, in my application the parts are a lots).
[
{
"createdAt": "2015-11-26T10:06:05.756Z",
"date": "2015-10-31T23:00:00.000Z",
"files": [],
"id": 1,
"metadata": {},
"notes": "note impianto 1",
"parentSubject": {
"code": "soggetto1",
"createdAt": "2015-11-26T10:05:38.765Z",
"id": 1,
"metadata": {},
"notes": "note soggetto 1",
"personalInfo": 1,
"sex": "M",
"tags": null,
"type": 1,
"updatedAt": "2015-11-26T10:05:38.765Z"
}
},
{
"createdAt": "2015-11-26T10:06:36.684Z",
"date": "2015-11-01T23:00:00.000Z",
"files": [],
"id": 2,
"metadata": {},
"notes": "note impianto 2",
"parentSubject": {
"code": "soggetto1",
"createdAt": "2015-11-26T10:05:38.765Z",
"id": 1,
"metadata": {},
"notes": "note soggetto 1",
"personalInfo": 1,
"sex": "M",
"tags": null,
"type": 1,
"updatedAt": "2015-11-26T10:05:38.765Z"
}
}
]
If for example I make this request:
...
conn.request("GET","/data?id=1",headers=header)
...
I obviously get only the first part. The problem is that I don't want to get all data that have id=1 but all data that have code=soggetto1. How can I do?
If API can't do this then you have to get all data and find on your own.
In example I changed one code to "soggetto2"
data = '''[
{
"createdAt": "2015-11-26T10:06:05.756Z",
"date": "2015-10-31T23:00:00.000Z",
"files": [],
"id": 1,
"metadata": {},
"notes": "note impianto 1",
"parentSubject": {
"code": "soggetto2",
"createdAt": "2015-11-26T10:05:38.765Z",
"id": 1,
"metadata": {},
"notes": "note soggetto 1",
"personalInfo": 1,
"sex": "M",
"tags": null,
"type": 1,
"updatedAt": "2015-11-26T10:05:38.765Z"
}
},
{
"createdAt": "2015-11-26T10:06:36.684Z",
"date": "2015-11-01T23:00:00.000Z",
"files": [],
"id": 2,
"metadata": {},
"notes": "note impianto 2",
"parentSubject": {
"code": "soggetto2",
"createdAt": "2015-11-26T10:05:38.765Z",
"id": 1,
"metadata": {},
"notes": "note soggetto 1",
"personalInfo": 1,
"sex": "M",
"tags": null,
"type": 1,
"updatedAt": "2015-11-26T10:05:38.765Z"
}
}
]'''
#------------------------------------------------------
import json
j = json.loads(data)
results = []
for x in j:
if x["parentSubject"]["code"] == "soggetto1":
results.append(x)
print results
You can do this with list comprehension
results = [ x for x in j if x["parentSubject"]["code"] == "soggetto1" ]
or filter()
results = filter(lambda x:x["parentSubject"]["code"] == "soggetto1", j)