How to map fields in NumPy - python

I have a script that writes a JSON web-service to an esri file geodatabase. There is a 1-M relationship between address and requests, the requests are read and written as 3 fields. Below is an example of JSON response and how the data appears in a table. I would like my data to read as one feature/row with multiple request types and quantities associated with the one address and Service Request number. i.e. Address, Type, E-Waste Item 1, Quantity 1, E-Waste Item 2, Quantity 2, etc..
{
"Response": {
"ListOfServiceRequest": {
"ServiceRequest": [
{
"ActionTaken": "",
"AddressVerified": "Y",
"Anonymous": "N",
"AssignTo": "EV",
"Assignee": "",
"CreatedByUserLogin": "MYLATHREEONEONE",
"CreatedDate": "02/17/2015 16:53:25",
"CustomerAccessNumber": "",
"Email": "mylathreeoneone#gmail.com",
"FirstName": "Myla",
"HomePhone": "2131234567",
"IntegrationId": "02172015165417667",
"LADWPAccountNo": "",
"Language": "",
"LastName": "Threeoneone",
"Latitude": "34.176277",
"ListOfAuditTrailItem2": {
"AuditTrailItem2": [
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Status",
"NewValue": "Closed",
"OldValue": "Open"
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Updated By User",
"NewValue": "EAIBOS",
"OldValue": "MYLATHREEONEONE"
}
]
},
"ListOfChildServiceRequest": {},
"ListOfLa311BarricadeRemoval": {},
"ListOfLa311BeesOrBeehive": {},
"ListOfLa311BillingCsscAdjustment": {},
"ListOfLa311BillingEccAdjustment": {},
"ListOfLa311BillingRsscAdjustment": {},
"ListOfLa311BillingRsscExemption": {},
"ListOfLa311BrushItemsPickup": {},
"ListOfLa311BulkyItem": {},
"ListOfLa311BusPadLanding": {},
"ListOfLa311Containers": {},
"ListOfLa311CurbRepair": {},
"ListOfLa311DeadAnimalRemoval": {},
"ListOfLa311DocumentLog": {},
"ListOfLa311ElectronicWaste": {
"La311ElectronicWaste": [
{
"CollectionLocation": "Gated Community/Multifamily Dw",
"DriverFirstName": "Moody",
"DriverLastName": "Frederick10/09/2014",
"ElectronicWestType": "Microwaves",
"GatedCommunityMultifamilyDwelling": "Curb",
"IllegalDumpCollectionLoc": "",
"IllegallyDumped": "N",
"ItemCount": "3",
"LastUpdatedBy": "",
"MobileHomeSpace": "",
"Name": "021720151654176711",
"OtherElectronicWestType": "",
"ServiceDateRendered": "",
"TruckNo": "SC Truck 10",
"Type": "Electronic Waste"
},
{
"CollectionLocation": "Gated Community/Multifamily Dw",
"DriverFirstName": "Moody",
"DriverLastName": "Frederick10/09/2014",
"ElectronicWestType": "Televisions (Any Size)",
"GatedCommunityMultifamilyDwelling": "Curb",
"IllegalDumpCollectionLoc": "",
"IllegallyDumped": "N",
"ItemCount": "6",
"LastUpdatedBy": "",
"MobileHomeSpace": "",
"Name": "021720151654176722",
"OtherElectronicWestType": "",
"ServiceDateRendered": "",
"TruckNo": "SC Truck 10SC Truck 10",
"Type": "Electronic Waste"
},
{
"CollectionLocation": "Gated Community/Multifamily Dw",
"DriverFirstName": "Moody",
"DriverLastName": "Frederick10/09/2014",
"ElectronicWestType": "VCR/DVD Players",
"GatedCommunityMultifamilyDwelling": "Curb",
"IllegalDumpCollectionLoc": "",
"IllegallyDumped": "N",
"ItemCount": "1",
"LastUpdatedBy": "",
"MobileHomeSpace": "",
"Name": "021720151654176723",
"OtherElectronicWestType": "",
"ServiceDateRendered": "",
"TruckNo": "SC Truck 10SC Truck 10",
"Type": "Electronic Waste"
}
]
},
"ListOfLa311Flooding": {},
"ListOfLa311GeneralStreetInspection": {},
"ListOfLa311GenericBc": {
"La311GenericBc": [
{
"ATTRIB_08": "N",
"ATTRIB_16": "",
"ListOfLa311GenericbcAuditTrail": {
"La311GenericbcAuditTrail": [
{
"Date": "02/27/2015 15:15:28",
"EmployeeLogin": "EAIBOS",
"Field": "Driver Last Name",
"NewValue": "Frederick10/09/2014",
"OldValue": "Frederick"
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Driver First Name",
"NewValue": "Moody",
"OldValue": ""
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Driver Last Name",
"NewValue": "Frederick",
"OldValue": ""
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Truck No",
"NewValue": "SC Truck 10",
"OldValue": ""
}
]
},
"NAME": "021720151654176711",
"PAR_ROW_ID": "1-24QH7",
"ROW_ID": "1-24QHR",
"TYPE": "Electronic Waste"
},
{
"ATTRIB_08": "N",
"ATTRIB_16": "",
"ListOfLa311GenericbcAuditTrail": {
"La311GenericbcAuditTrail": [
{
"Date": "02/27/2015 15:15:28",
"EmployeeLogin": "EAIBOS",
"Field": "Driver Last Name",
"NewValue": "Frederick10/09/2014",
"OldValue": "Frederick"
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Driver First Name",
"NewValue": "Moody",
"OldValue": ""
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Driver Last Name",
"NewValue": "Frederick",
"OldValue": ""
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Truck No",
"NewValue": "SC Truck 10SC Truck 10",
"OldValue": ""
}
]
},
"NAME": "021720151654176722",
"PAR_ROW_ID": "1-24QH7",
"ROW_ID": "1-24QHS",
"TYPE": "Electronic Waste"
},
{
"ATTRIB_08": "N",
"ATTRIB_16": "",
"ListOfLa311GenericbcAuditTrail": {
"La311GenericbcAuditTrail": [
{
"Date": "02/27/2015 15:15:28",
"EmployeeLogin": "EAIBOS",
"Field": "Driver Last Name",
"NewValue": "Frederick10/09/2014",
"OldValue": "Frederick"
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Driver First Name",
"NewValue": "Moody",
"OldValue": ""
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Driver Last Name",
"NewValue": "Frederick",
"OldValue": ""
},
{
"Date": "02/27/2015 11:45:46",
"EmployeeLogin": "EAIBOS",
"Field": "Truck No",
"NewValue": "SC Truck 10SC Truck 10",
"OldValue": ""
}
]
},
"NAME": "021720151654176723",
"PAR_ROW_ID": "1-24QH7",
"ROW_ID": "1-24QHT",
"TYPE": "Electronic Waste"
},
{
"ATTRIB_08": "",
"ATTRIB_16": "",
"ListOfLa311GenericbcAuditTrail": {},
"NAME": "02172015165417667100",
"PAR_ROW_ID": "1-24QH7",
"ROW_ID": "1-24QHU",
"TYPE": "GIS"
},
{
"ATTRIB_08": "",
"ATTRIB_16": "",
"ListOfLa311GenericbcAuditTrail": {},
"NAME": "021720151654176671",
"PAR_ROW_ID": "1-24QH7",
"ROW_ID": "1-24QHQ",
"TYPE": "SR Photo ID"
}
]
},
"ListOfLa311GisLayer": {
"La311GisLayer": [
{
"A_Call_No": "",
"Area": "0",
"BOSRadioHolderName": "",
"CommunityPlanningArea": "",
"Day": "THURSDAY",
"DirectionSuffix": "",
"DistrictAbbr": "",
"DistrictName": "EV",
"DistrictNumber": "",
"DistrictOffice": "",
"Fraction": "",
"LastUpdatedBy": "",
"Name": "02172015165417667100",
"R_Call_No": "",
"SectionId": "",
"ShortDay": "Thu",
"StreetFrom": "",
"StreetLightId": "",
"StreetLightStatus": "",
"StreetTo": "",
"Type": "GIS",
"Y_Call_No": ""
}
]
},
"ListOfLa311GraffitiRemoval": {},
"ListOfLa311GuardWarningRailMaintenance": {},
"ListOfLa311GutterRepair": {},
"ListOfLa311HomelessEncampment": {},
"ListOfLa311IllegalAutoRepair": {},
"ListOfLa311IllegalConstruction": {},
"ListOfLa311IllegalConstructionFence": {},
"ListOfLa311IllegalDischargeOfWater": {},
"ListOfLa311IllegalDumpingInProgress": {},
"ListOfLa311IllegalDumpingPickup": {},
"ListOfLa311IllegalExcavation": {},
"ListOfLa311IllegalSignRemoval": {},
"ListOfLa311IllegalVending": {},
"ListOfLa311InformationOnly": {},
"ListOfLa311LandMudSlide": {},
"ListOfLa311LeafBlowerViolation": {},
"ListOfLa311ManualPickup": {},
"ListOfLa311MedianIslandMaintenance": {},
"ListOfLa311MetalHouseholdAppliancesPickup": {},
"ListOfLa311MoveInMoveOut": {},
"ListOfLa311MultipleStreetlightIssue": {},
"ListOfLa311NewsRackViolation": {},
"ListOfLa311Obstructions": {},
"ListOfLa311Other": {},
"ListOfLa311OvergrownVegetationPlants": {},
"ListOfLa311PalmFrondsDown": {},
"ListOfLa311Pothole": {},
"ListOfLa311Resurfacing": {},
"ListOfLa311SanitationBillingBif": {},
"ListOfLa311SanitationBillingCssc": {},
"ListOfLa311SanitationBillingEcc": {},
"ListOfLa311SanitationBillingInquiry": {},
"ListOfLa311SanitationBillingLifeline": {},
"ListOfLa311SanitationBillingRssc": {},
"ListOfLa311SanitationBillingSrf": {},
"ListOfLa311ServiceNotComplete": {},
"ListOfLa311ServiceRequestNotes": {
"La311ServiceRequestNotes": [
{
"Comment": "Out on the sidewalk near the curb. Hopefully it is still there.",
"CommentType": "Address Comments",
"CreatedByUser": "MYLATHREEONEONE",
"CreatedDate": "02/17/2015 16:53:26",
"Date1": "",
"Date2": "",
"Date3": "",
"FeedbackSRType": "",
"IntegrationId": "021720151654176661",
"IsSrNoAvailable": "",
"ListOfLa311SrNotesAuditTrail": {},
"Notification": "N",
"Text1": ""
},
{
"Comment": "So glad to get rid of this old junk. Thanks.",
"CommentType": "External",
"CreatedByUser": "MYLATHREEONEONE",
"CreatedDate": "02/17/2015 16:53:26",
"Date1": "",
"Date2": "",
"Date3": "",
"FeedbackSRType": "",
"IntegrationId": "021720151654176662",
"IsSrNoAvailable": "",
"ListOfLa311SrNotesAuditTrail": {},
"Notification": "N",
"Text1": ""
}
]
},
"ListOfLa311SidewalkRepair": {},
"ListOfLa311SingleStreetlightIssue": {},
"ListOfLa311SrPhotoId": {
"La311SrPhotoId": [
{
"LastUpdatedBy": "",
"Name": "021720151654176671",
"PhotoId": "https://myla311.lacity.org/portal/docview?id=04b8ba678fe21d32b05673eb9ad7711b",
"Type": "SR Photo ID"
}
]
},
"ListOfLa311StreetSweeping": {},
"ListOfLa311StreetTreeInspection": {},
"ListOfLa311StreetTreeViolations": {},
"ListOfLa311SubscribeDuplicateSr": {},
"ListOfLa311TablesAndChairsObstructing": {},
"ListOfLa311TreeEmergency": {},
"ListOfLa311TreeObstruction": {},
"ListOfLa311TreePermits": {},
"ListOfLa311WeedAbatementForPrivateParcels": {},
"LoginUser": "",
"Longitude": "-118.455249",
"MobilOS": "",
"NewContactEmail": "",
"NewContactFirstName": "",
"NewContactLastName": "",
"NewContactPhone": "",
"Owner": "BOS",
"ParentSRLinkDate": "",
"ParentSRLinkUser": "",
"ParentSRNumber": "",
"ParentSRStatus": "",
"ParentSRType": "",
"Priority": "Normal",
"ReasonCode": "",
"RescheduleCounter": "",
"ResolutionCode": "SW",
"SRAddress": "5810 N WILLIS AVE, 91411",
"SRAddressName": "",
"SRAreaPlanningCommission": "South Valley APC",
"SRAreaPlanningCommissionId": "3",
"SRCity": "",
"SRCommunityPoliceStation": "VALLEY BUREAU",
"SRCommunityPoliceStationAPREC": "VAN NUYS",
"SRCommunityPoliceStationPREC": "9",
"SRCouncilDistrictMember": "Tom LaBonge",
"SRCouncilDistrictNo": "4",
"SRCrossStreet": "",
"SRDirection": "N",
"SRHouseNumber": "5810",
"SRNeighborhoodCouncilId": "20",
"SRNeighborhoodCouncilName": "VAN NUYS NC",
"SRNumber": "1-3580171",
"SRStreetName": "WILLIS",
"SRSuffix": "AVE",
"SRTBColumn": "J",
"SRTBMapGridPage": "561",
"SRTBRow": "1",
"SRType": "Electronic Waste",
"SRUnitNumber": "5810",
"SRXCoordinate": "6423983",
"SRYCoordinate": "1886848",
"ServiceDate": "02/19/2015 00:00:00",
"Source": "",
"Status": "Closed",
"UpdatedByUserLogin": "EAIBOS",
"UpdatedDate": "02/27/2015 15:14:22",
"Zipcode": "91411"
}
]
},
"NumOutputObjects": "1"
Script
import json
import jsonpickle
import requests
import arcpy
fc = "C:\MYLATesting.gdb\MYLA311"
if arcpy.Exists(fc):
arcpy.Delete_management(fc)
ListTable ="C:\MYLATesting.gdb\ListView"
if arcpy.Exists(ListTable):
arcpy.Delete_management(ListTable)
f2 = open('C:\Users\Administrator\Desktop\DetailView.json', 'r')
data2 = jsonpickle.encode( jsonpickle.decode(f2.read()) )
url2 = "myURL"
headers2 = {'Content-type': 'text/plain', 'Accept': '/'}
r2 = requests.post(url2, data=data2, headers=headers2)
decoded2 = json.loads(r2.text)
items = []
for sr in decoded2['Response']['ListOfServiceRequest']['ServiceRequest']:
SRAddress = sr['SRAddress']
latitude = sr['Latitude']
longitude = sr['Longitude']
for ew in sr["ListOfLa311ElectronicWaste"][u"La311ElectronicWaste"]:
CommodityType = ew['Type']
ItemType = ew['ElectronicWestType']
ItemCount = ew['ItemCount']
items.append((SRAddress,
latitude,
longitude,
CommodityType,
ItemType,
ItemCount))
import numpy as np #NOTE THIS
dt = np.dtype([('SRAddress', 'U40'),
('latitude', '<f8'),
('longitude', '<f8'),
('Type', 'U40'),
('ElectronicWestType', 'U40'),
('ItemCount', 'U40')])
arr = np.array(items,dtype=dt)
sr = arcpy.SpatialReference(4326)
arcpy.da.NumPyArrayToFeatureClass(arr, fc, ['longitude', 'latitude'], sr )
print json.dumps(decoded2, sort_keys=True, indent=4)

Related

Consumption list API returns unexpected service period date

I use python sdk to fetch resource usage from azure. However, I realized that API is returning unexpected service_period_start_date and service_period_end_date .
service_period_end_date: 0001-01-01 00:00:00+00:00
service_period_start_date: 0001-01-01 00:00:00+00:00
Code snippet is here .
I also checked the az rest interface and same thing is happening.
az rest --url https://management.azure.com/subscriptions/741ff54c-7bab-4522-bb47-37f0452178f6/providers/Microsoft.Consumption/usageDetails?api-version=2021-10-01 --url-parameters metric="AmortizedCost" &> resp.log
{
"nextLink": "https://management.azure.com/subscriptions/xxxxx/providers/Microsoft.Consumption/usageDetails?api-version=2021-10-01&metric=AmortizedCost&sessiontoken=&$skiptoken=317C313030307C313030307C46616C73657C547275657C30&skiptokenver=v1&id=2c65fabe-8dc2-4000-944e-ae199a7044e0",
"value": [
{
"id": "/subscriptions/xxxxx/providers/Microsoft.Consumption/usageDetails/b4db9023-a53d-dac8-7ef2-285bf751ccaf",
"kind": "modern",
"name": "b4db9023-a53d-dac8-7ef2-285bf751ccaf",
"properties": {
"additionalInfo": "",
"benefitId": "",
"benefitName": "",
"billingAccountId": "xxxxxx",
"billingAccountName": "",
"billingCurrencyCode": "JPY",
"billingPeriodEndDate": "0001-01-01T00:00:00Z",
"billingPeriodStartDate": "0001-01-01T00:00:00Z",
"billingProfileId": "",
"billingProfileName": "",
"chargeType": "Usage",
"consumedService": "Microsoft.Storage",
"costAllocationRuleName": "",
"costCenter": "",
"costInBillingCurrency": 0.0,
"costInPricingCurrency": 0.0,
"costInUSD": 0.0,
"customerName": "xxxxxx",
"customerTenantId": "xxxxxx",
"date": "2022-11-10T00:00:00Z",
"effectivePrice": 0.00036,
"exchangeRate": "146.0150000000000633997130000000275",
"exchangeRateDate": "2022-11-01T00:00:00Z",
"exchangeRatePricingToBilling": 146.015,
"frequency": "UsageBased",
"instanceName": "/subscriptions/xxxxxx/resourceGroups/catseye/providers/Microsoft.Storage/storageAccounts/storageaccountcatsebc14",
"invoiceId": "",
"invoiceSectionId": "",
"invoiceSectionName": "",
"isAzureCreditEligible": true,
"meterCategory": "Storage",
"meterId": "40551b4c-e8be-48ed-b70b-f8d25c7de724",
"meterName": "Read Operations",
"meterRegion": "",
"meterSubCategory": "General Block Blob",
"partnerEarnedCreditApplied": "1",
"partnerEarnedCreditRate": 0.0,
"partnerName": "xxxxx",
"partnerTenantId": "xxxxxx",
"payGPrice": 0.00036,
"paygCostInBillingCurrency": 0.0,
"paygCostInUSD": 0.0,
"previousInvoiceId": "",
"pricingCurrencyCode": "USD",
"pricingModel": "OnDemand",
"product": "General Block Blob - LRS",
"productIdentifier": "DZH318Z0BNZH006F",
"productOrderId": "a0633fc8-5233-43c9-d9f6-f94d7d8a70f5",
"productOrderName": "Azure plan",
"provider": "",
"publisherId": "",
"publisherName": "Microsoft",
"publisherType": "Microsoft",
"quantity": 0.0004,
"resellerMpnId": "",
"resellerName": "",
"reservationId": "",
"reservationName": "",
"resourceGroup": "catseye",
"resourceLocation": "japaneast",
"resourceLocationNormalized": "JA East",
"serviceFamily": "Storage",
"serviceInfo1": "",
"serviceInfo2": "",
"servicePeriodEndDate": "0001-01-01T00:00:00Z",
"servicePeriodStartDate": "0001-01-01T00:00:00Z",
"subscriptionGuid": "xxxxxx",
"subscriptionName": "xxxxxx",
"term": "",
"unitOfMeasure": "10K",
"unitPrice": 0.00036
},
"tags": {
"project": "xxxxx"
},
"type": "Microsoft.Consumption/usageDetails"
},
I have no clue why am I seeing such date in the API response. The doc does not seem to have explanation about this .
I appreciate if someone can help me out finding the reason.

How to compare 2 JSON strings and store what changed in a variable Python

I have a JSON string that is a response to a GET request:
[
{
"symbol": "cosmic2",
"name": "Cosmic Condos",
"description": "Some cosmic condos",
"image": "https://bafybeif3tacmhinsivylzrrxskwshwufysst3s6np3y2ar3qagpmliw374.ipfs.dweb.link/72.png?ext=png",
"twitter": "",
"discord": "",
"website": "",
"categories": []
}
]
And after a while it becomes :
[
{
"symbol": "cosmic2",
"name": "Cosmic Condos",
"description": "Some cosmic condos",
"image": "",
"twitter": "",
"discord": "",
"website": "",
"categories": []
},
{
"symbol": "test_lp_1",
"name": "Test Launchpad 1",
"description": "3 Stages",
"image": "",
"categories": [
"launchpad"
]
}
]
How do I store the second part that was added in a variable and later work with it?

How to append to the child node in JSON at every iteration using python provided new child node exists?

Sample
template = {
"Table": [
{
"level": 2,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"flag": "flag",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": [
{
"level": 3,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": []
}
]
}
]
}
Considering the above dictionary, I want to append to the last "children" and every time loop runs it should append to the children created in previous iteration.
Loop 1:
"children":{
"level": 4,
"value": {"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"},
"RHS": {},
"children":
[]
}
Loop 2:
iteration 2
"children":{
"level": 5,
"value": {"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"},
"RHS": {},
"children":
[]
}
and so on.
My code is:
Python code for loop
for _ in range(sublevels):
number = number + 1
child = {"level": sublevels + 2,
"value": {"element Name": "", "add Row": False,
"cost Type": "", "cost": "",
"actions": True}, "RHS": {}, "children": []}
template['Table'][0]['children'].append(child)
Output:
After iteration, the JSON should look like below
{
"Table": [
{
"level": 2,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"flag": "flag",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": [
{
"level": 3,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": [
[
{
"level": 4,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": [
[
{
"level": 5,
"value": {
"element Name": "",
"add Row": "False",
"cost Type": "",
"cost": "",
"actions": "True"
},
"RHS": {},
"children": []
}
]
]
}
]
]
}
]
}
]
}
Iteration 1: template['Table'][0]['children']
Iteration 2: template['Table'][0]['children'][0]['children']
Iteration 3: template['Table'][0]['children'][0]['children'][0]['children']
import json
template = {"Table": []}
sublevels = 5
for _ in range(sublevels):
#number = number + 1
child = {"level": _ + 2,
"value": {"element Name": "", "add Row": False,
"cost Type": "", "cost": "",
"actions": True}, "RHS": {}, "children": []}
cur_path = "[0]['children']"*_
if _ == 0:
template['Table'].append(child)
else:
exec(f"template['Table']{cur_path}.append(child)")
print(json.dumps(template, indent = 2))
Not the prettiest way, you should avoid using exec, but I was trying to call a JSON path from a dict, and it wasn't working so I used exec.
This works well and nests it tho..
Output I got from running this code:
{
"Table": [
{
"level": 2,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": [
{
"level": 3,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": [
{
"level": 4,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": [
{
"level": 5,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": [
{
"level": 6,
"value": {
"element Name": "",
"add Row": false,
"cost Type": "",
"cost": "",
"actions": true
},
"RHS": {},
"children": []
}
]
}
]
}
]
}
]
}
]
}

Parsing json into a list of values

I have a program that takes some file and transforms it into a json format.
Im trying to get all the values of certain keys into a list but, because the format of the json file has a bunch of keys that are present multiple times, I cant find a way to do it properly.
My json file looks like this
{
"data": {
"__schema": {
"queryType": {
"fields": [
{
"description": "",
"name": "project"
},
{
"description": "",
"name": "projectEventFeed"
},
{
"description": "",
"name": "projectEventFeedFetchMore"
},
{
"description": "",
"name": "projectRecentEventFeed"
},
{
"description": "",
"name": "unseenProjectActivityCount"
},
{
"description": "",
"name": "projectFiles"
},
{
"description": "",
"name": "projectFilesIdSet"
},
{
"description": "",
"name": "projectFileMessages"
},
{
"description": "",
"name": "projectUserStatus"
},
{
"description": "",
"name": "projectFileScribble"
},
{
"description": "",
"name": "user"
},
{
"description": "",
"name": "viewer"
},
{
"description": "",
"name": "profile"
},
{
"description": "",
"name": "site"
},
{
"description": "",
"name": "designers"
},
{
"description": "",
"name": "predictImageCategory"
},
{
"description": "",
"name": "getPortfolioDesign"
}
]
}
}
}
}
My goal is to get all the name values into a list.
Before turning the file into json, I tried getting that with regex but failed.
With json format I tried the following
map(lambda parsed_json: parsed_json['data']['__schema']['queryType']['fields']['name'], List)
Im getting List from typing
But when i want to turn the map into a list, I get
TypeError: Parameters to generic types must be types. Got 0.
From the conversion.
You could just use list comprehension on the nested 'fields' key in the dict you have converted from your json.
d = {"data": {"__schema": {"queryType": {"fields": [{"description": "", "name": "project"}, {"description": "", "name": "projectEventFeed"}, {"description": "", "name": "projectEventFeedFetchMore"}, {"description": "", "name": "projectRecentEventFeed"}, {"description": "", "name": "unseenProjectActivityCount"}, {"description": "", "name": "projectFiles"}, {"description": "", "name": "projectFilesIdSet"}, {"description": "", "name": "projectFileMessages"}, {"description": "", "name": "projectUserStatus"}, {"description": "", "name": "projectFileScribble"}, {"description": "", "name": "user"}, {"description": "", "name": "viewer"}, {"description": "", "name": "profile"}, {"description": "", "name": "site"}, {"description": "", "name": "designers"}, {"description": "", "name": "predictImageCategory"}, {"description": "", "name": "getPortfolioDesign"}]}}}}
fields = [f['name'] for f in d['data']['__schema']['queryType']['fields']]
print(fields)
# ['project', 'projectEventFeed', 'projectEventFeedFetchMore', 'projectRecentEventFeed', 'unseenProjectActivityCount', 'projectFiles', 'projectFilesIdSet', 'projectFileMessages', 'projectUserStatus', 'projectFileScribble', 'user', 'viewer', 'profile', 'site', 'designers', 'predictImageCategory', 'getPortfolioDesign']

Handling multiple JSON data sets for POST request in Python

I have 3 JSON methods which are two query methods and an update method. I would like to parse through this information with and execute a POST request and pass this data into a database using arcpy for GIS use for all three methods. I have a script which works and gets a response, however the problem is locating the keys and values for each object so that I can successfully send my values to to a database.
Additionally, I handle this task with three different methods, all of which I need data from.
For instance,
query 1 would allow me to parse and find the address, lat/lng, etc.
query 2 would allow me to parse and find customer info, type of request, etc.
query 3 would allow me to update a request.
My first question is how do I successfully extract only the data that I want from each output; I have tested Key/Values in POSTman to no avail, the server is expecting an entire JSON file.
My second question is how do I handle 3 different requests; I am assuming 3 different post methods and selecting the data that I want.
Example of JSON one query passed to server
{
"RequestSpecificDetail": {
"ParentSRNumberForLink": ""
},
"MetaData": {
"appVersion": "1.34",
"deviceModel": "x86_64",
"dateAndTime": "01/15/2015 12:46:36",
"deviceToken": "A2C1DD9D-D17D-4031-BA3E-977C250BFD58",
"osVersion": "8.1"
},
"SRData": {
"LoginUser": "User89",
"NewContactEmail": "abc#gmail.com",
"UpdatedDate": "02/05/2015"
}
}
Example of Query 1 Output
{
"status": {
"code": 311,
"message": "Service Request Successfully Queried.",
"cause": ""
},
"Response": {
"NumOutputObjects": "2",
"ListOfServiceRequest": {
"ServiceRequest": [
{
"SRAddress": "1200 W TEMPLE ST, 90026",
"SRNumber": "1-5099871",
"SRType": "Feedback",
"CreatedDate": "02/05/2015 22:55:58",
"UpdatedDate": "02/05/2015 22:55:58",
"Status": "Open",
"imageURL": ""
},
{
"SRAddress": "1200 W TEMPLE ST, 90026",
"SRNumber": "1-5133051",
"SRType": "Feedback",
"CreatedDate": "02/05/2015 23:03:54",
"UpdatedDate": "02/05/2015 23:03:54",
"Status": "Open",
"imageURL": "https://SERVER_END_POINT/portal/docview?id=fe083ae14b52b1af0945b4d756c296a5"
}
]
},
"LastUpdateDate": "02/05/2015"
}
}
Example of Query 2 passed to server
{
"RequestSpecificDetail": {
"ParentSRNumberForLink": ""
},
"MetaData": {
"appVersion": "1.34",
"deviceModel": "x86_64",
"dateAndTime": "01/15/2015 12:46:36",
"deviceToken": "A2C1DD9D-D17D-4031-BA3E-977C250BFD58",
"osVersion": "8.1"
},
"SRData": {
"SRNumber": "1-1080871"
}
}
Query two output
{
"status": {
"code": 311,
"message": "Service Request Successfully Queried.",
"cause": ""
},
"Response": {
"NumOutputObjects": "1",
"ListOfServiceRequest": {
"ServiceRequest": [
{
"AddressVerified": "Y",
"SRNumber": "1-1080871",
"SRType": "Homeless Encampment",
"CreatedDate": "12/31/2014 13:49:23",
"UpdatedDate": "12/31/2014 13:49:23",
"IntegrationId": "1420033765921",
"Status": "Open",
"CreatedByUserLogin": "User89",
"UpdatedByUserLogin": "User89",
"Anonymous": "N",
"Zipcode": "90026",
"Latitude": "34.064937",
"Longitude": "-118.252968",
"CustomerAccessNumber": "",
"LADWPAccountNo": "",
"NewContactFirstName": "",
"NewContactLastName": "",
"NewContactPhone": "",
"NewContactEmail": "",
"ParentSRNumber": "",
"Priority": "Normal",
"Language": "English",
"ReasonCode": "",
"ServiceDate": "12/31/2014 00:00:00",
"Source": "311",
"Email": "user#email.com",
"FirstName": "User",
"HomePhone": "3123123123",
"LastName": "Pp",
"LoginUser": "",
"ResolutionCode": "",
"SRUnitNumber": "",
"MobilOS": "iOS",
"SRAddress": "1200 W TEMPLE ST, 90026",
"SRAddressName": "",
"SRAreaPlanningCommission": "Central APC",
"SRCommunityPoliceStation": "",
"SRCouncilDistrictMember": "Gilbert Cedillo",
"SRCouncilDistrictNo": "1",
"SRDirection": "W",
"SRNeighborhoodCouncilId": "44",
"SRNeighborhoodCouncilName": "GREATER ECHO PARK ELYSIAN NC",
"SRStreetName": "TEMPLE",
"SRSuffix": "ST",
"SRTBColumn": "E",
"SRTBMapGridPage": "634",
"SRTBRow": "2",
"SRXCoordinate": "6485064",
"SRYCoordinate": "1846114",
"AssignTo": "North Central - 104 - IED",
"Assignee": "Siebel Administrator",
"Owner": "BSS",
"ParentSRStatus": "",
"ParentSRType": "",
"ParentSRLinkDate": "",
"ParentSRLinkUser": "",
"SRAreaPlanningCommissionId": "4",
"SRCommunityPoliceStationAPREC": "RAMPART",
"SRCommunityPoliceStationPREC": "2",
"SRCrossStreet": "",
"ActionTaken": "",
"SRCity": "",
"RescheduleCounter": "",
"SRHouseNumber": "",
"ListOfDataBarricadeRemoval": {},
"ListOfDataBulkyItem": {},
"ListOfDataDeadAnimalRemoval": {},
"ListOfDataGraffitiRemoval": {},
"ListOfDataInformationOnly": {},
"ListOfDataMultipleStreetlightIssue": {},
"ListOfDataSingleStreetlightIssue": {},
"ListOfDataSrPhotoId": {
"DataSrPhotoId": []
},
"ListOfDataBusPadLanding": {},
"ListOfDataCurbRepair": {},
"ListOfDataFlooding": {},
"ListOfDataGeneralStreetInspection": {},
"ListOfDataGuardWarningRailMaintenance": {},
"ListOfDataGutterRepair": {},
"ListOfDataLandMudSlide": {},
"ListOfDataPothole": {},
"ListOfDataResurfacing": {},
"ListOfDataSidewalkRepair": {},
"ListOfDataStreetSweeping": {},
"ListOfDataBeesOrBeehive": {},
"ListOfDataMedianIslandMaintenance": {},
"ListOfDataOvergrownVegetationPlants": {},
"ListOfDataPalmFrondsDown": {},
"ListOfDataStreetTreeInspection": {},
"ListOfDataStreetTreeViolations": {},
"ListOfDataTreeEmergency": {},
"ListOfDataTreeObstruction": {},
"ListOfDataTreePermits": {},
"ListOfDataBrushItemsPickup": {},
"ListOfDataContainers": {},
"ListOfDataElectronicWaste": {},
"ListOfDataIllegalDumpingPickup": {},
"ListOfDataManualPickup": {},
"ListOfDataMetalHouseholdAppliancesPickup": {},
"ListOfDataMoveInMoveOut": {},
"ListOfDataHomelessEncampment": {
"DataHomelessEncampment": [
{
"ApprovedBy": "",
"AssignedTo": "",
"CompletedBy": "",
"Contact": "",
"ContactDate": "",
"Crew": "",
"DateCompleted": "12/31/2014 00:00:00",
"InspectedBy": "",
"InspectionDate": "",
"Location": "Alley",
"Type": "Homeless Encampment",
"LastUpdatedBy": "",
"OptionalTrackingCode": "",
"Name": "a5b5b2b9-d2e7-400a-bf75-1138ff013caa"
}
]
},
"ListOfDataIllegalAutoRepair": {},
"ListOfDataIllegalConstruction": {},
"ListOfDataIllegalConstructionFence": {},
"ListOfDataIllegalDischargeOfWater": {},
"ListOfDataIllegalDumpingInProgress": {},
"ListOfDataIllegalExcavation": {},
"ListOfDataIllegalSignRemoval": {},
"ListOfDataIllegalVending": {},
"ListOfDataLeafBlowerViolation": {},
"ListOfDataNewsRackViolation": {},
"ListOfDataObstructions": {},
"ListOfDataTablesAndChairsObstructing": {},
"ListOfDataGisLayer": {
"DataGisLayer": [
{
"A_Call_No": "",
"Area": "",
"Day": "",
"DirectionSuffix": "",
"DistrictAbbr": "",
"DistrictName": "Central",
"DistrictNumber": "104",
"DistrictOffice": "North Central",
"Fraction": "",
"R_Call_No": "",
"SectionId": "5279800",
"ShortDay": "",
"StreetFrom": "BOYLSTON ST",
"StreetTo": "FIRMIN ST",
"StreetLightId": "",
"StreetLightStatus": "",
"Type": "GIS",
"Y_Call_No": "",
"Name": "41572025-3803-49c4-8561-6e7ef41775df",
"CommunityPlanningArea": "Westlake",
"LastUpdatedBy": "",
"BOSRadioHolderName": ""
}
]
},
"ListOfDataServiceRequestNotes": {
"DataServiceRequestNotes": [
{
"CreatedDate": "12/31/2014 13:49:23",
"Comment": "",
"CreatedByUser": "User89",
"IsSrNoAvailable": "N",
"CommentType": "External",
"Notification": "N",
"FeedbackSRType": "",
"IntegrationId": "1420033765921",
"Date1": "",
"Date2": "",
"Date3": "",
"Text1": "",
"ListOfDataSrNotesAuditTrail": {}
}
]
},
"ListOfDataSubscribeDuplicateSr": {
"DataSubscribeDuplicateSr": [
{
"Activeflag": "Y",
"EmailId": "pratik.desai#yoopmail.com",
"Name": "010420150405",
"Type": "Subscription",
"LastUpdatedBy": ""
}
]
},
"ListOfChildServiceRequest": {},
"ListOfDataBillingCsscAdjustment": {},
"ListOfDataBillingEccAdjustment": {},
"ListOfDataBillingRsscAdjustment": {},
"ListOfDataBillingRsscExemption": {},
"ListOfDataSanitationBillingBif": {},
"ListOfDataSanitationBillingCssc": {},
"ListOfDataSanitationBillingEcc": {},
"ListOfDataSanitationBillingInquiry": {},
"ListOfDataSanitationBillingLifeline": {},
"ListOfDataSanitationBillingRssc": {},
"ListOfDataSanitationBillingSrf": {},
"ListOfDataDocumentLog": {},
"ListOfAuditTrailItem2": {},
"ListOfDataGenericBc": {
"DataGenericBc": [
{
"ATTRIB_08": "",
"NAME": "41572025-3803-49c4-8561-6e7ef41775df",
"PAR_ROW_ID": "1-N607",
"ROW_ID": "1-N60A",
"TYPE": "GIS",
"ListOfDataGenericbcAuditTrail": {}
},
{
"ATTRIB_08": "",
"NAME": "a5b5b2b9-d2e7-400a-bf75-1138ff013caa",
"PAR_ROW_ID": "1-N607",
"ROW_ID": "1-N609",
"TYPE": "Homeless Encampment",
"ListOfDataGenericbcAuditTrail": {}
},
{
"ATTRIB_08": "",
"NAME": "010420150405",
"PAR_ROW_ID": "1-N607",
"ROW_ID": "1-RN2D",
"TYPE": "Subscription",
"ListOfDataGenericbcAuditTrail": {}
}
]
},
"ListOfDataServiceNotComplete": {},
"ListOfDataOther": {},
"ListOfDataWeedAbatementForPrivateParcels": {}
}
]
}
}
}
Query 3 input
{
"MetaData": {},
"RequestSpecificDetail": {
"ParentSRNumberForLink": ""
},
"SRData": {
"SRNumber":"1-5968841",
"Anonymous": "N",
"Assignee": "",
"CreatedByUserLogin": "KAHUNA30DEC",
"CustomerAccessNumber": "",
"LADWPAccountNo": "",
"Language": "English",
"ListOfDataGisLayer": {},
"ListOfDataServiceRequestNotes": {
"DataServiceRequestNotes": [
{
"Comment": "description 1245",
"CommentType": "Feedback",
"CreatedByUser": "KAHUNA30DEC",
"FeedbackSRType": "Illegal Dumping in Progress",
"IsSrNoAvailable": "N"
},
{
"Comment": "comments 123568",
"CommentType": "External",
"CreatedByUser": "",
"IsSrNoAvailable": "N"
}
]
},
"LoginUser": "KAHUNA30DEC",
"MobilOS": "Android",
"NewContactEmail": "",
"NewContactFirstName": "",
"NewContactLastName": "",
"NewContactPhone": "",
"Owner": "Other",
"ParentSRNumber": "",
"Priority": "Normal",
"SRCommunityPoliceStation": "RAMPART",
"UpdatedByUserLogin": "KAHUNA30DEC",
"Status": "Open",
"SRType": "Feedback",
"ServiceDate": "02/11/2015",
"Source": "Mobile App"
}
}
Query 3 output
{
"status": {
"code": 311,
"message": "Service Request Successfully Submited",
"cause": ""
},
"Response": {
"PrimaryRowId": "1-3JXL5",
"ListOfServiceRequest": {
"ServiceRequest": [
{
"SRNumber": "1-5968841"
}
]
}
}
}
Python Script responds with query two output
import json
import jsonpickle
import arcpy
import json
import numpy
import requests
f = open('C:\Users\Administrator\Desktop\myData.json', 'r')
data = jsonpickle.encode( jsonpickle.decode(f.read()) )
url = "https://myDatatest.lacity.org/myDatarouter/srbe/1/QuerySR"
headers = {'Content-type': 'text/plain', 'Accept': '/'}
r = requests.post(url, data=data, headers=headers)
sr = arcpy.SpatialReference(4326)
decoded = json.loads(r.text)
# pretty printing of json-formatted string
print json.dumps(decoded, sort_keys=True, indent=4)
f.close()
decoded is a dictionary containing the data you're looking for.
try:
print decoded
print decoded.keys()
print decoded.items()

Categories

Resources