create yaml with AWS CloudFormation references

create yaml with AWS CloudFormation references - python

I need a python code that would create the yaml code below.
Tags:
- Key: key1
Value: !Ref 'AWS::StackName'
- Key: Key2
Value: !Ref 'AWS::StackId'
Here is what I have that doesn't do the trick.
def generate_resource(ami, source_data):
resource = {
"Type": "AWS::EC2::Instance",
"Properties": {
"ImageId": ami["ImageId"],
"InstanceType": ami["InstanceType"],
"PrivateIpAddress": ami["PrivateIpAddress"],
"KeyName": ami["KeyName"]
"SubnetId": { "Ref": "SubnetId" },
"SecurityGroupIds": { "Ref": "SecurityGroupId" },
"Tags": [
{ "Key": "key1", "Value": "{!Ref 'AWS::StackName'}"},
{ "Key": "key2", "Value": "{!Ref 'AWS::StackId'}"}
]
}
}
The yaml output from this code is not properly formatted so it simply copies {!Ref 'AWS::StackName'}, as the value.
import os, sys
import lib.aws as aws, lib.cft as cft, lib.inventory as inventory
BUCKET_NAME = 'testbucket'
def generate_cft(commit_hash, file_dict, dry_run):
return (
"# Autogenerated CFT for commit hash " + commit_hash + "\n" +
cft.generate(inventory.read(file_dict["path"]))
)
def upload_cft(commit_hash, file_dict, cft_text):
target_key = commit_hash + "/" + file_dict["name"].split("_")[0] + ".yaml"
aws.upload(BUCKET_NAME, target_key, cft_text)
def show_cft(file_dict, cft_text):
print(file_dict["path"] + " generates the following cft:")
print("")
print(cft_text)
print("")
def generate_and_upload(commit_hash, file_dict, dry_run):
cft_text = generate_cft(commit_hash, file_dict, dry_run)
aws.validate_cft(cft_text)
if dry_run:
show_cft(file_dict, cft_text)
else:
upload_cft(commit_hash, file_dict, cft_text)
def generate_and_upload_all(commit_hash, dry_run):
for file_dict in inventory.list():
print("generating cft for " + file_dict["path"])
generate_and_upload(commit_hash, file_dict, dry_run)
if __name__ == "__main__":
if not os.getcwd().endswith("ci"):
print("Please run this script from the ci directory")
exit()
commit_hash = sys.argv[1] if len(sys.argv) >= 2 else "test"
generate_and_upload_all(commit_hash, False)

YES!!!! I played around with the code and figured it out. It was a syntax error during the CFT creation. now when I create a stack with the CFT generated, the tag value is replaced by the actual 'StackId'.. Thanks everyone for your help and guidance.
def generate_resource(ami, source_data):
resource = {
"Type": "AWS::EC2::Instance",
"Properties": {
"ImageId": ami["ImageId"],
"InstanceType": ami["InstanceType"],
"PrivateIpAddress": ami["PrivateIpAddress"],
"KeyName": ami["KeyName"],
"SubnetId": { "Ref": "SubnetId" },
"SecurityGroupIds": { "Ref": "SecurityGroupId" },
"Tags": [
{ "Key": "Name", "Value": ami["Name"] },
{ "Key": "BootUpDependsOn", "Value": ami["BootUpDependsOn"]},
{ "Key": "WaitTimeAfterBootUp", "Value": ami["WaitTimeAfterBootUp"]},
{ "Key": "Key1", "Value": { "Ref": "AWS::StackName" }},
{ "Key": "Key2", "Value": { "Ref": "AWS::StackId" }}
]
}
}

Related

Writing resilient recursive code that will return results from a big json file

I have written a recursive code. I want more experienced people to tell me how resillient and fail-safe is my code:
I have a json file (Json file can be as big as 300MB):
[
{
"modules": {
"webpages": []
},
"webpages": {
"ip_addr": {
"value": "127.0.0.1",
"tags": []
},
"http": {
"status": {
"value": "Unavailable",
"tags": []
},
"title": {
"value": "403 Forbidden",
"tags": [
{
"category": "Server Code",
"match": "403"
},
{
"category": "Interesting Words",
"match": "Forbidden"
}
]
},
"server": {
"value": "Apache",
"tags": [
{
"category": "Apache Server",
"match": "Apache"
}
]
}
},
"redirects": [],
"robottxt": null
}
},
{
"modules": {
"webpages": []
}
}
]
I want to return value keys where tags are populated.
So I want to ignore:
"status": {
"value": "Unavailable",
"tags": []
},
But I want to return the title and server values. I also want to return ip_addr.value
I have written this code:
def getAllValues(nestedDictionary, firstArray, firstObj, firstUseful):
returnedArray = firstArray
tempValue = firstObj
useful = firstUseful
for key, value in nestedDictionary.items():
ipString = nestedDictionary.get("ip_addr")
if ipString is not None:
ipValue = ipString.get("value")
useful = {"ip_add": ipValue}
if isinstance(value, dict):
temp = {
"Key": key,
"useful": useful,
}
getAllValues(value, returnedArray, temp, useful)
else:
if key == "value":
tempValue["value"] = value
if key == "tags" and isinstance(value, list) and len(value) > 0:
tempValue["tags"] = value
returnedArray.append(tempValue)
return returnedArray
The above code should return:
[
{
"Key": "title",
"value": "403 Forbidden",
"useful": { "ip_addr": "127.0.0.1" },
"tags": [
{
"category": "Server Code",
"match": "403"
},
{
"category": "Interesting Words",
"match": "Forbidden"
}
]
},
{
"Key": "server",
"value": "Apache",
"useful": { "ip_addr": "127.0.0.1" },
"tags": [
{
"category": "Apache Server",
"match": "Apache"
}
]
}
]
Its a long post, but hopefully, someone can give me some assurance :)

Transforming streaming data into a json

I have a streaming data coming from a source and I was able to capture few important variables like ID, Timestamp and Vital signs. What I am trying to do is create a json file object and unload it into a file system. This is one of the examples:
for k in streaming:
id = k[1] ----------> 1
timestamp = k[2] ------> 1652304692
vsigns = k[3] -------> Nested dictionary
This is how vsigns looks like
"vsigns":
{
"ECG":
{
"VPB_FREQ": "0",
"STI": "+0.00"
},
"HR":
{
"HR_EKG": "87",
"HR_PPG_1": "87",
"HR_PULSED": "87"
},
"NIBP":
{
"NIBPS": "119",
"NIBPD": "88",
"NIBPM": "95"
}
}
And I want a json structure in the following format:
[{
"id": "1",
"ECG":
{
"timestamp": 1652304692,
"VPB_FREQ": "0",
"STI": "+0.00",
}
},
{
"id": "1",
"HR":
{
"timestamp": 1652304692,
"HR_EKG": "87",
"HR_PPG_1": "87",
"HR_PULSED": "87"
}
},
{
"id": "1",
"NIBP":
{
"timestamp": 1652304692,
"NIBPS": "119",
"NIBPD": "88",
"NIBPM": "95"
},
}]
I tried an approach but doesn't give me what I want. How do I get this right.
for k in streaming:
id = k[1]
timestamp = k[2]
vsigns = k[3]
vlist = []
for k, v in vsigns.items():
vdict = {"id": id,
k:{"timestamp":timestamp,
"vsigns": v}}
vlist.append(vdict)
print(vlist)
Output:
[{
"id": "1",
"ECG":
{
"timestamp": 1652951054.0,
"vsigns":
{
"VPB_FREQ": "0"
}
}
},
{
"id": "1",
"HR":
{
"timestamp": 1652951054.0,
"vsigns":
{
"HR_EKG": "126",
"HR_PPG_1": "127",
"HR_PULSED": "127"
}
}
},
{
"id": "1",
"NIBP":
{
"timestamp": 1652951054.0,
"vsigns":
{
"NIBPS": "95",
"NIBPD": "46",
"NIBPM": "62"
}
}
}
}]

The following piece of code worked for me:
for k in streaming:
id = k[1]
timestamp = k[2]
vsigns = k[3]
vlist = []
for k, v in vsigns.items():
v['timestamp'] = epoch_time
vdict = {"id": id,
k:v}
vlist.append(vdict)

How to extract specific nested JSON value doing loop? (Python)

{
"127.0.0.1":{
"addresses":{
"ipv4":"127.0.0.1"
},
"hostnames":[
{
"name":"localhost",
"type":"PTR"
}
],
"status":{
"reason":"conn-refused",
"state":"up"
},
"tcp":{
"5000":{
"conf":"10",
"cpe":"cpe:/a:python:python:3.9.2",
"extrainfo":"Python 3.9.2",
"name":"http",
"product":"Werkzeug httpd",
"reason":"syn-ack",
"script":{
"vulners":"\n cpe:/a:python:python:3.9.2: \n \tCVE-2021-29921\t7.5\thttps://vulners.com/cve/CVE-2021-29921\n \tCVE-2021-23336\t4.0\thttps://vulners.com/cve/CVE-2021-23336\n \tMSF:ILITIES/DEBIAN-CVE-2021-3426/\t2.7\thttps://vulners.com/metasploit/MSF:ILITIES/DEBIAN-CVE-2021-3426/\t*EXPLOIT*\n \tCVE-2021-3426\t2.7\thttps://vulners.com/cve/CVE-2021-3426"
},
"state":"open",
"version":"1.0.1"
},
"6000":{
"conf":"10",
"cpe":"cpe:/a:python:python:3.9.2",
"extrainfo":"Python 3.9.2",
"name":"http",
"product":"Werkzeug httpd",
"reason":"syn-ack",
"script":{
"vulners":"\n cpe:/a:python:python:3.9.2: \n \tCVE-2021-29921\t7.5\thttps://vulners.com/cve/CVE-2021-29921\n \tCVE-2021-23336\t4.0\thttps://vulners.com/cve/CVE-2021-23336\n \tMSF:ILITIES/DEBIAN-CVE-2021-3426/\t2.7\thttps://vulners.com/metasploit/MSF:ILITIES/DEBIAN-CVE-2021-3426/\t*EXPLOIT*\n \tCVE-2021-3426\t2.7\thttps://vulners.com/cve/CVE-2021-3426"
},
"state":"open",
"version":"1.0.1"
}
},
"vendor":{
}
}
}
I want to extract "vulners" value here i tried this -
results = []
for x in collection.find({},{"scan": 1, "_id": 0 }):
results.append(json.loads(json_util.dumps(x)))
portnumber = []
datay = []
datapro = []
for result in results:
ips = result['scan']
for ip in ips:
ports = result['scan'][ip]['tcp']
ipdomain = result['scan'][ip]['hostnames']
for ip2 in ipdomain:
ip3 = ip2['name']
for port in ports:
portnumber.append(port)
datax = ports[port]['script']
datay.append(datax)
datapro2 = ports[port]['product']
datapro.append(datapro2)
date = datetime.datetime.now()
date_now = date.strftime("%x, %X")
pass_json_var = {'domain': ip3, 'ports': portnumber, 'product': datapro, 'vulnerabilities': datay, "date": date_now}
if isinstance(pass_json_var, list):
domaindata.insert_many(pass_json_var)
else:
domaindata.insert_one(pass_json_var)
Ok so here if the "results" output gives me one "vulners" value then it works fine but when it's multiple ports with vulners values it doesn't work!
How can i access the 'vulners' value? Hoping for someone to guide me also a bit, Please try to give a solution which is dynamic
Thanks a lot!

Model based approach
this approach is based on a model of your data you want to parse. From my point of view this is more work in the beginning. With the advantage, that you will have clean error messages and you can control the behaviour by adapting your data model.
make a model of the data you want to parse
from typing import Any, Optional
from pydantic import BaseModel, Field
class ExScript(BaseModel):
vulners:str = ""
class Ex30000(BaseModel):
script:ExScript = Field(default=Any)
class ExTcp(BaseModel):
root:Ex30000= Field(default=Any, alias="30000")
class ExRoot(BaseModel):
tcp:ExTcp = Field() # Required
class Base(BaseModel):
root:ExRoot = Field(default=Any, alias="35.0.0.0.0")
change your input data to a raw string outherwise you will have to escape \n and \t
input_will_work = r"""{
"35.0.0.0.0": {
"hostnames": [
{
"name": "domain.com",
"type": "PTR"
}
],
"addresses": {
"ipv4": "35.0.0.0"
},
"vendor": {},
"status": {
"state": "up",
"reason": "syn-ack"
},
"tcp": {
"30000": {
"state": "open",
"reason": "syn-ack",
"name": "http",
"product": "nginx",
"version": "1.20.0",
"extrainfo": "",
"conf": "10",
"cpe": "cpe:/a:igor_sysoev:nginx:1.20.0",
"script": {
"http-server-header": "nginx/1.20.0",
"vulners": "\n cpe:/a:igor_sysoev:nginx:1.20.0: \n \tNGINX:CVE-2021-23017\t6.8\thttps://vulners.com/nginx/NGINX:CVE-2021-23017\n \t9A14990B-D52A-56B6-966C-6F35C8B8EB9D\t6.8\thttps://vulners.com/githubexploit/9A14990B-D52A-56B6-966C-6F35C8B8EB9D\t*EXPLOIT*\n \t1337DAY-ID-36300\t6.8\thttps://vulners.com/zdt/1337DAY-ID-36300\t*EXPLOIT*\n \tPACKETSTORM:162830\t0.0\thttps://vulners.com/packetstorm/PACKETSTORM:162830\t*EXPLOIT*"
}
}
}
}
}
"""
input_will_fail = r"""{
"35.0.0.0.0": {}
}
"""
3.1 this should give you the expected result
obj1 = Base.parse_raw(input_will_work)
print(obj1.root.tcp.root.script.vulners)
3.2 this should throw an exception
obj2 = Base.parse_raw(input_will_fail)
Search data with jsonpath
should return all objects with the name vulners
from jsonpath_ng import jsonpath, parse
import json
obj = json.loads(input_will_work)
p = parse('$..vulners')
for match in p.find(obj):
print(match.value)

Update:
def extract_data(ip_address_data):
domains = ip_address_data["hostnames"]
ports_data = []
# Each port can have different products and vulners
# So that data is grouped together in a dictionary
for port in ip_address_data["tcp"].keys():
port_data = ip_address_data["tcp"][port]
product = port_data["product"]
vulners = port_data['script']['vulners']
ports_data.append({
"port": port,
"product": product,
"vulners": vulners
})
return {
"domains": domains,
"ports_data": ports_data
}
# Result is the data from mongo db
# result = collection.find({})["scan"]
result = {
"127.0.0.1": {
"addresses": {
"ipv4": "127.0.0.1"
},
"hostnames": [
{
"name": "localhost",
"type": "PTR"
}
],
"status": {
"reason": "conn-refused",
"state": "up"
},
"tcp": {
"5000": {
"conf": "10",
"cpe": "cpe:/a:python:python:3.9.2",
"extrainfo": "Python 3.9.2",
"name": "http",
"product": "Werkzeug httpd",
"reason": "syn-ack",
"script": {
"vulners": "\n cpe:/a:python:python:3.9.2: \n \tCVE-2021-29921\t7.5\thttps://vulners.com/cve/CVE-2021-29921\n \tCVE-2021-23336\t4.0\thttps://vulners.com/cve/CVE-2021-23336\n \tMSF:ILITIES/DEBIAN-CVE-2021-3426/\t2.7\thttps://vulners.com/metasploit/MSF:ILITIES/DEBIAN-CVE-2021-3426/\t*EXPLOIT*\n \tCVE-2021-3426\t2.7\thttps://vulners.com/cve/CVE-2021-3426"
},
"state": "open",
"version": "1.0.1"
},
"6000": {
"conf": "10",
"cpe": "cpe:/a:python:python:3.9.2",
"extrainfo": "Python 3.9.2",
"name": "http",
"product": "Werkzeug httpd",
"reason": "syn-ack",
"script": {
"vulners": "\n cpe:/a:python:python:3.9.2: \n \tCVE-2021-29921\t7.5\thttps://vulners.com/cve/CVE-2021-29921\n \tCVE-2021-23336\t4.0\thttps://vulners.com/cve/CVE-2021-23336\n \tMSF:ILITIES/DEBIAN-CVE-2021-3426/\t2.7\thttps://vulners.com/metasploit/MSF:ILITIES/DEBIAN-CVE-2021-3426/\t*EXPLOIT*\n \tCVE-2021-3426\t2.7\thttps://vulners.com/cve/CVE-2021-3426"
},
"state": "open",
"version": "1.0.1"
}
},
"vendor": {
}
}
}
def scandata():
for ip_address in result:
ip_address_data = extract_data(
result[ip_address]
)
print(ip_address, ip_address_data)
scandata()

def extract_data(ip_address_data):
domains = ip_address_data["hostnames"]
ports_data = []
# Each port can have different products and vulners
# So that data is grouped together in a dictionary
for port in ip_address_data["tcp"].keys():
port_data = ip_address_data["tcp"][port]
product = port_data["product"]
vulners = port_data['script']['vulners']
ports_data.append({
"port": port,
"product": product,
"vulners": vulners
})
return {
"domains": domains,
"ports_data": ports_data
}
#app.route('/api/vulnerableports', methods=['GET'])
def show_vulnerableports():
data = []
resultz = []
for x in collection.find({}, {"scan": 1, "_id": 0}):
resultz.append(json.loads(json_util.dumps(x)))
for resultx in resultz:
result = resultx['scan']
for ip_address in result:
ip_address_data = extract_data(
result[ip_address]
)
data.append({ip_address: ip_address_data})
return jsonify(data)
So this is the solution! i had to iterate over script and then access vulner!

JSON ID extraction from Array in Python

I wrote a script to pull data from Verizon's connectivity management API with the following. Below is the section that requests the line information based on the search item, in this case, the SIM or iccid. I did not include the previous parts because they are just to connect to the API and get credentials.
header = {
'accept': 'application/json',
'VZ-M2M-Token': session_token,
'Authorization': 'Bearer' + bearer_token,
'Content-Type': 'application/json',
}
data = '{ "deviceId": { "id": ' + SIM +', "kind": "ICCID" }}'
response = requests.post('https://thingspace.verizon.com/api/m2m/v1/devices/actions/list', headers=header, data=data)
And the response I get is a JSON Array which looks like
{
"hasMoreData": false,
"devices": [
{
"accountName": "123456789-00001",
"billingCycleEndDate": "2020-10-31T20:00:00-04:00",
"carrierInformations": [
{
"carrierName": "Verizon Wireless",
"servicePlan": "3rrrrx48wwwwrjgjtyjtyjtyjtyj",
"state": "active"
}
],
"connected": true,
"createdAt": "2016-11-04T11:06:28-04:00",
"deviceIds": [
{
"id": "5256694405",
"kind": "mdn"
},
{
"id": "3114949302094150",
"kind": "imsi"
},
{
"id": "35922505468230",
"kind": "imei"
},
{
"id": "891480000054957290575",
"kind": "iccId"
},
{
"id": "15256694405",
"kind": "msisdn"
},
{
"id": "5256694405",
"kind": "min"
}
],
"extendedAttributes": [
{
"key": "PrimaryPlaceOfUseTitle"
},
{
"key": "PrimaryPlaceOfUseFirstName",
"value": "5256694405",
},
{
"key": "PrimaryPlaceOfUseMiddleName"
},
{
"key": "PrimaryPlaceOfUseLastName",
"value": "ESN"
},
{
"key": "PrimaryPlaceOfUseSuffix"
},
{
"key": "PrimaryPlaceOfUseAddressLine1"
},
{
"key": "PrimaryPlaceOfUseAddressLine2"
},
{
"key": "PrimaryPlaceOfUseCity"
},
{
"key": "PrimaryPlaceOfUseState"
},
{
"key": "PrimaryPlaceOfUseCountry"
},
{
"key": "PrimaryPlaceOfUseZipCode"
},
{
"key": "PrimaryPlaceOfUseZipCode4"
},
{
"key": "PrimaryPlaceOfUseCBRPhone"
},
{
"key": "PrimaryPlaceOfUseCBRPhoneType"
},
{
"key": "PrimaryPlaceOfUseEmailAddress"
},
{
"key": "AccountNumber",
"value": "5256694405-00001"
},
{
"key": "SmsrOid"
},
{
"key": "ProfileStatus"
},
{
"key": "PromoCodes",
"value": ""
},
{
"key": "PromotionStartDate",
"value": ""
},
{
"key": "PromotionScheduledEndDate",
"value": ""
},
{
"key": "LeadId",
"value": ""
},
{
"key": "CustomerName",
"value": ""
},
{
"key": "CustomerAddressLine1",
"value": ""
},
{
"key": "CustomerAddressLine2",
"value": ""
},
{
"key": "CustomerAddressCity",
"value": ""
},
{
"key": "CustomerAddressState",
"value": ""
},
{
"key": "CustomerAddressZipCode",
"value": ""
},
{
"key": "ServiceZipCode",
"value": ""
},
{
"key": "SkuNumber",
"value": "VZW080000460053"
},
{
"key": "CostCenterCode"
},
{
"key": "PreIMEI",
"value": "3592254564568445"
},
{
"key": "PreSKU",
"value": "VZW080000100037"
},
{
"key": "SIMOTADate",
"value": "4/30/2020 1:22:18 PM"
},
{
"key": "RoamingStatus",
"value": "NotRoaming"
},
{
"key": "LastRoamingStatusUpdate",
"value": "9/24/2020 5:40:26 PM"
}
],
"groupNames": [
"Default: 0220433754-00001"
],
"ipAddress": "100.100.100.100",
"lastActivationBy": "User Verizon",
"lastActivationDate": "2016-11-04T11:06:28-04:00",
"lastConnectionDate": "2020-09-24T13:40:26-04:00"
}
]
}
I added a part to my script to pull the mdn, iccid and the imei from the array with the code that is below.
def puller(line_json):
line_data = json.loads(line_json)
mdn = (line_data['devices'][0]['deviceIds'][0]['id'])
iccid = (line_data['devices'][0]['deviceIds'][3]['id'])
imei = (line_data['devices'][0]['deviceIds'][2]['id'])
print('phone = ' ,mdn)
print('SIM = ' , iccid)
print('IMEI = ' , imei)
I tested this code and it works the way it should with one test ID. I then proceeded to test with another test ID and I learned that the array structure is not always the same. That second JSON array is below. I am wondering is there a better way to find the specific values that I want, but in the way that I am not specifically telling the script where in the structure the item will be as I did above.
{
"hasMoreData": false,
"devices": [
{
"accountName": "02234234234-00001",
"billingCycleEndDate": "2020-10-31T20:00:00-04:00",
"carrierInformations": [
{
"carrierName": "Verizon Wireless",
"servicePlan": "37776xdsfewsfwe576193",
"state": "active"
}
],
"connected": true,
"createdAt": "2016-05-24T15:55:06-04:00",
"deviceIds": [
{
"id": "0945437676404",
"kind": "esn"
},
{
"id": "1234565799",
"kind": "mdn"
},
{
"id": "31148454545458767",
"kind": "imsi"
},
{
"id": "01426786678211",
"kind": "imei"
},
{
"id": "89148000006456456454",
"kind": "iccId"
},
{
"id": "1234565799",
"kind": "min"
}
],
"extendedAttributes": [
{
"key": "PrimaryPlaceOfUseTitle"
},
{
"key": "PrimaryPlaceOfUseFirstName",
"value": "096114564506772"
},
{
"key": "PrimaryPlaceOfUseMiddleName"
},
{
"key": "PrimaryPlaceOfUseLastName",
"value": "096546454806772"
},
{
"key": "PrimaryPlaceOfUseSuffix"
},
{
"key": "PrimaryPlaceOfUseAddressLine1"
},
{
"key": "PrimaryPlaceOfUseAddressLine2"
},
{
"key": "PrimaryPlaceOfUseCity"
},
{
"key": "PrimaryPlaceOfUseState"
},
{
"key": "PrimaryPlaceOfUseCountry"
},
{
"key": "PrimaryPlaceOfUseZipCode"
},
{
"key": "PrimaryPlaceOfUseZipCode4"
},
{
"key": "PrimaryPlaceOfUseCBRPhone"
},
{
"key": "PrimaryPlaceOfUseCBRPhoneType"
},
{
"key": "PrimaryPlaceOfUseEmailAddress"
},
{
"key": "AccountNumber",
"value": "02242342354-00001"
},
{
"key": "SmsrOid"
},
{
"key": "ProfileStatus"
},
{
"key": "PromoCodes",
"value": ""
},
{
"key": "PromotionStartDate",
"value": ""
},
{
"key": "PromotionScheduledEndDate",
"value": ""
},
{
"key": "LeadId",
"value": ""
},
{
"key": "CustomerName",
"value": ""
},
{
"key": "CustomerAddressLine1",
"value": ""
},
{
"key": "CustomerAddressLine2",
"value": ""
},
{
"key": "CustomerAddressCity",
"value": ""
},
{
"key": "CustomerAddressState",
"value": ""
},
{
"key": "CustomerAddressZipCode",
"value": ""
},
{
"key": "ServiceZipCode",
"value": ""
},
{
"key": "SkuNumber",
"value": "VZW12000364343005"
},
{
"key": "CostCenterCode"
},
{
"key": "PreIMEI"
},
{
"key": "PreSKU",
"value": "VZW12000334340005"
},
{
"key": "SIMOTADate",
"value": "3/13/2020 10:52:07 AM"
},
{
"key": "RoamingStatus",
"value": "NotRoaming"
},
{
"key": "LastRoamingStatusUpdate",
"value": "10/20/2020 6:14:20 PM"
}
],
"groupNames": [
"Default: 02342343754-00001"
],
"ipAddress": "101.101.101.101",
"lastActivationBy": "User Verizon",
"lastActivationDate": "2016-05-24T15:55:16-04:00",
"lastConnectionDate": "2020-10-20T14:14:20-04:00"
}
]
}
I tried to use this block of code from some research I did to find the value that I was looking for; in this case, the mdn. Problem I have is that the response returns a blank set of brackets with no information, so I know there is something I probably did wrong.
def json_extract(obj, kind):
"""Recursively fetch values from nested JSON."""
arr = []
def extract(obj, arr, kind):
"""Recursively search for values of key in JSON tree."""
if isinstance(obj, dict):
for k, v in obj.items():
if isinstance(v, (dict, list)):
extract(v, arr, kind)
elif k == kind:
arr.append(v)
elif isinstance(obj, list):
for item in obj:
extract(item, arr, kind)
return arr
values = extract(obj, arr, kind)
return values
names = json_extract(response , 'mdn')
print(names)

I understood that you are trying to find, mdn, iccid and imei'id from the json object above,so, instead of recursion and the complicated coding that you have done there, it is easier to use python's inbuilt libraries to help you out:
You can use the next function for your purpose:
# load your json data
line_data = json.loads(data)
# narrow your focus on the array in question
device_ids = line_data['devices'][0]['deviceIds']
# This gets the first item's id attribute from the list that matches the condition, and returns None if no item matches.
mdn = next((x['id'] for x in device_ids if x['kind'] == "mdn"), None)
iccid = next((x['id'] for x in device_ids if x['kind'] == "iccid"), None)
imei = next((x['id'] for x in device_ids if x['kind'] == "imei"), None)
You will need to handle the None if it was unable to find such element in the array.
Reference : Find object in list that has attribute equal to some value (that meets any condition)

Problem with extrakting Data out of a List of JSONS

I want to evaluate json data using python. However, I get the error message: string indices must be integers
I have tested it with the following code, but it does not work. I tried it before with elasticsearch but im not that good in programming so i decided to try it with a homebrew solution
(sorry for the bad formating, i am new to stackoverflow)
```
import requests, json, os
from elasticsearch import Elasticsearch
directory = "C:\\Users\\Felix Bildstein\\Desktop\\Test1"
Dateien = os.listdir(directory)
index_len = len(Dateien) - 2
n = 1
# Create your dictionary class
class my_dictionary(dict):
# __init__ function
def __init__(self):
self = dict()
# Function to add key:value
def add(self, key, value):
self[key] = value
# Main Function
dict_obj = my_dictionary()
dict_obj.add(1, 'Geeks')
while n <= index_len:
try:
a, *Dateien = Dateien
n += 1
f = open(a, "r")
file_contents = (f.read())
f.close
dict_obj.add(n, file_contents)
finally:
print(n)
print(file_contents['unitPrice'])
output = dict_obj
print(output)
with open('result.json', 'w') as fp:
json.dump(output, fp)
f = open("dict.txt","w")
f.write( str(dict_obj) )
f.close()
```
It should spend the appropriate value
This is my Test Json
{
"merchantOrderId": "302-08423880-89823764",
"creationTime": {
"usecSinceEpochUtc": "15555040944000000",
"granularity": "MICROSECOND"
},
"transactionMerchant": {
"name": "Amazon.de"
},
"lineItem": [{
"purchase": {
"status": "ACCEPTED",
"unitPrice": {
"amountMicros": "4690000",
"currencyCode": {
"code": "EUR"
},
"displayString": "EUR 4,20"
},
"returnsInfo": {
"isReturnable": true,
"daysToReturn": 30
},
"fulfillment": {
"location": {
"address": [""]
},
"timeWindow": {
"startTime": {
"usecSinceEpochUtc": "155615040000000",
"granularity": "DAY"
},
"endTime": {
"usecSinceEpochUtc": "155615040000000",
"granularity": "DAY"
}
}
},
"landingPageUrl": {
"link": "https://www.amazon.de/gp/r.html?C\u003d3ILR4VQSVD3HI\u0026K\u0026M\u003durn:rtn:msg:20190417124222996c5bb6751e45b5ba12aff8d350p0eu\u0026R\u003d2FEGGCJMDBAOF\u0026T\u003dC\u0026U\u003dhttps%3A%2F%2Fwww.amazon.de%2Fdp%2FB001J8I7VG%2Fref%3Dpe_3044161_185740101_TE_item\u0026H\u003d6EXBPJA679MVNLICLRRO4K1XPFCA\u0026ref_\u003dpe_3044161_185740101_TE_item"
},
"productInfo": {
"name": "tesa Powerstrips Bildernagel, selbstklebend, weiß, 2 Stück"
}
},
"name": "tesa Powerstrips Bildernagel, selbstklebend, weiß, 2 Stück"
}],
"priceline": [{
"type": "SUBTOTAL",
"amount": {
"amountMicros": "4690000",
"currencyCode": {
"code": "EUR"
}
}
}, {
"type": "DELIVERY",
"amount": {
"amountMicros": "0",
"currencyCode": {
"code": "EUR"
},
"displayString": "EUR 0,00"
}
}]
}

Develop Reference

Python is a programming language that lets you work quickly and integrate systems more effectively.

create yaml with AWS CloudFormation references - python

Related

Writing resilient recursive code that will return results from a big json file

Transforming streaming data into a json

How to extract specific nested JSON value doing loop? (Python)

JSON ID extraction from Array in Python

Problem with extrakting Data out of a List of JSONS

Categories

Resources