Develop Reference

Changing .json file in python

I have the following .json file like this:
[
{
"2004": "0",
"2005": "0",
"2006": "0",
},
{
"2004": "0",
"2005": "0",
"2006": "0",
},
{
"2013": "0",
"2014": "0",
"2015": "--",
}
]
How to do in python to get a following .json file like this:
{
"Countries": [
{
"2004": "0",
"2005": "0",
"2006": "0"
},
{
"2004": "0",
"2005": "0",
"2006": "0"
},
{
"2013": "0",
"2014": "0",
"2015": "--"
}
]
}

For your case of using files change to use_files = True in the beginning of script and set correct files names instead of 'input.json' and 'output.json'.
Try it online!
import json
use_files = False
inp_fname, out_fname = 'input.json', 'output.json'
if not use_files:
text = """
[
{
"2004": "0",
"2005": "0",
"2006": "0"
},
{
"2004": "0",
"2005": "0",
"2006": "0"
},
{
"2013": "0",
"2014": "0",
"2015": "--"
}
]
"""
if use_files:
with open(inp_fname, 'r', encoding = 'utf-8') as f:
text = f.read()
obj = json.loads(text)
obj = {'Countries': obj}
rtext = json.dumps(obj, indent = 4, ensure_ascii = False)
if use_files:
with open(out_fname, 'w', encoding = 'utf-8') as f:
f.write(rtext)
else:
print(rtext)
the code outputs:
{
"Countries": [
{
"2004": "0",
"2005": "0",
"2006": "0"
},
{
"2004": "0",
"2005": "0",
"2006": "0"
},
{
"2013": "0",
"2014": "0",
"2015": "--"
}
]
}

Python Json Parsing Key Value Filtering

How can I only get the lines which do not have "popup:" in them?
json_data = json.loads(raw_json, strict=False)
This is the data:
{
"259655": { "params": ["OIL", "9,5"], "availability": "1", "reload": ""},
"259656": { "params": ["OIL", "10"], "availability": "1", "reload": ""},
"259659": { "params": ["OIL", "11,5"], "availability": "1", "reload": ""} ,
"259661": { "params": ["SALT", "5"], "availability": "1", "reload": "", "popup": "" },
"259662": { "params": ["SALT", "5,5"], "availability": "1", "reload": "", "popup": "" },
"259663": { "params": ["SALT", "6"], "availability": "1", "reload": "", "popup": "" },
}

Here's a simple solution using dict comprehension to create a new dict without values that contain "popup" keys
filtered_dict = {i: json_data[i] for i in json_data if "popup" not in json_data[i]}

Creating dict within the dict based on dynamic key(contains the numbers) e.g 'abx.123':'xyz'. Group similar numbers to the Dict and make single dict

I have a dict object in which key and value pairs are there. So I want to add the key on the basis of key
e.g.
{'abx.123.a':'name',
'abz.123.b':'address',
aby.123.c':'location',
'abc.231.a':'Postion',
'abc.231.b':'dob'}
Now i want to add the key who contains the 123 number to the dict and create a dict within dict. And add the key who contains the 231 number to the dict.
Data = {
"arn": "arn",
"description": "Security group for all coporate communications",
"egress.#": "1",
"egress.4820.cidr_blocks.#": "1",
"egress.4820.cidr_blocks.0": "0.0.0.0/0",
"egress.4820.description": "",
"egress.4820.from_port": "0",
"egress.4820.ipv6_cidr_blocks.#": "0",
"egress.4820.prefix_list_ids.#": "0",
"egress.4820.protocol": "-1",
"egress.4820.security_groups.#": "0",
"egress.4820.self": "False",
"egress.4820.to_port": "0",
"id": "sg-080b03",
"ingress.#": "4",
"ingress.1279476397.cidr_blocks.#": "0",
"ingress.1279476397.description": "self",
"ingress.1279476397.from_port": "0",
"ingress.1279476397.prefix_list_ids.#": "0",
"ingress.1279476397.protocol": "-1",
"ingress.1279476397.security_groups.#": "0",
"ingress.1279476397.self": "true",
"ingress.1279476397.to_port": "0",
"ingress.2455438834.cidr_blocks.0": "10.10.0.0/16",
"ingress.2455438834.description": "cluster VPC",
"ingress.2455438834.from_port": "443",
"ingress.2455438834.protocol": "tcp",
"ingress.2455438834.to_port": "443",
"ingress.3391123749.cidr_blocks.#": "0",
"ingress.3391123749.description": "eks-cluster-master",
"ingress.3391123749.from_port": "443",
"ingress.3391123749.protocol": "tcp",
"ingress.3391123749.to_port": "443",
"ingress.439086653.cidr_blocks.#": "0",
"ingress.439086653.description": "eks-cluster-master",
"ingress.439086653.from_port": "1025",
"ingress.439086653.ipv6_cidr_blocks.#": "0",
"ingress.439086653.prefix_list_ids.#": "0",
"ingress.439086653.protocol": "tcp",
"ingress.439086653.security_groups.#": "1",
"ingress.439086653.security_groups.3696519931": "sg-0007a603523411",
"ingress.439086653.self": "False",
"ingress.439086653.to_port": "65535",
"name":"xyz.abc.corporate",
"owner_id": "12345678",
"revoke_rules_on_delete": "False",
"tags.%": "2",
"tags.Name": "abc.xyz.pqr",
"tags.abc": "owned"
}
Create a dict who has same number within th data dict.
I want like this. The number should come dynamically in future
e.g.
Data = [{
"arn": "arn",
"description": "Security group for all coporate communications",
"egress.#": "1",
{
"egress.4820.cidr_blocks.#": "1",
"egress.4820.cidr_blocks.0": "0.0.0.0/0",
"egress.4820.description": "",
"egress.4820.from_port": "0",
"egress.4820.ipv6_cidr_blocks.#": "0",
"egress.4820.prefix_list_ids.#": "0",
"egress.4820.protocol": "-1",
"egress.4820.security_groups.#": "0",
"egress.4820.self": "False",
"egress.4820.to_port": "0",
},
"id": "sg-080b03",
"ingress.#": "4",
{
"ingress.1279476397.cidr_blocks.#": "0",
"ingress.1279476397.description": "self",
"ingress.1279476397.from_port": "0",
"ingress.1279476397.prefix_list_ids.#": "0",
"ingress.1279476397.protocol": "-1",
"ingress.1279476397.security_groups.#": "0",
"ingress.1279476397.self": "true",
"ingress.1279476397.to_port": "0"
},
{
"ingress.2455438834.cidr_blocks.0": "10.10.0.0/16",
"ingress.2455438834.description": "cluster VPC",
"ingress.2455438834.from_port": "443",
"ingress.2455438834.protocol": "tcp",
"ingress.2455438834.to_port": "443"
},
{
"ingress.3391123749.cidr_blocks.#": "0",
"ingress.3391123749.description": "eks-cluster-master",
"ingress.3391123749.from_port": "443",
"ingress.3391123749.protocol": "tcp",
"ingress.3391123749.to_port": "443"
},
{
"ingress.439086653.cidr_blocks.#": "0",
"ingress.439086653.description": "eks-cluster-master",
"ingress.439086653.from_port": "1025",
"ingress.439086653.ipv6_cidr_blocks.#": "0",
"ingress.439086653.prefix_list_ids.#": "0",
"ingress.439086653.protocol": "tcp",
"ingress.439086653.security_groups.#": "1",
"ingress.439086653.security_groups.3631": "sg-0007",
"ingress.439086653.self": "False",
"ingress.439086653.to_port": "65535"
},
"name":"xyz.abc.corporate",
"owner_id": "12345678",
"revoke_rules_on_delete": "False",
"tags.%": "2",
"tags.Name": "abc.xyz.pqr",
"tags.abc": "owned"
}]
Can you please suggest me if somebody know the logic for this?

As Christian König already mentioned:
Your desired result is not a valid python data structure.
because your code contains things like this:
{
# ...
"egress.#": "1", # <-- ok
{ # <-- not ok (key is missing)
"egress.4820.cidr_blocks.#": "1",
"egress.4820.cidr_blocks.0": "0.0.0.0/0",
"egress.4820.description": "",
"egress.4820.from_port": "0",
"egress.4820.ipv6_cidr_blocks.#": "0",
"egress.4820.prefix_list_ids.#": "0",
"egress.4820.protocol": "-1",
"egress.4820.security_groups.#": "0",
"egress.4820.self": "False",
"egress.4820.to_port": "0",
},
}
But if you want to completely unfold your data like this (probably not very efficient):
Data: {
"description": "Security group for all coporate communications"
"arn": "arn"
"name": "xyz.abc.corporate"
"owner_id": "12345678"
"tags": {
"Name": "abc.xyz.pqr"
"%": "2"
"abc": "owned"
}
"revoke_rules_on_delete": "False"
"egress": {
"4820": {
"description": ""
"ipv6_cidr_blocks": {
"#": "0"
}
"prefix_list_ids": {
"#": "0"
}
"to_port": "0"
"cidr_blocks": {
"#": "1"
"0": "0.0.0.0/0"
}
"security_groups": {
"#": "0"
}
"self": "False"
"protocol": "-1"
"from_port": "0"
}
"#": "1"
}
"id": "sg-080b03"
"ingress": {
"1279476397": {
"description": "self"
"prefix_list_ids": {
"#": "0"
}
"to_port": "0"
"cidr_blocks": {
"#": "0"
}
"security_groups": {
"#": "0"
}
"self": "true"
"protocol": "-1"
"from_port": "0"
}
"3391123749": {
"description": "eks-cluster-master"
"cidr_blocks": {
"#": "0"
}
"to_port": "443"
"protocol": "tcp"
"from_port": "443"
}
"439086653": {
"description": "eks-cluster-master"
"ipv6_cidr_blocks": {
"#": "0"
}
"prefix_list_ids": {
"#": "0"
}
"to_port": "65535"
"cidr_blocks": {
"#": "0"
}
"security_groups": {
"3696519931": "sg-0007a603523411"
"#": "1"
}
"self": "False"
"protocol": "tcp"
"from_port": "1025"
}
"2455438834": {
"to_port": "443"
"cidr_blocks": {
"0": "10.10.0.0/16"
}
"protocol": "tcp"
"description": "cluster VPC"
"from_port": "443"
}
"#": "4"
}
}
you can use the following code:
def transformData(data, separator = '.'):
def insertRecursive(dict_, key, value): # Recursive
''' inserts recursively:
mydict = makeDefaultDict()
insertRecursive(mydict, "deply.nested.value", 42)
print(mydict["deply"]["nested"]["value"]) # prints 42 '''
l, dot, r = key.partition(separator) # partition at first separator
if dot == "": # if there was no separator left
dict_[l] = value
else:
insertRecursive(dict_[l], r, value)
def makeDefaultDict(): # Recursive
''' creates a dictionary thats default value is another dict with the same properties
so you can do something like this:
mydict = makeDefaultDict()
mydict["deply"]["nested"]["value"] = 42 '''
return defaultdict(makeDefaultDict)
# actual code:
result = makeDefaultDict()
for key, value in data.items(): # for each entry in data:
insertRecursive(result, key, value)
return result

can't print the value of json object in django

I have this json object in ajax_data variable
{
"columns[0][data]": "0",
"columns[1][name]": "",
"columns[5][searchable]": "true",
"columns[5][name]": "",
"columns[4][search][regex]": "false",
"order[0][dir]": "asc",
"length": "10",
}
I have converted it using json.loads() function like.
ajax_data = json.loads(ajax_data)
I want to get the value if "order[0][dir]" and "columns[0][data]" but if i print it using
ajax_data['order'][0]['dir]
its giving error :
KeyError at /admin/help
'order'
But same code works if i access it for length key then it works.

The keys you have used are actually not a good way of implementation.
{
"columns[0][data]": "0",
"columns[1][name]": "",
"columns[5][searchable]": "true",
"columns[5][name]": "",
"columns[4][search][regex]": "false",
"order[0][dir]": "asc",
"length": "10",
}
Instead of this you should hav gone for
{
"columns": [
{"data": "0", "name": "", "searchable": "true", "name": "", "search": {
"regex": "false"}
},
{"data": "0", "name": "", "searchable": "true", "name": ""," search": {
"regex": "false"}},
{"data": "0", "name": "", "searchable": "true", "name": "", "search": {
"regex": "false"}},
{"data": "0", "name": "", "searchable": "true", "name": "", "search": {
"regex": "false"}},
{"data": "0", "name": "", "searchable": "true", "name": "", "search": {
"regex": "false"}},
{"data": "0", "name": "", "searchable": "true", "name": "", "search": {
"regex": "false"}},
],
"order": [
{"dir": "asc"}
],
"length": "10"
}
In this case ajax_data['order'][0]['dir] will result in value "asc"
For your current implementation the key is "order[0][dir]"
That is go for
ajax_data["order[0][dir]"]
Hope you understood the issue.
Structuring of json is very important when dealing with APIs. Try to restructure your json which will help for future too.

That's because length is a key in that json object, and order is not. The key names are the entire strings inside the quotes: columns[0][data], order[0][dir], etc.
Those are unusual key names, but perfectly valid.

Elastic Search Query value that is a dictionary

I have a index (7111) in elastic search that contains event log data. One of the fields (_source) contains a Dictionary of data. Is there any way using python to formulate a elastic search query that searches the key and values in the Dictionary?
For example let's say I want to do a query that searching for "ProcessID" = "2060" AND "EventID" = "2002". How would I formulate this query?
Note I am currently using pyes, and the best I have been able to do is return all data in the index:
from pyes import *
conn = ES('dbup:9200')
conn.default_indices=["7111"]
q = TermQuery("_type", "tzworks_evtwalk")
results = conn.search(query = q)
for r in results:
print results
Example json data:
"hits" : [ {
"_index" : "7111",
"_type" : "tzworks_evtwalk",
"_id" : "AU7cz4WnebFiST-VQOSA",
"_score" : 1.0,
"_source":{"ProcessID": "2060", "Time-UTC": " 14:04:14.071", "UserID": "S-1-5-19", "Version": "0", "RelatedActivityID": "0", "ThreadID": "6316", "SettingValue": "04 00 00 00", "ModifyingUser": "S-1-5-80-3088073201-1464728630-1879813800-1107566885-823218052", "Channel": "Microsoft-Windows-Windows Firewall With Advanced Security/Firewall", "guid": ["1e152f7025a311e5b1cf005056c00008"], "eventlog": "/home/xxxxx/xxxxx-dirs/workdir/Collection-070815-141327_7111/C/Windows/System32/winevt/Logs/Microsoft-Windows-Windows Firewall With Advanced Security%4Firewall.evtx", "Date": "05/26/2015", "Origin": "1", "Task": "0", "SettingValueSize": "4", "Name": "Microsoft-Windows-Windows Firewall With Advanced Security", "Level": "4", "Opcode": "0", "Security": "", "EventID": "2002", "Record#": "19937", "xmlns": "http://schemas.microsoft.com/win/2004/08/events/event", "SettingValueDisplay": "Public", "line_md5": "55dd6f2a29a43658c5904e39a2e66fc4", "ActivityID": "86432a0b-3c7d-4ddf-a89c-172faa90485d", "Computer": "xxxxxx-X7001.clients.us.xxxx.xxxxx.com", "ModifyingApplication": "", "SettingType": "2", "Keywords": "0x8000000000000000", "row_id": 9, "Guid": "d1bc9aff-2abf-4d71-9146-ecb2a986eb85", "Qualifiers": "0"}
}, {
"_index" : "7111",
"_type" : "tzworks_evtwalk",
"_id" : "AU7cz4WnebFiST-VQOSF",
"_score" : 1.0,
"_source":{"ProcessID": "2060", "Time-UTC": " 14:05:57.506", "UserID": "S-1-5-19", "Version": "0", "RelatedActivityID": "0", "ThreadID": "5988", "SettingValue": "05 00 00 00", "ModifyingUser": "S-1-5-80-3088073201-1464728630-1879813800-1107566885-823218052", "Channel": "Microsoft-Windows-Windows Firewall With Advanced Security/Firewall", "guid": ["1e152f7025a311e5b1cf005056c00008"], "eventlog": "/home/xxxxx/xxxxx-dirs/workdir/Collection-070815-141327_7111/C/Windows/System32/winevt/Logs/Microsoft-Windows-Windows Firewall With Advanced Security%4Firewall.evtx", "Date": "05/26/2015", "Origin": "1", "Task": "0", "SettingValueSize": "4", "Name": "Microsoft-Windows-Windows Firewall With Advanced Security", "Level": "4", "Opcode": "0", "Security": "", "EventID": "2002", "Record#": "19950", "xmlns": "http://schemas.microsoft.com/win/2004/08/events/event", "SettingValueDisplay": "Domain,Public", "line_md5": "0b20f373041ddb34a6f0fc61926dc5bc", "ActivityID": "0", "Computer": xxxxxxx-X7001.clients.us.xxxx.xxxxx.com", "ModifyingApplication": "", "SettingType": "2", "Keywords": "0x8000000000000000", "row_id": 14, "Guid": "d1bc9aff-2abf-4d71-9146-ecb2a986eb85", "Qualifiers": "0"}
}, {

You can use filtered query in python:
t1 = TermFilter(‘ProcessID’, ‘2060’)
t2 = TermFilter(‘EventID’, ‘2002’)
f = ANDFilter([t1, t2])
q = FilteredQuery(MatchAllQuery(), f)
results = conn.search(q)