I'm trying to build a dictionary that looks like this:
nodes = {
'var': {
'type': 'd',
'full_path': '/var'
'active': True
'www': {
'type': 'd',
'full_path': '/var/www',
'active': True
'index.html': {
'type': 'f',
'full_path': '/var/www/index.html',
'active': False
}
'log': {
'type': 'd',
'full_path': '/var/log',
'active': False
}
}
'srv': {
'type': 'd',
'full_path': '/srv',
'active': True
}
}
I need it to be built by two pieces... The first needs to be from the file system where everything is 'active'. The second needs to come from a listing of full paths of files where everything is inactive.
So...
nodes = {}
for f, d, r in os.walk(root_path):
# append active items to nodes
for f in os.system(command_that_gets_files)
# append inactive items to nodes; not overwriting active
I'm sure I'm missing details...
Here's one way to get the active files. I found it easier to recurse than to use os.walk()'s iterative data. You may uncomment the result['stat'] line if you need to preserve more information than file type.
Every file has a dict entry like:
filename : { 'active' : True,
'full_path' = '/path/to/filename',
'type' : 'f' }
Every directory has a dict entry like:
dirname : { 'active' : True,
'full_path' = '/path/to/dirname',
'type' : 'd',
items = { 'itemname' : {...}, ... } }
Here you go:
import sys
import os
from stat import *
import pprint
def PathToDict(path):
st = os.stat(path)
result = {}
result['active'] = True
#result['stat'] = st
result['full_path'] = path
if S_ISDIR(st.st_mode):
result['type'] = 'd'
result['items'] = {
name : PathToDict(path+'/'+name)
for name in os.listdir(path)}
else:
result['type'] = 'f'
return result
pprint.pprint(PathToDict(sys.argv[1]))
Result:
{'active': True,
'full_path': '/tmp/x',
'items': {'var': {'active': True,
'full_path': '/tmp/x/var',
'items': {'log': {'active': True,
'full_path': '/tmp/x/var/log',
'items': {},
'type': 'd'},
'www': {'active': True,
'full_path': '/tmp/x/var/www',
'items': {'index.html': {'active': True,
'full_path': '/tmp/x/var/www/index.html',
'type': 'f'}},
'type': 'd'}},
'type': 'd'}},
'type': 'd'}
Related
I am trying to check if a specific item in a json file is equal to one of my python variables.
{'data': {'redemption': {'channel_id': 'secret',
'id': 'secret',
'redeemed_at': '2021-02-08T09:46:22.637059711Z',
'reward': {'background_color': '#FA1ED2',
'channel_id': '145998001',
'cooldown_expires_at': None,
'cost': 500,
'default_image': {'url_1x': 'https://static-cdn.jtvnw.net/custom-reward-images/ghost-1.png',
'url_2x': 'https://static-cdn.jtvnw.net/custom-reward-images/ghost-2.png',
'url_4x': 'https://static-cdn.jtvnw.net/custom-reward-images/ghost-4.png'},
'global_cooldown': {'global_cooldown_seconds': 1,
'is_enabled': False},
'id': '123',
'image': None,
'is_enabled': True,
'is_in_stock': True,
'is_paused': False,
'is_sub_only': False,
'is_user_input_required': False,
'max_per_stream': {'is_enabled': False,
'max_per_stream': 1},
'max_per_user_per_stream': {'is_enabled': False,
'max_per_user_per_stream': 1},
'prompt': '*Dabs*',
'redemptions_redeemed_current_stream': None,
'should_redemptions_skip_request_queue': False,
'template_id': 'template:4425c37e-6881-442a-aa3d-fdc6998a29de',
'title': 'Dab!',
'updated_for_indicator_at': '2020-09-10T18:55:40.064177881Z'},
'status': 'UNFULFILLED',
'user': {'display_name': 'Androteex',
'id': 'secret',
'login': 'androteex'}},
'timestamp': '2021-02-08T09:46:22.637059711Z'},
'type': 'reward-redeemed'}
I want to find the second id: 'id': '123' and check if id is equal to 123. And if so I want to print that string. How could I do that?
You can use the JSON module.
import json
data = json.loads(my_json)
my_id = data['data']['redemption']['reward']['id']
if my_id == '123':
print(data)
Granted it was added to data (can be done with json.loads):
id = data['data']['redemption']['reward']['id']
idcheck = 123
if (int(id) == idcheck):
print ("YES")
I have been trying to do something simple yet something hard for me to solve it!
I have a json object that looks like:
jsonObject = {
'attributes': {
'192': { <--- This can be changed times to times meaning different number
'id': '192',
'code': 'hello',
'label': 'world',
'options': [
{
'id': '211',
'label': '5'
},
{
'id': '1202',
'label': '8.5'
},
{
'id': '54',
'label': '9'
},
{
'id': '1203',
'label': '9.5'
},
{
'id': '58',
'label': '10'
}
]
}
},
'template': '12345',
'basePrice': '51233',
'oldPrice': '51212',
'productId': 'hello',
}
and what I want to do is to get the values from options (To have both id and label saved into a list)
For now I only managed to do:
for att, value in jsonObject.items():
print(f"{att} - {value}"
How can I get the label and id?
You can try the following code:
attr = jsonObject['attributes']
temp = list(attr.values())[0] # It is same as "temp = attr['192']", but you said '192' can be changed.
options = temp['options']
for option in options:
print(f"id: {option['id']}, label: {option['label']}")
I am trying to merge two nested dictionaries that will merge different dictionaries by comparing values, If the key will be same it will be replaced with the recent value.
Im using always_merger.merge() for merging but is appending the different element.It supposed to compare and replace values with recent one.
dictionary_1 = dict([('modifyParameterReq', {
'managedObject': {
'distName': 'ethLink_1',
'operation': 'create_mo',
'parameter': [
{
'parameterName': 'houseName',
'newValue': 'Bhawan',
'prevValue': ''
},
{
'parameterName': 'subscribe',
'newValue': 'TRUE',
'prevValue': ''
},
{
'parameterName': 'remoteMacAddress',
'newValue': 'd6-68-05-5e-06-b9',
'prevValue': ''
}
],
'class': 'EthernetLink'
}
})]
)
dictionary_2 = dict([('modifyParameterReq', {
'managedObject': {
'distName': 'ethLink_1',
'operation': 'create_mo',
'parameter': [
{
'parameterName': 'subscribe',
'newValue': 'FALSE',
'prevValue': ''
},
{
'parameterName': 'remoteMacAddress',
'newValue': 'a1-b1-c3-d4-e5-f6',
'prevValue': ''
},
{
'parameterName': 'yourName',
'newValue': 'Vicky',
'prevValue': ''
}
],
'class': 'EthernetLink'
}
})
]
)
if the element is not present add the element else replace with new value.
Any help should be appreciated.
This should do it. I 've written some comments in between to make it more readable:
# Get all current parameter names from dictionary 1.
currentDictParameterNames = [elem['parameterName'] for elem in dictionary_1['modifyParameterReq']['managedObject']['parameter']]
# Iterate through parameters from dictionary 2.
for index, dictData in enumerate(dictionary_2['modifyParameterReq']['managedObject']['parameter']):
# If current key iterated from dictionary 2 exists in dictionary 1, update.
if dictData['parameterName'] in currentDictParameterNames:
# FInd the location of the parameter name in dictionary 1 and update.
for i in range(len(dictionary_1['modifyParameterReq']['managedObject']['parameter'])):
if dictionary_1['modifyParameterReq']['managedObject']['parameter'][i]['parameterName'] == dictData['parameterName']:
dictionary_1['modifyParameterReq']['managedObject']['parameter'][i]['prevValue'] = dictionary_1['modifyParameterReq']['managedObject']['parameter'][i]['newValue']
dictionary_1['modifyParameterReq']['managedObject']['parameter'][i]['newValue'] = dictData['newValue']
break
# Else create a new record.
else:
dictionary_1['modifyParameterReq']['managedObject']['parameter'].append(dictData)
print(dictionary_1)
The result should be:
{'modifyParameterReq': {'managedObject': {
'distName': 'ethLink_1',
'operation': 'create_mo',
'parameter': [
{'parameterName': 'houseName', 'newValue': 'Bhawan', 'prevValue': ''},
{'parameterName': 'subscribe', 'newValue': 'FALSE', 'prevValue': 'TRUE'},
{'parameterName': 'remoteMacAddress', 'newValue': 'a1-b1-c3-d4-e5-f6', 'prevValue': 'd6-68-05-5e-06-b9'},
{'parameterName': 'yourName', 'newValue': 'Vicky', 'prevValue': ''}],
'class': 'EthernetLink',
}}}
I'm trying to write a post commit hook script for svn to export repository to multiple directories (so when you commit a code, that latest version of the code gets copied to other directories also). For example I have a repo which has 4 paths (or directories) A,B,C,D. I want that if I commit in A then the revision should also go to B,C,D. Similarly if I commit in B then revision should go to C,D and likewise when committed on C it should reflect on D.
I came to know about post-commit but don't know how to use it. I have written this bunch of code which is working partially. i.e when I am committing in A it goes to B but not to C or D, similarly when I commit on B it goes to C but not D.
I have not done scripting before. I have made a test-repo Test for testing my work.
#!/usr/bin/python
import os
import re
import sys
from subprocess import Popen, PIPE
REPOS, REV, TXN_NAME = sys.argv[1:]
SVNADMIN = '/opt/softwares/csvn/bin/svnadmin'
MAPPINGS = [
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'A'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'B'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'A'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'C'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'A'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'D'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'B'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'C'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'B'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'D'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'C'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'D'}
},
]
TMP_MAPPINGS = MAPPINGS
MAPPINGS = []
for mapping in TMP_MAPPINGS:
if mapping not in MAPPINGS:
MAPPINGS.append(mapping)
del TMP_MAPPINGS
MAPPINGS.sort(key=lambda mapping: len(mapping['FROM']['PATH']), reverse=True)
def map_(revision_content, to_repos):
pattern = '\n(?:Node-path|Node-copyfrom-path): ([^\n]*)\n'
mapped = {'VALUE': False}
def repl(match):
path = match.group(1)
for mapping in MAPPINGS:
if os.path.samefile(mapping['FROM']['REPOS'], REPOS) \
and mapping['TO']['REPOS'] == to_repos \
and path.startswith(mapping['FROM']['PATH']):
path = mapping['TO']['PATH'] + path[len(mapping['FROM']['PATH']):]
mapped['VALUE'] = True
break
return re.sub(': [^\n]*', ': ' + path, match.group(0), 1)
return re.sub(pattern, repl, revision_content), mapped['VALUE']
there_were_errors = False
processed_to_repos = []
revision_content = None
for mapping in MAPPINGS:
if not os.path.samefile(mapping['FROM']['REPOS'], REPOS):
continue
to_repos = mapping['TO']['REPOS']
if to_repos in processed_to_repos:
continue
processed_to_repos.append(to_repos)
if revision_content is None:
dump_process = Popen([SVNADMIN, 'dump', REPOS, '-r', REV, '--incremental', '-q'], stdout=PIPE, stderr=PIPE)
revision_content, errors = dump_process.communicate()
if errors:
print >> sys.stderr, errors
there_were_errors = True
break
mapped_revision_content, mapped = map_(revision_content, to_repos)
if not mapped:
continue
load_process = Popen([SVNADMIN, 'load', to_repos, '-r', REV, '--ignore-uuid', '-q'], stdin=PIPE, stderr=PIPE)
_, errors = load_process.communicate(mapped_revision_content)
if errors:
print >> sys.stderr, errors
there_were_errors = True
if there_were_errors:
sys.exit(1)
sys.exit(0)
ok got it.. working with
#!/usr/bin/python
import os
import re
import sys
from subprocess import Popen, PIPE
REPOS, REV, TXN_NAME = sys.argv[1:]
SVNADMIN = '/opt/softwares/csvn/bin/svnadmin'
MAPPINGS = [
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'A'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'B'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'A'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'C'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'A'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'D'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'B'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'C'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'B'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'D'}
},
{
'FROM': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'C'},
'TO': {'REPOS': '/opt/softwares/csvn/data/repositories/Test', 'PATH': 'D'}
},
]
MAPPINGS.sort(key=lambda mapping: len(mapping['FROM']['PATH']), reverse=True)
def map_(revision_content, to_repos):
global MAPPINGS
pattern = '\n(?:Node-path|Node-copyfrom-path): ([^\n]*)\n'
mapped = {'VALUE': False}
def repl(match):
global MAPPINGS
path = match.group(1)
for mapping in MAPPINGS:
if os.path.samefile(mapping['FROM']['REPOS'], REPOS) \
and mapping['TO']['REPOS'] == to_repos \
and path.startswith(mapping['FROM']['PATH']):
path = mapping['TO']['PATH'] + path[len(mapping['FROM']['PATH']):]
mapped['VALUE'] = True
break
return re.sub(': [^\n]*', ': ' + path, match.group(0), 1)
return re.sub(pattern, repl, revision_content), mapped['VALUE']
there_were_errors = False
revision_content = None
dump_process = Popen([SVNADMIN, 'dump', REPOS, '-r', REV, '--incremental', '-q'], stdout=PIPE, stderr=PIPE)
revision_content, errors = dump_process.communicate()
if errors:
print >> sys.stderr, errors
there_were_errors = True
for mapping in MAPPINGS:
if there_were_errors:
break
if not os.path.samefile(mapping['FROM']['REPOS'], REPOS):
continue
to_repos = mapping['TO']['REPOS']
mapped_revision_content, mapped = map_(revision_content, to_repos)
if not mapped:
continue
load_process = Popen([SVNADMIN, 'load', to_repos, '-r', REV, '--ignore-uuid', '-q'], stdin=PIPE, stderr=PIPE)
_, errors = load_process.communicate(mapped_revision_content)
revision_content = mapped_revision_content
if errors:
print >> sys.stderr, errors
there_were_errors = True
if there_were_errors:
sys.exit(1)
sys.exit(0)
Now it does the required job as expected !!
Sorry for the length but tried to be complete.
I'm trying to get the following data -
(only small sampling from a much larger json file, same structure)
{
"count": 394,
"status": "ok",
"data": [
{
"md5": "cd042ba78d0810d86755136609793d6d",
"threatscore": 90,
"threatlevel": 0,
"avdetect": 0,
"vxfamily": "",
"domains": [
"dynamicflakesdemo.com",
"www.bountifulbreast.co.uk"
],
"hosts": [
"66.33.214.180",
"64.130.23.5",
],
"environmentId": "1",
},
{
"md5": "4f3a560c8deba19c5efd48e9b6826adb",
"threatscore": 65,
"threatlevel": 0,
"avdetect": 0,
"vxfamily": "",
"domains": [
"px.adhigh.net"
],
"hosts": [
"130.211.155.133",
"65.52.108.163",
"172.225.246.16"
],
"environmentId": "1",
}
]
}
if "threatscore" is over 70 I want to add it to this json structure -
Ex.
"data": [
{
"md5": "cd042ba78d0810d86755136609793d6d",
"threatscore": 90,
{
"Event":
{"date":"2015-11-25",
"threat_level_id":"1",
"info":"HybridAnalysis",
"analysis":"0",
"distribution":"0",
"orgc":"SOC",
"Attribute": [
{"type":"ip-dst",
"category":"Network activity",
"to_ids":True,
"distribution":"3",
"value":"66.33.214.180"},
{"type":"ip-dst",
"category":"Network activity",
"to_ids":True,
"distribution":"3",
"value":"64.130.23.5"}
{"type":"domain",
"category":"Network activity",
"to_ids":True,
"distribution":"3",
"value":"dynamicflakesdemo.com"},
{"type":"domain",
"category":"Network activity",
"to_ids":True,
"distribution":"3",
"value":"www.bountifulbreast.co.uk"}
{"type":"md5",
"category":"Payload delivery",
"to_ids":True,
"distribution":"3",
"value":"cd042ba78d0810d86755136609793d6d"}]
}
}
This is my code -
from datetime import datetime
import os
import json
from pprint import pprint
now = datetime.now()
testFile = open("feed.json")
feed = json.load(testFile)
for x in feed['data']:
if x['threatscore'] > 90:
data = {}
data['Event']={}
data['Event']["date"] = now.strftime("%Y-%m-%d")
data['Event']["threat_level_id"] = "1"
data['Event']["info"] = "HybridAnalysis"
data['Event']["analysis"] = 0
data['Event']["distribution"] = 3
data['Event']["orgc"] = "Malware"
data['Event']["Attribute"] = []
if 'hosts' in x:
data['Event']["Attribute"].append({'type': "ip-dst"})
data['Event']["Attribute"][0]["category"] = "Network activity"
data['Event']["Attribute"][0]["to-ids"] = True
data['Event']["Attribute"][0]["distribution"] = "3"
data["Event"]["Attribute"][0]["value"] =x['hosts']
if 'md5' in x:
data['Event']["Attribute"].append({'type': "md5"})
data['Event']["Attribute"][1]["category"] = "Payload delivery"
data['Event']["Attribute"][1]["to-ids"] = True
data['Event']["Attribute"][1]["distribution"] = "3"
data['Event']["Attribute"][1]['value'] = x['md5']
if 'domains' in x:
data['Event']["Attribute"].append({'type': "domain"})
data['Event']["Attribute"][2]["category"] = "Network activity"
data['Event']["Attribute"][2]["to-ids"] = True
data['Event']["Attribute"][2]["distribution"] = "3"
data['Event']["Attribute"][2]["value"] = x['domains']
attributes = data["Event"]["Attribute"]
data["Event"]["Attribute"] = []
for attribute in attributes:
for value in attribute["value"]:
if value == " ":
pass
else:
new_attr = attribute.copy()
new_attr["value"] = value
data["Event"]["Attribute"].append(new_attr)
pprint(data)
with open('output.txt', 'w') as outfile:
json.dump(data, outfile)
And now it seems to be cleaned up a little but the data['md5'] is being split on each letter and I think it's just like L3viathan said earlier I keep overwriting the first element in the dictionary... but I'm not sure how to get it to keep appending???
{'Event': {'Attribute': [{'category': 'Network activity',
'distribution': '3',
'to-ids': True,
'type': 'ip-dst',
'value': u'216.115.96.174'},
{'category': 'Network activity',
'distribution': '3',
'to-ids': True,
'type': 'ip-dst',
'value': u'64.4.54.167'},
{'category': 'Network activity',
'distribution': '3',
'to-ids': True,
'type': 'ip-dst',
'value': u'63.250.200.37'},
{'category': 'Payload delivery',
'distribution': '3',
'to-ids': True,
'type': 'md5',
'value': u'7'},
{'category': 'Payload delivery',
'distribution': '3',
'to-ids': True,
'type': 'md5',
'value': u'1'},
And still getting the following error in the end:
Traceback (most recent call last):
File "hybridanalysis.py", line 34, in
data['Event']["Attribute"][1]["category"] = "Payload delivery"
IndexError: list index out of range
The final goal is to get it set so that I can post the events into MISP but they have to go one at a time.
I think this should fix your problems. I added the attribute dictionary all in one go, and moved the data in a list (which is more appropriate), but you might want to remove the superfluous list which wraps the Events.
from datetime import datetime
import os
import json
from pprint import pprint
now = datetime.now()
testFile = open("feed.json")
feed = json.load(testFile)
data_list = []
for x in feed['data']:
if x['threatscore'] > 90:
data = {}
data['Event']={}
data['Event']["date"] = now.strftime("%Y-%m-%d")
data['Event']["threat_level_id"] = "1"
data['Event']["info"] = "HybridAnalysis"
data['Event']["analysis"] = 0
data['Event']["distribution"] = 3
data['Event']["orgc"] = "Malware"
data['Event']["Attribute"] = []
if 'hosts' in x:
data['Event']["Attribute"].append({
'type': 'ip-dst',
'category': 'Network activity',
'to-ids': True,
'distribution': '3',
'value': x['hosts']})
if 'md5' in x:
data['Event']["Attribute"].append({
'type': 'md5',
'category': 'Payload delivery',
'to-ids': True,
'distribution': '3',
'value': x['md5']})
if 'domains' in x:
data['Event']["Attribute"].append({
'type': 'domain',
'category': 'Network activity',
'to-ids': True,
'distribution': '3',
'value': x['domains']})
attributes = data["Event"]["Attribute"]
data["Event"]["Attribute"] = []
for attribute in attributes:
for value in attribute["value"]:
if value == " ":
pass
else:
new_attr = attribute.copy()
new_attr["value"] = value
data["Event"]["Attribute"].append(new_attr)
data_list.append(data)
with open('output.txt', 'w') as outfile:
json.dump(data_list, outfile)
In the json, "Attiribute" Holds the value of a list with a 1 item, a dict, in it, as shown here.
{'Event': {'Attribute': [{'category': 'Network activity',
'distribution': '3',
'to-ids': True,
'type': 'ip-dst',
'value': [u'54.94.221.70']}]
...
When you call data['Event']["Attribute"][1]["category"] you are getting the second item (index 1) in the list of attribute, while it only has one item, which is why you are getting the error.
Thanks L3viathan! Below is how I tweaked it to not iterate over MD5's.
attributes = data["Event"]["Attribute"]
data["Event"]["Attribute"] = []
for attribute in attributes:
if attribute['type'] == 'md5':
new_attr = attribute.copy()
new_attr["value"] = str(x['md5'])
data["Event"]["Attribute"].append(new_attr)
else:
for value in attribute["value"]:
new_attr = attribute.copy()
new_attr["value"] = value
data["Event"]["Attribute"].append(new_attr)
data_list.append(data)
Manipulating json seems to be the way to go to learn lists and dictionaries.