I'm writing traceroute in Python3 as an exercise. It's working quite well, but every now and then, the trace will be unable to receive an answer from a certain hop and keep incrementing the TTL without ever getting a response. The regular linux traceroute program does not suffer from this problem so it has to be something in my code.
I've tried different variations, but they all suffer from the same problem. I'll run the trace on 8.8.8.8 and 4 out of 5 times, it works perfectly.
I've tried creating sockets using the context manager (with socket as ...) and also manually.
import os
import random
import socket
import sys
import time
class Traceroute:
def __init__(self, dest, maxhops=30, port=random.randrange(33434,33534), host=''):
self.dest = dest
self.maxhops = maxhops
self.port = port
self.host = host
self.hops = []
# Function for creating new sockets
def get_socket(self,type,proto=0):
s = socket.socket(
family=socket.AF_INET,
type=type,
proto=proto
)
s.bind((self.host,self.port))
return s
# Do the trace and store hop information for later use
def get_path(self):
try:
self.dest_ip = socket.gethostbyname(self.dest)
self.dest_name = socket.gethostbyaddr(self.dest)[0]
except socket.gaierror as e:
print(f'Error: {self.dest} - {e}')
address = ['']
ttl = 1
while ttl < self.maxhops +1 and address[0] != self.dest_ip:
receiver_socket = self.get_socket(socket.SOCK_RAW,socket.getprotobyname('icmp'))
receiver_socket.settimeout(2)
sender_socket = self.get_socket(socket.SOCK_DGRAM,socket.getprotobyname('udp'))
sender_socket.setsockopt(socket.SOL_IP,socket.IP_TTL,ttl)
sender_socket.sendto(b'',(self.dest_ip,self.port))
tries = 3
while tries > 0:
starttime = time.time()
try:
data, address = receiver_socket.recvfrom(1024)
latency = [round((time.time()-starttime)*1000,2)]
try:
hostname = socket.gethostbyaddr(address[0])[0]
except socket.herror as e:
hostname = address[0]
self.hops.append({
'address':address[0],
'hostname':hostname,
'latency': latency
})
tries = 0
except socket.error:
tries -= 1
if tries == 0:
self.hops.append({
'address':None,
'hostname':ttl,
'latency':None
})
finally:
receiver_socket.close()
sender_socket.close()
ttl += 1
os.system('clear')
self.draw()
def draw(self):
name = f'{self.dest_ip} - {self.dest_name}' if self.dest_name != self.dest_ip else self.dest_ip
print(f'Traceroute to {name}')
# Print the full hop list, for debugging
for hop in self.hops:
print(hop)
# Print the hops in a readable fashion
for hop in self.hops:
if hop['address'] is None:
print(f"{self.hops.index(hop)+1} - {'*':<50} - {'*':<50} - {'*':<50}")
else:
latencylen = len(hop['latency'])
latencyavg = str(round(sum(hop['latency'])/latencylen,2)) + 'ms'
name = f'{hop["address"]} - {hop["hostname"]}' if hop['address'] != hop['hostname'] else hop['address']
print(f'{self.hops.index(hop)+1} - {name:<50} - {latencyavg:<50} - LEN:{latencylen:<50}')
def main():
if len(sys.argv) < 2:
print('Please enter a destination')
else:
trace = Traceroute(sys.argv[1])
trace.get_path()
if __name__ == '__main__':
main()
Here is the output when it completely bugs out:
self.hops is a list of dictionaries.
The stuff in {brackets} is the hop dictionary itself. I added it for debugging.
Note that hops 2 and 3 are identical. So the trace somehow ran against this hop twice. The number at the beginning of the line is the list index, which also represents the TTL. I usually reach 8.8.8.8 by the 9th hop, but in this case it just keeps going, incrementing the TTL until it reaches the maximum.
Traceroute to 8.8.8.8 - google-public-dns-a.google.com
{'address': '192.168.1.1', 'hostname': '_gateway', 'latency': [8.21]}
{'address': '104.195.128.122', 'hostname': '104-195-128-122.cpe.teksavvy.com', 'latency': [1572.38]}
{'address': '104.195.128.122', 'hostname': '104-195-128-122.cpe.teksavvy.com', 'latency': [15.97]}
{'address': '104.195.129.9', 'hostname': '104-195-129-9.cpe.teksavvy.com', 'latency': [27.22]}
{'address': '206.248.155.92', 'hostname': 'ae10-0-bdr01-tor2.teksavvy.com', 'latency': [20.05]}
{'address': '206.248.155.10', 'hostname': 'ae12-0-bdr01-tor.teksavvy.com', 'latency': [27.2]}
{'address': None, 'hostname': 7, 'latency': None}
{'address': None, 'hostname': 8, 'latency': None}
{'address': None, 'hostname': 9, 'latency': None}
{'address': None, 'hostname': 10, 'latency': None}
{'address': None, 'hostname': 11, 'latency': None}
{'address': None, 'hostname': 12, 'latency': None}
{'address': None, 'hostname': 13, 'latency': None}
{'address': None, 'hostname': 14, 'latency': None}
{'address': None, 'hostname': 15, 'latency': None}
1 - 192.168.1.1 - _gateway - 8.21ms - LEN:1
2 - 104.195.128.122 - 104-195-128-122.cpe.teksavvy.com - 1572.38ms - LEN:1
3 - 104.195.128.122 - 104-195-128-122.cpe.teksavvy.com - 15.97ms - LEN:1
4 - 104.195.129.9 - 104-195-129-9.cpe.teksavvy.com - 27.22ms - LEN:1
5 - 206.248.155.92 - ae10-0-bdr01-tor2.teksavvy.com - 20.05ms - LEN:1
6 - 206.248.155.10 - ae12-0-bdr01-tor.teksavvy.com - 27.2ms - LEN:1
7 - * - * - *
8 - * - * - *
9 - * - * - *
10 - * - * - *
11 - * - * - *
12 - * - * - *
13 - * - * - *
14 - * - * - *
15 - * - * - *
Thanks,
Related
I am trying to implement limit order book using flask and I am working on the backend part right now. I am new to flask so I am still learning and I am not much aware about how backend of trading works but I am trying to learn via this small project.
I have created 3 endpoints in my application which add order, remove order and give a response of the order status and these three endpoints are working fine checked them with postman. Now I am trying to run a function in background which will continuously check the new orders (buy/sell) from a json file which save all new orders. It will pick them one by one and will find a match based on price if a user's buy order matches a different user's sell order it will process and store it in a dict which I want to return or store all those successful order to the user.
Here is my code for the class I have created:
import json
import bisect
import random
import os
class Process(object):
def __init__(self):
self.trade_book = []
self.bid_prices = []
self.ask_prices = []
self.ask_book = {}
self.bid_book = {}
self.confirm_traded = []
self.orders_history = {}
self.traded = False
self.counter = 0
def save_userdata(self,order, newId):
orderid = order['order']['trader'] +"_"+ str(newId)
user_list = order
newJson = {
"orders":[
{ orderid: order['order']}
]
}
with open('data_user.json', 'a+') as jsonFile:
with open('data_user.json', 'r') as readableJson:
try:
jsonObj = json.load(readableJson)
except Exception as e:
jsonObj = {}
if jsonObj == {}:
json.dump(newJson, jsonFile)
else:
with open('data_user.json', 'w+') as writeFile:
exists = False
for item in jsonObj['orders']:
if item.get(orderid, None) is not None:
item[orderid] = order['order']
exists = True
break
if not exists:
jsonObj['orders'].append(newJson['orders'][0])
json.dump(jsonObj, writeFile)
return orderid
def get_userdata(self):
with open('data_user.json', 'r') as readableJson:
return json.load(readableJson)
def removeOrder(self, orderid):
order_id = list(orderid.values())[0]
with open('data_user.json') as data_file:
data = json.load(data_file)
newData = []
for item in data['orders']:
if item.get(order_id, None) is not None:
del item[order_id]
else:
newData.append(item)
data['orders'] = newData
with open('data_user.json', 'w') as data_file:
data = json.dump(data, data_file)
return order_id
def add_order_to_book(self, order):
index = list(order.keys())[0]
book_order = order[index]
print(index)
if order[index]['side'] == 'buy':
book_prices = self.bid_prices
book = self.bid_book
else: #order[index]['side'] == 'sell'
book_prices = self.ask_prices
book = self.ask_book
if order[index]['price'] in book_prices:
book[order[index]['price']]['num_orders'] += 1
book[order[index]['price']]['size'] += order[index]['quantity']
book[order[index]['price']]['order_ids'].append(index)
book[order[index]['price']]['orders'][index] = book_order
else:
bisect.insort(book_prices, order[index]['price'])
book[order[index]['price']] = {'num_orders': 1, 'size': order[index]['quantity'],'order_ids':
[index],
'orders': {index: book_order}}
def confirm_trade(self,order_id, timestamp, order_quantity, order_price, order_side):
trader = order_id.partition('_')[0]
self.confirm_traded.append({ 'trader': trader,'quantity': order_quantity, 'side': order_side,
'price': order_price,
'status': 'Successful'})
return self.confirm_traded
def process_trade_orders(self, order):
self.traded = False
index = list(order.keys())[0]
if order[index]['side'] == 'buy':
book = self.ask_book
if order[index]['price'] in self.ask_prices:
remainder = order[index]['quantity']
while remainder > 0:
book_order_id = book[order[index]['price']]['order_ids'][0]
book_order = book[order[index]['price']]['orders'][book_order_id]
if remainder >= book_order['quantity']:
self.trade_book.append({'order_id': book_order_id, 'timestamp': order[index]['timestamp'],
'price': order[index]['price'],
'quantity': order[index]['quantity'], 'side': book_order['side']})
self.confirm_trade(index, order[index]['timestamp'], order[index]['quantity'], order[index]['price'], order[index]['side'])
self.traded = True
remainder = remainder - book_order['quantity']
self.save_historty_orders(index, order[index])
break
else:
self.traded = True
self.trade_book.append({'order_id': index, 'timestamp': order[index]['timestamp'],
'price': order[index]['price'],
'quantity': order[index]['quantity'], 'side': order[index]['side']})
self.confirm_trade(index, order[index]['timestamp'], order[index]['quantity'], order[index]['price'], order[index]['side'])
self.save_historty_orders(index, order[index])
break
else:
self.add_order_to_book(order)
self.save_historty_orders(index, order[index])
else: #order['side'] == 'sell'
book = self.bid_book
if order[index]['price'] in self.bid_prices:
remainder = order[index]['quantity']
while remainder > 0:
book_order_id = book[order[index]['price']]['order_ids'][0]
book_order = book[order[index]['price']]['orders'][book_order_id]
if remainder >= book_order['quantity']:
self.trade_book.append({'order_id': book_order_id, 'timestamp': order[index]['timestamp'],
'price': order[index]['price'],
'quantity': order[index]['quantity'], 'side': order[index]['side']})
self.traded = True
self.confirm_trade(index, order[index]['timestamp'], order[index]['quantity'], order[index]['price'], order[index]['side'])
remainder = remainder - book_order['quantity']
self.save_historty_orders(index, order[index])
break
else:
self.traded = True
self.trade_book.append({'order_id': book_order_id, 'timestamp': order[index]['timestamp'],
'price': order[index]['price'],
'quantity': order[index]['quantity'], 'side': order[index]['side']})
self.confirm_trade(index, order[index]['timestamp'], order[index]['quantity'], order[index]['price'], order[index]['side'])
self.save_historty_orders(index, order[index])
break
else:
self.add_order_to_book(order)
self.save_historty_orders(index, order[index])
This class process I create object in my app.py and call the function process_trade_orders in that inside a function processing():
app = Flask(__name__)
app.config['DEBUG'] = True
newUser = Process()
succorder = Success()
#sched = BackgroundScheduler()
def generate_orderid():
num = 0
while num < 1000:
yield num
num = num + 1
genid = generate_orderid()
proc = Process()
sucorder = Success()
#Processing orders to find if they have a match
def processing():
get_orders_data = proc.get_userdata()
print(get_orders_data)
print("\n")
for data in get_orders_data['orders']:
index = list(data.keys())[0]
if data[index]['status'] == 'Successful':
sucorder.add_trader_orders(data[index],index)
else:
proc.process_trade_orders(data)
# sched = BackgroundScheduler()
# sched.add_job(func = processing, trigger="interval", seconds = 2)
# sched.start()
I did use APSbackground-scheduler for the same but I want to use thread for it. I was thinking of running a main thread in infinite loop as a daemon and use worker thread to run this function processing() in app.py which will be called after every few seconds to check if there are any successful order it will return the value to the main thread and those list of dict every new one I can return a response or some other way to the user about this successful order getting matched.
Note that this will be running is short intervals like 5 seconds and multiple add orders will be added and will be continuously running the checks asynchronously so I am not sure how will I return those values. I am just confused so if anyone can help me will be grateful.
If you want to make a threaded function that runs in background, just use the threading module, like this:
from threading import Thread
def bg_func():
doSomething
t = Thread(target=bg_func)
t.start() # will start the function and continue, even if the function still runs
doSomethingelseAtSameTime # runs with bg_func
You can also have multiple background threads.
Check the documentation for more info.
I'm using dbus api for NetworkManager 1.4.0 with python wrapper. This code scan wifi networks:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import NetworkManager as NM
def ap_sec_to_str2(sec):
list = []
if sec & (0x1 | 0x2 | 0x10 | 0x20):
list.append("WEP")
if sec & (0x4 | 0x40):
list.append("WPA")
if sec & (0x8 | 0x80):
list.append("WPA2")
if sec & 0x100:
list.append("PSK")
if sec & 0x200:
list.append("802.1X")
return list
def MHZ2Chan(mhz):
return {
2412: 1,
2417: 2,
2422: 3,
2427: 4,
2432: 5,
2437: 6,
2442: 7,
2447: 8,
2452: 9,
2457: 10,
2462: 11,
2467: 12,
2472: 13,
2484: 14
}.get(mhz)
"""WIFI NetwokList"""
def APList(iface):
check = lambda dev: dev.DeviceType == NM.NM_DEVICE_TYPE_WIFI and dev.Interface == iface
dev = next( (dev for dev in NM.NetworkManager.GetDevices() if check(dev) ))
rslt = []
dev = dev.SpecificDevice()
dev.RequestScan({}) #this is wrong!!!
for ap in dev.GetAllAccessPoints():
info = {
'ssid': ap.Ssid,
'channel': MHZ2Chan(ap.Frequency),
'level': ap.Strength,
'security': ap_sec_to_str2(ap.WpaFlags | ap.RsnFlags)
}
rslt.append(info)
return rslt
If I often call this function sometimes occurs error
org.freedesktop.NetworkManager.Device.NotAllowed:Scanningnotallowedimmediatelyfollowingpreviousscan.
Whether it's hardware features?
Platform ARM CortexA9 Yocto 1.6
WIFI usb dongle:
ID 20f4:648b TRENDnetTEW-648UBM802.11n150MbpsMicroWirelessNAdapter[RealtekRTL8188CUS]
I am trying to update a key in a nested dictionary of multiprocessing module's manager.dict() but not able to do so. It doesn't update the value and doesn't throw any error too.
Code:
import time
import random
from multiprocessing import Pool, Manager
def spammer_task(d, token, repeat):
success = 0
fail = 0
while success+fail<repeat:
time.sleep(random.random()*2.0)
if (random.random()*100)>98.0:
fail+=1
else:
success+=1
d[token] = {
'status': 'ongoing',
'fail': fail,
'success': success,
'repeat': repeat
}
print d[token]['status']
d[token]['status'] = 'complete'
return
p = Pool()
m = Manager()
d = m.dict()
p.apply_async(spammer_task (d, 'abc', 5))
print d
Output:
ongoing
{'abc': {'status': 'ongoing', 'fail': 0, 'repeat': 5, 'success': 5}}
My expectations are that as soon as while loop ends, it should make d['abc']['status'] = complete. But on final print it prints its status as 'ongoing' only.
not sure why, but the Manager DictProxy object can't seem to handle mutating a nested part. this code works:
import time
import random
from multiprocessing import Pool, Manager
def spammer_task(d, token, repeat):
success = 0
fail = 0
while success+fail<repeat:
time.sleep(random.random()*2.0)
if (random.random()*100)>98.0:
fail+=1
else:
success+=1
d[token] = {
'status': 'ongoing',
'fail': fail,
'success': success,
'repeat': repeat,
}
print d[token]['status']
foo = d[token]
foo['status'] = 'complete'
d[token] = foo
return
p = Pool()
m = Manager()
d = m.dict()
p.apply_async(spammer_task(d, 'abc', 5))
print d
Looks like this issue remains per code below:
import multiprocessing, sys;
if __name__ == '__main__':
print(sys.version);
mpd = multiprocessing.Manager().dict();
mpd['prcss'] = {'q' : 'queue_1', 'ctlg' : 'ctlg_1' };
# update 1 - doesn't work!
mpd['prcss'].update( { 'name': 'concfun_1'} );
print('Result of failed update 1:', mpd['prcss']);
# update 2 - doesn't work!
mpd['prcss']['name'] = 'concfun_1';
print('Result of failed update 2:', mpd['prcss']);
# update 3 - works!
mpd_prcss = mpd['prcss'];
mpd_prcss['name'] = 'concfun_1';
mpd['prcss'] = mpd_prcss;
print('Result of successful update 3:', mpd['prcss']);
Output:
3.6.1 (v3.6.1:69c0db5, Mar 21 2017, 17:54:52) [MSC v.1900 32 bit (Intel)]
Result of failed update 1: {'q': 'queue_1', 'ctlg': 'ctlg_1'}
Result of failed update 2: {'q': 'queue_1', 'ctlg': 'ctlg_1'}
Result of successful update 3: {'q': 'queue_1', 'ctlg': 'ctlg_1',
'name': 'concfun_1'}
In playbook I'm using variable {{excluded_service}}. I want to run ansible playbook from python and provide this variable. And I can't use external inventory script to provide this variable.
I am using to create inventory:
hosts = ["127.0.0.1"]
inventory=ansible.inventory.Inventory(hosts)
but I don't understand where I can add value of variable?
My code, that works with external inventory script:
import sys
import os
import stat
import json
import ansible.playbook
import ansible.constants as C
import ansible.utils.template
from ansible import errors
from ansible import callbacks
from ansible import utils
from ansible.color import ANSIBLE_COLOR, stringc
from ansible.callbacks import display
playbook="/opt/RDE/3p/ansible/loop/testloop.yml"
inventory="/opt/RDE/3p/ansible/loop/lxc.py"
connection="local"
timeout=10
subset=None
options={'subset': None, 'ask_pass': False, 'sudo': False, 'private_key_file': None, 'syntax': None, 'skip_tags': None, 'diff': False, 'check': False, 'remote_user': 'root', 'listtasks': None, 'inventory': '/opt/RDE/3p/ansible/loop/lxc.py', 'forks': 5, 'listhosts': None, 'start_at': None, 'tags': 'all', 'step': None, 'sudo_user': None, 'ask_sudo_pass': False, 'extra_vars': [], 'connection': 'smart', 'timeout': 10, 'module_path': None}
sshpass = None
sudopass = None
extra_vars = {}
def colorize(lead, num, color):
""" Print 'lead' = 'num' in 'color' """
if num != 0 and ANSIBLE_COLOR and color is not None:
return "%s%s%-15s" % (stringc(lead, color), stringc("=", color), stringc(str(num), color))
else:
return "%s=%-4s" % (lead, str(num))
def hostcolor(host, stats, color=True):
if ANSIBLE_COLOR and color:
if stats['failures'] != 0 or stats['unreachable'] != 0:
return "%-37s" % stringc(host, 'red')
elif stats['changed'] != 0:
return "%-37s" % stringc(host, 'yellow')
else:
return "%-37s" % stringc(host, 'green')
return "%-26s" % host
inventory = ansible.inventory.Inventory(options['inventory'])
hosts = ["127.0.0.1"]
#inventory=ansible.inventory.Inventory(hosts)
inventory.subset(options['subset'])
if len(inventory.list_hosts()) == 0:
raise errors.AnsibleError("provided hosts list is empty")
inventory.set_playbook_basedir(os.path.dirname(playbook))
stats = callbacks.AggregateStats()
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
if options['step']:
playbook_cb.step = options['step']
if options['start_at']:
playbook_cb.start_at = options['start_at']
runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
pb = ansible.playbook.PlayBook(
playbook=playbook,
module_path=None,
inventory=inventory,
forks=options['forks'],
remote_user=options['remote_user'],
remote_pass=sshpass,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats,
timeout=options['timeout'],
transport=options['connection'],
sudo=options['sudo'],
sudo_user=options['sudo_user'],
extra_vars=extra_vars,
private_key_file=options['private_key_file'],
check=options['check'],
diff=options['diff']
)
playnum = 0
failed_hosts = []
unreachable_hosts = []
try:
print pb.run()
hosts = sorted(pb.stats.processed.keys())
print hosts
display(callbacks.banner("PLAY RECAP"))
playbook_cb.on_stats(pb.stats)
for h in hosts:
t = pb.stats.summarize(h)
if t['failures'] > 0:
failed_hosts.append(h)
if t['unreachable'] > 0:
unreachable_hosts.append(h)
retries = failed_hosts + unreachable_hosts
if len(retries) > 0:
filename = pb.generate_retry_inventory(retries)
if filename:
display(" to retry, use: --limit #%s\n" % filename)
for h in hosts:
t = pb.stats.summarize(h)
display("%s : %s %s %s %s" % (
hostcolor(h, t),
colorize('ok', t['ok'], 'green'),
colorize('changed', t['changed'], 'yellow'),
colorize('unreachable', t['unreachable'], 'red'),
colorize('failed', t['failures'], 'red')),
screen_only=True
)
display("%s : %s %s %s %s" % (
hostcolor(h, t, False),
colorize('ok', t['ok'], None),
colorize('changed', t['changed'], None),
colorize('unreachable', t['unreachable'], None),
colorize('failed', t['failures'], None)),
log_only=True
)
except Exception as e:
print ("!!!!!!!ERROR: %s" % e)
Specify the vars in a host_vars file?
E.g. create a YAML file named /etc/ansible/host_vars/localhost with the vars you wanna put there.
I don't know, yet, how to specify it in Python code itself.
--- update ---
After a quick look the code, I don't think ansible supports specifying host variables when you specify hosts via host_list parameter. Hacks I can think of (if you must do this in python code) are:
[mis]use extra_vars parameter.
Write an inventory file (YAML or executable that just prints required JSON) from your python code and pass it's path as parameter inventory.
HTH
Hi i have an log data
IPADDRESS['192.168.0.10'] - - PlATFORM['windows'] - - BROWSER['chrome'] - - EPOCH['1402049518'] - - HEXA['0x3c0593ee']
I want to extract the corresponding value inside the bracket using regular expression
for example (this does not work)
ipaddress = re.findall(r"\[(IPADDRESS[^]]*)",output)
ouput:
ipaddress = 192.168.0.10
You can simply get all the elements as a dictionary like this
print dict(re.findall(r'([a-zA-Z]+)\[\'(.*?)\'\]', data))
Output
{'BROWSER': 'chrome',
'EPOCH': '1402049518',
'HEXA': '0x3c0593ee',
'IPADDRESS': '192.168.0.10',
'PlATFORM': 'windows'}
s1 = "IPADDRESS['192.168.0.10'] - - PlATFORM['windows'] - - BROWSER['chrome'] - - EPOCH['1402049518'] - - HEXA['0x3c0593ee']"
import re
print re.findall('\[\'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\'\]',s1)[0]
192.168.0.10
To get all in a list:
s1 = "IPADDRESS['192.168.0.10'] - - PlATFORM['windows'] - - BROWSER['chrome'] - - EPOCH['1402049518'] - - HEXA['0x3c0593ee']"
print re.findall('\[\'(.*?)\'\]',s1)
['192.168.0.10', 'windows', 'chrome', '1402049518', '0x3c0593ee']
result=re.findall('\[\'(.*?)\'\]',s1)
ipaddress, platform, browser, epoch, hexa = result # assign all variables
print "ipaddress = {}, platform = {}, browser = {}, epoch = {}, hexa = {}".format(ipaddress,platform,browser,epoch,hexa)
ipaddress = 192.168.0.10, platform = windows, browser = chrome, epoch = 1402049518, hexa = 0x3c0593ee