I have a ROUTER whose purpose is to accumulate image data from multiple DEALER clients and perform OCR on the complete image. I found that the most efficient way of handling the OCR is through the utilization of Python's multiprocessing library; the accumulated image bytes are put into a Queue for due procession in a separate Process. However, I need to ensure that when a client experiences a timeout that the Process is properly terminated and doesn't meaninglessly linger and hog resources.
In my current solution I insert each newly-connected client into a dict where the value is my ClientHandler class that possesses all image data and spawns a Thread that sets a boolean variable named "timeout" to True when 5 seconds have elapsed. Should a new message be received within that 5 second frame, bump is called & the timer is reset back to 0, otherwise I cleanup prior to thread termination and the reference is deleted from the dict in the main loop:
import threading
import time
import zmq
class ClientHandler(threading.Thread):
def __init__(self, socket):
self.elapsed = time.time()
self.timeout = False
self.socket = socket
super(ClientHandler, self).__init__()
def run(self):
while time.time() - self.elapsed < 5.0:
pass
self.timeout = True
# CLIENT TIMED OUT
# HANDLE TERMINATION AND CLEAN UP HERE
def bump(self):
self.elapsed = time.time()
def handle(self, id, header, data):
# HANDLE CLIENT DATA HERE
# ACCUMULATE IMAGE BYTES, ETC
self.socket.send_multipart([id, str(0)])
def server_task():
clients = dict()
context = zmq.Context.instance()
server = context.socket(zmq.ROUTER)
server.setsockopt(zmq.RCVTIMEO, 0)
server.bind("tcp://127.0.0.1:7777")
while True:
try:
id, header, data = server.recv_multipart()
client = clients.get(id)
if client == None:
client = clients[id] = ClientHandler(server)
client.start()
client.bump()
client.handle(id, header, data)
except zmq.Again:
for id in clients.keys():
if clients[id].timeout:
del clients[id]
context.term()
if __name__ == "__main__":
server_task()
But this entire method just doesn't feel right. Am I going about this improperly? If so, I would greatly appreciate if someone could point me in the right direction.
Figured it out myself, hoping it may be of assistance to others.
I instead have a ROUTER on an assigned port that distributes unique ports to each client, which thereafter connects to the newly-bound socket on said unique port. When a client disconnects, the port is recycled for reassignment.
import sys
import zmq
from multiprocessing import Process, Queue, Value
def server_task():
context = zmq.Context.instance()
server = context.socket(zmq.ROUTER)
server.bind("tcp://127.0.0.1:7777")
timeout_queue = Queue()
port_list = [ 1 ]
proc_list = [ ]
while True:
try:
id = server.recv_multipart()[0]
# Get an unused port from the list
# Ports from clients that have timed out are recycled here
while not timeout_queue.empty():
port_list.append(timeout_queue.get())
port = port_list.pop()
if len(port_list) == 0:
port_list.append(port + 1)
# Spawn a new worker task, binding the port to a socket
proc_running = Value("b", True)
proc_list.append(proc_running)
Process(target=worker_task, args=(proc_running, port, timeout_queue)).start()
# Send the new port to the client
server.send_multipart([id, str(7777 + port)])
except KeyboardInterrupt:
break
# Safely allow our worker processes to terminate
for proc_running in proc_list:
proc_running.value = False
context.term()
def worker_task(proc_running, port, timeout_queue):
context = zmq.Context.instance()
worker = context.socket(zmq.ROUTER)
worker.setsockopt(zmq.RCVTIMEO, 5000)
worker.bind("tcp://127.0.0.1:%d" % (7777 + port, ))
while proc_running.value:
try:
id, data = worker.recv_multipart()
worker.send_multipart([id, data])
except zmq.Again:
timeout_queue.put(port)
context.term()
break
print("Client on port %d disconnected" % (7777 + port, ))
Related
The bounty expires in 5 days. Answers to this question are eligible for a +50 reputation bounty.
Haley Mueller wants to draw more attention to this question.
I'm new to Python so this could be a simple fix.
I am using Flask and sockets for this Python project. I am starting the socket on another thread so I can actively listen for new messages. I have an array variable called 'SocketConnections' that is within my UdpComms class. The variable gets a new 'Connection' appended to it when a new socket connection is made. That works correctly. My issue is that when I try to read 'SocketConnections' from outside of the thread looking in, it is an empty array.
server.py
from flask import Flask, jsonify
import UdpComms as U
app = Flask(__name__)
#app.route('/api/talk', methods=['POST'])
def talk():
global global_server_socket
apples = global_server_socket.SocketConnections
return jsonify(message=apples)
global_server_socket = None
def start_server():
global global_server_socket
sock = U.UdpComms(udpIP="127.0.0.1", portTX=8000, portRX=8001, enableRX=True, suppressWarnings=True)
i = 0
global_server_socket = sock
while True:
i += 1
data = sock.ReadReceivedData() # read data
if data != None: # if NEW data has been received since last ReadReceivedData function call
print(data) # print new received data
time.sleep(1)
if __name__ == '__main__':
server_thread = threading.Thread(target=start_server)
server_thread.start()
app.run(debug=True,host='192.168.0.25')
UdpComms.py
import json
import uuid
class UdpComms():
def __init__(self,udpIP,portTX,portRX,enableRX=False,suppressWarnings=True):
self.SocketConnections = []
import socket
self.udpIP = udpIP
self.udpSendPort = portTX
self.udpRcvPort = portRX
self.enableRX = enableRX
self.suppressWarnings = suppressWarnings # when true warnings are suppressed
self.isDataReceived = False
self.dataRX = None
# Connect via UDP
self.udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # internet protocol, udp (DGRAM) socket
self.udpSock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # allows the address/port to be reused immediately instead of it being stuck in the TIME_WAIT state waiting for late packets to arrive.
self.udpSock.bind((udpIP, portRX))
# Create Receiving thread if required
if enableRX:
import threading
self.rxThread = threading.Thread(target=self.ReadUdpThreadFunc, daemon=True)
self.rxThread.start()
def __del__(self):
self.CloseSocket()
def CloseSocket(self):
# Function to close socket
self.udpSock.close()
def SendData(self, strToSend):
# Use this function to send string to C#
self.udpSock.sendto(bytes(strToSend,'utf-8'), (self.udpIP, self.udpSendPort))
def SendDataAddress(self, strToSend, guid):
# Use this function to send string to C#
print('finding connection: ' + guid)
if self.SocketConnections:
connection = self.GetConnectionByGUID(guid)
print('found connection: ' + guid)
if connection is not None:
self.udpSock.sendto(bytes(strToSend,'utf-8'), connection.Address)
def ReceiveData(self):
if not self.enableRX: # if RX is not enabled, raise error
raise ValueError("Attempting to receive data without enabling this setting. Ensure this is enabled from the constructor")
data = None
try:
data, _ = self.udpSock.recvfrom(1024)
print('Socket data recieved from: ', _)
if self.IsNewConnection(_) == True:
print('New socket')
self.SendDataAddress("INIT:" + self.SocketConnections[-1].GUID, self.SocketConnections[-1].GUID)
data = data.decode('utf-8')
except WindowsError as e:
if e.winerror == 10054: # An error occurs if you try to receive before connecting to other application
if not self.suppressWarnings:
print("Are You connected to the other application? Connect to it!")
else:
pass
else:
raise ValueError("Unexpected Error. Are you sure that the received data can be converted to a string")
return data
def ReadUdpThreadFunc(self): # Should be called from thread
self.isDataReceived = False # Initially nothing received
while True:
data = self.ReceiveData() # Blocks (in thread) until data is returned (OR MAYBE UNTIL SOME TIMEOUT AS WELL)
self.dataRX = data # Populate AFTER new data is received
self.isDataReceived = True
# When it reaches here, data received is available
def ReadReceivedData(self):
data = None
if self.isDataReceived: # if data has been received
self.isDataReceived = False
data = self.dataRX
self.dataRX = None # Empty receive buffer
if data != None and data.startswith('DIALOG:'): #send it info
split = data.split(':')[1]
return data
class Connection:
def __init__(self, gUID, address) -> None:
self.GUID = gUID
self.Address = address
def IsNewConnection(self, address):
for connection in self.SocketConnections:
if connection.Address == address:
return False
print('Appending new connection...')
connection = self.Connection(str(uuid.uuid4()),address)
self.SocketConnections.append(connection)
return True
def GetConnectionByGUID(self, guid):
for connection in self.SocketConnections:
if connection.GUID == guid:
return connection
return None
As mentioned above. When IsNewConnection() is called in UdpComms it does append a new object to SocketConnections. It's just trying to view the SocketConnections in the app.route that is empty. My plans are to be able to send socket messages from the app.routes
For interprocess communication you may try to use something like shared memory documented here
Instead of declaring your self.SocketConnections as a list = []
you'd use self.SocketConnections = Array('i', range(10)) (you are then limited to remembering only 10 connections though).
For refreshing skills for a new job i have been trying to implement a very simple price publisher + client + mtm_cache. In the code below in publishing.py the code stops at time.sleep() and doesnt seem to wakeup (say for example two clients connect, but they stop receiving the messages while publisher executes time.sleep()), how can i correct the behavior of the publisher.
Also there is another class MTMCache where i hope to store the MTM sent by the client, would a dictionary be a good thread safe cache which stores list of tuples (time,MTMValue) per client.
As a third point how can i change the implementation of this Publisher class using asyncio?
publishing.py
import threading
import socket
import time
import random
class Publisher(object):
def __init__(self):
self.list_subscribers = []#list of tuples with ip and port
self.mtm_cache = None
self.current_price = 50
self.listening_ports = [1100,1101,1102,1103,1104]
self.stop_threads = False
self.listening_port = 1100
self.sock = None
self.listening = False
#run listening function in another thread
def update_price(self):
tmp = random.randint(-10,10)
#print("Number generated is " + str(tmp))
self.current_price = self.current_price + tmp
def update_price_in_loop(self):
while(True):
self.update_price()
if(self.stop_threads):
break
def send_price_to_subscribers(self):
#in a parallel for loop send current price to subscribers
while(True):
#print(str(self.current_price))
if self.list_subscribers:
for cl in self.list_subscribers:
cl.send(str(self.current_price).encode())
if(self.stop_threads):
break
def start_listener_and_accept_connections(self):
if(not self.listening):
self.sock = socket.socket()
self.sock.bind(('',self.listening_port))
print("sock bound at ", self.listening_port)
while True:
if not self.listening:
print("listening for connections")
self.sock.listen(5)
#above is a blocking call, it is blocking other threads
self.listening = True # how to properly set and utilise this flag
c, addr = self.sock.accept()
print('Got connection from', addr)
self.list_subscribers.append(c)
if(self.stop_threads):
print("closing client connections")
for cl in self.list_subscribers:
cl.close()
self.sock.close()
break
class MTMCache(object):
def __init__(self,publisher):
self.publisher = publisher
self.cache = {} #subscriber token for each of the subscribers in publisher class, currenttime and mtm to be stored here, maybe some other structure besides dict could be used, explore that later
def receive_mtm(self,message):
#message could be string in json format
#have to design this method with appropriate data structures
pass
if __name__ == "__main__":
from threading import *
import time
pub = Publisher()
#pub.update_price()
#print(pub.current_price)
t = Thread(target = pub.update_price_in_loop)
t1 = Thread(target = pub.send_price_to_subscribers)
#t.start()
#t1.start()
t2 = Thread(target = pub.start_listener_and_accept_connections)
t2.start()
t.start()
t1.start()
time.sleep(5)
pub.stop_threads = True
#t2.raise_exception()
#t.raise_exception()
#t1.raise_exception()
t2.join()
t.join()
t1.join()
------
basic_client.py
import socket
s = socket.socket()
port = 1100
s.connect(('127.0.0.1', port))
while(True):
tmp = s.recv(1024).decode()
print(tmp)
if not tmp:
break
s.close()
----
in separate terminals:
python3 publishing.py
term2
python3 basic_client.py
term3
python3 basic_client.py
I want to build a port map tool in python from scratch. Basically it is a TCP proxy that transits all the traffic between the client and the objective service or application.
Specifically, For every connection I create two sockets who are responsible to communicate to the client and objective service respectively. To implement IO-multiplexing feature, I use module selectors to monitor the EVENT_READ and EVENT_WRITE event on these two sockets. However I find that the sockets are always writable so that the main loop doesn't block at all. Is it normal? My code is as follows:
import socket
import selectors
def recv_from(sock):
data = b''
try:
while True:
chunk = sock.recv(4096)
if not chunk:
break
data += chunk
except:
pass
return data
class RelayHandlder:
def __init__(self, client_sock, remote_sock, selector):
'''
client_sock and remote_sock have already finished the connection.
'''
self._client_sock = client_sock
self._remote_sock = remote_sock
self._selector = selector
self._send_buffer = b''
self._recv_buffer = b''
self._selector.register(self._client_sock, selectors.EVENT_READ|selectors.EVENT_WRITE, self._client_handler)
self._selector.register(self._remote_sock, selectors.EVENT_READ|selectors.EVENT_WRITE, self._remote_handler)
def _client_handler(self, client_sock, mask):
if mask & selectors.EVENT_READ:
data = recv_from(client_sock)
if data:
self._send_buffer = data
else:
self._close()
elif mask & selectors.EVENT_WRITE:
if self._recv_buffer:
try:
client_sock.send(self._recv_buffer)
self._recv_buffer = b''
except OSError:
self._close()
def _remote_handler(self, remote_sock, mask):
if mask & selectors.EVENT_READ:
data = recv_from(remote_sock)
if data:
self._recv_buffer = data
else:
self._close()
elif mask & selectors.EVENT_WRITE:
if self._send_buffer:
try:
remote_sock.send(self._send_buffer)
self._send_buffer = b''
except OSError:
self._close()
def _close(self):
print('Closing ...')
self._selector.unregister(self._client_sock)
self._client_sock.close()
self._selector.unregister(self._remote_sock)
self._remote_sock.close()
self._send_buffer = b''
self._recv_buffer = b''
class PortMapper:
'''
Map the remote port to local.
'''
def __init__(self, proxy_ip, proxy_port, remote_ip, remote_port):
self.proxy_ip = proxy_ip
self.proxy_port = proxy_port
self.remote_ip = remote_ip
self.remote_port = remote_port
self._selector = selectors.DefaultSelector()
self._proxy_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
while True:
try:
self._proxy_sock.bind((proxy_ip, proxy_port))
break
except OSError:
proxy_port += 1
self.proxy_port = proxy_port
self._proxy_sock.listen(10)
self._proxy_sock.setblocking(False)
self._selector.register(self._proxy_sock, selectors.EVENT_READ, self._accept_handler)
print('Listening at {}:{}'.format(proxy_ip, proxy_port))
def _accept_handler(self, proxy_sock, mask):
client_sock, addr = proxy_sock.accept()
client_sock.setblocking(False)
print('Accept from {}'.format(addr))
remote_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
remote_sock.setblocking(False)
try:
remote_sock.connect((self.remote_ip, self.remote_port))
except BlockingIOError:
pass
RelayHandlder(client_sock, remote_sock, self._selector)
def loop(self):
while True:
events = self._selector.select()
for key, mask in events:
callback = key.data
callback(key.fileobj, mask)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Port Map Tool.')
parser.add_argument('-H', '--remote-host', required=True, type=str, help='Remote host.')
parser.add_argument('-P', '--remote-port', required=True, type=int, help='Remote port.')
parser.add_argument('-p', '--local-port', default=1000, type=int, help='Local port.')
args = parser.parse_args()
PortMapper('0.0.0.0', args.local_port, args.remote_host, args.remote_port).loop()
In the code, self._send_buffer in RelayHandler is used to cache the data received from the client. If self._remote_sock is writable and self._send_buffer is not empty, the proxy will send self._send_buffer to the remote service. The logic is similar for self._client_sock. The main loop is defined on loop function in PortMapper.
I have two questions:
Is a non-block socket always writable after it finishes its connection in python?
In the above code, sockets are always writable while buffers are often empty. So for every single loop in the main loop, self._selector.select() will always return without blocking, and the callback is executed to do nothing but only to see if the buffer is empty, which may hurt the performance. Is there a better method or structure to do this?
A socket is normally writable until the system buffer is full. That is the reason why many simple select multiplexed system only considere the read part and assume that they will be able to write or accept the possibility of being blocked if they are not.
If you want to be super safe and ensure that you will be able to write, you should ignore EVENT_WRITE unless you are ready to write something. But to prevent your code to exhaust the local memory buffer, the relay should stop reading (also ignore EVENT_READ) if the other channel cannot write.
So I'm making a port scanner in python...
import socket
ip = "External IP"
s = socket.socket(2, 1) #socket.AF_INET, socket.SOCK_STREAM
def porttry(ip, port):
try:
s.connect((ip, port))
return True
except:
return None
for port in range(0, 10000):
value = porttry(ip, port)
if value == None:
print("Port not opened on %d" % port)
else:
print("Port opened on %d" % port)
break
raw_input()
But this is too slow, I want to somehow be able to some how close or break code after a period of time of not returning anything.
In addition to setting socket timeout, you can also apply multi-threading technique to turbo boost the process. It will be, at best, N times faster when you have N ports to scan.
# This script runs on Python 3
import socket, threading
def TCP_connect(ip, port_number, delay, output):
TCPsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
TCPsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
TCPsock.settimeout(delay)
try:
TCPsock.connect((ip, port_number))
output[port_number] = 'Listening'
except:
output[port_number] = ''
def scan_ports(host_ip, delay):
threads = [] # To run TCP_connect concurrently
output = {} # For printing purposes
# Spawning threads to scan ports
for i in range(10000):
t = threading.Thread(target=TCP_connect, args=(host_ip, i, delay, output))
threads.append(t)
# Starting threads
for i in range(10000):
threads[i].start()
# Locking the main thread until all threads complete
for i in range(10000):
threads[i].join()
# Printing listening ports from small to large
for i in range(10000):
if output[i] == 'Listening':
print(str(i) + ': ' + output[i])
def main():
host_ip = input("Enter host IP: ")
delay = int(input("How many seconds the socket is going to wait until timeout: "))
scan_ports(host_ip, delay)
if __name__ == "__main__":
main()
here is a quick and simple port scanner, it scans 100000 ports in 180 sec:
import threading
import socket
target = 'pythonprogramming.net'
#ip = socket.gethostbyname(target)
def portscan(port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(0.5)#
try:
con = s.connect((target,port))
print('Port :',port,"is open.")
con.close()
except:
pass
r = 1
for x in range(1,100):
t = threading.Thread(target=portscan,kwargs={'port':r})
r += 1
t.start()
Consider setting a timeout instead of a for loop by using socket.setdefaulttimeout(timeout).
This should be a bit faster.
#-*-coding:utf8;-*-
#qpy:3
#qpy:console
import socket
import os
# This is used to set a default timeout on socket
# objects.
DEFAULT_TIMEOUT = 0.5
# This is used for checking if a call to socket.connect_ex
# was successful.
SUCCESS = 0
def check_port(*host_port, timeout=DEFAULT_TIMEOUT):
''' Try to connect to a specified host on a specified port.
If the connection takes longer then the TIMEOUT we set we assume
the host is down. If the connection is a success we can safely assume
the host is up and listing on port x. If the connection fails for any
other reason we assume the host is down and the port is closed.'''
# Create and configure the socket.
sock = socket.socket()
sock.settimeout(timeout)
# the SO_REUSEADDR flag tells the kernel to reuse a local
# socket in TIME_WAIT state, without waiting for its natural
# timeout to expire.
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Like connect(address), but return an error indicator instead
# of raising an exception for errors returned by the C-level connect()
# call (other problems, such as “host not found,” can still raise exceptions).
# The error indicator is 0 if the operation succeeded, otherwise the value of
# the errnovariable. This is useful to support, for example, asynchronous connects.
connected = sock.connect_ex(host_port) is SUCCESS
# Mark the socket closed.
# The underlying system resource (e.g. a file descriptor)
# is also closed when all file objects from makefile() are closed.
# Once that happens, all future operations on the socket object will fail.
# The remote end will receive no more data (after queued data is flushed).
sock.close()
# return True if port is open or False if port is closed.
return connected
con = check_port('www.google.com', 83)
print(con)
One can use threading.Thread and threading.Condition to synchronize port check and spawning new threads.
Script example usage:
python port_scan.py google.com 70 90
Checking 70 - 80
Checking 80 - 84
Checking 84 - 90
Found active port 80
Checking 90 - 91
Checking 91 - 94
All threads started ...
port_scan.py:
# import pdb
import socket, threading
from traceback import print_exc
class AllThreadsStarted(Exception): pass
class IPv4PortScanner(object):
def __init__(self, domain, timeout=2.0, port_range=(1024, 65535), threadcount=10):
self.domain = domain
self.timeout = timeout
self.port_range = port_range
self.threadcount = threadcount
self._lock = threading.Lock()
self._condition = threading.Condition(self._lock)
self._ports_active = []
self._ports_being_checked = []
self._next_port = self.port_range[0]
def check_port_(self, port):
"If connects then port is active"
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.settimeout(self.timeout)
try:
sock.connect((self.domain, port))
with self._lock:
self._ports_active.append(port)
print ("Found active port {}".format(port))
sock.close()
except socket.timeout, ex:
return
except:
print_exc()
# pdb.set_trace()
def check_port(self, port):
"updates self._ports_being_checked list on exit of this method"
try:
self.check_port_(port)
finally:
self._condition.acquire()
self._ports_being_checked.remove(port)
self._condition.notifyAll()
self._condition.release()
def start_another_thread(self):
if self._next_port > self.port_range[1]:
raise AllThreadsStarted()
port = self._next_port
self._next_port += 1
t = threading.Thread(target=self.check_port, args=(port,))
# update books
with self._lock:
self._ports_being_checked.append(port)
t.start()
def run(self):
try:
while True:
self._condition.acquire()
while len(self._ports_being_checked) >= self.threadcount:
# we wait for some threads to complete the task
self._condition.wait()
slots_available = self.threadcount - len(self._ports_being_checked)
self._condition.release()
print ("Checking {} - {}".format(self._next_port, self._next_port+slots_available))
for i in xrange(slots_available):
self.start_another_thread()
except AllThreadsStarted, ex:
print ("All threads started ...")
except:
print_exc()
if __name__ == "__main__":
import sys
domain = sys.argv[1]
port_s = int(sys.argv[2])
port_e = int(sys.argv[3])
scanner = IPv4PortScanner(domain=domain, port_range=(port_s, port_e))
scanner.run()
I think that this one snippet could help you : http://www.coderholic.com/python-port-scanner/
socket.setdefaulttimeout(0.5)
This will make the program faster!
socket.setdefualttimeout (time)
is used to keep trying to connect with port for perticular time...when you send request and there is timeout set for 2 seconds so it will try to connect with port for 2 seconds....if there will be no response from that port in 2 seconds....it will be count as a dead port
The following port scanner has a few constants defined at the top that you can modify as needed:
PURPOSE -- help message for the command line
PORTS -- range of ports you would like scanned
POOL_SIZE -- number of processes to scan with
TIMEOUT -- how long to wait for server connection
Feel free to adapt this according to your requirements. Maybe add some command line arguments?
#! /usr/bin/env python3
import argparse
import collections
import itertools
import multiprocessing
import operator
import socket
PURPOSE = 'Scan for open ports on a computer.'
PORTS = range(1 << 16)
POOL_SIZE = 1 << 8
TIMEOUT = 0.01
def main():
"""Get computer to scan, connect with process pool, and show open ports."""
parser = argparse.ArgumentParser(description=PURPOSE)
parser.add_argument('host', type=str, help='computer you want to scan')
host = parser.parse_args().host
with multiprocessing.Pool(POOL_SIZE, socket.setdefaulttimeout, [TIMEOUT]) \
as pool:
results = pool.imap_unordered(test, ((host, port) for port in PORTS))
servers = filter(operator.itemgetter(0), results)
numbers = map(operator.itemgetter(1), servers)
ordered = sorted(numbers)
print(f'Ports open on {host}:', *format_ports(ordered), sep='\n ')
field_names = 'family', 'socket_type', 'protocol', 'canon_name', 'address'
AddressInfo = collections.namedtuple('AddressInfo', field_names)
del field_names
def test(address):
"""Try connecting to the server and return whether or not it succeeded."""
host, port = address
for info in itertools.starmap(AddressInfo, socket.getaddrinfo(host, port)):
try:
probe = socket.socket(info.family, info.socket_type, info.protocol)
except OSError:
pass
else:
try:
probe.connect(info.address)
except OSError:
pass
else:
probe.shutdown(socket.SHUT_RDWR)
return True, port
finally:
probe.close()
return False, port
def format_ports(ports):
"""Convert port numbers into strings and show all associated services."""
if ports:
for port in ports:
try:
service = socket.getservbyport(port)
except OSError:
service = '?'
yield f'{port:<5} = {service}'
else:
yield 'None'
if __name__ == '__main__':
main()
I've just finished tinkering with Concurrent Futures on a port scanner and by God it's fast:
import concurrent.futures
import socket
def scan_port(domainip: str, port: int) -> tuple:
try:
# Use a faster socket implementation
s = socket.create_connection((domainip, port), timeout=0.5)
# Check if the connection was successful
if s:
return (port, "open")
else:
return (port, "closed")
except Exception as e:
print(f"Error scanning port {port}: {e}")
return (port, "error")
openports = {}
# Scan the ports in parallel using the faster scanning code
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = [executor.submit(scan_port, domainip, port) for port in range(1, 1024)]
for future in concurrent.futures.as_completed(futures):
status = future.result()
if status[1] == "open":
openports[status[0]] = status[1]
So, I've been working on an application for a client that communicates with wireless devices via a Serial (RS-232) "Master". I've currently written the core of the app using threading (below). I've been noticing on #python that the consensus seems to be to NOT use threads and to use Twisted's asynchronous communication abilities.
I haven't been able to find any good examples of using twisted for serial port async I/O communication. However, I have found Dave Peticolas' 'Twisted Introduction' (thanks nosklo) that I'm currently working through, but, it uses sockets instead of serial communication (but the async concept is definitely very well explained).
How would I go about porting this app over to Twisted from using Threading, Queues? Are there any advantages/disadvantages (I have noticed that, on occasion, if a thread hangs it will BSOD the system)?
The Code (msg_poller.py)
from livedatafeed import LiveDataFeed
from msg_build import build_message_to_send
from utils import get_item_from_queue
from protocol_wrapper import ProtocolWrapper, ProtocolStatus
from crc16 import *
import time
import Queue
import threading
import serial
import gc
gc.enable()
PROTOCOL_HEADER = '\x01'
PROTOCOL_FOOTER = '\x0D\x0A'
PROTOCOL_DLE = '\x90'
INITIAL_MODBUS = 0xFFFF
class Poller:
"""
Connects to the serial port and polls nodes for data.
Reads response from node(s) and loads that data into queue.
Parses qdata and writes that data to database.
"""
def __init__(self,
port,
baudrate,
parity,
rtscts,
xonxoff,
echo=False):
try:
self.serial = serial.serial_for_url(port,
baudrate,
parity=parity,
rtscts=rtscts,
xonxoff=xonxoff,
timeout=.01)
except AttributeError:
self.serial = serial.Serial(port,
baudrate,
parity=parity,
rtscts=rtscts,
xonxoff=xonxoff,
timeout=.01)
self.com_data_q = None
self.com_error_q = None
self.livefeed = LiveDataFeed()
self.timer = time.time()
self.dtr_state = True
self.rts_state = True
self.break_state = False
def start(self):
self.data_q = Queue.Queue()
self.error_q = Queue.Queue()
com_error = get_item_from_queue(self.error_q)
if com_error is not None:
print 'Error %s' % (com_error)
self.timer = time.time()
self.alive = True
# start monitor thread
#
self.mon_thread = threading.Thread(target=self.reader)
self.mon_thread.setDaemon(1)
self.mon_thread.start()
# start sending thread
#
self.trans_thread = threading.Thread(target=self.writer)
self.trans_thread.setDaemon(1)
self.trans_thread.start()
def stop(self):
try:
self.alive = False
self.serial.close()
except (KeyboardInterrupt, SystemExit):
self.alive = False
def reader(self):
"""
Reads data from the serial port using self.mon_thread.
Displays that data on the screen.
"""
from rmsg_format import message_crc, message_format
while self.alive:
try:
while self.serial.inWaiting() != 0:
# Read node data from the serial port. Data should be 96B.
data = self.serial.read(96)
data += self.serial.read(self.serial.inWaiting())
if len(data) > 0:
# Put data in to the data_q object
self.data_q.put(data)
if len(data) == 96:
msg = self.data_q.get()
pw = ProtocolWrapper(
header=PROTOCOL_HEADER,
footer=PROTOCOL_FOOTER,
dle=PROTOCOL_DLE)
status = map(pw.input, msg)
if status[-1] == ProtocolStatus.IN_MSG:
# Feed all the bytes of 'msg' sequentially into pw.input
# Parse the received CRC into a 16-bit integer
rec_crc = message_crc.parse(msg[-4:]).crc
# Compute the CRC on the message
calc_crc = calcString(msg[:-4], INITIAL_MODBUS)
from datetime import datetime
ts = datetime.now().strftime('%Y/%m/%d %H:%M:%S')
if rec_crc != calc_crc:
print ts
print 'ERROR: CRC Mismatch'
print msg.encode('hex')
else:
#msg = message_format.parse(msg[1:])
#print msg.encode('hex') + "\r\n"
msg = message_format.parse(msg[1:])
print msg
#return msg
gc.collect()
time.sleep(.2)
except (KeyboardInterrupt, SystemExit, Exception, TypeError):
self.alive = False
self.serial.close()
raise
def writer(self):
"""
Builds the packet to poll each node for data.
Writes that data to the serial port using self.trans_thread
"""
import time
try:
while self.alive:
try:
dest_module_code = ['DRILLRIG',
'POWERPLANT',
'GENSET',
'MUDPUMP']
dest_ser_no = lambda x: x + 1
for code in dest_module_code:
if code != 'POWERPLANT':
msg = build_message_to_send(
data_len=0x10,
dest_module_code='%s' % (code),
dest_ser_no=dest_ser_no(0),
dest_customer_code='*****',
ret_ser_no=0x01,
ret_module_code='DOGHOUSE',
ret_customer_code='*****',
command='POLL_NODE',
data=[])
self.serial.write(msg)
time.sleep(.2)
gc.collect()
elif code == 'POWERPLANT':
msg = build_message_to_send(
data_len=0x10,
dest_module_code='POWERPLANT',
dest_ser_no=dest_ser_no(0),
dest_customer_code='*****',
ret_ser_no=0x01,
ret_module_code='DOGHOUSE',
ret_customer_code='*****',
command='POLL_NODE',
data=[])
self.serial.write(msg)
time.sleep(.2)
gc.collect()
msg = build_message_to_send(
data_len=0x10,
dest_module_code='POWERPLANT',
dest_ser_no=dest_ser_no(1),
dest_customer_code='*****',
ret_ser_no=0x01,
ret_module_code='DOGHOUSE',
ret_customer_code='*****',
command='POLL_NODE',
data=[])
self.serial.write(msg)
time.sleep(.2)
gc.collect()
except (KeyboardInterrupt, SystemExit):
self.alive = False
self.serial.close()
raise
except (KeyboardInterrupt, SystemExit):
self.alive = False
self.serial.close()
raise
def main():
poller = Poller(
port='COM4',
baudrate=115200,
parity=serial.PARITY_NONE,
rtscts=0,
xonxoff=0,
)
poller.start()
poller.reader()
poller.writer()
poller.stop()
if __name__ == '__main__':
main()
It is very difficult (if not impossible) to write a direct one-to-one mapping program between threading/queue approach and one that uses twisted.
I would suggest that, get a hang of twisted and its reactor way it's use of Protocol and the protocol specific methods. Think about it as as all the asynchronous things that you had been explicitly coding using threads and queues are given to you for free when you are using deferred using twisted.
twisted does seem to support SerialPort over it's reactor using SerialPort transport class and the basic structure seems to be somewhat like this.
from twisted.internet import reactor
from twisted.internet.serialport import SerialPort
SerialPort(YourProtocolClass(), Port, reactor, baudrate=baudrate))
reactor.run()
In YourProtocolClass() would you handle the various events that are specific to your Serial Port Communication requirements. The doc/core/examples directory contains examples such as gpsfix.py and mouse.py.