Develop Reference

Python Pyserial Asyncio

I'm using in my python script the pyserial-asyncio lib. I encountered that from writing to reading it takes around 1s which is in my opinion far to long. Should be some ms only. Does anybody also have this seen or any idea?

import asyncio
import serial_asyncio
import time
class Output(asyncio.Protocol):
def connection_made(self, transport):
global start
self.transport = transport
print('port opened', transport)
cmd = b'config A'
print("write cmd: " + str(cmd))
start = time.time()
transport.write(cmd) # Write serial data via transport
def data_received(self, data):
print('data received', repr(data))
end = time.time()
print("write-receive: {0:0.3f}".format(end - start))
if b'\n' in data:
self.transport.close()
def connection_lost(self, exc):
print('port closed')
self.transport.loop.stop()
if __name__ == '__main__':
port = '/dev/ttyACM0'
loop = asyncio.get_event_loop()
coro = serial_asyncio.create_serial_connection(loop, Output, port, baudrate=921600)
loop.run_until_complete(coro)
loop.run_forever()
loop.close()

Thread does not end cleanly due to socket recv callback

I am trying to close the socket and terminate the thread when shutdown handler callback is executed (I send SIGINT signal by CTRL+C)
main
from example import Example
if __name__ == '__main__':
exmpl = Example()
success = exmpl.connect('', 2000)
if success:
rospy.on_shutdown(exmpl.shutdown_handler)
rospy.spin()
imported class
import socket
import threading
class Example(object):
def connect(self, host='', port=2000):
try:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.settimeout(10)
self.sock.bind((host, port))
self.sock.listen(1)
self.conn, self.addr = self.sock.accept()
self.sock.settimeout(None)
self.t = threading.Thread(target=self.recv_msg)
self.t.start()
return True
except Exception as e:
return False
except Exception as timeout:
return False
def recv_msg(self):
while True:
recv_msg = self.conn.recv(1024)
print(recv_msg)
def shutdown_handler(self):
try:
self.sock.close()
self.t.join()
except Exception as e:
print(e)
In this application recv_msg is always executing because data are coming all the time. So, when I send SIGINT, shutdown handler starts and only executes the statement sock.close() but not t.join() and thread never ends

How destroy instance of Protocol on asyncio server after connection lost?

I have asynsio server on Python 3.7.
For each connection, asyncio creates a new EchoServerProtocol() object. After receiving the first packet, the server closes the connection, but the EchoServerProtocol() object remains in memory. Can you please tell me how to correctly remove it? I understand that somewhere in asyncio there are links to it.
server.py
import asyncio
class EchoServerProtocol(asyncio.Protocol):
def __init__(self):
self.__loop = asyncio.get_event_loop()
def connection_made(self, transport):
self.__loop.call_later(5, self.check_connection)
print('Connection made')
self.transport = transport
def connection_lost(self, exc):
print('Connection lost')
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
print('Close the client socket')
self.transport.close()
def check_connection(self):
print('check connection here')
self.__loop.call_later(5, self.check_connection)
async def main():
loop = asyncio.get_running_loop()
server = await loop.create_server(
lambda: EchoServerProtocol(),
'127.0.0.1', 8888)
async with server:
await server.serve_forever()
asyncio.run(main())
client.py
import asyncio
class EchoClientProtocol(asyncio.Protocol):
def __init__(self, message, on_con_lost, loop):
self.message = message
self.loop = loop
self.on_con_lost = on_con_lost
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
self.on_con_lost.set_result(True)
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
on_con_lost = loop.create_future()
message = 'Hello World!'
transport, protocol = await loop.create_connection(
lambda: EchoClientProtocol(message, on_con_lost, loop),
'127.0.0.1', 8888)
# Wait until the protocol signals that the connection
# is lost and close the transport.
try:
await on_con_lost
finally:
transport.close()
Output:
Connection made
Data received: 'Hello World!'
Send: 'Hello World!'
Close the client socket
Connection lost
check connection here
check connection here
check connection here
check connection here
check connection here
check connection here
check connection here
check connection here

After receiving the first packet, the server closes the connection, but the EchoServerProtocol() object remains in memory.
Looking at your code, it is check_connection that is keeping the object in memory. Specifically, the end of check_connection says:
self.__loop.call_later(5, self.check_connection)
This means: "after 5 seconds, invoke check_connection on self". The fact that self is a protocol that is no longer in use doesn't matter - the event loop is told to invoke a function 5 seconds later, and it will do exactly that.
If you want to have a monitoring task, you should make it a coroutine and cancel it when the connection is lost. For example:
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
loop = asyncio.get_event_loop()
self._checker = loop.create_task(self.check_connection())
print('Connection made')
self.transport = transport
def connection_lost(self, exc):
print('Connection lost')
self._checker.cancel()
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
print('Close the client socket')
self.transport.close()
async def check_connection(self):
while True:
print('check connection here')
await asyncio.sleep(5)

user4815162342 is right. Thanks for your reply. My stupid mistake.
If you want to have a monitoring task, you should make it a coroutine and cancel it when >the connection is lost. For example:
As for the check_connection method, I just changed my code and added the del handler to make sure that the object is being deleted.
def check_connection(self):
print('check connection here')
if not self.transport.is_closing():
self.__loop.call_later(5, self.check_connection)
def __del__(self):
print("destroy protocol")

Sending data from outside socket object in python

I made a Client socket object, which I instantiate and it keeps alive a connection with the server, which is working fine, but I'm wondering if there is a way to call the socket.send event from outside the instance. I was about to make a stack for the messages and check the stack in the while loop and if it's not empty then send the oldest data to the server, which would be just fine for me, but my problem is that the stack only updates after the while loop(I tried breaking out, then it updated).
So my question would be, is there a way to update the global stack simultaneously with the while loop running? Or is there any other way to call the socket.send event outside the object?
import socket
import sys
import select
import threading
SERVER_IP = '192.168.1.4'
PORT = 8686
TIMEOUT = 5
BUF_SIZE = 1024
MESSAGES = ['testdata1', 'testdata2']
class Client(threading.Thread):
def __init__(self, host=SERVER_IP, port=PORT):
threading.Thread.__init__(self)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = socket.create_connection((host, port), 1)
self.sock.setblocking(0)
while 1:
try:
global MESSAGES
ready = select.select([self.sock], [], [], TIMEOUT*1000)
if ready[0]:
buf = self.sock.recv(BUF_SIZE)
print buf
#TODO:do stuff with buf
print 'messages left:'+str(len(MESSAGES))
if len(MESSAGES)>0:
self.sock.send(MESSAGES.pop())
except KeyboardInterrupt:
self.sock.close()
sys.exit(1)
except Exception, e:
print '\n[ERR] %s' % e
self.sock.close()
sys.exit(1)
def run(self):
pass
def sendData(self, data):
global MESSAGES
print 'appending data:%s' % data
MESSAGES.append(data)
def main():
client = Client()
client.start()
client.sendData("test1")
client.sendData("test2")
client.sendData("test3")
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
sys.exit(1)

Client.__init__() does not return because it enters an infinite while loop. Hence control is never returned to the main thread, and the Client thread is not actually started.
Instead you should move the while loop into the run() method. Then the __init__() method will return control to the main thread, which can then start the thread, and request that the client send messages via sendData().
class Client(threading.Thread):
def __init__(self, host=SERVER_IP, port=PORT):
threading.Thread.__init__(self)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = socket.create_connection((host, port), 1)
self.sock.setblocking(0)
def run(self):
while 1:
try:
global MESSAGES
ready = select.select([self.sock], [], [], TIMEOUT*1000)
if ready[0]:
buf = self.sock.recv(BUF_SIZE)
print buf
#TODO:do stuff with buf
print 'messages left:'+str(len(MESSAGES))
if len(MESSAGES)>0:
self.sock.send(MESSAGES.pop())
except KeyboardInterrupt:
self.sock.close()
sys.exit(1)
except Exception, e:
print '\n[ERR] %s' % e
self.sock.close()
sys.exit(1)
def sendData(self, data):
global MESSAGES
print 'appending data:%s' % data
MESSAGES.append(data)
Instead of using the global MESSAGES list you should probably create a Queue for communicating between the main thread and the worker thread(s), particularly if more than one worker thread is running. Something like this (untested!):
import Queue
class Client(threading.Thread):
def __init__(self, msg_queue, host=SERVER_IP, port=PORT):
threading.Thread.__init__(self)
self.msg_queue = msg_queue
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = socket.create_connection((host, port), 1)
self.sock.setblocking(0)
def run(self):
while 1:
try:
ready = select.select([self.sock], [], [], TIMEOUT*1000)
if ready[0]:
buf = self.sock.recv(BUF_SIZE)
print buf
#TODO:do stuff with buf
print 'messages left:'+ str(self.msg_queue.qsize())
try:
msg = self.msg_queue.get_nowait()
self.sock.send(msg)
except Queue.Empty:
pass
except KeyboardInterrupt:
self.sock.close()
sys.exit(1)
except Exception, e:
print '\n[ERR] %s' % e
self.sock.close()
sys.exit(1)
def main():
# create a queue and pass it to the client
msg_queue = Queue.Queue()
client = Client(msg_queue)
client.start()
msg_queue.put("test1")
msg_queue.put("test2")
msg_queue.put("test3")

The thing should work if you move your loop from
__init__() into run()
method instead.
Your thread is not a thread this way, process blocks at client = Client(...).
Why do you mix select and threads? Is this really necessary? If you want asynchronous sending and receiving without threads use asyncore module.
Or remove select from your code. The socket.recv() will block until it receives data in blocking mode, but as this is a thread, I don't see anything wrong about that. If in nonblocking mode, recv() will just return None if there is no data to receive if I remember correctly. So you don't really need select. Just check if recv() returned None. If it does, sleep some time before trying again.
The way you did it troubles your OS twice. Once for reading a socket, and second time to get the status of a socket where timeout is used to simulate sleep() more than anything else. Then the loop checks again making select() system call right after timeout confirmed that there is nothing to do for that socket.

How can I send message to clients every x seconds when running a tornado server?

I have Python code like this:
class WebSocketHandler(tornado.websocket.WebSocketHandler):
def check_origin(self, origin):
print "origin: " + origin
return True
# the client connected
def open(self):
print "New client connected"
self.write_message("You are connected")
# the client sent the message
def on_message(self, message):
print "message: " + message
self.write_message(message)
# client disconnected
def on_close(self):
print "Client disconnected"
socket = tornado.web.Application([(r"/wss", WebSocketHandler),])
if __name__ == "__main__":
socket.listen(8888)
tornado.ioloop.IOLoop.instance().start()
while True:
readmydata()
#send message to all connected clients
time.sleep(3)
How can I start the websocket server, but continue with the python code that sends the message to all connected clients? The script blocks at tornado.ioloop.IOLoop.instance().start(), so my while True loop never runs.

You can use tornado.ioloop.IOLoop.add_timeout to call a method every X number of seconds from within the Tornado event loop. To send a message to all conncected clients, you'll need to maintain a global list of each connected client, which you update on each connection/disconnection.
clients = []
class WebSocketHandler(tornado.websocket.WebSocketHandler):
def check_origin(self, origin):
print "origin: " + origin
return True
# the client connected
def open(self):
print "New client connected"
self.write_message("You are connected")
clients.append(self)
# the client sent the message
def on_message(self, message):
print "message: " + message
self.write_message(message)
# client disconnected
def on_close(self):
print "Client disconnected"
clients.remove(self)
def send_message_to_clients():
try:
read_my_data()
for client in clients:
# Do whatever
finally:
tornado.ioloop.IOLoop.instance().add_timeout(timedelta(seconds=3),
send_message_to_clients)
socket = tornado.web.Application([(r"/wss", WebSocketHandler),])
if __name__ == "__main__":
socket.listen(8888)
tornado.ioloop.IOLoop.instance().add_timeout(timedelta(seconds=3),
send_message_to_clients)
tornado.ioloop.IOLoop.instance().start()

A simpler approach:
import tornado.httpserver
import tornado.websocket
import tornado.ioloop
import tornado.web
import socket
import random
import threading
import time
'''
This is a simple Websocket Echo server that uses the Tornado websocket handler.
Please run `pip install tornado` with python of version 2.7.9 or greater to install tornado.
This program will echo back the reverse of whatever it recieves.
Messages are output to the terminal for debuggin purposes.
'''
clients = []
def updateClients():
while True:
for c in clients:
c.write_message('your update data')
time.sleep(1) #in seconds you can use float point
class WSHandler(tornado.websocket.WebSocketHandler):
def open(self):
print 'new connection'
clients.append(self)
def on_message(self, message):
self.write_message('echo ' + message)
def on_close(self):
clients.remove(self)
def check_origin(self, origin):
return True
application = tornado.web.Application([
(r'/ws', WSHandler),
])
if __name__ == "__main__":
t = threading.Thread(target = updateClients)
t.start()
#start the thread before tornado
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(8888)
myIP = socket.gethostbyname(socket.gethostname())
print '*** Websocket Server Started at %s***' % myIP
tornado.ioloop.IOLoop.instance().start()