I'm building a Websocket server application in python 3. I'm using this implementation: https://websockets.readthedocs.io/
Basically I want to manage multiple client.
Also I want to send data from 2 different thread (one for GPS + one for IMU)
GPS thread is refreshed 1Hz, while IMU thread is refresh at 25Hz
My problem is in MSGWorker.sendData method: as soon as I uncomment the line #asyncio.coroutine and yield from websocket.send('{"GPS": "%s"}' % data) the whole method does nothing (no print("Send data: foo") in terminal)
However with these two line commented my code works as I expect except that I send nothing through the websocket.
But, of course, my goal is to send data through the websocket, I just don't understand why it doesn't work ? Any idea ?
server.py
#!/usr/bin/env python3
import signal, sys
sys.path.append('.')
import time
import websockets
import asyncio
import threading
connected = set()
stopFlag = False
class GPSWorker (threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.data = 0
self.lastData = 0
self.inc = 0
# Simulate GPS data
def run(self):
while not stopFlag:
self.data = self.inc
self.inc += 1
time.sleep(1)
def get(self):
if self.lastData is not self.data:
self.lastData = self.data
return self.data
class IMUWorker (threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.data = 0
self.lastData = 0
self.inc = 0
# Simulate IMU data
def run(self):
while not stopFlag:
self.data = self.inc
self.inc += 1
time.sleep(0.04)
def get(self):
if self.lastData is not self.data:
self.lastData = self.data
return self.data
class MSGWorker (threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while not stopFlag:
data = gpsWorker.get()
if data:
self.sendData('{"GPS": "%s"}' % data)
data = imuWorker.get()
if data:
self.sendData('{"IMU": "%s"}' % data)
time.sleep(0.04)
##asyncio.coroutine
def sendData(self, data):
for websocket in connected.copy():
print("Sending data: %s" % data)
#yield from websocket.send('{"GPS": "%s"}' % data)
#asyncio.coroutine
def handler(websocket, path):
global connected
connected.add(websocket)
#TODO: handle client disconnection
# i.e connected.remove(websocket)
if __name__ == "__main__":
print('aeroPi server')
gpsWorker = GPSWorker()
imuWorker = IMUWorker()
msgWorker = MSGWorker()
try:
gpsWorker.start()
imuWorker.start()
msgWorker.start()
start_server = websockets.serve(handler, 'localhost', 7700)
loop = asyncio.get_event_loop()
loop.run_until_complete(start_server)
loop.run_forever()
except KeyboardInterrupt:
stopFlag = True
loop.close()
print("Exiting program...")
client.html
<!doctype html>
<html>
<head>
<meta charset="UTF-8" />
</head>
<body>
</body>
</html>
<script type="text/javascript">
var ws = new WebSocket("ws://localhost:7700", 'json');
ws.onmessage = function (e) {
var data = JSON.parse(e.data);
console.log(data);
};
</script>
Thanks you for your help
Finally I got it !
It required Python 3.5.1 (while my distro provide only 3.4.3) and some help from Aymeric, the author of the websockets library (thanks to him).
#!/usr/bin/env python3
import signal, sys
sys.path.append('.')
import time
import websockets
import asyncio
import threading
stopFlag = False
class GPSWorker (threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.data = 0
self.lastData = 0
self.inc = 0
# Simulate GPS data
def run(self):
while not stopFlag:
self.data = self.inc
self.inc += 1
time.sleep(1)
def get(self):
if self.lastData is not self.data:
self.lastData = self.data
return self.data
class IMUWorker (threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.data = 0
self.lastData = 0
self.inc = 0
# Simulate IMU data
def run(self):
while not stopFlag:
self.data = self.inc
self.inc += 1
time.sleep(0.04)
def get(self):
if self.lastData is not self.data:
self.lastData = self.data
return self.data
class MSGWorker (threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.connected = set()
def run(self):
while not stopFlag:
data = gpsWorker.get()
if data:
self.sendData('{"GPS": "%s"}' % data)
data = imuWorker.get()
if data:
self.sendData('{"IMU": "%s"}' % data)
time.sleep(0.04)
async def handler(self, websocket, path):
self.connected.add(websocket)
try:
await websocket.recv()
except websockets.exceptions.ConnectionClosed:
pass
finally:
self.connected.remove(websocket)
def sendData(self, data):
for websocket in self.connected.copy():
print("Sending data: %s" % data)
coro = websocket.send(data)
future = asyncio.run_coroutine_threadsafe(coro, loop)
if __name__ == "__main__":
print('aeroPi server')
gpsWorker = GPSWorker()
imuWorker = IMUWorker()
msgWorker = MSGWorker()
try:
gpsWorker.start()
imuWorker.start()
msgWorker.start()
ws_server = websockets.serve(msgWorker.handler, '0.0.0.0', 7700)
loop = asyncio.get_event_loop()
loop.run_until_complete(ws_server)
loop.run_forever()
except KeyboardInterrupt:
stopFlag = True
#TODO: close ws server and loop correctely
print("Exiting program...")
Regards,
Clément
I know this is an old thread but I am trying to mimic the functionality and I figured someone may be willing to lend a hand.
I have two CAN busses so I want one thread for each. I will later have more threads for a web socket connection and possibly other computational tasks.
The code gets stuck in the infinite while loop when I call run on the can0worker. I'm not familiar enough with python multithreading to know how to fix it. I thought that the while loop, because it is on a separate thread, is non-blocking but mine is blocking from the testing that I have done.
class CANWorker (threading.Thread):
def __init__(self, channel_num):
threading.Thread.__init__(self)
self.msg = None
self.lastmsg = None
self.channel_num = channel_num
print("init", self.channel_num)
os.system(f'sudo ifconfig can{channel_num} down')
os.system(f'sudo ip link set can{channel_num} up type can bitrate 2000000 dbitrate 8000000 restart-ms 1000 berr-reporting on fd on')
os.system(f'sudo ifconfig can{channel_num} txqueuelen 65536')
self.can = can.interface.Bus(channel = f'can{channel_num}', bustype = 'socketcan', fd=True)
def run(self):
print("run", self.channel_num)
while not stopFlag:
self.msg = self.can.recv(0.01)
def get(self):
print("get", self.channel_num)
if self.lastmsg is not self.msg:
self.lastmsg = self.msg
return self.msg
Call this within if __name__ == "__main__":
can0worker = CANWorker(0)
can1worker = CANWorker(1)
can0worker.run()
can1worker.run()
Results after running
init 0
init 1
run 0
Related
I'm on Python 3.4 making a practice app to learn Python's asyncio module.
I've got several modules in my application:
app.py
import asyncio
import logging
class Client(asyncio.Protocol):
def __init__(self, loop):
self.loop = loop
def connection_made(self, transport):
print('Connection with server established')
self.transport = transport
def data_received(self, data):
data = data.decode()
print('Received: ', data)
if not int(data):
print('Stopping loop')
self.loop.stop()
else:
message = 'remove one'.encode()
self.transport.write(message)
print('Sent:', message)
def connection_lost(self, exc):
print('Connection with server lost.')
self.loop.stop()
loop = asyncio.get_event_loop()
fn = loop.create_connection(
lambda: Client(loop), '127.0.0.1', 9999
)
logging.basicConfig(level=logging.DEBUG)
client = loop.run_until_complete(fn)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
print('Loop ended')
loop.close()
counter.py
import asyncio
class Counter:
def __init__(self, maxcount):
self.maxcount = maxcount
#asyncio.coroutine
def count(self):
yield from asyncio.sleep(self.maxcount)
print('Done counting')
serie.py
import asyncio
class Serie:
def __init__(self, loop, items=None):
self.loop = loop
self.items = items or []
#asyncio.coroutine
def remove_one(self, counter):
if len(self.items) is not 0:
yield from counter.count()
item = self.items.pop(0)
print('Removed', item)
else:
print('Serie is empty')
#asyncio.coroutine
def start_removing(self, counter):
while self.items:
yield from self.remove_one(counter)
print('Serie.start_removing() has finished')
mission_control.py
import asyncio
import logging
from counter import Counter
from serie import Serie
class MissionControl(asyncio.Protocol):
def __init__(self, loop, counter, serie):
self.loop = loop
self.counter = counter
self.serie = serie
def connection_made(self, transport):
print('Connection established with', transport.get_extra_info('peername'))
self.transport = transport
self.transport.write(str(len(self.serie.items)).encode())
def data_received(self, data):
data = data.decode()
print('Received:', data)
if data == 'remove one':
yield from self.serie.remove_one()
print('Removed one: {}'.format(self.serie.items))
self.transport.write(str(len(self.serie.items)).encode())
else:
print('Done')
def connection_lost(self, exc):
print('Connection with {} ended'.format(self.transport.get_extra_info('peername')))
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
counter = Counter(2)
planets = Serie(loop, ['Mercúrio', 'Vênus', 'Terra', 'Marte',
'Júpiter', 'Saturno', 'Urano', 'Netuno'])
fn = loop.create_server(
lambda: MissionControl(loop, counter, planets), '127.0.0.1', 9999
)
server = loop.run_until_complete(fn)
print('Server started')
try:
loop.run_forever()
except KeyboardInterrupt:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.stop()
loop.close()
You can also find the source at this Github gist.
In mission_control.py, the method data_received() appears not to be called when the client (app.py) sends data via its self.transport property.
Where is the implementation error and how can I fix it?
The problem is that data_received is not (and cannot be) a coroutine, but you're using yield from inside it. Internally asyncio is just calling self.data_received(data) without any yield from call, which means the body of the method isn't being executed at all - instead a generator object is immediately returned. You need to refactor your implementation to not use yield from, which requires using callbacks instead:
class MissionControl(asyncio.Protocol):
def __init__(self, loop, counter, serie):
self.loop = loop
self.counter = counter
self.serie = serie
def connection_made(self, transport):
print('Connection established with', transport.get_extra_info('peername'))
self.transport = transport
self.transport.write(str(len(self.serie.items)).encode())
def data_received(self, data):
data = data.decode()
print('Received:', data)
if data == 'remove one':
fut = asyncio.async(self.serie.remove_one(self.counter))
fut.add_done_callback(self.on_removed_one)
else:
print('Done')
def on_removed_one(self, result):
print('Removed one: {}'.format(self.serie.items))
self.transport.write(str(len(self.serie.items)).encode())
def connection_lost(self, exc):
print('Connection with {} ended'.format(self.transport.get_extra_info('peername')))
Another option would be to use the asyncio Streams API instead of asyncio.Protocol, which will allow you to use coroutines:
import asyncio
import logging
from counter import Counter
from serie import Serie
#asyncio.coroutine
def mission_control(reader, writer):
counter = Counter(2)
serie = Serie(loop, ['Mercúrio', 'Vênus', 'Terra', 'Marte',
'Júpiter', 'Saturno', 'Urano', 'Netuno'])
writer.write(str(len(serie.items)).encode())
while True:
data = (yield from reader.read(100)).decode()
print('Received:', data)
if data == 'remove one':
result = yield from serie.remove_one(counter)
else:
print('Done')
return
print('Removed one: {}'.format(serie.items))
writer.write(str(len(serie.items)).encode())
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
coro = asyncio.start_server(mission_control, '127.0.0.1', 9999, loop=loop)
server = loop.run_until_complete(coro)
# The rest is the same
I am using tornado websocket for simple test code.
In the test code, i want to get tornado.websocket.WebSocketHandler.
For example, I used this way below.
class ConvPlayerInterface(object):
class WebsocketHandler(tornado.websocket.WebSocketHandler):
client = None
queue = ipcQueue.IpcQueue()
def open(self):
print 'new connection'
self.client = self #in my simple code, it handles only one client.
self.write_message("Connection Open")
def on_message(self, message):
self.queue.put(message)
def on_close(self):
print 'connection closed'
def __init__(self, url='/ws'):
self.application = tornado.web.Application([(url, self.WebsocketHandler),])
self.httpServer = tornado.httpserver.HTTPServer(self.application)
self.httpServer.listen(8888)
self.queue = self.WebsocketHandler.queue
self.ioLoop = threading.Thread(target = tornado.ioloop.IOLoop.instance().start)
def start(self):
self.ioLoop.start()
def get(self):
return self.queue.get()
def put(self, command):
self.WebsocketHandler.client.write_message(command)
But the point when it calls self.WebsocketHandler.client.write_message(command) in put() method, Python says client is Non type.
Any advice?
And how usually it is used to get client connection handler object in tornado?
In this part of your code
def put(self, command):
self.WebsocketHandler.client.write_message(command)
you are accessing to WebsocketHandler class, not a class member.
And the "client" attribute of WebsocketHandler is None, as expected.
WebsocketHandler instance will be created for each request tornado will accept, so there can be several websocket handlers simultaneously.
If you really want to have handle only one connection - you can do something like this:
class ConvPlayerInterface(object):
the_only_handler = None
class WebsocketHandler(tornado.websocket.WebSocketHandler):
client = None
queue = ipcQueue.IpcQueue()
def open(self):
print 'new connection'
ConvPlayerInterface.the_only_handler = self
self.write_message("Connection Open")
def on_message(self, message):
self.queue.put(message)
def on_close(self):
ConvPlayerInterface.the_only_handler = None
print 'connection closed'
def __init__(self, url='/ws'):
self.application = tornado.web.Application([(url, self.WebsocketHandler),])
self.httpServer = tornado.httpserver.HTTPServer(self.application)
self.httpServer.listen(8888)
self.queue = self.WebsocketHandler.queue
self.ioLoop = threading.Thread(target = tornado.ioloop.IOLoop.instance().start)
def start(self):
self.ioLoop.start()
def get(self):
return self.queue.get()
def put(self, command):
if self.the_only_handler is not None
self.the_only_handler.write_message(command)
This test doesn't work.
class PrintHandler(MessageHandler):
def handle_message(self, message):
print(message)
class FileHandler(MessageHandler):
def handle_message(self, message):
with open('nana', 'w') as f:
f.write(message)
class SubscribeProcess(Process):
def __init__(self, handler):
super(SubscribeProcess, self).__init__(group=None, target=None, name=None, args=(), kwargs={})
self.handler = handler
def run(self):
self.address = TcpAddress(host='127.0.0.1', port=5555)
subscriber = ZmqSubscriber(ZmqBlockingConnection(address=self.address, bind=False))
subscriber.set_message_handler(self.handler)
print('............')
class TestZmqSubscriber(TestCase):
def test_set_message_handler(self):
address = TcpAddress(host='127.0.0.1', port=5555)
pub_connection = ZmqBlockingConnection(address, bind=True)
publisher = ZmqPublisher(pub_connection)
p = SubscribeProcess(handler=PrintHandler())
p.start()
while True:
publisher.publish('Message number {}'.format(2))
I now that's this is not the unit test actually. But I want to see the received messages in console first. Then I will write proper test.
While this two scripts work perfectly.
connection = ZmqBlockingConnection(TcpAddress(host='127.0.0.1', port=5555), bind=False)
sub = ZmqSubscriber(connection)
sub.set_message_handler(PrintHandler())
address = TcpAddress(host='127.0.0.1', port=5555)
pub_connection = ZmqBlockingConnection(address, bind=True)
publisher = ZmqPublisher(pub_connection)
while True:
publisher.publish('Message number {}'.format(2))
Inside of subscriber.set_message_handler(handler) is actually this
def start_receiving_messages(self, message_handler):
while True:
message_handler.handle_message(self.socket.recv())
And in debugger I see that the code hangs infinitely in socket.recv()
Maybe I'm using multiprocessing wrong?
EDIT1
class ZmqBlockingConnection(Connection):
def start_receiving_messages(self, message_handler):
while True:
message_handler.handle_message(self.socket.recv())
def send_message(self, message):
self.socket.send(message)
def __init__(self, address, bind, hwm=1000):
self.hwm = hwm
self.bind = bind
self.address = address
self.socket = None
def set_hwm(self, hwm):
self.socket.set_hwm(hwm)
def configure(self, socket_type):
self.socket = zmq.Context().socket(socket_type)
if self.bind:
self.socket.bind(str(self.address))
else:
self.socket.connect(str(self.address))
self.set_hwm(self.hwm)
OK, the problem was in
def configure(self, socket_type):
self.socket = zmq.Context().instance().socket(socket_type)
if self.bind:
self.socket.bind(str(self.address))
else:
self.socket.connect(str(self.address))
self.set_hwm(self.hwm)
so instead of using singleton I started to create context instances and now it's working.
Can you anyone please help me (noob) call the broadcast function from class BroadcastServerFactory in class process, as per attached code
I have tried so many methods of call a function from another class, but no solution
import time, sys
from apscheduler.scheduler import Scheduler
import threading
import socket
from twisted.internet import reactor
from twisted.python import log
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, \
listenWS
class process(threading.Thread):
def __init__(self, buffer3):
threading.Thread.__init__(self)
self.setDaemon(True)
self.buffer3 = buffer3
def run(self):
factory.broadcast("I don't know what I'm doing!")
class BroadcastServerProtocol(WebSocketServerProtocol):
def onOpen(self):
self.factory.register(self)
def onMessage(self, msg, binary):
if not binary:
self.factory.broadcast("'%s' from %s" % (msg, self.peerstr))
def connectionLost(self, reason):
WebSocketServerProtocol.connectionLost(self, reason)
self.factory.unregister(self)
class BroadcastServerFactory(WebSocketServerFactory):
"""
Simple broadcast server broadcasting any message it receives to all
currently connected clients.
"""
def __init__(self, url, debug = False, debugCodePaths = False):
WebSocketServerFactory.__init__(self, url, debug = debug, debugCodePaths = debugCodePaths)
self.clients = []
self.tickcount = 0
self.tick()
def tick(self):
self.tickcount += 1
self.broadcast("'tick %d' from server" % self.tickcount)
reactor.callLater(1, self.tick)
def register(self, client):
if not client in self.clients:
print "registered client " + client.peerstr
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print "unregistered client " + client.peerstr
self.clients.remove(client)
def broadcast(self, msg):
print "broadcasting message '%s' .." % msg
for c in self.clients:
c.sendMessage(msg)
print "message sent to " + c.peerstr
class BroadcastPreparedServerFactory(BroadcastServerFactory):
"""
Functionally same as above, but optimized broadcast using
prepareMessage and sendPreparedMessage.
"""
def broadcast(self, msg):
print "broadcasting prepared message '%s' .." % msg
preparedMsg = self.prepareMessage(msg)
for c in self.clients:
c.sendPreparedMessage(preparedMsg)
print "prepared message sent to " + c.peerstr
def testing():
buffer2 - "hello"
myDisplay = process(buffer2)
myDisplay.start()
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == 'debug':
log.startLogging(sys.stdout)
debug = True
else:
debug = False
level_scheduler = Scheduler()
level_scheduler.add_interval_job(testing, seconds=5)
level_scheduler.start()
#ServerFactory = BroadcastServerFactory
ServerFactory = BroadcastPreparedServerFactory
factory = ServerFactory("ws://localhost:9000",
debug = debug,
debugCodePaths = debug)
factory.protocol = BroadcastServerProtocol
factory.setProtocolOptions(allowHixie76 = True)
listenWS(factory)
webdir = File(".")
web = Site(webdir)
reactor.listenTCP(8080, web)
reactor.run()
Thanks
Pass the class instance of BroadcastServerFactory to be called to the class instance that calls it process on creation
class process(threading.Thread):
def __init__(self, buffer3m, broadcast_server_factory):
threading.Thread.__init__(self)
self.setDaemon(True)
self.buffer3 = buffer3
self.factory = broadcast_server_factory
def run(self):
self.factory.broadcast("I don't know what I'm doing!")
and then call it (it's assigned as self.factory in the run statement. I can't see where you create a process class in your __main__ but it will be created with something like
p = process(buffer, factory)
Aside: Using capital letters for class names is considered good form in python process -> Process
I modified the code (found here) a bit (added class AsyncEventLoop)
import socket,asyncore
import threading
class forwarder(asyncore.dispatcher):
def __init__(self, ip, port, remoteip,remoteport,backlog=5):
asyncore.dispatcher.__init__(self)
self.remoteip=remoteip
self.remoteport=remoteport
self.create_socket(socket.AF_INET,socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((ip,port))
self.listen(backlog)
def handle_accept(self):
conn, addr = self.accept()
# print '--- Connect --- '
sender(receiver(conn),self.remoteip,self.remoteport)
def closef(self):
self.close()
class receiver(asyncore.dispatcher):
def __init__(self,conn):
asyncore.dispatcher.__init__(self,conn)
self.from_remote_buffer=''
self.to_remote_buffer=''
self.sender=None
def handle_connect(self):
pass
def handle_read(self):
read = self.recv(4096)
# print '%04i -->'%len(read)
self.from_remote_buffer += read
def writable(self):
return (len(self.to_remote_buffer) > 0)
def handle_write(self):
sent = self.send(self.to_remote_buffer)
# print '%04i <--'%sent
self.to_remote_buffer = self.to_remote_buffer[sent:]
def handle_close(self):
self.close()
if self.sender:
self.sender.close()
class sender(asyncore.dispatcher):
def __init__(self, receiver, remoteaddr,remoteport):
asyncore.dispatcher.__init__(self)
self.receiver=receiver
receiver.sender=self
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((remoteaddr, remoteport))
def handle_connect(self):
pass
def handle_read(self):
read = self.recv(4096)
# print '<-- %04i'%len(read)
self.receiver.to_remote_buffer += read
def writable(self):
return (len(self.receiver.from_remote_buffer) > 0)
def handle_write(self):
sent = self.send(self.receiver.from_remote_buffer)
# print '--> %04i'%sent
self.receiver.from_remote_buffer = self.receiver.from_remote_buffer[sent:]
def handle_close(self):
self.close()
self.receiver.close()
class AsyncEventLoop(threading.Thread):
def run(self):
asyncore.loop()
starting in the code
ser = forwarder('127.0.0.1', 7774, 'google.com.ua', 80)
evLoop = AsyncEventLoop()
Can I change remoteip, remoteport on the fly? If the fly is not possible to change, as well stop this thread to start with the new settings (remoteip, remoteport)?
yes, you can
ser.remoteip, ser.remoteport = IP, port