I'm working on a project which uses python asyncio socket server. The problem is that the implementation of the server doesn't call .close() on the transport when the server stops. This seems to leave clients connected and causes crashes in other parts of the code.
Python documents say that transports need to be closed explicitly, but in this project I don't know where I can close them because there is no reference to the transports that are created for each client.
https://docs.python.org/3/library/asyncio-dev.html#close-transports-and-event-loops
Here is the code:
"""
Socket server forwarding request to internal server
"""
import logging
try:
# we prefer to use bundles asyncio version, otherwise fallback to trollius
import asyncio
except ImportError:
import trollius as asyncio
from opcua import ua
from opcua.server.uaprocessor import UaProcessor
logger = logging.getLogger(__name__)
class BinaryServer(object):
def __init__(self, internal_server, hostname, port):
self.logger = logging.getLogger(__name__)
self.hostname = hostname
self.port = port
self.iserver = internal_server
self.loop = internal_server.loop
self._server = None
self._policies = []
def set_policies(self, policies):
self._policies = policies
def start(self):
class OPCUAProtocol(asyncio.Protocol):
"""
instanciated for every connection
defined as internal class since it needs access
to the internal server object
FIXME: find another solution
"""
iserver = self.iserver
loop = self.loop
logger = self.logger
policies = self._policies
def connection_made(self, transport):
self.peername = transport.get_extra_info('peername')
self.logger.info('New connection from %s', self.peername)
self.transport = transport
self.processor = UaProcessor(self.iserver, self.transport)
self.processor.set_policies(self.policies)
self.data = b""
def connection_lost(self, ex):
self.logger.info('Lost connection from %s, %s', self.peername, ex)
self.transport.close()
self.processor.close()
def data_received(self, data):
logger.debug("received %s bytes from socket", len(data))
if self.data:
data = self.data + data
self.data = b""
self._process_data(data)
def _process_data(self, data):
buf = ua.utils.Buffer(data)
while True:
try:
backup_buf = buf.copy()
try:
hdr = ua.Header.from_string(buf)
except ua.utils.NotEnoughData:
logger.info("We did not receive enough data from client, waiting for more")
self.data = backup_buf.read(len(backup_buf))
return
if len(buf) < hdr.body_size:
logger.info("We did not receive enough data from client, waiting for more")
self.data = backup_buf.read(len(backup_buf))
return
ret = self.processor.process(hdr, buf)
if not ret:
logger.info("processor returned False, we close connection from %s", self.peername)
self.transport.close()
return
if len(buf) == 0:
return
except Exception:
logger.exception("Exception raised while parsing message from client, closing")
self.transport.close()
break
coro = self.loop.create_server(OPCUAProtocol, self.hostname, self.port)
self._server = self.loop.run_coro_and_wait(coro)
print('Listening on {}:{}'.format(self.hostname, self.port))
def stop(self):
self.logger.info("Closing asyncio socket server")
self.loop.call_soon(self._server.close)
self.loop.run_coro_and_wait(self._server.wait_closed())
As you can see when we call stop() on this server class the asyncio server calls it's close method. However if clients are connected the created transports never get closed.
The project repository is here https://github.com/FreeOpcUa/python-opcua/ , you can take a look at Issue 137.
What is the correct way to close the transport object?
I solve this by applying this approach:
#self.OPCUAServer - this is my opcua server
nodes = []
nodes.append(self.OPCUAServer.get_node("ns=0; s=Measurements")) #Adding two root nodes
nodes.append(self.OPCUAServer.get_node("ns=1; s=Calibrations")) #to the list
self.OPCUAServer.delete_nodes(nodes, True) # Recursively call delete_nodes with this list
self.OPCUAServer.stop()
I've started Python a few times ago and now, I'm currently creating a socket server. I already have the server functioning with multiple threads with multiple clients (Hurray !) But I'm looking for functionality I can't call (i don't even know if it exists) I would like to create a kind of channel where client can send different type of message.
An example I create a channel INFO and if the server received this type of socket it just does a print
I create another channel DEBUG where I can send custom command which the server will execute
etc
In a non-programming language it will do this:
def socketDebug(command):
run command
def socketInfo(input):
print input
if socket == socketDebug:
socketDebug(socket.rcv)
else:
if socket == socketInfo:
socketInfo(socket.rcv)
I hope I'm clear.
Here is a quite simple implementation of a Channel class. It creates a socket, to accept
connections from clients and to send messages. It is also a client itself,
receiving messages from other Channel instances (in separate processes for example).
The communication is done in two threads, which is pretty bad (I would use async io). when
a message is received, it calls the registered function in the receiving thread which
can cause some threading issues.
Each Channel instance creates its own sockets, but it would be much more scalable to
have channel "topics" multiplexed by a single instance.
Some existing libraries provide a "channel" functionality, like nanomsg.
The code here is for educational purposes, if it can help...
import socket
import threading
class ChannelThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.clients = []
self.chan_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.chan_sock.bind(('',0))
_, self.port = self.chan_sock.getsockname()
self.chan_sock.listen(5)
self.daemon=True
self.start()
def run(self):
while True:
new_client = self.chan_sock.accept()
if not new_client:
break
self.clients.append(new_client)
def sendall(self, msg):
for client in self.clients:
client[0].sendall(msg)
class Channel(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self.channel_thread = ChannelThread()
def public_address(self):
return "tcp://%s:%d" % (socket.gethostname(), self.channel_thread.port)
def register(self, channel_address, update_callback):
host, s_port = channel_address.split("//")[-1].split(":")
port = int(s_port)
self.peer_chan_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.peer_chan_sock.connect((host, port))
self._callback = update_callback
self.start()
def deal_with_message(self, msg):
self._callback(msg)
def run(self):
data = ""
while True:
new_data = self.peer_chan_sock.recv(1024)
if not new_data:
# connection reset by peer
break
data += new_data
msgs = data.split("\n\n")
if msgs[-1]:
data = msgs.pop()
for msg in msgs:
self.deal_with_message(msg)
def send_value(self, channel_value):
self.channel_thread.sendall("%s\n\n" % channel_value)
Usage:
In process A:
c = Channel()
c.public_address()
In process B:
def msg_received(msg):
print "received:", msg
c = Channel()
c.register("public_address_string_returned_in_process_A", msg_received)
In process A:
c.send_value("HELLO")
In process B:
received: HELLO
I'm writing an asyncore server which fetches info from another module in same process and writes it back to client. The info basically is a dictionary where for each key we have a queue of messages. I'm required to dump the length of each queue. The code runs fine on a test machine but as soon as I install it on a production server I start getting following error message : "socket.error'>:[Errno 32] Broken pipe)".
This is the server:
class request_handler (asyncore.dispatcher):
def __init__(self, conn_sock, client_address, dict):
self.client_address = client_address
self.buffer = ""
self.dict = dict
asyncore.dispatcher.__init__(self, conn_sock)
def readable(self):
return True
def writable(self):
return False
def handle_read(self):
data = self.recv(SIZE)
mtats = "msgq-stats"
if data:
buffer = data
if buffer.lower() == mstats.lower():
msgout = "-- Message Queue Stats --\n"
for key, value in dict.items():
mq = 0
if dict[key].message_queue:
mq = len(dict[key].message_queue)
msgout += key + ":" + str(mq) + "\n"
self.send(msgout)
else: self.send("Invalid input\n")
else:
self.send("Invalid input\n")
def handle_write(self):
print ("--Handling read--\n")
def handle_close(self):
pass
# ---------------------------------------------------------------------
class monitor_server (asyncore.dispatcher):
def __init__ (self, ip, port, destination):
sys.path.append('/path/')
import dict
self.ip = ip
self.port = port
self.dict = dict
asyncore.dispatcher.__init__ (self)
self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind ((ip, port))
self.listen (5)
def writable (self):
return 0
def handle_read (self):
pass
def readable (self):
return self.accepting
def handle_connect (self):
pass
def handle_accept (self):
(conn_sock, client_address) = self.accept()
request_handler (conn_sock, client_address, self.destination)
and this is the client code:
class Client(asyncore.dispatcher_with_send):
def __init__(self, host, port, message):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host, port))
print "Message being sent is "
print message
self.out_buffer = message
def handle_close(self):
self.close()
def handle_read(self):
print self.recv(1024)
self.close()
c = Client('', 6000, 'msgq-stats')
asyncore.loop()
Thanks in advance.
That's an error case you have to handle. Sometimes connections will close. It's normal for different socket errors to be encountered during development as compared to in production, since there are so many different possible errors and they almost entirely depend on the execution environment and what the program on the other side of the connection is doing and on what all the routers between the client and the server decide to do.
So, the literal answer to your question is that you need to handle this and many other socket errors in your application code. That's part of your job when you use asyncore. Add the necessary exception handling and mark the connection as closed when something like this happens.
A slightly better answer is that there are higher level tools that make network programming easier, and you should probably consider using those. The big thing in this area is Twisted.
I'm trying to send a message (a string) to a server (say server A) as a response to a particular string message received from the server. I've used a client connection to send messages to the server. I've created my own server (say server B) using socket module to receive the messages from the the above mentioned server. If a certain message is received, to client should send a message to that server A. I used threads so that both my client and server can work concurrently.
Here's my client script (socket1.py).
import socket
import time
s = socket.socket()
s2 = socket.socket()
class MyClient():
def __init__(self):
self.s=socket.socket()
def start(self):
self.s.connect(('localhost',6000))
self.s.settimeout(1000)
self.s.send('JOIN#')
while True:
if self.s.gettimeout():
break
def move(self):
self.s.send('LEFT#')
def close(self):
self.s.close()
def socStart():
global s,s2
s.connect(('localhost',6000))
#s2.connect(('localhost',6000))
s.send('JOIN#')
#time.sleep(10)
#s.send('DOWN#')
#s.close()
def move():
global s1
s1.send('DOWN#')
def close():
global s,s2
s.close()
#s2.close()
Here's my server script (server1.py)
import socket
class MyServer():
def __init__(self):
self.s= socket.socket()
def serStart(self):
self.s.bind(('localhost',7000))
#self.s.settimeout(30)
self.s.listen(5)
self.started = False
while True:
self.c, self.addr = self.s.accept()
self.msg = self.c.recv(1024)
if self.msg[0] == 'S':
self.started = True
print self.addr, '>>', self.msg
self.c.close()
if self.s.gettimeout():
break
self.s.close()
def getStarted(self):
return self.started
Here's the script in which I used threads with both socket1 & server1 module.
import threading
import datetime
import socket1
import server1
import time
class ThreadClass(threading.Thread):
def initialize(self,client,server):
self.client = client
self.server = server
def run(self):
self.client.start()
if self.server.getStarted():
self.client.move()
self.client.close()
class ThreadServer(threading.Thread):
def initialize(self, client, server):
self.client = client
self.server = server
def run(self):
self.server.serStart()
client = socket1.MyClient()
server = server1.MyServer()
t = ThreadClass()
tS=ThreadServer()
t.initialize(client, server)
tS.initialize(client, server)
t.start()
tS.start()
Though server class changes it's started varialble to True the move method doesn't work in ThreadClass. How do I perform such event triggering and responding accordingly with a outside server in python?
Well, I'm trying to build a small python prgram with a SocketServer that is supposed to send messages it receives to all connected clients. I'm stuck, I don't know how to store clients on the serverside, and I don't know how to send to multiple clients. Oh and, my program fails everytime more then 1 client connects, and everytime a client sends more then one message...
Here's my code until now:
print str(self.client_address[0])+' connected.'
def handle(self):
new=1
for client in clients:
if client==self.request:
new=0
if new==1:
clients.append(self.request)
for client in clients:
data=self.request.recv(1024)
client.send(data)
class Host:
def __init__(self):
self.address = ('localhost', 0)
self.server = SocketServer.TCPServer(self.address, EchoRequestHandler)
ip, port = self.server.server_address
self.t = threading.Thread(target=self.server.serve_forever)
self.t.setDaemon(True)
self.t.start()
print ''
print 'Hosted with IP: '+ip+' and port: '+str(port)+'. Clients can now connect.'
print ''
def close(self):
self.server.socket.close()
class Client:
name=''
ip=''
port=0
def __init__(self,ip,port,name):
self.name=name
self.hostIp=ip
self.hostPort=port
self.s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((self.hostIp, self.hostPort))
def reco(self):
self.s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((self.hostIp, self.hostPort))
def nick(self,newName):
self.name=newName
def send(self,message):
message=self.name+' : '+message
len_sent=self.s.send(message)
response=self.s.recv(len_sent)
print response
self.reco()
def close(self):
self.s.close()
Obviously I have no idea what I'm doing, so any help would be great.
Thanks in advance!
Edit: I'm using Python 2.7 on Windows Vista.
You want to look at asyncore here. The socket operations you're calling on the client side are blocking (don't return until some data is received or a timeout occurs) which makes it hard to listen for messages sent from the host and let the client instances enqueue data to send at the same time. asyncore is supposed to abstract the timeout-based polling loop away from you.
Here's a code "sample" -- let me know if anything is unclear:
from __future__ import print_function
import asyncore
import collections
import logging
import socket
MAX_MESSAGE_LENGTH = 1024
class RemoteClient(asyncore.dispatcher):
"""Wraps a remote client socket."""
def __init__(self, host, socket, address):
asyncore.dispatcher.__init__(self, socket)
self.host = host
self.outbox = collections.deque()
def say(self, message):
self.outbox.append(message)
def handle_read(self):
client_message = self.recv(MAX_MESSAGE_LENGTH)
self.host.broadcast(client_message)
def handle_write(self):
if not self.outbox:
return
message = self.outbox.popleft()
if len(message) > MAX_MESSAGE_LENGTH:
raise ValueError('Message too long')
self.send(message)
class Host(asyncore.dispatcher):
log = logging.getLogger('Host')
def __init__(self, address=('localhost', 0)):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind(address)
self.listen(1)
self.remote_clients = []
def handle_accept(self):
socket, addr = self.accept() # For the remote client.
self.log.info('Accepted client at %s', addr)
self.remote_clients.append(RemoteClient(self, socket, addr))
def handle_read(self):
self.log.info('Received message: %s', self.read())
def broadcast(self, message):
self.log.info('Broadcasting message: %s', message)
for remote_client in self.remote_clients:
remote_client.say(message)
class Client(asyncore.dispatcher):
def __init__(self, host_address, name):
asyncore.dispatcher.__init__(self)
self.log = logging.getLogger('Client (%7s)' % name)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.name = name
self.log.info('Connecting to host at %s', host_address)
self.connect(host_address)
self.outbox = collections.deque()
def say(self, message):
self.outbox.append(message)
self.log.info('Enqueued message: %s', message)
def handle_write(self):
if not self.outbox:
return
message = self.outbox.popleft()
if len(message) > MAX_MESSAGE_LENGTH:
raise ValueError('Message too long')
self.send(message)
def handle_read(self):
message = self.recv(MAX_MESSAGE_LENGTH)
self.log.info('Received message: %s', message)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
logging.info('Creating host')
host = Host()
logging.info('Creating clients')
alice = Client(host.getsockname(), 'Alice')
bob = Client(host.getsockname(), 'Bob')
alice.say('Hello, everybody!')
logging.info('Looping')
asyncore.loop()
Which results in the following output:
INFO:root:Creating host
INFO:root:Creating clients
INFO:Client ( Alice):Connecting to host at ('127.0.0.1', 51117)
INFO:Client ( Bob):Connecting to host at ('127.0.0.1', 51117)
INFO:Client ( Alice):Enqueued message: Hello, everybody!
INFO:root:Looping
INFO:Host:Accepted client at ('127.0.0.1', 55628)
INFO:Host:Accepted client at ('127.0.0.1', 55629)
INFO:Host:Broadcasting message: Hello, everybody!
INFO:Client ( Alice):Received message: Hello, everybody!
INFO:Client ( Bob):Received message: Hello, everybody!
You can use socketserver to broadcast messages to all connected clients. However, the ability is not built into the code and will need to be implemented by extending some of the classes already provided. In the following example, this is implemented using the ThreadingTCPServer and StreamRequestHandler classes. They provide a foundation on which to build but still require some modifications to allow what you are trying to accomplish. The documentation should help explain what each function, class, and method are trying to do in order to get the job done.
Server
#! /usr/bin/env python3
import argparse
import pickle
import queue
import select
import socket
import socketserver
def main():
"""Start a chat server and serve clients forever."""
parser = argparse.ArgumentParser(description='Execute a chat server demo.')
parser.add_argument('port', type=int, help='location where server listens')
arguments = parser.parse_args()
server_address = socket.gethostbyname(socket.gethostname()), arguments.port
server = CustomServer(server_address, CustomHandler)
server.serve_forever()
class CustomServer(socketserver.ThreadingTCPServer):
"""Provide server support for the management of connected clients."""
def __init__(self, server_address, request_handler_class):
"""Initialize the server and keep a set of registered clients."""
super().__init__(server_address, request_handler_class, True)
self.clients = set()
def add_client(self, client):
"""Register a client with the internal store of clients."""
self.clients.add(client)
def broadcast(self, source, data):
"""Resend data to all clients except for the data's source."""
for client in tuple(self.clients):
if client is not source:
client.schedule((source.name, data))
def remove_client(self, client):
"""Take a client off the register to disable broadcasts to it."""
self.clients.remove(client)
class CustomHandler(socketserver.StreamRequestHandler):
"""Allow forwarding of data to all other registered clients."""
def __init__(self, request, client_address, server):
"""Initialize the handler with a store for future date streams."""
self.buffer = queue.Queue()
super().__init__(request, client_address, server)
def setup(self):
"""Register self with the clients the server has available."""
super().setup()
self.server.add_client(self)
def handle(self):
"""Run a continuous message pump to broadcast all client data."""
try:
while True:
self.empty_buffers()
except (ConnectionResetError, EOFError):
pass
def empty_buffers(self):
"""Transfer data to other clients and write out all waiting data."""
if self.readable:
self.server.broadcast(self, pickle.load(self.rfile))
while not self.buffer.empty():
pickle.dump(self.buffer.get_nowait(), self.wfile)
#property
def readable(self):
"""Check if the client's connection can be read without blocking."""
return self.connection in select.select(
(self.connection,), (), (), 0.1)[0]
#property
def name(self):
"""Get the client's address to which the server is connected."""
return self.connection.getpeername()
def schedule(self, data):
"""Arrange for a data packet to be transmitted to the client."""
self.buffer.put_nowait(data)
def finish(self):
"""Remove the client's registration from the server before closing."""
self.server.remove_client(self)
super().finish()
if __name__ == '__main__':
main()
Of course, you also need a client that can communicate with your server and use the same protocol the server speaks. Since this is Python, the decision was made to utilize the pickle module to facilitate data transfer among server and clients. Other data transfer methods could have been used (such as JSON, XML, et cetera), but being able to pickle and unpickle data serves the needs of this program well enough. Documentation is included yet again, so it should not be too difficult to figure out what is going on. Note that server commands can interrupt user data entry.
Client
#! /usr/bin/env python3
import argparse
import cmd
import pickle
import socket
import threading
def main():
"""Connect a chat client to a server and process incoming commands."""
parser = argparse.ArgumentParser(description='Execute a chat client demo.')
parser.add_argument('host', type=str, help='name of server on the network')
parser.add_argument('port', type=int, help='location where server listens')
arguments = parser.parse_args()
client = User(socket.create_connection((arguments.host, arguments.port)))
client.start()
class User(cmd.Cmd, threading.Thread):
"""Provide a command interface for internal and external instructions."""
prompt = '>>> '
def __init__(self, connection):
"""Initialize the user interface for communicating with the server."""
cmd.Cmd.__init__(self)
threading.Thread.__init__(self)
self.connection = connection
self.reader = connection.makefile('rb', -1)
self.writer = connection.makefile('wb', 0)
self.handlers = dict(print=print, ping=self.ping)
def start(self):
"""Begin execution of processor thread and user command loop."""
super().start()
super().cmdloop()
self.cleanup()
def cleanup(self):
"""Close the connection and wait for the thread to terminate."""
self.writer.flush()
self.connection.shutdown(socket.SHUT_RDWR)
self.connection.close()
self.join()
def run(self):
"""Execute an automated message pump for client communications."""
try:
while True:
self.handle_server_command()
except (BrokenPipeError, ConnectionResetError):
pass
def handle_server_command(self):
"""Get an instruction from the server and execute it."""
source, (function, args, kwargs) = pickle.load(self.reader)
print('Host: {} Port: {}'.format(*source))
self.handlers[function](*args, **kwargs)
def preloop(self):
"""Announce to other clients that we are connecting."""
self.call('print', socket.gethostname(), 'just entered.')
def call(self, function, *args, **kwargs):
"""Arrange for a handler to be executed on all other clients."""
assert function in self.handlers, 'You must create a handler first!'
pickle.dump((function, args, kwargs), self.writer)
def do_say(self, arg):
"""Causes a message to appear to all other clients."""
self.call('print', arg)
def do_ping(self, arg):
"""Ask all clients to report their presence here."""
self.call('ping')
def ping(self):
"""Broadcast to all other clients that we are present."""
self.call('print', socket.gethostname(), 'is here.')
def do_exit(self, arg):
"""Disconnect from the server and close the client."""
return True
def postloop(self):
"""Make an announcement to other clients that we are leaving."""
self.call('print', socket.gethostname(), 'just exited.')
if __name__ == '__main__':
main()
why use SocketServer? a simple client doesn't meet your needs?
import socket
HOST = ''
PORT = 8000
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((HOST, PORT))
sock.listen(5)
while True:
conn, addr = sock.accept()
print 'connecting to', addr
while True:
data = conn.recv(1024)
if not data:
break
conn.send(data)
To take multiple clients simultaneously, you will have to add SocketServer.ForkingMixIn or ThreadingMixIn.