I am creating a program that requires a client and a server to track player locations in pygame.
I use this class for the networking in the client:
class Network:
def __init__(self):
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server = "0.0.0.0"
self.port = 5555
self.addr = (self.server, self.port)
self.pos = self.connect()
def getPos(self):
return self.pos
def connect(self):
try:
self.client.connect(self.addr)
return self.client.recv(2048).decode()
except:
pass
def send(self, data):
try:
self.client.send(str.encode(data))
return self.client.recv(2048).decode()
except socket.error as e:
print(e)
To format the data to be sent and received by the server I am using:
def read_pos(str):
print(str)
print(type(str))
return json.loads(str) # returns as tuple
def make_pos(tup):
return json.dumps(tup)
Within the main function I create a network object (n) and then use n.connect() to the server, no errors occur. However, on the following line the following error occurs:
raise TypeError(f'the JSON object must be str, bytes or bytearray, '
TypeError: the JSON object must be str, bytes or bytearray, not NoneType
This is the main function where the code fails:
def main(): # asking server for updates, checking for events
n = Network()
n.connect()
startPos = read_pos(n.getPos())
Upon investigation, I can see the client has not connected to the server. Therefore, when the str being passed to read_pos is a NoneType which is where the error is coming from. What is preventing the client from connecting to the server?
I have a code below that does the client-server communication properly.
The client:
# Client
import socket
import pickle
class Model:
def __init__(self, host, port):
self.port = port
self.host = host
def snd_query(self, query):
received_data = []
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((self.host, self.port))
sock.sendall(pickle.dumps(query))
while True:
packet = sock.recv(4096)
if not packet or packet == b'':
break
received_data.append(packet)
try:
content = pickle.loads(b"".join(received_data))
return content
except EOFError:
return None
and a server:
# Server.py
import socketserver
import pickle
import configparser
from data_manipulation import DataManipulation
class MyTCPHandler(socketserver.BaseRequestHandler):
def handle(self):
db = DataManipulation("data.db")
request = pickle.loads(self.request.recv(1024))
if request['command'] == 'GET':
content = db.get_data()
#elif ...:
#... some other logic and database interactions
self.request.sendall(pickle.dumps((content)))
def main(HOST, PORT):
try:
my_server = socketserver.TCPServer((HOST, PORT), MyTCPHandler)
my_server.serve_forever()
except KeyboardInterrupt:
my_server.shutdown()
my_server.server_close()
if __name__ == "__main__":
config = configparser.ConfigParser()
config.read('../params.ini')
main(config['SERVER']['host'], int(config['SERVER']['port']))
In that form with every new handled request a database connection db instance is created.
I would like to avoid it. I would like to make db object ones and raise it with argument read from ini file. Obviously it is enough to read it once.
All examples that I found show simple echo server application. handle() doesn't do much, only prints some stuff.
I am not sure where is the correct place to call db object to have only one instance?
How to properly incorporate some advanced logic in handle() method?
I have previously managed to implement a client-server socket script which relays messages between a single client and the server and I'm now trying to implement a multiple-client system.
More specifically, I would like to use the server as some sort of medium between two clients which retrieves information from one client and relays it to the other. I had tried to attach and send the port number of the receiving client and then extract it from the message on the server side. After that, I would try and send it to whatever socket with that port number but I ran into some trouble (as port numbers are determined at the point of sending I believe?) so now I am simply just trying to relay the sent message back to all clients. However, the problem is that the message is only being sent to the server and not being relayed to the desired client.
I had previously tried to implement a peer-to-peer system but I ran into trouble so I decided to take a step back and do this instead.
Server.py:
import socket, _thread, threading
import tkinter as tk
SERVERPORT = 8600
HOST = 'localhost'
class Server():
def __init__(self):
self.Connected = True
self.ServerSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.ServerSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.ServerSocket.bind((HOST, SERVERPORT))
self.ServerSocket.listen(2)
self.Clients = []
def Listen(self):
print('Server is now running')
while self.Connected:
ClientSocket, Address = self.ServerSocket.accept()
self.Clients.append(Address)
print('\nNew user connected', Address)
t = threading.Thread(target=self.NewClient, args=(ClientSocket,
Address))
t.daemon = True
t.start()
self.Socket.close()
def NewClient(self, ClientSocket, Address):
while self.Connected:
if ClientSocket:
try:
ReceivedMsg = ClientSocket.recv(4096)
print('Message received from', Address, ':', ReceivedMsg)
self.Acknowledge(ClientSocket, Address)
if ReceivedMsg.decode('utf8').split()[-1] != 'message':
ReceiverPort = self.GetSendPort(ReceivedMsg)
self.SendToClient(ClientSocket,ReceivedMsg,ReceiverPort)
except:
print('Connection closed')
raise Exception
ClientSocket.close()
def Acknowledge(self, Socket, Address):
Socket.sendto(b'The server received your message', Address)
def GetSendPort(self, Msg):
MsgDigest = Msg.decode('utf8').split()
return int(MsgDigest[-1])
def SendToClient(self, Socket, Msg, Port):
Addr = (HOST, Msg)
for Client in self.Clients:
Socket.sendto(Msg, Client)
def NewThread(Func, *args):
if len(args) == 1:
t = threading.Thread(target=Func, args=(args,))
elif len(args) > 1:
t = threading.Thread(target=Func, args=args)
else:
t = threading.Thread(target=Func)
t.daemon = True
t.start()
t.join()
Host = Server()
NewThread(Host.Listen)
And the Client(.py):
import socket, threading
import tkinter as tk
Username = 'Ernest'
PORT = 8601
OtherPORT = 8602
SERVERPORT = 8600
HOST = '127.0.0.1'
class Client():
def __init__(self, Username):
self.Connected, self.Username = False, Username
self.Socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def Connect(self):
print('Trying to connect')
try:
self.Socket.connect((HOST, SERVERPORT))
self.Connected = True
print(self.Username, 'connected to server')
Msg = MsgUI(self.Username)
Msg.Display()
except Exception:
print('Could not connect to server')
raise Exception
def SendMsg(self):
if self.Connected:
Msg = '{} sent you a message {}'.format(self.Username, OtherPORT)
self.Socket.sendall(bytes(Msg, encoding='utf8'))
self.GetResponse()
def GetResponse(self, *args):
AckMsg = '\n{} received the message'.format(self.Username)
NMsg = '\n{} did not receive the message'.format(self.Username)
if self.Connected:
Msg = self.Socket.recv(4096)
print(Msg)
if Msg:
self.Socket.sendall(bytes(AckMsg, encoding='utf8'))
else:
self.Socket.sendall(bytes(NMsg, encoding='utf8'))
class MsgUI():
def __init__(self, Username):
self.Username = Username
self.entry = tk.Entry(win)
self.sendbtn = tk.Button(win, text='send', command=Peer.SendMsg)
def Display(self):
self.entry.grid()
self.sendbtn.grid()
win.mainloop()
win = tk.Tk()
Peer = Client(Username)
Peer.Connect()
I want a message to be sent whenever the user presses the send button in the tkinter window, but at the same time, it is continually 'listening' to see if it received any messages.
I also previously tried to run the GetResponse method in the Client in another thread and instead of if self.Connected I used while self.Connected and it still didn't work.
UPDATE
After some helpful comments, I have edited the two files as such:
The server now holds the two sockets for each client which is run first. The server file is imported into the client file as a module. Each client file is then run and each client runs a function in the server file, requesting to use the socket. If the request is allowed (i.e. no error was thrown), the socket is connected, added to a set of clients stored in the server file and then returned to the client file. The client then uses this socket to send and receive messages.
Server.py
import socket, _thread, threading
import tkinter as tk
SERVERPORT = 8600
HOST = 'localhost'
class Server():
def __init__(self):
self.Connected = True
self.ServerSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.ServerSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.ServerSocket.bind((HOST, SERVERPORT))
self.ServerSocket.listen(2)
self.Clients = {}
def ConnectClient(self, Username, Port):
Socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.Clients[Username] = [Socket, Port, False]
try:
self.Clients[Username][0].connect((HOST, SERVERPORT))
self.Clients[Username][2] = True
print('Opened port for user', Username)
return Socket
except Exception:
print('Could not open port for user', Username)
raise Exception
def Listen(self):
print('Server is now running')
while self.Connected:
ClientSocket, Address = self.ServerSocket.accept()
print('\nNew user connected', Address)
t = threading.Thread(target=self.NewClient, args=(ClientSocket,
Address))
t.daemon = True
t.start()
self.Socket.close()
def NewClient(self, ClientSocket, Address):
while self.Connected:
if ClientSocket:
try:
ReceivedMsg = ClientSocket.recv(4096)
if b'attempting to connect to the server' in ReceivedMsg:
ClientSocket.send(b'You are now connected to the server')
else:
print('Message received from', Address, ':',ReceivedMsg)
#self.Acknowledge(ClientSocket, Address)
ReceiverPort = self.GetSendPort(ReceivedMsg)
if ReceiverPort != None:
self.SendToClient(ClientSocket,ReceivedMsg,
ReceiverPort)
except:
print('Connection closed')
raise Exception
ClientSocket.close()
def Acknowledge(self, Socket, Address):
Socket.sendto(b'The server received your message', Address)
def GetSendPort(self, Msg):
MsgDigest = Msg.decode('utf8').split()
try:
Port = int(MsgDigest[-1])
except ValueError:
Port = None
return Port
def SendToClient(self, Socket, Msg, Port):
Addr = (HOST, Port)
Receiver = None
for Client, Vars in self.Clients.items():
if Vars[1] == Port:
Receiver = Client
self.Clients[Receiver][0].sendto(Msg, Addr)
def NewThread(Func, *args):
if len(args) == 1:
t = threading.Thread(target=Func, args=(args,))
elif len(args) > 1:
t = threading.Thread(target=Func, args=args)
else:
t = threading.Thread(target=Func)
t.daemon = True
t.start()
t.join()
Host = Server()
if __name__ == '__main__':
NewThread(Host.Listen)
And Client.py
import socket, threading, Server
import tkinter as tk
Username = 'Ernest'
PORT = 8601
OtherPORT = 8602
SERVERPORT = 8600
HOST = '127.0.0.1'
class Client():
def __init__(self, Username):
self.Connected, self.Username = False, Username
def Connect(self):
print('Requesting to connect to server')
try:
self.Socket = Server.Host.ConnectClient(self.Username, PORT)
self.Connected = Server.Host.Clients[self.Username][2]
Msg = '{} is attempting to connect to the server'.format(self.Username)
self.Socket.sendall(bytes(Msg, encoding='utf8'))
ReceivedMsg = self.Socket.recv(4096)
print(ReceivedMsg)
Msg = MsgUI(self.Username)
Msg.Display()
except Exception:
print('Could not connect to server')
raise Exception
def SendMsg(self):
try:
if self.Connected:
Msg = '{} sent you a message {}'.format(self.Username,OtherPORT)
self.Socket.sendall(bytes(Msg, encoding='utf8'))
self.GetResponse()
except Exception:
print('Connection closed')
raise Exception
def GetResponse(self, *args):
AckMsg = '\n{} received the message'.format(self.Username)
NMsg = '\n{} did not receive the message'.format(self.Username)
if self.Connected:
Msg = self.Socket.recv(4096)
print(Msg)
if Msg:
self.Socket.sendall(bytes(AckMsg, encoding='utf8'))
else:
self.Socket.sendall(bytes(NMsg, encoding='utf8'))
class MsgUI():
def __init__(self, Username):
self.Username = Username
self.entry = tk.Entry(win)
self.sendbtn = tk.Button(win, text='send', command=Peer.SendMsg)
def Display(self):
self.entry.grid()
self.sendbtn.grid()
win.mainloop()
win = tk.Tk()
Peer = Client(Username)
Peer.Connect()
Now the problem is more of a python and scope problem. When trying to relay the message back to the client, I was getting a KeyError as the Clients dictionary was still empty. When making the function call to the server in the client file, it's clear that the update to the dictionary happens in the client file rather than the server file - which is in a different instance. I need a method of changing the contents of the Clients dictionary that is called to action by the client file but takes effect in the server file.
Are you committed to multithreading? Threads don't run concurrently in python ( due to the GIL), and while they are one way to handle concurrent operations, they aren't the only way and usually they're not the best way, unless they're the only way. Consider this code, which doesn't handle failure cases well, but seems to work as a starting point.
import socket, select, Queue
svrsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
svrsock.setblocking(0)
svrsock.bind(('', 17654))
svrsock.listen(16)
client_queues = {}
write_ready=[] # we'll update this for clients only that have things in the queue
while client_queues.keys() + [svrsock] :
readable, writable, exceptional = select.select(client_queues.keys() + [svrsock] , write_ready, [])
for rd in readable:
if rd is svrsock: # reading listening socket == accepting connection
conn, addr = svrsock.accept()
print("Connection from {}".format(addr))
conn.setblocking(0)
client_queues[conn] = Queue.Queue()
else:
data = rd.recv(1024)
if data:
# TODO: send to all queues
print("Message from {}".format(rd.getpeername()))
for sock, q in client_queues.iteritems():
q.put("From {}: {}".format( rd.getpeername(), data))
if sock not in write_ready:
write_ready.append(sock)
for rw in writable:
try:
data = client_queues[rw].get_nowait()
rw.send(data)
except Queue.Empty:
write_ready.remove(rw)
continue
The concept is pretty simple. The server accepts connections; each connection (socket) is associated with a queue of pending messages. Each socket that's ready for reading is read from, and its message is added to each client's queue. The recipient client is added into the write_ready list of clients with data pending, if it's not already in there. Then each socket that's ready for writing has its next queued message written to it. If there are no more messages, the recipient is removed from the write_ready list.
This is very easy to orchestrate if you don't use multithreading because all coordination is inherent in the order of the application. With threads it would be more difficult and a lot more code, but probably not more performance due to the gil.
The secret to handling multiple I/O streams concurrently without multithreading is select. In principle it's pretty easy; we pass select() a list of possible sockets for reading, another list of possible sockets for writing, and a final list that for this simplified demo I completely ignore . The results of the select call will include one or more sockets that are actually ready for reading or writing, which allows me to block until one or more sockets are ready for activity. I then process all the sockets ready for activity every pass ( but they've already been filtered down to just those which wouldn't block).
There's a ton still to be done here. I don't cleanup after myself, don't track closed connections, don't handle any exceptions, and so on. but without having to worry about threading and concurrency guarantees, it's pretty easy to start addressing these deficiencies.
Here it is "in action". Here for the client side I use netcat, which is perfect for layer 3 testing without layer 4+ protocols ( in other words, raw tcp so to speak). It simply opens a socket to the given destination and port and sends its stdin through the socket and sends its socket data to stdout, which makes it perfect for demoing this server app!
I also wanted to point out, coupling code between server and client is inadvisable because you won't be able to roll out changes to either without breaking the other. It's ideal to have a "contract" so to speak between server and client and maintain it. Even if you implement the behavior of server and client in the same code base, you should use the tcp communications contract to drive your implementation, not code sharing. Just my 2 cents, but once you start sharing code you often start coupling server/client versions in ways you didn't anticipate.
the server:
$ python ./svr.py
Connection from ('127.0.0.1', 52059)
Connection from ('127.0.0.1', 52061)
Message from ('127.0.0.1', 52061)
Message from ('127.0.0.1', 52059)
Message from ('127.0.0.1', 52059)
First client ( 52059):
$ nc localhost 17654
hello
From ('127.0.0.1', 52061): hello
From ('127.0.0.1', 52059): hello
From ('127.0.0.1', 52059): hello
Second client:
$ nc localhost 17654
From ('127.0.0.1', 52061): hello
hello
From ('127.0.0.1', 52059): hello
hello
From ('127.0.0.1', 52059): hello
If you need more convincing on why select is way more compelling than concurrent execution, consider this: Apache is based on a threading model, in other words, the connections each get a worker thread . nginx is based on a select model, so you can see how much faster that can potentially be. Not to say that nginx is inherently better, as Apache benefits from the threading model because of its heavy use of modules to extend capabilities ( mod_php for example), whereas nginx doesn't have this limitation and can handle all requests from any thread. But the raw performance of nginx is typically considered far higher and far more efficient, and a big reason for this is that it avoids almost all the cpu context switches inherent in apache. It's a valid approach!
A word on scaling. Obviously, this wouldn't scale forever. Neither would a threading model; eventually you run out of threads. A more distributed and high throughput system would likely use a Pub/Sub mechanism of some kind, offloading the client connection tracking and message queueing from the server to a pub/sub data tier and allowing connections to be restored and queued data to be sent, as well as adding multiple servers behind a load balancer. Just throwing it out there. You might be pleasantly surprised how well select can scale ( cpu is so much faster than network anyway that it's likely not the bottleneck).
I'm writing a server in Python, which should basicaly work as a pipe between some serial device (like Arduino) and any client connected to it.
import SocketServer
import threading
import errno
def SerialThread():
# Serial init is here
while (True):
msg = read_from_serial(blocking = True)
#print(msg)
try:
server.socket.sendall(str(msg))
except IOError, e:
if e.errno == errno.EPIPE:
print('Server not ready')
serial_daemon = threading.Thread(name='SerialThread', target=SerialThread)
serial_daemon.daemon = True
serial_daemon.start()
class TCPHandler(SocketServer.StreamRequestHandler):
def handle(self):
print('#')
server = SocketServer.TCPServer(('127.0.0.1', 4242), TCPHandler)
print('Starting server')
server.serve_forever()
But I keep getting Server not ready spammed in my console. So what's wrong with my code, and how can I fix it?
TCPServer.socket is in "listen" mode, you can not use it to trans data directly, you need "connect" to its address with a new socket object:
import socket
while 1:
s = socket.create_connection(('127.0.0.1', 4242))
s.sendall(str(msg))
depends on how your server handle connection, you could reuse the same socket or create a new one every loop.
I'm working on a project which uses python asyncio socket server. The problem is that the implementation of the server doesn't call .close() on the transport when the server stops. This seems to leave clients connected and causes crashes in other parts of the code.
Python documents say that transports need to be closed explicitly, but in this project I don't know where I can close them because there is no reference to the transports that are created for each client.
https://docs.python.org/3/library/asyncio-dev.html#close-transports-and-event-loops
Here is the code:
"""
Socket server forwarding request to internal server
"""
import logging
try:
# we prefer to use bundles asyncio version, otherwise fallback to trollius
import asyncio
except ImportError:
import trollius as asyncio
from opcua import ua
from opcua.server.uaprocessor import UaProcessor
logger = logging.getLogger(__name__)
class BinaryServer(object):
def __init__(self, internal_server, hostname, port):
self.logger = logging.getLogger(__name__)
self.hostname = hostname
self.port = port
self.iserver = internal_server
self.loop = internal_server.loop
self._server = None
self._policies = []
def set_policies(self, policies):
self._policies = policies
def start(self):
class OPCUAProtocol(asyncio.Protocol):
"""
instanciated for every connection
defined as internal class since it needs access
to the internal server object
FIXME: find another solution
"""
iserver = self.iserver
loop = self.loop
logger = self.logger
policies = self._policies
def connection_made(self, transport):
self.peername = transport.get_extra_info('peername')
self.logger.info('New connection from %s', self.peername)
self.transport = transport
self.processor = UaProcessor(self.iserver, self.transport)
self.processor.set_policies(self.policies)
self.data = b""
def connection_lost(self, ex):
self.logger.info('Lost connection from %s, %s', self.peername, ex)
self.transport.close()
self.processor.close()
def data_received(self, data):
logger.debug("received %s bytes from socket", len(data))
if self.data:
data = self.data + data
self.data = b""
self._process_data(data)
def _process_data(self, data):
buf = ua.utils.Buffer(data)
while True:
try:
backup_buf = buf.copy()
try:
hdr = ua.Header.from_string(buf)
except ua.utils.NotEnoughData:
logger.info("We did not receive enough data from client, waiting for more")
self.data = backup_buf.read(len(backup_buf))
return
if len(buf) < hdr.body_size:
logger.info("We did not receive enough data from client, waiting for more")
self.data = backup_buf.read(len(backup_buf))
return
ret = self.processor.process(hdr, buf)
if not ret:
logger.info("processor returned False, we close connection from %s", self.peername)
self.transport.close()
return
if len(buf) == 0:
return
except Exception:
logger.exception("Exception raised while parsing message from client, closing")
self.transport.close()
break
coro = self.loop.create_server(OPCUAProtocol, self.hostname, self.port)
self._server = self.loop.run_coro_and_wait(coro)
print('Listening on {}:{}'.format(self.hostname, self.port))
def stop(self):
self.logger.info("Closing asyncio socket server")
self.loop.call_soon(self._server.close)
self.loop.run_coro_and_wait(self._server.wait_closed())
As you can see when we call stop() on this server class the asyncio server calls it's close method. However if clients are connected the created transports never get closed.
The project repository is here https://github.com/FreeOpcUa/python-opcua/ , you can take a look at Issue 137.
What is the correct way to close the transport object?
I solve this by applying this approach:
#self.OPCUAServer - this is my opcua server
nodes = []
nodes.append(self.OPCUAServer.get_node("ns=0; s=Measurements")) #Adding two root nodes
nodes.append(self.OPCUAServer.get_node("ns=1; s=Calibrations")) #to the list
self.OPCUAServer.delete_nodes(nodes, True) # Recursively call delete_nodes with this list
self.OPCUAServer.stop()