I've written a basic client/server interface using Python socket (quoted only relevant part of code, for full script: (SERVER: https://github.com/mydomo/ble-presence/blob/master/server.py)
(CLIENT: https://github.com/mydomo/ble-presence/blob/master/clients/DOMOTICZ/ble-presence/plugin.py)
The issue is when the script run from some hours and the result list is getting bigger sometimes the reply is exactly as it should be, other times it's cutted, not complete... it's random, like if the socket closed for no reason earlier or the reply is not fully read.
Can you please help me?
SERVER:
def client_thread(conn, ip, port, MAX_BUFFER_SIZE = 32768):
# the input is in bytes, so decode it
input_from_client_bytes = conn.recv(MAX_BUFFER_SIZE)
# MAX_BUFFER_SIZE is how big the message can be
# this is test if it's too big
siz = sys.getsizeof(input_from_client_bytes)
if siz >= MAX_BUFFER_SIZE:
print("The length of input is probably too long: {}".format(siz))
# decode input and strip the end of line
input_from_client = input_from_client_bytes.decode("utf8").rstrip()
res = socket_input_process(input_from_client)
#print("Result of processing {} is: {}".format(input_from_client, res))
vysl = res.encode("utf8") # encode the result string
conn.sendall(vysl) # send it to client
conn.close() # close connection
##########- END FUNCTION THAT HANDLE SOCKET'S TRANSMISSION -##########
def start_server():
global soc
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# this is for easy starting/killing the app
soc.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
#print('Socket created')
try:
soc.bind((socket_ip, socket_port))
# print('Socket bind complete')
except socket.error as msg:
# print('Bind failed. Error : ' + str(sys.exc_info()))
sys.exit()
#Start listening on socket
soc.listen(10)
#print('Socket now listening')
# for handling task in separate jobs we need threading
#from threading import Thread
# this will make an infinite loop needed for
# not reseting server for every client
while (not killer.kill_now):
conn, addr = soc.accept()
ip, port = str(addr[0]), str(addr[1])
#print('Accepting connection from ' + ip + ':' + port)
try:
Thread(target=client_thread, args=(conn, ip, port)).start()
except:
print("Terible error!")
import traceback
traceback.print_exc()
soc.close()
CLIENT:
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
SERV_ADDR = str(Parameters["Address"])
SERV_PORT = int(Parameters["Port"])
soc.connect((SERV_ADDR, SERV_PORT))
if BATTERY_REQUEST == True:
clients_input = str(BATTERY_DEVICE_REQUEST)
else:
clients_input = "beacon_data"
soc.send(clients_input.encode()) # we must encode the string to bytes
result_bytes = soc.recv(32768) # the number means how the response can be in bytes
result_string = result_bytes.decode("utf8") # the return will be in bytes, so decode
Method recv() does not guarantee receiving the full message in the first call so you have to try getting the full message by calling recv() multiple times.
If recv() does return an empty string, connection is closed in the client side.
Using this while loop you can get full stream from client into data:
data = b'' # recv() does return bytes
while True:
try:
chunk = conn.recv(4096) # some 2^n number
if not chunk: # chunk == ''
break
data += chunk
except socket.error:
conn.close()
break
TCP is a streaming protocol, meaning it has no concept of what constitutes a complete message. You have to implement your own message protocol layer on top of TCP to make sure you send and receive complete messages. You are responsible for buffering data received until you have a complete message, and you have to define what a complete message is. Some options:
Send fixed length messages.
Send a fixed number of bytes representing the length of the message, then the message.
Separate messages with a sentinel byte.
Then, call recv and accumulate the results until you have a complete message in the buffer.
Related
Good evening,
I am trying to make a basic implementation of the clients-to-server model using sockets.
Basically, I want the client to retrieve a small piece of information (IE: MESSAGE key) to the server. To accomplish this, I've used threading.
When the client sends the request to receive the info, there is no error on the client side.
However, the server sends an error
Exception in thread Thread-2:
Traceback (most recent call last):
File "C:\Users\Stock\AppData\Local\Continuum\anaconda3\lib\threading.py", line 926, in _bootstrap_inner
self.run()
File "C:\Users\Stock\AppData\Local\Continuum\anaconda3\lib\threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "server.py", line 40, in listeningForRetriever
dataBytes = conn.recv(1024)
ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine
After doing some research, this thread seemed pretty relevant, https://stackoverflow.com/a/49289545/6902431, but I had already implemented the suggested fixes.
server.py
import socket
import threading
from Utility import *
HOST = '127.0.0.1' # Standard loopback interface address (localhost)
PORT = 65432 # Port to listen on (non-privileged ports are > 1023)
PORT_A = 22221
PORT_B = 22222
PORT_C = 22223
bank = {'me':'you'}
def listeningForRetriever():
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind((HOST, PORT_B))
s.listen()
conn, addr = s.accept()
with conn:
while True:
dataBytes = conn.recv(1024)
dataString = bytesToString(dataBytes)
try:
retrievedValueBytes = bytesToString(bank[cleanGET(dataString)])
print(f"Value: {retrievedValueBytes} has been retrieved with Key {cleanGET(dataString)}")
conn.sendall(retrievedValueBytes)
except:
print("SERVER: Errored out")
conn.sendall(stringToBytes("ERROR! Value retrieval for command: " + dataString + " doesn't exist!"))
retrieverThread = threading.Thread(target=listeningForRetriever)
retrieverThread.start()
retriever.py
#Retrieves values from server using keys
#retriever.py --key=username
import sys
import socket
from Utility import *
# total arguments
n = len(sys.argv)
print("Total arguments passed:", n)
# Arguments passed
print("\nName of Python script:", sys.argv[0])
print("\nArguments passed:", end = " ")
key = ''
value = ''
for i in range(1, n):
print(sys.argv[i], end = " ")
if '--key=' in sys.argv[i]:
data = sys.argv[i].replace('--key=','')
key = data
print()
if key == '':
print("ERROR! Invalid syntax.")
print("Try something like: Bob.py --key=username")
else:
print(f"Key: {key} {type(key)}")
#TODO: Send key to storage of server
HOST = '127.0.0.1' # The server's hostname or IP address
PORT = 22222 # The port used by the server
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST, PORT))
s.sendall(stringToBytes("GET key"+"="+key))
print("DEBUG 1")
value = s.recv(1024)
print("DEBUG 2")
print(f"Retrieved Value: {value}")
print(f"Value Retrieved: {value}")
s.close()
What am I overlooking/misunderstanding?
I could (more or less...) reproduce and fix. But I may be wrong, because your code uses an Utility module that you failed to show so I could only guess from the names what bytesToString, stringToBytes and cleanGET were supposed to do.
First there is a misunderstanding about how TCP actually behaves. It is a streaming protocol, that guarantees that all sent bytes will be received an in same order. But there is no guarantee on the packets themselves: they may be splitted and/or reassembled by the network. So you must use a higher level way to delimit a message. It does not exhibit any problem here because short packets are often left unmodified, but the protocol does not guarantee that so you could experience weird errors later. Here as you send a single message, you could shut down the socket to signal the end of the message: the receiver will see a 0 size packet at that point.
Next (and the actual cause of your problem), you loop after accepting a connection. So you get a message, successfully process it, read again on a socket that will be closed by the peer and get a 0 size byte string. It probably causes a key error when looking in the bank dict, but you silently swallow that error message (which is BAD), so you try to write on a socket closed by the peer, which causes the connection to be aborted. And finally on next recv you get the error that you see.
What you should have learned from that:
never ever use a silent try: ... except: ... that swallows any exception without letting you know what has happened. Always limit the filtered exception to the smaller expected set, or (at least and at dev time) display the caught exception to make sure you have not inadvertently caught an unwanted one.
use a higher level protocol to delimit messages on TCP
Now for the fixes:
retriever.py
...
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST, PORT))
s.sendall(stringToBytes("GET key"+"="+key))
s.shutdown(socket.SHUT_WR) # signals the end of the message
print("DEBUG 1")
...
server.py
...
s.listen()
conn, addr = s.accept()
with conn:
# read until the end of a message signaled by a 0 size packet
dataBytes = b''
while True:
b = conn.recv(1024)
if len(b) == 0:
break
dataBytes += b
dataString = bytesToString(dataBytes)
try:
retrievedValueBytes = stringToBytes(bank[cleanGET(dataString)])
print(f"Value: {retrievedValueBytes} has been retrieved with Key {cleanGET(dataString)}")
conn.sendall(retrievedValueBytes)
except Exception as e:
print(e)
print("SERVER: Errored out")
conn.sendall(stringToBytes("ERROR! Value retrieval for command: " + dataString + " doesn't exist!"))
...
Here the thread stops after one single message. If you want the server to be able to accept many clients, you can loop over accept:
s.listen()
while True:
conn, addr = s.accept()
...
Your server is coded to listen in a loop (while True / conn.recv), but your
client only listens once, then exits, which closes the connection.
So the message on the server side is actually not vague at all, it's perfectly to
the point: it's telling you that the connection has been closed. It has indeed,
by the client.
(To be very specific, your retriever code uses a socket as a context manager,
meaning that when you exit the 'with' bock, the socket gets automatically
closed - you can remove the s.close() call, BTW)
Your code works fine, you should be getting the data back (I did, but your post
doesn't say anything about that). The only thing missing is the exception
handling on the server side to gracefully handle the client disconnection.
I am working on a program using TCP protocol to collect ADS-B messages from an antenna. Since I am new to Python, I used the following scripts to establish connection. The problem is that I receive several messages at the same time (since TCP is stream-oriented). I would like to separate each message using a "\n" delimiter for instance (each message has "#" at the beginning and ";" at the end and the length varies). I have no idea of how to tell Python to separate each message like this, do you have any idea ? Thanks a lot
Python version 3.7.6, Anaconda, Windows 10
import socketserver
class MyTCPHandler(socketserver.BaseRequestHandler):
"""
# The request handler class for our server.
# It is instantiated once per connection to the server, and must
# override the handle() method to implement communication to the
# client.
# """
def handle(self):
# self.rfile is a file-like object created by the handler;
# we can now use e.g. readline() instead of raw recv() calls
self.data = self.rfile.readline().strip()
print("{} wrote:".format(self.client_address[0]))
print(self.data)
# Likewise, self.wfile is a file-like object used to write back
# to the client
self.wfile.write(self.data.upper())
if __name__ == "__main__":
print ("Server online")
HOST, PORT = "localhost", 10100
# Create the server, binding to localhost on port 10002
with socketserver.TCPServer((HOST, PORT), MyTCPHandler) as server:
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
server.serve_forever()
import socket
import sys
def tcp_client():
HOST, PORT = "192.168.2.99", 10002
data = " ".join(sys.argv[1:])
# Create a socket (SOCK_STREAM means a TCP socket)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
# Connect to server and send data
sock.connect((HOST, PORT))
while True :
sock.sendall(bytes(data + "\n", "utf-8"))
# Receive data from the server
received = str(sock.recv(1024), "utf-8")
print("{}".format(received))
You could try using:
acumulator = ""
while True:
received = str(sock.recv(1024), "utf-8")
divided_message = received.split('\n')
if len(divided_message) >= 2:
print('One mesage: ', acumulator + divided_message[0].strip())
for i in range(1, len(divided_message) - 1):
print('One mesage: ', divided_message[i].strip())
if '\n' in divided_message[-1]:
print('One mesage: ', divided_message[-1].strip())
acumulator = ''
else:
acumulator = divided_message[-1]
else:
acumulator += divided_message[0]
If the message is separated by /n you can divide the message applying a selection technique, like the one presented above. If your messages have a fixed length you could just modify your delimiter.
I would suggest you use the following approach:
Assuming your messages can't be more than 4GB long, just send the
length, packed into exactly 4 bytes, and then you send the data
itself. So, the other side always knows how much to read: Read exactly
4 bytes, unpack it into a length, then read exactly as many bytes as
that:
def send_one_message(sock, data):
length = len(data)
sock.sendall(struct.pack('!I', length))
sock.sendall(data)
def recv_one_message(sock):
lengthbuf = recvall(sock, 4)
length, = struct.unpack('!I', lengthbuf)
return recvall(sock, length)
That's almost a complete protocol. The only problem is that Python
doesn't have a recvall counterpart to sendall, but you can write it
yourself:
def recvall(sock, count):
buf = b''
while count:
newbuf = sock.recv(count)
if not newbuf: return None
buf += newbuf
count -= len(newbuf)
return buf
More detailed description here
This question already has an answer here:
Python Bidirectional TCP Socket Hanging on socket.recv
(1 answer)
Closed 3 years ago.
I'm trying to make a client-server application in which:
the client sends a message (name of a function) to the server
the server receives the message, calls the corresponding function and returns the results to the client.
I'm able to do this only if the message from the client is one; if I want to call two or more functions, and therefore send two or more messages, I get some problems because I need to add a loop inside the server.
In particular, what is not clear to me is the socket function recv(). Usually, to receive data I write something like this (without setblocking()) and I never had any problems:
while True:
data = sock.recv(BUFFER)
results += data.decode()
if not data:
break
But if I add a while True: to the server (in order to wait for the other functions), then the client never exits from sock.rev(). Why? I expect that function to be always blocking or always non-blocking based on how it was set sock.setblocking().
I already tried with settimeout() and it worked, but performances are really important so I would like to avoid every possible delay.
Anyway, the main question is why recv() is behaving differently; then if you had any suggestions to solve my task it would be really appreciated.
Here's a simple simulation of the real application.
Client:
import socket
BUFFER = 1024
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_address= ('localhost', 14000)
sock.connect(bridge_address)
msg1 = "function 1".encode()
msg2 = "function 2".encode()
results1 = " "
results2 = " "
sock.sendall(msg1)
while True:
data = sock.recv(BUFFER)
results1 += data.decode()
if not data:
print('no more data')
break
sock.sendall(msg2)
while True:
data = sock.recv(BUFFER)
results2 += data.decode()
if not data:
print('no more data')
break
sock.close()
print('Results 1: ',results1)
print('Results 2: ',results2)
Server:
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_address = ('localhost', 14000)
sock.bind(server_address)
sock.listen(1)
msg = ""
for i in range(4096):
msg += str(i) + " "
while True:
print('waiting for a connection')
client, client_address = sock.accept()
try:
while True:
data = client.recv(128)
if not data:
print('no more data from')
break
client.sendall(msg.encode())
finally:
print('connection closed.\n')
client.close()
You are explicitly forcing the server side socket to recieve the message from the client, even when the client is not sending anything.
data = client.recv(128)
if not data:
print('no more data from')
break
if case in the code snippet above will only execute when the client socket goes down.
One method to solve this would be to first send the size of the message and then call recv accordingly for the specified number of bytes.
Im sure there are easier ways with particular python modules, but for an assignment I need to create a program that can act as a client/server. As of right now I have it working to the point of only being able to send a message if the reciever has responded. I need it to just send and appear on the respective client/server terminal when enter is pressed. Any help would be greatly appreciated!
These are pictures of what happens as of now
https://i.stack.imgur.com/T9CsJ.png
import sys
import socket
import getopt
def usage(script_name):
print('Usage: py' + script_name + '-l' +' <port number>' + '[<server>]')
def sockObj():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
return sock
def serversockConn(serversocket,port):
serversocket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
# bind the serversocket to address
serversocket.bind(('',int(port)))
# enable a server to accept connections
serversocket.listen(5)
# wait for a connection and accept it
sock,addr = serversocket.accept()
return sock
def connToServersock(sock,server, port):
# connect to a serversocket
if server:
sock.connect((server, int(port)))
else:
sock.connect(('localhost', int(port)))
return sock
if __name__ == '__main__':
## get the command line arguments
try:
options, non_option_args = getopt.getopt(sys.argv[1:],'l')
except getopt.GetoptError as err:
print(err)
sys.exit(2)
#check if '-l' is present in command line agrument
serverSide = False
for opt,arg in options:
if opt == "-l":
serverSide = True
# port number
port = non_option_args[0]
# server address
server = None
hostLen = len(non_option_args)
if hostLen == 2:
server = non_option_args[1]
# create a communicator object, and make a connection between server and client
# server
if serverSide:
serversocket = sockObj()
sock = serversockConn(serversocket,port)
# client
else:
sock = sockObj()
sock = connToServersock(sock,server,port)
while True:
## read a message from standard input
message = sys.stdin.readline().encode()
if len(message) != 0:
sock.send(message)
return_msg = sock.recv( 1024 )
if return_msg:
print("Message recieved: " + return_msg.decode())
else:
print("Other side shut down")
else:
try:
sock.shutdown(socket.SHUT_WR)
sock.close()
except:
pass
I think your issue is that there are two places in your event loop where you block:
message = sys.stdin.readline().encode()
Here, you block until the user has pressed return -- during this time, your program is unable to respond to any data received over the network, because it is blocked waiting for data from stdin.
... and:
return_msg = sock.recv( 1024 )
Here, you are waiting for data to be received from the network -- during this time, your program is unable to respond to any data received from stdin, because it is blocked waiting for data from the network.
The behavior you'd ideally like to have is for your program to wait for both stdin and network traffic simultaneously -- i.e. have it block until either the user has pressed return, or some network data has been received, whichever comes first.
The easiest way to achieve that behavior is to use select(); its purpose is to block until at least one of several file descriptors is ready to be acted on. (Note, however, that Windows does not support using select() on stdin, so if your program needs to run under Windows you will probably have to spawn a second thread instead).
To implement the event loop using select(), add import select to the top of your script, then replace your event loop with something like this instead:
while True:
## block here until either sock or sys.stdin has data ready for us
readable, writable, exceptional = select.select([sock, sys.stdin], [], [])
if sys.stdin in readable:
## read a message from standard input
message = sys.stdin.readline().encode()
if len(message) != 0:
sock.send(message)
if sock in readable:
## read a message from the network
try:
return_msg = sock.recv( 1024 )
if (return_msg):
print("Message received: " + return_msg.decode())
else:
print("Other side shut down")
break
except:
print("recv() threw an exception")
break
I wrote a python client to communicate with server side. Each time when I finished sanding out data, I have to call sock.shutdown(socket.SHUT_WR), otherwise the server would not do any response. But after calling sock.shutdown(socket.SHUT_WR), I have to reconnect the connection as sock.connect((HOST, PORT)), other wise I can not send data to server. So how can I keep the connection alive without close it.
My sample code as following:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.sendall(data)
sock.shutdown(socket.SHUT_WR)
received = sock.recv(1024)
while len(received)>0:
received = sock.recv(1024)
sock.sendall(newdata) # this would throw exception
The Server Side code as following:
def handle(self):
cur_thread = threading.current_thread()
while True:
self.data = self.rfile.read(bufsiz=100)
if not self.data:
print 'receive none!'
break
try:
data = self.data
print 'Received data, length: %d' % len(data)
self.wfile.write('get received data\n')
except Exception:
print 'exception!!'
You didn't show any server side code but I suspect it simply reads bytes until it gets none anymore.
You can't do this as you found out, because then the only way to tell the server the message is complete is by killing the connection.
Instead you'll have to add some form of framing in your protocol. Possible approaches include a designated stop character that the server recognises (such as a single newline character, or perhaps a 0-byte), sending frames in fixed sizes that your client and server agree upon, or send the frame size first as a network encoded integer followed by exactly the specified number of bytes. The server then first reads the integer and then exactly that same number of bytes from the socket.
That way you can leave the connection open and send multiple messages.