Develop Reference

Python UDP socket receive - server.recvfrom package size longer than bufsize

I'm using the socket module for a UDP server。The incoming packets always have a different size(0-65535), so client send package length first, then send the package；server receives data according to the package length and 1024 per server receives。but server didn't work like I thought，it can receives package only once。
# -*- coding: utf-8 -*-
import json
import socket
ip_port = ('127.0.0.1', 8080)
server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server.bind(ip_port)
print('server listen......')
def recv_header(s_socket):
try:
msg, addr = s_socket.recvfrom(32)
return json.loads(msg), addr
except Exception as e:
return str(e), None
while True:
header_msg, client_addr = recv_header(s_socket=server)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
receiverBufsize = 1024
if client_addr is None:
print (header_msg)
continue
data_body = ""
package_len = int(header_msg['length'])
print ("package_len: {}".format(package_len))
print ("client_addr: {}".format(str(client_addr)))
while package_len > 0:
if package_len > receiverBufsize:
body_part = server.recvfrom(receiverBufsize)
else:
body_part = server.recvfrom(package_len)
data_body += body_part[0]
package_len -= receiverBufsize
print ("data_body: {}".format(data_body))
# -*- coding: utf-8 -*-
import json
import socket
import random
ip_port = ('127.0.0.1', 8080)
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def get_data(length):
base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789'
return ''.join([random.choice(base_str) for _ in range(length)])
while 1:
msg = raw_input('>>>：')
if msg == 'q':
break
if not msg:
continue
len = int(msg)
client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
data = get_data(len)
header = json.dumps({"length": len})
aa = client.sendto(header.encode('utf-8'), ip_port)
print aa
aa = client.sendto(data.encode('utf-8'), ip_port)
print aa
If the packet length is less than 1024(my receiverBufsize), it works normally
enter image description here
enter image description here
If the packet length is more than 1024(my receiverBufsize), socket.recvfrom will block
enter image description here
enter image description here

UDP is a message oriented protocol, not a data stream like TCP. This means in UDP a single send is matched by a single recv.
This means you cannot use multiple recv with a small buffer (size 1024 in your case) to get data which were sent in a single send with a larger buffer - you need to use recv with a large enough buffer instead.
This means also that you don't actually need to send the length upfront, since you don't need to add message semantics to a protocol which already has message semantic. In contrary, sending the length in a separate message as you do could actually be harmful since UDP is an unreliable protocol and you cannot rely on packets getting delivered in order, at all or only once.

How do I troubleshoot this error: OSError: [Errno 9] Bad file descriptor

I am trying to figure out where the problem is coming from between my client and server files. The client receives the correct calculation done by the TCP server. However, the TCP server continues to throw an error after performing the task.
add_server.py
# This is add_server.py script
import socket
host = socket.gethostname()
port = 8096
s = socket.socket()
s.bind((host, port))
s.listen(5)
print('Waiting for connection...')
conn, addr = s.accept()
while True:
data = conn.recv(1024) # Receive data in bytes
print(data)
decoded_data = data.decode('utf-8') # Decode data from bystes to string
print(data)
d = decoded_data.split(",") # Split the received string using ',' as separator and store in list 'd'
add_data = int(d[0]) + int(d[1]) # add the content after converting to 'int'
conn.sendall(str(add_data).encode('utf-8')) # Stringify results and convert to bytes for transmission (String conversion is a must)
conn.close() # Close connection
add_client.py
# This add_client.py script
import socket
host = socket.gethostname()
port = 8096
s = socket.socket()
s.connect((host, port))
a = input('Enter first number: ')
b = input('Enter second number: ')
c = a + ', ' + b # Generate string from numbers
print('Sending string {} to sever for processing...'.format(c))
s.sendall(c.encode('utf-8')) # Converst string to bytes for transmission
data = s.recv(1024).decode('utf-8') # Receive server response (addition results) and convert from bystes to string
print(data) # Convert 'string' data to 'int'
s.close() # close connection
Full traceback
Traceback (most recent call last):
File "/home/sharhan/AWS/AWS-PERSONAL-COURSES/linux-networking-and-troubleshooting/python-networking/add_server.py", line 17, in <module>
data = conn.recv(1024) # Receive data in bytes
OSError: [Errno 9] Bad file descriptor

You are closing the socket inside the while loop in this line
while True:
data = conn.recv(1024)
# Rest of the code
conn.close()
So the next time you try to receive the data with conn.recv it results in an error.
To fix this simply close the connection after you're done receiving all the data.
while True:
data = conn.recv(1024)
# Rest of the code
conn.close()

can't send pickle object from linux to windows through sockets

I can't send pickle object from raspberry with Linux through socket as a client to an online windows server while I can send it from my computer without any problem. The weird thing that I can send the message length (the server receive the message length ) but can't send the pickle object and the server is stuck to the while loop after receiving that message.
client code:
message = pickle.dumps(faceBlob)
message_header = bytes(f"{len(message):<{HEADER_LENGTH}}", "utf-8")
client_socket.send(message_header + message)
server code:
try:
# Receive our "header" containing message length, it's size is defined and constant
message_header = client_socket.recv(HEADER_LENGTH)
# If we received no data, client gracefully closed a connection, for example using socket.close() or socket.shutdown(socket.SHUT_RDWR)
if not len(message_header):
return False
# Convert header to int value
message_length = int(message_header.decode('utf-8').strip())
fragments = []
print(message_length)
received_len = 0
while received_len < message_length:
chunk = client_socket.recv(message_length - received_len)
if not chunk:
break
fragments.append(chunk)
received_len += len(chunk)
data_arr = b"".join(fragments)
# Return an object of message header and message data
return {'header': message_header, 'data': data_arr}
except:
return False

Socket server not pausing on recv(). Immediately reads 0 bytes and closes

I have a server-client setup that works as follows:
The client connects to the server.
The client sends the server a 64-byte message telling the server how much data to read
The server reads that many bytes of data, responds, and the process repeats.
When the client is finished, it sends the server a null message
The server sees that the message length is 0 and closes the connection.
This seems to work fine for the first pass. After the server responds though, it doesn't wait for the client to send more data. Instead the server immediately reads 64 bytes. Since the client hasn't responded, the length of the message is 0 and the connection is closed.
I'm unsure why the server is not pausing until the client sends more data.
Here is the server loop:
self.__stop = False
while not self.__stop:
if self.client_sock:
# Check if the client is still connected and if data is available
try:
rdy_read, rdy_write, sock_err = select.select(
[self.client_sock, ], [self.client_sock, ], [], 5)
except select.error as err:
self.stop()
return
if len(rdy_read) > 0:
# msg length will be sent as bytes
read_data = self.client_sock.recv(64)
# Check if the socket has been closed
if read_data == 0:
self.stop()
else:
msg_length = int(read_data)
msg = self.client_sock.recv(msg_length)
response = f"{[12, 15, 66]}\n"
msg_size = padded_size_of_msg(response)
self.client_sock.send(msg_size)
self.client_sock.send(f"{response}".encode('utf-8'))
else:
print(
f"[THREAD {self.number}] No client connected.")
self.stop()
self.close()
The function padded_size_of_msg() is to calculate the length of the message, pad that number to be 64-bytes, then send that to the client:
def padded_size_of_msg(msg):
msg_length = len(msg)
send_length = str(msg_length).encode('utf-8')
send_length += b' ' * (64- len(send_length))
return send_length
The complete class declaration is below:
class ServerSocketThread(Thread):
def __init__(self, client_sock, client_addr, number):
Thread.__init__(self)
self.client_sock = client_sock
self.client_addr = client_addr
self.number = number
def run(self):
self.__stop = False
while not self.__stop:
if self.client_sock:
# Check if the client is still connected and if data is available
try:
rdy_read, rdy_write, sock_err = select.select(
[self.client_sock, ], [self.client_sock, ], [], 5)
except select.error as err:
print(
f"[THREAD {self.number}] Select() failed on socket with {self.client_addr}")
self.stop()
return
if len(rdy_read) > 0:
# msg length will be sent as bytes
read_data = self.client_sock.recv(64)
# Check if the socket has been closed
if read_data == 0:
print(
f"[THREAD {self.number}] {self.client_addr} closed the socket")
self.stop()
else:
msg_length = int(read_data)
# Client will send msg as bytes. No need to decode to str
msg = self.client_sock.recv(msg_length)
response = f"{[12, 15, 66]}\n"
# Send outputs to client as bytes
msg_size = padded_size_of_msg(response)
self.client_sock.send(msg_size)
self.client_sock.send(f"{response}".encode('utf-8'))
else:
print(
f"[THREAD {self.number}] No client connected.")
self.stop()
self.close()
def stop(self):
self.__stop = True
def close(self):
if self.client_sock:
print(
f"[THREAD {self.number}] Closing conn with {self.client_addr}")
self.client_sock.close()

This ended up being an issue with using nc from the command line.
If I were to try and send "test" from the client, it would register the size as 4-bytes and tell the server to read 4-bytes. From the terminal, nc would instead send "test\n". The newline character '\n' would not be read and instead waited in the queue. When the second recv() was called, it immediately read the '\n' and took that as an indication to close the connection.

Get a response fro server

I'm trying to make a server-client program, where server will listen to client's messages and depends on the message, will response. I send a message from client with username and content, server accept it and print a message to sending to client Till here is everything fine. But when it comes to sending a message server will throw and error:
`TypeError: byte indices must be integers or slices, not str`
It looks like this line is the problem, but I'm not sure....
`clientsocket.send(msg['header'] + msg['data'])`
here is a whole server code. Please let me know, if client code is necessary too please.
import socket
import time
import pickle
import select
HEADERSIZE = 10
IP = "127.0.0.1"
PORT = 1234
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((IP, PORT))
s.listen()
sockets_list = [s]
clients = {}
# Handles message receiving
def receive_message(clientsocket):
try:
message_header = clientsocket.recv(HEADERSIZE)
if not len(message_header):
return False
message_length = int(message_header.decode('utf-8').strip())
return {'header': message_header, 'data': clientsocket.recv(message_length)}
except:
return False
while True:
read_sockets, _, exception_socket = select.select(sockets_list, [], sockets_list)
for notified_socket in read_sockets:
if notified_socket == s:
clientsocket, address = s.accept()
user = receive_message(clientsocket)
if user is False:
continue
sockets_list.append(clientsocket)
clients[clientsocket] = user
print(f"Connection from {address[0]}:{address[1]} has been estabilished! User:{user['data'].decode('utf-8')}")
else:
message = receive_message(notified_socket)
if message is False:
print(f"Close connection from {clients[notified_socket]['data'].decode('utf-8')}")
sockets_list.remove(notified_socket)
del clients[notified_socket]
continue
user = clients[notified_socket]
#message_decoded = message['data'].decode('utf-8')
print(f'Received message from {user["data"].decode("utf-8")}: {message["data"].decode("utf-8")}')
for clientsocket in clients:
if clientsocket == notified_socket:
if message["data"].decode("utf-8") == "y":
#d = {1: "Hey", 2: "there"}
msg = pickle.dumps("th.jpeg")
print(msg)
msg = bytes(f'{len(msg):<{HEADERSIZE}}', "utf-8") + msg
clientsocket.send(msg['header'] + msg['data'])
else:
d = {1: "Hey", 2: "there"}
msg = pickle.dumps(d)
print(msg)
# msg = bytes(f'{len(msg):<{HEADERSIZE}}', "utf-8") + msg
clientsocket.send(msg['header'] + msg['data'])
for notified_socket in exception_socket:
sockets_list.remove(notified_socket)
del clients[notified_socket]
HERE is a whole error code:
Connection from 127.0.0.1:48480 has been estabilished! User:j
Received message from j: y
b'\x80\x03X\x07\x00\x00\x00th.jpegq\x00.'
Traceback (most recent call last):
File "server.py", line 69, in <module>
clientsocket.send(msg['header'] + msg['data'])
TypeError: byte indices must be integers or slices, not str
As You can see, it works till sending the message line

msg = pickle.dumps("th.jpeg") will encode the string "th.jpeg" as a bytes-object.
msg = bytes(f'{len(msg):<{HEADERSIZE}}', "utf-8") + msg just adds that bytes object to another bytes-object.
So msg is a simple bytes-object, not any kind of server packet or similar. Therefor it is not possible to subscribe msg with msg['header'] or any other string.
Your code seems a little weird but maybe just try this line:
clientsocket.send(msg)
Since you are already converting msg to a bytes-object, it can be sent to the client directly. You just have to decode it properly in the client.