Develop Reference

Python socket - struct.error: unpack requires a buffer of 4 bytes

I wrote a basic socket program where the client sends an image to a server.
client.py
import socket
from PIL import ImageFile
import struct
ImageFile.LOAD_TRUNCATED_IMAGES = True
HOST = "127.0.0.1" # The server's hostname or IP address
PORT = 65432 # The port used by the server
img_byte = None
with open('test-images/hummingbird-1.jpeg', 'rb') as fp:
img_byte = fp.read()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST, PORT))
length = len(img_byte)
s.sendall(struct.pack('!I', length))
s.sendall(img_byte)
, and server.py
import socket
import struct
HOST = "127.0.0.1" # Standard loopback interface address (localhost)
PORT = 65432 # Port to listen on (non-privileged ports are > 1023)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind((HOST, PORT))
s.listen()
conn, addr = s.accept()
with conn:
print(f"Connected by {addr}")
while True:
buf = conn.recv(4)
(length, ) = struct.unpack('!I', buf)
data = b''
while len(data) < length:
to_read = length - len(data)
data += conn.recv(4096 if to_read > 4096 else to_read)
if not data:
break
On ther server side, I'm getting this error:
(length, ) = struct.unpack('!I', buf)
struct.error: unpack requires a buffer of 4 bytes
I think I'm missing something decisive, but I can't figure it out.

Based on the suggestion by #Mark Tolonen, I was able to modify and run server.py as follows:
import socket
import struct
HOST = "127.0.0.1" # Standard loopback interface address (localhost)
PORT = 65432 # Port to listen on (non-privileged ports are > 1023)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind((HOST, PORT))
s.listen()
conn, addr = s.accept()
with conn:
print(f"Connected by {addr}")
data = b''
buf = b''
payload_size = struct.calcsize("!I")
print("payload_size: {}".format(payload_size))
while True:
while len(buf) < payload_size:
buf += conn.recv(4096)
length = struct.unpack('!I', buf[:payload_size])[0]
data = buf[payload_size:]
while len(data) < length:
to_read = length - len(data)
data += conn.recv(4096 if to_read > 4096 else to_read)
if not data:
break
file = open('rec-images/image.jpeg', 'wb')
file.write(data)
file.close()

Why I have some packets lost in localhost?

I try to develop a server and a client programs in Python 2.7, which can switch in UDP or in TCP, based on this echo program :
TCP vs. UDP socket latency benchmark
For now, i just try to program it as local host
When i run it in TCP (is_UDP = False), the server program shows me that there is no packet lost (total_perdu = 0)
But if i run it in UDP (is_UDP = True), it gives me some packets lost.
This is my code for the server :
import socket
from numpy import *
server_address = ("127.0.0.1", 4444)
client_address = ("127.0.0.1", 4445)
bufferSize = 4096
# is_UDP = True
is_UDP = False
# Create a datagram socket
if is_UDP == True:
UDP_Server_Socket_in = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
UDP_Server_Socket_in.bind(server_address)
UDP_Server_Socket_out = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
UDP_Server_Socket_out.connect(client_address)
connection = UDP_Server_Socket_in
print("UDP server is running...")
else :
TCP_Server_Socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
TCP_Server_Socket.bind(server_address)
TCP_Server_Socket.listen(1)
connection, client_address = TCP_Server_Socket.accept()
print("TCP server is running...")
t = 0
total_perdu = 0
i = 0
while(True):
i += 1
# Receive packet from client
data_2= connection.recv(bufferSize)
tab=fromstring(data_2,dtype="int32")
size=len(data_2)
while size<bufferSize:
data_2= connection.recv(bufferSize-size)
size+=len(data_2)
if data_2:
perdu=int(tab[0])-t-1
sperdu=""
if perdu>0:
total_perdu+=perdu
sperdu = "(%d)"%(perdu)
print "Receive data : %s %d %d %s" % (tab[0], len(tab), total_perdu,sperdu)
t=int(tab[0])
And this is my code for the client:
import socket
from numpy import *
import time
server_address = ("127.0.0.1", 4444)
client_address = ("127.0.0.1", 4445)
# Packets variables
packet_size = 1024
total_packet = 1000
bufferSize = 4*packet_size
# Variables initialization
error = 0
total_throughput = 0
total_latene = 0
total_ratio = 0
total_stop_time_1 = 0
total_stop_time_3 = 0
# Creation of a packet
send_tab = zeros(packet_size, int)
for i in range(0, packet_size):
send_tab[i] = i
data_size = (send_tab.size+8)*send_tab.itemsize
print "Data size : %d" % data_size
print "Tab : %s \n" % send_tab
# is_UDP = True
is_UDP = False
# Create a socket at client side
if is_UDP == True:
UDP_Client_Socket_out = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
UDP_Client_Socket_out.connect(server_address)
UDP_Client_Socket_in = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
UDP_Client_Socket_in.bind(client_address)
connection = UDP_Client_Socket_out
print("UDP client is running...")
else:
TCP_Client_Socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
TCP_Client_Socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 0)
TCP_Client_Socket.connect(server_address)
connection = TCP_Client_Socket
print("TCP client is running...")
start_time_0 = time.clock()
for packet_number in range(0,total_packet):
send_tab[0] = packet_number
# Send packet to server
start_time=time.clock()
sent = connection.send(send_tab)
if sent:
stop_time_1 = time.clock() - start_time
# Calculate throughput and ratio
throughput = data_size / (stop_time_1 * 1000000)
print "stop_time_1 \t%f" % stop_time_1
total_throughput += throughput
stop_time_3 = (time.clock() - start_time_0)
print "Results : \n"
print " Packet error : %d \n" % error
print " Thoughput: %f Mo/s \n " % (total_throughput/total_packet)
print " total_stop_time_1 : %f s \n " % (total_stop_time_1/total_packet)
print " stop_time_3 : %f \n" % stop_time_3
So, I have 3 questions about it :
Is it normal to have some packets which are lost even if I do it as a local host?
If yes, why?
Will I have the same problem if I program it in C?

From your code it looks like you expect to receive the UDP packets in the same order you are sending them. I don't think you are loosing packets, but rather the order in which they are received by the server is not the expected one (which is normal for UDP).
Also, you should take into consideration that UDP does not guarantee neither the order or the receiving of packets so your program should take this into consideration.
I would refactor the code and add the tab into a list, then sort it and check for the gaps (at the end of transmission).
Another way would be to send a reply from the server and make the check on the client (but this might increase the number if you deploy it on the Internet).

Python socket.error: [Errno 104] Connection reset by peer

I have a code where there are 13 clients that have to connect to the server. Then the server does some counting on the data given by the client. After that the roles turns around - the server becomes a client and clients become servers to receive the data.
The thing is that when trying to do the first connection, that is when the 13 clients try to connect to the server I keep getting this error: [Errno 104] Connection reset by peer. I tried some workarounds for example trying to connect 5 times in a second interval but nothing works.
Here my code:
server.py
import socket, pickle, numpy as np
import struct
import math
while 1:
HOST = ''
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(13)
adresses = []
ports = []
i = 0
print("receiving...")
while i < 13:
i += 1
#wait to accept a connection - blocking call
conn, addr = s.accept()
print ('Connected with ', addr)
adresses.append(addr[0])
buf = b''
while len(buf) < 4:
buf += conn.recv(4 - len(buf))
length = struct.unpack('>I', buf)[0]
data = b''
l = length
while l > 0:
d = conn.recv(l)
l -= len(d)
data += d
if not data: break
M = np.loads(data)
if i == 1:
L = M[0]
else:
L += M[0]
ports.append(M[1])
conn.close()
s.close()
L /= 993040
packet = pickle.dumps(L)
length = struct.pack('>I', len(packet))
packet = length + packet
print("sending...")
for kl, addr in enumerate(adresses):
HOST = addr
PORT = 50007 + ports[kl]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.sendall(packet)
s.close()
client.py
def connection(centers, kl):
HOST = "192.168.143.XX"
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(3600)
try:
s.connect((HOST, PORT)) # HERE IS AN ERROR
s.settimeout(None)
packet = pickle.dumps([centers, kl]) ## ???
length = struct.pack('>I', len(packet))
packet = length + packet
s.sendall(packet) # OR HERE IS AN ERROR
s.close()
except Exception as e:
print(e)
print('error ', kl)
s.close()
return np.zeros(centers.shape)
HOST = ''
PORT = 50007 + kl
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(2)
i = 0
while i < 1:
#wait to accept a connection - blocking call
conn, addr = s.accept()
i += 1
print ('Connected with ', addr)
buf = b''
while len(buf) < 4:
buf += conn.recv(4 - len(buf))
length = struct.unpack('>I', buf)[0]
data = b''
l = length
while l > 0:
d = conn.recv(l)
l -= len(d)
data += d
if not data: break
new_centers = np.loads(data)
conn.close()
s.close()
return new_centers
aa = 0
for k in range(99):
print(k)
centers = some_function(centers)
time.sleep(60)
centers1 = connection(centers, i)
aa = 0
while not (centers1.any()) and aa < 5:
time.sleep(1)
centers1 = connection(centers, i)
aa += 1
centers = centers1
The thing is all of the 13 clients HAVE TO connect to the server or it won't proceed to the next iteration.
I'm using Python 3.4.
Please help.
Update:
I have added threads but the error remains:
[Errno 104] Connection reset by peer
server.py
import socket, pickle, numpy as np
import struct
import math
from multiprocessing.pool import ThreadPool
def clientthread(conn, L):
buf = b''
while len(buf) < 4:
buf += conn.recv(4 - len(buf))
length = struct.unpack('>I', buf)[0]
data = b''
l = length
while l > 0:
d = conn.recv(l)
l -= len(d)
data += d
M = np.loads(data)
return(M)
j = 0
while 1:
HOST = ''
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(2)
#print('0')
adresses = []
ports = []
i = 0
print("receiving...")
while i < 13:
i += 1
#wait to accept a connection - blocking call
conn, addr = s.accept()
print ('Connected with ', addr)
adresses.append(addr[0])
pool = ThreadPool(processes=13)
async_result = pool.apply_async(clientthread, (conn, i,))
M = async_result.get()
conn.close()
if i == 1:
L = M[0]
else:
L += M[0]
ports.append(M[1])
s.close()
L /= 993040
packet = pickle.dumps(L)
length = struct.pack('>I', len(packet))
packet = length + packet
for kl, addr in enumerate(adresses):
HOST = addr
PORT = 50007 + ports[kl]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.sendall(packet)
s.close()

It seems that the clients were connected to the server but they encountered with " [Errno 104] Connection reset by peer" exception when they tried to send data. For the first time, Python raises "[Errno 104] Connection reset by peer" exception, then for the second time and more you would get "[Errno 32] Broken pipe" exception on the client side.
This can mean that the server is up and listening on the port (otherwise, you would get "[Errno 111] Connection refused" exception on the client side". This also means that the server is crashed before closing the connection since if the connection was closed on the server side before sending data on the client side, the client would encounter with "[Errno 32] Broken pipe" exception.
"Connection reset by peer" is the TCP/IP equivalent of slamming the phone back on the hook. It's more polite than merely not replying, leaving one hanging. But it's not the FIN-ACK expected of the truly polite TCP/IP converseur. (From another stackoverflow answer)

You are going to have to use threading on the server.
For a simple explanation see: http://www.binarytides.com/python-socket-server-code-example/
The guts of the simple example given there is:
import socket
import sys
from thread import *
HOST = '' # Symbolic name meaning all available interfaces
PORT = 8888 # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print 'Socket created'
#Bind socket to local host and port
try:
s.bind((HOST, PORT))
except socket.error as msg:
print 'Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1]
sys.exit()
print 'Socket bind complete'
#Start listening on socket
s.listen(10)
print 'Socket now listening'
#Function for handling connections. This will be used to create threads
def clientthread(conn):
#Sending message to connected client
conn.send('Welcome to the server. Type something and hit enter\n') #send only takes string
#infinite loop so that function do not terminate and thread do not end.
while True:
#Receiving from client
data = conn.recv(1024)
reply = 'OK...' + data
if not data:
break
conn.sendall(reply)
#came out of loop
conn.close()
#now keep talking with the client
while 1:
#wait to accept a connection - blocking call
conn, addr = s.accept()
print 'Connected with ' + addr[0] + ':' + str(addr[1])
#start new thread takes 1st argument as a function name to be run, second is the tuple of arguments to the function.
start_new_thread(clientthread ,(conn,))
s.close()

I had this exact error when trying to connect to a remote redis instance from python, because I had mistakenly left Fiddler running and it was interfering with the requests.

Socket Programming; File corrupts when transferring over multiple devices

The problem I'm having is to get a file from the server to client across devices. Everything works fine on localhost.
Lets say I want to "get ./testing.pdf" which sends the pdf from the server to the client. It sends but it is always missing bytes. Is there any problems with how I am sending the data. If so how can I fix it? I left out the code for my other functionalities since they are not used for this function.
sending a txt file with "hello" in it works perfectly
server.py
import socket, os, subprocess # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
#host = ''
port = 5000 # Reserve a port for your service.
bufsize = 4096
s.bind((host, port)) # Bind to the port
s.listen(5) # Now wait for client connection.
while True:
c, addr = s.accept() # Establish connection with client.
print 'Got connection from', addr
while True:
userInput = c.recv(1024)
.... CODE ABOUT OTHER FUNCTIONALITY
elif userInput.split(" ")[0] == "get":
print "inputed get"
somefile = userInput.split(" ")[1]
size = os.stat(somefile).st_size
print size
c.send(str(size))
bytes = open(somefile).read()
c.send(bytes)
print c.recv(1024)
c.close()
client.py
import socket, os # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
#host = '192.168.0.18'
port = 5000 # Reserve a port for your service.
bufsize = 1
s.connect((host, port))
print s.recv(1024)
print "Welcome to the server :)"
while 1 < 2:
userInput = raw_input()
.... CODE ABOUT OTHER FUNCTIONALITY
elif userInput.split(" ")[0] == "get":
print "inputed get"
s.send(userInput)
fName = os.path.basename(userInput.split(" ")[1])
myfile = open(fName, 'w')
size = s.recv(1024)
size = int(size)
data = ""
while True:
data += s.recv(bufsize)
size -= bufsize
if size < 0: break
print 'writing file .... %d' % size
myfile = open('Testing.pdf', 'w')
myfile.write(data)
myfile.close()
s.send('success')
s.close

I can see two problems right away. I don't know if these are the problems you are having, but they are problems. Both of them relate to the fact that TCP is a byte stream, not a packet stream. That is, recv calls do not necessarily match one-for-one with the send calls.
size = s.recv(1024) It is possible that this recv could return only some of the size digits. It is also possible that this recv could return all of the size digits plus some of the data. I'll leave it for you to fix this case.
data += s.recv(bufsize) / size -= bufsize There is no guarantee that that the recv call returns bufsize bytes. It may return a buffer much smaller than bufsize. The fix for this case is simple: datum = s.recv(bufsize) / size -= len(datum) / data += datum.

Cannot Get Strings Across in a Socket Program

I have been looking at some code for a small chat program that I found online. It was originally written for 2.7, but it seems to work with 3.2. The only problem is that I cannot send strings, only numbers:
The chat.py file source code:
from socket import *
HOST = ''
PORT = 8000
s = socket(AF_INET, SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print ('Connected by ' + str(addr))
i = True
while i is True:
data = conn.recv(1024)
print ("Received " + repr(data))
reply = str(input("Reply: "))
conn.send(reply)
conn.close()
And the client.py source file:
from socket import *
HOST = ''
PORT = 8000
s = socket(AF_INET, SOCK_STREAM)
s.connect((HOST, PORT)) # client-side, connects to a host
while True:
message = str(input("Your Message: "))
s.send(message)
print ("Awaiting reply...")
reply = s.recv(1024) # 1024 is max data that can be received
print ("Received " + repr(reply))
s.close()
When I run these using two separate terminals, they work, but do not send strings.
Thank you

When you work with sockets, the message you're passing around should probably be in bytes, b'bytes'. In Python 2.x, a str is actually what a bytes is in Python 3.x
So your message should be something like:
message = b'Message I want to pass'
Check here http://docs.python.org/3.3/library/stdtypes.html for more information.
According to http://docs.python.org/3/library/functions.html#input input returns a str, which means you'll have to encode message into bytes as such:
message = message.encode()
Do verify that this is the right approach to convert str to bytes by checking the type of message.

Your socket code is correct, it was just failing due to an unrelated error due to raw_input vs input. You probably intended to read a string from the shell instead of reading a string and trying to evaluate it as Python code which is what input does.
Try this instead:
chat.py
from socket import *
HOST = ''
PORT = 8000
s = socket(AF_INET, SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print ('Connected by ' + str(addr))
i = True
while i is True:
data = conn.recv(1024)
print ("Received " + repr(data))
reply = str(raw_input("Reply: "))
conn.send(reply)
conn.close()
client.py
from socket import *
HOST = ''
PORT = 8000
s = socket(AF_INET, SOCK_STREAM)
s.connect((HOST, PORT)) # client-side, connects to a host
while True:
message = str(raw_input("Your Message: "))
s.send(message)
print ("Awaiting reply...")
reply = s.recv(1024) # 1024 is max data that can be received
print ("Received " + repr(reply))
s.close()