Develop Reference

What is the proper way of sending a large amount of data over sockets in Python?

Recently I wrote some code (client and server) to send an image - the client simply uploads the image to the server, just using the socket module: Sending image over sockets (ONLY) in Python, image can not be open.
However, the image sending part is now what I am concerned with. This is the original image I'm using:
In my server code (which receives the images), I have these lines:
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
data = sock.recv(40960000)
if not data:
myfile.close()
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
But I don't think this is the best way of doing it. I think I should instead implement the server such that it receives the data in chunks:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = 0
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()
But when I do this, the server prints:
got size 54674
4096
8192
12288
16384
20480
24576
28672
32768
36864
40960
45056
49152
50578
And then seems to hang, and the client hangs too. However, at the server's side I can see only a piece of the image the client wanted to send:
It seems like there are some bytes missing. What is the proper way of sending a huge amount of data (images, other type of file) using ONLY sockets?

I'm assuming that you have a particular reason for doing this with naked sockets, such as self-edification, which means that I won't answer by saying "You accidentally forgot to just use HTTP and Twisted", which perhaps you've heard before :-P. But really you should look at higher-level libraries at some point as they're a lot easier!
Define a protocol
If all you want is to send an image, then it can be simple:
Client -> server: 8 bytes: big endian, length of image.
Client -> server: length bytes: all image data.
(Client <- server: 1 byte, value 0: indicate transmission received - optional step you may not care if you're using TCP and just assume that it's reliable.)
Code it
server.py
import os
from socket import *
from struct import unpack
class ServerProtocol:
def __init__(self):
self.socket = None
self.output_dir = '.'
self.file_num = 1
def listen(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.bind((server_ip, server_port))
self.socket.listen(1)
def handle_images(self):
try:
while True:
(connection, addr) = self.socket.accept()
try:
bs = connection.recv(8)
(length,) = unpack('>Q', bs)
data = b''
while len(data) < length:
# doing it in batches is generally better than trying
# to do it all in one go, so I believe.
to_read = length - len(data)
data += connection.recv(
4096 if to_read > 4096 else to_read)
# send our 0 ack
assert len(b'\00') == 1
connection.sendall(b'\00')
finally:
connection.shutdown(SHUT_WR)
connection.close()
with open(os.path.join(
self.output_dir, '%06d.jpg' % self.file_num), 'w'
) as fp:
fp.write(data)
self.file_num += 1
finally:
self.close()
def close(self):
self.socket.close()
self.socket = None
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
sp = ServerProtocol()
sp.listen('127.0.0.1', 55555)
sp.handle_images()
client.py
from socket import *
from struct import pack
class ClientProtocol:
def __init__(self):
self.socket = None
def connect(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.connect((server_ip, server_port))
def close(self):
self.socket.shutdown(SHUT_WR)
self.socket.close()
self.socket = None
def send_image(self, image_data):
# use struct to make sure we have a consistent endianness on the length
length = pack('>Q', len(image_data))
# sendall to make sure it blocks if there's back-pressure on the socket
self.socket.sendall(length)
self.socket.sendall(image_data)
ack = self.socket.recv(1)
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
cp = ClientProtocol()
image_data = None
with open('IMG_0077.jpg', 'r') as fp:
image_data = fp.read()
assert(len(image_data))
cp.connect('127.0.0.1', 55555)
cp.send_image(image_data)
cp.close()

A simple way is to send data size as the first 4 bytes of your data and then read complete data in one shot. Use the below functions on both client and server-side to send and receive data.
def send_data(conn, data):
serialized_data = pickle.dumps(data)
conn.sendall(struct.pack('>I', len(serialized_data)))
conn.sendall(serialized_data)
def receive_data(conn):
data_size = struct.unpack('>I', conn.recv(4))[0]
received_payload = b""
reamining_payload_size = data_size
while reamining_payload_size != 0:
received_payload += conn.recv(reamining_payload_size)
reamining_payload_size = data_size - len(received_payload)
data = pickle.loads(received_payload)
return data
you could find sample program at https://github.com/vijendra1125/Python-Socket-Programming.git

The problem is you are not incrementing amount_received for the first chunk of the data received.
Fix below:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = len(data) # The fix!
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()

print contents of file - python socket

I'm trying to print the contents of a file to the client using the defined command 'get'. I am not getting the contents of the file instead the contents are getting over written with Server Says... get test.txt.
Here is my client code:
import socket
import sys
import os
HOST = 'localhost'
PORT = 8082
size = 1024
def ls():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST,PORT))
s.send(userInput)
result = s.recv(size)
print result
s.close()
return
def put(commandName):
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName)
string = commandName.split(' ', 1)
inputFile = string[1]
with open(inputFile, 'rb') as file_to_send:
for data in file_to_send:
socket1.sendall(data)
print 'PUT Successful'
socket1.close()
return
def get(commandName):
socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))
socket1.send(commandName)
string = commandName.split(' ', 1)
inputFile = string[1]
with open(inputFile, 'wb') as file_to_write:
while True:
data = socket1.recv(1024)
print data
if not data:
break
print data
file_to_write.write(data)
file_to_write.close()
print 'GET Successful'
#socket1.close()
return
done = False
while not done:
userInput = raw_input()
if "quit" == userInput:
done = True
elif "ls" == userInput:
ls()
else:
string = userInput.split(' ', 1)
if (string[0] == 'put'):
put(userInput)
elif (string[0] == 'get'):
get(userInput)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST,PORT))
s.send(userInput)
data = s.recv(size)
s.close()
print 'Received:', data
And server code:
import socket
import os
import sys
host = ''
port = 8082
backlog = 5
size = 1024
serverID = socket.gethostbyname(socket.gethostname())
info = 'SERVER ID: {} port: {}'.format(serverID, port)
print info
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(backlog)
done = False
# Loop until client sends 'quit' to server
while not done:
client, address = s.accept()
data = client.recv(size)
print "Server received: ", data
if data:
client.send("Server Says... " + data)
if data == "quit":
done = True
elif data == "ls":
data = os.listdir("C://")
client.send(data[0])
else:
string = data.split(' ', 1)
dataFile = string[1]
if (string[0] == 'put'):
with open(dataFile, 'wb') as file_to_write:
while True:
data = client.recv(1024)
if not data:
break
file_to_write.write(data)
file_to_write.close()
break
print 'Receive Successful'
elif (string[0] == 'get'):
with open(dataFile, 'rb') as file_to_send:
for data in file_to_send:
client.send(data)
print 'Send Successful'
client.close()
s.close()
print "Server exiting."

You are getting the content of the file but it is empty... and your code works just because you probably run it in the same directory.
Your client open file to write (and then it is empty) and the server reads it (both code use the same file name). You probably wanted to read file from C:\ so you should change server code and replace line:
with open(dataFile, 'rb') as file_to_send:
with:
with open('C:\\' + dataFile, 'rb') as file_to_send:

No matter what you send to the server this condition is true so you get "server says..."
if data:
client.send("Server Says... " + data)
You have also several flaws in you client I think. Like your commands are sent 2 times: once in the client 'main' when you do:
s.send(userInput)
And once in the functions like get and ls :
socket1.send(commandName)

Why does the file not transfer completely? Python Socket Programming

The problem I'm having is to get a file from the server. Lets say I want to
"get ./testing.pdf" which sends the pdf from the server to the client. It sends but it is always missing bytes. Is there any problems with how I am sending the data. If so how can I fix it? I left out the code for my other functionalities since they are not used for this function.
server.py
import socket, os, subprocess # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
#host = ''
port = 5000 # Reserve a port for your service.
bufsize = 4096
s.bind((host, port)) # Bind to the port
s.listen(5) # Now wait for client connection.
while True:
c, addr = s.accept() # Establish connection with client.
print 'Got connection from', addr
while True:
userInput = c.recv(1024)
.... CODE ABOUT OTHER FUNCTIONALITY
elif userInput.split(" ")[0] == "get":
print "inputed get"
somefile = userInput.split(" ")[1]
size = os.stat(somefile).st_size
print size
c.send(str(size))
bytes = open(somefile).read()
c.send(bytes)
print c.recv(1024)
c.close()
client.py
import socket, os # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
#host = '192.168.0.18'
port = 5000 # Reserve a port for your service.
bufsize = 1
s.connect((host, port))
print s.recv(1024)
print "Welcome to the server :)"
while 1 < 2:
userInput = raw_input()
.... CODE ABOUT OTHER FUNCTIONALITY
elif userInput.split(" ")[0] == "get":
print "inputed get"
s.send(userInput)
fName = os.path.basename(userInput.split(" ")[1])
myfile = open(fName, 'w')
size = s.recv(1024)
size = int(size)
data = ""
while True:
data += s.recv(bufsize)
size -= bufsize
if size < 0: break
print 'writing file .... %d' % size
myfile = open('Testing.pdf', 'w')
myfile.write(data)
myfile.close()
s.send('success')
s.close

Python socket server receive image

I am trying to receive an image from Android to PC using socket in Python. My server code is as follows:
import socket
address = ("10.0.0.12", 5000)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(address)
s.listen(1000)
client, addr = s.accept()
print 'got connected from', addr
filename = open('tst.jpg', 'wb')
while True:
strng = client.recv(1024)
if not strng:
break
filename.write(strng)
filename.close()
print 'received, yay!'
client.close()
And it returns me a tst.jpg which is the same size of that on my Android. But I cannot open the pic.
Here is my Android code:
Socket photoSocket = new Socket(ipString, port);
DataOutputStream dos = new DataOutputStream(photoSocket.getOutputStream());
FileInputStream fis = new FileInputStream(PhotoActivity.filePath);
int size = fis.available();
byte[] data = new byte[size];
fis.read(data);
dos.writeInt(size);
dos.write(data);
dos.flush();
dos.close();
fis.close();
photoSocket.close();

so the android code is sending the size as int in front of the data, but the python code doesn't read it.
import socket
import struct
address = ("10.0.0.12", 5000)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(address)
s.listen(1000)
client, addr = s.accept()
print 'got connected from', addr
buf = ''
while len(buf)<4:
buf += client.recv(4-len(buf))
size = struct.unpack('!i', buf)
print "receiving %s bytes" % size
with open('tst.jpg', 'wb') as img:
while True:
data = client.recv(1024)
if not data:
break
img.write(data)
print 'received, yay!'
client.close()

Python file transfer only works in localhost and fails between two machines

I have written following python code for file transfer. It works fine in localhost environment. It fails between two different physical machines. I get the file but with incomplete data. There is also invalid literal error while converting string to long while sending file size to client. I can't figure out why?
server.py
from socket import *
import thread
import os
l = {}
def handler(clientsocket, clientaddr):
print "Accepted connection from: ", clientaddr
while 1:
data = clientsocket.recv(8192)
if not data:
break
else:
print "The following data was received - ",data
l[clientaddr] = data
print l
print "Opening file - ",data
fp = open(data,'r')
size = os.path.getsize(data)
clientsocket.send(str(size))
strng = "hi"
print size
while size > 0:
strng = fp.read(8192)
clientsocket.send (strng)
size = size - 8192
clientsocket.close()
if __name__ == "__main__":
host = 'localhost'
port = 55573
buf = 8192
addr = (host, port)
serversocket = socket(AF_INET, SOCK_STREAM)
serversocket.bind(addr)
serversocket.listen(5)
while 1:
print "Server is listening for connections\n"
clientsocket, clientaddr = serversocket.accept()
thread.start_new_thread(handler, (clientsocket, clientaddr))
serversocket.close()
client.py
from socket import *
import os
if __name__ == '__main__':
host = '10.1.99.176'
port = 55573
buf = 8192
addr = (host, port)
clientsocket = socket(AF_INET, SOCK_STREAM)
clientsocket.connect(addr)
while 1:
fname = raw_input("Enter the file name that u want>> ")
if not fname:
break
else:
clientsocket.send(fname)
print "\nThe file will be saved and opened- "
fname = '/home/coep/Downloads/'+fname
nf = open(fname,"a")
strng = "hi"
size = clientsocket.recv(16)
size = long(size)
print size
while size > 0:
strng = clientsocket.recv(8192)
if not strng:
break
nf.write(strng)
size = size - 8192
if size > 500000:
print size
nf.close()
fname = 'viewnior '+ fname
print fname
os.system(fname)

In server.py, you are using
host = 'localhost' and it will just bind the port to localhost Ip address i.e. 127.0.0.1.
Change it to host='0.0.0.0' and it will bind the specific port to all available interfaces.
Update: Another reason can be that buffer size is just too big and with 8192, it may get blocked forever because last transfer was never able to fill the buffer. To get around it, set the timeout to makesure to proceed if the buffer doesn't fill up. In client.py, try changing,
clientsocket.timeout(5)
while size > 0:
try:
strng = clientsocket.recv(8192)
if not strng:
break
nf.write(strng)
size = size - 8192
if size > 500000:
print size
except:
nf.write(string)
Also 8192 is quite big size and try to reduce the size to 1000-1300 bytes. Chossing 8192 is not giving you any advantage as the packets will still move in the fragments with upper limit of MTU which is generally 1436 bytes.