Develop Reference

Python Sockets: my recv protocol receive all the data when i debug it but not when i run it doesnt

So my project is that I need to send a jpg image from one computer to another computer in the same network. To send the data I split the data into chunks of at least 9999 bytes and then I create a length header that tells the length of the data and I attach it to the start of the massage. here is the code:
the protocol:
import os.path
LENGTH_FIELD_SIZE = 4
PORT = 8820
COMANDS_LIST = "TAKE_SCREENSHOT\nSEND_PHOTO\nDIR\nDELETE\nCOPY\nEXECUTE\nEXIT".split("\n")
def check_cmd(data):
"""
Check if the command is defined in the protocol, including all parameters
For example, DELETE c:\work\file.txt is good, but DELETE alone is not
"""
command = ""
file_location =""
splited_data = data.split(maxsplit=1)
if len(splited_data) == 2:
command, file_location = splited_data
return (command in COMANDS_LIST) and (file_location is not None)
elif len(splited_data) == 1:
command = splited_data[0]
return command in ["TAKE_SCREENSHOT","EXIT","SEND_PHOTO"]
return False
# (3)
def create_msg(data):
"""
Create a valid protocol message, with length field
"""
data_len = len(str(data))
if data_len > 9999 or data_len == 0:
print(f"data len is bigger then 9999 or is 0, data len = {data_len} ")
return False
len_field = str(data_len).zfill(4)
# (4)
print(len_field)
return True ,f"{len_field}{data}"
def get_msg(my_socket):
"""
Extract message from protocol, without the length field
If length field does not include a number, returns False, "Error"
"""
lenght_field = ""
data = ""
try:
while len(lenght_field) < 4:
lenght_field += my_socket.recv(4).decode()
except RuntimeError as exc_run:
return False, "header wasnt sent properly"
if not lenght_field.isdigit():
return False, "error, length header is not valid"
lenght_field = lenght_field.lstrip("0")
while len(data) < int(lenght_field):
data += my_socket.recv(int(lenght_field)).decode()
return True, data
now the protocol works fine when I use the same computer for both server and client and when I debug get_msg on the other computer. when I'm not, it seems that the problem is that the part that recv the header will recv something else after a few successful recv and return an error message.
here are the server parts:
import socket
import pyautogui as pyautogui
import protocol
import glob
import os.path
import shutil
import subprocess
import base64
IP = "0.0.0.0"
PORT = 8820
PHOTO_PATH = r"C:\Users\Innon\Pictures\Screenshots\screenShot.jpg"# The path + filename where the screenshot at the server should be saved
def check_client_request(cmd):
"""
Break cmd to command and parameters
Check if the command and params are good.
For example, the filename to be copied actually exists
Returns:
valid: True/False
command: The requested cmd (ex. "DIR")
params: List of the cmd params (ex. ["c:\\cyber"])
"""
# Use protocol.check_cmd first
cmd_arr = cmd.split(maxsplit=1)
command = cmd_arr[0]
file_location = None
if len(cmd_arr) == 2:
file_location = cmd_arr[1]
if file_location == None:
return protocol.check_cmd(cmd) ,command, file_location
else:
file_location = tuple(str(file_location).split())
if (os.path.exists(file_location[0])):
return protocol.check_cmd(cmd) , command , file_location
return False , command , file_location
# Then make sure the params are valid
# (6)
def handle_client_request(command,params):
"""Create the response to the client, given the command is legal and params are OK
For example, return the list of filenames in a directory
Note: in case of SEND_PHOTO, only the length of the file will be sent
Returns:
response: the requested data
"""
# (7)
response = "no server response"
if command == "DIR":
response = glob.glob(f"{params[0]}\\*.*" )
if command == "DELETE":
os.remove(params[0])
response = f"{params[0]} was deleted"
if command == "COPY":
try:
shutil.copy(params[0],params[1])
response = f"{params[0]} was copyed to {params[1]}"
except FileNotFoundError as ex1:
response = ex1
except IndexError as ex2:
response = ex2
if command == "EXECUTE":
subprocess.call(params[0])
response = f"{params[0]} was executed"
if command == "TAKE_SCREENSHOT":
#todo find a way to know and create the locatipn of screen shot to be saved
myScreenshot = pyautogui.screenshot()
myScreenshot.save(PHOTO_PATH)
response = f"screen shot have been taken and been saved at {PHOTO_PATH}"
if command == "SEND_PHOTO":
with open(PHOTO_PATH, "rb") as file:
file_data = base64.b64encode(file.read()).decode()
print(file_data)
is_vaild_response, img_length = protocol.create_msg(len(file_data))
print(img_length)
img_data = ""
if not is_vaild_response:
response = "img length data isnt valid"
return response
while len(file_data) > 0:
chunk_data = file_data[:9999]
is_vaild_response, data = protocol.create_msg(chunk_data)
if not is_vaild_response:
response = "img data isnt valid"
return response
img_data += data
file_data = file_data[9999:]
response = f"{img_length}{img_data}"
return response
def main():
# open socket with client
server_socket = socket.socket()
server_socket.bind((IP,PORT))
server_socket.listen(1)
# (1)
client_socket, addr = server_socket.accept()
# handle requests until user asks to exit
while True:
# Check if protocol is OK, e.g. length field OK
valid_protocol, cmd = protocol.get_msg(client_socket)
print(f"got message {valid_protocol}")
if valid_protocol:
# Check if params are good, e.g. correct number of params, file name exists
valid_cmd, command, params = check_client_request(cmd)
print(f"check_client_request {valid_cmd}")
if valid_cmd:
# (6)
if command == 'EXIT':
break
if command == 'SEND_PHOTO':
data = handle_client_request(command, params)
client_socket.sendall(data.encode())
continue
# prepare a response using "handle_client_request"
data = handle_client_request(command,params)
# add length field using "create_msg"
is_vaild_response , response = protocol.create_msg(data)
print(f"creat_msg {is_vaild_response}")
# send to client
if is_vaild_response:
client_socket.sendall(response.encode())
else:
# prepare proper error to client
resp = 'Bad command or parameters'
is_vaild_response , response = protocol.create_msg(resp)
# send to client
client_socket.sendall(response.encode())
else:
# prepare proper error to client
resp = 'Packet not according to protocol'
is_vaild_response, response = protocol.create_msg(resp)
#send to client
client_socket.sendall(response.encode())
# Attempt to clean garbage from socket
client_socket.recv(1024)
# close sockets
resp = "Closing connection"
print(resp)
is_vaild_response, response = protocol.create_msg(resp)
client_socket.sendall(response.encode())
client_socket.close()
server_socket.close()
if __name__ == '__main__':
main()
and the client:
import socket
import base64
import protocol
IP = "127.0.0.1"
SAVED_PHOTO_LOCATION = r'C:\Users\Innon\Pictures\Saved Pictures\screenShot.jpg' # The path + filename where the copy of the screenshot at the client should be saved
def handle_server_response(my_socket, cmd):
"""
Receive the response from the server and handle it, according to the request
For example, DIR should result in printing the contents to the screen,
Note- special attention should be given to SEND_PHOTO as it requires and extra receive
"""
# (8) treat all responses except SEND_PHOTO
if "SEND_PHOTO" not in cmd:
vaild_data, data = protocol.get_msg(my_socket)
if vaild_data:
return data
# (10) treat SEND_PHOTO
else:
pic_data = ""
vaild_pick_len, pic_len = protocol.get_msg(my_socket)
if pic_len.isdigit() == False:
print(f"picture length is not valid. got massage: {pic_len}")
return
with open(SAVED_PHOTO_LOCATION, "wb") as file:
while len(pic_data) < int(pic_len):
vaild_data, data = protocol.get_msg(my_socket)
if not vaild_data:
return f"img data isnt valid. {data}"
pic_data += data
print(pic_data)
file.write(base64.b64decode(pic_data.encode()))
return "img was recived succesfully "
def main():
# open socket with the server
my_socket = socket.socket()
my_socket.connect((IP,8820))
# (2)
# print instructions
print('Welcome to remote computer application. Available commands are:\n')
print('TAKE_SCREENSHOT\nSEND_PHOTO\nDIR\nDELETE\nCOPY\nEXECUTE\nEXIT')
# loop until user requested to exit
while True:
cmd = input("Please enter command:\n")
if protocol.check_cmd(cmd):
valid_pack , packet = protocol.create_msg(cmd)
if valid_pack:
my_socket.sendall(packet.encode())
print(handle_server_response(my_socket, cmd))
if cmd == 'EXIT':
break
else:
print("Not a valid command, or missing parameters\n")
my_socket.close()
if __name__ == '__main__':
main()
here is how the problem looks like:thi is how it looks
here is how to needs look like:
the right way
thank you.

the solution was to change get_msg function in the protocol:
while len(data) < int(lenght_field):
data += my_socket.recv(int(lenght_field) - len(data)).decode()
instead of:
while len(data) < int(lenght_field):
data += my_socket.recv(int(lenght_field)).decode()

I can't break a while loop in python

I'm trying to transfer files with python socket module, I can transfer and write the file but I can't stop the function when there are no more bytes to write
FORMAT="utf-8"
HEADER=512
CLIENT CODE
FORMAT="utf-8"
HEADER=512
def File(path):
name=os.path.basename(path)
client.send("file".encode(FORMAT))
client.send(name.encode(FORMAT))
print(client.recv(HEADER).decode(FORMAT))
f=open(path,"rb")
l=f.read(HEADER)
while (l):
client.send(l)
l=f.read(HEADER)
f.close()
print("Finish")
SERVER CODE
def Save(conn):
name=(conn.recv(HEADER).decode(FORMAT))
conn.send(f"Saving {name}".encode(FORMAT))
print(name)
Writing=True
with open(PATH+name,"wb") as f:
print("Writing file")
while Writing:
data=conn.recv(HEADER)
if not data:
Writing=False
f.write(data)
f.close()
print("File written")
conn.close()

To know when a socket has finished, take a look at How does the python socket.recv() method know that the end of the message has been reached?.
Anyway, receiving constant HEADER size is a bad practice, as the received amount can be smaller (due to network issues);
You'd better keep up with the actual amount received with respect to the desired amount, like in this example:
def receive(self, size):
from_client = b''
remaining_to_recive = size
while 0 < remaining_to_recive:
data = self.sock.recv(remaining_to_recive)
if not data: break
from_client += data
remaining_to_recive -= len(data)
if 0 < remaining_to_recive:
raise RuntimeError(Connection.NOT_ALL_DATA_RECEIVED_ERROR)
return from_client

ValueError when transmitting a file titled in Arabic over a Python tcp socket

I'm building a socket program to transmit files between two computers. Files with English titles are transmitted successfully, but when I try to send Arabic titled files (eg وثيقة.docx), I get a long list of ValueErrors, starting with:
invalid literal for int() with base 2: b'.docx000000000000000000010000001'
invalid literal for int() with base 2: b'10001PK\x03\x04\x14\x00\x08\x08\x08\x00\xe0'
My code is:
Server:
import socket
serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = 'localhost'
port = 9000
serversock.bind((host,port))
filename = ""
serversock.listen(10)
print("Waiting for a connection.....")
clientsocket, addr = serversock.accept()
print("Got a connection from %s" % str(addr))
while True:
try:
size = clientsocket.recv(16) # Note that you limit your filename length to 255 bytes.
if not size:
clientsocket, addr = serversock.accept()
print("Got a connection from %s" % str(addr))
continue
size = int(size, 2)
print('SIZE', size)
filename = clientsocket.recv(size)
print('filename', filename)
filesize = clientsocket.recv(32)
print('FILESIZE', filesize, 'TYPE', type(filesize))
filesize = int(filesize, 2) ##########
file_to_write = open(filename, 'wb')
chunksize = 4096
while filesize > 0:
if filesize < chunksize:
chunksize = filesize
data = clientsocket.recv(chunksize)
file_to_write.write(data)
filesize -= len(data)
file_to_write.close()
print('File (%s) received successfully' % filename.decode('utf-8'))
except ValueError as verr:
print(verr)
#continue
except FileNotFoundError:
print('FileNotFoundError')
#continue
serversock.close()
Client:
import
socket
import os
from file_walker import files_to_transmit
def transmit(host, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
directory = files_to_transmit()
for file in directory:
filename = file
size = len(filename.split('/')[-1]) # split() to get bare file name to transmit to server
size = bin(size)[2:].zfill(16) # encode filename size as 16 bit binary
s.send(size.encode('utf-8'))
s.send(filename.split('/')[-1].encode('utf-8')) # split() to get bare file name
filename = file
filesize = os.path.getsize(filename)
filesize = bin(filesize)[2:].zfill(32) # encode filesize as 32 bit binary
s.send(filesize.encode('utf-8'))
file_to_send = open(filename, 'rb')
l = file_to_send.read()
s.sendall(l)
file_to_send.close()
print('File Sent')
s.close()
except ConnectionRefusedError:
print('ConnectionRefusedError: Server may not be running')
except ValueError as e:
print(e)
transmit('localhost', 9000)
What is the problem here? Please help.

You send the size in unicode character of the title, and then try to send it encoded in utf-8. For example with your example:
title = 'وثيقة.docx'
print(len(title), len(title.encode('utf8')))
gives
10 15
The peer will only use 10 bytes for the file name, and will use the remaining 5 as the beginning of the file size. And will choke for .docx not being the beginning of a binary number. What happens...
Fix is easy, build the byte string before computing its length:
...
for file in directory:
filename = file.split('/')[-1].encode('utf_8') # split() to get bare file name
size = len(filename)
size = bin(size)[2:].zfill(16) # encode filename size as 16 bit binary
s.send(size.encode('utf-8'))
s.send(filename)
...

Simalar problem faced out in my experience.
What i'm suggesting is to use an algorithm to encode and decode files like base64.
Somewhere in your program you're reading the file to save him in memory and send in second steps. Well instead just reading encode also
import base64
with open("yourfile.ext", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
Obviously in other side you need to decode and write on disk. You can encode and decode also the title of file to avoid ValueError

The response time is decreasing for some tasks

I've written a code which is supposed to receive some images and make them black & white. I'm measuring the response time for each task (response time = the time each image is received and is turned to black & white). Here is the code:
from __future__ import print_function
import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)
from select import select
import socket
from struct import pack
from struct import unpack
#from collections import deque
import commands
from PIL import Image
import time
host = commands.getoutput("hostname -I")
port = 5005
backlog = 5
BUFSIZE = 4096
queueList = []
start = []
end = []
temp = []
def processP(q):
i = 0
while q:
name = q.pop(0)
col = Image.open(name)
gray = col.convert('L')
bw = gray.point(lambda x: 0 if x<128 else 255, '1')
bw.save("%d+30.jpg" % (i+1))
end.append(time.time())
#print(temp)
i = i + 1
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
name = receiver.get(name_size).decode()
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
queueList.append(name)
print('name', name)
start.append(time.time())
if (name == "sample.jpg"):
print('------------ok-------------')
processP(queueList)
print("Start: ", start)
print('--------------------------')
print("End: ", end)
while start:
temp.append(end.pop(0) - start.pop(0))
print('****************************')
print("Start: ", start)
print('--------------------------')
print("End: ", end)
print("Temp: ", temp)
i = 0
while i < len(temp)-1:
if (temp[i]<temp[i+1]):
print('yes')
else:
print('No')
i = i + 1
def read_udp(s):
data,addr = s.recvfrom(1024)
print("received message:", data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
tcp.bind((host,port))
except socket.error as err:
print('Bind failed', err)
return
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
print('***Socket now listening at***:', host, port)
input = [tcp,udp]
try:
while True:
inputready,outputready,exceptready = select(input,[],[])
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
raise
tcp.close()
udp.close()
if __name__ == '__main__':
run()
Now for some evaluation purposes, I send a single image many times. When I look at the response times I see that sometimes the response time of the 8th image, for example, is more than the response time of the 9th one.
So my question is that since the size and the time needed for processing each of images are the same (I'm sending a single image several times), Why is the response time for each image variable? Shouldn't the response time of the next image be longer (or at least equal) that the previous one (For example, the response time for 4th image > the response time for 3rd image)?

Your list contains the actual elapsed time it took for each image processing call. This value will be influenced by many things, including the amount of load on the system at that time.
When your program is running, it does not have exclusive access to all of the resources (cpu, ram, disk) of the system it's running on. There could be dozens, hundreds or thousands of other processes being managed by the OS vying for resources. Given this, it is highly unlikely that you would ever see even the same image processed in the exact same amount of time between two runs, when you are measuring with sub-second accuracy. The amount of time it takes can (and will) go up and down with each successive call.

Python file transfer only works in localhost and fails between two machines

I have written following python code for file transfer. It works fine in localhost environment. It fails between two different physical machines. I get the file but with incomplete data. There is also invalid literal error while converting string to long while sending file size to client. I can't figure out why?
server.py
from socket import *
import thread
import os
l = {}
def handler(clientsocket, clientaddr):
print "Accepted connection from: ", clientaddr
while 1:
data = clientsocket.recv(8192)
if not data:
break
else:
print "The following data was received - ",data
l[clientaddr] = data
print l
print "Opening file - ",data
fp = open(data,'r')
size = os.path.getsize(data)
clientsocket.send(str(size))
strng = "hi"
print size
while size > 0:
strng = fp.read(8192)
clientsocket.send (strng)
size = size - 8192
clientsocket.close()
if __name__ == "__main__":
host = 'localhost'
port = 55573
buf = 8192
addr = (host, port)
serversocket = socket(AF_INET, SOCK_STREAM)
serversocket.bind(addr)
serversocket.listen(5)
while 1:
print "Server is listening for connections\n"
clientsocket, clientaddr = serversocket.accept()
thread.start_new_thread(handler, (clientsocket, clientaddr))
serversocket.close()
client.py
from socket import *
import os
if __name__ == '__main__':
host = '10.1.99.176'
port = 55573
buf = 8192
addr = (host, port)
clientsocket = socket(AF_INET, SOCK_STREAM)
clientsocket.connect(addr)
while 1:
fname = raw_input("Enter the file name that u want>> ")
if not fname:
break
else:
clientsocket.send(fname)
print "\nThe file will be saved and opened- "
fname = '/home/coep/Downloads/'+fname
nf = open(fname,"a")
strng = "hi"
size = clientsocket.recv(16)
size = long(size)
print size
while size > 0:
strng = clientsocket.recv(8192)
if not strng:
break
nf.write(strng)
size = size - 8192
if size > 500000:
print size
nf.close()
fname = 'viewnior '+ fname
print fname
os.system(fname)

In server.py, you are using
host = 'localhost' and it will just bind the port to localhost Ip address i.e. 127.0.0.1.
Change it to host='0.0.0.0' and it will bind the specific port to all available interfaces.
Update: Another reason can be that buffer size is just too big and with 8192, it may get blocked forever because last transfer was never able to fill the buffer. To get around it, set the timeout to makesure to proceed if the buffer doesn't fill up. In client.py, try changing,
clientsocket.timeout(5)
while size > 0:
try:
strng = clientsocket.recv(8192)
if not strng:
break
nf.write(strng)
size = size - 8192
if size > 500000:
print size
except:
nf.write(string)
Also 8192 is quite big size and try to reduce the size to 1000-1300 bytes. Chossing 8192 is not giving you any advantage as the packets will still move in the fragments with upper limit of MTU which is generally 1436 bytes.