I have a code which is running on 2 hosts. They are processing some photos. When host1 receives a message, it will send some of its photos to host2 for processing. It will also send a list to host2 (it will convert it to string and then it will send it).
import pickle
import time, threading
host = commands.getoutput("hostname -I")
port = 5005
i = 0
backlog = 5
BUFSIZE = 4096
queueList = []
start = []
end = []
l = threading.Lock()
def read_udp(s):
data,addr = s.recvfrom(1024)
global start
if data.startswith('10.0.0'):
print("received message:", data)
data_split = data.split(" ")
address = data_split[0]
num = int(data_split[1])
ipToTransfer = address
l.acquire()
transferList = queueList[-num:]
del queueList[-num:]
transferStart = start[-num:]
del start[-num:]
l.release()
msg = pickle.dumps(transferStart)
#udp_send('New Transfer', ipToTransfer)
udp_send(msg, ipToTransfer)
send_file(ipToTransfer, transferList)
else:
recvStart = pickle.loads(data)
print("start before add::: ", start)
print("received::: ", recvStart)
l.acquire()
start = start + recvStart
l.release()
print("start after add::: ", start)
def udp_send(s, ip):
UDP_IP = ip
if(type(s) == str):
MESSAGE = s
else:
MESSAGE = pickle.dumps(s)
#print(MESSAGE)
print ("UDP target IP & port:", UDP_IP, port)
print ("message:", MESSAGE)
sock3 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
sock3.sendto(MESSAGE, (UDP_IP, port))
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def send(sock2, data2):
while data2:
sent = sock2.send(data2)
data2 = data2[sent:]
def send_file(ipToTransfer, transferList):
while transferList:
fname = transferList.pop(0)
print("transferring:", fname)
with open(fname, 'rb') as f:
sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock2.connect((ipToTransfer, 5005))
except socket.error as err:
print(err, ipToTransfer, 5005)
sock2.close()
return
# Send the file name length & the filename itself in one packet
send(sock2, pack('B', len(fname)) + fname.encode())
while True:
data2 = f.read(BUFSIZE)
if not data2:
break
send(sock2, data2)
sock2.close()
When host2 receives this string, it will convert it to list again, but I receive an EOFError in this part. My cmd doesn't have the copy capability, so I upload the photo from this error:
What's wrong?
you delete the pointer to what is being pickled
transferStart = start[-num:]
del start[-num:]
l.release()
msg = pickle.dumps(transferStart)
If you are trying to remove elements from a list that is not the way to do it. Consider popping or reassigning into another list that does not have that element, etc.
Related
I am creating a very simple rdt 2.2 socket program that transfers an image file dictated as "Cat.bmp" from client to server. Once the client reads the first line of data from the bmp file, it sends it to the server, and then the server will continue to repeat reading this same line in an infinite loop. I have no idea why this won't allow the client to send new data. Any suggestions on how to fix this would be very appreciated.
Client.py
import binascii
import struct
import sys
import hashlib
import base64
import time
from asyncio.tasks import sleep
def rdtSend(currentSequence , currentAck , data):
values = (currentACK, currentSequence, data)
UDPData = struct.Struct('I I 8s')
packedData = UDPData.pack(*values)
checksumVal = hashlib.md5(packedData).hexdigest().encode('utf-8')
sendPacket = makepacket(currentACK, currentSequence, data, checksumVal)
UDPSend(sendPacket)
def makepacket(currentACK, currentSequence, data, checksumVal):
values = (currentACK, currentSequence, data, checksumVal)
packetData = struct.Struct('I I 8s 32s')
packet = packetData.pack(*values)
return packet
def UDPSend(sendPacket):
senderSocket.sendto(sendPacket, (IP, Port))
def dataError(receivePacket):
checksum = makeChecksum(receivePacket[0], receivePacket[1], receivePacket[2])
# Compare calculated chechsum with checksum value in packet
if receivePacket[3] == checksum:
print('CheckSums is OK')
return False
else:
print('CheckSums Do Not Match')
return True
def makeChecksum(ACK, SEQ, DATA):
values = (ACK, SEQ, DATA)
packer = struct.Struct('I I 8s')
packedData = packer.pack(*values)
checksum = hashlib.md5(packedData).hexdigest().encode('utf-8')
return checksum
def isACK(receivePacket, ACKVal):
if (receivePacket[0] == ACKVal):
return True
else:
return False
IP = "127.0.0.1"
#Local Port for client and server
Port = 20001
#buffer to receive information from client
bufferSize = 1024
unpacker = struct.Struct('I I 8s 32s')
senderSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
senderSocket.connect((IP , Port))
print("UDP IP:", IP)
print("UDP port:", Port)
filename = 'Cat.bmp'
file = open(filename , 'rb')
# current data item being processed
data = file.read(bufferSize)
currentSequence = 0
currentACK = 0
while (data):
rdtSend(currentSequence, currentACK , data)
packet, addr = senderSocket.recvfrom(bufferSize)
print(packet)
print("Received from: ", addr)
receivePacket = unpacker.unpack(packet)
if(dataError(receivePacket) == False and isACK(receivePacket , currentACK) == True):
currentACK = currentACK + 1
currentSequence = (currentSequence + 1) % 2
data = file.read(bufferSize)
print("sending more data")
else:
print("Resending packet")
file.close()
senderSocket.close
Server.py
import socket
import binascii
import struct
import sys
import hashlib
import base64
import time
from asyncio.tasks import sleep
def rdtSend(currentSequence , currentAck , data):
values = (currentACK, currentSequence, data)
UDPData = struct.Struct('I I 8s')
packedData = UDPData.pack(*values)
checksumVal = hashlib.md5(packedData).hexdigest().encode('utf-8')
#This is where it gets the UDP packet
sendPacket = makepacket(currentACK, currentSequence, data, checksumVal)
UDPSend(sendPacket)
def makepacket(currentACK, currentSequence, data, checksumVal):
values = (currentACK, currentSequence, data, checksumVal)
packetData = struct.Struct('I I 8s 32s')
packet = packetData.pack(*values)
return packet
def UDPSend(sendPacket):
receiverSocket.sendto(sendPacket, (IP, Port))
def makeChecksum(ACK, SEQ, DATA):
values = (ACK, SEQ, DATA)
packer = struct.Struct('I I 8s')
packedData = packer.pack(*values)
checksum = hashlib.md5(packedData).hexdigest().encode('utf-8')
return checksum
#Function that checks the packet for corruption
def dataError(receivePacket):
# Calculate new checksum of the [ ACK, SEQ, DATA ]
checksum = makeChecksum(receivePacket[0], receivePacket[1], receivePacket[2])
# Compare calculated chechsum with checksum value in packet
if receivePacket[3] == checksum:
print('CheckSums is OK')
return False
else:
print('CheckSums Do Not Match')
return True
#IP Address for local communications
IP = "127.0.0.1"
#Local Port for client and server
Port = 20001
#buffer to receive information from client
bufferSize = 1024
# Integer, Integer, 8 letter char array, 32 letter char array
unpacker = struct.Struct('I I 8s 32s')
# Create the actual UDP socket for the server
receiverSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# Bind the socket to the local IP address and port
receiverSocket.bind((IP, Port))
currentACK = 0
currentSequence = 0
dataFile = open('receive.bmp' , 'wb')
print("Listening")
packet, addr = receiverSocket.recvfrom(bufferSize)
receivedPacket = unpacker.unpack(packet)
#Where the previous functions are used to send the packets back to the client
while receivedPacket[2]:
print("Received from:", addr)
print("Data Received:" , receivedPacket[2])
#This compares checksums to see if there are errors
if not dataError(receivedPacket):
dataFile.write(receivedPacket[2])
# Built checksum [ACK, SEQ, DATA]
ACK = receivedPacket[0]
SEQ = receivedPacket[1]
DATA = b''
print('Packeting')
rdtSend(currentSequence , currentACK , DATA)
print('Sent')
currentACK = currentACK + 1
currentSequence = (currentSequence + 1) % 2
packet, addr = receiverSocket.recvfrom(bufferSize)
receivedPacket = unpacker.unpack(packet)
else:
print('Packet error')
checksumVal = makeChecksum(packet[0] + 1, (packet[1] + 1) % 2, b'')
packet = makepacket(packet[0] + 1, (packet[1] + 1) % 2, b'', checksumVal)
print('Packeting')
receiverSocket.sendto(packet, addr)
print('Sent')
packet, addr = receiverSocket.recvfrom(bufferSize)
receivedPacket = unpacker.unpack(packet)
dataFile.close()
receiverSocket.close```
Whenever the client disconnect, the server will close itself. How can i make the server to run forever ?
What i'm doing
The server let one client to retrieve files with no issues. But the problem is when the client close the program, the server will also closed itself and wouldn't let another client to establish the connection . I had read a few articles about using while loops to make the session alive. Does anyone know how can I do that ?
Server.py
import socket, os, subprocess, shutil, pickle, struct
# Create a Socket ( connect two computers)
def create_socket():
try:
global host
global port
global s
host = ""
port = 9999
s = socket.socket()
except socket.error as msg:
create_socket()
# Binding the socket and listening for connections
def bind_socket():
try:
global host
global port
global s
s.bind((host, port))
s.listen(5)
except socket.error as msg:
bind_socket()
# send file list
def flist(conn):
try:
arr = pickle.dumps(os.listdir())
conn.send(arr)
except:
conn.send(('Error').encode("utf-8"))
# accept file from server
def fdown(filename, conn):
try:
data = conn.recv(1024).decode("utf-8")
if data[:6] == 'EXISTS':
filesize = data[6:]
conn.send("OK".encode("utf-8"))
f = open(filename, 'wb')
data = (conn.recv(1024))
totalRecv = len(data)
f.write(data)
while int(totalRecv) < int(filesize):
data = conn.recv(1024)
totalRecv += len(data)
f.write(data)
f.close()
except:
conn.send(('Error').encode("utf-8"))
# send file
def fup(filename, conn):
if os.path.isfile(filename):
conn.send(str.encode("EXISTS " + str(os.path.getsize(filename))))
filesize = int(os.path.getsize(filename))
userResponse = conn.recv(1024).decode("utf-8")
if userResponse[:2] == 'OK':
with open(filename, 'rb') as f:
bytesToSend = f.read(1024)
conn.send(bytesToSend)
totalSend = len(bytesToSend)
while int(totalSend) < int(filesize):
bytesToSend = f.read(1024)
totalSend += len(bytesToSend)
conn.send(bytesToSend)
else:
conn.send("ERROR".encode("utf-8"))
# main
def main(s):
while True:
data = (s.recv(1024)).decode("utf-8").split('~')
if data[0] == 'fdown':
fup(data[1], s)
elif data[0] == 'fup':
fdown(data[1], s)
elif data[0] == 'flist':
flist(s)
else:
s.send(".".encode('utf-8'))
# Establish connection with a client (socket must be listening)
def socket_accept():
conn, address = s.accept()
main(conn)
conn.close()
create_socket()
bind_socket()
socket_accept()
You should put accept in the loop, and you may need use a thread to handle read
sample code:
def handle_read(s):
while True:
data = s.recv(1024)
if not data:
#When the client closed, recv will return an empty data
s.close()
break
data = data.decode("utf-8").split('~')
if data[0] == 'fdown':
fup(data[1], s)
elif data[0] == 'fup':
fdown(data[1], s)
elif data[0] == 'flist':
flist(s)
else:
s.send(".".encode('utf-8'))
def socket_accept():
while True:
conn, address = s.accept()
t = threading.Thread(target = handle_read, args=(conn, ))
t.start()
I am pretty new to python and I had a script idea:
You have a listener which is waiting for functions and a client script which sends instructions to the server and in this case instructions are functions.
Like that I could execute functions code from the client on the other computer
I tried pickle but I can't figure out if it's working with functions
Shall I use Sockets or something else ?
here is the server code:
import socket
import time
HEADER_LENGHT = 10
IP = socket.gethostname()
PORT = 1234
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((IP, PORT))
s.listen(5)
while True:
clientsocket, address = s.accept()
print(f"Connection from {address} has been established ! ")
while True:
full_msg = b''
new_msg = True
while True:
msg = s.recv(16)
if new_msg:
print(f"New message lenght: {msg[:HEADERSIZE]}")
msglen = int(msg[:HEADERSIZE])
new_msg = False
full_msg += msg
if len(full_msg)- HEADERSIZE == msglen:
print("full msg recvd")
print(full_msg[HEADERSIZE:])
d = pickle.loads(full_msg[HEADERSIZE:])
print(d)
new_msg = True
full_msg = b''
print(full_msg)
The client code :
import socket
HEADERSIZE = 10
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((socket.gethostname(), 1235))
while True:
full_msg = b''
new_msg = True
while True:
msg = s.recv(16)
if new_msg:
print(f"New message lenght: {msg[:HEADERSIZE]}")
msglen = int(msg[:HEADERSIZE])
new_msg = False
full_msg += msg.decode("utf-8")
if len(full_msg)- HEADERSIZE == msglen:
print("full msg recvd")
print(full_msg[HEADERSIZE:])
new_msg = True
full_msg = b''
print(full_msg)
Here is the function code:
def generate(target, apikey, nbfriendly, nbteacher, nbleader, accounts):
acc_list = accounts
jsonFile = open("config.json", "r")
data = json.load(jsonFile)
lines_count = 0
if nbfriendly == "":
nbfriendly = 10
if nbteacher == "":
nbteacher = 10
if nbleader == "":
nbleader = 10
if apikey == "":
apikey = (data['steamWebAPIKey'])
if target == "":
target = (data['target'])
for l in accounts:
lines_count = lines_count + 1
conn = sqlite3.connect("accounts.sqlite")
c = conn.cursor()
max = accounts.count("\n")
counter = 0
print(accounts)
accounts = accounts.replace(":", " ")
accounts = accounts.replace("\n", " ")
accounts.split()
print(accounts)
splitted = accounts.split()
while counter < max:
username = splitted[counter]
password = splitted[(counter + 1)]
c.execute(
"INSERT INTO accounts (username, password, sharedSecret, lastCommend, operational) VALUES (? , ?, ?, ? , ?)",
(username, password, blank, mone, one)
)
conn.commit()
counter += 2
nbfriendly = int(nbfriendly)
nbteacher = int(nbteacher)
nbleader = int(nbleader)
target = str(target)
(data['commend']['friendly']) = nbfriendly
(data['commend']['teaching']) = nbteacher
(data['commend']['leader']) = nbleader
(data['target']) = target
(data['steamWebAPIKey']) = apikey
data = json.dumps(data, indent=4)
jsonFile.close()
with open('config.json', 'w') as outfile:
outfile.write(data)
outfile.close()
I'm not sure your code for sending messages using socket is correct (I don't see where you're sending bytes).
Concerning pickle is is indeed possible to use it as a function. Plus, the good thing is that it is dumped as a bytes object, so you don't even need to cast it before sendding it with sockets.
On the client side:
import pickle
def f(n):
return n * f(n - 1) if n else 1
to_send = pickle.dumps(f)
# Send to_send using sockets then
Then, on the server side, the only thing you need is to load the function:
import pickle
f = pickle.loads(received)
Since everything is an object in Python, you can use the exact same method to send the function's arguments using sockets and pickle.
I am trying to make an application which is accessible for a wide range of users.
Its some basic socket + python networking stuff.
I researched a bit recently and I tried a few things.
I implemented some UPNP stuff via miniupnpc but the thing is that you need a permission to use UPNP on most routers which is not ideal for my needs. I am trying to completely bypass things like port forwarding.
I also tried to completely switch from Python socket to pyp2p but I could't get it working because of the lack of documentation and maintenance on the project.
Is there an easy way to bypass NAT and if yes how ?
Client:
import socket
import struct
import os
import pickle
from pygame import mixer
def connect(host, port):
ourmusic = []
fileBRAIN = []
musicPath = './music'
s = socket.socket()
print('created socket')
s.connect((host,port))
print('conected to ',host)
#identify self
hostname = socket.gethostname()
IPAddr = socket.gethostbyname(hostname)
IPAddrBYTE = str(IPAddr).encode('utf-8')
send_msg(s, IPAddrBYTE)
if os.path.exists(str(host)) == True :
print('server seen previously')
#load host's song list
fileBRAIN = pickle.load(open( str(host), 'rb'))
#recieve new files from host
receiveFiles(s, fileBRAIN)
#compare files
saveMusicList(ourmusic, musicPath)
filesToTransmit = diff(ourmusic, fileBRAIN)
#transmit music which host dont have
transmitFiles(s , filesToTransmit, fileBRAIN)
#save host's song list
pickle.dump(fileBRAIN, open(str(host), 'wb'))
#start playing music
clientMainLoop(s)
else:
print('server not seen previously')
#recieve music files from host
receiveFiles(s, fileBRAIN)
#compare files
saveMusicList(ourmusic, musicPath)
filesToTransmit = diff(ourmusic, fileBRAIN)
#transmit music which host dont have
transmitFiles(s , filesToTransmit, fileBRAIN)
#save host's song list
pickle.dump(fileBRAIN, open(str(host), 'wb'))
#start playing music
clientMainLoop(s)
def clientMainLoop(socket):
mixer.init()
while True:
randomsongBYTE = recv_msg(socket)
randomsong = randomsongBYTE.decode('utf-8')
mixer.music.load('./music/' + randomsong)
mixer.music.play()
print('now playing: ' + randomsong)
while mixer.music.get_busy() == True:
temp = 0
else:
print('music ended receiving next...')
def transmitFiles(socket, files, fileBRAIN):
Transmissions = struct.pack('i',len(files))
send_msg(socket, Transmissions)
for x in files:
filename = str(x).encode('utf-8')
send_msg(socket, filename)
file = open('./music/' + x , 'rb')
file_data = file.read()
send_msg(socket, file_data)
fileBRAIN.append(str(x))
print(x + '\n' + 'has been send transmitted successfully')
print('')
def receiveFiles(s, fileBRAIN):
i = 1
transmissionsDATA = recv_msg(s)
transmissionsTUP = struct.unpack('i',transmissionsDATA)
transmissions = transmissionsTUP[0]
print('there are ' + str(transmissions) + ' awaiting receive
transmissions')
while i <= transmissions:
BYTEfilename = recv_msg(s)
filename = BYTEfilename.decode('utf-8')
file = open('./music/' + filename, 'wb')
file_data = recv_msg(s)
file.write(file_data)
file.close()
fileBRAIN.append(str(filename))
print('transmission ' + str(i) + ' recieived')
i = i + 1
def saveMusicList(musiclist, musicPath):
for entry in os.listdir(musicPath):
if os.path.isfile(os.path.join(musicPath, entry)):
musiclist.append(entry)
#find elements which are not in the other list
def diff(li1, li2):
return (list(set(li1) - set(li2)))
#TCP msg protocoll
def send_msg(sock, msg):
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock):
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
Server :
import socket
import os
import struct
import pickle
import miniupnpc
from random import randint
from pygame import mixer
def init(port):
fileBRAIN = []
ourmusic = []
files = []
musicPath = './music'
try:
#Network Gateway (port forwarding)
upnp = miniupnpc.UPnP()
upnp.discoverdelay = 10
upnp.discover()
upnp.selectigd()
upnp.addportmapping(port, 'TCP', upnp.lanaddr, port,
'OpenMCS', '')
except:
print('INFO:Port is already forwarded or you dont have the
premission to forward it')
s = socket.socket()
host = ('') #socket.gethostname()
s.bind((host,port))
s.listen(5)
print(host)
print('waiting for any incoming conections ... ')
conn,addr = s.accept()
#get client identity
ClientIPAddrBYTE = recv_msg(conn)
ClientIPAddr = ClientIPAddrBYTE.decode('utf-8')
if os.path.exists(ClientIPAddr) == True :
print('client seen previously')
#load client's song list
fileBRAIN = pickle.load(open( ClientIPAddr, 'rb'))
#compare files
saveMusicList(ourmusic, musicPath)
filesToTransmit = diff(ourmusic, fileBRAIN)
#transmit new files client doesn't have
transmitFiles(conn, filesToTransmit, fileBRAIN)
#recieve music files we dont have
receiveFiles(conn, fileBRAIN)
#save client's song list
pickle.dump(fileBRAIN, open(ClientIPAddr, 'wb'))
#start playing music
serverMainLoop(conn, ourmusic, musicPath)
else:
print('client not seen previously')
#transmit own music files to client
transmitFiles(conn, files, fileBRAIN)
#recieve music files we dont have
receiveFiles(conn, fileBRAIN)
#save client's song list
pickle.dump(fileBRAIN, open(ClientIPAddr, 'wb'))
#start playing music
serverMainLoop(conn, ourmusic, musicPath)
def serverMainLoop(socket, ourmusic, musicPath):
saveMusicList(ourmusic, musicPath)
mixer.init()
while True:
randomsong = ourmusic[randint(0, (len(ourmusic) - 1 ))]
randomsongBYTE = randomsong.encode('utf-8')
send_msg(socket, randomsongBYTE)
mixer.music.load('./music/' + randomsong)
mixer.music.play()
print('now playing: ' + randomsong)
while mixer.music.get_busy() == True:
temp = 0
else:
print('music ended picking next...')
def transmitFiles(socket, files, fileBRAIN):
Transmissions = struct.pack('i',len(files))
send_msg(socket, Transmissions)
for x in files:
filename = str(x).encode('utf-8')
send_msg(socket, filename)
file = open('./music/' + x , 'rb')
file_data = file.read()
send_msg(socket, file_data)
fileBRAIN.append(str(x))
print(x + '\n' + 'has been send transmitted successfully')
print('')
def receiveFiles(s, fileBRAIN):
i = 1
transmissionsDATA = recv_msg(s)
transmissionsTUP = struct.unpack('i',transmissionsDATA)
transmissions = transmissionsTUP[0]
print('there are ' + str(transmissions) + ' awaiting receive
transmissions')
while i <= transmissions:
BYTEfilename = recv_msg(s)
filename = BYTEfilename.decode('utf-8')
file = open('./music/' + filename, 'wb')
file_data = recv_msg(s)
file.write(file_data)
file.close()
fileBRAIN.append(str(filename))
print('transmission ' + str(i) + ' recieved')
i = i + 1
def saveMusicList(musiclist, musicPath):
for entry in os.listdir(musicPath):
if os.path.isfile(os.path.join(musicPath, entry)):
musiclist.append(entry)
#find elements which are not in the other list
def diff(li1, li2):
return (list(set(li1) - set(li2)))
#TCP msg proctocoll
def send_msg(sock, msg):
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock):
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
You can find the full code here if you need it : https://github.com/Finn1510/OpenMCS
Recently I wrote some code (client and server) to send an image - the client simply uploads the image to the server, just using the socket module: Sending image over sockets (ONLY) in Python, image can not be open.
However, the image sending part is now what I am concerned with. This is the original image I'm using:
In my server code (which receives the images), I have these lines:
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
data = sock.recv(40960000)
if not data:
myfile.close()
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
But I don't think this is the best way of doing it. I think I should instead implement the server such that it receives the data in chunks:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = 0
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()
But when I do this, the server prints:
got size 54674
4096
8192
12288
16384
20480
24576
28672
32768
36864
40960
45056
49152
50578
And then seems to hang, and the client hangs too. However, at the server's side I can see only a piece of the image the client wanted to send:
It seems like there are some bytes missing. What is the proper way of sending a huge amount of data (images, other type of file) using ONLY sockets?
I'm assuming that you have a particular reason for doing this with naked sockets, such as self-edification, which means that I won't answer by saying "You accidentally forgot to just use HTTP and Twisted", which perhaps you've heard before :-P. But really you should look at higher-level libraries at some point as they're a lot easier!
Define a protocol
If all you want is to send an image, then it can be simple:
Client -> server: 8 bytes: big endian, length of image.
Client -> server: length bytes: all image data.
(Client <- server: 1 byte, value 0: indicate transmission received - optional step you may not care if you're using TCP and just assume that it's reliable.)
Code it
server.py
import os
from socket import *
from struct import unpack
class ServerProtocol:
def __init__(self):
self.socket = None
self.output_dir = '.'
self.file_num = 1
def listen(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.bind((server_ip, server_port))
self.socket.listen(1)
def handle_images(self):
try:
while True:
(connection, addr) = self.socket.accept()
try:
bs = connection.recv(8)
(length,) = unpack('>Q', bs)
data = b''
while len(data) < length:
# doing it in batches is generally better than trying
# to do it all in one go, so I believe.
to_read = length - len(data)
data += connection.recv(
4096 if to_read > 4096 else to_read)
# send our 0 ack
assert len(b'\00') == 1
connection.sendall(b'\00')
finally:
connection.shutdown(SHUT_WR)
connection.close()
with open(os.path.join(
self.output_dir, '%06d.jpg' % self.file_num), 'w'
) as fp:
fp.write(data)
self.file_num += 1
finally:
self.close()
def close(self):
self.socket.close()
self.socket = None
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
sp = ServerProtocol()
sp.listen('127.0.0.1', 55555)
sp.handle_images()
client.py
from socket import *
from struct import pack
class ClientProtocol:
def __init__(self):
self.socket = None
def connect(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.connect((server_ip, server_port))
def close(self):
self.socket.shutdown(SHUT_WR)
self.socket.close()
self.socket = None
def send_image(self, image_data):
# use struct to make sure we have a consistent endianness on the length
length = pack('>Q', len(image_data))
# sendall to make sure it blocks if there's back-pressure on the socket
self.socket.sendall(length)
self.socket.sendall(image_data)
ack = self.socket.recv(1)
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
cp = ClientProtocol()
image_data = None
with open('IMG_0077.jpg', 'r') as fp:
image_data = fp.read()
assert(len(image_data))
cp.connect('127.0.0.1', 55555)
cp.send_image(image_data)
cp.close()
A simple way is to send data size as the first 4 bytes of your data and then read complete data in one shot. Use the below functions on both client and server-side to send and receive data.
def send_data(conn, data):
serialized_data = pickle.dumps(data)
conn.sendall(struct.pack('>I', len(serialized_data)))
conn.sendall(serialized_data)
def receive_data(conn):
data_size = struct.unpack('>I', conn.recv(4))[0]
received_payload = b""
reamining_payload_size = data_size
while reamining_payload_size != 0:
received_payload += conn.recv(reamining_payload_size)
reamining_payload_size = data_size - len(received_payload)
data = pickle.loads(received_payload)
return data
you could find sample program at https://github.com/vijendra1125/Python-Socket-Programming.git
The problem is you are not incrementing amount_received for the first chunk of the data received.
Fix below:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = len(data) # The fix!
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()