Related
I made two applications ('Client' and 'Server') written in Python.
On each of them beyond main thread I created two additonal threads, which are handling operations of sending / receiving data from each TCP / UDP parts.
So for client I have written code for:
clientTCP part:
class ClientTCP:
def __init__(self, host_address: str, port: int):
self.client_socket = None
self.host_address = host_address
self.port = port
def connect_to_server(self, is_nagle_disabled: bool):
try:
self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if is_nagle_disabled:
self.client_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True)
self.client_socket.connect((self.host_address, self.port))
except socket.error:
print('Error: probably wrong port passed for connection')
sys.exit(1)
def send_message_to_server(self, user_input: str):
try:
#print(f'MESSAGE TO SEND AS CLIENT TCP: {user_input}')
self.client_socket.sendall(bytes(user_input, 'utf-8'))
except socket.error as e:
print('OVER HERE SEND MESSAGE TCP ERROR')
print(f'Error: {e}')
self.client_socket.close()
sys.exit(1)
def receive_message_from_server(self, buffer_size: int):
data_from_server = []
while True:
byte_portion_of_data = self.client_socket.recv(buffer_size)
if byte_portion_of_data.decode("utf-8") in ['BUSY', 'READY']:
return data_from_server
elif byte_portion_of_data:
#print(f'TCP -> byte portion of data: {byte_portion_of_data.decode("utf-8")}')
data_from_server.append(byte_portion_of_data)
else:
print('Entirety of message from server received')
break
return data_from_server
def get_client_socket(self):
return self.client_socket
def __del__(self):
self.client_socket.close()
def tcp_send_data_to_server(client: ClientTCP, data_to_send: list[int], size_of_buffer: int, is_nagle_disabled: bool, stop):
try:
#print('OVER HERE TCP!!!')
client.connect_to_server(is_nagle_disabled)
client_connection_list = client.receive_message_from_server(10)
client_connection_message = ''.join([x.decode("utf-8") for x in client_connection_list])
if client_connection_message == 'BUSY':
return
elif client_connection_message == 'READY':
client.send_message_to_server(f"SIZE:{str(size_of_buffer)}")
while True:
#print("TCP!!!")
if stop():
break
message_to_send = ''.join([str(x) for x in data_to_send])
client.send_message_to_server(message_to_send)
time.sleep(1)
except ConnectionResetError:
print("Socket was closed due to some unknown reasons. Sorry. :(")
senderUDP
class SenderUDP:
def __init__(self, host_address: str, port: int):
self.client_socket = None
self.host_address = host_address
self.port = port
def connect_to_server(self):
try:
self.client_socket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
except socket.error as e:
print(str(e))
sys.exit(1)
def send_message_to_server(self, input_to_send: str):
try:
#print(input_to_send)
#print('OVER HERE 1 UDP send')
self.client_socket.sendto(input_to_send.encode(), (self.host_address, self.port))
#print('OVER HERE 2 UDP send')
except Exception as e:
print('Error: ' + str(e))
self.client_socket.close()
sys.exit(1)
def get_client_socket(self):
return self.client_socket
def __del__(self):
self.client_socket.close()
def sending(sender: SenderUDP, data_to_send: list[int], size_of_buffer: int, stop):
try:
sender.connect_to_server()
sender.send_message_to_server(f"SIZE:{size_of_buffer}")
while True:
#print("UDP!!!")
if stop():
sender.send_message_to_server('END')
break
message_to_send = ''.join([str(x) for x in data_to_send])
sender.send_message_to_server(message_to_send)
sleep(1)
except ConnectionResetError:
print("Socket was closed due to some unknown reasons. Sorry. :(")
As for the 'Server' part:
serverTCP
import socket
import sys
import re
import time
from datetime import datetime
class ServerTCP:
def __init__(self, address, port, buffer: int):
self.server_socket = None
self.host_address = address
self.port = port
self.number_of_clients = 0
self.buffer = buffer
self.client_socket = None
self.count_bytes_from_client = 0
self.count_time_in_seconds = 0
def create_socket(self):
try:
socket.inet_aton(self.host_address)
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.host_address, self.port))
except socket.error:
print('Error: ' + str(socket.error))
sys.exit(1)
def start_listening(self):
try:
self.server_socket.listen(1)
except socket.error as msg:
print('Error: ' + str(socket.error))
self.server_socket.close()
sys.exit(1)
def set_client_socket(self, client_socket):
self.client_socket = client_socket
self.count_bytes_from_client = 0
self.count_time_in_seconds = 0
self.client_socket.send('READY'.encode())
def handle_another_client(self, another_client_socket):
another_client_socket.send('BUSY'.encode())
another_client_socket.close()
def remove_client_socket(self):
self.client_socket.shutdown(socket.SHUT_WR)
self.client_socket.close()
self.buffer = 100
self.client_socket = None
def get_server_socket(self):
return self.server_socket
def get_client_socket(self):
return self.client_socket
def get_transmission_data(self):
return self.count_bytes_from_client / 1000, self.count_time_in_seconds
def __del__(self):
self.server_socket.shutdown(socket.SHUT_RDWR)
self.server_socket.close()
print("Server socket closed")
def __str__(self):
kbytes, time = self.get_transmission_data()
return f"TCP: {kbytes} per {time} seconds"
def tcp_get_data_from_client(server: ServerTCP):
# data_from_client = []
try:
start = datetime.now()
while True:
byte_portion_of_data = server.client_socket.recv(server.buffer)
if not byte_portion_of_data:
break
# data_from_client.append(byte_portion_of_data)
string_data_from_client = byte_portion_of_data.decode('utf-8')
if string_data_from_client.startswith('SIZE:'):
temp = re.findall(r'\d+', string_data_from_client[5:])
res = list(map(int, temp))
if len(res) != 0:
server.buffer = res[0]
start = datetime.now()
else:
server.count_bytes_from_client += len(string_data_from_client)
#print('End of the ServerTCP loop')
time.sleep(0)
end = datetime.now()
server.count_time_in_seconds += (end - start).total_seconds()
print(server)
server.remove_client_socket()
except ConnectionResetError:
print("Socket was closed due to some unknown reasons. Sorry. :(")
receiverUDP
import socket
import struct
import sys
import re
import time
from datetime import datetime
class ReceiverUDP:
def __init__(self, group, port: int, buffer: int):
self.group = group
self.port = port
self.socket = None
self.buffer = buffer
self.count_bytes_from_client = 0
self.count_time_in_seconds = 0
self.start = None
def starting(self):
try:
self.socket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
self.socket.bind((self.group, self.port))
except socket.error as e:
print(f'Error: {str(e)}')
def get_transmission_data(self):
return self.count_bytes_from_client / 1000, self.count_time_in_seconds
def clean_transmission_data(self):
self.count_bytes_from_client = 0
self.count_time_in_seconds = 0
def close_of_receiver(self):
try:
self.socket.close()
except socket.error as e:
print(f'Error: {str(e)}')
def __str__(self):
kbytes, time = self.get_transmission_data()
return f"UDP: {kbytes} per {time}"
def receiving(receiver: ReceiverUDP, stop_thread):
is_first_client = True
try:
#print('Start of ReceiverUDP')
while True:
#print('Before recvfrom, ReceiverUDP')
#print(f'Start buffer: {receiver.buffer}')
data = receiver.socket.recvfrom(receiver.buffer)
message = data[0].decode("utf-8")
print(f"Server UDP, message: {message}")
if message.startswith("SIZE:"):
temp = re.findall(r'\d+', message[5:])
res = list(map(int, temp))
print(f'New buffer: {res[0]}')
if len(res) != 0:
if is_first_client:
receiver.start = datetime.now()
is_first_client = False
receiver.buffer = res[0]
print(f'New buffer assigned: {receiver.buffer}')
elif message.__contains__("END"):
receiver.count_time_in_seconds += (datetime.now() - receiver.start).total_seconds()
receiver.start = datetime.now()
print(receiver)
else:
receiver.count_bytes_from_client += len(message)
#print(f"Message from Server: ")
time.sleep(0)
except ConnectionResetError:
print(receiver)
print("Socket was closed due to some unknown reasons. Sorry. :(")
Static methods are of course 'thread' methods also.
Now back into the problem. I read on the internet that UDP transmission should be much faster than TCP. But that's not the case, in fact it's complete opposite for me.
When I put Server part on the container and run it + launched client with 'typed host_address' of docker gateway (it's something like 172.16.0.1') I got the same thing as earlier on running both on my machine.
On server application output I got such statistics for both TCP and UDP:
TCP: 2.3kB per 15.004 sec
UDP: 1.5kB per 15.009 sec
So clearly even now UDP is much slower than TCP. Why is that and what I did wrong?
I would be grateful for all advices.
Python UDP throughput is much slower than TCP throughput
This is to be expected. TCP is optimized for low overhead and will combine multiple send into as few packets on the wire as possible. With UDP instead each send will result in a single packet with all the overhead.
This overhead is especially noticable if the datagrams (i.e. the payload of send) are significantly smaller than the MTU of the link. And from a short look at your code it looks like that your are sending several small datagrams.
Apart from that it looks like you assume that a single send in the sender will match a single recv in the recipient. This is not true for TCP since data can be combined to reduce overhead: TCP is not a message protocol but an unstructured byte stream. The assumption is more or less true with UDP, but contrary to TCP there might be packet loss, packet reordering and duplication which you currently don't account for.
This is the code in serverside:
#!/usr/bin/env python
import sys
import logging
import socket
import struct
import fcntl
import os
from util import *
logger = logging.getLogger()
clients = {}
def main(host='**client public ip**', port=3443):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, port))
s.listen(1)
s.settimeout(30)
while True:
try:
conn, addr = s.accept()
except socket.timeout:
continue
logger.info('connection address: %s', addr)
data = recv_msg(conn)
priv_addr = msg_to_addr(data)
send_msg(conn, addr_to_msg(addr))
data = recv_msg(conn)
data_addr = msg_to_addr(data)
if data_addr == addr:
logger.info('client reply matches')
clients[addr] = Client(conn, addr, priv_addr)
else:
logger.info('client reply did not match')
conn.close()
logger.info('server - received data: %s', data)
if len(clients) == 2:
(addr1, c1), (addr2, c2) = clients.items()
logger.info('server - send client info to: %s', c1.pub)
send_msg(c1.conn, c2.peer_msg())
logger.info('server - send client info to: %s', c2.pub)
send_msg(c2.conn, c1.peer_msg())
clients.pop(addr1)
clients.pop(addr2)
conn.close()
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
main(*addr_from_args(sys.argv))
and this is clientside:
#!/usr/bin/env python
import sys
import logging
import socket
import struct
from threading import Event, Thread
from util import *
logger = logging.getLogger('client')
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
STOP = Event()
def accept(port):
logger.info("accept %s", port)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
s.bind(('**server's public ip**', port))
s.listen(1)
s.settimeout(5)
while not STOP.is_set():
try:
conn, addr = s.accept()
except socket.timeout:
continue
else:
logger.info("Accept %s connected!", port)
# STOP.set()
def connect(local_addr, addr):
logger.info("connect from %s to %s", local_addr, addr)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
s.bind(local_addr)
while not STOP.is_set():
try:
s.connect(addr)
except socket.error:
continue
# except Exception as exc:
# logger.exception("unexpected exception encountered")
# break
else:
logger.info("connected from %s to %s success!", local_addr, addr)
# STOP.set()
def main(host='**client's local ip**', port=3443):
sa = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sa.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sa.connect((host, port))
priv_addr = sa.getsockname()
send_msg(sa, addr_to_msg(priv_addr))
data = recv_msg(sa)
logger.info("client %s %s - received data: %s", priv_addr[0], priv_addr[1], data)
pub_addr = msg_to_addr(data)
send_msg(sa, addr_to_msg(pub_addr))
data = recv_msg(sa)
pubdata, privdata = data.split(b'|')
client_pub_addr = msg_to_addr(pubdata)
client_priv_addr = msg_to_addr(privdata)
logger.info(
"client public is %s and private is %s, peer public is %s private is %s",
pub_addr, priv_addr, client_pub_addr, client_priv_addr,
)
threads = {
'0_accept': Thread(target=accept, args=(priv_addr[1],)),
'1_accept': Thread(target=accept, args=(client_pub_addr[1],)),
'2_connect': Thread(target=connect, args=(priv_addr, client_pub_addr,)),
'3_connect': Thread(target=connect, args=(priv_addr, client_priv_addr,)),
}
for name in sorted(threads.keys()):
logger.info('start thread %s', name)
threads[name].start()
while threads:
keys = list(threads.keys())
for name in keys:
try:
threads[name].join(1)
except TimeoutError:
continue
if not threads[name].is_alive():
threads.pop(name)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, message='%(asctime)s %(message)s')
main(*addr_from_args(sys.argv))
and this is the util.py code:
import struct
from collections import namedtuple
def addr_from_args(args, host='**server's local ip**', port=9999):
if len(args) >= 3:
host, port = args[1], int(args[2])
elif len(args) == 2:
host, port = host, int(args[1])
else:
host, port = host, port
return host, port
def msg_to_addr(data):
ip, port = data.decode('utf-8').strip().split(':')
return (ip, int(port))
def addr_to_msg(addr):
return '{}:{}'.format(addr[0], str(addr[1])).encode('utf-8')
def send_msg(sock, msg):
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = b''
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data += packet
return data
def recv_msg(sock):
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
class Client(namedtuple('Client', 'conn, pub, priv')):
def peer_msg(self):
return addr_to_msg(self.pub) + b'|' + addr_to_msg(self.priv)
and this is the error:
Traceback (most recent call last):
File "tcpclient.py", line 96, in <module>
main(*addr_from_args(sys.argv))
File "tcpclient.py", line 55, in main
sa.connect((host, port))
ConnectionRefusedError: [Errno 111] Connection refused
It's working in same network but in different one not!!
For adding more details I would say that we have one client and on server that wanna communicate to each other, client and server are both behind the nat and the server has webserver and client want to go to the webserver but because of the nat problem it can't. so we are trying to make a hole punching peer to peer connection to make client able to see the webserver.
we can't take any kind of ip static or commercial web service or isp or what ever that everyone say.
what's wrong with this code?
in util.py the ip should be public not private
import struct
from collections import namedtuple
def addr_from_args(args, host='**server's ip public**', port=9999):
if len(args) >= 3:
host, port = args[1], int(args[2])
elif len(args) == 2:
host, port = host, int(args[1])
else:
host, port = host, port
return host, port
def msg_to_addr(data):
ip, port = data.decode('utf-8').strip().split(':')
return (ip, int(port))
def addr_to_msg(addr):
return '{}:{}'.format(addr[0], str(addr[1])).encode('utf-8')
def send_msg(sock, msg):
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = b''
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data += packet
return data
def recv_msg(sock):
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
class Client(namedtuple('Client', 'conn, pub, priv')):
def peer_msg(self):
return addr_to_msg(self.pub) + b'|' + addr_to_msg(self.priv)
I have a TCP server and a Kivy app client, upon initialisation I send the server code 90010 and get all the data fine however when I move to another screen where I need to receive data again I only receive b'' however on other screens I only need to send data and that works perfectly, I honestly can't wrap my head around why its sending empty byte strings any help is really appreciated
I know the code is reaching the if server code == 919 point so it should be sending the data but nothing is being received!, I have also put a s.connect on the initialisation of every screen maybe this is causing problems however im completely unsure
TCP code this SERVERCODE is in the class clientthread and it uses the same method as the client with GetLength() to get server code
class clientthread(thread)
def run(self)
while true:
SERVERCODE = self.getlength
if SERVERCODE == b''
break
else:
while true:
elif SERVERCODE == 90010:
global Todays_refgroup
print(Todays_refgroup)
print(statuslist)
TF_Pickles = pickle.dumps(Todays_refgroup)
INIT_Length = self.Pack_data(TF_Pickles)
TF_Status = pickle.dumps(statuslist)
INIT_Status = self.Pack_data(TF_Status)
conn.sendall(INIT_Length)
conn.sendall(TF_Pickles)
conn.sendall(INIT_Status)
conn.sendall(TF_Status)
break
elif SERVERCODE == 919:
pickle_status = pickle.dumps(statuslist)
len_statpick = self.Pack_data(pickle_status)
conn.sendall(len_statpick)
conn.sendall(pickle_status)
break
TCP_IP = '127.0.0.1'
TCP_PORT = 8079
BUFFER_SIZE = 1024
tcpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcpServer.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcpServer.bind((TCP_IP, TCP_PORT))
threads = []
while True:
tcpServer.listen(4)
print(" : Waiting for connections from TCP clients...")
(conn, (ip, port)) = tcpServer.accept()
newthread = ClientThread(ip, port)
newthread.start()
threads.append(newthread)
Client Code
servercode = StructPack(919)
self.s.sendall(servercode)
buf = GetLength(self.s)
stats = GetSpecificAmount(self.s, buf)
print(stats)
def GetLength(socket):
buf = b''
while len(buf) < 4:
# print('waiting...')
tbuf = socket.recv(4 - len(buf))
if tbuf == '':
raise RuntimeError("Lost connection with peer")
buf += tube
SERVERCODE = struct.unpack('!I', buf)[0]
return SERVERCODE
def GetSpecificAmount(socket, Amount_2Recieve):
data = []
len_recvd = 0
while len_recvd < Amount_2Recieve:
buf = socket.recv(Amount_2Recieve - len_recvd)
if buf == '':
raise RuntimeError("Lost connection with peer")
data.append(buf)
len_recvd += len(buf)
# print(len_recvd)
df = pickle.loads(b''.join(data))
return df
I made an app to send to python socket the message "Oleft" when i tilt the phone but the result on console is like:
connection from('192.168.0.101', 33313)
b'Oleft'
b'OleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'Oleft'
b'OrightOrightOrightOright'
b'OleftOleftOleftOleftOleftOrightOrightOrightOrightOrightOrightOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'OleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'OleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'OleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'OleftOleft'
It has no way to receive only an b'Oleft' ?
wgremote.py
import socket
class WGRemote:
def __init__(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.host = socket.gethostname()
self.port = 10000
self.received = None
def connect(self):
global c, addr
self.sock.bind((self.host, self.port))
self.sock.listen(5)
c, addr = self.sock.accept()
print('connection from' + str(addr))
def setMode(self,mode):
sent = c.send(mode.encode("utf-8"))
def receive(self):
self.received = c.recv(1024)
return self.received
def close(self):
c.close
testApp.py
import socket
import sys
from wgremote import WGRemote
remote = WGRemote()
remote.connect()
remote.setMode('orient')
while True:
data = remote.receive()
print(data)
Because, You wrote print(data) in the while loop :)
while True:
data = remote.receive()
print(data)
If you want to receive only an b'Oleft you can make a filter :
count = 0
while True:
data = remote.receive()
if data == "b'Oleft":
if count < 1:
count = count + 1
print(data)
Or you must send the only an b'Oleft :)
Recently I wrote some code (client and server) to send an image - the client simply uploads the image to the server, just using the socket module: Sending image over sockets (ONLY) in Python, image can not be open.
However, the image sending part is now what I am concerned with. This is the original image I'm using:
In my server code (which receives the images), I have these lines:
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
data = sock.recv(40960000)
if not data:
myfile.close()
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
But I don't think this is the best way of doing it. I think I should instead implement the server such that it receives the data in chunks:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = 0
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()
But when I do this, the server prints:
got size 54674
4096
8192
12288
16384
20480
24576
28672
32768
36864
40960
45056
49152
50578
And then seems to hang, and the client hangs too. However, at the server's side I can see only a piece of the image the client wanted to send:
It seems like there are some bytes missing. What is the proper way of sending a huge amount of data (images, other type of file) using ONLY sockets?
I'm assuming that you have a particular reason for doing this with naked sockets, such as self-edification, which means that I won't answer by saying "You accidentally forgot to just use HTTP and Twisted", which perhaps you've heard before :-P. But really you should look at higher-level libraries at some point as they're a lot easier!
Define a protocol
If all you want is to send an image, then it can be simple:
Client -> server: 8 bytes: big endian, length of image.
Client -> server: length bytes: all image data.
(Client <- server: 1 byte, value 0: indicate transmission received - optional step you may not care if you're using TCP and just assume that it's reliable.)
Code it
server.py
import os
from socket import *
from struct import unpack
class ServerProtocol:
def __init__(self):
self.socket = None
self.output_dir = '.'
self.file_num = 1
def listen(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.bind((server_ip, server_port))
self.socket.listen(1)
def handle_images(self):
try:
while True:
(connection, addr) = self.socket.accept()
try:
bs = connection.recv(8)
(length,) = unpack('>Q', bs)
data = b''
while len(data) < length:
# doing it in batches is generally better than trying
# to do it all in one go, so I believe.
to_read = length - len(data)
data += connection.recv(
4096 if to_read > 4096 else to_read)
# send our 0 ack
assert len(b'\00') == 1
connection.sendall(b'\00')
finally:
connection.shutdown(SHUT_WR)
connection.close()
with open(os.path.join(
self.output_dir, '%06d.jpg' % self.file_num), 'w'
) as fp:
fp.write(data)
self.file_num += 1
finally:
self.close()
def close(self):
self.socket.close()
self.socket = None
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
sp = ServerProtocol()
sp.listen('127.0.0.1', 55555)
sp.handle_images()
client.py
from socket import *
from struct import pack
class ClientProtocol:
def __init__(self):
self.socket = None
def connect(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.connect((server_ip, server_port))
def close(self):
self.socket.shutdown(SHUT_WR)
self.socket.close()
self.socket = None
def send_image(self, image_data):
# use struct to make sure we have a consistent endianness on the length
length = pack('>Q', len(image_data))
# sendall to make sure it blocks if there's back-pressure on the socket
self.socket.sendall(length)
self.socket.sendall(image_data)
ack = self.socket.recv(1)
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
cp = ClientProtocol()
image_data = None
with open('IMG_0077.jpg', 'r') as fp:
image_data = fp.read()
assert(len(image_data))
cp.connect('127.0.0.1', 55555)
cp.send_image(image_data)
cp.close()
A simple way is to send data size as the first 4 bytes of your data and then read complete data in one shot. Use the below functions on both client and server-side to send and receive data.
def send_data(conn, data):
serialized_data = pickle.dumps(data)
conn.sendall(struct.pack('>I', len(serialized_data)))
conn.sendall(serialized_data)
def receive_data(conn):
data_size = struct.unpack('>I', conn.recv(4))[0]
received_payload = b""
reamining_payload_size = data_size
while reamining_payload_size != 0:
received_payload += conn.recv(reamining_payload_size)
reamining_payload_size = data_size - len(received_payload)
data = pickle.loads(received_payload)
return data
you could find sample program at https://github.com/vijendra1125/Python-Socket-Programming.git
The problem is you are not incrementing amount_received for the first chunk of the data received.
Fix below:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = len(data) # The fix!
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()