Develop Reference

Python socket server, limiting data received

When trying to send a file with sockets from one device on my network to another it is throttling the data to 2760 bytes and not receiving the whole file on the server side
Here is the code of my server on my raspberry pi:
import socket
import os
host = "my_ip"
port = "my_port"
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
s.listen()
def send(mess):
con.send(bytes(mess, "utf-8"))
def receive():
global message
message = con.recv(22000).decode('utf-8')
while True:
try:
con, address = s.accept()
print(f"Connection to {address} made successfully")
receive()
if message == "quit":
break
else:
send("received")
print(len(message))
with open("file.py", 'w', encoding='utf-8')as a:
a.write(message)
os.system("python3 file.py")
except:
pass
And here is the code of my client on another device
with open('file.py', 'r', encoding = 'utf-8')as f:
python_file = f.read()
print(len(python_file))
#returns 20940
import socket
host = "my_ip"
port = my_port
def send(mess):
s.send(bytes(mess, 'utf-8'))
def receive():
message = s.recv(1024).decode('utf-8')
print(message)
while True:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
send_it = input("Send File?[y/n]: ")
if send_it == "y":
try:
s.connect((host, port))
send(python_file)
recieve()
except:
pass
if send_it == "quit":
try:
s.connect((host, port))
send(send_it)
recieve()
except:
pass
else:
pass
When i take off the try loop it doesn't give any errors or reason as to why im only receiving part (2760 bytes) of the file. Also it will randomly(rarely) send more than 2760 bytes or it will sometimes send all the bytes. Its pretty consistently only sending 2760 bytes though.

Maybe you want to use sendall() instead of send(). Basically sendall() guarantees that all data will be sent unless an exception is raised and it's a python feature that is offered by other high level languages as well.
In order to receive all bytes sent, it's a good idea to do that using a loop:
data = bytearray(1)
# loop until there is no more data to receive.
while data:
data = socket.recv(1024) # Receive 1024 bytes at a time.
There is a well explained answer on what the difference between send() and sendall() is in python here.

How to send and receive from the same socket in Python?

I'm am trying to write a client program in Python that can send and receive from the same socket, but it is always giving me the same error which address is already in use. Here is the function I'm trying to write.
def Login():
username=raw_input()
password=raw_input()
message=raw_input()
array=[username,password,message]
TCP_IP = '127.0.0.1'
TCP_PORT = 5563
BUFFER_SIZE = 1024 # Normally 1024, but we want fast response
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((TCP_IP, TCP_PORT))
array_string=pickle.dumps(array)
sock.send(array_string)
sock.close()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((TCP_IP, TCP_PORT))
sock.listen(1)
conn, info = sock.accept()
while 1:
data = serverSocket.recv(1024)
if not data:break
conn.send(data)
conn.close()

There is a bunch of truly newbie errors here.
You can't ever connect a TCP socket to itself. There must be two different sockets.
If you really want to get the data you sent earlier at a listening socket, this listening socket must be created, bound and configured to listen before the client side connects (or, at least, in parallel to this connect attempt, in a few seconds, so the connect attempt will try - but this very likely won't work on localhost).
You can't wait on connect and on accept in the same thread if both are blocking. The simplest approach is to separate the client side and the server side to 2 different programs and run them manually in parallel. Then, after successful debugging, you will be able to do this in different threads of the same process, or using an event-driven engine.

While you may not be able to connect a socket to itself to send and receive data, you might be able to learn from the following example inspired by your code that attempts to do something similar.
import _thread
import pickle
import socket
import time
def main():
"""Run a server in a thread and start a client to talk to it."""
_thread.start_new_thread(run_server, ('', 5563))
run_client('localhost', 5563)
def run_server(host, port):
"""Handle all incoming connections by spawning worker threads."""
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind((host, port))
server.listen(5)
while True:
_thread.start_new_thread(handle_connection, server.accept())
def handle_connection(client, address):
"""Answer an incoming question from the connected client."""
print('Incoming connection from', address)
client.settimeout(0.1)
data = recvall(client)
client.shutdown(socket.SHUT_RD)
question = pickle.loads(data)
answer = '''len(username) = {}
len(password) = {}
len(message) = {}'''.format(*map(len, question))
client.sendall(answer.encode())
client.shutdown(socket.SHUT_WR)
client.close()
print('Finished with', address)
def recvall(connection):
"""Receive all data from a socket and return as a bytes object."""
buffer = bytearray()
while True:
try:
data = connection.recv(1 << 12)
except socket.timeout:
pass
else:
if data:
buffer.extend(data)
else:
return bytes(buffer)
def run_client(host, port):
"""Collect information from question and display returned answer."""
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
time.sleep(0.1) # wait for server to start listening for clients
client.connect((host, port))
time.sleep(0.1) # wait for handler thread to display connection
username = input('Username: ')
password = input('Password: ')
message = input('Message: ')
question = pickle.dumps((username, password, message))
client.sendall(question)
client.shutdown(socket.SHUT_WR)
answer = recvall(client)
client.shutdown(socket.SHUT_RD)
client.close()
print(answer.decode())
time.sleep(0.1) # wait for handler to cleanly terminate execution
if __name__ == '__main__':
main()

Python error: [Errno 111] Connection refused

I am trying to send and receive data using TCP connection using Python. My server and client are in the same file, defined and used as follows.
In the constructor, I define the server as:
self.sock_in = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock_in.bind((self.host_ip, self.host_port))
self.sock_in.listen(1)
Do not worry about the host_ip and host_port variables, they are all fine.
In a function, I am trying to send data as follows:
sock_out = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP Connection
sock_out.connect((self.remote_ip, self.remote_port))
sock_out.send(self.navigation_data.get_message())
sock_out.close()
And this is my main:
def main(self):
rospy.logwarn("Starting...")
while not rospy.is_shutdown():
conn = self.sock_in.accept()
try:
recv_buffer = conn.recv(BUFFERSIZE_IN)
if recv_buffer != "":
msg = recv_buffer.decode('utf-8')
msg_type = msg[:msg.find(',')]
if msg_type == self.pilot_control.MESSAGE_ID:
self.pilot_control_handler(msg, self.pilot_control_publisher)
else:
rospy.logwarn("Received an unimplemented message type '%s'", msg_type)
except socket.error as socket_error:
rospy.logerr("SocketError: %s", str(socket_error))
And the error I get is:
line 230, in send_83b_package
sock_out.connect((self.remote_ip, self.remote_port))
File "/usr/lib/python2.7/socket.py", line 224, in meth
return getattr(self._sock,name)(*args)
error: [Errno 111] Connection refused
I put some print commands to see where it collapses, and apparently it does not run the accept command. Until there I can see the print commands working, but after the accept method nothing is printed, which means it collapses there.
I suspect the problem is about synchronization. That is, the server does not start fast enough.
Any thoughts?
EDIT:
One of the suggestions was to run the server on a separate thread, which I tried as follows:
def my_tcp_server(self):
# Establish a TCP Connection
self.sock_in = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock_in.bind((self.host_ip, self.host_port))
self.sock_in.listen(1)
rospy.logwarn("ready")
while not rospy.is_shutdown():
rospy.logwarn("before accept")
conn, address = self.sock_in.accept()
rospy.logwarn("after accept")
try:
recv_buffer = conn.recv(BUFFERSIZE_IN)
rospy.logwarn("recv works!")
if recv_buffer != "":
msg = recv_buffer.decode('utf-8')
msg_type = msg[:msg.find(',')]
if msg_type == self.pilot_control.MESSAGE_ID:
self.pilot_control_handler(msg, self.pilot_control_publisher)
else:
rospy.logwarn("Received an unimplemented message type '%s'", msg_type)
except socket.error as socket_error:
rospy.logerr("SocketError: %s", str(socket_error))
conn.close()
def main(self):
rospy.logwarn("Starting..")
threading.Thread(target=self.my_tcp_server).start()
And in my constructor, the order of calls are as follows:
self.main()
self.sendDataFunction()
Which should be okay. However, the accept function is still not working, hence there is no connection.

As you didn't provide a complete executable code example I took your snippet and removed the class declaration aspects, added definitions for ip/port etc. Also added socket timeout. Anywayt this code works for me on Windows 7x64 with 32-bit Python 2.7.8:
import threading
import socket
is_shutdown = False
BUFFERSIZE_IN = 32768
def my_tcp_server():
# Establish a TCP Connection
sock_in = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock_in.bind((host_ip, host_port))
sock_in.settimeout(10000)
sock_in.listen(1)
print "ready"
while not is_shutdown:
print "before accept"
conn, address = sock_in.accept()
print "after accept"
try:
recv_buffer = conn.recv(BUFFERSIZE_IN)
print "recv works!"
if recv_buffer != "":
msg = recv_buffer.decode('utf-8')
print "Received",msg
except socket.error as socket_error:
print "SocketError: %s", str(socket_error)
conn.close()
print "Shutting down server"
sock_in.close()
def main():
print "Starting.."
threading.Thread(target=my_tcp_server).start()
def sendData():
sock_out = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP Connection
sock_out.connect((remote_ip, remote_port))
sock_out.send("ASD")
sock_out.close()
host_ip="127.0.0.1"
remote_ip = host_ip
host_port = 8073
remote_port = host_port
main()
print "Sending"
sendData()
print "Completed"
is_shutdown = True
The output is:
Starting..
Sending
ready
before accept
Completedafter accept
recv works!
Received ASD
Shutting down server
I guess there is something in your class/constructor/something I can't see which is making your code not work.
HTH
barny

Python Socket Scripting. What am i doing wrong?

My socket program hangs at clientsocket, address) = serversocket.accept() and doesn't spit our an error or anything.
I followed directions on https://docs.python.org/3/howto/sockets.html
I've been trying to figure it out for an hour now, but to no avail. I'm using python3 btw. What am i doing wrong? EDIT: My intedentation is all screwed up because I pasted it wrong, but other than that my code is as I have it in my file.
#import socket module
import socket
#creates an inet streaming socket.
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print('socket created')
#binds socket to a public host, and a well known port
serversocket.bind(('127.0.0.1', 1024))
#print(socket.gethostname())# on desktop prints 'myname-PC')
#become a server socket
serversocket.listen(5) # listens for up to 5 requests
while True:
#accept connections from outside
#print('In while true loop') This works, but we never get to the next print statement. Why the hell is it catching at line 20?
(clientsocket, address) = serversocket.accept()
#clientsocket = serversocket.accept()
print('Ready to serve')
#now we do something with client socket...
try:
message = clientsocket.recv(1024)
filename = message.split()[1]
f = open(filename[1:])
outputdata = f.read()
#send an http header line
clientsocket.send('HTTP/1.1 200 OK\nContent-Type: text/html\n\n')
for i in range(0, len(outputdata)):
clientsocket.send(outputdata[i])
clientsocket.close()
except IOERROR:
clientsocket.send('HTTP/1.1 404 File not found!')
clientsocket.close()

If you haven't written a client script / program to connect to the socket and send it data, it's also going to hang on serversocket.accept() due to there being nothing to accept. But assuming you have...
while True:
#accept connections from outside
#print('In while true loop') This works, but we never get to the next print statement. Why the hell is it catching at line 20?
(clientsocket, address) = serversocket.accept()
#clientsocket = serversocket.accept()
It hangs because the loop never exits due to True always being True. In the example provided, once a connection is accepted they pretend that the server is threaded and the idea is to create a separate thread to begin reading and processing data received allowing the socket to continue to listen for more connections.
while True:
# accept connections from outside
(clientsocket, address) = serversocket.accept()
# now do something with the clientsocket
# in this case, we'll pretend this is a threaded server
ct = client_thread(clientsocket)
ct.run()

How to keep a socket open until client closes it?

I have simple python server and client.
Server:
import SocketServer
import threading
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
print str(self.client_address[0]) + " wrote: "
print self.data
self.request.send(self.data.upper())
if __name__ == "__main__":
HOST, PORT = "localhost", 3288
server = SocketServer.TCPServer((HOST, PORT), MyTCPHandler)
server.serve_forever()
Client:
import socket
import sys
from time import sleep
HOST, PORT = "localhost", 3288
data = "hello"
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
sock.send(data + "\n")
received = sock.recv(1024)
sleep(10)
sock.send(data + "\n")
received = sock.recv(1024)
sleep(10)
sock.send(data + "\n")
received = sock.recv(1024)
finally:
sock.close()
Here is the output I get:
Server:
>python server.py
127.0.0.1 wrote:
hello
Client:
>python client.py
Traceback (most recent call last):
File "client.py", line 18, in <module>
received = sock.recv(1024)
socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
I tried it on a linux machine as well. The server only receives one message and then I get an error on the recv statement of second message. I have just started learning networking on python but I think the server is closing the socket for some reason. How do I correct this?

A MyTcpHandler object is created for each connection, and handle is called to deal with the client. The connection is closed when handle returns, so you have to handle the complete communication from the client within the handle method:
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
while 1:
self.data = self.request.recv(1024)
if not self.data:
break
self.data = self.data.strip()
print str(self.client_address[0]) + " wrote: "
print self.data
self.request.send(self.data.upper())
NOTE: recv returns '' when the client closes the connection, so I moved .strip() after the recv so there is no false alarm due to the client sending only white space.

I'll first admit that it's been years since I last used SocketServer, so there might be more idiomatic approaches to solve your problem.
Note that your client opens a single connection and sends three sets of data and receives three sets of data. (Hopefully the TCP stack will send buffered data once you call receive() on the socket.)
Your server is expecting to handle a client connection completely, from start to finish, when it is called from the SocketServer callback mechanism. Your current class does a little bit of IO and then quits. You just need to extend your server callback to do more:
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
print str(self.client_address[0]) + " wrote: "
print self.data
self.request.send(self.data.upper())
foo = self.request.recv(1024).strip()
self.request.send(foo.lower())
bar = self.request.recv(1024).strip()
self.request.send("goodbye " + bar)

TO a similar problem here error: [Errno 10053]
I also tried the same thing and got the same error.
If there is a simple code like this to demonstrate this error:
import socket
host = 'localhost'
port = 5001
size = 102400
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,port))
for msg in ['Hello, world','Test','anything goes here']:
s.send(msg)
data = s.recv(size)
print 'Received:', data
s.close()
If you create a socket object and the amt it can send and echo back from server to see how much it receivers, if you vary that, say 1024 to 102400(in this code);
Which means the socket should not get closed but again in my Windows OS, the server side keeps listening and printing any data that client sends but on the Client side you get this error;
However if it is that the client can connect only once and send and receive only once, then that is how it was designed. Trying this works without any errors:
for msg in ['Hello, world','Test','anything goes here']:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,port))
s.send(msg)
data = s.recv(size)
s.close()
print 'Received:', data
I am not sure if one socket object works only once to send and recieve data.
UPDATE
I think the issue was the capacity per client socket to receive data as per the buffersize fixed;
That's why the second code snippet above works thus creating new client connection sockets on the server. But that way lots of sockets are going to get used up.
Instead the following code fixed that problem by checking the amt of size being used up. If it exceeds the given amount, it creates a new socket at clients' but makes sure the message is sent; Actually the problem was with the server code but fixed it.
size = 10
This is a quick baby attempt at the code. I am sure you would understand and optimize it for the better!
client code:
messag = ['Hello, world', 'Test', 'anything goes here']
def client_to_server(messag,host,port,size):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
countmsg = 0
restmsg = ''
for msg in messag:
strl = tmsg = msg
if len(restmsg):
tmsg = restmsg + ' ' + msg
countmsg = len(tmsg)
if countmsg <= size:
pass
else:
restmsg = tmsg[size:]
tmsg = tmsg[:size]
#s.close()
countmsg = len(tmsg)
#s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.connect((host, port))
print 'Sending to server msg {}'.format(tmsg)
s.send(tmsg)
# s.settimeout(1)
try:
data = s.recv(size)
print 'Received:', data
if strl == data:
print strl,data
countmsg = 0
restmsg = ''
except (socket.error), e:
print e.args,e.message
s.close()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
s.close()
if restmsg:
client_to_server([restmsg],host,port,size)
return
client_to_server(messag,host,port,size)
Server Code:
size = 1024 #This has to be bigger than client buf size!
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
s.listen(backlog)
while True:
#this is what accepts and creates a P2P dedicated client socket per socket
client, address = s.accept()
try:
data = client.recv(size)
while data or 0:
print "Client sent {} | Server sending data to client address {}".format(data, address)
client.send(data)
data = client.recv(size)
else: client.close()
except (socket.error), e:
client.close()
print e.args, e.message
Try it out. This uses the same socket.