I'm trying to implement UDP socket's threading.
I want to be able to wait for clients to send me some data in a thread and wait for first datas in an other.
import threading
import socket
class Broker():
def __init__(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.bind(('127.0.0.1', 4242))
self.clients_list = []
def talkToClient(self, ip):
self.sock.sendto("ok", ip)
def listen_clients(self):
while True:
msg, client = self.sock.recvfrom(1024)
t = threading.Thread(None, self.talkToClient, None, (client,), None)
b = Broker()
b.listen_clients()
and my client
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
sock.sendto("connection", ('127.0.0.1', 4242))
while True:
msg, b = sock.recvfrom(1024)
print msg
Problem is that my client is never receiving "ok"
Your main problem is that you are not starting the thread that you have created.
t.start()
Should do it. Please make sure you are using four spaces for indentation as well.
I didn't see the error first myself, but once I added some logging statements it was pretty obvious. The code ended up looking like this:
import threading
import socket
import logging
class Broker():
def __init__(self):
logging.info('Initializing Broker')
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.bind(('127.0.0.1', 4242))
self.clients_list = []
def talkToClient(self, ip):
logging.info("Sending 'ok' to %s", ip)
self.sock.sendto("ok", ip)
def listen_clients(self):
while True:
msg, client = self.sock.recvfrom(1024)
logging.info('Received data from client %s: %s', client, msg)
t = threading.Thread(target=self.talkToClient, args=(client,))
t.start()
if __name__ == '__main__':
# Make sure all log messages show up
logging.getLogger().setLevel(logging.DEBUG)
b = Broker()
b.listen_clients()
I'm afraid you will run into other problems however, because of your threaded solution. Most python modules are not thread-safe by default, unfortunately this is true for the socket module as well. I'm pretty sure that eventually your socket's internal state will be corrupted since you are reading in one thread and writing in another, or potentially in many others since you spawn a new process for each client.
If you look at multi-threaded socket code examples in Python, a socket is usually owned and used by only one thread. The key is to not reuse the listening socket for clients, but to use socket.accept to create a new socket for each client once it has connected.
Related
I just started programming Python.
My goal is to built a digital Picture Frame with three Screens. Therefore I use 3 Raspis, one for each Monitor.
For the communication of these Raspis I need to program a server and a Client.
For a first test I want to built a server which is able to send and receive messages to/from multiple clients.
So I started with a few socket tutorials an created the following program.
Server Class (TcpServer.py)
class TcpServer:
clients = []
serverIsRunning = 0
port = 0
def __init__(self, port):
self.port = port
self.serverIsRunning = 0
self.serverRunning = 0
def startServer (self):
print("start Server...")
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind(("", self.port))
self.server.listen(1)
self.serverRunning = 1
while self.serverRunning:
read, write, oob = select.select([self.server] + self.clients, [], [])
for sock in read:
if sock is self.server:
client, addr = self.server.accept()
self.clients.append(client)
print ("+++ Client ", addr[0], " verbunden")
else:
nachricht = sock.recv(1024)
ip = sock.getpeername()[0]
if nachricht:
print (ip, nachricht)
else:
print ("+++ Verbindung zu ", ip , " beendet")
sock.close()
self.clients.remove(sock)
for c in self.clients:
c.close()
self.clients.remove(c)
self.server.close()
def send(self, message):
message = message.encode()
self.server.send(message)
Client class (TcpClient.py)
import socket
class TcpClient:
def __init__(self, ip, port):
self.serverAdress = (ip, port)
self.connected = 0
self.connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.connection.connect(self.serverAdress)
print ("connectet to ", self.serverAdress)
def send(self, message):
message = message.encode()
self.connection.send(message)
Server:
import threading
import TcpServer
tcpServer = TcpServer.TcpServer(50000)
threadTcpServer = threading.Thread(target = tcpServer.startServer)
threadTcpServer.start()
while True:
tcpServer.send(input("Nachricht eingeben: "))
Client:
import threading
import TcpClient
tcpClient = TcpClient.TcpClient("192.168.178.49", 50000)
while True:
tcpClient.send(input("Nachricht eingeben: "))
I can send messages from the Client to the server, but when I want to send a Message from the server to the client it generates the following error:
BrokenPipeError: [Errno 32] Broken pipe
I assume it is because the server thread blocks the socket while waiting of a incoming message. But I have no idea how to handle this.
How can I program a server who can send and receive messages? Can you recommend a tutorial? I didn't found a tutorial who describes a solution for my problem.
Edit:
Now I tried to solve the problem with the socketserver library, but I still can't solve may problem.
here is my new code for the server:
import socketserver
import threading
import time
class MyTCPHandler(socketserver.BaseRequestHandler):
"""
The RequestHandler class for our server.
It is instantiated once per connection to the server, and must
override the handle() method to implement communication to the
client.
"""
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
print("{} wrote:".format(self.client_address[0]))
print(self.data)
# just send back the same data, but upper-cased
self.request.sendall(self.data.upper())
if __name__ == "__main__":
HOST, PORT = "localhost", 9999
# Create the server, binding to localhost on port 9999
server = socketserver.TCPServer((HOST, PORT), MyTCPHandler)
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
threadTcpServer = threading.Thread(target = server.serve_forever)
threadTcpServer.start()
print("server started")
time.sleep(10)
print("sending Data")
server.request.sendall("Server is sending...")
it generates the error:
AttributeError: 'TCPServer' object has no attribute 'request'
My goal is to write a server with a thread who receives Data and still be able to send data from a other thread.
Is this even possible with only one socket?
You should use the provided socketserver rather than writing all the handling of sockets and select etc.
There are multiple problems with your code -
1 - The server is trying to write to the listening socket!! The client communication socket is the one that you get from the accept() call and that is the one you have to use for reading and writing.
2 - The client is sending the data and completing immediately, but it should really wait for getting a response. Otherwise, the python / OS will close the client socket as soon as the program completes and it will mostly be before the server gets a chance to respond.
I believe with the Handler code you are able to receive the data sent by the client on the server and are also able to send some data back from the Handler to the client? You must have understood that the server cannot send any data back unless there is a client connected to it?
Now, to send data to the client (or clients) from "another" thread, you will need a way to make the handler objects or the client sockets (available inside the Handler object as self.request) available to the "another" thread.
One way is to override the def __init__(self, request, client_address, server): method and save this object's reference in a global list. Remember to do the below as the last line of the overridden init -
# BaseRequestHandler __init__ must be the last statement as all request processing happens in this method
socketserver.BaseRequestHandler.__init__(self, request, client_address, server)
Once you have all the client handlers in the global list, you can easily write to all the clients from any thread as per your needs. You must read about synchronization (Locks) and understand that using same object / socket from multiple threads can create some logical / data issues with your application.
Another thing that you have to worry about and code for is cleaning up this global list whenever a client closes the connection.
I have following code, server accept net connection, pass it to child to process with Manager().Queue():
q = Manager().Queue()
class Server:
def run(self, host, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
s.listen(1)
print('parent', os.getpid())
while True:
c, _ = s.accept()
q.put(c)
c.close()
def handle_request():
print('child', os.getpid())
while True:
c = q.get()
time.sleep(1)
print(c.recv(4))
c.close()
Process(target=handle_request, args=()).start()
Server().run('127.0.0.1', 10000)
close doesn't work as expected, I think it is because Manager's server process sill have a reference on that socket, lsof -i confirmed. How to deal with this? I found there is not a way to close the socket in Manager process, shutdown could do the trick but not what I want.
Interesting problem.
I am not sure if this is of any help, but I found your code somewhat odd in the beginning, as sending socket objects using Manager().Queue() to another process does not sound like it is supported. It may be, but sending a file descriptor to another process needs a couple of hoops. I changed your code a bit to do it as I would do it - basically reducing and reconstructing handles.
from multiprocessing import Manager, Process
from multiprocessing.reduction import reduce_handle, rebuild_handle
import socket
import os
from time import sleep
q = Manager().Queue()
class Server:
def run(self, host, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, port))
s.listen(1)
print('parent', os.getpid())
while True:
c, _ = s.accept()
foo = reduce_handle(c.fileno())
q.put(foo)
c.close()
def handle_request():
print('child', os.getpid())
while True:
bar = q.get()
sleep(1)
barbar = rebuild_handle(bar)
c = socket.fromfd(barbar, socket.AF_INET, socket.SOCK_STREAM)
print(c.recv(4))
c.shutdown(socket.SHUT_RDWR)
Process(target=handle_request, args=()).start()
Server().run('127.0.0.1', 10000)
This does not leave any sockets behind in CLOSE_WAIT at least when I ran it, and it works as I would expect it to work.
I'm am trying to write a client program in Python that can send and receive from the same socket, but it is always giving me the same error which address is already in use. Here is the function I'm trying to write.
def Login():
username=raw_input()
password=raw_input()
message=raw_input()
array=[username,password,message]
TCP_IP = '127.0.0.1'
TCP_PORT = 5563
BUFFER_SIZE = 1024 # Normally 1024, but we want fast response
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((TCP_IP, TCP_PORT))
array_string=pickle.dumps(array)
sock.send(array_string)
sock.close()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((TCP_IP, TCP_PORT))
sock.listen(1)
conn, info = sock.accept()
while 1:
data = serverSocket.recv(1024)
if not data:break
conn.send(data)
conn.close()
There is a bunch of truly newbie errors here.
You can't ever connect a TCP socket to itself. There must be two different sockets.
If you really want to get the data you sent earlier at a listening socket, this listening socket must be created, bound and configured to listen before the client side connects (or, at least, in parallel to this connect attempt, in a few seconds, so the connect attempt will try - but this very likely won't work on localhost).
You can't wait on connect and on accept in the same thread if both are blocking. The simplest approach is to separate the client side and the server side to 2 different programs and run them manually in parallel. Then, after successful debugging, you will be able to do this in different threads of the same process, or using an event-driven engine.
While you may not be able to connect a socket to itself to send and receive data, you might be able to learn from the following example inspired by your code that attempts to do something similar.
import _thread
import pickle
import socket
import time
def main():
"""Run a server in a thread and start a client to talk to it."""
_thread.start_new_thread(run_server, ('', 5563))
run_client('localhost', 5563)
def run_server(host, port):
"""Handle all incoming connections by spawning worker threads."""
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind((host, port))
server.listen(5)
while True:
_thread.start_new_thread(handle_connection, server.accept())
def handle_connection(client, address):
"""Answer an incoming question from the connected client."""
print('Incoming connection from', address)
client.settimeout(0.1)
data = recvall(client)
client.shutdown(socket.SHUT_RD)
question = pickle.loads(data)
answer = '''len(username) = {}
len(password) = {}
len(message) = {}'''.format(*map(len, question))
client.sendall(answer.encode())
client.shutdown(socket.SHUT_WR)
client.close()
print('Finished with', address)
def recvall(connection):
"""Receive all data from a socket and return as a bytes object."""
buffer = bytearray()
while True:
try:
data = connection.recv(1 << 12)
except socket.timeout:
pass
else:
if data:
buffer.extend(data)
else:
return bytes(buffer)
def run_client(host, port):
"""Collect information from question and display returned answer."""
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
time.sleep(0.1) # wait for server to start listening for clients
client.connect((host, port))
time.sleep(0.1) # wait for handler thread to display connection
username = input('Username: ')
password = input('Password: ')
message = input('Message: ')
question = pickle.dumps((username, password, message))
client.sendall(question)
client.shutdown(socket.SHUT_WR)
answer = recvall(client)
client.shutdown(socket.SHUT_RD)
client.close()
print(answer.decode())
time.sleep(0.1) # wait for handler to cleanly terminate execution
if __name__ == '__main__':
main()
I want to test a complex class, which wraps some methods of the socket module: connect, sendall and recv. Especially, I want to test the recv method of this class.
The working example code below shows how I could do that (in a basic, underlying form to keep it simple, testsocket would correspond to the complex wrapper class):
import socket
# This is just a socket for testing purposes, binds to the loopback device
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("127.0.0.1", 1234))
sock.listen(5)
# This is the socket later part of the complex socket wrapper.
# It just contains calls to connect, sendall and recv
testsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
testsocket.connect(("127.0.0.1", 1234))
testsocket.sendall("test_send")
# The testing socket connects to a client
(client, adr) = sock.accept()
print client.recv(1024)
# Now I can do the actual test: Test the receive method of the socket
# wrapped in the complex class
client.sendall("test_recv")
print testsocket.recv(1024) # <-- This is what I want to test !!
# close everything
testsocket.close()
client.close()
sock.close()
But in order to test testsocket.recv I need to use testsocket.sendall before.
Is it possible to modify this code in a simple way (without forks or threads) in order to test testsocket.recv without using the method testsocket.sendall?
How about using socket.socketpair? :
import socket
client, testsocket = socket.socketpair()
client.sendall("test_recv")
print testsocket.recv(1024)
testsocket.close()
client.close()
NOTE only available in Unix.
Using mock
import mock
testsocket = mock.Mock()
testsocket.configure_mock(**{'recv.return_value': 'test_recv'})
print testsocket.recv(1024)
You can't run the client and the server socket in the same process/thread since the server.recv() is a blocking call
my routine :
import socket, threading
# Protocols supported
TCP = (0x00)
UDP = (0x01)
UDP_Multicast = (0x02)
# Client/ Server mode
CLIENT = (0x00)
SERVER = (0x01)
# Data to be sent
data = 'Test. Please Ignore'
# Server processing
def simple_processing(data):
print "messsage : ", data
def start_socket( protocol, client, processing_callback):
# switch on protocol
if protocol == TCP:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
else:
return
# switch on client
if client == SERVER:
# Server mode = listening to incoming datas
sock.bind(("127.0.0.1", 1234))
sock.listen(5)
(sock, adr) = sock.accept()
processing_callback( sock.recv(1024) ) # processing data
elif client == CLIENT:
# Client mode : connecting and sending data
sock.connect(("127.0.0.1", 1234))
sock.sendall(data)
else:
return
sock.close()
def test():
# Thread creations
server = threading.Thread( target = start_socket,
args=( TCP,
SERVER,
simple_processing, )
)
client = threading.Thread( target= start_socket,
args=( TCP,
CLIENT,
None)
)
server.start()
client.start()
# Join : wait on every thread to finish
client.join()
server.join()
if __name__ == '__main__':
# Launch the test
test()
I'm completely lost trying to create a UDP server/client for my game in python. I'm new to the language and only have limited experience with networking. Right now, the server runs, but doesn't seem to be getting any messages from the client.
Server:
class GameServer:
class GameServerUDPHandler(socketserver.BaseRequestHandler):
def handle(self):
data = self.request[0].strip()
socket = self.request[1]
print("{} wrote:".format(self.client_address[0]))
print(data)
socket.sendto(data.upper(), self.client_address)
def __init__(self, port):
self.server = socketserver.UDPServer(("localhost", port), self.GameServerUDPHandler)
def start_server(self):
self.server.serve_forever(
Client:
import socket
import sys
class GameClient:
def __init__(self, port, host):
self.port = port
self.host = host
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def register(self):
self.socket.sendto(bytes("register\n", "utf-8"), (self.host, self.port))
self.numberID = int(self.socket.recv(1024))
print("Received: {}".format(self.numberID))
-Main/Start of program
import gameserver
import gameclient
if __name__ == "__main__":
server = gameserver.GameServer(1300)
server.start_server()
client = gameclient.GameClient(1300, "localhost")
client.register()
NOTE: I'm most likely to multiple things wrong and may be violating several best practices in the language. I really have no clue.
The problem is that some of these calls are blocking. In particular, the serve_forever() method will run forever, so you need to put that on a separate thread if you want the rest of your program to continue:
import threading
if __name__ == "__main__":
server = GameServer(1300)
server_thread = threading.Thread(target=lambda: server.start_server())
server_thread.start()
time.sleep(1) # Give it time to start up; not production quality code of course
client = GameClient(1300, "localhost")
client.register()
socket.recv() is also a blocking call but that might be okay in this case.
Seems like this library isn't asynchronous so your first call to serve_forever will not return and your client never gets started. You can create a new thread to launch the server on or split your client and server into seperate processes.