Develop Reference

How do multi-threading and infinite loops work together

I'm trying to learn python, doing exercise watching video online. I'm currently trying to do a multi-threading chat server but I think I have trouble understanding the way threads work.
Here's my code:
<!-- language: lang-py -->
import socket
import threading
class ThreadedServer(threading.Thread):
def __init__(self, host, port, buff_size):
threading.Thread.__init__(self)
self.host = host
self.port = port
self.buff_size = buff_size
self.sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind((self.host, self.port))
def listen(self):
self.sock.listen(5)
while True:
print('Waiting for a connexion...')
self.c, self.addr = self.sock.accept()
listening = listenToClient(self.c, self.addr)
class listenToClient(threading.Thread):
def __init__(self, client, address):
threading.Thread.__init__(self)
global socks
socks.append(client)
client.send(("Bienvenue sur le serveur, entre ton pseudo: ").encode('utf-8'))
self.name = client.recv(buff_size).decode('utf-8')
welcome = "Welcome " + self.name + "! If ever you want to quit, type {quit}"
client.send(welcome.encode('utf-8'))
msg = self.name + " has joined the chat"
broadcast(msg)
while True:
msg = client.recv(buff_size).decode('utf-8')
if msg != '{quit}':
broadcast(msg, self.name + ": ")
print('TEST6')
else:
client.close()
leave_msg = self.name + " has left"
broadcast(leave_msg)
break
def broadcast(msg, prefix=''):
for sck in socks:
content = prefix + msg
sck.send(content.encode('utf-8'))
def Main():
ThreadedServer(host,port,buff_size).listen()
host = '127.0.0.1'
port = 10000
buff_size = 1024
socks = []
if __name__ == '__main__':
Main()
The code is working ok for only one user.
I think the problem is in my comprehension of threads.
In my mind, when I type
listening = listenToClient(self.c, self.addr)
a new thread is created running in parallel with the first one.
What I see in reality is that my new thread is indeed executing and ends in the infinite loop in here(managing whatever my first user connected is typing):
while True:
msg = client.recv(buff_size).decode('utf-8')
if msg != '{quit}':
broadcast(msg, self.name + ": ")
print('TEST6')
But I thought as the original thread would have continued looping here:
while True:
print('Waiting for a connexion...')
self.c, self.addr = self.sock.accept()
listening = listenToClient(self.c, self.addr)
Printing 'Waiting for a connection' and waiting for a second connection.
But it is not the case.
I'm pretty sure it's a misunderstanding on my side, I spent hours trying and trying changing everything etc, at some point the listenToClient() thread was just a method in the ThreadedServer(). But now I'm very lost and I don't know what to do, so I came here for help.
Thanks in advance,
Vincent

Getting data from a task to a connection with asyncio

So this should be fairly simple, i have a client that connects to a server, and it can receive messages from the server just fine. However i need to be able to send messages to the server. I am using asyncio to handle these asynchronously but i have a problem with that. How do i get the user input to my client so it can use its transport to send the data up to the server. Heres the code.
import asyncio, zen_utils, json
from struct import *
class ChatClient(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
self.address = transport.get_extra_info('peername')
self.data = b''
print('Accepted connection from {}'.format(self.address))
self.username = b'jacksonm'
json_name = b'{"USERNAME": "'+self.username+b'"}'
length = len(json_name)
code_len = pack(b'!I', length)
message = code_len + json_name
self.json_loaded = False
self.next_length = -1
self.transport.write(message)
def data_received(self, data):
self.data += data
if (self.json_loaded == False):
self.compile_server_data(data)
elif(self.json_loaded):
if (len(self.data) > 4 and self.next_length == -1):
self.next_length = self.data[:4]
self.next_length = unpack(b'!I', self.next_length)[0]
print("LENGTH: ", self.next_length)
elif (len(self.data) >= self.next_length):
self.data = self.data[4:]
print("MESSAGE: ", self.data)
self.next_length = -1
def compile_server_data(self, data):
if (self.data.find(b'}') != -1):
start_index = self.data.find(b'{')
end_index = self.data.find(b'}')
self.json_data = self.data[start_index:end_index + 1]
self.data = self.data[end_index + 1:]
self.json_data = self.json_data.decode('ascii')
self.json_data = json.loads(self.json_data)
self.json_loaded = True
self.print_server_status()
def send_message(self, message):
message = message.encode('ascii')
length = len(message)
code_len = pack(b'!I', length)
message = code_len + message
def parse_message(self, raw_message):
message = {}
message['SRC'] = self.username
message['DEST'] = b'ALL'
message['TIMESTAMP'] = int(time.time())
message['CONTENT'] = b'test_message'
json_message = json.loads(message)
print (json_message)
def print_server_status(self):
print ("USERS:")
for user in self.json_data["USER_LIST"]:
print(user)
print()
print("MESSAGES:")
for message in self.json_data["MESSAGES"][-10:]:
print ("From: ", message[0], " ", "To: ", message[1])
print ("Message: ", message[3])
print()
def get_inital_data(self):
pass
def connection_lost(self, exc):
if exc:
print('Client {} error: {}'.format(self.address, exc))
elif self.data:
print('Client {} sent {} but then closed'
.format(self.address, self.data))
else:
print('Client {} closed socket'.format(self.address))
#asyncio.coroutine
def handle_user_input(loop):
"""reads from stdin in separate thread
if user inputs 'quit' stops the event loop
otherwise just echos user input
"""
while True:
message = yield from loop.run_in_executor(None, input, "> ")
if message == "quit":
loop.stop()
return
print(message)
if __name__ == '__main__':
address = zen_utils.parse_command_line('asyncio server using callbacks')
loop = asyncio.get_event_loop()
coro = loop.create_connection(ChatClient, *address)
server = loop.run_until_complete(coro)
# Start a task which reads from standard input
asyncio.async(handle_user_input(loop))
print('Listening at {}'.format(address))
try:
loop.run_forever()
finally:
server.close()
loop.close()

This is working in my tests by changing one line, using Python 3.4:
Change the run_until_complete() call to receive a (transport,protocol) pair instead of a single value.
Specifically, change server = loop.run_until_complete(coro) to transport, protocol = loop.run_until_complete(coro), because the call executes the coroutine and returns its value which is a pair of items. The first item is the server object, and the second entry is a protocol object which is an instance of ChatClient.
Run the server then the client in different command windows.
Client code:
import asyncio, json #, zen_utils
from struct import *
class ChatClient(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
self.address = transport.get_extra_info('peername')
self.data = b''
print('Accepted connection from {}'.format(self.address))
def data_received(self, data):
pass
def compile_server_data(self, data):
pass
def send_message(self, message):
message = message.encode('ascii')
self.transport.write(message)
def parse_message(self, raw_message):
pass
def print_server_status(self):
pass
def get_inital_data(self):
pass
def connection_lost(self, exc):
if exc:
print('Client {} error: {}'.format(self.address, exc))
elif self.data:
print('Client {} sent {} but then closed'
.format(self.address, self.data))
else:
print('Client {} closed socket'.format(self.address))
#asyncio.coroutine
def handle_user_input(loop, protocol):
"""reads from stdin in separate thread
if user inputs 'quit' stops the event loop
otherwise just echos user input
"""
while True:
message = yield from loop.run_in_executor(None, input, "> ")
if message == "quit":
loop.stop()
return
print(message)
protocol.send_message(message)
if __name__ == '__main__':
address = "127.0.0.1"
port = 82
loop = asyncio.get_event_loop()
coro = loop.create_connection(ChatClient, address, port) #*address
transport, protocol = loop.run_until_complete(coro)
# Start a task which reads from standard input
asyncio.async(handle_user_input(loop,protocol))
print('Listening at {}'.format(address))
try:
loop.run_forever()
finally:
transport.close()
loop.close()
Server code, credit goes to this site:
import socket
host = '' # Symbolic name meaning all available interfaces
port = 82 # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
s.listen(1)
conn, addr = s.accept()
print('Connected by', addr)
while True:
data = conn.recv(1024)
if not data: break
print( 'Received ', repr(data) )
if repr(data)=="stop": break
conn.sendall(data)
conn.close()

Python Socketserver client timeout

I have a socketserver in Python which has to handle multiple clients using the select.select method, as seen in the code below:
import socket
import select
class Server:
def __init__(self):
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind(('localhost', 2000))
self.socket_list = [self.server]
self.addresses = ['localhost']
self.commands = [""]
self.running = True
self.server.listen(10)
def listen(self):
while self.running:
read, write, error = select.select(self.socket_list, [], self.socket_list, 0)
for sock in read:
if sock == self.server and self.running:
try:
conn, address = self.server.accept()
conn.settimeout(30)
self.socket_list.append(conn)
self.addresses.append(address[0])
self.commands.append("")
except:
self.shutdown()
break
elif self.running:
try:
packet = sock.recv(60)
if not packet:
self.close_conn(sock)
index = self.socket_list.index(sock)
self.commands[index] += packet
if '\n' in self.commands[index]:
#handle command
except:
self.close_conn(sock)
def close_conn(self, conn):
#close client conn
def shutdown(self):
#shutdown server
if __name__ == "__main__":
Server().listen()
The problem i currently have is that the client's connection should be closed after 30 seconds, but that doesn't happen, even though i declared that by using conn.settimeout(30). I haven't found any explanation yet as to why this happens.
Note: comments were used to replace parts of the code that didn't mater to the problem.

Python - multithreaded sockets

From my understanding python can only run 1 thread at a time so if I were to do something like this
import socket, select
from threading import Thread
import config
class Source(Thread):
def __init__(self):
self._wait = False
self._host = (config.HOST, config.PORT + 1)
self._socket = socket.socket()
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock = None
self._connections = []
self._mount = "None"
self._writers = []
self._createServer()
Thread.__init__(self)
def _createServer(self):
self._socket.bind(self._host)
self._socket.listen(2)
self._connections.append(self._socket)
self._audioPackets=[]
def _addPacket(self, packet):
self._audioPackets.append(packet)
def _removePacket(self, packet):
self._audioPackets.remove(packet)
def _getPacket(self):
if len(self._audioPackets) > 0:
return self._audioPackets[0]
else:
return None
def _sendOK(self, sock):
sock.send("OK")
def _sendDenied(self, sock):
sock.send("DENIED")
def _sendMount(self, sock):
sock.send("mount:{0}".format(self._mount))
def _sendBufPacket(self, sock, packet):
packet = "buffer:%s" % packet
sock.send(packet)
def recv(self, sock, data):
data = data.split(":", 1)
if data[0] == "WAIT": self._wait = True
elif data[0] == "STOP_WAITING": self._wait = False
elif data[0] == "LOGIN":
if data[1] == config.SOURCE_AUTH:
self._source = sock
self._sendOK(sock)
else:
self._sendClose(sock)
elif data[0] == "MOUNT":
if self._source == sock:
self._mount = data[1]
else:
self._sendClose(sock)
elif data[0] == "CLIENT":
self._sendMount(sock)
self._writers.append(sock)
def _sendCloseAll(self):
for sock in self._connections:
sock.send("CLOSE")
sock.close()
def _sendClose(self, sock):
sock.send("CLOSE")
sock.close()
def main(self):
while True:
rl, wl, xl = select.select(self._connections, self._writers, [], 0.2)
for sock in rl:
if sock == self._socket:
con, ip = sock.accept()
self._connections.append(con)
else:
data = sock.recv(config.BUFFER)
if data:
self.recv(sock, data)
else:
if sock in self._writers:
self._writers.remove(sock)
if sock in self._connections:
self._connections.remove(sock)
for sock in wl:
packet = self._getPacket()
if packet != None:
self._sendBufPacket(sock, packet)
def run(self):
self.main()
class writeThread(Thread):
def __init__(self):
self.running = False
def make(self, client):
self.client = client
self.running = True
def run(self):
host = (config.HOST, config.PORT+1)
sock = socket.socket()
sock.connect(host)
sock.send("CLIENT")
sock.send("MOUNT:mountpoint")
while self.running:
data = sock.recv(config.BUFFER)
if data:
data = data.split(":", 1)
if data[0] == "buffer":
self.client.send(data[1])
elif data[0] == "CLOSE":
self.client.close()
break
if __name__=="__main__":
source = Source()
source.start()
webserver = WebServer()
webserver.runloop()
if I need to build the webserver part I will. But, I'll explain it.
Okay, so basically when someone connects to the websever under the mountpoint that was set, They will get there own personal thread that then grabs the data from Source() and sends it to them. Now say another person connects to the mount point and the last client as well as the source is still going. Wouldn't the new client be blocked from getting the Source data considering there are two active threads?

Your understanding of how Threads work in Python seems to be incorrect, based on the question you are asking. If used correctly, threads will not be blocking: you can instantiate multiple thread with Python. The limitation is that, due to the Global Interpreter Lock (GIL), you cannot get the full parallelism expected in thread programming (e.g. simultaneous execution and thus, reduced runtime).
What is going to happen in your case is that the two threads will take, together, the same amount of time that they would take if they were executed sequentially (although that is not necessarily what happens in practice).

Okay, I have copy and pasted some Python3 code that I have already written for a project that I am currently working on. With modification, you can make this code serve your purposes.
The code uses multiprocessing and multithreading. For my purposes, I am using multiprocessing so that sockets will run on one processor, and I can run a GUI program on another processor. You can remove the multiprocessor part if you prefer. The code below runs a socket message server. The server will listen for clients one at a time. Once a client has connected, a new thread will be initiated to handle all the communications between the server and each client. The server will then continue to search for for clients. At the moment however, the server only listens to data being sent from each client, and then it prints it to the terminal. With a small amount of effort, you can modify my code to send information from the server to each client individually.
import multiprocessing
import threading
from threading import Thread
class ThreadedServer(object):
def __init__(self, host, port):
self.host = host
self.port = port
self.sock = socket(AF_INET, SOCK_STREAM)
self.sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.sock.bind((self.host, self.port))
def listen(self):
self.sock.listen(3) #Allow 3 Clients to connect to this server
while True:
#The program will search for one client at a time
print("Searching for Client")
client, address = self.sock.accept()
print(address, " is connected")
#client.settimeout(60)
#Once a client has been found, start a individual client thread
d = threading.Thread(target = self.listenToClient, args=(client, address))
d.daemon = True
d.start()
def listenToClient(self, client, address):
size = 1024
while True:
try:
data = client.recv(size)
if not data:
break
if data:
print(data)
#client.send(response)
else:
raise error('Client disconnected')
except:
client.close()
return False
def dataSharingHost():
#Using Sockets to send information between Processes
#This is the server Function
#ThreadServer(Host_IP, Port_Number), for LocalHost use ''
ThreadedServer('', 8000).listen()
def Main():
commServer = multiprocessing.Process(target=dataSharingHost, args=())
commServer.daemon = True
commServer.start()
if __name__== '__main__':
Main()
And to be fair, my code is modified from https://www.youtube.com/watch?v=qELZAi4yra8 . The client code is covered in those videos. I think the 3rd video covers the multiple client connects.

How can I stop udp server in threading?

I wrote a udp no echo server in my program, I used thread to running and listen some message send by others. But it seems I couldn't stop it use tl.stop(), when I input q or quit. Some of my code is following:
class treadListen(threading.Thread):
def __init__(self):
self.running = True
threading.Thread.__init__(self)
def run(self):
address = ('localhost', 16666)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(address)
while True:
data = sock.recv(65535)
print "MESSAGE:{0}".format(data)
sock.close()
def stop(self):
self.running = False
# end of class thread_clock
if __name__ == "__main__":
tl = treadListen()
tl.start()
while True:
message = raw_input("CMD>")
if not message:
print "Please input command!"
continue
elif (message == 'quit') or (message == 'q'):
tl.stop()
break
else:
print "input is {0}".format(message)
# do something
continue
print "[CONNECTION CLOSED!]"
I was trying to add sock.shutdown(socket.SHUT_RDWR) and sock.close() to def stop of class, but it doesn't work.
How can I stop the thread safety? Thanks!

Your while loop while True: works forever, so I guess your close or shutdown calls to the socket never can gets in way to work.
You should change while True: to while self.running: and that should do the trick.

Thanks ntki,rbp,st.
The problem solved use following code:
class treadListen(threading.Thread):
def __init__(self):
**self.running = True**
threading.Thread.__init__(self)
def run(self):
address = ('localhost', 16666)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
**sock.settimeout(1)**
sock.bind(address)
**while self.running:
try:
data = sock.recv(65535)
print "MESSAGE:{0}".format(data)
except Exception, e:
continue**
sock.close()
def stop(self):
**self.running = False**
# end of class thread_clock