I try to understand concurrency from David Beazley talks. But when executing the server and client and try to submit the number 20 fro client, is seem that the futur object block forever when calling futur.result(). I can't understand why:
# server.py
# Fib microservice
from socket import *
from fib import fib
from threading import Thread
from concurrent.futures import ProcessPoolExecutor as Pool
pool = Pool(4)
def fib_server(address):
sock = socket(AF_INET, SOCK_STREAM)
sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sock.bind(address)
sock.listen(5)
while True:
client, addr = sock.accept()
print("Connection", addr)
Thread(target=fib_handler, args=(client,), daemon=True).start()
def fib_handler(client):
while True:
req = client.recv(100)
if not req:
break
n = int(req)
future = pool.submit(fib, n)
#Next line will block!!!!
result = future.result()
resp = str(result).encode('ascii') + b'\n'
client.send(resp)
print("Closed")
fib_server(('',25000))
#client.py
import socket
s = socket.socket()
s.connect(('localhost',25000))
while True:
num=input("number?")
s.send(str(num).encode('ascii') + b'\n')
res = s.recv(1000)
print('res:',res)
server> python server.py
client> python client.py
We see in order:
server> Connection ('127.0.0.1', 57876)
client> number?20
server>[freeze]
Finally this post help me to solve the problem: All example concurrent.futures code is failing with "BrokenProcessPool".
"Under Windows, it is important to protect the main loop of code to avoid recursive spawning of subprocesses when using processpoolexecutor or any other parallel code which spawns new processes.
Basically, all your code which creates new processes must be under if name == 'main': , for the same reason you cannot execute it in interpreter."
Related
I use a client-server socket connexion to transfer some data from my Python server. The problem I have at the moment is that the creation of the server socket block the programm because it can't connect to the client.
I tried to use async but I didn't succeed
from flask import *
import random
import socket
import json
app = Flask(__name__, static_url_path='')
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.bind(('',55555))
async def acceptConnexion():
while True:
socket.listen(10)
client, address = socket.accept()
print("{} connected".format( address ))
#app.route('/getInfos')
def getInfos():
global infosThymio
return json.dumps(infosThymio)
if __name__ == '__main__':
app.run()
I don't know where I could call my acceptConnexion() and I don't know how to manage to let this method turn in background until it can do the connexion with the client.
You can separate the accept call to a separate thread, in this way and let the main thread continue while the side thread waits on accept.
import socket
from threading import Thread
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.bind(('',55555))
def acceptConnexion():
print("running in thread")
while True:
socket.listen(10)
address = socket.accept()
print("{} connected".format( address ))
if __name__ == "__main__":
thread = Thread(target = acceptConnexion)
print("you can here do bla bla")
x = 1
print("x", x)
print("Main thread will wait here for thread to exit")
thread.join()
print("thread finished...exiting")
I have a server with main process acepting socket connections and put them in Queue stack and another process monitoring this stack and applying it to pool processes handling connections. All works fine except for one thing:
last connection allways at stuck until another connection appears, it's look like last connection can't be closed, but why?
from multiprocessing import Queue, Process, Pool, Manager
import datetime
import socket
def get_date():
return datetime.datetime.now().strftime('%H:%M:%S')
class Server:
def __init__(self, host, port):
self.server_address = host, port
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def run(self):
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_socket.bind(self.server_address)
self.server_socket.listen(1)
print('listen at: %s:%s' % self.server_address)
q = Manager().Queue()
Process(target=self.handle_request, args=(q,)).start()
while True:
client_socket, adress = self.server_socket.accept()
print('\n[%s] request from: %s:%s' % (get_date(), *adress))
q.put(client_socket)
client_socket.close()
del client_socket # client_socket.close() not working
def help(self, client_socket):
data = client_socket.recv(512)
client_socket.send(data)
client_socket.close()
print(data[:50])
def handle_request(self, q):
with Pool(processes=2) as pool:
while True:
pool.apply_async(self.help, (q.get(),))
Server('localhost', 8000).run()
close doesn't realy close connection unless no other process holding a reference, but shutdown will affect all processes. you could call client_socket.shutdown(socket.SHUT_WR) before client_socket.close().
Update:
the reason close doesn't fully close connection is there is a process started by Manager() is holding a reference. Use Queue instead would make close works as you expected.
the server-side code (tcp_server.py):
from SocketServer import TCPServer, ThreadingMixIn, StreamRequestHandler
class Server(ThreadingMixIn, TCPServer):
pass
class Handler(StreamRequestHandler):
def handle(self):
print 'got a connection from: ', self.request.getpeername()
print self.rfile.read(1024)
msg = 'hello'
self.wfile.write(msg)
server = Server(('127.0.0.1', 8888), Handler)
server.serve_forever()
the client-side code (tcp_client.py):
from socket import *
import threading
def do_connection():
s = socket(AF_INET, SOCK_STREAM)
s.connect(('127.0.0.1', 8888))
s.sendall('this is client')
print s.recv(1024)
ts = []
for x in xrange(100):
print x
t = threading.Thread(target=do_connection())
t.daemon = True
ts.append(t)
for t in ts:
t.start()
I runned tcp_server.py, and then tcp_client.py. tcp_client.py should have been over soon. However, tcp_client.py seemed just run only one thread and blocked, and tcp_server.py got only one connection. When I interrupted tcp_client.py，tcp_server.py got one message this is client。
Is there any mistake in my code ?
This line:
t = threading.Thread(target=do_connection())
Should be
t = threading.Thread(target=do_connection)
When you use do_connection(), you end up executing do_connection in the main thread, and then pass the return value of that call to the Thread object. What you want to do is pass the do_connection function object to Thread, so that the Thread object can execute do_connection in a new thread when you call t.start.
Also, note that starting 100 threads concurrently to connect to your server may not perform very well. You may want to consider starting with fewer threads, and working your way up to a higher number once you know things are working properly.
because the server is blocked by the first request, I try to change the read(1024) to
readline in the server.py and add a '\n' to the content sended from the client, it
works.
it seems the rfile.read(1024) will block the how process, so the goodway is use readline
or use the self.request.recv(1024)
server.py
from SocketServer import TCPServer, ThreadingMixIn, StreamRequestHandler
class Server(ThreadingMixIn, TCPServer):
pass
class Handler(StreamRequestHandler):
def handle(self):
print 'got a connection from: ', self.request.getpeername()
print self.rfile.readline()
#print self.request.recv(1024).strip()
msg = 'hello'
self.wfile.write(msg)
# Create the server, binding to localhost on port 9999
server = Server(("127.0.0.1", 8888), Handler)
server.serve_forever()
client.py
from socket import *
import threading
def do_connection():
print "start"
s = socket(AF_INET, SOCK_STREAM)
s.connect(('127.0.0.1', 8888))
s.sendall('this is client\n')
print s.recv(1024)
ts = []
for x in xrange(100):
print x
t = threading.Thread(target=do_connection)
ts.append(t)
for t in ts:
print "start t"
t.start()
I have been trying to write a python script that initiates a thread to listen on a socket and send HTTP data to another application to be launched by the same program. There is a requirement for the socket server to be running prior to executing the application. However, the thread running the socket server blocks further execution of the program and it freezes where it is listening. Putting some dummy code.
In module 1:
def runServer(Port, Host, q):
HTTPServerObj = HTTPServer((Host, Port), RequestHandler)
HTTPServerObj.handle_request()
HTTPServerObj.server_close()
q.put((True, {'messageDoNotDuplicate': 'Data sent successfully by the server'}))
class SpoofHTTPServer(object):
def runServerThread(self):
q = Queue.Queue()
serverThread=Thread(target=runServer, args=(self.Port, self.Host, q))
serverThread.daemon=True
serverThread.start()
result = q.get()
print result
return result
In module 2:
from module1 import SpoofHTTPServer
spoofHTTPServer = SpoofHTTPServer()
result = spoofHTTPServer.runServerThread()
rc = myApp.start()
The myApp.start() never gets executed as the thread is blocking it.
It looks to me like the method that blocks execution is not the thread but q.get(). It will listen to the Queue until an item is available, but since it's executed before running the client application nothing ever gets posted into the queue. Maybe you should return q instead and listen to the queue in module 2 after calling myApp.start()?
This may work for you from Python 3. Make a connection to ('localhost', 8080) to see it work.
import queue as Queue
from threading import Thread
from http.server import HTTPServer
from socketserver import BaseRequestHandler as RequestHandler
def runServer(Port, Host, q):
HTTPServerObj = HTTPServer((Host, Port), RequestHandler)
HTTPServerObj.handle_request()
HTTPServerObj.server_close()
q.put((True, {'messageDoNotDuplicate':
'Data sent successfully by the server'}))
class SpoofHTTPServer(object):
Port = 8080
Host = ''
def runServerThread(self):
q = Queue.Queue()
serverThread=Thread(target=runServer, args=(self.Port, self.Host, q))
serverThread.daemon=True
serverThread.start()
result = q.get()
print(result)
return result
spoofHTTPServer = SpoofHTTPServer()
result = spoofHTTPServer.runServerThread()
##rc = myApp.start()
I'm writing a server which uses multiprocessing.Process for each client. socket.accept() is being called in a parent process and the connection object is given as an argument to the Process.
The problem is that when calling socket.close() the socket does not seem to be closing. The client's recv() should return immediately after close() has been called on the server. This is the case when using threading.Thread or just handle the requests in the main thread, however when using multiprocessing, the client's recv seem to be hanging forever.
Some sources indicate that socket objects should be shared as handles with multiprocessing.Pipes and multiprocess.reduction but it does not seem to make a difference.
EDIT: I am using Python 2.7.4 on Linux 64 bit .
Below are the sample implementation demonstrating this issue.
server.py
import socket
from multiprocessing import Process
#from threading import Thread as Process
s = socket.socket()
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('', 5001))
s.listen(5)
def process(s):
print "accepted"
s.close()
print "closed"
while True:
print "accepting"
c, _ = s.accept()
p = Process(target=process, args=(c,))
p.start()
print "started process"
client.py
import socket
s = socket.socket()
s.connect(('', 5001))
print "connected"
buf = s.recv(1024)
print "buf: '" + buf +"'"
s.close()
The problem is that the socket is not closed in the parent process. Therefore it remains open, and causes the symptom you are observing.
Immediately after forking off the child process to handle the connection, you should close the parent process' copy of the socket, like so:
while True:
print "accepting"
c, _ = s.accept()
p = Process(target=process, args=(c,))
p.start()
print "started process"
c.close()