Develop Reference

The set pool.Pool in Gevent doesn't have effect

I try to test pool method in gevent module, so write some code like this:
# server
import sys
import socket
import time
import gevent
from gevent import socket,monkey,pool #导入pool
monkey.patch_all()
def server(port, pool):
s = socket.socket()
s.bind(('0.0.0.0', port))
s.listen()
while True:
cli, addr = s.accept()
#print("Welcome %s to SocketServer" % str(addr[0]))
pool.spawn(handle_request, cli) #通过pool.spawn()运行协程
def handle_request(conn):
try:
data = conn.recv(1024)
print("recv:", data)
data = 'From SockeServer:---%s' % data.decode("utf8")
conn.sendall(bytes(data, encoding="utf8"))
if not data:
conn.shutdown(socket.SHUT_WR)
except Exception as ex:
print(ex)
finally:
conn.close()
if __name__ == '__main__':
pool = pool.Pool(2) #限制并发协程数量5
server(8888, pool)
#client
import socket
import gevent
from gevent import socket, monkey
from gevent.pool import Pool
import time
monkey.patch_all()
HOST = '192.168.2.202'
PORT = 8888
def sockclient(i):
s = socket.socket()
s.connect((HOST, PORT))
#print(gevent.getcurrent())
msg = bytes(("This is gevent: %s" % i),encoding="utf8")
s.sendall(msg)
data = s.recv(1024)
print("Received", data.decode())
s.close()
pool = Pool(50)
threads = [pool.spawn(sockclient, i) for i in range(2000)]
gevent.joinall(threads)
The program are running properly without any error. but I think the server's code should receive an error, because i set pool = pool.Pool(2) in it.
I try to set pool.Pool(1) and open multi client running together but the result have no change.

ConnectionRefusedError in Python Sockets

I have some code that acts as a socket server:
import threading
import socketserver
class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
ip, port = self.client_address
print("{}:{} connected.".format(ip, port))
try:
while True:
data = str(self.request.recv(1024), 'ascii')
print("{}:{} - {}".format(ip, port, data))
cur_thread = threading.current_thread()
response = bytes("{}: {}".format(cur_thread.name, data), 'ascii')
self.request.sendall(response)
except ConnectionError:
print("{}:{} lost connection.".format(ip, port))
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
pass
def make_server() -> ThreadedTCPServer:
host, port = "127.0.0.1", 9191
server = ThreadedTCPServer((host, port), ThreadedTCPRequestHandler, False)
server.allow_reuse_address = True
server.server_bind()
# Start a thread with the server -- that thread will then start one
# more thread for each request
server_thread = threading.Thread(target=server.serve_forever)
# Exit the server thread when the main thread terminates
server_thread.daemon = True
server_thread.start()
print("Server started.")
return server
And I have tried to connect to the server in my terminal:
>>> from socket import socket
>>> s = socket()
>>> s.connect("127.0.0.1", 9191)
ConnectionRefusedError: [WinError 10061] No connection could be made because the target machine actively refused it
What is the cause of this?
Note: this code is imported and ran from another python file - "Server started." is printed when I run the code.

Sending data from outside socket object in python

I made a Client socket object, which I instantiate and it keeps alive a connection with the server, which is working fine, but I'm wondering if there is a way to call the socket.send event from outside the instance. I was about to make a stack for the messages and check the stack in the while loop and if it's not empty then send the oldest data to the server, which would be just fine for me, but my problem is that the stack only updates after the while loop(I tried breaking out, then it updated).
So my question would be, is there a way to update the global stack simultaneously with the while loop running? Or is there any other way to call the socket.send event outside the object?
import socket
import sys
import select
import threading
SERVER_IP = '192.168.1.4'
PORT = 8686
TIMEOUT = 5
BUF_SIZE = 1024
MESSAGES = ['testdata1', 'testdata2']
class Client(threading.Thread):
def __init__(self, host=SERVER_IP, port=PORT):
threading.Thread.__init__(self)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = socket.create_connection((host, port), 1)
self.sock.setblocking(0)
while 1:
try:
global MESSAGES
ready = select.select([self.sock], [], [], TIMEOUT*1000)
if ready[0]:
buf = self.sock.recv(BUF_SIZE)
print buf
#TODO:do stuff with buf
print 'messages left:'+str(len(MESSAGES))
if len(MESSAGES)>0:
self.sock.send(MESSAGES.pop())
except KeyboardInterrupt:
self.sock.close()
sys.exit(1)
except Exception, e:
print '\n[ERR] %s' % e
self.sock.close()
sys.exit(1)
def run(self):
pass
def sendData(self, data):
global MESSAGES
print 'appending data:%s' % data
MESSAGES.append(data)
def main():
client = Client()
client.start()
client.sendData("test1")
client.sendData("test2")
client.sendData("test3")
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
sys.exit(1)

Client.__init__() does not return because it enters an infinite while loop. Hence control is never returned to the main thread, and the Client thread is not actually started.
Instead you should move the while loop into the run() method. Then the __init__() method will return control to the main thread, which can then start the thread, and request that the client send messages via sendData().
class Client(threading.Thread):
def __init__(self, host=SERVER_IP, port=PORT):
threading.Thread.__init__(self)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = socket.create_connection((host, port), 1)
self.sock.setblocking(0)
def run(self):
while 1:
try:
global MESSAGES
ready = select.select([self.sock], [], [], TIMEOUT*1000)
if ready[0]:
buf = self.sock.recv(BUF_SIZE)
print buf
#TODO:do stuff with buf
print 'messages left:'+str(len(MESSAGES))
if len(MESSAGES)>0:
self.sock.send(MESSAGES.pop())
except KeyboardInterrupt:
self.sock.close()
sys.exit(1)
except Exception, e:
print '\n[ERR] %s' % e
self.sock.close()
sys.exit(1)
def sendData(self, data):
global MESSAGES
print 'appending data:%s' % data
MESSAGES.append(data)
Instead of using the global MESSAGES list you should probably create a Queue for communicating between the main thread and the worker thread(s), particularly if more than one worker thread is running. Something like this (untested!):
import Queue
class Client(threading.Thread):
def __init__(self, msg_queue, host=SERVER_IP, port=PORT):
threading.Thread.__init__(self)
self.msg_queue = msg_queue
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = socket.create_connection((host, port), 1)
self.sock.setblocking(0)
def run(self):
while 1:
try:
ready = select.select([self.sock], [], [], TIMEOUT*1000)
if ready[0]:
buf = self.sock.recv(BUF_SIZE)
print buf
#TODO:do stuff with buf
print 'messages left:'+ str(self.msg_queue.qsize())
try:
msg = self.msg_queue.get_nowait()
self.sock.send(msg)
except Queue.Empty:
pass
except KeyboardInterrupt:
self.sock.close()
sys.exit(1)
except Exception, e:
print '\n[ERR] %s' % e
self.sock.close()
sys.exit(1)
def main():
# create a queue and pass it to the client
msg_queue = Queue.Queue()
client = Client(msg_queue)
client.start()
msg_queue.put("test1")
msg_queue.put("test2")
msg_queue.put("test3")

The thing should work if you move your loop from
__init__() into run()
method instead.
Your thread is not a thread this way, process blocks at client = Client(...).
Why do you mix select and threads? Is this really necessary? If you want asynchronous sending and receiving without threads use asyncore module.
Or remove select from your code. The socket.recv() will block until it receives data in blocking mode, but as this is a thread, I don't see anything wrong about that. If in nonblocking mode, recv() will just return None if there is no data to receive if I remember correctly. So you don't really need select. Just check if recv() returned None. If it does, sleep some time before trying again.
The way you did it troubles your OS twice. Once for reading a socket, and second time to get the status of a socket where timeout is used to simulate sleep() more than anything else. Then the loop checks again making select() system call right after timeout confirmed that there is nothing to do for that socket.

Why are my threads hanging in this socket test on windows?

I wrote this script to test socket behavior in Windows, and I'm not sure why it hangs in Windows and not in Ubuntu. The script makes three listening sockets bound to '127.0.0.1', and it makes 60 threads which each connect to the listening sockets 10 times, 20 threads per listening socket.
import threading
import socket
import logging
import os
ports = [60003, 60004, 60005]
class ServerTest(threading.Thread):
log_lock = threading.Lock()
def __init__(self, port):
super(ServerTest, self).__init__(name=('socktest_%d'%port))
self.port = port
self._init_logger()
def _init_logger(self):
self.logger = logging.getLogger(self.name)
handler = logging.FileHandler('socktest.log')
formatter = logging.Formatter(
'%(levelname)s -- %(asctime)s:\n%(message)s',
datefmt='%m/%d/%Y %I:%M:%S %a')
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
self.logger.addHandler(handler)
self.logger.setLevel(logging.INFO)
def log(self, junk):
self.log_lock.acquire()
if isinstance(junk, Exception):
self.logger.exception(junk)
else:
self.logger.info(str(junk))
self.log_lock.release()
def run(self):
try:
listener = socket.socket()
listener.bind(('127.0.0.1', self.port))
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.listen(100)
except Exeption as exc:
self.log(exc)
return
while True:
try:
c, a = listener.accept()
self.log('accepted connection from '+str(a)+' to '+self.name)
data = c.recv(4096)
if data == 'stop':
break
self.log('data:\n'+data)
c.sendall(data)
c.close()
except Exception as exc:
self.log(exc)
listener.close()
class ClientTest(threading.Thread):
def __init__(self, port):
super(ClientTest, self).__init__()
self.port = port
def run(self):
try:
for i in range(10):
c = socket.create_connection(('127.0.0.1', self.port))
data = os.urandom(256)
c.sendall(data)
c.recv(256)
c.close()
except Exception as exc:
return
def main():
print 'Starting test'
server_threads = [ServerTest(p) for p in ports]
for thread in server_threads:
thread.start()
print 'started thread', thread
client_threads = []
for p in ports:
for i in range(20):
client_threads.append(ClientTest(p))
for thread in client_threads:
thread.start()
print 'started thread', thread
for thread in client_threads:
thread.join()
print 'joined thread', thread
for p in ports:
c = socket.create_connection(('127.0.0.1', p))
c.sendall('stop')
c.close()
for thread in server_threads:
thread.join()
print 'joined thread', thread
print 'Finished'
if __name__ == '__main__':
main()
I've tried it with and without the line listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) and the behavior is the same. I also tried it with an without the lock and the behavior was still the same.
EDIT
I forgot to mention that it does print out all the the threads, it seems to be stuck in the thread.join() for the client_threads, but I don't know why.

My psychic powers tell me os.urandom() can block on Windows if there is insufficient entropy available (it should never block on Unix). I haven't been able to confirm this one way or the other via MSDN, but given the reliance on hardware entropy (unlike, say, FreeBSD, which uses a pure software implementation), I don't think blocking is entirely inconceivable. Try replacing the os.urandom(256) call with (say) b'\x00' * 256.

There is an error in the class do not understand written with python

I get the following error when I want to run this code. I made a mistake I do not understand where all the normal
Where do you think the error
import socket,time
import thread
class http():
def __init__(self):
self.socket = socket
self.port = 5000
self.buffsize = 1024
self.listen = 5
self._header = ("HTTP/1.1 200 OK\r\n"
"Content-Type: text/html; charset=utf-8\r\n\r\n")
def _worker(self,socket,sleep):
# Client connect for thread worker
while True:
time.sleep(sleep)
client,address = socket.accept()
data = client.recv(1024)
if data:
client.send(self._header)
client.send(data)
client.close()
def httpHandler(self):
# Create Socket For Connection
try:
self.socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.socket.bind(('127.0.0.1',self.port))
self.socket.listen(self.listen)
self.socket.setblocking(False)
except socket.error as error:
print error
finally:
print "HTTP Server Running - 127.0.0.1:5000"
self._worker(self.socket,1)
if __name__ == "__main__":
application = http()
application.httpHandler()
When I want to run on the terminal, the error
but how can it be said there is the problem of self-
HTTP Server Running - 127.0.0.1:5000
Traceback (most recent call last):
File "/Users/batuhangoksu/http.py", line 44, in <module>
application.httpHandler()
File "/Users/batuhangoksu/http.py", line 40, in httpHandler
self._worker(self.socket,1)
File "/Users/batuhangoksu/http.py", line 22, in _worker
client,address = socket.accept()
AttributeError: 'module' object has no attribute 'accept'

Use self.socket, not socket:
client,address = self.socket.accept()
socket is the name of the module. self.socket is a socket._socketobject, the value returned by a call to socket.socket. Verily, there are too many things named "socket" :).
I suggest changing self.socket to something else to help separate the ideas.
You also need to save the return value when you call socket.socket. Currently, you have
self.socket = socket
which sets the instance attribute self.socket to the module socket. That's not useful, since you can already access the module with plain old socket. Instead, use something like
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
import multiprocessing as mp
import socket
import time
HOST = '' # Symbolic name meaning all available interfaces
PORT = 50007 # Arbitrary non-privileged port
def server():
header = ("HTTP/1.1 200 OK\r\n"
"Content-Type: text/html; charset=utf-8\r\n\r\n")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(5)
while True:
conn, addr = s.accept()
data = conn.recv(1024)
if not data: break
conn.send(header)
conn.send(data)
conn.close()
def client():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.send('Hello, world')
data = s.recv(1024)
s.close()
print 'Received', repr(data)
sproc = mp.Process(target = server)
sproc.daemon = True
sproc.start()
while True:
time.sleep(.5)
client()