I have a simple web server in python which responds to the requests based on some configurations. Configurations define the percent of OK, NOK, Timeout and Null responses:
import socket
import sys
import os
import datetime
import random
import time
# define globals
global log_file
global configs
dash = '-'
sep = '\n' + 100 * dash + '\n'
ok_message = 'HTTP/1.0 200 OK\n\n'
nok_message = 'HTTP/1.0 404 NotFound\n\n'
def initialize():
if not os.path.isdir('./logs'):
os.mkdir(os.path.abspath('./logs'))
path = os.path.abspath(os.path.join(os.path.abspath('./logs'),
datetime.datetime.now().strftime('%d-%m-%Y %H-%M-%S')))
os.mkdir(path)
log_file = open(os.path.join(path, 'received_packets.log'), 'a')
def finalize():
log_file.close()
def select_resp_type():
percents = {}
for key, val in configs.items():
if key.endswith('Percent'):
percents.update({key: int(val)})
items = [x.replace('Percent', '') for x, v in percents.items()
if (float(counts[x.replace('Percent', '')]) / counts['all_packets']) * 100 < v]
print items
print [(float(counts[x.replace('Percent', '')]) / counts['all_packets']) * 100 for x, v in percents.items()]
if len(items):
selected = random.choice(items)
counts[selected] += 1
return selected
sys.stdout('Everything is done!')
sys.exit(0)
def get_response():
resp_type = select_resp_type()
if resp_type == 'ok':
return ok_message
elif resp_type == 'nok':
return nok_message
elif resp_type == 'nok':
time.sleep(int(configs['timeoutAmount']))
return ok_message
elif resp_type == 'nok':
time.sleep(int(configs['timeoutAmount']))
return None
def load_configs(config):
if not os.path.isfile(config):
log_file.write('No such file ' + os.path.abspath(config))
sys.exit(1)
config_lines = open(config, 'r').readlines()
configs = {}
for line in config_lines:
if line.strip() == '' or line.strip().startswith('#'):
continue
configs.update({line.split('=')[0].strip(): line.split('=')[1].strip()})
if __name__ == '__main__':
initialize()
config = sys.argv[3]
load_configs(config)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((str(configs['host']), int(configs['port'])))
s.listen(1)
try:
while True:
s_sock, s_addr = s.accept()
sfile = s_sock.makefile('rw', 0)
content = sfile.readline().strip()
while content != '':
log_file.write(content + sep)
resp = get_response()
if resp:
sfile.write(resp)
sfile = s_sock.makefile('rw', 0)
content = sfile.readline().strip()
sfile.close()
s_sock.close()
except:
print 'an exception occurred!'
sys.exit(1)
finally:
finalize()
This is my configuration file:
# server configurations
host = 127.0.0.1
port = 8000
okPercent = 80
nokPercent = 20
nullPercent = 0
timeoutPercent = 0
timeoutAmount = 120
maxClients = 10
I want to change this script to be a multiprocessing (by which I mean non-blocking, so that multiple requests can be processed) web server, but I don't know where to start and how to do that. Any help?
EDIT 1:
According to #Jan-Philip Gehrcke's answer, I changed my script to use gevent library:
def answer(s):
try:
gevent.sleep(1)
s_sock, s_addr = s.accept()
print conn_sep + 'Receive a connection from ' + str(s_addr)
while True:
content = s_sock.recv(1024)
counts['all_packets'] += 1
log_file.write(packet_sep + content)
resp = get_response()
if resp:
s_sock.send(resp)
except:
print 'An error occurred in connection with ', s_addr, '; quiting...'
if __name__ == '__main__':
log_dir = sys.argv[2]
log_file = initialize(sys.argv[2])
config = sys.argv[1]
configs = load_configs(config)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((str(configs['host']), int(configs['port'])))
s.listen(int(configs['maxClients']))
threads = [gevent.spawn(answer, s) for i in xrange(int(configs['maxClients']))]
gevent.joinall(threads)
Nothing changed. Still if I run multiple clients to connect to the server, each one should wait for previous ones to be disconnected. Maybe I missed something. Any idea?
EDIT 2:
I also tried accepting requests in the main block as #Paul Rooney said:
def answer(server_sock):
try:
gevent.sleep(1)
while True:
content = server_sock.recv(1024)
counts['all_packets'] += 1
log_file.write(packet_sep + content)
resp = get_response()
if resp:
server_sock.send(resp)
except:
print 'An error occurred in connection with ', s_addr, '; quiting...'
if __name__ == '__main__':
log_dir = sys.argv[2]
log_file = initialize(sys.argv[2])
config = sys.argv[1]
configs = load_configs(config)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((str(configs['host']), int(configs['port'])))
s.listen(int(configs['maxClients']))
s_sock, s_addr = s.accept()
print conn_sep + 'Receive a connection from ' + str(s_addr)
threads = [gevent.spawn(answer, s_sock) for i in xrange(int(configs['maxClients']))]
gevent.joinall(threads)
First, I have the same result about concurrent connections; no requests will be answered till previous clients are dead. Second, when the first client disconnects, I get following error in the server and it terminates:
Traceback (most recent call last):
File "/opt/python2.7/lib/python2.7/site-packages/gevent-1.0.1-py2.7-linux-x86_64.egg/gevent/greenlet.py", line 327, in run
result = self._run(*self.args, **self.kwargs)
File "main.py", line 149, in answer
server_sock.send(resp)
error: [Errno 32] Broken pipe
<Greenlet at 0x1e202d0: answer(<socket._socketobject object at 0x1dedad0>)> failed with error
It seems when the first client disconnects, it closes its socket and that socket is no longer available for use; so other connected waiting clients can not be answered anymore.
At the very simplest level what you can do is spawn a new process every time your accept call returns and pass the process the client socket, which is returned by accept.
You are effectively offloading the processing of the request to the child process and leaving the main process free to process new requests and likewise offload them to new child processes.
The way I have found to do this and I am not saying it the perfect answer but it works for me (Debian Python 2.7.3).
Simple example that bears some resemblance to your original code and is intended only to demonstrate when to spawn the process.
import socket
import sys
import time
import errno
from multiprocessing import Process
ok_message = 'HTTP/1.0 200 OK\n\n'
nok_message = 'HTTP/1.0 404 NotFound\n\n'
def process_start(s_sock):
content = s_sock.recv(32)
s_sock.send(ok_message)
s_sock.close()
#time.sleep(10)
sys.exit(0) # kill the child process
if __name__ == '__main__':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((sys.argv[1], int(sys.argv[2])))
print 'listen on address %s and port %d' % (sys.argv[1], int(sys.argv[2]))
s.listen(1)
try:
while True:
try:
s_sock, s_addr = s.accept()
p = Process(target=process_start, args=(s_sock,))
p.start()
except socket.error:
# stop the client disconnect from killing us
print 'got a socket error'
except Exception as e:
print 'an exception occurred!',
print e
sys.exit(1)
finally:
s.close()
The things to take note of are
s_sock, s_addr = s.accept()
p = Process(target=process_start, args=(s_sock,))
p.start()
Here is where you spawn a process in response to accept returning.
def process_start(s_sock):
content = s_sock.recv(32)
s_sock.send(ok_message)
s_sock.close()
#time.sleep(10)
sys.exit(0) # kill the child process
Here is the function that starts the new process, takes the socket passed to it and sends the response (you would do a bit more here). and then kills the child. I'm not 100% sure that this is the correct way to kill the child process or that killing it is even required. Maybe someone can correct me or edit the answer if required.
I can see that even if I uncomment the time.sleep calls that I can get responses from multiple client sockets pretty much instantly.
The greenlets way is no doubt a better way to do it in terms of system resource and performance.
"I want to change this script to be a multiprocessing (by which I mean non-blocking, so that multiple requests can be processed)"
Indeed, you mean "non-blocking", that is the right term. Before doing anything, you need to appreciate that this is a complex topic and that you need to learn a bit about concurrency architectures.
"concurrency" is the concept of making multiple things happen at the same time (whereas often times we actually need efficient usage of a single CPU core instead of real simultaneity).
Believe me, this is not a trivial topic. One approach many would take here is to monkey-patch the socket module via gevent (search for that). This would allow for many network connections to be processed concurrently, without changing your code. Actually, your problem is a prime example for gevent. Have a look into it.
How this works? Gevent installs a greenlet-based machinery behind the scenes and monitors your open sockets for I/O events via libev. Each network connection is handled within its own execution context (a so-called coroutine, as implemented by greenlet). Behind the scenes, the execution flow then jumps between coroutines, depending on the order of I/O events on your sockets. That's actually a complicated topic and you cannot understand it within 5 minutes.
The core concept with gevent/greenlet/coroutines/even-driven architectures is:
Instantaneously detect when your program would wait for I/O
Do some other work instead
For this to realize one does not need multiple CPU cores, which is why "multiprocessing" is not a good term in your title.
Related
Hi i have an exercise to build with sockets select and msvcrt, server and clients of mltiplie chat(the server and the clients need to be built non-blocking) that every client will send message and the server will send the message to all the clients except the one who sent it, the server:
import socket
import select
IP = "192.168.1.154"
port = 123
default_buffer_size = 1024
open_client_sockets = []
messages_to_send = []
def send_waiting_messages(wlist):
for message in messages_to_send:
(client_sock, data) = message
if client_sock in wlist:
for sock in open_client_sockets:
if sock is not client_sock:
sock.send(data)
messages_to_send.remove(message)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((IP, port))
sock.listen(5)
print("The server is on and waiting for client...")
while True:
rlist, wlist, xlist = select.select([sock] + open_client_sockets, open_client_sockets, [])
for current_socket in rlist:
if current_socket is sock:
(new_socket, addr) = sock.accept()
open_client_sockets.append(new_socket)
else:
data = current_socket.recv(default_buffer_size)
if data == "":
open_client_sockets.remove(current_socket)
print("Connection with client closed")
else:
messages_to_send.append((current_socket, 'Hello ' + data))
send_waiting_messages(wlist)
if __name__ == '__main__':
main()
Building the server wasnt hard because it was guided(if it was not guided i would never got this code working) by the book but i have problem building the client and the main reason is that i dont understand how select.select works, couldn't find answer that will simplify enough this module for me.
this is what i did with the client:
import socket
import select
import msvcrt
IP = "192.168.1.154"
port = 123
sockets = []
def write():
pass
def main():
sock = socket.socket()
sock.connect((IP, port))
while True:
rlist, wlist, xlist = select.select(sockets, sockets, [])
for current_socket in rlist:
if current_socket is sock:
data = current_socket.recv(1024)
print(data)
else:
sockets.append(current_socket)
write()
if __name__ == '__main__':
main()
This probably shows you that I have low understanding of the module select and the exercise actually. I saw some threads that has similar question but I understand nothing from them so I realy need good explantion.
In conclusion I realy am lost...
select takes as parameters a list of sockets to wait for readablity, a list of sockets to wait for writability, and a list of sockets to wait for errors. It returns lists of ready to read, ready to write, and error sockets. From help:
>>> help(select.select)
Help on built-in function select in module select:
select(...)
select(rlist, wlist, xlist[, timeout]) -> (rlist, wlist, xlist)
Wait until one or more file descriptors are ready for some kind of I/O.
The first three arguments are sequences of file descriptors to be waited for:
rlist -- wait until ready for reading
wlist -- wait until ready for writing
xlist -- wait for an ``exceptional condition''
If only one kind of condition is required, pass [] for the other lists.
A file descriptor is either a socket or file object, or a small integer
gotten from a fileno() method call on one of those.
The optional 4th argument specifies a timeout in seconds; it may be
a floating point number to specify fractions of seconds. If it is absent
or None, the call will never time out.
The return value is a tuple of three lists corresponding to the first three
arguments; each contains the subset of the corresponding file descriptors
that are ready.
*** IMPORTANT NOTICE ***
On Windows, only sockets are supported; on Unix, all file
descriptors can be used.
So to fix your client, you need to add the socket you opened (sock) to the sockets list. Your write function can then be called if your socket is ready to be written.
In write, use msvcrt.kbhit() to test for characters typed. You can't just use input because it will block. Then read the character if one has been typed. Collect up the characters until you hit enter, then build a message and write it to the socket. Something like:
message = []
def write(sock):
if msvcrt.kbhit():
c = msvcrt.getche()
if c == '\r':
data = ''.join(message)
print 'sending:',data
sock.sendall(data)
message.clear()
else:
message.append(c)
Im sure there are easier ways with particular python modules, but for an assignment I need to create a program that can act as a client/server. As of right now I have it working to the point of only being able to send a message if the reciever has responded. I need it to just send and appear on the respective client/server terminal when enter is pressed. Any help would be greatly appreciated!
These are pictures of what happens as of now
https://i.stack.imgur.com/T9CsJ.png
import sys
import socket
import getopt
def usage(script_name):
print('Usage: py' + script_name + '-l' +' <port number>' + '[<server>]')
def sockObj():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
return sock
def serversockConn(serversocket,port):
serversocket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
# bind the serversocket to address
serversocket.bind(('',int(port)))
# enable a server to accept connections
serversocket.listen(5)
# wait for a connection and accept it
sock,addr = serversocket.accept()
return sock
def connToServersock(sock,server, port):
# connect to a serversocket
if server:
sock.connect((server, int(port)))
else:
sock.connect(('localhost', int(port)))
return sock
if __name__ == '__main__':
## get the command line arguments
try:
options, non_option_args = getopt.getopt(sys.argv[1:],'l')
except getopt.GetoptError as err:
print(err)
sys.exit(2)
#check if '-l' is present in command line agrument
serverSide = False
for opt,arg in options:
if opt == "-l":
serverSide = True
# port number
port = non_option_args[0]
# server address
server = None
hostLen = len(non_option_args)
if hostLen == 2:
server = non_option_args[1]
# create a communicator object, and make a connection between server and client
# server
if serverSide:
serversocket = sockObj()
sock = serversockConn(serversocket,port)
# client
else:
sock = sockObj()
sock = connToServersock(sock,server,port)
while True:
## read a message from standard input
message = sys.stdin.readline().encode()
if len(message) != 0:
sock.send(message)
return_msg = sock.recv( 1024 )
if return_msg:
print("Message recieved: " + return_msg.decode())
else:
print("Other side shut down")
else:
try:
sock.shutdown(socket.SHUT_WR)
sock.close()
except:
pass
I think your issue is that there are two places in your event loop where you block:
message = sys.stdin.readline().encode()
Here, you block until the user has pressed return -- during this time, your program is unable to respond to any data received over the network, because it is blocked waiting for data from stdin.
... and:
return_msg = sock.recv( 1024 )
Here, you are waiting for data to be received from the network -- during this time, your program is unable to respond to any data received from stdin, because it is blocked waiting for data from the network.
The behavior you'd ideally like to have is for your program to wait for both stdin and network traffic simultaneously -- i.e. have it block until either the user has pressed return, or some network data has been received, whichever comes first.
The easiest way to achieve that behavior is to use select(); its purpose is to block until at least one of several file descriptors is ready to be acted on. (Note, however, that Windows does not support using select() on stdin, so if your program needs to run under Windows you will probably have to spawn a second thread instead).
To implement the event loop using select(), add import select to the top of your script, then replace your event loop with something like this instead:
while True:
## block here until either sock or sys.stdin has data ready for us
readable, writable, exceptional = select.select([sock, sys.stdin], [], [])
if sys.stdin in readable:
## read a message from standard input
message = sys.stdin.readline().encode()
if len(message) != 0:
sock.send(message)
if sock in readable:
## read a message from the network
try:
return_msg = sock.recv( 1024 )
if (return_msg):
print("Message received: " + return_msg.decode())
else:
print("Other side shut down")
break
except:
print("recv() threw an exception")
break
I need a little help on my port scanner in Python. I have to learn how to add the threading module to it, but have no clue and do not really understand the few tutorials and help I've found. Here is a little test script:
from socket import *
remote = raw_input ("Website: ")
remote_ip = gethostbyname(remote)
print "scaning at:",remote_ip
for i in range (20,100):
s = socket(AF_INET,SOCK_STREAM)
result = s.connect_ex((remote,i))
if result == 0):
print "port %d: open"%(i)
s.close
Something like the below. You force each function into a separate thread and kick it off. You also keep a reference to each thread so you can block your main thread until they have all executed. You also need a lock because "print" in Python isn't thread safe and you'll end up getting the output from each thread mangled up otherwise (you can use the logger instead which is thread safe and neater). This will also generate a lot of threads as I haven't even bothered to show you pooling. This is just a rough and ready example to get you going on your journey to discover multi threading in Python.
print_lock = Lock()
def socket_test(address, port):
s = socket(AF_INET,SOCK_STREAM)
result = s.connect_ex((address,port))
if result == 0:
with print_lock:
print "port %s: open" % port
s.close()
def main():
remote = raw_input("Website: ")
remote_ip = gethostbyname(remote)
print "scaning at:",remote_ip
threads = []
for i in range(20, 100):
new_thread = threading.Thread(socket_test(remote_ip, i))
new_thread.start()
threads.append(new_thread)
[this_thread.join() for this_thread in threads]
I have a few computers on a network and I'm trying to coordinate work between them by broadcasting instructions and receiving replies from individual workers. When I use zmq to assign a single socket to each program it works fine, but when I try to assign another, none of them work. For example, the master program runs on one machine. With the code as such it works fine as a publisher, but when I uncomment the commented lines neither socket works. I've seen example code extremely similar to this so I believe it should work, but I must be missing something.
Here's some example code, first with the master program and then the worker program. The idea is to control the worker programs from the master based on input from the workers to the master.
import zmq
import time
import sys
def master():
word = sys.argv[1]
numWord = sys.argv[2]
port1 = int(sys.argv[3])
port2 = int(sys.argv[4])
context = zmq.Context()
publisher = context.socket(zmq.PUB)
publisher.bind("tcp://*:%s" % port1)
#receiver = context.socket(zmq.REP)
#receiver.bind("tcp://*:%s" % port2)
for i in range(int(numWord)):
print str(i)+": "+word
print "Publishing 1"
publisher.send("READY_FOR_NEXT_WORD")
print "Publishing 2"
publisher.send(word)
#print "Published. Waiting for REQ"
#word = receiver.recv()
#receiver.send("Master IRO")
time.sleep(1)
print "Received: "+word
publisher.send("EXIT_NOW")
master()
Ditto for the workers:
import zmq
import random
import zipfile
import sys
def worker(workerID, fileFirst, fileLast):
print "Worker "+ str(workerID) + " started"
port1 = int(sys.argv[4])
port2 = int(sys.argv[5])
# Socket to talk to server
context = zmq.Context()
#pusher = context.socket(zmq.REQ)
#pusher.connect("tcp://10.122.102.45:%s" % port2)
receiver = context.socket(zmq.SUB)
receiver.connect ("tcp://10.122.102.45:%s" % port1)
receiver.setsockopt(zmq.SUBSCRIBE, '')
found = False
done = False
while True:
print "Ready to receive"
word = receiver.recv()
print "Received order: "+word
#pusher.send("Worker #"+str(workerID)+" IRO "+ word)
#pusher.recv()
#print "Confirmed receipt"
worker(sys.argv[1], sys.argv[2], sys.argv[3])
Well, PUB-SUB patterns are not meant to be reliable specially on initialization (while the connection is established).
Your "master" publishes the first two messages in that loop and then waits for a request from the "worker". Now, if those messages get lost (something that may happen with the first messages sent with PUB-SUB patterns), then the "worker" will be stuck waiting for a publication from the "master". So, basically, they are both stuck waiting for an incoming message.
Apart from that, notice that you are publishing 2 messages from the "master" node while only processing 1 from the "worker". Your "worker" wont be able to catch-up with your "master" and, therefore, messages will be dropped or you'll get a crash.
I made a better chat client following help from people:
They told me that if I didn't want to be blocked on .recv when waiting for messages, I would need to use threads, classes, functions, and queues to do so.
So I followed some help a specific person gave me where I created a thread from a class and then defined a function that was supposed to read incoming messages and print them.
I also created a function that allows you to enter stuff to be sent off.
Thing is, when I run the program. Nothing happens.
Can somebody help point out what is wrong? (I've asked questions and researched for 3 days, without getting anywhere, so I did try)
from socket import *
import threading
import json
import select
print("Client Version 3")
HOST = input("Connect to: ")
PORT = int(input("On port: "))
# Create Socket
s = socket(AF_INET,SOCK_STREAM)
s.connect((HOST,PORT))
print("Connected to: ",HOST,)
#-------------------Need 2 threads for handling incoming and outgoing messages--
# 1: Create out_buffer:
Buffer = []
rlist,wlist,xlist = select.select([s],Buffer,[])
class Incoming(threading.Thread):
# made a function a thread
def Incoming_messages():
while True:
for i in rlist:
data = i.recv(1024)
if data:
print(data.decode())
# Now for outgoing data.
def Outgoing():
while True:
user_input=("Your message: ")
if user_input is True:
Buffer += [user_input.encode()]
for i in wlist:
s.sendall(Buffer)
Buffer = []
Thanks for taking a look, thanks also to Tony The Lion for suggesting this
Take a look at this revised version of your code: (in python3.3)
from socket import *
import threading
import json
import select
print("client")
HOST = input("connect to: ")
PORT = int(input("on port: "))
# create the socket
s = socket(AF_INET, SOCK_STREAM)
s.connect((HOST, PORT))
print("connected to:", HOST)
#------------------- need 2 threads for handling incoming and outgoing messages--
# 1: create out_buffer:
out_buffer = []
# for incoming data
def incoming():
rlist,wlist,xlist = select.select([s], out_buffer, [])
while 1:
for i in rlist:
data = i.recv(1024)
if data:
print("\nreceived:", data.decode())
# now for outgoing data
def outgoing():
global out_buffer
while 1:
user_input=input("your message: ")+"\n"
if user_input:
out_buffer += [user_input.encode()]
# for i in wlist:
s.send(out_buffer[0])
out_buffer = []
thread_in = threading.Thread(target=incoming, args=())
thread_out = threading.Thread(target=outgoing, args=())
thread_in.start() # this causes the thread to run
thread_out.start()
thread_in.join() # this waits until the thread has completed
thread_out.join()
in your program you had various problems, namely you need to call the threads; to just define them isn't enough.
you also had forgot the function input() in the line: user_input=input("your message: ")+"\n".
the "select()" function was blocking until you had something to read, so the program didn't arrive to the next sections of the code, so it's better to move it to the reading thread.
the send function in python doesn't accept a list; in python 3.3 it accepts a group of bytes, as returned by the encoded() function, so that part of the code had to be adapted.