Develop Reference

ZeroMQ: Waiting for a message without consuming thread

I have some simple python code which waits for messages for a topic.
However...when running this, the python process will hog CPU. I know with other languages, such as with sockets, there is a way to wait for messages without eating the entire processing power. Essentially the thread just remains halted waiting for a response. Is that possible with ZeroMQ?
import zmq
import sys
port = "5556"
if len(sys.argv) > 1:
port = sys.argv[1]
int(port)
# Socket to talk to server
context = zmq.Context()
socket = context.socket(zmq.SUB)
print "Collecting updates from weather server..."
socket.connect ("tcp://localhost:%s" % port)
# Subscribe to zipcode, default is NYC, 10001
topicfilter = "10001"
socket.setsockopt(zmq.SUBSCRIBE, topicfilter)
# Process 5 updates
total_value = 0
for update_nbr in range (5):
string = socket.recv()
topic, messagedata = string.split()
total_value += int(messagedata)
print ('{} {}'.format(topic, message)

Yes, this is possible.
The code above is not a reproducible MCVE/MWE-representation of the claimed problem.
First : the code explicitly blocks. Using a .recv()-method is blocking until any new, matching message arrives ( if ever ).
That does not overload the CPU. It just keeps sitting on the SUB-side waiting, until a first such message ( if any ) arrives.
If interested in a better scheme, do not use .recv()-method this way, but rather detect any such POSACK-acknowledgement with a .poll()-method & read a POSACK-ed case with .recv( zmq.NOBLOCK )-method.

Python ZMQ examples over WLAN network

For a project I need to communicate between C++ and Python via ZMQ over the WLAN network.
If I use my C++ implementation, everything works fine. I just type in the IP+Port number at the client.bind("tcp:// ...) and I can send messages via WLAN.
If I try the same with the Python Code, it does not work.
So I just tested the python examples (so no C++ anymore): http://zguide.zeromq.org/py:durapub
http://zguide.zeromq.org/py:durasub
I replaced the >localhost< in the client with the IP of my host computer. I do not receive any messages. I am using exactly the code from the example, except the replacement.
Here is the Code:
PUBLISHER:
import zmq
import time
context = zmq.Context()
# Subscriber tells us when it's ready here
sync = context.socket(zmq.PULL)
sync.bind("tcp://*:5564")
# We send updates via this socket
publisher = context.socket(zmq.PUB)
publisher.bind("tcp://*:5565")
# Wait for synchronization request
sync_request = sync.recv()
# Now broadcast exactly 10 updates with pause
for n in xrange(10):
msg = "Update %d" % n
publisher.send(msg)
time.sleep(1)
publisher.send("END")
time.sleep(1) # Give 0MQ/2.0.x time to flush output
SUBSCRIBER
import zmq
import time
context = zmq.Context()
# Connect our subscriber socket
subscriber = context.socket(zmq.SUB)
subscriber.setsockopt(zmq.IDENTITY, "Hello")
subscriber.setsockopt(zmq.SUBSCRIBE, "")
subscriber.connect("tcp://192.168.2.119:5565")
# Syncronize with the publisher
sync = context.socket(zmq.PUSH)
sync.connect("tcp://192.168.2.119:5564")
sync.send("")
# Get updates, expect random Ctrl-C death
while True:
data = subscriber.recv()
print data
if data == "END":
break
Its exactly the example code, except that I changed localhost to the IP Adress of my publisher in the Subscriber-Code. Btw, I did the same in the C++ example Code and it works.

How to write a multiprocessing web server in python

I have a simple web server in python which responds to the requests based on some configurations. Configurations define the percent of OK, NOK, Timeout and Null responses:
import socket
import sys
import os
import datetime
import random
import time
# define globals
global log_file
global configs
dash = '-'
sep = '\n' + 100 * dash + '\n'
ok_message = 'HTTP/1.0 200 OK\n\n'
nok_message = 'HTTP/1.0 404 NotFound\n\n'
def initialize():
if not os.path.isdir('./logs'):
os.mkdir(os.path.abspath('./logs'))
path = os.path.abspath(os.path.join(os.path.abspath('./logs'),
datetime.datetime.now().strftime('%d-%m-%Y %H-%M-%S')))
os.mkdir(path)
log_file = open(os.path.join(path, 'received_packets.log'), 'a')
def finalize():
log_file.close()
def select_resp_type():
percents = {}
for key, val in configs.items():
if key.endswith('Percent'):
percents.update({key: int(val)})
items = [x.replace('Percent', '') for x, v in percents.items()
if (float(counts[x.replace('Percent', '')]) / counts['all_packets']) * 100 < v]
print items
print [(float(counts[x.replace('Percent', '')]) / counts['all_packets']) * 100 for x, v in percents.items()]
if len(items):
selected = random.choice(items)
counts[selected] += 1
return selected
sys.stdout('Everything is done!')
sys.exit(0)
def get_response():
resp_type = select_resp_type()
if resp_type == 'ok':
return ok_message
elif resp_type == 'nok':
return nok_message
elif resp_type == 'nok':
time.sleep(int(configs['timeoutAmount']))
return ok_message
elif resp_type == 'nok':
time.sleep(int(configs['timeoutAmount']))
return None
def load_configs(config):
if not os.path.isfile(config):
log_file.write('No such file ' + os.path.abspath(config))
sys.exit(1)
config_lines = open(config, 'r').readlines()
configs = {}
for line in config_lines:
if line.strip() == '' or line.strip().startswith('#'):
continue
configs.update({line.split('=')[0].strip(): line.split('=')[1].strip()})
if __name__ == '__main__':
initialize()
config = sys.argv[3]
load_configs(config)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((str(configs['host']), int(configs['port'])))
s.listen(1)
try:
while True:
s_sock, s_addr = s.accept()
sfile = s_sock.makefile('rw', 0)
content = sfile.readline().strip()
while content != '':
log_file.write(content + sep)
resp = get_response()
if resp:
sfile.write(resp)
sfile = s_sock.makefile('rw', 0)
content = sfile.readline().strip()
sfile.close()
s_sock.close()
except:
print 'an exception occurred!'
sys.exit(1)
finally:
finalize()
This is my configuration file:
# server configurations
host = 127.0.0.1
port = 8000
okPercent = 80
nokPercent = 20
nullPercent = 0
timeoutPercent = 0
timeoutAmount = 120
maxClients = 10
I want to change this script to be a multiprocessing (by which I mean non-blocking, so that multiple requests can be processed) web server, but I don't know where to start and how to do that. Any help?
EDIT 1:
According to #Jan-Philip Gehrcke's answer, I changed my script to use gevent library:
def answer(s):
try:
gevent.sleep(1)
s_sock, s_addr = s.accept()
print conn_sep + 'Receive a connection from ' + str(s_addr)
while True:
content = s_sock.recv(1024)
counts['all_packets'] += 1
log_file.write(packet_sep + content)
resp = get_response()
if resp:
s_sock.send(resp)
except:
print 'An error occurred in connection with ', s_addr, '; quiting...'
if __name__ == '__main__':
log_dir = sys.argv[2]
log_file = initialize(sys.argv[2])
config = sys.argv[1]
configs = load_configs(config)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((str(configs['host']), int(configs['port'])))
s.listen(int(configs['maxClients']))
threads = [gevent.spawn(answer, s) for i in xrange(int(configs['maxClients']))]
gevent.joinall(threads)
Nothing changed. Still if I run multiple clients to connect to the server, each one should wait for previous ones to be disconnected. Maybe I missed something. Any idea?
EDIT 2:
I also tried accepting requests in the main block as #Paul Rooney said:
def answer(server_sock):
try:
gevent.sleep(1)
while True:
content = server_sock.recv(1024)
counts['all_packets'] += 1
log_file.write(packet_sep + content)
resp = get_response()
if resp:
server_sock.send(resp)
except:
print 'An error occurred in connection with ', s_addr, '; quiting...'
if __name__ == '__main__':
log_dir = sys.argv[2]
log_file = initialize(sys.argv[2])
config = sys.argv[1]
configs = load_configs(config)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((str(configs['host']), int(configs['port'])))
s.listen(int(configs['maxClients']))
s_sock, s_addr = s.accept()
print conn_sep + 'Receive a connection from ' + str(s_addr)
threads = [gevent.spawn(answer, s_sock) for i in xrange(int(configs['maxClients']))]
gevent.joinall(threads)
First, I have the same result about concurrent connections; no requests will be answered till previous clients are dead. Second, when the first client disconnects, I get following error in the server and it terminates:
Traceback (most recent call last):
File "/opt/python2.7/lib/python2.7/site-packages/gevent-1.0.1-py2.7-linux-x86_64.egg/gevent/greenlet.py", line 327, in run
result = self._run(*self.args, **self.kwargs)
File "main.py", line 149, in answer
server_sock.send(resp)
error: [Errno 32] Broken pipe
<Greenlet at 0x1e202d0: answer(<socket._socketobject object at 0x1dedad0>)> failed with error
It seems when the first client disconnects, it closes its socket and that socket is no longer available for use; so other connected waiting clients can not be answered anymore.

At the very simplest level what you can do is spawn a new process every time your accept call returns and pass the process the client socket, which is returned by accept.
You are effectively offloading the processing of the request to the child process and leaving the main process free to process new requests and likewise offload them to new child processes.
The way I have found to do this and I am not saying it the perfect answer but it works for me (Debian Python 2.7.3).
Simple example that bears some resemblance to your original code and is intended only to demonstrate when to spawn the process.
import socket
import sys
import time
import errno
from multiprocessing import Process
ok_message = 'HTTP/1.0 200 OK\n\n'
nok_message = 'HTTP/1.0 404 NotFound\n\n'
def process_start(s_sock):
content = s_sock.recv(32)
s_sock.send(ok_message)
s_sock.close()
#time.sleep(10)
sys.exit(0) # kill the child process
if __name__ == '__main__':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((sys.argv[1], int(sys.argv[2])))
print 'listen on address %s and port %d' % (sys.argv[1], int(sys.argv[2]))
s.listen(1)
try:
while True:
try:
s_sock, s_addr = s.accept()
p = Process(target=process_start, args=(s_sock,))
p.start()
except socket.error:
# stop the client disconnect from killing us
print 'got a socket error'
except Exception as e:
print 'an exception occurred!',
print e
sys.exit(1)
finally:
s.close()
The things to take note of are
s_sock, s_addr = s.accept()
p = Process(target=process_start, args=(s_sock,))
p.start()
Here is where you spawn a process in response to accept returning.
def process_start(s_sock):
content = s_sock.recv(32)
s_sock.send(ok_message)
s_sock.close()
#time.sleep(10)
sys.exit(0) # kill the child process
Here is the function that starts the new process, takes the socket passed to it and sends the response (you would do a bit more here). and then kills the child. I'm not 100% sure that this is the correct way to kill the child process or that killing it is even required. Maybe someone can correct me or edit the answer if required.
I can see that even if I uncomment the time.sleep calls that I can get responses from multiple client sockets pretty much instantly.
The greenlets way is no doubt a better way to do it in terms of system resource and performance.

"I want to change this script to be a multiprocessing (by which I mean non-blocking, so that multiple requests can be processed)"
Indeed, you mean "non-blocking", that is the right term. Before doing anything, you need to appreciate that this is a complex topic and that you need to learn a bit about concurrency architectures.
"concurrency" is the concept of making multiple things happen at the same time (whereas often times we actually need efficient usage of a single CPU core instead of real simultaneity).
Believe me, this is not a trivial topic. One approach many would take here is to monkey-patch the socket module via gevent (search for that). This would allow for many network connections to be processed concurrently, without changing your code. Actually, your problem is a prime example for gevent. Have a look into it.
How this works? Gevent installs a greenlet-based machinery behind the scenes and monitors your open sockets for I/O events via libev. Each network connection is handled within its own execution context (a so-called coroutine, as implemented by greenlet). Behind the scenes, the execution flow then jumps between coroutines, depending on the order of I/O events on your sockets. That's actually a complicated topic and you cannot understand it within 5 minutes.
The core concept with gevent/greenlet/coroutines/even-driven architectures is:
Instantaneously detect when your program would wait for I/O
Do some other work instead
For this to realize one does not need multiple CPU cores, which is why "multiprocessing" is not a good term in your title.

Multithreading - Alternating between two threads using Conditions and Events in Python

I am trying to write a program using which I wish to alternate between two threads, thread1 and thread2. The tricky part is that the thread should begin execution first must be thread1.
This is the code I have so far:
Class Client:
#member variables
def sendFile(self,cv1,lock1):
sent=0;
while (i<self.size):
message = self.data[i:1024+i]
cv1.acquire()
BadNet.transmit(self.clientSocket,message,self.serverIP,self.serverPort)
cv1.notify()
cv1.release()
i = i+1024
sent+=1
lock1.wait()
print "File sent successfully !"
self.clientSocket.close()
def receiveAck(self,cv1,lock2):
i=0
while (1):
lock1.clear()
cv1.acquire()
cv1.wait()
print "\nentered ack !\n"
self.ack, serverAddress = self.clientSocket.recvfrom(self.buf)
cv1.release()
lock1.set()
if __name__ == "__main__":
lock1 = Event()
cv1 = Condition()
cv2= Condition()
client = Client();
client.readFile();
thread1 = Thread(target = client.sendFile, args=[cv1,lock1])
thread2 = Thread(target = client.receiveAck, args=[cv1,lock1])
thread1.start()
thread2.start()
thread1.join()
thread2.join()
The problem I am currently facing is that initially the program does alternate between two threads (confirmed by the output on the console. But after an arbitrary number of iterations (usually between 20 and 80) the program just hangs and no further iterations are performed.

There are at least two problems with your synchronization.
First, you're using cv1 wrong. Your receive thread has to loop around its cv, checking the condition and calling wait each time. Otherwise, you're just using a cv as a broken event + lock combination. You don't have such a loop. More importantly, you don't even have a condition to wait for.
Second, you're using lock1 wrong. Your receive thread sets the event and then immediately clears it. But there's no guarantee that the send thread has gotten to the wait yet. (The race from the previous problem makes this more of a problem, but it's still a problem even if you fix that.) On a multi-core machine, it will usually get there in time, but "usually" is even worse than never in threaded programming. So, eventually the send thread will get to the wait after the receive thread has already done the clear, and therefore it will wait forever. The receive thread, meanwhile, will be waiting to be notified by the send thread, which will never happen. So you're deadlocked.
For future reference, adding print statements before and after every blocking operation, especially sync operations, would make this a lot to debug: you would see the receive thread's last message was "receive waiting on cv1", while the send thread's last message was "send waiting on lock1", and it would be obvious where the deadlock was.
Anyway, I'm not sure what it would even mean to "fix" a cv with no condition, or an event that you're trying to use as a cv, so instead I'll show how to write something sensible with two cvs. In this case, we might as well just use a flag that we flip back and forth as the condition for both cvs.
While I'm at it, I'll fix a couple other problems that made your code not even testable (e.g., i is never initialized), and include the debugging information, and what I had to fill in to make this a complete example, but otherwise I'll try to leave your structure and irrelevant problems (like Client being an old-style class) intact.
class Client:
def __init__(self):
self.clientSocket = socket(AF_INET, SOCK_DGRAM)
self.serverIP = '127.0.0.1'
self.serverPort = 11111
self.buf = 4
self.waitack = False
def readFile(self):
self.data = ', '.join(map(str, range(100000)))
self.size = len(self.data)
#member variables
def sendFile(self,cv1,lock1):
i = 0
sent=0
while (i<self.size):
message = self.data[i:1024+i]
print "s cv1 acquire"
with cv1:
print "s sendto"
self.clientSocket.sendto(message, (self.serverIP, self.serverPort))
self.waitack = True
print "s cv1 notify"
cv1.notify()
i = i+1024
sent+=1
print "s cv2 acquire"
with cv2:
print "s cv2 wait"
while self.waitack:
cv2.wait()
print "File sent successfully !"
self.clientSocket.close()
def receiveAck(self,cv1,lock2):
i=0
while (1):
print "r cv1 acquire"
with cv1:
while not self.waitack:
print "r cv1 wait"
cv1.wait()
print "r recvfrom"
self.ack, serverAddress = self.clientSocket.recvfrom(self.buf)
i += 1
print self.ack, i
print "r cv2 acquire"
with cv2:
self.waitack = False
print "r cv2 notify"
cv2.notify()
And here's a test server for it:
from itertools import *
from socket import *
s = socket(AF_INET, SOCK_DGRAM)
s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
s.bind(('127.0.0.1', 11111))
for i in count():
data, addr = s.recvfrom(1024)
print(i)
s.sendto('ack\n', addr)
Start the server, start the client, the server will count up to 672, the client will count up to 673 (since your code counts 1-based) with 673 balanced pairs of messages and a "File sent successfully !" at the end. (Of course the client will then hang forever because receiveAck has no way to finish, and the server because I wrote it as an infinite loop.)

My chat client freezes up after beginning threads

I made a better chat client following help from people:
They told me that if I didn't want to be blocked on .recv when waiting for messages, I would need to use threads, classes, functions, and queues to do so.
So I followed some help a specific person gave me where I created a thread from a class and then defined a function that was supposed to read incoming messages and print them.
I also created a function that allows you to enter stuff to be sent off.
Thing is, when I run the program. Nothing happens.
Can somebody help point out what is wrong? (I've asked questions and researched for 3 days, without getting anywhere, so I did try)
from socket import *
import threading
import json
import select
print("Client Version 3")
HOST = input("Connect to: ")
PORT = int(input("On port: "))
# Create Socket
s = socket(AF_INET,SOCK_STREAM)
s.connect((HOST,PORT))
print("Connected to: ",HOST,)
#-------------------Need 2 threads for handling incoming and outgoing messages--
# 1: Create out_buffer:
Buffer = []
rlist,wlist,xlist = select.select([s],Buffer,[])
class Incoming(threading.Thread):
# made a function a thread
def Incoming_messages():
while True:
for i in rlist:
data = i.recv(1024)
if data:
print(data.decode())
# Now for outgoing data.
def Outgoing():
while True:
user_input=("Your message: ")
if user_input is True:
Buffer += [user_input.encode()]
for i in wlist:
s.sendall(Buffer)
Buffer = []
Thanks for taking a look, thanks also to Tony The Lion for suggesting this

Take a look at this revised version of your code: (in python3.3)
from socket import *
import threading
import json
import select
print("client")
HOST = input("connect to: ")
PORT = int(input("on port: "))
# create the socket
s = socket(AF_INET, SOCK_STREAM)
s.connect((HOST, PORT))
print("connected to:", HOST)
#------------------- need 2 threads for handling incoming and outgoing messages--
# 1: create out_buffer:
out_buffer = []
# for incoming data
def incoming():
rlist,wlist,xlist = select.select([s], out_buffer, [])
while 1:
for i in rlist:
data = i.recv(1024)
if data:
print("\nreceived:", data.decode())
# now for outgoing data
def outgoing():
global out_buffer
while 1:
user_input=input("your message: ")+"\n"
if user_input:
out_buffer += [user_input.encode()]
# for i in wlist:
s.send(out_buffer[0])
out_buffer = []
thread_in = threading.Thread(target=incoming, args=())
thread_out = threading.Thread(target=outgoing, args=())
thread_in.start() # this causes the thread to run
thread_out.start()
thread_in.join() # this waits until the thread has completed
thread_out.join()
in your program you had various problems, namely you need to call the threads; to just define them isn't enough.
you also had forgot the function input() in the line: user_input=input("your message: ")+"\n".
the "select()" function was blocking until you had something to read, so the program didn't arrive to the next sections of the code, so it's better to move it to the reading thread.
the send function in python doesn't accept a list; in python 3.3 it accepts a group of bytes, as returned by the encoded() function, so that part of the code had to be adapted.