Develop Reference

The publishing code below is stalling on time.sleep() how can i change this behaviour + what would be a good way to implement a thread safe cach

For refreshing skills for a new job i have been trying to implement a very simple price publisher + client + mtm_cache. In the code below in publishing.py the code stops at time.sleep() and doesnt seem to wakeup (say for example two clients connect, but they stop receiving the messages while publisher executes time.sleep()), how can i correct the behavior of the publisher.
Also there is another class MTMCache where i hope to store the MTM sent by the client, would a dictionary be a good thread safe cache which stores list of tuples (time,MTMValue) per client.
As a third point how can i change the implementation of this Publisher class using asyncio?
publishing.py
import threading
import socket
import time
import random
class Publisher(object):
def __init__(self):
self.list_subscribers = []#list of tuples with ip and port
self.mtm_cache = None
self.current_price = 50
self.listening_ports = [1100,1101,1102,1103,1104]
self.stop_threads = False
self.listening_port = 1100
self.sock = None
self.listening = False
#run listening function in another thread
def update_price(self):
tmp = random.randint(-10,10)
#print("Number generated is " + str(tmp))
self.current_price = self.current_price + tmp
def update_price_in_loop(self):
while(True):
self.update_price()
if(self.stop_threads):
break
def send_price_to_subscribers(self):
#in a parallel for loop send current price to subscribers
while(True):
#print(str(self.current_price))
if self.list_subscribers:
for cl in self.list_subscribers:
cl.send(str(self.current_price).encode())
if(self.stop_threads):
break
def start_listener_and_accept_connections(self):
if(not self.listening):
self.sock = socket.socket()
self.sock.bind(('',self.listening_port))
print("sock bound at ", self.listening_port)
while True:
if not self.listening:
print("listening for connections")
self.sock.listen(5)
#above is a blocking call, it is blocking other threads
self.listening = True # how to properly set and utilise this flag
c, addr = self.sock.accept()
print('Got connection from', addr)
self.list_subscribers.append(c)
if(self.stop_threads):
print("closing client connections")
for cl in self.list_subscribers:
cl.close()
self.sock.close()
break
class MTMCache(object):
def __init__(self,publisher):
self.publisher = publisher
self.cache = {} #subscriber token for each of the subscribers in publisher class, currenttime and mtm to be stored here, maybe some other structure besides dict could be used, explore that later
def receive_mtm(self,message):
#message could be string in json format
#have to design this method with appropriate data structures
pass
if __name__ == "__main__":
from threading import *
import time
pub = Publisher()
#pub.update_price()
#print(pub.current_price)
t = Thread(target = pub.update_price_in_loop)
t1 = Thread(target = pub.send_price_to_subscribers)
#t.start()
#t1.start()
t2 = Thread(target = pub.start_listener_and_accept_connections)
t2.start()
t.start()
t1.start()
time.sleep(5)
pub.stop_threads = True
#t2.raise_exception()
#t.raise_exception()
#t1.raise_exception()
t2.join()
t.join()
t1.join()
------
basic_client.py
import socket
s = socket.socket()
port = 1100
s.connect(('127.0.0.1', port))
while(True):
tmp = s.recv(1024).decode()
print(tmp)
if not tmp:
break
s.close()
----
in separate terminals:
python3 publishing.py
term2
python3 basic_client.py
term3
python3 basic_client.py

python multiprocessing doesn't seem to be working with class

I am trying to implement zmq client wrapper class. Hope the comments written are useful to understand. Please see the __main__ for the different ways I tried.
import zmq
class Client(object):
"""
This is class for passing message using pipeline(push-pull) pattern of zmq.
"""
def __init__(self, pull_addr="tcp://127.0.0.1:5757",\
push_addr="tcp://127.0.0.1:5858"):
"""
The constructor
Parameters:
pull_addr (string) : Local endpoint for pushing messages and pulling from
push_addr (string) : Remote endpoint for pushing messages.
"""
super(Client, self).__init__()
print("Initialized...")
self.context = zmq.Context()
self.push_addr = push_addr
self.pull_addr = pull_addr
def producer(self, msg):
"""
Produces messages.
It's a wrapper and will receive the message which will be queued locally by
pushing to self.pull_addr.
Parameters:
msg (json) : Messages to be queued.
Returns:
None
"""
print("Production started.... ")
zmq_socket = self.context.socket(zmq.PUSH)
zmq_socket.bind(self.pull_addr)
zmq_socket.send_json(msg)
def consumer(self):
"""
Consumes messages produced by self.producer.
Reads messages queued by self.producer and pushes to the self.push_addr
where it assumes the collector would be expecting messages.
Parameters:
None
Returns:
None
"""
print("consumption is awaiting...")
receiver = self.context.socket(zmq.PULL)
receiver.connect(self.pull_addr)
sender = self.context.socket(zmq.PUSH)
sender.connect(self.push_addr)
while True:
msg = receiver.recv_json()
sender.send_json(msg)
if __name__ == '__main__':
import multiprocessing
# con_pool = multiprocessing.Pool()
# # pro_pool = multiprocessing.Pool()
zmq_client = Client()
# consumer = con_pool.apply_async(zmq_client.consumer, ())
# # producer = pro_pool.apply_async(zmq_client.producer,({"msg" : "Hello World!"}))
# print("DONE>>>>")
# from multiprocessing import Process
# consumer = Process(target=zmq_client.consumer)
# consumer.start()
# producer = Process(target=zmq_client.producer, args=({'msg' : "Hello World!"}))
# producer.start()
# producer.join()
# consumer.join()
pool = multiprocessing.Pool()
p = pool.apply_async(zmq_client.consumer)
p2 = pool.apply_async(zmq_client.producer, ({"msg" : "Hello World!"}))
Try 1
pool = multiprocessing.Pool()
p = pool.apply_async(zmq_client.consumer)
p2 = pool.apply_async(zmq_client.producer, ({"msg" : "Hello World!"}))
Output 1
calls the constructor and terminates.
Try 2
con_pool = multiprocessing.Pool()
pro_pool = multiprocessing.Pool()
zmq_client = Client()
consumer = con_pool.apply_async(zmq_client.consumer, ())
producer = pro_pool.apply_async(zmq_client.producer,({"msg" : "Hello World!"}))
Output 2
calls the constructor and terminates.
Try 3
from multiprocessing import Process
zmq_client = Client()
con = Process(target=zmq_client.consumer)
con.start()
pro = Process(target=zmq_client.producer, args=({'msg' : "Hello World!"}))
pro.start()
pro.join()
con.join()
Output 3
calls the constructor
calls the consumer
calls the producer
keeps hanging
Here I was expecting zmq server to receive the message. Any suggestions to get this class working as expected?

Zeromq (pyzmq) ROUTER procession of multiple clients' data and subsequent timeout handling

I have a ROUTER whose purpose is to accumulate image data from multiple DEALER clients and perform OCR on the complete image. I found that the most efficient way of handling the OCR is through the utilization of Python's multiprocessing library; the accumulated image bytes are put into a Queue for due procession in a separate Process. However, I need to ensure that when a client experiences a timeout that the Process is properly terminated and doesn't meaninglessly linger and hog resources.
In my current solution I insert each newly-connected client into a dict where the value is my ClientHandler class that possesses all image data and spawns a Thread that sets a boolean variable named "timeout" to True when 5 seconds have elapsed. Should a new message be received within that 5 second frame, bump is called & the timer is reset back to 0, otherwise I cleanup prior to thread termination and the reference is deleted from the dict in the main loop:
import threading
import time
import zmq
class ClientHandler(threading.Thread):
def __init__(self, socket):
self.elapsed = time.time()
self.timeout = False
self.socket = socket
super(ClientHandler, self).__init__()
def run(self):
while time.time() - self.elapsed < 5.0:
pass
self.timeout = True
# CLIENT TIMED OUT
# HANDLE TERMINATION AND CLEAN UP HERE
def bump(self):
self.elapsed = time.time()
def handle(self, id, header, data):
# HANDLE CLIENT DATA HERE
# ACCUMULATE IMAGE BYTES, ETC
self.socket.send_multipart([id, str(0)])
def server_task():
clients = dict()
context = zmq.Context.instance()
server = context.socket(zmq.ROUTER)
server.setsockopt(zmq.RCVTIMEO, 0)
server.bind("tcp://127.0.0.1:7777")
while True:
try:
id, header, data = server.recv_multipart()
client = clients.get(id)
if client == None:
client = clients[id] = ClientHandler(server)
client.start()
client.bump()
client.handle(id, header, data)
except zmq.Again:
for id in clients.keys():
if clients[id].timeout:
del clients[id]
context.term()
if __name__ == "__main__":
server_task()
But this entire method just doesn't feel right. Am I going about this improperly? If so, I would greatly appreciate if someone could point me in the right direction.

Figured it out myself, hoping it may be of assistance to others.
I instead have a ROUTER on an assigned port that distributes unique ports to each client, which thereafter connects to the newly-bound socket on said unique port. When a client disconnects, the port is recycled for reassignment.
import sys
import zmq
from multiprocessing import Process, Queue, Value
def server_task():
context = zmq.Context.instance()
server = context.socket(zmq.ROUTER)
server.bind("tcp://127.0.0.1:7777")
timeout_queue = Queue()
port_list = [ 1 ]
proc_list = [ ]
while True:
try:
id = server.recv_multipart()[0]
# Get an unused port from the list
# Ports from clients that have timed out are recycled here
while not timeout_queue.empty():
port_list.append(timeout_queue.get())
port = port_list.pop()
if len(port_list) == 0:
port_list.append(port + 1)
# Spawn a new worker task, binding the port to a socket
proc_running = Value("b", True)
proc_list.append(proc_running)
Process(target=worker_task, args=(proc_running, port, timeout_queue)).start()
# Send the new port to the client
server.send_multipart([id, str(7777 + port)])
except KeyboardInterrupt:
break
# Safely allow our worker processes to terminate
for proc_running in proc_list:
proc_running.value = False
context.term()
def worker_task(proc_running, port, timeout_queue):
context = zmq.Context.instance()
worker = context.socket(zmq.ROUTER)
worker.setsockopt(zmq.RCVTIMEO, 5000)
worker.bind("tcp://127.0.0.1:%d" % (7777 + port, ))
while proc_running.value:
try:
id, data = worker.recv_multipart()
worker.send_multipart([id, data])
except zmq.Again:
timeout_queue.put(port)
context.term()
break
print("Client on port %d disconnected" % (7777 + port, ))

python multiprocessing agents and udp listeners

I have a master class (server for want of a better term) that creates multiple clients (from a client class) usingthge mulitproccessing library.
class mantransact:
def __init__(self,runMode,f_xml):
#call the build nodes function
self.buildNodes()
sockLisProcess = multiprocessing.Process(target=self.sockListener())
sockLisProcess.start()
self.initiateTransactions()
def buildNodes(self,):
n_source = self.f_xml.getElement("nodeSource")
self.log.addToLog ("node source is - %s" % n_source)
self.n_file = load_source.load_source(n_source,"csv")
#remove header from node list
del self.n_file.data_list[0]
self.nodes = [self.mkproc(node, l) for l in self.n_file.data_list]
self.log.addToLog(self.nodes)
def mkproc(self, func, line):
l = "-".join(line)
p = multiprocessing.Process(target=func, args=(l, self.f_xml))
p.start()
return (line[0], p)
def sockListener(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_addresss = ('localhost',10099)
self.sock.bind(server_addresss)
while True:
self.log.addToLog("server is waitin")
data, address = self.sock.recvfrom(1024)
self.log.addToLog(data, address)
def sockSender(self,client_address,d):
self.sock2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock2.bind(('localhost',10098))
recip = ('localhost',int(client_address))
self.sock2.sendto(str(d),recip)
self.sock2.close()
def initiateTransactions(self,):
#loop through transaction and then loop node list to match at match transmit transaction
#using UDP ports
for t in self.t_file.data_list:
for n in self.nodes:
if t[0] == n[0]:
for l in self.n_file.data_list:
if l[0] == n[0]:
self.log.addToLog ("gonna transmit UDP transaction to node - %s" % n[0])
client_address = l[1]
pip = n[2]
t.insert(0, "nTransaction")
self.sockSender(client_address, t)
I am trying to create UDP listeners at both the client and the nodes:
class node:
def __init__(self,child_conn, line, f_xml):
l = line.split("-")
"""extract calues from array and use f_xml for config"""
self.proofProcess = multiprocessing.Process(target=self.runProof(self.waitingTransactions))
self.proofProcess.start()
self.listenProcess = Multiprocessing.Process(target=self.udpListener())
self.listenProcess.start()
def udpListener(self):
lsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
lsock.bind(("localhost",int(self.IP)))
while 1 ==1:
data, addr = lsock.recvfrom(1024)
print ("received message", data)
"""do some things with data"""
I have two issues:
1 with my server I want my code to kick off these processes and then continue on instantiating or performing other tasks but the code just hangs waiting for the listener to receive a packet. Am i instantiating processing incorrectly:
2 with my clients they are performing a task to solve a problem and don't start the listeners until that task is complete. Can they not start their task an listen in parallel? The listener is meant to interrupt the calculation if another clietn solves it first and then it will receive a new task from the server

I found the solution.
By putting the multiprocessing element into a seperate process:
def loopProcesses(self,procedureName):
processX = multiprocessing.Process(target=procedureName)
processX.start()
return processX
And putting the names of the processes to be used into an array which is looped through to call the loopProcesses() process both processes were started in parallel.
m_processes = [self.sockListener(), self.initiateTransactions()]
l_processes = [self.loopProcesses(mp) for mp in m_processes]

The above did not work as the functions called are in a continuous loop until they have found a number of solutions. A problem was occuring when the first function was called it would start without the start command. I later found that I have to call the function without using the '()' and then the funciton will wait. ammended code is:
p = [multiprocessing.Process(target=self.sockListener),multiprocessing.Process(target=self.initiateTransactions)]
for prc in p:
prc.start()
I found this after a lot of seraching and came accross this: Socketserver multiprocessing.Process is starting without calling start()

Wrapping Pyro's NameServer the correct way

I'm trying to warp Pyro's name server into a more convenient object that would allow me to start and stop it as I wish. For example, I would like to be able to do something like
nameServer = NameServer("localhost")
nameServer.startNS()
[... make some other operations...]
nameServer.stopNS()
nameServer = None
[... make some other operations...]
nameServer = NameServer("localhost")
nameServer.startNS()
using the following definition for the NameServer class:
class NameServer(threadutil.Thread):
def __init__(self, host, isDeamon=True, port=None, enableBroadcast=True,
bchost=None, bcport=None, unixsocket=None, nathost=None, natport=None):
super(NameServer,self).__init__()
self.setDaemon(isDeamon)
self.host=host
self.started=threadutil.Event()
self.unixsocket = unixsocket
self.port = port
self.enableBroadcast = enableBroadcast
self.bchost = bchost
self.bcport = bcport
self.nathost = nathost
self.natport = natport
"""
This code is taken from Pyro4.naming.startNSloop
"""
self.ns_daemon = naming.NameServerDaemon(self.host, self.port, self.unixsocket,
nathost=self.nathost, natport=self.natport)
self.uri = self.ns_daemon.uriFor(self.ns_daemon.nameserver)
internalUri = self.ns_daemon.uriFor(self.ns_daemon.nameserver, nat=False)
self.bcserver=None
if self.unixsocket:
hostip = "Unix domain socket"
else:
hostip = self.ns_daemon.sock.getsockname()[0]
if hostip.startswith("127."):
enableBroadcast=False
if enableBroadcast:
# Make sure to pass the internal uri to the broadcast responder.
# It is almost always useless to let it return the external uri,
# because external systems won't be able to talk to this thing anyway.
bcserver=naming.BroadcastServer(internalUri, self.bchost, self.bcport)
bcserver.runInThread()
def run(self):
try:
self.ns_daemon.requestLoop()
finally:
self.ns_daemon.close()
if self.bcserver is not None:
self.bcserver.close()
def startNS(self):
self.start()
def stopNS(self):
self.ns_daemon.shutdown()
if self.bcserver is not None:
self.bcserver.shutdown()
So far, so good. It works as expected. However, if I run a command Pyro4.naming.locateNS() from another thread when the name server is running, then the next time I call nameServer.stopNS(), the program freezes. Anyone has an idea why? And what would be the best (at least a better) way to write such a NameServer wrapper.

There is an example in the Pyro4 repository that you could adapt.
https://github.com/delmic/Pyro4/blob/master/examples/eventloop/server.py
from __future__ import print_function
import socket
import select
import sys
import Pyro4.core
import Pyro4.naming
if sys.version_info<(3,0):
input=raw_input
print("Make sure that you don't have a name server running already.")
servertype=input("Servertype thread/multiplex (t/m)?")
if servertype=='t':
Pyro4.config.SERVERTYPE="thread"
else:
Pyro4.config.SERVERTYPE="multiplex"
hostname=socket.gethostname()
class EmbeddedServer(object):
def multiply(self, x, y):
return x*y
print("initializing services... servertype=%s" % Pyro4.config.SERVERTYPE)
# start a name server with broadcast server as well
nameserverUri, nameserverDaemon, broadcastServer = Pyro4.naming.startNS(host=hostname)
assert broadcastServer is not None, "expect a broadcast server to be created"
print("got a Nameserver, uri=%s" % nameserverUri)
print("ns daemon location string=%s" % nameserverDaemon.locationStr)
print("ns daemon sockets=%s" % nameserverDaemon.sockets)
print("bc server socket=%s (fileno %d)" % (broadcastServer.sock, broadcastServer.fileno()))
# create a Pyro daemon
pyrodaemon=Pyro4.core.Daemon(host=hostname)
print("daemon location string=%s" % pyrodaemon.locationStr)
print("daemon sockets=%s" % pyrodaemon.sockets)
# register a server object with the daemon
serveruri=pyrodaemon.register(EmbeddedServer())
print("server uri=%s" % serveruri)
# register it with the embedded nameserver directly
nameserverDaemon.nameserver.register("example.embedded.server",serveruri)
print("")
# below is our custom event loop.
while True:
print("Waiting for events...")
# create sets of the socket objects we will be waiting on
# (a set provides fast lookup compared to a list)
nameserverSockets = set(nameserverDaemon.sockets)
pyroSockets = set(pyrodaemon.sockets)
rs=[broadcastServer] # only the broadcast server is directly usable as a select() object
rs.extend(nameserverSockets)
rs.extend(pyroSockets)
rs,_,_ = select.select(rs,[],[],3)
eventsForNameserver=[]
eventsForDaemon=[]
for s in rs:
if s is broadcastServer:
print("Broadcast server received a request")
broadcastServer.processRequest()
elif s in nameserverSockets:
eventsForNameserver.append(s)
elif s in pyroSockets:
eventsForDaemon.append(s)
if eventsForNameserver:
print("Nameserver received a request")
nameserverDaemon.events(eventsForNameserver)
if eventsForDaemon:
print("Daemon received a request")
pyrodaemon.events(eventsForDaemon)
nameserverDaemon.close()
broadcastServer.close()
pyrodaemon.close()
print("done")