Develop Reference

Connecting multiple processes on the same socket

In this question, the questioner said that we can not use the same socket for the same process (Monitor creates a new process for each message with unseen id), but he did the opposite in the code as each time he created a monitor, he created it on the same socket? I guess there should be no problem if we create multiple processes and connect them to the same socket using socket.bind("localhost:8888") please?
import zmq
from zmq.eventloop import ioloop
from zmq.eventloop.zmqstream import ZMQStream
class Monitor(object):
def __init(self)
self.context = zmq.Context()
self.socket = self.context.socket(zmq.DEALER)
self.socket.connect("tcp//127.0.0.1:5055")
self.stream = ZMQStream(self._socket)
self.stream.on_recv(self.somefunc)
def initialize(self,id)
self._id = id
def somefunc(self, something)
"""work here and send back results if any """
import json
jdecoded = json.loads(something)
if self_id == jdecoded['_id']
""" good im the right monitor for you """
work = jdecoded['message']
results = algorithm (work)
self.socket.send(json.dumps(results))
else:
"""let some other process deal with it, not mine """
pass
class Prefect(object):
def __init(self, id)
self.context = zmq.Context()
self.socket = self.context.socket(zmq.DEALER)
self.socket.bind("tcp//127.0.0.1:5055")
self.stream = ZMQStream(self._socket)
self.stream.on_recv(self.check_if)
self._id = id
self.monitors = []
def check_if(self,message):
"""find out from message's id whether we have
started a proces for it previously"""
import json
jdecoded = json.loads(message)
this_id = jdecoded['_id']
if this_id in self.monitors:
pass
else:
"""start new process for it should have its won socket """
new = Monitor()
import Process
newp = Process(target=new.initialize,args=(this_id) )
newp.start()
self.monitors.append(this_id) ## ensure its remembered
Original question: here.

Python class variable not updated by callback method

I am struggling with a class variable that does not
get updated properly.
What I do is instantiate a class object and call one of its methods. this method will start a Process for async result handling. The process is given a callback method, which is an instance method from my object.
When my process is finished, a class variable should have been updated by the callback method. This didn't happen. During callback executing, everything looks fine.
Can someone point me at what I am doing wrong here?
Here is my code:
import subprocess
import logging
try:
from multiprocessing import Process
except ImportError:
# For pre 2.6 releases
from threading import Thread as Process
def runner(printout, callback):
command = "ls"
data = subprocess.Popen(
command,
bufsize=100000,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait until finished
# get output
(last_output, myerr) = data.communicate()
last_output = bytes.decode(last_output)
myerr = bytes.decode(myerr)
if len(myerr) > 0:
logging.debug(myerr)
if printout and callable(callback):
callback(last_output)
return
class MyClass(object):
def __init__(self, myres):
if myres is None:
logging.warning("No result object given. Results will be printed to stdout only!")
self.results = myres
def _result_callback(self, scan_results):
if scan_results:
for line in scan_results.splitlines():
self.results = self.results + line
print(self.results)
def task_starter(self):
process = Process(
target=runner,
args=(True, self._result_callback),
)
process.daemon = True
process.start()
while process.is_alive():
logging.debug("Still running...")
process.join(2)
result = ""
myjob = MyClass(result)
myjob.task_starter()
print("\n")
print("SAME RESULT: %s" % myjob.results)
Thanks in advance,
ChocoRolls

How to communicate with multiple servers using the single class defined in python and run it parallel

I have a python class that communicates with a server. That python class has many functions, i was using few functions. But i want to access multiple servers data at same time using the python class i have.
Am trying something like this, but it'll run one after the other. And i wanted to get the data at same time.
import threading
from server_class import server
class runMonitor(threading.Thread):
def __init__(self,func):
self.func = func
threading.Thread.__init__(self)
def run(self):
self.func()
def monitorSB(ipAddr):
sb = server(ipAddr)
sb.readInfo()
print ('\nReading Registers...\n')
sb.read_rx()
sb.read_tx()
i = 0
while(1):
if i == 0:
print 'Monitoring Registers...'
i = 1
sb.monitor_tx()
sb.monitor_rx()
t = runMonitor(monitorSB('192.168.10.78'))
q = runMonitor(monitorSB('192.168.10.101'))
t.start()
q.start()
print ('\nTest Done...\n')
In the above code, i wanted to access both servers at same time. Help me how to run in parallel

The thing is by saying monitorSB('192.168.10.78') it will execute it before passing it to your thread. Try this:
import threading
from server_class import server
class runMonitor(threading.Thread):
def __init__(self,func, param):
self.func = func
self.param = param
threading.Thread.__init__(self)
def run(self):
self.func(self.param)
def monitorSB(ipAddr):
sb = server(ipAddr)
sb.readInfo()
print ('\nReading Registers...\n')
sb.read_rx()
sb.read_tx()
i = 0
while(1):
if i == 0:
print 'Monitoring Registers...'
i = 1
sb.monitor_tx()
sb.monitor_rx()
t = runMonitor(monitorSB, '192.168.10.78')
q = runMonitor(monitorSB, '192.168.10.101')
t.start()
q.start()
print ('\nTest Done...\n')

Changing mutables inside Socketserver.handle() - Python 3.3

I have a problem to change the data variable in the class NetworkManagerData. Everytime a request with 'SIT' comes to the server the variable 'master_ip' and 'time_updated' are updated. I have chosen a dictionary for my values as a container because it is mutable. But everytime i get a new request it has it old values in it.
Like:
First Request:
>>False
>>True
Second Request:
>>False
>>True
Third Request without 'SIT':
>>False
>>False
Do I have some missunderstanding with these mutables. Or are there some special issues with using dictionarys in multiprocessing?
Code to start the server:
HOST, PORT = "100.0.0.1", 11880
network_manager = NetworkManagerServer((HOST, PORT), NetworkManagerHandler)
network_manager_process =
multiprocessing.Process(target=network_manager.serve_forever)
network_manager_process.daemon = True
network_manager_process.start()
while True:
if '!quit' in input():
network_manager_process.terminate()
sys.exit()
Server:
from multiprocessing import Lock
import os
import socketserver
class NetworkManagerData():
def __init__(self):
self.lock = Lock()
self.data = {'master_ip': '0.0.0.0', 'time_updated': False}
class NetworkManagerServer(socketserver.ForkingMixIn, socketserver.TCPServer):
def __init__(self, nmw_server, RequestHandlerClass):
socketserver.TCPServer.__init__(self, nmw_server, RequestHandlerClass)
self.nmd = NetworkManagerData()
def finish_request(self, request, client_address):
self.RequestHandlerClass(request, client_address, self, self.nmd)
class NetworkManagerHandler(socketserver.StreamRequestHandler):
def __init__(self, request, client_address, server, nmd):
self.request = request
self.client_address = client_address
self.server = server
self.setup()
self.nmd = nmd
try:
self.handle(self.nmd)
finally:
self.finish()
def handle(self, nmd):
print(nmd.data.get('time_updated')) # <<<- False ->>>
while True:
self.data = self.rfile.readline()
if self.data:
ds = self.data.strip().decode('ASCII')
header = ds[0:3]
body = ds[4:]
if 'SIT' in header:
# ...
nmd.lock.acquire()
nmd.data['master_ip'] = self.client_address[0] # <-
nmd.data['time_updated'] = True # <-
nmd.lock.release()
# ...
print(nmd.data.get('time_updated')) # <<<- True ->>>
else:
print("Connection closed: " + self.client_address[0] + ":" +
str(self.client_address[1]))
return
Thanks!

Ok, the use of multiprocessing.Value and multiprocessing.Array have solved my problem. :)
If you give some variables that are not part of the multiprocessing library to a process it will only copy the variables for its own process, there is no connection between the original and the copy. The variable is still mutable, but only in its own copy.
To work on the original variable in the memory you have to use multiprocessing.Array or multiprocessing.Value. There are other things like variable managers or queues to get this done. What you want to use depends on your case.
So I changed the datamanager class:
class NetworkManagerData():
def __init__(self):
self.lock = multiprocessing.Lock()
self.master_ip = multiprocessing.Array('B', (255,255,255,255))
self.time_updated = multiprocessing.Value('B', False)
To set the IP I am using this now:
nmd.lock.acquire()
ip_array = []
for b in self.client_address[0].split('.'):
ip_array.append(int(b))
nmd.master_ip[:] = ip_array
nmd.lock.release()
To read the IP I am using this:
self.wfile.write(("GIP|%s.%s.%s.%s" %
(nmd.master_ip[0], nmd.master_ip[1], nmd.master_ip[2],
nmd.master_ip[3]) + '\n').encode('ASCII'))

Python equivalent of Perl's HTTP::Async->next_response

I'm looking for a way to do the equivalent of Perl's HTTP::Async module's next_response method
The HTTP::Async module doesn't spawn any background threads, nor does it use any callbacks. Instead, every time anyone (in my case, the main thread) calls next_response on the object, all the data that has been received by the OS so far is read (blocking, but instantaneous since it only processes data that's already been received). If this is the end of the response, then next_response returns an HTTP::Response object, otherwise it returns undef.
Usage of this module looks something like (pseudocode):
request = HTTP::Async(url)
do:
response = request->next_response()
if not response:
sleep 5 # or process events or whatever
while not response
# Do things with response
As far as I can see, Python's urllib or http.client don't support this style. As for why I want to do it in this style:
This is for an embedded Python environment where I can't spawn threads, nor have Python spawn any.
I'm restricted to a single thread that is actually the embedding application's thread. This means I cannot have any delayed callbacks either - the application decides when to let my Python code run. All I can do is request the embedding application to invoke a callback of my choosing every 50 milliseconds, say.
Is there a way to do this in Python?
For reference, this is an example of the Perl code I have right now and that I'm looking to port to Python:
httpAsync = HTTP::Async->new()
sub httpRequestAsync {
my ($url, $callback) = #_; # $callback will be called with the response text
$httpAsync->add(new HTTP::Request(GET => $url));
# create_timer causes the embedding application to call the supplied callback every 50ms
application::create_timer(50, sub {
my $timer_result = application::keep_timer;
my $response = $httpAsync->next_response;
if ($response) {
my $responseText = $response->decoded_content;
if ($responseText) {
$callback->($responseText);
}
$timer_result = application::remove_timer;
}
# Returning application::keep_timer will preserve the timer to be called again.
# Returning application::remove_timer will remove the timer.
return $timer_result;
});
}
httpRequestAsync('http://www.example.com/', sub {
my $responseText = $_[0];
application::display($responseText);
});
Edit: Given that this is for an embedded Python instance, I'll take all the alternatives I can get (part of the standard library or otherwise) as I'll have to evaluate all of them to make sure they can run under my particular constraints.

Note: If you're interested in only retrieving data when YOU call for data to be received, simply add a flag to handle_receive and add it to the sleep block inside handle_receive thus giving you data only when you call your function.
#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
import asyncore, errno
from socket import AF_INET, SOCK_STREAM
from time import sleep
class sender():
def __init__(self, sock_send):
self.s = sock_send
self.bufferpos = 0
self.buffer = {}
self.alive = 1
def send(self, what):
self.buffer[len(self.buffer)] = what
def writable(self):
return (len(self.buffer) > self.bufferpos)
def run(self):
while self.alive:
if self.writable():
logout = str([self.buffer[self.bufferpos]])
self.s(self.buffer[self.bufferpos])
self.bufferpos += 1
sleep(0.01)
class SOCK(asyncore.dispatcher):
def __init__(self, _s=None, config=None):
self.conf = config
Thread.__init__(self)
self._s = _s
self.inbuffer = ''
#self.buffer = ''
self.lockedbuffer = False
self.is_writable = False
self.autounlockAccounts = {}
if _s:
asyncore.dispatcher.__init__(self, _s)
self.sender = sender(self.send)
else:
asyncore.dispatcher.__init__(self)
self.create_socket(AF_INET, SOCK_STREAM)
#if self.allow_reuse_address:
# self.set_resue_addr()
self.bind((self.conf['SERVER'], self.conf['PORT']))
self.listen(5)
self.sender = None
self.start()
def parse(self):
self.lockedbuffer = True
## Parse here
print self.inbuffer
self.inbuffer = ''
self.lockedbuffer = False
def readable(self):
return True
def handle_connect(self):
pass
def handle_accept(self):
(conn_sock, client_address) = self.accept()
if self.verify_request(conn_sock, client_address):
self.process_request(conn_sock, client_address)
def process_request(self, sock, addr):
x = SOCK(sock, config={'PARSER' : self.conf['PARSER'], 'ADDR' : addr[0], 'NAME' : 'CORE_SUB_SOCK_('+str(addr[0]) + ')'})
def verify_request(self, conn_sock, client_address):
return True
def handle_close(self):
self.close()
if self.sender:
self.sender.alive = False
def handle_read(self):
data = self.recv(8192)
while self.lockedbuffer:
sleep(0.01)
self.inbuffer += data
def writable(self):
return True
def handle_write(self):
pass
def run(self):
if not self._s:
asyncore.loop()
imap = SOCK(config={'SERVER' : '', 'PORT' : 6668})
imap.run()
while 1
sleep(1)
Something along the lines of this?
Asyncore socket that always appends to the inbuffer when there's data to recieve.
You can modify it however you want to, i just pasted a piece of code from another project that happens to be Threaded :)
Last attempt:
class EchoHandler(asyncore.dispatcher_with_send):
def handle_read(self):
data = self.recv(8192)
if data:
self.send(data)