Develop Reference

Python - multithreaded sockets

From my understanding python can only run 1 thread at a time so if I were to do something like this
import socket, select
from threading import Thread
import config
class Source(Thread):
def __init__(self):
self._wait = False
self._host = (config.HOST, config.PORT + 1)
self._socket = socket.socket()
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock = None
self._connections = []
self._mount = "None"
self._writers = []
self._createServer()
Thread.__init__(self)
def _createServer(self):
self._socket.bind(self._host)
self._socket.listen(2)
self._connections.append(self._socket)
self._audioPackets=[]
def _addPacket(self, packet):
self._audioPackets.append(packet)
def _removePacket(self, packet):
self._audioPackets.remove(packet)
def _getPacket(self):
if len(self._audioPackets) > 0:
return self._audioPackets[0]
else:
return None
def _sendOK(self, sock):
sock.send("OK")
def _sendDenied(self, sock):
sock.send("DENIED")
def _sendMount(self, sock):
sock.send("mount:{0}".format(self._mount))
def _sendBufPacket(self, sock, packet):
packet = "buffer:%s" % packet
sock.send(packet)
def recv(self, sock, data):
data = data.split(":", 1)
if data[0] == "WAIT": self._wait = True
elif data[0] == "STOP_WAITING": self._wait = False
elif data[0] == "LOGIN":
if data[1] == config.SOURCE_AUTH:
self._source = sock
self._sendOK(sock)
else:
self._sendClose(sock)
elif data[0] == "MOUNT":
if self._source == sock:
self._mount = data[1]
else:
self._sendClose(sock)
elif data[0] == "CLIENT":
self._sendMount(sock)
self._writers.append(sock)
def _sendCloseAll(self):
for sock in self._connections:
sock.send("CLOSE")
sock.close()
def _sendClose(self, sock):
sock.send("CLOSE")
sock.close()
def main(self):
while True:
rl, wl, xl = select.select(self._connections, self._writers, [], 0.2)
for sock in rl:
if sock == self._socket:
con, ip = sock.accept()
self._connections.append(con)
else:
data = sock.recv(config.BUFFER)
if data:
self.recv(sock, data)
else:
if sock in self._writers:
self._writers.remove(sock)
if sock in self._connections:
self._connections.remove(sock)
for sock in wl:
packet = self._getPacket()
if packet != None:
self._sendBufPacket(sock, packet)
def run(self):
self.main()
class writeThread(Thread):
def __init__(self):
self.running = False
def make(self, client):
self.client = client
self.running = True
def run(self):
host = (config.HOST, config.PORT+1)
sock = socket.socket()
sock.connect(host)
sock.send("CLIENT")
sock.send("MOUNT:mountpoint")
while self.running:
data = sock.recv(config.BUFFER)
if data:
data = data.split(":", 1)
if data[0] == "buffer":
self.client.send(data[1])
elif data[0] == "CLOSE":
self.client.close()
break
if __name__=="__main__":
source = Source()
source.start()
webserver = WebServer()
webserver.runloop()
if I need to build the webserver part I will. But, I'll explain it.
Okay, so basically when someone connects to the websever under the mountpoint that was set, They will get there own personal thread that then grabs the data from Source() and sends it to them. Now say another person connects to the mount point and the last client as well as the source is still going. Wouldn't the new client be blocked from getting the Source data considering there are two active threads?

Your understanding of how Threads work in Python seems to be incorrect, based on the question you are asking. If used correctly, threads will not be blocking: you can instantiate multiple thread with Python. The limitation is that, due to the Global Interpreter Lock (GIL), you cannot get the full parallelism expected in thread programming (e.g. simultaneous execution and thus, reduced runtime).
What is going to happen in your case is that the two threads will take, together, the same amount of time that they would take if they were executed sequentially (although that is not necessarily what happens in practice).

Okay, I have copy and pasted some Python3 code that I have already written for a project that I am currently working on. With modification, you can make this code serve your purposes.
The code uses multiprocessing and multithreading. For my purposes, I am using multiprocessing so that sockets will run on one processor, and I can run a GUI program on another processor. You can remove the multiprocessor part if you prefer. The code below runs a socket message server. The server will listen for clients one at a time. Once a client has connected, a new thread will be initiated to handle all the communications between the server and each client. The server will then continue to search for for clients. At the moment however, the server only listens to data being sent from each client, and then it prints it to the terminal. With a small amount of effort, you can modify my code to send information from the server to each client individually.
import multiprocessing
import threading
from threading import Thread
class ThreadedServer(object):
def __init__(self, host, port):
self.host = host
self.port = port
self.sock = socket(AF_INET, SOCK_STREAM)
self.sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.sock.bind((self.host, self.port))
def listen(self):
self.sock.listen(3) #Allow 3 Clients to connect to this server
while True:
#The program will search for one client at a time
print("Searching for Client")
client, address = self.sock.accept()
print(address, " is connected")
#client.settimeout(60)
#Once a client has been found, start a individual client thread
d = threading.Thread(target = self.listenToClient, args=(client, address))
d.daemon = True
d.start()
def listenToClient(self, client, address):
size = 1024
while True:
try:
data = client.recv(size)
if not data:
break
if data:
print(data)
#client.send(response)
else:
raise error('Client disconnected')
except:
client.close()
return False
def dataSharingHost():
#Using Sockets to send information between Processes
#This is the server Function
#ThreadServer(Host_IP, Port_Number), for LocalHost use ''
ThreadedServer('', 8000).listen()
def Main():
commServer = multiprocessing.Process(target=dataSharingHost, args=())
commServer.daemon = True
commServer.start()
if __name__== '__main__':
Main()
And to be fair, my code is modified from https://www.youtube.com/watch?v=qELZAi4yra8 . The client code is covered in those videos. I think the 3rd video covers the multiple client connects.

Why can I launch a thread (with the python threading module) only once?

I have simple script for watchdog on network device. Script monitors response from PING command. If there is no answer then second thread executes and first thread is stopped. If second thread is finished then first thread is resumed (checking ping). If there is no answer then following message appears:
RuntimeError: threads can only be started once
Here is my code:
#!/usr/bin/python
import os
import time
import sqlite3
from ablib import Pin
import threading
led=Pin('W9','OUTPUT')
class threadout1(threading.Thread):
def run(self):
while True:
conn = sqlite3.connect('database/database.db')
cur = conn.cursor()
cur.execute("SELECT * FROM watchdog")
rows_output = cur.fetchall()
time.sleep(1)
if rows_output[0][1] == "ping":
response = os.system("ping -c 1 " + rows_output[0][2])
if response != 0:
print "bad"
rest.start()
rest.join()
class restart(threading.Thread):
def run(self):
led.on()
time.sleep(15)
led.off()
thr = threadout1()
rest = restart()
thr.start()

You can either create the restart thread every time you need it
if response != 0:
print "bad"
restart_thread = restart()
restart_thread.start()
restart_thread.join()
or use Events
class restart_thread(threading.Thread):
def __init__(self, evt):
self.evt = evt
def run(self):
self.evt.wait()
# do stuff
self.evt.clear()
class threadout(threading.Thread):
def __init__(self, evt):
self.evt = evt
def run(self):
if #other thread needs to run once
self.evt.set()
evt = threading.Event()
restart_thread = restart(evt)
restart_thread.start()
pinging_thread = threadout(evt)
pinging_thread.start()
To make the pinging_thread wait for the restart_thread to finish, you could use another Event.

Run function in background and continue with program

I am trying to run a function in the background, whilst continuing with said code in python.
The function I want to run in the background is from socket. Looking for specific data to cut the program off.
Here is the function:
def receive():
host = ""
port = 13000
buf = 1024
addr = (host,port)
Sock = socket(AF_INET, SOCK_DGRAM)
Sock.bind(addr)
(data, addr) = Sock.recvfrom(buf)
return data
Here is the code I want to run:
while True:
r = receive()
if r == "stop":
break
#Cannot get past here, because of the function running.
#Should loop over and over, until stop data is received
print "Running program"
I have tried threading, with r = threading.Thread(target=receive()) with no joy.

Rookie error:
r = threading.Thread(target=receive())
I did not take the brackets off the receive():
r = threading.Thread(target=receive)

You can't return to the invoking thread from an invoked thread's target function. Instead, you need some inter-thread communication system. Below, is an example using Python's Queue to pass received datagrams between the two threads. I've used a threading.Event to signal when the receiver thread should stop.
#!/usr/bin/env python
import socket
import threading
from queue import Empty, Queue
class DatagramReceiver(threading.Thread):
def __init__(self, stop, queue):
super().__init__()
self._stop = stop
self._queue = queue
def run(self):
with socket.socket(AF_INET, SOCK_DGRAM) as sock:
sock.bind(('', 13000))
while not self._stop.is_set():
data = sock.recvfrom(1024)[0]
if data == 'stop':
self._stop.set()
break
self._queue.put(data)
def main():
stop = threading.Event()
queue = Queue()
reader = DatagramReceiver(stop, queue)
reader.deamon = True
reader.start()
while not stop.is_set():
user_input = input('Press RETURN to print datagrams, or q quit')
if user_input == 'q':
break
while True:
try:
datagram = queue.get_nowait()
except Empty:
break
print(datagram)
stop.set()
reader.join()

pika, stop_consuming does not work

I'm new to rabbitmq and pika, and is having trouble with stopping consuming.
channel and queue setting:
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue=new_task_id, durable=True, auto_delete=True)
Basically, consumer and producer are like this:
consumer:
def task(task_id):
def callback(channel, method, properties, body):
if body != "quit":
print(body)
else:
print(body)
channel.stop_consuming(task_id)
channel.basic_consume(callback, queue=task_id, no_ack=True)
channel.start_consuming()
print("finish")
return "finish"
producer:
proc = Popen(['app/sample.sh'], shell=True, stdout=PIPE)
while proc.returncode is None: # running
line = proc.stdout.readline()
if line:
channel.basic_publish(
exchange='',
routing_key=self.request.id,
body=line
)
else:
channel.basic_publish(
exchange='',
routing_key=self.request.id,
body="quit"
)
break
consumer task gave me output:
# ... output from sample.sh, as expected
quit
�}q(UstatusqUSUCCESSqU tracebackqNUresultqNUtask_idqU
1419350416qUchildrenq]u.
However, "finish" didn't get printed, so I'm guessing it's because channel.stop_consuming(task_id) didn't stop consuming. If so, what is the correct way to do? Thank you.

I had the same problem. It seems to be caused by the fact that internally, start_consuming calls self.connection.process_data_events(time_limit=None). This time_limit=None makes it hang.
I managed to workaround this problem by replacing the call to channel.start_consuming() with its implemenation, hacked:
while channel._consumer_infos:
channel.connection.process_data_events(time_limit=1) # 1 second

I have a class defined with member variables of channel and connection. These are initialized by a seperate thread. The consumer of MyClient Class uses the close() method and the the connection and consumer is stopped!
class MyClient:
def __init__(self, unique_client_code):
self.Channel = None
self.Conn: pika.BlockingConnection = None
self.ClientThread = self.init_client_driver()
def _close_callback(self):
self.Channel.stop_consuming()
self.Channel.close()
self.Conn.close()
def _client_driver_thread(self, tmout=None):
print("Starting Driver Thread...")
self.Conn = pika.BlockingConnection(pika.ConnectionParameters("localhost"))
self.Channel = self.Conn.channel()
def init_client_driver(self, tmout=None):
kwargs = {'tmout': tmout}
t = threading.Thread(target=self._client_driver_thread, kwargs=kwargs)
t.daemon = True
t.start()
return t
def close(self):
self.Conn.add_callback_threadsafe(self._close_callback)
self.ClientThread.join()

How to do dynamic creation of per-process queues in Python multiprocessing

I want to dynamically create multiple Processes, where each instance has a queue for incoming messages from other instances, and each instance can also create new instances. So we end up with a network of processes all sending to each other. Every instance is allowed to send to every other.
The code below would do what I want: it uses a Manager.dict() to store the queues, making sure updates are propagated, and a Lock() to protect write-access to the queues. However when adding a new queue it throws "RuntimeError: Queue objects should only be shared between processes through inheritance".
The problem is that when starting-up, we don't know how many queues will eventually be needed, so we have to create them dynamically. But since we can't share queues except at construction time, I don't know how to do that.
I know that one possibility would be to make queues a global variable instead of a managed one passed-in to __init__: the problem then, as I understand it, is that additions to the queues variable wouldn't be propagated to other processes.
EDIT I'm working on evolutionary algorithms. EAs are a type of machine learning technique. An EA simulates a "population", which evolves by survival of the fittest, crossover, and mutation. In parallel EAs, as here, we also have migration between populations, corresponding to interprocess communication. Islands can also spawn new islands, and so we need a way to send messages between dynamically-created processes.
import random, time
from multiprocessing import Process, Queue, Lock, Manager, current_process
try:
from queue import Empty as EmptyQueueException
except ImportError:
from Queue import Empty as EmptyQueueException
class MyProcess(Process):
def __init__(self, queues, lock):
super(MyProcess, self).__init__(target=lambda x: self.run(x),
args=tuple())
self.queues = queues
self.lock = lock
# acquire lock and add a new queue for this process
with self.lock:
self.id = len(list(self.queues.keys()))
self.queues[self.id] = Queue()
def run(self):
while len(list(self.queues.keys())) < 10:
# make a new process
new = MyProcess(self.lock)
new.start()
# send a message to a random process
dest_key = random.choice(list(self.queues.keys()))
dest = self.queues[dest_key]
dest.put("hello to %s from %s" % (dest_key, self.id))
# receive messages
message = True
while message:
try:
message = self.queues[self.id].get(False) # don't block
print("%s received: %s" % (self.id, message))
except EmptyQueueException:
break
# what queues does this process know about?
print("%d: I know of %s" %
(self.id, " ".join([str(id) for id in self.queues.keys()])))
time.sleep(1)
if __name__ == "__main__":
# Construct MyProcess with a Manager.dict for storing the queues
# and a lock to protect write access. Start.
MyProcess(Manager().dict(), Lock()).start()

I'm not entirely sure what your use case actually is here. Perhaps if you elaborate a bit more on why you want to have each process dynamically spawn a child with a connected queue it'll be a bit more clear what the right solution would be in this situation.
Anyway, with the question as is it seems that there is not really a good way to dynamically create pipes or queues with Multiprocessing right now.
I think that if you're willing to spawn threads within each of your processes you may be able to use multiprocessing.connection.Listener/Client to communicate back and forth. Rather than spawning threads I took an approach using network sockets and select to communicate between threads.
Dynamic process spawning and network sockets may still be flaky depending on how multiprocessing cleans up your file descriptors when spawning/forking a new process and your solution will most likely work more easily on *nix derivatives. If you're concerned about socket overhead you could use unix domain sockets to be a little more lightweight at the cost of added complexity running nodes on multiple worker machines.
Anyway, here's an example using network sockets and a global process list to accomplish this since I was unable to find a good way to make multiprocessing do it.
import collections
import multiprocessing
import random
import select
import socket
import time
class MessagePassingProcess(multiprocessing.Process):
def __init__(self, id_, processes):
self.id = id_
self.processes = processes
self.queue = collections.deque()
super(MessagePassingProcess, self).__init__()
def run(self):
print "Running"
inputs = []
outputs = []
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
address = self.processes[self.id]["address"]
print "Process %s binding to %s"%(self.id, address)
server.bind(address)
server.listen(5)
inputs.append(server)
process = self.processes[self.id]
process["listening"] = True
self.processes[self.id] = process
print "Process %s now listening!(%s)"%(self.id, process)
while inputs:
readable, writable, exceptional = select.select(inputs,
outputs,
inputs,
0.1)
for sock in readable:
print "Process %s has a readable scoket: %s"%(self.id,
sock)
if sock is server:
print "Process %s has a readable server scoket: %s"%(self.id,
sock)
conn, addr = sock.accept()
conn.setblocking(0)
inputs.append(conn)
else:
data = sock.recv(1024)
if data:
self.queue.append(data)
print "non server readable socket with data"
else:
inputs.remove(sock)
sock.close()
print "non server readable socket with no data"
for sock in exceptional:
print "exception occured on socket %s"%(sock)
inputs.remove(sock)
sock.close()
while len(self.queue) >= 1:
print "Received:", self.queue.pop()
# send a message to a random process:
random_id = random.choice(list(self.processes.keys()))
print "%s Attempting to send message to %s"%(self.id, random_id)
random_process = self.processes[random_id]
print "random_process:", random_process
if random_process["listening"]:
random_address = random_process["address"]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(random_address)
except socket.error:
print "%s failed to send to %s"%(self.id, random_id)
else:
s.send("Hello World!")
finally:
s.close()
time.sleep(1)
if __name__=="__main__":
print "hostname:", socket.getfqdn()
print dir(multiprocessing)
manager = multiprocessing.Manager()
processes = manager.dict()
joinable = []
for n in xrange(multiprocessing.cpu_count()):
mpp = MessagePassingProcess(n, processes)
processes[n] = {"id":n,
"address":("127.0.0.1",7000+n),
"listening":False,
}
print "processes[%s] = %s"%(n, processes[n])
mpp.start()
joinable.append(mpp)
for process in joinable:
process.join()
With a lot of polish and testing love this might be a logical extension to multiprocessing.Process and/or multiprocessing.Pool as this does seem like something people would use if it were available in the standard lib. It may also be reasonable to create a DynamicQueue class that uses sockets to be discoverable to other queues.
Anyway, hope it helps. Please update if you figure out a better way to make this work.

This code is based on the accepted answer. It's in Python 3 since OSX Snow Leopard segfaults on some uses of multiprocessing stuff.
#!/usr/bin/env python3
import collections
from multiprocessing import Process, Manager, Lock, cpu_count
import random
import select
import socket
import time
import pickle
class Message:
def __init__(self, origin):
self.type = "long_msg"
self.data = "X" * 3000
self.origin = origin
def __str__(self):
return "%s %d" % (self.type, self.origin)
class MessagePassingProcess(Process):
def __init__(self, processes, lock):
self.lock = lock
self.processes = processes
with self.lock:
self.id = len(list(processes.keys()))
process_dict = {"id": self.id,
"address": ("127.0.0.1", 7000 + self.id),
"listening": False
}
self.processes[self.id] = process_dict
print("new process: processes[%s] = %s" % (self.id, processes[self.id]))
self.queue = collections.deque()
super(MessagePassingProcess, self).__init__()
def run(self):
print("Running")
self.processes[self.id]["joinable"] = True
inputs = []
outputs = []
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
address = self.processes[self.id]["address"]
print("Process %s binding to %s" % (self.id, address))
server.bind(address)
server.listen(5)
inputs.append(server)
process = self.processes[self.id]
process["listening"] = True
self.processes[self.id] = process
print("Process %s now listening!(%s)" % (self.id, process))
while inputs and len(list(self.processes.keys())) < 10:
readable, writable, exceptional = select.select(inputs,
outputs,
inputs,
0.1)
# read incoming messages
for sock in readable:
print("Process %s has a readable socket: %s" % (self.id, sock))
if sock is server:
print("Process %s has a readable server socket: %s" %
(self.id, sock))
conn, addr = sock.accept()
conn.setblocking(0)
inputs.append(conn)
else:
data = True
item = bytes() # empty bytes object, to be added to
recvs = 0
while data:
data = sock.recv(1024)
item += data
recvs += 1
if len(item):
self.queue.append(item)
print("non server readable socket: recvd %d bytes in %d parts"
% (len(item), recvs))
else:
inputs.remove(sock)
sock.close()
print("non server readable socket: nothing to read")
for sock in exceptional:
print("exception occured on socket %s" % (sock))
inputs.remove(sock)
sock.close()
while len(self.queue):
msg = pickle.loads(self.queue.pop())
print("received:" + str(msg))
# send a message to a random process:
random_id = random.choice(list(self.processes.keys()))
print("%s attempting to send message to %s" % (self.id, random_id))
random_process = self.processes[random_id]
if random_process["listening"]:
random_address = random_process["address"]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(random_address)
except socket.error:
print("%s failed to send to %s"%(self.id, random_id))
else:
item = pickle.dumps(Message(self.id))
print("sending a total of %d bytes" % len(item))
s.sendall(item)
finally:
s.close()
# make a new process
if random.random() < 0.1:
mpp = MessagePassingProcess(self.processes, self.lock)
mpp.start()
else:
time.sleep(1.0)
print("process %d finished looping" % self.id)
if __name__=="__main__":
manager = Manager()
processes = manager.dict()
lock = Lock()
# make just one process: it will make more
mpp = MessagePassingProcess(processes, lock)
mpp.start()
# this doesn't join on all the other processes created
# subsequently
mpp.join()

The standard library socketserver is provided to help avoid programming select() manually. In this version, we start a socketserver in a separate thread so that each Process can do (well, pretend to do) computation in its main loop.
#!/usr/bin/env python3
# Each Node is an mp.Process. It opens a client-side socket to send a
# message to another Node. Each Node listens using a separate thread
# running a socketserver (so avoiding manual programming of select()),
# which itself starts a new thread to handle each incoming connection.
# The socketserver puts received messages on an mp.Queue, where they
# are picked up by the Node for processing once per loop. This setup
# allows the Node to do computation in its main loop.
import multiprocessing as mp
import threading, random, socket, socketserver, time, pickle, queue
class Message:
def __init__(self, origin):
self.type = "long_message"
self.data = "X" * random.randint(0, 2000)
self.origin = origin
def __str__(self):
return "Message of type %s, length %d from %d" % (
self.type, len(self.data), self.origin)
class Node(mp.Process):
def __init__(self, nodes, lock):
super().__init__()
# Add this node to the Manager.dict of node descriptors.
# Write-access is protected by a Lock.
self.nodes = nodes
self.lock = lock
with self.lock:
self.id = len(list(nodes.keys()))
host = "127.0.0.1"
port = 7022 + self.id
node = {"id": self.id, "address": (host, port), "listening": False}
self.nodes[self.id] = node
print("new node: nodes[%s] = %s" % (self.id, nodes[self.id]))
# Set up socketserver.
# don't know why collections.deque or queue.Queue don't work here.
self.queue = mp.Queue()
# This MixIn usage is directly from the python.org
# socketserver docs
class ThreadedTCPServer(socketserver.ThreadingMixIn,
socketserver.TCPServer):
pass
class HandlerWithQueue(socketserver.BaseRequestHandler):
# Something of a hack: using class variables to give the
# Handler access to this Node-specific data
handler_queue = self.queue
handler_id = self.id
def handle(self):
# could receive data in multiple chunks, so loop and
# concatenate
item = bytes()
recvs = 0
data = True
if data:
data = self.request.recv(4096)
item += data
recvs += 1
if len(item):
# Receive a pickle here and put it straight on
# queue. Will be unpickled when taken off queue.
print("%d: socketserver received %d bytes in %d recv()s"
% (self.handler_id, len(item), recvs))
self.handler_queue.put(item)
self.server = ThreadedTCPServer((host, port), HandlerWithQueue)
self.server_thread = threading.Thread(target=self.server.serve_forever)
self.server_thread.setDaemon(True) # Tell it to exit when Node exits.
self.server_thread.start()
print("%d: server loop running in thread %s" %
(self.id, self.server_thread.getName()))
# Now ready to receive
with self.lock:
# Careful: if we assign directly to
# self.nodes[self.id]["listening"], the new value *won't*
# be propagated to other Nodes by the Manager.dict. Have
# to use this hack to re-assign the Manager.dict key.
node = self.nodes[self.id]
node["listening"] = True
self.nodes[self.id] = node
def send(self):
# Find a destination. All listening nodes are eligible except self.
dests = [node for node in self.nodes.values()
if node["id"] != self.id and node["listening"]]
if len(dests) < 1:
print("%d: no node to send to" % self.id)
return
dest = random.choice(dests)
print("%d: sending to %s" % (self.id, dest["id"]))
# send
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(dest["address"])
except socket.error:
print("%s: failed to send to %s" % (self.id, dest["id"]))
else:
item = pickle.dumps(Message(self.id))
s.sendall(item)
finally:
s.close()
# Check our queue for incoming messages.
def receive(self):
while True:
try:
message = pickle.loads(self.queue.get(False))
print("%d: received %s" % (self.id, str(message)))
except queue.Empty:
break
def run(self):
print("%d: in run()" % self.id)
# Main loop. Loop until at least 10 Nodes exist. Because of
# parallel processing we might get a few more
while len(list(self.nodes.keys())) < 10:
time.sleep(random.random() * 0.5) # simulate heavy computation
self.send()
time.sleep(random.random() * 0.5) # simulate heavy computation
self.receive()
# maybe make a new node
if random.random() < 0.1:
new = Node(self.nodes, self.lock)
new.start()
# Seems natural to call server_thread.shutdown() here, but it
# hangs. But since we've set the thread to be a daemon, it
# will exit when this process does.
print("%d: finished" % self.id)
if __name__=="__main__":
manager = mp.Manager()
nodes = manager.dict()
lock = mp.Lock()
# make just one node: it will make more
node0 = Node(nodes, lock)
node0.start()
# This doesn't join on all the other nodes created subsequently.
# But everything seems to work out ok.
node0.join()