I'm trying to understand the basics of threading and concurrency. I want a simple case where two threads repeatedly try to access one shared resource.
The code:
import threading
class Thread(threading.Thread):
def __init__(self, t, *args):
threading.Thread.__init__(self, target=t, args=args)
self.start()
count = 0
lock = threading.Lock()
def increment():
global count
lock.acquire()
try:
count += 1
finally:
lock.release()
def bye():
while True:
increment()
def hello_there():
while True:
increment()
def main():
hello = Thread(hello_there)
goodbye = Thread(bye)
while True:
print count
if __name__ == '__main__':
main()
So, I have two threads, both trying to increment the counter. I thought that if thread 'A' called increment(), the lock would be established, preventing 'B' from accessing until 'A' has released.
Running the makes it clear that this is not the case. You get all of the random data race-ish increments.
How exactly is the lock object used?
Additionally, I've tried putting the locks inside of the thread functions, but still no luck.
You can see that your locks are pretty much working as you are using them, if you slow down the process and make them block a bit more. You had the right idea, where you surround critical pieces of code with the lock. Here is a small adjustment to your example to show you how each waits on the other to release the lock.
import threading
import time
import inspect
class Thread(threading.Thread):
def __init__(self, t, *args):
threading.Thread.__init__(self, target=t, args=args)
self.start()
count = 0
lock = threading.Lock()
def incre():
global count
caller = inspect.getouterframes(inspect.currentframe())[1][3]
print "Inside %s()" % caller
print "Acquiring lock"
with lock:
print "Lock Acquired"
count += 1
time.sleep(2)
def bye():
while count < 5:
incre()
def hello_there():
while count < 5:
incre()
def main():
hello = Thread(hello_there)
goodbye = Thread(bye)
if __name__ == '__main__':
main()
Sample output:
...
Inside hello_there()
Acquiring lock
Lock Acquired
Inside bye()
Acquiring lock
Lock Acquired
...
import threading
# global variable x
x = 0
def increment():
"""
function to increment global variable x
"""
global x
x += 1
def thread_task():
"""
task for thread
calls increment function 100000 times.
"""
for _ in range(100000):
increment()
def main_task():
global x
# setting global variable x as 0
x = 0
# creating threads
t1 = threading.Thread(target=thread_task)
t2 = threading.Thread(target=thread_task)
# start threads
t1.start()
t2.start()
# wait until threads finish their job
t1.join()
t2.join()
if __name__ == "__main__":
for i in range(10):
main_task()
print("Iteration {0}: x = {1}".format(i,x))
Related
with some help I could run a process in python, Now I wan't to share a value betwenn the two tasks. I can set the value inside the init, but I can't change it inside the run method.
And by the way: how to kill the process when the main process stops?
from multiprocessing import Process, Value
import serial
import time
class P(Process):
def __init__(self, num):
num.value = 15
super(P, self).__init__()
def run(self):
while True:
num.value = num.value + 1
print("run simple process")
time.sleep(0.5)
def main():
while True:
print("run main")
print (num.value)
time.sleep(2.5)
if __name__ == "__main__":
num = Value('d', 0.0)
p = P(num)
p.start()
#p.join()
main()
In your simplified case you just passed num value upon initialization time.
To be able to access that value in other process's methods - set it as a state of the process:
class P(Process):
def __init__(self, num):
self.num = num
self.num.value = 15
super(P, self).__init__()
def run(self):
while True:
self.num.value += 1
print("run simple process")
time.sleep(0.5)
For a more "serious" cases - consider using Managers and Synchronization primitives.
Hi i need to create 2 threads one which repeatedly writes the time of day as an
HH:MM:SS string into a global variable 100 times per second. The second thread will repeatedly read the time of day
string from that variable twice per second and try to display it to screen but code in that thread should ensure the same
string is never written twice in a row. The result is that second thread really displays to screen only once per second. i have tried following code but its not working
import threading
import time
c = threading.Condition()
flag = 0 #shared between Thread_A and Thread_B
val = ''
class Thread_A(threading.Thread):
def __init__(self, name):
threading.Thread.__init__(self)
self.name = name
def run(self):
global flag
global val #made global here
while True:
c.acquire()
if flag == 0:
time.sleep(0)
flag = 1
a=range(1,101)
for i in a:
val=time.strftime("%H:%M:%S", time.localtime(time.time()))
c.notify_all()
else:
c.wait()
c.release()
class Thread_B(threading.Thread):
def __init__(self, name):
threading.Thread.__init__(self)
self.name = name
def run(self):
global flag
global val #made global here
while True:
c.acquire()
if flag == 1:
#time.sleep(1)
flag = 0
a=range(0,2)
for i in a:
print str(val)
#val = 20
c.notify_all()
else:
c.wait()
c.release()
a = Thread_A("myThread_name_A")
b = Thread_B("myThread_name_B")
b.start()
a.start()
a.join()
b.join()
You're making this more complicated than it needs to be. You can use a simple Lock object to make sure that only one thread can access val at a time.
The code below will run on Python 2 or Python 3. To stop it, hit Enter
import time
from threading import Thread, Lock
# Rename Python 2's raw_input to input
try:
input = raw_input
except NameError:
pass
val = ''
lock = Lock()
def set_time(delay=0.01):
''' Write the current time to val '''
global val
while True:
lock.acquire()
val = time.strftime("%H:%M:%S")
lock.release()
time.sleep(delay)
def get_time(delay=0.5):
''' Read the current time from val and print
it if it hasn't been printed already
'''
oldval = ''
while True:
lock.acquire()
if val != oldval:
print(val)
oldval = val
lock.release()
time.sleep(delay)
# Start the threads
for func in (set_time, get_time):
t = Thread(target=func)
t.setDaemon(True)
t.start()
#Wait until we get some input
s = input()
some typical output
02:22:04
02:22:05
02:22:06
02:22:07
02:22:08
I am using the class that was provided by Python Thread Pool (Python recipe) in order to simulate thread pooling. I am trying to increment the value counter in function test. The problem is that it is remaining 0. I used lock that was explained in Is this simple python code thread safe but still it's not working.
Source code
#! /usr/bin/python
# -*- coding: utf-8 -*-
from Queue import Queue
from threading import Thread
import threading
lock = threading.Lock()
class Worker(Thread):
"""Thread executing tasks from a given tasks queue"""
def __init__(self, tasks):
Thread.__init__(self)
self.tasks = tasks
self.daemon = True
self.start()
def run(self):
while True:
func, args, kargs = self.tasks.get()
try: func(*args, **kargs)
except Exception, e: print e
self.tasks.task_done()
class ThreadPool:
"""Pool of threads consuming tasks from a queue"""
def __init__(self, num_threads):
self.tasks = Queue(num_threads)
for _ in range(num_threads): Worker(self.tasks)
def add_task(self, func, *args, **kargs):
"""Add a task to the queue"""
self.tasks.put((func, args, kargs))
def wait_completion(self):
"""Wait for completion of all the tasks in the queue"""
self.tasks.join()
def exp1_thread(counter):
with lock:
print counter
counter = counter + 1
def test():
# 1) Init a Thread pool with the desired number of threads
pool = ThreadPool(6)
counter = 0
for i in range(0, 10):
pool.add_task(exp1_thread,counter)
# 3) Wait for completion
pool.wait_completion()
if __name__ == "__main__":
test()
output
counter 0
counter 0
counter 0
counter 0
counter 0
counter 0
counter 0
counter 0
counter 0
counter 0
The reason why you are getting all zeros is because the integer value counter is passed by value to the threads. Each thread receives a copy of the counter and then goes to town.
You can fix this by finding a way to pass the value by reference.
Option 1. Define counter as a list you pass around:
def exp1_thread(counter):
with lock:
print counter[0]
counter[0] = counter[0] + 1
def test():
# 1) Init a Thread pool with the desired number of threads
pool = ThreadPool(6)
counter = [0]
for i in range(0, 10):
pool.add_task(exp1_thread, counter)
# 3) Wait for completion
pool.wait_completion()
Option 2. Create an object you pass around.
class Counter:
def __init__(self, initial_count):
self.count = initial_count
def exp1_thread(counter):
with lock:
print counter.count
counter.count = counter.count + 1
def test():
# 1) Init a Thread pool with the desired number of threads
pool = ThreadPool(6)
counter = Counter(0)
for i in range(0, 10):
pool.add_task(exp1_thread, counter)
# 3) Wait for completion
pool.wait_completion()
This has nothing to do with threading. The actual reason is that int is immutable in Python.
A function that just increments an int would not have the desired effect.
def inc(x):
x +=1
y = 0
inc(y)
print y # 0
If you want to increment the number you can store it in a mutable datatype (such as a list or dict) and manipulate the list.
int work differently from dictionaries, I'm sure someone well versed in Python logic can explain the difference. This is the logic I usually use and it works. As I just realized it's because you're passing the object (dict, list etc) and not the value itself.
Either declare your variables as globals (but be careful) or use say a dictionary with individual key slots for the different threads and sum them up at the end.
from threading import *
from time import sleep
myMap = {'counter' : 0}
class worker(Thread):
def __init__(self, counterMap):
Thread.__init__(self)
self.counterMap = counterMap
self.start()
def run(self):
self.counterMap['counter'] += 1
worker(myMap)
sleep(0.2)
worker(myMap)
sleep(0.2)
print(myMap)
I have a function to update a global/class variable.
So, What should care after regularly invoke such function as subthread?(in asynchronous way)
Or, any suggestions to avoid using this pattern? (the pathonic way)
import time
import threading
# through global variable or class variable
_a = 123
def update_a(): # may be called more than once
"slow updating process"
time.sleep(3)
global _a
_a += 10
return
if __name__ == '__main__':
print(_a)
th = threading.Thread(target=update_a)
th.setDaemon(True)
th.start()
print(_a)
# updating aynchrounously
time.sleep(5)
print(_a)
First of all, threads are a thing to avoid in Python altogether, but if you really want to, I'd do it like this. First, create a thread-safe object with a lock:
class ThreadSafeValue(object):
def __init__(self, init):
self._value = init
self._lock = threading.Lock()
def atomic_update(self, func):
with self._lock:
self._value = func(self._value)
#property
def value(self):
return self._value
then I'd pass that to the thread target function:
def update(val):
time.sleep(3)
val.atomic_update(lambda v: v + 10)
def main():
a = ThreadSaveValue(123)
print a.value
th = threading.Thread(target=update, args=(a,))
th.daemon = True
th.start()
print a.value
th.join()
print a.value
if __name__ == '__main__':
main()
That way you will avoid global variables and ensure the thread-safety.
This demonstrates that addition is not threadsafe (See Josiah Carlson' comment. effbot.org seems to be down right now; you can check out an archived version of the page through the wayback machine here.):
import threading
x = 0
def foo():
global x
for i in xrange(1000000):
x += 1
threads = [threading.Thread(target=foo), threading.Thread(target=foo)]
for t in threads:
t.daemon = True
t.start()
for t in threads:
t.join()
print(x)
yields some number less than 2000000. This shows that some calls to x += 1 did not properly update the variable.
The solution is to protect assignment to your global variable with a lock:
lock = threading.Lock()
def safe_foo():
global x
for i in xrange(1000000):
with lock:
x += 1
x = 0
threads = [threading.Thread(target=safe_foo), threading.Thread(target=safe_foo)]
for t in threads:
t.daemon = True
t.start()
for t in threads:
t.join()
print(x)
yields 2000000.
The following code executes two threads (multithread), each with different time delays so that each thread will finish at a different time.
Once both threads are finished module display1.py issues a print statement saying they are BOTH finished.
I would like module display1.py to issue a 'finished' statement for EACH thread AS EACH thread finishes
How can i do this ... amendments to my working code appreciated! I'd like to change as little of the current code as possible so a better form of variable transfer between the two modules might be what I'm after
display1.py
from threads1 import *
manager = ThreadManager()
manager.start(False)
print (manager.GetResults())
threads1.py
from threading import Thread
import time
class ThreadManager:
def __init__(self):
pass
def start(self, answer):
self.answer = answer
thread_refs = []
t1 = MyThread(70, 'Not finished')
t1.daemon = True
t1.start()
t2 = MyThread(2, 'Not finished')
t2.daemon = True
t2.start()
while True:
if t1.AskFinished == 'Finished' and t2.AskFinished == 'Finished': #If I break the loop after EACH site, Only the first to finish will be sent via GetResults to display1.py
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
def GetResults(self):
global results
return(results)
class MyThread(Thread):
def __init__(self, SleepWait, AskFinished):
Thread.__init__(self)
self.SleepWait = SleepWait
self.AskFinished = AskFinished
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'
What you have here (entering a very tight check loop in the main thread) is a very naive approach to threading in many languages, but especially in python where GIL contention will just slow the threads down a great bit.
What is a better idea is instead using queue.Queue to push info when a thread is completed. This allows the main thread to block on the queue instead, which is less CPU intensive as well as allowing you to know (out of order) which one is finished.
The changes you would need to make:
at the top of the module threads1.py:
import queue
finished_queue = queue.Queue()
in your start():
num_finished = 0
while True:
info = finished_queue.get()
num_finished += 1
if info is t1:
print("t1 finished")
else:
print("t2 finished")
if num_finished == 2:
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
and finally in run():
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'
finished_queue.put(self)
Some more fundamental modifications I'd make is actually pushing the result into the queue and then fetching the results out, skipping the extra step before GetResults. Furthermore, if GetResults had to stay, I'd pass them through a field on self e.g. self.results = [t1.AskFinished, t2.AskFinished]
Update:
Ok, so you want to know more about how to have display1.py print the results. It would be helpful if you could explain why it matters, because that might make a difference in how you should do this, but here's a first approach:
# threads1.py
from threading import Thread
import time
class ThreadManager:
def __init__(self):
self.threads = {}
def start(self):
t1 = MyThread(4)
t1.daemon = True
t1.start()
self.threads[1] = t1
t2 = MyThread(1)
t2.daemon = True
t2.start()
self.threads[2] = t2
def is_alive(self, thread_id):
return self.threads[thread_id].is_alive()
def GetResults(self): # or you could just access results directly
return self.results
class MyThread(Thread):
def __init__(self, SleepWait):
Thread.__init__(self)
self.SleepWait = SleepWait
def run(self):
time.sleep(self.SleepWait)
And then...
# display1.py
from threads1 import *
manager = ThreadManager()
manager.start()
t1_state = t2_state = True
while manager.is_alive(1) or manager.is_alive(2):
time.sleep(1)
if manager.is_alive(1) != t1_state:
print("t1 finished")
t1_state = manager.is_alive(1)
if manager.is_alive(2) != t2_state:
print("t2 finished")
t2_state = manager.is_alive(2)
if not manager.is_alive(1) and not manager.is_alive(2):
print("Both Finished")
break
You should eventually consider using a Queue as suggested by Crast; but let's focus on getting this right first.
Original Post:
There are a number of problems with this code.
First, you should use t1.is_alive() to check if a thread is finished. There's no need to reimplement it with AskFinished.
Second, the while True: loop in threads1.py is doing nothing at a furious rate while it waits for your threads to terminate. Take a look at the cpu usage while this is running if you don't believe me. You should throw a time.sleep(1) statement in there.
Third, why are you using a global var to return your results? That's a really strange thing to do. Just store it in self!
And finally, why does display1.py have to print the messages? Why can't thread1.py do that?
With these four points in mind, here's a thread1.py that works more sensibly:
from threading import Thread
import time
class ThreadManager:
def __init__(self):
self.results = None
def start(self, answer): # why is "answer" here?
self.answer = answer
thread_refs = []
t1 = MyThread(4, 'Not finished')
t1.daemon = True
t1.start()
t2 = MyThread(1, 'Not finished')
t2.daemon = True
t2.start()
t1_state = t2_state = True
while t1.is_alive() or t2.is_alive():
time.sleep(1)
if t1.is_alive() != t1_state:
print("t1 finished")
t1_state = t1.is_alive()
if t2.is_alive() != t2_state:
print("t2 finished")
t2_state = t2.is_alive()
if not t1.is_alive() and not t2.is_alive():
self.results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
def GetResults(self): # or you could just access results directly
return self.results
class MyThread(Thread):
def __init__(self, SleepWait, AskFinished):
Thread.__init__(self)
self.SleepWait = SleepWait
self.AskFinished = AskFinished
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'
Now, this still doesn't do exactly what you wanted, because you asked for display.py to do the displaying. To make that work, you'd have to put your while True loop in display.py and add an ThreadManager.is_alive() method that display.py could use to check whether a thread is alive or not. If you want to see how to do that let me know.
Im not familiar with threading but since no answers yet ill give it a shot.
In this:
Cant you just add two if statements before hand?
while True:
if t1.askFinished == 'Finished':
print("t1 Finished")
if t2.askFinished == 'Finished':
print("t2 Finished")
if t1.AskFinished == 'Finished' and t2.AskFinished == 'Finished': #If I break the loop after EACH site, Only the first to finish will be sent via GetResults to display1.py
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
edit: I tried changing your code as little as possible... it's not very well written though tbh. (No offense)