I am trying to gulp threading, and started with Python Module of the week examples:
according to below code
import threading
def worker(arg=None):
"""thread worker function"""
print 'Worker thread: %s\n' % arg
return
threads = []
for i in range(5):
t = threading.Thread(target=worker, args=str(i), name="threadingPrac")
threads.append(t)
t.start()
does this mean that I am starting 5 threads ?
I have just started with threading so want to understand it better.
Yes.
Add import time and time.sleep(5) after the print statement to better see it.
import threading
import time
def worker(arg=None):
"""thread worker function"""
print 'Worker thread: %s\n' % arg
time.sleep(5)
return
threads = []
for i in range(5):
t = threading.Thread(target=worker, args=str(i), name="threadingPrac")
threads.append(t)
t.start()
Yes you can check the length of the list threads by adding this line at the bottom of your code:
print len(threads)
Output:
5 #Number of threads
Related
My multi-threading script raising this error:
thread.error : can't start new thread
when it reached 460 threads:
threading.active_count() = 460
I assume the old threads keeps stack up, since the script didn't kill them. This my code:
import threading
import Queue
import time
import os
import csv
def main(worker):
#Do Work
print worker
return
def threader():
while True:
worker = q.get()
main(worker)
q.task_done()
def main_threader(workers):
global q
global city
q = Queue.Queue()
for x in range(20):
t = threading.Thread(target=threader)
t.daemon = True
print "\n\nthreading.active_count() = " + str(threading.active_count()) + "\n\n"
t.start()
for worker in workers:
q.put(worker)
q.join()
How do I kill the old threads when their job is done? (Is the function returning not enough?)
Python threading API doesn't have any function to kill a thread (nothing like threading.kill(PID)).
That said, you should code some thread-stopping algorithm yourself. For example, your thread should somehow decide that is should terminate (e.g. check some global variable or check whether some signal has been sent) and simply return.
For example:
import threading
nthreads = 7
you_should_stop = [0 for _ in range(nthreads)]
def Athread(number):
while True:
if you_should_stop[number]:
print "Thread {} stopping...".format(number)
return
print "Running..."
for x in range(nthreads):
threading.Thread(target = Athread, args = (x, )).start()
for x in range(nthreads):
you_should_stop[x] = 1
print "\nStopped all threads!"
I try threading in python. I have some code and hear that my program is waiting for threads if i use the .join method. But in the following code I get the print 'done' earlier then the prints in my thread. But why?
def getresults(seed):
print("get results now")
results[seed]
i = 0
threads = []
for suggestengine in suggestengines.keys():
i += 1
t = threading.Thread(target=getSuggestengineResult, args = (suggestengine, seed, i))
threads.append(t)
print('threads initialized')
for thread in threads:
thread.start()
for thread in threads:
thread.join
print('done')
I want to do a infinite loop function.
Here is my code
def do_request():
# my code here
print(result)
while True:
do_request()
When use while True to do this, it's a little slow, so I want to use a thread pool to concurrently execute the function do_request(). How to do this ?
Just like use ab (Apache Bench) to test HTTP server.
Finally, I've solved this problem. I use a variable to limit the thread number.
Here is my final code, solved my problem.
import threading
import time
thread_num = 0
lock = threading.Lock()
def do_request():
global thread_num
# -------------
# my code here
# -------------
with lock:
thread_num -= 1
while True:
if thread_num <= 50:
with lock:
thread_num += 1
t = threading.Thread(target=do_request)
t.start()
else:
time.sleep(0.01)
Thanks for all replies.
You can use threading in Python to implement this.
Can be something similar to this (when using two extra threads only):
import threading
# define threads
task1 = threading.Thread(target = do_request)
task2 = threading.Thread(target = do_request)
# start both threads
task1.start()
task2.start()
# wait for threads to complete
task1.join()
task2.join()
Basically, you start as many threads as you need (make sure you don't get too many, so your system can handle it), then you .join() them to wait for tasks to complete.
Or you can get fancier with multiprocessing Python module.
Try the following code:
import multiprocessing as mp
import time
def do_request():
while(True):
print('I\'m making requests')
time.sleep(0.5)
p = mp.Process(target=do_request)
p.start()
for ii in range(10):
print 'I\'m also doing other things though'
time.sleep(0.7)
print 'Now it is time to kill the service thread'
p.terminate()
The main thread stars a service thread that does the request and goes on until it has to, and then it finishes up the service thread.
Maybe you can use the concurrent.futures.ThreadPoolExecutor
from concurrent.futures import ThreadPoolExecutor
import time
def wait_on_b(hello):
time.sleep(1)
print(hello) # b will never complete because it is waiting on a.
return 5
def wait_on_a():
time.sleep(1)
print(a.result()) # a will never complete because it is waiting on b.
return 6
executor = ThreadPoolExecutor(max_workers=2)
a = executor.submit(wait_on_b, 3)
b = executor.submit(wait_on_a)
How about this?
from threading import Thread, Event
class WorkerThread(Thread):
def __init__(self, logger, func):
Thread.__init__(self)
self.stop_event = Event()
self.logger = logger
self.func = func
def run(self):
self.logger("Going to start the infinite loop...")
#Your code
self.func()
concur_task = WorkerThread(logger, func = do_request)
concur_task.start()
To end this thread...
concur_task.stop_event.set()
concur_task.join(10) #or any value you like
Why doesn't this code "act" threaded? (Please see the output.)
import time
from threading import Thread
def main():
for nums in [range(0,5), range(5,10)]:
t = Spider(nums)
t.start()
print 'started a thread'
t.join()
print "done"
class Spider(Thread):
def __init__(self, nums):
Thread.__init__(self)
self.nums = nums
def run(self): # this is an override
for num in self.nums:
time.sleep(3) # or do something that takes a while
print 'finished %s' % (num, )
if __name__ == '__main__':
main()
Output:
started a thread
finished 0
finished 1
finished 2
finished 3
finished 4
started a thread
finished 5
finished 6
finished 7
finished 8
finished 9
done
When you say t.join(), you're telling it to wait for the thread to end.
This means, you're asking it to make a thread, start it, then wait for the thread to end before making a new one.
If you want it to act multithreaded, you'll need to move the join()s outside of the loop.
def main():
# We will store the running threads in this
threads = []
# Start the threads
for nums in [range(0,5), range(5,10)]:
t = Spider(nums)
t.start()
print 'started a thread'
threads.append(t)
# All the threads have been started
# Now we wait for them to finish
for t in threads:
t.join()
print "done"
See also:
Documentation of Thread.join()
Your Thread join t.join blocks the main thread until the thread completes execution ( http://docs.python.org/library/threading.html#threading.Thread.join ). Change your code to look something like this:
def main():
threads = []
for nums in [range(0,5), range(5,10)]:
t = Spider(nums)
t.start()
print 'started a thread'
threads.append(t)
for t in threads: t.join()
print "done"
You need to start both the threads first, and then join with them once they are both running.
I have a script and I want one function to run at the same time as the other.
The example code I have looked at:
import threading
def MyThread (threading.thread):
# doing something........
def MyThread2 (threading.thread):
# doing something........
MyThread().start()
MyThread2().start()
I am having trouble getting this working. I would prefer to get this going using a threaded function rather than a class.
This is the working script:
from threading import Thread
class myClass():
def help(self):
os.system('./ssh.py')
def nope(self):
a = [1,2,3,4,5,6,67,78]
for i in a:
print i
sleep(1)
if __name__ == "__main__":
Yep = myClass()
thread = Thread(target = Yep.help)
thread2 = Thread(target = Yep.nope)
thread.start()
thread2.start()
thread.join()
print 'Finished'
You don't need to use a subclass of Thread to make this work - take a look at the simple example I'm posting below to see how:
from threading import Thread
from time import sleep
def threaded_function(arg):
for i in range(arg):
print("running")
sleep(1)
if __name__ == "__main__":
thread = Thread(target = threaded_function, args = (10, ))
thread.start()
thread.join()
print("thread finished...exiting")
Here I show how to use the threading module to create a thread which invokes a normal function as its target. You can see how I can pass whatever arguments I need to it in the thread constructor.
There are a few problems with your code:
def MyThread ( threading.thread ):
You can't subclass with a function; only with a class
If you were going to use a subclass you'd want threading.Thread, not threading.thread
If you really want to do this with only functions, you have two options:
With threading:
import threading
def MyThread1():
pass
def MyThread2():
pass
t1 = threading.Thread(target=MyThread1, args=[])
t2 = threading.Thread(target=MyThread2, args=[])
t1.start()
t2.start()
With thread:
import thread
def MyThread1():
pass
def MyThread2():
pass
thread.start_new_thread(MyThread1, ())
thread.start_new_thread(MyThread2, ())
Doc for thread.start_new_thread
I tried to add another join(), and it seems worked. Here is code
from threading import Thread
from time import sleep
def function01(arg,name):
for i in range(arg):
print(name,'i---->',i,'\n')
print (name,"arg---->",arg,'\n')
sleep(1)
def test01():
thread1 = Thread(target = function01, args = (10,'thread1', ))
thread1.start()
thread2 = Thread(target = function01, args = (10,'thread2', ))
thread2.start()
thread1.join()
thread2.join()
print ("thread finished...exiting")
test01()
Python 3 has the facility of Launching parallel tasks. This makes our work easier.
It has for thread pooling and Process pooling.
The following gives an insight:
ThreadPoolExecutor Example
import concurrent.futures
import urllib.request
URLS = ['http://www.foxnews.com/',
'http://www.cnn.com/',
'http://europe.wsj.com/',
'http://www.bbc.co.uk/',
'http://some-made-up-domain.com/']
# Retrieve a single page and report the URL and contents
def load_url(url, timeout):
with urllib.request.urlopen(url, timeout=timeout) as conn:
return conn.read()
# We can use a with statement to ensure threads are cleaned up promptly
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
# Start the load operations and mark each future with its URL
future_to_url = {executor.submit(load_url, url, 60): url for url in URLS}
for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future]
try:
data = future.result()
except Exception as exc:
print('%r generated an exception: %s' % (url, exc))
else:
print('%r page is %d bytes' % (url, len(data)))
Another Example
import concurrent.futures
import math
PRIMES = [
112272535095293,
112582705942171,
112272535095293,
115280095190773,
115797848077099,
1099726899285419]
def is_prime(n):
if n % 2 == 0:
return False
sqrt_n = int(math.floor(math.sqrt(n)))
for i in range(3, sqrt_n + 1, 2):
if n % i == 0:
return False
return True
def main():
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)):
print('%d is prime: %s' % (number, prime))
if __name__ == '__main__':
main()
You can use the target argument in the Thread constructor to directly pass in a function that gets called instead of run.
Did you override the run() method? If you overrided __init__, did you make sure to call the base threading.Thread.__init__()?
After starting the two threads, does the main thread continue to do work indefinitely/block/join on the child threads so that main thread execution does not end before the child threads complete their tasks?
And finally, are you getting any unhandled exceptions?
the simple way to implement multithread process using threading
code snippet for the same
import threading
#function which takes some time to process
def say(i):
time.sleep(1)
print(i)
threads = []
for i in range(10):
thread = threading.Thread(target=say, args=(i,))
thread.start()
threads.append(thread)
#wait for all threads to complete before main program exits
for thread in threads:
thread.join()