I'm very much a beginner in both python and programming. Trying to get multithreading work but haven't managed so far. Grateful for any help or tips.
from threading import Thread
import time
import requests
class crawler:
def get_urls(self):
while True:
#r = self.s.get('http:\\someurl')
time.sleep(1)
print 'Thread 1'
def thread_test(self):
while True:
print 'Thread 2'
time.sleep(1)
crawl = crawler()
if __name__ == '__main__':
Thread(target=crawl.get_urls()).start()
Thread(target=crawl.thread_test()).start()
It has been a while since i have done thread programming in python but I remembered that you will have to call .join() on each thread or else the main thread will exit before your spawn thread get a chance to execute.
T1 = Thread(target=crawl.get_urls()).start()
T2 = Thread(target=crawl.thread_test()).start()
T1.join()
T2.join()
should do the trick
edit I just looked into python threading library the issue is that target is expecting a callable object. when you call target=crawl.get_urls() the method was being evaluated rather than passing the method.
from threading import Thread
import time
import requests
class crawler:
def get_urls(self):
while True:
#r = self.s.get('http:\\someurl')
time.sleep(1)
print 'Thread 1'
def thread_test(self):
while True:
print 'Thread 2'
time.sleep(1)
crawl = crawler()
if __name__ == '__main__':
Thread(target=crawl.get_urls).start()
Thread(target=crawl.thread_test).start()
ref: https://docs.python.org/3/library/threading.html#threading.Thread
Related
I have the following toy example for Python threading module
from __future__ import print_function
import threading
import time
import signal
import sys
import os
import time
class ThreadShutdown(Exception):
# Custom exception to allow clean thread exit
pass
def thread_shutdown(signum, frame):
print(" o Signal {} caught and raising ThreadShutdown exception".format(signum))
raise ThreadShutdown
def main():
"""
Register the signal handlers needed to stop
cleanly the child accessing thread
"""
signal.signal(signal.SIGTERM, thread_shutdown)
signal.signal(signal.SIGINT, thread_shutdown)
test_run_seconds = 120
try:
thread = ChildThread()
thread.start()
time.sleep(1)
while test_run_seconds > 0:
test_run_seconds -= 1
print(" o [{}] remaining time is {} seconds".format(time.asctime( time.localtime(time.time()) ), test_run_seconds))
time.sleep(1)
except ThreadShutdown:
thread.shutdown_flag.set()
thread.join()
print(" o ThreadShutdown procedure complete")
return
proc.terminate()
thread.shutdown_flag.set()
thread.join()
print(" o Test terminated")
class ChildThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.shutdown_flag = threading.Event()
def run(self):
while not self.shutdown_flag.is_set():
print(" o [{}] is the current time in child, sleep for 10s".format(time.asctime( time.localtime(time.time()))))
time.sleep(10)
return
if __name__ == "__main__":
sys.exit(main())
which behaves as expected (the main thread counts every second while the spawned thread prints only every 10 seconds.
I was trying to understand the behaviour of the same code snippet in presence of blocking waits in kernel mode in the spawned thread. For example, assume that the spawned thread now goes into a killable wait in an ioctl with a timeout of 10s, I would still expect to have the main thread counting every second. For some reason, it instead counts every 10s, as if it was blocked as well in the wait of the spawned thread. What is the reason?
I'm trying to stop the code from running if the user presses ctrl+shift+c. I use the code below. Unfortunately sys.exit() stops only "wait_for_ctrl_shift_c" function, but not "main_func". What should I use to stop them both?
Thanks.
def wait_for_ctrl_shift_c():
print ('wait_for_ctrl_shift_c is working')
keyboard.wait('ctrl+shift+c')
print('wait_for_ctrl_shift_c was pressed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
sys.exit()
def main_func():
a=0
while True:
print ('Working2 ',a)
a=a+1
sleep(1)
if __name__ == '__main__':
Thread(target = wait_for_ctrl_shift_c).start()
Thread(target = main_func).start()
There are multiple ways to do it. First of all you have 3 threads, one main thread and the other 2 (infinite loop & keyboard one) you create.
You can register signals and handle it, also you can call interrupt_main to interrupt main thread (not the while loop thread). Interrupt will go to main exception handler. Also instead of True i changed the second thread to have an attribute to check if it should run for clean exit.
import os
import threading
import time
import sys
import _thread
def wait_for_ctrl_shift_c():
print ('wait_for_ctrl_shift_c is working')
keyboard.wait('ctrl+shift+c')
print ('exiting thread')
_thread.interrupt_main()
sys.exit()
def main_func():
a=0
t = threading.currentThread()
while getattr(t, "run", True):
print ('Working2 ',a)
a=a+1
time.sleep(1)
print ('exiting main_func')
if __name__ == '__main__':
try:
t1 = threading.Thread(target = wait_for_ctrl_shift_c)
t2 = threading.Thread(target = main_func)
t1.start()
t2.start()
t1.join()
t2.join()
except:
print ('main exiting')
t2.run = False
sys.exit()
Open shell in another window, type ps to list running processes, and kill the Python one (via kill 3145, if 3145 is its PID) to stop them both. This way we kill the process within which these threads run.
This code:
import multiprocessing as mp
from threading import Thread
import subprocess
import time
class WorkerProcess(mp.Process):
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
self.code = self.subprocess.wait()
class ControlThread(Thread):
def run():
jobs = []
for _ in range(2):
job = WorkerProcess()
jobs.append(job)
job.start()
# wait for a while and then kill jobs
time.sleep(2)
for job in jobs:
job.terminate()
if __name__ == "__main__":
controller = ControlThread()
controller.start()
When I terminate the spawned WorkerProcess instances. They die just fine, however the subprocesses python -c 'import time; time.sleep(1000) runs until completition. This is well documented in the official docs, but how do I kill the child processes of a killed process?
A possbile soultion might be:
Wrap WorkerProcess.run() method inside try/except block catching SIGTERM, and terminating the subprocess.call call. But I am not sure how to catch the SIGTERM in the WorkerProcess
I also tried setting signal.signal(signal.SIGINT, handler) in the WorkerProcess, but I am getting ValuError, because it is allowed to be set only in the main thread.
What do I do now?
EDIT: As #svalorzen pointed out in comments this doesn't really work since the reference to self.subprocess is lost.
Finally came to a clean, acceptable solution. Since mp.Process.terminate is a method, we can override it.
class WorkerProcess(mp.Process):
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
self.code = self.subprocess.wait()
# HERE
def terminate(self):
self.subprocess.terminate()
super(WorkerProcess, self).terminate()
You can use queues to message to your subprocesses and ask them nicely to terminate their children before exiting themselves. You can't use signals in anywhere else but your main thread, so signals are not suitable for this.
Curiously, when I modify the code like this, even if I interrupt it with control+C, subprocesses will die as well. This may be OS related thing, though.
import multiprocessing as mp
from threading import Thread
import subprocess
import time
from Queue import Empty
class WorkerProcess(mp.Process):
def __init__(self,que):
super(WorkerProcess,self).__init__()
self.queue = que
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
while True:
a = self.subprocess.poll()
if a is None:
time.sleep(1)
try:
if self.queue.get(0) == "exit":
print "kill"
self.subprocess.kill()
self.subprocess.wait()
break
else:
pass
except Empty:
pass
print "run"
else:
print "exiting"
class ControlThread(Thread):
def run(self):
jobs = []
queues = []
for _ in range(2):
q = mp.Queue()
job = WorkerProcess(q)
queues.append(q)
jobs.append(job)
job.start()
# wait for a while and then kill jobs
time.sleep(5)
for q in queues:
q.put("exit")
time.sleep(30)
if __name__ == "__main__":
controller = ControlThread()
controller.start()
Hope this helps.
Hannu
All the examples I've been able to get to don't really address my problem, of having a certain procedure in the background constantly looping, while the rest of the program continues.
Here is a simple example of a method that works, using _thread:
import _thread
import time
def countSeconds():
time.sleep(1)
print("Second")
_thread.start_new(countSeconds, ())
def countTenSeconds():
time.sleep(10)
print("Ten seconds passed")
_thread.start_new(countTenSeconds, ())
_thread.start_new(countSeconds, ())
_thread.start_new(countTenSeconds, ())
Ignoring the obvious fact that we could track the seconds, and just print something different if it's a multiple of ten, how would I go about creating this more efficiently.
In my actual program, the threading seems to be guzzling RAM, I assume from creating multiple instance of the thread. Do I have to "start_new" thread at the end of each procedure?
Thanks for any help.
All the examples I've been able to get to don't really address my problem
Which examples?
This does work for me.
import threading
def f():
import time
time.sleep(1)
print "Function out!"
t1 = threading.Thread(target=f)
print "Starting thread"
t1.start()
time.sleep(0.1)
print "Something done"
t1.join()
print "Thread Done"
You're asking for a repeated thread, I don't get what exactly you need, this might work:
import threading
var = False
def f():
import time
counter = 0
while var:
time.sleep(0.1)
print "Function {} run!".format(counter)
counter+=1
t1 = threading.Thread(target=f)
print "Starting thread"
var = True
t1.start()
time.sleep(3)
print "Something done"
var = False
t1.join()
print "Thread Done"
use the threading.timer to continue launching a new background thread
import threading
import time
def countSeconds():
print("Second")
threading.Timer(1, countSeconds).start()
def countTenSeconds():
print("Ten seconds passed")
threading.Timer(10, countTenSeconds).start()
threading.Timer(1, countSeconds).start()
threading.Timer(10, countTenSeconds).start()
I'm trying out threads in python. I want a spinning cursor to display while another method runs (for 5-10 mins). I've done out some code but am wondering is this how you would do it? i don't like to use globals, so I assume there is a better way?
c = True
def b():
for j in itertools.cycle('/-\|'):
if (c == True):
sys.stdout.write(j)
sys.stdout.flush()
time.sleep(0.1)
sys.stdout.write('\b')
else:
return
def a():
global c
#code does stuff here for 5-10 minutes
#simulate with sleep
time.sleep(2)
c = False
Thread(target = a).start()
Thread(target = b).start()
EDIT:
Another issue now is that when the processing ends the last element of the spinning cursor is still on screen. so something like \ is printed.
You could use events:
http://docs.python.org/2/library/threading.html
I tested this and it works. It also keeps everything in sync. You should avoid changing/reading the same variables in different threads without synchronizing them.
#!/usr/bin/python
from threading import Thread
from threading import Event
import time
import itertools
import sys
def b(event):
for j in itertools.cycle('/-\|'):
if not event.is_set():
sys.stdout.write(j)
sys.stdout.flush()
time.sleep(0.1)
sys.stdout.write('\b')
else:
return
def a(event):
#code does stuff here for 5-10 minutes
#simulate with sleep
time.sleep(2)
event.set()
def main():
c = Event()
Thread(target = a, kwargs = {'event': c}).start()
Thread(target = b, kwargs = {'event': c}).start()
if __name__ == "__main__":
main()
Related to 'kwargs', from Python docs (URL in the beginning of the post):
class threading.Thread(group=None, target=None, name=None, args=(), kwargs={})
...
kwargs is a dictionary of keyword arguments for the target invocation. Defaults to {}.
You're on the right track mostly, except for the global variable. Normally you'd needed to coordinate access to shared data like that with a lock or semaphore, but in this special case you can take a short-cut and just use whether one of the threads is running or not instead. This is what I mean:
from threading import Thread
from threading import Event
import time
import itertools
import sys
def monitor_thread(watched_thread):
chars = itertools.cycle('/-\|')
while watched_thread.is_alive():
sys.stdout.write(chars.next())
sys.stdout.flush()
time.sleep(0.1)
sys.stdout.write('\b')
def worker_thread():
# code does stuff here - simulated with sleep
time.sleep(2)
if __name__ == "__main__":
watched_thread = Thread(target=worker_thread)
watched_thread.start()
Thread(target=monitor_thread, args=(watched_thread,)).start()
This is not properly synchronized. But I will not try to explain it all to you right now because it's a whole lot of knowledge. Try to read this: http://effbot.org/zone/thread-synchronization.htm
But in your case it's not that bad that things aren't synchronized correctyl. The only thing that could happen, is that the spining bar spins a few ms longer than the background task actually needs.