We want to know if the executor has reached max_workers. Is there a simple way to find out how many workers are running in the following code?
import concurrent.futures
def dummy_process(arg_a, arg_b):
print("ml_process", arg_a, arg_b)
time.sleep(5)
executor = concurrent.futures.ProcessPoolExecutor(max_workers=2)
def main():
while True:
executor.submit(dummy_process, "test_a", "test_b")
if __name__ == "__main__":
main()
use try except
import concurrent.futures
def dummy_process(arg_a, arg_b):
print("ml_process", arg_a, arg_b)
time.sleep(5)
executor = concurrent.futures.ProcessPoolExecutor(max_workers=2)
def main():
while True:
try:
executor.submit(dummy_process, "test_a", "test_b")
except:
print("Pool Executer is Full")
if __name__ == "__main__":
main()
Related
I want to share a variable among multiple processes.
I read this one: Shared variable in concurrent.futures.ProcessPoolExecutor() python but it didn't really help my code. I am also not an expert in this and just starting since a few weeks (first year student) :)
How is is possible to share the variable x among (all) threads as soon as it becomes available? This is what I have so far:
import concurrent.futures, time
def share():
time.sleep(1)
global x
x = "hello!"
def printshare():
while True:
time.sleep(0.5)
try:
print(x)
except Exception as e:
print(f"printshare {e}")
def main():
with concurrent.futures.ProcessPoolExecutor() as executor:
executor.submit(share)
executor.submit(printshare)
if __name__ == '__main__':
main()
And it gives me the error:
printshare name 'x' is not defined
Got it to work:
def foo(x):
time.sleep(1)
x.string = 'hello'
def foo2(x):
time.sleep(1.5)
print(x.string)
def main():
x = Value('i')
with concurrent.futures.ProcessPoolExecutor() as executor:
executor.submit(foo(x))
executor.submit(foo2(x))
if __name__ == '__main__':
main()
I want to do something like that:
def foo(tickers):
executor = ProcessPoolExecutor(2)
loop = asyncio.get_event_loop()
for ticker in tickers:
loop.run_in_executor(executor, get_data,ticker)
schedule.every().minute.at(":00").do(foo)
while True:
schedule.run_pending()
I'm using the "schedule" package to execute a function foo at a precise time of the minute.
But right now, I can only execute the threaded function with:
if __name__ == "__main__":
foo(tickers)
How can I have it executed every minute with that schedule object?
EDIT: Here is my modified code:
executor = ProcessPoolExecutor(2)
def foo(tickers):
args = (ticker for ticker in tickers)
executor.map(get_data,args)
if __name__ == "__main__":
foo(tickers)
See my comment to your question. Here is an example of what I mean using the built-in sched.scheduler class. But you can the schedule module instead with little change:
import multiprocessing
from sched import scheduler
def worker(x):
print(x, flush=True)
def foo():
pool.map(worker, (1,2))
def main():
global pool
import time
pool = multiprocessing.Pool(2)
s = scheduler(time.time, time.sleep)
s.enter(delay=1, priority=1, action=foo)
s.enter(delay=2, priority=1, action=foo)
s.run()
if __name__ == '__main__':
main()
I have this async worker functionality using tornado's ioloop.
I'm trying to shutdown the loop gracefully on Ctrl+C but getting the following error
tornado.ioloop.TimeoutError: Operation timed out after None seconds
I know I can catch it but I do want to finish the process in a graceful way, how can I achieve that?
#!/usr/bin/env python
import time
import signal
import random
from tornado import gen, ioloop, queues
concurrency = 10
def sig_exit(signum, frame):
ioloop.IOLoop.current().add_callback_from_signal(shutdown)
def shutdown():
print('Will shutdown in few seconds ...')
io_loop = ioloop.IOLoop.current()
deadline = time.time() + 3
def stop_loop():
now = time.time()
if now < deadline and (io_loop._callbacks or io_loop._timeouts):
io_loop.add_timeout(now + 1, stop_loop)
else:
io_loop.stop()
print('Shutdown')
stop_loop()
#gen.coroutine
def main():
q = queues.Queue()
q.put(1)
#gen.coroutine
def do_stuff():
print("doing stuff")
yield gen.Task(ioloop.IOLoop.instance().add_timeout, time.time() + random.randint(1, 5))
print("done doing stuff")
#gen.coroutine
def worker():
while True:
yield do_stuff()
for _ in range(concurrency):
worker()
yield q.join()
if __name__ == '__main__':
signal.signal(signal.SIGTERM, sig_exit)
signal.signal(signal.SIGINT, sig_exit)
io_loop = ioloop.IOLoop.instance()
io_loop.run_sync(main)
If you're using run_sync, you can no longer call IOLoop.stop - run_sync is now responsible for that. So if you want to make this shutdown "graceful" (instead of just raising a KeyboardInterrupt at the point where you now call stop() and exiting with a stack trace), you need to change the coroutine passed to run_sync so it exits.
One possible solution is a tornado.locks.Event:
# Create a global Event
shutdown_event = tornado.locks.Event()
def shutdown():
# Same as in the question, but instead of `io_loop.stop()`:
shutdown_event.set()
#gen.coroutine
def main():
# Use a WaitIterator to exit when either the queue
# is done or shutdown is triggered.
wait_iter = gen.WaitIterator(q.join(), shutdown_event.wait())
# In this case we just want to wait for the first one; we don't
# need to actually iterate over the WaitIterator.
yield wait_iter.next()
async def main():
tornado.options.parse_command_line()
...
app = Application(db)
app.listen(options.port)
shutdown_event = tornado.locks.Event()
def shutdown( signum, frame ):
print("shutdown database !!!!")
db.close()
shutdown_event.set()
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
await shutdown_event.wait()
print("\n\nshutdown -h now")
if __name__ == "__main__":
tornado.ioloop.IOLoop.current().run_sync(main)
I want to terminate my program when press Ctrl-C, code as follow:
#!/usr/bin/env python
# encoding: utf-8
import multiprocessing
import time
import signal
import sys
def init_worker():
signal.signal(signal.SIGINT, signal.SIG_IGN)
def worker():
while(True):
time.sleep(1.1234)
print "Working..."
if __name__ == "__main__":
pool = multiprocessing.Pool(50, init_worker)
try:
for i in range(50):
pool.apply_async(worker)
# time.sleep(10)
pool.close()
pool.join()
except KeyboardInterrupt:
print "Caught KeyboardInterrupt, terminating workers"
pool.terminate()
pool.join()
but this can not work right
You can try this way:
import multiprocessing
import time
import signal
def init_worker():
signal.signal(signal.SIGINT, signal.SIG_IGN)
def worker():
while(True):
time.sleep(1.1234)
print "Working..."
if __name__ == "__main__":
pool = multiprocessing.Pool(10, init_worker)
result = []
for i in range(10):
result.append(pool.apply_async(worker))
try:
while True:
time.sleep(0.5)
if all([r.ready() for r in result]):
break
except KeyboardInterrupt:
pool.terminate()
pool.join()
else:
pool.close()
pool.join()
How to give a variable from _While.py to __scheduler.py in this multithreading Szenario?
I tried so much stuff but nothing seems to be working.
Thread.py
from multiprocessing import Process
import _While
import _Scheduler
if __name__ == '__main__':
p1 = Process(target=_While.main)
p1.start()
p2 = Process(target=_Scheduler.main)
p2.start()
_While.py
import time
def main():
while True:
print "while"
time.sleep(0.5)
"""getting button status"""
"""giving button status to _Scheudler._scheduler"""
__Scheduler.py
import logging
import time
from apscheduler.scheduler import Scheduler
from _While import """button status"""
def _scheduler():
print "scheduler"
while """button status"""==True:
print "Button is pressed"
time.sleep(0.5)
def main():
logging.basicConfig()
scheduler = Scheduler(standalone=True)
scheduler.add_interval_job(_scheduler, seconds=2)
scheduler.start()
if __name__ == '__main__':
main()
Solution:
Thread.py
from multiprocessing import Process, Value, Array
import time
import _While
import _Scheduler
if __name__ == '__main__':
num = Value('d', 0.0)
arr = Array('i', range(10))
p1 = Process(target=_While.main, args=(num, arr))
p1.start()
p2 = Process(target=_Scheduler.main, args=(num, arr))
p2.start()
p1.join()
p2.join()
print num.value
_While
import time
def main(num, arr):
while True:
print "while"
num.value = 1
time.sleep(10)
"""getting button status"""
"""giving button status to _Scheudler._scheduler"""
__Scheduler.py
import logging
from apscheduler.scheduler import Scheduler
def _scheduler(num, arr):
while True:
print num.value
if num.value == 1:
print "mongo"
num.value = 0
break
def main(num, arr):
logging.basicConfig()
scheduler = Scheduler(standalone=True)
scheduler.add_interval_job(_scheduler, args=(num, arr), seconds=2)
scheduler.start()
if __name__ == '__main__':
main()
The only problem left is that I can't use Value without using Array
Create an instance of multiprocessing.Value in Threads.py just before you create p1 and p2, then pass the instance of Value as args to both p1 and p2 and change the main() method of _While.py and _Scheduler.py to accept the new Value parameter.
Similar to how it is done here http://docs.python.org/2/library/multiprocessing.html#sharing-state-between-processes
You could also use Queues or Pipes as suggested by Euegene C. an example can be found here http://docs.python.org/2/library/multiprocessing.html#exchanging-objects-between-processes