Import threading module
import threading
Import time module to implement sleep functionality
import time
Import datetime to get start time, end time and elapsed time
from datetime import datetime
#Function to loop 100000 times
def loop_fun(name):
for i in range(0, 10):
time.sleep(0.0001)
print(name)
print("Loop executed!")
#Function to execute loop_fun without multithreading
num =0
def without_thread(num):
starting_time = datetime.now()
for i in range(0, num):
loop_fun("Called from without_thread")
ending_time = datetime.now()
elapsed_time = (ending_time - starting_time).total_seconds()
print("\n\nTime Elapsed without_thread: "+ str(elapsed_time)+"\n\n\n\n\n")
#Function to execute loop_fun with multithreading"
def with_thread(num):
start_time = datetime.now()
threads_lst = []
# Creating threads to call loop_fun
for i in range(0, num):
threads_lst.append(threading.Thread(target=loop_fun, args=("Called from with_thread",)))
# Running threads
for threads_ in threads_lst:
threads_.start()
# Waiting for completion of all threads
for threads_ in threads_lst:
threads_.join()
end_time = datetime.now()
elapsed_time = (end_time - start_time).total_seconds()
print("\n\nTime Elapsed with_thread: "+ str(elapsed_time)+"\n\n\n\n\n")
if __name__ == "__main__":
without_thread(10)
with_thread(10)
Your indentation is messed up. Copy/paste issue or does your code look like that ?
Missing Indentation in Python makes it hard to guess what you are trying to achieve.
Nevertheless I tried and came up with this code working w/o errors. Does it do what you want ? I frankly don't know.
I added these lines, which got messed up by the forum I think
from datetime import datetime
import time
import threading
I "fixed" indentation
#Function to loop 100000 times
def loop_fun(name):
for i in range(0, 10):
time.sleep(0.0001)
print(name)
print("Loop executed!")
#Function to execute loop_fun without multithreading
num =0
def without_thread(num):
starting_time = datetime.now()
for i in range(0, num):
loop_fun("Called from without_thread")
ending_time = datetime.now()
elapsed_time = (ending_time - starting_time).total_seconds()
print("\n\nTime Elapsed without_thread: "+ str(elapsed_time)+"\n\n\n\n\n")
#Function to execute loop_fun with multithreading"
def with_thread(num):
start_time = datetime.now()
threads_lst = []
# Creating threads to call loop_fun
for i in range(0, num):
threads_lst.append(threading.Thread(target=loop_fun, args=("Called from with_thread",)))
# Running threads
for threads_ in threads_lst:
threads_.start()
# Waiting for completion of all threads
for threads_ in threads_lst:
threads_.join()
end_time = datetime.now()
elapsed_time = (end_time - start_time).total_seconds()
print("\n\nTime Elapsed with_thread: "+ str(elapsed_time)+"\n\n\n\n\n")
if __name__ == "__main__":
without_thread(10)
with_thread(10)
It's printing stuff like that
Loop executed!
Called from without_thread
Loop executed!
Called from without_thread
Loop executed!
Time Elapsed without_thread: 6.378763
I am trying to figure out how to do the following:
If "some logic" occurs for more than 15 seconds:
"execute some code"
Im not quite sure how to ask python to do this. I am working with stock prices and my index column is dtype='datetime64[ns].
I tried searching other threads for hints on how to do this but i wasn't even quite sure what to search for.
Sorry in advance if its a very elementary question.
try takinng the current time and comparing it every tick
One way to accomplish this is to use threads (multiprocessing) as they would be 2 tasks "some logic" and the "scheduler".
This can be addressed easily with the following logic:
import threading
import time
lock = threading.Lock()
cond = threading.Condition(threading.Lock())
def waitLock(timeout):
with cond:
current_time = start_time = time.time()
while current_time < start_time + timeout:
if lock.acquire(False):
return True
else:
cond.wait(timeout - current_time + start_time)
current_time = time.time()
return False
Source: How to implement a Lock with a timeout in Python 2.7
import time
def check():
if not some_logic(): return False
start_time = time.time()
while time.time() - start_time <= 15:
if not some_logic(): return False
return True
if check():
... # do stuff
This runs some_logic as many times as possible to ensure that it returns True throughout the whole 15-second period.
I've never had much opportunity to play with threading but I need it for a project i'm working on. I've significantly simplified my problem for the example I present below, but I'm pretty sure the solution to the simpler problem will get me most of the way to solving the more complex problem.
That said, here's my reduced case: I have a class whose job it is to monitor an incoming data stream that is accessed through a class method. I am calculating statistics from the stream. I would like to print a report about the incoming data to one terminal window and short intervals, and also print a summary report to another window at regular (longer) intervals.
This demo below generates data and prints reports to the same window: how would I rework this to print the separate reports to two different windows?
from __future__ import division
from random import gauss
import time
class MyStreamMonitor(object):
def __init__(self):
self.sum = 0
self.count = 0
#property
def mu(self):
return outv = self.sum/self.count
def generate_values(self):
while True:
yield gauss(0,1)
def monitor(self, report_interval=1):
start1 = time.time()
start2 = time.time()
for x in self.generate_values():
self.sum += x
self.count += 1
# print this to terminal 1
if time.time() - start1 > report_interval:
start1 = time.time()
print self.count, x
# print this to terminal 2
if time.time() - start2 > 5*report_interval:
start2 = time.time()
print self.mu
if __name__ == '__main__':
stream = MyStreamMonitor()
stream.monitor()
One approach is to write to a text file, and then in second terminal, just tail the file:
def monitor(self, report_interval=1):
second = open('report.txt', 'wt')
start1 = time.time()
start2 = time.time()
for x in self.generate_values():
self.sum += x
self.count += 1
# print this to terminal 1
if time.time() - start1 > report_interval:
start1 = time.time()
print self.count, x
second.write('%d, %s\n' % (self.count, x))
# print this to terminal 2
if time.time() - start2 > 5*report_interval:
start2 = time.time()
print self.mu
second.write('%s\n', self.mu)
second.flush()
Then on the second terminal:
$ tail -f report.txt
I ended up taking #reptilicus' advice and built this as a client/server application with redis. Here's a minimum working example:
server.py
from __future__ import division
from random import gauss
import time
import redis
class MyStreamMonitor(object):
def __init__(self):
self.sum = 0
self.count = 0
self.r = redis.StrictRedis()
#property
def mu(self):
if self.count >1:
outv = self.sum/self.count
else:
outv = 0
return outv
def generate_values(self):
while True:
yield gauss(0,1)
def monitor(self):
for x in self.generate_values():
self.sum += x
self.count += 1
# This is the magic here
self.r.publish('count', self.count)
self.r.publish('mu', self.mu)
if __name__ == '__main__':
stream = MyStreamMonitor()
stream.monitor()
listener.py
import redis
import time
import sys
r = redis.StrictRedis()
channel = sys.argv[1]
interval = float(sys.argv[2])
while True:
# Resubscribe at each iteration to ensure we are only receiving
# the newest message
pubsub = r.pubsub()
pubsub.subscribe(channel)
_ = pubsub.listen().next() # the first message is always just a "1"
message = pubsub.listen().next()
print message['data']
time.sleep(interval )
The server publishes data to the "count" and "mu" channels. So to run this, first we need to open up a terminal and kick off the server:
$ python server.py
Then we can open up a separate terminal for each channel we want to listen on, passing in the channel we want to listen to and the sleep interval as arguments.
First terminal:
$ python listener.py mu 1
Second terminal:
$ python listener.py count 2
For the record: installing redis was extremely painless and didn't really require any configuration at all. Depending on your needs installation/configuration may be more complex, but at least for this toy example I didn't need to do anything fancy.
I am running pool.map on big data array and i want to print report in console every minute.
Is it possible? As i understand, python is synchronous language, it can't do this like nodejs.
Perhaps it can be done by threading.. or how?
finished = 0
def make_job():
sleep(1)
global finished
finished += 1
# I want to call this function every minute
def display_status():
print 'finished: ' + finished
def main():
data = [...]
pool = ThreadPool(45)
results = pool.map(make_job, data)
pool.close()
pool.join()
You can use a permanent threaded timer, like those from this question: Python threading.timer - repeat function every 'n' seconds
from threading import Timer,Event
class perpetualTimer(object):
# give it a cycle time (t) and a callback (hFunction)
def __init__(self,t,hFunction):
self.t=t
self.stop = Event()
self.hFunction = hFunction
self.thread = Timer(self.t,self.handle_function)
def handle_function(self):
self.hFunction()
self.thread = Timer(self.t,self.handle_function)
if not self.stop.is_set():
self.thread.start()
def start(self):
self.stop.clear()
self.thread.start()
def cancel(self):
self.stop.set()
self.thread.cancel()
Basically this is just a wrapper for a Timer object that creates a new Timer object every time your desired function is called. Don't expect millisecond accuracy (or even close) from this, but for your purposes it should be ideal.
Using this your example would become:
finished = 0
def make_job():
sleep(1)
global finished
finished += 1
def display_status():
print 'finished: ' + finished
def main():
data = [...]
pool = ThreadPool(45)
# set up the monitor to make run the function every minute
monitor = PerpetualTimer(60,display_status)
monitor.start()
results = pool.map(make_job, data)
pool.close()
pool.join()
monitor.cancel()
EDIT:
A cleaner solution may be (thanks to comments below):
from threading import Event,Thread
class RepeatTimer(Thread):
def __init__(self, t, callback, event):
Thread.__init__(self)
self.stop = event
self.wait_time = t
self.callback = callback
self.daemon = True
def run(self):
while not self.stop.wait(self.wait_time):
self.callback()
Then in your code:
def main():
data = [...]
pool = ThreadPool(45)
stop_flag = Event()
RepeatTimer(60,display_status,stop_flag).start()
results = pool.map(make_job, data)
pool.close()
pool.join()
stop_flag.set()
One way to do this, is to use main thread as the monitoring one. Something like below should work:
def main():
data = [...]
results = []
step = 0
pool = ThreadPool(16)
pool.map_async(make_job, data, callback=results.extend)
pool.close()
while True:
if results:
break
step += 1
sleep(1)
if step % 60 == 0:
print "status update" + ...
I've used .map() instead of .map_async() as the former is synchronous one. Also you probably will need to replace results.extend with something more efficient. And finally, due to GIL, speed improvement may be much smaller than expected.
BTW, it is little bit funny that you wrote that Python is synchronous in a question that asks about ThreadPool ;).
Consider using the time module. The time.time() function returns the current UNIX time.
For example, calling time.time() right now returns 1410384038.967499. One second later, it will return 1410384039.967499.
The way I would do this would be to use a while loop in the place of results = pool(...), and on every iteration to run a check like this:
last_time = time.time()
while (...):
new_time = time.time()
if new_time > last_time+60:
print "status update" + ...
last_time = new_time
(your computation here)
So that will check if (at least) a minute has elapsed since your last status update. It should print a status update approximately every sixty seconds.
Sorry that this is an incomplete answer, but I hope this helps or gives you some useful ideas.
I want to create a python function to test the time spent in each function and print its name with its time, how i can print the function name and if there is another way to do so please tell me
def measureTime(a):
start = time.clock()
a()
elapsed = time.clock()
elapsed = elapsed - start
print "Time spent in (function name) is: ", elapsed
First and foremost, I highly suggest using a profiler or atleast use timeit.
However if you wanted to write your own timing method strictly to learn, here is somewhere to get started using a decorator.
Python 2:
def timing(f):
def wrap(*args):
time1 = time.time()
ret = f(*args)
time2 = time.time()
print '%s function took %0.3f ms' % (f.func_name, (time2-time1)*1000.0)
return ret
return wrap
And the usage is very simple, just use the #timing decorator:
#timing
def do_work():
#code
Python 3:
def timing(f):
def wrap(*args, **kwargs):
time1 = time.time()
ret = f(*args, **kwargs)
time2 = time.time()
print('{:s} function took {:.3f} ms'.format(f.__name__, (time2-time1)*1000.0))
return ret
return wrap
Note I'm calling f.func_name to get the function name as a string(in Python 2), or f.__name__ in Python 3.
After playing with the timeit module, I don't like its interface, which is not so elegant compared to the following two method.
The following code is in Python 3.
The decorator method
This is almost the same with #Mike's method. Here I add kwargs and functools wrap to make it better.
def timeit(func):
#functools.wraps(func)
def new_func(*args, **kwargs):
start_time = time.time()
result = func(*args, **kwargs)
elapsed_time = time.time() - start_time
print('function [{}] finished in {} ms'.format(
func.__name__, int(elapsed_time * 1_000)))
return result
return new_func
#timeit
def foobar():
mike = Person()
mike.think(30)
The context manager method
from contextlib import contextmanager
#contextmanager
def timeit_context(name):
start_time = time.time()
yield
elapsed_time = time.time() - start_time
print('[{}] finished in {} ms'.format(name, int(elapsed_time * 1_000)))
For example, you can use it like:
with timeit_context('My profiling code'):
mike = Person()
mike.think()
And the code within the with block will be timed.
Conclusion
Using the first method, you can easily comment out the decorator to get the normal code. However, it can only time a function. If you have some part of code that you don't what to make it a function, then you can choose the second method.
For example, now you have
images = get_images()
big_image = ImagePacker.pack(images, width=4096)
drawer.draw(big_image)
Now you want to time the big_image = ... line. If you change it to a function, it will be:
images = get_images()
big_image = None
#timeit
def foobar():
nonlocal big_image
big_image = ImagePacker.pack(images, width=4096)
drawer.draw(big_image)
Looks not so great...What if you are in Python 2, which has no nonlocal keyword.
Instead, using the second method fits here very well:
images = get_images()
with timeit_context('foobar'):
big_image = ImagePacker.pack(images, width=4096)
drawer.draw(big_image)
I don't see what the problem with the timeit module is. This is probably the simplest way to do it.
import timeit
timeit.timeit(a, number=1)
Its also possible to send arguments to the functions. All you need is to wrap your function up using decorators. More explanation here: http://www.pythoncentral.io/time-a-python-function/
The only case where you might be interested in writing your own timing statements is if you want to run a function only once and are also want to obtain its return value.
The advantage of using the timeit module is that it lets you repeat the number of executions. This might be necessary because other processes might interfere with your timing accuracy. So, you should run it multiple times and look at the lowest value.
Timeit has two big flaws: it doesn't return the return value of the function, and it uses eval, which requires passing in extra setup code for imports. This solves both problems simply and elegantly:
def timed(f):
start = time.time()
ret = f()
elapsed = time.time() - start
return ret, elapsed
timed(lambda: database.foo.execute('select count(*) from source.apachelog'))
(<sqlalchemy.engine.result.ResultProxy object at 0x7fd6c20fc690>, 4.07547402381897)
There is an easy tool for timing. https://github.com/RalphMao/PyTimer
It can work like a decorator:
from pytimer import Timer
#Timer(average=False)
def matmul(a,b, times=100):
for i in range(times):
np.dot(a,b)
Output:
matmul:0.368434
matmul:2.839355
It can also work like a plug-in timer with namespace control(helpful if you are inserting it to a function which has a lot of codes and may be called anywhere else).
timer = Timer()
def any_function():
timer.start()
for i in range(10):
timer.reset()
np.dot(np.ones((100,1000)), np.zeros((1000,500)))
timer.checkpoint('block1')
np.dot(np.ones((100,1000)), np.zeros((1000,500)))
np.dot(np.ones((100,1000)), np.zeros((1000,500)))
timer.checkpoint('block2')
np.dot(np.ones((100,1000)), np.zeros((1000,1000)))
for j in range(20):
np.dot(np.ones((100,1000)), np.zeros((1000,500)))
timer.summary()
for i in range(2):
any_function()
Output:
========Timing Summary of Default Timer========
block2:0.065062
block1:0.032529
========Timing Summary of Default Timer========
block2:0.065838
block1:0.032891
Hope it will help
Decorator method using decorator Python library:
import decorator
#decorator
def timing(func, *args, **kwargs):
'''Function timing wrapper
Example of using:
``#timing()``
'''
fn = '%s.%s' % (func.__module__, func.__name__)
timer = Timer()
with timer:
ret = func(*args, **kwargs)
log.info(u'%s - %0.3f sec' % (fn, timer.duration_in_seconds()))
return ret
See post on my Blog:
post on mobilepro.pl Blog
my post on Google Plus
My way of doing it:
from time import time
def printTime(start):
end = time()
duration = end - start
if duration < 60:
return "used: " + str(round(duration, 2)) + "s."
else:
mins = int(duration / 60)
secs = round(duration % 60, 2)
if mins < 60:
return "used: " + str(mins) + "m " + str(secs) + "s."
else:
hours = int(duration / 3600)
mins = mins % 60
return "used: " + str(hours) + "h " + str(mins) + "m " + str(secs) + "s."
Set a variable as start = time() before execute the function/loops, and printTime(start) right after the block.
and you got the answer.
Elaborating on #Jonathan Ray I think this does the trick a bit better
import time
import inspect
def timed(f:callable):
start = time.time()
ret = f()
elapsed = 1000*(time.time() - start)
source_code=inspect.getsource(f).strip('\n')
logger.info(source_code+": "+str(elapsed)+" seconds")
return ret
It allows to take a regular line of code, say a = np.sin(np.pi) and transform it rather simply into
a = timed(lambda: np.sin(np.pi))
so that the timing is printed onto the logger and you can keep the same assignment of the result to a variable you might need for further work.
I suppose in Python 3.8 one could use the := but I do not have 3.8 yet
Below is a Timer class that:
Easy to use: use directly or as decorator function, < 100 lines
Measures a lot: total calls, total time, average time, and std. deviation.
Prints pretty time
Thread-safe
This is how you use it:
# Create the timer
timer1 = Timer("a name", log_every=2)
# Use "with"
with timer1:
print("timer1")
# Reuse as a decorator
#timer1
def my_func():
print("my_func")
# Instantiate as a decorator
#Timer("another timer", log_every=1)
def my_func2():
print("my_func2")
my_func()
my_func2()
my_func()
Below is the class
from datetime import datetime
import time, logging, math, threading
class Timer(object):
'''A general timer class. Does not really belong in a judicata file here.'''
def __init__(self, name, log_every = 1):
self.name = name
self.log_every = 1
self.calls = 0
self.total_time = 0
self.total_squared_time = 0
self.min, self.max = None, 0
# Make timer thread-safe by storing the times in thread-local storage.
self._local = threading.local()
self._lock = threading.Lock()
def __enter__(self):
"""Start a new timer"""
self._local.start = datetime.utcnow()
def __exit__(self, exc_type, exc_val, exc_tb):
"""Stop the timer, and report the elapsed time"""
elapsed_time = (datetime.utcnow() - self._local.start).total_seconds()
with self._lock:
self.calls += 1
self.total_time += elapsed_time
if self.min == None or elapsed_time < self.min:
self.min = elapsed_time
if elapsed_time > self.max:
self.max = elapsed_time
self.total_squared_time += elapsed_time * elapsed_time
if self.log_every and (self.calls % self.log_every) == 0:
self.log()
def __call__(self, fn):
'''For use as a decorator.'''
def decorated_timer_function(*args, **kwargs):
with self:
return fn(*args, **kwargs)
return decorated_timer_function
#classmethod
def time_str(cls, secs):
if isinstance(secs, six.string_types):
try:
secs = float(secs)
except:
return "(bad time: %s)"%secs
sign = lambda x: x
if secs < 0:
secs = -secs
sign = lambda x: ("-" + x)
return sign("%d secs"%int(secs) if secs >= 120 else
"%.2f secs" % secs if secs >= 1 else
"%d ms" % int(secs * 1000) if secs >= .01 else
"%.2f ms" % (secs * 1000) if secs >= .0001 else
"%d ns" % int(secs * 1000 * 10000) if secs >= 1e-9 else
"%s" % secs)
def log(self):
if not self.calls:
logging.info("<Timer %s: no calls>"%self.name)
return
avg = 1.0 * self.total_time / self.calls
var = 1.0 * self.total_squared_time / self.calls - avg*avg
std_dev = self.time_str(math.sqrt(var))
total = self.time_str(self.total_time)
min, max, avg = [self.time_str(t) for t in [self.min, self.max, avg]]
logging.info("<Timer %s: N=%s, total=%s, avg=%s, min/max=%s/%s, std=%s>"
%(self.name, self.calls, total, avg, min, max, std_dev))
You can use timeit.default_timer along with a contextmanager:
from timeit import default_timer
from contextlib import contextmanager
#contextmanager
def timer():
start_time = default_timer()
try:
yield
finally:
print("--- %s seconds ---" % (default_timer() - start_time))
Use it with with statement:
def looper():
for i in range(0, 100000000):
pass
with timer():
looper()
Output:
--- 2.651526927947998 seconds ---
Here is a generic solution
def timed(fn):
# make sure wherever u used this, imports will be ready
from time import perf_counter
from functools import wraps
# wraps preserves the metadata of fn
#wraps(fn)
def inner(*args, **kwargs):
start = perf_counter()
result = fn(*args, **kwargs)
end = perf_counter()
elapsed = end - start
args_ = [str(a) for a in args]
kwargs_ = ["{0}={1}".format(k, v) for (k, v) in kwargs.items()]
all_args = args_ + kwargs_
args_str = ",".join(all_args)
print("{0} ({1}) took {2:.6f} to run.".format(fn.__name__, args_str, elapsed))
return result
return inner
define a function:
#timed
def sum_up(a,b):
return a+b
now call it:
sum_up(2,9)
For the case using timeit.timeit, if command
timeit.timeit(function_to_test, n=10000)
raise error ValueError: stmt is neither a string nor callable
or command
timeit.timeit('function_to_test', n=10000)
raise error name 'function_to_test' is not defined, then you need:
replace function_to_test or 'function_to_test' with str(function_to_test), that is
timeit.timeit(str(function_to_test), n=10000)
or if Python version >= 3.6, another way is using f string as
timeit.timeit(f'{function_to_test}', n=10000)
About version use lambda, i.e. timeit.timeit(lambda: function_to_test, n=10000), it work but, from my test, it take much longer time.
Here, is a concrete example:
import timeit
def function_to_test(n):
s = 1
for i in range(n):
s += 1
return s
print("time run function_to_test: ", timeit.timeit(str(function_to_test(1000000)), number=10000))
print("time run function_to_test: ", timeit.timeit(f'{function_to_test(1000000)}', number=10000))