So I have a loop which takes a while to run and I wanted to print a rough estimation of the ETA (time required to end), I have written some code to do this but it's quite intricate and I'm sure there must be a cleaner way of doing it, any tip?
my class:
from timeit import default_timer as timer
from datetime import datetime, timedelta
import sys
class Measure:
def __init__(self):
self.initial = None
self.end = None
self.average = None
def start_end(self):
if self.initial is not None:
self.end = timer()
else:
self.initial = timer()
def avg(self):
self.start_end()
if self.end is not None:
if self.average is None:
self.average = round((self.end - self.initial), 3)
else:
self.average = round((self.average + (self.end - self.initial)) / 2, 3)
self.initial = timer()
def avg_time(self):
self.avg()
if self.average is not None:
sys.stdout.write('\r' + "Avg. Time elapsed: " + str(self.average) + " seconds")
def how_long(self):
self.start_end()
if self.end is not None:
print("Time elapsed: " + str(round((self.end - self.initial), 3)) + " seconds")
self.initial = timer()
def estimated_time(self, length):
self.avg()
if self.average is not None:
x = datetime.now() + timedelta(seconds=self.average * length)
sys.stdout.write('\r' + "I still need to work for at least...: " + str(round(self.average * length, 3))
+ " seconds. Which means I'll be done by: " + x.strftime("%d/%m/%Y %H:%M:%S"))
You can time any aspect of your code in regards to how long it takes to run, not just a particular chunk of it. But if you want to get just the time taken for the loop itself to compute, you can do something like:
import time
starttime= time.time()
**add your loop/other code here**
endtime= time.time()
print("time elapsed:", endtime - starttime)
Related
This question already has answers here:
Python: Why is threaded function slower than non thread
(2 answers)
Closed 2 years ago.
I need to implement search function for array of complex class but when I changed to multi thread, I figured out that it became worse than past!
So I test simple code and saw it is true.
My code:
import numpy as np
import threading
import time
class Test(object):
def __init__(self):
self.num_workers = 1
self.workers = []
self.poses = []
self.arr = None
def search(self, arr, val):
self.poses = []
for i, a in enumerate(arr):
if a == val:
self.poses.append(i)
return self.poses
def search_worker(self, val, ID):
search_len = int(len(self.arr) / self.num_workers)
prefix = ID * search_len
if ID == self.num_workers - 1:
search_len = int(len(self.arr) - prefix)
for i in range(search_len):
if self.arr[prefix + i] == val:
self.poses.append(i)
def search_multi_thread(self, arr, val):
self.arr = arr
self.poses = []
self.num_workers = 5
for i in range(self.num_workers):
worker = threading.Thread(target=self.search_worker, args=(val, i,))
worker.start()
self.workers.append(worker)
for i in range(self.num_workers):
self.workers[i].join()
return self.poses
if __name__ == '__main__':
t = Test()
sample = np.random.randint(1000, size=50000000)
t1 = time.perf_counter()
res = t.search(sample, 65)
t2 = time.perf_counter()
print(F'Elapsed time to search = {t2 - t1}')
t1 = time.perf_counter()
res = t.search_multi_thread(sample, 65)
t2 = time.perf_counter()
print(F'Elapsed time to search with multiple thread = {t2 - t1}')
result :
Elapsed time to search = 13.291269699999999
Elapsed time to search with multiple thread = 17.8231911
Environment:
OS = windows 10
python = 3.7.7
CPU = Intel core i7 6700HQ
Whats I wrong?
How can I solve this problem?
(I read about multiprocessing but it seems that each process has different stack so they cant access to single array)
Note, while working with threads, one thing to be kept in mind is that threads tend to increase the efficiency of your program when there are possibilities of the processr sitting idle (whatever be the cause like i/o, sleeps, user interactions, etc.) during the processing of your work. If this is not the case, then the overhead of thread switching will simply further degrade the performance of your program.
In your case, there are very low possibilities of processor sitting idle for significant time. Moreover, you are using far too many threads for this task. So, the thread switching simply overweights whatever performance you achieve from the use of threads. Hence, the performance of your program degrades further.
Massive performance boost after using multiprocessing.
import numpy as np
import time
import multiprocessing
class Test(object):
def __init__(self):
self.num_workers = 1
self.workers = []
self.poses = []
self.arr = None
def search(self, arr, val):
self.poses = []
for i, a in enumerate(arr):
if a == val:
self.poses.append(i)
return self.poses
def search_worker(self, val, ID):
search_len = int(len(self.arr) / self.num_workers)
prefix = ID * search_len
if ID == self.num_workers - 1:
search_len = int(len(self.arr) - prefix)
for i in range(search_len):
if self.arr[prefix + i] == val:
self.poses.append(i)
def search_multi_thread(self, arr, val):
self.arr = arr
self.poses = []
self.num_workers = 5
for i in range(self.num_workers):
worker = multiprocessing.Process(target=self.search_worker, args=(val, i,))
worker.start()
self.workers.append(worker)
for i in range(self.num_workers):
self.workers[i].join()
return self.poses
if __name__ == '__main__':
t = Test()
sample = np.random.randint(1000, size=50000000)
t1 = time.perf_counter()
res = t.search(sample, 65)
t2 = time.perf_counter()
print(F'Elapsed time to search = {t2 - t1}')
t1 = time.perf_counter()
res = t.search_multi_thread(sample, 65)
t2 = time.perf_counter()
print(F'Elapsed time to search with multiprocessing = {t2 - t1}')
The below code simulates a stock price and calculates its payoff. I am trying to use multiprocessing to speed up the simulations. The problem is that in CallUpAndOut where I have pool.map, I am not sure how to access total from simulations
I have tried several things like self.Simulations.Total or self.total but it doesn't work.
import numpy as np
from multiprocessing import Pool
import time
class PricingSimulatedBarrierOption:
def __init__(self, spot, strike, barrier, rate, sigma, time, sims, steps):
self.spot = spot
self.strike = strike
self.barrier = barrier
self.rate = rate
self.sigma = sigma
self.time = time
self.sims = sims
self.steps = steps
self.dt = self.time / self.steps
def Simulations(self):
total = np.zeros((self.sims,self.steps+1),float)
pathwiseS= np.zeros((self.steps+1),float)
for j in range(self.sims):
pathwiseS[0] =self.spot
total[j,0] = self.spot
for i in range(1,self.steps+1):
phi = np.random.normal()
pathwiseS[i] = pathwiseS[i-1]*(1+self.rate*self.dt+self.sigma*phi*np.sqrt(self.dt))
total[j,i]= pathwiseS[i]
return total.reshape(self.sims, self.steps+1)
def CallUpAndOut(self):
start_time = time.time()
p = Pool()
getpayoff = p.map(self.Simulations(),self.total) ###How to pass total here?
p.close()
p.join()
end_time = time.time()-start_time
print(end_time)
# getpayoff = self.Simulations()
callpayoff = np.zeros((self.sims),float)
for j in range(self.sims):
if max(getpayoff[j,])>=self.barrier:
callpayoff[j] = 0
else:
callpayoff[j] = max(getpayoff[j,self.steps-1]-self.strike,0)
return np.exp(-self.rate*self.time)*np.average(callpayoff)
c = PricingSimulatedBarrierOption(100,100,170,0.05,0.2,1,10000,252)
print(c.CallUpAndOut())
In function definition add parameter see below example:
def CallUpAndOut(self, total):
And pass array of total values in map see below example:
total = [1,2,3]
getpayoff = p.map(self.Simulations,total)
To work this I had to move the declaration outside. Below code is now able to accept variable in the Pool function.
import numpy as np
from multiprocessing import Pool
import time
class PricingSimulatedBarrierOption:
def __init__(self, spot, strike, barrier, rate, sigma, time, sims, steps):
self.spot = spot
self.strike = strike
self.barrier = barrier
self.rate = rate
self.sigma = sigma
self.time = time
self.sims = sims
self.steps = steps
self.dt = self.time / self.steps
self.pathwiseS= np.zeros((self.steps+1),float)
def Simulations(self):
print("Called")
total = np.zeros((self.sims,self.steps+1),float)
self.pathwiseS= np.zeros((self.steps+1),float)
for j in range(self.sims):
self.pathwiseS[0] =self.spot
total[j,0] = self.spot
for i in range(1,self.steps+1):
phi = np.random.normal()
self.pathwiseS[i] = self.pathwiseS[i-1]*(1+self.rate*self.dt+self.sigma*phi*np.sqrt(self.dt))
total[j,i]= self.pathwiseS[i]
return total.reshape(self.sims, self.steps+1)
def CallUpAndOut(self):
start_time = time.time()
p = Pool()
getpayoff = p.map(self.Simulations(),self.pathwiseS)
p.close()
p.join()
end_time = time.time()-start_time
print(end_time)
# getpayoff = self.Simulations()
callpayoff = np.zeros((self.sims),float)
for j in range(self.sims):
if max(getpayoff[j,])>=self.barrier:
callpayoff[j] = 0
else:
callpayoff[j] = max(getpayoff[j,self.steps-1]-self.strike,0)
return np.exp(-self.rate*self.time)*np.average(callpayoff)
I am trying to implement datetime module to my Clock OOP, but the data is not being updated with the current hour, minute and second. Please have a look at the code. I can't see what I am doing wrong.
import datetime
import time
# Python class to emulate CLock
class ClockNow2:
def __init__(self):
self.theHour = 0
self.theMinute = 0
self.theSecond = 0
self.theLocale ="Unknow"
self.AM = True
#setters and getters (transformers and accessors)
#one setter and getter for each attribute
def setHour(self):
self.theHour =datetime.timezone
def getHour(self):
return self.theHour
def setMinute(self):
self.theMinute = now.minute
def getMinute(self):
return self.theMinute
def setSecond(self):
self.theSecond = now.second
def getSecond(self):
return self.theSecond
def setLocale(self,newval):
self.theLocale = newval
def getLocale(self):
return self.theLocale
def toString(self):
clockstring = "The time in " +self.theLocale + " is " +str(self.theHour) \
+ ":" +str(self.theMinute) + ":" + str(self.theSecond)
return clockstring
I got this code for an assignment:
from stop_watch import StopWatch
size = 1000000
stopWatch = StopWatch()
sum = 0
for i in range(1, size + 1):
sum += i
stopWatch.stop()
print("The loop time is", stopWatch.get_elapsed_time(), "milliseconds")
I have to create a class which generates a stopwatch and this is my code:
import time
class StopWatch:
def __init__(self):
pass
def start(self):
self.start = time.time()
return self.start
def stop(self):
self.stop = time.time()
return self.stop
def get_elapsed_time(self):
print(str(self.stop-self.start))
I get this error:
File "week33.py", line 10, in <module>
print("The loop time is", stopWatch.get_elapsed_time(), "milliseconds")
File "/Users/Marinkton/Desktop/stop_watch.py", line 16, in get_elapsed_time
print(str(self.stop-self.start))
TypeError: unsupported operand type(s) for -: 'float' and 'method'
What am I doing wrong? I could not discover a mistake.
You can't name your functions and your properties the same thing. When you do self.stop = time.time(), you overwrite the function stop.
You need to change the name of the internal fields.
import time
class StopWatch:
def __init__(self):
self.start_time = 0
self.stop_time = 0
def start(self):
self.start_time = time.time()
return self.start_time
def stop(self):
self.stop_time = time.time()
return self.stop_time
def get_elapsed_time(self):
print(str(self.stop_time - self.start_time))
PS: You're never calling start in your code.
There are many questions relating to printing on the same line but there aren't any for printing multiple lines on the same line within the terminal.
For example:
ogeno#OH-ogeno-MBP:~|⇒ python my_script.py
Process 1: 5%
Process 2: 14%
Process 3: 55%
I want the progress of these processes to update on the same line rather than printing over and over again. I have looked at other questions that say to use the return carriage character \r and sys.stdout.flush() but it doesn't seem to change the caret to go up a line, just to the end of the current line.
EDIT: My question is different because it's to do with printing MULTIPLE lines on the same lines in the terminal. It's easy if it's just one line.
This can easily be done by using backspace. Following is the sample code that will print the percentage on the same line.
import time
print "Work in progress(0%%)", # Python 2 print without newline
for work_done in range(10):
print "\b\b\b\b\b%2d%%)" % work_done, # Backspace then overwrite
time.sleep(1)
One approach is to use the ANSI escape-code "\033[F" for going to the beginning of the previous line. The following worked well in all my terminals, just writing to the next two lines from the current terminal position:
import time
import sys
progress_1 = 'Process 1: {}%'
progress_2 = 'Process 2: {}%'
print
print
for i in range(100):
sys.stdout.write('\033[F')
sys.stdout.write('\033[F')
print(progress_1.format(i))
print(progress_2.format(i))
time.sleep(0.02)
for python 2.7 you can use,
print 2%, 3% # Using comma will print it in same line
for python 3.x
print('2%', end=" ")
Or you can use sys.stdout.write for doing it with sys.stdout.flush()
Please check my below code, I have created a demo progress bar.
"""ProgressBar Module."""
import sys
import time
class ProgressBar(object):
"""Main class for the ProgressBa."""
DEFAULT_BAR_LENGTH = float(30)
def __init__(self, start=0, step=1):
"""Init for the class."""
self.end = ProgressBar.DEFAULT_BAR_LENGTH
self.start = start
self.step = step
self.total = self.end - self.start
self.counts = self.total / self.step
self._barLength = ProgressBar.DEFAULT_BAR_LENGTH
self.set_level(self.start)
self._plotted = False
def set_level_old(self, level, initial=False):
"""Setting Level."""
self._level = level
if level < self.start:
self._level = self.start
if level > self.end:
self._level = self.end
self._ratio = float(
self._level - self.start) / float(self.end - self.start)
self._levelChars = int(self._ratio * self._barLength)
def set_level(self, level, initial=False):
"""Setting Level."""
self._level = level
if level < self.start:
self._level = self.start
if level > self.end:
self._level = self.end
self._ratio = float(self._level) / float(self._barLength)
self._levelChars = int(self._ratio * self._barLength) * self.step
def plot_progress(self):
"""Plotting the bar."""
sys.stdout.write("\r %3i%% |%s%s|" % (
int(self._ratio * self.step * 100.0),
u'\u2588' * int(self._levelChars),
' ' * int(self._barLength - self._levelChars),
))
sys.stdout.flush()
self._plotted = True
def set_and_plot(self, level):
"""Call the plot."""
old_chars = self._levelChars
self.set_level(level)
if (not self._plotted) or (old_chars != self._levelChars):
self.plot_progress()
def __del__(self):
"""Del for the class."""
sys.stdout.write("\n")
if __name__ == "__main__":
pb = ProgressBar(0, 1)
curProgress = 0
pb.plot_progress()
while curProgress <= pb.counts:
pb.set_and_plot(curProgress)
curProgress += 1
time.sleep(0.1)
del pb