Develop Reference

Alternative to global variables when logging stats about requests

I have a program that logs some messages about data that I download. Besides that, I would like to display some stats about the requests with every k-requests that I make to a site (k is 10 in my case) + some overall stats at the end of the execution.
At the moment I have an implementation that I am not happy with, as it uses global variables. I am looking for a cleaner alternative. It looks like this (Note: please ignore the fact that I am using print instead of logging and that I am measuring the passing of time using time.time instead of time.perf_counter (read here that the latter would be a better option):
import time
import pprint
def f2(*args, **kwargs):
global START_TIME
global NO_REQUESTS
global TOTAL_TIME_FOR_REQUESTS
global MAX_TIME_FOR_REQUEST
global AVERAGE_TIME_FOR_REQUESTS
global TOTAL_TIME_FOR_DECODING
global TOTAL_TIME_FOR_INTERSECT
# ... logic that changes values of most of these global variables
if NO_REQUESTS % 10 == 0:
AVERAGE_TIME_FOR_REQUESTS = TOTAL_TIME_FOR_REQUESTS / NO_REQUESTS
print()
print('no requests so far: ' + str(NO_REQUESTS))
print('average request time: {:.2f}s'.format(AVERAGE_TIME_FOR_REQUESTS))
print('max request time: {:.2f}s'.format(MAX_TIME_FOR_REQUEST))
elapsed = time.time() - START_TIME
hours_elapsed = elapsed // 3600
minutes_elapsed = (elapsed % 3600) // 60
seconds_elapsed = ((elapsed % 3600) % 60)
print('time elapsed so far: {}h {}m {:.2f}s'.format(hours_elapsed, minutes_elapsed, seconds_elapsed))
print()
time5 = time.time()
decoded = some_module.decode(res.content)
time6 = time.time()
elapsed2 = time6 - time5
TOTAL_TIME_FOR_DECODING += elapsed2
return something
def f1(*args, **kwargs):
global START_TIME
global TOTAL_TIME_FOR_REQUESTS
TOTAL_TIME_FOR_REQUESTS = 0
global MAX_TIME_FOR_REQUEST
MAX_TIME_FOR_REQUEST = 0
global NO_REQUESTS
NO_REQUESTS = 0
global AVERAGE_TIME_FOR_REQUESTS
AVERAGE_TIME_FOR_REQUESTS = 0
global TOTAL_TIME_FOR_DECODING
TOTAL_TIME_FOR_DECODING = 0
global TOTAL_TIME_FOR_INTERSECT
TOTAL_TIME_FOR_INTERSECT = 0
f2() # notice call to other function!
# ... some logic
return some_results
def output_final_stats(elapsed, results, precision='{:.3f}'):
print()
print('=============================')
hours_elapsed = elapsed // 3600
minutes_elapsed = (elapsed % 3600) // 60
seconds_elapsed = ((elapsed % 3600) % 60)
print("TIME ELAPSED: {:.3f}s OR {}h {}m {:.3f}s".format(
elapsed, hours_elapsed, minutes_elapsed, seconds_elapsed))
print("out of which:")
# print((precision+'s for requests)'.format(TOTAL_TIME_FOR_REQUESTS)))
print('{:.3f}s for requests'.format(TOTAL_TIME_FOR_REQUESTS))
print('{:.3f}s for decoding'.format(TOTAL_TIME_FOR_DECODING))
print('{:.3f}s for intersect'.format(TOTAL_TIME_FOR_INTERSECT))
total = TOTAL_TIME_FOR_REQUESTS + TOTAL_TIME_FOR_DECODING + TOTAL_TIME_FOR_INTERSECT
print('EXPECTED: {:.3f}s'.format(total))
print('DIFF: {:.3f}s'.format(elapsed - total))
print()
print('AVERAGE REQUEST TIME: {:.3f}s'.format(AVERAGE_TIME_FOR_REQUESTS))
print('TOTAL NO. REQUESTS: ' + str(NO_REQUESTS))
print('MAX REQUEST TIME: {:.3f}s'.format(MAX_TIME_FOR_REQUEST))
print('TOTAL NO. RESULTS: ' + str(len(results)))
pprint('RESULTS: {}'.format(results), indent=4)
if __name__ == '__main__':
START_TIME = time.time()
results = f1(some_params)
final_time = time.time()
elapsed = final_time - START_TIME
output_final_stats(elapsed, results)
The way I thought of it (not sure if the best option, open to alternatives) is to somehow have a listener on the NO_REQUESTS variable and whenever that number reaches a multiple of 10 trigger the logging of the variables that I am interested in. Nonetheless, where would I store those variables, what would be their namespace?
Another alternative would be to maybe have a parametrised decorator for one of my functions, but in this case I am not sure how easy it would be to pass the values that I am interested in from one function to another.

I think the cleanest way is to use a parametrized class decorator.
class LogEveryN:
def __init__(self, n=10):
self.n = n
self.number_of_requests = 0
self.total_time_for_requests = 0
self.max_time_for_request = 0
self.average_time_for_request = 0
def __call__(self, func, *args, **kwargs):
def wrapper(*args, **kwargs):
self.number_of_request += 1
if self.number_of_request % self.n:
# Do your computation and logging
return func(*args, **kwargs)
return wrapper
#LogEveryN(n=5)
def request_function():
pass

Using Python multiprocessing library inside nested objects

I'm trying to use the multiprocessing library to parallelize some expensive calculations without blocking some others, much lighter. The both need to interact through some variables, although the may run with different paces.
To show this, I have created the following example, that works fine:
import multiprocessing
import time
import numpy as np
class SumClass:
def __init__(self):
self.result = 0.0
self.p = None
self.return_value = None
def expensive_function(self, new_number, return_value):
# Execute expensive calculation
#######
time.sleep(np.random.random_integers(5, 10, 1))
return_value.value = self.result + new_number
#######
def execute_function(self, new_number):
print(' New number received: %f' % new_number)
self.return_value = multiprocessing.Value("f", 0.0, lock=True)
self.p = multiprocessing.Process(target=self.expensive_function, args=(new_number, self.return_value))
self.p.start()
def is_executing(self):
if self.p is not None:
if not self.p.is_alive():
self.result = self.return_value.value
self.p = None
return False
else:
return True
else:
return False
if __name__ == '__main__':
sum_obj = SumClass()
current_value = 0
while True:
if not sum_obj.is_executing():
# Randomly determine whether the function must be executed or not
if np.random.rand() < 0.25:
print('Current sum value: %f' % sum_obj.result)
new_number = np.random.rand(1)[0]
sum_obj.execute_function(new_number)
# Execute other (light) stuff
#######
print('Executing other stuff')
current_value += sum_obj.result * 0.1
print('Current value: %f' % current_value)
time.sleep(1)
#######
Basically, in the main loop some light function is executed, and depending on a random condition, some heavy work is sent to another process if it has already finished the previous one, carried out by an object which needs to store some data between executions. Although expensive_function needs some time, the light function keeps on executing without being blocked.
Although the above code gets the job done, I'm wondering: is it the best/most appropriate method to do this?
Besides, let us suppose the class SumClass has an instance of another object, which also needs to store data. For example:
import multiprocessing
import time
import numpy as np
class Operator:
def __init__(self):
self.last_value = 1.0
def operate(self, value):
print(' Operation, last value: %f' % self.last_value)
self.last_value *= value
return self.last_value
class SumClass:
def __init__(self):
self.operator_obj = Operator()
self.result = 0.0
self.p = None
self.return_value = None
def expensive_function(self, new_number, return_value):
# Execute expensive calculation
#######
time.sleep(np.random.random_integers(5, 10, 1))
# Apply operation
number = self.operator_obj.operate(new_number)
# Apply other operation
return_value.value = self.result + number
#######
def execute_function(self, new_number):
print(' New number received: %f' % new_number)
self.return_value = multiprocessing.Value("f", 0.0, lock=True)
self.p = multiprocessing.Process(target=self.expensive_function, args=(new_number, self.return_value))
self.p.start()
def is_executing(self):
if self.p is not None:
if not self.p.is_alive():
self.result = self.return_value.value
self.p = None
return False
else:
return True
else:
return False
if __name__ == '__main__':
sum_obj = SumClass()
current_value = 0
while True:
if not sum_obj.is_executing():
# Randomly determine whether the function must be executed or not
if np.random.rand() < 0.25:
print('Current sum value: %f' % sum_obj.result)
new_number = np.random.rand(1)[0]
sum_obj.execute_function(new_number)
# Execute other (light) stuff
#######
print('Executing other stuff')
current_value += sum_obj.result * 0.1
print('Current value: %f' % current_value)
time.sleep(1)
#######
Now, inside the expensive_function, a function member of the object Operator is used, which needs to store the number passed.
As expected, the member variable last_value does not change, i.e. it does not keep any value.
Is there any way of doing this properly?
I can imagine I could arrange everything so that I only need to use one class level, and it would work well. However, this is a toy example, in reality there are different levels of complex objects and it would be hard.
Thank you very much in advance!

from concurrent.futures import ThreadPoolExecutor
from numba import jit
import requests
import timeit
def timer(number, repeat):
def wrapper(func):
runs = timeit.repeat(func, number=number, repeat=repeat)
print(sum(runs) / len(runs))
return wrapper
URL = "https://httpbin.org/uuid"
#jit(nopython=True, nogil=True,cache=True)
def fetch(session, url):
with session.get(url) as response:
print(response.json()['uuid'])
#timer(1, 1)
def runner():
with ThreadPoolExecutor(max_workers=25) as executor:
with requests.Session() as session:
executor.map(fetch, [session] * 100, [URL] * 100)
executor.shutdown(wait=True)
executor._adjust_thread_count
Maybe this might help.
I'm using ThreadPoolExecutor for multithreading. you can also use ProcessPoolExecutor.
For your compute expensive operation you can use numba for making cached byte code of your function for faster exeution.

simpy traffic light simulation

I am trying to simulate a circular road with a series of traffic lights in sequence. Vehicles enter the system via a Poisson process. Once in the system, they queue at each light. It takes one time unit for them to go through each queue. They exit the system when they have traveled a number of queues equal to their trip length. The queues only operate during a green phase. Cars are represented by integers.
The problem is that I keep getting the error "pop from an empty deque" even though this part of the code is only reachable by an if statement that checks whether the deque has cars in it. I am not very familiar with simpy and so I think the problem has to do with the timeout. If I move the timeout after the pop operation, the code works. But this is not quiet what I want.
import simpy
from simpy.util import start_delayed
# import numpy.random
from collections import deque,namedtuple
from numpy import random
NUM_INT = 3
ARRIVAL_TIME_MEAN = 1.1
TRIP_LENGTH = 4
GREEN_TIME = 3.0
RED_TIME = 3.0
class Simulation(object):
def __init__(self,env):
self.env = env
self.intersections = [Intersection(env,i) for i in range(NUM_INT)]
for (i,intersection) in enumerate(self.intersections):
intersection.set_next_intersection(self.intersections[(i+1)%NUM_INT])
self.env.process(self.light())
self.env.process(self.arrivals())
def arrivals(self):
while True:
yield self.env.timeout(random.exponential(ARRIVAL_TIME_MEAN))
intersection = random.choice(self.intersections)
intersection.receive(TRIP_LENGTH)
def light(self):
while True:
for intersection in self.intersections:
intersection.start_departing()
yield self.env.timeout(GREEN_TIME)
for intersection in self.intersections:
intersection.turn_red()
yield env.timeout(RED_TIME)
class Intersection(object):
def __init__(self,env,index):
self.index = index
self.queue = deque()
self.env = env
self.start_departing()
def set_next_intersection(self,intersection):
self.next_intersection = intersection
def start_departing(self):
self.is_departing = True
self.action = env.process(self.departure())
def turn_red(self):
if self.is_departing:
self.is_departing = False
self.action.interrupt('red light')
def receive(self,car):
self.queue.append(car)
if not self.is_departing:
self.start_departing()
def departure(self):
while True:
try:
if len(self.queue)==0:
self.is_departing = False
self.env.exit('no more cars in %d'%self.index)
else:
yield self.env.timeout(1.0)
car = self.queue.popleft()
car = car - 1
if car > 0:
self.next_intersection.receive(car)
except simpy.Interrupt as i:
print('interrupted by',i.cause)
env = simpy.Environment()
sim = Simulation(env)
env.run(until=15.0)

not a fix as much as a workaround...
import simpy
from simpy.util import start_delayed
# import numpy.random
from collections import deque,namedtuple
from numpy import random
NUM_INT = 3
ARRIVAL_TIME_MEAN = 1.1
TRIP_LENGTH = 4
GREEN_TIME = 3.0
RED_TIME = 3.0
class Simulation(object):
def __init__(self,env):
self.env = env
self.intersections = [Intersection(env,i) for i in range(NUM_INT)]
for (i,intersection) in enumerate(self.intersections):
intersection.set_next_intersection(self.intersections[(i+1)%NUM_INT])
self.env.process(self.light())
self.env.process(self.arrivals())
def arrivals(self):
while True:
yield self.env.timeout(random.exponential(ARRIVAL_TIME_MEAN))
intersection = random.choice(self.intersections)
intersection.receive(TRIP_LENGTH)
def light(self):
while True:
for intersection in self.intersections:
intersection.start_departing()
yield self.env.timeout(GREEN_TIME)
for intersection in self.intersections:
intersection.turn_red()
yield env.timeout(RED_TIME)
class Intersection(object):
def __init__(self,env,index):
self.index = index
self.queue = deque()
self.env = env
self.start_departing()
def set_next_intersection(self,intersection):
self.next_intersection = intersection
def start_departing(self):
self.is_departing = True
self.action = env.process(self.departure())
def turn_red(self):
if self.is_departing:
self.is_departing = False
self.action.interrupt('red light')
def receive(self,car):
self.queue.append(car)
if not self.is_departing:
self.start_departing()
def departure(self):
while True:
try:
if len(self.queue)==0:
self.is_departing = False
self.env.exit('no more cars in %d'%self.index)
else:
yield self.env.timeout(1.0)
if len(self.queue)>0:
if len(self.queue)==1:
car=self.queue[0]
self.queue.clear()
else:
car = self.queue.popleft()
car = car - 1
if car > 0:
self.next_intersection.receive(car)
except simpy.Interrupt as i:
print('interrupted by',i.cause)
env = simpy.Environment()
sim = Simulation(env)
env.run(until=15.0)
tell me if this works for you

In departure, in the case where the queue is not empty, you immediately yield, which lets the calling process run, which can cause other events to cause the queue to be empty and raise your exception. This is a bit like cooperative multitasking, but without locks, so you have to be a bit careful with where you place yields.
Moving the yield to the end of that if fixes it for me.
def departure(self):
while True:
try:
if len(self.queue)==0:
self.is_departing = False
self.env.exit('no more cars in %d'%self.index)
else:
car = self.queue.popleft()
car = car - 1
if car > 0:
self.next_intersection.receive(car)
yield self.env.timeout(1.0)
except simpy.Interrupt as i:
print('interrupted by',i.cause)
yield self.env.timeout(1.0)

Use a class to monitor a data stream and print reports to two different windows at different time intervals in python

I've never had much opportunity to play with threading but I need it for a project i'm working on. I've significantly simplified my problem for the example I present below, but I'm pretty sure the solution to the simpler problem will get me most of the way to solving the more complex problem.
That said, here's my reduced case: I have a class whose job it is to monitor an incoming data stream that is accessed through a class method. I am calculating statistics from the stream. I would like to print a report about the incoming data to one terminal window and short intervals, and also print a summary report to another window at regular (longer) intervals.
This demo below generates data and prints reports to the same window: how would I rework this to print the separate reports to two different windows?
from __future__ import division
from random import gauss
import time
class MyStreamMonitor(object):
def __init__(self):
self.sum = 0
self.count = 0
#property
def mu(self):
return outv = self.sum/self.count
def generate_values(self):
while True:
yield gauss(0,1)
def monitor(self, report_interval=1):
start1 = time.time()
start2 = time.time()
for x in self.generate_values():
self.sum += x
self.count += 1
# print this to terminal 1
if time.time() - start1 > report_interval:
start1 = time.time()
print self.count, x
# print this to terminal 2
if time.time() - start2 > 5*report_interval:
start2 = time.time()
print self.mu
if __name__ == '__main__':
stream = MyStreamMonitor()
stream.monitor()

One approach is to write to a text file, and then in second terminal, just tail the file:
def monitor(self, report_interval=1):
second = open('report.txt', 'wt')
start1 = time.time()
start2 = time.time()
for x in self.generate_values():
self.sum += x
self.count += 1
# print this to terminal 1
if time.time() - start1 > report_interval:
start1 = time.time()
print self.count, x
second.write('%d, %s\n' % (self.count, x))
# print this to terminal 2
if time.time() - start2 > 5*report_interval:
start2 = time.time()
print self.mu
second.write('%s\n', self.mu)
second.flush()
Then on the second terminal:
$ tail -f report.txt

I ended up taking #reptilicus' advice and built this as a client/server application with redis. Here's a minimum working example:
server.py
from __future__ import division
from random import gauss
import time
import redis
class MyStreamMonitor(object):
def __init__(self):
self.sum = 0
self.count = 0
self.r = redis.StrictRedis()
#property
def mu(self):
if self.count >1:
outv = self.sum/self.count
else:
outv = 0
return outv
def generate_values(self):
while True:
yield gauss(0,1)
def monitor(self):
for x in self.generate_values():
self.sum += x
self.count += 1
# This is the magic here
self.r.publish('count', self.count)
self.r.publish('mu', self.mu)
if __name__ == '__main__':
stream = MyStreamMonitor()
stream.monitor()
listener.py
import redis
import time
import sys
r = redis.StrictRedis()
channel = sys.argv[1]
interval = float(sys.argv[2])
while True:
# Resubscribe at each iteration to ensure we are only receiving
# the newest message
pubsub = r.pubsub()
pubsub.subscribe(channel)
_ = pubsub.listen().next() # the first message is always just a "1"
message = pubsub.listen().next()
print message['data']
time.sleep(interval )
The server publishes data to the "count" and "mu" channels. So to run this, first we need to open up a terminal and kick off the server:
$ python server.py
Then we can open up a separate terminal for each channel we want to listen on, passing in the channel we want to listen to and the sleep interval as arguments.
First terminal:
$ python listener.py mu 1
Second terminal:
$ python listener.py count 2
For the record: installing redis was extremely painless and didn't really require any configuration at all. Depending on your needs installation/configuration may be more complex, but at least for this toy example I didn't need to do anything fancy.

Why doesn't SIGVTALRM trigger inside time.sleep()?

I'm trying to use SIGVTALRM to snapshot profile my Python code, but it doesn't seem to be firing inside blocking operations like time.sleep() and socket operations.
Why is that? And is there any way to address that, so I can collect samples while I'm inside blocking operations?
I've also tried using ITIMER_PROF/SIGPROF and ITIMER_REAL/SIGALRM and both seem to produce similar results.
The code I'm testing with follows, and the output is something like:
$ python profiler-test.py
<module>(__main__:1);test_sampling_profiler(__main__:53): 1
<module>(__main__:1);test_sampling_profiler(__main__:53);busyloop(__main__:48): 1509
Note that the timesleep function isn't shown at all.
Test code:
import time
import signal
import collections
class SamplingProfiler(object):
def __init__(self, interval=0.001, logger=None):
self.interval = interval
self.running = False
self.counter = collections.Counter()
def _sample(self, signum, frame):
if not self.running:
return
stack = []
while frame is not None:
formatted_frame = "%s(%s:%s)" %(
frame.f_code.co_name,
frame.f_globals.get('__name__'),
frame.f_code.co_firstlineno,
)
stack.append(formatted_frame)
frame = frame.f_back
formatted_stack = ';'.join(reversed(stack))
self.counter[formatted_stack] += 1
signal.setitimer(signal.ITIMER_VIRTUAL, self.interval, 0)
def start(self):
if self.running:
return
signal.signal(signal.SIGVTALRM, self._sample)
signal.setitimer(signal.ITIMER_VIRTUAL, self.interval, 0)
self.running = True
def stop(self):
if not self.running:
return
self.running = False
signal.signal(signal.SIGVTALRM, signal.SIG_IGN)
def flush(self):
res = self.counter
self.counter = collections.Counter()
return res
def busyloop():
start = time.time()
while time.time() - start < 5:
pass
def timesleep():
time.sleep(5)
def test_sampling_profiler():
p = SamplingProfiler()
p.start()
busyloop()
timesleep()
p.stop()
print "\n".join("%s: %s" %x for x in sorted(p.flush().items()))
if __name__ == "__main__":
test_sampling_profiler()

Not sure about why time.sleep works that way (could it be using SIGALRM for itself to know when to resume?) but Popen.wait does not block signals so worst case you can call out to OS sleep.

Another approach is to use a separate thread to trigger the sampling:
import sys
import threading
import time
import collections
class SamplingProfiler(object):
def __init__(self, interval=0.001):
self.interval = interval
self.running = False
self.counter = collections.Counter()
self.thread = threading.Thread(target=self._sample)
def _sample(self):
while self.running:
next_wakeup_time = time.time() + self.interval
for thread_id, frame in sys._current_frames().items():
if thread_id == self.thread.ident:
continue
stack = []
while frame is not None:
formatted_frame = "%s(%s:%s)" % (
frame.f_code.co_name,
frame.f_globals.get('__name__'),
frame.f_code.co_firstlineno,
)
stack.append(formatted_frame)
frame = frame.f_back
formatted_stack = ';'.join(reversed(stack))
self.counter[formatted_stack] += 1
sleep_time = next_wakeup_time - time.time()
if sleep_time > 0:
time.sleep(sleep_time)
def start(self):
if self.running:
return
self.running = True
self.thread.start()
def stop(self):
if not self.running:
return
self.running = False
def flush(self):
res = self.counter
self.counter = collections.Counter()
return res
def busyloop():
start = time.time()
while time.time() - start < 5:
pass
def timesleep():
time.sleep(5)
def test_sampling_profiler():
p = SamplingProfiler()
p.start()
busyloop()
timesleep()
p.stop()
print "\n".join("%s: %s" %x for x in sorted(p.flush().items()))
if __name__ == "__main__":
test_sampling_profiler()
When doing it this way the result is:
$ python profiler-test.py
<module>(__main__:1);test_sampling_profiler(__main__:62);busyloop(__main__:54): 2875
<module>(__main__:1);test_sampling_profiler(__main__:62);start(__main__:37);start(threading:717);wait(threading:597);wait(threading:309): 1
<module>(__main__:1);test_sampling_profiler(__main__:62);timesleep(__main__:59): 4280
Still not totally fair, but better than no samples at all during sleep.

The absence of SIGVTALRM during a sleep() doesn't surprise me, since ITIMER_VIRTUAL "runs only when the process is executing."
(As an aside, CPython on non-Windows platforms implements time.sleep() in terms of select().)
With a plain SIGALRM, however, I expect a signal interruption and indeed I observe one:
<module>(__main__:1);test_sampling_profiler(__main__:62);busyloop(__main__:54): 4914
<module>(__main__:1);test_sampling_profiler(__main__:62);timesleep(__main__:59): 1
I changed the code somewhat, but you get the idea:
class SamplingProfiler(object):
TimerSigs = {
signal.ITIMER_PROF : signal.SIGPROF,
signal.ITIMER_REAL : signal.SIGALRM,
signal.ITIMER_VIRTUAL : signal.SIGVTALRM,
}
def __init__(self, interval=0.001, timer = signal.ITIMER_REAL): # CHANGE
self.interval = interval
self.running = False
self.counter = collections.Counter()
self.timer = timer # CHANGE
self.signal = self.TimerSigs[timer] # CHANGE
....