I have the following decorator running fine using a parameter
from functools import wraps
def sumproduct(cnt):
def dec(f):
#wraps(f)
def wrap(*args):
print('inside wrapper')
_sum = 0
for i in range(cnt):
_sum = _sum + sum([i * a for a in args])
f(_sum)
return wrap
return dec
cnt = 3
#sumproduct(cnt)
def myfunc(num):
print(num)
if __name__ == "__main__":
myfunc(10)
The output is 30 which is 0*10 + 1*10+ 2*10
However, I would like to import this module somewhere else, for example into a test module. I would like to do something like the following so that cnt is not defined in global scope:
from functools import wraps
def sumproduct(cnt):
def dec(f):
#wraps(f)
def wrap(*args):
print('inside wrapper')
_sum = 0
for i in range(cnt):
_sum = _sum + sum([i * a for a in args])
f(_sum)
return wrap
return dec
#sumproduct(cnt)
def myfunc(num):
print(num)
if __name__ == "__main__":
cnt = 3
myfunc(10)
How can I define cnt so that
cnt is always 3 when code is executed?
and cnt is not imported when module is imported?
Note: This is just a sample representation of the code. Suppose that cnt is database connection which connects to production database. I would like to use a different database connection for tests, hence I don't want to import production database connection into test module.
You could use #sumproduct(lambda: cnt). That way the execution is delayed.
For example:
from functools import wraps
def sumproduct(cnt):
def dec(f):
#wraps(f)
def wrap(*args):
print('inside wrapper')
_sum = 0
for i in range(cnt()): # <---- Note the ()
_sum = _sum + sum([i * a for a in args])
f(_sum)
return wrap
return dec
#sumproduct(lambda: cnt) # <--- put lambda: here
def myfunc(num):
print(num)
if __name__ == "__main__":
cnt = 3
myfunc(10)
Prints:
inside wrapper
30
Related
I'm trying to use the multiprocessing library to parallelize some expensive calculations without blocking some others, much lighter. The both need to interact through some variables, although the may run with different paces.
To show this, I have created the following example, that works fine:
import multiprocessing
import time
import numpy as np
class SumClass:
def __init__(self):
self.result = 0.0
self.p = None
self.return_value = None
def expensive_function(self, new_number, return_value):
# Execute expensive calculation
#######
time.sleep(np.random.random_integers(5, 10, 1))
return_value.value = self.result + new_number
#######
def execute_function(self, new_number):
print(' New number received: %f' % new_number)
self.return_value = multiprocessing.Value("f", 0.0, lock=True)
self.p = multiprocessing.Process(target=self.expensive_function, args=(new_number, self.return_value))
self.p.start()
def is_executing(self):
if self.p is not None:
if not self.p.is_alive():
self.result = self.return_value.value
self.p = None
return False
else:
return True
else:
return False
if __name__ == '__main__':
sum_obj = SumClass()
current_value = 0
while True:
if not sum_obj.is_executing():
# Randomly determine whether the function must be executed or not
if np.random.rand() < 0.25:
print('Current sum value: %f' % sum_obj.result)
new_number = np.random.rand(1)[0]
sum_obj.execute_function(new_number)
# Execute other (light) stuff
#######
print('Executing other stuff')
current_value += sum_obj.result * 0.1
print('Current value: %f' % current_value)
time.sleep(1)
#######
Basically, in the main loop some light function is executed, and depending on a random condition, some heavy work is sent to another process if it has already finished the previous one, carried out by an object which needs to store some data between executions. Although expensive_function needs some time, the light function keeps on executing without being blocked.
Although the above code gets the job done, I'm wondering: is it the best/most appropriate method to do this?
Besides, let us suppose the class SumClass has an instance of another object, which also needs to store data. For example:
import multiprocessing
import time
import numpy as np
class Operator:
def __init__(self):
self.last_value = 1.0
def operate(self, value):
print(' Operation, last value: %f' % self.last_value)
self.last_value *= value
return self.last_value
class SumClass:
def __init__(self):
self.operator_obj = Operator()
self.result = 0.0
self.p = None
self.return_value = None
def expensive_function(self, new_number, return_value):
# Execute expensive calculation
#######
time.sleep(np.random.random_integers(5, 10, 1))
# Apply operation
number = self.operator_obj.operate(new_number)
# Apply other operation
return_value.value = self.result + number
#######
def execute_function(self, new_number):
print(' New number received: %f' % new_number)
self.return_value = multiprocessing.Value("f", 0.0, lock=True)
self.p = multiprocessing.Process(target=self.expensive_function, args=(new_number, self.return_value))
self.p.start()
def is_executing(self):
if self.p is not None:
if not self.p.is_alive():
self.result = self.return_value.value
self.p = None
return False
else:
return True
else:
return False
if __name__ == '__main__':
sum_obj = SumClass()
current_value = 0
while True:
if not sum_obj.is_executing():
# Randomly determine whether the function must be executed or not
if np.random.rand() < 0.25:
print('Current sum value: %f' % sum_obj.result)
new_number = np.random.rand(1)[0]
sum_obj.execute_function(new_number)
# Execute other (light) stuff
#######
print('Executing other stuff')
current_value += sum_obj.result * 0.1
print('Current value: %f' % current_value)
time.sleep(1)
#######
Now, inside the expensive_function, a function member of the object Operator is used, which needs to store the number passed.
As expected, the member variable last_value does not change, i.e. it does not keep any value.
Is there any way of doing this properly?
I can imagine I could arrange everything so that I only need to use one class level, and it would work well. However, this is a toy example, in reality there are different levels of complex objects and it would be hard.
Thank you very much in advance!
from concurrent.futures import ThreadPoolExecutor
from numba import jit
import requests
import timeit
def timer(number, repeat):
def wrapper(func):
runs = timeit.repeat(func, number=number, repeat=repeat)
print(sum(runs) / len(runs))
return wrapper
URL = "https://httpbin.org/uuid"
#jit(nopython=True, nogil=True,cache=True)
def fetch(session, url):
with session.get(url) as response:
print(response.json()['uuid'])
#timer(1, 1)
def runner():
with ThreadPoolExecutor(max_workers=25) as executor:
with requests.Session() as session:
executor.map(fetch, [session] * 100, [URL] * 100)
executor.shutdown(wait=True)
executor._adjust_thread_count
Maybe this might help.
I'm using ThreadPoolExecutor for multithreading. you can also use ProcessPoolExecutor.
For your compute expensive operation you can use numba for making cached byte code of your function for faster exeution.
In order to control the code I am working on, I have tried to create a single tracking across many tasks that occur in different threads.
I know at the beginning of the run the amount of tasks (and workers).
For demonstration (does not work, toy example):
from multiprocessing import Pool
from tqdm import tqdm
def work(i, t):
for _ in range(10**6):
t.update()
return i
def wrapped_work(params):
work(*params)
def main(n=1):
# another loop:
with Pool(processes=8) as p:
with tqdm(total=n * 10**6) as t:
return sum(p.map(work, ((i, t) for i in range(1, n+1))))
if __name__ == "__main__":
main(5)
I tried to implies this topic with pool, but without success.
I would greatly appreciate your help.
based on this post:
from multiprocessing import Pool, Process, Value
from ctypes import c_bool, c_long
from tqdm.auto import tqdm
class TqdmMultiprocessing:
max_processes = 64
def __init__(self, static_func, processes=64):
self.counter = Value(c_long, lock=False)
self.pool = Pool(
processes=min(processes, self.max_processes),
initializer=self.worker_init,
initargs=(static_func, self.counter)
)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.pool.close()
def tqdm(self, static_func, iterable, **kwargs):
done_value = Value(c_bool)
proc = Process(target=self.listener, args=(self.counter, done_value, kwargs,))
proc.start()
result = self.pool.map(static_func, iterable)
done_value.value = True
proc.join()
self.counter.value = 0
return result
#staticmethod
def listener(counter: Value, is_done: Value, kwargs):
with tqdm(**kwargs) as tqdm_bar:
old_counter = 0
while not is_done.value:
new_counter = counter.value
tqdm_bar.update(new_counter - old_counter)
old_counter = new_counter
tqdm_bar.update(tqdm_bar.total - old_counter)
#staticmethod
def worker_init(static_func, counter: Value):
static_func.counter = counter
def work(i):
for _ in range(10**6):
work.counter.value += 1
return i
def main(n=1):
with TqdmMultiprocessing(work, processes=3) as p:
p.tqdm(work, range(n), total=n * 10 ** 6)
p.tqdm(work, range(n), total=n * 10 ** 6)
if __name__ == "__main__":
main(5)
I'm trying to answer a competitive programming question on kattis that can be found here My algorithm is correct, however there is one of the test cases that has a lot of inputs and my code times out. Is there a more optimized way to do IO in python?
from sys import stdin, stdout
import atexit, io, sys
buffer = io.BytesIO()
sys.stdout = buffer
#atexit.register
def write():
sys.__stdout__.write(buffer.getvalue())
def main():
teque = []
for i in range(int(stdin.readline())):
l = stdin.readline().split()
if l[0] == 'push_back':
teque.append(int(l[1]))
if l[0] == 'push_front':
teque.insert(0, int(l[1]))
if l[0] == 'push_middle':
if len(teque)%2==0:
mid = len(teque)/2
else:
mid = (len(teque)+1)/2
teque.insert(int(mid), int(l[1]))
if l[0] == 'get':
stdout.write(str(teque[int(l[1])])+'\n')
if __name__ == "__main__":
main()
So as I learned in the comments, I wasn't actually doing the program in O(1) time for insert, I fixed invoking 2 queues. This solution still isn't fast enough for python 3 run time, however it passes the python 2 run time.
from sys import stdin, stdout
from collections import deque
class Teque:
def __init__(self):
self._teque1 = deque()
self._teque2 = deque()
def push_back(self, x):
self._teque2.append(x)
if len(self._teque2) > len(self._teque1):
self._teque1.append((self._teque2.popleft()))
def push_front(self, x):
self._teque1.appendleft(x)
if len(self._teque1) > len(self._teque2):
self._teque2.appendleft((self._teque1.pop()))
def push_middle(self, x):
if len(self._teque2) > len(self._teque1):
self._teque1.append(self._teque2.popleft())
self._teque2.appendleft(x)
def get(self, i):
if i >= len(self._teque1):
return self._teque2[i-len(self._teque1)]
return self._teque1[i]
def main():
teque = Teque()
for i in range(int(stdin.readline())):
l = stdin.readline().split()
if l[0] == 'push_back':
teque.push_back(int(l[1]))
elif l[0] == 'push_front':
teque.push_front(int(l[1]))
elif l[0] == 'push_middle':
teque.push_middle(int(l[1]))
else:
stdout.write(str(teque.get(int(l[1])))+'\n')
if __name__ == "__main__":
main()
I want to test async_who function by pytest.
How do I test callback is called and the return value is 'Bob'
import threading
def async_who(callback):
t = threading.Thread(target=_who, args=(callback,))
t.start()
def _who(callback):
return callback('Bob')
def callback(name):
print(name)
return name
async_who(callback)
Because the async_who didn't return value. I can't do this,
def test_async_who():
res = async_who(callback)
assert res == 'Bob'
ThreadPool from multiprocessing module or ThreadPoolExecutor (for python version >= 3.2)
are ways to get the return value of a thread.
With concurrent.futures.ThreadPoolExecutor
from concurrent.futures import ThreadPoolExecutor
def async_who(callback):
executor = ThreadPoolExecutor(max_workers=2)
res = executor.submit(_who, callback)
return res.result()
def _who(callback):
return callback('Bob')
def callback(name):
print(name)
return name
def test_async_who():
res = async_who(callback)
assert res == 'Bob'
With multiprocessing.pool.ThreadPool
from multiprocessing.pool import ThreadPool
pool = ThreadPool(processes=2)
def async_who(callback):
res = pool.apply_async(_who, args=(callback,))
return res.get()
def _who(callback):
return callback('Bob')
def callback(name):
print(name)
return name
def test_async_who():
res = async_who(callback)
assert res == 'Bob'
I have a python file called myGlobal.py, in this file I declared count as a global variable.
import datetime
import time
import os
count = 0
failCount = 0
def counterstart:
global count
count +=1
I am calling this counter in another file. So every time I call a function, I want the counter to increase by 1. That file is scripts.py
import os
from selenium import webdriver
import datetime
import time
from myGlobal import *
def main():
fnfirst()
fnsecond()
fnthird()
def fnfirst():
global count
print count
def fnsecond():
global count
print count
def fnthird():
global count
print count
main()
But whenever I run script.py, the count is shown as 0 only.
Why is this happening?
import myGlobal
print myGlobal.count # 0
myGlobal.count += 1
def foo():
myGlobal.count += 1
foo()
print myGlobal.count # 2
So here is a solution based on the Python's decorators. It will count the number of call on specific functions :
# File script.py
import myGlobal
def count_call(func):
def _wrapper(*args, **kwargs):
myGlobal.count += 1
func(*args, **kwargs)
return _wrapper
#count_call
def fnfirst():
pass # do something
#count_call
def fnsecond():
pass # do something
def main():
print myGlobal.count # 0
fnfirst()
print myGlobal.count # 1
fnsecond()
print myGlobal.count # 2
fnfirst()
print myGlobal.count # 3
if __name__ == "__main__":
main()