Given a function with a parameter a and two other parameters (pickle_from, pickle_to), I'd like to:
Load and return the pickled object located at pickle_from, if pickle_from is not None. If it is None, compute some function of a and return it.
Dump the result of the above to pickle_to if pickle_to is not None.
With a single function this is straightforward. If pickle_from isn't null, the function just loads the pickled result and returns it. Otherwise, it performs some time-intensive calculation with a, dumps that to pickle_to, and returns the calculation result.
try:
import cPickle as pickle
except:
import pickle
def somefunc(a, pickle_from=None, pickle_to=None):
if pickle_from:
with open(pickle_from + '.pickle', 'rb') as f
res = pickle.load(f)
else:
# Re-calcualte some time-intensive func call
res = a ** 2
if pickle_to:
# Update pickled data with newly calculated `res`
with open(pickle_to + '.pickle', 'wb') as f:
pickle.dump(res, f)
return res
My question is regarding how to build a decorator so that this process can form a shell around multiple functions similar to somefunc, cutting down on source code in the process.
I'd like to be able to write something like:
#pickle_option
def somefunc(a, pickle_from=None, pickle_to=None)
# or do params need to be in the decorator call?
# remember, "the files are in the computer"
res = a ** 2
return res
Is this possible? Something about decorators makes my head explode, so I will politely decline to post here "what I have tried."
This decorator requires a little bit of introspection. Specifically, I've made use of inspect.Signature to extract the pickle_from and pickle_to parameters.
Other than that, it's a very straightforward decorator: It keeps a reference to the decorated function, and calls it if necessary.
import inspect
from functools import wraps
def pickle_option(func):
sig = inspect.signature(func)
#wraps(func)
def wrapper(*args, **kwargs):
# get the value of the pickle_from and pickle_to parameters
# introspection magic, don't worry about it or read the docs
bound_args = sig.bind(*args, **kwargs)
pickle_from = bound_args.arguments.get('pickle_from', \
sig.parameters['pickle_from'].default)
pickle_to = bound_args.arguments.get('pickle_to', \
sig.parameters['pickle_to'].default)
if pickle_from:
with open(pickle_from + '.pickle', 'rb') as f:
result = pickle.load(f)
else:
result = func(*args, **kwargs)
if pickle_to:
with open(pickle_to + '.pickle', 'wb') as f:
pickle.dump(result, f)
return result
return wrapper
Given your use case, I think it would be clearer to use just a generic wrapper:
def pickle_call(fun, *args, pickle_from=None, pickle_to=None, **kwargs):
if pickle_from:
with open(pickle_from + '.pickle', 'rb') as f
res = pickle.load(f)
else:
res = fun(*args, **kwargs)
if pickle_to:
# Update pickled data with newly calculated `res`
with open(pickle_to + '.pickle', 'wb') as f:
pickle.dump(res, f)
return res
Then you'd use it like:
res = pickle_call(somefunc, a, pickle_from="from", pickle_to="to")
This avoids having to add a decorator everywhere you want to use this feature, and in fact works with any callable (not just functions), from your code or else.
I just found that a bunch of unit tests are failing, due a developer hasn't mocked out the dependency to a redis client within the test. I'm trying to give a hand in this matter but have difficulties myself.
The method writes to a redis client:
redis_client = get_redis_client()
redis_client.set('temp-facility-data', cPickle.dumps(df))
Later in the assert the result is retrieved:
res = cPickle.loads(get_redis_client().get('temp-facility-data'))
expected = pd.Series([set([1, 2, 3, 4, 5])], index=[1])
assert_series_equal(res.variation_pks, expected)
I managed to patch the redis client's get() and set() successfully.
#mock.patch('redis.StrictRedis.get')
#mock.patch('redis.StrictRedis.set')
def test_identical(self, mock_redis_set, mock_redis_get):
mock_redis_get.return_value = ???
f2 = deepcopy(self.f)
f3 = deepcopy(self.f)
f2.pk = 2
f3.pk = 3
self.one_row(f2, f3)
but I don't know how to set the return_value of get() to what the set() would set in the code, so that the test would pass.
Right now this line fails the test:
res = cPickle.loads(get_redis_client().get('temp-facility-data'))
TypeError: must be string, not MagicMock
Any advice please?
Think you can use side effect to set and get value in a local dict
data = {}
def set(key, val):
data[key] = val
def get(key):
return data[key]
mock_redis_set.side_effect = set
mock_redis_get.side_effect = get
not tested this but I think it should do what you want
If you want something more complete, you can try fakeredis
#patch("redis.Redis", return_value=fakeredis.FakeStrictRedis())
def test_something():
....
I think you can do something like this.
redis_cache = {
"key1": (b'\x80\x04\x95\x08\x00\x00\x00\x00\x00\x00\x00\x8c\x04test\x94.', "test"),
"key2": (None, None),
}
def get(redis_key):
if redis_key in redis_cache:
return redis_cache[redis_key][0]
else:
return None
mock = MagicMock()
mock.get = Mock(side_effect=get)
with patch('redis.StrictRedis', return_value=mock) as p:
for key in redis_cache:
result = self.MyClass.my_function(key)
self.assertEqual(result, redis_cache[key][1])
I am a newb to Python and I understand testing, however, I cannot wrap my head around working with Mocked Objects and side_effects.
Here is my method:
#retry(every=RETRY_EVERY, until=RETRY_UNTIL)
#unique()
#sessionized(0)
def record_click(session, queue, mailing_id, member_id, link_id, timestamp, user_agent):
message = session.query(Message).get((mailing_id, member_id))
mailing = session.query(Mailing).get(mailing_id)
# More code here
Here is my test:
#mock.patch("audience.jobs.EventProvider")
#mock.patch("audience.jobs.enqueue_webhook")
#mock.patch("logging.exception")
#mock.patch("audience.jobs.audience_queues")
#mock.patch("audience.jobs.Session")
#mock.patch("audience.jobs.DatabaseConnector")
def test_track_click_publishes_event_to_sns(self, DatabaseConnector, Session, audience_queues, logger, enqueue_webhook, EventProvider):
message_mock = mock.Mock(account_id=77)
message_mock.record_open.return_value = True
mailing_mock = mock.Mock(mailing_id=123)
mailing_mock.recipient_groups.return_value = [111]
session_query = Session.return_value.query.return_value
session_query.side_effect = lambda arg: message_mock if isinstance(arg, tuple) else mailing_mock
result = jobs.record_click(
888,
9999,
2048,
datetime.datetime(1999, 12, 31, 23, 59, 59, 999999).isoformat(),
"Mozilla/5.0")
self.assertIsNone(result)
self.assertListEqual(EventProvider.mock_calls, [
mock.call(),
mock.call().publish_link_clicked(
headers={'User-Agent': 'Mozilla/5.0'},
mailing_id=888,
account_id=77,
contact_id=9999,
link_id=2048,
group_ids=[111]
)
])
self.assertListEqual(logger.mock_calls, [])
There error I keep receiving is:
Instead of
call().publish_link_clicked(group_ids=[111], account_id=77, **etc)
This is what is called in the UnitTest
call().publish_link_clicked(group_ids=<MagicMock name='Session().query().get().recipient_groups' id='4557662736'>, account_id=<MagicMock name='Session().query().get().account_id' id='4557652048'>, **etc)
What am I doing wrong?
Don't call Session() or query(); use the Mock.return_value attribute instead to traverse the call graph:
Session.return_value.query.return_value.side_effect = lambda arg: message_mock if isinstance(arg, tuple) else mailing_mock
I usually use intermediary names to hold a return value:
session_query = Session.return_value.query.return_value
session_query.side_effect = lambda arg: message_mock if isinstance(arg, tuple) else mailing_mock
You also need to patch the right Session class; this depends entirely how your code produces the session argument to record_click. See Where to Patch for more details. If the #sessionized decorator produces this argument, and it doesn't live in the audience.jobs module, you are not patching the right location.
Is there a way to memoize the output of a function to disk?
I have a function
def getHtmlOfUrl(url):
... # expensive computation
and would like to do something like:
def getHtmlMemoized(url) = memoizeToFile(getHtmlOfUrl, "file.dat")
and then call getHtmlMemoized(url), so as to do the expensive computation only once for each url.
Python offers a very elegant way to do this - decorators. Basically, a decorator is a function that wraps another function to provide additional functionality without changing the function source code. Your decorator can be written like this:
import json
def persist_to_file(file_name):
def decorator(original_func):
try:
cache = json.load(open(file_name, 'r'))
except (IOError, ValueError):
cache = {}
def new_func(param):
if param not in cache:
cache[param] = original_func(param)
json.dump(cache, open(file_name, 'w'))
return cache[param]
return new_func
return decorator
Once you've got that, 'decorate' the function using #-syntax and you're ready.
#persist_to_file('cache.dat')
def html_of_url(url):
your function code...
Note that this decorator is intentionally simplified and may not work for every situation, for example, when the source function accepts or returns data that cannot be json-serialized.
More on decorators: How to make a chain of function decorators?
And here's how to make the decorator save the cache just once, at exit time:
import json, atexit
def persist_to_file(file_name):
try:
cache = json.load(open(file_name, 'r'))
except (IOError, ValueError):
cache = {}
atexit.register(lambda: json.dump(cache, open(file_name, 'w')))
def decorator(func):
def new_func(param):
if param not in cache:
cache[param] = func(param)
return cache[param]
return new_func
return decorator
Check out joblib.Memory. It's a library for doing exactly that.
from joblib import Memory
memory = Memory("cachedir")
#memory.cache
def f(x):
print('Running f(%s)' % x)
return x
A cleaner solution powered by Python's Shelve module. The advantage is the cache gets updated in real time via well-known dict syntax, also it's exception proof(no need to handle annoying KeyError).
import shelve
def shelve_it(file_name):
d = shelve.open(file_name)
def decorator(func):
def new_func(param):
if param not in d:
d[param] = func(param)
return d[param]
return new_func
return decorator
#shelve_it('cache.shelve')
def expensive_funcion(param):
pass
This will facilitate the function to be computed just once. Next subsequent calls will return the stored result.
There is also diskcache.
from diskcache import Cache
cache = Cache("cachedir")
#cache.memoize()
def f(x, y):
print('Running f({}, {})'.format(x, y))
return x, y
The Artemis library has a module for this. (you'll need to pip install artemis-ml)
You decorate your function:
from artemis.fileman.disk_memoize import memoize_to_disk
#memoize_to_disk
def fcn(a, b, c = None):
results = ...
return results
Internally, it makes a hash out of input arguments and saves memo-files by this hash.
Check out Cachier. It supports additional cache configuration parameters like TTL etc.
Simple example:
from cachier import cachier
import datetime
#cachier(stale_after=datetime.timedelta(days=3))
def foo(arg1, arg2):
"""foo now has a persistent cache, trigerring recalculation for values stored more than 3 days."""
return {'arg1': arg1, 'arg2': arg2}
Something like this should do:
import json
class Memoize(object):
def __init__(self, func):
self.func = func
self.memo = {}
def load_memo(filename):
with open(filename) as f:
self.memo.update(json.load(f))
def save_memo(filename):
with open(filename, 'w') as f:
json.dump(self.memo, f)
def __call__(self, *args):
if not args in self.memo:
self.memo[args] = self.func(*args)
return self.memo[args]
Basic usage:
your_mem_func = Memoize(your_func)
your_mem_func.load_memo('yourdata.json')
# do your stuff with your_mem_func
If you want to write your "cache" to a file after using it -- to be loaded again in the future:
your_mem_func.save_memo('yournewdata.json')
Assuming that you data is json serializable, this code should work
import os, json
def json_file(fname):
def decorator(function):
def wrapper(*args, **kwargs):
if os.path.isfile(fname):
with open(fname, 'r') as f:
ret = json.load(f)
else:
with open(fname, 'w') as f:
ret = function(*args, **kwargs)
json.dump(ret, f)
return ret
return wrapper
return decorator
decorate getHtmlOfUrl and then simply call it, if it had been run previously, you will get your cached data.
Checked with python 2.x and python 3.x
You can use the cache_to_disk package:
from cache_to_disk import cache_to_disk
#cache_to_disk(3)
def my_func(a, b, c, d=None):
results = ...
return results
This will cache the results for 3 days, specific to the arguments a, b, c and d. The results are stored in a pickle file on your machine, and unpickled and returned next time the function is called. After 3 days, the pickle file is deleted until the function is re-run. The function will be re-run whenever the function is called with new arguments. More info here: https://github.com/sarenehan/cache_to_disk
Most answers are in a decorator fashion. But maybe I don't want to cache the result every time when calling the function.
I made one solution using context manager, so the function can be called as
with DiskCacher('cache_id', myfunc) as myfunc2:
res=myfunc2(...)
when you need the caching functionality.
The 'cache_id' string is used to distinguish data files, which are named [calling_script]_[cache_id].dat. So if you are doing this in a loop, will need to incorporate the looping variable into this cache_id, otherwise data will be overwritten.
Alternatively:
myfunc2=DiskCacher('cache_id')(myfunc)
res=myfunc2(...)
Alternatively (this is probably not quite useful as the same id is used all time time):
#DiskCacher('cache_id')
def myfunc(*args):
...
The complete code with examples (I'm using pickle to save/load, but can be changed to whatever save/read methods. NOTE that this is also assuming the function in question returns only 1 return value):
from __future__ import print_function
import sys, os
import functools
def formFilename(folder, varid):
'''Compose abspath for cache file
Args:
folder (str): cache folder path.
varid (str): variable id to form file name and used as variable id.
Returns:
abpath (str): abspath for cache file, which is using the <folder>
as folder. The file name is the format:
[script_file]_[varid].dat
'''
script_file=os.path.splitext(sys.argv[0])[0]
name='[%s]_[%s].nc' %(script_file, varid)
abpath=os.path.join(folder, name)
return abpath
def readCache(folder, varid, verbose=True):
'''Read cached data
Args:
folder (str): cache folder path.
varid (str): variable id.
Keyword Args:
verbose (bool): whether to print some text info.
Returns:
results (tuple): a tuple containing data read in from cached file(s).
'''
import pickle
abpath_in=formFilename(folder, varid)
if os.path.exists(abpath_in):
if verbose:
print('\n# <readCache>: Read in variable', varid,
'from disk cache:\n', abpath_in)
with open(abpath_in, 'rb') as fin:
results=pickle.load(fin)
return results
def writeCache(results, folder, varid, verbose=True):
'''Write data to disk cache
Args:
results (tuple): a tuple containing data read to cache.
folder (str): cache folder path.
varid (str): variable id.
Keyword Args:
verbose (bool): whether to print some text info.
'''
import pickle
abpath_out=formFilename(folder, varid)
if verbose:
print('\n# <writeCache>: Saving output to:\n',abpath_out)
with open(abpath_out, 'wb') as fout:
pickle.dump(results, fout)
return
class DiskCacher(object):
def __init__(self, varid, func=None, folder=None, overwrite=False,
verbose=True):
'''Disk cache context manager
Args:
varid (str): string id used to save cache.
function <func> is assumed to return only 1 return value.
Keyword Args:
func (callable): function object whose return values are to be
cached.
folder (str or None): cache folder path. If None, use a default.
overwrite (bool): whether to force a new computation or not.
verbose (bool): whether to print some text info.
'''
if folder is None:
self.folder='/tmp/cache/'
else:
self.folder=folder
self.func=func
self.varid=varid
self.overwrite=overwrite
self.verbose=verbose
def __enter__(self):
if self.func is None:
raise Exception("Need to provide a callable function to __init__() when used as context manager.")
return _Cache2Disk(self.func, self.varid, self.folder,
self.overwrite, self.verbose)
def __exit__(self, type, value, traceback):
return
def __call__(self, func=None):
_func=func or self.func
return _Cache2Disk(_func, self.varid, self.folder, self.overwrite,
self.verbose)
def _Cache2Disk(func, varid, folder, overwrite, verbose):
'''Inner decorator function
Args:
func (callable): function object whose return values are to be
cached.
varid (str): variable id.
folder (str): cache folder path.
overwrite (bool): whether to force a new computation or not.
verbose (bool): whether to print some text info.
Returns:
decorated function: if cache exists, the function is <readCache>
which will read cached data from disk. If needs to recompute,
the function is wrapped that the return values are saved to disk
before returning.
'''
def decorator_func(func):
abpath_in=formFilename(folder, varid)
#functools.wraps(func)
def wrapper(*args, **kwargs):
if os.path.exists(abpath_in) and not overwrite:
results=readCache(folder, varid, verbose)
else:
results=func(*args, **kwargs)
if not os.path.exists(folder):
os.makedirs(folder)
writeCache(results, folder, varid, verbose)
return results
return wrapper
return decorator_func(func)
if __name__=='__main__':
data=range(10) # dummy data
#--------------Use as context manager--------------
def func1(data, n):
'''dummy function'''
results=[i*n for i in data]
return results
print('\n### Context manager, 1st time call')
with DiskCacher('context_mananger', func1) as func1b:
res=func1b(data, 10)
print('res =', res)
print('\n### Context manager, 2nd time call')
with DiskCacher('context_mananger', func1) as func1b:
res=func1b(data, 10)
print('res =', res)
print('\n### Context manager, 3rd time call with overwrite=True')
with DiskCacher('context_mananger', func1, overwrite=True) as func1b:
res=func1b(data, 10)
print('res =', res)
#--------------Return a new function--------------
def func2(data, n):
results=[i*n for i in data]
return results
print('\n### Wrap a new function, 1st time call')
func2b=DiskCacher('new_func')(func2)
res=func2b(data, 10)
print('res =', res)
print('\n### Wrap a new function, 2nd time call')
res=func2b(data, 10)
print('res =', res)
#----Decorate a function using the syntax sugar----
#DiskCacher('pie_dec')
def func3(data, n):
results=[i*n for i in data]
return results
print('\n### pie decorator, 1st time call')
res=func3(data, 10)
print('res =', res)
print('\n### pie decorator, 2nd time call.')
res=func3(data, 10)
print('res =', res)
The outputs:
### Context manager, 1st time call
# <writeCache>: Saving output to:
/tmp/cache/[diskcache]_[context_mananger].nc
res = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90]
### Context manager, 2nd time call
# <readCache>: Read in variable context_mananger from disk cache:
/tmp/cache/[diskcache]_[context_mananger].nc
res = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90]
### Context manager, 3rd time call with overwrite=True
# <writeCache>: Saving output to:
/tmp/cache/[diskcache]_[context_mananger].nc
res = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90]
### Wrap a new function, 1st time call
# <writeCache>: Saving output to:
/tmp/cache/[diskcache]_[new_func].nc
res = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90]
### Wrap a new function, 2nd time call
# <readCache>: Read in variable new_func from disk cache:
/tmp/cache/[diskcache]_[new_func].nc
res = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90]
### pie decorator, 1st time call
# <writeCache>: Saving output to:
/tmp/cache/[diskcache]_[pie_dec].nc
res = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90]
### pie decorator, 2nd time call.
# <readCache>: Read in variable pie_dec from disk cache:
/tmp/cache/[diskcache]_[pie_dec].nc
res = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90]
Here's a solution I came up with which can:
memoize mutable objects (memoized functions should have no side effects that change mutable parameters or it won't work as expected)
writes to a separate cache file for each wrapped function (easy to delete the file to purge that particular cache)
compresses the data to make it much smaller on disk (a LOT smaller)
It will create cache files like:
cache.__main__.function.getApiCall.db
cache.myModule.function.fixDateFormat.db
cache.myOtherModule.function.getOtherApiCall.db
Here's the code. You can choose a compression library of your choosing, but I've found LZMA works best for the pickle storage we are using.
import dbm
import hashlib
import pickle
# import bz2
import lzma
# COMPRESSION = bz2
COMPRESSION = lzma # better with pickle compression
# Create a #memoize_to_disk decorator to cache a memoize to disk cache
def memoize_to_disk(function, cache_filename=None):
uniqueFunctionSignature = f'cache.{function.__module__}.{function.__class__.__name__}.{function.__name__}'
if cache_filename is None:
cache_filename = uniqueFunctionSignature
# print(f'Caching to {cache_file}')
def wrapper(*args, **kwargs):
# Convert the dictionary into a JSON object (can't memoize mutable fields, this gives us an immutable, hashable function signature)
if cache_filename == uniqueFunctionSignature:
# Cache file is function-specific, so don't include function name in params
params = {'args': args, 'kwargs': kwargs}
else:
# add module.class.function name to params so no collisions occur if user overrides cache_file with the same cache for multiple functions
params = {'function': uniqueFunctionSignature, 'args': args, 'kwargs': kwargs}
# key hash of the json representation of the function signature (to avoid immutable dictionary errors)
params_json = json.dumps(params)
key = hashlib.sha256(params_json.encode("utf-8")).hexdigest() # store hash of key
# Get cache entry or create it if not found
with dbm.open(cache_filename, 'c') as db:
# Try to retrieve the result from the cache
try:
result = pickle.loads(COMPRESSION.decompress(db[key]))
# print(f'CACHE HIT: Found {key[1:100]=} in {cache_file=} with value {str(result)[0:100]=}')
return result
except KeyError:
# If the result is not in the cache, call the function and store the result
result = function(*args, **kwargs)
db[key] = COMPRESSION.compress(pickle.dumps(result))
# print(f'CACHE MISS: Stored {key[1:100]=} in {cache_file=} with value {str(result)[0:100]=}')
return result
return wrapper
To use the code, use the #memoize_to_disk decorator (with an optional filename parameter if you don't like "cache." as a prefix)
#memoize_to_disk
def expensive_example(n):
// expensive operation goes here
return value