I am having a weird bug that seems related to Djangos caching.
I have a 3-step registration process:
insert personal data
insert company data
summary view and submit all data for registration
If person A walks through the process to the summary part but does not submit the form and
person B does the same, person B gets the data of person A in the summary view.
The data gets stored in a Storage object which carries the data through each step. Every new registration instanciates a new Storage object (at least it should).
While debugging I've found that Django does not call any method in the corresponding views when the cache is already warmed up (by another running registration) and I guess that's why there is no new Storage instance. Hence the cross-polution of data.
Now I'm perfectly aware that I can decorate the method with #never_cache() (which it already was) but that doesn't do the trick.
I've also found that the #never_cache decorator does not work properly prior to Django 1.9(?) as it misses some headers.
One solution that I've found was to set these headers myself with #cache_control(max_age=0, no_cache=True, no_store=True, must_revalidate=True). But that also doesn't work.
So how can I properly disable caching for these methods?
Here is some relevant code:
# views.py
def _request_storage(request, **kwargs):
try:
return getattr(request, '_registration_storage')
except AttributeError:
from .storage import Storage
storage = Storage(request, 'registration')
setattr(request, '_registration_storage', storage)
return storage
...
# Route that gets called by clicking the "register" button
#secure_required
#cache_control(max_age=0, no_cache=True, no_store=True, must_revalidate=True)
# also does not work with #never_cache()
def registration_start(request, **kwargs):
storage = _request_storage(request)
storage.clear()
storage.store_data('is_authenticated', request.user.is_authenticated())
storage.store_data('user_pk', request.user.pk if request.user.is_authenticated() else None)
return HttpResponseRedirect(reverse_i18n('registration_package', kwargs={'pk': 6}))
def registration_package(request, pk=None, **kwargs):
"""stores the package_pk in storage"""
...
def registration_personal(request, pk=None, **kwargs):
from .forms import PersonalForm
storage = _request_storage(request)
if request.method == 'POST':
form = PersonalForm(request.POST)
if form.is_valid():
storage.store_form('personal_form_data', form)
return HttpResponseRedirect(reverse_i18n('registration_company'))
else:
if request.GET.get('revalidate', False):
form = storage.retrieve_form('personal_form_data', PersonalForm)
else:
form = storage.retrieve_initial_form('personal_form_data', PersonalForm)
return render_to_response('registration/personal.html', {
'form': form,
'step': 'personal',
'previous': _previous_steps(request),
}, context_instance=RequestContext(request))
# the other steps are pretty much the same
# storage.py
class Storage(object):
def __init__(self, request, prefix):
self.request = request
self.prefix = prefix
def _debug(self):
from pprint import pprint
self._init_storage()
pprint(self.request.session[self.prefix])
def exists(self):
return self.prefix in self.request.session
def has(self, key):
if self.prefix in self.request.session:
return key in self.request.session[self.prefix]
return False
def _init_storage(self,):
if not self.prefix in self.request.session:
self.request.session[self.prefix] = {}
self.request.session.modified = True
def clear(self):
self.request.session[self.prefix] = {}
self.request.session.modified = True
def store_data(self, key, data):
self._init_storage()
self.request.session[self.prefix][key] = data
self.request.session.modified = True
def update_data(self, key, data):
self._init_storage()
if key in self.request.session[self.prefix]:
self.request.session[self.prefix][key].update(data)
self.request.session.modified = True
else:
self.store_data(key, data)
def retrieve_data(self, key, fallback=None):
self._init_storage()
return self.request.session[self.prefix].get(key, fallback)
def store_form(self, key, form):
self.store_data(key, form.data)
def retrieve_form_data(self, key):
return self.retrieve_data(key)
def retrieve_form(self, key, form_class):
data = self.retrieve_form_data(key)
form = form_class(data=data)
return form
def retrieve_initial_form(self, key, form_class):
data = self.retrieve_form_data(key)
form = form_class(initial=self.convert_form_data_to_initial(data))
return form
def convert_form_data_to_initial(self, data):
result = {}
if data is None:
return result
for key in data:
try:
values = data.getlist(key)
if len(values) > 1:
result[key] = values
else:
result[key] = data.get(key)
except AttributeError:
result[key] = data.get(key)
return result
def retrieve_process_form(self, key, form_class, initial=None):
if request.method == 'POST':
return form_class(data=request.POST)
else:
data = self.get_form_data(key)
initial = self.convert_form_data_to_initial(data) or initial
return form_class(initial=initial)
I've tested this across browser, computers and networks. When I clear the cache manually it works again.
(Easy to see with just debugging print() statements which do not get called with a warm cache.)
It can't be too hard to selectively disable caching, right?
Additional question:
Could it be that the #never_cache() decorator just prevents browser-caching and has nothing to do with the Redis cache?
Related
I have two def validate(self) functions within the RegistrationForm(FlaskForm). First function validates the user input if the car range is over 400km. The other function validates if the user lives far or close to work by calculating the distance between the user's address and work address by means of geopy module. I've been told that def validate_field(self, field): function takes only one attribute which must be the same as the field name. In that function I have an extra variable total_travel_km that stores distance between work and user's home. If user is validated than I need to store total_travel_km variable in DB.
Now the server brings me this message: 'RegistrationForm' object has no attribute 'total_travel_km'
How can I call that total_travel_km variable correctly and write it inside DB
Here is the cut out from my code:
class RegistrationForm(FlaskForm):
car_range = IntegerField('Car Range, km')
home_address = StringField('Home Address')
submit = SubmitField('Register User/Car')
def validate_car_range(self, car_range):
if car_range.data > 400:
raise ValidationError('Your car range does not meet our requirements')
def validate_home_address(self, home_address):
user_loc = locator.geocode(home_address.data)
user_coords = (user_loc.latitude, user_loc.longitude)
one_way = geopy.distance.geodesic(tco_coords, user_coords).km
total_travel_km = one_way*4
if total_travel_km < self.car_range.data:
raise ValidationError('You live too close to work')
return total_travel_km
#app.route("/register", methods=['POST', 'GET'])
def register():
form = RegistrationForm()
if form.validate_on_submit():
user = User(car_range=form.car_range.data, home_address=form.home_address.data,
car_travel_km=form.total_travel_km)
db.session.add(user)
db.session.commit()
EDIT: I figured out. I had to pass form. instead of self. in validation method. Below is the correct code:
def validate_home_address(form, home_address):
user_loc = locator.geocode(home_address.data)
user_coords = (user_loc.latitude, user_loc.longitude)
one_way = geopy.distance.geodesic(tco_coords, user_coords).km
form.total_travel_km = one_way*4
if form.total_travel_km < self.car_range.data:
raise ValidationError('You live too close to work')
return form.total_travel_km
Set a form instance variable in the validation method:
class RegistrationForm(FlaskForm):
def __init__(self, *args, **kwargs):
super(RegistrationForm, self).__init__(*args, **kwargs)
self.total_travel_km = None
def validate_home_address(self, home_address):
user_loc = locator.geocode(home_address.data)
user_coords = (user_loc.latitude, user_loc.longitude)
one_way = geopy.distance.geodesic(tco_coords, user_coords).km
total_travel_km = one_way * 4
if total_travel_km < self.car_range.data:
raise ValidationError('You live too close to work')
# all good save the total travel in the form
self.total_travel_km = total_travel_km
Also, you might not want to hard code the car range limit, you can pass a value when you instance the form:
class RegistrationForm(FlaskForm):
def __init__(self, *args, **kwargs):
super(RegistrationForm, self).__init__(*args, **kwargs)
self.total_travel_km = None
self.required_car_range = kwargs.get('required_car_range', 400)
def validate_car_range(self, car_range):
if car_range.data > self.required_car_range:
raise ValidationError('Your car range does not meet our requirements')
and use as follows:
#app.route("/register", methods=['POST', 'GET'])
def register():
form = RegistrationForm(required_car_range=600)
if form.validate_on_submit():
user = User(car_range=form.car_range.data, home_address=form.home_address.data,
car_travel_km=form.total_travel_km)
db.session.add(user)
db.session.commit()
I have this Python script to control a PfSense router via FauxAPI. The problem is that when i call a function it gives an error. I think i'm calling the function wrong. Does anyone know how to call them?
Here is a link to the API i'm using: https://github.com/ndejong/pfsense_fauxapi
I have tried calling config_get(self, section=none) but that does not seem to work.
import os
import json
import base64
import urllib
import requests
import datetime
import hashlib
class PfsenseFauxapiException(Exception):
pass
class PfsenseFauxapi:
host = '172.16.1.1'
proto = None
debug = None
version = None
apikey = 'key'
apisecret = 'secret'
use_verified_https = None
def __init__(self, host, apikey, apisecret, use_verified_https=False, debug=False):
self.proto = 'https'
self.base_url = 'fauxapi/v1'
self.version = __version__
self.host = host
self.apikey = apikey
self.apisecret = apisecret
self.use_verified_https = use_verified_https
self.debug = debug
if self.use_verified_https is False:
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
def config_get(self, section=None):
config = self._api_request('GET', 'config_get')
if section is None:
return config['data']['config']
elif section in config['data']['config']:
return config['data']['config'][section]
raise PfsenseFauxapiException('Unable to complete config_get request, section is unknown', section)
def config_set(self, config, section=None):
if section is None:
config_new = config
else:
config_new = self.config_get(section=None)
config_new[section] = config
return self._api_request('POST', 'config_set', data=config_new)
def config_patch(self, config):
return self._api_request('POST', 'config_patch', data=config)
def config_reload(self):
return self._api_request('GET', 'config_reload')
def config_backup(self):
return self._api_request('GET', 'config_backup')
def config_backup_list(self):
return self._api_request('GET', 'config_backup_list')
def config_restore(self, config_file):
return self._api_request('GET', 'config_restore', params={'config_file': config_file})
def send_event(self, command):
return self._api_request('POST', 'send_event', data=[command])
def system_reboot(self):
return self._api_request('GET', 'system_reboot')
def system_stats(self):
return self._api_request('GET', 'system_stats')
def interface_stats(self, interface):
return self._api_request('GET', 'interface_stats', params={'interface': interface})
def gateway_status(self):
return self._api_request('GET', 'gateway_status')
def rule_get(self, rule_number=None):
return self._api_request('GET', 'rule_get', params={'rule_number': rule_number})
def alias_update_urltables(self, table=None):
if table is not None:
return self._api_request('GET', 'alias_update_urltables', params={'table': table})
return self._api_request('GET', 'alias_update_urltables')
def function_call(self, data):
return self._api_request('POST', 'function_call', data=data)
def system_info(self):
return self._api_request('GET', 'system_info')
def _api_request(self, method, action, params=None, data=None):
if params is None:
params = {}
if self.debug:
params['__debug'] = 'true'
url = '{proto}://{host}/{base_url}/?action={action}&{params}'.format(
proto=self.proto, host=self.host, base_url=self.base_url, action=action, params=urllib.parse.urlencode(params))
if method.upper() == 'GET':
res = requests.get(
url,
headers={'fauxapi-auth': self._generate_auth()},
verify=self.use_verified_https
)
elif method.upper() == 'POST':
res = requests.post(
url,
headers={'fauxapi-auth': self._generate_auth()},
verify=self.use_verified_https,
data=json.dumps(data)
)
else:
raise PfsenseFauxapiException('Request method not supported!', method)
if res.status_code == 404:
raise PfsenseFauxapiException('Unable to find FauxAPI on target host, is it installed?')
elif res.status_code != 200:
raise PfsenseFauxapiException('Unable to complete {}() request'.format(action), json.loads(res.text))
return self._json_parse(res.text)
def _generate_auth(self):
# auth = apikey:timestamp:nonce:HASH(apisecret:timestamp:nonce)
nonce = base64.b64encode(os.urandom(40)).decode('utf-8').replace('=', '').replace('/', '').replace('+', '')[0:8]
timestamp = datetime.datetime.utcnow().strftime('%Y%m%dZ%H%M%S')
hash = hashlib.sha256('{}{}{}'.format(self.apisecret, timestamp, nonce).encode('utf-8')).hexdigest()
return '{}:{}:{}:{}'.format(self.apikey, timestamp, nonce, hash)
def _json_parse(self, data):
try:
return json.loads(data)
except json.JSONDecodeError:
pass
raise PfsenseFauxapiException('Unable to parse response data!', data)
Without having tested the above script myself, I can conclude that yes you are calling the function wrong. The above script is rather a class that must be instantiated before any function inside can be used.
For example you could first create an object with:
pfsense = PfsenseFauxapi(host='<host>', apikey='<API key>', apisecret='<API secret>')
replacing <host>, <API key> and <API secret> with the required values
Then call the function with:
pfsense.config_get() # self is not passed
where config_get can be replaced with any function
Also note
As soon as you call pfsense = PfsenseFauxapi(...), all the code in
the __init__ function is also run as it is the constructor (which
is used to initialize all the attributes of the class).
When a function has a parameter which is parameter=something, that something is the default value when nothing is passed for that parameter. Hence why use_verified_https, debug and section do not need to be passed (unless you want to change them of course)
Here is some more information on classes if you need.
You need to create an object of the class in order to call the functions of the class. For example
x = PfsenseFauxapi() (the init method is called during contructing the object)
and then go by x.'any function'. Maybe name the variable not x for a good naming quality.
I am using the cms Plone to build a form that contains two other schemas.
With the Group Forms, I have been able to include both of the fields of the two schemas. However, they lose all of their properties such as hidden or when I use datagridfield to build the table. What I want to be able to do is have both of these forms with their fields and on a save be able to save them in the object which the link was clicked as parent -> object 1 [top of the form] -> object 2 [bottom of the form]
Here is my python code:
class QuestionPart(group.Group):
label = u'Question Part'
fields = field.Fields(IQuestionPart)
template = ViewPageTemplateFile('questionpart_templates/view.pt')
class Question(group.Group):
label = u'Question'
fields = field.Fields(IQuestion)
template = ViewPageTemplateFile('question_templates/view.pt')
class QuestionSinglePart(group.GroupForm, form.AddForm):
grok.name('register')
grok.require('zope2.View')
grok.context(ISiteRoot)
label = u"Question with Single Part"
ignoreContext = True
enable_form_tabbing = False
groups = (Question,QuestionPart)
def update(self):
super(QuestionSinglePart, self).update()
This code displays both the fields of IQuestion, IQuestionPart without the regard to things like: form.mode(contype='hidden') or DataGridField widget.
I found a way that displays the correct form with field hints.
class QuestionSinglePart(AutoExtensibleForm, form.AddForm):
grok.require('zope2.View')
grok.context(ISiteRoot)
label = u"Question"
schema = IQuestion
additionalSchemata = (IQuestionPart,)
I feel I am still a long way off. I have talked to some people. I am now trying to use a separate form and view.
So far, I am at this point with my code:
class QuestionSinglePartForm(AutoExtensibleForm, form.Form):
ignoreContext = True
autoGroups = True
template = ViewPageTemplateFile('questionsinglepart_templates/questionsinglepartform.pt')
#property
def additionalSchemata(self):
return self._additionalSchemata
def __init__(self, context, request, schema, additional=()):
self.context = context
self.request = request
if not IInterface.providedBy(schema):
raise ValueError('Schema is not interface object')
self._schema = schema
if not all(IInterface.providedBy(s) for s in additional):
raise ValueError('Additional schema is not interface')
self._additionalSchemata = additional
class QuestionSinglePartView(object):
schema = IQuestion
additional = (IQuestionPart,)
def __init__(self, context, request):
self.context = context
self.request = request
self.form = QuestionSinglePartForm(context, request, self.schema, self.additional)
def magic(self, data, errors):
pass
"""
question = Question()
question.number = data['number']
question.questionContent = data['questionContent']
questionPart = QuestionPart()
questionPart.typeOfQuestion = data['IQuestionPart.typeOfQuestion']
questionPart.explanation = data['IQuestionPart.explanation']
questionPart.fileSize = data['IQuestionPart.fileSize']
questionPart.fileType = data['IQuestionPart.fileType']
questionPart.hints = data['IQuestionPart.hints']
questionPart.table = data['IQuestionPart.table']
questionPart.contype = data['IQuestionPart.contype']
questionPart.content = data['IQuestionPart.content']
"""
def update(self, *args, **kwargs):
if self.request.get('REQUEST_METHOD') == 'POST':
data, errors = self.form.extractData()
self.magic(data, errors)
self.formdisplay = self.form.render()
def __call__(self, *args, **kwargs):
self.update(*args, **kwargs)
return self.index(*args, **kwargs)
I am struggling with the rendering of the form and that the QuestionSinglePart object does not have an index attribute.
After a couple of hours of working with some plone devs, we have figured out what was going on.
I had left out:
#property
def schema(self):
return self._schema
I needed to define an index in the view like so:
index = ViewPageTemplateFile('questionsinglepart_templates/questionsinglepart.pt')
I needed to add this to the views init:
alsoProvides(self.form, IWrappedForm)
In the update method for view I needed to call this before the formdisplay. I could also remove the data extraction and move that to the form.
def update(self, *args, **kwargs):
self.form.update(*args, **kwargs)
self.formdisplay = self.form.render()
I am still currently working to get the data to save into objects.
Only form classes that include the plone.autoform.base.AutoExtensibleForm mixin pay attention to schema form hints. Try using this as a mixin to your Group form class (and provide the schema attribute that this mixin looks for instead of fields):
from plone.autoform.base import AutoExtensibleForm
class QuestionPart(AutoExtensibleForm, group.Group):
label = u'Question Part'
schema = IQuestionPart
template = ViewPageTemplateFile('questionpart_templates/view.pt')
Here is my final code with the changes made above. There were issues with the index for the object. I needed to create a simple custom view. I forgot the property for schema on the form. I need to changed my update method for the view as well.
class QuestionSinglePartForm(AutoExtensibleForm, form.Form):
ignoreContext = True
autoGroups = False
#property
def schema(self):
return self._schema
#property
def additionalSchemata(self):
return self._additionalSchemata
def __init__(self, context, request, schema, additional=()):
self.context = context
self.request = request
if not IInterface.providedBy(schema):
raise ValueError('Schema is not interface object')
self._schema = schema
if not all(IInterface.providedBy(s) for s in additional):
raise ValueError('Additional schema is not interface')
self._additionalSchemata = additional
#button.buttonAndHandler(u'Save')
def handleSave(self, action):
data, errors = self.extractData()
if errors:
return False
obj = self.createAndAdd(data)
if obj is not None:
# mark only as finished if we get the new object
self._finishedAdd = True
IStatusMessage(self.request).addStatusMessage(_(u"Changes saved"), "info")
print data
#button.buttonAndHandler(u'Cancel')
def handleCancel(self, action):
print 'cancel'
class QuestionSinglePartView(object):
schema = IQuestion
additional = (IQuestionPart,)
index = ViewPageTemplateFile('questionsinglepart_templates/questionsinglepart.pt')
def __init__(self, context, request):
self.context = context
self.request = request
self.form = QuestionSinglePartForm(context, request, self.schema, self.additional)
alsoProvides(self.form, IWrappedForm)
def magic(self, data, errors):
pass
"""
question = Question()
question.number = data['number']
question.questionContent = data['questionContent']
questionPart = QuestionPart()
questionPart.typeOfQuestion = data['IQuestionPart.typeOfQuestion']
questionPart.explanation = data['IQuestionPart.explanation']
questionPart.fileSize = data['IQuestionPart.fileSize']
questionPart.fileType = data['IQuestionPart.fileType']
questionPart.hints = data['IQuestionPart.hints']
questionPart.table = data['IQuestionPart.table']
questionPart.contype = data['IQuestionPart.contype']
questionPart.content = data['IQuestionPart.content']
"""
def update(self, *args, **kwargs):
self.form.update(*args, **kwargs)
self.formdisplay = self.form.render()
def __call__(self, *args, **kwargs):
self.update(*args, **kwargs)
return self.index(*args, **kwargs)
How do I use Flask-Cache #cache.cached() decorator with Flask-Restful? For example, I have a class Foo inherited from Resource, and Foo has get, post, put, and delete methods.
How can I can invalidate cached results after a POST?
#api.resource('/whatever')
class Foo(Resource):
#cache.cached(timeout=10)
def get(self):
return expensive_db_operation()
def post(self):
update_db_here()
## How do I invalidate the value cached in get()?
return something_useful()
As Flask-Cache implementation doesn't give you access to the underlying cache object, you'll have to explicitly instantiate a Redis client and use it's keys method (list all cache keys).
The cache_key method is used to override the default key generation in your cache.cached decorator.
The clear_cache method will clear only the portion of the cache corresponding to the current resource.
This is a solution that was tested only for Redis and the implementation will probably differ a little when using a different cache engine.
from app import cache # The Flask-Cache object
from config import CACHE_REDIS_HOST, CACHE_REDIS_PORT # The Flask-Cache config
from redis import Redis
from flask import request
import urllib
redis_client = Redis(CACHE_REDIS_HOST, CACHE_REDIS_PORT)
def cache_key():
args = request.args
key = request.path + '?' + urllib.urlencode([
(k, v) for k in sorted(args) for v in sorted(args.getlist(k))
])
return key
#api.resource('/whatever')
class Foo(Resource):
#cache.cached(timeout=10, key_prefix=cache_key)
def get(self):
return expensive_db_operation()
def post(self):
update_db_here()
self.clear_cache()
return something_useful()
def clear_cache(self):
# Note: we have to use the Redis client to delete key by prefix,
# so we can't use the 'cache' Flask extension for this one.
key_prefix = request.path
keys = [key for key in redis_client.keys() if key.startswith(key_prefix)]
nkeys = len(keys)
for key in keys:
redis_client.delete(key)
if nkeys > 0:
log.info("Cleared %s cache keys" % nkeys)
log.info(keys)
Yes, you can use like that.
Maybe you will still need to read: flask-cache memoize URL query string parameters as well
You can invalidate cache using cache.clear() method.
For more detials see: https://pythonhosted.org/Flask-Cache/#flask.ext.cache.Cache.clear and Clearing Cache section in https://pythonhosted.org/Flask-Cache/
##create a decarator
from werkzeug.contrib.cache import SimpleCache
CACHE_TIMEOUT = 300
cache = SimpleCache()
class cached(object):
def __init__(self, timeout=None):
self.timeout = timeout or CACHE_TIMEOUT
def __call__(self, f):
def decorator(*args, **kwargs):
response = cache.get(request.path)
if response is None:
response = f(*args, **kwargs)
cache.set(request.path, response, self.timeout)
return response
return decorator
#add this decarator to your views like below
#app.route('/buildingTotal',endpoint='buildingTotal')
#cached()
def eventAlert():
return 'something'
#app.route('/buildingTenants',endpoint='buildingTenants')
#cached()
def buildingTenants():
return 'something'
Answer from #JahMyst didn't work for me.
Flask-Cache doesn’t work with Flask restful framework. #cache.Cached & #cache.memoize can’t handle mutable objects per their documentation.
Using mutable objects (classes, etc) as part of the cache key can become tricky. It is suggested to not pass in an object instance into a memoized function. However, the memoize does perform a repr() on the passed in arguments so that if the object has a __repr__ function that returns a uniquely identifying string for that object, that will be used as part of the cache key.
Had to come-up with my own implementation. Leaving this code snippet incase someone else gets stuck with the same issue.
cache_key function converts the user req into hash.
cache_res_pickled function is being used to pickle or unpickle the data
|-flask-app
|-app.py
|-resource
|--some_resource.py
import json
import logging
import pickle
import time
import urllib
from flask import Response, abort, request
from redis import Redis
redis_client = Redis("127.0.0.1", "6379")
exp_setting_s = 1500
def json_serial(obj):
"""
JSON serializer for objects not serializable by default json code"
Args:
obj: JSON serialized object for dates
Returns:
serialized JSON data
"""
if isinstance(obj, datetime.datetime):
return obj.__str__()
def cache_key():
""" ""
Returns: Hashed string of request made by the user.
"""
args = request.args
key = (
request.path
+ "?"
+ urllib.parse.urlencode(
[(k, v) for k in sorted(args) for v in sorted(args.getlist(k))]
)
)
key_hashed = hashlib.sha256(key.encode())
return key_hashed.hexdigest()
def cache_res_pickled(data, encode):
"""
Args:
data (dict): Data in dict format
encode (Boolean): Encode (true) or decode (false) the data
Returns: Result after pickling
"""
if encode:
return pickle.dumps(data)
else:
data = pickle.loads(data)
return data
class SomeResource(Resource):
#auth.login_required
def get(self):
# Get the key for request in hashed format SHA256
key = cache_key()
result = redis_client.get(key)
def generate():
"""
A lagging generator to stream JSON so we don't have to hold everything in memory
This is a little tricky, as we need to omit the last comma to make valid JSON,
thus we use a lagging generator, similar to http://stackoverflow.com/questions/1630320/
"""
releases = res.__iter__()
try:
prev_release = next(releases) # get first result
# We have some releases. First, yield the opening json
yield '{"data": ['
# Iterate over the releases
for release in releases:
yield json.dumps(prev_release, default=json_serial) + ", "
prev_release = release
logging.info(f"For {key} # records returned = {len(res)}")
# Now yield the last iteration without comma but with the closing brackets
yield json.dumps(prev_release, default=json_serial) + "]}"
except StopIteration:
# StopIteration here means the length was zero, so yield a valid releases doc and stop
logging.info(f"For {key} # records returned = {len(res)}")
yield '{"data": []}'
if result is None:
# Secure a key on Redis server.
redis_client.set(key, cache_res_pickled({}, True), ex=exp_setting_s)
try:
# Do the querying to the DB or math here to get res. It should be in dict format as shown below
res = {"A": 1, "B": 2, "C": 2}
# Update the key on Redis server with the latest data
redis_client.set(key, cache_res_pickled(res, True), ex=exp_setting_s)
return Response(generate(), content_type="application/json")
except Exception as e:
logging.exception(e)
abort(505, description="Resource not found. error - {}".format(e))
else:
res = cache_res_pickled(result, False)
if res:
logging.info(
f"The data already exists!😊 loading the data form Redis cache for Key - {key} "
)
return Response(generate(), content_type="application/json")
else:
logging.info(
f"There is already a request for this key. But there is no data in it. Key: {key}."
)
s = time.time()
counter = 0
# loops aimlessly till the data is available on the Redis
while not any(res):
result = redis_client.get(key)
res = cache_res_pickled(result, False)
counter += 1
logging.info(
f"The data was available after {time.time() - s} seconds. Had to loop {counter} times.🤦"
)
return Response(generate(), content_type="application/json")
Inspired from durga's answer I wrote a very basic decorator which uses redis directly instead of any library.
from src.consts import config
from src.utils.external_services import redis_connector
import json
import jsons
import base64
class cached(object):
def __init__(self, req, timeout=None):
self.timeout = timeout or config.CACHE_DEFAULT_TIMEOUT
self.request = req
self.cache = redis_connector.get_redis_instance()
def __call__(self, f):
def decorator(*args, **kwargs):
redis_healthy = True
if self.cache is not None:
try:
self.cache.ping()
except Exception as ex:
redis_healthy = False
else:
redis_healthy = False
if self.request is not None and self.request.values is not None and self.request.path is not None and redis_healthy:
cache_key = "{}-{}".format(self.request.path, json.dumps(jsons.dump(self.request.values), sort_keys=True))
cache_key_base_64 = base64.b64encode(cache_key.encode("ascii")).decode("ascii")
response = self.cache.get(cache_key_base_64)
if response is None:
response = f(*args, **kwargs)
self.cache.setex(cache_key_base_64, self.timeout, jsons.dumps(response))
else:
response = json.loads(response)
else:
response = f(*args, **kwargs)
return response
return decorator
Now use this decorator on your api functions
from flask import g, request
from flask_restful import Resource
from webargs.flaskparser import use_args
class GetProducts(Resource):
#use_args(gen_args.argsGetProducts)
#cached(request)
def get(self, args):
return "hello from products"
I have built an application that has a lot of similar views that should be able to use the same base code. However each method has certain unique characteristics at various inflection points within the methods such that I can't figure out a way to structure this to actually reuse any code. Instead I've created a cut-and-paste methodology and tweaked each method individually. This part of the application was some of the first Python code I ever wrote and know there must be a better way to do this, but I got locked into doing it this way and "it works" so I can't see a way out.
Here's what the base view template essentially looks like:
def view_entity(request, entity_id=None):
if request.method == 'POST':
return _post_entity(request, entity_id)
else:
return _get_entity(request, entity_id)
def _get_entity(request, entity_id):
data = _process_entity(request, entity_id)
if 'redirect' in data:
return data['redirect']
else:
return _render_entity(request, data['form'])
def _post_entity(request, entity_id):
data = _process_entity(request, entity_id)
if 'redirect' in data:
return data['redirect']
elif data['form'].is_valid():
# custom post processing here
instance = data['form'].save()
return HttpResponseRedirect(reverse('entity', args=[instance.id]))
else:
return _render_entity(request, data['form'])
def _process_entity(request, entity_id):
data = {}
if entity_id != 'new': # READ/UPDATE
# sometimes there's custom code to retrieve the entity
e = entity_id and get_object_or_404(Entity.objects, pk=entity_id)
# sometimes there's custom code here that deauthorizes e
# sometimes extra values are added to data here (e.g. parent entity)
if e:
if request.method == 'POST':
data['form'] = EntityForm(request.POST, instance=e)
# sometimes there's a conditional here for CustomEntityForm
else:
data['form'] = EntityForm(instance=e)
else: # user not authorized for this entity
return {'redirect': HttpResponseRedirect(reverse('home'))}
# sometimes there's custom code here for certain entity types
else: # CREATE
if request.method == 'POST':
data['form'] = EntityForm(request.POST)
else:
data['form'] = EntityForm()
# sometimes extra key/values are added to data here
return data
I didn't even include all the possible variations, but as you can see, the _process_entity method requires a lot of individual customization based upon the type of entity being processed. This is the primary reason I can't figure out a DRY way to handle this.
Any help is appreciated, thanks!
Use class based views. You can use inheritance and other features from classes to make your views more reusable. You can also use built-in generic views for simplifying some of the basic tasks.
Check class-based views documentation. You can also read this this
So I did end up refactoring the code into a base class that all my views inherit from. I didn't end up refactoring into multiple views (yet), but instead solved the problem of having custom processing methods by inserting hooks within the processing method.
Here's the gist of the base class that inherits from DetailView:
class MyDetailView(DetailView):
context = {}
def get(self, request, *args, **kwargs):
self._process(request, *args, **kwargs)
if 'redirect' in self.context:
return HttpResponseRedirect(self.context['redirect'])
else:
return self._render(request, *args, **kwargs)
def post(self, request, *args, **kwargs):
self._process(request, *args, **kwargs)
if 'redirect' in self.context:
return HttpResponseRedirect(self.context['redirect'])
elif self.context['form'].is_valid():
self._get_hook('_pre_save')(request, *args, **kwargs)
return self._save(request, *args, **kwargs)
else:
return self._render(request, *args, **kwargs)
def _process(self, request, *args, **kwargs):
form = getattr(app.forms, '%sForm' % self.model.__name__)
if kwargs['pk'] != 'new': # READ/UPDATE
self.object = self.get_object(request, *args, **kwargs)
self._get_hook('_auth')(request, *args, **kwargs)
if not self.object: # user not authorized for this entity
return {'redirect': reverse(
'%s_list' % self.model.__name__.lower())}
self.context['form'] = form(
data=request.POST if request.method == 'POST' else None,
instance=self.object if hasattr(self, 'object') else None)
self._get_hook('_post_process')(request, *args, **kwargs)
def _get_hook(self, hook_name):
try:
return getattr(self, '%s_hook' % hook_name)
except AttributeError, e:
def noop(*args, **kwargs):
pass
return noop
The key part to note is the _get_hook method and the places within the other methods that I use it. That way, in some complex view I can inject custom code like this:
class ComplexDetailView(MyDetailView):
def _post_process_hook(self, request, *args, **kwargs):
# here I can add stuff to self.context using
# self.model, self.object, request.POST or whatever
This keeps my custom views small since they inherit the bulk of the functionality but I can add whatever tweaks are necessary for that specific view.