I have a simple Python program that uses GAE, and I am using GAE's memcache module to make sure that I do not screw up when updating the cache:
from google.appengine.api import memcache
class NewPost(BlogHandler):
def get(self):
self.render("newpost.html")
def post(self):
#update cache for the front page
val, unique = memcache.gets(FRONT_PAGE_KEY)
for p in val:
logging.warning(p)
Now this code should run without problem, but instead when I use the method post, it blows up:
Traceback (most recent call last):
File "/home/pedro/google_appengine/google/appengine/runtime/wsgi.py", line 266, in Handle
result = handler(dict(self._environ), self._StartResponse)
File "/home/pedro/google_appengine/lib/webapp2-2.3/webapp2.py", line 1519, in __call__
response = self._internal_error(e)
File "/home/pedro/google_appengine/lib/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/home/pedro/google_appengine/lib/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/home/pedro/google_appengine/lib/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/home/pedro/google_appengine/lib/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/home/pedro/google_appengine/lib/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/home/pedro/google_appengine/lib/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/home/pedro/google_appengine/hw6/blog.py", line 172, in post
val, unique = memcache.gets(FRONT_PAGE_KEY)
AttributeError: 'module' object has no attribute 'gets'
INFO 2014-05-08 13:36:49,525 module.py:639] default: "POST /blog/newpost HTTP/1.1" 500 -
This makes no sense at all, specially because I know that memcache has a method called gets(key):
https://developers.google.com/appengine/docs/python/memcache/
Based on my research on stackoverflow I found this discussion:
App Engine Python: AttributeError: 'module' object has no attribute 'Stock'
And so I have flushed my cache and deleted all my local DB's content, but I am still getting the error.
What am I doing wrong?
You're mistaken: the memcache module does not have a gets function. See the documentation.
gets is a method of the memcache Client object: again, see the docs.
Related
I get the following error when i try to implement some kind of dynamic class loading using __import__ ...:
No module named pip._vendor.requests.status_codes
Traceback (most recent call last):
File"/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1536, in __call__
rv = self.handle_exception(request, response, e)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1530, in __call__
rv = self.router.dispatch(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~ticatestdev/2.386059027148196174/tica/tasks.py", line 36, in post
m = __import__(handler['module'], fromlist=[handler['name']])
File "/base/data/home/apps/s~ticatestdev/2.386059027148196174/tica/sources/processors.py", line 18, in <module>
from pip._vendor.requests.status_codes import codes
ImportError: No module named pip._vendor.requests.status_codes
... or importlib.import_module :
No module named pip._vendor.requests.status_codes
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1536, in __call__
rv = self.handle_exception(request, response, e)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1530, in __call__
rv = self.router.dispatch(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~ticatestdev/2.386058813285719320/tica/tasks.py", line 36, in post
m = importlib.import_module(handler['module'])
File "/base/data/home/runtimes/python27/python27_dist/lib/python2.7/importlib/__init__.py", line 37, in import_module
__import__(name)
File "/base/data/home/apps/s~ticatestdev/2.386058813285719320/tica/sources/processors.py", line 18, in <module>
from pip._vendor.requests.status_codes import codes
ImportError: No module named pip._vendor.requests.status_codes
I import importlib of course.
I deploy my program on the Google AppEngine server to test it (I have a production URL and a Development URL).
Here is the code :
import importlib
import json
import logging
class Process(handlers.BaseHandler):
def post(self):
parameters = json.loads(self.request.get('parameters'))
if parameters:
if 'handler' in parameters:
handler = parameters['handler']
m = __import__(handler['module'], fromlist=[handler['name']])
#m = importlib.import_module(handler['module'])
task_handler = getattr(m, handler['name'])(parameters)
#task_handler = getattr(m, handler['name'])(parameters)
task_handler.startProcessing(parameters)
else:
logging.error("python.tasks.Process(): No handler information provided!")
else:
logging.error("python.tasks.Process(): No parameters provided!")
and the parameters contents:
{"dictionary_list": [...], "handler": {"name": "SourceProcessingHandler", "module": "sources.processors"}, "command": "process", "fsm": {}, "language_code": "ja"}
I could not find a similar error on google,
I wonder now if I am not wasting my time trying to outsmart myself with this complex solution ...
May be I should use a simple dictionary associating a name to the python classes?
The tica/sources/processors.py file appears to be part of your application. If so it explicitly depends on the pip package, which doesn't appear to be part of the standard python library and is not included in the GAE Runtime-Provided Libraries. This means it would need to be vendored in.
All the packages/modules that you want to dynamically load together with all their dependencies may similarly need to be vendored in.
I'd also add a lot more sanity checks before attempting to execute the dynamically loaded code. At least catching and handling ImportError.
Side note: just in case task_handler.startProcessing(parameters) can take too long and cause DeadlineExceededError for the post request you may want to delegate the execution onto a task queue or a backend execution module instead.
I'm trying to insert an instance of a model into an ndb database.
It keeps on giving me the "Model is not immutable" error.
I have tried with different model names, but still the same error.
class User(ndb.Model):
username = ndb.StringProperty()
email = ndb.StringProperty()
lwr_username = ndb.ComputedProperty(lambda self: self.username.lower())
lwr_email = ndb.ComputedProperty(lambda self: self.email.lower())
Here's my insert code:
entity = User()
entity.email = ""
entity.username = "bob"
logging.info(entity)
#Commit data asynchronously
entities = [entity]
futures = ndb.put_multi_async(entities)
#Build Response whilst database is committing
response = {
}
#Wait for commits to finish
ndb.Future.wait_all(entities)
This is the full stack trace
Model is not immutable
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~myapp-api/1.377037445874907069/v1/handler/userHandler.py", line 11, in post
responseCode, response = UserService.create(locale, json.loads(self.request.body), **kwargs)
File "/base/data/home/apps/s~myapp-api/1.377037445874907069/v1/service/userService.py", line 37, in create
ndb.Future.wait_all(entities)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 345, in wait_all
waiting_on = set(futures)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/model.py", line 3017, in __hash__
raise TypeError('Model is not immutable')
TypeError: Model is not immutable
You need to use the futures list, not entities to wait for the async process to complete:
ndb.Future.wait_all(futures)
The wait_all() function stores these in a set() object, and sets require the contents to be hashable. Since mutable objects cannot be stored in sets or dictionaries, Google engineers added an explicit TypeError to the Model.__hash__ method, which is what you see raised here.
My application experiences over quota issues and I would like to handle such cases properly in my code. The limit is reached just for Datastore Read Operations, but I get TypeError: ConjunctionNode() requires at least one node exception when try to read the data from the memcache - entries = memcache.get('mykey').
Why this exception happens?
Exception details:
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/runtime/wsgi.py", line 266, in Handle
result = handler(dict(self._environ), self._StartResponse)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1519, in __call__
response = self._internal_error(e)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~myapp/1.373233284460557570/myapp.py", line 595, in get
entries = memcache.get('mykey')
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/memcache/__init__.py", line 559, in get
results = rpc.get_result()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/apiproxy_stub_map.py", line 612, in get_result
return self.__get_result_hook(self)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/memcache/__init__.py", line 624, in __get_hook
self._do_unpickle)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/memcache/__init__.py", line 271, in _decode_value
return do_unpickle(value)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/memcache/__init__.py", line 401, in _do_unpickle
return unpickler.load()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/query.py", line 550, in __new__
raise TypeError('ConjunctionNode() requires at least one node.')
TypeError: ConjunctionNode() requires at least one node.
Upd. here is what I save to memcache:
entries = MyModel.query()
entries = entries.fetch(keys_only=True)
entries = random.sample(entries, 10)
entries = [list_key.get() for list_key in entries]
memcache.set('mykey', entries, 60*60*24)
Upd2. Online memcache viewer shows the following value stored (just first part is shown below):
Type: Object
..cgoogle.appengine.ext.ndb.query.Query.q.).q.}q.(U._Query__projectionq.NU._Query__filtersq.cgoogle.appengine.ext.ndb.query.ConjunctionNode.q.).q.}q.U._ConjunctionNode__nodesq.]q.(cgoogle.appengine.ext.ndb.query.FilterNode.q.).q.}q.(U._Filt
I believe that this happens simply because you are trying to put a pickled Query object into memcache, but Query objects are not pickle-able.
What I do instead in this case is to convert the Query to a list prior to putting it into the cache. If your code doesn't care about the distinction (that is, if it's not calling .filter() or something like that) then it'll probably work fine for you too.
I have this piece of python code in gae.
def post(self):
cases=self.request.get('cases')
while cases:
logging.info("cases: %s " % cases)
case=cases.pop()
Which produces this log.
INFO 2012-09-19 20:23:50,690 views.py:674] cases: [u'court1150']
ERROR 2012-09-19 20:23:50,690 webapp2.py:1553] 'unicode' object has no attribute 'pop'
Traceback (most recent call last):
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 1536, in __call__
rv = self.handle_exception(request, response, e)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 1530, in __call__
rv = self.router.dispatch(request, response)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "/Users/brian/googleapps/scheduler/views.py", line 675, in post
case=cases.pop()
AttributeError: 'unicode' object has no attribute 'pop'
On the other hand with this almost identical code in the interactive console.
cases = [u'court1150']
while cases:
case=cases.pop()
print case
print cases
I get no error and the following print out.
court1150
[]
Why am I getting a unicode error in the gae launcher and how can I fix it?
self.request.get('cases') does not return a list of values, it returns just one value. Try using get_all('cases') instead:
def post(self):
cases=self.request.get_all('cases')
while cases:
logging.info("cases: %s " % cases)
case=cases.pop()
There is no need to use a while loop, a for loop works much better:
def post(self):
cases=self.request.get_all('cases')
logging.info("cases: %s " % cases)
for case in cases:
# do something with case.
I'm trying to implement the code that Guido posted as an answer to the question Avoiding Memcache 1M limit of values. Everything seems(?) to be working when I first load the page and add values to memcache, but when I re-load it and retrieve values from the memcache I get a strange error:
Traceback (most recent call last):
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/runtime/wsgi.py", line 194, in Handle
for chunk in result:
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/ext/appstats/recording.py", line 1036, in appstats_wsgi_wrapper
result = app(environ, appstats_start_response)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1519, in __call__
response = self._internal_error(e)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/appengineurlhere/mypythoncode.py", line 87, in get
entitylist = retrieve("entitylist")
File "/appengineurlhere/mypythoncode.py", line 53, in retrieve
return pickle.loads(serialized)
File "/base/python27_runtime/python27_dist/lib/python2.7/pickle.py", line 1382, in loads
return Unpickler(file).load()
File "/base/python27_runtime/python27_dist/lib/python2.7/pickle.py", line 858, in load
dispatch[key](self)
KeyError: ':'
Here's the store/retrieve code:
def store(key, value, chunksize=750000):
serialized = pickle.dumps(value, 2)
values = {}
for i in xrange(0, len(serialized), chunksize):
values['%s.%s' % (key, i//chunksize)] = serialized[i : i+chunksize]
memcache.set_multi(values)
def retrieve(key):
result = memcache.get_multi(['%s.%s' % (key, i) for i in xrange(32)])
serialized = ''.join([v for v in result.values() if v is not None])
return pickle.loads(serialized)
And this is how I'm using it:
try:
entitylist = retrieve("entitylist")
except EOFError:
entitylist = MyModel.all().fetch(None)
store("entitylist", entitylist)
Other oddities:
I don't see this on my development server; just the production App Engine site. (Though the data on my development server is slightly different; I believe it all fits into the standard 1MB memcache size, where the production data is larger.)
When I search for "entitylist" on both my development and production admin panel, App Engine tells me "no such key". Yet based on the total size of the memcache shown (this is the only place I've implemented memcache), it absolutely appears that something is getting cached.
Can anyone please point me in the direction of something I should look at or fix?
Is it possible that you have some old instances of MyModel that don't match with the current definition of MyModel? You could get errors while pickling if there are unrecognized or missing attributes.