Related
I am using JSON serializer helper function to easy access of dictionary(basically received as JSON) objects.
jsondict.py
"""Utilities for working with JSON and json-like structures - deeply nested Python dicts and lists
This lets us iterate over child nodes and access elements with a dot-notation.
"""
import sys
isPy3 = sys.version_info[0]==3
if isPy3:
def __alt_str__(v,enc='utf8'):
return v if isinstance(v,bytes) else v.encode(enc)
__strTypes__ = (str,bytes)
else:
__alt_str__ = unicode
__strTypes__ = (str,unicode)
class MyLocals(object):
pass
mylocals = MyLocals()
def setErrorCollect(collect):
mylocals.error_collect = collect
setErrorCollect(False)
def errorValue(x):
if isinstance(x,__strTypes__):
return repr(x) if ' ' in x else x
return 'None' if x is None else str(x)
def condJSON(v,__name__=''):
return JSONDict(v,__name__=__name__) if isinstance(v,dict) else JSONList(v,__name__=__name__) if isinstance(v,list) else v
def condJSONSafe(v,__name__=''):
return JSONDictSafe(v,__name__=__name__) if isinstance(v,dict) else JSONListSafe(v,__name__=__name__) if isinstance(v,list) else v
class JSONListIter(object):
def __init__(self, lst, conv):
self.lst = lst
self.i = -1
self.conv = conv
def __iter__(self):
return self
def next(self):
if self.i<len(self.lst)-1:
self.i += 1
return self.conv(self.lst[self.i])
else:
raise StopIteration
if isPy3:
__next__ = next
del next
class JSONList(list):
def __init__(self,v,__name__=''):
list.__init__(self,v)
self.__name__ = __name__
def __getitem__(self,x):
return condJSON(list.__getitem__(self,x),__name__='%s\t%s'%(self.__name__,errorValue(x)))
def __iter__(self):
return JSONListIter(self,condJSON)
class JSONListSafe(JSONList):
def __getitem__(self,x):
__name__='%s\t%s'%(self.__name__,errorValue(x))
try:
return condJSONSafe(list.__getitem__(self,x),__name__=__name__)
except:
if mylocals.error_collect:
mylocals.error_collect(__name__)
return JSONStrSafe('')
def __iter__(self):
return JSONListIter(self,condJSONSafe)
class JSONStrSafe(str):
def __getattr__(self, attr):
return self
__getitem__ = __getattr__
class JSONDict(dict):
"Allows dotted access"
def __new__(cls,*args,**kwds):
__name__ = kwds.pop('__name__')
self = dict.__new__(cls,*args,**kwds)
self.__name__ = __name__
return self
def __init__(self,*args,**kwds):
kwds.pop('__name__','')
dict.__init__(self,*args,**kwds)
def __getattr__(self, attr, default=None):
if attr in self:
return condJSON(self[attr],__name__='%s\t%s'%(self.__name__,errorValue(attr)))
elif __alt_str__(attr) in self:
return condJSON(self[__alt_str__(attr)],__name__='%s\t%s'%(self.__name__,errorValue(attr)))
elif attr=='__safe__':
return JSONDictSafe(self,__name__=self.__name__)
else:
raise AttributeError("No attribute or key named '%s'" % attr)
def sorted_items(self,accept=None, reject=lambda i: i[0]=='__name__'):
if accept or reject:
if not accept:
f = lambda i: not reject(i)
elif not reject:
f = accept
else: #both
f = lambda i: accept(i) and not reject(i)
return sorted(((k,condJSON(v,__name__==k)) for k,v in self.iteritems() if f((k,v))))
else:
return sorted(((k,condJSON(v,__name__==k)) for k,v in self.iteritems()))
def sorted_keys(self):
return sorted(self.keys())
class JSONDictSafe(JSONDict):
"Allows dotted access"
def __getattr__(self, attr, default=None):
if attr in self:
return condJSONSafe(self[attr],__name__='%s\t%s'%(self.__name__,errorValue(attr)))
elif __alt_str__(attr) in self:
return condJSONSafe(self[__alt_str__(attr)],__name__='%s\t%s'%(self.__name__,errorValue(attr)))
elif attr=='__safe__':
return self
else:
return JSONStrSafe('')
def __getitem__(self,x):
__name__='%s\t%s'%(self.__name__,errorValue(x))
try:
return condJSONSafe(dict.__getitem__(self,x),__name__=__name__)
except KeyError:
if mylocals.error_collect:
mylocals.error_collect(__name__)
return JSONStrSafe('')
def sorted_items(self,accept=None, reject=lambda i: i[0]=='__name__'):
if accept or reject:
if not accept:
f = lambda i: not reject(i)
elif not reject:
f = accept
else: #both
f = lambda i: accept(i) and not reject(i)
return sorted(((k,condJSONSafe(v,__name__==k)) for k,v in self.iteritems() if f((k,v))))
else:
return sorted(((k,condJSONSafe(v,__name__==k)) for k,v in self.iteritems()))
If JSON object passed like below.
data = {'name': 'john', 'age': 20, 'address': {'city':'xyz', 'country':'XZ', 'zip': 1223}}
json_obj = condJSONSafe(data)
I am able to access data with dot notation.
print(json_obj.name) --> john
print(json_obj.address.country) --> XZ
It was working well until I implementing multiprocessing in my code to improve the performance.
I have extracted a certain number of data from JSON (after made it as dot notation accessible data with the above helper function) and store it into separate lists, like list a,b,c.
And then, I passed into multiprocessing threads,
with mp.Pool(processes=mp.cpu_count()) as pool:
res = pool.starmap(self.process_records, zip(self.a, self.b, self.c))
pool.join()
end up with
TypeError: 'JSONStrSafe' object is not callable
I tried this answer, but it does not work for me. Appreciate your help. Thanks in advance.
EDIT:
reproduce example:
test.py
import jsondict
import multiprocessing as mp
import itertools
def process_records(data, metadata):
print(data.name)
print(metadata)
#code to requirment
if __name__ == '__main__':
data = {
"metadata": "test_data",
"cust_list": [
{
'name': 'john',
'age': 20,
'address': {
'city':'xyz',
'country':'XZ',
'zip': 1223
}
},
{
'name': 'michal',
'age': 25,
'address': {
'city':'abc',
'country':'CX',
'zip': 3435
}
},
{
'name': 'david',
'age': 30,
'address': {
'city':'mnl',
'country':'TD',
'zip': 6767
}
}
]
}
json_obj = jsondict.condJSONSafe(data)
print(json_obj.metadata) #will print 'test_data'
print(json_obj.cust_list[0].name) #will print 'john'
print(json_obj.cust_list[2].address.city) #will print 'mnl'
with mp.Pool(processes=mp.cpu_count()) as pool:
res = pool.starmap(process_records, zip(json_obj.cust_list, itertools.repeat(json_obj.metadata))) # --> not working
#res = pool.map(process_records, zip(json_obj.cust_list, itertools.repeat(json_obj.metadata))) --> not working
#res = [pool.apply_async(process_records, d, json_obj.metadata) for d in json_obj.cust_list] --> not working
#apply --> not working
pool.join()
Output:
test_data
john
mnl
Traceback (most recent call last):
File "c:/Users/mohanlal/Desktop/Mock/json_err/test_app.py", line 53, in <module>
res = pool.starmap(process_records, zip(json_obj.cust_list, itertools.repeat(json_obj.metadata))) # --> not working
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 268, in starmap
return self._map_async(func, iterable, starmapstar, chunksize).get()
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 608, in get
raise self._value
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 385, in _handle_tasks
put(task)
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\connection.py", line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
TypeError: 'JSONStrSafe' object is not callable
Tried with startmap, map, apply_async, apply, getting the same error for all.
I have tried with solution given in similar question attached link above. Modified as below where this error raised.
import re
dunder_pattern = re.compile("__.*__")
protected_pattern = re.compile("_.*")
classJSONStrSafe(str):
def__getattr__(self, attr):
if dunder_pattern.match(attr) or protected_pattern.match(attr):
return super().__getattr__(attr)
return self
def__getstate__(self): returnself.__dict__
def__setstate__(self, d): self.__dict__.update(d)
__getitem__ = __getattr__
But issue persists.
As suggested in the comments, I changed in all 3 places for getattr and tried. Getting different error as below
Process SpawnPoolWorker-1:
Traceback (most recent call last):
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\process.py", line 249, in _bootstrap
self.run()
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 108, in worker
task = get()
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\queues.py", line 345, in get
return _ForkingPickler.loads(res)
File "c:\Users\mohanlal\Desktop\Mock\json_err\jsondict.py", line 89, in __new__
__name__ = kwds.pop('__name__')
Process SpawnPoolWorker-2:
Process SpawnPoolWorker-4:
Traceback (most recent call last):
Traceback (most recent call last):
KeyError: '__name__'
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\process.py", line 249, in _bootstrap
self.run()
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 108, in worker
task = get()
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\queues.py", line 345, in get
return _ForkingPickler.loads(res)
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\process.py", line 249, in _bootstrap
self.run()
File "c:\Users\mohanlal\Desktop\Mock\json_err\jsondict.py", line 89, in __new__
__name__ = kwds.pop('__name__')
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
KeyError: '__name__'
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 108, in worker
task = get()
File "C:\Users\mohanlal\AppData\Local\Programs\Python\Python36\lib\multiprocessing\queues.py", line 345, in get
return _ForkingPickler.loads(res)
File "c:\Users\mohanlal\Desktop\Mock\json_err\jsondict.py", line 89, in __new__
__name__ = kwds.pop('__name__')
KeyError: '__name__'
The problem is you are in a "pickle". Forgive the pun -- you have a pickle problem. When you are doing multiprocessing, the arguments to your worker functions/methods are pickled. Usually, the defaults used to serialize and de-serialize states are OK, but not in your case. See Pickling Class Instances. The default save and load operations for serializing and de-serializing an object are:
def save(obj):
return (obj.__class__, obj.__dict__)
def load(cls, attributes):
obj = cls.__new__(cls)
obj.__dict__.update(attributes)
return obj
Note that when de-serializing the object the object's __init__ method is not called but its __new__ method is, and therein lies the problem. I had to modify your __new__ method of class JSONDict to try to recognize that it was being called by de-serialization and therefore '__name__' may not be present among the keyword arguments and then had to add to that class customized __getstate__ and __setstate__ methods to override the default way it saves and restores the object's attributes (method __init__ remains unmodified):
class JSONDict(dict):
"Allows dotted access"
def __new__(cls,*args,**kwds):
self = dict.__new__(cls,*args,**kwds)
if kwds and '__name__' in kwds:
__name__ = kwds.pop('__name__')
self.__name__ = __name__
return self
def __init__(self,*args,**kwds):
kwds.pop('__name__','')
dict.__init__(self,*args,**kwds)
def __getstate__(self):
return self.__dict__
def __setstate__(self, d):
self.__dict__ = d
""" The other methods remain unmodified """
Prints:
test_data
john
mnl
john
test_data
michal
david
test_data
test_data
Update
I was scratching my head figuring out why it should be necessary to provide the __getstate__ and __setstate__ pickle methods since what they are doing should be the default action anyway. If you modify the program just to test the pickling without even running the Pool methods by inserting the following line:
json_obj = condJSONSafe(data)
# insert this line:
import pickle; print(pickle.dumps(json_obj)); sys.exit(0)
It prints:
Traceback (most recent call last):
File "test.py", line 205, in <module>
import pickle; print('pickle'); print(pickle.dumps(json_obj)); sys.exit(0)
TypeError: 'JSONStrSafe' object is not callable
After adding a print statement in the right place, it became clear that the problem was in the __getattr__ method of class JSONDictSafe. When pickle checks to see if the class implements methods __getstate__ and __setstate__, when there are no implementations __getattr__ is ultimately called and returns as the default value for these attributes a JSONStrSafe instance. So instead of providing these attributes by defining these methods as I have done, one can alternatively add a simple check as follows:
class JSONDictSafe(JSONDict):
"Allows dotted access"
def __getattr__(self, attr, default=None):
if attr in ('__getstate__', '__setstate__'):
raise AttributeError(f'Missing attribute: {attr}')
""" rest of the method is unmodified """
I'm trying to create a custom validator for one field in the AddSongForm. I used the inline validator and it uses two methods from my Songs class. When I try running the code, I get the following trace back:
form = AddSongForm()
[2017-05-16 13:44:11,547] ERROR in app: Exception on /addsong [POST]
Traceback (most recent call last):
File "C:\Python36-32\lib\site-packages\flask\app.py", line 1982, in wsgi_app
response = self.full_dispatch_request()
File "C:\Python36-32\lib\site-packages\flask\app.py", line 1614, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Python36-32\lib\site-packages\flask\app.py", line 1517, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Python36-32\lib\site-packages\flask\_compat.py", line 33, in reraise
raise value
File "C:\Python36-32\lib\site-packages\flask\app.py", line 1612, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Python36-32\lib\site-packages\flask\app.py", line 1598, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "C:\Python36-32\lib\site-packages\flask_login\utils.py", line 228, in decorated_view
return func(*args, **kwargs)
File "C:/Users/Pellissari/Desktop/files/projects/Musical 9gag/view.py", line 40, in addsong
if form.validate_on_submit():
File "C:\Python36-32\lib\site-packages\flask_wtf\form.py", line 101, in validate_on_submit
return self.is_submitted() and self.validate()
File "C:\Python36-32\lib\site-packages\wtforms\form.py", line 310, in validate
return super(Form, self).validate(extra)
File "C:\Python36-32\lib\site-packages\wtforms\form.py", line 152, in validate
if not field.validate(self, extra):
File "C:\Python36-32\lib\site-packages\wtforms\fields\core.py", line 204, in validate
stop_validation = self._run_validation_chain(form, chain)
File "C:\Python36-32\lib\site-packages\wtforms\fields\core.py", line 224, in _run_validation_chain
validator(form, self)
File "C:\Users\Pellissari\Desktop\files\projects\app\forms.py", line 20, in validate_song_link
if Songs.get_provider(url) in valid_providers:
TypeError: get_provider() missing 1 required positional argument: 'url'
127.0.0.1 - - [16/May/2017 13:44:11] "POST /addsong HTTP/1.1" 500 -
This is my form class
class AddSongForm(Form):
song_title = StringField('song_title', validators=[DataRequired()])
song_artist = StringField('song_artist', validators=[DataRequired()])
song_genre = StringField('song_genre')
song_link = StringField('song_link', validators=[DataRequired()])
def validate_song_link(form, song_link):
valid_providers = ['youtube.com', 'www.youtube.com', 'soundcloud.com', 'www.soundcloud.com']
url = song_link.data
if Songs.get_provider(url) in valid_providers:
if Songs.get_embed_code(url) is not False:
return True
else:
print("couldn't get your content")
return False
else:
print("unsupported provider")
return False
And here is the class I used the methods
class Songs(db.Model):
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
title = db.Column(db.String(50))
artist = db.Column(db.String(30))
genre = db.Column(db.String(40))
author = db.Column(db.String(40))
timestamp = db.Column(db.DateTime)
likes_num = db.Column(db.Integer)
song_link = db.Column(db.String(120))
def get_provider(self, url):
return urllib.parse.urlsplit(url)[1]
def get_embed_code(self, url):
code = None
vars = {'url': url, 'format': 'json', 'iframe': 'true', 'maxwidth': '450', 'show_comments': 'false'}
provider = self.get_provider(url)
endpoint = "http://"+provider+"/oembed?"
source = endpoint+urllib.parse.urlencode(vars)
try:
request = urlopen(source)
code = json.load(request)['html']
return code
except:
print("impossible to get your content. Check the link")
return False
I'm quite new with Python and this is my first time writing OO code, so I have no idea what could be happening here. Besides the problem, I would be happy if you also could give me some feedback on the code and if there are room to improve in some sense.
I'm not convinced these should be methods on Songs. They don't refer to anything to do with the Song class.
However if you want to keep them as methods, but still want to call them from the class, they need to be classmethods, not instance methods.
#classmethod
def get_provider(cls, url):
return urllib.parse.urlsplit(url)[1]
#classmethod
def get_embed_code(cls, url):
...
provider = cls.get_provider(url)
First of all correct the indentation for the method validate_song_link. It should be like this:
def validate_song_link(form, song_link):
valid_providers = ['youtube.com', 'www.youtube.com', 'soundcloud.com', 'www.soundcloud.com']
url = song_link.data
if Songs.get_provider(url) in valid_providers:
if Songs.get_embed_code(url) is not False:
return True
else:
print("couldn't get your content")
return False
else:
print("unsupported provider")
return False
To solve your problem you can try to change the get_provider method to a class method like this:
#classmethod
def get_provider(cls, url):
return urllib.parse.urlsplit(url)[1]
And then you can call this way:
songs = Songs()
if Songs.get_provider(songs, url) in valid_providers:
if Songs.get_embed_code(url) is not False:
return True
[solved]
i mixed up class method and instance method before. so when i
tried to call strptime(cls.ts,format) inside #classmethod function , i was actually calling
ts = ndb.StringProperty() or StringProperty('ts'),but
not the string value inside ts.
the correct one is to remove # decorator and call
strptime(self.ts,format) .
original question below
1:when the Token instance is created , init_created() is called to inititalize ts to a string.
2:user request the verification handler , with token as a arg , the token is used to retrieve
the Token instance.
3:i use the Token instance to call is_valid() .In is_valid , i convert ts back to datetime obj,
to compare to other datetime obj.
error 1 :
when i set( with the str() method )
delta = (now-datetime.strptime(str(cls.ts),'%Y-%b-%d / %H:%M:%S:%f')).total_seconds()
i get
ValueError: time data "StringProperty('ts')" does not match format '%Y-%b-%d / %H:%M:%S:%f'
error 2 :
so i try another way. i set (without str() )
delta = (now-datetime.strptime(cls.ts,'%Y-%b-%d / %H:%M:%S:%f')).total_seconds()
i get
TypeError: must be string, not StringProperty
so my question is how to correctly pass a stringproperty to strptime method.
thank you very much .
below is my code:
class Token(ndb.Model):
ts = ndb.StringProperty()
#classmethod
def init_created(cls):
ts = (datetime.utcnow() + timedelta(hours=8)).strftime(format='%Y-%b-%d / %H:%M:%S:%f')
return ts
#classmethod
def is_valid(cls):
now = (datetime.utcnow() + timedelta(hours=8))
delta = (now-datetime.strptime(cls.ts,'%Y-%b-%d / %H:%M:%S:%f')).total_seconds()
return (delta<expire_time)
class Verification(webapp2.RequestHandler):
def get(self , token ):
token_cls = Token.query(Token.token == token ).get()
if not (token_cls and token_cls.is_valid() ) :
template = jinja_environment.get_template('verification_error.html' )
pass
must be string, not StringProperty
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1536, in __call__
rv = self.handle_exception(request, response, e)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1530, in __call__
rv = self.router.dispatch(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.1/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s/1.374948928179626607/main.py", line 216, in get
if not (token_cls and token_cls.is_valid() ) :
File "/base/data/home/apps/s~/1.374948928179626607/main.py", line 86, in is_valid
delta = (now-datetime.strptime(cls.ts,'%Y-%b-%d / %H:%M:%S:%f')).total_seconds()
TypeError: must be string, not StringProperty
Your is_valid method shouldn't have the #classmethod decorator, as with it you're operating on the Token class, and not on the entity returned from your query. Once you've removed the decorator, it would be idiomatic to change cls to self.
Im trying to use alchimia for get asynchronous API for DB. Trying to make a simple request to DB, like that:
def authorization(self, data):
"""
Checking user with DB
"""
def __gotResult(user):
yield engine.execute(sqlalchemy.select([Users]).where(Users.name == user))
result = __gotResult(data['user'])
log.msg("[AUTH] User=%s trying to auth..." % data['user'])
data, result_msg = commands.AUTH(result, data)
log.msg(result_msg)
return data
And cant understand - what i doing wrong? Maybe issue in option for engine (where reactor=[])?
Source code:
import sys
from json import dumps, loads
import sqlalchemy
from twisted.internet import reactor, ssl
from twisted.python import log, logfile
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol, listenWS
import commands
from db.tables import Users
from alchimia import TWISTED_STRATEGY
log_file = logfile.LogFile("service.log", ".")
log.startLogging(log_file)
engine = sqlalchemy.create_engine('postgresql://test:test#localhost/testdb', pool_size=20, max_overflow=0,strategy=TWISTED_STRATEGY, reactor=[])
class DFSServerProtocol(WebSocketServerProtocol):
commands = commands.commands_user
def __init__(self):
self.commands_handlers = self.__initHandlersUser()
def __initHandlersUser(self):
handlers = commands.commands_handlers_server
handlers['AUTH'] = self.authorization
handlers['READ'] = None
handlers['WRTE'] = None
handlers['DELT'] = None
handlers['RNME'] = None
handlers['SYNC'] = None
handlers['LIST'] = None
return handlers
def authorization(self, data):
"""
Checking user with DB
"""
def __gotResult(user):
yield engine.execute(sqlalchemy.select([Users]).where(Users.name == data['user']))
result = __gotResult(data['user'])
log.msg("[AUTH] User=%s trying to auth..." % data['user'])
data, result_msg = commands.AUTH(result, data)
log.msg(result_msg)
return data
def onMessage(self, payload, isBinary):
json_data = loads(payload)
json_auth = json_data['auth']
json_cmd = json_data['cmd']
if json_auth == False:
if json_cmd == 'AUTH':
json_data = self.commands_handlers['AUTH'](json_data)
# for authorized users
else:
if json_cmd in commands.commands_user.keys():
if self.commands_handlers[json_cmd] is not None:
json_data = self.commands_handlers[json_cmd](json_data)
else:
json_data['error'] = '%s command is not already realized...' % json_cmd
else:
json_data['auth'] = False
json_data['error'] = 'This command is not supported on server...'
response = dumps(json_data)
self.sendMessage(str(response))
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == 'debug':
log.startLogging(sys.stdout)
debug = True
else:
debug = False
contextFactory = ssl.DefaultOpenSSLContextFactory('keys/server.key', 'keys/server.crt')
factory = WebSocketServerFactory("wss://localhost:9000", debug = debug, debugCodePaths = debug)
factory.protocol = DFSServerProtocol
factory.setProtocolOptions(allowHixie76 = True)
listenWS(factory, contextFactory)
webdir = File("./web/")
webdir.contentTypes['.crt'] = 'application/x-x509-ca-cert'
web = Site(webdir)
reactor.listenSSL(8080, web, contextFactory)
#reactor.listenTCP(8080, web)
reactor.run()
Traceback:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/twisted/python/log.py", line 88, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/usr/local/lib/python2.7/dist-packages/twisted/python/log.py", line 73, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/local/lib/python2.7/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python2.7/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
--- <exception caught here> ---
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite
why = selectable.doRead()
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 215, in doRead
return self._dataReceived(data)
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 221, in _dataReceived
rval = self.protocol.dataReceived(data)
File "/usr/local/lib/python2.7/dist-packages/twisted/protocols/tls.py", line 419, in dataReceived
self._flushReceiveBIO()
File "/usr/local/lib/python2.7/dist-packages/twisted/protocols/tls.py", line 389, in _flushReceiveBIO
ProtocolWrapper.dataReceived(self, bytes)
File "/usr/local/lib/python2.7/dist-packages/twisted/protocols/policies.py", line 120, in dataReceived
self.wrappedProtocol.dataReceived(data)
File "/usr/local/lib/python2.7/dist-packages/autobahn/twisted/websocket.py", line 78, in dataReceived
self._dataReceived(data)
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1270, in _dataReceived
self.consumeData()
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1286, in consumeData
while self.processData() and self.state != WebSocketProtocol.STATE_CLOSED:
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1445, in processData
return self.processDataHybi()
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1758, in processDataHybi
fr = self.onFrameEnd()
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1887, in onFrameEnd
self._onMessageEnd()
File "/usr/local/lib/python2.7/dist-packages/autobahn/twisted/websocket.py", line 107, in _onMessageEnd
self.onMessageEnd()
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 734, in onMessageEnd
self._onMessage(payload, self.message_is_binary)
File "/usr/local/lib/python2.7/dist-packages/autobahn/twisted/websocket.py", line 110, in _onMessage
self.onMessage(payload, isBinary)
File "server.py", line 84, in onMessage
json_data = self.commands_handlers['AUTH'](json_data)
File "server.py", line 68, in authorization
data, result_msg = commands.AUTH(result, data)
File "/home/relrin/code/Helenae/helenae/commands.py", line 68, in AUTH
if result['name'] == data['user']:
exceptions.TypeError: 'generator' object has no attribute '__getitem__'
I think you are missing an #inlineCallbacks around __gotResult() That might not help you quite enough, though; since a single statement generator wrapped with inlineCallbacks is sort of pointless. You should get used to working with explicit deferred handling anyway. Lets pull this apart:
def authorization(self, data):
"""
Checking user with DB
"""
# engine.execute already gives us a deferred, will grab on to that.
user = data['user']
result_d = engine.execute(sqlalchemy.select([Users]).where(Users.name == user))
# we don't have the result in authorization,
# we need to wrap any code that works with its result int a callback.
def result_cb(result):
data, result_msg = commands.AUTH(result, data)
return data
result_d.addCallback(result_cb)
# we want to pass the (asynchronous) result out, it's hiding in our deferred;
# so we return *that* instead; callers need to add more callbacks to it.
return result_d
If you insist; we can squish this down into an inline callbacks form:
from twisted.internet.defer import inlineCallbacks, returnValue
#inlineCallbacks
def authorization(self, data):
user = data['user']
result = yield engine.execute(sqlalchemy.select([Users]).where(Users.name == user))
data, result_msg = commands.AUTH(result, data)
yield returnValue(data)
as before, though, authorization() is asynchronous; and must be since engine.execute is async. to use it, you must attach a callback to the deferred it returns (although the caller may also yield it if it is also wrapped in inlineCallbacks
I persist such an entity in one RequestHandler and can verify through SDK console:
class Moment(db.Model):
user = db.IntegerProperty()
index = db.IntegerProperty()
EDIT: I previously only included the partial class definition, and didn't show how I was writing the model. The following is the full example:
class Moment(db.Model):
user = db.IntegerProperty()
index = db.IntegerProperty()
date = db.DateTimeProperty()
qx = db.FloatProperty()
qy = db.FloatProperty()
qz = db.FloatProperty()
qw = db.FloatProperty()
latitude = db.FloatProperty()
longitude = db.FloatProperty()
image = db.BlobProperty()
def __init__(self, obj):
super(Moment,self).__init__()
self.user = obj['user']
self.index = obj['index']
self.date = obj['date']
self.qx = obj['qx']
self.qy = obj['qy']
self.qz = obj['qz']
self.qw = obj['qw']
self.latitude = obj['latitude']
self.longitude = obj['longitude']
self.image = obj['image']
class UploadHandler(webapp2.RequestHandler):
def post(self):
obj = biplist.readPlistFromString(self.request.body)
Moment(obj).put()
When I try filtered get...
class ServeHandler(webapp2.RequestHandler):
def get(self):
params = {}
params['user'] = int(self.request.get('user'))
params['index'] = int(self.request.get('index'))
q = Moment.all()
q.filter("user =", params['user'])
q.filter("index =", params['index'])
print q.get()
I get the following:
ERROR 2012-11-04 06:56:04,846 webapp2.py:1553] __init__() got an unexpected keyword argument 'index'
Traceback (most recent call last):
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 1536, in __call__
rv = self.handle_exception(request, response, e)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 1530, in __call__
rv = self.router.dispatch(request, response)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "<path>/main.py", line 51, in get
print q.get()
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/google/appengine/ext/db/__init__.py", line 2102, in get
return results.next()
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/google/appengine/ext/db/__init__.py", line 2314, in next
return self.__model_class.from_entity(self.__iterator.next())
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/google/appengine/ext/db/__init__.py", line 1442, in from_entity
return cls(None, _from_entity=entity, **entity_values)
TypeError: __init__() got an unexpected keyword argument 'index'
INFO 2012-11-04 06:56:04,849 dev_appserver.py:3092] "GET /serve?user=0&index=0 HTTP/1.1" 500 -
However if I do print q.count() instead of print q.get() I will get 1. Seems I'm doing things correctly according to documentation, and I tried suggestions from similar questions I found but to no avail.
SDK v1.7.3
Inspired by a comment left to my question, and some notion that I was doing something wrong, with the addition of some other similar questions/answers found, I re-evaluated the way I was handling the Model class. It turns out at least the way I was overriding the class constructor, I believe I was breaking something in the way the superclass handles initialization. The following works OK for me now:
class Moment(db.Model):
user = db.IntegerProperty()
index = db.IntegerProperty()
date = db.DateTimeProperty()
qx = db.FloatProperty()
qy = db.FloatProperty()
qz = db.FloatProperty()
qw = db.FloatProperty()
latitude = db.FloatProperty()
longitude = db.FloatProperty()
image = db.BlobProperty()
class UploadHandler(webapp2.RequestHandler):
def post(self):
obj = biplist.readPlistFromString(self.request.body)
Moment(user = obj['user'],
index = obj['index'],
date = obj['date'],
qx = obj['qx'],
qy = obj['qy'],
qz = obj['qz'],
qw = obj['qw'],
latitude = obj['latitude'],
longitude = obj['longitude'],
image = obj['image']).put()