while I'm working on the front end, somethimes the server breaks and give me this error.
all i was doing is modifing the front end and refresh the page to see the results.
not sure what is causing it but it happens every few minutes, i don't think flask is suppose to break with only 1 connection/user is connected to it and bearly using it, on the top of that I'm only hosting it locally.
I'm running a flask app with socket io , flask-sqlalchemy and Postgresql as my database
if __name__ == "__main__":
socketio.run(app,debug=True )
lunching the app using
note: sqlalchemy drops first , so that could be useful.
python app.py
Exception in thread Thread-425:
Traceback (most recent call last):
File "<hidden_python_directory>\lib\threading.py", line 954, in _bootstrap_inner
self.run()
File "<hidden_python_directory>\lib\threading.py", line 892, in run
self._target(*self._args, **self._kwargs)
File "<hidden_python_directory>\lib\site-packages\socketio\server.py", line 680, in _handle_event_internal
r = server._trigger_event(data[0], namespace, sid, *data[1:])
File "<hidden_python_directory>\lib\site-packages\socketio\server.py", line 704, in _trigger_event
return self.handlers[namespace][event](*args)
File "<hidden_python_directory>\lib\site-packages\flask_socketio\__init__.py", line 283, in _handler
return self._handle_event(handler, message, namespace, sid,
File "<hidden_python_directory>\lib\site-packages\flask_socketio\__init__.py", line 751, in _handle_event
ret = handler(*args)
File "<hidden_applocation>\app.py", line 271, in connect_user
if current_user.is_authenticated:
File "<hidden_python_directory>\lib\site-packages\werkzeug\local.py", line 347, in __getattr__
return getattr(self._get_current_object(), name)
File "<hidden_python_directory>\lib\site-packages\werkzeug\local.py", line 306, in _get_current_object
return self.__local()
File "<hidden_python_directory>\lib\site-packages\flask_login\utils.py", line 26, in <lambda>
current_user = LocalProxy(lambda: _get_user())
File "<hidden_python_directory>\lib\site-packages\flask_login\utils.py", line 346, in _get_user
current_app.login_manager._load_user()
File "<hidden_python_directory>\lib\site-packages\flask_login\login_manager.py", line 318, in _load_user
user = self._user_callback(user_id)
File "<hidden_applocation>\app.py", line 30, in load_user
return users.query.get(int(id))
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\query.py", line 1018, in get
return self._get_impl(ident, loading.load_on_pk_identity)
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\query.py", line 1135, in _get_impl
return db_load_fn(self, primary_key_identity)
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\loading.py", line 286, in load_on_pk_identity
return q.one()
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\query.py", line 3490, in one
ret = self.one_or_none()
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\query.py", line 3459, in one_or_none
ret = list(self)
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\query.py", line 3535, in __iter__
return self._execute_and_instances(context)
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\query.py", line 3556, in _execute_and_instances
conn = self._get_bind_args(
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\query.py", line 3571, in _get_bind_args
return fn(
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\query.py", line 3550, in _connection_from_session
conn = self.session.connection(**kw)
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\session.py", line 1142, in connection
return self._connection_for_bind(
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\session.py", line 1150, in _connection_for_bind
return self.transaction._connection_for_bind(
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\orm\session.py", line 433, in _connection_for_bind
conn = bind._contextual_connect()
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\engine\base.py", line 2302, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\engine\base.py", line 2336, in _wrap_pool_connect
return fn()
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\pool\base.py", line 364, in connect
return _ConnectionFairy._checkout(self)
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\pool\base.py", line 778, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\pool\base.py", line 495, in checkout
rec = pool._do_get()
File "<hidden_python_directory>\lib\site-packages\sqlalchemy\pool\impl.py", line 128, in _do_get
raise exc.TimeoutError(
sqlalchemy.exc.TimeoutError: QueuePool limit of size 5 overflow 10 reached, connection timed out, timeout 30 (Background on this error at: http://sqlalche.me/e/13/3o7r)
sqlalchemy.exc.TimeoutError: QueuePool limit of size 5 overflow 10 reached, connection timed out, timeout 30 (Background on this error at: http://sqlalche.me/e/13/3o7r)
then after couple of seconds later it attempts to refresh the page i'm already on but another error keeps showing up.
Traceback (most recent call last):
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 573, in handle_one_response
result = self.application(self.environ, start_response)
File "<hidden_python_directory>\lib\site-packages\flask\app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\flask_socketio\__init__.py", line 45, in __call__
return super(_SocketIOMiddleware, self).__call__(environ,
File "<hidden_python_directory>\lib\site-packages\engineio\middleware.py", line 60, in __call__
return self.engineio_app.handle_request(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\socketio\server.py", line 571, in handle_request
return self.eio.handle_request(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\engineio\server.py", line 392, in handle_request
packets = socket.handle_get_request(
File "<hidden_python_directory>\lib\site-packages\engineio\socket.py", line 103, in handle_get_request
return getattr(self, '_upgrade_' + transport)(environ,
File "<hidden_python_directory>\lib\site-packages\engineio\socket.py", line 158, in _upgrade_websocket
return ws(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\engineio\async_drivers\eventlet.py", line 20, in __call__
return super(WebSocketWSGI, self).__call__(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 130, in __call__
self.handler(ws)
File "<hidden_python_directory>\lib\site-packages\engineio\socket.py", line 184, in _websocket_handler
pkt = ws.wait()
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 788, in wait
for i in self.iterator:
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 643, in _iter_frames
message = self._recv_frame(message=fragmented_message)
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 669, in _recv_frame
header = recv(2)
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 578, in _get_bytes
d = self.socket.recv(numbytes - len(data))
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 365, in recv
return self._recv_loop(self.fd.recv, b'', bufsize, flags)
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 359, in _recv_loop
self._read_trampoline()
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 327, in _read_trampoline
self._trampoline(
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 208, in _trampoline
return trampoline(fd, read=read, write=write, timeout=timeout,
File "<hidden_python_directory>\lib\site-packages\eventlet\hubs\__init__.py", line 159, in trampoline
return hub.switch()
File "<hidden_python_directory>\lib\site-packages\eventlet\hubs\hub.py", line 313, in switch
return self.greenlet.switch()
socket.timeout: timed out
127.0.0.1 - - [12/Mar/2021 13:36:55] "GET /socket.io/?EIO=4&transport=websocket&sid=Wu2Ou1x4dEJnfaynAAAo HTTP/1.1" 500 0 30.117126
Traceback (most recent call last):
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 573, in handle_one_response
result = self.application(self.environ, start_response)
File "<hidden_python_directory>\lib\site-packages\flask\app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\flask_socketio\__init__.py", line 45, in __call__
return super(_SocketIOMiddleware, self).__call__(environ,
File "<hidden_python_directory>\lib\site-packages\engineio\middleware.py", line 60, in __call__
return self.engineio_app.handle_request(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\socketio\server.py", line 571, in handle_request
return self.eio.handle_request(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\engineio\server.py", line 392, in handle_request
packets = socket.handle_get_request(
File "<hidden_python_directory>\lib\site-packages\engineio\socket.py", line 103, in handle_get_request
return getattr(self, '_upgrade_' + transport)(environ,
File "<hidden_python_directory>\lib\site-packages\engineio\socket.py", line 158, in _upgrade_websocket
return ws(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\engineio\async_drivers\eventlet.py", line 20, in __call__
return super(WebSocketWSGI, self).__call__(environ, start_response)
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 130, in __call__
self.handler(ws)
File "<hidden_python_directory>\lib\site-packages\engineio\socket.py", line 184, in _websocket_handler
pkt = ws.wait()
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 788, in wait
for i in self.iterator:
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 643, in _iter_frames
message = self._recv_frame(message=fragmented_message)
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 669, in _recv_frame
header = recv(2)
File "<hidden_python_directory>\lib\site-packages\eventlet\websocket.py", line 578, in _get_bytes
d = self.socket.recv(numbytes - len(data))
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 365, in recv
return self._recv_loop(self.fd.recv, b'', bufsize, flags)
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 359, in _recv_loop
self._read_trampoline()
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 327, in _read_trampoline
self._trampoline(
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 208, in _trampoline
return trampoline(fd, read=read, write=write, timeout=timeout,
File "<hidden_python_directory>\lib\site-packages\eventlet\hubs\__init__.py", line 159, in trampoline
return hub.switch()
File "<hidden_python_directory>\lib\site-packages\eventlet\hubs\hub.py", line 313, in switch
return self.greenlet.switch()
socket.timeout: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 357, in __init__
self.handle()
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 390, in handle
self.handle_one_request()
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 466, in handle_one_request
self.handle_one_response()
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 616, in handle_one_response
write(err_body)
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 538, in write
wfile.flush()
File "<hidden_python_directory>\lib\socket.py", line 722, in write
return self._sock.send(b)
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 396, in send
return self._send_loop(self.fd.send, data, flags)
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 383, in _send_loop
return send_method(data, *args)
BrokenPipeError: [WinError 10058] A request to send or receive data was disallowed because the socket had already been shut down in that direction with a previous shutdown call
BrokenPipeError: [WinError 10058] A request to send or receive data was disallowed because the socket had already been shut down in that direction with a previous shutdown call
app.py configrations
# for app
app = Flask(__name__)
app.secret_key= #//secret
app.config['WTF_CSRF_SECRET_KEY'] = #//secret
# for database
_string = #//secret
app.config['SQLALCHEMY_DATABASE_URI']= _string
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
# for login manager
login = LoginManager(app)
login.init_app(app)
( Update ) I've added this part based on waynetech's answer
#I've added that part
def get_db():
if 'db' not in g:
g.db = db.session.connect_to_database()
return g.db
#app.teardown_appcontext
def teardown_db(exception):
db = g.pop('db', None)
if db is not None:
db.close()
#app.teardown_appcontext
def shutdown_session(exception=None):
print(f'\n now shutting down request. \n')
db.session.remove()
how I add objects to the database
class1_obj = class1(name=current_user.name ,age=12)
db.session.add(class1_obj)
db.session.commit()
(update) now I get this error as well
now shutting down request.
Traceback (most recent call last):
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 598, in handle_one_response
write(b''.join(towrite))
File "<hidden_python_directory>\lib\site-packages\eventlet\wsgi.py", line 537, in write
wfile.writelines(towrite)
File "<hidden_python_directory>\lib\socket.py", line 722, in write
return self._sock.send(b)
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 396, in send
return self._send_loop(self.fd.send, data, flags)
File "<hidden_python_directory>\lib\site-packages\eventlet\greenio\base.py", line 383, in _send_loop
return send_method(data, *args)
ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine
(update) now I tried removeing the Flask_Login as #waynetech suggested in the comments, it might be doing something to the g element of flask.
so I did disable it but ofcourse I had to remove many other things like all Ajax and most of the SocketIO routs as well.
Then I spamed refresh as fast as i could and the server was stable and never broke!.
So it could be casued by the Flask_Login, but I dont know how to go about that.
(Update) everytime I refresh I get the following messages on the power shell console
note :
now shutting down request is showing up due to the print command I have under
#app.teardown_appcontext
def shutdown_session(exception=None):
====
127.0.0.1 - - [17/Mar/2021 18:20:54] "GET /static/webfonts/fa-regular-400.woff2 HTTP/1.1" 200 13894 0.006981
127.0.0.1 - - [17/Mar/2021 11:20:54] "GET /socket.io/?EIO=4&transport=polling&t=NX0_N3U HTTP/1.1" 200 277 0.000997
127.0.0.1 - - [17/Mar/2021 11:20:54] "GET /socket.io/?EIO=4&transport=polling&t=NX0_N3V HTTP/1.1" 200 277 0.001995
127.0.0.1 - - [17/Mar/2021 11:20:54] "POST /socket.io/?EIO=4&transport=polling&t=NX0_N4P&sid=jxmFQEApPI_OD9OZAAAE HTTP/1.1" 200 219 0.000000
(13996) accepted ('127.0.0.1', 58165)
127.0.0.1 - - [17/Mar/2021 11:20:54] "GET /socket.io/?EIO=4&transport=polling&t=NX0_N4R&sid=jxmFQEApPI_OD9OZAAAE HTTP/1.1" 200 181 0.001001
(13996) accepted ('127.0.0.1', 58166)
127.0.0.1 - - [17/Mar/2021 11:20:54] "POST /socket.io/?EIO=4&transport=polling&t=NX0_N8d&sid=CF1mhCp5AmERFJGZAAAF HTTP/1.1" 200 219 0.000000
127.0.0.1 - - [17/Mar/2021 11:20:54] "GET /socket.io/?EIO=4&transport=polling&t=NX0_N8e&sid=CF1mhCp5AmERFJGZAAAF HTTP/1.1" 200 181 0.000000
127.0.0.1 - - [17/Mar/2021 11:20:55] "GET /socket.io/?EIO=4&transport=polling&t=NX0_N9N&sid=jxmFQEApPI_OD9OZAAAE HTTP/1.1" 200 181 0.000000
127.0.0.1 - - [17/Mar/2021 11:20:55] "GET /socket.io/?EIO=4&transport=polling&t=NX0_N9S&sid=CF1mhCp5AmERFJGZAAAF HTTP/1.1" 200 181 0.000000
now shutting down request
127.0.0.1 - - [17/Mar/2021 18:20:55] "GET /favicon.ico HTTP/1.1" 404 2083 0.013965
now shutting down request
====
Note: On every refresh, there are few querys and few socketIO's being send to the connected each user
Queries :
1 - pull all new messages.
2 - pull all new notifications.
socketIO:
1- pull the some connection data and update the user profile.
(Update) I FINALLY KNOW WHAT IS CAUSING THIS OVERFLOW
I have a socketIO route that checks if user is authonticated (( using flask_login )) then query data from the database (( using Flask_SQLAlchemy )) and send it to the user
as long as either flask_login or SQLAlchemy involved in one way or another with socketIO , i get that error above
here is an example of what its like.
#socketio.on('custom_emit')
def custom_emit(data):
if current_user.is_authenticated:
_id = current_user.get_id()
x = users.query.filter_by(id=_id).first()
socketio.emit('update_me',{'my_id':x })
the Most effective answer for this
Not to query or use flask_login under SocektIO Decorators.
The server is at it's best without linking SocketIO with flask_login or querying commands
until someone explains the logic behind it, that's the best answer I've right now
few more steps that might help:
adding the following lines of code.
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
socketio = SocketIO(app, manage_session=True)
i am using pysaml2 library in python. which has a method defined as below:
def _store_request(self, saml_msg):
key = sha1(saml_msg["SAMLRequest"]).hexdigest()
IDP.ticket[key] = saml_msg
return key
where, IDP.ticket = {} is an application wide simple key value mapping(in a raw WSGI application).here IDP.ticket persists between multiple requests . some context below.
The route that is called is SSO().redirect,
Initially self.user is False so key is being set by
self._store_request(saml_msg)
Then user is returned with a form with the key set in form so that the request can be identified. On form submit SSO().redirect is called again & value set in IDP.ticket during previous request was obtained.
i want to port this method to a flask application but when using flask with Flask-Cache when i do cache.set(key,saml_msg) i get error:
TypeError: can't pickle CompiledFFI objects
I gues its because saml_msg contains a key
req_info': <saml2.request.AuthnRequest object at 0x1072302d0>
How can i store state between multiple requests in flask other than using FLask-Cache?
EDIT:.
this question is similar to this question but I have already tried that solution .I need to save an object not a simple data type so when i save it using Flask-Cache i get `can't pickle CompiledFFI objects(maybe this is something with AuthnRequest object ,it can't be pickled)
Using flask.session to store key between requests, it gives TypeError: Object of type AuthnRequest is not JSON serializable.
full stack trace.
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2463, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2449, in wsgi_app
response = self.handle_exception(e)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1866, in handle_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1952, in full_dispatch_request
return self.finalize_request(rv)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1969, in finalize_request
response = self.process_response(response)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2268, in process_response
self.session_interface.save_session(self, ctx.session, response)
File "/usr/local/lib/python3.7/site-packages/flask/sessions.py", line 378, in save_session
val = self.get_signing_serializer(app).dumps(dict(session))
File "/usr/local/lib/python3.7/site-packages/itsdangerous/serializer.py", line 166, in dumps
payload = want_bytes(self.dump_payload(obj))
File "/usr/local/lib/python3.7/site-packages/itsdangerous/url_safe.py", line 42, in dump_payload
json = super(URLSafeSerializerMixin, self).dump_payload(obj)
File "/usr/local/lib/python3.7/site-packages/itsdangerous/serializer.py", line 133, in dump_payload
return want_bytes(self.serializer.dumps(obj, **self.serializer_kwargs))
File "/usr/local/lib/python3.7/site-packages/flask/json/tag.py", line 305, in dumps
return dumps(self.tag(value), separators=(",", ":"))
File "/usr/local/lib/python3.7/site-packages/flask/json/__init__.py", line 211, in dumps
rv = _json.dumps(obj, **kwargs)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/__init__.py", line 238, in dumps
**kw).encode(obj)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/usr/local/lib/python3.7/site-packages/flask/json/__init__.py", line 100, in default
return _json.JSONEncoder.default(self, o)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type AuthnRequest is not JSON serializable
I want to send bytes in a string to Celery's task using Redis as broker, but I'm receiving error shown below:
[2017-06-17 21:27:13,826] ERROR in app: Exception on /endpoint_method [POST]
Traceback (most recent call last):
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/flask/app.py", line 1982, in wsgi_app
response = self.full_dispatch_request()
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/flask/app.py", line 1614, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/flask/app.py", line 1517, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/flask/app.py", line 1612, in full_dispatch_request
rv = self.dispatch_request()
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/flask/app.py", line 1598, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/Users/developer/my_project/application.py", line 23, in endpoint_method
task = my_task.execute.delay(request.data)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/celery/app/task.py", line 412, in delay
return self.apply_async(args, kwargs)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/celery/app/task.py", line 535, in apply_async
**options
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/celery/app/base.py", line 737, in send_task
amqp.send_task_message(P, name, message, **options)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/celery/app/amqp.py", line 558, in send_task_message
**properties
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/kombu/messaging.py", line 169, in publish
compression, headers)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/kombu/messaging.py", line 252, in _prepare
body) = dumps(body, serializer=serializer)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/kombu/serialization.py", line 221, in dumps
payload = encoder(data)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/contextlib.py", line 35, in __exit__
self.gen.throw(type, value, traceback)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/kombu/serialization.py", line 54, in _reraise_errors
reraise(wrapper, wrapper(exc), sys.exc_info()[2])
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/kombu/serialization.py", line 50, in _reraise_errors
yield
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/kombu/serialization.py", line 221, in dumps
payload = encoder(data)
File "/Users/developer/my_project/venv/lib/python2.7/site-packages/kombu/utils/json.py", line 72, in dumps
**dict(default_kwargs, **kwargs))
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/__init__.py", line 250, in dumps
sort_keys=sort_keys, **kw).encode(obj)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/encoder.py", line 207, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/encoder.py", line 270, in iterencode
return _iterencode(o, 0)
EncodeError: 'utf8' codec can't decode byte 0x80 in position 5: invalid start byte
Shortened (mostly it's up to 1 mln characters string) example data which I want to send to:
'RIFF$\x80\r\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x02\x00D\xac\x00\x00\x10\xb1\x02\x00\x04\x00\x10\x00data\x00\x80\r\x00z\xefz\xef\xd5\xec\xd5\xec\xc1\xee\xc1\xee\xe6\xf3\xe6\xf3\xb4\xfa\xb4\xfa\x92\x03\x92\x03\xab\x0e\xab\x0e\xf9\x18\xf9\x18D\x1eD\x1ev\x1dv\x1d#\x19#\x19\x86\x13\x86\x13w\nw\n\xf0\xfd\xf0\xfd\xf9\xf2\xf9\xf2\xd0\xed\xd0\xedK\xedK\xed{\xed{\xed%\xee%\xeeP\xf3P\xf3\xeb\xfc\xeb\xfc!\x05!\x05\xa4\x08\xa4\x08\xe5\t\xe5\t\x84\x0b\x84\x0bF\x0bF\x0b\xfb\x04\xfb\x040\xfa0\xfa\x86\xf1\x86\xf1T\xeeT'
Part of the code responsible for calling Celery's task:
task = my_task.execute.delay(request.data)
This is how tasks/my_task.py file looks like:
from __future__ import absolute_import, unicode_literals
from my_module.celery import app
#app.task
def execute(request_data):
return <some operations on request_data>
and celery.py:
from __future__ import absolute_import, unicode_literals
from celery import Celery
app = Celery(
'my_module',
broker='redis://127.0.0.1:6379/1',
backend='redis://127.0.0.1:6379/2',
include=[
'my_module.tasks.my_task'
]
)
app.conf.update(
result_expires=3600,
)
if __name__ == '__main__':
app.start()
Maybe is there some clever way to encode that string before sending it to Redis and decode it in while running the task?
Finally, I came up with a solution using binascii methods binascii.b2a_base64(data) and binascii.a2b_base64(string). This is what I had to change to make this working:
task = my_task.execute.delay(binascii.b2a_base64(request.data))
and in tasks/my_task.py I had to convert data back to be able to make proper operations on it:
binascii.a2b_base64(request_data)
I am using python-raven as my sentry client. Unfortunately, when I got an error 500 on my server (caught by raven) my request times out:
E 17:05:57.743 2015-05-13 500 0 B 59.99s /mobile_backend/device/2EBE5887-03EA-458A-B4C4-E57253966231/
137.194.58.229 - - [13/May/2015:08:05:57 -0700] "GET /mobile_backend/device/2EBE5887-03EA-458A-B4C4-E57253966231/ HTTP/1.1" 500 0 - "Apache-HttpClient/4.2.6 (java 1.5)" "v1.blueberry.xxx-staging.appspot.com" ms=59989 cpu_ms=0 cpm_usd=0.000281 exit_code=119 instance=00c61b117cce7ef8bc8fb86f8aa5315fc96a4d app_engine_release=1.9.20
E 17:05:57.741 Thread running after request. Creation traceback:
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/runtime/runtime.py", line 152, in HandleRequest
error)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/runtime/wsgi.py", line 329, in HandleRequest
return WsgiRequest(environ, handler_name, url, post_data, error).Handle()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/runtime/wsgi.py", line 267, in Handle
result = handler(dict(self._environ), self._StartResponse)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/django/core/handlers/wsgi.py", line 187, in __call__
response = self.get_response(request)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/django/core/handlers/base.py", line 198, in get_response
signals.got_request_exception.send(sender=self.__class__, request=request)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/django/dispatch/dispatcher.py", line 198, in send
response = receiver(signal=self, sender=sender, **named)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/contrib/django/models.py", line 177, in sentry_exception_handler
client.captureException(exc_info=sys.exc_info(), request=request)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/base.py", line 679, in captureException
'raven.events.Exception', exc_info=exc_info, **kwargs)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/contrib/django/client.py", line 157, in capture
result = super(DjangoClient, self).capture(event_type, **kwargs)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/base.py", line 526, in capture
self.send(**data)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/base.py", line 613, in send
return self.send_encoded(message, auth_header=auth_header)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/base.py", line 640, in send_encoded
self.send_remote(url=url, data=message, headers=headers)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/base.py", line 598, in send_remote
failed_send)
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/transport/threaded.py", line 169, in async_send
self.get_worker().queue(
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/transport/threaded.py", line 157, in get_worker
self._worker = AsyncWorker()
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/transport/threaded.py", line 36, in __init__
self.start()
File "/base/data/home/apps/s~xxx-staging/blueberry:v1.384273266847676695/lib/raven/transport/threaded.py", line 113, in start
self._thread.start()
File "/base/data/home/runtimes/python27/python27_dist/lib/python2.7/threading.py", line 505, in start
_start_new_thread(self.__bootstrap, ())
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/runtime/runtime.py", line 82, in StartNewThread
return base_start_new_thread(Run, ())
W 17:05:57.743 Threads started by this request continued executing past the hard deadline.
The 500 error is correctly reported on Sentry. How can I configure raven to not make timeout my request?
Switching from:
SENTRY_DSN = 'http://public:secret#example.com/1'
to:
SENTRY_DSN = 'sync+http://public:secret#example.com/1'
did the trick !
This worked for me as well:
from raven import Client
from raven.transport.threaded import HTTPTransport
client = Client(
'https://public_key:secret_key#sentry.io/project_id',
transport=HTTPTransport
)
try:
1 / 0
except ZeroDivisionError:
client.captureException()
I followed the tutorial for Werkzeug "Shortly" here
And I get this error message after submitting a valid url.
Traceback (most recent call last)
File "/home/sadik/NLM/shortly/shortly.py", line 87, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/lib/python2.7/dist-packages/Werkzeug-0.9.4-py2.7.egg/werkzeug/wsgi.py", line 579, in __call__
return self.app(environ, start_response)
File "/home/sadik/NLM/shortly/shortly.py", line 83, in wsgi_app
response = self.dispatch_request(request)
File "/home/sadik/NLM/shortly/shortly.py", line 33, in dispatch_request
return getattr(self, 'on_' + endpoint)(request, **values)
File "/home/sadik/NLM/shortly/shortly.py", line 45, in on_new_url
short_id = self.insert_url(url)
File "/home/sadik/NLM/shortly/shortly.py", line 72, in insert_url
short_id = self.redis.get('reverse-url:' + url)
File "/usr/local/lib/python2.7/dist-packages/redis-2.9.1-py2.7.egg/redis/client.py", line 705, in get
return self.execute_command('GET', name)
File "/usr/local/lib/python2.7/dist-packages/redis-2.9.1-py2.7.egg/redis/client.py", line 464, in execute_command
connection.send_command(*args)
File "/usr/local/lib/python2.7/dist-packages/redis-2.9.1-py2.7.egg/redis/connection.py", line 334, in send_command
self.send_packed_command(self.pack_command(*args))
File "/usr/local/lib/python2.7/dist-packages/redis-2.9.1-py2.7.egg/redis/connection.py", line 316, in send_packed_command
self.connect()
File "/usr/local/lib/python2.7/dist-packages/redis-2.9.1-py2.7.egg/redis/connection.py", line 253, in connect
raise ConnectionError(self._error_message(e))
ConnectionError: Error 111 connecting localhost:6379. Connection refused.
The error message indicates that there is something wrong with localhost:6379
The relevant part of code is here:
def create_app(redis_host='localhost', redis_port=6379, with_static=True):
app = Shortly({
'redis_host': redis_host,
'redis_port': redis_port
})
if with_static:
app.wsgi_app = SharedDataMiddleware(app.wsgi_app, {
'/static': os.path.join(os.path.dirname(__file__), 'static')
})
return app
if __name__ == '__main__':
from werkzeug.serving import run_simple
app = create_app()
run_simple('127.0.0.1', 5000, app, use_debugger=True, use_reloader=True)
That means of course that the server is running on localhost:5000. So why is there another port number in the create_app function? That confuses me a bit.
I'm not familiar with shortly or werkzeug but it looks like you're missing the redis server, install one using your favourite package manager and try again.