I'm searching a wsgi middleware which I can warp around a wsgi applications and which lets me monitor incoming and outgoing http requests and header fields.
Something like firefox live headers, but for the server side.
The middleware
from wsgiref.util import request_uri
import sys
def logging_middleware(application, stream=sys.stdout):
def _logger(environ, start_response):
stream.write('REQUEST\n')
stream.write('%s %s\n' %(
environ['REQUEST_METHOD'],
request_uri(environ),
))
for name, value in environ.items():
if name.startswith('HTTP_'):
stream.write(' %s: %s\n' %(
name[5:].title().replace('_', '-'),
value,
))
stream.flush()
def _start_response(code, headers):
stream.write('RESPONSE\n')
stream.write('%s\n' % code)
for data in headers:
stream.write(' %s: %s\n' % data)
stream.flush()
start_response(code, headers)
return application(environ, _start_response)
return _logger
The test
def application(environ, start_response):
start_response('200 OK', [
('Content-Type', 'text/html')
])
return ['Hello World']
if __name__ == '__main__':
logger = logging_middleware(application)
from wsgiref.simple_server import make_server
httpd = make_server('', 1234, logger)
httpd.serve_forever()
See also the werkzeug debugger Armin wrote, it's usefull for interactive debugging.
That shouldn't be too hard to write yourself as long as you only need the headers. Try that:
import sys
def log_headers(app, stream=None):
if stream is None:
stream = sys.stdout
def proxy(environ, start_response):
for key, value in environ.iteritems():
if key.startswith('HTTP_'):
stream.write('%s: %s\n' % (key[5:].title().replace('_', '-'), value))
return app(environ, start_response)
return proxy
If you want Apache-style logs, try paste.translogger
But for something more complete, though not in a very handy or stable location (maybe copy it into your source) is wsgifilter.proxyapp.DebugHeaders
And writing one using WebOb:
import webob, sys
class LogHeaders(object):
def __init__(self, app, stream=sys.stderr):
self.app = app
self.stream = stream
def __call__(self, environ, start_response):
req = webob.Request(environ)
resp = req.get_response(self.app)
print >> self.stream, 'Request:\n%s\n\nResponse:\n%s\n\n\n' % (req, resp)
return resp(environ, start_response)
The mod_wsgi documentation provides various tips on debugging which are applicable to any WSGI hosting mechanism and not just mod_wsgi. See:
http://code.google.com/p/modwsgi/wiki/DebuggingTechniques
This includes an example WSGI middleware that captures request and response.
My WebCore project has a bit of middleware that logs the entire WSGI environment (thus Beaker sessions, headers, etc.) for the incoming request, headers for outbound responses, as well as performance information to a MongoDB database. Average overhead is around 4ms.
The module has been removed from the core package, but hasn’t yet been integrated into its own. The current version as of this answer is available in the Git history:
http://github.com/GothAlice/WebCore/blob/cd1d6dcbd081323869968c51a78eceb1a32007d8/web/extras/cprofile.py
Related
Recently, I'm trying to make Dixit online server for school festival from https://github.com/arvoelke/Dixit, But I have encountered a problem. when I reload my web app on my own Pythonanywhere web tab, an error log says,
'Typeerror: call() takes exactly 2 arguments (3 given)'.
I tried to find where 'call()' function is in every Dixit code, but I can't find it. What should I do?
Here's a code related to WSGI, this one is 'server.py'.
class Application(tornado.web.Application):
"""Main application class for holding all state."""
def __init__(self, *args, **kwargs):
"""Initializes the users, games, chat log, and cards."""
self.users = Users()
self.games = []
self.chat_log = ChatLog()
# Specifies where to find all the card images for each set.
self.card_sets = [CardSet(name, find_cards(folder), enabled)
for name, (folder, enabled) in kwargs['card_sets'].iteritems()]
self.admin_password = kwargs['admin_password']
super(Application, self).__init__(*args, **kwargs)
settings = {
'static_path' : os.path.join(os.path.dirname(__file__), 'static'),
'template_path' : os.path.join(os.path.dirname(__file__), 'templates'),
'debug' : True
}
configFilename = (sys.argv + ['config.json'])[1]
settings.update(config.parse(os.path.join(os.path.dirname(__file__),
configFilename)))
application = Application([
(r'/', MainHandler),
(r'/admin', AdminHandler),
(r'/main.js', MainJSHandler),
(r'/main.css', MainCSSHandler),
(r'/setusername', SetUsernameHandler),
(r'/create', CreateHandler),
(r'/getgames', GetGamesHandler),
(r'/getusers', GetUsersHandler),
(r'/game/([0-9]+)/(.+)', GameHandler),
(r'/chat', ChatHandler),
], **settings)
if __name__ == "__main__":
application.listen(settings['port'])
tornado.ioloop.IOLoop.instance().start()
and this one is 'jangheunghsstudentserver_pythonanywhere_com_wsgi.py'.
import tornado.web
import tornado.wsgi
def application(environ, start_response):
if environ.get('PATH_INFO') == '/':
status = '200 OK'
# content = HELLO_WORLD
else:
status = '404 NOT FOUND'
content = 'Page not found.'
response_headers = [('Content-Type', 'text/html'), ('Content-Length', str(len(content)))]
start_response(status, response_headers)
yield content.encode('utf8')
import sys
path = '/home/JangheungHSStudentServer/DixitOnline'
if path not in sys.path:
sys.path.append(path)
from server import application
note. I don't use any python web framework such as django, flask.
After the first Python3 application is created in a new RedHat OpenShift account, wsgi.py looks like this template:
def application(environ, start_response):
ctype = 'text/plain'
if environ['PATH_INFO'] == '/health':
response_body = "1"
elif environ['PATH_INFO'] == '/env':
response_body = ['%s: %s' % (key, value)
for key, value in sorted(environ.items())]
response_body = '\n'.join(response_body)
else:
ctype = 'text/html'
response_body = '<lots of html>'
response_body = response_body.encode('utf-8')
status = '200 OK'
response_headers = [('Content-Type', ctype),
('Content-Length', str(len(response_body)))]
#
start_response(status, response_headers)
return [response_body ]
#
# Below for testing only
#
if __name__ == '__main__':
from wsgiref.simple_server import make_server
httpd = make_server('localhost', 8051, application)
# Wait for a single request, serve it and quit.
httpd.handle_request()
My question is this: During the execution of this code, the following information appears in the log, "[28/Jun/2016 18:35:08] "GET / HTTP/1.1" 200 14". How do I get at that information from within the python program? Have examined the available variables but can not see it. Is there a function call I can make?
Many thanks for your insights.
.... later ....
From reader fat fantasma I see that I should provide additional information.
The user connects to the website using a URL that looks like this:
http://www.website.com/help?
The "help?" appears after the GET in "GET / HTTP/1.1", when the extension to the base URL appears. The GET phrase appears in the system log but I need access to it within wsgi.py's app() function. I do not see how to do that.
If I have a spyne application that inherits from spyne.Application and am serving that through a spyne.WsgiApplication object, how would I add custom HTTP endpoints to the WSGI server such as / or /info?
The basic structure mirrors the one found on spyne.io
class HelloWorldService(ServiceBase):
#srpc(Unicode, Integer, _returns=Iterable(Unicode))
def say_hello(name, times):
for i in range(times):
yield 'Hello, %s' % name
application = Application([HelloWorldService], # <--- spyne.Application
tns='spyne.examples.hello',
in_protocol=Soap11(validator='lxml'),
out_protocol=JsonDocument()
)
if __name__ == '__main__':
from wsgiref.simple_server import make_server
wsgi_app = WsgiApplication(application) # <--- spyne.WsgiApplication
server = make_server('0.0.0.0', 8000, wsgi_app)
server.serve_forever()
In spyne importing from spyne.util.wsgi_wrapper import WsgiMounter (Source) will allow you to call the WsgiMounter function with a single dictionary parameter. The keys of the dictionary represent an extension of the root endpoint, and the values are the WSGI-compatible application.
For example:
def create_web_app(config):
app = Flask(__name__)
#app.route('/about')
def about():
return 'About Page'
return app
wsgi_app = WsgiMounter({
'': SpyneAppWsgi(app),
'www': create_web_app(config)
})
..will configure one server, where the spyne application will be served from the root, and everything from the create_web_app app will be served from /www. (To get to the /about page, you would route to http://localhost:8080/www/about)
In this example create_web_app returns a Flask application.
In a python project with multiple threads my logging works well to write to a logger file. Basically based on Logging, StreamHandler and standard streams
Part of my project is a bottle web server which runs well also. But every bottle call writes a log to the console like this:
192.168.178.20 - - [26/Jun/2015 20:22:17] "GET /edit?addJob HTTP/1.1" 200 48028
How to handle this the same way as with the other code, so the bottle logs go also to the logger file?
If you're rolling your own solution, you should write a simple Bottle plugin that emits log lines to a logging logger. Here's an example that sets up a basic logger, defines the logging plugin, and creates a Bottle app with that plugin installed on all routes.
from bottle import Bottle, request, response
from datetime import datetime
from functools import wraps
import logging
logger = logging.getLogger('myapp')
# set up the logger
logger.setLevel(logging.INFO)
file_handler = logging.FileHandler('myapp.log')
formatter = logging.Formatter('%(msg)s')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
def log_to_logger(fn):
'''
Wrap a Bottle request so that a log line is emitted after it's handled.
(This decorator can be extended to take the desired logger as a param.)
'''
#wraps(fn)
def _log_to_logger(*args, **kwargs):
request_time = datetime.now()
actual_response = fn(*args, **kwargs)
# modify this to log exactly what you need:
logger.info('%s %s %s %s %s' % (request.remote_addr,
request_time,
request.method,
request.url,
response.status))
return actual_response
return _log_to_logger
app = Bottle()
app.install(log_to_logger)
#app.route('/')
def home():
return ['hello, world']
app.run(host='0.0.0.0', port='8080', quiet=True)
Running that code yields what you want:
% python myapp.py &
% curl -v http://localhost:8080/
% tail myapp.log
127.0.0.1 2015-06-27 16:57:09.983249 GET http://localhost:8080/ 200 OK
I'm trying to use Ron's solution with starting the bottle program on a thread:
tWeb = Thread(target=runWeb, args=('192.168.178.16', 5003)).start()
with
def runWeb(aserver, aport):
run(host=aserver, port=aport, debug=True)
but that fails. Any 'print' goes to the file, but not the 'yield' (see above), it goes to the console.
Also changing "debug=True" to "quiet=True" only changes to: there is no output on the console at all.
You running builtin server right ? Then you can make a simple plugin :
from bottle import request, response, route, install, run
from datetime import datetime
def logger(func):
def wrapper(*args, **kwargs):
log = open('log.txt', 'a')
log.write('%s %s %s %s %s \n' % (request.remote_addr, datetime.now().strftime('%H:%M'),
request.method, request.url, response.status))
log.close()
req = func(*args, **kwargs)
return req
return wrapper
install(logger)
#route('/')
def index():
return 'Hello, World'
run(quiet=True)
Or try this one
I'm trying to route certain URLs to a grafted WSGI app and also route sub URLs to a normal cherrypy page handler.
I need the following routes to work. All the other routes should return 404.
/api -> WSGI
/api?wsdl -> WSGI
/api/goodurl -> Page Handler
/api/badurl -> 404 Error
The WSGI app mounted at /api is a legacy SOAP based application. It needs to accept ?wsdl parameters but that is all.
I'm writing a new RESTful api at /api/some_resource.
The issue I'm having is that if the resource doesn't exist, it ends up sending the bad request to the legacy soap application. The final example "/api/badurl" ends up going to the WSGI app.
Is there a way to tell cherrypy to only send the first two routes to the WSGI app?
I wrote up a simple example of my issue:
import cherrypy
globalConf = {
'server.socket_host': '0.0.0.0',
'server.socket_port': 8080,
}
cherrypy.config.update(globalConf)
class HelloApiWsgi(object):
def __call__(self, environ, start_response):
start_response('200 OK', [('Content-Type', 'text/html')])
return ['Hello World from WSGI']
class HelloApi(object):
#cherrypy.expose
def index(self):
return "Hello from api"
cherrypy.tree.graft(HelloApiWsgi(), '/api')
cherrypy.tree.mount(HelloApi(), '/api/hello')
cherrypy.engine.start()
cherrypy.engine.block()
Here's some unit tests:
import unittest
import requests
server = 'localhost:8080'
class TestRestApi(unittest.TestCase):
def testWsgi(self):
r = requests.get('http://%s/api?wsdl'%(server))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.text, 'Hello World from WSGI')
r = requests.get('http://%s/api'%(server))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.text, 'Hello World from WSGI')
def testGoodUrl(self):
r = requests.get('http://%s/api/hello'%(server))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.text, 'Hello from api')
def testBadUrl(self):
r = requests.get('http://%s/api/badurl'%(server))
self.assertEqual(r.status_code, 404)
Outputs:
nosetests test_rest_api.py
F..
======================================================================
FAIL: testBadUrl (webserver.test_rest_api.TestRestApi)
----------------------------------------------------------------------
Traceback (most recent call last):
File line 25, in testBadUrl
self.assertEqual(r.status_code, 404)
AssertionError: 200 != 404
-------------------- >> begin captured stdout << ---------------------
Hello World from WSGI
Preface: I can not avoid mentioning that I wish everyone would ask questions in such a complete form with means to validate the answer :-)
Solutions out of CherryPy's scope:
do URL pre-processing at front-end server, e.g. nginx
create own WSGI middleware, i.e. wrap you legacy WSGI app in another app that will filter URLs
The latter is probably the preferred way to do it, but here's CherryPy's way. Documentation section host a foreign WSGI application in CherryPy says:
You cannot use tools with a foreign WSGI application.
Also you cannot set a custom dispatcher. But you can subclass the application tree.
#!/usr/bin/env python
import cherrypy
class Tree(cherrypy._cptree.Tree):
def __call__(self, environ, start_response):
# do more complex check likewise
if environ['PATH_INFO'].startswith('/api/badurl'):
start_response('404 Not Found', [])
return []
return super(Tree, self).__call__(environ, start_response)
cherrypy.tree = Tree()
globalConf = {
'server.socket_host': '0.0.0.0',
'server.socket_port': 8080,
}
cherrypy.config.update(globalConf)
class HelloApiWsgi:
def __call__(self, environ, start_response):
start_response('200 OK', [('Content-Type', 'text/html')])
return ['Hello World from WSGI']
class HelloApi:
#cherrypy.expose
def index(self):
return "Hello from api"
cherrypy.tree.graft(HelloApiWsgi(), '/api')
cherrypy.tree.mount(HelloApi(), '/api/hello')
if __name__ == '__main__':
cherrypy.engine.signals.subscribe()
cherrypy.engine.start()
cherrypy.engine.block()