Develop Reference

int treated as `self' in python3

I want to build a simple server that dispatches requests depending on their path. So I wrote the following code:
from http.server import HTTPServer, BaseHTTPRequestHandler
import json
import socket
class Handler:
"""处理对应资源的请求"""
def __init__(self, dispatcher):
self.dispatcher = dispatcher
def handle(self):
"""根据dispatcher解析出来的path，调用对应的方法"""
command = 'do_' + self.dispatcher.command
print(command)
if not hasattr(self, command):
return;
method = getattr(self, command)
method(self)
def do_GET(self):
response = {
'message': 'message'
}
dispatcher.protocol_version = 'HTTP/1.1'
dispatcher.send_response(200, 'OK')
dispatcher.send_header('Content-type', 'text/plain')
dispatcher.wfile.write(bytes(json.dumps(response), 'utf-8'))
class dispatcher(BaseHTTPRequestHandler):
"""
根据url的不同分发请求
"""
def handle_one_request(self):
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(HTTPStatus.REQUEST_URI_TOO_LONG)
return
if not self.raw_requestline:
self.close_connection = True
return
if not self.parse_request():
# An error code has been sent, just exit
return
print(self.command)
print(self.path)
if self.path.startswith('/wrong'):
Handler(self).handle()
self.wfile.flush() #actually send the response if not already done.
except socket.timeout as e:
#a read or a write timed out. Discard this connection
self.log_error("Request timed out: %r", e)
self.close_connection = True
return
if __name__ == '__main__':
server = HTTPServer(('', 8080), dispatcher)
server.serve_forever()
The dispatcher will parse the incoming request and get the path so that it can decide which handler to call(though there is only one here for now).
In the handler class, it will call corresponding method based on the http method. In the do_GET method, it will call some methods in the dispatcher and that's where things go wrong.
When I ran this program and execute curl http://localhost:8080/wrong, I had the following exception:
GET
/wrong
do_GET
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 50072)
Traceback (most recent call last):
File "/usr/lib/python3.5/socketserver.py", line 313, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python3.5/socketserver.py", line 341, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python3.5/socketserver.py", line 354, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python3.5/socketserver.py", line 681, in __init__
self.handle()
File "/usr/lib/python3.5/http/server.py", line 422, in handle
self.handle_one_request()
File "hello.py", line 51, in handle_one_request
Handler(self).handle()
File "hello.py", line 18, in handle
method()
File "hello.py", line 25, in do_GET
dispatcher.send_response(200, 'OK')
File "/usr/lib/python3.5/http/server.py", line 487, in send_response
self.log_request(code)
AttributeError: 'int' object has no attribute 'log_request'
----------------------------------------
log_request is defined as follows in the super class of dispatcher:
def log_request(self, code='-', size='-'):
# blablabla
and it is called in dispatcher.send_response.
I guess the problem is that the 200 is treated as self by the interpreter.
If my guess is right, why is this happening?
If not, what causes this exception?
I know this question is still in the grammar level, but I'd appreciate it if someone can help me.

How to mock stompest send, and connect methods

I am writing a log handler for ActiveMQ. I have one class, Messenger, to publish a message to ActiveMQ. Also, I added class Handler to handle this and get_logger to get this logger.
import json
import logging
from stompest.config import StompConfig
from stompest.sync import Stomp
class Messanger(object):
def __init__(self, uri):
self.cfg = StompConfig(uri)
def publish(self, queue, data):
data = json.dumps(data)
client = Stomp(self.cfg)
client.connect()
try:
client.send(queue, data)
except Exception, exc:
print "Error: ", exc
client.disconnect()
class Handler(logging.Handler):
def __init__(self, amq_uri, out_queue):
logging.Handler.__init__(self)
self.queue = queue
self.uri = uri
def emit(self, record):
msg = self.format(record)
messanger = Messanger(self.uri)
messanger.send(self.queue, msg)
def get_logger(uri, queue):
logger = logging.getLogger('testlogger')
logger.addHandler(Handler(uri, queue))
return logger
I have written a UT for this.
from unittest import TestCase
from mock import patch, Mock
from my_package import get_logger
class TestHandler(TestCase):
#patch('stompest.sync.client.Stomp')
def test_activemq_handler(self, mock_stomp):
URI = "tcp://localhost:61613"
QUEUE = "/queue/logger_queue"
mock_stomp.connect = Mock(return_value=1)
mock_stomp.send = Mock(return_value=2)
data = "This is test logging"
LOG = get_logger(URI, QUEUE)
LOG.error(data)
But still it goes to the original send method and tries to connect to the server.
Traceback (most recent call last):
File ".my_package/venv/lib/python2.7/site-packages/mock/mock.py", line 1305, in patched
return func(*args, **keywargs)
File "./my_package/tests/test_activemq_handler.py", line 23, in test_activemq_handler
LOG.error(data)
File "/System/Library/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 1191, in error
self._log(ERROR, msg, args, **kwargs)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 1284, in _log
self.handle(record)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 1294, in handle
self.callHandlers(record)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 1334, in callHandlers
hdlr.handle(record)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 757, in handle
self.emit(record)
File "./my_package/__init__.py", line 28, in emit
messanger.send(self.queue, msg)
File "./my_package/clients/stomp_messanger.py", line 41, in send
client.connect()
File "./my_package/venv/lib/python2.7/site-packages/stompest/sync/client.py", line 85, in connect
for (broker, connectDelay) in self._failover:
File "./my_package/venv/lib/python2.7/site-packages/stompest/protocol/failover.py", line 48, in __iter__
yield broker, self._delay()
File "./my_package/venv/lib/python2.7/site-packages/stompest/protocol/failover.py", line 66, in _delay
raise StompConnectTimeout('Reconnect timeout: %d attempts' % self._maxReconnectAttempts)
StompConnectTimeout: Reconnect timeout: 0 attempts
How can I mock this?

The basic principle is that you patch where an object is looked up, which is not necessarily the same place as where it is defined.
You are adding the patch to the wrong place. The Messenger class module imports Stomp from stompest.sync before the tests run, and that's why your patching has no effect. You should patch the reference to Stomp in your module.
from unittest import TestCase
from mock import patch, Mock
from my_package import get_logger
class TestHandler(TestCase):
#patch('path.to.your.Messenger_class_module.Stomp') #path to your module
def test_activemq_handler(self, mock_stomp):
URI = "tcp://localhost:61613"
QUEUE = "/queue/logger_queue"
...
This will help if you need further clarification on why this is happening.

Test working differently when launched alone than with unittest

I want to test my application
I have these two files (Not my real application but this presents the same problem).
File 1: testServer.py
import SocketServer
import SimpleHTTPServer
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
f = self.wfile
self.send_response(200)
self.end_headers()
f.write("ok")
def runServer(port):
Handler = ServerHandler
httpd = SocketServer.TCPServer(("", port), Handler)
httpd.serve_forever()
if __name__ == "__main__":
runServer(8000)
File 2: test_testServer.py
import testServer
from multiprocessing import Process
import httplib
import unittest
class Test(unittest.TestCase):
#classmethod
def setUpClass(cls):
cls.serverProcess = Process(target = testServer.runServer, args = (8000,))
cls.serverProcess.daemon = True
cls.serverProcess.start()
#classmethod
def tearDownClass(cls):
cls.serverProcess.terminate()
def test_Server(self):
conn = httplib.HTTPConnection("localhost", 8000)
conn.request("GET", "")
response = conn.getresponse().read()
self.assertTrue(response, "ok")
if __name__ == "__main__":
unittest.main()
So my goal is to start the server I coded in a process and then test its replies to different requests.
When I launch the test alone, it works perfectly fine:
C:\Users\r.bunel\workspace\Sandbox>python test_testServer.py
127.0.0.1 - - [23/Dec/2013 09:23:50] "GET / HTTP/1.1" 200 -
----------------------------------------------------------------------
Ran 1 test in 1.009s
OK
But if I start it as a part of my all test suite, by using unittest's discover, it doesn't work:
C:\Users\r.bunel\workspace\Sandbox>python -m unittest discover
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Python27\lib\multiprocessing\forking.py", line 380, in main
prepare(preparation_data)
File "C:\Python27\lib\multiprocessing\forking.py", line 488, in prepare
assert main_name not in sys.modules, main_name
AssertionError: __main__
E
======================================================================
ERROR: test_Server (test_testServer.Test)
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\r.bunel\workspace\Sandbox\test_testServer.py", line 22, in test_Server
conn.request("GET", "")
File "C:\Python27\lib\httplib.py", line 973, in request
self._send_request(method, url, body, headers)
File "C:\Python27\lib\httplib.py", line 1007, in _send_request
self.endheaders(body)
File "C:\Python27\lib\httplib.py", line 969, in endheaders
self._send_output(message_body)
File "C:\Python27\lib\httplib.py", line 829, in _send_output
self.send(msg)
File "C:\Python27\lib\httplib.py", line 791, in send
self.connect()
File "C:\Python27\lib\httplib.py", line 772, in connect
self.timeout, self.source_address)
File "C:\Python27\lib\socket.py", line 571, in create_connection
raise err
error: [Errno 10061] Aucune connexion nÆa pu Ûtre Útablie car lÆordinateur cible lÆa expressÚment refusÚe
----------------------------------------------------------------------
Ran 1 test in 2.009s
FAILED (errors=1)
So, my question is whether or not there is a reason for unittest to behave this way and what are my options to make these tests work inside unittest?

Perhaps the unittest is trying to connect to the server before it has had time to start up properly in the second thread? Try adding time.sleep(1) after cls.serverProcess.start() in setupClass(cls) ?
#classmethod
def setUpClass(cls):
cls.serverProcess = Process(target = testServer.runServer, args = (8000,))
cls.serverProcess.daemon = True
cls.serverProcess.start()
time.sleep(1)

Ok, after realising with the help of #Alo Sarv that the problem lied in the multiprocessing module, I found a solution.
Here it is in case someone encounters the same problem:
file 1:testServer.py
import SocketServer
import SimpleHTTPServer
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
f = self.wfile
self.send_response(200)
self.end_headers()
f.write("ok")
def runServer(port):
Handler = ServerHandler
httpd = SocketServer.TCPServer(("", port), Handler)
httpd.serve_forever()
if __name__ == "__main__":
runServer(8000)
file 2: test_testServer.py
from testServer import ServerHandler
import SocketServer
from threading import Thread
import httplib
import unittest
class serverThread(Thread):
def __init__(self, port):
self.Handler = ServerHandler
self.httpd = SocketServer.TCPServer(("", port), self.Handler)
Thread.__init__(self)
def run(self):
self.httpd.serve_forever()
class Test(unittest.TestCase):
#classmethod
def setUpClass(cls):
cls.serverThread = serverThread(8000)
cls.serverThread.daemon = True
cls.serverThread.start()
#classmethod
def tearDownClass(cls):
cls.serverThread.httpd.__shutdown_request = True
def test_Server(self):
conn = httplib.HTTPConnection("localhost", 8000)
conn.request("GET", "")
response = conn.getresponse().read()
self.assertTrue(response, "ok")
if __name__ == "__main__":
unittest.main()
In short, the solution was to replace the use of multiprocessing by the use of threads. Then, adapting the server in order to be able to stop the thread properly (because thread can not be killed as easily as Process)

HTTP server hangs while accepting packets

I have written a simple http server to handle POST requests:
class MyHandler( BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST( self ):
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
postvars = {}
try:
if ctype == 'application/x-www-form-urlencoded':
length = int(self.headers.getheader('content-length'))
postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1)
self.send_response( 200 )
self.send_header( "Content-type", "text")
self.send_header( "Content-length", str(len(body)) )
self.end_headers()
self.wfile.write(body)
except:
print "Error"
def httpd(handler_class=MyHandler, server_address = ('2.3.4.5', 80)):
try:
print "Server started"
srvr = BaseHTTPServer.HTTPServer(server_address, handler_class)
srvr.serve_forever() # serve_forever
except KeyboardInterrupt:
server.socket.close()
if __name__ == "__main__":
httpd( )
The server runs fine but sometimes it just hangs. When I press CTRL+C it gives the following error and then continues receiving data:
Exception happened during processing of request from ('1.1.1.2', 50928)
Traceback (most recent call last):
File "/usr/lib/python2.6/SocketServer.py", line 281, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python2.6/SocketServer.py", line 307, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python2.6/SocketServer.py", line 320, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python2.6/SocketServer.py", line 615, in __init__
self.handle()
File "/usr/lib/python2.6/BaseHTTPServer.py", line 329, in handle
self.handle_one_request()
File "/usr/lib/python2.6/BaseHTTPServer.py", line 312, in handle_one_request
self.raw_requestline = self.rfile.readline()
File "/usr/lib/python2.6/socket.py", line 406, in readline
data = self._sock.recv(self._rbufsize)
KeyboardInterrupt
Can someone tell me how to correct this? I can't make sense of the errors.

I completely rewrote my solution in order to fix two disadvantages:
It can treat timeout even if client has opened only a connection but did not start communicate.
If the client opens a connection, a new server process is forked. A slow client can not block other.
It is based on your code as most as possible. It is now a complete independent demo. If you open a browser on http://localhost:8000/ and write 'simulate error' to form and press submit, it simulates runtime error.
from http.server import HTTPServer, BaseHTTPRequestHandler
from socketserver import ForkingMixIn
from urllib.parse import parse_qs
class MyHandler(BaseHTTPRequestHandler):
def do_POST(self):
ctype, pdict = self.headers.get_params(header='content-type')[0]
postvars = {}
try:
if ctype == 'application/x-www-form-urlencoded':
length = int(self.headers['content-length'])
postvars = parse_qs(self.rfile.read(length), keep_blank_values=1
encoding='utf-8')
assert postvars.get('foo', '') != ['bar'] # can simulate error
body = 'Something\n'.encode('ascii')
self.send_response(200)
self.send_header("Content-type", "text")
self.send_header("Content-length", str(len(body)))
self.end_headers()
self.wfile.write(body)
except Exception:
self.send_error(500)
raise
def do_GET(self):
# demo for testing POST by web browser - without valid html
body = ('Something\n<form method="post" action="http://%s:%d/">\n'
'<input name="foo" type="text"><input type="submit"></form>\n'
% self.server.server_address).encode('ascii')
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-length", str(len(body)))
self.end_headers()
self.wfile.write(body)
class ForkingHTTPServer(ForkingMixIn, HTTPServer):
def finish_request(self, request, client_address):
request.settimeout(30)
# "super" can not be used because BaseServer is not created from object
HTTPServer.finish_request(self, request, client_address)
def httpd(handler_class=MyHandler, server_address=('localhost', 8000)):
try:
print("Server started")
srvr = ForkingHTTPServer(server_address, handler_class)
srvr.serve_forever() # serve_forever
except KeyboardInterrupt:
srvr.socket.close()
if __name__ == "__main__":
httpd()
Forking can be disabled for e.g. debugging purposes by removing class SocketServer.ForkingMixIn from code.
EDIT Updated for Python 3, but a warning from Python 3 docs should be noted:
Warning: http.server is not recommended for production. It only implements basic security checks.

Selenium in Python

I've been using urllib2 to access webpages, but it doesn't support javascript, so I took a look at Selenium, but I'm quite confused even having read its docs.
I downloaded Selenium IDE add-on for firefox and I tried some simple things.
from selenium import selenium
import unittest, time, re
class test(unittest.TestCase):
def setUp(self):
self.verificationErrors = []
self.selenium = selenium("localhost", 4444, "*chrome", "http://www.wikipedia.org/")
self.selenium.start()
def test_test(self):
sel = self.selenium
sel.open("/")
sel.type("searchInput", "pacific ocean")
sel.click("go")
sel.wait_for_page_to_load("30000")
def tearDown(self):
self.selenium.stop()
self.assertEqual([], self.verificationErrors)
if __name__ == "__main__":
unittest.main()
I just access wikipedia.org and type pacific ocean in the search field, but when I try to compile it, it gives me a lot of errors.

If running the script results in a [Errno 111] Connection refused error such as this:
% test.py
E
======================================================================
ERROR: test_test (__main__.test)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/unutbu/pybin/test.py", line 11, in setUp
self.selenium.start()
File "/data1/unutbu/pybin/selenium.py", line 189, in start
result = self.get_string("getNewBrowserSession", [self.browserStartCommand, self.browserURL, self.extensionJs])
File "/data1/unutbu/pybin/selenium.py", line 219, in get_string
result = self.do_command(verb, args)
File "/data1/unutbu/pybin/selenium.py", line 207, in do_command
conn.request("POST", "/selenium-server/driver/", body, headers)
File "/usr/lib/python2.6/httplib.py", line 898, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python2.6/httplib.py", line 935, in _send_request
self.endheaders()
File "/usr/lib/python2.6/httplib.py", line 892, in endheaders
self._send_output()
File "/usr/lib/python2.6/httplib.py", line 764, in _send_output
self.send(msg)
File "/usr/lib/python2.6/httplib.py", line 723, in send
self.connect()
File "/usr/lib/python2.6/httplib.py", line 704, in connect
self.timeout)
File "/usr/lib/python2.6/socket.py", line 514, in create_connection
raise error, msg
error: [Errno 111] Connection refused
----------------------------------------------------------------------
Ran 1 test in 0.063s
FAILED (errors=1)
then the solution is most likely that you need get the selenium server running first.
In the download for SeleniumRC you will find a file called selenium-server.jar (as of a few months ago, that file was located at SeleniumRC/selenium-server-1.0.3/selenium-server.jar).
On Linux, you could run the selenium server in the background with the command
java -jar /path/to/selenium-server.jar 2>/dev/null 1>&2 &
You will find more complete instructions on how to set up the server here.

I would suggest you to use a webdriver, you can find it here: http://code.google.com/p/selenium/downloads/list. If you want to write tests as a coder (and not with the use of your mouse), that thing would work better then the RC version you're trying to use, at least because it would not ask you for an SeleniumRC Jar Instance. You would simply have a binary of a browser or use those ones that are already installed on your system, for example, Firefox.

I faced with this issue in my project and found that problem was in few webdriver.get calls with very small time interval between them. My fix was not to put delay, just remove unneeded calls and error disappears.
Hope, it can helps for somebody.