Develop Reference

Mock DNS server using Twisted

I'm trying to write a Twisted-based mock DNS server to do some tests.
Taking inspiration from this guide I wrote a very simple server that just resolves everything to 127.0.0.1:
from twisted.internet import defer, reactor
from twisted.names import dns, error, server
class MockDNSResolver:
def _doDynamicResponse(self, query):
name = query.name.name
record = dns.Record_A(address=b"127.0.0.1")
answer = dns.RRHeader(name=name, payload=record)
authority = []
additional = []
return [answer], authority, additional
def query(self, query, timeout=None):
print("Incoming query for:", query.name)
if query.type == dns.A:
return defer.succeed(self._doDynamicResponse(query))
else:
return defer.fail(error.DomainError())
if __name__ == "__main__":
clients = [MockDNSResolver()]
factory = server.DNSServerFactory(clients=clients)
protocol = dns.DNSDatagramProtocol(controller=factory)
reactor.listenUDP(10053, protocol)
reactor.listenTCP(10053, factory)
reactor.run()
The above is working just fine with dig and nslookup (from a different terminal):
$ dig -p 10053 #localhost something.example.org A +short
127.0.0.1
$ nslookup something.else.example.org 127.0.0.1 -port=10053
Server: 127.0.0.1
Address: 127.0.0.1#10053
Non-authoritative answer:
Name: something.else.example.org
Address: 127.0.0.1
I'm also getting the corresponding hits on the terminal that's running the server:
Incoming query for: something.example.org
Incoming query for: something.else.example.org
Then, I wrote the following piece of code, based on this section about making requests and this section about installing a custom resolver:
from twisted.internet import reactor
from twisted.names.client import createResolver
from twisted.web.client import Agent
d = Agent(reactor).request(b'GET', b'http://does.not.exist')
reactor.installResolver(createResolver(servers=[('127.0.0.1', 10053)]))
def callback(result):
print('Result:', result)
d.addBoth(callback)
d.addBoth(lambda _: reactor.stop())
reactor.run()
But this fails (and I get no lines in the server terminal). It appears as if the queries are not going to the mock server, but to the system-defined server:
Result: [Failure instance: Traceback: <class 'twisted.internet.error.DNSLookupError'>: DNS lookup failed: no results for hostname lookup: does.not.exist.
/.../venv/lib/python3.6/site-packages/twisted/internet/_resolver.py:137:deliverResults
/.../venv/lib/python3.6/site-packages/twisted/internet/endpoints.py:921:resolutionComplete
/.../venv/lib/python3.6/site-packages/twisted/internet/defer.py:460:callback
/.../venv/lib/python3.6/site-packages/twisted/internet/defer.py:568:_startRunCallbacks
--- <exception caught here> ---
/.../venv/lib/python3.6/site-packages/twisted/internet/defer.py:654:_runCallbacks
/.../venv/lib/python3.6/site-packages/twisted/internet/endpoints.py:975:startConnectionAttempts
]
I'm using:
macOS 10.14.6 Python 3.6.6, Twisted 18.9.0
Linux Mint 19.1, Python 3.6.9, Twisted 19.7.0
I appreciate any help, please let me know if additional information is required.
Thanks!

The solution was:
(Client) swap the order of the lines that install the resolver and create the deferred for the request, as suggested by #Hadus. I thought this didn't matter, since the reactor was not running yet.
(Server) implement lookupAllRecords, reusing the existing _doDynamicResponse method.
# server
from twisted.internet import defer, reactor
from twisted.names import dns, error, server
class MockDNSResolver:
"""
Implements twisted.internet.interfaces.IResolver partially
"""
def _doDynamicResponse(self, name):
print("Resolving name:", name)
record = dns.Record_A(address=b"127.0.0.1")
answer = dns.RRHeader(name=name, payload=record)
return [answer], [], []
def query(self, query, timeout=None):
if query.type == dns.A:
return defer.succeed(self._doDynamicResponse(query.name.name))
return defer.fail(error.DomainError())
def lookupAllRecords(self, name, timeout=None):
return defer.succeed(self._doDynamicResponse(name))
if __name__ == "__main__":
clients = [MockDNSResolver()]
factory = server.DNSServerFactory(clients=clients)
protocol = dns.DNSDatagramProtocol(controller=factory)
reactor.listenUDP(10053, protocol)
reactor.listenTCP(10053, factory)
reactor.run()
# client
from twisted.internet import reactor
from twisted.names.client import createResolver
from twisted.web.client import Agent
reactor.installResolver(createResolver(servers=[('127.0.0.1', 10053)]))
d = Agent(reactor).request(b'GET', b'http://does.not.exist:8000')
def callback(result):
print('Result:', result)
d.addBoth(callback)
d.addBoth(lambda _: reactor.stop())
reactor.run()
$ python client.py
Result: <twisted.web._newclient.Response object at 0x101077f98>
(I'm running a simple web server with python3 -m http.server in another terminal, otherwise I get a reasonable twisted.internet.error.ConnectionRefusedError exception).

This is how you should make a request to it:
from twisted.internet import reactor
from twisted.names import client
resolver = client.createResolver(servers=[('127.0.0.1', 10053)])
d = resolver.getHostByName('does.not.exist')
def callback(result):
print('Result:', result)
d.addBoth(callback)
reactor.callLater(4, reactor.stop)
reactor.run()
And the MockDNSResolvershould be a subclass of twisted.names.common.ResolverBase and then you only have to implement _lookup.
class MockDNSResolver(common.ResolverBase):
def _lookup(self, name, cls, type, timeout):
print("Incoming query for:", name)
record = dns.Record_A(address=b"127.0.0.5")
answer = dns.RRHeader(name=name, payload=record)
return defer.succeed(([answer], [], []))
Full code: https://pastebin.com/MfGahtAV

Opening Webbrowser after Running A Server with Bottle

I am trying to create a program that will start a server with the Bottle module and then open the server url with the webbrowser module. I am having issues opening the webbrowser AFTER starting the bottle server. If i open webbrowser before starting the bottle server the webbrowser successfully opens. However, the webbrowser never opens if I start the bottle server before opening the webbrowser.
import webbrowser
from bottle import route, run, request
def step1():
url = 'http://localhost:8080'
chrome_path = 'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe %s --incognito'
webbrowser.get(chrome_path).open_new(url)
run(host='', port=8080)
step1()
I expect the server to start and then for the webbrowser to open. However, the webbrowser just doesn't open and no errors are thrown.
In this example, opening the webbrowser before running the server works and the connection is successful. However, if I wanted to make a more complex webbrowser function that requires feedback from the server it wouldn't work.
Is there a way to open a webbrowser AFTER starting a bottle server?
Thanks!

Rus this script in console, open you Chrome and point it to http://localhost:8181/hello/Russell
import webbrowser
from bottle import route, run, template
#route('/hello/<name>')
def index(name):
url = 'http://localhost:8181/hello/Russell'
chrome_path = '"C:/Program Files (x86)/Google/Chrome/Application/chrome.exe" %s --incognito'
webbrowser.get(chrome_path).open_new(url)
return template('<b>Hello, <font color=red size=+2>{{name}}</font></b>!', name=name)
run(host='localhost', port=8181)

Async is your friend here.
import gevent
from gevent import monkey, joinall, spawn, signal
monkey.patch_all()
from gevent.pywsgi import WSGIServer
import webbrowser
import bottle
from bottle import route, run, request, get
#get('/')
def home():
return 'Hello World'
def step1():
url = 'http://localhost:8080'
chrome_path = 'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe %s --incognito'
webbrowser.get(chrome_path).open_new(url)
if __name__ == '__main__':
print('Started...')
botapp = bottle.app()
server = WSGIServer(("0.0.0.0", int(8080)), botapp )
def shutdown():
print('Shutting down ...')
server.stop(timeout=60)
exit(signal.SIGTERM)
gevent.signal(signal.SIGTERM, shutdown)
gevent.signal(signal.SIGINT, shutdown) #CTRL C
threads = []
threads.append(spawn(server.serve_forever))
threads.append(spawn(step1))
joinall(threads)

Flask session value not persisting on shared hosting

I developed a Flask application on localhost (running Python 3). It works, but when transferred to my shared hosting account (running Python 2), it doesn't. I fixed all the issues related to Python versions. But the session is not working. Its value does not persists between requests.
I tried to recreate the problem with simpler code (test.py), the commented out part is how session is configured in my application:
import sys
sys.path.insert(0, '/home/user_name/public_html')
from flask import Flask, request, session
from flask.ext.session import Session
from tempfile import mkdtemp
from cs50 import SQL
from constants import *
app = Flask(__name__)
#app.config["SESSION_TYPE"] = "filesystem"
#app.config["SESSION_PERMANENT"] = False
#app.config["SESSION_FILE_DIR"] = mkdtemp()
Session(app)
#app.route('/set/')
def set():
session['key'] = 'value'
return 'ok'
#app.route('/get/')
def get():
return "{}".format(session.get('key'))
If you go to /set/, you will see ok. But on /get/ you will see None.
And here's my CGI file (which I need only for shared hosting, not for localhost):
#!/home/user_name/public_html/cgi-bin/flask/bin/python
import sys
sys.path.insert(0, '/home/user_name/public_html')
# Enable CGI error reporting
import cgitb
cgitb.enable()
import os
from wsgiref.handlers import CGIHandler
app = None
try:
import test
app = test.app
except Exception, e:
print "Content-type: text/html"
print
cgitb.handler()
exit()
#os.environ['SERVER_PORT'] = '80'
#os.environ['REQUEST_METHOD'] = 'POST'
class ScriptNameStripper(object):
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
environ['SCRIPT_NAME'] = ''
return self.app(environ, start_response)
app = ScriptNameStripper(app)
try:
CGIHandler().run(app)
except Exception, e:
print "Content-type: text/html"
print
cgitb.handler()
exit()
In case .htaccess file is needed:
RewriteEngine On
RewriteCond %{REQUEST_FILENAME} !-f
RewriteRule ^(.*)$ /cgi-bin/mas.cgi/$1 [L]
Googling didn't help, and other similar questions on Stackoverflow don't fix it. Any solution/help is welcomed. Thanks!

I am still not sure if session was being written or not (as #Danila Ganchar pointed out), but only commenting out the 3rd configuration line solved the issue.
So the changes made in test.py are:
app.config["SESSION_TYPE"] = "filesystem"
app.config["SESSION_PERMANENT"] = False
#app.config["SESSION_FILE_DIR"] = mkdtemp()
I guess mkdtemp() wasn't working as it was on localhost.

django, wsgi_intercept and behave

Hi I am trying to implement behave test framework in my django python app. However not sure what the problem is and I keep getting connection refused.
Following is the content of features/environment.py:
import os
import django
import urlparse
os.environ['DJANGO_SETTINGS_MODULE'] = 'tilesproj.settings'
django.setup()
def before_all(context):
from django.test.runner import DiscoverRunner
context.runner = DiscoverRunner()
import wsgi_intercept
from django.core.handlers.wsgi import WSGIHandler
host = context.host = 'localhost'
port = context.port = 8000
from django.core.wsgi import get_wsgi_application
from wsgiref.simple_server import make_server
application = get_wsgi_application()
wsgi_intercept.add_wsgi_intercept(host, port, lambda: application)
import mechanize
context.browser = mechanize.Browser()
def browser_url(url):
return urlparse.urljoin('http://%s:%d/' % (host, port), url)
context.browser_url = browser_url
from BeautifulSoup import BeautifulSoup
def parse_soup():
r = context.browser.response()
html = r.read()
r.seek(0)
return BeautifulSoup(html)
context.parse_soup = parse_soup
def before_scenario(context, scenario):
context.runner.setup_test_environment()
context.old_db_config = context.runner.setup_databases()
def after_scenario(context, scenario):
context.runner.teardown_databases(context.old_db_config)
context.runner.teardown_test_environment()
I am trying to figure out how to bootstrap my django app when behave script is run so can test my web app.

You should try the example Django Behave configuration from the website. Essentially you use a fake browser to perform requests which don't actually go through a real website. Your connections are refused because there is no real server to connect to.
So you should get the mechanize browser from wsgi_intercept instead of from mechanize directly:
### Set up the Mechanize browser.
from wsgi_intercept import mechanize_intercept
# MAGIC: All requests made by this monkeypatched browser to the magic
# host and port will be intercepted by wsgi_intercept via a
# fake socket and routed to Django's WSGI interface.
browser = context.browser = mechanize_intercept.Browser()

django unittest without database connection

I'm trying to write a unittest that will check if the correct error message is returned in case the database connection hits exception. I've tried to use connection.creation.destroy_test_db(':memory:') but it didn't work as I expected. I suppose I should either remove the tables or somehow cut the db connection. Is any of those possible?

I found my answer in the presentation Testing and Django by Carl Meyer. Here is how I did it:
from django.db import DatabaseError
from django.test import TestCase
from django.test.client import Client
import mock
class NoDBTest(TestCase):
cursor_wrapper = mock.Mock()
cursor_wrapper.side_effect = DatabaseError
#mock.patch("django.db.backends.util.CursorWrapper", cursor_wrapper)
def test_no_database_connection(self):
response = self.client.post('/signup/', form_data)
self.assertEqual(message, 'An error occured with the DB')

Sounds like this is a job for mocking. For example, if you are using MySQL, you can put a side_effect on connect method, like this:
from django.test import TestCase
from mock import patch
import MySQLdb
class DBTestCase(TestCase):
def test_connection_error(self):
with patch.object(MySQLdb, 'connect') as connect_method:
connect_method.side_effect = Exception("Database Connection Error")
# your assertions here
Hope that helps.

Since dec, 2021 there is the library Django Mockingbird.
With this you can mock the object that would be retrieved from db.
from djangomockingbird import mock_model
#mock_model('myapp.myfile.MyModel')
def test_my_test():
some_test_query = MyModel.objects.filter(bar='bar').filter.(foo='foo').first()
#some more code
#assertions here

I was looking for django's actual http response code in case of a database connection timeout when using pymysql. The following test confirmed it's a 401 Unauthorized when pymysql raises an OperationalError.
from unittest.mock import patch
import pymysql
from django.test import TestCase, Client
class TestDatabaseOutage(TestCase):
client = None
def setUp(self):
self.client = Client()
def test_database_connection_timeout_returns_401(self):
with patch.object(pymysql, 'connect') as connect_method:
message = "Can't connect to MySQL server on 'some_database.example.com' ([Errno 110] Connection timed out)"
connect_method.side_effect = pymysql.OperationalError(2003, message)
response = self.client.get('/')
self.assertEqual(response.status_code, 401)