Develop Reference

Integration test with native SQLAlchemy

I have some trouble with integration test. I'm using python 3.5, SQLAlchemy 1.2.0b3, latest docker image of postgresql. So, I wrote test:
# tests/integration/usecases/test_users_usecase.py
class TestGetUsersUsecase(unittest.TestCase):
def setUp(self):
Base.metadata.reflect(_pg)
Base.metadata.drop_all(_pg)
Base.metadata.create_all(_pg)
self._session = sessionmaker(bind=_pg, autoflush=True, autocommit=False, expire_on_commit=True)
self.session = self._session()
self.session.add(User(id=1, username='user1'))
self.session.commit()
self.pg = PostgresService(session=self.session)
def test_get_user(self):
expected = User(id=1, username='user1')
boilerplates.get_user_usecase(storage_service=self.pg, id=1, expected=expected)
# tests/boilerplates/usecases/user_usecases.py
def get_user_usecase(storage_service, id, expected):
u = GetUser(storage_service=storage_service)
actual = u.apply(id=id)
assert expected == actual
In usecase I did next:
# usecases/users.py
class GetUser(object):
"""
Usecase for getting user from storage service by Id
"""
def __init__(self, storage_service):
self.storage_service = storage_service
def apply(self, id):
user = self.storage_service.get_user_by_id(id=id)
if user is None:
raise UserDoesNotExists('User with id=\'%s\' does not exists' % id)
return user
storage_service.get_user_by_id looks like:
# infrastructure/postgres.py (method of Postgres class)
def get_user_by_id(self, id):
from anna.domain.user import User
return self.session.query(User).filter(User.id == id).one_or_none()
And it does not work in my integration test. But if I add print(actual) in test before assert - all is OK. I thought that my test is bad, I try many variants and all does not works. Also I tried return generator from storage_service.get_user_by_id() and it also does not work. What I did wrong? It works good only if print() was called in test.

Python doctest: how to testing a database insert or delete function?

The question is simple, the answer I dont know...
I'm newbie with testing and I have problems testing class for drive a sql3 database. What is the best way for test a class like this? Test the class or test the init function is not a problem, but the others? the test insert a test row?
import sqlite3
class DataBase:
def __init__(self):
self._database_path = 'data.sql'
self._conn = sqlite3.connect(self._database_path)
self._cursor = self._conn.cursor()
def get(self, sql):
# select
self._cursor.execute(sql)
dataset = []
for row in self._cursor:
dataset.append(row)
return dataset
def post(self, sql):
# insert
self._cursor.execute(sql)
self._conn.commit()
Thank you for all of you, thank you for all your answers!!

You can use the rollback function of the database.
Just replace self._conn.commit() with self._conn.rollback() and you can test the validity of your sql with no effects on the data.
If you need to test a series of actions (i.e: get data->modify data->insert new data->remove some data->get data again) you can remove all the _conn.commit() in your code, run the tests and finally call _conn.rollback().
Example:
import sqlite3
class DataBase:
def __init__(self):
self._database_path = 'data.sql'
self._conn = sqlite3.connect(self._database_path)
self._cursor = self._conn.cursor()
def get(self, sql):
# select
self._cursor.execute(sql)
dataset = []
for row in self._cursor:
dataset.append(row)
return dataset
def post(self, sql):
# insert
self._cursor.execute(sql)
def delete(self, sql):
# delete
self._cursor.execute(sql)
def rollback(self):
self._conn.rollback()
# You do your tests:
db = DataBase()
data = db.get('select name from table')
new_data = ['new' + name for name in data]
db.post('insert into table values {}'.format(','.join('({})'.format(d) for d in new_data)))
db.delete('delete from table where name = \'newMario\'')
check = bool(db.get('select name from table where name = \'newMario\''))
if check:
print('delete ok')
# You make everything as before the test:
db.rollback()

I think the CursorTests in official sqlite3 tests is a good example.
https://github.com/python/cpython/blob/master/Lib/sqlite3/test/dbapi.py#L187
You can write setUp and tearDown methods to set up and rollback the database.
from unittest import TestCase
class TestDataBase(TestCase):
def setUp(self):
self.db = DataBase()
def test_get(self):
pass # your code here
def test_post(self):
pass # your code here

Python - Mocking chained function calls

I have the following statement in one of the methods under unit test.
db_employees = self.db._session.query(Employee).filter(Employee.dept ==
new_employee.dept).all()
I want db_employees to get mock list of employees. I tried to achieve this using:
m = MagickMock()
m.return_value.filter().all().return_value = employees
where employees is a list of employee object. But this did not work. When I try to print the value of any attribute, it has a mock value. This is how the code looks:
class Database(object):
def __init__(self, user=None, passwd=None, db="sqlite:////tmp/emp.db"):
try:
engine = create_engine(db)
except Exception:
raise ValueError("Database '%s' does not exist." % db)
def on_connect(conn, record):
conn.execute('pragma foreign_keys=ON')
if 'sqlite://' in db:
event.listen(engine, 'connect', on_connect)
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
self._session = DBSession()
class TestEmployee(MyEmployee):
def setUp(self):
self.db = emp.database.Database(db=options.connection)
self.db._session._autoflush()
#mock.patch.object(session.Session, 'add')
#mock.patch.object(session.Session, 'query')
def test_update(self, mock_query, mock_add):
employees = [{'id': 1,
'name': 'Pradeep',
'department': 'IT',
'manager': 'John'}]
mock_add.side_effect = self.add_side_effect
mock_query.return_value = self.query_results()
self.update_employees(employees)
def add_side_effect(self, instance, _warn=True):
// Code to mock add
// Values will be stored in a dict which will be used to
// check with expected value.
def query_results(self):
m = MagicMock()
if self.count == 0:
m.return_value.filter.return_value.all.return_value = [employee]
elif:
m.return_value.filter.return_value.all.return_value = [department]
return m
I have query_results as the method under test calls query twice. First the employee table and next the department table.
How do I mock this chained function call?

m = Mock()
m.session.query().filter().all.return_value = employees
https://docs.python.org/3/library/unittest.mock.html

I found a solution to a similar problem where I needed to mock out a nested set of filtering calls.
Given code under test similar to the following:
interesting_cats = (session.query(Cats)
.filter(Cat.fur_type == 'furry')
.filter(Cat.voice == 'meowrific')
.filter(Cat.color == 'orande')
.all())
You can setup mocks like the following:
mock_session_response = MagicMock()
# This is the magic - create a mock loop
mock_session_response.filter.return_value = mock_session_response
# We can exit the loop with a call to 'all'
mock_session_response.all.return_value = provided_cats
mock_session = MagicMock(spec=Session)
mock_session.query.return_value = mock_session_response

You should patch query() method of _session's Database attribute and configure it to give you the right answer. You can do it in a lot of way, but IMHO the cleaner way is to patch DBSession's query static reference. I don't know from witch module you imported DBSession so I'll patch the local reference.
The other aspect is the mock configuration: we will set query's return value that in your case become the object that have filter() method.
class TestEmployee(MyEmployee):
def setUp(self):
self.db = emp.database.Database(db=options.connection)
self.db._session._autoflush()
self.log_add = {}
#mock.patch.object(__name__.'DBSession.add')
#mock.patch.object(__name__.'DBSession.query')
def test_update(self, mock_query, mock_add):
employees = [{'id': 1,
'name': 'Pradeep',
'department': 'IT',
'manager': 'John'}]
mock_add.side_effect = self.add_side_effect
mock_query.return_value = self.query_results()
self.update_employees(employees)
.... your test here
def add_side_effect(self, instance, _warn=True):
# ... storing data
self.log_add[...] = [...]
def query_results(self):
m = MagicMock()
value = "[department]"
if not self.count:
value = "[employee]"
m.filter.return_value.all.return_value = value
return m

What's the correct way to unit test Pyramid views with a SQLAlchemy DBSession?

When writing a Pyramid unit test suite, what is the proper or appropriate way to unit test a view that does a SQLAlchemy call. Example:
def my_view(request):
DBSession.query(DeclarativeBase).all()
Would I use Mock() and patch to override the scope of DBSession to a DummyDB class of the sorts?

You can, and I'll be blogging/speaking/exampling about this soon. This is totally new stuff. Here's a sneak peek:
import mock
from sqlalchemy.sql import ClauseElement
class MockSession(mock.MagicMock):
def __init__(self, *arg, **kw):
kw.setdefault('side_effect', self._side_effect)
super(MockSession, self).__init__(*arg, **kw)
self._lookup = {}
def _side_effect(self, *arg, **kw):
if self._mock_return_value is not mock.sentinel.DEFAULT:
return self._mock_return_value
else:
return self._generate(*arg, **kw)
def _get_key(self, arg, kw):
return tuple(self._hash(a) for a in arg) + \
tuple((k, self._hash(kw[k])) for k in sorted(kw))
def _hash(self, arg):
if isinstance(arg, ClauseElement):
expr = str(arg.compile(compile_kwargs={"literal_binds": True}))
return expr
else:
assert hash(arg)
return arg
def _generate(self, *arg, **kw):
key = self._get_key(arg, kw)
if key in self._lookup:
return self._lookup[key]
else:
self._lookup[key] = ret = MockSession()
return ret
if __name__ == '__main__':
from sqlalchemy import Column, Integer
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Foo(Base):
__tablename__ = 'foo'
id = Column(Integer, primary_key=True)
x = Column(Integer)
y = Column(Integer)
sess = MockSession()
# write out queries as they would in the code, assign return_value
sess.query(Foo.x).filter(Foo.x == 5).first.return_value = 5
sess.query(Foo.x).filter(Foo.x == 2).first.return_value = 2
sess.query(Foo).filter(Foo.x == 2).filter_by(y=5).all.return_value = [Foo(x=2, y=5)]
sess.query(Foo).filter(Foo.x == 9).all.return_value = [Foo(x=9, y=1), Foo(x=9, y=2)]
# those queries are now replayable and will return what was assigned.
print sess.query(Foo.x).filter(Foo.x == 5).first()
print sess.query(Foo.x).filter(Foo.x == 2).first()
print sess.query(Foo).filter(Foo.x == 2).filter_by(y=5).all()
print sess.query(Foo).filter(Foo.x == 9).all()
I've actually assigned this into the global ScopedSession within setup/teardown and it works amazingly:
from myapp.model import MyScopedSession
def setUp(self):
MyScopedSession.registry.set(MockSession())
# set up some queries, we can usually use scoped_session's proxying
MyScopedSession.query(User).filter_by(id=1).first.return_value = User(id=1)
def tearDown(self):
MyScopedSession.remove()
def some_test(self):
# to get at mock methods and accessors, call the scoped_session to get at
# the registry
# ... test some thing
# test a User was added to the session
self.assertEquals(
MyScopedSession().add.mock_calls,
[mock.call(User(name='someuser'))]
)

Concept: Mocking DB python

I'm working with postgresql and I have used MagicMock to test, but I'm not sure that I have understanded the mock's concepts. This is my example code (I have a dbname=test, table=py_test and user = simone):
import psycopg2
import sys
from mock import Mock, patch
import unittest
from mock import MagicMock
from collections import Counter
import doctest
class db(object):
def __init__(self,database, user):
self.con = None
self.database = database
self.user = user
def test_connection(self):
"""Connection DB"""
try:
self.con = psycopg2.connect(database=self.database, user=self.user)
return True
except psycopg2.DatabaseError, e:
print 'Error %s' % e
return False
def test_empty_table(self,table):
"""empty table?"""
try:
cur = self.con.cursor()
cur.execute('SELECT * from ' + table )
ver = cur.fetchone()
return ver
except psycopg2.DatabaseError, e:
print 'Error %s' % e
def test_data_type(self, table, column):
"""data type"""
try:
cur = self.con.cursor()
cur.execute("SELECT data_type from information_schema.columns where table_name = '"+ table + "' and column_name= '"+column+"'")
ver = cur.fetchone()
return ver
except psycopg2.DatabaseError, e:
print 'Error %s' % e
def __del__(self):
if self.con:
self.con.close()
class test_db(unittest.TestCase):
def testing(self):
tdb = db('test','simone')
self.assertTrue(tdb.test_connection(), 1)
self.assertTrue(tdb.test_empty_table('py_test'), 1)
self.assertTrue(tdb.test_data_type('py_test','id'), int)
class test_mock(object):
def __init__(self, db):
self.db = db
def execute(self, nomedb, user, table, field):
self.db(nomedb, user)
self.db.test_connection()
self.db.test_empty_table(table)
self.db.test_data_type(table, field)
if __name__ == "__main__":
c = MagicMock()
d = test_mock(c)
d.execute('test','simone','py_test','id')
method_count = Counter([str(method) for method in c.method_calls])
print c.method_calls
print method_count
print c.mock_calls

Maybe I'll give You some other example of mocking using Mockito package:
import sphinxsearch
import unittest
from mockito import mock, when, unstub, verify
class SearchManagerTest(unittest.TestCase):
def setUp(self):
self.sphinx_client = mock()
when(sphinxsearch).SphinxClient().thenReturn(self.sphinx_client)
def tearDown(self):
unstub()
def test_search_manager(self):
# given
value = {'id': 142564}
expected_result = 'some value returned from SphinxSearch'
# when
search_manager = SearchManager()
result = search_manager.get(value)
# then
verify(self.sphinx_client).SetServer('127.0.0.1', 9312)
verify(self.sphinx_client).SetMatchMode(sphinxsearch.SPH_MATCH_ALL)
verify(self.sphinx_client).SetRankingMode(sphinxsearch.SPH_RANK_WORDCOUNT)
self.assertEqual(result, expected_result)
Main concept is to replace some module (mock) that is tested some where else (it has it's own unittest module) and record some behavior.
Replace module You use with mock:
self.sphinx_client = mock()
and then record on this mock that if You call specific method, this method will return some data - simple values like strings or mocked data if You need to check behavior:
when(sphinxsearch).SphinxClient().thenReturn(self.sphinx_client)
In this case You tell that if You import sphinxsearch module and call SphinxClient() on it, You get mocked object.
Then the main test comes in. You call method or object to test (SearchManager here). It's body is tested with some given values:
self.search_manager = SearchManager()
When section verifies if some actions where made:
verify(self.sphinx_client).SetServer('127.0.0.1', 9312)
verify(self.sphinx_client).SetMatchMode(sphinxsearch.SPH_MATCH_ALL)
verify(self.sphinx_client).SetRankingMode(sphinxsearch.SPH_RANK_WORDCOUNT)
Here - if SetServer was called on self.sphinx_client with parameters '127.0.0.1' and 9312. Two other lines are self explanatory like above.
And here we do normal checks:
self.assertEqual(result, expected_result)