Develop Reference

Python Mysql class to call a stored procedure

I am very new. All of my experience is on the DB side so I am lost on the Python side of things. That said I am trying to create a class that I can use to execute stored procedures. I am using Python 3.4.3. I found a mysql class on github and simplified/modified it to make a proc call and it is not working.
mysqlquery.py
import mysql.connector, sys
from collections import OrderedDict
class MysqlPython(object):
__host = None
__user = None
__password = None
__database = None
__procname = None
__inputvals = None
def __init__(self, host='localhost', user='root', password='', database=''):
self.__host = host
self.__user = user
self.__password = password
self.__database = database
## End def __init__
def __open(self):
cnx = mysql.connector.connect(self.__host, self.__user, self.__password, self.__database)
self.__connection = cnx
self.__session = cnx.cursor()
## End def __open
def __close(self):
self.__session.close()
self.__connection.close()
## End def __close
def proc(self,procname,inputvals):
self.__open()
self.__session.callproc(procname, inputvals)
## End for proc
## End class
test.py
from mysqlquery import MysqlPython
connect_mysql = MysqlPython()
result = connect_mysql.proc ('insertlink','1,www.test.com')
I get this error
TypeError: __init__() takes 1 positional argument but 5 were given
Looking at my init, it take 5 args as it should. Not sure why I am getting this. Again, I am very new so it could be a simple problem.
Thanks for any help.
G

mysql.connector.connect() takes named arguments, not positional arguments, and you're missing the names.
cnx = mysql.connector.connect(host=self.__host, user=self.__user, password=self.__password, database=self.__database)

Python - Mocking chained function calls

I have the following statement in one of the methods under unit test.
db_employees = self.db._session.query(Employee).filter(Employee.dept ==
new_employee.dept).all()
I want db_employees to get mock list of employees. I tried to achieve this using:
m = MagickMock()
m.return_value.filter().all().return_value = employees
where employees is a list of employee object. But this did not work. When I try to print the value of any attribute, it has a mock value. This is how the code looks:
class Database(object):
def __init__(self, user=None, passwd=None, db="sqlite:////tmp/emp.db"):
try:
engine = create_engine(db)
except Exception:
raise ValueError("Database '%s' does not exist." % db)
def on_connect(conn, record):
conn.execute('pragma foreign_keys=ON')
if 'sqlite://' in db:
event.listen(engine, 'connect', on_connect)
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
self._session = DBSession()
class TestEmployee(MyEmployee):
def setUp(self):
self.db = emp.database.Database(db=options.connection)
self.db._session._autoflush()
#mock.patch.object(session.Session, 'add')
#mock.patch.object(session.Session, 'query')
def test_update(self, mock_query, mock_add):
employees = [{'id': 1,
'name': 'Pradeep',
'department': 'IT',
'manager': 'John'}]
mock_add.side_effect = self.add_side_effect
mock_query.return_value = self.query_results()
self.update_employees(employees)
def add_side_effect(self, instance, _warn=True):
// Code to mock add
// Values will be stored in a dict which will be used to
// check with expected value.
def query_results(self):
m = MagicMock()
if self.count == 0:
m.return_value.filter.return_value.all.return_value = [employee]
elif:
m.return_value.filter.return_value.all.return_value = [department]
return m
I have query_results as the method under test calls query twice. First the employee table and next the department table.
How do I mock this chained function call?

m = Mock()
m.session.query().filter().all.return_value = employees
https://docs.python.org/3/library/unittest.mock.html

I found a solution to a similar problem where I needed to mock out a nested set of filtering calls.
Given code under test similar to the following:
interesting_cats = (session.query(Cats)
.filter(Cat.fur_type == 'furry')
.filter(Cat.voice == 'meowrific')
.filter(Cat.color == 'orande')
.all())
You can setup mocks like the following:
mock_session_response = MagicMock()
# This is the magic - create a mock loop
mock_session_response.filter.return_value = mock_session_response
# We can exit the loop with a call to 'all'
mock_session_response.all.return_value = provided_cats
mock_session = MagicMock(spec=Session)
mock_session.query.return_value = mock_session_response

You should patch query() method of _session's Database attribute and configure it to give you the right answer. You can do it in a lot of way, but IMHO the cleaner way is to patch DBSession's query static reference. I don't know from witch module you imported DBSession so I'll patch the local reference.
The other aspect is the mock configuration: we will set query's return value that in your case become the object that have filter() method.
class TestEmployee(MyEmployee):
def setUp(self):
self.db = emp.database.Database(db=options.connection)
self.db._session._autoflush()
self.log_add = {}
#mock.patch.object(__name__.'DBSession.add')
#mock.patch.object(__name__.'DBSession.query')
def test_update(self, mock_query, mock_add):
employees = [{'id': 1,
'name': 'Pradeep',
'department': 'IT',
'manager': 'John'}]
mock_add.side_effect = self.add_side_effect
mock_query.return_value = self.query_results()
self.update_employees(employees)
.... your test here
def add_side_effect(self, instance, _warn=True):
# ... storing data
self.log_add[...] = [...]
def query_results(self):
m = MagicMock()
value = "[department]"
if not self.count:
value = "[employee]"
m.filter.return_value.all.return_value = value
return m

Concept: Mocking DB python

I'm working with postgresql and I have used MagicMock to test, but I'm not sure that I have understanded the mock's concepts. This is my example code (I have a dbname=test, table=py_test and user = simone):
import psycopg2
import sys
from mock import Mock, patch
import unittest
from mock import MagicMock
from collections import Counter
import doctest
class db(object):
def __init__(self,database, user):
self.con = None
self.database = database
self.user = user
def test_connection(self):
"""Connection DB"""
try:
self.con = psycopg2.connect(database=self.database, user=self.user)
return True
except psycopg2.DatabaseError, e:
print 'Error %s' % e
return False
def test_empty_table(self,table):
"""empty table?"""
try:
cur = self.con.cursor()
cur.execute('SELECT * from ' + table )
ver = cur.fetchone()
return ver
except psycopg2.DatabaseError, e:
print 'Error %s' % e
def test_data_type(self, table, column):
"""data type"""
try:
cur = self.con.cursor()
cur.execute("SELECT data_type from information_schema.columns where table_name = '"+ table + "' and column_name= '"+column+"'")
ver = cur.fetchone()
return ver
except psycopg2.DatabaseError, e:
print 'Error %s' % e
def __del__(self):
if self.con:
self.con.close()
class test_db(unittest.TestCase):
def testing(self):
tdb = db('test','simone')
self.assertTrue(tdb.test_connection(), 1)
self.assertTrue(tdb.test_empty_table('py_test'), 1)
self.assertTrue(tdb.test_data_type('py_test','id'), int)
class test_mock(object):
def __init__(self, db):
self.db = db
def execute(self, nomedb, user, table, field):
self.db(nomedb, user)
self.db.test_connection()
self.db.test_empty_table(table)
self.db.test_data_type(table, field)
if __name__ == "__main__":
c = MagicMock()
d = test_mock(c)
d.execute('test','simone','py_test','id')
method_count = Counter([str(method) for method in c.method_calls])
print c.method_calls
print method_count
print c.mock_calls

Maybe I'll give You some other example of mocking using Mockito package:
import sphinxsearch
import unittest
from mockito import mock, when, unstub, verify
class SearchManagerTest(unittest.TestCase):
def setUp(self):
self.sphinx_client = mock()
when(sphinxsearch).SphinxClient().thenReturn(self.sphinx_client)
def tearDown(self):
unstub()
def test_search_manager(self):
# given
value = {'id': 142564}
expected_result = 'some value returned from SphinxSearch'
# when
search_manager = SearchManager()
result = search_manager.get(value)
# then
verify(self.sphinx_client).SetServer('127.0.0.1', 9312)
verify(self.sphinx_client).SetMatchMode(sphinxsearch.SPH_MATCH_ALL)
verify(self.sphinx_client).SetRankingMode(sphinxsearch.SPH_RANK_WORDCOUNT)
self.assertEqual(result, expected_result)
Main concept is to replace some module (mock) that is tested some where else (it has it's own unittest module) and record some behavior.
Replace module You use with mock:
self.sphinx_client = mock()
and then record on this mock that if You call specific method, this method will return some data - simple values like strings or mocked data if You need to check behavior:
when(sphinxsearch).SphinxClient().thenReturn(self.sphinx_client)
In this case You tell that if You import sphinxsearch module and call SphinxClient() on it, You get mocked object.
Then the main test comes in. You call method or object to test (SearchManager here). It's body is tested with some given values:
self.search_manager = SearchManager()
When section verifies if some actions where made:
verify(self.sphinx_client).SetServer('127.0.0.1', 9312)
verify(self.sphinx_client).SetMatchMode(sphinxsearch.SPH_MATCH_ALL)
verify(self.sphinx_client).SetRankingMode(sphinxsearch.SPH_RANK_WORDCOUNT)
Here - if SetServer was called on self.sphinx_client with parameters '127.0.0.1' and 9312. Two other lines are self explanatory like above.
And here we do normal checks:
self.assertEqual(result, expected_result)

Returning useable values from ConfigParse

Using the code described bellow, i can sucessfully retrieve the properties stored into the file.cfg, but how can i use the output into others variables?
from ConfigParser import SafeConfigParser
class Main:
def get_properties(self, section, *variables):
cfgFile = 'c:\file.cfg'
parser = SafeConfigParser()
parser.read(cfgFile)
properties= variables
return {
variable : parser.get(section,variable) for variable in properties
}
def run_me(self):
config_vars= self.get_properties('database','host','dbname')
print config_vars
op=Main()
op.run_me()
Im still learning Python, but i'm not sure what i need to do to set the output into individual variables:
current output:
{'host': 'localhost', 'dbname': 'sample'}
what i would like to have:
db_host = localhost
db_name = sample

def run_me(self):
config_vars= self.get_properties('database','host','dbname')
for key, value in config_vars.items():
print key, "=", value
You recieved dict-object config_vars, so your can using config variables as values of the dict:
>>> print config_vars["dbname"]
sample
>>> print config_vars["host"]
localhost
Read more about python dictionaries in documentation.

I would suggest this approach:
import ConfigParser
import inspect
class DBConfig:
def __init__(self):
self.host = 'localhost'
self.dbname = None
def foo(self): pass
class ConfigProvider:
def __init__(self, cfg):
self.cfg = cfg
def update(self, section, cfg):
for name, value in inspect.getmembers(cfg):
if name[0:2] == '__' or inspect.ismethod(value):
continue
#print name
if self.cfg.has_option(section, name):
setattr(cfg, name, self.cfg.get(section, name))
class Main:
def __init__(self, dbConfig):
self.dbConfig = dbConfig
def run_me(self):
print('Connecting to %s:%s...' % (self.dbConfig.host, self.dbConfig.dbname))
config = ConfigParser.RawConfigParser()
config.add_section('Demo')
#config.set('Demo', 'host', 'domain.com')
config.set('Demo', 'dbname', 'sample')
configProvider = ConfigProvider(config)
dbConfig = DBConfig()
configProvider.update('Demo', dbConfig)
main = Main(dbConfig)
main.run_me()
The idea is that you collect all important properties in a class (where you can also set the defaults).
The method ConfigProvider.update() will then overwrite those with the values from the config (if they exist).
This allows you to access properties with the simple obj.name syntax.
gist

python memcache search string

any idea how i can search a string in memcache,i have a list of domains which are loaded onto memcache
what i would like to do is search a string on those domains...
[root#server python]# cat memtest.py
#!/usr/bin/env python
import os
import sys
import memcache
domain = "http://www.yahoo.com/images.txt"
s = memcache.Client(["127.0.0.1:11211"])
def addData():
proc = open("domains.txt","r")
for i in proc.readlines():
d = i.rstrip("\n");
s.set(d,1)
def getData():
name = s.get("yahoo.com")
print name
name = s.get("xaa.com")
print name
##dummy code, just an example
if domain in s.get(domain):
print found
def main():
addData()
getData()
if __name__ == "__main__":
main()

def addData():
proc = open("domains.txt","r")
for i in proc.readlines():
d = i.rstrip("\n");
s.set(d,d)
def getData():
name = s.get("yahoo.com")
print name
name = s.get("xaa.com")
print name
##dummy code, just an example
if s.get("yahoo.com"):
if domain.find(s.get("yahoo.com")) > 0:
print found

To retrieve a value from memcache, you have to know the exact key under which the value was stored.
s.get("yahoo.com")
will work only if
s.set("yahoo.com", <some value>)
was executed before.
looks like you don't know the exact keys because they're being retrieved from a file.
You can try using a regular expression to get the base domain from the file, and make sure "yahoo.com" is used as a key.