I need to test a code that has a global variable which is populated by a function that returns a value after making a connection to external server. For the pytest, I need to figure out a way so that the function does not get called.
I've tried patching both the global variable and the function, but neither worked.
Here's the python code - my_module.py:
import datetime
# %%
def server_function_1 ():
try:
return_val = "Assume the external server function returns a string"
except Exception as e:
print("Failed")
print(e)
raise e
else:
return return_val
finally:
# assume when called from the testing environment, the server connection will raise a exception
raise Exception("Cannot connect to server")
global_result_of_server_func = server_function_1()
# %%
def get_current_datetime_str():
return datetime.datetime.now().strftime('%Y%m%d.%H%M%S.%f')
# %%
def some_function():
return global_result_of_server_func, get_current_datetime_str()
Here's the pytest file - test_my_module.py:
# %%
import pytest
from unittest import mock
import datetime
import logging
import sys
# %%
import my_module
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
frozen_time = datetime.datetime(year=2022, month=6, day=1, hour=1, minute=0, second=0, microsecond=0)
mock_server_str = "Mock Server String"
# %%
class TestMyModule () :
def test_handler_1 (self):
with mock.patch("my_module.get_current_datetime_str", return_value=frozen_time.strftime('%Y%m%d.%H%M%S.%f')), \
mock.patch("my_module.global_result_of_server_func", new=mock_server_str):
test_server_val, test_frozen_time = my_module.some_function()
assert test_frozen_time == frozen_time.strftime('%Y%m%d.%H%M%S.%f')
assert test_server_val == mock_server_str
def test_handler_2 (self):
with mock.patch("my_module.get_current_datetime_str", return_value=frozen_time.strftime('%Y%m%d.%H%M%S.%f')), \
mock.patch("my_module.server_function_1", return_value=mock_server_str):
test_server_val, test_frozen_time = my_module.some_function()
assert test_frozen_time == frozen_time.strftime('%Y%m%d.%H%M%S.%f')
assert test_server_val == mock_server_str
What I am trying to achieve is that variable global_result_of_server_func gets the mock value, and the function server_function_1 doesn't get called and tries to make a connection to the server.
Thanks.
Delaying the import like the suggestion in this question didn't seem to make any difference.
Related
I'd like to print 't' and 's' on localhost. But it doesn't work with timer function. Here is what I tried.
import requests
import pandas as pd
import time
from bottle import route, run
def timer(n):
while True:
url2 = 'https://api.bittrex.com/api/v1.1/public/getticker?market=USDT-BTC'
res = requests.get(url2).json()
BTC = pd.json_normalize(res['result'])
t = time.ctime()
s = BTC.to_html(index=False)
time.sleep(n)
timer(5)
#route('/')
def index():
return t, s
run(host='localhost', port=8080)
Without more details around what exactly isn't working, it's tough to give a comprehensive answer. But one thing that surely needs to change is where you define t and s. As #Pygirl states in her comment, you'll need to define those globally:
import requests
import pandas as pd
import time
from bottle import route, run
t = ''
s = ''
def timer(n):
while True:
url2 = 'https://api.bittrex.com/api/v1.1/public/getticker?market=USDT-BTC'
res = requests.get(url2).json()
BTC = pd.json_normalize(res['result'])
# Must declare t and s as globals since you're changing their referents.
global t
global s
t = time.ctime()
s = BTC.to_html(index=False)
time.sleep(n)
timer(5)
#route('/')
def index():
return '{}: {}'.format(t, s)
run(host='localhost', port=8080)
An alternative to defining them globally is to create a top-level dict with two items in it (t and s), and simply modify those items as needed; no need for global in that case.
Question is as in the heading, how can I mock select.select by example to test my thread run function.
the test function fails with
ready = select.select([self.sock], [], [], 5)
TypeError: fileno() returned a non-integer
and the type print gives
type 'builtin_function_or_method'
so clearly select.select is NOT mocked in the thread's scope while in the test it is...(assert isinstance)
import select
import threading
RECEIVE_BYTES = 256
class Bar(threading.Thread):
def __init__(self, sock):
threading.Thread.__init__(self)
self.sock = sock
def run(self):
print type(select.select)
ready = select.select([self.sock],[],[],5)
if ready[0]:
print self.sock.recv(RECEIVE_BYTES)
the test is as follows in another module
def test_run(self):
with patch("select.select"):
select.select.return_value = [True]
mock_sock = MagicMock()
foo = Bar(mock_sock)
assert isinstance(select.select, MagicMock)
foo.start()
tests are run via nose
The short answer is to call foo.join() to wait for the thread to finish before leaving the with patch(...) block. The error was caused by removing the patch before the thread had finished.
By the way, it's much easier for people to help you if you post an example that can be run. Your example was incomplete and had syntax errors.
Here's the fixed up test. I added the loop to make it easier to reproduce the error.
import select
from mock import patch, MagicMock
from time import sleep
from scratch import Bar
IS_FIXED = True
def test_run():
for _ in range(20):
with patch("select.select"):
select.select.return_value = [True]
mock_sock = MagicMock()
foo = Bar(mock_sock)
assert isinstance(select.select, MagicMock)
foo.start()
if IS_FIXED:
foo.join()
sleep(0.1)
And here's the Bar class with some syntax fixes.
import select
import threading
RECEIVE_BYTES = 256
class Bar(threading.Thread):
def __init__(self, sock):
threading.Thread.__init__(self)
self.sock = sock
def run(self):
print type(select.select)
ready = select.select([self.sock],[],[],5)
if ready[0]:
print self.sock.recv(RECEIVE_BYTES)
My questions are interlaced within my code:
#!/usr/bin/python
import threading
import logging, logging.handlers
import hpclib
import json
import time
from datetime import datetime
from features import *
import sys
if len(sys.argv) != 3:
print "Please provide the correct inputs"
print "Usage: rest_test.py <controllerip> <counter>"
sys.exit()
controller = sys.argv[1]
counter = int(sys.argv[2])
class FuncThread(threading.Thread):
def __init__(self, target, *args):
self._target = target
self._args = args
threading.Thread.__init__(self)
def run(self):
self._target(*self._args)
def datapath_thread(ipaddress, testlogfile,count):
#initialize logging system
testlogger = logging.getLogger("testlogger")
testlogger.setLevel(logging.DEBUG)
file = open(testlogfile,'w')
file.close()
# This handler writes everything to a file.
h1 = logging.FileHandler(testlogfile)
f = logging.Formatter("%(levelname)s %(asctime)s %(funcName)s %(lineno)d %(message)s")
h1.setFormatter(f)
h1.setLevel(logging.DEBUG)
testlogger.addHandler(h1)
mylib = hpclib.hpclib(ipaddress)
success_count = 0
failure_count = 0
for i in range(count):
t1=datetime.now()
try:
(code, val) = datapaths.listDatapaths(mylib)
I want to pass this function datapaths.listDatapaths(mylib) as a argument from a thread below, something like (code,val)=functionname
if code == 200:
success_count +=1
else:
testlogger.debug("Return Code other than 200 received with code = %d, value = %s"%(code,val))
failure_count +=1
except:
failure_count += 1
testlogger.debug ("Unexpected error: %s"%sys.exc_info()[0])
continue
t2=datetime.now()
diff=t2-t1
testlogger.debug('RETURN code: %d. Time taken in sec = %s,Iteration = %d, Success = %d, Failure = %d'%(code,diff.seconds,i+1,success_count,failure_count))
time.sleep(1)
testlogger.removeHandler(h1)
# Passing ipadress of controller and log file name
t1 = FuncThread(datapath_thread, controller, "datapaths.log",counter)
Here I would like to pass function name as one of the argument,something like t1 = FuncThread(datapath_thread, controller, datapaths.listDatapaths(mylib),"datapaths.log",counter)
t1.start()
t1.join()
I have many functions to call like this,so want a easy way to call all the functions from one single function using many threads
Firstly, FuncThread is not very useful - FuncThread(func, *args) can be spelt Thread(target=lambda: func(*args)) or Thread(target=func, args=args).
You're pretty close - instead of passing in the result of calling the function, pass in the function itself
def datapath_thread(ipaddress, test_func, testlogfile, count):
# ...
for i in range(count):
# ...
try:
(code, val) = test_func(mylib)
#...
thread = Thread(target=datapath_thread, args=(
controller,
datapaths.listDatapaths,
"datapaths.log",
counter
))
I am having an issue with accessing the queue object within my class in multiple functions, the following code might show you what i am trying to do;
import MySQLdb
import socket, sys
from struct import *
import threading
import sched, time
from datetime import datetime
from copy import deepcopy
import Queue
IP = {}
class QP:
def __init__(self):
self.jobs = Queue.Queue()
# this function runs every 10 seconds
# and processes any queued data.
def processQueue(self):
print(self.jobs.qsize())
time.sleep(5)
if self.jobs.empty():
print("No items in queue")
else:
IP_TEMP = {}
IP_TEMP = self.jobs.get()
IP_TEMP_QUEUE = {}
IP_TEMP_QUEUE = IP_TEMP
try:
cnx = #connect to database
cursor = cnx.cursor()
for k, v in IP_TEMP.iteritems():
#there is some code here, but its not the issue
try:
cursor.execute(add_packet, data_packet)
cnx.commit()
print("Task Done")
except:
print("Query failed, skipping")
break
except:
self.queueJobs(IP_TEMP_QUEUE)
IP = {}
self.jobs.task_done()
self.processQueue()
# this function is called by other modules to add data
def queueJobs(self, data):
self.jobs.put(data)
print(self.jobs.qsize())
return True
This is how i call the queueJobs method in the other modules:
self.process.queueJobs(IP_TEMP_QUEUE)
Now, the main issue is that the processQueue() function always returns that there is no jobs in the queue, even though they have been added below in the queueJobs() function.
Any help would be great!
I tried to get support on this but I am TOTALLY confused.
Here's my code:
from twisted.internet import reactor
from twisted.web.client import getPage
from twisted.web.error import Error
from twisted.internet.defer import DeferredList
from sys import argv
class GrabPage:
def __init__(self, page):
self.page = page
def start(self, *args):
if args == ():
# We apparently don't need authentication for this
d1 = getPage(self.page)
else:
if len(args) == 2:
# We have our login information
d1 = getPage(self.page, headers={"Authorization": " ".join(args)})
else:
raise Exception('Missing parameters')
d1.addCallback(self.pageCallback)
dl = DeferredList([d1])
d1.addErrback(self.errorHandler)
dl.addCallback(self.listCallback)
def errorHandler(self,result):
# Bad thingy!
pass
def pageCallback(self, result):
return result
def listCallback(self, result):
print result
a = GrabPage('http://www.google.com')
data = a.start() # Not the HTML
I wish to get the HTML out which is given to pageCallback when start() is called. This has been a pita for me. Ty! And sorry for my sucky coding.
You're missing the basics of how Twisted operates. It all revolves around the reactor, which you're never even running. Think of the reactor like this:
(source: krondo.com)
Until you start the reactor, by setting up deferreds all you're doing is chaining them with no events from which to fire.
I recommend you give the Twisted Intro by Dave Peticolas a read. It's quick and it really gives you all the missing information that the Twisted documentation doesn't.
Anyways, here is the most basic usage example of getPage as possible:
from twisted.web.client import getPage
from twisted.internet import reactor
url = 'http://aol.com'
def print_and_stop(output):
print output
if reactor.running:
reactor.stop()
if __name__ == '__main__':
print 'fetching', url
d = getPage(url)
d.addCallback(print_and_stop)
reactor.run()
Since getPage returns a deferred, I'm adding the callback print_and_stop to the deferred chain. After that, I start the reactor. The reactor fires getPage, which then fires print_and_stop which prints the data from aol.com and then stops the reactor.
Edit to show a working example of OP's code:
class GrabPage:
def __init__(self, page):
self.page = page
########### I added this:
self.data = None
def start(self, *args):
if args == ():
# We apparently don't need authentication for this
d1 = getPage(self.page)
else:
if len(args) == 2:
# We have our login information
d1 = getPage(self.page, headers={"Authorization": " ".join(args)})
else:
raise Exception('Missing parameters')
d1.addCallback(self.pageCallback)
dl = DeferredList([d1])
d1.addErrback(self.errorHandler)
dl.addCallback(self.listCallback)
def errorHandler(self,result):
# Bad thingy!
pass
def pageCallback(self, result):
########### I added this, to hold the data:
self.data = result
return result
def listCallback(self, result):
print result
# Added for effect:
if reactor.running:
reactor.stop()
a = GrabPage('http://google.com')
########### Just call it without assigning to data
#data = a.start() # Not the HTML
a.start()
########### I added this:
if not reactor.running:
reactor.run()
########### Reference the data attribute from the class
data = a.data
print '------REACTOR STOPPED------'
print
########### First 100 characters of a.data:
print '------a.data[:100]------'
print data[:100]