file.py
import datetime
ENV_NUMBER_DAYS_RESERVED = 3
def check_date(entity):
entity_created_date = entity.get('createdDatetime').date()
utc_date = datetime.datetime.utcnow().date()
print((utc_date - entity_created_date).days) #<MagicMock name='datetime.datetime.utcnow().date().__sub__().days' id='4899349456'>
print(type((utc_date - entity_created_date).days))#<class 'unittest.mock.MagicMock'>
return True if (utc_date - entity_created_date).days > int(ENV_NUMBER_DAYS_RESERVED) else False
test.py
import file
#patch('file.datetime')
def test_check_date_False(self, date_now):
entity = MagicMock()
entity.get.return_value = datetime.datetime(2022,1,1)
date_now.datetime.utcnow.date.return_value = datetime.datetime(2022,1,1)
self.assertFalse(check_date(entity=entity))
I get an Error:
TypeError: '>' not supported between instances of 'MagicMock' and 'int'
Is there anyway to test this function?
Probably the problem is with this part of the last line:
(utc_date - namespace_created_date).days
I could reproduce and fix.
Here is a working unittest file (assuming file.py is accessible):
import unittest
from unittest.mock import patch, MagicMock
import datetime
import file
class TestFile(unittest.TestCase):
#patch('file.datetime')
def test_check_date_False(self, date_now):
entity = MagicMock()
entity.get.return_value = datetime.datetime(2022,1,1)
date_now.datetime.utcnow.return_value = datetime.datetime(2022,1,1)
self.assertFalse(file.check_date(None, entity=entity))
unittest.main(verbosity=2)
If I use date_now.datetime.utcnow.date.return_value = datetime.datetime(2022,1,1) I have the same error as you have, but removing .date is enough to have the test to pass.
Related
I want to mock HTTP request that returns an image (bytes) with MagicMock. So, I have simple function here:
import io
import urllib.request
def blah():
try:
req = urllib.request.Request(url='<image url/base64>', data=None, method='GET')
response = urllib.request.urlopen(req)
except Exception as e:
return str(e)
body = io.BytesIO(response.read()) # error here
return 'something'
And I want test it, like this:
import unittest
from blah import blah
from unittest.mock import patch, MagicMock
class TestBlah(unittest.TestCase):
def test_blah(self):
with patch('urllib.request.urlopen') as mock_urlopen:
cm = MagicMock()
cm.getcode.return_value = 200
cm.read.return_value = open('./download.jpeg') # an image file
cm.__enter__.return_value = cm
mock_urlopen.return_value = cm
self.assertEqual(blah(), 'something')
When I execute my code, it produces error:
TypeError: a bytes-like object is required, not '_io.TextIOWrapper'
Can anyone help me?
It should be like this,
cm.read.return_value = open('./download.jpeg', 'rb').read()
I downloaded the code from here https://github.com/SpaceNetChallenge/SpaceNet_SAR_Buildings_Solutions, specifically using model 1. I downloaded the weights corresponding and created the following file to load the model and test. First, I copy the Unet part in main.py into a separate file umodel.py and the test file as follows
import torch
exec(open("./umodel.py").read())
network_data = torch.load('snapshot_fold_8_best')
print(network_data.keys())
import sys
sys.path.append("geffnet")
class Namespace:
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
args = Namespace(extra_num = 1,
dec_ch = [32, 64, 128, 256, 1024],
stride=32,
net='b5',
bot1x1=True,
glob=True,
bn=True,
aspp=True,
ocr=True,
aux_scale=True)
def load_state_dict(model, state_dict):
missing_keys = []
# from UnetOS.umodel import Unet
exec(open("./umodel.py").read())
try:
from torch.hub import load_state_dict_from_url
except ImportError:
from torch.utils.model_zoo import load_url as load_state_dict_from_url
# from UnetOS.umodel import *
model = Unet(extra_num = args.extra_num, dec_ch = args.dec_ch, stride=args.stride, net=args.net, bot1x1 = args.bot1x1, glob=args.glob, bn=args.bn, aspp=args.aspp,
ocr=args.ocr, aux = args.aux_scale > 0).cuda()
load_state_dict(model, network_data)
My question is, why exec(open("./umodel.py").read()) works nicely but whenever I tried to import from umodel import Unet it has errors
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_10492/1282530406.py in <module>
9 # ah
10 model = Unet(extra_num = args.extra_num, dec_ch = args.dec_ch, stride=args.stride, net=args.net, bot1x1 = args.bot1x1, glob=args.glob, bn=args.bn, aspp=args.aspp,
---> 11 ocr=args.ocr, aux = args.aux_scale > 0).cuda()
12 #model = Unet()
13 #print(network_data.key())
D:\hines\Pretrained\1-zbigniewwojna\UnetOS\umodel.py in __init__(self, extra_num, dec_ch, stride, net, bot1x1, glob, bn, aspp, ocr, aux)
238 ['ir_r4_k5_s2_e6_c192_se0.25'],
239 ['ir_r1_k3_s1_e6_c320_se0.25']]
--> 240 enc = GenEfficientNet(in_chans=3, block_args=decode_arch_def(arch_def, depth_multiplier),
241 num_features=round_channels(1280, channel_multiplier, 8, None), stem_size=32,
242 channel_multiplier=channel_multiplier, act_layer=resolve_act_layer({}, 'swish'),
NameError: name 'decode_arch_def' is not defined
The main file is as follow https://github.com/SpaceNetChallenge/SpaceNet_SAR_Buildings_Solutions/blob/master/1-zbigniewwojna/main.py
From the error message, it appears that decode_arch_def is not available and looking at your imports, that has to come from from geffnet.efficientnet_builder import * (it does https://github.com/rwightman/gen-efficientnet-pytorch/blob/master/geffnet/efficientnet_builder.py)
Your exec must have worked because it followed a similar import, that brought decode_arch_def in scope - exec() executes code in the current scope, so it will work because in that scope decode_arch_def is already defined.
However, when you import, the imported module itself doesn't have the function you need in scope. You should add the required import statements to the file you're importing to bring them into scope and it should work.
For example, with a mod.py containing this:
def my_func():
print(datetime.now())
This works:
from datetime import datetime
exec(open("./mod.py").read())
my_func()
But this does not:
from datetime import datetime
import mod
mod.my_func()
To make that work, mod.py would have to be:
from datetime import datetime
def my_func():
print(datetime.now())
And the import of datetime wouldn't be needed in the main program, since it's not referenced there. Your code has a similar issue - you need to determine all the dependencies of your Unet class and import them.
I want to test a function which import another module.
So i want to mock module.
So my test function :
def test_validate_transaction():
dataset_id = "idx"
transaction_id = "idy"
transaction = {
"transaction_rid" : transaction_id,
"is_successful" : True
}
with mock.patch('table_management.get_last_transaction_id', return_value=transaction) :
assert validate_transaction(dataset_rid,transaction_rid) == False
and my function that i want to test is
import json
import os
from table_management import get_last_transaction_id
def validate_transaction(dataset_id,transaction_id):
try:
transaction = get_last_transaction_id(dataset_rid)
return True if transaction['transaction_id'] != transaction_rid or transaction['is_successful']==False else False
except Exception as e:
print("An exception occurred " + str(e))
return {}
But by doing this, i receive the error :
..\..\..\env\lib\site-packages\mock\mock.py:1378: in __enter__
self.target = self.getter() ..\..\..\env\lib\site-packages\mock\mock.py:1548: in <lambda>
getter = lambda: _importer(target)
target = 'table_management'
def _importer(target):
components = target.split('.')
import_path = components.pop(0)
thing = __import__(import_path)
E ImportError: No module named table_management
..\..\..\env\lib\site-packages\mock\mock.py:1231: ImportError
do you have any idea what is missing ?
I'm not sure that you actually don't have some other issues related to import but you are trying to mock a wrong object. You need to mock get_last_transaction_id imported in the module where validate_transaction is. You didn't mention its name so let's assume it's called xyz:
from full_path.to.xyz import validate_transaction
def test_validate_transaction():
dataset_id = "idx"
transaction_id = "idy"
transaction = {
"transaction_rid" : transaction_id,
"is_successful" : True
}
with mock.patch('xyz.get_last_transaction_id', return_value=transaction):
assert validate_transaction(dataset_rid,transaction_rid) == False
I'd like to mock an OverflowError because I'd like to test the value of a variable is after that exception is raised. However, I do not know how to replicate an OverflowError with the libraries I am using. Libraries I am using in this specific test is pysolar.solar specifically the get_altitude, get_azimuth and radiation methods
After mindlessly trying out different numbers to try to simulate an OverflowError I decided to try out Mocking the function and introducing a side effect.
Code I am testing sunposition.py
import numpy as np
import pandas as pd
from pysolar.solar import get_altitude, get_azimuth, radiation as radiation_module
def sun_position(lat: float, lon: float, time: pd.Timestamp = None) -> List[float]:
if time is None:
time = pd.Timestamp.now(tz='UTC')
dt = time.to_pydatetime()
altitude = get_altitude(lat, lon, dt)
azimuth = get_azimuth(lat, lon, dt)
try:
radiation = radiation_module.get_radiation_direct(dt, altitude)
except OverflowError:
radiation = np.nan
return pd.Series([altitude, azimuth, radiation], index=['Alt', 'Azi', 'Rad'])
**What I started to do with patch **
"""Test sunposition module"""
import unittest
import numpy as np
import pandas as pd
from unittest.mock import MagicMock, patch, Mock
from bigfolder.sun import sunposition
class TestSunposition(unittest.TestCase):
"""Test functions in sunposition."""
def test_sun_position_overflow_error(self):
error_lat = 23
error_lon = 12
error_time = pd.Timestamp('2007-02-18 15:13:05', tz="UTC")
mock_args = {'side_effect': OverflowError}
with patch('bigfolder.sun.sunposition.sun_position', **mock_args):
# run the test
self.assertRaises(OverflowError, sunposition.sun_position(lat=error_lat, lon=error_lon, time=error_time))
if __name__ == '__main__':
unittest.main()
I was expecting it to give me and OverFlow error... and it did, however my assertion failed anyways with an OverflowError My guess is that I patched at the wrong place? I don't really understand why the test failed regardless when the error was still an OverFlow Error
This is what was printed out
_mock_self = <MagicMock name='sun_position' id='102333856'>, args = ()
kwargs = {'lat': 23, 'lon': 12, 'time': Timestamp('2007-02-18 15:13:05+0000', tz='UTC')}
self = <MagicMock name='sun_position' id='102333856'>, _new_name = ''
_new_parent = None
_call = call(lat=23, lon=12, time=Timestamp('2007-02-18 15:13:05+0000', tz='UTC'))
seen = set(), skip_next_dot = False, do_method_calls = False
name = 'sun_position'
def _mock_call(_mock_self, *args, **kwargs):
self = _mock_self
self.called = True
self.call_count += 1
_new_name = self._mock_new_name
_new_parent = self._mock_new_parent
_call = _Call((args, kwargs), two=True)
self.call_args = _call
self.call_args_list.append(_call)
self.mock_calls.append(_Call(('', args, kwargs)))
seen = set()
skip_next_dot = _new_name == '()'
do_method_calls = self._mock_parent is not None
name = self._mock_name
while _new_parent is not None:
this_mock_call = _Call((_new_name, args, kwargs))
if _new_parent._mock_new_name:
dot = '.'
if skip_next_dot:
dot = ''
skip_next_dot = False
if _new_parent._mock_new_name == '()':
skip_next_dot = True
_new_name = _new_parent._mock_new_name + dot + _new_name
if do_method_calls:
if _new_name == name:
this_method_call = this_mock_call
else:
this_method_call = _Call((name, args, kwargs))
_new_parent.method_calls.append(this_method_call)
do_method_calls = _new_parent._mock_parent is not None
if do_method_calls:
name = _new_parent._mock_name + '.' + name
_new_parent.mock_calls.append(this_mock_call)
_new_parent = _new_parent._mock_new_parent
# use ids here so as not to call __hash__ on the mocks
_new_parent_id = id(_new_parent)
if _new_parent_id in seen:
break
seen.add(_new_parent_id)
ret_val = DEFAULT
effect = self.side_effect
if effect is not None:
if _is_exception(effect):
> raise effect
E OverflowError
So then I thought I must have patched in the wrong place and introduced the side effect earlier than I should have? So I instead patched the method int the try block. This was my following code.
def test_sun_position_overflow_error(self):
error_lat = 23
error_lon = 12
error_time = pd.Timestamp('2007-02-18 15:13:05', tz="UTC")
mock_args = {'side_effect': OverflowError}
with patch('bigfolder.sun.sunposition.sun_position.radiation_module.get_radiation_direct', **mock_args):
# run the test
self.assertRaises(OverflowError, sunposition.sun_position(lat=error_lat, lon=error_lon, time=error_time))
And now my error is " ModuleNotFoundError: No module named 'bigfolder.sun.sunposition.sun_position'; 'bigfolder.sun.sunposition' is not a package"
I then just changed the path to 'sun_position.radiation_module.get_radiation_direct' but there's no module found.
So my questions are: How to replicate an OverflowError so that then I can check the value of a variable I set once that Exception is raised. Why is the first OverflowError I introduce still not pass my assertion?
Thanks
UPDATE TEST passes
After following #Gang 's suggestion, the OverFlowError was reproduced. I realized that in order to test the block for the exception, specifically that radiation was then np.nan I had to patch the method that I wanted to have the OverFlowError and not the whole method of sun_position. When I tried doing that I imported it incorrectly, as I thought the external library was part of the code. So I changed bigfolder.sun.sunposition.sun_position.radiation_module.get_radiation_direct to pysolar.solar.radiation.get_radiation_direct which is the external library that has the get_radiation_direct method I wanted to mock.
def test_sun_position_overflow_error(self):
lat = 23
lon = 12
time = pd.Timestamp('2007-02-18 15:13:05', tz="UTC")
# get_radiation_direct will now produce an OverFlowError(regardless of coordinates)
mock_args = {'side_effect': OverflowError}
# mock get_radiation_direct and produce OverFlowError
with patch('pysolar.solar.radiation.get_radiation_direct', **mock_args):
# Check radiation column is nan value
assert math.isnan(sunposition.sun_position(lat=lat, lon=lon, time=time)[2])
Why is the first OverflowError I introduce still not pass my assertion?
Almost there. The right way to assertRaises:
def test_sun_position_overflow_error(self):
# This has to be here first and without func call
with self.assertRaises(OverflowError):
# patch the function to have an exception no matter what
mock_args = {'side_effect': OverflowError}
with patch('bigfolder.sun.sunposition.sun_position', **mock_args):
# call this func to trigger an exception
sunposition.sun_position(lat=error_lat, lon=error_lon, time=error_time)
After review the document, it is how to call a func inside assertRaises
assertRaises(exception, callable, *args, **kwds)
fun(*args, **kwds) raises exc
This usage is wrong:
self.assertRaises(OverflowError, sunposition.sun_position(lat=error_lat, lon=error_lon, time=error_time))
It should be a func name with kwargs:
self.assertRaises(OverflowError, sunposition.sun_position, lat=error_lat, lon=error_lon, time=error_time)
I have copied all the codes to the work directory on all my engine machines. And my code are:
my_test.py
my_startegy.py
main.py
So, main.py will be ran on Client Machine, the codes in main.py are:
from ipyparallel import Client
import my_test
import my_strategy as strategy
class Beck_Test_Parallel(object):
"""
"""
def __init__(self):
self.rc = None
self.dview = None
def start_client(self, path):
self.rc = Client(path)
self.dview = self.rc[:]
#self.dview.push(dict(
# Account=my_test.Account,
# dataImport=my_test.dataImport
# ))
def parallel_map(self, deal_function, accounts):
import my_test
return self.dview.map_sync(deal_function, accounts)
def create_accounts(time_list, account):
accounts = []
for index, time in enumerate(time_list):
acc = my_test.Account(
strategy.start,
strategy.end,
strategy.freq,
strategy.universe_code,
strategy.capital_base,
strategy.short_capital,
strategy.benchmark,
strategy.self_defined
)
account.share_data(acc)
acc.iniData2()
acc.iniData3()
acc.current_time = time
acc.days_counts = index+1
acc.dynamic_record['capital'] = acc.capital_base
del acc.connect
accounts.append(acc)
return accounts
def let_us_deal(account):
account = strategy.handle_data(account)
print ' >>>', account.current_time
return account
if __name__ == '__main__':
account = my_test.Account(
strategy.start,
strategy.end,
strategy.freq,
strategy.universe_code,
strategy.capital_base,
strategy.short_capital,
strategy.benchmark,
strategy.self_defined
)
account.iniData()
account.iniData2()
account.iniData3()
time_list = my_test.get_deal_time_list(account)
accounts = parallel.create_accounts(time_list, account)
back_test_parallel = parallel.Beck_Test_Parallel()
back_test_parallel.start_client(
'/home/fit/.ipython/profile_default/security/ipcontroller-client.json')
back_test_parallel.dview.execute('import my_test')
back_test_parallel.dview.execute('import my_strategy as strategy')
# get the result
result = back_test_parallel.parallel_map(let_us_deal, accounts)
for acc in result.get():
print acc.reselected_stocks, acc.current_time
And I have imported my_test module in parallel_map() function in Class Back_Test_Parallel and I have also imported my_test module in back_test_parallel.dview.execute('import my_test').
And the corresponding modules are on the engine machine's work directory. I have copied the ipcontroller-client.json and ipcontroller-engine.json to the work directory on engine machine.
But when it runs, the error is ImportError: No module named my_test, since the module my_test.py is already on the work directory. It really made me feel frustrated!
---------------------------------------------------------------------------
CompositeError Traceback (most recent call last)
/home/fit/log/1027/back_test/main.py in <module>()
119 import ipdb
120 ipdb.set_trace()
--> 121 for acc in result.get():
122 print acc.reselected_stocks, acc.current_time
123
/usr/local/lib/python2.7/dist-packages/ipyparallel/client/asyncresult.pyc in get(self, timeout)
102 return self._result
103 else:
--> 104 raise self._exception
105 else:
106 raise error.TimeoutError("Result not ready.")
CompositeError: one or more exceptions from call to method: let_us_deal
[0:apply]: ImportError: No module named my_test
[1:apply]: ImportError: No module named my_test
something about result:
In [2]: result
Out[2]: <AsyncMapResult: finished>
In [3]: type(result)
Out[3]: ipyparallel.client.asyncresult.AsyncMapResult
Note that, when it runs on single machine by using ipcluster start -n 8, it works fine, without any error.
Thanks in advance
I think my CWD is not in the right directory. So you can check your CWD
>>> import os
>>> print(dview.apply_sync(os.getcwd).get())
If it is in wrong directory, before parallel computing, you can set the right CWD to make sure you ipyparallel env is in the right work directory:
>>> import os
>>> dview.map(os.chdir, ['/path/to/my/project/on/engine']*number_of_engines)
>>> print(dview.apply_sync(os.getcwd).get())
You can also check your engines' name by
>>> import socket
>>> print(dview.apply_sync(socket.gethostname))
And it works fine!
pip install ipyparallel --upgrade