Develop Reference

Multiprocessing In Django Function

Is it possible to use multi processing in Django on a request.
#so if I send a request to http://127.0.0.1:8000/wallet_verify
def wallet_verify(request):
walelts = botactive.objects.all()
#here I check if the user want to be included in the process or not so if they set it to True then i'll include them else ignore.
for active in walelts:
check_active = active.active
if check_active == True:
user_is_active = active.user
#for the ones that want to be included I then go to get their key data.
I need to get both api and secret so then I loop through to get the data from active users.
database = Bybitapidatas.objects.filter(user=user_is_active)
for apikey in database:
apikey = apikey.apikey
for apisecret in database:
apisecret = apisecret.apisecret
#since I am making a request to an exchange endpoint I can only include one API and secret at a time . So for 1 person at a time this is why I want to run in parallel.
for a, b in zip(list(Bybitapidatas.objects.filter(user=user_is_active).values("apikey")), list(Bybitapidatas.objects.filter(user=user_is_active).values("apisecret"))):
session =spot.HTTP(endpoint='https://api-testnet.bybit.com/', api_key=a['apikey'], api_secret=b['apisecret'])
#here I check to see if they have balance to open trades if they have selected to be included.
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
for i in GET_USDT_BALANCE:
if 'USDT' in i.values():
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
idx_USDT = GET_USDT_BALANCE.index(i)
GET_USDTBALANCE = session.get_wallet_balance()['result']['balances'][idx_USDT]['free']
print(round(float(GET_USDTBALANCE),2))
#if they don't have enough balance I skip the user.
if round(float(GET_USDTBALANCE),2) < 11 :
pass
else:
session.place_active_order(
symbol="BTCUSDT",
side="Buy",
type="MARKET",
qty=10,
timeInForce="GTC"
)
How can I run this process in parallel while looping through the database to also get data for each individual user.
I am still new to coding so hope I explained that it makes sense.
I have tried multiprocessing and pools but then I get that the app has not started yet and I have to run it outside of wallet_verify is there a way to do it in wallet_verify
and when I send the Post Request.
Any help appreciated.

Filtering the Database to get Users who have set it to True
Listi - [1,3](these are user ID's Returned
processess = botactive.objects.filter(active=True).values_list('user')
listi = [row[0] for row in processess]
Get the Users from the listi and perform the action.
def wallet_verify(listi):
# print(listi)
database = Bybitapidatas.objects.filter(user = listi)
print("---------------------------------------------------- START")
for apikey in database:
apikey = apikey.apikey
print(apikey)
for apisecret in database:
apisecret = apisecret.apisecret
print(apisecret)
start_time = time.time()
session =spot.HTTP(endpoint='https://api-testnet.bybit.com/', api_key=apikey, api_secret=apisecret)
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
for i in GET_USDT_BALANCE:
if 'USDT' in i.values():
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
idx_USDT = GET_USDT_BALANCE.index(i)
GET_USDTBALANCE = session.get_wallet_balance()['result']['balances'][idx_USDT]['free']
print(round(float(GET_USDTBALANCE),2))
if round(float(GET_USDTBALANCE),2) < 11 :
pass
else:
session.place_active_order(
symbol="BTCUSDT",
side="Buy",
type="MARKET",
qty=10,
timeInForce="GTC"
)
print ("My program took", time.time() - start_time, "to run")
print("---------------------------------------------------- END")
return HttpResponse("Wallets verified")
Verifyt is what I use for the multiprocessing since I don't want it to run without being requested to run. also initialiser starts apps for each loop
def verifyt(request):
with ProcessPoolExecutor(max_workers=4, initializer=django.setup) as executor:
results = executor.map(wallet_verify, listi)
return HttpResponse("done")
```

Mock.patch returning MagicMock object causing AssertionError?

I have a function that I am trying to test in querySomething.py:
class QuerySomething:
def retrieveIssues(self,token):
responses = []
if "customFields" in self._event:
if not self.custom_fields:
fields = []
else:
fields = self.custom_fields
else:
fields = []
for issueTypeKey, issueTypeValue in self.issueTypes.items():
print(issueTypeKey, ":", issueTypeValue)
query = self.getQuery(issueTypeValue, self.status, fields)
respons = httpClient.get_request(query, token)
responses.append(respons)
return responses
And the test file:
def mock_getQuery():
return "QUERY"
def mock_response(state):
if state=="unauth":
with open("src/tests/mockdata/unauthorized_api_response.json","r") as response_file:
unauth_error = response_file.read()
return json.dumps(unauth_error)
elif state=="auth":
with open("src/tests/mockdata/success_api_response.json","r") as response_file:
success_message = response_file.read()
return json.dumps(success_message)
return "No message"
class test_query(unittest.TestCase):
#mock.patch("querySomething.QuerySomething.getQuery", side_effect=mock_getQuery)
#mock.patch("httpClient.get_request", side_effect=mock_response)
def test_retreiveIssues_unauth_response(self,mock_get,QuerySomething):
self.assertEqual(QuerySomething.retrieveIssues("token"),mock_response("unauth"))
if __name__ == "__main__":
unittest.main()
I am trying to mock the httpClient.get_request so that it gets the JSON file instead of reaching out to the API. We want to test an unauthorized response and a success response which explains the mock_response function. However, when I run the test, I get the following:
AssertionError: <MagicMock name='getQuery.retri[36 chars]712'> != '"{\\n \\"errorMessages\\": [\\n [131 chars]\n}"'
which is somewhat correct, but we need just the text, not the object. I read that I need to call the function, but when I try to call the function it throws a ModuleNotFound or NotAPackage error. What do I need to do to mock the httpClient.get_request and return the JSON string in the retrieveIssues function?

Updated, I was able to pull the JSON from the other file, and then was able to mock the return value as follows:
QuerySomething.retrieveIssues.return_value=load_json("unauth")
where load_json("unauth") pulls from the JSON response file.

Testing cursor.execute with sql script called using pytest

Function to test
def get_adgroups_not_taked_share(
campaign_ids: List[str], src_table: str, spend_src_table: str
) -> List[Tuple[str, str]]:
loses_adgroups: List[Tuple[str, str]] = []
with RedshiftCursor() as cursor:
cursor.execute(
"""
SELET some_data from some_table WHERE some_condition
"""
)
for row in cursor.fetchall():
loses_adgroups.append((row[0], str(row[1])))
return loses_adgroups
There is a test for this function
import pytest
from my_ap import get_adgroups_not_taked_share
#pytest.fixture
def campaigns_redshift_cursor_mock(mocker):
cursor_mock = mocker.MagicMock()
cursor_mock.fetchall.return_value = [
('hs_video544', '123123123', 100),
('hs_video547', '123123123', 50),
]
rs_cursor_creator = mocker.patch('google_panel.logic.clean_creative.RedshiftCursor')
rs_cursor_creator.return_value.__enter__.return_value = cursor_mock
return rs_cursor_creator
#pytest.mark.django_db
def test_get_adgroups_not_taked_share(
campaigns_redshift_cursor_mock,
):
campaign_ids = ['1111', '2222', '3333']
result = get_adgroups_not_taked_share(campaign_ids, 'test_table', 'spend_src_table')
assert result == [('hs_video544', '123123123'), ('hs_video547', '123123123')]
Now I want to add a new feature to test the sql script. Checking that the correct sql query is being called. something like
def test_get_adgroups_not_taked_share(
campaigns_redshift_cursor_mock,
):
......
query = """SELET some_data from some_table WHERE some_condition"""
campaigns_redshift_cursor_mock.execute.assert_called_with(query)
But got
E AssertionError: Expected call: execute('query')
E Not called

The short answer is that you need to assert this: campaigns_redshift_cursor_mock.return_value.__enter__.return_value.execute.assert_called_once_with(query). The reason is that you're using RedshiftCursor as a context manager (hence the return_value.__enter__.return_value part) and only then calling the execute method.
A slightly longer answer is how I got this assert.
I wrote this library which adds the mg pytest fixture. To use it, pip install it and then add the mg fixture to your test and execute its generate_asserts method like so:
#pytest.mark.django_db
def test_get_adgroups_not_taked_share(
campaigns_redshift_cursor_mock,
mg
):
campaign_ids = ['1111', '2222', '3333']
result = get_adgroups_not_taked_share(campaign_ids, 'test_table', 'spend_src_table')
mg.generate_asserts(campaigns_redshift_cursor_mock)
assert result == [('hs_video544', '123123123'), ('hs_video547', '123123123')]
Then run the test as usual and you would get this output to the console:
assert 1 == campaigns_redshift_cursor_mock.call_count
campaigns_redshift_cursor_mock.assert_called_once_with()
campaigns_redshift_cursor_mock.return_value.__enter__.assert_called_once_with()
campaigns_redshift_cursor_mock.return_value.__enter__.return_value.execute.assert_called_once_with('\n SELET some_data from some_table WHERE some_condition\n ')
campaigns_redshift_cursor_mock.return_value.__enter__.return_value.fetchall.assert_called_once_with()
campaigns_redshift_cursor_mock.return_value.__exit__.assert_called_once_with(None, None, None)
These are all the calls made to the mock, you can filter the ones which are relevant for you.

Preserve changes in multiple function when testing a Flask app

I'm following a talk on Flask about creating an API. I want to write some tests for it. Upon testing creating a resource does testing deleting the resource in another function work? How do I make the creation of a resource persist to be tested for deletion and editing?
David Baumgold - Prototyping New APIs with Flask - PyCon 2016
The talk shows how to make an API for the names and image urls of puppies.
you create a puppy at the index page with a POST request
you get a single puppy with a GET from '/puppy_name'
you get the list of puppies with a GET from '/'
you edit a puppy with a PUT from '/puppy_name' (along with the new data of course)
you delete a puppy with a DELETE from '/puppy_name'
import py.test
import unittest
from requests import get, post, delete, put
localhost = 'http://localhost:5000'
class TestApi(unittest.TestCase):
def test_list_puppies(self):
index = get(localhost)
assert index.status_code == 200
def test_get_puppy(self):
puppy1 = get(localhost + '/rover')
puppy2 = get(localhost + '/spot')
assert puppy1.status_code == 200 and puppy2.status_code == 200
def test_create_puppy(self):
create = post(localhost, data={
'name': 'lassie', 'image_url': 'lassie_url'})
assert create.status_code == 201
#py.test.mark.skip('cannot fix it')
def test_edit_puppy(self):
ret = put(localhost + '/lassie',
data={'name': 'xxx', 'image_url': 'yyy'})
assert ret.status_code == 201
def test_puppy_exits(self):
lassie = get(localhost + '/lassie').status_code
assert lassie == 200
def test_delete_puppy(self):
ret = delete(localhost + '/lassie')
assert ret.status_code == 200
#app.route('/', methods=['POST'])
def create_puppy():
puppy, errors = puppy_schema.load(request.form)
if errors:
response = jsonify(errors)
response.status_code = 400
return response
puppy.slug = slugify(puppy.name)
# create in database
db.session.add(puppy)
db.session.commit()
# return an HTTP response
response = jsonify( {'message': 'puppy created'} )
response.status_code = 201
location = url_for('get_puppy', slug=puppy.slug)
response.headers['location'] = location
return response
#app.route('/<slug>', methods=['DELETE'])
def delete_puppy(slug):
puppy = Puppy.query.filter(Puppy.slug == slug).first_or_404()
db.session.delete(puppy)
db.session.commit()
return jsonify( {'message': '{} deleted'.format(puppy.name)} )
The assert statements in both 'test_edit_puppy' and 'test_puppy_exists' fails. I get a 404 status code instead of 201 and 200.

You're testing the wrong thing. You can persist changes to a database, simply by committing it, when you run a test, but you really don't want to do that.
With unit testing you're testing simple units. With integration testing, which is what you're talking about here, you still want each test to have a specific focus. In this particular case you would want to do something like:
def test_delete_puppy(self):
create = post(localhost, data={
'name': 'lassie', 'image_url': 'lassie_url'})
lassie = get(localhost + '/lassie')
# not sure if there's an "assume" method, but I would
# use that here - the test is not a valid test
# if you can't create a puppy and retrieve the puppy
# then there's really no way to delete something that
# does not exist
assert create.status_code == 201
assert lassie.status_code == 200
ret = delete(localhost + '/lassie')
assert ret.status_code == 200
lassie = get(localhost + '/lassie')
assert lassie.status_code == 404
The focus of this test is to test that deleting a puppy works fine. But you want to setup the puppy in the database as either part of your test or part of the setup of your test. It's the arrange portion of arrange, act, assert. You're arranging the state of the world in its proper order before you actually perform the test. The difference between integration testing and unit testing is that with unit testing you'd be mocking out all the endpoints that you're calling, while with integration tests you're going to actually setup all the data that you need before you run the test portion. Which is what you want to do here.

How to mock a redis client in Python?

I just found that a bunch of unit tests are failing, due a developer hasn't mocked out the dependency to a redis client within the test. I'm trying to give a hand in this matter but have difficulties myself.
The method writes to a redis client:
redis_client = get_redis_client()
redis_client.set('temp-facility-data', cPickle.dumps(df))
Later in the assert the result is retrieved:
res = cPickle.loads(get_redis_client().get('temp-facility-data'))
expected = pd.Series([set([1, 2, 3, 4, 5])], index=[1])
assert_series_equal(res.variation_pks, expected)
I managed to patch the redis client's get() and set() successfully.
#mock.patch('redis.StrictRedis.get')
#mock.patch('redis.StrictRedis.set')
def test_identical(self, mock_redis_set, mock_redis_get):
mock_redis_get.return_value = ???
f2 = deepcopy(self.f)
f3 = deepcopy(self.f)
f2.pk = 2
f3.pk = 3
self.one_row(f2, f3)
but I don't know how to set the return_value of get() to what the set() would set in the code, so that the test would pass.
Right now this line fails the test:
res = cPickle.loads(get_redis_client().get('temp-facility-data'))
TypeError: must be string, not MagicMock
Any advice please?

Think you can use side effect to set and get value in a local dict
data = {}
def set(key, val):
data[key] = val
def get(key):
return data[key]
mock_redis_set.side_effect = set
mock_redis_get.side_effect = get
not tested this but I think it should do what you want

If you want something more complete, you can try fakeredis
#patch("redis.Redis", return_value=fakeredis.FakeStrictRedis())
def test_something():
....

I think you can do something like this.
redis_cache = {
"key1": (b'\x80\x04\x95\x08\x00\x00\x00\x00\x00\x00\x00\x8c\x04test\x94.', "test"),
"key2": (None, None),
}
def get(redis_key):
if redis_key in redis_cache:
return redis_cache[redis_key][0]
else:
return None
mock = MagicMock()
mock.get = Mock(side_effect=get)
with patch('redis.StrictRedis', return_value=mock) as p:
for key in redis_cache:
result = self.MyClass.my_function(key)
self.assertEqual(result, redis_cache[key][1])