Develop Reference

How do I test start_server_http is working for Prometheus

I have created this monitoring class that updates some Counter Metrics according to some logic. I have attached the code. Please can someone explain to me why would my registry be empty even after I add the test metric.
import logging
from prometheus_client import (
CollectorRegistry,
Counter,
start_http_server
)
class Reporter:
def __init__(self):
self._set_counters()
start_http_server(8080, registry=self.registry)
def _set_counters(self):
self.registry = CollectorRegistry()
self.bycounter = Counter(
'bycounter',
'blah blah',
['by', 'level0top'],
registry=self.registry
)
self.bycounter.labels(by='test', level0top='test').inc()
I am trying to test the metrics like
import unittest
from sc_eol.monitoring import TodayDataReporter
from sc_eol.sc_eol_utils import generate_query_url
reporter = TodayDataReporter()
class TestTodayDataReporter(unittest.TestCase):
#staticmethod
def test_publish():
by = 'level1'
parse_query = {'level0top' : 'WSJ2', 'date' : '2021-11-01'}
start = '2021-11-01'
print(dir(reporter.registry))
reporter.registry.collect()
before = reporter.registry.get_sample_value('bycounter', ['level1', 'WSJ2'])
print("BEFOREEE", before)
reporter.registry.collect()
generate_query_url(by, start, parse_query, reporter)
before = reporter.registry.get_sample_value('bycounter', {'by':'level1', 'level0top': 'WSJ2'})
reporter.registry.collect()
print("After", before)
if __name__ == "__main__":
unittest.main()
Why is the bycounter None?
How do I test if a server is running at port 8080 or not

before = reporter.registry.get_sample_value('bycounter', ['level1', 'WSJ2'])
The counter is being renamed as bycounter_total

How to test response of service using mock?

I have the following service with a post request:
import requests
class Attt():
def get_req(self):
pload = {'username': 'Olivia', 'password': '123'}
r = requests.post('https://httpbin.org/post', data=pload)
print(r.text)
print(r.content)
obj=Attt()
obj.get_req()
I would like to mock the request so that I can create the needed response of the service.
I have written a test like below:
import unittest
from unittest.mock import patch
from mock_tutorial.attt import Attt
import pandas as pd
class TestMockService(unittest.TestCase):
def test_mock(self):
fake_json = [{'test': "mock"}]
with patch('mock_tutorial.attt.requests.post') as mock_get:
mock_get.return_value.status_code = 200
mock_get.return_value.json.return_value = fake_json
obj = Attt()
response = obj.get_req()
print('response json', response.json())
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), fake_json)
if __name__ == "__main__":
unittest.main()
But as the get_req method doesn't return the reesponse, the assertions fail: AttributeError: 'NoneType' object has no attribute 'json'. How can I modify the assertions to check that the response of the get_req method has been mocked correspondingly, to the fake_json variable?

It is because you are not returning anything from Attt.get_req thus it returns None. So in your test, it is like calling None.json(). You should return the response of the request:
...
class Attt():
def get_req(self):
...
return r
...
Also as documented, you should wrap your code in attt.py so that it doesn't get executed when imported:
if __name__ == "__main__":
obj=Attt()
obj.get_req()
This seems fine since this is just for tutorial. But take note that there are already libraries you can use that mocks the requests module such as requests-mock.
Update
If the method wouldn't return anything, then all we can do to check is by adding a spy to the target functionality. But take note that this wouldn't make sense in reality because what we are spying on is a mocked/patched functionality which is requests.post. This would only be good for learning purposes but in reality, better yet not add the test as it doesn't add any value at all. We can use pytest-mock for this purpose:
mock_tutorial/attt.py
import requests
class Attt():
def get_req(self):
pload = {'username': 'Olivia', 'password': '123'}
r = requests.post('https://httpbin.org/post', data=pload)
print(r.text)
print(r.content)
test_attt.py
from unittest.mock import patch
from mock_tutorial import attt
from mock_tutorial.attt import Attt
import pytest
def test_mock(mocker): # Run <pip install pytest-mock>
fake_json = [{'test': "mock"}]
with patch('mock_tutorial.attt.requests.post') as mock_get:
mock_get.return_value.status_code = 200
mock_get.return_value.json.return_value = fake_json
post_spy = mocker.spy(attt.requests, "post")
obj = Attt()
response = obj.get_req()
assert post_spy.spy_return.status_code == 200
assert post_spy.spy_return.json() == fake_json

Issue in mocking python unit test

I have written a test as below:
class TestLoader(TestCase):
#pytest.fixture(autouse=True)
#patch('loaders.myloader.DSFactory')
def _initialize_(self, mock_ds_factory):
self.loader = MyLoader()
mock_ds = Mock()
mock_ds_factory.get_ds_for_env.return_value = mock_ds
self.loader.ds = mock_ds
def test_load(self):
self.loader.ds.read_file.return_value = json.dumps(self.get_data())
self.loader.load("test_s3_key") #####IN THIS LINE I AM GETTING ERROR AS MENTIONED BELOW##
#staticmethod
def get_data():
return {"key1":"value1","key2":"value2"}
Associated source is located here: loaders->myloader.py. myloader.py is as follows:
from common.ds_factory import DSFactory
class MyLoader:
def __init__(self):
self.ds = DSFactory.get_ds_for_env()
def load(self, file_key):
print(f"ds : {self.ds}")
print(f"file read is : {self.ds.read_file(S3_BUCKET, file_key)}"}
data_dict = json.loads(self.ds.read_file(S3_BUCKET, file_key))
But while testing, I am getting error as follows:
ds is :<MagicMock name='DSFactory.get_ds_for_env()' id='140634163567528'>
file read is :<MagicMock name='DSFactory.get_ds_for_env().read_file()' id='140635257259568'>
E TypeError: the JSON object must be str, bytes or bytearray, not 'MagicMock'
I don't understand why, even after mocking return value of read_file with
self.loader.ds.read_file.return_value = json.dumps(self.get_data())
I am getting MagickMock object. I am stuck, not getting any clue how to resolve this.

Your code:
from common.ds_factory import DSFactory
class MyLoader:
def __init__(self):
self.ds = DSFactory.get_ds_for_env()
def load(self, file_key):
data_dict = json.loads(self.datastore.read_file(S3_BUCKET, file_key))
Issue here i can see is, data-store is not present, it should be self.ds.read_file
Please print self.datastore.read_file(S3_BUCKET, file_key) and verify the output.
This is the error coming from AWS_S3 bucket Json structure. It seems its not sending the Json value in string format rather than in Magic Mock format.
To more about Magic Mock format, please visit here: https://medium.com/ryans-dev-notes/python-mock-and-magicmock-b3295c2cc7eb

python class managing in another class

I have a class called martCrawler which import other 3 crawlers from other files
However, the code becomes too long as the crawler imported increased.
Is there a better practice managing the code like this?
Thanks in advance~
from Scripts_mart.wat_ver2 import crawler_watsons
from Scripts_mart.cosmed_ver1 import crawler_cosmed
from Scripts_mart.pxmart_ver1 import crawler_pxmart
import datetime
class martCrawler():
def __init__(self):
self.wat = crawler_watsons()
self.cos = crawler_cosmed()
self.pxm = crawler_pxmart()
def crawler_testing(self):
result_pack = {}
wat_result = self.wat.run_before_insert()
cos_result = self.cos.run_before_insert()
pxm_result = self.pxm.run_before_insert()
result_pack['wat'] = wat_result
result_pack['cos'] = cos_result
result_pack['pxm'] = pxm_result
return result_pack
...

Then why not store all the crawlers in a dict from the beginning?
As an example:
from Scripts_mart.wat_ver2 import crawler_watsons
from Scripts_mart.cosmed_ver1 import crawler_cosmed
from Scripts_mart.pxmart_ver1 import crawler_pxmart
class MartCrawler():
def __init__(self, *args):
self.crawlers = {}
for crawler in args:
# use some introspection to get the name of the classes.
# The names should be "crawler_watsons", etc
self.crawlers[crawler.__name__] = crawler()
def crawler_testing(self):
result_pack = {}
for name, crawler in self.crawlers.items():
result_back[name] = crawler.run_before_insert()
return result_back
martCrawler = MartCrawler(crawler_watsons, crawler_cosmed, crawler_pxmart)
Just have in mind that the names in your dict will be the actual names of the imported classes, not 'wat', 'pos' and 'pxm'. But if this is a problem you can use some string manipulation and/or regexes to fix it.
For example you could replace crawler.__name__ with crawler.__name__[8:11]

Just put them into a dictionary:
from Scripts_mart.wat_ver2 import crawler_watsons
from Scripts_mart.cosmed_ver1 import crawler_cosmed
from Scripts_mart.pxmart_ver1 import crawler_pxmart
import datetime
class martCrawler():
def __init__(self):
self.crawlers = {
"wat": crawler_watsons(),
"cos": crawler_cosmed(),
"pxm": crawler_pxmart()
}
def crawler_testing(self):
result_pack = {}
for key in self.crawlers:
result_pack[key] = self.crawlers[key].run_before_insert()
return result_pack

Django unittest and mocking the requests module

I am new to Mock and am writing a unit test for this function:
# utils.py
import requests
def some_function(user):
payload = {'Email': user.email}
url = 'http://api.example.com'
response = requests.get(url, params=payload)
if response.status_code == 200:
return response.json()
else:
return None
I am using Michael Foord's Mock library as part of my unit test and am having difficulty mocking the response.json() to return a json structure. Here is my unit test:
# tests.py
from .utils import some_function
class UtilsTestCase(unittest.TestCase):
def test_some_function(self):
with patch('utils.requests') as mock_requests:
mock_requests.get.return_value.status_code = 200
mock_requests.get.return_value.content = '{"UserId":"123456"}'
results = some_function(self.user)
self.assertEqual(results['UserId'], '123456')
I have tried numerous combinations of different mock settings after reading the docs with no luck. If I print the results in my unit test it always displays the following instead of the json data structure I want:
<MagicMock name=u'requests.get().json().__getitem__().__getitem__()' id='30315152'>
Thoughts on what I am doing wrong?

Patch json method instead of content. (content is not used in some_function)
Try following code.
import unittest
from mock import Mock, patch
import utils
class UtilsTestCase(unittest.TestCase):
def test_some_function(self):
user = self.user = Mock()
user.email = 'user#example.com'
with patch('utils.requests') as mock_requests:
mock_requests.get.return_value = mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {"UserId":"123456"}
results = utils.some_function(self.user)
self.assertEqual(results['UserId'], '123456')

Another pattern I like to use that is a little more reusable would be to start the patcher in your unit test's setUp method. It's also important to check that mock request was called with the expected parameters:
class UtilsTestCase(TestCase):
def setUp(self):
self.user = Mock(id=123, email='foo#bar.com')
patcher = patch('utils.requests.get')
self.mock_response = Mock(status_code=200)
self.mock_response.raise_for_status.return_value = None
self.mock_response.json.return_value = {'UserId': self.user.id}
self.mock_request = patcher.start()
self.mock_request.return_value = self.mock_response
def tearDown(self):
self.mock_request.stop()
def test_request(self):
results = utils.some_function(self.user)
self.assertEqual(results['UserId'], 123)
self.mock_request.assert_called_once_with(
'http://api.example.com'
payload={'Email': self.user.email},
)
def test_bad_request(self):
# override defaults and reassign
self.mock_response.status_code = 500
self.mock_request.return_value = self.mock_response
results = utils.some_function(self.user)
self.assertEqual(results, None)
self.mock_request.assert_called_once_with(
'http://api.example.com'
payload={'Email': user.email},
)

Another way that I believe is more clear and straight forward:
import unittest
from mock import Mock, patch
import utils
class UtilsTestCase(unittest.TestCase):
def test_some_function(self):
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {"UserId": "123456"}
with patch('utils.requests.get') as mock_requests:
results = utils.some_function(self.user)
self.assertEqual(results['UserId'], '123456')