Develop Reference

How to mock a method of a dependent class?

I'm trying to test a method (Creator.do_a_call) which calls another class's method (Requester.make_request) which makes external calls (all the code is below). So I want to mock Request.make_request to return a mocked out response object so it doesn't do any external calls, but when I do that it gives me an error (see below)
Here's the code:
Requester.py:
from requests import Response, Session, Request
class Requester:
url: str = ''
def __init__(self, url: str):
self.url = url
def make_request(self, path: str, method: str = 'get', body: dict = None, custom_headers: dict = None) -> Response:
print("make_request")
url = self.url + path
session = Session()
request = Request(method, url, json=body)
response = session.send(request.prepare())
return response
Creator.py
from Requester import Requester
from http import HTTPStatus as status
from requests import Response
class Creator:
def do_a_call(self, url) -> Response:
print("do_a_call")
requester = Requester(url)
response = requester.make_request(
"/",
"GET"
)
print(type(response))
if response.status_code != status.OK and response.status_code != status.ACCEPTED:
raise Exception(response.status_code, response.text)
return response
main.py
from Creator import Creator
print("main")
creator = Creator()
response = creator.do_a_call("https://google.com")
print(response)
test.py
import unittest
from unittest.mock import patch
from Creator import Creator
from requests import Session
import requests_mock
class CreatorTest(unittest.TestCase):
def mocked_make_request(self, two, three):
session = Session()
mocker = requests_mock.Mocker(session=session)
adapter = requests_mock.Adapter()
adapter.register_uri('POST', 'mock://test.com', text="Invalid policy number supplied", status_code=400)
return mocker.post("mock://test.com", status_code=400)
#patch('Requester.Requester.make_request')
def test_do_a_call_side_effect(self, make_request):
creator = Creator()
make_request.side_effect = self.mocked_make_request
response = creator.do_a_call("mock://test.com")
self.assertEqual(200, response.status_code)
#patch('Requester.Requester.make_request')
def test_do_a_call_return_value(self, make_request):
creator = Creator()
make_request.return_value = self.mocked_make_request("", "")
response = creator.do_a_call("mock://test.com")
self.assertEqual(200, response.status_code)
if __name__ == "__main__":
unittest.main()
In my research I've found two ways that should work - changing the return_value of the mock, or changing the side_effect of the mock, both of which give me the same error:
Traceback (most recent call last):
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/unittest/mock.py", line 1325, in patched
return func(*newargs, **newkeywargs)
File "test.py", line 19, in test_do_a_call_side_effect
response = creator.do_a_call("mock://test.com")
File "/Users/citrja/Documents/git/python_test/Creator.py", line 14, in do_a_call
if response.status_code != status.OK and response.status_code != status.ACCEPTED:
AttributeError: '_Matcher' object has no attribute 'status_code'

I've found out what was wrong! I Was not attaching the adapter to the session, I had thought this happened automagically but apparently not. Here's the updated mocked_make_request function:
def mocked_make_request(self, two, three):
session = Session()
mocker = requests_mock.Mocker(session=session)
adapter = requests_mock.Adapter()
adapter.register_uri('GET', 'mock://test.com', text="Invalid policy number supplied", status_code=400)
session.mount("mock://", adapter)
return session.get("mock://test.com")

python extract few lines from the output

I've created a python class to execute a curl command which in turn generates a token. The code works as expected.The script generates the expected output.Now, need few lines from the below mentioned output and not the entire output.
How do I extract the following lines from the output.
lines starting after
Set-Cookie: SESSION= till =;
. How do I extract these lines.
import requests
import json
from akamai.edgegrid import EdgeGridAuth, EdgeRc
from tabulate import tabulate
import os
import re
import base64
class generate_token(object):
"""
This library is to generate token for the test application
"""
def __init__(self,kwargs):
self.username = 'user'
self.password = 'abcdef'
self.contractName = kwargs.get("Names")
self.contractType = kwargs.get("Type")
self.accountId = kwargs.get("Id")
self.env = kwargs.get("env")
def requestSession(self):
if self.env == "test":
ping = os.popen('ping -c 1 hostname.com').read()
ip_addr = re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",ping)[0]
else:
ping = os.popen('ping -c 1 different_hostname.com').read()
ip_addr = re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",ping)[0]
url = "https://%s/EdgeAuth/asyncUserLogin" %(ip_addr)
command = "curl -is -v -k -d username=%s -d password=%s --request POST '%s' " %(self.username,self.password,url)
self.session = os.popen(command).read()
if __name__ == "__main__":
initData = {
'Names': 'names',
'Type': 'type',
'Id': 'ID',
'env' : 'test'
}
c = generate_token(kwargs=initData)
result = c.requestSession()
Entire output :
* TCP_NODELAY set
* WARNING: disabling hostname validation also disables SNI.
> POST /EdgeAuth/asyncUserLogin HTTP/1.1
> Host: 143.111.112.202
> User-Agent: curl/7.54.0
> Accept: */*
> Content-Length: 33
> Content-Type: application/x-www-form-urlencoded
>
} [33 bytes data]
* upload completely sent off: 33 out of 33 bytes
< HTTP/1.1 302 Moved Temporarily
< Date: Thu, 26 Jul 2018 15:54:07 GMT
< Server: Apache-Coyote/1.1
< Location: https://192.168.93.201/homeng/view/main
< Content-Length: 0
< Set-Cookie: SESSION=1-7FOmRFhEcci+8fAfh+DvLg==-1bk61eksykUHbWc9kx3LzL0KdkaxyDodxsEsu6siMR/1cpR7yLaD7HtTjXEr+XDowzpYcug4bfKIMjk2/T0aqJiHg5BrbRbnBeZnTcAUIpdyKGvtnmEQkSADoaxvgfYGrxqQctrVPdqJtPziqvIbjO1X8xh0Xtj/ZNvJXaSFs//w9ISNZ5hvIDBygUo+0EuYT8PSTYVPrcBCLaJUkC8ACg==-jdN937gosTsMx8fdaJqP5sA7wKPRgqv1RxRkX65SWeE=;
Path=/; Secure
< Via: 1.1 %{HTTP_HOST}
< X-Content-Type-Options: nosniff
< Content-Type: text/plain
Required output :
17FOmRFhEcci+8fAfh+DvLg==-1bk61eksykUHbWc9kx3LzL0KdkaxyDodxsEsu6siMR/1cpR7yLaD7HtTjXEr+XDowzpYcug4bfKIMjk2/T0aqJiHg5BrbRbnBeZnTcAUIpdyKGvtnmEQkSADoaxvgfYGrxqQctrVPdqJtPziqvIbjO1X8xh0Xtj/ZNvJXaSFs//w9ISNZ5hvIDBygUo+0EuYT8PSTYVPrcBCLaJUkC8ACg==-jdN937gosTsMx8fdaJqP5sA7wKPRgqv1RxRkX65SWeE=;

Don't use os.popen and friends to execute external programs - use the subprocess module for that:
ping = subprocess.check_output(['ping', '-c', '1', 'hostname.com'])
In the same way, don't execute a subprocess just to get a cookie from a url - you're already importing the excellent requests library in your code, why not use it!
r = requests.post(url, data={'username': self.username, 'password': self.password})
print (r.cookies['SESSION'])

Why am I getting a http status code of 200 instead of 302 [duplicate]

I've write this function:
def download_mp3(url,name):
opener1 = urllib2.build_opener()
page1 = opener1.open(url)
mp3 = page1.read()
filename = name+'.mp3'
fout = open(filename, 'wb')
fout.write(mp3)
fout.close()
This function take an url and a name both as string.
Then will download and save an mp3 from the url with the name of the variable name.
the url is in the form http://site/download.php?id=xxxx where xxxx is the id of an mp3
if this id does not exist the site redirects me to another page.
So, the question is: how Can I check if this id exist? I've tried to check if the url exist with a function like this:
def checkUrl(url):
p = urlparse(url)
conn = httplib.HTTPConnection(p.netloc)
conn.request('HEAD', p.path)
resp = conn.getresponse()
return resp.status < 400
But it's seems not working..
Thank you

Something like this, and check code:
import urllib2, urllib
class NoRedirectHandler(urllib2.HTTPRedirectHandler):
def http_error_302(self, req, fp, code, msg, headers):
infourl = urllib.addinfourl(fp, headers, req.get_full_url())
infourl.status = code
infourl.code = code
return infourl
http_error_300 = http_error_302
http_error_301 = http_error_302
http_error_303 = http_error_302
http_error_307 = http_error_302
opener = urllib2.build_opener(NoRedirectHandler())
urllib2.install_opener(opener)
response = urllib2.urlopen('http://google.com')
if response.code in (300, 301, 302, 303, 307):
print('redirect')

My answer to this looked like
req = urllib2.Request(url)
try:
response = urllib2.urlopen(url)
except urllib2.HTTPError as e:
# Do something about it
raise HoustonWeHaveAProblem
else:
if response.url != url:
print 'We have redirected!'

Python MJPEG stream don't work outside VLC

I'm creating a streaming mjpeg from some IP cameras,the streaming with VLC works perfectly, just do not work inside a web page and other enterprise devices where other MJPEG streams taken from the internet work perfectly
here is my code:
class Streamer(Thread):
def __init__(self, *args, **kwargs):
self.container = kwargs.pop("container", None)
# self.q = kwargs.pop("queue", None)
# self.stream = kwargs.pop("stream", None)
super(Streamer, self).__init__(*args, **kwargs)
def run(self):
try:
print("dichiarazione handler")
server = ThreadedHTTPServer(('192.168.1.182', 8080), HandlerArguments(self.container))
print('started httpserver...')
server.serve_forever()
except KeyboardInterrupt:
print('^C received, shutting down server')
server.socket.close()
def HandlerArguments(init_args):
class MyHandler(BaseHTTPRequestHandler):
def __init__(self, *args, **kwargs):
self.container = init_args
super(MyHandler, self).__init__(*args, **kwargs)
def do_GET(self):
print("GET")
if self.path.endswith('.mjpeg'):
self.container.stream = True
self.send_response(200)
self.send_header('Pragma:', 'no-cache')
self.send_header('Cache-Control:', 'no-cache')
self.send_header('Content-type:', 'multipart/x-mixed-replace;boundary=--myboundary')
self.end_headers()
try:
while self.container.stream == True:
# print(self.queue.qsize())
print("ciclo")
img = self.container.finalq.get()
ret, img_jpg = cv2.imencode(".jpeg", img, (cv2.IMWRITE_JPEG_QUALITY, 90))
self.wfile.write('--myboundary'.encode('utf-8'))
self.send_header('Content-type:','image/jpeg')
# self.send_header('Content-length:', str(len(img_jpg)))
self.end_headers()
img_str = img_jpg.tostring()
self.wfile.write(img_str)
# self.wfile.write('--myboundary\r\n'.encode('utf-8'))
# self.send_response(200)
# time.sleep(0.05)
except:
self.container.stream = False
while not self.container.finalq.empty():
self.container.finalq.get()
elif self.path.endswith('.jpg'):
self.send_response(200)
self.send_header('Pragma:', 'no-cache')
self.send_header('Cache-Control:', 'no-cache')
self.send_header('Content-type:','image/jpeg')
self.end_headers()
img = self.container.finalq.get()
ret, img_jpg = cv2.imencode(".jpeg", img, (cv2.IMWRITE_JPEG_QUALITY, 90))
# self.send_header('Content-length:', str(len(img_jpg)))
img_str = img_jpg.tostring()
self.wfile.write(img_str)
return MyHandler
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
#class ThreadedHTTPServer(HTTPServer):
"""Handle requests in a separate thread."""
I think the problem is in the request header, in particular this is a stream taken from the internet compared with my stream
curl -I http://85.90.40.19/mjpg/video.mjpg
HTTP/1.0 200 OK
Cache-Control: no-cache
Pragma: no-cache
Expires: Thu, 01 Dec 1994 16:00:00 GMT
Connection: close
Content-Type: multipart/x-mixed-replace; boundary=--myboundary
///////
curl -I http://192.168.1.182:8080/stream.mjpeg
HTTP/1.0 501 Unsupported method ('HEAD')
Server: BaseHTTP/0.6 Python/3.4.0
Date: Mon, 20 Jul 2015 10:00:54 GMT
Content-Type: text/html;charset=utf-8
Connection: close
Content-Length: 474
Especially in my request i see:
Content-Type: text/html;charset=utf-8
while in my code i run
self.send_header('Content-type:','image/jpeg')
Thanks in advance guys :)

curl -I sends a HTTP HEAD request.
In your code you're handling only the HTTP GET requests:
def do_GET(self):
See the error message: HTTP/1.0 501 Unsupported method ('HEAD')
Handle HEAD requests too:
def do_HEAD(self):
and output the expected headers since the clients may issue a HEAD before a GET.

How to force http.client to send chunked-encoding HTTP body in python?

I want to send chunked HTTP body to test my own HTTP server.
So I wrote this python code:
import http.client
body = 'Hello World!' * 80
conn = http.client.HTTPConnection("some.domain.com")
url = "/some_path?arg=true_arg"
conn.request("POST", url, body, {"Transfer-Encoding":"chunked"})
resp = conn.getresponse()
print(resp.status, resp.reason)
I expect the HTTP request's body is transferrd chunked,
but I capture the network package with Wireshark, the HTTP request's body is not transferred chunked.
How to transfer chunked body by http.client lib in python?

OK, I get it.
First, write my own chunked encode function.
Then use putrequest(), putheader(), endheaders() and send() instead of request()
import http.client
def chunk_data(data, chunk_size):
dl = len(data)
ret = ""
for i in range(dl // chunk_size):
ret += "%s\r\n" % (hex(chunk_size)[2:])
ret += "%s\r\n\r\n" % (data[i * chunk_size : (i + 1) * chunk_size])
if len(data) % chunk_size != 0:
ret += "%s\r\n" % (hex(len(data) % chunk_size)[2:])
ret += "%s\r\n" % (data[-(len(data) % chunk_size):])
ret += "0\r\n\r\n"
return ret
conn = http.client.HTTPConnection(host)
url = "/some_path"
conn.putrequest('POST', url)
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
conn.send(chunk_data(body, size_per_chunk).encode('utf-8'))
resp = conn.getresponse()
print(resp.status, resp.reason)
conn.close()

I'd suggest that if you already know the size of your data like in the answer given you could just set the Content-Length and send it all back in one hit, which is kind of what you're doing with the single call to conn.send anyway.
Chunked transfer encoding is most useful when you don't know how big the data is e.g. dynamically generated content. I've modified your code to illustrate:
import httplib
def write_chunk(conn, data):
conn.send("%s\r\n" % hex(len(data))[2:])
conn.send("%s\r\n" % data)
def dynamically_generate_data():
for i in range(80):
yield "hello world"
conn = httplib.HTTPConnection("localhost")
url = "/some_path"
conn.putrequest('POST', url)
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
for new_chunk in dynamically_generate_data():
write_chunk(conn, new_chunk)
conn.send('0\r\n')
resp = conn.getresponse()
print(resp.status, resp.reason)
conn.close()