I have a Python class that sends payloads to AWS with boto3 and requests library. However, sometimes the http requests fail with various codes, so I wanted to write a wrapper function inside the class that will retry to send the payload 5 times if it gets certain codes, and raise an exception if it completely fails. Here is the class method(assume method calls work as expected):
import requests
from boto3 import Session
def update_status(self, status):
payload = status
auth = self.sign_request()
response = requests.patch(self.url, auth=auth, data=payload)
status_code = response.status_code
response_text = response.text
if not response.ok:
logging.error("Failed updating status of request: " + str(
{'host': self.host, 'region': self.region,
'service': self.service, 'url': self.url, 'status': str(status)}))
raise IOError('Update training status failed with status code: ' + str(status_code) + '\n' + response_text)
logging.info("Updated status")
Sometimes this api call will fail with status 504. I would like to write a wrapper retry method around this class method that will by default retry 5 times with a wait of retry^2 between every try, and exit the loop if it's a success with code 200.
I found this code which seems to be along the lines of what I would use, I'm just not sure how to wrap my current method inside this and call it:
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
def requests_retry_session(
retries=5,
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
session=None,
):
session = session or requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
Issue with the above code is it is using requests.session and returning it while my class is already using boto3.Session. Any help would be appreciated!
I'd try something like this:
import time
import requests
from functools import wraps
import logging
logging.basicConfig(level=logging.DEBUG)
def retry(delay=10, retries=4):
def retry_decorator(f):
#wraps(f)
def f_retry(*args, **kwargs):
opt_dict = {'retries': retries, 'delay': delay}
while opt_dict['retries'] > 1:
try:
return f(*args, **kwargs)
except Exception as e:
msg = "Exception: {}, Retrying in {} seconds...".format(e, delay)
print(msg)
time.sleep(opt_dict['delay'])
opt_dict['retries'] -= 1
return f(*args, **kwargs)
return f_retry
return retry_decorator
class YourClass:
# JUST MOCK FOR PROOF OF CONCEPT
url = 'YOUR URL'
status = 'YOUR STATUS'
def sign_request(self):
return ''
host = 'YOUR HOST'
region = 'YOUR REGION'
service = 'YOUR SERVICE'
# MOCK END
def update_status(self, status):
payload = status
auth = self.sign_request()
#retry(1, 5)
def get_status():
response = requests.patch(self.url, auth=auth, data=payload)
if not response.ok:
logging.error("Failed updating status of request: " + str(
{'host': self.host, 'region': self.region,
'service': self.service, 'url': self.url, 'status': str(status)}))
raise IOError('Update training status failed with status code: ' + str(response.status_code) + '\n' + response.text)
return response
res = get_status()
status_code = res.status_code
response_text = res.text
logging.info("Updated status")
x = YourClass()
x.url = 'https://httpstat.us/200'
x.update_status('')
x.url = 'https://httpstat.us/504'
x.update_status('')
Of course you may want to adjust it to your needs.
Related
I'm trying to set up a simple reverse proxy with Falcon in Python.
I have:
import falcon
import requests
class ReverseProxyResource:
def on_get(self, req, resp, text=None):
print("GET")
if(text):
destination = "[destination_url]/" + text
else:
destination = "[destination_url]"
result = requests.get(destination)
resp.body = result.text
resp.status = result.status_code
def on_post(self, req, resp, text=None):
print("POST")
if(text):
destination = "[destination_url]/" + text
else:
destination = "[destination_url]"
result = requests.post(destination, data=req.bounded_stream.read())
resp.body = result.text
resp.status = result.status_code
proxy_api = application = falcon.API()
proxy_api.req_options.auto_parse_form_urlencoded = True
proxy_api.add_route('/{text}', ReverseProxyResource())
proxy_api.add_route('/', ReverseProxyResource())
Get requests to the proxy are returned correctly.
However, Post requests are only returned a 404 error from the api. The "POST" print statement is not shown, indicating on_post isn't called at all. (The post requests only included Header Content-Type: application/json and a simple JSON body, which work correctly when called directly against the destination url)
EDIT: Interestingly enough, if I change GET call in postman to POST (ie: no body, headers, or anything else added) on_post() is called when I hit the endpoint. So it seems like an issue where post requests that contain a body are being automtically 404'ed without calling on_post()
Try adding user agent and content type before making the post call
headers = {"Content-Type": "text/plain", "User-Agent": "PostmanRuntime/7.30.0"}
result = requests.post(url = destination, data=req.bounded_stream.read(), headers=headers)
below code works for me
import falcon
import requests
class ReverseProxyResource:
def on_get(self, req, resp, text=None):
print("GET")
if(text):
destination = "https://cat-fact.herokuapp.com/" + text
else:
destination = "https://cat-fact.herokuapp.com/facts/"
result = requests.get(destination)
resp.body = result.text
resp.status = result.status_code
def on_post(self, req, resp, text=None):
print("POST")
if(text):
destination = "https://dummy.restapiexample.com/api/v1/"+ text
else:
destination = "https://dummy.restapiexample.com/api/v1/create"
headers = {"Content-Type": "text/plain", "User-Agent": "PostmanRuntime/7.30.0"}
result = requests.post(url = destination, data=req.bounded_stream.read(), headers=headers)
resp.text = result.text
resp.status = result.status_code
proxy_api = application = falcon.App()
proxy_api.req_options.auto_parse_form_urlencoded = True
proxy_api.add_route('/{text}', ReverseProxyResource())
proxy_api.add_route('/', ReverseProxyResource())
How do I create a request to rentry.co in order to create pastes?
I've tried to solve this in Python but I get the following response:
403 reason: Forbidden ...
I tried changing the URL and adding my cookie.
My code looks currently as follows.
import requests
text = "Hello World!"
data = {"text":text}
r = requests.post("https://rentry.co/api", data=data)
print(f"status code: {r.status_code}")
print(f"reason: {r.reason}") ```
try this
#!/usr/bin/env python3
import http.cookiejar
import sys
import urllib.parse
import urllib.request
from http.cookies import SimpleCookie
from json import loads as json_loads
_headers = {"Referer": 'https://rentry.co'}
class UrllibClient:
"""Simple HTTP Session Client, keeps cookies."""
def __init__(self):
self.cookie_jar = http.cookiejar.CookieJar()
self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cookie_jar))
urllib.request.install_opener(self.opener)
def get(self, url, headers={}):
request = urllib.request.Request(url, headers=headers)
return self._request(request)
def post(self, url, data=None, headers={}):
postdata = urllib.parse.urlencode(data).encode()
request = urllib.request.Request(url, postdata, headers)
return self._request(request)
def _request(self, request):
response = self.opener.open(request)
response.status_code = response.getcode()
response.data = response.read().decode('utf-8')
return response
def new(url, edit_code, text):
client, cookie = UrllibClient(), SimpleCookie()
cookie.load(vars(client.get('https://rentry.co'))['headers']['Set-Cookie'])
csrftoken = cookie['csrftoken'].value
payload = {
'csrfmiddlewaretoken': csrftoken,
'url': url,
'edit_code': edit_code,
'text': text
}
return json_loads(client.post('https://rentry.co/api/new', payload, headers=_headers).data)
def get_rentry_link(text):
url, edit_code = '', ''
response = new(url, edit_code, text)
if response['status'] != '200':
print('error: {}'.format(response['content']))
try:
for i in response['errors'].split('.'):
i and print(i)
sys.exit(1)
except:
sys.exit(1)
else:
pastebin_link = response['url']
print('Url: {}\nEdit code: {}'.format(response['url'], response['edit_code']))
return pastebin_link
if __name__ == '__main__':
link_list = ['https://stackoverflow.com/', 'https://www.youtube.com/', 'https://www.google.com/']
pastebin_link = get_rentry_link('\n'.join(map(str, link_list)))
I have been trying to optimize my code, which fetches the data from a given ip address and has pagination involved in it. I have tried applying multiprocessing/multithreading but can't able to implement them successfully.
My goal of using multiprocessing is to reduce the execution time. Please help me out in this.
has_pagination = True
session_obj = some_value #like this <requests.sessions.Session object at 0x7fac952c4fa0>
headers = {'X-XSRF-TOKEN':token, 'Content-type':'application/json', 'Accept':'application/json', 'Cookie':jsessionid}
while has_pagination:
url = f"https://{self.ip}/data/page?count=100&scrollId={scroll_id}"
response = session_object.get(url=url, headers=headers, verify=False)
try:
resp_json = response.json()
data = resp_json['data']
has_pagination = resp_json['pageInfo']['hasMoreData']
except Exception as e:
print(f'status code: {response.status_code}, {e}')
# # Logging out if session is established
logout_url = f'https://{self.ip}/logout?nocache=123456'
logout_resp = session_obj.get(url=logout_url, headers=headers, verify=False, allow_redirects=False)
print(f'Logging out. Response code: {str(logout_resp.status_code)}')
# # Relogin
print(f'2. Trying to re-login with connnection details Token: {str(token)}. Jsessionid: {str(jsessionid)}. Headers: {str(headers)}.')
login_result = self.login()
if login_result == False:
return False
else:
number_of_relogin += 1
session_obj, token, jsessionid = login_result[0], login_result[1], login_result[2]
headers = {'X-XSRF-TOKEN':token, 'Content-type':'application/json', 'Accept':'application/json', 'Cookie':jsessionid}
logger.info(f'2. New session established with Token: {str(token)}. Jsessionid: {str(jsessionid)}. Headers: {str(headers)}')
I have this python function and i wish to execute this using lambda handler function hence I have written this code. When I execute in Pycharm I don't see any output in console. Can someone guide what is the problem with below code?
import json
from json import loads
import requests
from requests import exceptions
from requests.auth import HTTPBasicAuth
def lambda_handler(event, context):
test_post_headers_body_json()
return {"statusCode": 200, "body": json.dumps("Hello from Lambda!")}
def test_post_headers_body_json():
client_id = "WJRYDHNGROIZHL8B"
client_secret = "V5VXK6FLG1YI0GD2XY3H"
user = "automation-store-admin1#abc.com"
password = "c0Ba5PBdvVl2"
access_point = "https://api.platform.abc.com/auth/oauth/token"
grant_type = "password"
headers = {"Content-Type": "application/x-www-form-urlencoded"}
# auth = auth.HTTPBasicAuth(client_id, client_secret)
data = {"grant_type": grant_type, "username": user, "password": password}
resp = None
try:
resp = requests.post(
access_point,
auth=HTTPBasicAuth(client_id, client_secret),
data=data,
headers=headers,
)
except exceptions.ConnectionError:
exit(1)
if resp.status_code == 200:
resp = loads(resp.text)
if "access_token" in resp:
print(resp["access_token"])
exit(0)
exit(1)
It is normal because when running you code, Python only declare the function not using it. You should add a __main__ entry point at the end of your file:
import json
from json import loads
import requests
from requests import exceptions
from requests.auth import HTTPBasicAuth
def lambda_handler(event, context):
test_post_headers_body_json()
return {"statusCode": 200, "body": json.dumps("Hello from Lambda!")}
def test_post_headers_body_json():
client_id = "WJRYDHNGROIZHL8B"
client_secret = "V5VXK6FLG1YI0GD2XY3H"
user = "automation-store-admin1#abc.com"
password = "c0Ba5PBdvVl2"
access_point = "https://api.platform.abc.com/auth/oauth/token"
grant_type = "password"
headers = {"Content-Type": "application/x-www-form-urlencoded"}
# auth = auth.HTTPBasicAuth(client_id, client_secret)
data = {"grant_type": grant_type, "username": user, "password": password}
resp = None
try:
resp = requests.post(
access_point,
auth=HTTPBasicAuth(client_id, client_secret),
data=data,
headers=headers,
)
except exceptions.ConnectionError:
exit(1)
if resp.status_code == 200:
resp = loads(resp.text)
if "access_token" in resp:
print(resp["access_token"])
exit(0)
exit(1)
# added part
if __name__ == '__main__':
test_post_headers_body_json()
I have following code
import unittest
import requests
import time
import json
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
token = ''
headers = {'Content-Type': 'application/json',
'Token': token}
class ChromeSearch(unittest.TestCase):
#classmethod
def setUpClass(cls):
cls.driver = webdriver.Chrome()
driver = cls.driver
base_url = 'http://127.0.0.1:8080/'
driver.get(base_url)
print("Generating Token")
usernameStr = 'user1a'
passwordStr = 'user'
response = requests.get('http://127.0.0.1:8080/api/auth/token', auth=(usernameStr, passwordStr))
print(response)
data = response.json()
print(data)
if response.status_code == 200 and data["status"] == "SUCCESS":
token = data["token"]
else:
token = None
print("The request was not successful.")
print(token)
def test_update_user_infp(self):
print("Planning to update user's info")
datap = {'firstname': 'Newfirstname',
'lastname': 'Newname',
'phone': '111000111'}
r = requests.put('http://127.0.0.1:8080/api/users/user1a', data=json.dumps(datap), headers=headers)
print(r.text)
def test_get_users(self):
print("Planning to get users")
r = requests.get('http://127.0.0.1:8080/api/users', headers=headers)
print"Get request for api Users"
print(r.text)
#classmethod
def tearDownClass(cls):
cls.driver.close()
if __name__ == "__main__":
unittest.main()
Log are as below, if you see in logs it says Generating Token Planning to get users but gives a . and .Planning to update user's info authentication required, why ? It just works fine if I dont use class and def.
DevTools listening on ws://127.0.0.1:55429/devtools/browser/cc53d84c-3847-40a3-b3fb-159892c71ac9
Generating Token
<Response [200]>
{u'status': u'SUCCESS', u'token': u'MjU5NzMzMDY1NTU1NzYxMTE4NjQ4NDgxMTc0OTkyMjI4NTg0NTE5'}
MjU5NzMzMDY1NTU1NzYxMTE4NjQ4NDgxMTc0OTkyMjI4NTg0NTE5
Planning to get users
Get request for api Users
{"payload":["user1a"],"status":"SUCCESS"}
.Planning to update user's info
{"message":"Token authentication required","status":"FAILURE"}
.
----------------------------------------------------------------------
Ran 2 tests in 5.938s
OK
Following up from the comments section:
Here's a sample answer on how to declare and initialize token in one method and access in another.
import unittest
class Example(unittest.TestCase):
#classmethod
def setUpClass(cls):
print("Print me only once")
cls.token = 100 # Initilze your token here using auth route
def test1(self):
print(1)
print('Token = ', self.token) # Prints Token = 100
def test2(myself): # Although self is preferred
print(2)
print('Token = ', myself.token) # Prints Token = 100
unittest.main()