There is a REST API with which I want to communicate via the request library.
First, I have to authenticate with a username and password (BasicAuth).
After a failed attempt I want to wait 3 seconds and repeat the whole process (maximum 3 times).
In case of success, I get a JSON string as a response.
You can see my current implementation below. The method get_order() is just an example. There are about 12 methods with a similar structure (they use partly POST-, partly GET-requests).
I think that my implementation is very complicated and I hope for suggestions and ideas how to optimize it.
def session_wrapper(self, func, *args, **kwargs) -> Union[bool, dict]:
has_authorization = False
attempts = 3
timeout_s = 3
for attempt in range(attempts):
try:
resp = func(*args, **kwargs)
if resp.status_code == requests.status_codes.codes.UNAUTHORIZED:
self.session.auth = self.login_details
result = self.session.get(f"{self.base_uri}/auth/token", verify=False)
if not result.ok:
print("Cannot reach the authentication url.")
else:
has_authorization = True
raise UnauthorizedException
try:
return resp.json()
except json.JSONDecodeError as e:
print(f"Could not parse json: {e}")
return False
except requests.exceptions.RequestException as e:
print(f"RequestException: {e}")
except UnauthorizedException:
print("The session is expired and needs to be reestablished.")
except Exception as e:
print(f"Something unexpected went wrong: {e}")
time.sleep(timeout_s)
if not has_authorization:
print("The session couldn't be reestablished.")
return False
def get_order(self, id: str) -> Optional[dict]:
payload = {'id': f"{id}"}
return self.session_wrapper(
self.session.get,
f"{self.base_uri}/order",
params=payload,
verify=True
)
def get_customor(self, id: str) -> Optional[dict]:
# ...
Related
The code below works for other OIDs such as hostname (1.3.6.1.2.1.1.5.0) however I am having trouble with pulling the SNMP communities table (list of allowed ips for snmp).
I searched for "communities" for cisco in http://www.mibdepot.com/ and found 5 OIDs. All of which did not pull anything:
.1.3.6.1.4.1.224.2.3.6.3.1
.1.3.6.1.4.1.224.2.3.6.1.0
.1.3.6.1.4.1.224.2.3.6.3
.1.3.6.1.4.1.224.2.3.6.4.1
.1.3.6.1.4.1.224.2.3.6.4
Any guidance on this would be much appreciated. Thank you!
from pysnmp import hlapi
def get(target, oids, credentials, port=161, engine=hlapi.SnmpEngine(), context=hlapi.ContextData()):
handler = hlapi.getCmd(
engine,
credentials,
hlapi.UdpTransportTarget((target, port)),
context,
*construct_object_types(oids)
)
return fetch(handler, 1)[0]
def construct_object_types(list_of_oids):
object_types = []
for oid in list_of_oids:
object_types.append(hlapi.ObjectType(hlapi.ObjectIdentity(oid)))
return object_types
def fetch(handler, count):
result = []
for i in range(count):
try:
error_indication, error_status, error_index, var_binds = next(handler)
if not error_indication and not error_status:
items = {}
for var_bind in var_binds:
items[str(var_bind[0])] = cast(var_bind[1])
result.append(items)
else:
raise RuntimeError('Got SNMP error: {0}'.format(error_indication))
except StopIteration:
break
return result
def cast(value):
try:
return int(value)
except (ValueError, TypeError):
try:
return float(value)
except (ValueError, TypeError):
try:
return str(value)
except (ValueError, TypeError):
pass
return value
def getSNMPCommunities(ip):
try:
communities = get(ip, ['1.3.6.1.4.1.224.2.3.6.1.0'], hlapi.CommunityData('public'))
return communities.get('1.3.6.1.4.1.224.2.3.6.1.0')
except:
return None
snmpCommunities = getSNMPCommunities('10.0.0.1')
print(type(snmpCommunities))
print(snmpCommunities)
This is not possible since SNMP read-only shouldn't have access to this information.
The solution I came up with was to login via SSH and read in the stdout.
I'm trying to get some data from a web page. To speed up this process (they allow me to make 1000 requests per minute), I use ThreadPool.
Since there is a huge amount of data, the process is quite vulnerable to connection fails etc. so I try to log everything I can to be able to detect each mistake I did in code.
The problem is that program sometimes just stops without any exception (it acts like it is running but with no effect - I use PyCharm). I log catched exceptions everywhere I can but I can't see any exception in any log.
I assume that if there were a timeout reached, the exception would be raised and logged.
I've found out where the problem could be. Here is the code:
As a pool, I use: from multiprocessing.pool import ThreadPool as Pool
And lock: from threading import Lock
The download_category function is being used in loop.
def download_category(url):
# some code
#
# ...
log('Create pool...')
_pool = Pool(_workers_number)
with open('database/temp_produkty.txt') as f:
log('Spracovavanie produktov... vytvaranie vlakien...') # I see this in log
for url_product in f:
x = _pool.apply_async(process_product, args=(url_product.strip('\n'), url))
_pool.close()
_pool.join()
log('Presuvanie produktov z temp export do export.csv...') # I can't see this in log
temp_export_to_export_csv()
set_spracovanie_kategorie(url)
except Exception as e:
logging.exception('Got exception on download_one_category: {}'.format(url))
And process_product function:
def process_product(url, cat):
try:
data = get_product_data(url)
except:
log('{}: {} exception while getting product data... #') # I don't see this in log
return
try:
print_to_temp_export(data, cat) # I don't see this in log
except:
log('{}: {} exception while printing to csv... #') # I don't see this in log
raise
LOG function:
def log(text):
now = datetime.now().strftime('%d.%m.%Y %H:%M:%S')
_lock.acquire()
mLib.printToFile('logging/log.log', '{} -> {}'.format(now, text))
_lock.release()
I use logging module too. In this log, I see that probably 8 (number of workers) times request was sent but no answer hasn't been recieved.
EDIT1:
def get_product_data(url):
data = defaultdict(lambda: '-')
root = load_root(url)
try:
nazov = root.xpath('//h1[#itemprop="name"]/text()')[0]
except:
nazov = root.xpath('//h1/text()')[0]
under_block = root.xpath('//h2[#id="lowest-cost"]')
if len(under_block) < 1:
under_block = root.xpath('//h2[contains(text(),"Naj")]')
if len(under_block) < 1:
return False
data['nazov'] = nazov
data['url'] = url
blocks = under_block[0].xpath('./following-sibling::div[#class="shp"]/div[contains(#class,"shp")]')
i = 0
for block in blocks:
i += 1
data['dat{}_men'.format(i)] = eblock.xpath('.//a[#class="link"]/text()')[0]
del root
return data
LOAD ROOT:
class RedirectException(Exception):
pass
def load_url(url):
r = requests.get(url, allow_redirects=False)
if r.status_code == 301:
raise RedirectException
if r.status_code == 404:
if '-q-' in url:
url = url.replace('-q-','-')
mLib.printToFileWOEncoding('logging/neexistujuce.txt','Skusanie {} kategorie...'.format(url))
return load_url(url) # THIS IS NOT LOOPING
else:
mLib.printToFileWOEncoding('logging/neexistujuce.txt','{}'.format(url))
html = r.text
return html
def load_root(url):
try:
html = load_url(url)
except Exception as e:
logging.exception('load_root_exception')
raise
return etree.fromstring(html, etree.HTMLParser())
I'm trying to use the Xively API with python to update a datastream but occasionally I get a 504 error which seems to end my script.
How can I catch that error and more importantly delay and try again so the script can keep going and upload my data a minute or so later?
Here's the block where I'm doing the uploading.
# Upload to Xivity
api = xively.XivelyAPIClient("[MY_API_KEY")
feed = api.feeds.get([MY_DATASTREAM_ID])
now = datetime.datetime.utcnow()
feed.datastreams = [xively.Datastream(id='temps', current_value=tempF, at=now)]
feed.update()
And here's the error I see logged when my script fails:
Traceback (most recent call last):
File "C:\[My Path] \ [My_script].py", line 39, in <module>
feed = api.feeds.get([MY_DATASTREAM_ID])
File "C:\Python34\lib\site-packages\xively_python-0.1.0_rc2-py3.4.egg\xively\managers.py", >line 268, in get
response.raise_for_status()
File "C:\Python34\lib\site-packages\requests-2.3.0-py3.4.egg\requests\models.py", line 795, >in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 504 Server Error: Gateway Time-out
Thanks,
P.S. I've replaced my personal info with [MY_INFO] but obviously the correct data appears in my code.
I usually use a decorator for this:
from functools import wraps
from requests.exceptions import HTTPError
import time
def retry(func):
""" Call `func` with a retry.
If `func` raises an HTTPError, sleep for 5 seconds
and then retry.
"""
#wraps(func)
def wrapper(*args, **kwargs):
try:
ret = func(*args, **kwargs)
except HTTPError:
time.sleep(5)
ret = func(*args, **kwargs)
return ret
return wrapper
Or, if you want to retry more than once:
def retry_multi(max_retries):
""" Retry a function `max_retries` times. """
def retry(func):
#wraps(func)
def wrapper(*args, **kwargs):
num_retries = 0
while num_retries <= max_retries:
try:
ret = func(*args, **kwargs)
break
except HTTPError:
if num_retries == max_retries:
raise
num_retries += 1
time.sleep(5)
return ret
return wrapper
return retry
Then put your code in a function like this
##retry
#retry_multi(5) # retry 5 times before giving up.
def do_call():
# Upload to Xivity
api = xively.XivelyAPIClient("[MY_API_KEY")
feed = api.feeds.get([MY_DATASTREAM_ID])
now = datetime.datetime.utcnow()
feed.datastreams = [xively.Datastream(id='temps', current_value=tempF, at=now)]
feed.update()
You could throw in a try/except statement in a loop that has a sleep timer for however long you want to wait between tries. Something like this:
import time
# Upload to Xivity
api = xively.XivelyAPIClient("[MY_API_KEY")
feed = api.feeds.get([MY_DATASTREAM_ID])
now = datetime.datetime.utcnow()
feed.datastreams = [xively.Datastream(id='temps', current_value=tempF, at=now)]
### Try loop
feed_updated = False
while feed_updated == False:
try:
feed.update()
feed_updated=True
except: time.sleep(60)
EDIT As Dano pointed out, it would be better to have a more specific except statement.
### Try loop
feed_updated = False
while feed_updated == False:
try:
feed.update()
feed_updated=True
except HTTPError: time.sleep(60) ##Just needs more time.
except: ## Otherwise, you have bigger fish to fry
print "Unidentified Error"
## In such a case, there has been some other kind of error.
## Not sure how you prefer this handled.
## Maybe update a log file and quit, or have some kind of notification,
## depending on how you are monitoring it.
Edit a general except statement.
### Try loop
feed_updated = False
feed_update_count = 0
while feed_updated == False:
try:
feed.update()
feed_updated=True
except:
time.sleep(60)
feed_update_count +=1 ## Updates counter
if feed_update_count >= 60: ## This will exit the loop if it tries too many times
feed.update() ## By running the feed.update() once more,
## it should print whatever error it is hitting, and crash
Suppose my django/flask application pulls in information from API's, how can I test that connection exceptions are caught and handled properly?
So for example here is a function that calls an API:
import requests
def call_the_api():
url = 'http://httpbin.org/get'
try:
req = requests.get(url)
if req.json().get('errors'):
logger.warn("API error response")
return {'request_error': 'api_error_response'}
except requests.exceptions.ConnectionError:
logger.warn('ConnectionError')
return {'request_error': 'ConnectionTimeout'}
except requests.exception.Timeout:
logger.warn('API request timed out')
return {'request_error': 'Timeout'}
except Exception, ex:
logger.warn("API request Exception: %s", ex)
return {'request_error': ex}
else:
return req.json()
For testing responses from the API I found mock to be very useful.
def mock_get_request():
response = requests.get.return_value
json_file = 'sample_response.json'
json_file_path = os.path.join(os.path.dirname(__file__), json_file)
with open(json_file_path, 'r') as f:
response.content = response.text = f.read()
response.status_code = 200
response.encoding = 'utf-8'
response.json = lambda: json.loads(response.content.decode(response.encoding))
response.url = u'%s' % args[0]
return response
class TestSuitabilityFunctions(TestCase):
def test_call_the_api(self):
requests.get = MagicMock(side_effect=mock_get_request)
resp = call_the_api()
self.assertEqual(resp.get('url'), "http://httpbin.org/get")
So my question is how would I go about simulating a connection timeout or error?
Untested code but...
def connection_error():
raise requests.exceptions.ConnectionError
class TestSuitabilityFunctions(TestCase):
#patch.object(module_that_youre_testing, "requests")
def test_connection_error(self, mock_requests):
mock_requests.get = MagicMock(side_effect=connection_error)
with self.assertRaises(requests.exceptions.ConnectionError) as cm:
resp = call_the_api()
exception = cm.exception
self.assertEqual(resp, {'request_error': 'ConnectionTimeout'})
... or similar should do the trick. Off the top of my head I can't remember how assertRaises interacts with errors that are caught. Maybe you don't even need the assertRaises part.
I wrote a little tool, that gathers data from facebook, using api. Tool uses multiprocessing, queues and httplib modules. Here, is a part of code:
main process:
def extract_and_save(args):
put_queue = JoinableQueue()
get_queue = Queue()
for index in range(args.number_of_processes):
process_name = u"facebook_worker-%s" % index
grabber = FacebookGrabber(get_queue=put_queue, put_queue=get_queue, name=process_name)
grabber.start()
friend_list = get_user_friends(args.default_user_id, ["id"])
for index, friend_id in enumerate(friend_list):
put_queue.put(friend_id)
put_queue.join()
if not get_queue.empty():
... save to database ...
else:
logger.info(u"There is no data to save")
worker process:
class FacebookGrabber(Process):
def __init__(self, *args, **kwargs):
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2)
self.get_queue = kwargs.pop("get_queue")
self.put_queue = kwargs.pop("put_queue")
super(FacebookGrabber, self).__init__(*args, **kwargs)
self.daemon = True
def run(self):
while True:
friend_id = self.get_queue.get(block=True)
try:
friend_obj = self.get_friend_obj(friend_id)
except Exception, e:
logger.info(u"Friend id %s: facebook responded with an error (%s)", friend_id, e)
else:
if friend_obj:
self.put_queue.put(friend_obj)
self.get_queue.task_done()
common code:
def get_json_from_facebook(connection, url, kwargs=None):
url_parts = list(urlparse.urlparse(url))
query = dict(urlparse.parse_qsl(url_parts[4]))
if kwargs:
query.update(kwargs)
url_parts[4] = urllib.urlencode(query)
url = urlparse.urlunparse(url_parts)
try:
connection.request("GET", url)
except Exception, e:
print "<<<", e
response = connection.getresponse()
data = json.load(response)
return data
This code perfectly works on Ubuntu. But when I tried to run it on Windows 7 I got message "There is no data to save". The problem is here:
try:
connection.request("GET", url)
except Exception, e:
print "<<<", e
I get next error: <<< a float is required
Do anybody know, how to fix this problem?
Python version: 2.7.5
One of the "gotcha's" that occasionally happens with socket timeout values is that most operating systems expect them as floats. I believe this has been accounted for with later versions of the linux kernel.
Try changing:
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2)
to:
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2.0)
That's 2 seconds, by the way. Default is typically 5 seconds. Might be a little low.