I'm trying to use the Xively API with python to update a datastream but occasionally I get a 504 error which seems to end my script.
How can I catch that error and more importantly delay and try again so the script can keep going and upload my data a minute or so later?
Here's the block where I'm doing the uploading.
# Upload to Xivity
api = xively.XivelyAPIClient("[MY_API_KEY")
feed = api.feeds.get([MY_DATASTREAM_ID])
now = datetime.datetime.utcnow()
feed.datastreams = [xively.Datastream(id='temps', current_value=tempF, at=now)]
feed.update()
And here's the error I see logged when my script fails:
Traceback (most recent call last):
File "C:\[My Path] \ [My_script].py", line 39, in <module>
feed = api.feeds.get([MY_DATASTREAM_ID])
File "C:\Python34\lib\site-packages\xively_python-0.1.0_rc2-py3.4.egg\xively\managers.py", >line 268, in get
response.raise_for_status()
File "C:\Python34\lib\site-packages\requests-2.3.0-py3.4.egg\requests\models.py", line 795, >in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 504 Server Error: Gateway Time-out
Thanks,
P.S. I've replaced my personal info with [MY_INFO] but obviously the correct data appears in my code.
I usually use a decorator for this:
from functools import wraps
from requests.exceptions import HTTPError
import time
def retry(func):
""" Call `func` with a retry.
If `func` raises an HTTPError, sleep for 5 seconds
and then retry.
"""
#wraps(func)
def wrapper(*args, **kwargs):
try:
ret = func(*args, **kwargs)
except HTTPError:
time.sleep(5)
ret = func(*args, **kwargs)
return ret
return wrapper
Or, if you want to retry more than once:
def retry_multi(max_retries):
""" Retry a function `max_retries` times. """
def retry(func):
#wraps(func)
def wrapper(*args, **kwargs):
num_retries = 0
while num_retries <= max_retries:
try:
ret = func(*args, **kwargs)
break
except HTTPError:
if num_retries == max_retries:
raise
num_retries += 1
time.sleep(5)
return ret
return wrapper
return retry
Then put your code in a function like this
##retry
#retry_multi(5) # retry 5 times before giving up.
def do_call():
# Upload to Xivity
api = xively.XivelyAPIClient("[MY_API_KEY")
feed = api.feeds.get([MY_DATASTREAM_ID])
now = datetime.datetime.utcnow()
feed.datastreams = [xively.Datastream(id='temps', current_value=tempF, at=now)]
feed.update()
You could throw in a try/except statement in a loop that has a sleep timer for however long you want to wait between tries. Something like this:
import time
# Upload to Xivity
api = xively.XivelyAPIClient("[MY_API_KEY")
feed = api.feeds.get([MY_DATASTREAM_ID])
now = datetime.datetime.utcnow()
feed.datastreams = [xively.Datastream(id='temps', current_value=tempF, at=now)]
### Try loop
feed_updated = False
while feed_updated == False:
try:
feed.update()
feed_updated=True
except: time.sleep(60)
EDIT As Dano pointed out, it would be better to have a more specific except statement.
### Try loop
feed_updated = False
while feed_updated == False:
try:
feed.update()
feed_updated=True
except HTTPError: time.sleep(60) ##Just needs more time.
except: ## Otherwise, you have bigger fish to fry
print "Unidentified Error"
## In such a case, there has been some other kind of error.
## Not sure how you prefer this handled.
## Maybe update a log file and quit, or have some kind of notification,
## depending on how you are monitoring it.
Edit a general except statement.
### Try loop
feed_updated = False
feed_update_count = 0
while feed_updated == False:
try:
feed.update()
feed_updated=True
except:
time.sleep(60)
feed_update_count +=1 ## Updates counter
if feed_update_count >= 60: ## This will exit the loop if it tries too many times
feed.update() ## By running the feed.update() once more,
## it should print whatever error it is hitting, and crash
Related
There is a REST API with which I want to communicate via the request library.
First, I have to authenticate with a username and password (BasicAuth).
After a failed attempt I want to wait 3 seconds and repeat the whole process (maximum 3 times).
In case of success, I get a JSON string as a response.
You can see my current implementation below. The method get_order() is just an example. There are about 12 methods with a similar structure (they use partly POST-, partly GET-requests).
I think that my implementation is very complicated and I hope for suggestions and ideas how to optimize it.
def session_wrapper(self, func, *args, **kwargs) -> Union[bool, dict]:
has_authorization = False
attempts = 3
timeout_s = 3
for attempt in range(attempts):
try:
resp = func(*args, **kwargs)
if resp.status_code == requests.status_codes.codes.UNAUTHORIZED:
self.session.auth = self.login_details
result = self.session.get(f"{self.base_uri}/auth/token", verify=False)
if not result.ok:
print("Cannot reach the authentication url.")
else:
has_authorization = True
raise UnauthorizedException
try:
return resp.json()
except json.JSONDecodeError as e:
print(f"Could not parse json: {e}")
return False
except requests.exceptions.RequestException as e:
print(f"RequestException: {e}")
except UnauthorizedException:
print("The session is expired and needs to be reestablished.")
except Exception as e:
print(f"Something unexpected went wrong: {e}")
time.sleep(timeout_s)
if not has_authorization:
print("The session couldn't be reestablished.")
return False
def get_order(self, id: str) -> Optional[dict]:
payload = {'id': f"{id}"}
return self.session_wrapper(
self.session.get,
f"{self.base_uri}/order",
params=payload,
verify=True
)
def get_customor(self, id: str) -> Optional[dict]:
# ...
i need a script to make it like a cpanel checker, with more than 1 url and the url is stored in a txt file.
usage : python script.py list.txt
format in file list.txt : https://demo.cpanel.net:2083|democom|DemoCoA5620
this is my code but it doesn't work, can someone help me?
Thanks.
import requests, sys
from multiprocessing.dummy import Pool as ThreadPool
try:
with open(sys.argv[1], 'r') as f:
list_data = [line.strip() for line in f if line.strip()]
except IOError:
pass
def cpanel(url):
try:
data = {'user':'democom', 'pass':'DemoCoA5620'}
r = requests.post(url, data=data)
if r.status_code==200:
print "login success"
else:
print "login failed"
except:
pass
def chekers(url):
try:
cpanel(url)
except:
pass
def Main():
try:
start = timer()
pp = ThreadPool(25)
pr = pp.map(chekers, list_data)
print('Time: ' + str(timer() - start) + ' seconds')
except:
pass
if __name__ == '__main__':
Main()
I fixed your code in a way that it will return an actual array containing a boolean array indicating the success of the cpanel function.
from __future__ import print_function
import requests
from multiprocessing.pool import ThreadPool
try:
list_data = ["https://demo.cpanel.net:2083|democom|DemoCoA5620",
"https://demo.cpanel.net:2083|UserDoesNotExist|WRONGPASSWORD",
]
except IOError:
pass
def cpanel(url):
try:
# try to split that url to get username / password
try:
url, username, password = url.split('|')
except Exception as e:
print("Url {} seems to have wrong format. Concrete error: {}".format(url, e))
return False
# build the correct url
url += '/login/?login_only=1'
# build post parameters
params = {'user': username,
'pass': password}
# make request
r = requests.post(url, params)
if r.status_code==200:
print("login for user {} success".format(username))
return True
else:
print("login for user {} failed due to Status Code {} and message \"{}\"".format(username, r.status_code, r.reason))
return False
except Exception as e:
print("Error occured for url {} ".format(e))
return False
def chekers(url):
return cpanel(url)
def Main():
try:
# start = timer()
pp = ThreadPool(1)
pr = pp.map(chekers, list_data)
print(pr)
# print('Time: ' + str(timer() - start) + ' seconds')
except:
pass
if __name__ == '__main__':
Main()
Output:
login for user democom success
login for user UserDoesNotExist failed due to Status Code 401 and message "Access Denied"
[True, False]
Be aware that I replaced your file read operation by some fixed urls.
Since you use request.post I guess you actually want to POST something to that urls. Your code does not do that. If you just want to send a request, use the requests.get method.
See the official documentation for the requests packet: https://2.python-requests.org/en/master/user/quickstart/#make-a-request for more details.
Also note that
"but it doesn't work"
is NOT a question.
I'm trying to get some data from a web page. To speed up this process (they allow me to make 1000 requests per minute), I use ThreadPool.
Since there is a huge amount of data, the process is quite vulnerable to connection fails etc. so I try to log everything I can to be able to detect each mistake I did in code.
The problem is that program sometimes just stops without any exception (it acts like it is running but with no effect - I use PyCharm). I log catched exceptions everywhere I can but I can't see any exception in any log.
I assume that if there were a timeout reached, the exception would be raised and logged.
I've found out where the problem could be. Here is the code:
As a pool, I use: from multiprocessing.pool import ThreadPool as Pool
And lock: from threading import Lock
The download_category function is being used in loop.
def download_category(url):
# some code
#
# ...
log('Create pool...')
_pool = Pool(_workers_number)
with open('database/temp_produkty.txt') as f:
log('Spracovavanie produktov... vytvaranie vlakien...') # I see this in log
for url_product in f:
x = _pool.apply_async(process_product, args=(url_product.strip('\n'), url))
_pool.close()
_pool.join()
log('Presuvanie produktov z temp export do export.csv...') # I can't see this in log
temp_export_to_export_csv()
set_spracovanie_kategorie(url)
except Exception as e:
logging.exception('Got exception on download_one_category: {}'.format(url))
And process_product function:
def process_product(url, cat):
try:
data = get_product_data(url)
except:
log('{}: {} exception while getting product data... #') # I don't see this in log
return
try:
print_to_temp_export(data, cat) # I don't see this in log
except:
log('{}: {} exception while printing to csv... #') # I don't see this in log
raise
LOG function:
def log(text):
now = datetime.now().strftime('%d.%m.%Y %H:%M:%S')
_lock.acquire()
mLib.printToFile('logging/log.log', '{} -> {}'.format(now, text))
_lock.release()
I use logging module too. In this log, I see that probably 8 (number of workers) times request was sent but no answer hasn't been recieved.
EDIT1:
def get_product_data(url):
data = defaultdict(lambda: '-')
root = load_root(url)
try:
nazov = root.xpath('//h1[#itemprop="name"]/text()')[0]
except:
nazov = root.xpath('//h1/text()')[0]
under_block = root.xpath('//h2[#id="lowest-cost"]')
if len(under_block) < 1:
under_block = root.xpath('//h2[contains(text(),"Naj")]')
if len(under_block) < 1:
return False
data['nazov'] = nazov
data['url'] = url
blocks = under_block[0].xpath('./following-sibling::div[#class="shp"]/div[contains(#class,"shp")]')
i = 0
for block in blocks:
i += 1
data['dat{}_men'.format(i)] = eblock.xpath('.//a[#class="link"]/text()')[0]
del root
return data
LOAD ROOT:
class RedirectException(Exception):
pass
def load_url(url):
r = requests.get(url, allow_redirects=False)
if r.status_code == 301:
raise RedirectException
if r.status_code == 404:
if '-q-' in url:
url = url.replace('-q-','-')
mLib.printToFileWOEncoding('logging/neexistujuce.txt','Skusanie {} kategorie...'.format(url))
return load_url(url) # THIS IS NOT LOOPING
else:
mLib.printToFileWOEncoding('logging/neexistujuce.txt','{}'.format(url))
html = r.text
return html
def load_root(url):
try:
html = load_url(url)
except Exception as e:
logging.exception('load_root_exception')
raise
return etree.fromstring(html, etree.HTMLParser())
so I have the below loop that works great until it hits certain hosts that simply cause a connection error. Unfortunately, instead of skipping over these instances, it causes the script to crash. I know to catch and avoid this exception it is best to through the troubling statement (serveroutput = tn.read_until(b'STARTTLS')) in a try: except block. I can do that, however I am not sure how to catch the error and tell it to move on. If I add a break, it will break the loop and cause the script to stop prematurely anyway. How can I continue iterating through j? I've heard I can use 'continue' as a way to continue iteration, but am I even catching the right exception here?
My Code:
def getServers():
fp = open("mailserverdata.csv", "r")
pt = from_csv(fp)
fp.close()
domains = txt_domains.get(1.0, 'end').splitlines()
symbols = txt_symbols.get(1.0, 'end').splitlines()
for x in range(len(domains)):
#Start Get MX Record
answers = dns.resolver.query(str(domains[x]), 'MX')
#End Get MX Record
#Start Get Employees
if symbols[x]!='':
xml = urllib.request.urlopen('https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.stocks%20where%20symbol%3D%22'+symbols[x]+'%22&diagnostics=true&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys')
dom = parse(xml)
numemployees = dom.getElementsByTagName('FullTimeEmployees')
if len(numemployees)!=0:
numemployees = numemployees[0].firstChild.nodeValue
else:
numemployees = 0
#End Get Employees
j=0
tlsbool = 'N'
verified = 'N'
for rdata in answers:
#Start Trim Domains
output = str(rdata.exchange)
output = output[:len(output)-1]
print(output)
#End Trim Domains
#Start Telnet
tn = telnetlib.Telnet(output,25)
tn.write(b'ehlo a.com\r\n')
serveroutput = tn.read_until(b'STARTTLS')
checkvar = "STARTTLS"
for checkvar in serveroutput:
tlsbool = 'Y'
break
#End Telnet
#Start verification
if output.find(domains[x])>-1:
verified = 'Y'
#End verification
if j==0:
pt.add_row([domains[x],output,tlsbool,numemployees,verified])
else:
pt.add_row(['',output,tlsbool,'',verified])
j = j + 1
txt_tableout.delete(1.0, 'end')
txt_tableout.insert('end',pt)
root.ptglobal = pt
Try Catch Code:
try:
serveroutput = tn.read_until(b'STARTTLS')
except SocketError as e:
if e.errno != errno.ECONNRESET:
raise # Not error we are looking for
pass # Handle error here.
Full Stack Error:
Traceback (most recent call last):
File "C:\Python34\lib\tkinter\__init__.py", line 1487, in __call__
return self.func(*args)
File "C:\Users\kylec\Desktop\Data Motion\Mail Server Finder\mailserverfinder.py", line 58, in getServers
serveroutput = tn.read_until(b'STARTTLS')
File "C:\Python34\lib\telnetlib.py", line 317, in read_until
self.fill_rawq()
File "C:\Python34\lib\telnetlib.py", line 526, in fill_rawq
buf = self.sock.recv(50)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
UPDATE:
I tried the following code but I received the following error.
Code:
try:
serveroutput = tn.read_until(b'STARTTLS')
except tn.ConnectionsResetError:
continue
Error:
AttributeError: 'Telnet' object has no attribute 'ConnectionsResetError'
What ended up working for me in the end was a modification of what #user3570335 had suggested.
try:
serveroutput = tn.read_until(b'STARTTLS')
except Exception as e:
tlsbool = '?'
I wrote a little tool, that gathers data from facebook, using api. Tool uses multiprocessing, queues and httplib modules. Here, is a part of code:
main process:
def extract_and_save(args):
put_queue = JoinableQueue()
get_queue = Queue()
for index in range(args.number_of_processes):
process_name = u"facebook_worker-%s" % index
grabber = FacebookGrabber(get_queue=put_queue, put_queue=get_queue, name=process_name)
grabber.start()
friend_list = get_user_friends(args.default_user_id, ["id"])
for index, friend_id in enumerate(friend_list):
put_queue.put(friend_id)
put_queue.join()
if not get_queue.empty():
... save to database ...
else:
logger.info(u"There is no data to save")
worker process:
class FacebookGrabber(Process):
def __init__(self, *args, **kwargs):
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2)
self.get_queue = kwargs.pop("get_queue")
self.put_queue = kwargs.pop("put_queue")
super(FacebookGrabber, self).__init__(*args, **kwargs)
self.daemon = True
def run(self):
while True:
friend_id = self.get_queue.get(block=True)
try:
friend_obj = self.get_friend_obj(friend_id)
except Exception, e:
logger.info(u"Friend id %s: facebook responded with an error (%s)", friend_id, e)
else:
if friend_obj:
self.put_queue.put(friend_obj)
self.get_queue.task_done()
common code:
def get_json_from_facebook(connection, url, kwargs=None):
url_parts = list(urlparse.urlparse(url))
query = dict(urlparse.parse_qsl(url_parts[4]))
if kwargs:
query.update(kwargs)
url_parts[4] = urllib.urlencode(query)
url = urlparse.urlunparse(url_parts)
try:
connection.request("GET", url)
except Exception, e:
print "<<<", e
response = connection.getresponse()
data = json.load(response)
return data
This code perfectly works on Ubuntu. But when I tried to run it on Windows 7 I got message "There is no data to save". The problem is here:
try:
connection.request("GET", url)
except Exception, e:
print "<<<", e
I get next error: <<< a float is required
Do anybody know, how to fix this problem?
Python version: 2.7.5
One of the "gotcha's" that occasionally happens with socket timeout values is that most operating systems expect them as floats. I believe this has been accounted for with later versions of the linux kernel.
Try changing:
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2)
to:
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2.0)
That's 2 seconds, by the way. Default is typically 5 seconds. Might be a little low.