Here is part of my code:
class KafkaProducer:
def __init__(self):
pass
bootstrap_server_host = system_config.get_kafka_bootstrap_server()
producer = Producer({'bootstrap.servers': bootstrap_server_host, "log.connection.close":False})
#classmethod
def send(cls, topic, key, value, data_type=None, uid=None):
try:
data = {"data": value, "createTime": long(time.time() * 1000)}
if data_type is not None:
data["type"] = int(data_type)
if uid is not None:
data["uid"] = long(uid)
cls.producer.produce(topic, json.dumps(data), key)
cls.producer.poll(0)
except BufferError as e:
logger.error('%% Local producer queue is full ' \
'(%d messages awaiting delivery): try again\n' %
len(cls.producer))
raise e
class new_application_scanner():
#classmethod
def scan_new_application(cls):
db_source = None
try:
db_source = DBConnector().connect()
db_cur = db_source.cursor()
...
KafkaProducer.send("RiskEvent", str(uid),
{"uid": uid, "country_id": user_info[1], "event_id": constant.RISK_EVENT_NEW_APPLICATION})
...
except Exception as e:
logger.error(traceback.format_exc())
finally:
if db_source is not None:
db_source.close()
def run_scan_new_application():
while is_scan_new_application_active:
try:
logging.info("scan_new_application starts at %s",time.time())
new_application_scanner.scan_new_application()
logging.info("scan_new_application ends at %s", time.time())
except Exception as e:
logging.error("new_application_scanner Error:%s",format(e))
logging.error(traceback.format_exc())
time.sleep(10)
t1 = threading.Thread(target=run_scan_new_application, name='run_scan_new_application', args=([]))
t1.start()
I have a kafka group of two servers. when I restart two servers one by one ,KafkaProducer.send() throws KafkaException(maybe some bug in confluent_kafka), and there are some exception logs.
The strange thing is the Exception continues to throw out of scan_new_application and there are exception logs in run_scan_new_application too. Even the thread stopped.Here is the exception logs:
2017-12-21 07:11:49 INFO pre_risk_control_flow.py:71 pid-16984 scan_new_application starts at 1513840309.6
2017-12-21 07:11:49 ERROR new_application_scan.py:165 pid-16984 Traceback (most recent call last):
File "/home/ubuntu/data/code/risk/Feature_Engine/data_retrive/pre_risk_control_flow/new_application_scan.py", line 163, in scan_new_application
{"uid": uid, "country_id": user_info[1], "event_id": constant.RISK_EVENT_NEW_APPLICATION})
File "/home/ubuntu/data/code/risk/Feature_Engine/data_retrive/kafka_client/Producer.py", line 27, in send
cls.producer.produce(topic, json.dumps(data), key)
KafkaException: KafkaError{code=_UNKNOWN_TOPIC,val=-188,str="Unable to produce message: Local: Unknown topic"}
2017-12-21 07:11:49 ERROR pre_risk_control_flow.py:75 pid-16984 new_application_scanner Error:KafkaError{code=_UNKNOWN_TOPIC,val=-188,str="Unable to produce message: Local: Unknown topic"}
The underlying client is raising KafkaException KafkaError{code=_UNKNOWN_TOPIC..} because it (now) knows the requested topic does not exist in the cluster (and auto topic creation is disabled). This is expected.
You are seeing the exception in run_scan_new_application because you are not catching KafkaException in send().
Related
I am writing a python script to process an operation through the ThreadPoolExecutor. My requirement is to stop script execution if the exception is raised by any of the workers. I have used the exit method but it only exit/stops the particular thread, not the whole script.
Below is the piece of code:
def create_dump():
tenants = get_all_tenants()
with ThreadPoolExecutor(max_workers=8) as executor:
executor.map(process_create_dump, tenants, chunksize=1)
file = open(dump_file_path, 'a')
json.dump(json_dump, file, indent=1)
file.close()
def process_create_dump(tenant):
r_components = dict()
print("processing.....%s" % tenant)
try:
add_clients(r_components, tenant)
except Exception:
print("Unexpected exception occurred while processing")
exit(1)
I found a solution using threading.Event and shutdown method of ThreadPoolExecutor. Below is the code:
event_object = threading.Event()
def create_dump():
tenants = get_all_tenants()
with ThreadPoolExecutor(max_workers=8) as executor:
executor.submit(terminate_executor, executor)
executor.map(process_create_dump, tenants, chunksize=1)
executor.submit(set_event_on_success)
file = open(dump_file_path, 'a')
json.dump(json_dump, file, indent=1)
file.close()
def process_create_dump(tenant):
r_components = dict()
print("processing.....%s" % tenant)
try:
add_clients(r_components, tenant)
except Exception:
event_object.set()
def terminate_executor(executor):
event_object.wait()
executor.shutdown(wait=False, cancel_futures=True)
def set_event_on_success():
while not is_process_completed():
pass
event_object.set()
I'm running a load of DNS lookups through dns.asyncresolver and around 0.05% of them are throwing:
Exception in callback _SelectorDatagramTransport._read_ready()
handle: <Handle _SelectorDatagramTransport._read_ready()>
Traceback (most recent call last):
File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "/usr/lib/python3.8/asyncio/selector_events.py", line 1021, in _read_ready
self._protocol.datagram_received(data, addr)
File "/usr/local/lib/python3.8/dist-packages/dns/_asyncio_backend.py", line 30, in datagram_received
self.recvfrom.set_result((data, addr))
asyncio.exceptions.InvalidStateError: invalid state
If I edit /usr/local/lib/python3.8/dist-packages/dns/_asyncio_backend.py from:
def datagram_received(self, data, addr):
if self.recvfrom:
self.recvfrom.set_result((data, addr))
self.recvfrom = None
To:
def datagram_received(self, data, addr):
if self.recvfrom:
try:
self.recvfrom.set_result((data, addr))
except Exception as e:
print(self.recvfrom)
self.recvfrom = None
I instead get:
# python3 asynctest.py
<Future cancelled>
<Future cancelled>
<Future cancelled>
Is there any way I can catch this without editing Python library files, which I'd really rather not do?
Test code below:
from dns.asyncresolver import Resolver
import asyncio
import sqlite3
async def getIPs():
conn = sqlite3.connect('hosts.db')
c = conn.cursor()
c.execute('SELECT DISTINCT job.host FROM job')
# sqlite> SELECT DISTINCT job.host FROM job LIMIT 10;
# .
# 163mx00.mxmail.netease.com.
# 163mx01.mxmail.netease.com.
# 163mx02.mxmail.netease.com.
# 163mx03.mxmail.netease.com.
# a.dailymotion.com.
# a.iana-servers.net.
# a.icann-servers.net.
# a.mx.openstreetmap.org.
# a.ns.apple.com.
# Total: 10,259
lookups = c.fetchall()
res = Resolver()
sema = asyncio.Semaphore(500)
output = [get_ips(domain[0], res, sema) for domain in lookups]
await asyncio.gather(*output, return_exceptions=False)
async def get_ips(domain, res, sema):
async with sema:
try:
ip = await res.resolve(domain, 'A')
for result in ip:
result = result.to_text()
break
except Exception as e:
pass
try:
ip = await res.resolve(domain, 'AAAA')
for result in ip:
result = result.to_text()
break
except Exception as e:
pass
asyncio.run(getIPs())
I am a decently new python coder and i wish to create a twitter bot in which everytime it retweets, it favourites the tweet as well. I am not exactly sure how to do that but when the bot searches, it sends out an error message of 'list index out of range'.
import tweepy, time, traceback
from tweepy.auth import OAuthHandler
from tweepy.streaming import StreamListener, Stream
ckey = ''
csecret = ''
atoken = ''
asecret = ''
auths = OAuthHandler(ckey, csecret)
auths.set_access_token(atoken, asecret)
api = tweepy.API(auths)
class listener(StreamListener):
def on_data(self, raw_data):
try:
tweet_text = raw_data.lower().split('"text":')[1].split('","source":"')[0].replace(",", "")
screen_name = raw_data.lower().split('"screen_name":"')[1].split('","location"')[0].replace(",", "")
tweet_cid = raw_data.split('"id:')[1].split('"id_str":')[0].replace(",", "")
#there is ment to be 4 spaces at tweet_text
accs = [''] # banned accounts screen name goes in here
words = ['hate' , 'derp' , 'racist' , 'evil' , 'keemstar' , 'mario' , 'kirby'] #banned words goes in here
if not any(acc in screen_name.lower() for acc in accs):
if not any(word in tweet_text.lower() for word in words):
fav(tweet_cid)
follow(screen_name)
retweet(tweet_cid)
tweet(myinput)
#call what u want to do here
#fav(tweet_cid)
#retweet(tweet_cid)
return True
except Exception as e:
print (str(e)) # prints the error message, if you dont want it to comment it out.
pass
def on_error(self, status_code):
try:
print( "error" + status_code)
except Exception as e:
print(str(e))
pass
def retweet(tweet_cid):
try:
api.retweet(tweet_cid)
time.sleep(random.randit(range(50,900)))
except Exception as e:
print(str(e))
pass
def follow(screen_name):
try:
api.create_friendship(screen_name)
time.sleep(random.randit(range(50,900)))
except Exception as e:
print(str(e))
pass
def fav(tweet_cid):
try:
api.create_favourite(tweet_cid)
time.sleep(random.randit(range(600,1100)))
except Exception as e:
print(str(e))
pass
def unfav(tweet_cid):
try:
api.destroy_tweet(tweet_cid)
time.sleep(random.randit(range(8000,9000)))
except Exception as e:
print(str(e))
pass
def tweet(myinput):
try:
api.update_status(myinput)
time.sleep(random.randit(range(1000,4000)))
except Exception as e:
print(str(e))
pass
# tags below
track_words = [""] #deleted all tags so easier to read
follow_acc = [] # all username converted to user ids
try:
twt = Stream(auths, listener())
twt.filter(track=track_words, follow = follow_acc)
except Exception as e:
print (str(e))
pass
Is this what you are asking for? It gives the stack trace of the exception.
import traceback
try:
s='hi'
s=s+1
except Exception as e:
print(traceback.format_exc())
Output:
Traceback (most recent call last):
File "<stdin>", line 3, in <module>
TypeError: cannot concatenate 'str' and 'int' objects
Hope this helps! :)
I am trying to implement a decorator to retry a urllib2.urlopen n times.
I cannot get the decorator to work. When I run it I get the followin error:
Traceback (most recent call last):
File "F:\retry\dec_class.py", line 60, in
x.getURLdata('127.0.0.1')
TypeError: 'NoneType' object is not callable
Can anyone give me hand please?
import serial, urllib2, time
from functools import wraps
import xml.etree.cElementTree as ET
from xml.etree.cElementTree import parse
class Retry(object):
default_exceptions = (Exception)
def __init__(self, tries, exceptions=None, delay=0):
self.tries = tries
if exceptions is None:
exceptions = Retry.default_exceptions
self.exceptions = exceptions
self.delay = delay
def __call__(self, f):
def fn(*args, **kwargs):
tried = 0
exception = None
while tried <= self.tries:
try:
return f(*args, **kwargs)
except self.exceptions, e:
print "Retry, exception: "+str(e)
time.sleep(self.delay)
tried += 1
exception = e
#if no success after tries, raise last exception
raise exception
return fn
class getURL(object):
#Retry(2 )
def getURLdata(self, IPaddress):
try:
f = urllib2.urlopen(''.join(['http://', IPaddress]))
f = ET.parse(f)
return f
except IOError, err:
print("L112 IOError is %s" %err)
except urllib2.URLError, err:
print("L114 urllib2.URLError is %s" %err)
except urllib2.HTTPError, err:
print("L116 urllib2.HTTPError is %s" %err)
except Exception, err :
print("L118 Exception is %s" %err)
x = getURL()
x.getURLdata('127.0.0.1')
Your __call__ method doesn't return fn. Instead, it implicitly returns None and so None is bound to getURLdata.
I wrote a little tool, that gathers data from facebook, using api. Tool uses multiprocessing, queues and httplib modules. Here, is a part of code:
main process:
def extract_and_save(args):
put_queue = JoinableQueue()
get_queue = Queue()
for index in range(args.number_of_processes):
process_name = u"facebook_worker-%s" % index
grabber = FacebookGrabber(get_queue=put_queue, put_queue=get_queue, name=process_name)
grabber.start()
friend_list = get_user_friends(args.default_user_id, ["id"])
for index, friend_id in enumerate(friend_list):
put_queue.put(friend_id)
put_queue.join()
if not get_queue.empty():
... save to database ...
else:
logger.info(u"There is no data to save")
worker process:
class FacebookGrabber(Process):
def __init__(self, *args, **kwargs):
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2)
self.get_queue = kwargs.pop("get_queue")
self.put_queue = kwargs.pop("put_queue")
super(FacebookGrabber, self).__init__(*args, **kwargs)
self.daemon = True
def run(self):
while True:
friend_id = self.get_queue.get(block=True)
try:
friend_obj = self.get_friend_obj(friend_id)
except Exception, e:
logger.info(u"Friend id %s: facebook responded with an error (%s)", friend_id, e)
else:
if friend_obj:
self.put_queue.put(friend_obj)
self.get_queue.task_done()
common code:
def get_json_from_facebook(connection, url, kwargs=None):
url_parts = list(urlparse.urlparse(url))
query = dict(urlparse.parse_qsl(url_parts[4]))
if kwargs:
query.update(kwargs)
url_parts[4] = urllib.urlencode(query)
url = urlparse.urlunparse(url_parts)
try:
connection.request("GET", url)
except Exception, e:
print "<<<", e
response = connection.getresponse()
data = json.load(response)
return data
This code perfectly works on Ubuntu. But when I tried to run it on Windows 7 I got message "There is no data to save". The problem is here:
try:
connection.request("GET", url)
except Exception, e:
print "<<<", e
I get next error: <<< a float is required
Do anybody know, how to fix this problem?
Python version: 2.7.5
One of the "gotcha's" that occasionally happens with socket timeout values is that most operating systems expect them as floats. I believe this has been accounted for with later versions of the linux kernel.
Try changing:
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2)
to:
self.connection = httplib.HTTPSConnection("graph.facebook.com", timeout=2.0)
That's 2 seconds, by the way. Default is typically 5 seconds. Might be a little low.