I'm having issues with my program just closing at random stages and am not sure why.
At first, I thought it was because it was getting an error but I added an error handle. still for some reason it just closes after say a few days of running and no error is displayed. code below
import requests
import lxml.html as lh
import sys
import time
from clint.textui import puts, colored
API_URL = "http://urgmsg.net/livenosaas/ajax/update.php"
class Scraper (object):
id_stamp = 0
def __init__(self, timeout, recent_messages=True):
self.timeout = timeout
self.handlers = []
self.recent_messages = recent_messages
def register_handler(self, handler):
self.handlers.append(handler)
return handler
def scrape(self):
try:
resp = requests.get(API_URL, params={'f': self.id_stamp}).json()
except requests.exceptions.ConnectionError as e:
puts("Error encountered when connecting to urgmsg: ", newline=False)
puts(colored.red(e.__class__.__name__), newline=False)
puts(" " + e.message)
return
if not resp['updated']:
return
old_id_stamp = self.id_stamp
self.id_stamp = resp['IDstamp']
# if old_id_stamp is 0, this is the first scrape
# which will return a whole bunch of recent past messages
if not self.recent_messages and old_id_stamp == 0: return
# Pager messages are returned newest to oldest, we want to
# process them oldest to newest
frags = lh.fragments_fromstring(resp['data'])[::-1]
for frag in frags:
msg = PagerMessage(frag)
for handler in self.handlers:
handler(msg)
def run(self):
while True:
self.scrape()
time.sleep(self.timeout)
class PagerMessage:
def __init__(self, fragment):
children = fragment.getchildren()
self.datetime = children[0].text
self.text = children[1].text
# channel starts with `- `
self.channel = children[1].getchildren()[0].text[2:]
self.response = 'CFSRES' in self.text
def __str__(self):
return "{} [{}]: {}".format(self.channel, self.datetime, self.text)
if __name__ == "__main__":
scraper = Scraper(5)
#scraper.register_handler
def handler(msg):
puts(colored.yellow(msg.channel), newline=False)
puts(" [", newline=False)
puts(colored.green(msg.datetime), newline=False)
puts("] ", newline=False)
if msg.response:
puts(colored.red(msg.text))
else:
puts(msg.text)
scraper.run()
Have I set this part out wrong ?
except requests.exceptions.ConnectionError as e:
puts("Error encountered when connecting to urgmsg: ", newline=False)
puts(colored.red(e.__class__.__name__), newline=False)
puts(" " + e.message)
return
As suggested by #sobolevn change
except: as e:
puts("Error encountered", newline=False)
puts(colored.red(e.__class__.__name__), newline=False)
puts(" " + e.message)
return
Related
I am trying to write a script so when an IP address can't be seen a message gets sent with a telegram letting me know which computer is offline
I have been able to get the telegram side working but i have not been able to pass the data from the main script where it is testing the ip address , at the moment i have test data in there but i would like it to send the error with the computer name
main.py
import socket
import ssl
from datetime import datetime
import pickle
import subprocess
import platform
class Server:
def __init__(self, name, port, connection, priority):
self.name = name
self.port = port
self.connection = connection.lower()
self.priority = priority.lower()
self.history = []
self.alert = False
def check_connection(self):
msg = ""
success = False
now = datetime.now().strftime("%d-%m-%Y %H:%M")
try:
if self.connection == "plain":
socket.create_connection((self.name, self.port), timeout=10)
msg = f"{self.name} is up. On port {self.port} with {self.connection}"
success = True
self.alert = False
elif self.connection == "ssl":
ssl.wrap_socket(socket.create_connection((self.name, self.port), timeout=10))
msg = f"{self.name} is up. On port {self.port} with {self.connection}"
success = True
self.alert = False
else:
if self.ping():
msg = f"{self.name} is up. On port {self.port} with {self.connection}"
success = True
self.alert = False
except socket.timeout:
msg = f"server: {self.name} timeout. On port {self.port}"
except (ConnectionRefusedError, ConnectionResetError) as e:
msg = f"server: {self.name} {e}"
except Exception as e:
msg = f"No Clue??: {e}"
if success == False and self.alert == False:
# Send Alert
self.alert = True
import tg_start
tg_start.send_message("Happy days")
self.create_history(msg, success, now)
def create_history(self, msg, success, now):
history_max = 100
self.history.append((msg, success, now))
while len(self.history) > history_max:
self.history.pop(0)
def ping(self):
try:
output = subprocess.check_output("ping -{} 1 {}".format('n' if platform.system().lower(
) == "windows" else 'c', self.name), shell=True, universal_newlines=True)
if 'unreachable' in output:
return False
else:
return True
except Exception:
return False
if __name__ == "__main__":
try:
servers = pickle.load(open("servers.pickle", "rb"))
except:
servers = [
# Server("ifmc-repserver", 80, "plain", "high"),
# Server("ifmc-repserver", 80, "plain", "high"),
# Server("ifmc-repserver", 465, "ssl", "high"),
# Server("ifmc-repserver", 80, "ping", "high"),
Server("ifmc-repserver", 80, "ping", "high")
]
for server in servers:
server.check_connection()
print(len(server.history))
print(server.history[-1])
pickle.dump(servers, open("servers.pickle", "wb"))
and tg_start.py
import requests
message = "global"
alert = ""
def send_message(text):
global alert
global message
print ("this is text messsage" + " " + text)
#text = "Superman"
alert = text
print("Sending ALERT ...")
token = "token"
chat_id = "chat_id"
print("test message" + " " + alert)
url_req = "https://api.telegram.org/bot" + token + "/sendMessage" + "?chat_id=" + chat_id + "&text=" + alert
print(url_req)
#results = requests.get(url_req)
results = requests.post(url_req) # this request is a post, not a get
print(results.json())
# text = "my name" + text
send_message(alert)
You code worked with a slight change, sendMessage require a POST request, not a GET request.
def send_message(text):
global alert
global message
print ("this is text messsage" + " " + text)
alert = text
print("Sending ALERT ...")
token = "token"
chat_id = "chat_id"
print("test message" + " " + alert)
url_req = f"https://api.telegram.org/bot{token}/sendMessage?chat_id={chat_id}&text={alert}"
print(url_req)
results = requests.post(url_req) # this request is a post, not a get
print(results.json())
# text = "my name" + text
I have been experimenting with QuickFix but I am getting a "AttributeError:SessionID". I honestly have no idea why it's happening, I tinkered with the code a bit but problem remains. Also , google has failed me big on this one so I hope you guys can help me.
Code :
import sys
import time
import thread
import argparse
from datetime import datetime
import quickfix as fix
class Application(fix.Application):
orderID = 0
execID = 0
def gen_ord_id(self):
global orderID
orderID+=1
return orderID
def onCreate(self, sessionID):
return
def onLogon(self, sessionID):
self.sessionID = sessionID
print ("Successful Logon to session '%s'." % sessionID.toString())
return
def onLogout(self, sessionID): return
def toAdmin(self, sessionID, message):
return
def fromAdmin(self, sessionID, message):
return
def toApp(self, sessionID, message):
print "Sent the following message: %s" % message.toString()
return
def fromApp(self, message, sessionID):
print "Received the following message: %s" % message.toString()
return
def genOrderID(self):
self.orderID = self.orderID+1
return `self.orderID`
def genExecID(self):
self.execID = self.execID+1
return `self.execID`
def put_order(self):
print("Creating the following order: ")
trade = fix.Message()
trade.getHeader().setField(fix.BeginString(fix.BeginString_FIX50)) #
trade.getHeader().setField(fix.MsgType(fix.MsgType_NewOrderSingle)) #39=D
trade.setField(fix.ClOrdID(self.genExecID())) #11=Unique order
trade.setField(fix.HandlInst(fix.HandlInst_MANUAL_ORDER_BEST_EXECUTION)) #21=3 (Manual order, best executiona)
trade.setField(fix.Symbol('SMBL')) #55=SMBL ?
trade.setField(fix.Side(fix.Side_BUY)) #43=1 Buy
trade.setField(fix.OrdType(fix.OrdType_LIMIT)) #40=2 Limit order
trade.setField(fix.OrderQty(100)) #38=100
trade.setField(fix.Price(10))
print trade.toString()
fix.Session.sendToTarget(trade, self.sessionID)
def main(config_file):
try:
settings = fix.SessionSettings("initiatorsettings.cfg")
application = Application()
storeFactory = fix.FileStoreFactory(settings)
logFactory = fix.FileLogFactory(settings)
initiator = fix.SocketInitiator(application, storeFactory, settings, logFactory)
initiator.start()
while 1:
input = raw_input()
if input == '1':
print "Putin Order"
application.put_order()
if input == '2':
sys.exit(0)
if input == 'd':
import pdb
pdb.set_trace()
else:
print "Valid input is 1 for order, 2 for exit"
continue
except (fix.ConfigError, fix.RuntimeError), e:
print e
if __name__=='__main__':
parser = argparse.ArgumentParser(description='FIX Client')
parser.add_argument('file_name', type=str, help='Name of configuration file')
args = parser.parse_args()
main(args.file_name)
I get the following error message :
Traceback (most recent call last):
File "initiator.py", line 97, in main application.put_order()
File "initiator.py", line 80, in put_order fix.Session.sendToTarget(trade, self.sessionID)
File "C:\Users\gribeiro\Anaconda\lib\site-packages\quickfix.py", line 30939, in <lambda>__getattr__ = lambda self, name: _swig_getattr(self, Application, name)
File "C:\Users\gribeiro\Anaconda\lib\site-packages\quickfix.py", line 57, in _swig_getattr raise AttributeError(name)
AttributeError: sessionID
------------------------------------------------------------
Update :
Code producing the error:
import sys
import time
import thread
import argparse
from datetime import datetime
import quickfix as fix
class Application(fix.Application):
orderID = 0
execID = 0
def gen_ord_id(self):
global orderID
orderID+=1
return orderID
def onCreate(self, sessionID):
return
def onLogon(self, sessionID):
# self.sessionID = sessionID
# print ("Successful Logon to session '%s'." % sessionID.toString())
return
def onLogout(self, sessionID): return
def toAdmin(self, sessionID, message):
return
def fromAdmin(self, sessionID, message):
return
def toApp(self, sessionID, message):
print "Sent the following message: %s" % message.toString()
return
def fromApp(self, message, sessionID):
print "Received the following message: %s" % message.toString()
return
def genOrderID(self):
self.orderID = self.orderID+1
return `self.orderID`
def genExecID(self):
self.execID = self.execID+1
return `self.execID`
def put_order(self):
print("Creating the following order: ")
trade = fix.Message()
trade.getHeader().setField(fix.BeginString(fix.BeginString_FIX50)) #
trade.getHeader().setField(fix.MsgType(fix.MsgType_NewOrderSingle)) #39=D
trade.setField(fix.ClOrdID(self.genExecID())) #11=Unique order
trade.setField(fix.HandlInst(fix.HandlInst_MANUAL_ORDER_BEST_EXECUTION)) #21=3 (Manual order, best executiona)
trade.setField(fix.Symbol('SMBL')) #55=SMBL ?
trade.setField(fix.Side(fix.Side_BUY)) #43=1 Buy
trade.setField(fix.OrdType(fix.OrdType_LIMIT)) #40=2 Limit order
trade.setField(fix.OrderQty(100)) #38=100
trade.setField(fix.Price(10))
print trade.toString()
fix.Session_sendToTarget(trade)
def main(config_file):
try:
settings = fix.SessionSettings(config_file)
application = Application()
storeFactory = fix.FileStoreFactory(settings)
logFactory = fix.FileLogFactory(settings)
initiator = fix.SocketInitiator(application, storeFactory, settings, logFactory)
initiator.start()
while 1:
input = raw_input()
if input == '1':
print "Putin Order"
application.put_order()
if input == '2':
sys.exit(0)
if input == 'd':
import pdb
pdb.set_trace()
else:
print "Valid input is 1 for order, 2 for exit"
continue
except (fix.ConfigError, fix.RuntimeError), e:
print e
if __name__=='__main__':
parser = argparse.ArgumentParser(description='FIX Client')
parser.add_argument('file_name', type=str, help='Name of configuration file')
args = parser.parse_args()
main(args.file_name)
Config file :
[DEFAULT]
ConnectionType=initiator
ReconnectInterval=60
FileStorePath=store
FileLogPath=log
StartTime=00:00:00
EndTime=00:00:00
UseDataDictionary=Y
DataDictionary=spec/FIX42.xml
HttpAcceptPort=9911
ValidateUserDefinedFields=N
ResetOnLogout=Y
ResetOnLogon=Y
ValidateFieldsOutOfOrder=N
DefaultApplVerID=FIX.5.0SP2
# standard config elements
[SESSION]
# inherit ConnectionType, ReconnectInterval and SenderCompID from default
BeginString=FIX.4.2
SenderCompID=BRANZOL
TargetCompID=FIXSIM
SocketConnectHost=host
SocketConnectPort=port
HeartBtInt=30
Output from ScreenLogFactory >
<20151216-17:09:43.426, FIX.4.2:BRANZOL->FIXSIM, event>
(Created session)
<20151216-17:09:43.434, FIX.4.2:BRANZOL->FIXSIM, event>
(Connecting to hostX on port Y)
<20151216-17:09:44.159, FIX.4.2:BRANZOL->FIXSIM, outgoing>
(8=FIX.4.29=7435=A34=149=BRANZOL52=20151216-17:09:43.88256=FIXSIM98=0108=30141=Y10=032)
<20151216-17:09:44.159, FIX.4.2:BRANZOL->FIXSIM, event>
(Initiated logon request)
<20151216-17:09:44.264, FIX.4.2:BRANZOL->FIXSIM, event>
(Socket Error: Connection reset by peer.)
<20151216-17:09:44.264, FIX.4.2:BRANZOL->FIXSIM, event>
(Disconnecting)
Valid input is 1 for order, 2 for exit
Putin Order
Creating the following order:
8=FIX.5.09=4635=D11=121=338=10040=244=1054=155=SMBL10=081
Try cutting this line entirely self.sessionID = sessionID
EDIT -- OKay, thanks for including the traceback. You should replace the line fix.Session.sendToTarget(trade, self.sessionID) with the line fix.Session_sendToTarget(trade) and let me know how that goes.
Background:
I would like to integrate yowsup to my home automation project. I have seen a simple sample on how to receive messages and after some minor changes it is working fine.
Issue:
My problem starts when it comes to integrate the send message feature. Those are the two files I am using:
run.py
from layer import EchoLayer
from yowsup.layers.auth import YowAuthenticationProtocolLayer
from yowsup.layers.protocol_messages import YowMessagesProtocolLayer
from yowsup.layers.protocol_receipts import YowReceiptProtocolLayer
from yowsup.layers.protocol_acks import YowAckProtocolLayer
from yowsup.layers.protocol_presence import YowPresenceProtocolLayer
from yowsup.layers.network import YowNetworkLayer
from yowsup.layers.coder import YowCoderLayer
from yowsup.common import YowConstants
from yowsup.layers import YowLayerEvent
from yowsup.stacks import YowStack, YOWSUP_CORE_LAYERS
from yowsup import env
CREDENTIALS = ("phone", "pwd")
if __name__ == "__main__":
layers = (
EchoLayer,
(YowAuthenticationProtocolLayer, YowMessagesProtocolLayer, YowReceiptProtocolLayer, YowAckProtocolLayer, YowPresenceProtocolLayer)
) + YOWSUP_CORE_LAYERS
stack = YowStack(layers)
# Setting credentials
stack.setProp(YowAuthenticationProtocolLayer.PROP_CREDENTIALS, CREDENTIALS)
# WhatsApp server address
stack.setProp(YowNetworkLayer.PROP_ENDPOINT, YowConstants.ENDPOINTS[0])
stack.setProp(YowCoderLayer.PROP_DOMAIN, YowConstants.DOMAIN)
stack.setProp(YowCoderLayer.PROP_RESOURCE, env.CURRENT_ENV.getResource())
# Sending connecting signal
stack.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_CONNECT))
# Program main loop
stack.loop()
layer.py
from yowsup.layers.interface import YowInterfaceLayer, ProtocolEntityCallback
from yowsup.layers.protocol_messages.protocolentities import TextMessageProtocolEntity
from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity
from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity
from yowsup.layers.protocol_presence.protocolentities import PresenceProtocolEntity
import threading
import logging
logger = logging.getLogger(__name__)
class EchoLayer(YowInterfaceLayer):
#ProtocolEntityCallback("message")
def onMessage(self, messageProtocolEntity):
#send receipt otherwise we keep receiving the same message over and over
print str(messageProtocolEntity.getFrom()) + ' - ' + str(messageProtocolEntity.getBody())
receipt = OutgoingReceiptProtocolEntity(messageProtocolEntity.getId(), messageProtocolEntity.getFrom())
self.toLower(receipt)
#ProtocolEntityCallback("send_message")
def sendMessage(self, destination, message, messageProtocolEntity):
outgoingMessageProtocolEntity = TextMessageProtocolEntity(
message,
to = destination + "#s.whatsapp.net")
self.toLower(outgoingMessageProtocolEntity)
#ProtocolEntityCallback("receipt")
def onReceipt(self, entity):
ack = OutgoingAckProtocolEntity(entity.getId(), "receipt", "delivery")
self.toLower(ack)
# List of (jid, message) tuples
PROP_MESSAGES = "org.openwhatsapp.yowsup.prop.sendclient.queue"
def __init__(self):
super(EchoLayer, self).__init__()
self.ackQueue = []
self.lock = threading.Condition()
#ProtocolEntityCallback("success")
def onSuccess(self, successProtocolEntity):
self.lock.acquire()
for target in self.getProp(self.__class__.PROP_MESSAGES, []):
phone, message = target
if '#' in phone:
messageEntity = TextMessageProtocolEntity(message, to = phone)
elif '-' in phone:
messageEntity = TextMessageProtocolEntity(message, to = "%s#g.us" % phone)
else:
messageEntity = TextMessageProtocolEntity(message, to = "%s#s.whatsapp.net" % phone)
self.ackQueue.append(messageEntity.getId())
self.toLower(messageEntity)
self.lock.release()
#ProtocolEntityCallback("ack")
def onAck(self, entity):
self.lock.acquire()
if entity.getId() in self.ackQueue:
self.ackQueue.pop(self.ackQueue.index(entity.getId()))
if not len(self.ackQueue):
logger.info("Message sent")
#raise KeyboardInterrupt()
self.lock.release()
Questions:
Where am I supposed to call the send_message method, so I can send messages wherever I need it?
Is there a regular event (triggering every second or something) which I could use to send my messages?
#ProtocolEntityCallback("send_message")
def sendMessage(self, destination, message, messageProtocolEntity):
outgoingMessageProtocolEntity = TextMessageProtocolEntity(
message,
to = destination + "#s.whatsapp.net")
self.toLower(outgoingMessageProtocolEntity)
In the avove code sendMessage to be called, protocolEntity.getTag() == "send_message" has to be True. You don't need it to send message.
layer.py
from yowsup.layers.interface import YowInterfaceLayer, ProtocolEntityCallback
from yowsup.layers.protocol_messages.protocolentities import TextMessageProtocolEntity
from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity
from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity
from yowsup.layers.protocol_presence.protocolentities import PresenceProtocolEntity
import threading
import logging
logger = logging.getLogger(__name__)
recv_msg = []
class EchoLayer(YowInterfaceLayer):
def __init__(self):
super(EchoLayer, self).__init__()
self.ackQueue = []
self.lock = threading.Condition()
#ProtocolEntityCallback("message")
def onMessage(self, messageProtocolEntity):
if messageProtocolEntity.getType() == 'text':
recv_msg.append((messageProtocolEntity.getFrom(),messageProtocolEntity.getBody()))
#send receipt otherwise we keep receiving the same message over and over
receipt = OutgoingReceiptProtocolEntity(messageProtocolEntity.getId(), messageProtocolEntity.getFrom())
self.toLower(receipt)
#ProtocolEntityCallback("receipt")
def onReceipt(self, entity):
ack = OutgoingAckProtocolEntity(entity.getId(), "receipt", "delivery")
self.toLower(ack)
# List of (jid, message) tuples
PROP_MESSAGES = "org.openwhatsapp.yowsup.prop.sendclient.queue"
#ProtocolEntityCallback("success")
def onSuccess(self, successProtocolEntity):
self.lock.acquire()
for target in self.getProp(self.__class__.PROP_MESSAGES, []):
phone, message = target
if '#' in phone:
messageEntity = TextMessageProtocolEntity(message, to = phone)
elif '-' in phone:
messageEntity = TextMessageProtocolEntity(message, to = "%s#g.us" % phone)
else:
messageEntity = TextMessageProtocolEntity(message, to = "%s#s.whatsapp.net" % phone)
self.ackQueue.append(messageEntity.getId())
self.toLower(messageEntity)
self.lock.release()
#ProtocolEntityCallback("ack")
def onAck(self, entity):
self.lock.acquire()
if entity.getId() in self.ackQueue:
self.ackQueue.pop(self.ackQueue.index(entity.getId()))
if not len(self.ackQueue):
self.lock.release()
logger.info("Message sent")
raise KeyboardInterrupt()
self.lock.release()
To send message define a function send_message in the stack run.py. You can also import run.py and use it's function from other script.
from layer import EchoLayer, recv_msg
CREDENTIALS = ("phone", "pwd")
def send_message(destination, message):
'''
destination is <phone number> without '+'
and with country code of type string,
message is string
e.g send_message('11133434343','hello')
'''
messages = [(destination, message)]
layers = (EchoLayer,
(YowAuthenticationProtocolLayer,
YowMessagesProtocolLayer,
YowReceiptProtocolLayer,
YowAckProtocolLayer,
YowPresenceProtocolLayer)
) + YOWSUP_CORE_LAYERS
stack = YowStack(layers)
stack.setProp(EchoLayer.PROP_MESSAGES, messages)
stack.setProp(YowAuthenticationProtocolLayer.PROP_PASSIVE, True)
# Setting credentials
stack.setProp(YowAuthenticationProtocolLayer.PROP_CREDENTIALS, CREDENTIALS)
# WhatsApp server address
stack.setProp(YowNetworkLayer.PROP_ENDPOINT, YowConstants.ENDPOINTS[0])
stack.setProp(YowCoderLayer.PROP_DOMAIN, YowConstants.DOMAIN)
stack.setProp(YowCoderLayer.PROP_RESOURCE, env.CURRENT_ENV.getResource())
# Sending connecting signal
stack.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_CONNECT))
try:
# Program main loop
stack.loop()
except AuthError as e:
print('Authentication error %s' % e.message)
sys.exit(1)
def recv_message():
layers = ( EchoLayer,
(YowAuthenticationProtocolLayer, YowMessagesProtocolLayer,
YowReceiptProtocolLayer, YowAckProtocolLayer,
YowPresenceProtocolLayer)
) + YOWSUP_CORE_LAYERS
stack = YowStack(layers)
# Setting credentials
stack.setProp(YowAuthenticationProtocolLayer.PROP_CREDENTIALS, CREDENTIALS)
# WhatsApp server address
stack.setProp(YowNetworkLayer.PROP_ENDPOINT, YowConstants.ENDPOINTS[0])
stack.setProp(YowCoderLayer.PROP_DOMAIN, YowConstants.DOMAIN)
stack.setProp(YowCoderLayer.PROP_RESOURCE, env.CURRENT_ENV.getResource())
# Sending connecting signal
stack.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_CONNECT))
try:
# Program main loop
stack.loop()
except AuthError as e:
print('Authentication error %s' % e.message)
sys.exit(1)
if __name__ == '__main__':
if len(sys.argv) == 1:
print('%s send number message\nrecv\n' % sys.argv[0])
sys.exit(1)
if sys.argv[1] == 'send':
try:
send_message(sys.argv[2],sys.argv[3])
except KeyboardInterrupt:
print('closing')
sys.exit(0)
if sys.argv[1] == 'recv':
try:
recv_message()
except KeyboardInterrupt:
print('closing')
sys.exit(0)
for m in recv_msg:
print('From %s:\n%s\n' % m)
Now you can send message by calling send_message('1234567890','Howdy') and recieve message by calling recv_message().
I'm a php guy and I don't understand python very well... so .. excuse me if my question is stupid.
I have 2 php scripts (client.php and worker.php) using Gearman, that I need to convert to python versions. I have been able to do it partially, but I'm stuck. First, here are my two scripts:
client.py
#!/usr/bin/env python
import gearman
import json
RBLS = [
'b.barracudacentral.org',
'bl.emailbasura.org'
]
IP = '1.2.3.4'
def check_request_status(job_request):
if job_request.complete:
print "Job %s finished! Result: %s - %s" % (job_request.job.unique, job_request.state, job_request.result)
elif job_request.timed_out:
print "Job %s timed out!" % job_request.unique
elif job_request.state == JOB_UNKNOWN:
print "Job %s connection failed!" % job_request.unique
data = {"ip": IP, "rbls": RBLS}
serialized_data = json.dumps(data)
gm_client = gearman.GearmanClient(['localhost:4730'])
completed_job_request = gm_client.submit_job("runcheck", serialized_data)
check_request_status(completed_job_request)
worker.py
#!/usr/bin/env python
import gearman
import sys
import socket
import re
import json
from dns.resolver import Resolver, NXDOMAIN, NoNameservers, Timeout, NoAnswer
from threading import Thread
# This hardcoded RBLS need to be passed by gearman client script
# RBLS = ['xbl.spamhaus.org', 'zen.spamhaus.org']
class Lookup(Thread):
def __init__(self, host, dnslist, listed, resolver):
Thread.__init__(self)
self.host = host
self.listed = listed
self.dnslist = dnslist
self.resolver = resolver
def run(self):
try:
host_record = self.resolver.query(self.host, "A")
if len(host_record) > 0:
self.listed[self.dnslist]['LISTED'] = True
self.listed[self.dnslist]['HOST'] = host_record[0].address
text_record = self.resolver.query(self.host, "TXT")
if len(text_record) > 0:
self.listed[self.dnslist]['TEXT'] = "\n".join(text_record[0].strings)
self.listed[self.dnslist]['ERROR'] = False
except NXDOMAIN:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NXDOMAIN
except NoNameservers:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NoNameservers
except Timeout:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = Timeout
except NameError:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NameError
except NoAnswer:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NoAnswer
class RBLSearch(object):
def __init__(self, lookup_host):
self.lookup_host = lookup_host
self._listed = None
self.resolver = Resolver()
self.resolver.timeout = 0.2
self.resolver.lifetime = 1.0
def search(self):
if self._listed is not None:
pass
else:
host = self.lookup_host.split(".")
host = ".".join(list(reversed(host)))
self._listed = {'SEARCH_HOST': self.lookup_host}
threads = []
for LIST in RBLS:
self._listed[LIST] = {'LISTED': False}
query = Lookup("%s.%s" % (host, LIST), LIST, self._listed, self.resolver)
threads.append(query)
query.start()
for thread in threads:
thread.join()
return self._listed
listed = property(search)
def print_results(self):
listed = self.listed
print("")
print("--- DNSBL Report for %s ---" % listed['SEARCH_HOST'])
for key in listed:
if key == 'SEARCH_HOST':
continue
if not listed[key].get('ERROR'):
if listed[key]['LISTED']:
print("Results for %s: %s" % (key, listed[key]['LISTED']))
print(" + Host information: %s" % \
(listed[key]['HOST']))
if 'TEXT' in listed[key].keys():
print(" + Additional information: %s" % \
(listed[key]['TEXT']))
else:
#print "*** Error contacting %s ***" % key
pass
def task_listener_runcheck(gearman_worker, gearman_job):
jdata = json.loads(gearman_job.data)
host = jdata['ip']
ip = host
RBLS = jdata['rbls']
print("Looking up: %s (please wait)" % host)
pat = re.compile("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}")
is_ip_address = pat.match(host)
if not is_ip_address:
try:
ip = socket.gethostbyname(host)
print("Hostname %s resolved to ip %s" % (host,ip))
except socket.error:
print("Hostname %s can't be resolved" % host)
ip = ""
if ip:
searcher = RBLSearch(ip)
searcher.print_results()
return "RunCheck was successfull"
gm_worker = gearman.GearmanWorker(['localhost:4730'])
gm_worker.set_client_id('python-worker')
gm_worker.register_task('runcheck', task_listener_runcheck)
gm_worker.work()
Here is how this 2 scripts are working: client.py passes the IP address and array of rbl's to the worker.py. Then worker gets the IP address and check it against all the rbl's.
The problem is that I don't know how to use the RBLS inside the RBLSearch class. It's working if I hardcode the RBLS in the beginning of the script (See worker.py, Line 12), but it's not working if I define RBLS in task_listener_runcheck
I have been able to solve it. Here is the edited version (in case anyone want it):
worker.py
#!/usr/bin/env python
import gearman
import sys
import socket
import re
import json
from dns.resolver import Resolver, NXDOMAIN, NoNameservers, Timeout, NoAnswer
from threading import Thread
class Lookup(Thread):
def __init__(self, host, dnslist, listed, resolver):
Thread.__init__(self)
self.host = host
self.listed = listed
self.dnslist = dnslist
self.resolver = resolver
def run(self):
try:
host_record = self.resolver.query(self.host, "A")
if len(host_record) > 0:
self.listed[self.dnslist]['LISTED'] = True
self.listed[self.dnslist]['HOST'] = host_record[0].address
text_record = self.resolver.query(self.host, "TXT")
if len(text_record) > 0:
self.listed[self.dnslist]['TEXT'] = "\n".join(text_record[0].strings)
self.listed[self.dnslist]['ERROR'] = False
except NXDOMAIN:
self.listed[self.dnslist]['ERROR'] = False
self.listed[self.dnslist]['ERRORTYPE'] = NXDOMAIN
except NoNameservers:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NoNameservers
self.listed[self.dnslist]['TEXT'] = "%s - The operation timed out." % self.host
except Timeout:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = Timeout
self.listed[self.dnslist]['TEXT'] = "%s - The operation timed out." % self.host
except NameError:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NameError
self.listed[self.dnslist]['TEXT'] = "%s - NameError" % self.host
except NoAnswer:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NoAnswer
self.listed[self.dnslist]['TEXT'] = "%s - The response did not contain an answer to the question." % self.host
class RBLSearch(object):
def __init__(self, lookup_host, rbls):
self.lookup_host = lookup_host
self.rbls = rbls
self._listed = None
self.resolver = Resolver()
self.resolver.timeout = 0.2
self.resolver.lifetime = 1.0
def search(self):
if self._listed is not None:
pass
else:
host = self.lookup_host.split(".")
host = ".".join(list(reversed(host)))
self._listed = {'SEARCH_HOST': self.lookup_host}
threads = []
for LIST in self.rbls:
self._listed[LIST] = {'LISTED': False}
query = Lookup("%s.%s" % (host, LIST), LIST, self._listed, self.resolver)
threads.append(query)
query.start()
for thread in threads:
thread.join()
return self._listed
listed = property(search)
def print_results(self):
listed = self.listed
print("")
print("--- DNSBL Report for %s ---" % listed['SEARCH_HOST'])
for key in listed:
if key == 'SEARCH_HOST':
continue
if not listed[key].get('ERROR'):
if listed[key]['LISTED']:
print("Results for %s: %s" % (key, listed[key]['LISTED']))
print(" + Host information: %s" % \
(listed[key]['HOST']))
if 'TEXT' in listed[key].keys():
print(" + Additional information: %s" % \
(listed[key]['TEXT']))
else:
print("Not listed in %s" % (key))
else:
#print "*** Error contacting %s ***" % key
pass
def task_listener_runcheck(gearman_worker, gearman_job):
jdata = json.loads(gearman_job.data)
host = jdata['ip']
rbls = jdata['rbls']
ip = host
print("Looking up: %s (please wait)" % host)
pat = re.compile("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}")
is_ip_address = pat.match(host)
if not is_ip_address:
try:
ip = socket.gethostbyname(host)
print("Hostname %s resolved to ip %s" % (host,ip))
except socket.error:
print("Hostname %s can't be resolved" % host)
ip = ""
if ip:
searcher = RBLSearch(ip, rbls)
searcher.print_results()
return "RunCheck was successfull"
gm_worker = gearman.GearmanWorker(['localhost:4730'])
gm_worker.set_client_id('python-worker')
gm_worker.register_task('runcheck', task_listener_runcheck)
gm_worker.work()
I am trying to implement a simple threaded SocketServer (using SocketServer.ThreadedMixIn). However, my server stops receiving further messages. Here is the code:
#!/usr/bin/python -u
import SocketServer
import sys
class MYAgentHandler(SocketServer.BaseRequestHandler):
def handle(self):
try:
data = self.request.recv(1024)
print "Received request " + str(data) + "\n"
reply = str(agent.processAgentMessage(data))
self.request.send(reply)
self.request.close()
except Exception, instr:
print "While processing data " + data + " error encountered " + str(instr) + "\n"
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
daemon_threads = True
allow_reuse_address = True
def __init__(self, server_address, RequestHandlerClass):
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
class MYAgent:
def processAgentMessage(self, msg):
try:
tokens = msg.split('^')
if tokens[0] == "CreateSession":
return("New session")
elif tokens[0] == "GetStatus":
return("Init")
except Exception, instr:
print "Error while processing message " + str(instr) + "\n"
agent = MYAgent()
def main():
MYServer = sys.argv[1]
MYAgentPort = sys.argv[2]
agent.listener = ThreadedTCPServer((MYServer, int(MYAgentPort)), MYAgentHandler)
agent.listener.serve_forever()
if __name__ == '__main__':
main()
And here is my client:
#!/usr/bin/python -u
import socket
import time
if __name__ == "__main__":
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', 15222))
try:
sock.send("CreateSession")
sessionID = str(sock.recv(1024))
print "Received: " + sessionID
sock.send("GetStatus^"+sessionID)
print "Sent Getstatus\n"
time.sleep(1)
response = str(sock.recv(1024))
print "status of " + str(sessionID) + " is " + str(response) + "\n"
sock.close()
except Exception, instr:
print "Error occurred " + str(instr) + "\n"
Here is one session. Server output:
$ ./t.py localhost 15222
Received request CreateSession
Client output:
$ ./client.py
Received: New session
Sent Getstatus
status of New session is
$
Any ideas why this is happening?
You have to remove self.request.close() (which closes the connection) and wrap everything with while True: (so it will continue to read from the same socket).