I'm a php guy and I don't understand python very well... so .. excuse me if my question is stupid.
I have 2 php scripts (client.php and worker.php) using Gearman, that I need to convert to python versions. I have been able to do it partially, but I'm stuck. First, here are my two scripts:
client.py
#!/usr/bin/env python
import gearman
import json
RBLS = [
'b.barracudacentral.org',
'bl.emailbasura.org'
]
IP = '1.2.3.4'
def check_request_status(job_request):
if job_request.complete:
print "Job %s finished! Result: %s - %s" % (job_request.job.unique, job_request.state, job_request.result)
elif job_request.timed_out:
print "Job %s timed out!" % job_request.unique
elif job_request.state == JOB_UNKNOWN:
print "Job %s connection failed!" % job_request.unique
data = {"ip": IP, "rbls": RBLS}
serialized_data = json.dumps(data)
gm_client = gearman.GearmanClient(['localhost:4730'])
completed_job_request = gm_client.submit_job("runcheck", serialized_data)
check_request_status(completed_job_request)
worker.py
#!/usr/bin/env python
import gearman
import sys
import socket
import re
import json
from dns.resolver import Resolver, NXDOMAIN, NoNameservers, Timeout, NoAnswer
from threading import Thread
# This hardcoded RBLS need to be passed by gearman client script
# RBLS = ['xbl.spamhaus.org', 'zen.spamhaus.org']
class Lookup(Thread):
def __init__(self, host, dnslist, listed, resolver):
Thread.__init__(self)
self.host = host
self.listed = listed
self.dnslist = dnslist
self.resolver = resolver
def run(self):
try:
host_record = self.resolver.query(self.host, "A")
if len(host_record) > 0:
self.listed[self.dnslist]['LISTED'] = True
self.listed[self.dnslist]['HOST'] = host_record[0].address
text_record = self.resolver.query(self.host, "TXT")
if len(text_record) > 0:
self.listed[self.dnslist]['TEXT'] = "\n".join(text_record[0].strings)
self.listed[self.dnslist]['ERROR'] = False
except NXDOMAIN:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NXDOMAIN
except NoNameservers:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NoNameservers
except Timeout:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = Timeout
except NameError:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NameError
except NoAnswer:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NoAnswer
class RBLSearch(object):
def __init__(self, lookup_host):
self.lookup_host = lookup_host
self._listed = None
self.resolver = Resolver()
self.resolver.timeout = 0.2
self.resolver.lifetime = 1.0
def search(self):
if self._listed is not None:
pass
else:
host = self.lookup_host.split(".")
host = ".".join(list(reversed(host)))
self._listed = {'SEARCH_HOST': self.lookup_host}
threads = []
for LIST in RBLS:
self._listed[LIST] = {'LISTED': False}
query = Lookup("%s.%s" % (host, LIST), LIST, self._listed, self.resolver)
threads.append(query)
query.start()
for thread in threads:
thread.join()
return self._listed
listed = property(search)
def print_results(self):
listed = self.listed
print("")
print("--- DNSBL Report for %s ---" % listed['SEARCH_HOST'])
for key in listed:
if key == 'SEARCH_HOST':
continue
if not listed[key].get('ERROR'):
if listed[key]['LISTED']:
print("Results for %s: %s" % (key, listed[key]['LISTED']))
print(" + Host information: %s" % \
(listed[key]['HOST']))
if 'TEXT' in listed[key].keys():
print(" + Additional information: %s" % \
(listed[key]['TEXT']))
else:
#print "*** Error contacting %s ***" % key
pass
def task_listener_runcheck(gearman_worker, gearman_job):
jdata = json.loads(gearman_job.data)
host = jdata['ip']
ip = host
RBLS = jdata['rbls']
print("Looking up: %s (please wait)" % host)
pat = re.compile("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}")
is_ip_address = pat.match(host)
if not is_ip_address:
try:
ip = socket.gethostbyname(host)
print("Hostname %s resolved to ip %s" % (host,ip))
except socket.error:
print("Hostname %s can't be resolved" % host)
ip = ""
if ip:
searcher = RBLSearch(ip)
searcher.print_results()
return "RunCheck was successfull"
gm_worker = gearman.GearmanWorker(['localhost:4730'])
gm_worker.set_client_id('python-worker')
gm_worker.register_task('runcheck', task_listener_runcheck)
gm_worker.work()
Here is how this 2 scripts are working: client.py passes the IP address and array of rbl's to the worker.py. Then worker gets the IP address and check it against all the rbl's.
The problem is that I don't know how to use the RBLS inside the RBLSearch class. It's working if I hardcode the RBLS in the beginning of the script (See worker.py, Line 12), but it's not working if I define RBLS in task_listener_runcheck
I have been able to solve it. Here is the edited version (in case anyone want it):
worker.py
#!/usr/bin/env python
import gearman
import sys
import socket
import re
import json
from dns.resolver import Resolver, NXDOMAIN, NoNameservers, Timeout, NoAnswer
from threading import Thread
class Lookup(Thread):
def __init__(self, host, dnslist, listed, resolver):
Thread.__init__(self)
self.host = host
self.listed = listed
self.dnslist = dnslist
self.resolver = resolver
def run(self):
try:
host_record = self.resolver.query(self.host, "A")
if len(host_record) > 0:
self.listed[self.dnslist]['LISTED'] = True
self.listed[self.dnslist]['HOST'] = host_record[0].address
text_record = self.resolver.query(self.host, "TXT")
if len(text_record) > 0:
self.listed[self.dnslist]['TEXT'] = "\n".join(text_record[0].strings)
self.listed[self.dnslist]['ERROR'] = False
except NXDOMAIN:
self.listed[self.dnslist]['ERROR'] = False
self.listed[self.dnslist]['ERRORTYPE'] = NXDOMAIN
except NoNameservers:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NoNameservers
self.listed[self.dnslist]['TEXT'] = "%s - The operation timed out." % self.host
except Timeout:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = Timeout
self.listed[self.dnslist]['TEXT'] = "%s - The operation timed out." % self.host
except NameError:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NameError
self.listed[self.dnslist]['TEXT'] = "%s - NameError" % self.host
except NoAnswer:
self.listed[self.dnslist]['ERROR'] = True
self.listed[self.dnslist]['ERRORTYPE'] = NoAnswer
self.listed[self.dnslist]['TEXT'] = "%s - The response did not contain an answer to the question." % self.host
class RBLSearch(object):
def __init__(self, lookup_host, rbls):
self.lookup_host = lookup_host
self.rbls = rbls
self._listed = None
self.resolver = Resolver()
self.resolver.timeout = 0.2
self.resolver.lifetime = 1.0
def search(self):
if self._listed is not None:
pass
else:
host = self.lookup_host.split(".")
host = ".".join(list(reversed(host)))
self._listed = {'SEARCH_HOST': self.lookup_host}
threads = []
for LIST in self.rbls:
self._listed[LIST] = {'LISTED': False}
query = Lookup("%s.%s" % (host, LIST), LIST, self._listed, self.resolver)
threads.append(query)
query.start()
for thread in threads:
thread.join()
return self._listed
listed = property(search)
def print_results(self):
listed = self.listed
print("")
print("--- DNSBL Report for %s ---" % listed['SEARCH_HOST'])
for key in listed:
if key == 'SEARCH_HOST':
continue
if not listed[key].get('ERROR'):
if listed[key]['LISTED']:
print("Results for %s: %s" % (key, listed[key]['LISTED']))
print(" + Host information: %s" % \
(listed[key]['HOST']))
if 'TEXT' in listed[key].keys():
print(" + Additional information: %s" % \
(listed[key]['TEXT']))
else:
print("Not listed in %s" % (key))
else:
#print "*** Error contacting %s ***" % key
pass
def task_listener_runcheck(gearman_worker, gearman_job):
jdata = json.loads(gearman_job.data)
host = jdata['ip']
rbls = jdata['rbls']
ip = host
print("Looking up: %s (please wait)" % host)
pat = re.compile("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}")
is_ip_address = pat.match(host)
if not is_ip_address:
try:
ip = socket.gethostbyname(host)
print("Hostname %s resolved to ip %s" % (host,ip))
except socket.error:
print("Hostname %s can't be resolved" % host)
ip = ""
if ip:
searcher = RBLSearch(ip, rbls)
searcher.print_results()
return "RunCheck was successfull"
gm_worker = gearman.GearmanWorker(['localhost:4730'])
gm_worker.set_client_id('python-worker')
gm_worker.register_task('runcheck', task_listener_runcheck)
gm_worker.work()
Related
I am running a simple python script to log the accessed url using squid url_rewriter_program.
However every time it runs, rewriter crashes with broken pipe error at sys.stdout.flush().
Please suggest a specific solution.
python code is:
import sys
import os
import io
line = sys.stdin.readline()
fo=open("/home/linux/Desktop/foo1.txt","a")
fo.write(line)
fo.close()
sys.stdout.write("\n")
sys.stdout.flush()
This is a squid redirector file, written on python, can you get, or compare with your script:
Redirector:
#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-
#-----------------------------------------------------------------------------
# Name: redirector_master.py
# Purpose: SiSCont checker cuote
#
# Author: Ernesto Licea Martin
#
# Created: 2011/11/24
# RCS-ID: $Id: redirector_master.py $
# Copyright: (c) 2011
# Licence: GPL
#-----------------------------------------------------------------------------
import sys, string, libSiSCont
__name__= "redirector_master"
query="SELECT accounts_proxyquotatype.name, accounts_proxyaccount.proxy_quota, accounts_proxyaccount.proxy_quota_extra, accounts_proxyaccount.proxy_active, accounts_proxyaccounttype.name FROM accounts_proxyaccount INNER JOIN accounts_proxyaccounttype INNER JOIN accounts_proxyquotatype ON (accounts_proxyaccount.proxy_quota_type_id = accounts_proxyquotatype.id) AND (accounts_proxyaccount.proxy_account_type_id = accounts_proxyaccounttype.id) WHERE accounts_proxyaccount.proxy_username=%s"
class RedirMaster:
def __init__(self):
obj = libSiSCont.ParceConf()
obj.parcecfgfile()
self.__listModules = obj.getModList()
self.__redirDicc = obj.getRedirectURL()
self.__penalURL = obj.getPenalizedURL()
self.__confDicc = obj.getConfParam()
self.__dbDicc = obj.getDBDicc()
self.__proxyDicc = obj.getProxyCacheParam()
self.__dbParam = []
def getProxyTypes(self):
db=libSiSCont.connectDB(dbDicc=self.__dbDicc)
c=db.cursor()
c.execute("SELECT accounts_proxy")
def run(self):
modules=[]
for mod in self.__listModules:
try:
m=__import__(mod)
modules.append(m)
except Exception, e:
libSiSCont.reportLogs("%s.run" %__name__, 'Unable to load redirector module %s; the error was: %s' % (mod,str(e)))
if len(modules) == 0:
libSiSCont.reportLogs("%s.run" %__name__, 'No usable redirector module found; switching to trasparent behavour')
while 1:
try:
data_redir=raw_input()
data_redir=data_redir.split()
url,ip_host,user,method,urlgroup = data_redir[0:5]
ip=ip_host.split("/")[0]
host_name=ip_host.split("/")[1]
uri = url
mode=""
#Don't check cache access
if string.find(url,"cache_object") == 0:
sys.stdout.write("%s%s\n" %(mode,uri))
sys.stdout.flush()
continue
db=libSiSCont.connectDB(dbDicc=self.__dbDicc)
c=db.cursor()
c.execute(query,user)
cuote_type,cuote, ext_cuote, active, acc_type = c.fetchall()[0]
self.__dbParam=[cuote_type,int(cuote), int(ext_cuote), active, acc_type]
for module in modules:
try:
uri = module.redir(url = url, ip = ip, host_name = host_name, user = user, method = method, urlgroup = urlgroup, redirDicc = self.__redirDicc, penalURL = self.__penalURL, confDicc = self.__confDicc, proxyDicc = self.__proxyDicc, dbParam = self.__dbParam)
except Exception, e:
libSiSCont.reportLogs("%s.run" %__name__, 'Error while running module: %s -- The error was: %s' % (module,str(e)))
if uri != url:
mode = "301:"
break
sys.stdout.write("%s%s\n" %(mode,uri))
sys.stdout.flush()
except Exception, e:
if not string.find('%s' % e,'EOF') >= 0:
sys.stdout.write('%s\n' % uri)
sys.stdout.flush()
libSiSCont.reportLogs("%s.run" %__name__, '%s: data received from parent: %s' % (str(e),string.join(data_redir)))
else:
sys.exit()
obj=RedirMaster()
obj.run()
Helper:
#!/usr/bin/env python
import sys, syslog, libSiSCont, string,crypt
__name__ = "Helper"
query = "SELECT accounts_proxyaccount.proxy_username FROM accounts_proxyaccount WHERE accounts_proxyaccount.proxy_username=%s AND accounts_proxyaccount.proxy_password=%s"
class BasicAuth:
def __init__(self):
obj = libSiSCont.ParceConf()
obj.parcecfgfile()
self.__dbDicc = obj.getDBDicc()
def run(self):
while 1:
try:
user_pass = string.split(raw_input())
user = user_pass[0].strip("\n")
passwd = user_pass[1].strip("\n")
crypt_passwd = crypt.crypt(passwd,user)
db = libSiSCont.connectDB(self.__dbDicc)
c = db.cursor()
c.execute(query,(user,crypt_passwd))
if c.fetchone() == None:
libSiSCont.reportLogs('%s.run' %__name__,'User Authentication Fail, user = %s password= %s, Access Denied' %(user,passwd) )
sys.stdout.write("ERR\n")
sys.stdout.flush()
else:
libSiSCont.reportLogs('%s.run' %__name__, 'User Authentication Success, user = %s, Access Granted' %user)
sys.stdout.write("OK\n")
sys.stdout.flush()
except Exception, e:
if not string.find("%s" %e, "EOF") >= 0:
sys.stdout.write("ERR\n")
sys.stdout.flush()
libSiSCont.reportLogs('%s.run' %__name__, 'Authenticator error, user will navigate without authentication: %s' %str(e))
else:
sys.exit()
obj = BasicAuth()
obj.run()
I hope help you ;-)
I'd like to know how to write TCP tunnel/relay/bridge/proxy (you name it) using twisted.
I did some research in google, twisted doc/forum etc. etc but couldn't find anwser.
I already done it in pure python using socket, threading and select.
Here is code:
#!/usr/bin/env python
import socket
import sys
import select
import threading
import logging
import time
class Client(threading.Thread):
def __init__(self, client, address, id_number, dst_ip, dst_port):
self.log = logging.getLogger(__name__+'.client-%s' % id_number)
self.running = False
self.cl_soc = client
self.cl_adr = address
self.my_id = id_number
self.dst_ip = dst_ip
self.dst_port = dst_port
threading.Thread.__init__(self)
def stop(self):
self.running = 0
def run(self):
try:
self.dst_soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.dst_soc.connect((self.dst_ip,self.dst_port))
except:
self.log.error('Can\'t connect to %s:%s' % (self.dst_ip, self.dst_port))
else:
self.running = True
self.log.info('Bridge %s <-> %s created' % (
'%s:%s' % self.dst_soc.getpeername(), '%s:%s' % self.cl_adr))
try:
while self.running:
iRdy = select.select([self.cl_soc, self.dst_soc],[],[], 1)[0]
if self.cl_soc in iRdy:
buf = self.cl_soc.recv(4096)
if not buf:
info = 'Ended connection: client'
self.running = False
else:
self.dst_soc.send(buf)
if self.dst_soc in iRdy:
buf = self.dst_soc.recv(4096)
if not buf:
info = 'Ended connection: destination'
self.running = False
else:
self.cl_soc.send(buf)
except:
self.log.error('Sth bad happend', exc_info=True)
self.running = False
self.log.debug('Closing sockets')
try:
self.dst_soc.close()
except:
pass
try:
self.cl_soc.close()
except:
pass
class Server(threading.Thread):
def __init__(self, l_port=25565, d_ip='127.0.0.1', d_port=None):
self.log = logging.getLogger(__name__+'.server-%s:%s' % (d_ip,d_port))
threading.Thread.__init__(self)
self.d_ip = d_ip
if d_port == None:
self.d_port = l_port
else:
self.d_port = d_port
self.port = l_port
binding = 1
wait = 30
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
while binding:
try:
self.s.bind(('',self.port))
except:
self.log.warning('Cant bind. Will wait %s sec' % wait)
time.sleep(wait)
else:
binding = 0
self.log.info('Server ready for connections - port %s' % d_port)
def run(self):
self.s.listen(5)
input = [self.s, sys.stdin]
running = 1
self.cl_threads = []
id_nr = 0
while running:
iRdy = select.select(input, [], [],1)[0]
if self.s in iRdy:
c_soc, c_adr = self.s.accept()
c = Client(c_soc, c_adr, id_nr, self.d_ip, self.d_port)
c.start()
self.cl_threads.append(c)
id_nr += 1
if sys.stdin in iRdy:
junk = sys.stdin.readline()
print junk
running = 0
try:
self.s.close()
except:
pass
for c in self.cl_threads:
c.stop()
c.join(5)
del c
self.cl_threads = None
self.log.info('Closing server')
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
s = Server(1424, '192.168.1.6', 1424)
s.run()
this is actually, already built into twisted, from the command line you can type:
$ twistd --nodaemon portforward --port 1424 --host 192.168.1.6
to get the exact behavior you seem to be looking for.
If you'd like to roll your own, you can still use all of the bits, in twisted.protocols.portforward
I have written a simple class using urllib and urllib2 to send http requests and get the response. However, any request to localhost using its IP Address is very slow.
IP Address of LOCALHOST is = 192.168.158.27
import urllib2,urllib,re,datetime,time
class HTTPRequest():
def __init__(self,**kargs):
self._response = None
self._buffer = None
self._conn = urllib2.build_opener(urllib2.HTTPCookieProcessor())
urllib2.install_opener(self._conn)
def _encode_url(self,**kargs):
try:
params = urllib.urlencode(kargs)
except:
raise HTTPError("Failed to encode URL parameters..")
return str(params)
def _request(self,url=None,params=None):
try:
self._buffer = self._conn.open(url,params)
self._response = self._buffer.read()
except ValueError:
raise HTTPError("Invalid URL %s" % url)
except:
raise HTTPError("Failed to send HTTP(s) Request")
return str(self._response)
class HTTPError(Exception):
pass
PARAM_PASSWORD = 'password'
PARAM_USER = 'userName'
PARAM_ACTION = 'a'
PARAM_RID = 'rid'
PARAM_XO = 'xo'
PARAM_START_TIME = 't1'
PARAM_END_TIME = 't2'
PARAM_PATH = 'path'
BOOLEAN_TRUE = 'true'
BOOLEAN_FALSE = 'false'
ACTION_SIGNIN = 'signIn'
ACTION_SEARCH = 'search'
ACTION_GET_NEXT_RESULTS = 'getNextResults'
STATUS_SUCCEEDED = 'succeeded'
DEFAULT_WAIT = 5
host = "192.168.158.27"
user = "admin"
password = "admin"
protocol = "https"
port = 8443
query = "vm[=name rx (?i) *]&[#cpuUsage rx b .+][#cpuUsagemhz rx b .+]"
start_time = "10/05/2013 16:16:00"
end_time = "10/05/2013 17:16:00"
base_url = "%s://%s:%d" % (protocol,host,port)
login_url = "%s/user" % base_url
http = HTTPRequest()
attributes = {PARAM_PASSWORD : password,
PARAM_USER : user,
PARAM_ACTION : ACTION_SIGNIN,
PARAM_RID : 1000,
PARAM_XO : BOOLEAN_TRUE}
params = http._encode_url(**attributes)
if not http._request(login_url,params):
print "Login Failed.."
else:
print "Login Successful.. \n"
rid = 1000
search_url = "%s/Search" % base_url
status = STATUS_SUCCEEDED
hasMoreData = BOOLEAN_TRUE
completed = BOOLEAN_FALSE
total = 0
processed = 1
responseContent = ""
xml_dict = {}
_response = ""
attributes = {PARAM_START_TIME : start_time,
PARAM_END_TIME : end_time,
PARAM_ACTION : ACTION_SEARCH,
PARAM_RID : rid,
PARAM_PATH : query}
print "URL PARAMETERS :"
print "\tBase url = %s" % base_url
for param in attributes:
print "\t%s = %s" % (param,attributes[param])
#Query Execution Start Time
start = datetime.datetime.now()
while True:
params = http._encode_url(**attributes)
if hasMoreData == BOOLEAN_TRUE:
#Delay 10ms
time.sleep(10/1000)
#Send HTTP Request
response = http._request(search_url,params)
pattern = re.match(".*?hasMoreData=\"(.*?)\".*?",response)
if pattern:
hasMoreData = pattern.group(1)
pattern = re.match(".*?status=\"(.*?)\".*?",response)
if pattern:
status = pattern.group(1)
pattern = re.match(".*?completed=\"(.*?)\".*?",response)
if pattern:
completed = pattern.group(1)
pattern = re.match(".*?processed=\"(.*?)\".*?",response)
if pattern:
processed = pattern.group(1)
pattern = re.match(".*?total=\"(.*?)\".*?",response)
if pattern:
total = pattern.group(1)
pattern = re.match(".*?matched=\"(.*?)\".*?",response)
if pattern:
matched = pattern.group(1)
attributes = {PARAM_ACTION : ACTION_GET_NEXT_RESULTS,
PARAM_RID : rid}
if matched != "0":
response = re.sub(r'\n',"",response)
matchObj = re.search(r'(<Resource.*</Resource>)',response)
resp_match = ""
if matchObj:
resp_match = matchObj.group(1)
responseContent = str(responseContent) + str(resp_match)
else:
#Query Execution Completed
#Query Execution End Time
end = datetime.datetime.now()
print "RESULTS : "
print "\tStatus = %s"%status
print "\tHas More Data = %s"%hasMoreData
print "\tCompleted = %s"%completed
print "\tProcessed = %s"%processed
print "\tTotal = %s"%total
print "\tMatched = %s"%matched
print "\nQuery Execution Started : %s" % start
print "Query Execution Ended : %s\n" % end
if total != processed:
err = "The number records processed did not match"
err += " with the number of records completed."
print err
if not status == STATUS_SUCCEEDED:
err = "The responce status is not 'succeeded'"
print err
if completed == BOOLEAN_FALSE:
err = "The repsponse is completed. "
err += "However, the flag is set to 'False'"
print err
break
Instead of your local network IP, try using 127.0.0.1 instead.
When I have a except in memcache.Client, I can capture the exception, but mc.getstats still exec, what should I do to stop the main when have an exception?
def main():
if 'host' not in dir():
host = '127.0.0.1'
if 'port' not in dir():
port = '11211'
server = host + ':' + port
try:
mc = memcache.Client([server], debug=1,socket_timeout=3)
result = mc.get_stats()
mcstat = result[0][0]
print mcstat
except Exception,e:
print e
sys.exit(3)
if __name__ == "__main__":
try:
main()
except:
sys.exit(2)
import memcache
import sys
def main():
host = "127.0.0.1"
port = 11211
my_server = "{}:{}".format(host, port)
try:
mc = memcache.Client(
my_server #<**** Should be an array
)
result = mc.get_stats()
mcstat = result[0][0]
print mcstat
except ValueError, e:
print "Mission control: There was a problem..."
print e
sys.exit(3)
--output:--
Mission control: There was a problem...
Unable to parse connection string: "1"
I have a interval of 60 and wanted to print 6 events every 1 minutes. But it prints 11,12 and 13 events randomly every 1 minutes. Why is that so ? Is it because of my codes or what other factors can cause this ?
My code is -
import logging
import httplib
import simplejson as json
import socket
import time
import datetime
import urllib2
import sys
import xml.dom.minidom
from bs4 import BeautifulSoup as soup
SCHEME = """<scheme>
<title>testingCurrentWeatherSG</title>
<description>Get data from forecast.</description>
<use_external_validation>true</use_external_validation>
<streaming_mode>simple</streaming_mode>
<endpoint>
<args>
<arg name="intervalone">
<title>Intervalone</title>
<description>How long to refresh this query?</description>
</arg>
</args>
</endpoint>
</scheme>
"""
def do_scheme():
print SCHEME
## Utility functions
def fahrenheit(fahren):
return (fahren-32) * 5.0/9.0
def get_percent(num):
return num * 100.
## Responses
def get_response(conn, url):
try:
conn.request('GET', url)
result = conn.getresponse()
data = result.read()
return json.loads(data)
except socket.timeout:
return None
## Printing
def print_forecast(name, di):
# Print the forcast from 'di', for location 'name'
# name is the name of the location, di is the api response
psi_avg=20
current = di['currently']
for key, value in sorted(current.iteritems()):
if key in ['cloudCover', 'icon', 'ozone', 'precipIntensity', # time
'precipProbability', 'precipType', 'pressure', 'summary',
'visibility', 'windBearing', 'windSpeed']:
print '{0} : {1}'.format(key, value)
elif key in ['temperature', 'dewPoint']:
print '%s: %.2f' % (key, fahrenheit(value))
elif key == 'humidity':
print '%s: %.2f' % (key, get_percent(value))
print 'psiAverage : ' + str(psi_avg)
print 'latitude : ' + str(di['latitude'])
print 'longitude : ' + str(di['longitude'])
print 'location : ' + str(name)
print
def weather_Connection(intervalone):
host = 'api.forecast.io'
conn = httplib.HTTPSConnection(host, timeout=60) # adjust timeout as desired
try:
urlnyp = '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.37871,103.848808'
conn.request('GET', urlnyp)
resultnyp = conn.getresponse()
contentnyp = resultnyp.read()
except socket.timeout:
print 'socket timeout'
return
# the locations and urls for the api calls
urls = {
'Choa Chu Kang': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.394557,103.746396',
'Kallang': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.311469,103.871399',
'Jurong West': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.352008,103.698599',
'Redhill': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.289732,103.81675',
'Tampines': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.353092,103.945229',
'Yishun': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.429463,103.84022',
}
responses = {}
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
return
responses[name] = response
conn.close()
# print the forecast
for name, data in responses.iteritems():
print_forecast(name, data)
def get_config():
#Read XML Configuration data passed from splunkd on stdin
config = {}
try:
# read everything from stdin
config_str = sys.stdin.read()
# parse the config XML
doc = xml.dom.minidom.parseString(config_str)
root = doc.documentElement
conf_node = root.getElementsByTagName("configuration")[0]
if conf_node:
logging.debug("XML: found configuration")
stanza = conf_node.getElementsByTagName("stanza")[0]
if stanza:
stanza_name = stanza.getAttribute("name")
if stanza_name:
logging.debug("XML: found stanza " + stanza_name)
config["name"] = stanza_name
params = stanza.getElementsByTagName("param")
for param in params:
param_name = param.getAttribute("name")
logging.debug("XML: found param '%s'" % param_name)
if param_name and param.firstChild and \
param.firstChild.nodeType == param.firstChild.TEXT_NODE:
data = param.firstChild.data
config[param_name] = data
logging.debug("XML: '%s' -> '%s'" % (param_name, data))
if not config:
raise Exception, "Invalid configuration received from Splunk."
except Exception, e:
raise Exception, "Error getting Splunk configuration via STDIN: %s" % str(e)
return config
def run():
#The Main function that starts the action. The thread will sleep for however many seconds are configured via the Input.
# config = get_config()
#
#
# intervalone = config["intervalone"]
intervalone =60
while True:
weather_Connection(intervalone)
logging.info("Sleeping for %s seconds" %(intervalone))
time.sleep(float(intervalone))
if __name__ == '__main__':
if len(sys.argv) > 1:
if sys.argv[1] == "--scheme":
do_scheme()
else:
run()
sys.exit(0)
I've checked and tried your code and it works fine. Try replacing
logging.info("Sleeping for %s seconds" %(intervalone))
with
print("Sleeping for %s seconds" % (intervalone))
You should see this statement every 6 forecasts.
Note: why returning from weather_Connection() here
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
return
responses[name] = response
You can just skip it with continue
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
continue
responses[name] = response