Develop Reference

Threaded python socket server

I have a Multihtreaded Server with python that can handle clients request, but i have a problem with this.
In my Server Class I have a start function that start listening to clients like this:
class Server:
def __init__(self, clients={}):
self.clients = clients
self.ip = 'localhost'
self.port = ****
self.pattern = '(C\d)'
def start(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.s.bind((self.ip, self.port))
self.s.listen(10)
while 1:
clientsock, addr = self.s.accept()
print ('Connected with ' + addr[0] + ':' + str(addr[1]))
_thread.start_new_thread(self.handler, (clientsock, addr))
def handler(self, clientsock, addr):
data = clientsock.recv(BUFF)
print ('Data : ' + repr(data))
data = data.decode("UTF-8")
result = re.match(self.pattern, data)
print (data)
if(result):
self.registerClient(clientsock, data)
if(data == "Exit"):
self.exitClient(clientsock)
def server_response(self, message, flag, err):
if(flag):
res = message.encode('utf-8')
return res
else:
res = message.encode('utf-8')+ "[ ".encode('utf-8')+err.encode('utf-8')+ " ]".encode('utf-8')
return res
def registerClient(self, clientsock, data):
if(data in self.clients):
err = "Error : Client Name Exist!"
clientsock.send(self.server_response('Reg#NOK#', 0, err))
clientsock.close()
sys.exit(1)
self.clients[clientsock] = data
clientsock.send(self.server_response('Reg#OK', 1, ''))
def exitClient(self, clientsock):
try:
f = self.clients.pop(clientsock)
clientsock.send(self.server_response('BYE#OK', 1, ''))
clientsock.close()
except KeyError:
err = "Error : Client Doesn't Connected To Server!"
clientsock.send(self.server_response('BYE#NOK#', 0, err))
clientsock.close()
sys.exit(1)
And this is my client Class:
class Client:
def __init__(self, name):
self.name = name
self.ip = '127.0.0.1'
self.next_client = None
self.s = ""
try:
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except:
print ('Reg#NOK#[ ' + msg[1] + ' ]')
sys.exit()
def register(self, server):
self.s.connect((server.ip, server.port))
message = self.name
try:
self.s.sendall(bytes(message, 'UTF-8'))
except (socket.error):
print ('Send Failed')
sys.exit()
reply = self.s.recv(4096)
print ("Respose From Server : " + reply.decode("utf-8") )
def exitFromServer(self, server):
message = "Exit".encode('utf-8')
try:
a = self.s.sendall(message)
except (socket.error):
print ('Send Failed')
sys.exit()
reply = self.s.recv(4096)
And this is the main file:
from server import *
from client import *
import _thread
a = Server()
_thread.start_new_thread(a.start, ())
b = Client("C1")
b.register(a)
b.exitFromServer(a)
As you can see when start function from Server class called there is no thread that can handle create Client , I mean when I use start function like this with out thread there is no way that program can go ahead in main file , I know I should use another thread here but where and how, I use Python 3.4, Thanks for helping me.
Edit
the Problem with start was Solved , Thanks from tdelaney,
but when I run this only the register function works and exitFromServer dont do anything can you tell me where is the problem.the program dosent do anything after execute register function and it seems that its wating for something.

This mean?
import threading
from server import *
from client import *
global a
a = Server()
def sServer():
global a
a.start()
def cClient():
global a
b = Client("C1")
b.register(a)
s = threading.Thread(name='server', target=sServer)
c = threading.Thread(name='client', target=cClient)
s.start()
c.start()

In Server Class I must add another while True loop after handler function cause it shuould do all client request till client has request:
def handler(self, clientsock, addr):
while 1:
data = clientsock.recv(BUFF)
print ('Data : ' + repr(data))
data = data.decode("UTF-8")
result = re.match(self.pattern, data)
if(result):
self.registerClient(clientsock, data)
if(data == "Exit"):
self.exitClient(clientsock)
break

Many clients and many servers in RabbitMQ

I read this and can you explain few things?
For exaple I have run rpc_server.py in different tabs (3 tabs) of terminal.
rpc_server.py from that tutorial:
#!/usr/bin/env python
import pika
connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='rpc_queue')
def fib(n):
if n == 0:
return 0
elif n == 1:
return 1
else:
return fib(n-1) + fib(n-2)
def on_request(ch, method, props, body):
n = int(body)
print " [.] fib(%s)" % (n,)
response = fib(n)
ch.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(correlation_id = \
props.correlation_id),
body=str(response))
ch.basic_ack(delivery_tag = method.delivery_tag)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(on_request, queue='rpc_queue')
print " [x] Awaiting RPC requests"
channel.start_consuming()
Nice, I need to send in send.py 3 requests:
#!/usr/bin/env python
import pika
import uuid
class FibonacciRpcClient(object):
def __init__(self):
self.connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
self.channel = self.connection.channel()
result = self.channel.queue_declare(exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(self.on_response, no_ack=True,
queue=self.callback_queue)
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = body
def call(self, n):
self.response = None
self.corr_id = str(uuid.uuid4())
self.channel.basic_publish(exchange='',
routing_key='rpc_queue',
properties=pika.BasicProperties(
reply_to = self.callback_queue,
correlation_id = self.corr_id,
),
body=str(n))
while self.response is None:
self.connection.process_data_events()
return int(self.response)
fibonacci_rpc = FibonacciRpcClient()
print " [x] Requesting fib(30)"
response = fibonacci_rpc.call(30)
print " [.] Got %r" % (response,)
fibonacci_rpc1 = FibonacciRpcClient()
print " [x] Requesting fib(30)"
response1 = fibonacci_rpc1.call(30)
print " [.] Got %r" % (response1,)
fibonacci_rpc2 = FibonacciRpcClient()
print " [x] Requesting fib(30)"
response2 = fibonacci_rpc2.call(30)
print " [.] Got %r" % (response2,)
Does it mean that script will waiting response from first request, then send second requests, wait again response, and then send third request?
I want to do 3 request in one moment, not to wait response and then send new request. How to do this?
How I need to change send.py or use another technique? Must I use threading or multiprocessing? Does RabbitMQ supports this?
Thanks!

You would need to use threads if you want all three requests sent at once. A simple solution would look something like this:
import threading
from time import sleep
def make_rpc_call(value):
fibonacci_rpc = FibonacciRpcClient()
print " [x] Requesting fib({0})".format(value)
response = fibonacci_rpc.call(value)
print " [.] Got %r" % (response,)
for index in xrange(5):
thread_ = threading.Thread(target=make_rpc_call, args=(30, ))
thread_.start()
sleep(0.1)
Keep in mind that Pika is not thread-safe, so you need to create one connection per thread. As an alternative you could look at my Flask example here for Pika. It is easy enough to modify to allow you to execute multiple requests asynchronously.
def rpc_call():
# Fire of all three requests.
corr_id1 = rpc_client.send_request('1')
corr_id2 = rpc_client.send_request('2')
corr_id3 = rpc_client.send_request('3')
# Wait for the response on all three requests.
while not rpc_client.queue[corr_id1] \
or not rpc_client.queue[corr_id2] \
or not rpc_client.queue[corr_id3]:
sleep(0.1)
# Print the result of all three requests.
print rpc_client.queue[corr_id1]
print rpc_client.queue[corr_id2]
print rpc_client.queue[corr_id3]
if __name__ == '__main__':
rpc_client = RpcClient('rpc_queue')
sleep(1)
print rpc_call()

exiting thread application with signal interrupt in python

I am on OSx and I am trying to exit the program by pressing CTRL+C.
but it seems like even if I have a signal handler registered in main thread it doesn't exit while the thread is executing on pressing CTRL+C.
Here is a piece of cake where I am trying to download 3 mp3 files from internet each in separate thread.
import Queue
import urllib2
import os
import signal
import sys
import time
import threading
from socket import error as _SocketError
urls = ["http://broadcast.lds.org/churchmusic/MP3/1/2/nowords/271.mp3",
"http://s1.fans.ge/mp3/201109/08/John_Legend_So_High_Remix(fans_ge).mp3",
"http://megaboon.com/common/preview/track/786203.mp3"]
queue = Queue.Queue()
def do_exit(sigNum, stack):
# handle unix signal recived and exit
sys.stderr.write("Received signal %d " % (sigNum))
raise SystemExit("Exiting")
class ThreadedFetch(threading.Thread):
""" docstring for ThreadedFetch
"""
def __init__(self, queue, count = 1):
super(ThreadedFetch, self).__init__()
self.queue = queue
def run(self):
while True:
# grabs url of link and path to saveTo and save to lst
host = self.queue.get()
# submit the url for download and location where to save.
self._downloadFile(host[0], host[1])
def _downloadFile(self, url, saveTo=None):
file_name = url.split('/')[-1]
self.setName("Parent_%s_thread" % file_name.split(".")[0])
if not saveTo:
saveTo = '/Users/sanjeevkumar/Desktop'
try:
u = urllib2.urlopen(url)
except urllib2.URLError , er:
print("%s for %s failed to download." % (er.reason, file_name))
self.queue.task_done()
print "Exiting: %s" % self.getName()
except _SocketError , err:
print("%s \n %s failed to download." % (err, file_name))
self.queue.task_done()
else:
th = threading.Thread(
target=self._fileWriteToDisk,
args=(saveTo, u, file_name),
name="fileWrite_Child_of_%s" % self.getName(),
)
# if the user clicks close while the thread is still running,
# then the programme will wait till the save is done,
# then it will close.
th.daemon = False
th.start()
time.sleep(0.1)
print "Writing to disk using child: %s " % th.name
def _fileWriteToDisk(self, saveTo, urlObject, file_name):
path = os.path.join(saveTo, file_name)
try:
f = open(path, 'wb')
except IOError , er:
self.queue.task_done()
print er
return
meta = urlObject.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s : %s " % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
while True:
buffer = urlObject.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8)*(len(status)+1)
sys.stdout.write('%s\r' % status)
time.sleep(.05)
sys.stdout.flush()
if file_size_dl == file_size:
print r"Download Completed %s%% for file %s, saved to %s" % (file_size_dl * 100. / file_size, file_name, saveTo)
f.close()
# signals to queue job is done
self.queue.task_done()
def main():
# Register signal in main thread
signal.signal(signal.SIGINT, do_exit)
try:
# spawn a pool of threads, and pass them queue instance
for i in range(len(urls)):
t = ThreadedFetch(queue)
t.setDaemon(True)
time.sleep(0.1)
t.start()
urls_saveTo = {urls[0]: None, urls[1]: None, urls[2]: None}
# populate queue with data
for item, value in urls_saveTo.iteritems():
queue.put([item, value])
# wait on the queue until everything has been processed
queue.join()
print '*** Done'
except (KeyboardInterrupt, SystemExit):
print '\n! Received keyboard interrupt, quitting threads.\n'
if __name__ == "__main__":
main()
Feel free to point what else can be improved strategically.
any help would be greatly appreciated.

Threaded SocketServer not receiving second message

I am trying to implement a simple threaded SocketServer (using SocketServer.ThreadedMixIn). However, my server stops receiving further messages. Here is the code:
#!/usr/bin/python -u
import SocketServer
import sys
class MYAgentHandler(SocketServer.BaseRequestHandler):
def handle(self):
try:
data = self.request.recv(1024)
print "Received request " + str(data) + "\n"
reply = str(agent.processAgentMessage(data))
self.request.send(reply)
self.request.close()
except Exception, instr:
print "While processing data " + data + " error encountered " + str(instr) + "\n"
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
daemon_threads = True
allow_reuse_address = True
def __init__(self, server_address, RequestHandlerClass):
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
class MYAgent:
def processAgentMessage(self, msg):
try:
tokens = msg.split('^')
if tokens[0] == "CreateSession":
return("New session")
elif tokens[0] == "GetStatus":
return("Init")
except Exception, instr:
print "Error while processing message " + str(instr) + "\n"
agent = MYAgent()
def main():
MYServer = sys.argv[1]
MYAgentPort = sys.argv[2]
agent.listener = ThreadedTCPServer((MYServer, int(MYAgentPort)), MYAgentHandler)
agent.listener.serve_forever()
if __name__ == '__main__':
main()
And here is my client:
#!/usr/bin/python -u
import socket
import time
if __name__ == "__main__":
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', 15222))
try:
sock.send("CreateSession")
sessionID = str(sock.recv(1024))
print "Received: " + sessionID
sock.send("GetStatus^"+sessionID)
print "Sent Getstatus\n"
time.sleep(1)
response = str(sock.recv(1024))
print "status of " + str(sessionID) + " is " + str(response) + "\n"
sock.close()
except Exception, instr:
print "Error occurred " + str(instr) + "\n"
Here is one session. Server output:
$ ./t.py localhost 15222
Received request CreateSession
Client output:
$ ./client.py
Received: New session
Sent Getstatus
status of New session is
$
Any ideas why this is happening?

You have to remove self.request.close() (which closes the connection) and wrap everything with while True: (so it will continue to read from the same socket).

python calling apis events and printing randomly

I have a interval of 60 and wanted to print 6 events every 1 minutes. But it prints 11,12 and 13 events randomly every 1 minutes. Why is that so ? Is it because of my codes or what other factors can cause this ?
My code is -
import logging
import httplib
import simplejson as json
import socket
import time
import datetime
import urllib2
import sys
import xml.dom.minidom
from bs4 import BeautifulSoup as soup
SCHEME = """<scheme>
<title>testingCurrentWeatherSG</title>
<description>Get data from forecast.</description>
<use_external_validation>true</use_external_validation>
<streaming_mode>simple</streaming_mode>
<endpoint>
<args>
<arg name="intervalone">
<title>Intervalone</title>
<description>How long to refresh this query?</description>
</arg>
</args>
</endpoint>
</scheme>
"""
def do_scheme():
print SCHEME
## Utility functions
def fahrenheit(fahren):
return (fahren-32) * 5.0/9.0
def get_percent(num):
return num * 100.
## Responses
def get_response(conn, url):
try:
conn.request('GET', url)
result = conn.getresponse()
data = result.read()
return json.loads(data)
except socket.timeout:
return None
## Printing
def print_forecast(name, di):
# Print the forcast from 'di', for location 'name'
# name is the name of the location, di is the api response
psi_avg=20
current = di['currently']
for key, value in sorted(current.iteritems()):
if key in ['cloudCover', 'icon', 'ozone', 'precipIntensity', # time
'precipProbability', 'precipType', 'pressure', 'summary',
'visibility', 'windBearing', 'windSpeed']:
print '{0} : {1}'.format(key, value)
elif key in ['temperature', 'dewPoint']:
print '%s: %.2f' % (key, fahrenheit(value))
elif key == 'humidity':
print '%s: %.2f' % (key, get_percent(value))
print 'psiAverage : ' + str(psi_avg)
print 'latitude : ' + str(di['latitude'])
print 'longitude : ' + str(di['longitude'])
print 'location : ' + str(name)
print
def weather_Connection(intervalone):
host = 'api.forecast.io'
conn = httplib.HTTPSConnection(host, timeout=60) # adjust timeout as desired
try:
urlnyp = '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.37871,103.848808'
conn.request('GET', urlnyp)
resultnyp = conn.getresponse()
contentnyp = resultnyp.read()
except socket.timeout:
print 'socket timeout'
return
# the locations and urls for the api calls
urls = {
'Choa Chu Kang': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.394557,103.746396',
'Kallang': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.311469,103.871399',
'Jurong West': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.352008,103.698599',
'Redhill': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.289732,103.81675',
'Tampines': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.353092,103.945229',
'Yishun': '/forecast/59ff8cb7661d231f2967c2663c0a3bdc/1.429463,103.84022',
}
responses = {}
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
return
responses[name] = response
conn.close()
# print the forecast
for name, data in responses.iteritems():
print_forecast(name, data)
def get_config():
#Read XML Configuration data passed from splunkd on stdin
config = {}
try:
# read everything from stdin
config_str = sys.stdin.read()
# parse the config XML
doc = xml.dom.minidom.parseString(config_str)
root = doc.documentElement
conf_node = root.getElementsByTagName("configuration")[0]
if conf_node:
logging.debug("XML: found configuration")
stanza = conf_node.getElementsByTagName("stanza")[0]
if stanza:
stanza_name = stanza.getAttribute("name")
if stanza_name:
logging.debug("XML: found stanza " + stanza_name)
config["name"] = stanza_name
params = stanza.getElementsByTagName("param")
for param in params:
param_name = param.getAttribute("name")
logging.debug("XML: found param '%s'" % param_name)
if param_name and param.firstChild and \
param.firstChild.nodeType == param.firstChild.TEXT_NODE:
data = param.firstChild.data
config[param_name] = data
logging.debug("XML: '%s' -> '%s'" % (param_name, data))
if not config:
raise Exception, "Invalid configuration received from Splunk."
except Exception, e:
raise Exception, "Error getting Splunk configuration via STDIN: %s" % str(e)
return config
def run():
#The Main function that starts the action. The thread will sleep for however many seconds are configured via the Input.
# config = get_config()
#
#
# intervalone = config["intervalone"]
intervalone =60
while True:
weather_Connection(intervalone)
logging.info("Sleeping for %s seconds" %(intervalone))
time.sleep(float(intervalone))
if __name__ == '__main__':
if len(sys.argv) > 1:
if sys.argv[1] == "--scheme":
do_scheme()
else:
run()
sys.exit(0)

I've checked and tried your code and it works fine. Try replacing
logging.info("Sleeping for %s seconds" %(intervalone))
with
print("Sleeping for %s seconds" % (intervalone))
You should see this statement every 6 forecasts.
Note: why returning from weather_Connection() here
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
return
responses[name] = response
You can just skip it with continue
for i, (name, url) in enumerate(sorted(urls.iteritems())):
response = get_response(conn, url)
if not response:
print 'socket timeout on url#%d: %s' % (i, url)
continue
responses[name] = response