I read this and can you explain few things?
For exaple I have run rpc_server.py in different tabs (3 tabs) of terminal.
rpc_server.py from that tutorial:
#!/usr/bin/env python
import pika
connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='rpc_queue')
def fib(n):
if n == 0:
return 0
elif n == 1:
return 1
else:
return fib(n-1) + fib(n-2)
def on_request(ch, method, props, body):
n = int(body)
print " [.] fib(%s)" % (n,)
response = fib(n)
ch.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(correlation_id = \
props.correlation_id),
body=str(response))
ch.basic_ack(delivery_tag = method.delivery_tag)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(on_request, queue='rpc_queue')
print " [x] Awaiting RPC requests"
channel.start_consuming()
Nice, I need to send in send.py 3 requests:
#!/usr/bin/env python
import pika
import uuid
class FibonacciRpcClient(object):
def __init__(self):
self.connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
self.channel = self.connection.channel()
result = self.channel.queue_declare(exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(self.on_response, no_ack=True,
queue=self.callback_queue)
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = body
def call(self, n):
self.response = None
self.corr_id = str(uuid.uuid4())
self.channel.basic_publish(exchange='',
routing_key='rpc_queue',
properties=pika.BasicProperties(
reply_to = self.callback_queue,
correlation_id = self.corr_id,
),
body=str(n))
while self.response is None:
self.connection.process_data_events()
return int(self.response)
fibonacci_rpc = FibonacciRpcClient()
print " [x] Requesting fib(30)"
response = fibonacci_rpc.call(30)
print " [.] Got %r" % (response,)
fibonacci_rpc1 = FibonacciRpcClient()
print " [x] Requesting fib(30)"
response1 = fibonacci_rpc1.call(30)
print " [.] Got %r" % (response1,)
fibonacci_rpc2 = FibonacciRpcClient()
print " [x] Requesting fib(30)"
response2 = fibonacci_rpc2.call(30)
print " [.] Got %r" % (response2,)
Does it mean that script will waiting response from first request, then send second requests, wait again response, and then send third request?
I want to do 3 request in one moment, not to wait response and then send new request. How to do this?
How I need to change send.py or use another technique? Must I use threading or multiprocessing? Does RabbitMQ supports this?
Thanks!
You would need to use threads if you want all three requests sent at once. A simple solution would look something like this:
import threading
from time import sleep
def make_rpc_call(value):
fibonacci_rpc = FibonacciRpcClient()
print " [x] Requesting fib({0})".format(value)
response = fibonacci_rpc.call(value)
print " [.] Got %r" % (response,)
for index in xrange(5):
thread_ = threading.Thread(target=make_rpc_call, args=(30, ))
thread_.start()
sleep(0.1)
Keep in mind that Pika is not thread-safe, so you need to create one connection per thread. As an alternative you could look at my Flask example here for Pika. It is easy enough to modify to allow you to execute multiple requests asynchronously.
def rpc_call():
# Fire of all three requests.
corr_id1 = rpc_client.send_request('1')
corr_id2 = rpc_client.send_request('2')
corr_id3 = rpc_client.send_request('3')
# Wait for the response on all three requests.
while not rpc_client.queue[corr_id1] \
or not rpc_client.queue[corr_id2] \
or not rpc_client.queue[corr_id3]:
sleep(0.1)
# Print the result of all three requests.
print rpc_client.queue[corr_id1]
print rpc_client.queue[corr_id2]
print rpc_client.queue[corr_id3]
if __name__ == '__main__':
rpc_client = RpcClient('rpc_queue')
sleep(1)
print rpc_call()
Related
I'm setting up a little server with commands and other kind of stuff, but ,I don't get the thread functionement, when I connect my server it seems like everything is okay, I can connect a first client "without" problems, but when I want to connect another client it never get conected , the code runs but I can send anything I want it never shows up on the other client or the server.
I've already read the Threading documentation but even with the exemples, I don't get it, can someone give me some clues about how to proceed to handle more than just one client?
the server code :
#!/usr/bin/python3+x
import socket
import sys
from time import sleep
import threading
import random
HOST = "localhost"
PORT = 33700
MSG_SIZE = 32768
serveur_on = True
CLIENT_NICK_CHAN = {} #clients" nicks dict like {nickname : channel} -> needed to easily know who is where
CLIENT_NICK_SOCKET = {} #clients" dict like {nickname : socket} -> needed to send private message to the nickname's socket easily
CLIENT_NICK_THREAD = {} #clients" dict with like {nick : thread}
Rand_disconnection_msg = [" has drown in the abyss.", " is disconnected.", " is now in a better place.", " is now a part of our past", " passed away, in really tragic circumstances..."]
CHANNELS = ["main lobby", "test"]
CMD_LIST = [b"HELP",b"NICK",b"JOIN",b"CHANNELS",b"LEAVE"]
COMMANDS = ["/NICK <nickname>: Use only when you\'re connecting, allow you to choose a unique nickname",
"/JOIN <channel_name>: Allow you to join or create a channel, you can\'t use this command if your are already in another channel than the" + CHANNELS[0],
"/CHANNELS : Allow you to see every channels on the server with every connected people",
"/LEAVE : You leave the channel your within and get bringed back to the" + CHANNELS[0],
"/HELP : Gives you the whole command list",
"/BYE : Disconnect ou from the server, you have to in the " + CHANNELS[0] + " to use this command"
]
class Colors:
Blue, Cyan, Green, Red, Magenta, Yellow, White =b"\033[94m", b"\033[96m", b"\033[92m", b"\033[91m", b"\033[95m", b"\033[93m", b"\033[0m"
Normal, Bold, Italics, Thin = b"\033[0m", b"\033[1m", b"\x1B[3m", b"\033[2m"
class thread_client(threading.Thread):
def __init__(self,conn):
self.nom = ""
if(self.nom == ""):
nickname_input(connexion, self)
print("nom : " + self.nom.decode("utf8"))
self.channel = CHANNELS[0]
self.admin = False
self.adress = ""
threading.Thread.__init__(self)
self.connexion = conn
print("init done")
def run(self):
while True:
msgClient = self.connexion.recv(MSG_SIZE)
if not msgClient or msgClient == b"BYE":
break
print(type(self.nom))
print(type(msgClient))
str_name = self.nom.decode("utf8")
msg = str_name + " > " + msgClient.decode("utf8")
print("string type name is : " + str_name + "\n")
str_msg = msgClient.decode("utf8")
print("{} > {}".format(str_name, str_msg))
for clients in CLIENT_NICK_SOCKET:
if clients != self.nom:
CLIENT_NICK_SOCKET[clients].send(bytes(str_msg,"utf8"))
self.connexion.send(b"You are now disconnected.\n")
self.connexion.close()
del CLIENT_NICK_SOCKET[self.nom.decode("utf8")]
del CLIENT_NICK_CHAN[self.nom.decode("utf8")]
rand_leave = random.randint(0, len(Rand_disconnection_msg)-1)
leaving_msg = Rand_disconnection_msg[rand_leave]
print(str_name + leaving_msg + "\n")
def nickname_input(client_socket, thread):
print("now in input nickname")
msg_nom = client_socket.recv(MSG_SIZE)
print("msg_nom = " + msg_nom.decode("utf8"))
msg_nom_arr = msg_nom.split()
if not msg_nom_arr[0]:
client_socket.send(b"Please send a non void message")
nickname_input(client_socket, thread)
print("msg_nom_arr[0] = " + str(msg_nom_arr[0]))
if(msg_nom_arr[0] == b"NICK"):
if(len(msg_nom_arr)== 1):
client_socket.send(b"Please do not just enter '/NICK' -> you have to type '/NICK <your_nickname>', please proceed again :\n")
nickname_input(client_socket, thread)
else:
thread.nom = msg_nom_arr[1]
else:
client_socket.send(b"It seems like you forgot to use '/NICK' before entering your nickname, please proceed again:\n")
nickname_input(client_socket, thread)
return
def print_channels(client_socket, thread):
client_socket.send(b"Here\'s the current channel list :\n\n")
for chan in CHANNELS:
sleep(0.70)
client_socket.send( bytes(chan,"utf8") + b":\n current members :\n")
for chan_user in CLIENT_NICK_CHAN:
if(CLIENT_NICK_CHAN[chan_user] == chan):
sleep(0.35)
if(chan_user == thread.nom):
if(thread.admin):
client_socket.send(b" " +Colors.Bold + Colors.Yellow + b"#"+ thread.nom + b"#" + Colors.Normal + b"\n")
else:
client_socket.send(b" " +Colors.Bold + Colors.Yellow + thread.nom + Colors.Normal + b"\n")
else:
client_socket.send(b" " +bytes(chan_user,"utf8") + b"#\n")
client_socket.send(b"\n")
client_socket.send(b"\n")
return
def join_channel(client_socket, thread, data, data_array):
if(not data_arr[1]):
connexion.send(b"Please select a channel you want to join using '/JOIN <channel_name>'\nNote that if the channel you asked for doesn\'t exists a new channel <channel_name> will be created and you will be the administrator of this channel")
return
else:
asked_channel = data_arr[1]
if( not (asked_channel in CHANNELS)):
thread.channel = asked_channel
thread.admin = True
connexion.send(b"Welcome in " + asked_channel + b" channel, since you\'re the on who created this channel you are granted as administrator for this channel")
connexion.send(b"Note that being administrator allow you tu use some new commands as '/GRANT', '/REVOKE' or '/REN', for more information please use '/HELP'")
else:
thread.channel = asked_channel
connexion.send(b"Welcome in " + asked_channel + b" channel !")
return
SERVER = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
SERVER.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
try:
SERVER.bind((HOST,PORT))
except socket.error:
print("Server connexion failed")
sys.exit()
print("Server is now connected\nWaiting for connexions...\n")
SERVER.listen(5)
connexion, adresse = SERVER.accept()
thread = thread_client(connexion)
thread.start()
print("thread type = " +str(type(thread)) +"\n")
print("thread = ")
print(thread)
connexion.send(bytes("Welcome ","utf8") + Colors.Yellow + Colors.Bold + thread.nom + Colors.Normal)
nick = thread.nom #type -> bytes
str_nick = nick.decode("utf8")
CLIENT_NICK_CHAN[str_nick] = thread.channel
CLIENT_NICK_SOCKET[str_nick] = connexion
CLIENT_NICK_THREAD[str_nick] = thread
print("client list : ")
print(CLIENT_NICK_CHAN)
print("\n")
print("CLIENT_NICK_SOCKET = ")
print(CLIENT_NICK_SOCKET)
print("\n")
while serveur_on:
conn_msg = str_nick + " joined the chat\n"
print(conn_msg)
connexion.send(b"hello world 3\n\n")
connexion.send(b"*" * 80 + b"\n")
connexion.send(Colors.Red + Colors.Bold + b"\nWELCOME IN THE MAIN LOBBY \n" + Colors.Normal+b"\nTo enter a channel use '/JOIN <channel_name>'\nthe <channel_name> have to be composed by one world or use underscores to join words\nIf the channel does not exists a new one will be created\n\nNote that you have to be in another channel than the main lobby to chat\n")
print_channels(connexion, thread)
connexion.send(b"*" * 80 + b"\n\n")
while True:
print("thread list = ")
print(CLIENT_NICK_THREAD)
data = connexion.recv(MSG_SIZE) #receiving data from client
data_arr= data.split() #convert data into an array to check if the first word in the message is "MSG" or not
print(str_nick +" is now in -> " + thread.channel + "\n")
if(data_arr[0] in CMD_LIST):
if(data.startswith(b"HELP")): #HELP CMD
for command in COMMANDS:
connexion.send(bytes(command,"utf") + b"\n")
if(data.startswith(b"CHANNELS")): #Channels + current members CMD
connexion.send(b"\n")
print_channel(connexion, thread)
connexion.send(b"\n")
if(data.startswith(b"JOIN")):
join_channel(connexion, thread, data, data_arr)
connexion.send(b"\n")
else:
if ((thread.channel != CHANNELS[0]) and (data.startswith("MSG"))):
for chan_user in thread.channel:
chan_user.send(nick + b" > " + bytes(data,"utf8"))
print("data = " + data)
elif (thread.channel == CHANNELS[0]):
connexion.send(b"You have to be in another channel than the " + bytes(CHANNELS[0], "utf8") + b" to start chating !\nPlease use '/JOIN <channel_name>' or '/HELP' to learn how to join another channel.\n\n")
and the client code:
#!/usr/bin/python3+x
host = ''
port = 33700
MSG_SIZE = 32768
emission_stop = False
import socket
import sys
import threading
import time
def Tsend():
while True:
msg_envoi = input("> ")
if msg_envoi.startswith("/"):
msg_envoi = msg_envoi.replace("/","",1)
else:
msg_envoi = msg_envoi
CLIENT.send(bytes(msg_envoi,"utf8"))
if emission_stop:
CLIENT.close()
def Trecv():
while True:
msg_recu = CLIENT.recv(MSG_SIZE).decode("utf8")
print("\n" + msg_recu)
if not msg_recu:
break
emission_stop = True
print("connexion lost\n")
CLIENT.close()
CLIENT = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
try:
CLIENT.connect((host,port))
except socket.error:
print("connexion failed\n")
sys.exit()
print("Now connected to the server on port: {}\n".format(port))
print("Please now enter your nickname using '/NICK'\n")
thread_emission = threading.Thread(target = Tsend)
thread_reception = threading.Thread(target = Trecv)
thread_emission.start()
thread_reception.start()
What I want is just to have multiple clients that are allowed to talk to each other but I can't even get two clients.
The biggest problem I can see is that you are only calling SERVER.accept() once. This means you will only ever accept 1 client connection. When using blocking sockets as you are, a typical approach is to keep doing SERVER.accept() inside a loop so that you can keep accepting all client sockets. After you accept() a new socket, you create new thread(s) dedicated to sending/receiving for that socket, so that you don't block the accepting thread. And then you continue accepting more connections. Something like this:
#SERVER:
while serveur_on:
connexion, adresse = SERVER.accept()
# Possibly do some limited IO with client socket here, but be careful not
# to block this thread too long because that will prevent more clients from
# connecting.
thread = thread_client(connexion)
thread.start()
# No more client IO on this thread, it's the client thread's job now.
You seem to have code that communicates with the client (receiving messages and sending responses) in 2 different places: on the main thread after you SERVER.accept(), and up in thread_client.run(). It doesn't make sense that way, it should all be in thread_client.run().
I'm attempting to add an irc client to a django web application I'm working on. I'd like the server side code for the irc communication to use python and connect through the sockets library. I'll then use gevent-socketio to send and receive data from the irc channel to the client side in the browser. So far I have a very basic gevent-socketio server and client that can be used for realtime broadcasting amongst multiple clients, however, when I start to connect clients to IRC they connect to the IRC server successfully using a nick passed from the client, but then it appears that this is ongoing keepalive/listening process is blocking be from sending any messages from the client.
Python IRC code
import socket
class pycli:
def __init__(self,user):
self.nick = user
self.chan = "#testchannel"
self.owner = "Bluebot"
self.sock = socket.socket()
print "irc conn"
def send_msg(self,message):
self.sock.send("PRIVMSG " + self.chan + " : " + message + "\r\n")
## misc setup
def keep_alive(self):
self.sock.connect(("irc.freenode.net",6667))
self.sock.send("USER " + self.nick + " 0 * :" + self.owner + "\r\n")
self.sock.send("NICK " + self.nick + "\r\n")
while 1:
## keep checking for data
data = self.sock.recv(512)
datalen = len(data.split(" "))
sender = ""
msg_type = ""
msg_rcpt = ""
message = ""
###----------------------------------------------
## reply to keep alive pings
if data[0:4] == "PING":
self.sock.send(data.replace("PING", "PONG"))
if data[0]!=':':
continue
if data.split(" ")[1] == "001":
self.sock.send("MODE " + self.nick + " +B\r\n")
self.sock.send("JOIN " + self.chan + "\r\n")
###-----------------------------------------------
##split and assign data parts
## parse out the actual sender
send_data = data.split(" ")[0]
send_data = send_data[1:]
sender = send_data.split('!')[0]
## mode
msg_type = data.split(" ")[1]
## if it exists get the recipient (room or private)
if datalen > 2:
msg_rcpt = data.split(" ")[2]
## get the actual message body
if datalen > 3:
message = (" ".join(data.split(" ")[3:])).lower()[1:]
print data
I know this functionality is super basic, but I can expand on it once I get it working through the client.
The relevant parts of my server code basically looks like:
def on_login(self, nick):
if self.nick:
self._broadcast('exit', self.nick)
self.nick = nick
self._broadcast('enter', nick)
self.emit('users',
[ ns.nick
for ns in self._registry.values()
if ns.nick is not None ])
t = threading.Thread(target=self.make_start_irc(nick),daemon=True)
t.start()
def on_chat(self, message):
if self.nick:
self._broadcast('chat', dict(u=self.nick, m=message))
self._irc_nicks[self.nick].send_msg("this is a test")
else:
self.emit('chat', dict(u='SYSTEM', m='You must first login'))
def make_start_irc(self,nick):
if nick not in self._irc_nicks.values():
self._irc_nicks[nick] = pycli.pycli(nick)
print self._irc_nicks
self._irc_nicks[nick].keep_alive()
def _broadcast(self, event, message):
for s in self._registry.values():
s.emit(event, message)
def chat(environ, start_response):
if environ['PATH_INFO'].startswith('/socket.io'):
return socketio_manage(environ, { '/chat': ChatNamespace })
else:
return serve_file(environ, start_response)
def serve_file(environ, start_response):
path = os.path.normpath(
os.path.join(public, environ['PATH_INFO'].lstrip('/')))
assert path.startswith(public), path
if os.path.exists(path):
start_response('200 OK', [('Content-Type', 'text/html')])
with open(path) as fp:
while True:
chunk = fp.read(4096)
if not chunk: break
yield chunk
else:
start_response('404 NOT FOUND', [])
yield 'File not found'
if __name__ == "__main__":
from gevent import monkey
monkey.patch_all()
sio_server = SocketIOServer(
('', 8080), chat,
policy_server=False)
t2 = threading.Thread(target=sio_server.serve_forever())
t2.start()
When I eventually give up and use ctrl-C, I see the following stacktrace which leads me to believe something about the way I'm threading is blocking.
Traceback (most recent call last):
File "socketio_test.py", line 92, in <module>
t2 = threading.Thread(target=sio_server.serve_forever())
File "/Users/andrewscott/Desktop/wham/pycli/wham/lib/python2.7/site-packages/gevent/baseserver.py", line 284, in serve_forever
self._stop_event.wait()
File "/Users/andrewscott/Desktop/wham/pycli/wham/lib/python2.7/site-packages/gevent/event.py", line 77, in wait
result = self.hub.switch()
File "/Users/andrewscott/Desktop/wham/pycli/wham/lib/python2.7/site-packages/gevent/hub.py", line 338, in switch
return greenlet.switch(self)
KeyboardInterrupt
If anyone has any idea how I can change the irc process to be non-blocking, or any general suggestions they'd be greatly appreciated.
You should remove the call:
t2 = threading.Thread(target=sio_server.serve_forever())
And properly pass the method:
t2 = threading.Thread(target=sio_server.serve_forever)
I'm building a proxy checker using multithreads, specificly a thread pool from:
from multiprocessing.dummy import Pool as ThreadPool.
The http request is by using urllib2.
What I want to do is for each proxy run 20 requests. If it was 1 threaded it would take too much time. thats where the multithreads power comes to help. So once I set up the proxy I want to run those 20 requests, and manage 2 things. One is to count the exceptions and dump the proxy if too many occurs. 2nd Is to save the average response time and present it later.
I just don't manage to implement the above. But I have implemented it with 1 thread:
import socket
import ssl
import time
import urllib
import urllib2
import httplib
proxyList = []
def loadProxysFromFile(fileName):
global proxyList
with open(fileName) as f:
proxyList = [line.rstrip('\n') for line in f]
def setUrllib2Proxy(proxyAddress):
proxy = urllib2.ProxyHandler({
'http': "http://" + proxyAddress,
'https': "https://" + proxyAddress
})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
def timingRequest(proxy, url):
error = False
setUrllib2Proxy(proxy)
start = time.time()
try:
req = urllib2.Request(url)
urllib2.urlopen(req, timeout=5) #opening the request (getting a response)
except (urllib2.URLError, httplib.BadStatusLine, ssl.SSLError, socket.error) as e:
error = True
end = time.time()
timing = end - start
if error:
print "Error with proxy " + proxy
return 0
else:
print proxy + " Request to " + url + " took: %s" %timing + " seconds."
return timing
# Main
loadProxysFromFile("proxyList.txt")
for proxy in proxyList:
print "Testing: " + proxy
print "\n"
REQUEST_NUM = 20
ERROR_TOLERANCE_NUM = 3
resultList = []
for proxy in proxyList:
avgTime = 0
errorCount = 0
for x in range(0, REQUEST_NUM):
result = timingRequest(proxy, 'https://www.google.com')
if (result == 0):
errorCount += 1
if (errorCount >= ERROR_TOLERANCE_NUM):
break
else:
avgTime += result
if (errorCount < ERROR_TOLERANCE_NUM):
avgTime = avgTime/(REQUEST_NUM-errorCount)
resultList.append(proxy + " has an average response time of: %s" %avgTime)
print '\n'
print "Results Summery: "
print "-----------------"
for res in resultList:
print res
Things that must be done are:
for every proxy: wait until all 20 requests are over before changing proxy. Sync somehow the threads when they adding up to calculate the average response time (includes not to take in account the exceptions)
The best solutions I've read so far is using from multiprocessing.dummy import Pool as ThreadPool and pool.map(func, iterable) but I cant figure out how to implement it in my code.
I am trying to get tweets related to the keyword in the code But at the python shell there is nothing its just curson only No traceback nothing.The code is here
import time
import pycurl
import urllib
import json
import oauth2 as oauth
API_ENDPOINT_URL = 'https://stream.twitter.com/1.1/statuses/filter.json'
USER_AGENT = 'TwitterStream 1.0' # This can be anything really
# You need to replace these with your own values
OAUTH_KEYS = {'consumer_key': 'ABC',
'consumer_secret': 'ABC',
'access_token_key': 'ABC',
'access_token_secret': 'ABC'}
# These values are posted when setting up the connection
POST_PARAMS = {'include_entities': 0,
'stall_warning': 'true',
'track': 'iphone,ipad,ipod'}
class TwitterStream:
def __init__(self, timeout=False):
self.oauth_token = oauth.Token(key=OAUTH_KEYS['access_token_key'], secret=OAUTH_KEYS['access_token_secret'])
self.oauth_consumer = oauth.Consumer(key=OAUTH_KEYS['consumer_key'], secret=OAUTH_KEYS['consumer_secret'])
self.conn = None
self.buffer = ''
self.timeout = timeout
self.setup_connection()
def setup_connection(self):
""" Create persistant HTTP connection to Streaming API endpoint using cURL.
"""
if self.conn:
self.conn.close()
self.buffer = ''
self.conn = pycurl.Curl()
# Restart connection if less than 1 byte/s is received during "timeout" seconds
if isinstance(self.timeout, int):
self.conn.setopt(pycurl.LOW_SPEED_LIMIT, 1)
self.conn.setopt(pycurl.LOW_SPEED_TIME, self.timeout)
self.conn.setopt(pycurl.URL, API_ENDPOINT_URL)
self.conn.setopt(pycurl.USERAGENT, USER_AGENT)
# Using gzip is optional but saves us bandwidth.
self.conn.setopt(pycurl.ENCODING, 'deflate, gzip')
self.conn.setopt(pycurl.POST, 1)
self.conn.setopt(pycurl.POSTFIELDS, urllib.urlencode(POST_PARAMS))
self.conn.setopt(pycurl.HTTPHEADER, ['Host: stream.twitter.com',
'Authorization: %s' % self.get_oauth_header()])
# self.handle_tweet is the method that are called when new tweets arrive
self.conn.setopt(pycurl.WRITEFUNCTION, self.handle_tweet)
def get_oauth_header(self):
""" Create and return OAuth header.
"""
params = {'oauth_version': '1.0',
'oauth_nonce': oauth.generate_nonce(),
'oauth_timestamp': int(time.time())}
req = oauth.Request(method='POST', parameters=params, url='%s?%s' % (API_ENDPOINT_URL,
urllib.urlencode(POST_PARAMS)))
req.sign_request(oauth.SignatureMethod_HMAC_SHA1(), self.oauth_consumer, self.oauth_token)
return req.to_header()['Authorization'].encode('utf-8')
def start(self):
""" Start listening to Streaming endpoint.
Handle exceptions according to Twitter's recommendations.
"""
backoff_network_error = 0.25
backoff_http_error = 5
backoff_rate_limit = 60
while True:
self.setup_connection()
try:
self.conn.perform()
except:
# Network error, use linear back off up to 16 seconds
print 'Network error: %s' % self.conn.errstr()
print 'Waiting %s seconds before trying again' % backoff_network_error
time.sleep(backoff_network_error)
backoff_network_error = min(backoff_network_error + 1, 16)
continue
# HTTP Error
sc = self.conn.getinfo(pycurl.HTTP_CODE)
if sc == 420:
# Rate limit, use exponential back off starting with 1 minute and double each attempt
print 'Rate limit, waiting %s seconds' % backoff_rate_limit
time.sleep(backoff_rate_limit)
backoff_rate_limit *= 2
else:
# HTTP error, use exponential back off up to 320 seconds
print 'HTTP error %s, %s' % (sc, self.conn.errstr())
print 'Waiting %s seconds' % backoff_http_error
time.sleep(backoff_http_error)
backoff_http_error = min(backoff_http_error * 2, 320)
def handle_tweet(self, data):
""" This method is called when data is received through Streaming endpoint.
"""
self.buffer += data
if data.endswith('\r\n') and self.buffer.strip():
# complete message received
message = json.loads(self.buffer)
self.buffer = ''
msg = ''
if message.get('limit'):
print 'Rate limiting caused us to miss %s tweets' % (message['limit'].get('track'))
elif message.get('disconnect'):
raise Exception('Got disconnect: %s' % message['disconnect'].get('reason'))
elif message.get('warning'):
print 'Got warning: %s' % message['warning'].get('message')
else:
print 'Got tweet with text: %s' % message.get('text')
if __name__ == '__main__':
ts = TwitterStream()
ts.setup_connection()
ts.start()
please help me to resolve the issue with code
I am trying to implement a simple threaded SocketServer (using SocketServer.ThreadedMixIn). However, my server stops receiving further messages. Here is the code:
#!/usr/bin/python -u
import SocketServer
import sys
class MYAgentHandler(SocketServer.BaseRequestHandler):
def handle(self):
try:
data = self.request.recv(1024)
print "Received request " + str(data) + "\n"
reply = str(agent.processAgentMessage(data))
self.request.send(reply)
self.request.close()
except Exception, instr:
print "While processing data " + data + " error encountered " + str(instr) + "\n"
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
daemon_threads = True
allow_reuse_address = True
def __init__(self, server_address, RequestHandlerClass):
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
class MYAgent:
def processAgentMessage(self, msg):
try:
tokens = msg.split('^')
if tokens[0] == "CreateSession":
return("New session")
elif tokens[0] == "GetStatus":
return("Init")
except Exception, instr:
print "Error while processing message " + str(instr) + "\n"
agent = MYAgent()
def main():
MYServer = sys.argv[1]
MYAgentPort = sys.argv[2]
agent.listener = ThreadedTCPServer((MYServer, int(MYAgentPort)), MYAgentHandler)
agent.listener.serve_forever()
if __name__ == '__main__':
main()
And here is my client:
#!/usr/bin/python -u
import socket
import time
if __name__ == "__main__":
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', 15222))
try:
sock.send("CreateSession")
sessionID = str(sock.recv(1024))
print "Received: " + sessionID
sock.send("GetStatus^"+sessionID)
print "Sent Getstatus\n"
time.sleep(1)
response = str(sock.recv(1024))
print "status of " + str(sessionID) + " is " + str(response) + "\n"
sock.close()
except Exception, instr:
print "Error occurred " + str(instr) + "\n"
Here is one session. Server output:
$ ./t.py localhost 15222
Received request CreateSession
Client output:
$ ./client.py
Received: New session
Sent Getstatus
status of New session is
$
Any ideas why this is happening?
You have to remove self.request.close() (which closes the connection) and wrap everything with while True: (so it will continue to read from the same socket).