I'm using the example bellow with Thread, a simple http server to listen to post request and collect the data for the main process.
I want to convert it to use multiprocessing and passing a Queue in order to collect the POST data but I'm not sure how to pass it, any idea on how to do so?
from threading import Thread
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from io import BytesIO
KEEP_RUNNING = True
COLLECT_POST_DATA = []
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
def do_POST(self):
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
print("body: {}".format(body))
COLLECT_POST_DATA.append(body)
self.send_response(200)
self.end_headers()
response = BytesIO()
response.write(b'This is POST request. ')
response.write(b'Received: ')
response.write(body)
self.wfile.write(response.getvalue())
def run_server(port):
httpd = HTTPServer(('0.0.0.0', port), SimpleHTTPRequestHandler)
httpd.timeout = 1
while KEEP_RUNNING:
httpd.handle_request()
# httpd.serve_forever()
t = Thread(target = run_server, args=(8000,))
t.start()
# to kill
KEEP_RUNNING = False
So I worked it out:
import multiprocessing
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from io import BytesIO
class QueuingHTTPServer(HTTPServer):
# def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True, queue=False):
def __init__(self, server_address, RequestHandlerClass, queue, bind_and_activate=True):
HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.queue = queue
class PostHTTPRequestHandler(BaseHTTPRequestHandler):
def do_POST(self):
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
print("body: {}".format(body))
self.send_response(200)
self.end_headers()
response = BytesIO()
response.write(b'This is POST request. ')
response.write(b'Received: ')
response.write(body)
self.wfile.write(response.getvalue())
self.server.queue.put(body)
def run_server(port, queue):
print('run_server')
httpd = QueuingHTTPServer(('0.0.0.0', port), PostHTTPRequestHandler, queue)
httpd.timeout = 2
while True:
httpd.handle_request()
# httpd.serve_forever()
queue = multiprocessing.Queue()
p = multiprocessing.Process(target=run_server, name='serve', args=(8000, queue))
p.start()
try it with:
curl hostip:8000 -m 5 -d"12.12.12.12"
Then from the script:
data = queue.get(timeout=2)
kill by:
p.terminate()
Related
I am currently running into an issue where the following code only reads the first 1028 bytes from a socket, and then hangs waiting for more even though the rest has been sent. I thought it could be the code causing the problem, but it only happens when I am receiving data from the client to the server (which is multi-threaded). When I run the code in reverse (server sending data to client) it runs without an issue.
This is what is currently failing:
The clients code:
with open(commands[1], "rb") as file:
data = file.read()
# Sends the length of the file to the server
serverSocket.sendall(str(len(data)).encode())
# Sends the data to the server
data = file.read(1024)
while data:
data = file.read(1024)
serverSocket.sendall(data)
Along with the server code:
# Gets the total length of the file
dataLen = int(clientsocket.recv(1024).decode())
totalData = ""
while len(totalData) < dataLen:
# Reads in and appends the data to the variable
data = clientsocket.recv(1024).decode()
totalData += data
What else have I tried?
The original code for the client was:
with open(commands[1], "rb") as file:
data = file.read()
# Sends the length of the file to the server
serverSocket.sendall(str(len(data)).encode())
# Sends the data to the server
serverSocket.sendall(data)
Which I changed because I thought there may be an issue with the recv not getting all of the packets from the client, but the client believes that everything was sent.
Minimum Reproducible Code:
SERVER.PY
from socket import socket, AF_INET, SOCK_STREAM, error
from threading import Thread, Lock
from _thread import interrupt_main
import logging
from sys import stdout
from os import makedirs
from pathlib import Path
class Server:
def __init__(self, host, port):
logging.basicConfig(stream=stdout, level=logging.INFO, format='%(asctime)s - %(message)s')
self.host = host
self.port = port
self.quitVal = False
self.threads = []
self.lock = Lock()
self.sock = None
self.open()
def open(self):
sock = socket(AF_INET, SOCK_STREAM)
sock.bind((self.host, self.port))
sock.listen(5)
self.sock = sock
def listen(self):
try:
while self.quitVal != True:
(clientsocket, address) = self.sock.accept()
# This creates a threaded connection as this is the choke for the connection
thread = Thread(target=self.connection, args=(clientsocket, address), daemon=True)
self.threads.append(thread)
thread.start()
except OSError:
quit()
def connection(self, clientsocket, address):
while self.quitVal != True:
# We are going to allow multiple file transfers at one time on a connection
requestThreads = []
# The client innitially sends a command "s<command> [<fileLocation>]"
data = clientsocket.recv(4096).decode("utf-8")
data = data.split()
data[0] = data[0].lower()
if data[0] == "get":
thread = Thread(target=self.get, args=(clientsocket, data[1]), daemon=True)
requestThreads.append(thread)
thread.start()
elif data[0] == "put":
thread = Thread(target=self.put, args=(clientsocket, data[1]), daemon=True)
requestThreads.append(thread)
thread.start()
clientsocket.close()
def get(self, clientsocket, fileLocation):
# Tries to get the file, and sends it all to the client
with self.lock:
try:
with open(fileLocation, "rb") as file:
data = file.read()
# Sends the length of the file to the client
clientsocket.sendall(str(len(data)).encode())
# Byte to separate the input from length and file
response = self.sock.recv(1).decode()
# Sends the data to the client
clientsocket.sendall(data)
except IOError as e:
print("\nFile doesn't exist")
def put(self, clientsocket, fileLocation):
# Gets the total length of the file
with self.lock:
dataLen = int(clientsocket.recv(1024).decode())
totalData = ""
# Gets the total length of the file
dataLen = int(clientsocket.recv(1024).decode())
totalData = ""
# Byte to separate the input from length and file
clientsocket.sendall("0".encode())
while len(totalData) < dataLen:
# Reads in and appends the data to the variable
data = clientsocket.recv(2048).decode()
totalData += data
logging.info(totalData)
server = Server("127.0.0.1", 10002)
server.listen()
CLIENT.PY
from socket import socket, AF_INET, SOCK_STREAM, error
from threading import Thread, Lock
from _thread import interrupt_main
import logging
from sys import stdout
from os import makedirs
from pathlib import Path
class Client:
def __init__(self, host, port):
self.host = host
self.port = port
self.quitVal = False
self.sock = self.open()
self.threads = []
self.lock = Lock()
def open(self):
try:
sock = socket(AF_INET, SOCK_STREAM)
sock.connect((self.host, self.port))
except error as err:
return None
return sock
def get(self, commands):
# Sends the request for the file to the server using the main Sock
with self.lock:
command = "GET " + commands[0]
self.sock.send(command.encode())
dataLen = int(self.sock.recv(1024).decode())
totalData = ""
self.sock.sendall("0".encode())
while len(totalData) < dataLen:
data = self.sock.recv(2048).decode()
totalData += data
def put(self, commands):
with self.lock:
# Sends the put call command to the server
command = "PUT " + commands[1]
self.sock.sendall(command.encode())
try:
with open(commands[1], "rb") as file:
data = file.read()
# Sends the length of the file to the server
self.sock.sendall(str(len(data)).encode())
# Byte to separate the input from length and file
response = self.sock.recv(1).decode()
# Sends the data to the server
self.sock.sendall(data)
except IOError as e:
logging.info("\nFile doesn't exist")
client = Client("127.0.0.1", 10002)
print("foo")
client.get(["foo.txt", "bar.txt"])
client.put(["foo.txt", "bar1.txt"])
I am implementing stomp consumer as a library. By calling this library in other application i should be able to get the data in ActiveMQ. I am implementing it as below, but I have a problem in returning the frame.body. I am not able to retrieve the data from outside the class.
from twisted.internet import defer
from stompest.async import Stomp
from stompest.async.listener import SubscriptionListener
from stompest.config import StompConfig
from socket import gethostname
from uuid import uuid1
import json
class Consumer(object):
def __init__(self, amq_uri):
self.amq_uri = amq_uri
self.hostname = gethostname()
self.config = StompConfig(uri=self.amq_uri)
#defer.inlineCallbacks
def run(self, in_queue):
client = yield Stomp(self.config)
headers = {
StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL,
StompSpec.ID_HEADER: self.hostname,
'activemq.prefetchSize': '1000',
}
yield client.connect(headers=self._return_client_id())
client.subscribe(
in_queue,
headers,
listener=SubscriptionListener(self.consume)
)
try:
client = yield client.disconnected
except StompConnectionError:
yield client.connect(headers=self._return_client_id())
client.subscribe(
in_queue,
headers,
listener=SubscriptionListener(self.consume)
)
while True:
try:
yield client.disconnected
except StompProtocolError:
pass
except StompConnectionError:
yield client.connect(headers=self._return_client_id())
client.subscribe(
in_queue,
headers,
listener=SubscriptionListener(self.consume)
)
def _return_client_id(self):
client_id = {}
client_id['client-id'] = gethostname() + '-' + str(uuid1())
return client_id
def consume(self, client, frame):
data = json.loads(frame.body)
print 'Received Message Type {}'.format(type(data))
print 'Received Message {}'.format(data)
## I want to return data here. I am able to print the frame.body here.
# Call from another application
import Queue
from twisted.internet import reactor
amq_uri = 'tcp://localhost:61613'
in_queue = '/queue/test_queue'
c = Consumer(amq_uri)
c.run(in_queue)
print "data is from outside function", data # Should be able to get the data which is returned by consume here
reactor.run()
Can someone please let me know how can i achieve this.
Thanks
I found a solution to my problem. Instead of using async stomp library, i used sync stomp library. Implemented it as below,
class Consumer(object):
def __init__(self, amq_uri):
self.amq_uri = amq_uri
self.hostname = gethostname()
self.config = StompConfig(uri=self.amq_uri)
def run(self, in_queue, return_dict):
client = Stomp(self.config)
headers = {
StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL,
StompSpec.ID_HEADER: self.hostname
}
client.connect()
client.subscribe(in_queue, headers)
try:
frame = client.receiveFrame()
data = json.dumps(frame.body)
except Exception as exc:
print exc
client.ack(frame)
client.disconnect()
return_dict['data'] = data
return data
I am using below code from the below link. My understanding is asyncore.loop() will print LOOP_DONE instead of waiting for the data to be delivered. Instead, the LOOP_DONE is printed only after all the data has been read. Why is loop() blocking?
http://broadcast.oreilly.com/2009/03/pymotw-asyncore.html
import asyncore
import logging
import socket
from cStringIO import StringIO
import urlparse
class HttpClient(asyncore.dispatcher):
def __init__(self, url):
self.url = url
self.logger = logging.getLogger(self.url)
self.parsed_url = urlparse.urlparse(url)
asyncore.dispatcher.__init__(self)
self.write_buffer = 'GET %s HTTP/1.0\r\n\r\n' % self.url
self.read_buffer = StringIO()
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.parsed_url.netloc, 80)
self.logger.debug('connecting to %s', address)
self.connect(address)
def handle_connect(self):
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def writable(self):
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
clients = []
for i in range(10):
clients.append(HttpClient('http://www.python.org/'))
#clients = [
# HttpClient('http://www.python.org/'),
# HttpClient('http://www.doughellmann.com/PyMOTW/contents.html'),
#]
logging.debug('LOOP STARTING')
asyncore.loop()
logging.debug('LOOP DONE')
for c in clients:
response_body = c.read_buffer.getvalue()
print c.url, 'got', len(response_body), 'bytes'
iv created a simple async client and server but im unable to get the client to reply after receiving the first time. It seems the server can send back a reply after receiving from the client but the client cant:
here is the client's session:
[mike#mike Public]$ python cl.py
buf got your stuff
dded callback ## this is a log addded to see if execution got where i wanted
and here is the server's log:
[mike#mike Public]$ python that.py
buf ehlo localhost
i was expecting some sort of ping pong effect where one send then the other then rinse lather repeat.
here is the client's code:
import socket
import fcntl, os, io, time, functools
from tornado import ioloop
class Punk(object):
def __init__(self):
self.loop = ioloop.IOLoop.instance()
self.address = 'blah.sock'
self.authkey = "none"
self.sock = socket.socket(socket.AF_UNIX)
def setup(self):
self.sock.connect(self.address)
fcntl.fcntl(self.sock, fcntl.F_SETFL, os.O_NONBLOCK)
self.sock.sendall("ehlo localhost")
self.fd = self.sock.fileno()
self.loop.add_handler(self.fd,self.reader,self.loop.READ)
self.loop.start()
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
try:
while True:
servrep = self.sock.recv(1024)
if not servrep:
break
result += servrep
self.prnt(result)
break
except Exception as e:
print "this %s happend"%e
return
def prnt(self,buf):
print "buf %s"%buf
tim = time.time() + 2
self.loop.instance().add_timeout(tim, self.reply)
#callbac = functools.partial(self.loop.add_timeout,tim,self.reply)
#self.loop.add_callback(self.reply) ### i tried this too
print "added callback"
def reply(self):
self.sock.sendall(" clent got your stuff")
if __name__ == "__main__":
bob = Punk()
bob.setup()
and here is the server:
import socket
import fcntl, os, io, time, functools
from array import array
from tornado import ioloop
class Player(object):
def __init__(self):
self.loop = ioloop.IOLoop.instance()
self.address = 'blah.sock'
self.authkey = "none"
self.sock = socket.socket(socket.AF_UNIX)
def setup(self):
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.sock.bind(self.address)
fcntl.fcntl(self.sock, fcntl.F_SETFL, os.O_NONBLOCK)
self.sock.listen(1)
self.fd = self.sock.fileno()
self.loop.add_handler(self.fd,self.reader,self.loop.READ)
self.loop.start()
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
self.conn, self.addr = self.sock.accept()
try:
while True:
maxrep = self.conn.recv(1024)
if not maxrep:
break
result += maxrep
self.prnt(result)
break
except Exception as e:
print "this %s happend"%e
return
def prnt(self,buf):
print "buf %s"%buf
tim = time.time() + 2
self.loop.instance().add_timeout(tim, self.reply)
#callbac = functools.partial(self.loop.add_timeout,tim,self.reply)
#self.loop.add_callback(callbac)
def reply(self):
self.conn.sendall("got your stuff")
if __name__ == "__main__":
bob = Player()
bob.setup()
i had set my sockets to nonblock mode, but i did not catch an error when accepting from
a nonblock state when there is no connection:
here:
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
self.conn, self.addr = self.sock.accept()
should be
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
try:
self.conn, self.addr = self.sock.accept() # we get stuck here
self.connl.append(self.conn)
except Exception as e:
pass
How can you load test gevent sockets?
I've written a simple server and I want to load test this before I tweak the code to find out what improvements can be done.
Are there any specifics to load testing a socket server?
How would I go about load testing this code so I can compare the results after applying tweaks?
from gevent import server
from gevent.monkey import patch_all; patch_all()
import gevent
class Client(object):
def __init__(self, socket, address, server_handler):
self.socket = socket
self.address = address
self.server_handler = server_handler
gevent.spawn(self.listen)
def listen(self):
f = self.socket.makefile()
while True:
line = f.readline()
if not line:
print 'client died'
break
line = line.rstrip()
for addr, c in self.server_handler.users.iteritems():
msg = '<%s> says: %s' % (self.address, line)
c.send(msg)
print '<%s>: %s' % (self.address, line)
def send(self, msg):
f = self.socket.makefile()
f.write('%s\r\n' % msg)
f.flush()
class ServerHandler(object):
def __init__(self):
self.users = {}
def __call__(self, socket, address):
client = Client(socket, address, self)
self.users[address] = client
self.socket = socket
self.address = address
print 'connection made'
if __name__ == '__main__':
server = server.StreamServer(('0.0.0.0', 5000), ServerHandler())
server.serve_forever()