Develop Reference

Multiple call/response messages between Twisted server/client

I have a basic server and client implemented in Twisted. My goal is to have the client process some data, report its progress back to the server, and repeat until all the data is processed. The client is able to send an initial message to the server but it is not receiving the server's response letting it know it is ok to start processing that data. Here is the code I have.
Server:
from twisted.internet import reactor, protocol
PORT = 9000
progress = 0
class MyServer(protocol.Protocol):
def dataReceived(self, data):
global progress
print(data)
if data != "Ready?":
progress = int(data)
self.transport.write("Got it.")
self.transport.loseConnection()
def connectionLost(self, reason):
global progress
if progress == 10:
print("Completed.")
reactor.stop()
class MyServerFactory(protocol.Factory):
protocol = MyServer
factory = MyServerFactory()
reactor.listenTCP(PORT, factory)
reactor.run()
Client:
from twisted.internet import reactor, protocol
import time
HOST = 'localhost'
PORT = 9000
progress = 0
class MyClient(protocol.Protocol):
def connectionMade(self):
self.transport.write("Ready?")
self.transport.loseConnection()
def dataReceived(self, data):
global progress
progress += 1
print(progress)
self.transport.write(str(progress))
self.loseConnection()
def connectionLost(self, reason):
global progress
if progress == 10:
print("Completed.")
reactor.stop()
class MyClientFactory(protocol.ClientFactory):
protocol = MyClient
factory = MyClientFactory()
reactor.connectTCP(HOST, PORT, factory)
reactor.run()

I figured out that my issue was that I was prematurely closing the connection. In some earlier testing I was trying to send multiple messages within the dataReceived function which were getting concatenated into a single message. This led me to believe that you must lose the connection for each message to go through. Rather, you simply need to let the function finish before sending another message. Here is the updated code that is working as intended.
Server:
from twisted.internet import reactor, protocol
PORT = 9000
progress = 0
class MyServer(protocol.Protocol):
def dataReceived(self, data):
global progress
print(data)
if data != "Ready?":
progress = int(data)
self.transport.write("Got it")
if progress == 10:
self.transport.loseConnection()
def connectionLost(self, reason):
print("Completed.")
reactor.stop()
class MyServerFactory(protocol.Factory):
protocol = MyServer
factory = MyServerFactory()
reactor.listenTCP(PORT, factory)
reactor.run()
Client:
from twisted.internet import reactor, protocol
import time
HOST = 'localhost'
PORT = 9000
progress = 0
class MyClient(protocol.Protocol):
def connectionMade(self):
self.transport.write("Ready?")
def dataReceived(self, data):
global progress
progress += 1
print(progress)
self.transport.write(str(progress))
if progress == 10:
self.transport.loseConnection()
def connectionLost(self, reason):
print("Completed.")
reactor.stop()
class MyClientFactory(protocol.ClientFactory):
protocol = MyClient
factory = MyClientFactory()
reactor.connectTCP(HOST, PORT, factory)
reactor.run()

Using Twisted and Python to create a Router

When a client (ie. web browser) points to localhost:8080, its request should be sent to an address defined by SERVER_ADDR such as http://www.yahoo.com. It's more like a router or load balancer, than a proxy.
Problem: When my web browser points to localhost:8080, nothing is returned to it. On the Python console, the HTTP request header can be seen to have reached the Twisted app. Maybe this cannot be used for redirecting the browser's request?
from twisted.internet import protocol, reactor
LISTEN_PORT = 8080
SERVER_PORT = 80
SERVER_ADDR = 'http://www.yahoo.com'
class ServerProtocol(protocol.Protocol):
def __init__(self):
self.buffer = None
self.client = None
def connectionMade(self):
factory = protocol.ClientFactory()
factory.protocol = ClientProtocol
factory.server = self
reactor.connectTCP(SERVER_ADDR, SERVER_PORT, factory)
# Client => Proxy
def dataReceived(self, data):
print 'Data received from Client:'
if self.client:
self.client.write(data)
else:
data = data.replace('localhost:8080', SERVER_ADDR)
print data
self.buffer = data
# Proxy => Client
def write(self, data):
self.transport.write(data)
class ClientProtocol(protocol.Protocol):
def connectionMade(self):
self.factory.server.client = self
self.write(self.factory.server.buffer)
self.factory.server.buffer = ''
# Server => Proxy
def dataReceived(self, data):
print 'Data received from Server: '
print data
self.factory.server.write(data)
# Proxy => Server
def write(self, data):
if data:
self.transport.write(data)
def main():
factory = protocol.ServerFactory()
factory.protocol = ServerProtocol
reactor.listenTCP(LISTEN_PORT, factory)
reactor.run()
if __name__ == '__main__':
main()

It partially works for me: I get an HTTP error 400 bad request from the Yahoo server in my browser when I visit http://localhost:8080. This is because you're replacing the Host: localhost:8080 section of the GET request with Host: http://www.yahoo.com and the protocol here is invalid. It should just be Host: www.yahoo.com.
That said, it then responds with a HTTP/1.1 301 Moved Permanently redirect and it you run into other problems.
Edit: you should take a look at the proxy example on the twisted website (https://twistedmatrix.com/documents/12.3.0/web/examples/#auto2) and dive into the source to see how it's implemented.
from twisted.web import proxy, http
from twisted.internet import reactor
class ProxyFactory(http.HTTPFactory):
def buildProtocol(self, addr):
return proxy.Proxy()
reactor.listenTCP(8080, ProxyFactory())
reactor.run()

SocketServer Python

So I found the following sample code which allows for a basic python HTTP server to be established at a given url and port. I am quite inexperienced with web servers and am trying to create handlers for certain GET requests to this server. However, I cannot figure out how to actually create handlers for a GET request made by another computer when accessing this URL remotely. Any suggestions?
import SocketServer
class MyTCPHandler(SocketServer.BaseRequestHandler):
"""
The RequestHandler class for our server.
It is instantiated once per connection to the server, and must
override the handle() method to implement communication to the
client.
"""
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
print "{} wrote:".format(self.client_address[0])
print self.data
# just send back the same data, but upper-cased
self.request.sendall(self.data.upper())
if __name__ == "__main__":
HOST, PORT = "url" , PORT
# Create the server, binding to localhost on port 9999
server = SocketServer.TCPServer((HOST, PORT), MyTCPHandler)
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
server.serve_forever()

Here is very simple example of how it could work. You would start this and call it with this, for example:
curl -i 'http://localhost:5001/foo/bar?foo=bar' -X POST -d '{"Foo":"Bar"}'
HTTP/1.1 200 OK
Some response%
It is missing tons of things, but this should at least give you some sort of idea.
import SocketServer
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
print self.data
self.parse_request(self.data)
func, args = self.path.split("/", 1)
args = args.split("/")
resp = getattr(self, func)(*args)
self.request.sendall("HTTP/1.1 200 OK\n")
self.request.sendall("\n")
self.request.sendall(resp)
def parse_request(self, req):
headers = {}
lines = req.splitlines()
inbody = False
body = ''
for line in lines[1:]:
if line.strip() == "":
inbody = True
if inbody:
body += line
else:
k, v = line.split(":", 1)
headers[k.strip()] = v.strip()
method, path, _ = lines[0].split()
self.path = path.lstrip("/")
self.method = method
self.headers = headers
self.body = body
self.path, self.query_string = self.path.split("?")
def foo(self, *args):
print self.path
print self.query_string
print self.body
print self.headers
print self.method
return "Some response"
if __name__ == "__main__":
server = SocketServer.TCPServer(("localhost", 5001), MyTCPHandler)
server.serve_forever()

How do I achieve a webpush with data coming from a socket?

Here is my current code:
#!/usr/bin/env python
from twisted.application import internet, service
from twisted.application.service import IServiceMaker, MultiService
from twisted.protocols import basic
from twisted.internet import reactor, protocol, defer
from twisted.internet.protocol import DatagramProtocol
import datetime
class WebPUSH(basic.LineReceiver):
logTemplate = '''
<script type="text/javascript">
pushHandler.addLi('%s')
</script>
'''
def __init__(self):
self.gotRequest = False
def lineReceived(self, line):
if not self.gotRequest:
self.startResponse()
self.gotRequest = True
def startResponse(self):
self.sendLine('HTTP/1.1 200 OK')
self.sendLine('Content-Type: text/html; charset=utf-8')
self.sendLine('')
f = open('index.html', 'r')
self.transport.write( ''.join(f.read()) )
f.close()
self.logTime()
def logTime(self):
self.sendLine( self.logTemplate % datetime.datetime.now() )
#reactor.callLater(2, self.logTime)
class Echo(DatagramProtocol):
def datagramReceived(self, data, (host, port)):
WebPUSH.logTime()
print "received %r from %s:%d" % (data, host, port)
self.transport.write(data, (host, port))
if __name__ == '__main__':
f = protocol.ServerFactory()
f.protocol = WebPUSH
reactor.listenTCP(8080, f)
reactor.listenUDP(9999, Echo())
reactor.run()
As you can see, I am trying to call a method in WebPUSH from Echo when data is received. Because I never actually instantiate WebPUSH it doesn't look like I can easily call this method. I tried converting this to use a multiservice method but that didn't seem to work although I am sure I am doing something wrong.
There aren't (as far as I could google) any good examples on multiservice with twisted or atleast one like this.
Any help will be appreciated.

Echo instance needs an instance of WebPUSH protocol to call .logTime().
You need to maintain loggers list, e.g., in main():
...
f.loggers = []
echo = Echo()
echo.loggers = f.loggers
...
Add to it in WebPUSH.connectionMade, remove in WebPUSH.connectionLost:
def connectionMade(self):
self.factory.loggers.append(self)
def connectionLost(self, reason):
self.factory.loggers.remove(self)
then in Echo.datagramReceived:
for time_logger in self.loggers:
time_logger.logTime()

user970077, the example in my blog was meant to be a simplified demo to show how the webpush work. Here is what you are trying to do (incorporating JFB suggestions):
#!/usr/bin/env python
from twisted.protocols import basic
from twisted.internet import reactor, protocol, defer
import datetime
class WebPUSH(basic.LineReceiver):
logTemplate = '''
<script type="text/javascript">
pushHandler.addLi('%s')
</script>
'''
def connectionMade(self):
self.factory.listeners.append(self)
self.startResponse()
def connectionLost(self, reason):
self.factory.listeners.remove(self)
def lineReceived(self, line):
self.sendLine( self.logTemplate % line )
def startResponse(self):
self.sendLine('HTTP/1.1 200 OK')
self.sendLine('Content-Type: text/html; charset=utf-8')
self.sendLine('')
with open('index.html', 'r') as f:
self.transport.write( ''.join(f.read()) )
class WebPushFactory(protocol.ServerFactory):
protocol = WebPUSH
def __init__(self):
self.listeners = []
class Echo(protocol.DatagramProtocol):
def __init__(self, listeners):
self.listeners = listeners
def datagramReceived(self, data, (host, port)):
msg = '[%s:%s] %s' % (host, port, data)
for listener in self.listeners:
# udp is not necessarily line-oriented
# so we can:
# 1) feed dataReceived and wait until the line
# delimiter arrives in an udp package
# triggering lineReceived:
#listener.dataReceived( msg )
# or 2) fake a line by calling lineReceived direclty:
listener.lineReceived( msg )
print msg
self.transport.write(data, (host, port))
if __name__ == '__main__':
web = WebPushFactory()
reactor.listenTCP(8080, web)
reactor.listenUDP(9999, Echo(web.listeners))
reactor.run()
And a client to test it (taken from UDP client and server with Twisted Python):
#!/usr/bin/env python
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor
from twisted.internet.task import LoopingCall
import sys, time
class HeartbeatSender(DatagramProtocol):
def __init__(self, name, host, port):
self.name = name
self.loopObj = None
self.host = host
self.port = port
def startProtocol(self):
# Called when transport is connected
# I am ready to send heart beats
self.loopObj = LoopingCall(self.sendHeartBeat)
self.loopObj.start(2, now=False)
def stopProtocol(self):
"Called after all transport is teared down"
pass
def datagramReceived(self, data, (host, port)):
print "received %r from %s:%d" % (data, host, port)
def sendHeartBeat(self):
self.transport.write(self.name, (self.host, self.port))
if __name__ == '__main__':
sender = HeartbeatSender("sender", "127.0.0.1", 9999)
reactor.listenMulticast(9998, sender, listenMultiple=True)
reactor.run()

Python SocketServer: sending to multiple clients?

Well, I'm trying to build a small python prgram with a SocketServer that is supposed to send messages it receives to all connected clients. I'm stuck, I don't know how to store clients on the serverside, and I don't know how to send to multiple clients. Oh and, my program fails everytime more then 1 client connects, and everytime a client sends more then one message...
Here's my code until now:
print str(self.client_address[0])+' connected.'
def handle(self):
new=1
for client in clients:
if client==self.request:
new=0
if new==1:
clients.append(self.request)
for client in clients:
data=self.request.recv(1024)
client.send(data)
class Host:
def __init__(self):
self.address = ('localhost', 0)
self.server = SocketServer.TCPServer(self.address, EchoRequestHandler)
ip, port = self.server.server_address
self.t = threading.Thread(target=self.server.serve_forever)
self.t.setDaemon(True)
self.t.start()
print ''
print 'Hosted with IP: '+ip+' and port: '+str(port)+'. Clients can now connect.'
print ''
def close(self):
self.server.socket.close()
class Client:
name=''
ip=''
port=0
def __init__(self,ip,port,name):
self.name=name
self.hostIp=ip
self.hostPort=port
self.s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((self.hostIp, self.hostPort))
def reco(self):
self.s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((self.hostIp, self.hostPort))
def nick(self,newName):
self.name=newName
def send(self,message):
message=self.name+' : '+message
len_sent=self.s.send(message)
response=self.s.recv(len_sent)
print response
self.reco()
def close(self):
self.s.close()
Obviously I have no idea what I'm doing, so any help would be great.
Thanks in advance!
Edit: I'm using Python 2.7 on Windows Vista.

You want to look at asyncore here. The socket operations you're calling on the client side are blocking (don't return until some data is received or a timeout occurs) which makes it hard to listen for messages sent from the host and let the client instances enqueue data to send at the same time. asyncore is supposed to abstract the timeout-based polling loop away from you.
Here's a code "sample" -- let me know if anything is unclear:
from __future__ import print_function
import asyncore
import collections
import logging
import socket
MAX_MESSAGE_LENGTH = 1024
class RemoteClient(asyncore.dispatcher):
"""Wraps a remote client socket."""
def __init__(self, host, socket, address):
asyncore.dispatcher.__init__(self, socket)
self.host = host
self.outbox = collections.deque()
def say(self, message):
self.outbox.append(message)
def handle_read(self):
client_message = self.recv(MAX_MESSAGE_LENGTH)
self.host.broadcast(client_message)
def handle_write(self):
if not self.outbox:
return
message = self.outbox.popleft()
if len(message) > MAX_MESSAGE_LENGTH:
raise ValueError('Message too long')
self.send(message)
class Host(asyncore.dispatcher):
log = logging.getLogger('Host')
def __init__(self, address=('localhost', 0)):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind(address)
self.listen(1)
self.remote_clients = []
def handle_accept(self):
socket, addr = self.accept() # For the remote client.
self.log.info('Accepted client at %s', addr)
self.remote_clients.append(RemoteClient(self, socket, addr))
def handle_read(self):
self.log.info('Received message: %s', self.read())
def broadcast(self, message):
self.log.info('Broadcasting message: %s', message)
for remote_client in self.remote_clients:
remote_client.say(message)
class Client(asyncore.dispatcher):
def __init__(self, host_address, name):
asyncore.dispatcher.__init__(self)
self.log = logging.getLogger('Client (%7s)' % name)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.name = name
self.log.info('Connecting to host at %s', host_address)
self.connect(host_address)
self.outbox = collections.deque()
def say(self, message):
self.outbox.append(message)
self.log.info('Enqueued message: %s', message)
def handle_write(self):
if not self.outbox:
return
message = self.outbox.popleft()
if len(message) > MAX_MESSAGE_LENGTH:
raise ValueError('Message too long')
self.send(message)
def handle_read(self):
message = self.recv(MAX_MESSAGE_LENGTH)
self.log.info('Received message: %s', message)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
logging.info('Creating host')
host = Host()
logging.info('Creating clients')
alice = Client(host.getsockname(), 'Alice')
bob = Client(host.getsockname(), 'Bob')
alice.say('Hello, everybody!')
logging.info('Looping')
asyncore.loop()
Which results in the following output:
INFO:root:Creating host
INFO:root:Creating clients
INFO:Client ( Alice):Connecting to host at ('127.0.0.1', 51117)
INFO:Client ( Bob):Connecting to host at ('127.0.0.1', 51117)
INFO:Client ( Alice):Enqueued message: Hello, everybody!
INFO:root:Looping
INFO:Host:Accepted client at ('127.0.0.1', 55628)
INFO:Host:Accepted client at ('127.0.0.1', 55629)
INFO:Host:Broadcasting message: Hello, everybody!
INFO:Client ( Alice):Received message: Hello, everybody!
INFO:Client ( Bob):Received message: Hello, everybody!

You can use socketserver to broadcast messages to all connected clients. However, the ability is not built into the code and will need to be implemented by extending some of the classes already provided. In the following example, this is implemented using the ThreadingTCPServer and StreamRequestHandler classes. They provide a foundation on which to build but still require some modifications to allow what you are trying to accomplish. The documentation should help explain what each function, class, and method are trying to do in order to get the job done.
Server
#! /usr/bin/env python3
import argparse
import pickle
import queue
import select
import socket
import socketserver
def main():
"""Start a chat server and serve clients forever."""
parser = argparse.ArgumentParser(description='Execute a chat server demo.')
parser.add_argument('port', type=int, help='location where server listens')
arguments = parser.parse_args()
server_address = socket.gethostbyname(socket.gethostname()), arguments.port
server = CustomServer(server_address, CustomHandler)
server.serve_forever()
class CustomServer(socketserver.ThreadingTCPServer):
"""Provide server support for the management of connected clients."""
def __init__(self, server_address, request_handler_class):
"""Initialize the server and keep a set of registered clients."""
super().__init__(server_address, request_handler_class, True)
self.clients = set()
def add_client(self, client):
"""Register a client with the internal store of clients."""
self.clients.add(client)
def broadcast(self, source, data):
"""Resend data to all clients except for the data's source."""
for client in tuple(self.clients):
if client is not source:
client.schedule((source.name, data))
def remove_client(self, client):
"""Take a client off the register to disable broadcasts to it."""
self.clients.remove(client)
class CustomHandler(socketserver.StreamRequestHandler):
"""Allow forwarding of data to all other registered clients."""
def __init__(self, request, client_address, server):
"""Initialize the handler with a store for future date streams."""
self.buffer = queue.Queue()
super().__init__(request, client_address, server)
def setup(self):
"""Register self with the clients the server has available."""
super().setup()
self.server.add_client(self)
def handle(self):
"""Run a continuous message pump to broadcast all client data."""
try:
while True:
self.empty_buffers()
except (ConnectionResetError, EOFError):
pass
def empty_buffers(self):
"""Transfer data to other clients and write out all waiting data."""
if self.readable:
self.server.broadcast(self, pickle.load(self.rfile))
while not self.buffer.empty():
pickle.dump(self.buffer.get_nowait(), self.wfile)
#property
def readable(self):
"""Check if the client's connection can be read without blocking."""
return self.connection in select.select(
(self.connection,), (), (), 0.1)[0]
#property
def name(self):
"""Get the client's address to which the server is connected."""
return self.connection.getpeername()
def schedule(self, data):
"""Arrange for a data packet to be transmitted to the client."""
self.buffer.put_nowait(data)
def finish(self):
"""Remove the client's registration from the server before closing."""
self.server.remove_client(self)
super().finish()
if __name__ == '__main__':
main()
Of course, you also need a client that can communicate with your server and use the same protocol the server speaks. Since this is Python, the decision was made to utilize the pickle module to facilitate data transfer among server and clients. Other data transfer methods could have been used (such as JSON, XML, et cetera), but being able to pickle and unpickle data serves the needs of this program well enough. Documentation is included yet again, so it should not be too difficult to figure out what is going on. Note that server commands can interrupt user data entry.
Client
#! /usr/bin/env python3
import argparse
import cmd
import pickle
import socket
import threading
def main():
"""Connect a chat client to a server and process incoming commands."""
parser = argparse.ArgumentParser(description='Execute a chat client demo.')
parser.add_argument('host', type=str, help='name of server on the network')
parser.add_argument('port', type=int, help='location where server listens')
arguments = parser.parse_args()
client = User(socket.create_connection((arguments.host, arguments.port)))
client.start()
class User(cmd.Cmd, threading.Thread):
"""Provide a command interface for internal and external instructions."""
prompt = '>>> '
def __init__(self, connection):
"""Initialize the user interface for communicating with the server."""
cmd.Cmd.__init__(self)
threading.Thread.__init__(self)
self.connection = connection
self.reader = connection.makefile('rb', -1)
self.writer = connection.makefile('wb', 0)
self.handlers = dict(print=print, ping=self.ping)
def start(self):
"""Begin execution of processor thread and user command loop."""
super().start()
super().cmdloop()
self.cleanup()
def cleanup(self):
"""Close the connection and wait for the thread to terminate."""
self.writer.flush()
self.connection.shutdown(socket.SHUT_RDWR)
self.connection.close()
self.join()
def run(self):
"""Execute an automated message pump for client communications."""
try:
while True:
self.handle_server_command()
except (BrokenPipeError, ConnectionResetError):
pass
def handle_server_command(self):
"""Get an instruction from the server and execute it."""
source, (function, args, kwargs) = pickle.load(self.reader)
print('Host: {} Port: {}'.format(*source))
self.handlers[function](*args, **kwargs)
def preloop(self):
"""Announce to other clients that we are connecting."""
self.call('print', socket.gethostname(), 'just entered.')
def call(self, function, *args, **kwargs):
"""Arrange for a handler to be executed on all other clients."""
assert function in self.handlers, 'You must create a handler first!'
pickle.dump((function, args, kwargs), self.writer)
def do_say(self, arg):
"""Causes a message to appear to all other clients."""
self.call('print', arg)
def do_ping(self, arg):
"""Ask all clients to report their presence here."""
self.call('ping')
def ping(self):
"""Broadcast to all other clients that we are present."""
self.call('print', socket.gethostname(), 'is here.')
def do_exit(self, arg):
"""Disconnect from the server and close the client."""
return True
def postloop(self):
"""Make an announcement to other clients that we are leaving."""
self.call('print', socket.gethostname(), 'just exited.')
if __name__ == '__main__':
main()

why use SocketServer? a simple client doesn't meet your needs?
import socket
HOST = ''
PORT = 8000
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((HOST, PORT))
sock.listen(5)
while True:
conn, addr = sock.accept()
print 'connecting to', addr
while True:
data = conn.recv(1024)
if not data:
break
conn.send(data)

To take multiple clients simultaneously, you will have to add SocketServer.ForkingMixIn or ThreadingMixIn.