Is it possible to share socket objects between 2 running processes?
To simplify the issue, assume that we have two files s1.py and s2.py. Both are tcp servers listening on different ports.
s1.py body
from twisted.internet import reactor, protocol
CLIENTS = []
class Echo(protocol.Protocol):
def connectionMade(self):
CLIENTS.append(self)
def dataReceived(self, data):
self.transport.write(data)
def connectionLost(self):
CLIENTS.remove(self)
def main():
factory = protocol.ServerFactory()
factory.protocol = Echo
reactor.listenTCP(8000, factory)
reactor.run()
main()
and s2.py
from twisted.internet import reactor, protocol
class Echo(protocol.Protocol):
def dataReceived(self, data):
for socket in list_of_serialized_redis_CLIENTS_socket_object:
socket.transport.write(data)
def main():
factory = protocol.ServerFactory()
factory.protocol = Echo
reactor.listenTCP(8001, factory)
reactor.run()
main()
Is it possible to share CLIENTS from s1.py with s2.py?
Maybe there is some way to serialize CLIENTS and store it in redis or so.
Using socket.share() might not be as straight forward as you might think.
First off, it's a Windows exclusive feature and Windows is the one platform that gets the least attention by Twisted.
Getting access to the low-level socket object in Twisted can be tricky because there are a lot of things happening behind the scenes and might cause strange outcomes if changed.
So in other words, it's certainly possible to go this route, but it's not recommended.
If you're open to new solutions, I think an RPC-style server will solve your issue.
My thinking is, since the connection objects live in s1, simply create a remote process that will "do stuff" to those connections.
Other processes can connect to that server and execute functions on objects local to s1. Twisted's Perspective Broker is a ready-made solution you could leverage.
s1.py
from twisted.internet import reactor, protocol, endpoints
from twisted.spread import pb
class Echo(protocol.Protocol):
def connectionMade(self):
self.factory.client_set.add(self)
def connectionLost(self, reason):
self.factory.client_set.remove(self)
class Remote(pb.Root):
def __init__(self, client_set):
self.client_set = client_set
def remote_s1_write(self, data):
msg_tmpl = 'From s1...{0}'
for client in self.client_set:
response = msg_tmpl.format(data).encode('utf-8')
client.transport.write(response)
def main():
client_set = set() # container will be shared between the 2 servers
# setup Echo server
factory = protocol.ServerFactory()
factory.protocol = Echo
factory.client_set = client_set
echo_server = endpoints.TCP4ServerEndpoint(reactor, interface='localhost', port=8000)
echo_server.listen(factory)
# setup PB server
pb_root = Remote(client_set)
pb_factory = pb.PBServerFactory(pb_root)
pb_server = endpoints.TCP4ServerEndpoint(reactor, interface='localhost', port=7999)
pb_server.listen(pb_factory)
reactor.run()
main()
s2.py
from twisted.internet import reactor, protocol, endpoints
from twisted.spread import pb
class Echo(protocol.Protocol):
def dataReceived(self, data):
remote = self.factory.pb_factory.getRootObject()
remote.addCallback(lambda obj: obj.callRemote('s1_write', data))
remote.addCallback(lambda echo: 's1 said: {0}'.format(data))
remote.addErrback(lambda reason: 'error: {0}'.format(str(reason.value)))
remote.addCallback(lambda response: print(response))
self.transport.write(data)
def main():
# connect to s1's PB server
pb_factory = pb.PBClientFactory()
pb_connection = endpoints.TCP4ClientEndpoint(reactor, host='localhost', port=7999)
pb_connection.connect(pb_factory)
# setup Echo server
factory = protocol.ServerFactory()
factory.protocol = Echo
factory.pb_factory = pb_factory
echo_server = endpoints.TCP4ServerEndpoint(reactor, interface='localhost', port=8001)
echo_server.listen(factory)
reactor.run()
main()
I've taken the liberty and updated the syntax. Try this code and see if it works and ask if you have any issues.
Related
I am learning the socketserver module and I am following the example but I modified the handle function a bit
class CustomServer(socketserver.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
print(f">{self.client_address[0]}: {self.data}")
def send(self, targets=[]):
if not targets:
return
if __name__ == "__main__":
HOST, PORT = "localhost", 6666
with socketserver.TCPServer((HOST, PORT), CustomServer) as server:
server.serve_forever()
Now when I try to use netcat and send sth to the server I don't see anything being outputted to the console
nc -v 10.0.0.112 6666
How do you properly edit the handle method so that it will print the address of the client each time
It is really important to understand the OOP concept and how to use it
Looking at the source code for socketserver I realized that I can create a class that inherits the BaseRequestHandler than I modified the handler method and passed my class to the TCPServer
class CustomHandler(BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
print(f">{self.client_address[0]}: {self.data}")
if __name__ == "__main__":
HOST, PORT = "0.0.0.0", 6666
server = TCPServer(((HOST, PORT)), CustomHandler)
server.serve_forever()
I am trying to reimplement netcat in Python:
#!/usr/bin/env python2
from sys import stdin, stdout
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol
class NcClient(Protocol):
def dataReceived(self, data):
stdout.write(data)
def sendData(self, data):
self.transport.write(data)
self.transport.write("\n")
client = NcClient()
def cmdloop():
while True:
line = stdin.readline()
if line == "":
break
else:
client.sendData(line)
if reactor.running:
reactor.stop()
point = TCP4ClientEndpoint(reactor, "localhost", 6004)
connectProtocol(point, client)
reactor.callInThread(cmdloop)
reactor.run()
When cmdloop detects end of input, it calls reactor.stop.
As I understand, reactor.stop sends shutdown events to all things managed by Twisted, such as threads, connections etc. In response to these events connections are closed, threads wait for the completion of their procedures etc. So when reactor.stop() is called, the connection to localhost:6004 should close, and the program should exit.
However, it doesn't happen immediately, but only when NcClient receives a message from server. As if it is reading those messages blockingly, in a loop, and only when it receives one does it proceed to handle shutdown requests.
How to make it shut down before receiving a message? I know of reactor.crash(), but is there a more polite option?
Your main problem is that cmdloop is running in a non-reactor thread, and yet it is calling reactor methods other than callFromThread (specifically: transport.write via client.sendData). The documentation is quite clear on this:
Methods within Twisted may only be invoked from the reactor thread unless otherwise noted. Very few things within Twisted are thread-safe.
Luckily, implementing the equivalent of netcat doesn't require threading at all. You can simply use Twisted's built-in support for standard I/O as a source of data. Here's an example version:
import sys
from twisted.internet import reactor
from twisted.internet.protocol import Protocol, Factory
from twisted.internet.endpoints import clientFromString
from twisted.internet.stdio import StandardIO
class NcClient(Protocol):
def __init__(self, forwardTo):
self.forwardTo = forwardTo
def connectionMade(self):
self.transport.registerProducer(self.forwardTo.transport, True)
self.forwardTo.transport.resumeProducing()
self.forwardTo.transport.registerProducer(self.transport, True)
def dataReceived(self, data):
self.forwardTo.transport.write(data)
def connectionLost(self, reason):
reactor.stop()
class StdIo(Protocol):
def connectionMade(self):
self.transport.pauseProducing()
f4p = Factory.forProtocol(lambda: NcClient(self))
d = endpoint.connect(f4p)
#d.addCallback
def connected(proto):
self.client = proto
def dataReceived(self, data):
self.client.transport.write(data)
def connectionLost(self, reason):
reactor.stop()
endpoint = clientFromString(reactor, sys.argv[1])
output = StandardIO(proto=StdIo(), reactor=reactor)
reactor.run()
If you still want to use threads for some reason, you can modify NcClient to use callFromThread to invoke sendData instead of calling it directly.
I have modified Jean-Paul Calderone's code from this question to achieve that two server are run asynchronously in one thread and the second server is the client of the first one (he's a broker). This is my current code:
from twisted.internet import protocol, reactor
from twisted.protocols import basic
class BaseServerProtocol(basic.LineReceiver):
def lineReceived(self, line):
self.sendLine("I can see that you said: %s" % (line,))
self.transport.loseConnection()
##############################################################
class BrokerServerProtocol(basic.LineReceiver):
def lineReceived(self, line):
factory = protocol.ClientFactory()
factory.protocol = BrokerClientProtocol
reactor.connectTCP('localhost', 12345, factory)
self.sendLine("I did connect but have nothing to return")
# TODO: return broker client's response
class BrokerClientProtocol(basic.LineReceiver):
def connectionMade(self):
self.sendLine("Hello!")
# TODO: receive the message from broker
self.transport.loseConnection()
##############################################################
def main():
import sys
from twisted.python import log
log.startLogging(sys.stdout)
factory = protocol.ServerFactory()
factory.protocol = BaseServerProtocol
reactor.listenTCP(12345, factory)
factory = protocol.ServerFactory()
factory.protocol = BrokerServerProtocol
reactor.listenTCP(12346, factory)
reactor.run()
if __name__ == '__main__':
main()
What I would like to modify here is to replace both # TODO lines - first, I'd like to fetch BaseServer's response by the BrokerClient (second todo) and then fetch it by the BrokerServer (which has just created the BrokerClient, first todo). Can anyone help?
I've created a class for a server with the declaration:
class myServer(socketserver.BaseRequestHandler):
def handle(self):
pass
And started it with:
someServer = socketserver.UDPServer((HOST, PORT), myServer)
someServer.serve_forever()
My question is: how can I get the server to shutdown itself? I've seen it has a base class (of a base class) called BaseServer with a shutdown method. It can be called on someServer with someServer.shutdown() but this is from the outside of the server itself.
By using threads. Serving by one thread and going via another after your timeout.
Consider this working example. Modify it for your UDPServer
import threading
import time
import SimpleHTTPServer
import SocketServer
PORT = 8000
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", PORT), Handler)
def worker():
# minimal web server. serves files relative to the
print "serving at port", PORT
httpd.serve_forever()
def my_service():
time.sleep(3)
print "I am going down"
httpd.shutdown()
h = threading.Thread(name='httpd', target=worker)
t = threading.Thread(name='timer', target=my_service)
h.start()
t.start()
You could use twisted. Its about the best networking lib for python, here is an example of a UDP server here is (taken from the twisted documentation) the simplest UDP server ever written;
#!/usr/bin/env python
# Copyright (c) 2001-2009 Twisted Matrix Laboratories.
# See LICENSE for details.
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor
# Here's a UDP version of the simplest possible protocol
class EchoUDP(DatagramProtocol):
def datagramReceived(self, datagram, address):
self.transport.write(datagram, address)
def main():
reactor.listenUDP(8000, EchoUDP())
reactor.run()
if __name__ == '__main__':
main()
You can then close this down by calling self.transport.loseConnection() When you are ready or a specific event happens.
The server instance is available as self.server in the handler class. So you can call self.server.shutdown() in the handle method.
How do I create a twisted server that's also a client?
I want the reactor to listen while at the same time it can also be use to connect to the same server instance which can also connect and listen.
Call reactor.listenTCP and reactor.connectTCP. You can have as many different kinds of connections - servers or clients - as you want.
For example:
from twisted.internet import protocol, reactor
from twisted.protocols import basic
class SomeServerProtocol(basic.LineReceiver):
def lineReceived(self, line):
host, port = line.split()
port = int(port)
factory = protocol.ClientFactory()
factory.protocol = SomeClientProtocol
reactor.connectTCP(host, port, factory)
class SomeClientProtocol(basic.LineReceiver):
def connectionMade(self):
self.sendLine("Hello!")
self.transport.loseConnection()
def main():
import sys
from twisted.python import log
log.startLogging(sys.stdout)
factory = protocol.ServerFactory()
factory.protocol = SomeServerProtocol
reactor.listenTCP(12345, factory)
reactor.run()
if __name__ == '__main__':
main()