I am trying to reimplement netcat in Python:
#!/usr/bin/env python2
from sys import stdin, stdout
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol
class NcClient(Protocol):
def dataReceived(self, data):
stdout.write(data)
def sendData(self, data):
self.transport.write(data)
self.transport.write("\n")
client = NcClient()
def cmdloop():
while True:
line = stdin.readline()
if line == "":
break
else:
client.sendData(line)
if reactor.running:
reactor.stop()
point = TCP4ClientEndpoint(reactor, "localhost", 6004)
connectProtocol(point, client)
reactor.callInThread(cmdloop)
reactor.run()
When cmdloop detects end of input, it calls reactor.stop.
As I understand, reactor.stop sends shutdown events to all things managed by Twisted, such as threads, connections etc. In response to these events connections are closed, threads wait for the completion of their procedures etc. So when reactor.stop() is called, the connection to localhost:6004 should close, and the program should exit.
However, it doesn't happen immediately, but only when NcClient receives a message from server. As if it is reading those messages blockingly, in a loop, and only when it receives one does it proceed to handle shutdown requests.
How to make it shut down before receiving a message? I know of reactor.crash(), but is there a more polite option?
Your main problem is that cmdloop is running in a non-reactor thread, and yet it is calling reactor methods other than callFromThread (specifically: transport.write via client.sendData). The documentation is quite clear on this:
Methods within Twisted may only be invoked from the reactor thread unless otherwise noted. Very few things within Twisted are thread-safe.
Luckily, implementing the equivalent of netcat doesn't require threading at all. You can simply use Twisted's built-in support for standard I/O as a source of data. Here's an example version:
import sys
from twisted.internet import reactor
from twisted.internet.protocol import Protocol, Factory
from twisted.internet.endpoints import clientFromString
from twisted.internet.stdio import StandardIO
class NcClient(Protocol):
def __init__(self, forwardTo):
self.forwardTo = forwardTo
def connectionMade(self):
self.transport.registerProducer(self.forwardTo.transport, True)
self.forwardTo.transport.resumeProducing()
self.forwardTo.transport.registerProducer(self.transport, True)
def dataReceived(self, data):
self.forwardTo.transport.write(data)
def connectionLost(self, reason):
reactor.stop()
class StdIo(Protocol):
def connectionMade(self):
self.transport.pauseProducing()
f4p = Factory.forProtocol(lambda: NcClient(self))
d = endpoint.connect(f4p)
#d.addCallback
def connected(proto):
self.client = proto
def dataReceived(self, data):
self.client.transport.write(data)
def connectionLost(self, reason):
reactor.stop()
endpoint = clientFromString(reactor, sys.argv[1])
output = StandardIO(proto=StdIo(), reactor=reactor)
reactor.run()
If you still want to use threads for some reason, you can modify NcClient to use callFromThread to invoke sendData instead of calling it directly.
Related
I have no idea how to use something like twisted.internet.loopingCall() in twisted.internet.ClientFactory
I need to write python script that scans directory for incoming files with phone numbers, reads them, and makes call using YATE yaypm python module that uses twisted library.
client_factory = yaypm.TCPDispatcherFactory(start_client)
reactor.connectTCP(host, port, client_factory)
reactor.run()
Where yaypm.TCPDispatcherFactory derived from twisted.internet.ClientFactory and start_client is the function that will be executed after successfull connection.
If start_client only makes demonstration call:
def start_client(client_yate):
d = dialer(client_yate)
d.call(caller, target)
Everything is OK.
(dialer is the object that implements yaypm.flow logic, full description placed in http://docs.yate.ro/wiki/YAYPM:Bridge_and_then_unbridge)
I need to write something like this in start_client
d = dialer(client_yate)
files = os.listdir(input_directory)
for filename in files:
<read caller and target numbers from file>
d.call(caller, target)
time.sleep(interval)
I know that using sleep function in the main thread leads to deadlock.
How should I implement the algorithm above?
twisted.internet.task.deferLater behaves like a sleep() call if you use it with the inlineCallbacks decorator. Here is a simplified example that uses ClientFactory:
from twisted.internet import reactor, task
from twisted.internet.defer import inlineCallbacks
from twisted.internet.protocol import Protocol, ClientFactory
class DoNothing(Protocol):
def __init__(self, connection_callback):
self.connection_callback = connection_callback
def connectionMade(self):
self.connection_callback()
return
class ConnectionClientFactory(ClientFactory):
def __init__(self, connection_callback):
self.connection_callback = connection_callback
def buildProtocol(self, addr):
return DoNothing(self.connection_callback)
def sleep(delay):
# Returns a deferred that calls do-nothing function
# after `delay` seconds
return task.deferLater(reactor, delay, lambda: None)
#inlineCallbacks
def repeat_forever(message):
while True:
print(message)
yield sleep(1)
if __name__ == '__main__':
repeat_forever('running')
factory = ConnectionClientFactory(lambda: repeat_forever('connected'))
reactor.connectTCP('example.com', 80, factory)
reactor.run()
The above code is essentially what your library does with the callback you pass in. As you can see, the call to repeat_forever('running') runs concurrently to the one called after the client connects.
How can I create an HTTP echo server from Tornado?
#!/usr/bin/env python
import signal
from tornado.ioloop import IOLoop
from tornado.tcpserver import TCPServer
import tornado.web
def handle_signal(sig, frame):
IOLoop.instance().add_callback(IOLoop.instance().stop)
class EchoServer(TCPServer):
def handle_stream(self, stream, address):
self._stream = stream
self._read_line()
def _read_line(self):
self._stream.read_until('\n' ,self._handle_read)
def _handle_read(self, data):
self._stream.write(data, '\n')
self._read_line()
if __name__ == '__main__':
signal.signal(signal.SIGINT, handle_signal)
signal.signal(signal.SIGTERM, handle_signal)
server = EchoServer()
server.bind(8001)
server.start(25)
IOLoop.instance().start()
IOLoop.instance().close()
How do I make of this http echo server
what's wrong? not much i am newbie
Thanks!
Your question would be more clear if you explained what happened when you run this code, and what you expected instead.
One TCPServer object may handle many connections, so instead of assigning to self.stream in handle_stream, you should make a new object to handle this stream.
The second argument to stream.write is a callback; it looks like you meant self._stream.write(data + '\n').
I am creating a robot which is going to be driven by the commands received over TCP connection. Therefore, I will have a robot class with methods (e.g. sense(), drive()...) and the class for TCP connection.
To establish TCP connection, I looked at examples from twisted. On the client side, I have written a client.py script for connection handling:
from twisted.internet import reactor, protocol
import random
from eventhook import EventHook
import common
#from Common.socketdataobjects import response
# a client protocol
class EchoClient(protocol.Protocol):
"""Once connected, send a message, then print the result."""
def connectionMade(self):
self.transport.write("hello, world!")
#the server should be notified that the connection to the robot has been established
#along with robot state (position)
#eventConnectionEstablishedHook.fire()
def dataReceived(self, data):
print "Server said:", data
self.transport.write("Hello %s" % str(random.randint(1,10)))
'''
serverMessage = common.deserializeJson(data)
command = serverMessage.command
arguments = serverMessage.arguments
#here we get for example command = "DRIVE"
#arguments = {motor1Speed: 50, motor2Speed: 40}
instead of above response, used for testing purposes,
the commands should be extracted from the data and according to the command,
the method in Robot instance should be called.
When the command execution finishes, the self.transport.write() method should be called
to notify the server that the command execution finished
'''
def connectionLost(self, reason):
print "connection lost"
class EchoFactory(protocol.ClientFactory):
protocol = EchoClient
def clientConnectionFailed(self, connector, reason):
print "Connection failed - goodbye!"
reactor.stop()
def clientConnectionLost(self, connector, reason):
print "Connection lost - goodbye!"
reactor.stop()
# this connects the protocol to a server runing on port 8000
def initializeEventHandlers(connectionEstablishedHook):
global connection
connection.established = 0
global eventConnectionEstablishedHook
eventConnectionEstablishedHook = connectionEstablishedHook
def main():
f = EchoFactory()
reactor.connectTCP("localhost", 8000, f)
reactor.run()
# this only runs if the module was *not* imported
if __name__ == '__main__':
main()
Beside this script, I have a robot class:
Class Robot(object():
def __init(self)__:
self.position = (0,0)
def drive(self, speedMotor1, speedMotor2, driveTime)
updateMotor1State(speedMotor1)
updateMotor2State(speedMotor2)
time.sleep(driveTime)
#when the execution finished, the finish status should be sent to client in order to inform the server
return "Finished"
def sense(self)
#logic to get the data from the environment
What I would like to do, is to receive the data(commands) from TCP connection and then call the according method in Robot instance. Some procedures might take longer (e.g. driving), so I tried to use events, but haven't figured out the appropriate way to communicate between TCP client and robot using events:
if __name__ == '__main__':
robotController = Robot()
eventController = Controller()
connectionEstablishedHook = EventHook()
client.initializeEventHandlers(connectionEstablishedHook)
eventController.connection = connectionEstablishedHook
client.main()
I tried to create ClientMainProgram script, where I wanted to create an instance of a robot, an instance of TCP client and implement the communication between them using events.
Previously I have managed to implement event handling using Michael Foord's events pattern on a simpler example. I would be very thankful if anyone could provide the solution to this question or any similar example which might be helpful to solve this problem.
Events are easily represented using regular Python function calls.
For example, if your protocol looks like this:
from twisted.internet.protocol import Protocol
class RobotController(Protocol):
def __init__(self, robot):
self.robot = robot
def dataReceived(self, data):
for byte in data:
self.commandReceived(byte)
def commandReceived(self, command):
if command == "\x00":
# drive:
self.robot.drive()
elif command == "\x01":
# sense:
self.robot.sense()
...
(The specifics of the protocol used in this example are somewhat incidental. I picked this protocol because it's very simple and has almost no parsing logic. For your real application I suggest you use twisted.protocols.amp.)
Then all you need to do is make sure the robot attribute is properly initialized. You can do this easily using the somewhat newer endpoint APIs that can often replace use of factories:
from sys import argv
from twisted.internet.endpoints import clientFromString, connectProtocol
from twisted.internet.task import react
def main(reactor, description):
robot = ...
endpoint = clientFromString(reactor, description)
connecting = connectProtocol(endpoint, RobotController(robot))
def connected(controller):
...
connecting.addCallback(connected)
return connecting
react(main, argv[1:])
I am using Twisted to create a simple TCP server. With the Twisted Protocol, Is it ipossible to specific the exact number of bytes returned by dataReceived?
from twisted.internet.protocol import Factory, Protocol
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.internet import reactor
class TestProtocol(Protocol):
def connectionMade(self):
print(id(self))
self.transport.write('hello')
def connectionLost(self, reason):
print('connection lost called')
def dataReceived(self, data):
# is it possible to specify size of "data"?
print('data received called')
class TestFactory(Factory):
protocol = TestProtocol
endpoint = TCP4ServerEndpoint(reactor, 8007)
endpoint.listen(TestFactory())
reactor.run()
I ask this question because if I can control the number of bytes coming in, I can avoid dealing with partial protocol messages or multiple protocol messages coming in a single dataReceived callback.
I was able to achieve this with asynccore by specifying the exact number of bytes in the recv() method. It would be great to be able to do this in Twisted as well.
Thanks
... Alan
Why not define a buffer and use that to grab messages of the size you want. LineReciever does something similar.
Something along the lines of:
from twisted.internet.protocol import Factory, Protocol
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.internet import reactor
class TestProtocol(Protocol):
def __init__(self):
self.__buffer = ""
self.frame_size = 3 #FRAME SIZE HERE
def connectionMade(self):
print(id(self))
self.transport.write('hello')
def connectionLost(self, reason):
print('connection lost called')
def dataReceived(self, data):
# is it possible to specify size of "data"?
print('data received called')
self.__buffer = self.__buffer+data
frame_size = self.frame_size
while len(self.__buffer) >= frame_size:
self.frame_received(self.__buffer[0:frame_size])
self.__buffer=self.__buffer[frame_size:]
def frame_received(self,data):
print data
class TestFactory(Factory):
protocol = TestProtocol
endpoint = TCP4ServerEndpoint(reactor, 8007)
endpoint.listen(TestFactory())
reactor.run()
Just deal with the partial protocol messages.
Twisted provides numerous classes to help you do this in the twisted.protocols.basic package. Normally one of those should just work for you; it's pretty unusual to have to implement your own custom framing protocol.
If you're designing your own protocol, you should probably just use twisted's built-in protocol construction kit, the Asynchronous Messaging Protocol. (Support for other langauges and frameworks available at amp-protocol.net.
I want to transmit data from a Queue using Twisted. I currently use a push producer to poll the queue for items and write to the transport.
class Producer:
implements(interfaces.IPushProducer)
def __init__(self, protocol, queue):
self.queue = queue
self.protocol = protocol
def resumeProducing(self):
self.paused = False
while not self.paused:
try:
data = self.queue.get_nowait()
logger.debug("Transmitting: '%s'", repr(data))
data = cPickle.dumps(data)
self.protocol.transport.write(data + "\r\n")
except Empty:
pass
def pauseProducing(self):
logger.debug("Transmitter paused.")
self.paused = True
def stopProducing(self):
pass
The problem is, that the data are sent very irregularly and if only one item was in the queue, the data is never going to be sent. It seems that Twisted waits until the data to be transmitted has grown to a specific value until it transmits it. Is the way I implemented my producer the right way? Can I force Twisted to transmit data now?
I've also tried using a pull producer, but Twisted does not call the resumeProducing() method of it at all. Do I have to call the resumeProducer() method from outside, when using a pull producer?
It's hard to say why your producer doesn't work well without seeing a complete example (that is, without also seeing the code that registers it with a consumer and the code which is putting items into that queue).
However, one problem you'll likely have is that if your queue is empty when resumeProducing is called, then you will write no bytes at all to the consumer. And when items are put into the queue, they'll sit there forever, because the consumer isn't going to call your resumeProducing method again.
And this generalizes to any other case where the queue does not have enough data in it to cause the consumer to call pauseProducing on your producer. As a push producer, it is your job to continue to produce data on your own until the consumer calls pauseProducing (or stopProducing).
For this particular case, that probably means that whenever you're going to put something in that queue - stop: check to see if the producer is not paused, and if it is not, write it to the consumer instead. Only put items in the queue when the producer is paused.
Here are two possible solutions:
1) Periodically poll your local application to see if you have additional data to send.
NB. This relies on a periodic async callback from the deferLater method in twisted. If you need a responsive application that sends data on demand, or a long running blocking operation (eg. ui that uses its own event loop) it may not be appropriate.
Code:
from twisted.internet.protocol import Factory
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.internet.interfaces import IPushProducer
from twisted.internet.task import deferLater, cooperate
from twisted.internet.protocol import Protocol
from twisted.internet import reactor
from zope.interface import implementer
import time
# Deferred action
def periodically_poll_for_push_actions_async(reactor, protocol):
while True:
protocol.send(b"Hello World\n")
yield deferLater(reactor, 2, lambda: None)
# Push protocol
#implementer(IPushProducer)
class PushProtocol(Protocol):
def connectionMade(self):
self.transport.registerProducer(self, True)
gen = periodically_poll_for_push_actions_async(self.transport.reactor, self)
self.task = cooperate(gen)
def dataReceived(self, data):
self.transport.write(data)
def send(self, data):
self.transport.write(data)
def pauseProducing(self):
print 'Workload paused'
self.task.pause()
def resumeProducing(self):
print 'Workload resumed'
self.task.resume()
def stopProducing(self):
print 'Workload stopped'
self.task.stop()
def connectionLost(self, reason):
print 'Connection lost'
try:
self.task.stop()
except:
pass
# Push factory
class PushFactory(Factory):
def buildProtocol(self, addr):
return PushProtocol()
# Run the reactor that serves everything
endpoint = TCP4ServerEndpoint(reactor, 8089)
endpoint.listen(PushFactory())
reactor.run()
2) Manually keep track of Protocol instances and use reactor.callFromThread() from a different thread. Lets you get away with a long blocking operation in the other thread (eg. ui event loop).
Code:
from twisted.internet.protocol import Factory
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.internet.interfaces import IPushProducer
from twisted.internet.task import deferLater, cooperate
from twisted.internet.protocol import Protocol
from twisted.internet import reactor, threads
import time
import random
import threading
# Connection
protocol = None
# Some other thread that does whatever it likes.
class SomeThread(threading.Thread):
def run(self):
while True:
print("Thread loop")
time.sleep(random.randint(0, 4))
if protocol is not None:
reactor.callFromThread(self.dispatch)
def dispatch(self):
global protocol
protocol.send("Hello World\n")
# Push protocol
class PushProtocol(Protocol):
def connectionMade(self):
global protocol
protocol = self
def dataReceived(self, data):
self.transport.write(data)
def send(self, data):
self.transport.write(data)
def connectionLost(self, reason):
print 'Connection lost'
# Push factory
class PushFactory(Factory):
def buildProtocol(self, addr):
return PushProtocol()
# Start thread
other = SomeThread()
other.start()
# Run the reactor that serves everything
endpoint = TCP4ServerEndpoint(reactor, 8089)
endpoint.listen(PushFactory())
reactor.run()
Personally, I find the fact that IPushProducer and IPullProducer require a periodic callback, makes them less useful. Others disagree... shrug. Take your pick.