Howto connect two programs with Twisted - python

I have an echoserver in Twisted that needs to get input from an echoclient. The echocient is a GUI (Panda3D). The client just sends a short message when a button is clicked.
So I have messages to send at irregular times (only when button is clicked).
How can I have a permanent connection (the reactor.run() is already started at the beginning of the client program) and send messages.
I don't want to write a polling mechanism in EchoClient/connectionMade. I saw an example of gtk+ but cannot translate it to Panda. How to go about. The code hereunder is not working at all but gives you an idea of what I want (basically permanent connection and once in a while the user sends something when pressing a button).
from direct.showbase.ShowBase import ShowBase
from direct.gui.DirectButton import DirectButton
from panda3d.core import Vec3
from direct.task import Task
from twisted.internet import protocol, reactor, defer
from twisted.internet.task import LoopingCall
from twisted.spread import pb
FRAMERATE = 32
class LoginDialog:
def __init__(self, deferred):
self.deferredResult = deferred
class EchoClient(ShowBase, protocol.Protocol):
def __init__(self):
ShowBase.__init__(self)
self.echoer = echoer
self.button = DirectButton(pos = Vec3(.1,0,.1), text = "Send request",
scale = .1, pad = (.5, .5),
rolloverSound = None, clickSound = None,
command = self.Request)
def Request():
self.echoer.transport.write("Message from client")
def dataReceived(self, data):
print "Server said: ", data
cf = pb.PBClientFactory()
cf.getRootObject().addCallback(EchoClient)
reactor.connectTCP("localhost", 17000, cf)
LoopingCall(taskMgr.step).start(1 / FRAMERATE)
reactor.run()

If you make the client and the server part of the same process then you'll always (well, practically always) be able to pass information from one to the other.
If you make the client and the server different processes then there is probably no solution that satisfies your requirements.
You always have to do something to establish a connection between two processes and there's always a chance it might go away. Sorry.

have a look at multiprocessing: http://docs.python.org/2/library/multiprocessing.html
you can start and join different processes as well as message between them.

I found the answer. Now I have a program that sends a message to a server once a button is pushed. I used a ClientCreator to create an EchoClient. Like this:
self.clientcreate = protocol.ClientCreator(reactor, EchoClient)
self.clientcreate.connectTCP(host, port).addCallbacks(self.connectionMade,
self.connectionFailed)
Now when I connectTCP, and make a connection it calls self.ConnectionMade upon success.
Thereby it also gives the EchoClient to the function. So I can store it (self.client = echoclient), and use it when I want (could not achieve this via Factories).
For instance the button calls the function Request and there I can directly use the transport.write command, because I have the client now. So self.client.transport.write('whatever').
If people did not understand what I wanted before, have better ideas to get this done, I would like to see their comments (i want to learn).
The complete code (Client only):
from twisted.internet import protocol, reactor, defer
from twisted.internet.task import LoopingCall
from direct.showbase.ShowBase import ShowBase
from direct.gui.DirectButton import DirectButton
from panda3d.core import Vec3
from direct.task import Task
FRAMERATE = 32
class ButtonDialog(ShowBase):
def __init__(self):
ShowBase.__init__(self)
self.button = DirectButton(pos = Vec3(.1,0,.1), text = "Send request",
scale = .1, pad = (.5, .5),
rolloverSound = None, clickSound = None,
command = self.Request)
host = "localhost"
port = 17001
self.clientcreate = protocol.ClientCreator(reactor, EchoClient)
self.clientcreate.connectTCP(host, port).addCallbacks(self.connectionMade,
self.connectionFailed)
def connectionFailed(self, f):
print "Connection Failed:", f
reactor.stop()
def connectionMade(self, echoclient):
self.client = echoclient
def Request(self):
self.client.transport.write("Message from button")
class EchoClient(protocol.Protocol):
def dataReceived(self, data):
print "Server said: ", data
def connectionLost(self, err):
print "Connection is lost: " + str(err)
reactor.stop()
ButtonDialog()
LoopingCall(taskMgr.step).start(1 / FRAMERATE)
reactor.run()

Related

Twisted Python - Push data to websocket

I've a web-socket server which connects with the clients. Following is the code:-
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
from twisted.internet import reactor
class Chat(LineReceiver):
def __init__(self, users):
self.users = users
self.name = None
self.state = "GETNAME"
def connectionMade(self):
self.sendLine("What's your name?")
def connectionLost(self, reason):
if self.users.has_key(self.name):
del self.users[self.name]
def lineReceived(self, line):
if self.state == "GETNAME":
self.handle_GETNAME(line)
else:
self.handle_CHAT(line)
def handle_GETNAME(self, name):
if self.users.has_key(name):
self.sendLine("Name taken, please choose another.")
return
self.sendLine("Welcome, %s!" % (name,))
self.name = name
self.users[name] = self
self.state = "CHAT"
def handle_CHAT(self, message):
# Need to send the message to the connected clients.
class ChatFactory(Factory):
def __init__(self):
self.users = {} # maps user names to Chat instances
def buildProtocol(self, addr):
return Chat(self.users)
reactor.listenTCP(8123, ChatFactory())
reactor.run()
Clients get connected to the above code(server), and sends the data to the server.
Now, I've another python script, basically a scraper which scrapes the web, processes it and finally need to send the data to the connected clients.
script.py
while True:
# call `send_message` function and send data to the connected clients.
How can I achieve it?? Any example would be of great help!!
UPDATE
After using Autobahn
I've a server that fetches data from 3rd party API. I want to send this data to all the connected web-socket clients. Here is my code:-
class MyServerProtocol(WebSocketServerProtocol):
def __init__(self):
self.connected_users = []
self.send_data()
def onConnect(self, request):
print("Client connecting: {0}".format(request.peer))
def onOpen(self):
print("WebSocket connection open.")
self.connected_users.append(self) # adding users to the connected_list
def send_data(self):
# fetch data from the API and forward it to the connected_users.
for u in self.users:
print 1111
u.sendMessage('Hello, Some Data from API!', False)
def onClose(self, wasClean, code, reason):
connected_users.remove(self) # remove user from the connected list of users
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
import sys
from twisted.python import log
from twisted.internet import reactor
factory = WebSocketServerFactory(u"ws://127.0.0.1:9000")
factory.protocol = MyServerProtocol
reactor.listenTCP(9000, factory)
reactor.run()
My Server will never receive a message or probably will receive, but as of right now there's no such use-case, hence no need for OnMessage event for this example).
How do I write my send_data function in order to send data to all my connected clients??
You need to avoid this pattern when writing software with Twisted:
while True:
# call `send_message` function and send data to the connected clients.
Twisted is a cooperative multitasking system. "Cooperative" means that you have to give up control of execution periodically so that other tasks get a chance to run.
twisted.internet.task.LoopingCall can be used to replace many while ... loops (particularly while True loops):
from twisted.internet.task import LoopingCall
LoopingCall(one_iteration).start(iteration_interval)
This will call one_iteration every iteration_interval seconds. In between, it will give up control of execution so other tasks can run.
Making one_iteration send a message to a client is just a matter of giving one_iteration a reference to that client (or those clients, if there are many).
This is a variation on the FAQ How do I make Input on One Connection Result in Output on Another.
If you have a ChatFactory with a dict containing all your clients, just pass that factory to one_iteration:
LoopingCall(one_iteration, that_factory)
or
LoopingCall(lambda: one_iteration(that_factory))

How to call a method from an already running python script

I need to check if the python script is already running then calling a method from the same running python script. But it must be on same process(pid), no new process. Is this possible?
I tried some codes but not worked.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import Tkinter as tk
from Tkinter import *
import socket
class Main():
def mainFunc(self):
self.root = tk.Tk()
self.root.title("Main Window")
self.lbl = Label(self.root, text = "First Text")
self.lbl.pack()
openStngs = Button(self.root, text = "Open Settings", command=self.settingsFunc)
openStngs.pack()
def settingsFunc(self):
stngsRoot = Toplevel()
stngsRoot.title("Settings Window")
changeTextOfLabel = Button(stngsRoot, text = "Change Main Window Text", command=self.change_text)
changeTextOfLabel.pack()
def change_text(self):
self.lbl.config(text="Text changed")
# the get_lock from http://stackoverflow.com/a/7758075/3254912
def get_lock(process_name):
lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
try:
print lock_socket
lock_socket.bind('\0' + process_name)
print 'I got the lock'
m.mainFunc()
mainloop()
except socket.error:
print 'lock exists'
m.settingsFunc()
mainloop()
# sys.exit()
if __name__ == '__main__':
m=Main()
get_lock('myPython.py')
You either need:
A proactive check in your running process to look at the environment (for instance, the contents of a file or data coming through a socket) to know when to fire the function,
or for your running process to receive unix signals or some other IPC (possibly one of the user-defined signals) and perform a function when one is received.
Either way you can't just reach into a running process and fire a function inside that process (it MIGHT not be literally impossible if you hook the running process up to a debugger, but I wouldn't recommend it).
Tkinter necessarily has its own event loop system, so I recommend reading up on how that works and how to either run something on a timer in that event loop system, or set up a callback that responds to a signal. You could also wrap a non-event loop based system in a try/except block that will catch an exception generated by a UNIX signal, but it may not be straightforward to resume the operation of the rest of the program after that signal is caught, in that case.
Sockets are a good solution to this kind of interprocess communication problem.
One possible approach would be to set up a socket server in a thread in your original process, this can be used as an entry point for external input. A (rather stupid) example might be:
# main.py
import socket
import SocketServer # socketserver in Python 3+
import time
from Queue import Queue
from threading import Thread
# class for handling requests
class QueueHandler(SocketServer.BaseRequestHandler):
def __init__(self, request, client_address, server):
self.server = server
server.client_address = client_address
SocketServer.BaseRequestHandler.__init__(self,request, client_address, server)
# receive a block of data
# put it in a Queue instance
# send back the block of data (redundant)
def handle(self):
data = self.request.recv(4096)
self.server.recv_q.put(data)
self.request.send(data)
class TCPServer(SocketServer.TCPServer):
def __init__(self, ip, port, handler_class=QueueHandler):
SocketServer.TCPServer.__init__(self, (ip, port), handler_class, bind_and_activate=False)
self.recv_q = Queue() # a Queue for data received over the socket
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_bind()
self.server_activate()
def shutdown(self):
SocketServer.TCPServer.shutdown(self)
def __del__(self):
self.server_close()
# This is the equivalent of the main body of your original code
class TheClassThatLovesToAdd(object):
def __init__(self):
self.value = 1
# create and instance of the server attached to some port
self.server = TCPServer("localhost",9999)
# start it listening in a separate control thread
self.server_thread = Thread(target=self.server.serve_forever)
self.server_thread.start()
self.stop = False
def add_one_to_value(self):
self.value += 1
def run(self):
while not self.stop:
print "Value =",self.value
# if there is stuff in the queue...
while not self.server.recv_q.empty():
# read and parse the message from the queue
msg = self.server.recv_q.get()
# perform some action based on the message
if msg == "add":
self.add_one_to_value()
elif msg == "shutdown":
self.server.shutdown()
self.stop = True
time.sleep(1)
if __name__ == "__main__":
x = TheClassThatLovesToAdd()
x.run()
When you start this running, it should just loop over and over printing to the screen. Output:
Value = 1
Value = 1
Value = 1
...
However the TCPServer instance attached to the TheClassThatLovesToAdd instance now gives us a control path. The simplest looking snippet of control code would be:
# control.py
import socket
import sys
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.settimeout(2)
sock.connect(('localhost',9999))
# send some command line argument through the socket
sock.send(sys.argv[1])
sock.close()
So if I run main.py in one terminal window and call python control.py add from another, the output of main.py will change:
Value = 1
Value = 1
Value = 1
Value = 2
Value = 2
...
Finally to kill it all we can run python control.py shutdown, which will gently bring main.py to a halt.
This is by no means the only solution to your problem, but it is likely to be one of the simplest.
One can try GDB, but not sure how to call a function from within [an idle thread].
Perhaps someone very versed with gdb and debugging/calling Python functions from within GDB can improve this answer.
One solution would be to use a messaging service (such as ActiveMQ or RabbitMQ). Your application subscribes to a queue/topic and whenever you want to send it a command, you write a message to it's queue. I'm not going to go into details because there are thousands of examples on-line. Queues/messaging/MQTT etc. are very simple to implement and are how most business systems (and modern control systems) communicate. Do a search for paho-mqtt.

How to handle TCP connection events in order to call methods within other class?

I am creating a robot which is going to be driven by the commands received over TCP connection. Therefore, I will have a robot class with methods (e.g. sense(), drive()...) and the class for TCP connection.
To establish TCP connection, I looked at examples from twisted. On the client side, I have written a client.py script for connection handling:
from twisted.internet import reactor, protocol
import random
from eventhook import EventHook
import common
#from Common.socketdataobjects import response
# a client protocol
class EchoClient(protocol.Protocol):
"""Once connected, send a message, then print the result."""
def connectionMade(self):
self.transport.write("hello, world!")
#the server should be notified that the connection to the robot has been established
#along with robot state (position)
#eventConnectionEstablishedHook.fire()
def dataReceived(self, data):
print "Server said:", data
self.transport.write("Hello %s" % str(random.randint(1,10)))
'''
serverMessage = common.deserializeJson(data)
command = serverMessage.command
arguments = serverMessage.arguments
#here we get for example command = "DRIVE"
#arguments = {motor1Speed: 50, motor2Speed: 40}
instead of above response, used for testing purposes,
the commands should be extracted from the data and according to the command,
the method in Robot instance should be called.
When the command execution finishes, the self.transport.write() method should be called
to notify the server that the command execution finished
'''
def connectionLost(self, reason):
print "connection lost"
class EchoFactory(protocol.ClientFactory):
protocol = EchoClient
def clientConnectionFailed(self, connector, reason):
print "Connection failed - goodbye!"
reactor.stop()
def clientConnectionLost(self, connector, reason):
print "Connection lost - goodbye!"
reactor.stop()
# this connects the protocol to a server runing on port 8000
def initializeEventHandlers(connectionEstablishedHook):
global connection
connection.established = 0
global eventConnectionEstablishedHook
eventConnectionEstablishedHook = connectionEstablishedHook
def main():
f = EchoFactory()
reactor.connectTCP("localhost", 8000, f)
reactor.run()
# this only runs if the module was *not* imported
if __name__ == '__main__':
main()
Beside this script, I have a robot class:
Class Robot(object():
def __init(self)__:
self.position = (0,0)
def drive(self, speedMotor1, speedMotor2, driveTime)
updateMotor1State(speedMotor1)
updateMotor2State(speedMotor2)
time.sleep(driveTime)
#when the execution finished, the finish status should be sent to client in order to inform the server
return "Finished"
def sense(self)
#logic to get the data from the environment
What I would like to do, is to receive the data(commands) from TCP connection and then call the according method in Robot instance. Some procedures might take longer (e.g. driving), so I tried to use events, but haven't figured out the appropriate way to communicate between TCP client and robot using events:
if __name__ == '__main__':
robotController = Robot()
eventController = Controller()
connectionEstablishedHook = EventHook()
client.initializeEventHandlers(connectionEstablishedHook)
eventController.connection = connectionEstablishedHook
client.main()
I tried to create ClientMainProgram script, where I wanted to create an instance of a robot, an instance of TCP client and implement the communication between them using events.
Previously I have managed to implement event handling using Michael Foord's events pattern on a simpler example. I would be very thankful if anyone could provide the solution to this question or any similar example which might be helpful to solve this problem.
Events are easily represented using regular Python function calls.
For example, if your protocol looks like this:
from twisted.internet.protocol import Protocol
class RobotController(Protocol):
def __init__(self, robot):
self.robot = robot
def dataReceived(self, data):
for byte in data:
self.commandReceived(byte)
def commandReceived(self, command):
if command == "\x00":
# drive:
self.robot.drive()
elif command == "\x01":
# sense:
self.robot.sense()
...
(The specifics of the protocol used in this example are somewhat incidental. I picked this protocol because it's very simple and has almost no parsing logic. For your real application I suggest you use twisted.protocols.amp.)
Then all you need to do is make sure the robot attribute is properly initialized. You can do this easily using the somewhat newer endpoint APIs that can often replace use of factories:
from sys import argv
from twisted.internet.endpoints import clientFromString, connectProtocol
from twisted.internet.task import react
def main(reactor, description):
robot = ...
endpoint = clientFromString(reactor, description)
connecting = connectProtocol(endpoint, RobotController(robot))
def connected(controller):
...
connecting.addCallback(connected)
return connecting
react(main, argv[1:])

Trial unittests using Autobahn WebSocket

I'm trying to write unittests for my application that uses Autobahn.
I want to test my controllers which gets received data from protocol, parses it and reacts to it.
But when my test comes to a point when protocol should be disconnected (self.sendClose) then it raises error
exceptions.AttributeError: 'MyProtocol' object has no attribute 'state'.
I was trying to makeConnection using proto_helpers.StringTransport but then I have errors too
exceptions.AttributeError: StringTransport instance has no attribute 'setTcpNoDelay'`
I'm using trial and I don't want to run dummy server/client for testing purposes only, because it's not recommended.
How should I write my tests so I can test functions that sends data, read data, disconnects etc. using fake connection and trial ?
It is difficult to say exactly what is going on without having a peek at MyProtocol class. The problem sounds a lot like it is caused by the fact that you are directly messing round with low level functions and therefore also the state attribute of WebSocket class, which is, well, a representation of the internal state of the WebSocket connection.
According to the autobahn reference doc, the APIs from the WebSicketProtocol that you could directly use and override are:
onOpen
onMessage
onClose
sendMessage
sendClose
Your approach of using the StringTransport to test your protocol is not ideal. The problem lays in the fact that MyProtocol is a tiny layer on top of the WebSocketProtocol framework provided by autobahn which, for better or worse, hides the details about managing the connection, the transport and the internal protocol state.
If you think about it, you want to test your stuff, not WebSocketProtocol and therefore if you do not want to embed a dummy server or client, your best bet is to test directly the methods that MyProtocol overrides.
An example of what I am saying is the following
class MyPublisher(object):
cbk=None
def publish(self, msg):
if self.cbk:
self.cbk(msg)
class MyProtocol(WebSocketServerProtocol):
def __init__(self, publisher):
WebSocketServerProtocol.__init__(self)
#Defining callback for publisher
publisher.cbk = self.sendMessage
def onMessage(self, msg, binary)
#Stupid echo
self.sendMessage(msg)
class NotificationTest(unittest.TestCase):
class MyProtocolFactory(WebSocketServerFactory):
def __init__(self, publisher):
WebSocketServerFactory.__init__(self, "ws://127.0.0.1:8081")
self.publisher = publisher
self.openHandshakeTimeout = None
def buildProtocol(self, addr):
protocol = MyProtocol(self.listener)
protocol.factory = self
protocol.websocket_version = 13 #Hybi version 13 is supported by pretty much everyone (apart from IE <8 and android browsers)
return protocol
def setUp(self):
publisher = task.LoopingCall(self.send_stuff, "Hi there")
factory = NotificationTest.MyProtocolFactory(listener)
protocol = factory.buildProtocol(None)
transport = proto_helpers.StringTransport()
def play_dumb(*args): pass
setattr(transport, "setTcpNoDelay", play_dumb)
protocol.makeConnection(transport)
self.protocol, self.transport, self.publisher, self.fingerprint_handler = protocol, transport, publisher, fingerprint_handler
def test_onMessage(self):
#Following 2 lines are the problematic part. Here you are manipulating explicitly a hidden state which your implementation should not be concerned with!
self.protocol.state = WebSocketProtocol.STATE_OPEN
self.protocol.websocket_version = 13
self.protocol.onMessage("Whatever")
self.assertEqual(self.transport.value()[2:], 'Whatever')
def test_push(self):
#Following 2 lines are the problematic part. Here you are manipulating explicitly a hidden state which your implementation should not be concerned with!
self.protocol.state = WebSocketProtocol.STATE_OPEN
self.protocol.websocket_version = 13
self.publisher.publish("Hi there")
self.assertEqual(self.transport.value()[2:], 'Hi There')
As you might have noticed, using the StringTransport here is very cumbersome. You must have knowledge of the underline framework and bypass its state management, something you don't really want to do. Unfortunately autobahn does not provide a ready-to-use test object that would permit easy state manipulation and therefore my suggestion of using dummy servers and clients is still valid
Testing your server WITH network
The test provided shows how you can test server push, asserting that what your are getting is what you expect, and using also a hook on how to determine when to finish.
The server protocol
from twisted.trial.unittest import TestCase as TrialTest
from autobahn.websocket import WebSocketServerProtocol, WebSocketServerFactory, WebSocketClientProtocol, WebSocketClientFactory, connectWS, listenWS
from twisted.internet.defer import Deferred
from twisted.internet import task
START="START"
class TestServerProtocol(WebSocketServerProtocol):
def __init__(self):
#The publisher task simulates an event that triggers a message push
self.publisher = task.LoopingCall(self.send_stuff, "Hi there")
def send_stuff(self, msg):
#this method sends a message to the client
self.sendMessage(msg)
def _on_start(self):
#here we trigger the task to execute every second
self.publisher.start(1.0)
def onMessage(self, message, binary):
#According to this stupid protocol, the server starts sending stuff when the client sends a "START" message
#You can plug other commands in here
{
START : self._on_start
#Put other keys here
}[message]()
def onClose(self, wasClean, code, reason):
#After closing the connection, we tell the task to stop sending messages
self.publisher.stop()
The client protocol and factory
Next class is the client protocol. It basically tells the server to start pushing messages. It calls the close_condition on them to see if it is time to close the connection and as a last thing, it calls the assertion function on the messages it received to see if the test was successful or not
class TestClientProtocol(WebSocketClientProtocol):
def __init__(self, assertion, close_condition, timeout, *args, **kwargs):
self.assertion = assertion
self.close_condition = close_condition
self._received_msgs = []
from twisted.internet import reactor
#This is a way to set a timeout for your test
#in case you never meet the conditions dictated by close_condition
self.damocle_sword = reactor.callLater(timeout, self.sendClose)
def onOpen(self):
#After the connection has been established,
#you can tell the server to send its stuff
self.sendMessage(START)
def onMessage(self, msg, binary):
#Here you get the messages pushed from the server
self._received_msgs.append(msg)
#If it is time to close the connection
if self.close_condition(msg):
self.damocle_sword.cancel()
self.sendClose()
def onClose(self, wasClean, code, reason):
#Now it is the right time to check our test assertions
self.assertion.callback(self._received_msgs)
class TestClientProtocolFactory(WebSocketClientFactory):
def __init__(self, assertion, close_condition, timeout, **kwargs):
WebSocketClientFactory.__init__(self, **kwargs)
self.assertion = assertion
self.close_condition = close_condition
self.timeout = timeout
#This parameter needs to be forced to None to not leave the reactor dirty
self.openHandshakeTimeout = None
def buildProtocol(self, addr):
protocol = TestClientProtocol(self.assertion, self.close_condition, self.timeout)
protocol.factory = self
return protocol
The trial based test
class WebSocketTest(TrialTest):
def setUp(self):
port = 8088
factory = WebSocketServerFactory("ws://localhost:{}".format(port))
factory.protocol = TestServerProtocol
self.listening_port = listenWS(factory)
self.factory, self.port = factory, port
def tearDown(self):
#cleaning up stuff otherwise the reactor complains
self.listening_port.stopListening()
def test_message_reception(self):
#This is the test assertion, we are testing that the messages received were 3
def assertion(msgs):
self.assertEquals(len(msgs), 3)
#This class says when the connection with the server should be finalized.
#In this case the condition to close the connectionis for the client to get 3 messages
class CommunicationHandler(object):
msg_count = 0
def close_condition(self, msg):
self.msg_count += 1
return self.msg_count == 3
d = Deferred()
d.addCallback(assertion)
#Here we create the client...
client_factory = TestClientProtocolFactory(d, CommunicationHandler().close_condition, 5, url="ws://localhost:{}".format(self.port))
#...and we connect it to the server
connectWS(client_factory)
#returning the assertion as a deferred purely for demonstration
return d
This is obviously just an example, but as you can see I did not have to mess around with makeConnection or any transport explicitly

Irregular Transmission Problem with Python Twisted Push Producer

I want to transmit data from a Queue using Twisted. I currently use a push producer to poll the queue for items and write to the transport.
class Producer:
implements(interfaces.IPushProducer)
def __init__(self, protocol, queue):
self.queue = queue
self.protocol = protocol
def resumeProducing(self):
self.paused = False
while not self.paused:
try:
data = self.queue.get_nowait()
logger.debug("Transmitting: '%s'", repr(data))
data = cPickle.dumps(data)
self.protocol.transport.write(data + "\r\n")
except Empty:
pass
def pauseProducing(self):
logger.debug("Transmitter paused.")
self.paused = True
def stopProducing(self):
pass
The problem is, that the data are sent very irregularly and if only one item was in the queue, the data is never going to be sent. It seems that Twisted waits until the data to be transmitted has grown to a specific value until it transmits it. Is the way I implemented my producer the right way? Can I force Twisted to transmit data now?
I've also tried using a pull producer, but Twisted does not call the resumeProducing() method of it at all. Do I have to call the resumeProducer() method from outside, when using a pull producer?
It's hard to say why your producer doesn't work well without seeing a complete example (that is, without also seeing the code that registers it with a consumer and the code which is putting items into that queue).
However, one problem you'll likely have is that if your queue is empty when resumeProducing is called, then you will write no bytes at all to the consumer. And when items are put into the queue, they'll sit there forever, because the consumer isn't going to call your resumeProducing method again.
And this generalizes to any other case where the queue does not have enough data in it to cause the consumer to call pauseProducing on your producer. As a push producer, it is your job to continue to produce data on your own until the consumer calls pauseProducing (or stopProducing).
For this particular case, that probably means that whenever you're going to put something in that queue - stop: check to see if the producer is not paused, and if it is not, write it to the consumer instead. Only put items in the queue when the producer is paused.
Here are two possible solutions:
1) Periodically poll your local application to see if you have additional data to send.
NB. This relies on a periodic async callback from the deferLater method in twisted. If you need a responsive application that sends data on demand, or a long running blocking operation (eg. ui that uses its own event loop) it may not be appropriate.
Code:
from twisted.internet.protocol import Factory
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.internet.interfaces import IPushProducer
from twisted.internet.task import deferLater, cooperate
from twisted.internet.protocol import Protocol
from twisted.internet import reactor
from zope.interface import implementer
import time
# Deferred action
def periodically_poll_for_push_actions_async(reactor, protocol):
while True:
protocol.send(b"Hello World\n")
yield deferLater(reactor, 2, lambda: None)
# Push protocol
#implementer(IPushProducer)
class PushProtocol(Protocol):
def connectionMade(self):
self.transport.registerProducer(self, True)
gen = periodically_poll_for_push_actions_async(self.transport.reactor, self)
self.task = cooperate(gen)
def dataReceived(self, data):
self.transport.write(data)
def send(self, data):
self.transport.write(data)
def pauseProducing(self):
print 'Workload paused'
self.task.pause()
def resumeProducing(self):
print 'Workload resumed'
self.task.resume()
def stopProducing(self):
print 'Workload stopped'
self.task.stop()
def connectionLost(self, reason):
print 'Connection lost'
try:
self.task.stop()
except:
pass
# Push factory
class PushFactory(Factory):
def buildProtocol(self, addr):
return PushProtocol()
# Run the reactor that serves everything
endpoint = TCP4ServerEndpoint(reactor, 8089)
endpoint.listen(PushFactory())
reactor.run()
2) Manually keep track of Protocol instances and use reactor.callFromThread() from a different thread. Lets you get away with a long blocking operation in the other thread (eg. ui event loop).
Code:
from twisted.internet.protocol import Factory
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.internet.interfaces import IPushProducer
from twisted.internet.task import deferLater, cooperate
from twisted.internet.protocol import Protocol
from twisted.internet import reactor, threads
import time
import random
import threading
# Connection
protocol = None
# Some other thread that does whatever it likes.
class SomeThread(threading.Thread):
def run(self):
while True:
print("Thread loop")
time.sleep(random.randint(0, 4))
if protocol is not None:
reactor.callFromThread(self.dispatch)
def dispatch(self):
global protocol
protocol.send("Hello World\n")
# Push protocol
class PushProtocol(Protocol):
def connectionMade(self):
global protocol
protocol = self
def dataReceived(self, data):
self.transport.write(data)
def send(self, data):
self.transport.write(data)
def connectionLost(self, reason):
print 'Connection lost'
# Push factory
class PushFactory(Factory):
def buildProtocol(self, addr):
return PushProtocol()
# Start thread
other = SomeThread()
other.start()
# Run the reactor that serves everything
endpoint = TCP4ServerEndpoint(reactor, 8089)
endpoint.listen(PushFactory())
reactor.run()
Personally, I find the fact that IPushProducer and IPullProducer require a periodic callback, makes them less useful. Others disagree... shrug. Take your pick.

Categories

Resources