I tried to get support on this but I am TOTALLY confused.
Here's my code:
from twisted.internet import reactor
from twisted.web.client import getPage
from twisted.web.error import Error
from twisted.internet.defer import DeferredList
from sys import argv
class GrabPage:
def __init__(self, page):
self.page = page
def start(self, *args):
if args == ():
# We apparently don't need authentication for this
d1 = getPage(self.page)
else:
if len(args) == 2:
# We have our login information
d1 = getPage(self.page, headers={"Authorization": " ".join(args)})
else:
raise Exception('Missing parameters')
d1.addCallback(self.pageCallback)
dl = DeferredList([d1])
d1.addErrback(self.errorHandler)
dl.addCallback(self.listCallback)
def errorHandler(self,result):
# Bad thingy!
pass
def pageCallback(self, result):
return result
def listCallback(self, result):
print result
a = GrabPage('http://www.google.com')
data = a.start() # Not the HTML
I wish to get the HTML out which is given to pageCallback when start() is called. This has been a pita for me. Ty! And sorry for my sucky coding.
You're missing the basics of how Twisted operates. It all revolves around the reactor, which you're never even running. Think of the reactor like this:
(source: krondo.com)
Until you start the reactor, by setting up deferreds all you're doing is chaining them with no events from which to fire.
I recommend you give the Twisted Intro by Dave Peticolas a read. It's quick and it really gives you all the missing information that the Twisted documentation doesn't.
Anyways, here is the most basic usage example of getPage as possible:
from twisted.web.client import getPage
from twisted.internet import reactor
url = 'http://aol.com'
def print_and_stop(output):
print output
if reactor.running:
reactor.stop()
if __name__ == '__main__':
print 'fetching', url
d = getPage(url)
d.addCallback(print_and_stop)
reactor.run()
Since getPage returns a deferred, I'm adding the callback print_and_stop to the deferred chain. After that, I start the reactor. The reactor fires getPage, which then fires print_and_stop which prints the data from aol.com and then stops the reactor.
Edit to show a working example of OP's code:
class GrabPage:
def __init__(self, page):
self.page = page
########### I added this:
self.data = None
def start(self, *args):
if args == ():
# We apparently don't need authentication for this
d1 = getPage(self.page)
else:
if len(args) == 2:
# We have our login information
d1 = getPage(self.page, headers={"Authorization": " ".join(args)})
else:
raise Exception('Missing parameters')
d1.addCallback(self.pageCallback)
dl = DeferredList([d1])
d1.addErrback(self.errorHandler)
dl.addCallback(self.listCallback)
def errorHandler(self,result):
# Bad thingy!
pass
def pageCallback(self, result):
########### I added this, to hold the data:
self.data = result
return result
def listCallback(self, result):
print result
# Added for effect:
if reactor.running:
reactor.stop()
a = GrabPage('http://google.com')
########### Just call it without assigning to data
#data = a.start() # Not the HTML
a.start()
########### I added this:
if not reactor.running:
reactor.run()
########### Reference the data attribute from the class
data = a.data
print '------REACTOR STOPPED------'
print
########### First 100 characters of a.data:
print '------a.data[:100]------'
print data[:100]
Related
I have a callback chain with an errback at the end. If any of the callbacks fail, I need to pass an object to be used on errBack.
How can I pass an object from callback to the errback?
The following code exemplifies what I want to do:
from twisted.internet.defer import FAILURE
from twisted.internet import defer
class CodMsg(object):
def __init__(self, code, msg):
self.code = code
self.msg = msg
class Resource(object):
#classmethod
def checkCondition(cls, result):
if result == "error":
cdm = CodMsg(1, 'Error 1')
raise FAILURE, cdm
else:
return "ok"
#classmethod
def erBackTst (cls, result):
####### How to get the value of cdm here? ######## <<<===
print 'Error:'
print result
return result
d = defer.Deferred()
d.addCallback(Resource.checkCondition)
d.addErrback(Resource.erBackTst)
d.callback("error")
print d.result
In this case you can just raise an exception, containing all info you need
For example:
from twisted.internet import defer
class MyCustomException(Exception):
def __init__(self, msg, code):
self.code = code
self.message = msg
def callback(result):
print result
raise MyCustomException('Message', 23)
def errback(failure):
# failure.value is an exception instance that you raised in callback
print failure.value.message
print failure.value.code
d = defer.Deferred()
d.addCallback(callback)
d.addErrback(errback)
d.callback("error")
Also for better understanding deffereds and async programming you can read this nice twisted tutorial http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/.
It uses a little bit outdated twisted version in examples but it is still an exellent source to start learning twisted
Here it is
There is class Body which sends the message to remote Perspective
Broker (PB) Friend.
The test is checking whether the Me instance can communicate with
Friend by calling *speak_ to _friend* or tell methods.
speak_ to_ friend is wrapper for Soul.speak virtual function, which is overwritten by Body.tell method.
Soul class is encapsulated into Body
tell method simulate blocking call by waiting for gotAnswer event from received callback function.
According to printouts, Friend communicates with Me in the same way for unittest and standalone cases.
The problem
... is that when running the unittest (unittest-me.py) with trial, the received callback is never called. Instead of that the silly error message printed on client side:
Connection was closed cleanly
For me it looks that reactor is closed before the deferred queue was completely processed.
If I am running standalone test test-body.py, then everything works as expected: Friend talks with me!
The questions
Why unittest behaves differently in this case?
How to fix it?
unittest output
Python2.7\Scripts\trial.py ..\Soapbox\Node\unittest-me.py
unittest-me
Trial_TestCase
test_conversation ... wake upConnecting to localhost:18800
DBG: speak
DBG: say
[FAIL]
Friend not willing to talk with me because: [Failure instance: Traceback (failure with no frames): <class 'twisted.internet.error.ConnectionDone
'>: Connection was closed cleanly.
]
===============================================================================
[FAIL]
Traceback (most recent call last):
File "..\Soapbox\Node\unittest-me.py", line 50, in test_conversation
self.assertTrue(answer is not None, "Friend gave no answer")
twisted.trial.unittest.FailTest: Friend gave no answer
unittest-me.Trial_TestCase.test_conversation
-------------------------------------------------------------------------------
Ran 1 tests in 3.096s
FAILED (failures=1)
test-body output
python test-body.py
connector: <twisted.internet.tcp.Connector instance at 0x01C87940>
Press any key to continue...Starting reactor
wake upDBG: speak
DBG: say
DBG: received
Friend's reply: <username>
Press any key to continue...
That's all
freind's output
Dr. Livesey: I got your words: whoami
DBG: answer: <username>
Dr. Livesey: I got your words: whoami
DBG: answer: <username>
unittest-me.py
from twisted.trial import unittest
from twisted.spread import pb
from twisted.internet import reactor
from body import Body
class Trial_TestCase(unittest.TestCase):
def setUp(self):
self.ref = None
self.Me = Body('localhost', 18800)
self.assertTrue(self.Me is not None, "Creating Me assertion failed")
self.Me.start()
self.Me.pbFactory = pb.PBClientFactory(self.Me)
print "Connecting to " + self.Me.hostName + ":" + str(self.Me.portNo)
connector = reactor.connectTCP(self.Me.hostName,
self.Me.portNo,
self.Me.pbFactory)
self.addCleanup(connector.disconnect)
# =====================================================================
# unittest related parts
# =====================================================================
def gotRoot(ref):
self.ref = ref
df = self.Me.pbFactory.getRootObject()
'''
Without this dummy call, I can say no word - say not called
'''
obj = df.addCallback(gotRoot)
return obj
def tearDown(self):
self.Me.remoteObj.broker.transport.loseConnection()
self.Me.pbFactory.disconnect()
pass
def test_conversation(self):
# answer = self.Me.speak_to_friend()
answer = self.Me.tell(None)
self.assertTrue(answer is not None, "Friend gave no answer")
return self.Me.dm
if __name__ == "__main__":
#import sys;sys.argv = ['', 'Test.testName']
unittest.main()
test-body.py
if __name__ == '__main__':
pass
from twisted.spread import pb
from twisted.internet import reactor, threads
import threading
import time
from body import Body
def controlThread(name, body):
print "Press any key to continue..."
raw_input()
body.start()
result = body.tell(None)
print "Friend's reply: " + str(result)
print "Press any key to continue..."
raw_input()
reactor.stop()
pass
body = Body('127.0.0.1', 18800)
f = pb.PBClientFactory(body)
body.pbFactory = f
connector = reactor.connectTCP("127.0.0.1", 18800, f)
print "connector: ", connector
threading.Thread(target=controlThread, args=("Control", body)).start()
print "Starting reactor"
reactor.run()
print "That's all"
body.py
import threading
import types
class Soul():
def speak(self):
'''
virtual method must be overwritten by derived class
'''
raise NotImplementedError("I cannot speak without a body. "\
"Please give me body first")
pass
class Body(threading.Thread):
def __init__(self, hostname, portno):
self.hostName = hostname
self.portNo = portno
self.df = None
self.dg = None
self.dm = None
self.remoteObj = None
self.pbFactory = None
self.answer = None
self.gotAnswer = threading.Event()
self.Soul = Soul()
self.Soul.speak = types.MethodType(self.tell, self.Soul)
threading.Thread.__init__(self)
pass
def run(self):
print "wake up"
pass
def speak_to_friend(self):
print 'DBG: speak_to_friend'
return self.Soul.speak()
def tell(self, soul):
print 'DBG: speak'
self.df = self.pbFactory.getRootObject()
self.answer = None
self.gotAnswer.clear()
self.dg = self.df.addCallback(self.say)
self.gotAnswer.wait(timeout=3)
return self.answer
def say(self, remoteObj):
print 'DBG: say'
self.remoteObj = remoteObj
self.dm = remoteObj.callRemote('talk', 'whoami')
self.dm.addCallback(self.received)
self.dm.addErrback(self.broken)
pass
def received(self, (answer, result)):
print 'DBG: received'
self.answer = '\n\t'.join(str.splitlines(answer))
self.gotAnswer.set()
pass
def broken(self,reason):
print "Friend not willing to talk with me because: ", reason.getErrorMessage()
pass
friend.py
from twisted.spread import pb
from twisted.internet import reactor
from twisted.internet import defer
from subprocess import Popen
import subprocess
from twisted.web import _newclient
class BrokenError(pb.Error): pass
class Friend(pb.Root):
def __init__(self,name):
self.counter = 0
self.name = str(name)
def remote_broken(self):
msg = "I don't hear you"
print "Talk louder" % msg
raise BrokenError(msg)
def broken(self,reason):
print "Communication is broken: ", reason.getErrorMessage()
# print "printError > %r" % failure
if reason.check(_newclient.RequestGenerationFailed):
print "printError: RequestGenerationFailed"
for f in reason.value.reasons:
print "printError > %r" % f
print f.getTraceback()
pass
def remote_talk(self,words):
print self.name + ": I got your words: " + words
process = Popen(words,stdout=subprocess.PIPE)
answer = process.stdout.read()
process.wait()
print "DBG: answer: ", answer
return (answer, process.returncode)
def main():
reactor.listenTCP(18800, pb.PBServerFactory(Friend("Dr. Livesey")))
reactor.run()
if __name__ == '__main__':
main()
My goal here is to make a simple server handling TCP message like "process My String" that send "My String" to be processed by a quite time taking operation called (here slowFunction). Here I'm calling this function through deferToThread but nothing seems to happened : the defered's callback messages doesn't show up anywhere (breakpoints show it's never called) and the print in the function aren't displayed (breakpoints show it's never called)
from twisted import protocols
from twisted.protocols import basic
from twisted.internet import threads, protocol, reactor
from twisted.application import service, internet
import re
import time
def slowFunction(arg):
print "starting process"
time.sleep(1)
print "processed "+arg
class MySimpleServer(basic.LineReceiver):
PROCESS_COMMAND = "process (.*)" #re pattern
processFunction = slowFunction
clients = []
def connectionMade(self):
print "Client connected"
MySimpleServer.clients.append(self)
def connectionLost(self, reason):
print "Client gone"
MySimpleServer.clients.remove(self)
def onProcessDone(self):
self.message("Process done")
def onError(self):
self.message("Process fail with error")
def lineReceived(self, line):
processArgumentResult = re.search(MySimpleServer.PROCESS_COMMAND, line)
if not processArgumentResult == None:
processArgument = processArgumentResult.groups()[0]
deferred = threads.deferToThread(MySimpleServer.processFunction, processArgument)
deferred.addCallback(self.onProcessDone)
deferred.addErrback(self.onError)
self.message("processing your request")
else:
print "Unknown message line: "+line
def message(self, message):
self.transport.write(message + '\n')
if __name__ == '__main__':
factory = protocol.ServerFactory()
factory.protocol = MySimpleServer
factory.client = []
reactor.listenTCP(8000, factory)
reactor.run()
I've got some help by the guys of twisted irc
The points are : the callback (onProcessDone and onError) are supposed to take a result argument, and the function called by deferToThread will receive self as an argument (it should one of the MySimpleServer class' method).
The final code is now :
from twisted import protocols
from twisted.protocols import basic
from twisted.internet import threads, protocol, reactor
from twisted.application import service, internet
import re
import time
def slowFunction(arg):
print "starting process"
time.sleep(20)
print "processed "+arg
class MySimpleServer(basic.LineReceiver):
PROCESS_COMMAND = "process (.*)" #re pattern
clients = []
def connectionMade(self):
print "Client connected"
MySimpleServer.clients.append(self)
def connectionLost(self, reason):
print "Client gone"
MySimpleServer.clients.remove(self)
def onProcessDone(self, result):
self.message("Process done")
def onError(self, result):
self.message("Process fail with error")
def processFunction(self, processArgument):
slowFunction(processArgument)
def lineReceived(self, line):
processArgumentResult = re.search(MySimpleServer.PROCESS_COMMAND, line)
if not processArgumentResult == None:
processArgument = processArgumentResult.groups()[0]
deferred = threads.deferToThread(self.processFunction, processArgument)
deferred.addCallback(self.onProcessDone)
deferred.addErrback(self.onError)
self.message("processing your request")
else:
print "Unknown message line: "+line
def message(self, message):
self.transport.write(message + '\n')
if __name__ == '__main__':
factory = protocol.ServerFactory()
factory.protocol = MySimpleServer
factory.client = []
reactor.listenTCP(8000, factory)
reactor.run()
Another way you could've done this is with staticmethod; this is the only legitimate use for it.
class MySimpleServer(basic.LineReceiver):
processFunction = staticmethod(slowFunction)
Learn Twisted. I decided to write a server and client that once a second to share data.
Wrote one implementation, but it seems to me that it is not correct.
# -*- coding: utf-8 -*-
from twisted.spread import pb
from twisted.internet import reactor, task
from twisted.cred import credentials
from win32com.server import factory
class login_send:
def __init__(self):
self.count=0
self.timeout = 1.0
self.factory = pb.PBClientFactory()
reactor.connectTCP("localhost", 8800, self.factory)
def testTimeout(self):
self.count+=1
print self.count
def1 = self.factory.login(credentials.UsernamePassword("test1","bb1b"))
def1.addCallbacks(self.good_connected, self.bad_connected)
def1.addCallback(self.send_data)
def1.addErrback(self.disconnect)
if self.count>10:def1.addBoth(self.disconnect)
def start(self):
l = task.LoopingCall(self.testTimeout)
l.start(self.timeout)
reactor.run()
def good_connected(self, perspective):
print 'good login and password', perspective
return perspective
def bad_connected(self, perspective):
print 'bad login or password', perspective
return perspective
def send_data(self, perspective):
print 'send'
return perspective.callRemote("foo", self.count)
def disconnect(self, perspective):
print 'disconnect'
reactor.stop()
if __name__ == "__main__":
st=login_send()
st.start()
Code: if login and password True -> send self.count, if login or password False -> disconnect, if self.count>10 -> disconnect
The first mistake, in my opinion is that I have to login every time.
def1 = self.factory.login(credentials.UsernamePassword("test1", "bb1b"))
How to make one authorization, and continue to send data every second?
simple test server code:
from zope.interface import implements
from twisted.spread import pb
from twisted.cred import checkers, portal
from twisted.internet import reactor
class MyPerspective(pb.Avatar):
def __init__(self, name):
self.name = name
def perspective_foo(self, arg):
print "I am", self.name, "perspective_foo(",arg,") called on", self
return arg
class MyRealm:
implements(portal.IRealm)
def requestAvatar(self, avatarId, mind, *interfaces):
if pb.IPerspective not in interfaces:
print 'qqqq'
raise NotImplementedError
return pb.IPerspective, MyPerspective(avatarId), lambda:None
p = portal.Portal(MyRealm())
c = checkers.InMemoryUsernamePasswordDatabaseDontUse(test1="bbb",
user2="pass2")
p.registerChecker(c)
reactor.listenTCP(8800, pb.PBServerFactory(p))
reactor.run()
I believe this should do the trick.
# Upper case first letter of class name is good policy.
class Login_send:
def __init__(self):
# initialize the state variable to False.
self.connection = False
self.count=0
self.timeout = 1.0
self.factory = pb.PBClientFactory()
reactor.connectTCP("localhost", 8800, self.factory)
def testTimeout(self):
self.count+=1
print self.count
# no connection -- create one.
if not self.connection:
self.assign_connection()
# cached connection exists, call send_data manually.
elif self.count > 10:
self.disconnect(self.connection)
else:
#you probably want to send data only if it it should be valid.
self.send_data(self.connection)
def assign_connection(self):
''' Creates and stores a Deffered which represents the connection to
the server. '''
# cache the connection.
self.connection = self.factory.login(
credentials.UsernamePassword("test1","bb1b"))
# add connection callbacks as normal.
self.connection.addCallbacks(
self.good_connected, self.bad_connected)
self.connection.addCallback(self.send_data)
self.connection.addErrback(self.disconnect)
def disconnect(self, perspective):
# be sure to cleanup after yourself!
self.connection = False
print 'disconnect'
reactor.stop()
# the rest of your class goes here.
from twisted.internet import reactor
from twisted.internet import threads
from twisted.internet import defer
import time
def worker(arg):
print 'Hello world'
time.sleep(10)
return 1
def run():
print 'Starting workers'
l = []
for x in range(2):
l.append(threads.deferToThread(worker, x))
return defer.DeferredList(l)
def res(results):
print results
reactor.stop()
d = run()
d.addCallback(res)
reactor.run()
How to stop workers by timeout ?
Threads cannot be interrupted unless they cooperate with you. time.sleep(10) is not going to cooperate, so I don't think you can interrupt this worker. If you have another kind of worker that has several discrete phases, or operates in a loop over some tasks, then you can do something like this:
def worker(stop, jobs):
for j in jobs:
if stop:
break
j.do()
stop = []
d = deferToThread(worker)
# This will make the list eval to true and break out of the loop.
stop.append(None)
This isn't Twisted specific, either. This is just how threads work in Python.
While it may not be possible to interrupt the threads, the Deferred can be stopped via the cancel function, which I think is available in Twisted 10.1.0 and later.
I've used the following class to make Deferreds that callback a particular function if the Deferred hasn't fired after some time. It might be useful for someone that has the same question as that posed in the subject of the OP.
EDIT: As suggested by the comments below, it's best not to inherit from defer.Deferred. Therefore, I've changed the code to use a wrapper that achieves the same effect.
class DeferredWrapperWithTimeout(object):
'''
Holds a deferred that allows a specified function to be called-back
if the deferred does not fire before some specified timeout.
'''
def __init__(self, canceller=None):
self._def = defer.Deferred(canceller)
def _finish(self, r, t):
'''
Function to be called (internally) after the Deferred
has fired, in order to cancel the timeout.
'''
if ( (t!=None) and (t.active()) ):
t.cancel()
return r
def getDeferred(self):
return self._def
def addTimeoutCallback(self, reactr, timeout,
callUponTimeout, *args, **kw):
'''
The function 'callUponTimeout' (with optional args or keywords)
will be called after 'timeout' seconds, unless the Deferred fires.
'''
def timeoutCallback():
self._def.cancel()
callUponTimeout(*args, **kw)
toc = reactr.callLater(timeout, timeoutCallback)
return self._def.addCallback(self._finish, toc)
Example callback before timeout:
from twisted.internet import reactor
from DeferredWithTimeout import *
dw = DeferredWrapperWithTimeout()
d = dw.getDeferred()
def testCallback(x=None):
print "called"
def testTimeout(x=None):
print "timedout"
d.addCallback(testCallback)
dw.addTimeoutCallback(reactor, 20, testTimeout, "to")
reactor.callLater(2, d.callback, "cb")
reactor.run()
Prints "called" and nothing else.
Example timeout before callback:
from twisted.internet import reactor
from DeferredWithTimeout import *
dw = DeferredWrapperWithTimeout()
d = dw.getDeferred()
def testCallback(x=None):
print "called"
def testTimeout(x=None):
print "timedout"
d.addCallback(testCallback)
dw.addTimeoutCallback(reactor, 20, testTimeout, "to")
reactor.run()
Prints "timedout" after 20 seconds, and nothing else.
We do it like this using a decorator. This method has the advantage that the deferred is cancelled when the timeout is reached. This should somehow become part of the Twisted library imho
from twisted.internet import defer, reactor
def timeout(secs):
"""Decorator to add timeout to Deferred calls"""
def wrap(func):
#defer.inlineCallbacks
def _timeout(*args, **kwargs):
raw_d = func(*args, **kwargs)
if not isinstance(raw_d, defer.Deferred):
defer.returnValue(raw_d)
timeout_d = defer.Deferred()
times_up = reactor.callLater(secs, timeout_d.callback, None)
try:
raw_result, timeout_result = yield defer.DeferredList(
[raw_d, timeout_d], fireOnOneCallback=True, fireOnOneErrback=True,
consumeErrors=True)
except defer.FirstError as e: # Only raw_d should raise an exception
assert e.index == 0
times_up.cancel()
e.subFailure.raiseException()
else: # timeout
if timeout_d.called:
raw_d.cancel()
raise Exception("%s secs have expired" % secs)
# no timeout
times_up.cancel()
defer.returnValue(raw_result)
return _timeout
return wrap
Well, my answer is not about threads, but as it was said, you can implement timeout functionality as a separate helper:
from twisted.internet import defer
def add_watchdog(deferred, timeout=0.05):
def callback(value):
if not watchdog.called:
watchdog.cancel()
return value
deferred.addBoth(callback)
from twisted.internet import reactor
watchdog = reactor.callLater(timeout, defer.timeout, deferred)
d = defer.Deferred()
add_watchdog(d)
Then you can trap defer.TimeoutError in deferred's errback if you need.