I am writing a log handler for ActiveMQ. I have one class, Messenger, to publish a message to ActiveMQ. Also, I added class Handler to handle this and get_logger to get this logger.
import json
import logging
from stompest.config import StompConfig
from stompest.sync import Stomp
class Messanger(object):
def __init__(self, uri):
self.cfg = StompConfig(uri)
def publish(self, queue, data):
data = json.dumps(data)
client = Stomp(self.cfg)
client.connect()
try:
client.send(queue, data)
except Exception, exc:
print "Error: ", exc
client.disconnect()
class Handler(logging.Handler):
def __init__(self, amq_uri, out_queue):
logging.Handler.__init__(self)
self.queue = queue
self.uri = uri
def emit(self, record):
msg = self.format(record)
messanger = Messanger(self.uri)
messanger.send(self.queue, msg)
def get_logger(uri, queue):
logger = logging.getLogger('testlogger')
logger.addHandler(Handler(uri, queue))
return logger
I have written a UT for this.
from unittest import TestCase
from mock import patch, Mock
from my_package import get_logger
class TestHandler(TestCase):
#patch('stompest.sync.client.Stomp')
def test_activemq_handler(self, mock_stomp):
URI = "tcp://localhost:61613"
QUEUE = "/queue/logger_queue"
mock_stomp.connect = Mock(return_value=1)
mock_stomp.send = Mock(return_value=2)
data = "This is test logging"
LOG = get_logger(URI, QUEUE)
LOG.error(data)
But still it goes to the original send method and tries to connect to the server.
Traceback (most recent call last):
File ".my_package/venv/lib/python2.7/site-packages/mock/mock.py", line 1305, in patched
return func(*args, **keywargs)
File "./my_package/tests/test_activemq_handler.py", line 23, in test_activemq_handler
LOG.error(data)
File "/System/Library/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 1191, in error
self._log(ERROR, msg, args, **kwargs)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 1284, in _log
self.handle(record)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 1294, in handle
self.callHandlers(record)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 1334, in callHandlers
hdlr.handle(record)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 757, in handle
self.emit(record)
File "./my_package/__init__.py", line 28, in emit
messanger.send(self.queue, msg)
File "./my_package/clients/stomp_messanger.py", line 41, in send
client.connect()
File "./my_package/venv/lib/python2.7/site-packages/stompest/sync/client.py", line 85, in connect
for (broker, connectDelay) in self._failover:
File "./my_package/venv/lib/python2.7/site-packages/stompest/protocol/failover.py", line 48, in __iter__
yield broker, self._delay()
File "./my_package/venv/lib/python2.7/site-packages/stompest/protocol/failover.py", line 66, in _delay
raise StompConnectTimeout('Reconnect timeout: %d attempts' % self._maxReconnectAttempts)
StompConnectTimeout: Reconnect timeout: 0 attempts
How can I mock this?
The basic principle is that you patch where an object is looked up, which is not necessarily the same place as where it is defined.
You are adding the patch to the wrong place. The Messenger class module imports Stomp from stompest.sync before the tests run, and that's why your patching has no effect. You should patch the reference to Stomp in your module.
from unittest import TestCase
from mock import patch, Mock
from my_package import get_logger
class TestHandler(TestCase):
#patch('path.to.your.Messenger_class_module.Stomp') #path to your module
def test_activemq_handler(self, mock_stomp):
URI = "tcp://localhost:61613"
QUEUE = "/queue/logger_queue"
...
This will help if you need further clarification on why this is happening.
Related
I'm trying to write some unittest for functions that uses openshift python client so I need to mock some openshift client calls.
https://github.com/openshift/openshift-restclient-python
from kubernetes import client, config
from openshift.dynamic import DynamicClient, exceptions as openshift_exceptions
_openshift_cert_manager_api_version = 'cert-manager.io/v1'
class OCPException(Exception):
pass
def certificate_exists(dyn_client: DynamicClient, namespace: str, certificate_name: str) -> bool:
""" Checks if certificate already exists in namespace. """
v1_certificates = dyn_client.resources.get(api_version=_openshift_cert_manager_api_version, kind='Certificate')
try:
v1_certificates.get(namespace=namespace, name=certificate_name)
except openshift_exceptions.NotFoundError:
return False
except openshift_exceptions as Error:
raise OCPException(Error)
return True
import unittest
from unittest import mock
from openshift.dynamic import DynamicClient
from awscertmanager import ocp
class TestCertificateExists(unittest.TestCase):
def test_certificate_exists(self):
mock_client = mock.create_autospec(DynamicClient)
ocp.certificate_exists(mock_client, namespace='dummy', certificate_name='tls-wildcard-dummy')
mock_client.resources.get.assert_called_with(
api_version=ocp._openshift_cert_manager_api_version,
kind='Certificate'
)
if __name__ == '__main__':
unittest.main()
This test works fine with the first call but I've tried to know if the second call (v1_certificates.get) is called with no success.
I've tried mocking Resource class but I get this error.
import unittest
from unittest import mock
from openshift.dynamic import DynamicClient, Resource
from awscertmanager import ocp
class TestCertificateExists(unittest.TestCase):
#mock.patch.object(Resource, 'get')
def test_certificate_exists(self, mock_get):
mock_client = mock.create_autospec(DynamicClient)
ocp.certificate_exists(mock_client, namespace='dummy', certificate_name='tls-wildcard-dummy')
mock_client.resources.get.assert_called_with(
api_version=ocp._openshift_cert_manager_api_version,
kind='Certificate'
)
mock_get.assert_called_with(namespace='dummy', name='tls-wildcard-dummy')
if __name__ == '__main__':
unittest.main()
Error
Traceback (most recent call last):
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/unittest/case.py", line 60, in testPartExecutor
yield
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/unittest/case.py", line 676, in run
self._callTestMethod(testMethod)
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/unittest/case.py", line 633, in _callTestMethod
method()
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/unittest/mock.py", line 1322, in patched
with self.decoration_helper(patched,
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/unittest/mock.py", line 1304, in decoration_helper
arg = exit_stack.enter_context(patching)
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/contextlib.py", line 425, in enter_context
result = _cm_type.__enter__(cm)
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/unittest/mock.py", line 1393, in __enter__
original, local = self.get_original()
File "/usr/local/Cellar/python#3.8/3.8.6_2/Frameworks/Python.framework/Versions/3.8/lib/python3.8/unittest/mock.py", line 1366, in get_original
raise AttributeError(
AttributeError: <class 'openshift.dynamic.resource.Resource'> does not have the attribute 'get'
Finally it's working mocking DynamicClient class and some calls:
#mock.patch('awscertmanager.ocp.DynamicClient')
def test_certificate_exists_true(self, mock_dynamicclient):
mock_resource = mock.Mock()
mock_resource.get.return_value = None
mock_dynamicclient.resources.get.return_value = mock_resource
result = ocp.certificate_exists(mock_dynamicclient, namespace=self.namespace, certificate_name=self.certificate_name)
mock_resource.get.assert_called_with(namespace=self.namespace, name=self.certificate_name)
self.assertEqual(result, True)
I want to build a simple server that dispatches requests depending on their path. So I wrote the following code:
from http.server import HTTPServer, BaseHTTPRequestHandler
import json
import socket
class Handler:
"""处理对应资源的请求"""
def __init__(self, dispatcher):
self.dispatcher = dispatcher
def handle(self):
"""根据dispatcher解析出来的path，调用对应的方法"""
command = 'do_' + self.dispatcher.command
print(command)
if not hasattr(self, command):
return;
method = getattr(self, command)
method(self)
def do_GET(self):
response = {
'message': 'message'
}
dispatcher.protocol_version = 'HTTP/1.1'
dispatcher.send_response(200, 'OK')
dispatcher.send_header('Content-type', 'text/plain')
dispatcher.wfile.write(bytes(json.dumps(response), 'utf-8'))
class dispatcher(BaseHTTPRequestHandler):
"""
根据url的不同分发请求
"""
def handle_one_request(self):
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(HTTPStatus.REQUEST_URI_TOO_LONG)
return
if not self.raw_requestline:
self.close_connection = True
return
if not self.parse_request():
# An error code has been sent, just exit
return
print(self.command)
print(self.path)
if self.path.startswith('/wrong'):
Handler(self).handle()
self.wfile.flush() #actually send the response if not already done.
except socket.timeout as e:
#a read or a write timed out. Discard this connection
self.log_error("Request timed out: %r", e)
self.close_connection = True
return
if __name__ == '__main__':
server = HTTPServer(('', 8080), dispatcher)
server.serve_forever()
The dispatcher will parse the incoming request and get the path so that it can decide which handler to call(though there is only one here for now).
In the handler class, it will call corresponding method based on the http method. In the do_GET method, it will call some methods in the dispatcher and that's where things go wrong.
When I ran this program and execute curl http://localhost:8080/wrong, I had the following exception:
GET
/wrong
do_GET
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 50072)
Traceback (most recent call last):
File "/usr/lib/python3.5/socketserver.py", line 313, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python3.5/socketserver.py", line 341, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python3.5/socketserver.py", line 354, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python3.5/socketserver.py", line 681, in __init__
self.handle()
File "/usr/lib/python3.5/http/server.py", line 422, in handle
self.handle_one_request()
File "hello.py", line 51, in handle_one_request
Handler(self).handle()
File "hello.py", line 18, in handle
method()
File "hello.py", line 25, in do_GET
dispatcher.send_response(200, 'OK')
File "/usr/lib/python3.5/http/server.py", line 487, in send_response
self.log_request(code)
AttributeError: 'int' object has no attribute 'log_request'
----------------------------------------
log_request is defined as follows in the super class of dispatcher:
def log_request(self, code='-', size='-'):
# blablabla
and it is called in dispatcher.send_response.
I guess the problem is that the 200 is treated as self by the interpreter.
If my guess is right, why is this happening?
If not, what causes this exception?
I know this question is still in the grammar level, but I'd appreciate it if someone can help me.
I'm inheriting ReconnectingClientFactory and use SSL connection. We have it all running, but I am getting the following error after a few minute. Once this error happend, the connection is lost
Unhandled Error
Traceback (most recent call last):
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/python/log.py", line 103, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/python/log.py", line 86, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
--- <exception caught here> ---
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/internet/posixbase.py", line 597, in _doReadOrWrite
why = selectable.doRead()
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/internet/tcp.py", line 208, in doRead
return self._dataReceived(data)
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/internet/tcp.py", line 214, in _dataReceived
rval = self.protocol.dataReceived(data)
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/protocols/tls.py", line 430, in dataReceived
self._flushReceiveBIO()
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/protocols/tls.py", line 400, in _flushReceiveBIO
self._flushSendBIO()
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/twisted/protocols/tls.py", line 352, in _flushSendBIO
bytes = self._tlsConnection.bio_read(2 ** 15)
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/OpenSSL/SSL.py", line 1384, in bio_read
self._handle_bio_errors(self._from_ssl, result)
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/OpenSSL/SSL.py", line 1365, in _handle_bio_errors
_raise_current_error()
File "/usr/local/hikvision/ezops/python/lib/python2.7/site-packages/OpenSSL/_util.py", line 48, in exception_from_error_queue
raise exception_type(errors)
OpenSSL.SSL.Error: []
my client code like this:
reactor.connectSSL(opscenter_addr, int(opscenter_port), NoahAgentFactory(), ssl.ClientContextFactory())
class NoahAgentFactory like this:
import logging
import traceback
from OpenSSL import SSL
from twisted.internet import reactor,ssl
from twisted.internet import task
from twisted.internet.protocol import Protocol
from twisted.internet.protocol import ReconnectingClientFactory
import DataPackageCodecFactory as Codec
from Header import MessageHandlerMap
from Handler import HBReq, DefaultRspHandler
from Util import Connection
logger = logging.getLogger('opsagent')
class NoahAgentProtocol(Protocol):
t = None
def connectionMade(self):
'''
客户端连接成功之后会自动调用该方法
:return:
'''
self.transport.setTcpKeepAlive(True) # maintain the TCP connection
self.transport.setTcpNoDelay(True) # allow Nagle algorithm
# 连接成功后保存连接信息
Connection.AgentTcpConnection = self
global t
logger.info('is already connect to the server')
self.recv_data_buffer = ''
# 创建定时的心跳任务,每隔30秒执行一次
t = task.LoopingCall(HBReq.execute, *[self])
t.start(30)
def dataReceived(self, data):
logger.debug("Received Message: %s", repr(data))
###code handler packages##########
def connectionLost(self, reason):
'''
当客户端连接断开的时候,会自动调用该方法
:param reason:
:return:
'''
try:
t.stop()
# 清空全局链接信息
Connection.AgentTcpConnection = None
except:
logger.error(traceback.format_exc())
logger.info('Connection is lost and Task stopped,Resaon =>%s', reason)
class NoahAgentFactory(ReconnectingClientFactory):
def startedConnecting(self, connector):
logger.info('Started to connect.')
def buildProtocol(self, addr):
logger.info('Connected.')
logger.info('Resetting reconnection delay')
self.resetDelay()
return NoahAgentProtocol()
def clientConnectionLost(self, connector, reason):
logger.info('Lost connection. Reason:%s', reason)
self.resetDelay()
self.retry(connector)
# ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
logger.info('Connection failed. Reason:%s', reason)
self.resetDelay()
self.retry(connector)
I think it is not a bug in Twisted's TLS other than a mistake,I send the message not in the main reactor thread,as we know,twisted is single thread mode,so there is no need to consider thread safety issues.But I use reactor.callInThread to deal long time cost bussiness and send message directly,and the write() is not thread safety,so invoke that error,once I use reactor.callFromThread on send message,it's running right.
I have this code snippet which I am using to establish a connection with a client and then writing all his text to a file. Here is the code
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor
import ctypes # An included library with Python install.
class DataTransfer(Protocol):
def connectionMade(self):
#self.transport.write("""connected""")
fwrite = open('/Applications/mamp/htdocs/datatest.txt','r+')
self.factory.clients.append(self)
print "clients are ", self.factory.clients
self.username = ""
self.password = ""
self.auth = False
self.ipaddress = self.transport.getPeer()
print self.ipaddress
def connectionLost(self, reason):
fwrite.close()
self.factory.clients.remove(self)
print reason
def dataReceived(self, data):
print data
fwrite.write(data)
a = data.split(':')
if len(a) > 1:
command = a[0]
content = a[1]
msg = ""
self.message(msg)
def message(self, message):
self.transport.write(message + '\n')
factory = Factory()
factory.protocol = DataTransfer
factory.clients = []
reactor.listenTCP(8889, factory)
print "Server started"
reactor.run()
I am executing this command pythonw socketListner.py. The server starts successfully. But when I connect to it using netcat command, I get the following in my terminal and the connection closes.
Unhandled Error
Traceback (most recent call last):
File "socketListner.py", line 42, in <module>
reactor.run()
File "/Library/Python/2.7/site-packages/twisted/internet/base.py", line 1194, in run
self.mainLoop()
File "/Library/Python/2.7/site-packages/twisted/internet/base.py", line 1206, in mainLoop
self.doIteration(t)
File "/Library/Python/2.7/site-packages/twisted/internet/selectreactor.py", line 143, in doSelect
_logrun(selectable, _drdw, selectable, method)
--- <exception caught here> ---
File "/Library/Python/2.7/site-packages/twisted/python/log.py", line 101, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/Library/Python/2.7/site-packages/twisted/python/log.py", line 84, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/Library/Python/2.7/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/Library/Python/2.7/site-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/Library/Python/2.7/site-packages/twisted/internet/selectreactor.py", line 154, in _doReadOrWrite
self._disconnectSelectable(selectable, why, method=="doRead")
File "/Library/Python/2.7/site-packages/twisted/internet/posixbase.py", line 258, in _disconnectSelectable
selectable.connectionLost(failure.Failure(why))
File "/Library/Python/2.7/site-packages/twisted/internet/tcp.py", line 293, in connectionLost
protocol.connectionLost(reason)
File "socketListner.py", line 17, in connectionLost
fwrite.close()
exceptions.NameError: global name 'fwrite' is not defined
Your variable fwrite is defined locally in connectionMade. Try to make it an instance variable by defining it as self.fwrite everywhere you use it within your class.
I'm trying to make this simple server script where multiple clients connect to it and are stored in the factory.connected_clients dictionary. I then want to show a menu so the user executing the server can select what client does he want to work with and what commands does he want to send them (like in a reverse shell). Here's the code:
class RshServer(protocol.Protocol):
def __init__(self, factory, addr):
self.addr = addr
self.factory = factory
def connectionMade(self):
address = self.addr.host + ':' + str(self.addr.port)
self.factory.connected_clients[address] = self
print '[*] Connection made with ' + address
class RshFactory(Factory):
def __init__(self):
self.connected_clients = {}
def buildProtocol(self, addr):
return RshServer(self, addr)
def show_server_menu():
print '1. Show connected clients'
print '2. Open client interface'
msg = raw_input('Select an option: ')
return msg
def show_client_menu():
print '1. Show all processes'
msg = raw_input('Select an option: ')
return msg
def server_interface():
while 1:
msg = show_server_menu()
command = server_commands.get(msg, None)
if command: command()
def client_interface():
protocol_name = raw_input('Enter client address: ')
protoc = rsh_factory.connected_clients.get(protocol_name, None)
if protoc:
msg = show_client_menu()
protoc.transport.write(msg)
rsh_factory = RshFactory()
server_commands = {
'1' : lambda: sys.stdout.write(str(rsh_factory.connected_clients)+'\n'),
'2' : client_interface
}
reactor.listenTCP(8000, rsh_factory)
reactor.callInThread(server_interface)
reactor.run()
However, when I select a client and execute the client interface function I get this error:
Unhandled Error
Traceback (most recent call last):
File "rsh_twisted_serv.py", line 58, in <module>
reactor.run()
File "/usr/lib/python2.7/dist-packages/twisted/internet/base.py", line 1192, in run
self.mainLoop()
File "/usr/lib/python2.7/dist-packages/twisted/internet/base.py", line 1204, in mainLoop
self.doIteration(t)
File "/usr/lib/python2.7/dist-packages/twisted/internet/epollreactor.py", line 396, in doPoll
log.callWithLogger(selectable, _drdw, selectable, fd, event)
--- <exception caught here> ---
File "/usr/lib/python2.7/dist-packages/twisted/python/log.py", line 88, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/usr/lib/python2.7/dist-packages/twisted/python/log.py", line 73, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/lib/python2.7/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python2.7/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/usr/lib/python2.7/dist-packages/twisted/internet/posixbase.py", line 627, in _doReadOrWrite
self._disconnectSelectable(selectable, why, inRead)
File "/usr/lib/python2.7/dist-packages/twisted/internet/posixbase.py", line 257, in _disconnectSelectable
selectable.readConnectionLost(f)
File "/usr/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 279, in readConnectionLost
self.connectionLost(reason)
File "/usr/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 293, in connectionLost
abstract.FileDescriptor.connectionLost(self, reason)
File "/usr/lib/python2.7/dist-packages/twisted/internet/abstract.py", line 207, in connectionLost
self.stopWriting()
File "/usr/lib/python2.7/dist-packages/twisted/internet/abstract.py", line 429, in stopWriting
self.reactor.removeWriter(self)
File "/usr/lib/python2.7/dist-packages/twisted/internet/epollreactor.py", line 344, in removeWriter
EPOLLOUT, EPOLLIN)
File "/usr/lib/python2.7/dist-packages/twisted/internet/epollreactor.py", line 321, in _remove
self._poller.unregister(fd)
exceptions.IOError: [Errno 2] No such file or directory
I believe this is error is due to the transport.write() being called from a non-reactor thread. Am I right? How should I solve this?
SOLUTION: as #Glyph stated, the problem was indeed that I was calling transport.write() from a non-reactor thread (more details in his answer). I used the solution he proposed by changing protoc.transport.write() to reactor.callFromThread(lambda: protoc.transport.write(msg))
I think you mean "called from a non-reactor thread". No code in Twisted is thread-safe unless explicitly specified. See the threading documentation for more information. Use reactor.callFromThread to invoke Twisted methods from your stdio-reading thread, or use StandardIO to tell Twisted to read from standard input instead.