Develop Reference

Mocked method socket.accept() is not called

I am trying to mock server side of socket. This example is not working and I don't know why (it still uses original socket.socket.accept() method).
Any ideas how to mock listening socket correctly?
class MySocket:
def __init__:
self.socket = socket.socket()
def listen(self):
self.socket.accept() # original socket method
class MockConn:
def recv(self, int):
return b''
def close(self):
pass
def sendall(self, str):
pass
with mock.patch('socket.socket') as mock_socket:
conn = MockConn()
mock_socket.return_value.accept.return_value = (conn, 'localhost')
s = MySocket(None)
s.listen() # this should call accept from the mock

Not sure what you want, but this works:
import socket
from unittest import mock
class MySocket:
def listen(self):
print('Accept result:', socket.socket().accept())
# Accept result: (<__main__.MockConn object at 0x000001E592B36B38>, 'localhost')
class MockConn:
def recv(self, int):
return b''
def close(self):
pass
def sendall(self, str):
pass
with mock.patch('socket.socket') as mock_socket:
conn = MockConn()
mock_socket.return_value.accept.return_value = (conn, 'localhost')
s = MySocket()
s.listen()

With some very minor changes to your current code this works:
from unittest import mock
import socket
class MySocket:
def __init__(self): # init takes self
self.socket = socket.socket()
def listen(self):
return self.socket.accept() # original socket method
class MockConn:
def recv(self, int):
return b''
def close(self):
pass
def sendall(self, str):
pass
with mock.patch('socket.socket') as mock_socket:
conn = MockConn()
mock_socket.return_value.accept.return_value = (conn, 'localhost')
s = MySocket()
print(s.listen()) # this should call accept from the mock
Output:
(<__main__.MockConn object at 0x00000255C5A53B00>, 'localhost')

Python asyncore.loop in client

I've googled many articles, but still without significant information.
My question is about how to stop asyncore.loop() in client side.
My code is the following
import socket, asyncore
class Client(asyncore.dispatcher):
def __init__(self, host, port):
self.buffer = ''
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host, port))
def writable(self):
return len(self.buffer) > 0
def handle_connect(self):
print "handle_connect"
def handle_close(self):
print "handle_close"
self.close()
def handle_write(self):
print "handle_write"
sent = self.send(self.buffer)
self.buffer = self.buffer[sent:]
print sent
def command(self, data):
self.buffer = data
def close(self):
self.close()
mysocket = Client("127.0.0.1", 8888)
mysocket.command("asfas\n")
asyncore.loop()
mysocket.close()
Thanks in advance!

Why is this zmq code not working?

This test doesn't work.
class PrintHandler(MessageHandler):
def handle_message(self, message):
print(message)
class FileHandler(MessageHandler):
def handle_message(self, message):
with open('nana', 'w') as f:
f.write(message)
class SubscribeProcess(Process):
def __init__(self, handler):
super(SubscribeProcess, self).__init__(group=None, target=None, name=None, args=(), kwargs={})
self.handler = handler
def run(self):
self.address = TcpAddress(host='127.0.0.1', port=5555)
subscriber = ZmqSubscriber(ZmqBlockingConnection(address=self.address, bind=False))
subscriber.set_message_handler(self.handler)
print('............')
class TestZmqSubscriber(TestCase):
def test_set_message_handler(self):
address = TcpAddress(host='127.0.0.1', port=5555)
pub_connection = ZmqBlockingConnection(address, bind=True)
publisher = ZmqPublisher(pub_connection)
p = SubscribeProcess(handler=PrintHandler())
p.start()
while True:
publisher.publish('Message number {}'.format(2))
I now that's this is not the unit test actually. But I want to see the received messages in console first. Then I will write proper test.
While this two scripts work perfectly.
connection = ZmqBlockingConnection(TcpAddress(host='127.0.0.1', port=5555), bind=False)
sub = ZmqSubscriber(connection)
sub.set_message_handler(PrintHandler())
address = TcpAddress(host='127.0.0.1', port=5555)
pub_connection = ZmqBlockingConnection(address, bind=True)
publisher = ZmqPublisher(pub_connection)
while True:
publisher.publish('Message number {}'.format(2))
Inside of subscriber.set_message_handler(handler) is actually this
def start_receiving_messages(self, message_handler):
while True:
message_handler.handle_message(self.socket.recv())
And in debugger I see that the code hangs infinitely in socket.recv()
Maybe I'm using multiprocessing wrong?
EDIT1
class ZmqBlockingConnection(Connection):
def start_receiving_messages(self, message_handler):
while True:
message_handler.handle_message(self.socket.recv())
def send_message(self, message):
self.socket.send(message)
def __init__(self, address, bind, hwm=1000):
self.hwm = hwm
self.bind = bind
self.address = address
self.socket = None
def set_hwm(self, hwm):
self.socket.set_hwm(hwm)
def configure(self, socket_type):
self.socket = zmq.Context().socket(socket_type)
if self.bind:
self.socket.bind(str(self.address))
else:
self.socket.connect(str(self.address))
self.set_hwm(self.hwm)

OK, the problem was in
def configure(self, socket_type):
self.socket = zmq.Context().instance().socket(socket_type)
if self.bind:
self.socket.bind(str(self.address))
else:
self.socket.connect(str(self.address))
self.set_hwm(self.hwm)
so instead of using singleton I started to create context instances and now it's working.

How to use ioloops with multithreading in python?

I have problem figuring out how to use zmq with ioloops and multithreading. Whatever I'm doing I have some exceptions.
__author__ = 'michael'
class ZmqLoopRunner(Thread):
def __init__(self, callback):
super(ZmqLoopRunner, self).__init__()
self.loop = IOLoop.current()
self.callback = callback
def run(self):
self.loop.start()
print('loop have been stopped')
self.callback()
def stop(self):
self.loop.stop()
class BaseZmqNode():
__metaclass__ = ABCMeta
def __init__(self, host, port, bind, hwm):
self.node = self.create_node()
self.node.host = host
self.port = port
self.context = zmq.Context().instance()
self.socket = self.create_socket()
if bind:
self.socket.bind(self.build_address(host, port))
else:
self.socket.connect(self.build_address(host, port))
self.set_hwm(hwm)
def set_hwm(self, hwm):
self.socket.set_hwm(hwm)
def send_multipart(self, message):
self.socket.send_multipart(message)
def send_json(self, json):
self.socket.send_json(json)
def create_node(self):
return BaseMessagingNode
def close(self):
self.socket.close()
#staticmethod
def build_address(host, port):
strings = [host, ':', str(port)]
return ''.join(strings)
#abstractmethod
def create_socket(self):
pass
class BaseZmqReceiver(BaseZmqNode):
__metaclass__ = ABCMeta
def __init__(self, host, port, hwm, bind, on_receive_callback):
super(BaseZmqReceiver, self).__init__(host=host, port=port, bind=bind, hwm=hwm)
self.node.on_message_callback = on_receive_callback
self.stream = ZMQStream(self.socket)
self.stream.on_recv(self.on_message_received)
self.runner = ZmqLoopRunner(self.stream.close)
self.runner.start()
def on_message_received(self, message):
return self.node.on_message_callback(message)
def create_node(self):
return ReceivingNode(None, None)
def close(self):
# super(BaseZmqReceiver, self).close()
self.runner.stop()
# self.socket.close()
Here is how my code looks right now. I have exceptions in test 'Address already in use' when running my tests.
Here is stacktrace
Traceback (most recent call last):
File "/opt/leos/code/messaging_system/tests/ZmqTest.py", line 51, in test_send_json
publisher = ZmqPublisher('tcp://*', 6000)
File "/opt/leos/code/messaging_system/zeromq/ZmqPublisher.py", line 11, in __init__
super(ZmqPublisher, self).__init__(host=host, port=port, bind=bind, hwm=hwm)
File "/opt/leos/code/messaging_system/zeromq/BaseZmqNode.py", line 18, in __init__
self.socket.bind(self.build_address(host, port))
File "socket.pyx", line 434, in zmq.backend.cython.socket.Socket.bind (zmq/backend/cython/socket.c:3928)
File "checkrc.pxd", line 21, in zmq.backend.cython.checkrc._check_rc (zmq/backend/cython/socket.c:6058)
ZMQError: Address already in use
class ZmqTest(AbstractMessagingTest):
def setUp(self):
super(ZmqTest, self).setUp()
self.multipart_messages = self.create_multipart_messages(10)
def tearDown(self):
super(ZmqTest, self).tearDown()
def test_request_reply(self):
requester = ZmqReq(host='tcp://localhost', port=6000)
self.request = 'Hello'
self.reply = 'World!'
replier = ZmqRep(host='tcp://*', port=6000, request_processor=self.on_request_received)
self.assertEqual(self.reply, requester.execute(request=self.request))
# requester.close()
replier.close()
requester.close()
def test_address_creation(self):
full_address = "tcp://localhost:5559"
self.assertEqual(full_address, ZmqSubscriber.build_address("tcp://localhost", 5559))
self.assertEqual('tcp://*:6000', ZmqPublisher.build_address("tcp://*", 6000))
def test_publisher_subscriber(self):
publisher = ZmqPublisher('tcp://*', 6000)
subscriber = ZmqSubscriber('tcp://localhost', 6000, self.handle_message)
self.send_messages(publisher, wait=False)
sleep(0.5)
self.assertSequenceEqual(self.test_messages, self.received_messages)
publisher.close()
subscriber.close()
def handle_message(self, message):
self.base_handle_message(message[0])
def test_send_json(self):
publisher = ZmqPublisher('tcp://*', 6000)
subscriber = ZmqSubscriber('tcp://localhost', 6000, self.handle_json_message)
md = {'1' : 1}
publisher.send_json(md)
publisher.close()
subscriber.close()
def create_multipart_messages(self, size):
messages = []
for i in range(size):
messages.append(['Multipart test message', str(i)])
return messages
def send_multipart_messages(self, sender):
for message in self.multipart_messages:
sender.send_multipart(message)
def test_multipart_messages(self):
publisher = ZmqPublisher('tcp://*', 6000)
subscriber = ZmqSubscriber('tcp://localhost', 6000, self.base_handle_message)
self.send_multipart_messages(publisher)
sleep(0.5)
self.assertSequenceEqual(self.multipart_messages, self.received_messages)
publisher.close()
subscriber.close()
def test_push_pull_multipart(self):
ventilator = ZmqPush('tcp://*', 6000)
worker = ZmqPull('tcp://localhost', 6000, self.base_handle_message)
self.send_multipart_messages(ventilator)
sleep(0.5)
self.assertSequenceEqual(self.multipart_messages, self.received_messages)
ventilator.close()
worker.close()
def handle_json_message(self, json):
print(str(json))
def test_push_pull(self):
ventilator = ZmqPush('tcp://*', 6000)
worker = ZmqPull('tcp://localhost', 6000, self.handle_message)
self.send_messages(ventilator, wait=False)
sleep(0.5)
self.assertSequenceEqual(self.test_messages, self.received_messages)
ventilator.close()
worker.close()
def on_request_received(self, message):
if message[0] == self.request:
return self.reply
else:
return 'ERROR'
And I've tried many variants of this code. Like as you can see right now I'm trying to close the stream after the loop.start() method returned. I've tried to close the stream after the stop method and it just don't work.

So you're getting the error that the socket address is already open. This is probably because you have run up the program, an exception was thrown and you haven't closed down the socket.
I'd suggest some try, except and finally blocks:
try:
requester = ZmqReq(host='tcp://localhost', port=6000)
self.request = 'Hello'
self.reply = 'World!'
replier = ZmqRep(host='tcp://*', port=6000, request_processor=self.on_request_received)
self.assertEqual(self.reply, requester.execute(request=self.request))
except Exception as e:
# You can catch exceptions here
pass
finally:
# Once the code completes or exceptions are thrown, clean up
replier.close()
requester.close()

Python socket hangs program

I am working on a simple HTTP server in Python. I am taking bits and pieces from here: http://hg.python.org/cpython/file/3.3/Lib/socketserver.py to see how Python's standard library handles it.
My problem is that as soon as I try to accept requests my program hangs. Here is my code its only 100 lines so I'll just post it directly here.
I have a process() function which is in a loop that loops forever and it's suppose to handle new connections. Inside I have a print statement that only gets printed once.
print('processing') in TCPServer.process()
I have tried threading off process() but I get the same result.
"""."""
import socket
import select
from abc import abstractmethod, ABCMeta
class BaseServer(metaclass=ABCMeta):
def __init__(self, server_address, server_port, RequestHandlerClass):
self._server_address = server_address
self._server_port = server_port
self._RequestHandlerClass = RequestHandlerClass
self._running = False
def serve_forever(self):
self._running = True
while self._running:
self.process()
#abstractmethod
def process(self):
pass
def shutdown(self):
self._running = False
class TCPServer(BaseServer):
def __init__(self,
server_address,
server_port,
RequestHandlerClass,
address_family=socket.AF_INET,
socket_type=socket.SOCK_STREAM,
request_queue_size=1,
bind=True):
super(TCPServer, self).__init__(server_address,
server_port,
RequestHandlerClass)
self._address_family = address_family
self._socket_type = socket_type
self._request_queue_size = request_queue_size
self._socket = socket.socket(self._address_family, self._socket_type)
self._read_list = [self._socket]
if bind:
self.bind()
def bind(self):
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.bind((self._server_address, self._server_port))
self._socket.listen(self._request_queue_size)
def shutdown(self):
super().shutdown()
self._socket.close()
def process(self):
print('processing')
readable, writeable, errored = select.select(self._read_list, [], [])
for socket in readable:
if socket is self._socket:
client_socket, client_address = self._socket.accept()
self._read_list.append(client_socket)
print('connection from: ', client_address)
else:
self._RequestHandlerClass(client_socket)
self._read_list.remove(client_socket)
class BaseRequestHandler(metaclass=ABCMeta):
def __init__(self, client_socket):
self._client_socket = client_socket
self.setup()
try:
self.handle()
finally:
self.finish()
#abstractmethod
def setup(self):
pass
#abstractmethod
def handle(self):
pass
#abstractmethod
def finish(self):
pass
class HTTPRequestHandler(BaseRequestHandler):
def setup(self):
print('REQUEST SETUP')
print(self._client_socket.recv(2048))
def handle(self):
print('REQUEST HANDLE')
def finish(self):
print('REQUEST FINISH')
self._client_socket.close()
if __name__ == '__main__':
tcp_server = TCPServer(server_address='',
server_port=9000,
RequestHandlerClass=HTTPRequestHandler)
tcp_server.serve_forever()

I ran your code but couldn't make it hang. However, there is a fatal error in your process() function where you refer to client_socket in the else: branch, but client_socket is not defined at that point. You probably meant to refer to socket.
I was able to make two connections to the server on port 9000, and get "connection from:" lines for each. As soon as one of those connections sent something, your server would crash for the above reason.