Develop Reference

How do I send input from python cmd to autobahn websocket client running in the same interpreter?

I'm trying to take input from an interactive prompt via python's cmd library and pass the input to an autobahn websocket client to send up to a websocket server. The cmd loop and the autobahn websocket client loop are running in the same interpreter. I'm trying to use crochet to make that work. The websocket client connects to the server successfully but when I enter something at the cmd prompt to invoke sendMessage, I get the exception shown at the bottom of this post. Any guidance on where I may have messed up would be much appreciated. If there's a better approach to accomplish what I'm trying to do, I'm all ears.
These are the relevant imports and setup:
from cmd import Cmd
from crochet import setup, run_in_reactor, wait_for, retrieve_result, TimeoutError
# Setup crochet before importing twisted
setup()
from twisted.internet import reactor, ssl
from twisted.python import log
from autobahn.twisted.websocket import WebSocketClientFactory, \
WebSocketClientProtocol, \
connectWS
This is the websocket client protocol class:
class MyClientProtocol(WebSocketClientProtocol):
def __init__(self, *args, **kwargs):
super(MyClientProtocol, self).__init__(*args, **kwargs)
def onConnect(self, response):
print("Connected")
def onMessage(self, payload, isBinary):
if not isBinary:
print('Message received: {}'.format(payload.decode('utf8')))
def sendTask(self, payload):
payload = json.dumps(payload, ensure_ascii = False).encode('utf8')
self.sendMessage(payload)
This is the websocket client factory class:
class MyClientFactory(WebSocketClientFactory):
def __init__(self, *args, **kwargs):
super(MyClientFactory, self).__init__(*args, **kwargs)
def buildFactory(self, uri, headers):
factory = WebSocketClientFactory(uri, headers=headers)
factory.protocol = MyClientProtocol
return factory
This is the cmd class that sends input to the websocket client:
class mycmd(Cmd):
def do_send(self, inp):
payload = {'task': inp}
m = MyClientProtocol()
reactor.callFromThread(m.sendTask, payload)
This is how I'm calling the websocket client and cmd loop:
if __name__ == '__main__':
#run_in_reactor
def start_connectWS():
headers = {'header1': 'value1'}
f = MyClientFactory()
connectStatement = f.buildFactory(uri, headers)
if connectStatement.isSecure:
contextFactory = ssl.ClientContextFactory()
else:
contextFactory = None
connectWS(connectStatement, contextFactory)
start_connectWS()
mycmd().cmdloop()
This is the exception:
Unhandled Error
Traceback (most recent call last):
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.7/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/Users/tomd/project/lib/python3.7/site-packages/crochet/_eventloop.py", line 412, in <lambda>
target=lambda: self._reactor.run(installSignalHandlers=False),
File "/Users/tomd/project/lib/python3.7/site-packages/twisted/internet/base.py", line 1283, in run
self.mainLoop()
File "/Users/tomd/project/lib/python3.7/site-packages/twisted/internet/base.py", line 1292, in mainLoop
self.runUntilCurrent()
--- <exception caught here> ---
File "/Users/tomd/project/lib/python3.7/site-packages/twisted/internet/base.py", line 886, in runUntilCurrent
f(*a, **kw)
File "./client.py", line 62, in sendTask
self.sendMessage(payload)
File "/Users/tomd/project/lib/python3.7/site-packages/autobahn/websocket/protocol.py", line 2215, in sendMessage
if self.state != WebSocketProtocol.STATE_OPEN:
builtins.AttributeError: 'MyClientProtocol' object has no attribute 'state'

Your copmmand class creates a new, unconnected protocol instance and then tries to use it as if it were connected:
class mycmd(Cmd):
def do_send(self, inp):
payload = {'task': inp}
m = MyClientProtocol()
reactor.callFromThread(m.sendTask, payload)
Specifically, this creates a new instance of your protocol class:
m = MyClientProtocol()
And this tries to use it as if it were connected:
reactor.callFromThread(m.sendTask, payload)
Later on you have code that actually connects a protocol to something:
connectWS(connectStatement, contextFactory)
However this code is not connected to your command class in any useful way.
Instead of creating a new MyClientProtocol instance you need to use the connection that results from calling connectWS.
There are many ways you could go about this which different trade-offs. One way which happens to be easy to explain is to use mutable state shared between the websocket code and the command interpreter code.
For example, MyClientProtocol.onConnect can set itself as an attribute on the factory instance and your command line code could accept the factory instance as an argument, then read the connected protocol instance from the attribute.
class MyClientProtocol(...):
def onConnect(self, response):
self.factory.connectedProtocol = self
...
class mycmd(Cmd):
# ... __init__ that accepts factory and sets it on self
def do_send(self, inp):
payload = {'task': inp}
m = self.factory.connectedProtocol
if m is None:
print("No connection")
else:
reactor.callFromThread(m.sendTask, payload)

TypeError: __init__() missing 3 required positional arguments

I'm building a class to handle HTTP GET requests using http.server, this is what i wrote so far:
class webServerHandler(BaseHTTPRequestHandler):
__HOST = "localhost"
__PORT = 8080
# Custom GET response
def do_GET(self):
page_content = self.htmlHandler()
self.wfile.write(page_content) # Send web page
# HTML code
def htmlHandler(self):
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
msg = '''
<html><head><title>Test</title></head>
<body><h1><center>Test</center></h1></body>
</html>
'''
return bytes(msg, "UTF-8") # UTF-8 Format
# Run the server
def serverStart(self):
# init HTTP Daemon
http_daemon = HTTPServer((self.__HOST, self.__PORT), webServerHandler)
http_daemon.serve_forever()
print("Info: Server started")
and i execute it with:
server = webServerHandler()
server.serverStart()
When i try to execute it it give me this error:
TypeError: __init__() missing 3 required positional arguments: 'request', 'client_address', and 'server'
What i'm doing wrong?

This error is happening because the BaseHTTPRequestHandler has 3 required arguments, and thus your webServerHandler class, which extends BaseHTTPRequestHandler and doesn't override the constructor also has those same required arguments.
You are calling webServerHandler() with no arguments, when you need to call webServerHandler with request, client_address, and server arguments.

Let's follow the MRO:
In [351]: http.server.BaseHTTPRequestHandler.__mro__
Out[351]:
(http.server.BaseHTTPRequestHandler,
socketserver.StreamRequestHandler,
socketserver.BaseRequestHandler,
object)
and the __init__ is defined in socketserver.BaseRequestHandler:
def __init__(self, request, client_address, server):
self.request = request
self.client_address = client_address
self.server = server
self.setup()
try:
self.handle()
finally:
self.finish()
As you can see you need to provide the needed 3 positional arguments (request, client_address, server) to instantiate the instance as mentioned in the exception.

Threaded UDP server bug

I've broken this code somehow and I can't fix it. The server/client code was written by someone else (mostly from the examples in py manuals), and I can't work out what's wrong.
I'm getting issues with super and init and that jazz, mostly because I don't fully understand and find most documentation on the subject leaves me more confused than when I started. For now, I'll be happy enough to get it working. It's likely to be some silly issue, fixed in one line.
Any ideas? I've tried not to paste in code not relevant, but I can add more or provide the whole file if it helps. The code falls over specifically when a handle thread is created. My test case is running code instances, and passing messages between them and it falls over on receipt of the first UDP message.
# Library imports
import threading
import SocketServer
import multiprocessing
# .. More code here ...
class ThreadedUDPServer(SocketServer.ThreadingMixIn, SocketServer.UDPServer):
pass
class NodeDaemon(ThreadedUDPServer):
def __init__(self, host, port, modules):
ThreadedUDPServer.__init__(self, (host, port), NodeProtocolHandler)
# Store parameters in the class
self.modules = modules
self.port = port
self.ip = host
# Check if we have enabled multithreaded listen daemon
if settings.MULTI:
self.server_thread = multiprocessing.Process(target=self.serve_forever)
else:
self.server_thread = threading.Thread(target=self.serve_forever)
# Make the server thread daemonic
self.server_thread.daemon = True
# Start the server thread
self.server_thread.start()
# Update our identity node info
self.modules.identity = NodeInfo(host, port)
def fetch_modules(self):
return self.modules
class NodeProtocolHandler(SocketServer.BaseRequestHandler):
"""
Handles nody things.
Check https://docs.python.org/2/library/socketserver.html
For more sweet deets.
"""
def __init__(self,*args,**kwargs):
super(SocketServer.BaseRequestHandler,self).__init__(args,kwargs)
# Grab modules references
self.modules = self.server.fetch_modules()
# ... More code here .. #
def handle(self):
"""
Main routine to handle all incoming UDP packets.
"""
# Grab the raw message data received
raw_data = self.request[0].strip()
# ... More code here ...
The error generated is:
Exception happened during processing of request from ('127.0.0.1', 60377)
----------------------------------------
Traceback (most recent call last):
File "C:\Python27\lib\SocketServer.py", line 593, in process_request_thread
self.finish_request(request, client_address)
File "C:\Python27\lib\SocketServer.py", line 334, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "C:\some_dir\node_daemon.py", line 60, in __init__
super(SocketServer.BaseRequestHandler,self).__init__(args,kwargs)
TypeError: must be type, not classobj

def __init__(self,*args,**kwargs):
- super(SocketServer.BaseRequestHandler,self).__init__(args, kwargs)
+ SocketServer.BaseRequestHandler.__init__(self,*args, **kwargs)

Python ForkingTCPServer error

Help to build python ForkingTCPServer. This server returns an error when client connects to the server. When I use ThreadingTCPServer instead of ForkingTCPServer it works fine. socketFileIO module contains functions for pickle to/from socket:
http://code.activestate.com/recipes/577667-pickle-tofrom-socket/
server.py:
import SocketServer, time
from socketFileIO import write, read
class MyClientHandler(SocketServer.BaseRequestHandler):
def handle(self):
print self.client_address
data=read(self.request)
print 'Client:', data
time.sleep(10) # for testing
write(self.request,data)
self.request.close()
HOST, PORT = '', 50007
server = SocketServer.ForkingTCPServer((HOST, PORT), MyClientHandler)
server.serve_forever()
client.py:
import socket
from socketFileIO import write, read
HOST = '127.0.0.1'
PORT = 50007
data=["A","B","C","End"]
for x in data:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
write(s,x)
y=read(s)
print 'Server:', y
s.close()
Error:
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 1469)
Traceback (most recent call last):
File "c:\Python26\lib\SocketServer.py", line 283, in _handle_request_noblock
self.process_request(request, client_address)
File "c:\Python26\lib\SocketServer.py", line 525, in process_request
pid = os.fork()
AttributeError: 'module' object has no attribute 'fork'
----------------------------------------
I have a several other questions:
How to shutdown the server by sending data "End" from client ?
How to restrict the number of connections (for example no more then 5) ?

Your fork question was already answered in comments.
For "how to shut down the server": in a similar application (but much older Python version and using xmlrpc), I used a loop invoking one request at a time until a "stop" variable was set. This appears to be supported directly in Python 2.7 now: in your threaded server, call shutdown() on the server instance, from one of the request-handler threads. (Not sure if this is available in 2.6, I de-installed my old 2.6 version a couple of months ago.) I.e.:
class MyClientHandler(SocketServer.BaseRequestHandler):
def handle(self):
...
# if we're supposed to shut down the server, do that
if somecond:
self.server.shutdown()
For limiting the number of connections: there's nothing built-in that I can see, but if you redefined the process_request handler you could count the number of active threads and wait for one to finish if there are "too many" outstanding (or work it however else you like). See the ThreadingMixIn class in SocketServer.py and compare it to the max_children code in ForkingMixIn.

Python Sharing a network socket with multiprocessing.Manager

I am currently writing a nginx proxy server module with a Request queue in front, so the requests are not dropped when the servers behind the nginx can't handle the requests (nginx is configured as a load balancer).
I am using
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
The idea is to put the request in a queue before handling them. I know multiprocessing.Queue supports only simple object and cannot support raw sockets, so I tried using a multiprocess.Manager to make a shared dictionary. The Manager also uses sockets for connection, so this method failed too. Is there a way to share network sockets between processes?
Here is the problematic part of the code:
class ProxyServer(Threader, HTTPServer):
def __init__(self, server_address, bind_and_activate=True):
HTTPServer.__init__(self, server_address, ProxyHandler,
bind_and_activate)
self.manager = multiprocessing.Manager()
self.conn_dict = self.manager.dict()
self.ticket_queue = multiprocessing.Queue(maxsize= 10)
self._processes = []
self.add_worker(5)
def process_request(self, request, client):
stamp = time.time()
print "We are processing"
self.conn_dict[stamp] = (request, client) # the program crashes here
#Exception happened during processing of request from ('172.28.192.34', 49294)
#Traceback (most recent call last):
# File "/usr/lib64/python2.6/SocketServer.py", line 281, in _handle_request_noblock
# self.process_request(request, client_address)
# File "./nxproxy.py", line 157, in process_request
# self.conn_dict[stamp] = (request, client)
# File "<string>", line 2, in __setitem__
# File "/usr/lib64/python2.6/multiprocessing/managers.py", line 725, in _callmethod
# conn.send((self._id, methodname, args, kwds))
#TypeError: expected string or Unicode object, NoneType found
self.ticket_queue.put(stamp)
def add_worker(self, number_of_workers):
for worker in range(number_of_workers):
print "Starting worker %d" % worker
proc = multiprocessing.Process(target=self._worker, args = (self.conn_dict,))
self._processes.append(proc)
proc.start()
def _worker(self, conn_dict):
while 1:
ticket = self.ticket_queue.get()
print conn_dict
a=0
while a==0:
try:
request, client = conn_dict[ticket]
a=1
except Exception:
pass
print "We are threading!"
self.threader(request, client)

U can use multiprocessing.reduction to transfer the connection and socket objects between processes
Example Code
# Main process
from multiprocessing.reduction import reduce_handle
h = reduce_handle(client_socket.fileno())
pipe_to_worker.send(h)
# Worker process
from multiprocessing.reduction import rebuild_handle
h = pipe.recv()
fd = rebuild_handle(h)
client_socket = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
client_socket.send("hello from the worker process\r\n")

Looks like you need to pass file descriptors between processes (assuming Unix here, no clue about Windows). I've never done this in Python, but here is link to python-passfd project that you might want to check.

You can look at this code - https://gist.github.com/sunilmallya/4662837 which is
multiprocessing.reduction socket server with parent processing passing connections to client after accepting connections