I would like to stay in the std lib if possible. All of my threads run, and the sockets will listen appropriately. I need to be able to gracefully close the sockets, but I can't seem to get back to them to shutdown properly.
Problem is, when I get to server_close() it blows up on me (see exception trace output below).
Thanks for looking!
import sys
import threading
import SocketServer
class EchoRequestHandler(SocketServer.BaseRequestHandler):
def __init__(self, request, client_address, server):
SocketServer.BaseRequestHandler.__init__(self, request, client_address, server)
return
def setup(self):
return SocketServer.BaseRequestHandler.setup(self)
def handle(self):
# Echo the back to the client
data = self.request.recv(1024)
return
def finish(self):
return SocketServer.BaseRequestHandler.finish(self)
class EchoServer(SocketServer.TCPServer):
def __init__(self, server_address, handler_class=EchoRequestHandler):
SocketServer.TCPServer.__init__(self, server_address, handler_class)
return
def server_activate(self):
SocketServer.TCPServer.server_activate(self)
return
def serve_forever(self):
while True:
self.handle_request()
return
def server_close(self):
return SocketServer.TCPServer.server_close(self)
def bind_sockets(port_list):
server_list = []
for port in port_list:
server = EchoServer(("", int(port)), EchoRequestHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.setDaemon(True)
server_thread.start()
server_list.append(server)
print "listening on port", port
return server_list
def main():
port_list = [12345,54321]
active_servers = bind_sockets(port_list)
print "Hit Q to quit or R to reload."
while 1:
key_press = sys.stdin.read(1)
if key_press == "q":
sys.exit(2)
elif key_press == "r":
#We need to close gracefully so we can go on to do other stuff.
print active_servers.__len__()
for server in active_servers:
server.server_close()
print "closed"
if __name__ == "__main__":
main()
The exception stack-trace I get is:
Exception in thread Thread-1: Traceback (most recent call last):
File "~snip~/threading.py", line 810, in __bootstrap_inner self.run()
File "~snip~/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs)
File "kill_thread.py", line 36, in serve_forever self.handle_request()
File "~snip~/SocketServer.py", line 276, in handle_request fd_sets = _eintr_retry(select.select, [self], [], [], timeout)
File "~snip~/SocketServer.py", line 155, in _eintr_retry return func(*args) error: (9, 'Bad file descriptor')
You have two problems:
you need to call shutdown()
You shouldn't override server_forever()
like this:
# don't overrule server_forever, the default implementation
# handles the shutdown()
#def serve_forever(self):
# while True:
# self.handle_request()
# return
def server_close(self):
self.shutdown()
return SocketServer.TCPServer.server_close(self)
Related
I'm trying to implement a simple ui to show network messages. client.connect() returns a string.
However, I get out of space exception and of course no UI updates
allText = ''
class Receive():
def __init__(self, client):
while 1:
try:
text = client.connect()
if not text:
pass
else:
global allText
allText = allText + text + '\n'
except Exception as e: print(e)
class App(thr.Thread):
client = cvClient()
client.login()
client.register()
client.setWorking()
def __init__(self, master):
thr.Thread.__init__(self)
frame = TK.Frame(master)
frame.pack()
self.gettext = tkst.ScrolledText(frame, height=10,width=100)
self.gettext.pack()
self.gettext.insert(TK.END,'Welcome to Chat\n')
self.gettext.configure(state='disabled')
sframe = TK.Frame(frame)
sframe.pack(anchor='w')
self.pro = TK.Label(sframe, text="Client>>");
self.sendtext = TK.Entry(sframe,width=80)
self.sendtext.focus_set()
self.sendtext.bind(sequence="<Return>", func=self.Send)
self.pro.pack(side=TK.LEFT)
self.sendtext.pack(side=TK.LEFT)
def Send(self, args):
self.gettext.configure(state='normal')
text = self.sendtext.get()
if text=="": text=" "
self.gettext.insert(TK.END,'Me >> %s\n'%text)
self.sendtext.delete(0,END)
self.client.send(text)
self.sendtext.focus_set()
self.gettext.configure(state='disabled')
self.gettext.see(TK.END)
def callback(self):
print ("in callback")
self.gettext.configure(state='normal')
global allText
self.gettext.insert(TK.END,'Server >> %s\n'%allText)
self.gettext.configure(state='disabled')
self.gettext.see(TK.END)
root.after(1000, self.callback)
def run(self):
self.callback()
Receive(self.client)
root = TK.Tk()
root.title('Client Chat')
app = App(root).start()
root.mainloop()
This is the error I get
Exception in thread Thread-1:
Traceback (most recent call last):
File "/home/innereye/anaconda2/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run()
File "cvClient.py", line 332, in run
self.callback()
File "cvClient.py", line 325, in callback
self.gettext.configure(state='normal')
File "/home/innereye/anaconda2/lib/python2.7/lib-tk/Tkinter.py", line 1331, in configure
return self._configure('configure', cnf, kw)
File "/home/innereye/anaconda2/lib/python2.7/lib-tk/Tkinter.py", line 1322, in _configure
self.tk.call(_flatten((self._w, cmd)) + self._options(cnf))
TclError: out of stack space (infinite loop?)
Any suggestions? I'm not blocking the UI thread or calling it in an infinite loop, so I don't really understand why this is happening.
I run mosquitto (docker image - on external server)
After some period of time, my mosquitto does not work and show me this error (I do not know why)
A network protocol error occurred when communicating with the broker.
When this mosquitto error occures and I try connect/publish, (python) paho.mqtt throws me this:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
2913, in _thread_main
self.loop_forever(retry_first_connection=True)
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
1578, in loop_forever
rc = self.loop(timeout, max_packets)
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
1072, in loop
rc = self.loop_read(max_packets)
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
1374, in loop_read
rc = self._packet_read()
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
2071, in _packet_read
rc = self._packet_handle()
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
2560, in _packet_handle
return self._handle_publish()
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
2728, in _handle_publish
(topic, packet) = struct.unpack(pack_format, packet) struct.error: bad char in struct format
It looks like my paho.mqtt.client is still connected but even if I restart mosquitto and I use mosquitto_sub (mosquitto error it's gone) I still do not see messages from my program. I need to restart program but it is not satisfying solution.
Could you tell me please, why mosquitto shows me this error and/or how can I catch (python) paho.mqtt socket.error (It's possible?)
I think that this part of code should be enought :)
mqtt_client.py
class MQttClient:
client = None
connected_flag = False
loop_is_working_flag = False
broker_is_online_flag = True
subscribe_command_channel_flag = True
mosquitto_ip = None
def __init__(self, config):
self.config = config
self.logger = Logger(config, self)
def connect(self, ip_address='127.0.0.1', port=1883, keepalive=4):
self.mosquitto_ip = ip_address
self.client = mqtt.Client(self.config.get(REQUIRE_INI_SECTION, DEVICE_ID_INI_PARAMETER), clean_session=False)
self.client.on_connect = self.on_connect
self.client.on_disconnect = self.on_disconnect
try:
self.client.connect(ip_address, port, keepalive=keepalive)
self.client.loop_start()
self.wait_for_connect()
self.loop_is_working_flag = True
except (OSError, ConnectionRefusedError):
self.logger.error(CAN_NOT_CONNECT_WITH_IP_ADDRESS_ERROR.format(ip_address), send_to_mosquitto=False)
def publish_message(self, message, topic, retain=False):
return self.client.publish(self.config.get(REQUIRE_INI_SECTION, BUILDING_NAME_INI_PARAMETER)
+ '/' + topic + '/' +
self.config.get(REQUIRE_INI_SECTION, DEVICE_ID_INI_PARAMETER), message, qos=1,
retain=retain)
def on_connect(self, client, userdata, flags, rc):
if rc is 0:
self.connected_flag = True
self.logger.info(CONNECTED_TO_THE_MOSQUITO_INFO.format(self.mosquitto_ip), send_to_mosquitto=True)
def on_disconnect(self, client, userdata, rc):
if rc is not 0:
self.connected_flag = False
self.logger.error(DISCONNECTED_FROM_THE_MOSQUITO_ERROR.format(
self.mosquitto_ip, str(rc)), send_to_mosquitto=True)
def wait_for_connect(self):
while not self.connected_flag:
pass
...
And usage:
controller.py
class Controller:
def __init__(self):
self.config = configparser.ConfigParser()
self.config.read(CONFIG_FILE_PATH)
self.client = MQttClient(self.config)
self.__connect_to_mosquitto()
def __connect_to_mosquitto(self):
if Network.host_is_available(self.config.get(NETWORK_INI_SECTION, BROKER_IP_INI_PARAMETER)):
if not self.client.loop_is_working_flag:
self.client.connect(ip_address=self.config.get(NETWORK_INI_SECTION, BROKER_IP_INI_PARAMETER))
time.sleep(0.2)
self.client.publish_message(self.json_parser.get_json_welcome_data(), MOSQUITTO_TOPIC_WELCOME_DATA)
else:
self.logger.error(NO_CONNECTION_WITH_BROKER_ERROR, send_to_mosquitto=False)
Network.restart_network_interface_if_there_is_no_connection(
self.config.get(NETWORK_INI_SECTION, ROUTER_IP_INI_PARAMETER))
...
def execute(self):
while self.client.connected_flag and self.client.broker_is_online_flag:
...
if self.client.publish_message(data, MOSQUITTO_TOPIC_DATA)[0] is not 0:
...
self.logger.error(CAN_NOT_SEND_DATA_TO_MOSQUITTO_ERROR)
break
I checked and this method returns 0 (success). But I do not get messages on mosquitto_sub
self.client.publish_message(data, MOSQUITTO_TOPIC_DATA)[0]
We have a db_connection.py module which looks like below.
from os import getenv
from collections import OrderedDict
import asyncpg
from tornado import gen
from util.utils import custom_exception
import time
db_name = getenv("DB_NAME", "XXX")
db_user = getenv("DB_USER", "YYY")
db_password = getenv("DB_PASSWORD", "ZZZ")
db_host = getenv("DB_HOST", "localhost")
db_port = getenv("DB_PORT", "5432")
db_args = dict(user=db_user, password=db_password,
database=db_name, host=db_host, port=db_port)
class db_connection(object):
def __init__(self, **db_args):
self.db_pool = []
self.init(**db_args)
# create a pool ref coroutine where we can get connections from
# needs user, password, database, host, port
#gen.coroutine
def init(self,**db_args):
self.db_pool = yield asyncpg.create_pool(**db_args)
#gen.coroutine
def exit(self):
yield self.db_pool.close()
# run a queer
async def run(self,q):
if not self.db_pool:
self.init() #try again
if not self.db_pool:
raise custom_exception(reason='Database connection error', status_code=500)
async with self.db_pool.acquire() as connection:
ret = await connection.fetch(q)
# format to match the old data types
return [OrderedDict(e) for e in ret]
def __exit__(self, exc_type, exc_val, exc_tb):
self.exit()
self.db_pool = []
In the app, we initialize the db_connection object as follows:
from util.db_connection import db_args, db_connection
(...)
if __name__ == "__main__":
AsyncIOMainLoop().install()
dbc = db_connection(**db_args)
# What we have here is a rather basic tornado app
app = make_app()
app.listen(port=8080)
asyncio.get_event_loop().run_forever()
The problem we saw is that, and this is something I can not explain, it seems that the connection string (db_args) is not always set and sometimes appears to be an empty dictionary; asyncpg then falls back and tries to get a connection string from the linux user associated to the container via os.
Mar 5 16:59:50 ABC: ERROR:
tornado.application:Future <tornado.concurrent.Future object at 0x7eff6005c320>
exception was never retrieved:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
yielded = self.gen.throw(*exc_info)
File "/abc/def/db_connection.py", line 26, in init
self.db_pool = yield asyncpg.create_pool(**db_args)
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
value = future.result()
File "/usr/local/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info) Mar 5 16:59:50 ABC: File "<string>", line 4, in raise_exc_info
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 307, in wrapper
yielded = next(result)
File "<string>", line 6, in _wrap_awaitable
File "/usr/local/lib/python3.6/site-packages/asyncpg/pool.py", line 356, in _async__init__
await first_ch.connect()
File "/usr/local/lib/python3.6/site-packages/asyncpg/pool.py", line 126, in connect
**self._connect_kwargs)
File "/usr/local/lib/python3.6/site-packages/asyncpg/connection.py", line 1498, in connect
max_cacheable_statement_size=max_cacheable_statement_size)
File "/usr/local/lib/python3.6/site-packages/asyncpg/connect_utils.py", line 296, in _connect
addrs, params, config=_parse_connect_arguments(timeout=timeout, **kwargs)
File "/usr/local/lib/python3.6/site-packages/asyncpg/connect_utils.py", line 242, in _parse_connect_arguments
server_settings=server_settings)
File "/usr/local/lib/python3.6/site-packages/asyncpg/connect_utils.py", line 152, in _parse_connect_dsn_and_args
user=getpass.getuser()
File "/usr/local/lib/python3.6/getpass.py", line 169, in getuser
return pwd.getpwuid(os.getuid())[0]
KeyError: 'getpwuid(): uid not found: 10001
I wrote a simple http server with Tornado, but sometimes it raise exception below, self.session.response runs in a threadpool.
code of server:
class AuToServer(object):
module_path = os.path.split(os.path.realpath(__file__))[0] + '/../module/'
def __init__(self, prefix=DEFAULT_PREFIX, work_root=module_path, module=[], **kwargs):
self._prefix = prefix
self._thread_pool = ExecutorDelegate(20)
self._module_manager = ModuleManager(work_root)
self.initilize(module, kwargs)
def initilize(self, modules, kwargs):
self._module_manager.set_args(kwargs)
if not self._module_manager.prepare(modules):
print "initilize module fail. [%s]" % (str(modules))
raise AppInitError("initilize %s fail" % (modules))
def __call__(self, request): # request handler
session = Session(request)
if not session.parse(self._prefix):
print "parse query fail. [%s]" % (session.query)
session.response("url parse fail [%s]\n" % session.query)
return
self._thread_pool.submit(WorkerHander(self._module_manager, session))
def start(self, port=8880):
http_server = tornado.httpserver.HTTPServer(self, max_header_size=128*1024)
http_server.listen(port)
tornado.ioloop.IOLoop.instance().start()
code of WorkerHander:
class WorkerHander(BaseExecutor):
def __init__(self, module_manage, session):
self.module_manage = module_manage
self.session = session
super(WorkerHander, self).__init__(self)
def run(self):
method = self.session.get_command()
if not method:
self.session.response("invalid url [%s]\n" % (self.session.query))
return
context = self.module_manage.run(method, self.session)
try:
if context:
self.session.response(context) # raise exception
else:
self.session.response("None\n")
except Exception, error:
Logger.warning("session response fail. [%s][%s]" % (self.session.query, traceback.format_exc()))
code of session.response
def response(self, context):
if not self.request:
return False
if isinstance(context, unicode):
context = context.encode("utf8")
self.request.write(utf8("%s %d\r\n\r\n%s" % (
HTTP_HEADER,
len(context),
context)))
Logger.debug("QUERY:[cmd:%s][%s] [%s] [%s]" % (
self.command,
time.time() - self.begin_time,
self.request.remote_ip,
self.query))
self.request.finish()
return True
exception:
2016-02-29 16:29:26,852 WARNING: server.py:run:94: session response fail. [/auto?start_timestamp=1456730772&_cmd=get_index_info][Traceback (most recent call last):
File "/home/admin/autobots/lib/python2.7/site-packages/autobase-0.0.1-py2.7.egg/autobase/service/server.py", line 90, in run
self.session.response(context)
File "/home/admin/autobots/lib/python2.7/site-packages/autobase-0.0.1-py2.7.egg/autobase/service/session.py", line 43, in response
self.request.finish()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/httputil.py", line 407, in finish
self.connection.finish()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 459, in finish
self._pending_write.add_done_callback(self._finish_request)
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/concurrent.py", line 243, in add_done_callback
fn(self)
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 491, in _finish_request
self.close()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 297, in close
self.stream.close()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/iostream.py", line 427, in close
self.close_fd()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/iostream.py", line 995, in close_fd
self.socket.close()
AttributeError: 'NoneType' object has no attribute 'close'
]
Do I use Tornado in a wrong way? ask for help. thanks~
Tornado is not thread-safe by design. You can use a thread pool to do your own work, but you must call all Tornado methods (including in this case self.request.write and self.request.finish) from the IOLoop thread.
from twisted.internet.protocol import Protocol,Factory
from twisted.internet import reactor
class ChatServer(Protocol):
def connectionMade(self):
print "A Client Has Connected"
self.factory.clients.append(self)
print"clients are ",self.factory.clients
self.transport.write('Hello,Welcome to the telnet chat to sign in type aim:YOUR NAME HERE to send a messsage type msg:YOURMESSAGE '+'\n')
def connectionLost(self,reason):
self.factory.clients.remove(self)
self.transport.write('Somebody was disconnected from the server')
def dataReceived(self,data):
#print "data is",data
a = data.split(':')
if len(a) > 1:
command = a[0]
content = a[1]
msg=""
if command =="iam":
self.name + "has joined"
elif command == "msg":
ma=sg = self.name + ":" +content
print msg
for c in self.factory.clients:
c.message(msg)
def message(self,message):
self.transport.write(message + '\n')
factory = Factory()
factory.protocol = ChatServer
factory.clients = []
reactor.listenTCP(80,factory)
print "Iphone Chat server started"
reactor.run()
The above code is running succesfully...but when i connect the client (by typing telnet localhost 80) to this chatserver and try to write message ,connection gets lost and following errors occurs :
Iphone Chat server started
A Client Has Connected
clients are [<__main__.ChatServer instance at 0x024AC0A8>]
Unhandled Error
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\twisted\python\log.py", line 84, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "C:\Python27\lib\site-packages\twisted\python\log.py", line 69, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "C:\Python27\lib\site-packages\twisted\python\context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "C:\Python27\lib\site-packages\twisted\python\context.py", line 81, in callWithContext
return func(*args,**kw)
--- ---
File "C:\Python27\lib\site-packages\twisted\internet\selectreactor.py", line 150, in _doReadOrWrite
why = getattr(selectable, method)()
File "C:\Python27\lib\site-packages\twisted\internet\tcp.py", line 199, in doRead
rval = self.protocol.dataReceived(data)
File "D:\chatserverultimate.py", line 21, in dataReceived
content = a[1]
exceptions.IndexError: list index out of range
Where am I going wrong?
You indented content = a[1] line incorrectly. You need more space(s)! :)