I have a long running Python script which collect tweets from Twitter, and I would like to know how its doing every once in awhile.
Currently, I am using the signal library to catch interrupts, at which point I call my print function. Something like this:
import signal
def print_info(count):
print "#Tweets:", count
#Print out the process ID so I can interrupt it for info
print 'PID:', os.getpid()
#Start listening for interrupts
signal.signal(signal.SIGUSR1, functools.partial(print_info, tweet_count))
And whenever I want my info, I open up a new terminal and issue my interrupt:
$kill -USR1 <pid>
Is there a better way to do this? I am aware I could have my script something at scheduled intervals, but I am more interested in knowing on demand, and potentially issuing other commands as well.
Sending a signal to process would interrupt the process. Below you will find an approach that uses dedicated thread to emulate python console. The console is exposed as a unix socket.
import traceback
import importlib
from code import InteractiveConsole
import sys
import socket
import os
import threading
from logging import getLogger
# template used to generate file name
SOCK_FILE_TEMPLATE = '%(dir)s/%(prefix)s-%(pid)d.socket'
log = getLogger(__name__)
class SocketConsole(object):
'''
Ported form :eventlet.backdoor.SocketConsole:.
'''
def __init__(self, locals, conn, banner=None): # pylint: diable=W0622
self.locals = locals
self.desc = _fileobject(conn)
self.banner = banner
self.saved = None
def switch(self):
self.saved = sys.stdin, sys.stderr, sys.stdout
sys.stdin = sys.stdout = sys.stderr = self.desc
def switch_out(self):
sys.stdin, sys.stderr, sys.stdout = self.saved
def finalize(self):
self.desc = None
def _run(self):
try:
console = InteractiveConsole(self.locals)
# __builtins__ may either be the __builtin__ module or
# __builtin__.__dict__ in the latter case typing
# locals() at the backdoor prompt spews out lots of
# useless stuff
import __builtin__
console.locals["__builtins__"] = __builtin__
console.interact(banner=self.banner)
except SystemExit: # raised by quit()
sys.exc_clear()
finally:
self.switch_out()
self.finalize()
class _fileobject(socket._fileobject):
def write(self, data):
self._sock.sendall(data)
def isatty(self):
return True
def flush(self):
pass
def readline(self, *a):
return socket._fileobject.readline(self, *a).replace("\r\n", "\n")
def make_threaded_backdoor(prefix=None):
'''
:return: started daemon thread running :main_loop:
'''
socket_file_name = _get_filename(prefix)
db_thread = threading.Thread(target=main_loop, args=(socket_file_name,))
db_thread.setDaemon(True)
db_thread.start()
return db_thread
def _get_filename(prefix):
return SOCK_FILE_TEMPLATE % {
'dir': '/var/run',
'prefix': prefix,
'pid': os.getpid(),
}
def main_loop(socket_filename):
try:
log.debug('Binding backdoor socket to %s', socket_filename)
check_socket(socket_filename)
sockobj = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sockobj.bind(socket_filename)
sockobj.listen(5)
except Exception, e:
log.exception('Failed to init backdoor socket %s', e)
return
while True:
conn = None
try:
conn, _ = sockobj.accept()
console = SocketConsole(locals=None, conn=conn, banner=None)
console.switch()
console._run()
except IOError:
log.debug('IOError closing connection')
finally:
if conn:
conn.close()
def check_socket(socket_filename):
try:
os.unlink(socket_filename)
except OSError:
if os.path.exists(socket_filename):
raise
Example program:
make_threaded_backdoor(prefix='test')
while True:
pass
Example session:
mmatczuk#cactus:~$ rlwrap nc -U /var/run/test-3196.socket
Python 2.7.6 (default, Mar 22 2014, 22:59:56)
[GCC 4.8.2] on linux2
Type "help", "copyright", "credits" or "license" for more information.
(InteractiveConsole)
>>> import os
>>> os.getpid()
3196
>>> quit()
mmatczuk#cactus:~$
This is a pretty robust tool that can be used to:
dump threads,
inspect process memory,
attach debugger on demand, pydev debugger (work for both eclipse and pycharm),
force GC,
monkeypatch function definition on the fly
and even more.
I personally write information to a file so that I have it afterwards, although this has the disadvantage of perhaps being slightly slower because it has to write to a file every time or every few times it retrieves a tweet.
Anyways, if you write it to a file "output.txt", you can open up bash and either type in tail output.txt for the latest 10 lines printed in the file, or you can type tail -f output.txt, which continuously updates the terminal prompt with the lines that you are writing to the file. If you wish to stop, just Ctrl-C.
Here's an example long-running program that also maintains a status socket. When a client connects to the socket, the script writes some status information to the socket.
#!/usr/bin/python
import os
import sys
import argparse
import random
import threading
import socket
import time
import select
val1 = 0
val2 = 0
lastupdate = 0
quit = False
# This function runs in a separate thread. When a client connects,
# we write out some basic status information, close the client socket,
# and wait for the next connection.
def connection_handler(sock):
global val1, val2, lastupdate, quit
while not quit:
# We use select() with a timeout here so that we are able to catch the
# quit flag in a timely manner.
rlist, wlist, xlist = select.select([sock],[],[], 0.5)
if not rlist:
continue
client, clientaddr = sock.accept()
client.send('%s %s %s\n' % (lastupdate, val1, val2))
client.close()
# This function starts the listener thread.
def start_listener():
sock = socket.socket(socket.AF_UNIX)
try:
os.unlink('/var/tmp/myprog.socket')
except OSError:
pass
sock.bind('/var/tmp/myprog.socket')
sock.listen(5)
t = threading.Thread(
target=connection_handler,
args=(sock,))
t.start()
def main():
global val1, val2, lastupdate
start_listener()
# Here is the part of our script that actually does "work".
while True:
print 'updating...'
lastupdate = time.time()
val1 = val1 + random.randint(1,10)
val2 = val2 + random.randint(100,200)
print 'sleeping...'
time.sleep(5)
if __name__ == '__main__':
try:
main()
except (Exception,KeyboardInterrupt,SystemExit):
quit=True
raise
You could write a simple Python client to connect to the socket, or you could use something like socat:
$ socat - unix:/var/tmp/myprog.sock
1403061693.06 6 152
I had write a similar application before.
Here is what I did:
When there are only a few commands needed, I just use signal as you did, just for not making it too complicated. By command, I mean something that you want you application to do, such as print_info in your post.
But when application updated, there are more different commands needed, I began to use a special thread listening on a socket port or reading a local file for accepting commands. Suppose the application need to support prinf_info1 print_info2 print_info3, so you can use a client connect to the target port and write print_info1 to make the application execute command print_info1 (Or just write print_info1 to a local file if you are using the reading local file mechanism).
When using the listening on a socket port mechanism, the disadvantage is it will take a bit more work to write a client to give commands, the advantage is you can give orders anywhere.
When using the reading a local file mechanism, the disadvantage is you have to make the thread check the file in a loop and it will use a bit resource, the advantage is giving orders is very simple (just write a string to a file) and you don't need to write a client and socket listen server.
rpyc is the perfect tool for this task.
In short, you define a rpyc.Service class which exposes the commands you want to expose, and start an rpyc.Server thread.
Your client then connects to your process, and calls the methods which are mapped to the commands your service exposes.
It's as simple and clean as that. No need to worry about sockets, signals, object serialization.
It has other cool features as well, for example the protocol being symmetric.
Your question relates to interprocess communication. You can achieve this by communicating over a unix socket or TCP port, by using a shared memory, or by using a message queue or cache system such as RabbitMQ and Redis.
This post talks about using mmap to achieve shared memory interprocess communication.
Here's how to get started with redis and RabbitMQ, both are rather simple to implement.
Related
I am trying to use GLib.IOChannels to send data from a client to a server running a Glib.Mainloop.
The file used for the socket should be located at /tmp/so/sock, and the server should simply run a function whenever it receives data.
This is the code I've written:
import sys
import gi
from gi.repository import GLib
ADRESS = '/tmp/so/sock'
def server():
loop = GLib.MainLoop()
with open(ADRESS, 'r') as sock_file:
sock = GLib.IOChannel.unix_new(sock_file.fileno())
GLib.io_add_watch(sock, GLib.IO_IN,
lambda *args: print('received:', args))
loop.run()
def client(argv):
sock_file = open(ADRESS, 'w')
sock = GLib.IOChannel.unix_new(sock_file.fileno())
try:
print(sock.write_chars(' '.join(argv).encode('utf-8'), -1))
except GLib.Error:
raise
finally:
sock.shutdown(True)
# sock_file.close() # calling close breaks the script?
if __name__ == '__main__':
if len(sys.argv) > 1:
client(sys.argv[1:])
else:
server()
When called without arguments, it acts as the server, if called with arguments, it sends them to a running server.
When starting the server, I immediately get the following output:
received: (<GLib.IOChannel object at 0x7fbd72558b80 (GIOChannel at 0x55b8397905c0)>, <flags G_IO_IN of type GLib.IOCondition>)
I don't know why that is. Whenever I send something, I get an output like (<enum G_IO_STATUS_NORMAL of type GLib.IOStatus>, bytes_written=4) on the client side, while nothing happens server-side.
What am I missing? I suspect I understood the documentation wrong, as I did not find a concrete example.
I got the inspiration to use the IOChannel instead of normal sockets from this post: How to listen socket, when app is running in gtk.main()?
I have a Python daemon running on a Linux system.
I would like to feed information such as "Bob", "Alice", etc. and have the daemon print "Hello Bob." and "Hello Alice" to a file.
This has to be asynchronous. The Python daemon has to wait for information and print it whenever it receives something.
What would be the best way to achieve this?
I was thinking about a named pipe or the Queue library but there could be better solutions.
Here is how you can do it with a fifo:
# receiver.py
import os
import sys
import atexit
# Set up the FIFO
thefifo = 'comms.fifo'
os.mkfifo(thefifo)
# Make sure to clean up after ourselves
def cleanup():
os.remove(thefifo)
atexit.register(cleanup)
# Go into reading loop
while True:
with open(thefifo, 'r') as fifo:
for line in fifo:
print "Hello", line.strip()
You can use it like this from a shell session
$ python receiver.py &
$ echo "Alice" >> comms.fifo
Hello Alice
$ echo "Bob" >> comms.fifo
Hello Bob
There are several options
1) If the daemon should accept messages from other systems, make the daemon an RPC server - Use xmlrpc/jsonrpc.
2) If it is all local, you can use either TCP sockets or Named PIPEs.
3) If there will be a huge set of clients connecting concurrently, you can use select.epoll.
python has a built-in rpc library (using xml for data encoding). the documentation is well written; there is a complete example there:
https://docs.python.org/2.7/library/xmlrpclib.html
(python 2.7) or
https://docs.python.org/3.3/library/xmlrpc.server.html#module-xmlrpc.server
(python 3.3)
that may be worth considering.
Everyone mentioned FIFO-s (that's named pipes in Linux terminology) and XML-RPC, but if you learning these things right now, you have to check TCP/UDP/Unix sockets as well, since they are platform independent (at least, TCP/UDP sockets are). You can check this tutorial for a working example or the Python documentation if you want to go deper in this direction. It's also useful since most of the modern communication platforms (XML-RPC, SOAP, REST) uses these basic things.
There are a few mechanisms you could use, but everything boils down to using IPC (inter-process communication).
Now, the actual mechanism you will use depends on the details of what you can achieve, a good solution though would be to use something like zmq.
Check the following example on pub/sub on zmq
http://learning-0mq-with-pyzmq.readthedocs.org/en/latest/pyzmq/patterns/pubsub.html
also this
http://learning-0mq-with-pyzmq.readthedocs.org/en/latest/pyzmq/multisocket/zmqpoller.html
for the non-blocking way.
I'm not good in python so I would like to share
**Universal Inter process communcation **
nc a.k.a netcat is a server client model program which allow to send data such as text,files over network.
Advantages of nc
Very easy to use
IPC even between different programming langauges
Inbuilt on most linux OS
Example
On deamon
nc -l 1234 > output.txt
From other program or shell/terminal/script
echo HELLO | nc 127.0.0.1 1234
nc can be python by using the system command calling function ( may be os.system ) and read the stdout.
Why not use signals?
I am not a python programmer but presumably you can register a signal handler within your daemon and then signal it from the terminal. Just use SIGUSR or SIGHUP or similar.
This is the usual method you use to rotate logfiles or similar.
One solution could be to use the asynchat library which simplify calls between a server and a client.
Here is an example you could use (adapted from this site)
In deamon.py, a ChatServer object is created. Each time a connection is done, a ChatHandler object is created, inherited from asynchat.async_chat. This object collects data and fills it in self.buffer.
When a special string call the terminator is encountered, data is supposed to be complete and method found_terminator is called. It is in this method that you write your own code.
In sender.py, you create a ChatClient object, inherited from asynchat.async_chat, setup the connection in the constructor, define the terminator (in case the server answers !) and call the push method to send your data. You must append your terminator string to your data for the server to know when it can stop reading data.
daemon.py :
import asynchat
import asyncore
import socket
# Terminator string can be changed here
TERMINATOR = '\n'
class ChatHandler(asynchat.async_chat):
def __init__(self, sock):
asynchat.async_chat.__init__(self, sock=sock)
self.set_terminator(TERMINATOR)
self.buffer = []
def collect_incoming_data(self, data):
self.buffer.append(data)
def found_terminator(self):
msg = ''.join(self.buffer)
# Change here what the daemon is supposed to do when a message is retrieved
print 'Hello', msg
self.buffer = []
class ChatServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind((host, port))
self.listen(5)
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
handler = ChatHandler(sock)
server = ChatServer('localhost', 5050)
print 'Serving on localhost:5050'
asyncore.loop()
sender.py :
import asynchat
import asyncore
import socket
import threading
# Terminator string can be changed here
TERMINATOR = '\n'
class ChatClient(asynchat.async_chat):
def __init__(self, host, port):
asynchat.async_chat.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host, port))
self.set_terminator(TERMINATOR)
self.buffer = []
def collect_incoming_data(self, data):
pass
def found_terminator(self):
pass
client = ChatClient('localhost', 5050)
# Data sent from here
client.push("Bob" + TERMINATOR)
client.push("Alice" + TERMINATOR)
I need to check if the python script is already running then calling a method from the same running python script. But it must be on same process(pid), no new process. Is this possible?
I tried some codes but not worked.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import Tkinter as tk
from Tkinter import *
import socket
class Main():
def mainFunc(self):
self.root = tk.Tk()
self.root.title("Main Window")
self.lbl = Label(self.root, text = "First Text")
self.lbl.pack()
openStngs = Button(self.root, text = "Open Settings", command=self.settingsFunc)
openStngs.pack()
def settingsFunc(self):
stngsRoot = Toplevel()
stngsRoot.title("Settings Window")
changeTextOfLabel = Button(stngsRoot, text = "Change Main Window Text", command=self.change_text)
changeTextOfLabel.pack()
def change_text(self):
self.lbl.config(text="Text changed")
# the get_lock from http://stackoverflow.com/a/7758075/3254912
def get_lock(process_name):
lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
try:
print lock_socket
lock_socket.bind('\0' + process_name)
print 'I got the lock'
m.mainFunc()
mainloop()
except socket.error:
print 'lock exists'
m.settingsFunc()
mainloop()
# sys.exit()
if __name__ == '__main__':
m=Main()
get_lock('myPython.py')
You either need:
A proactive check in your running process to look at the environment (for instance, the contents of a file or data coming through a socket) to know when to fire the function,
or for your running process to receive unix signals or some other IPC (possibly one of the user-defined signals) and perform a function when one is received.
Either way you can't just reach into a running process and fire a function inside that process (it MIGHT not be literally impossible if you hook the running process up to a debugger, but I wouldn't recommend it).
Tkinter necessarily has its own event loop system, so I recommend reading up on how that works and how to either run something on a timer in that event loop system, or set up a callback that responds to a signal. You could also wrap a non-event loop based system in a try/except block that will catch an exception generated by a UNIX signal, but it may not be straightforward to resume the operation of the rest of the program after that signal is caught, in that case.
Sockets are a good solution to this kind of interprocess communication problem.
One possible approach would be to set up a socket server in a thread in your original process, this can be used as an entry point for external input. A (rather stupid) example might be:
# main.py
import socket
import SocketServer # socketserver in Python 3+
import time
from Queue import Queue
from threading import Thread
# class for handling requests
class QueueHandler(SocketServer.BaseRequestHandler):
def __init__(self, request, client_address, server):
self.server = server
server.client_address = client_address
SocketServer.BaseRequestHandler.__init__(self,request, client_address, server)
# receive a block of data
# put it in a Queue instance
# send back the block of data (redundant)
def handle(self):
data = self.request.recv(4096)
self.server.recv_q.put(data)
self.request.send(data)
class TCPServer(SocketServer.TCPServer):
def __init__(self, ip, port, handler_class=QueueHandler):
SocketServer.TCPServer.__init__(self, (ip, port), handler_class, bind_and_activate=False)
self.recv_q = Queue() # a Queue for data received over the socket
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_bind()
self.server_activate()
def shutdown(self):
SocketServer.TCPServer.shutdown(self)
def __del__(self):
self.server_close()
# This is the equivalent of the main body of your original code
class TheClassThatLovesToAdd(object):
def __init__(self):
self.value = 1
# create and instance of the server attached to some port
self.server = TCPServer("localhost",9999)
# start it listening in a separate control thread
self.server_thread = Thread(target=self.server.serve_forever)
self.server_thread.start()
self.stop = False
def add_one_to_value(self):
self.value += 1
def run(self):
while not self.stop:
print "Value =",self.value
# if there is stuff in the queue...
while not self.server.recv_q.empty():
# read and parse the message from the queue
msg = self.server.recv_q.get()
# perform some action based on the message
if msg == "add":
self.add_one_to_value()
elif msg == "shutdown":
self.server.shutdown()
self.stop = True
time.sleep(1)
if __name__ == "__main__":
x = TheClassThatLovesToAdd()
x.run()
When you start this running, it should just loop over and over printing to the screen. Output:
Value = 1
Value = 1
Value = 1
...
However the TCPServer instance attached to the TheClassThatLovesToAdd instance now gives us a control path. The simplest looking snippet of control code would be:
# control.py
import socket
import sys
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.settimeout(2)
sock.connect(('localhost',9999))
# send some command line argument through the socket
sock.send(sys.argv[1])
sock.close()
So if I run main.py in one terminal window and call python control.py add from another, the output of main.py will change:
Value = 1
Value = 1
Value = 1
Value = 2
Value = 2
...
Finally to kill it all we can run python control.py shutdown, which will gently bring main.py to a halt.
This is by no means the only solution to your problem, but it is likely to be one of the simplest.
One can try GDB, but not sure how to call a function from within [an idle thread].
Perhaps someone very versed with gdb and debugging/calling Python functions from within GDB can improve this answer.
One solution would be to use a messaging service (such as ActiveMQ or RabbitMQ). Your application subscribes to a queue/topic and whenever you want to send it a command, you write a message to it's queue. I'm not going to go into details because there are thousands of examples on-line. Queues/messaging/MQTT etc. are very simple to implement and are how most business systems (and modern control systems) communicate. Do a search for paho-mqtt.
I try to write a daemon in python. But I have no idea how can I use a thread to start parallel tcp server in this daemon. And even what type of server I should use : asyncore?SocketServer?socket?
this is part of my code:
import os
def demonized():
child_pid = os.fork()
if child_pid == 0:
child_pid = os.fork()
if child_pid == 0: #fork twice for demonize
file = open('###', "r") # open file
event = file.read()
while event:
#TODO check for changes put changes in list variable
event = file.read()
file.close()
else:
sys.exit(0)
else:
sys.exit(0)
if __name__ == "__main__":
demonized()
So in a loop I have a list variable with some data appended every circle, and I want to start a thread with tcp server that wait for connection in the loop and if client connects send it this data(with zeroing variable). So I do not need to handle multiple clients, the client will be only one at time. What is the optimal way to implement this?
Thank you.
In case you want to avoid repeating boilerplate, Python will soon have a standard module that does the fork() pair and standard-I/O manipulations (which you have not added to your program yet?) that make it a daemon. You can download and use this module right now, from:
http://pypi.python.org/pypi/python-daemon
Running a TCP server in a separate thread is often as simple as:
import threading
def my_tcp_server():
sock = socket.socket(...)
sock.bind(...)
sock.listen()
while True:
conn, address = sock.accept()
...
... talk on the connection ...
...
conn.close()
def main():
...
threading.Thread(target=my_tcp_server).start()
...
I strongly recommend against trying to get your file-reader thread and your socket-answering thread talking with a list and lock of your own devising; such schemes are hard to get working and hard to keep working. Instead, use the standard library's Queue.Queue() class which does all of the locking and appending correctly for you.
Do you want to append items to the list in while event:... loop and serving this list simultaneously? If so then you have two writers and you must somehow protect your list.
In the sample SocketServer.TCPServer and threading.Lock was used:
import threading
import SocketServer
import time
class DataHandler(SocketServer.StreamRequestHandler):
def handle(self):
self.server.list_block.acquire()
self.wfile.write(', '.join(self.server.data))
self.wfile.flush()
self.server.data = []
self.server.list_block.release()
if __name__ == '__main__':
data = []
list_block = threading.Lock()
server = SocketServer.TCPServer(('localhost', 0), DataHandler)
server.list_block = list_block
server.data = data
t = threading.Thread(target=server.serve_forever)
t.start()
while True:
list_block.acquire()
data.append(1)
list_block.release()
time.sleep(1)
I'm working on a python script to query a few remote databases over an established ssh tunnel every so often. I'm fairly familiar with the paramiko library, so that was my choice of route. I'd prefer to keep this in complete python so I can use paramiko to deal with key issues, as well as uses python to start, control, and shutdown the ssh tunnels.
There have been a few related questions around here about this topic, but most of them seemed incomplete in answers. My solution below is a hacked together of the solutions I've found so far.
Now for the problem: I'm able to create the first tunnel quite easily (in a separate thread) and do my DB/python stuff, but when attempting to close the tunnel the localhost won't release the local port I binded to. Below, I've included my source and the relevant netstat data through each step of the process.
#!/usr/bin/python
import select
import SocketServer
import sys
import paramiko
from threading import Thread
import time
class ForwardServer(SocketServer.ThreadingTCPServer):
daemon_threads = True
allow_reuse_address = True
class Handler (SocketServer.BaseRequestHandler):
def handle(self):
try:
chan = self.ssh_transport.open_channel('direct-tcpip', (self.chain_host, self.chain_port), self.request.getpeername())
except Exception, e:
print('Incoming request to %s:%d failed: %s' % (self.chain_host, self.chain_port, repr(e)))
return
if chan is None:
print('Incoming request to %s:%d was rejected by the SSH server.' % (self.chain_host, self.chain_port))
return
print('Connected! Tunnel open %r -> %r -> %r' % (self.request.getpeername(), chan.getpeername(), (self.chain_host, self.chain_port)))
while True:
r, w, x = select.select([self.request, chan], [], [])
if self.request in r:
data = self.request.recv(1024)
if len(data) == 0:
break
chan.send(data)
if chan in r:
data = chan.recv(1024)
if len(data) == 0:
break
self.request.send(data)
chan.close()
self.request.close()
print('Tunnel closed from %r' % (self.request.getpeername(),))
class DBTunnel():
def __init__(self,ip):
self.c = paramiko.SSHClient()
self.c.load_system_host_keys()
self.c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.c.connect(ip, username='someuser')
self.trans = self.c.get_transport()
def startTunnel(self):
class SubHandler(Handler):
chain_host = '127.0.0.1'
chain_port = 5432
ssh_transport = self.c.get_transport()
def ThreadTunnel():
global t
t = ForwardServer(('', 3333), SubHandler)
t.serve_forever()
Thread(target=ThreadTunnel).start()
def stopTunnel(self):
t.shutdown()
self.trans.close()
self.c.close()
Although I will end up using a stopTunnel() type method, I've realize that code isn't entirely correct, but more so an experimentation of trying to get the tunnel to shutdown properly and test my results.
When I first call create the DBTunnel object and call startTunnel(), netstat yields the following:
tcp4 0 0 *.3333 *.* LISTEN
tcp4 0 0 MYIP.36316 REMOTE_HOST.22 ESTABLISHED
tcp4 0 0 127.0.0.1.5432 *.* LISTEN
Once I call stopTunnel(), or even delete the DBTunnel object itself..I'm left with this connection until I exit python all together, and what I assume to be the garbage collector takes care of it:
tcp4 0 0 *.3333 *.* LISTEN
It would be nice to figure out why this open socket is hanging around independent of the DBConnect object, and how to close it properly from within my script. If I try and bind a different connection to different IP using the same local port before completely exiting python (time_wait is not the issue), then I get the infamous bind err 48 address in use. Thanks in advance :)
It appears the SocketServer's shutdown method isn't properly shutting down/closing the socket. With the below changes in my code, I retain access to the SocketServer object and access the socket directly to close it. Note that socket.close() works in my case, but others might be interested in socket.shutdown() followed by a socket.close() if other resources are accessing that socket.
[Ref: socket.shutdown vs socket.close
def ThreadTunnel():
self.t = ForwardServer(('127.0.0.1', 3333), SubHandler)
self.t.serve_forever()
Thread(target=ThreadTunnel).start()
def stopTunnel(self):
self.t.shutdown()
self.trans.close()
self.c.close()
self.t.socket.close()
Note that you don't have do the Subhandler hack as shown in the demo code. The comment is wrong. Handlers do have access to their Server's data. Inside a handler you can use self.server.instance_data.
If you use the following code, in your Handler, you would use
self.server.chain_host
self.server.chain_port
self.server.ssh_transport
class ForwardServer(SocketServer.ThreadingTCPServer):
daemon_threads = True
allow_reuse_address = True
def __init__(
self, connection, handler, chain_host, chain_port, ssh_transport):
SocketServer.ThreadingTCPServer.__init__(self, connection, handler)
self.chain_host = chain_host
self.chain_port = chain_port
self.ssh_transport = ssh_transport
...
server = ForwardServer(('', local_port), Handler,
remote_host, remote_port, transport)
server.serve_forever()
You may want to add some synchronization between the spawned thread and the caller so that you don't try to use the tunnel before it is ready. Something like:
from threading import Event
def startTunnel(self):
class SubHandler(Handler):
chain_host = '127.0.0.1'
chain_port = 5432
ssh_transport = self.c.get_transport()
mysignal = Event()
mysignal.clear()
def ThreadTunnel():
global t
t = ForwardServer(('', 3333), SubHandler)
mysignal.set()
t.serve_forever()
Thread(target=ThreadTunnel).start()
mysignal.wait()
You can also try sshtunnel it has two cases to close tunnel .stop() if you want to wait until the end of all active connections or .stop(force=True) to close all active connections.
If you don't want to use it you can check the source code for this logic here: https://github.com/pahaz/sshtunnel/blob/090a1c1/sshtunnel.py#L1423-L1456