Develop Reference

Python - MQTT mulitiple subscription: Which topic the message came from

I have a logger to record all MQTT messages arrive to local broker.
This logger have multiple subscriptions, and one of them is "Alerts" - which will additionally send SMS to user's phone ( not showing is attached code ).
My question ( I guess it is a bit newbie ) - but is there a way to filter the origin of a message arrived ?
from sys import path
path.append('/home/guy/.local/lib/python3.5/site-packages')
import paho.mqtt.client as mqtt
from threading import Thread
import datetime
import os
class LogMQTTactivity(Thread):
def __init__(self, sid=None, mqtt_server="192.168.2.113", username=None, password=None, topics=None, topic_qos=None,
filename='/home/guy/MQTTlogger.log'):
Thread.__init__(self)
self.sid = sid
self.mqtt_server = mqtt_server
self.filename = filename
self.username = username
self.password = password
self.topics = topics
self.topic_qos = topic_qos
self.output2screen = 1
self.client, self.arrived_msg = None, None
self.check_logfile_valid()
self.log_header()
def log_header(self):
text = ' Connect to following topics '
x = 12
self.append_log('*' * x + text + x * "*")
for topic in self.topics:
self.append_log(topic)
self.append_log('*' * 2 * x + len(text) * "*")
def run(self):
self.client = mqtt.Client(str(self.sid))
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
if self.username is not None and self.password is not None:
self.client.username_pw_set(self.username, self.password)
self.client.connect(self.mqtt_server, 1883, 60)
self.client.loop_forever()
def on_connect(self, client, obj, flags, rc):
self.append_log(">> Connecting to MQTT mqtt_server %s: %d" % (self.mqtt_server, rc))
for topic in self.topics:
self.append_log(">> Subscribe topic: %s" % topic)
self.client.subscribe(topic, qos=self.topic_qos)
def on_message(self, client, obj, msg):
self.arrived_msg = msg.payload.decode()
self.append_log(self.arrived_msg)
#staticmethod
def timeStamp():
return str(datetime.datetime.now())[:-5]
def check_logfile_valid(self):
if os.path.isfile(self.filename) is True:
self.valid_logfile = True
else:
open(self.filename, 'a').close()
self.valid_logfile = os.path.isfile(self.filename)
if self.valid_logfile is True:
msg = '>>Log file %s was created successfully' % self.filename
else:
msg = '>>Log file %s failed to create' % self.filename
print(msg)
self.append_log(msg)
def append_log(self, log_entry=''):
self.msg = '[%s] %s' % (self.timeStamp(), log_entry)
if self.valid_logfile is True:
myfile = open(self.filename, 'a')
myfile.write(self.msg + '\n')
myfile.close()
else:
print('Log err')
if self.output2screen == 1:
print(self.msg)
if __name__ == "__main__":
a = LogMQTTactivity(sid="MQTTlogger", topics=['Alerts', 'notifications'], topic_qos=0,
mqtt_server="192.168.2.200", username="guy", password="12345678")
a.start()

The msg object passed into the on_message callback has a topic field that contains the topic the message was published to.
def on_message(self, client, obj, msg):
print(msg.topic)
self.arrived_msg = msg.payload.decode()
self.append_log(self.arrived_msg)
As mentioned in the doc here

Threaded python socket server

I have a Multihtreaded Server with python that can handle clients request, but i have a problem with this.
In my Server Class I have a start function that start listening to clients like this:
class Server:
def __init__(self, clients={}):
self.clients = clients
self.ip = 'localhost'
self.port = ****
self.pattern = '(C\d)'
def start(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.s.bind((self.ip, self.port))
self.s.listen(10)
while 1:
clientsock, addr = self.s.accept()
print ('Connected with ' + addr[0] + ':' + str(addr[1]))
_thread.start_new_thread(self.handler, (clientsock, addr))
def handler(self, clientsock, addr):
data = clientsock.recv(BUFF)
print ('Data : ' + repr(data))
data = data.decode("UTF-8")
result = re.match(self.pattern, data)
print (data)
if(result):
self.registerClient(clientsock, data)
if(data == "Exit"):
self.exitClient(clientsock)
def server_response(self, message, flag, err):
if(flag):
res = message.encode('utf-8')
return res
else:
res = message.encode('utf-8')+ "[ ".encode('utf-8')+err.encode('utf-8')+ " ]".encode('utf-8')
return res
def registerClient(self, clientsock, data):
if(data in self.clients):
err = "Error : Client Name Exist!"
clientsock.send(self.server_response('Reg#NOK#', 0, err))
clientsock.close()
sys.exit(1)
self.clients[clientsock] = data
clientsock.send(self.server_response('Reg#OK', 1, ''))
def exitClient(self, clientsock):
try:
f = self.clients.pop(clientsock)
clientsock.send(self.server_response('BYE#OK', 1, ''))
clientsock.close()
except KeyError:
err = "Error : Client Doesn't Connected To Server!"
clientsock.send(self.server_response('BYE#NOK#', 0, err))
clientsock.close()
sys.exit(1)
And this is my client Class:
class Client:
def __init__(self, name):
self.name = name
self.ip = '127.0.0.1'
self.next_client = None
self.s = ""
try:
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except:
print ('Reg#NOK#[ ' + msg[1] + ' ]')
sys.exit()
def register(self, server):
self.s.connect((server.ip, server.port))
message = self.name
try:
self.s.sendall(bytes(message, 'UTF-8'))
except (socket.error):
print ('Send Failed')
sys.exit()
reply = self.s.recv(4096)
print ("Respose From Server : " + reply.decode("utf-8") )
def exitFromServer(self, server):
message = "Exit".encode('utf-8')
try:
a = self.s.sendall(message)
except (socket.error):
print ('Send Failed')
sys.exit()
reply = self.s.recv(4096)
And this is the main file:
from server import *
from client import *
import _thread
a = Server()
_thread.start_new_thread(a.start, ())
b = Client("C1")
b.register(a)
b.exitFromServer(a)
As you can see when start function from Server class called there is no thread that can handle create Client , I mean when I use start function like this with out thread there is no way that program can go ahead in main file , I know I should use another thread here but where and how, I use Python 3.4, Thanks for helping me.
Edit
the Problem with start was Solved , Thanks from tdelaney,
but when I run this only the register function works and exitFromServer dont do anything can you tell me where is the problem.the program dosent do anything after execute register function and it seems that its wating for something.

This mean?
import threading
from server import *
from client import *
global a
a = Server()
def sServer():
global a
a.start()
def cClient():
global a
b = Client("C1")
b.register(a)
s = threading.Thread(name='server', target=sServer)
c = threading.Thread(name='client', target=cClient)
s.start()
c.start()

In Server Class I must add another while True loop after handler function cause it shuould do all client request till client has request:
def handler(self, clientsock, addr):
while 1:
data = clientsock.recv(BUFF)
print ('Data : ' + repr(data))
data = data.decode("UTF-8")
result = re.match(self.pattern, data)
if(result):
self.registerClient(clientsock, data)
if(data == "Exit"):
self.exitClient(clientsock)
break

Twisted tcp tunnel/bridge/relay

I'd like to know how to write TCP tunnel/relay/bridge/proxy (you name it) using twisted.
I did some research in google, twisted doc/forum etc. etc but couldn't find anwser.
I already done it in pure python using socket, threading and select.
Here is code:
#!/usr/bin/env python
import socket
import sys
import select
import threading
import logging
import time
class Client(threading.Thread):
def __init__(self, client, address, id_number, dst_ip, dst_port):
self.log = logging.getLogger(__name__+'.client-%s' % id_number)
self.running = False
self.cl_soc = client
self.cl_adr = address
self.my_id = id_number
self.dst_ip = dst_ip
self.dst_port = dst_port
threading.Thread.__init__(self)
def stop(self):
self.running = 0
def run(self):
try:
self.dst_soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.dst_soc.connect((self.dst_ip,self.dst_port))
except:
self.log.error('Can\'t connect to %s:%s' % (self.dst_ip, self.dst_port))
else:
self.running = True
self.log.info('Bridge %s <-> %s created' % (
'%s:%s' % self.dst_soc.getpeername(), '%s:%s' % self.cl_adr))
try:
while self.running:
iRdy = select.select([self.cl_soc, self.dst_soc],[],[], 1)[0]
if self.cl_soc in iRdy:
buf = self.cl_soc.recv(4096)
if not buf:
info = 'Ended connection: client'
self.running = False
else:
self.dst_soc.send(buf)
if self.dst_soc in iRdy:
buf = self.dst_soc.recv(4096)
if not buf:
info = 'Ended connection: destination'
self.running = False
else:
self.cl_soc.send(buf)
except:
self.log.error('Sth bad happend', exc_info=True)
self.running = False
self.log.debug('Closing sockets')
try:
self.dst_soc.close()
except:
pass
try:
self.cl_soc.close()
except:
pass
class Server(threading.Thread):
def __init__(self, l_port=25565, d_ip='127.0.0.1', d_port=None):
self.log = logging.getLogger(__name__+'.server-%s:%s' % (d_ip,d_port))
threading.Thread.__init__(self)
self.d_ip = d_ip
if d_port == None:
self.d_port = l_port
else:
self.d_port = d_port
self.port = l_port
binding = 1
wait = 30
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
while binding:
try:
self.s.bind(('',self.port))
except:
self.log.warning('Cant bind. Will wait %s sec' % wait)
time.sleep(wait)
else:
binding = 0
self.log.info('Server ready for connections - port %s' % d_port)
def run(self):
self.s.listen(5)
input = [self.s, sys.stdin]
running = 1
self.cl_threads = []
id_nr = 0
while running:
iRdy = select.select(input, [], [],1)[0]
if self.s in iRdy:
c_soc, c_adr = self.s.accept()
c = Client(c_soc, c_adr, id_nr, self.d_ip, self.d_port)
c.start()
self.cl_threads.append(c)
id_nr += 1
if sys.stdin in iRdy:
junk = sys.stdin.readline()
print junk
running = 0
try:
self.s.close()
except:
pass
for c in self.cl_threads:
c.stop()
c.join(5)
del c
self.cl_threads = None
self.log.info('Closing server')
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
s = Server(1424, '192.168.1.6', 1424)
s.run()

this is actually, already built into twisted, from the command line you can type:
$ twistd --nodaemon portforward --port 1424 --host 192.168.1.6
to get the exact behavior you seem to be looking for.
If you'd like to roll your own, you can still use all of the bits, in twisted.protocols.portforward

how do i get my tornado ioloop socket client to send back respose after receiving?

iv created a simple async client and server but im unable to get the client to reply after receiving the first time. It seems the server can send back a reply after receiving from the client but the client cant:
here is the client's session:
[mike#mike Public]$ python cl.py
buf got your stuff
dded callback ## this is a log addded to see if execution got where i wanted
and here is the server's log:
[mike#mike Public]$ python that.py
buf ehlo localhost
i was expecting some sort of ping pong effect where one send then the other then rinse lather repeat.
here is the client's code:
import socket
import fcntl, os, io, time, functools
from tornado import ioloop
class Punk(object):
def __init__(self):
self.loop = ioloop.IOLoop.instance()
self.address = 'blah.sock'
self.authkey = "none"
self.sock = socket.socket(socket.AF_UNIX)
def setup(self):
self.sock.connect(self.address)
fcntl.fcntl(self.sock, fcntl.F_SETFL, os.O_NONBLOCK)
self.sock.sendall("ehlo localhost")
self.fd = self.sock.fileno()
self.loop.add_handler(self.fd,self.reader,self.loop.READ)
self.loop.start()
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
try:
while True:
servrep = self.sock.recv(1024)
if not servrep:
break
result += servrep
self.prnt(result)
break
except Exception as e:
print "this %s happend"%e
return
def prnt(self,buf):
print "buf %s"%buf
tim = time.time() + 2
self.loop.instance().add_timeout(tim, self.reply)
#callbac = functools.partial(self.loop.add_timeout,tim,self.reply)
#self.loop.add_callback(self.reply) ### i tried this too
print "added callback"
def reply(self):
self.sock.sendall(" clent got your stuff")
if __name__ == "__main__":
bob = Punk()
bob.setup()
and here is the server:
import socket
import fcntl, os, io, time, functools
from array import array
from tornado import ioloop
class Player(object):
def __init__(self):
self.loop = ioloop.IOLoop.instance()
self.address = 'blah.sock'
self.authkey = "none"
self.sock = socket.socket(socket.AF_UNIX)
def setup(self):
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.sock.bind(self.address)
fcntl.fcntl(self.sock, fcntl.F_SETFL, os.O_NONBLOCK)
self.sock.listen(1)
self.fd = self.sock.fileno()
self.loop.add_handler(self.fd,self.reader,self.loop.READ)
self.loop.start()
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
self.conn, self.addr = self.sock.accept()
try:
while True:
maxrep = self.conn.recv(1024)
if not maxrep:
break
result += maxrep
self.prnt(result)
break
except Exception as e:
print "this %s happend"%e
return
def prnt(self,buf):
print "buf %s"%buf
tim = time.time() + 2
self.loop.instance().add_timeout(tim, self.reply)
#callbac = functools.partial(self.loop.add_timeout,tim,self.reply)
#self.loop.add_callback(callbac)
def reply(self):
self.conn.sendall("got your stuff")
if __name__ == "__main__":
bob = Player()
bob.setup()

i had set my sockets to nonblock mode, but i did not catch an error when accepting from
a nonblock state when there is no connection:
here:
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
self.conn, self.addr = self.sock.accept()
should be
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
try:
self.conn, self.addr = self.sock.accept() # we get stuck here
self.connl.append(self.conn)
except Exception as e:
pass

load testing gevent sockets

How can you load test gevent sockets?
I've written a simple server and I want to load test this before I tweak the code to find out what improvements can be done.
Are there any specifics to load testing a socket server?
How would I go about load testing this code so I can compare the results after applying tweaks?
from gevent import server
from gevent.monkey import patch_all; patch_all()
import gevent
class Client(object):
def __init__(self, socket, address, server_handler):
self.socket = socket
self.address = address
self.server_handler = server_handler
gevent.spawn(self.listen)
def listen(self):
f = self.socket.makefile()
while True:
line = f.readline()
if not line:
print 'client died'
break
line = line.rstrip()
for addr, c in self.server_handler.users.iteritems():
msg = '<%s> says: %s' % (self.address, line)
c.send(msg)
print '<%s>: %s' % (self.address, line)
def send(self, msg):
f = self.socket.makefile()
f.write('%s\r\n' % msg)
f.flush()
class ServerHandler(object):
def __init__(self):
self.users = {}
def __call__(self, socket, address):
client = Client(socket, address, self)
self.users[address] = client
self.socket = socket
self.address = address
print 'connection made'
if __name__ == '__main__':
server = server.StreamServer(('0.0.0.0', 5000), ServerHandler())
server.serve_forever()