Develop Reference

MQTT: Python script should subscribe to subtopics

we're working on a project where some sensors send data to a mqtt broker and we wrote a python script which takes this data and stores it in a csv file.
now when we're adding more sensors our topics vary like this:
topic/sensor1
topic/sensor2
and so on. Now we want to get the subtopics automated in this script to not hardcode it when a sensor is added or removed.
Have you any suggestions, how we can subscribe in a loop to all subtopics?
We have the following so far:
import paho.mqtt.client as mqtt
import logging
from datetime import datetime
def on_subscribe(mosq, obj, mid, granted_qos):
print("Subscribed: " + str(mid))
def on_log(client, userdata, level, buf):
print(buf)
def on_connect(client, userdata, flags, rc):
print("Connected with result code "+str(rc))
client.subscribe("envdata/#")
def on_message(client, userdata, msg):
f = open("log.csv", "a")
msg_decoded = str(msg.payload, 'utf-8')
msg_decoded = msg_decoded.replace("\n","")
msg_decoded = msg_decoded + "\ttime:" + datetime.now().strftime("%d/%m/%Y %H:%M:%S") + "\n"
f.write(msg_decoded)
f.close()
print(msg.topic+" "+msg_decoded)
client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message
client.on_log = on_log
client.on_subscribe = on_subscribe
client.username_pw_set(user, password=password)
client.connect(url, 1883, 60)
client.loop_forever()

We figured it out. So as we subscribe to each subtopic with client.subscribe("topic/#") we can access all topcs in on_message callback with msg.topic.
And now we can store this topic in a string which represents the .csv file.
So our on_message callback now looks like the following:
def on_message(client, userdata, msg):
filename = msg.topic
filename = filename.replace("/","-")
f = open(filename + ".csv", "a")
msg_decoded = str(msg.payload, 'utf-8')
msg_decoded = msg_decoded.replace("\n","")
msg_decoded = msg_decoded + "\ttime:" + datetime.now().strftime("%d/%m/%Y %H:%M:%S") + "\n"
f.write(msg_decoded)
f.close()
print(msg.topic+" "+msg_decoded)
This works fine for what we wanted to reach with this script. Now when new messages are recognised by the script it opens or creates a file like topic-subtopic.csv and fills it with the expected data.

How to set up proxy in your script and publish a topic using mqtt-paho in python?

The below code will run perfectly in my home network but when i am using my office network it wont, throwing some error. So i thought to use proxy in my script, again after that getting some proxy related error .
import paho.mqtt.client as paho
import time
import socket
import socks
broker = 'iot.eclipse.org'
def on_connect(client, userdata, flags, rc):
if rc == 0:
print('Connected OK')
else:
print('Bad Connection returned code=', rc)
# print("CONNACK received with code %d." % (rc))
def on_log(client, userdata, level, buf):
print('log :' + buf)
def on_disconnect(client, userdata, flags, rc=0):
print('Disconnected result code ', str(rc))
def on_message(client, userdata, msg):
print('topic: '+msg.topic, 'payload: '+str(msg.payload))
client = paho.Client('python2')
client.on_connect = on_connect
client.on_disconnect = on_disconnect
client.on_message = on_message
client.on_log = on_log
socks.set_default_proxy(socks.SOCKS5, 'proxyhost.com', 1180)
socket.socket = socks.socksocket
print('Connecting to broker', broker)
client.connect(broker, 1883, 60)
client.loop_start()
client.subscribe('test/iotglab')
client.publish('test/iotglab', 'Hello Client')
time.sleep(4)
client.loop_stop()
client.disconnect()
I got proxy which is look like socks5h://hostname.com.

python mqtt-paho disconnect after publish

I am trying to publish single message to MQTT and disconnect with following code. But it works sometime, sometime doesn't work as expected. I want to listen for a topic for if the switch1 os on then turn off, of its of then turn on based on the received data and disconnect.
#!/usr/bin/env python2.7
import json
import time
import os
import paho.mqtt.client as mqtt
mqtt_host = os.getenv('HOST', 'xxxx')
mqtt_port = os.getenv('PORT', 1883)
mqtt_username = os.getenv('USERNAME', 'xxxx')
mqtt_password = os.getenv('PASSWORD', 'xxxx')
mqtt_subacribe_topic = os.getenv('SUBSCRIBE_TOPIC', 'xxxx')
mqtt_publish_topic = os.getenv('PUBLISH_TOPIC', 'xxxx')
sleep_time = os.getenv('SLEEP_TIME', 15)
CLIENT_ID = "lambda"
SWITCH1_ON = { "SWITCH1": "on" }
SWITCH1_OFF = { "SWITCH1": "off" }
def on_publish(client, userdata, mid):
print ("Message Published...")
client.disconnect()
def on_subscribe(client, userdata, mid, granted_qos):
print("Subscribed: " + str(message.topic) + " " + str(mid) + " " + str(granted_qos))
def on_connect(client, userdata, flags, rc):
if rc == 0:
print("Connected to broker")
client.subscribe(mqtt_subacribe_topic)
else:
print("Connection failed")
def on_message(client, userdata, msg):
payload = json.loads(msg.payload)
if payload.get('switch1') == 1:
client.publish(mqtt_publish_topic,json.dumps(SWITCH1_ON))
elif payload.get('switch1') == 0:
client.publish(mqtt_publish_topic,json.dumps(SWITCH1_OFF))
def main():
client = mqtt.Client(CLIENT_ID)
client.username_pw_set(mqtt_username, password=mqtt_password)
# Register publish callback function
client.on_publish = on_publish
client.on_connect = on_connect
client.on_message = on_message
# Connect with MQTT Broker
client.connect(mqtt_host, port=mqtt_port)
# Loop forever
client.loop_start()
time.sleep(sleep_time)
client.loop_stop()
client.disconnect()
if __name__ == "__main__":
main()
Based on hardillb's answer I tried :
#!/usr/bin/env python2.7
import json
import time
import os
import paho.mqtt.subscribe as subscribe
import paho.mqtt.publish as publish
mqtt_host = os.getenv('HOST', 'xxx.cloudmqtt.com')
mqtt_port = os.getenv('PORT', 1883)
mqtt_username = os.getenv('USERNAME', 'xxx')
mqtt_password = os.getenv('PASSWORD', 'xxx')
mqtt_subacribe_topic = os.getenv('SUBSCRIBE_TOPIC', 'xxx')
mqtt_publish_topic = os.getenv('PUBLISH_TOPIC', 'xxx')
sleep_time = os.getenv('SLEEP_TIME', 14)
CLIENT_ID = "lambda"
SWITCH1_ON = { "SWITCH1": "on" }
SWITCH1_OFF = { "SWITCH1": "off" }
auth = {'username':mqtt_username, 'password':mqtt_password}
def on_message(client, userdata, msg):
payload = json.loads(msg.payload)
print(payload)
if payload.get('switch1') == 1:
publish.single(mqtt_publish_topic,json.dumps(SWITCH1_ON),hostname=mqtt_host,auth=auth)
print "Turning switch1 ON"
elif payload.get('switch1') == 0:
publish.single(mqtt_publish_topic,json.dumps(SWITCH1_OFF),hostname=mqtt_host,auth=auth)
print "Turning switch1 OFF"
def main():
subscribe.callback(on_message, mqtt_subacribe_topic,hostname=mqtt_host,auth=auth)
if __name__ == "__main__":
main()
But script is keep running, I have to kill it to stop. Is it possible to just subscribed to a topic and once first message is received, process it, publish to another topic and end the execution.

If you just want to publish a single message then the Paho client has a built in method to do this. You can find the doc here
import paho.mqtt.publish as publish
publish.single("paho/test/single", "payload", hostname="iot.eclipse.org")
There is an equivalent method for also subscribing to a topic and receiving a single message.

Python - MQTT mulitiple subscription: Which topic the message came from

I have a logger to record all MQTT messages arrive to local broker.
This logger have multiple subscriptions, and one of them is "Alerts" - which will additionally send SMS to user's phone ( not showing is attached code ).
My question ( I guess it is a bit newbie ) - but is there a way to filter the origin of a message arrived ?
from sys import path
path.append('/home/guy/.local/lib/python3.5/site-packages')
import paho.mqtt.client as mqtt
from threading import Thread
import datetime
import os
class LogMQTTactivity(Thread):
def __init__(self, sid=None, mqtt_server="192.168.2.113", username=None, password=None, topics=None, topic_qos=None,
filename='/home/guy/MQTTlogger.log'):
Thread.__init__(self)
self.sid = sid
self.mqtt_server = mqtt_server
self.filename = filename
self.username = username
self.password = password
self.topics = topics
self.topic_qos = topic_qos
self.output2screen = 1
self.client, self.arrived_msg = None, None
self.check_logfile_valid()
self.log_header()
def log_header(self):
text = ' Connect to following topics '
x = 12
self.append_log('*' * x + text + x * "*")
for topic in self.topics:
self.append_log(topic)
self.append_log('*' * 2 * x + len(text) * "*")
def run(self):
self.client = mqtt.Client(str(self.sid))
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
if self.username is not None and self.password is not None:
self.client.username_pw_set(self.username, self.password)
self.client.connect(self.mqtt_server, 1883, 60)
self.client.loop_forever()
def on_connect(self, client, obj, flags, rc):
self.append_log(">> Connecting to MQTT mqtt_server %s: %d" % (self.mqtt_server, rc))
for topic in self.topics:
self.append_log(">> Subscribe topic: %s" % topic)
self.client.subscribe(topic, qos=self.topic_qos)
def on_message(self, client, obj, msg):
self.arrived_msg = msg.payload.decode()
self.append_log(self.arrived_msg)
#staticmethod
def timeStamp():
return str(datetime.datetime.now())[:-5]
def check_logfile_valid(self):
if os.path.isfile(self.filename) is True:
self.valid_logfile = True
else:
open(self.filename, 'a').close()
self.valid_logfile = os.path.isfile(self.filename)
if self.valid_logfile is True:
msg = '>>Log file %s was created successfully' % self.filename
else:
msg = '>>Log file %s failed to create' % self.filename
print(msg)
self.append_log(msg)
def append_log(self, log_entry=''):
self.msg = '[%s] %s' % (self.timeStamp(), log_entry)
if self.valid_logfile is True:
myfile = open(self.filename, 'a')
myfile.write(self.msg + '\n')
myfile.close()
else:
print('Log err')
if self.output2screen == 1:
print(self.msg)
if __name__ == "__main__":
a = LogMQTTactivity(sid="MQTTlogger", topics=['Alerts', 'notifications'], topic_qos=0,
mqtt_server="192.168.2.200", username="guy", password="12345678")
a.start()

The msg object passed into the on_message callback has a topic field that contains the topic the message was published to.
def on_message(self, client, obj, msg):
print(msg.topic)
self.arrived_msg = msg.payload.decode()
self.append_log(self.arrived_msg)
As mentioned in the doc here

MQTT: How to send a message without to call loop_forever()

I'm using MQTT since a while to monitor some channels I subscribed. Now I want to implement to send a message as reaction to a state. I got it running with the code below, where I just react in the on_message callback (code 1 at the end). But this code uses
loop_forever()
in the main code which is blocking.
What I would like to do is to just send a single message to MQTT. When I try the following (with all different loop-functions), nothing is received by the MQTT server:
import paho.mqtt.client as mqtt
if __name__ == "__main__":
mqtt_client = mqtt.Client()
mqtt_client.connect("192.168.178.204", 1883, 60)
mqtt_client.username_pw_set(username="test", password="test")
mqtt_client.publish(topic='TEST', payload='CCCCCCCCC', retain=False)
mqtt_client.loop_write()
# mqtt_client.loop()
# mqtt_client.loop_start()
mqtt_client.disconnect()
How can I send a message to MQTT which does not block the process?
Code 1:
import paho.mqtt.client as mqtt
def on_connect(client, userdata, rc):
topic_list = [("TEST_MS", 1)]
if rc == 0:
print("Successful connected and subscribed to: {}".format(topic_list))
client.subscribe(topic_list)
def on_message(client, userdata, msg):
print(msg.payload)
client.publish(topic='TEST_MS2', payload=msg.payload, retain=False)
def on_publish(client, userdata, mid):
print("message published")
def on_subscribe(mosq, obj, mid, granted_qos):
print("Subscribed: " + str(mid) + " " + str(granted_qos))
if __name__ == "__main__":
mqtt_client = mqtt.Client()
mqtt_client.on_connect = on_connect
mqtt_client.on_message = on_message
mqtt_client.on_publish = on_publish
mqtt_client.on_subscribe = on_subscribe
mqtt_client.connect("192.168.178.204", 1883, 60)
mqtt_client.username_pw_set(username="test", password="test")
# mqtt_client.publish(topic='TEST_MS', payload='CCCCCCCCC', retain=False)
mqtt_client.loop_forever()
mqtt_client.disconnect()

If you want to send just a single message and then exit use the API specifically for that. Docs here
import paho.mqtt.publish as publish
publish.single("paho/test/single", "payload", hostname="iot.eclipse.org")