we're working on a project where some sensors send data to a mqtt broker and we wrote a python script which takes this data and stores it in a csv file.
now when we're adding more sensors our topics vary like this:
topic/sensor1
topic/sensor2
and so on. Now we want to get the subtopics automated in this script to not hardcode it when a sensor is added or removed.
Have you any suggestions, how we can subscribe in a loop to all subtopics?
We have the following so far:
import paho.mqtt.client as mqtt
import logging
from datetime import datetime
def on_subscribe(mosq, obj, mid, granted_qos):
print("Subscribed: " + str(mid))
def on_log(client, userdata, level, buf):
print(buf)
def on_connect(client, userdata, flags, rc):
print("Connected with result code "+str(rc))
client.subscribe("envdata/#")
def on_message(client, userdata, msg):
f = open("log.csv", "a")
msg_decoded = str(msg.payload, 'utf-8')
msg_decoded = msg_decoded.replace("\n","")
msg_decoded = msg_decoded + "\ttime:" + datetime.now().strftime("%d/%m/%Y %H:%M:%S") + "\n"
f.write(msg_decoded)
f.close()
print(msg.topic+" "+msg_decoded)
client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message
client.on_log = on_log
client.on_subscribe = on_subscribe
client.username_pw_set(user, password=password)
client.connect(url, 1883, 60)
client.loop_forever()
We figured it out. So as we subscribe to each subtopic with client.subscribe("topic/#") we can access all topcs in on_message callback with msg.topic.
And now we can store this topic in a string which represents the .csv file.
So our on_message callback now looks like the following:
def on_message(client, userdata, msg):
filename = msg.topic
filename = filename.replace("/","-")
f = open(filename + ".csv", "a")
msg_decoded = str(msg.payload, 'utf-8')
msg_decoded = msg_decoded.replace("\n","")
msg_decoded = msg_decoded + "\ttime:" + datetime.now().strftime("%d/%m/%Y %H:%M:%S") + "\n"
f.write(msg_decoded)
f.close()
print(msg.topic+" "+msg_decoded)
This works fine for what we wanted to reach with this script. Now when new messages are recognised by the script it opens or creates a file like topic-subtopic.csv and fills it with the expected data.
Related
I'm usuing paho.mqtt.client to forward a local mqtt broker to another public mqtt broker,
so what I did is, have two client, and subscribe on both, and on receive mqtt message, forward to another borker, the issue I have is it will always has a additional forward slash in the forwarded mqtt message topic.
the original message topic:
elevator/11112901/cabin/load/actual-load
the forward message topic:
elevator/11112901/cabin/load/actual-load/
Appreciate if you could share your comments,thanks!
The code
#!/usr/bin/env python
#coding=utf-8
import hmac
import queue
import base64
from hashlib import sha1
import time
from paho.mqtt.client import MQTT_LOG_INFO, MQTT_LOG_NOTICE, MQTT_LOG_WARNING, MQTT_LOG_ERR, MQTT_LOG_DEBUG
from paho.mqtt import client as mqtt
import random
import string
dev_key = ''.join(random.sample(string.ascii_letters + string.digits, 12))
# 实例 ID，购买后从产品控制台获取
instanceId ='xxx'
#账号AccessKey 从阿里云账号控制台获取
accessKey = 'xxx'
#账号secretKey 从阿里云账号控制台获取
secretKey = 'xxx'
#MQTT GroupID,创建实例后从 MQTT 控制台创建
groupId = 'xxx'
#MQTT ClientID，由 GroupID 和后缀组成，需要保证全局唯一
client_id=groupId+'###'+ dev_key
#MQTT 接入点域名，实例初始化之后从控制台获取
brokerUrl='xxx'
topic_elevator_command = 'commands/#'
topic_elevator_status = 'elevator/#'
q = queue.Queue()
def on_log_aliyun(client, userdata, level, buf):
if level == MQTT_LOG_INFO:
head = 'INFO'
elif level == MQTT_LOG_NOTICE:
head = 'NOTICE'
elif level == MQTT_LOG_WARNING:
head = 'WARN'
elif level == MQTT_LOG_ERR:
head = 'ERR'
elif level == MQTT_LOG_DEBUG:
head = 'DEBUG'
else:
head = level
print('%s: %s' % (head, buf))
def on_connect_cube(client, userdata, flags, rc):
print('Connected with result code ' + str(rc))
client.subscribe(topic_elevator_status, 1)
client.publish("elevator/test", "32q14324")
def on_message_cube(client, userdata, msg):
print(msg.topic + ' ' + str(msg.payload))
if msg.topic.startswith(topic_elevator_status[:len(topic_elevator_status)-2]):
q.put(("Cube", msg))
def on_disconnect_cube(client, userdata, rc):
if rc != 0:
print('Unexpected disconnection %s' % rc)
def on_connect_aliyun(client, userdata, flags, rc):
print('Connected with result code ' + str(rc))
client.subscribe(topic_elevator_command, 0)
client.publish("elevator/test", "32q14324")
def on_message_aliyun(client, userdata, msg):
print(msg.topic + ' ' + str(msg.payload))
if msg.topic.startswith(topic_elevator_command[:len(topic_elevator_command)-2]):
q.put(("Aliyun", msg))
def on_disconnect_aliyun(client, userdata, rc):
if rc != 0:
print('Unexpected disconnection %s' % rc)
client_aliyun = mqtt.Client(client_id, protocol=mqtt.MQTTv311, clean_session=True)
#client_aliyun.on_log = on_log_aliyun
client_aliyun.on_connect = on_connect_aliyun
client_aliyun.on_message = on_message_aliyun
client_aliyun.on_disconnect = on_disconnect_aliyun
## username和 Password 签名模式下的设置方法，参考文档 https://help.aliyun.com/document_detail/48271.html?spm=a2c4g.11186623.6.553.217831c3BSFry7
userName ='Signature'+'|'+accessKey+'|'+instanceId
password = base64.b64encode(hmac.new(secretKey.encode(), client_id.encode(), sha1).digest()).decode()
client_aliyun.username_pw_set(userName, password)
# ssl设置，并且port=8883
#client.tls_set(ca_certs=None, certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLS, ciphers=None)
client_aliyun.connect(brokerUrl, 1883, 60)
client_aliyun.loop_start()
client_cube = mqtt.Client(dev_key, clean_session=True)
#client_cube.on_log = on_log_aliyun
client_cube.on_connect = on_connect_cube
client_cube.on_message = on_message_cube
client_cube.on_disconnect = on_disconnect_cube
client_cube.connect("xxx",1883)
client_cube.loop_start()
while 1:
qmsg = q.get()
dir, msg = qmsg
print(dir, msg.topic)
if dir == "Cube":
client_aliyun.publish(msg.topic, msg.payload)
if dir == "Aliyun":
client_cube.publish(msg.topic, msg.payload)
I need to build a network with multiple publishers, one broker, and one subscriber.
Issue:
I have a working code with one to one option (one pub - one sub). However, when I run more publishers the subscriber receives messages only from the last connected publisher and I have no idea why it is happening.
More information: Broker (mosquitto) is working in a docker container, each publisher is a separate script and the target is to run multiple docker containers with one publisher in each container, but now I need to work around the communication issue.
Does anyone have any clues or ideas on how to debug or solved this?
This is the publisher script:
import time
import paho.mqtt.client as mqtt
import datetime
import random
import json
import logging
from multiprocessing import Process
CLEAN_SESSION = False
def on_connect(client, userdata, flags, rc):
logging.info(f"New connection {client}, {rc}")
def sensor(client_id):
localhost = '172.17.0.2'
port = 1883
timeout = 60
topic = "/mia/sensor"
client_id = f"sensor_{client_id}"
def check_sensor():
time.sleep(1)
rand = random.randint(0, 10)
if rand > 5:
current_time = datetime.datetime.now()
current_time = current_time.strftime('%Y-%m-%d %H:%M:%S')
return {"time": current_time, "signal": 1, "id": client_id}
else:
return 0
client = mqtt.Client(client_id, clean_session=CLEAN_SESSION)
client.on_connect = on_connect
client.connect(localhost, port, timeout)
while True:
check_info = check_sensor()
if check_info:
message_payload = json.dumps(check_info)
logging.info(message_payload)
client.publish(topic, message_payload, qos=2)
client.loop()
client.disconnect()
if __name__ == "__main__":
p = Process(target=sensor, args=(1,))
p.start()
print("new publisher created!")
This is the subscriber script:
import paho.mqtt.client as mqtt
import paho.mqtt.subscribe as sub
import json
import logging
localhost = '172.17.0.2'
port = 1883
timeout = 60
topic = "/mia/sensor"
def on_connect(client, userdata, flags, rc):
logging.info(f"New connection {client}, {rc}")
client.subscribe(topic, qos=2)
def on_message(client, userdata, msg):
data = json.loads(msg.payload.decode('utf-8'))
logging.debug(f"new message from {client} - {data}")
print(data)
client = mqtt.Client("python", clean_session=False)
client.on_connect = on_connect
client.on_message = on_message
client.connect_async(localhost, port, timeout)
client.loop_forever()
Thanks in advance
Your client_id needs to be unique. Check that.
The below code will run perfectly in my home network but when i am using my office network it wont, throwing some error. So i thought to use proxy in my script, again after that getting some proxy related error .
import paho.mqtt.client as paho
import time
import socket
import socks
broker = 'iot.eclipse.org'
def on_connect(client, userdata, flags, rc):
if rc == 0:
print('Connected OK')
else:
print('Bad Connection returned code=', rc)
# print("CONNACK received with code %d." % (rc))
def on_log(client, userdata, level, buf):
print('log :' + buf)
def on_disconnect(client, userdata, flags, rc=0):
print('Disconnected result code ', str(rc))
def on_message(client, userdata, msg):
print('topic: '+msg.topic, 'payload: '+str(msg.payload))
client = paho.Client('python2')
client.on_connect = on_connect
client.on_disconnect = on_disconnect
client.on_message = on_message
client.on_log = on_log
socks.set_default_proxy(socks.SOCKS5, 'proxyhost.com', 1180)
socket.socket = socks.socksocket
print('Connecting to broker', broker)
client.connect(broker, 1883, 60)
client.loop_start()
client.subscribe('test/iotglab')
client.publish('test/iotglab', 'Hello Client')
time.sleep(4)
client.loop_stop()
client.disconnect()
I got proxy which is look like socks5h://hostname.com.
I am trying to publish single message to MQTT and disconnect with following code. But it works sometime, sometime doesn't work as expected. I want to listen for a topic for if the switch1 os on then turn off, of its of then turn on based on the received data and disconnect.
#!/usr/bin/env python2.7
import json
import time
import os
import paho.mqtt.client as mqtt
mqtt_host = os.getenv('HOST', 'xxxx')
mqtt_port = os.getenv('PORT', 1883)
mqtt_username = os.getenv('USERNAME', 'xxxx')
mqtt_password = os.getenv('PASSWORD', 'xxxx')
mqtt_subacribe_topic = os.getenv('SUBSCRIBE_TOPIC', 'xxxx')
mqtt_publish_topic = os.getenv('PUBLISH_TOPIC', 'xxxx')
sleep_time = os.getenv('SLEEP_TIME', 15)
CLIENT_ID = "lambda"
SWITCH1_ON = { "SWITCH1": "on" }
SWITCH1_OFF = { "SWITCH1": "off" }
def on_publish(client, userdata, mid):
print ("Message Published...")
client.disconnect()
def on_subscribe(client, userdata, mid, granted_qos):
print("Subscribed: " + str(message.topic) + " " + str(mid) + " " + str(granted_qos))
def on_connect(client, userdata, flags, rc):
if rc == 0:
print("Connected to broker")
client.subscribe(mqtt_subacribe_topic)
else:
print("Connection failed")
def on_message(client, userdata, msg):
payload = json.loads(msg.payload)
if payload.get('switch1') == 1:
client.publish(mqtt_publish_topic,json.dumps(SWITCH1_ON))
elif payload.get('switch1') == 0:
client.publish(mqtt_publish_topic,json.dumps(SWITCH1_OFF))
def main():
client = mqtt.Client(CLIENT_ID)
client.username_pw_set(mqtt_username, password=mqtt_password)
# Register publish callback function
client.on_publish = on_publish
client.on_connect = on_connect
client.on_message = on_message
# Connect with MQTT Broker
client.connect(mqtt_host, port=mqtt_port)
# Loop forever
client.loop_start()
time.sleep(sleep_time)
client.loop_stop()
client.disconnect()
if __name__ == "__main__":
main()
Based on hardillb's answer I tried :
#!/usr/bin/env python2.7
import json
import time
import os
import paho.mqtt.subscribe as subscribe
import paho.mqtt.publish as publish
mqtt_host = os.getenv('HOST', 'xxx.cloudmqtt.com')
mqtt_port = os.getenv('PORT', 1883)
mqtt_username = os.getenv('USERNAME', 'xxx')
mqtt_password = os.getenv('PASSWORD', 'xxx')
mqtt_subacribe_topic = os.getenv('SUBSCRIBE_TOPIC', 'xxx')
mqtt_publish_topic = os.getenv('PUBLISH_TOPIC', 'xxx')
sleep_time = os.getenv('SLEEP_TIME', 14)
CLIENT_ID = "lambda"
SWITCH1_ON = { "SWITCH1": "on" }
SWITCH1_OFF = { "SWITCH1": "off" }
auth = {'username':mqtt_username, 'password':mqtt_password}
def on_message(client, userdata, msg):
payload = json.loads(msg.payload)
print(payload)
if payload.get('switch1') == 1:
publish.single(mqtt_publish_topic,json.dumps(SWITCH1_ON),hostname=mqtt_host,auth=auth)
print "Turning switch1 ON"
elif payload.get('switch1') == 0:
publish.single(mqtt_publish_topic,json.dumps(SWITCH1_OFF),hostname=mqtt_host,auth=auth)
print "Turning switch1 OFF"
def main():
subscribe.callback(on_message, mqtt_subacribe_topic,hostname=mqtt_host,auth=auth)
if __name__ == "__main__":
main()
But script is keep running, I have to kill it to stop. Is it possible to just subscribed to a topic and once first message is received, process it, publish to another topic and end the execution.
If you just want to publish a single message then the Paho client has a built in method to do this. You can find the doc here
import paho.mqtt.publish as publish
publish.single("paho/test/single", "payload", hostname="iot.eclipse.org")
There is an equivalent method for also subscribing to a topic and receiving a single message.
I'm using MQTT since a while to monitor some channels I subscribed. Now I want to implement to send a message as reaction to a state. I got it running with the code below, where I just react in the on_message callback (code 1 at the end). But this code uses
loop_forever()
in the main code which is blocking.
What I would like to do is to just send a single message to MQTT. When I try the following (with all different loop-functions), nothing is received by the MQTT server:
import paho.mqtt.client as mqtt
if __name__ == "__main__":
mqtt_client = mqtt.Client()
mqtt_client.connect("192.168.178.204", 1883, 60)
mqtt_client.username_pw_set(username="test", password="test")
mqtt_client.publish(topic='TEST', payload='CCCCCCCCC', retain=False)
mqtt_client.loop_write()
# mqtt_client.loop()
# mqtt_client.loop_start()
mqtt_client.disconnect()
How can I send a message to MQTT which does not block the process?
Code 1:
import paho.mqtt.client as mqtt
def on_connect(client, userdata, rc):
topic_list = [("TEST_MS", 1)]
if rc == 0:
print("Successful connected and subscribed to: {}".format(topic_list))
client.subscribe(topic_list)
def on_message(client, userdata, msg):
print(msg.payload)
client.publish(topic='TEST_MS2', payload=msg.payload, retain=False)
def on_publish(client, userdata, mid):
print("message published")
def on_subscribe(mosq, obj, mid, granted_qos):
print("Subscribed: " + str(mid) + " " + str(granted_qos))
if __name__ == "__main__":
mqtt_client = mqtt.Client()
mqtt_client.on_connect = on_connect
mqtt_client.on_message = on_message
mqtt_client.on_publish = on_publish
mqtt_client.on_subscribe = on_subscribe
mqtt_client.connect("192.168.178.204", 1883, 60)
mqtt_client.username_pw_set(username="test", password="test")
# mqtt_client.publish(topic='TEST_MS', payload='CCCCCCCCC', retain=False)
mqtt_client.loop_forever()
mqtt_client.disconnect()
If you want to send just a single message and then exit use the API specifically for that. Docs here
import paho.mqtt.publish as publish
publish.single("paho/test/single", "payload", hostname="iot.eclipse.org")