Develop Reference

python JSON format invalid

I'm trying to get date and time flowing into Azure IoT hub to enable me to analyze using Azure DX as time series. I can get the temperature and humidity (humidity at the moment is just a random number). If I use this code, all works well and the JSON is well formatted and flows into IoT hub and onto Azure DX:
The basis for the code is taken from the Microsoft examples here - https://github.com/Azure-Samples/azure-iot-samples-python/blob/master/iot-hub/Quickstarts/simulated-device/SimulatedDeviceSync.py
import asyncio
import random
from azure.iot.device import Message
from azure.iot.device.aio import IoTHubDeviceClient
import time
from datetime import datetime
from w1thermsensor import W1ThermSensor
sensor = W1ThermSensor()
import json
CONNECTION_STRING = "xxxxx"
HUMIDITY = 60
MSG_TXT = '{{"temperature": {temperature},"humidity": {humidity}}}'
async def main():
try:
# Create instance of the device client
client = IoTHubDeviceClient.create_from_connection_string(CONNECTION_STRING)
print("Simulated device started. Press Ctrl-C to exit")
while True:
humidity = round(HUMIDITY + (random.random() * 20), 2)
temperature = sensor.get_temperature()
msg_txt_formatted = MSG_TXT.format(temperature=temperature, humidity=humidity)
message = Message(msg_txt_formatted)
# Send a message to the IoT hub
print(f"Sending message: {message}")
await client.send_message(message)
await asyncio.sleep(1)
except KeyboardInterrupt:
print("Simulated device stopped")
if __name__ == '__main__':
asyncio.run(main())
The JSON format is valid and works well -
{ "temperature": 7, "humidity": 66.09 }
If I try to add a date/time field like this:
import asyncio
import random
from azure.iot.device import Message
from azure.iot.device.aio import IoTHubDeviceClient
import time
from datetime import datetime
from w1thermsensor import W1ThermSensor
sensor = W1ThermSensor()
import json
CONNECTION_STRING = "xxxxx"
HUMIDITY = 60
x = datetime.now()
timesent = str(x)
MSG_TXT = '{{"temperature": {temperature},"humidity": {humidity},"timesent": {timesent}}}'
async def main():
try:
# Create instance of the device client
client = IoTHubDeviceClient.create_from_connection_string(CONNECTION_STRING)
print("Simulated device started. Press Ctrl-C to exit")
while True:
humidity = round(HUMIDITY + (random.random() * 20), 2)
temperature = sensor.get_temperature()
msg_txt_formatted = MSG_TXT.format(temperature=temperature, humidity=humidity, timesent=timesent)
message = Message(msg_txt_formatted)
# Send a message to the IoT hub
print(f"Sending message: {message}")
await client.send_message(message)
await asyncio.sleep(1)
except KeyboardInterrupt:
print("Simulated device stopped")
if __name__ == '__main__':
asyncio.run(main())
The output from the JSON is no longer valid and Azure DX will not map. The invalid JSON I get is:
"{\"temperature\": 7,\"humidity\": 72.88, \"timesent\": 2022-11-08 14:21:04.021812}"
I suspect this is something to do with the date/time being formatted as a string, but I'm totally lost.
Would anyone have any ideas how I can send this data?

#JoeHo, thank you for pointing the sources that helped you resolve the issue. I am posting the solution here so that other community members facing similar issue would benefit. Making the below modifications to the code helped me resolve the issue.
def json_serial(obj):
if isinstance(obj, (datetime, date)):
return obj.isoformat()
raise TypeError ("Type %s not serializable" % type(obj))
x = datetime.now().isoformat();
timesent = dumps(datetime.now(), default=json_serial);
MSG_TXT = '{{"temperature": {temperature},"humidity": {humidity}, "timesent": {timesent}}}'
My table on the Azure data explorer has the following filed definitions defined.
.create table jsondata (temperature: real, humidity: real, timesent: datetime)
My data mapping query is as below
.create table jsondata ingestion json mapping 'jsonMapping' '[{"column":"humidity","path":"$.humidity","datatype":"real"},{"column":"temperature","path":"$.temperature","datatype":"real"}, {"column":"timesent","path":"$.timesent","datatype":"datetime"}]'
I then connected the Azure Data Explorer table to IoT Hub using the steps outlined in the following resource Connect Azure Data Explorer table to IoT hub
When I execute the program, I could see the Azure IoT Hub telemetry data flow getting bound to the Azure Data explorer table without any issues.

Python FIX API - no response at Logon

I've been trying to set up a connection to the FIX API for the GDAX crpyto exchange, but I can't seem to logon properly. I'm using the code below to generate the message:
import time
import simplefix
import socket
import base64
import hmac
import hashlib
from datetime import datetime
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 4197))
API_KEY = "KEY_GOES_HERE"
PASSPHRASE = "PASSPHRASE_GOES_HERE"
API_SECRET = "SECRET_GOES_HERE"
seq_num = "1"
sendingTime = str(datetime.utcnow()).replace("-","").replace(" ", "-")[:-3]
rawSig = "\x01".join([sendingTime, "A", seq_num, API_KEY, "Coinbase", PASSPHRASE]).encode("utf-8")
hmac_key = base64.b64decode(API_SECRET)
signature = hmac.new(hmac_key, rawSig, hashlib.sha256)
sign_b64 = base64.b64encode(signature.digest()).decode()
msg = simplefix.FixMessage()
msg.append_pair(8, "FIX.4.2")
msg.append_pair(35, "A")
msg.append_pair(49, API_KEY)
msg.append_pair(52, sendingTime)
msg.append_pair(56, "Coinbase")
msg.append_pair(98, "0")
msg.append_pair(108, "30")
msg.append_pair(554, PASSPHRASE)
msg.append_pair(96, sign_b64)
msg.append_pair(8013, "Y")
print(msg.encode())
s.sendall(msg.encode("ascii"))
print(s.recv(4096))
And I'm getting 0 bytes of response from the server. As far as I can tell, the stunnel is working properly (connects and validates certificates successfully, but disconnects after sending my logon message).
Have just tried with a newly generated API key, passphrase and secret but to no avail.
For reference, I was working from the question asked here: How to send FIX logon message with Python to GDAX but I'm not allowed to comment there.
If anyone has any ideas, would be appreciated. Below is an example of the fix message generated, passwords removed in post:
8=FIX.4.2\x019=161\x0135=A\x0149=[KEY_REMOVED]\x0152=20180113-18:24:07.889\x0156=Coinbase\x0198=0\x01108=30\x01554=[PASSPHRASE_REMOVED]\x0196=jueSJHoSNQM2BOCN3KM0mgB2/9tXpICbg4amqDKc2wY=\x018013=Y\x0110=053\x01

Connect to Watson Speech-to-Text API using websockets for real-time transcription

I am trying to write a script to call the Watson Speech-to-Text (STT) API to continually transcribe speech being recorded through a microphone word-for-word in real-time. I read that this should be possible using the Websockets version of the API.
I have a Python script that should be able to do this on Linux (assuming the dependencies are installed), however, it does not work on Mac OS X.
from ws4py.client.threadedclient import WebSocketClient
import base64, json, ssl, subprocess, threading, time
class SpeechToTextClient(WebSocketClient):
def __init__(self):
ws_url = "wss://stream.watsonplatform.net/speech-to-text/api/v1/recognize"
username = "your username"
password = "your password"
auth_string = "%s:%s" % (username, password)
base64string = base64.encodestring(auth_string).replace("\n", "")
self.listening = False
try:
WebSocketClient.__init__(self, ws_url,
headers=[("Authorization", "Basic %s" % base64string)])
self.connect()
except: print "Failed to open WebSocket."
def opened(self):
self.send('{"action": "start", "content-type": "audio/l16;rate=16000"}')
self.stream_audio_thread = threading.Thread(target=self.stream_audio)
self.stream_audio_thread.start()
def received_message(self, message):
message = json.loads(str(message))
if "state" in message:
if message["state"] == "listening":
self.listening = True
print "Message received: " + str(message)
def stream_audio(self):
while not self.listening:
time.sleep(0.1)
reccmd = ["arecord", "-f", "S16_LE", "-r", "16000", "-t", "raw"]
p = subprocess.Popen(reccmd, stdout=subprocess.PIPE)
while self.listening:
data = p.stdout.read(1024)
try: self.send(bytearray(data), binary=True)
except ssl.SSLError: pass
p.kill()
def close(self):
self.listening = False
self.stream_audio_thread.join()
WebSocketClient.close(self)
try:
stt_client = SpeechToTextClient()
raw_input()
finally:
stt_client.close()
Ideally, I wouldn't even be doing this in Python, but R, which is my native language which I will have to transfer the results back to to be processed anyway.
Could anyone provide me with a solution for how I can get a streamed transcription?

Not sure if this answer is exactly what you want, but sounds like an issue with the parameter continuous.
As you can see, has the lib Python SDK inside Watson-developer-cloud.
You can install with: pip install watson-developer-cloud
import json
from os.path import join, dirname
from watson_developer_cloud import SpeechToTextV1
speech_to_text = SpeechToTextV1(
username='YOUR SERVICE USERNAME',
password='YOUR SERVICE PASSWORD',
x_watson_learning_opt_out=False
)
print(json.dumps(speech_to_text.models(), indent=2))
print(json.dumps(speech_to_text.get_model('en-US_BroadbandModel'), indent=2))
with open(join(dirname(__file__), '../resources/speech.wav'),
'rb') as audio_file:
data = json.dumps(speech_to_text.recognize(audio_file, content_type='audio/wav', timestamps=False, word_confidence=False, continuous=True), indent=2)
print(data)
Obs.: The service returns an array of results with one per utterance.
In the line #L44, has the params that you can use, so, for continuous transcription, you need to use the parameter continuous and set to true like the example above.
See the Official Documentation talking about Websockets for keeping the connection alive. (Maybe this is what you need).

For some good examples of how to do this using R, check out these great blog posts by Ryan Anderson.
Voice Controlled Music Machine
Python as a tool to help with Continuous Audio - This shows how you can use R for your main logic, and use Python to deal with handling just the audio.
Ryan has done a lot of work with R and the Watson API's - he shares a lot of his knowledge on his blog.

Python - Poloniex Push API

I am trying to get live data in Python 2.7.13 from Poloniex through the push API.
I read many posts (including How to connect to poloniex.com websocket api using a python library) and I arrived to the following code:
from autobahn.twisted.wamp import ApplicationSession
from autobahn.twisted.wamp import ApplicationRunner
from twisted.internet.defer import inlineCallbacks
import six
class PoloniexComponent(ApplicationSession):
def onConnect(self):
self.join(self.config.realm)
#inlineCallbacks
def onJoin(self, details):
def onTicker(*args):
print("Ticker event received:", args)
try:
yield self.subscribe(onTicker, 'ticker')
except Exception as e:
print("Could not subscribe to topic:", e)
def main():
runner = ApplicationRunner(six.u("wss://api.poloniex.com"), six.u("realm1"))
runner.run(PoloniexComponent)
if __name__ == "__main__":
main()
Now, when I run the code, it looks like it's running successfully, but I don't know where I am getting the data. I have two questions:
I would really appreciate if someone could walk me through the process of subscribing and getting ticker data, that I will elaborate in python, from step 0: I am running the program on Spyder on Windows. Am I supposed to activate somehow Crossbar?
How do I quit the connection? I simply killed the process with Ctrl+c and now when I try to run it agan, I get the error: ReactorNonRestartable.

I ran into a lot of issues using Poloniex with Python2.7 but finally came to a solution that hopefully helps you.
I found that Poloniex has pulled support for the original WAMP socket endpoint so I would probably stray from this method altogether. Maybe this is the entirety of the answer you need but if not here is an alternate way to get ticker information.
The code that ended up working best for me is actually from the post you linked to above but there was some info regarding currency pair ids I found elsewhere.
import websocket
import thread
import time
import json
def on_message(ws, message):
print(message)
def on_error(ws, error):
print(error)
def on_close(ws):
print("### closed ###")
def on_open(ws):
print("ONOPEN")
def run(*args):
# ws.send(json.dumps({'command':'subscribe','channel':1001}))
ws.send(json.dumps({'command':'subscribe','channel':1002}))
# ws.send(json.dumps({'command':'subscribe','channel':1003}))
# ws.send(json.dumps({'command':'subscribe','channel':'BTC_XMR'}))
while True:
time.sleep(1)
ws.close()
print("thread terminating...")
thread.start_new_thread(run, ())
if __name__ == "__main__":
websocket.enableTrace(True)
ws = websocket.WebSocketApp("wss://api2.poloniex.com/",
on_message = on_message,
on_error = on_error,
on_close = on_close)
ws.on_open = on_open
ws.run_forever()
I commented out the lines that pull data you don't seem to want, but for reference here is some more info from that previous post:
1001 = trollbox (you will get nothing but a heartbeat)
1002 = ticker
1003 = base coin 24h volume stats
1010 = heartbeat
'MARKET_PAIR' = market order books
Now you should get some data that looks something like this:
[121,"2759.99999999","2759.99999999","2758.000000‌​00","0.02184376","12‌​268375.01419869","44‌​95.18724321",0,"2767‌​.80020000","2680.100‌​00000"]]
This is also annoying because the "121" at the beginning is the currency pair id, and this is undocumented and also unanswered in the other stack overflow question referred to here.
However, if you visit this url: https://poloniex.com/public?command=returnTicker it seems the id is shown as the first field, so you could create your own mapping of id->currency pair or parse the data by the ids you want from this.
Alternatively, something as simple as:
import urllib
import urllib2
import json
ret = urllib2.urlopen(urllib2.Request('https://poloniex.com/public?command=returnTicker'))
print json.loads(ret.read())
will return to you the data that you want, but you'll have to put it in a loop to get constantly updating information. Not sure of your needs once the data is received so I will leave the rest up to you.
Hope this helps!

I made, with the help of other posts, the following code to get the latest data using Python 3.x. I hope this helps you:
#TO SAVE THE HISTORICAL DATA (X MINUTES/HOURS) OF EVERY CRYPTOCURRENCY PAIR IN POLONIEX:
from poloniex import Poloniex
import pandas as pd
from time import time
import os
api = Poloniex(jsonNums=float)
#Obtains the pairs of cryptocurrencies traded in poloniex
pairs = [pair for pair in api.returnTicker()]
i = 0
while i < len(pairs):
#Available candle periods: 5min(300), 15min(900), 30min(1800), 2hr(7200), 4hr(14400), and 24hr(86400)
raw = api.returnChartData(pairs[i], period=86400, start=time()-api.YEAR*10)
df = pd.DataFrame(raw)
# adjust dates format and set dates as index
df['date'] = pd.to_datetime(df["date"], unit='s')
df.set_index('date', inplace=True)
# Saves the historical data of every pair in a csv file
path=r'C:\x\y\Desktop\z\folder_name'
df.to_csv(os.path.join(path,r'%s.csv' % pairs[i]))
i += 1

How to POST data from Zenoss to a URL using a python script

With the risk to be a little off-topic I'll give it a try:
I have a small script written in python, which is supposed to POST some data on a specific URL. The problem is:
I am integrating this script in zenoss:
Here is the path from zenoss (GUI)
`Events(http://ip:port/zport/dmd/Events/evconsole)->click on some event class->settings->transform(here I copy-paste my script)`
When the events are triggered, the script is called, but no data it's POSTed to the other IP (a different server) even if the script is running correctly - I don't get any error messages in logs or in the GUI. So, I guess there are some restrictions at the network level and somehow the POSTed data isn't sent by Zenoss.
Could you please guys tell me what should I modify (perhaps a .conf file or hosts file) or what should I do to be able to POST some data on another server ?
The code is here (if relevant):
import urllib2 as urllib
from urllib import urlencode
from os.path import join as joinPath
from traceback import print_tb
from os.path import isfile
from sys import exc_info
gh_url = 'http://ip/txsms.cgi'
APPLICATION_PATH = '/srv/sms_alert/'
ALERT_POINT_PATH = joinPath(APPLICATION_PATH, 'alert_contact')
try:
evt.data = 'Some text'
isInProduction = False
if evt.prodState == 1000:
isInProduction = True
if isInProduction and isfile(ALERT_POINT_PATH):
alertContactContent = None
with open(ALERT_POINT_PATH, 'r') as alertContactFile:
alertContactContent = alertContactFile.read()
alertContactContent = alertContactContent.splitlines()
if alertContactContent:
evt.summary = '#[ SMS ALERT ]# {}'.format(evt.summary)
for alertContactContentLine in alertContactContent:
data = {'phonenr': alertContactContentLine, 'smstxt': evt.summary, 'Submit': 'SendSms'}
urllib.urlencode(data)
req = urllib.Request(gh_url, data)
password_manager = urllib.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, gh_url, 'admin', 'password')
auth_manager = urllib.HTTPBasicAuthHandler(password_manager)
opener = urllib.build_opener(auth_manager)
urllib.install_opener(opener)
handler = urllib.urlopen(req)
else:
evt.summary = '#[ ERROR: SMS ALERT NO CONTACT ]# {}'.format(evt.summary)
except Exception as e:
ex_type, ex, tb = exc_info()
print('\n #[ERROR]# TRANSFORM: exception: {ex}\n'.format(ex=e))
print('\n #[ERROR]# TRANSFORM: exception traceback: {trace}\n'.format(trace=print_tb(tb)))
So to be more specific:
I am trying to authenticate and then POST some data to: http://ip/txsms.cgi. If I am running the script directly from my CentOS machine (and remove the events - as they can be used only inside zenoss) it works and it accomplishes what I want, but when I copy-paste the code in Zenoss, the data isn't POSTed.
Any ideas on this ? (how can I have this done)