So I've been looking for a working solution, but I can't just find it...
I currently have a little program which fetches data from my serial device. This works 100%, but I have trouble with some datetime stuff....
The date and time is in UTC, but I want it in another timezone (Europe/Brussels).
Now, I have written some code that works to convert the input strings, but when I try to replace the data in my json with new data, it doesn't do anything...
Full code:
import serial
import string
import pynmea2
import datetime
import pytz
import math
import time
def utc_to_local(utc_date, utc_time):
tz_eastern = pytz.timezone('UTC')
tz_brussels = pytz.timezone('Europe/Brussels')
return tz_eastern.localize(datetime.datetime.strptime(str(utc_date) + str(utc_time), '%Y-%m-%d%H:%M:%S')).astimezone(tz_brussels).strftime("%Y-%m-%d %H:%M:%S")
def knots_to_km(knots):
return round(math.floor(knots * 1.852))
def open_serial_connection():
ser = serial.Serial()
ser.port = "/dev/ttyS0"
ser.baudrate = 9600
ser.timeout = 1
ser.open()
return ser
def readGPS(serialObject):
try:
json = dict()
latitude = 0.0
longitude = 0.0
speed = 0.0
datetimestamp = ''
sentence = serialObject.readline().decode('utf-8')
if sentence.startswith('$GPGGA'):
data = pynmea2.parse(sentence)
latitude = data.latitude
longitude = data.longitude
if sentence.startswith('$GPVTG'):
data = pynmea2.parse(sentence)
speed = data.spd_over_grnd_kmph
if sentence.startswith('$GPRMC'):
data = pynmea2.parse(sentence)
datetimestamp = utc_to_local(data.datestamp, data.timestamp)
json["lat"] = latitude
json["lon"] = longitude
json["speed"] = speed
json["time"] = datetimestamp
if json["lat"] != 0.0 and json["lon"] != 0.0:
return json
except:
pass
# Test code on pi
ser = open_serial_connection()
while True:
data = readGPS(ser)
if data is not None:
print(data)
time.sleep(0.1)
The output here is the following:
{'lat': xx.xxxxxxxxxxxxxx, 'lon': x.xxxx, 'speed': 0.0, 'time': ''}
The X-es are the correct Latitude and Longitude, they also change.
As you can see, the time is empty, but should change according to the code, right?
What am I missing here?
So the issue here was, as mentioned before, the utc_to_local, which wanted to parse "wrong" data and thus crashed the whole program...
Now the working code stays the same as before, except for the readGPS(), this has now a loop for the data, using a set():
def readGPS(serialObject):
try:
result = {}
result_set = set()
request_set = {'GPGGA', 'GPVTG', 'GPRMC'}
while not request_set.issubset(result_set) and len(result_set) != 10: # 10 == max sentences
sentence = serialObject.readline().decode('utf-8')
key = sentence[1:6]
result_set.add(key)
data = pynmea2.parse(sentence)
if key == 'GPGGA':
result["lat"] = data.latitude
result["lon"] = data.longitude
if key == 'GPVTG':
result["speed"] = data.spd_over_grnd_kmph
if key == 'GPRMC':
result["datetime"] = utc_to_local(data.datestamp, data.timestamp)
return result
except:
pass
It is also good practice to not just pass on errors, but log them or some kind of handling, but for my case, I don't really need to have error loggging (except in debugging), since I'll have alot of "wrong" responses.
Related
An inefficient version of what I'm trying to do is this:
while(True):
dynamic_variable = http(request) # where http request is subject to change values
method(dynamic_variable)
Where method(dynamic_variable isn't guaranteed to finish, and if http(request) returns a different value than method(dynamic_variable) becomes a useless function.
It seems like I should be able to change dynamic_variable more efficiently by having it "automatically" update whenever http(request) changes value.
I think what I want to do is called the "observer pattern" but I'm not quite fluent enough in code to know if that's the correct pattern or how to implement it.
A simple example would be much appreciated!
Edit:
from web3 import Web3
import json
from hexbytes import HexBytes
import numpy as np
import os
import time
INFURA_ROPSTEN_URL = "https://ropsten.infura.io/v3/<api_key>"
# metamask account information
PUBLIC_KEY = "0x3FaD9AccC3A39aDbd9887E82F94602cEA6c7F86f"
PRIVATE_KEY = "myprivatekey"
UNITS_ADDRESS = "units_address"
# from truffle build. For ABI
JSON_PATH = "../truffle/build/contracts/Units.json"
def set_up_web3():
web3 = Web3(Web3.HTTPProvider(INFURA_ROPSTEN_URL))
web3.eth.defaultAccount = PUBLIC_KEY
return web3
def get_units_contract_object():
with open(JSON_PATH, 'r') as json_file:
abi = json.load(json_file)['abi']
return web3.eth.contract(address=UNITS_ADDRESS,abi=abi)
def solve(web3, units):
nonce = np.random.randint(0,1e10)
while True:
challenge_number_hex = HexBytes(units.functions.getChallengeNumber().call()).hex()
my_digest_hex = web3.solidityKeccak(
['bytes32','address','uint256'],
[challenge_number_hex, PUBLIC_KEY, nonce]).hex()
my_digest_number = int(my_digest_hex,0)
target = units.functions.getMiningTarget().call()
if my_digest_number < target:
return (nonce, my_digest_hex)
else:
nonce += 1
def build_transaction(units, nonce, digest_hex, txn_count):
return units.functions.mint(
nonce,
digest_hex
).buildTransaction({
"nonce" : txn_count,
})
if __name__ == "__main__":
web3 = set_up_web3()
txn_count = web3.eth.getTransactionCount(PUBLIC_KEY)
units = get_units_contract_object()
_cycle_goal = 20
_prev_finish = time.time()
_wait_time = 0
while True:
target = units.functions.getMiningTarget().call()
nonce, digest_hex = solve(web3, units)
mint_txn = build_transaction(units, nonce,digest_hex, txn_count)
signed_txn = web3.eth.account.sign_transaction(mint_txn, private_key=PRIVATE_KEY)
txn_address = web3.eth.sendRawTransaction(signed_txn.rawTransaction)
txn_count += 1
print(f"Solution found! nonce={nonce}, digest_hex={digest_hex}")
_finished = time.time()
_elapsed = _finished - _prev_finish
_additional_wait = _cycle_goal - _elapsed
_wait_time += _additional_wait
print(f"Waiting {_wait_time}")
_prev_finish = _finished
time.sleep(_wait_time)
challenge_hex_number is the variable I want to update
I have a piece of Python Code running as a service that pulls weather data via API.
The code itself runs perfectly fine when everything is hunky dory, ie the network, but I have noticed that sometimes the WiFi on the Pi that is pulling the API data will drop and then the python codes seems to stop.
I have a small line of code providing the most basic of logs, but I would like to improve upon it greatly. The log code just provides me with the datetime.now so I can see when the last time the code ran was.
#!/usr/bin/python3
#import modules
import cymysql
from time import sleep
from urllib.request import urlopen
import json
import datetime
#set MySQl Variables
host = "localhost"
user = "xxx"
password = "xxx"
schema = "xxx"
#connect to MySQL DB
db = cymysql.connect(host, user, password, schema)
curs = db.cursor()
#set api key for DarkSky API
apikey="xxx"
# Latitude & longitude
lati="-26.20227"
longi="28.04363"
# Add units=si to get it in sensible ISO units.
url="https://api.forecast.io/forecast/"+apikey+"/"+lati+","+longi+"?units=si"
#begin infinite loop
while True:
#convert API reading to json and readable array 'weather'
meteo=urlopen(url).read()
meteo = meteo.decode('utf-8')
weather = json.loads(meteo)
#set variables for current weather
cTemp = (weather['currently']['temperature'])
cCond = (weather['currently']['summary'])
cRain1 = (weather['currently']['precipProbability'])
cRain2 = cRain1*100
cIcon = (weather['currently']['icon'])
oaSum = (weather['daily']['summary'])
#print variables - for testing purposes
#print (cTemp)
#print (cCond)
#print (cRain2)
#print (cIcon)
#print (oaSum)
#extract daily data from 'weather' array
daily = (weather['daily']['data'])
#create new arrays for daily variables
listHigh = []
listLow = []
listCond = []
listRain = []
listIcon = []
#set daily variables
for i in daily:
listHigh.append(i['temperatureHigh'])
for i in range(0,len(listHigh)):
high1 = listHigh[0]
high2 = listHigh[1]
high3 = listHigh[2]
high4 = listHigh[3]
high5 = listHigh[4]
high6 = listHigh[5]
high7 = listHigh[6]
high8 = listHigh[7]
for o in daily:
listLow.append(o['temperatureLow'])
for o in range(0,len(listLow)):
low1 = listLow[0]
low2 = listLow[1]
low3 = listLow[2]
low4 = listLow[3]
low5 = listLow[4]
low6 = listLow[5]
low7 = listLow[6]
low8 = listLow[7]
for p in daily:
listCond.append(p['summary'])
for p in range(0,len(listCond)):
cond1 = listCond[0]
cond2 = listCond[1]
cond3 = listCond[2]
cond4 = listCond[3]
cond5 = listCond[4]
cond6 = listCond[5]
cond7 = listCond[6]
cond8 = listCond[7]
for m in daily:
listRain.append(m['precipProbability'])
for m in range(0,len(listRain)):
rain1 = listRain[0]
rain2 = listRain[1]
rain3 = listRain[2]
rain4 = listRain[3]
rain5 = listRain[4]
rain6 = listRain[5]
rain7 = listRain[6]
rain8 = listRain[7]
#convert rain chance to readable percentage
prain1 = rain1*100
prain2 = rain2*100
prain3 = rain3*100
prain4 = rain4*100
prain5 = rain5*100
prain6 = rain6*100
prain7 = rain7*100
prain8 = rain8*100
for l in daily:
listIcon.append(l['icon'])
for l in range (0,len(listIcon)):
icon1 = listIcon[0]
icon2 = listIcon[1]
icon3 = listIcon[2]
icon4 = listIcon[3]
icon5 = listIcon[4]
icon6 = listIcon[5]
icon7 = listIcon[6]
icon8 = listIcon[7]
#print daily variables - for testing purposes
#print (high1)
#print (low1)
#print (cond1)
#print (prain1)
#print (icon1)
#print (high2)
#print (low2)
#print (cond2)
#print (prain2)
#print (icon2)
#update data in DataBase
try:
sql_update_query = """UPDATE weather SET current_temp = %s, cur$
varis = (cTemp, cCond, cRain2, cIcon, high1, low1, cond1, prain$
curs.execute(sql_update_query, varis)
db.commit()
except db.Error as error:
print("Error: {}".format(error))
db.rollback()
#write date to log file
with open ("/home/pi/CoRo/Projects/WeatherMan/weatherlog.txt", mode="w") as file:
file.write('Last Data was pulled at: %s' %(datetime.datetime.now()))
#set loop to sleep for 10 minutes and go again
sleep(600)
I understand that the Database Code is snipped, but it is just the variables being put in to the database, which I can see works.
However if the network disconnects, the code stops and the database is left with the last polled API data.
How would I restart the python code if the API get fails?
Thanks in advance,
You could rewrite the portion of your code that pulls the weather data as a function or separate module. This would allow you to call it only when the network connection is working. Some pseudo code below:
if network_connection:
pull_weather_data()
else:
do_something()
do_something() could be an effort to reconnect to the network, such as resetting your network adapter.
You could determine the state of the network connection by trying to ping your router or an external IP like one of Google's DNS server (8.8.8.8 or 8.8.4.4).
To avoid nested loops you could use the continue clause. For example:
while True:
if network_connection:
pull_weather_data()
else:
reset_network_connection()
time.sleep(180) # Sleep for 3 minutes.
continue
The continue will send the interpreter back to the start of the while loop. From there it will check the network connection and either pull data or reset the network connection and sleep for another 3 minutes.
Using Quernons answer above the code has been edited as follows:
#!/usr/bin/python3
#import modules
import os
import cymysql
from time import sleep
from urllib.request import urlopen
import json
import datetime
#set MySQl Variables
host = "localhost"
user = "xxx"
password = "xxx"
schema = "xxx"
#connect to MySQL DB
db = cymysql.connect(host, user, password, schema)
curs = db.cursor()
#set api key for DarkSky API
apikey="xxx"
# Latitude & longitude
lati="-26.20227"
longi="28.04363"
# Add units=si to get it in sensible ISO units not stupid Fahreneheit.
url="https://api.forecast.io/forecast/"+apikey+"/"+lati+","+longi+"?units=si"
#begin infinite loop
while True:
#function to check if there is an internet connection
def check_ping():
hostname = "8.8.8.8"
response = os.system("ping -c 1 " + hostname)
#and then check the response...
if response == 0:
pingstatus = 0
else:
pingstatus = 1
return pingstatus
networkstatus = check_ping()
#print check_ping() - for testing purposes
#print (networkstatus)
#function to pull weather data from API
def get_weather():
#insert weather data here with no changes
if networkstatus == 0:
get_weather()
else:
print ("Resetting Network Adapters")
dwnnw = 'ifconfig wlan0 down'
upnw = 'ifconfig wlan0 up'
os.system(dwnnw)
os.system(upnw)
sleep(180)
continue
I'm trying to transfer ca. 10GB of json data (tweets in my case) to a collection in arangodb. I'm also trying to use joblib for it:
from ArangoConn import ArangoConn
import Userdata as U
import encodings
from joblib import Parallel,delayed
import json
from glob import glob
import time
def progress(total, prog, start, stri = ""):
if(prog == 0):
print("")
prog = 1;
perc = prog / total
diff = time.time() - start
rem = (diff / prog) * (total - prog)
bar = ""
for i in range(0,int(perc*20)):
bar = bar + "|"
for i in range(int(perc*20),20):
bar = bar + " "
print("\r"+"progress: " + "[" + bar + "] " + str(prog) + " of " +
str(total) + ": {0:.1f}% ".format(perc * 100) + "- " +
time.strftime("%H:%M:%S", time.gmtime(rem)) + " " + stri, end="")
def processfile(filepath):
file = open(filepath,encoding='utf-8')
s = file.read()
file.close()
data = json.loads(s)
Parallel(n_jobs=12, verbose=0, backend="threading"
(map(delayed(ArangoConn.createDocFromObject), data))
files = glob(U.path+'/*.json')
i = 1
j = len(files)
starttime = time.time()
for f in files:
progress(j,i,starttime,f)
i = i+1
processfile(f)
and
from pyArango.connection import Connection
import Userdata as U
import time
class ArangoConn:
def __init__(self,server,user,pw,db,collectionname):
self.server = server
self.user = user
self.pw = pw
self.db = db
self.collectionname = collectionname
self.connection = None
self.dbHandle = self.connect()
if not self.dbHandle.hasCollection(name=self.collectionname):
coll = self.dbHandle.createCollection(name=collectionname)
else:
coll = self.dbHandle.collections[collectionname]
self.collection = coll
def db_createDocFromObject(self, obj):
data = obj.__dict__()
doc = self.collection.createDocument()
for key,value in data.items():
doc[key] = value
doc._key= str(int(round(time.time() * 1000)))
doc.save()
def connect(self):
self.connection = Connection(arangoURL=self.server + ":8529",
username=self.user, password=self.pw)
if not self.connection.hasDatabase(self.db):
db = self.connection.createDatabase(name=self.db)
else:
db = self.connection.databases.get(self.db)
return db
def disconnect(self):
self.connection.disconnectSession()
def getAllData(self):
docs = []
for doc in self.collection.fetchAll():
docs.append(self.doc_to_result(doc))
return docs
def addData(self,obj):
self.db_createDocFromObject(obj)
def search(self,collection,search,prop):
docs = []
aql = """FOR q IN """+collection+""" FILTER q."""+prop+""" LIKE
"%"""+search+"""%" RETURN q"""
results = self.dbHandle.AQLQuery(aql, rawResults=False, batchSize=1)
for doc in results:
docs.append(self.doc_to_result(doc))
return docs
def doc_to_result(self,arangodoc):
modstore = arangodoc.getStore()
modstore["_key"] = arangodoc._key
return modstore
def db_createDocFromJson(self,json):
for d in json:
doc = self.collection.createDocument()
for key,value in d.items():
doc[key] = value
doc._key = str(int(round(time.time() * 1000)))
doc.save()
#staticmethod
def createDocFromObject(obj):
c = ArangoConn(U.url, U.user, U.pw, U.db, U.collection)
data = obj
doc = c.collection.createDocument()
for key, value in data.items():
doc[key] = value
doc._key = doc["id"]
doc.save()
c.connection.disconnectSession()
It kinda works like that. My problem is that the data that lands in the database is somehow mixed up.
as you can see in the screenshot "id" and "id_str" are not the same - as they should be.
what i investigated so far:
I thought that at some points the default keys in the databese may "collide"
because of the threading so I set the key to the tweet id.
I tried to do it without multiple threads. the threading doesn't seem to be
the problem
I looked at the data I send to the database... everything seems to be fine
But as soon as I communicate with the db the data mixes up.
My professor thought that maybe something in pyarango isn't threadsafe and it messes up the data but I don't think so as threading doesn't seem to be the problem.
I have no ideas left where this behavior could come from...
Any ideas?
The screenshot shows the following values:
id : 892886691937214500
id_str : 892886691937214465
It looks like somewhere along the way the value is converted to an IEEE754 double, which cannot safely represent the latter value. So there is potentially some precision loss due to conversion.
A quick example in node.js (JavaScript is using IEEE754 doubles for any number values greater than 0xffffffff) shows that this is likely the problem cause:
$ node
> 892886691937214500
892886691937214500
> 892886691937214465
892886691937214500
So the question is where the conversion does happen. Can you check whether the python client program is correctly sending the expected values to ArangoDB, or does it already send the converted/truncated values?
In general, any integer number that exceeds 0x7fffffffffffffff will be truncated when stored in ArangoDB, or converted to an IEEE754 double. This can be avoided by storing the number values inside a string, but of course comparing two number strings will produce different results than comparing two numbers (e.g. "10" < "9" vs. 10 > 9).
So I'm making an application using a GPS module, Python and a MySQL database.
So I have written some code, to try to capture the data from the GPS and store it in the database. I'm using a plugin called "pynmea2" to parse some of the data (longitude and latitude). However, I need more data then that, so, I already tried ALOT different things, but my program keeps crashing the whole time. Could someone help me out with this?
Most of the time I get all the data from the Serial connection, but I want to be able to strip data from it. So example of what I get:[b'$GPGGA,093512.000,,,,,0,3,,,M,,M,,*47\r\n', b'$GPGLL,,,,,093512.000,V,N*76\r\n', b'$GPGSA,A,1,,,,,,,,,,,,,,,*1E\r\n', b'$GPGSV,3,1,11,15,72,214,,24,52,276,,13,48,141,,17,31,093,29*70\r\n', b'$GPGSV,3,2,11,18,28,292,,28,27,049,25,19,24,120,24,12,23,211,13*7E\r\n', b'$GPGSV,3
Well, it's not that simple to extract data from it,but it works out just fine with the pynmea2 library (only library I'm allowed to use.
So, I need the speed, the latitude and longitude for now, but the speed is bothering me now. It gives ValueError: could not convert string to float: "22*49\\r\\n'"
alot of times because I don't do a proper way of finding the data and then "parsing" it.
Here is my code I'm currently using;
from model.GPSParser import GPSParser
from model.DB import DB
import serial
import time
import datetime
import pynmea2
#########################################
# This is the main code to setup the
# serial connection with the GPS module.
# it needs to be OR runt as root OR as
# pi with all the root rights.
#########################################
port = "/dev/ttyAMA0"
ser = serial.Serial(port, 9600, timeout=0)
#########################################
# These are all the global variables
# to be used. All defined and set to
# zero or their standard 'Null' value.
#########################################
lat = 0.0
lon = 0.0
cur_speed = 0.0
while True:
try:
# Get the data from the serial monitor.
data = str(ser.readlines()).lstrip("b'")[:-3]
# print(data)
#########################################
# Find the speed, to check if we're
# standing still or not. Save it in a
# #var speed
#########################################
if data.find('$GPVTG') != -1:
cur_speed = data.split(",")[7]
#########################################
# Get the Latitude and Longitude
#########################################
if data.find('$GPGGA') != -1:
print(data)
# Check whether the data strings are empty or not.
if GPSParser.parseLatitude(data) != "" and GPSParser.parseLongitude(data) != "":
lat = GPSParser.parseLatitude(data)
lon = GPSParser.parseLongitude(data)
# Debug printing
# print("Latitude: " + GPSParser.parseLatitude(data))
# print("Longitude: " + GPSParser.parseLongitude(data))
# print("Speed: " + cur_speed)
#########################################
# Insert the coordinates into the database
# Be sure to check of we are really driving
# So when the speed is higher then 5 km/u
# Store everything into the database.
#########################################
if float(cur_speed) > 5.0:
db = DB()
db.insertCoordinates(lat, lon, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
# Wait a bit to not overload the Serial port
time.sleep(0.5)
############################################################
# The error handling
############################################################
except serial.serialutil.SerialException:
ser.close()
port = "/dev/ttyAMA0"
ser = serial.Serial(port, 9600, timeout=0)
continue
except pynmea2.ParseError:
# print("Error on parsing object, continuing...")
continue
except BlockingIOError:
# print("Blocking I/O error, continuing...")
continue
except TypeError:
# print("Type error, continuing...")
continue
except IndexError:
# print("To catch an error...")
continue
except KeyboardInterrupt:
print("\nProgram stopped.")
exit()
So the import from model doesn't do much, only the database connection and the "gps parser" is only the pynmea that parses a string of data and then returns it.
So what I want is something like:
It gets all the data it pulses once per second,
it then splits it all into chucks where it starts with the $GP variable, then I can search for the second variable part, for example VTG or GGA. And then I can use that string to make conversions to the right value to extract the speed, latitude, longitude and other data if needed.
Hope you guys can understand me well and can help me out.
Not sure if that solves your problem, but pynmea2 has speed attributes, defined in talker.py.
import pynmea2
for i, line in enumerate(open('/tmp/nmea.txt').readlines()):
# parsing via pynmea
msg = pynmea2.parse(line.strip())
if msg.sentence_type == 'VTG':
print ('parsing line %s with pynmea:' % i, float(msg.spd_over_grnd_kmph))
# parsing via manually
if line.startswith('$GPVTG'):
cur_speed = line.split(",")[7]
print ('parsing line %s manually:' % i, float(cur_speed))
Returns:
parsing line 1 with pynmea: 91.626
parsing line 1 manually: 91.626
parsing line 10 with pynmea: 90.842
parsing line 10 manually: 90.842
parsing line 19 with pynmea: 89.676
parsing line 19 manually: 89.676
I am trying to consume hp nnmi web services. But here you can see I am using milliseconds as value in filter2.value = d_in_ms, and this is not working for me. I am able to see result when I use values like filter2.value = "1493078400000", Please tell me if we can use int values like below:
#!/usr/bin/python
from suds.client import Client
from suds.transport.http import HttpAuthenticated
import datetime
import time
now = datetime.datetime.now()
currenttime = now - datetime.timedelta(hours=12)
epochtime = time.mktime(currenttime.timetuple())
print epochtime
d_in_ms = int(epochtime)*1000
t = HttpAuthenticated(username='xxxxx', password='xxxx')
url = 'http://example.com/IncidentBeanService/IncidentBean?wsdl'
client = Client(url, transport=t)
filter1 = client.factory.create('ns2:condition')
filter1.name = "sourceNodeName"
filter1.operator = "EQ"
filter1.value = "DEVICE"
filter2 = client.factory.create('ns2:condition')
filter2.name = "lastOccurrenceTime"
filter2.operator = "GT"
filter2.value = d_in_ms
filter = client.factory.create('ns2:expression')
filter.operator = "AND"
filter.subFilters = [filter1, filter2]
allincidents = client.service.getIncidents(filter)
print "Nodes in topology:", len(allincidents.item)
for i in allincidents.item[:]:
print i
I am able to see result when I use values like filter2.value = "1493078400000"
According to this statement, it looks filter2.value should be a string. That would suggest that you need to use:
filter2.value = str(d_in_ms)