Develop Reference

Script Python to log values from modbus tcp to Mysql

I'm trying to make a script to read holding registers from modbus TCP devices and insert the UINT32 decoded into MySQL :
Read 2 registers with pymodbus
Decode into uint32
Insert decoded values into mySQL
Start again every x minutes
I'm not a programmer and discovering Python.
I tried this code which seems to work but I do not understand everything, I do not know if it is the right solution to use "threading". Do you have something better to offer me?
Thank you
import time
import pymysql.cursors
from pymodbus.constants import Endian
from pymodbus.payload import BinaryPayloadDecoder
from threading import Thread, Lock
from pyModbusTCP.client import ModbusClient
# connect database
#sqlconnection = pymysql.connect(host='127.0.0.1',
# user='root',
# password='',
# database='database',
# charset='utf8mb4',
# cursorclass=pymysql.cursors.DictCursor)
# read devices informations (ID, IP, Port, Register address)
#with sqlconnection:
# with sqlconnection.cursor() as cursor:
# sql = "SELECT `dev_id`, H0.hw_address, H0.hw_port FROM `devicestatus` LEFT JOIN `hardware` H0 ON H0.hw_id = devicestatus.dev_hw_id WHERE H0.hw_enabled = 1 AND devicestatus.dev_used = 1 ORDER BY H0.hw_address, dev_name"
# cursor.execute(sql,)
# result = cursor.fetchall()
# sample of result
result = [{'dev_id': 7, 'hw_address': '127.0.0.1', 'hw_port': 502}, {'dev_id': 8, 'hw_address': '127.0.0.1', 'hw_port': 502}, {'dev_id': 9, 'hw_address': '127.0.0.1', 'hw_port': 502}]
# set global variables
regs = []
sf_value = 0
# init a thread lock
regs_lock = Lock()
# modbus polling thread
def polling_thread():
global regs
# polling loop
while True:
REGISTER_ADDRESS = 98 #First device is at address %MD100 (in Schneider M221 simulator), i dont' understand why i need to do -2
# For each dictionary of the list, "extract" each value of element : information useful to launch Modbus requests, one per device.
for val in result:
# set modbusTCP variables :
DEV_ID = val['dev_id']
SERVER_HOST = val['hw_address']
SERVER_PORT = val['hw_port']
SERVER_ID = 1
REGISTER_ADDRESS = REGISTER_ADDRESS + 2 # to read %MD100 %MD102 %MD104 ...
REGISTER_COUNT = 2
print("Target :", SERVER_HOST, SERVER_PORT, REGISTER_ADDRESS)
client = ModbusClient(host=SERVER_HOST, port=int(SERVER_PORT), unit_id=int(SERVER_ID), auto_open=True, auto_close=True, timeout=2)
# keep TCP open
if not client.is_open():
client.open()
# do modbus reading on socket
reg_read = client.read_holding_registers(REGISTER_ADDRESS, REGISTER_COUNT)
# if read is ok, store result in regs (with thread lock synchronization)
if reg_read:
# decode the 2 registers into a 32 bit integer
decoder = BinaryPayloadDecoder.fromRegisters(reg_read, byteorder=Endian.Big, wordorder=Endian.Little)
sf_value = decoder.decode_32bit_int()
with regs_lock:
regs = sf_value
print(regs)
# To do : insert each data into sql table
#with sqlconnection:
#with sqlconnection.cursor() as cursor:
# Create a new record
#sql = "INSERT INTO ...
# x sec before next polling
time.sleep(2)
# start polling thread
tp = Thread(target=polling_thread)
# set daemon: polling thread will exit if main thread exit
tp.daemon = True
tp.start()
# display loop (in main thread)
while True:
# print regs (with thread lock synchronization)
with regs_lock:
#print(regs)
print("What is this part ? everything is done in the polling_thread")
# x sec before next print
time.sleep(2)

AttributeError: module 'urllib3' has no attribute 'urlopen' in python

I am trying to send temperature data over onto one of my website currently online. This code consists of measuring the temperature through a sensor(ds18b20), sending that data onto a mysql databse entitled temp_pi and specifically onto a table intitled TAB_CLASSROOM and lastly sending that data onto a webpage of mine. Everything in this code runs except for the sendDataToServer() part. I specify the error right before this particular line. I have the PHP set up on my website for this to work.
import os
import glob
import time
import MySQLdb
import datetime
import mysql.connector
from mysql.connector import Error
#define db and cur
db = MySQLdb.connect(host = "127.0.0.1", user = "root", passwd = "xB7O4fXmuMpF6M0u", db = "temp_pi")
cur = db.cursor()
#connection to the database
try:
connection = mysql.connector.connect(host='127.0.0.1',
database='temp_pi',
user='root',
password='xB7O4fXmuMpF6M0u')
if connection.is_connected():
db_Info = connection.get_server_info()
print("Connected to MySQL database... MySQL Server version on ",db_Info)
cursor = connection.cursor()
cursor.execute("select database();")
record = cursor.fetchone()
print ("Your connected to - ", record)
except Error as e :
print ("Error while connecting to MySQL", e)
#obtaining the temperature through the ds18b20 sensor
os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')
base_dir = '/sys/bus/w1/devices/'
device_folder = glob.glob(base_dir + '28*')[0]
device_file = device_folder + '/w1_slave'
def read_temp_raw():
f = open(device_file, 'r')
lines = f.readlines()
f.close()
return lines
def read_temp():
lines = read_temp_raw()
while lines[0].strip()[-3:] != 'YES':
time.sleep(0.2)
lines = read_temp_raw()
equals_pos = lines[1].find('t=')
if equals_pos != -1:
temp_string = lines[1][equals_pos+2:]
temp_c = float(temp_string) / 1000.0
temp_f = temp_c * 9.0 / 5.0 + 32.0
return temp_c
#Defining sendDataToServer() and trying to send this data towards my website
def sendDataToServer():
global temperature
threading.Timer(600,sendDataToServer).start()
print("Mesuring...")
read_temp()
temperature = read_temp()
print(temperature)
temp= read_temp()
urllib3.urlopen("http://francoouesttemp.tech/weather/add_data.php?temp="+temp).read()
#insertion of data into the mysql database
while True:
print("putting temperature data into temp_pi database")
i = datetime.datetime.now()
year = str(i.year)
month = str(i.month)
day = str(i.day)
date = day + "-" + month + "-" + year
hour = str(i.hour)
minute = str(i.minute)
second = str(i.second)
timestr = hour + ":" + minute + ":" + second
valT = str(read_temp())
try:
cur.execute("""INSERT INTO TAB_CLASSROOM(temp_c,T_Date,T_Time) VALUES(%s,%s,%s)""",(valT,i,timestr))
db.commit()
except:
db.rollback()
time.sleep(5)
#this is the part where my code tells me : NameError : name 'urllib3' is not defined ----- I want this part of the code to send the temperature, date and time over to my website.
sendDataToServer()
cur.close()
db.close()

import urllib
import requests
url = '....'
response = urllib.request.urlopen(url)

If you want to send requests using urllib3, you need to create a pool manager first.
Alternatively, you could use the HTTP client in the Python standard library. Its urlopen function is called urllib.request.urlopen. Depending on what you are trying to do, the requests package might also be an option, but it has certain disadvantages when it comes to certificate management for HTTPS URLs (the built-in client will automatically use the system certificate store).

Python script partially inactive after few hours - Raspberry Pi

I wrote a Python script for the Raspberry Pi in order to monitor changes of 4 GPIO inputs, write them into a databse and send a message to my phone via a telegram bot.
Everything works fine, except that after a couple of hours, some processes of the script shut down. The script is still running (knwoing from a log file), but especially the GPIO event detect and the telegram bot API stop working. This mostly occurs over night.
Because it works perfectly fine for the first few hours after startup, I guess it shouldn't be a bug in the program, but you never know.
I started up the script via rc.local, but switched to System.d for running on startup instead. Works fine for that.
So in a nutshell: Script runs fine, but after a couple of hours some functions in the script become inactive (GPIO event detect, log messages, telegram API - kinda feels like the Pi is falling asleep).
#!/usr/bin/env python
#Import libraries
import time
import RPi.GPIO as GPIO
import telepot
import MySQLdb
import os.path
import logging
import datetime
#start logging
logfile = '/home/pi/Betrieb/logs/log_'+datetime.datetime.now().strftime("%Y-%m-%d_%H-%M")
logging.basicConfig(filename=logfile,level=logging.DEBUG,format='%(asctime)s %(message)s')
logging.debug('=========================================================')
logging.debug('Logging started!')
#initialize connection to Telegram bot
bot = telepot.Bot('##########################################')
#Connect to MySQL-database
con = MySQLdb.connect('localhost',"root","###########","################")
c = con.cursor()
###########################################################################################################
class User:
def __init__(self, id, name, user_id, enable):
self.id = id
self.name = name
self.chat_id = user_id
self.enable = enable
class Machine:
def __init__(self, id, number, name, pin, enable):
self.id = id
self.number = number
self.name = name
self.pin = pin
self.enable = enable
###########################################################################################################
def my_callback(pin):
c.execute("SELECT name FROM machines WHERE pin=%s" % pin)
con.commit()
data = c.fetchall()
for set in data:
machine_name=set[0]
### Attention: multiple sets with same ID can exist
if GPIO.input(pin):
print "Rising edge detected "+str(pin)
string = "INSERT INTO malfunction(machine_name,machine_pin,status) VALUES('%s',%s,1)" % (machine_name,pin)
c.execute(string)
con.commit()
for i in range(0,len(user)):
if user[i].enable:
bot.sendMessage(user[i].chat_id,"Stoerung "+ machine_name)
print "Sent message to", user[i].name
logging.debug('Detected malfunction on Pin %s and sent message to %s', str(pin), user[i].name)
else:
print "Falling edge detected on "+str(pin)
string = "INSERT INTO malfunction(machine_name,machine_pin,status) VALUES('%s',%s,0)" % (machine_name,pin)
c.execute(string)
con.commit()
for i in range(0,len(user)):
if user[i].enable:
bot.sendMessage(user[i].chat_id,"Stoerung behoben "+ machine_name)
print "Sent message to", user[i].name
logging.debug('Solved malfunction on Pin %s and sent message to %s', str(pin), user[i].name)
def updateData():
global machine
global user
logging.debug('Update data.')
#Update user data
c.execute("SELECT * FROM telegram_users")
con.commit()
data = c.fetchall()
user = []
del user[:]
for set in data:
newUser = User(set[0],set[1],set[2],set[3])
user.append(newUser)
#Update machine data
try:
machine
except NameError:
machine = []
else:
for i in range(0,len(machine)):
if machine[i].enable:
GPIO.remove_event_detect(machine[i].pin)
del machine[:]
c.execute("SELECT * FROM machines")
con.commit()
data = c.fetchall()
GPIO.setmode(GPIO.BCM)
for set in data:
# 0 = id / 1 = number / 2 = name / 3 = pin / 4 = enable #
newMachine = Machine(set[0],set[1],set[2],set[3],set[4])
machine.append(newMachine)
if set[4]:
GPIO.setup(set[3], GPIO.IN, pull_up_down = GPIO.PUD_DOWN)
GPIO.add_event_detect(set[3], GPIO.BOTH, callback=my_callback, bouncetime=300)
logging.debug('Added Event Detect on Pin %s', set[3])
###########################################################################################################
updateData()
logging.debug('Initial Update finished. Start looping.')
Counter = 0
while True:
Counter +=1
if Counter == 600:
logging.debug('Still running!')
Counter = 0
time.sleep(1);
lockfile = "/home/pi/Betrieb/lockfiles/request_update"
if os.path.isfile(lockfile):
os.remove(lockfile)
updateData()
print "Data updated."
logging.debug('Deleted lockfile and updated data.')

Python for loop dropping after 1 iteration

I am new to python and thought I would practice what I have been learning to complete a little task. Essentially I am inserting a cognimatic cameras data into a database from a .csv file that I pulled from the web. sadly I have had to omit all the connection details as it can only be accessed from my works computer which means the script cannot be run.
To the problem!
I have a for loop that iterates through the cameras in the system, running this script:
#!/usr/bin/python
import pymssql
import urllib2
import sys
import getpass
import csv
import os
attempts = 0 #connection attempt counter
#check db connection with tsql -H cabernet.ad.uow.edu.au -p 1433 -U ADUOW\\mbeavis -P mb1987 -D library_gate_counts
server = "*****" #sever address
#myUser = 'ADUOW\\' + raw_input("User: ")# User and password for server. Will this be needed when the script runs on the server? # Ask David
#passw = getpass.getpass("Password: ")
while attempts < 3: # attempt to connect 3 times
try: #try connection
conn = pymssql.connect(server = server, user = '****', password = '****', database = "***", port='1433',timeout = 15, login_timeout = 15)
break
except pymssql.Error as e: #if connection fails print error information
attempts += 1
print type(e)
print e.args
camCursor = conn.cursor() #creates a cursor on the database
camCursor.execute("SELECT * FROM dbo.CAMERAS") #Selects the camera names and connection details
for rows in camCursor:
print rows
Everything is fine and the loop runs as it should, however when I actually try and do anything with the data the loop runs once and ends, this is the full script:
#!/usr/bin/python
import pymssql
import urllib2
import sys
import getpass
import csv
import os
attempts = 0 #connection attempt counter
#check db connection with tsql -H cabernet.ad.uow.edu.au -p 1433 -U ADUOW\\mbeavis -P mb1987 -D library_gate_counts
server = "*****" #sever address
#myUser = 'ADUOW\\' + raw_input("User: ")# User and password for server. Will this be needed when the script runs on the server? # Ask David
#passw = getpass.getpass("Password: ")
while attempts < 3: # attempt to connect 3 times
try: #try connection
conn = pymssql.connect(server = server, user = '****', password = '****', database = "***", port='1433',timeout = 15, login_timeout = 15)
break
except pymssql.Error as e: #if connection fails print error information
attempts += 1
print type(e)
print e.args
camCursor = conn.cursor() #creates a cursor on the database
camCursor.execute("SELECT * FROM dbo.CAMERAS") #Selects the camera names and connection details
for rows in camCursor:
print rows
cameraName = str(rows[0]) #converts UNICODE camera name to string
connectionDetails = str(rows[1]) #converts UNICODE connection details to string
try: #try connection
#connect to webpage, this will be changed to loop through the entire range of cameras, which will
#have their names and connection details stored in a seperate database table
prefix = "***"
suffix = "**suffix"
response = urllib2.urlopen(prefix + connectionDetails + suffix, timeout = 5)
content = response.read() #read the data for the csv page into content
f = open( "/tmp/test.csv", 'w' ) #open a file for writing (test phase only)
f.write( content ) #write the data stored in content to file
f.close() #close file
print content #prints out content
with open( "/tmp/test.csv", 'rb' ) as csvFile: #opens the .csv file previously created
reader = csv.DictReader(csvFile) #reader object of DictReader, allows for the first row to be the dictionary keys for the following rows
for row in reader: #loop through each row
start = row['Interval start']
end = row['Interval stop']
camName = row['Counter name']
pplIn = int(row['Pedestrians coming in'])
pplOut = int(row['Pedestrians going out'])
insertCursor = conn.cursor()
insert = "INSERT INTO dbo.COUNTS VALUES (%s, %s, %d, %d)"
insertCursor.execute(insert, (camName, start, pplIn, pplOut))
conn.commit()
except urllib2.URLError as e: #catch URL errors
print type(e)
print e.args
except urllib2.HTTPError as e: #catch HTTP erros
print type(e)
print e.code
I have been scratching my head as I cannot see why there is a problem, but maybe I just need some fresh eyes on it. Any help would be great cheers!

Have you tried to do something like
queryResult = camCursor.execute("SELECT * FROM dbo.CAMERAS")
for rows in queryResult:
...
I guess this might solve the problem, which is probably the fact that you're trying to iterate over a cursor instead of the results.
You might find this way interesting as well:
camCursor.execute("SELECT * FROM dbo.CAMERAS")
for rows in camCursor.fetchall():
...
Source: https://docs.python.org/2/library/sqlite3.html

Looping in Time Python

I'm new in python programming. I have a Question. I want to save every input data (raw data to list) every 15 minutes. after 15 minutes the list wil be delete and write input data again.can anyone help me please?thank you for your kindness.
from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory
def WriteListtoCSV (data):
with open ('tesdata.csv','a') as csvfile:
writer=csv.writer(csvfile)
for val in data:
writer.writerow([val])
class MyServerProtocol(WebSocketServerProtocol):
def onConnect(self, request):
print("Client connecting: {0}".format(request.peer))
def onOpen(self):
print("WebSocket connection open.")
def onMessage(self, payload, isBinary):
if isBinary:
print("Binary message received: {0} bytes".format(len(payload)))
else:
print("Text message received: {0}".format(payload.decode('utf8')))
# echo back message verbatim
self.sendMessage(payload, isBinary)
mins = 0
data_nilai = [ ]
while mins != 60: #change value with seconds
data_nilai.append(payload.decode('utf8'))
time.sleep(1)
mins+=1
WriteListtoCSV(data_nilai)
#ClearCSV()
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
import sys
import csv
import time
from twisted.python import log
from twisted.internet import reactor
log.startLogging(sys.stdout)
factory = WebSocketServerFactory(u"ws://192.168.1.23:9000", debug=False)
factory.protocol = MyServerProtocol
# factory.setProtocolOptions(maxConnections=2)
reactor.listenTCP(9000, factory)
reactor.run()
my focus is only onMessage

Following is Algo with small code.
Algo:
Set detail file path where we are saving data.
Get Input from the user and process to create list.
Save data to file.
Wait for some time.
Delete File.
Code:
import pickle
import time
import os
detail_file = "/tmp/test.txt"
while(1):
# Get input from User and split to List.
user_input = raw_input("Enter item of the list separated by comma:")
user_input = user_input.split(",")
print "User List:- ", user_input
#- Save process, We can save your data i.e. list into file or database or any where
with open(detail_file, "wb") as fp:
pickle.dump(user_input, fp)
# Wait for 15 minutes.
time.sleep(900) # 15 * 60 = 900
# delete Save details.
os.remove(detail_file)
Note:
Use input() to get user information for Python 3.x
Use raw_input() to get user information for Python 2.x
[Edit 1]
Crontab
Ref: http://www.thegeekstuff.com/2011/07/cron-every-5-minutes/
Ref: http://www.computerhope.com/unix/ucrontab.htm
OS: CentOS
To edit the crontab, use this command:
crontab -e
*/15 * * * * python /tmp/script.py
where crontab entry structure is:
m h dom mon dow command