I am new to python and thought I would practice what I have been learning to complete a little task. Essentially I am inserting a cognimatic cameras data into a database from a .csv file that I pulled from the web. sadly I have had to omit all the connection details as it can only be accessed from my works computer which means the script cannot be run.
To the problem!
I have a for loop that iterates through the cameras in the system, running this script:
#!/usr/bin/python
import pymssql
import urllib2
import sys
import getpass
import csv
import os
attempts = 0 #connection attempt counter
#check db connection with tsql -H cabernet.ad.uow.edu.au -p 1433 -U ADUOW\\mbeavis -P mb1987 -D library_gate_counts
server = "*****" #sever address
#myUser = 'ADUOW\\' + raw_input("User: ")# User and password for server. Will this be needed when the script runs on the server? # Ask David
#passw = getpass.getpass("Password: ")
while attempts < 3: # attempt to connect 3 times
try: #try connection
conn = pymssql.connect(server = server, user = '****', password = '****', database = "***", port='1433',timeout = 15, login_timeout = 15)
break
except pymssql.Error as e: #if connection fails print error information
attempts += 1
print type(e)
print e.args
camCursor = conn.cursor() #creates a cursor on the database
camCursor.execute("SELECT * FROM dbo.CAMERAS") #Selects the camera names and connection details
for rows in camCursor:
print rows
Everything is fine and the loop runs as it should, however when I actually try and do anything with the data the loop runs once and ends, this is the full script:
#!/usr/bin/python
import pymssql
import urllib2
import sys
import getpass
import csv
import os
attempts = 0 #connection attempt counter
#check db connection with tsql -H cabernet.ad.uow.edu.au -p 1433 -U ADUOW\\mbeavis -P mb1987 -D library_gate_counts
server = "*****" #sever address
#myUser = 'ADUOW\\' + raw_input("User: ")# User and password for server. Will this be needed when the script runs on the server? # Ask David
#passw = getpass.getpass("Password: ")
while attempts < 3: # attempt to connect 3 times
try: #try connection
conn = pymssql.connect(server = server, user = '****', password = '****', database = "***", port='1433',timeout = 15, login_timeout = 15)
break
except pymssql.Error as e: #if connection fails print error information
attempts += 1
print type(e)
print e.args
camCursor = conn.cursor() #creates a cursor on the database
camCursor.execute("SELECT * FROM dbo.CAMERAS") #Selects the camera names and connection details
for rows in camCursor:
print rows
cameraName = str(rows[0]) #converts UNICODE camera name to string
connectionDetails = str(rows[1]) #converts UNICODE connection details to string
try: #try connection
#connect to webpage, this will be changed to loop through the entire range of cameras, which will
#have their names and connection details stored in a seperate database table
prefix = "***"
suffix = "**suffix"
response = urllib2.urlopen(prefix + connectionDetails + suffix, timeout = 5)
content = response.read() #read the data for the csv page into content
f = open( "/tmp/test.csv", 'w' ) #open a file for writing (test phase only)
f.write( content ) #write the data stored in content to file
f.close() #close file
print content #prints out content
with open( "/tmp/test.csv", 'rb' ) as csvFile: #opens the .csv file previously created
reader = csv.DictReader(csvFile) #reader object of DictReader, allows for the first row to be the dictionary keys for the following rows
for row in reader: #loop through each row
start = row['Interval start']
end = row['Interval stop']
camName = row['Counter name']
pplIn = int(row['Pedestrians coming in'])
pplOut = int(row['Pedestrians going out'])
insertCursor = conn.cursor()
insert = "INSERT INTO dbo.COUNTS VALUES (%s, %s, %d, %d)"
insertCursor.execute(insert, (camName, start, pplIn, pplOut))
conn.commit()
except urllib2.URLError as e: #catch URL errors
print type(e)
print e.args
except urllib2.HTTPError as e: #catch HTTP erros
print type(e)
print e.code
I have been scratching my head as I cannot see why there is a problem, but maybe I just need some fresh eyes on it. Any help would be great cheers!
Have you tried to do something like
queryResult = camCursor.execute("SELECT * FROM dbo.CAMERAS")
for rows in queryResult:
...
I guess this might solve the problem, which is probably the fact that you're trying to iterate over a cursor instead of the results.
You might find this way interesting as well:
camCursor.execute("SELECT * FROM dbo.CAMERAS")
for rows in camCursor.fetchall():
...
Source: https://docs.python.org/2/library/sqlite3.html
Related
Basically, I have my code, which finds the Lobby ID of a video-game, which then sends to my laptop. This then opens the exe with a argument and that certain Lobby ID. I want it to change when I start up another game/server. I have the script to output when the lobby ID changes, but when I plug it into my socket script. It only outputs the first one.
If I hash out all the s= socket... part and under it functions correctly
import socket
import glob
import os
import re
placeholder = ''
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = '192.168.0.17'
port = 50502
s.connect((host,port))
while True:
GAME_DIRECTORY = 'C:/Program Files/Oculus/Software/Software/ready-at-dawn-echo-arena'
logs = glob.glob(GAME_DIRECTORY + '/_local/r14logs/*')
log = max(logs, key = os.path.getctime)
with open(log, 'r') as f:
file = f.read()
lobbyid = re.findall(r'........-....-....-....-............', file)[-1]
if lobbyid != placeholder:
if lobbyid != ('00000000-0000-0000-0000-000000000000'):
placeholder = lobbyid
print (lobbyid)
def ts(str):
s.send(lobbyid.encode())
data = ''
data = s.recv(1024).decode()
s.close ()
ts(s)
I am trying to send temperature data over onto one of my website currently online. This code consists of measuring the temperature through a sensor(ds18b20), sending that data onto a mysql databse entitled temp_pi and specifically onto a table intitled TAB_CLASSROOM and lastly sending that data onto a webpage of mine. Everything in this code runs except for the sendDataToServer() part. I specify the error right before this particular line. I have the PHP set up on my website for this to work.
import os
import glob
import time
import MySQLdb
import datetime
import mysql.connector
from mysql.connector import Error
#define db and cur
db = MySQLdb.connect(host = "127.0.0.1", user = "root", passwd = "xB7O4fXmuMpF6M0u", db = "temp_pi")
cur = db.cursor()
#connection to the database
try:
connection = mysql.connector.connect(host='127.0.0.1',
database='temp_pi',
user='root',
password='xB7O4fXmuMpF6M0u')
if connection.is_connected():
db_Info = connection.get_server_info()
print("Connected to MySQL database... MySQL Server version on ",db_Info)
cursor = connection.cursor()
cursor.execute("select database();")
record = cursor.fetchone()
print ("Your connected to - ", record)
except Error as e :
print ("Error while connecting to MySQL", e)
#obtaining the temperature through the ds18b20 sensor
os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')
base_dir = '/sys/bus/w1/devices/'
device_folder = glob.glob(base_dir + '28*')[0]
device_file = device_folder + '/w1_slave'
def read_temp_raw():
f = open(device_file, 'r')
lines = f.readlines()
f.close()
return lines
def read_temp():
lines = read_temp_raw()
while lines[0].strip()[-3:] != 'YES':
time.sleep(0.2)
lines = read_temp_raw()
equals_pos = lines[1].find('t=')
if equals_pos != -1:
temp_string = lines[1][equals_pos+2:]
temp_c = float(temp_string) / 1000.0
temp_f = temp_c * 9.0 / 5.0 + 32.0
return temp_c
#Defining sendDataToServer() and trying to send this data towards my website
def sendDataToServer():
global temperature
threading.Timer(600,sendDataToServer).start()
print("Mesuring...")
read_temp()
temperature = read_temp()
print(temperature)
temp= read_temp()
urllib3.urlopen("http://francoouesttemp.tech/weather/add_data.php?temp="+temp).read()
#insertion of data into the mysql database
while True:
print("putting temperature data into temp_pi database")
i = datetime.datetime.now()
year = str(i.year)
month = str(i.month)
day = str(i.day)
date = day + "-" + month + "-" + year
hour = str(i.hour)
minute = str(i.minute)
second = str(i.second)
timestr = hour + ":" + minute + ":" + second
valT = str(read_temp())
try:
cur.execute("""INSERT INTO TAB_CLASSROOM(temp_c,T_Date,T_Time) VALUES(%s,%s,%s)""",(valT,i,timestr))
db.commit()
except:
db.rollback()
time.sleep(5)
#this is the part where my code tells me : NameError : name 'urllib3' is not defined ----- I want this part of the code to send the temperature, date and time over to my website.
sendDataToServer()
cur.close()
db.close()
import urllib
import requests
url = '....'
response = urllib.request.urlopen(url)
If you want to send requests using urllib3, you need to create a pool manager first.
Alternatively, you could use the HTTP client in the Python standard library. Its urlopen function is called urllib.request.urlopen. Depending on what you are trying to do, the requests package might also be an option, but it has certain disadvantages when it comes to certificate management for HTTPS URLs (the built-in client will automatically use the system certificate store).
I make project read RFID tag using python on raspberry pi and using reader RDM880.
My idea is to take the time in and time out to check with the staff to work on time or not.
I try to add card_ID, time_in, time_out to local mysql and remote mysql (IP: 192.168.137.1) using python.
It has the same table in remote and local mysql.
If mysql remote is broken, I want only add to local mysql.
Here is my code:
import serial
import time
import RPi.GPIO as GPIO
import MySQLdb
from datetime import datetime
from binascii import hexlify
serial=serial.Serial("/dev/ttyAMA0",
baudrate=9600,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
bytesize=serial.EIGHTBITS,
timeout=0.1)
db_local = MySQLdb.connect("localhost","root","root","luan_van") #connect local
db = MySQLdb.connect("192.168.137.1", "root_a","","luan_van") #connect remote
ID_rong = 128187 # reader respone if no card
chuoi= "\xAA\x00\x03\x25\x26\x00\x00\xBB"
def RFID(str): #function read RFID via uart
serial.write(chuoi)
data = serial.readline()
tach_5 = data[5]
tach_6 = data[6]
hex_5 = hexlify(tach_5)
hex_6= hexlify(tach_6)
num_5 = int(hex_5,16)
num_6 = int(hex_6,16)
num_a = num_5 * 1000 + num_6
if(num_a != ID_rong):
tach_7 = data[7]
tach_8 = data[7]
hex_7 = hexlify(tach_7)
hex_8= hexlify(tach_8)
num_7 = int(hex_7,16)
num_8 = int(hex_8,16)
num = num_8 + num_7 * 1000 + num_6 * 1000000 + num_5 * 1000000000
else:
num = num_5 * 1000 + num_6
return num
def add_database(): # add card_ID and time_in to remote mysql
with db:
cur = db.cursor()
cur.execure("INSERT INTO tt_control(Card_ID,Time_in) VALUES ('%d',NOW()) " %num)
return
def add_database_local(): # add card_ID and time_in to remote mysql
with db_local:
cur = db_local.cursor()
cur.execure("INSERT INTO tt_control(Card_ID,Time_in) VALUES ('%d',NOW()) " %num)
return
def have_ID(int): #check ID in table tt_control
with db_local:
cur = db_local.cursor(MySQLdb.cursors.DictCursor)
cur.execute("SELECT * FROM tt_control WHERE Card_ID = '%d'" %num)
rows = cur.fetchall()
ID=""
for row in rows:
ID = row['Card_ID']
return ID
def add_time_out(): #add time out to remote mysql
with db:
cur = db.cursor(MySQLdb.cursors.DictCursor)
cur.execute("UPDATE tt_control SET Time_out = NOW() WHERE Card_ID = '%d'" %num)
return
def add_time_out_local(): #add time out to local mysql
with db_local:
cur = db_local.cursor(MySQLdb.cursors.DictCursor)
cur.execute("UPDATE tt_control SET Time_out = NOW() WHERE Card_ID = '%d'" %num)
return
def add_OUT(): #increase Card_ID to distinguish second check
with db:
cur = db.cursor(MySQLdb.cursors.DictCursor)
cur.execute("UPDATE tt_control SET Card_ID = Card_ID + 1 WHERE Card_ID = '%d'" %num)
return
def add_OUT_local(): #increase Card_ID to distinguish second check
with db_local:
cur = db_local.cursor(MySQLdb.cursors.DictCursor)
cur.execute("UPDATE tt_control SET Card_ID = Card_ID + 1 WHERE Card_ID = '%d'" %num)
return
while 1:
num = RFID(chuoi)
time.sleep(1)
Have_ID =have_ID(num)
if(num != ID_rong):
if(Have_ID ==""):
add_database() #---> it will error if remote broken, how can i fix it?
add_database_local()
else:
add_time_out() #---> it will error if remote broken, how can i fix it? I think connection alive can fix, but I don't know
add_time_out_local()
add_OUT()
add_OUT_local() #---> it will error if remote broken, how can i fix it?
You have a couple choices:
(not as good) Ping the server regularly to keep the connection alive.
(best) Handle the MySQLdb exception when calling cur.execute by re-establishing your connection and trying the call again. Here's an excellent and concise answer for how to do just that. From that article, you handle the exception yourself:
def __execute_sql(self,sql,cursor):
try:
cursor.execute(sql)
return 1
except MySQLdb.OperationalError, e:
if e[0] == 2006:
self.logger.do_logging('info','DB', "%s : Restarting db" %(e))
self.start_database()
return 0
(lastly) Establish a new database connection just before you actually call the database entries. In this case, move the db and db_local definitions into a function which you call just before your cursor. If you're making thousands of queries, this isn't the best. However, if it's only a few database queries, it's probably fine.
I use the following method:
def checkConn(self):
sq = "SELECT NOW()"
try:
self.cur.execute( sq )
except pymysql.Error as e:
if e.errno == 2006:
return self.connect()
else:
print ( "No connection with database." )
return False
I used a simple technique. Initially, I connected to DB using:
conect = mysql.connector.connect(host=DB_HOST, user=DB_USER, password=DB_PASS, database=DB_NAME)
Whenever I need to check if the DB is still connected, I used a line:
conect.ping(reconnect=True, attempts=3, delay=2)
This will check if the DB connection is still alive. If not, it will restart the connection which solves the problem.
It just makes sense not to use a status checker function before executing a SQL. Best practice shall handle the exception afterward and reconnect to the server.
Since the client library is always on the client side, there is no way to know the server status (connect status does depend on server status of course) unless we ping it or connect it.
Even if you ping the server and make sure the connection is fine and let the code execute down to the following line, the connection theoretically still could be down within that glimpse of time. So it's still not guaranteed that you will have a good connection right after you check the connection status.
On the other hand, ping is as expensive as most operations. If your operation fails because of a bad connection, then it's as good as using the ping to check the status.
Considering these, why bother to use ping or other no-matter built-in or not-built-in functions to check the connection status? Just execute your command as if it is up, then handle the exception in case it is down. This might be the reason the mysqlclient library does not provide a built-in status checker in the first place.
I am writing a python script to get logged in using ftp and download a file.But whenever I run this script,it says I have provided wrong user name or passwd.I am inputting right password still i am unable to run this script.My code is:
import os,getpass
from urllib.request import urlopen
filename='68544.jpg'
password=getpass.getpass('??')
At this line below the script is failed to run and whenever i run this address in browser it runs fine.
remoteaddr='ftp://Kamal:%s#localhost/%s;type=i'%(password,filename)
remotefile=urlopen(remoteaddr)
localfile=open(filename,'wb')
localfile.write(remotefile.read())
localfile.close()
remotefile.close()
def ftp_connect(path):
link = FTP(host = 'example.com', timeout = 5) #Keep low timeout
link.login(passwd = 'ftppass', user = 'ftpuser')
debug("%s - Connected to FTP" % strftime("%d-%m-%Y %H.%M"))
link.cwd(path)
return link
downloaded = open('/local/path/to/file.tgz', 'wb')
def debug(txt):
print txt
link = ftp_connect(path)
file_size = link.size(filename)
max_attempts = 5 #I dont want death loops.
while file_size != downloaded.tell():
try:
debug("%s while > try, run retrbinary\n" % strftime("%d-%m-%Y %H.%M"))
if downloaded.tell() != 0:
link.retrbinary('RETR ' + filename, downloaded.write, downloaded.tell())
else:
link.retrbinary('RETR ' + filename, downloaded.write)
except Exception as myerror:
if max_attempts != 0:
debug("%s while > except, something going wrong: %s\n \tfile lenght is: %i > %i\n"(strftime("%d-%m-%Y %H.%M"), myerror, file_size, downloaded.tell()))
link = ftp_connect(path)
max_attempts -= 1
else:
break
debug("Done with file, attempt to download m5dsum")
[...]
Use Paramiko library
import paramiko
paramiko.util.log_to_file('/tmp/paramiko.log')
# Open a transport
host = "example.com"
port = 22
transport = paramiko.Transport((host, port))
# Auth
password = "foo"
username = "bar"
transport.connect(username = username, password = password)
# Go!
sftp = paramiko.SFTPClient.from_transport(transport)
# Download
### It is relative path from the folder path on which this sftp user has default rights.
filepath = 'folder/file1.txt'
localpath = '/opt/backup/file.txt'
sftp.get(filepath, localpath)
# Close
sftp.close()
transport.close()
I have a server with socket, when client connect a new thread start to record the recive data in a mysql database, the script does it fine but i have a infinite while and the script only work for 4 hours i need the server work at least for 24 hours.
What can i do?
import time
import MySQLdb
import sys
from socket import *
myHost = ''
myPort = 6250
sockobj = socket(AF_INET, SOCK_STREAM)
sockobj.bind((myHost, myPort))
sockobj.listen(10)
print 'Server On'
def handleClient(connection):
time.sleep(0.1)
while True:
try:
data = connection.recv(256)
except timeout:
break
if not data:
break
if 'q' not in data:
if ',' in data:
dato = data.split(',')
try:
db=MySQLdb.connect(host='localhost',user='',passwd='',db='')
cursor=db.cursor()
sql = "INSERT INTO imei"+dato[1]+" (imei, estado, fecha, numero) VALUES ('%s','%s','%s','%s')"%(dato[0],dato[1],dato[2],dato[3])
cursor.execute(sql)
db.close()
except MySQLdb.Error:
db.close()
break
try:
connection.send('LOAD')
except:
break
connection.close( )
thread.exit()
def dispatcher( ):
while True:
connection, address = sockobj.accept( )
try:
thread.start_new(handleClient, (connection,))
except:
dispatcher( )
dispatcher( )
if i do the same script with fork it works for 4 hours too.
I suggest you to use https://pypi.python.org/pypi/supervisor to make sure that it is running 24 hours.
Also, could you please copy-paste the traceback after this script crashes?