sqlite3 python unexpected termination - python

I have set a sqlite3 database. I filled it with some data(~4million records ~1.2Gb of data).
Then I do some queries (selects/deletes/updates).
The problem is that sometimes after the insertions the script stops without an error. Sometimes it runs normal until the end.
These are the type of queries I run:
from __future__ import print_function
import sqlite3
import csv
import os
import glob
import sys
import time
db = 'test.db'
conn = sqlite3.connect(db)
conn.text_factory = str # allows utf-8 data to be stored
c = conn.cursor()
i = 0
### traverse the directory and process each .csv file
##print("debug")
csvfile =('/home/Desktop/Untitled Folder/Crimes_-_2001_to_present.csv')
with open(csvfile, "rb") as f:
reader = csv.reader(f)
t = time.time()
header = True
for row in reader:
if header:
# gather column names from the first row of the csv
header = False
sql = "DROP TABLE IF EXISTS test_table"
c.execute(sql)
#print("debug 1")
sql = "CREATE TABLE test_table (ID INTEGER,FBI_Code INTEGER,Updated_On TEXT,District TEXT,Beat INTEGER,Primary_Type TEXT,Location BLOB,Latitude REAL,Arrest INTEGER,Domestic INTEGER,Longitude REAL,Community_Area INTEGER,Case_Number INTEGER,Block TEXT,Location_Description TEXT,Ward INTEGER,IUCR INTEGER,Year INTEGER, Date TEXT,Y_Coordinate INTEGER,Description TEXT,X_Coordinate INTEGER);"
c.execute(sql)
#print("debug 2")
insertsql = "INSERT INTO test_table VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
rowlen = len(row)
#print("debug 3")
else:
# skip lines that don't have the right number of columns
#print("debug 4")
#if len(row) == rowlen:
#print("debug 5")
try:
c.execute(insertsql, row)
except:
print("problem in row %d" % i)
print(row)
continue
# print("debug 6")
i +=1
if i == 1000:
conn.commit()
#### if i == 4000000:
#### break
## #print (row)
conn.commit()
print('\nTime for Insertions TOTAL~> \n')
print(float( time.time() -t ))
print('\nTime for Insertions per Query~> \n')
print(float( time.time() -t )/i)
del rows
rows = list()
print('\nTime for Selections ~> Domestic\n')
t = time.time()
c.execute("SELECT * FROM test_table WHERE Domestic == 'false'")
rows = c.fetchall()
print(float( time.time() -t ))
print(len(rows))
del rows
rows = list()
print('\nTime for Selections ~> Arrests\n')
t = time.time()
c.execute("SELECT * FROM test_table WHERE Arrest == 'false'")
rows = c.fetchall()
print(float( time.time() -t ))
print(len(rows))
del rows
rows = list()
print('\nTime for Selections ID~> \n')
t = time.time()
c.execute("SELECT * FROM test_table WHERE ID < 9938614")
rows = c.fetchall()
print(float( time.time() -t ))
print(len(rows))
del rows
rows = list()
print('\nTime for Selections ~> Primary_Type\n')
t = time.time()
c.execute("SELECT * FROM test_table WHERE Primary_Type == 'BATTERY'")
rows = c.fetchall()
print(float( time.time() -t ))
print(len(rows))
del rows
rows = list()
print('\nTime for Selections Year~> \n')
t = time.time()
c.execute("SELECT * FROM test_table WHERE Year <= 2014")
rows = c.fetchall()
print(float( time.time() -t ))
print(len(rows))
del rows
rows = []
print('\nTime for Updates ~> YEAR\n')
t = time.time()
c.execute("UPDATE test_table SET Year = '2016' WHERE Year == '2014'")
print(float( time.time() -t ))
print('\nTime for Selections Year~> \n')
t = time.time()
c.execute("SELECT * FROM test_table WHERE Year <= 2014")
rows = c.fetchall()
print(float( time.time() -t ))
print(len(rows))
print('\nTime for DELETIONS ~> Domestic\n')
t = time.time()
c.execute("DELETE FROM test_table WHERE Domestic == 'false'")
rows = c.fetchall()
print(float( time.time() -t ))
print(len(rows))
del rows
c.close()
conn.close()
Every time I reassign the row list because after some queries I run out of memory. But I do not think that's the problem(just in case I used del rows & reassigned it, it was slower that way). Anyway after some of these queries the script stops without an error and I can not figure out why because there are some times in which it runs OK.
Edit
I have included the code above. The problem is that after the insertion part when I do the queries the script is terminated without any error.
For example it goes till here:
...
Time for Selections ~> Arrests
123.231
3928182
and then it terminates. In the 1st approach I did not delete the list and Cython produced core dump errors when I tried to re-declare the list. Now that I delete and then declare the list Cython runs ok. My question is why does not python catch any exceptions?
After the reassignment of the list, garbage collector clears the trash data (and it does, as I saw from linux monitor) but it crashes without errors. And the most annoying is that sometimes it runs ok till the end.


I had exactly the same problem, my solution was to create a new cursor, then I handled the SQL Select with one cursor and inserts/deletes with the other one.
conn = sqlite3.connect(db)
c = conn.cursor()
c2 = conn.cursor()

Related

Why wont this loop?

Newby working my way through python and sql with mariadb.
Why wont this loop? It updates the first record only. Im pretty hack at this...
cursor1.execute("select recordid, mvavgvol, quote_usd_volume_change_24h from pumped")
records = cursor1.fetchall()
for x in records:
rid = (x[0])
m = (x[1])
o = (x[2])
if (m >= o):
result = 0
else:
result = 1
upd_data=(result,rid)
sql1 = ("UPDATE pumped SET mvavgvolcheck = %s WHERE recordid = %s")
cursor2.execute(sql1,upd_data)
conn.commit()

Since you are fetching multiple rows you have to store the fetched values in an array and use cursor's executemany() method instead.
✂
data= []
for x in records:
rid = (x[0])
result= int(x[1] > x[2])
data+= [(result, rid)]
cursor.executemany(UPDATE pumped SET mvavgvolcheck = %s WHERE recordid = %s", data);
✂
When using mariadb python module (MariaDB Connector/Python) this is much more effective since it reduces network traffic: instead of sending n update commands in a loop (where n is the number of rows in table pumped) only one command will be send to the server.

conn = msql.connect(host=Host,port=Port, user=User, password=Password, database=database)
cursor1 = conn.cursor()
cursor2 = conn.cursor()
cursor1.execute("select recordid, mvavgvol, quote_usd_volume_change_24h from pumped")
records = cursor1.fetchall()
for x in records:
rid = (x[0])
m = (x[1])
o = (x[2])
if (m >= o):
result = 0
cursor2.execute("UPDATE pumped SET mvavgvolcheck = %s WHERE recordid = %s",(result, rid))
conn.commit()
else:
result = 1
cursor2.execute("UPDATE pumped SET mvavgvolcheck = %s WHERE recordid = %s",(result, rid))
conn.commit()

MySQL update or insert based on fetchall results in Python

I need to set some user meta in my wordpress through local python script. Hence I can't use the WP update_user_meta for it - it has to be done manually.
import mysql.connector as mysql
cnx = mysql.connect(host=HOST, database=DATABASE, user=USER, password=PASSWORD)
cursor = cnx.cursor()
get_meta = ("SELECT * FROM `ff_usermeta` WHERE `user_id`= 1 AND (`meta_key`='nickname' OR `meta_key`='info' OR `meta_key`='bg' OR `meta_key`='avatar' OR `meta_key`='profile_updated')")
cursor.execute(get_meta)
meta = cursor.fetchall()
#some processing of the result
cursor.execute(q, (...))
cnx.commit()
cursor.close()
cnx.close()
Now I need to check if the result has meta with each of the keys.
If the key already exists for this user, it needs to run UPDATE for this meta.
If this user still has no meta of this key, it has to INSERT new row.
if(there's no 'nickname' in meta_key on either of 5 or less rows):
q = ("INSERT INTO `ff_usermeta` ...")
else:
q = ("UPDATE `ff_usermeta` ...")
...and 4 more times like that?.. Seems like a good place for a cycle, but I don't really like the idea to make it 5x queues, especially since there might be more fields in the future.
I was thinking along the lines of searching the fetchall result for matches in meta_key, and if found, adding required data to one array, if not - to another. And then just running one update and one insert at the end, assuming both are not empty. If I were to write it in semi-php style, it would look roughly like this:
if(in_array("nickname", meta))
for_update .= "`nickname`='"+data[0]+"', "
else:
fields .= "`nickname`, "
vals .= "'"+data[0]+"', "
if(in_array("bg", meta)):
for_update .= "`bg`='"+data[1]+"', "
else:
fields .= "`bg`, "
vals .= "'"+data[1]+"', "
if(for_update):
update = ("UPDATE `ff_usermeta` SET "+for_update+" WHERE 1")
if(fields):
insert = ("INSERT INTO `ff_usermeta`("+fields+") VALUES ("+vals+")")
But absolutely no clue how to translate it correctly to python. I had to google it up to things like "why dot not working to add one string to another". Any advice? Or perhaps there is a better way? Thanks!

It is not complete, you can not update your rows in that way.
But with this you can start to make your query
The frist select gets exactly 1 row, if the user_id exists.
The user_id doesn't seem the right choice for this, but to get what you can do it is enough.
If the query doesn't have an entry, the it will insert some data you get from anywhere
The update as the insert are in that form wrong as you have to insert 5 new orws or update max 5 rows, but that is more for you to programm
import mysql.connector as mysql
HOST = "localhost"
DATABASE = ""
USER = "root"
PASSWORD = "mypassword"
cnx = mysql.connect(host=HOST, database=DATABASE, user=USER, password=PASSWORD)
cnx = mysql.connect(host=HOST, database=DATABASE, user=USER, password=PASSWORD)
cursor = cnx.cursor()
user_id = 1
get_meta = ("""SELECT umeta_id, user_id , MAX(IF( `meta_key`='nickname', meta_value,'')) AS 'nickname' , MAX(IF( `meta_key`='info', meta_value,'')) AS 'info' , MAX(IF( `meta_key`='bg', meta_value,'')) AS 'bg' , MAX(IF( `meta_key`='avatar', meta_value,''NULL'')) AS 'avatar' , MAX(IF (`meta_key`='profile_updated', meta_value,'')) AS 'profile_updated' FROM `ff_usermeta` WHERE `user_id`= %s GROUP BY umeta_id, user_id:""")
result = cursor.execute(get_meta,(user_id,))
if result > 0:
data = cursor.fetchone()
for_update = "";
#some processing of the result
if not data["nickname"]:
for_update += "`nickname`='"+data["nickname"]+"', "
if not data["bg"]:
for_update += "`bg`='"+data["bg"]+"', "
query = ("UPDATE `ff_usermeta` SET "+for_update+" WHERE user_id = " + user_id)
else:
#here are no data to be gathered as there is no user_id present add new user
nickname = ""
bg= ""
info = ""
avatar = ""
profile_updated = ""
fields= ""
vals = ""
fields += "`nickname`,`info`, `bg`,`avatar`,`profile_updated`"
vals += "'"+nickname+"', "+"'"+info+"', "+"'"+bg+"', "+"'"+avatar+"', "+"'"+profile_updatedfo+"'"
query = ("INSERT INTO `ff_usermeta`("+fields+") VALUES ("+vals+")")
cursor.execute(query)
cnx.commit()
cursor.close()
cnx.close()

I tried my best to adapt the suggestion above, but couldn't figure out how to make it work. Eventually I went another way, and it seems to work somehow, so I'll post the full code in case anyone would find it useful.
What it does: checks the queue in table with validation request, then parses a page (separate function) and updates user profile accodringly.
import mysql.connector as mysql
import time
from datetime import datetime
cnx = mysql.connect(host=HOST, database=DATABASE, user=USER, password=PASSWORD)
while True: #endless loop as a temporary scheduler
cursor = cnx.cursor()
#getting first request in the queue - 0: id, 1: url, 2: parse, 3: status, 4: user, 5: user_page, 6: req_date, 7: action
cursor.execute("SELECT * FROM `ff_qq` WHERE status = 0 LIMIT 1")
row = cursor.fetchone()
if row:
status = 1 #processed
if row[7] == "verify":
get_user = ("SELECT * FROM `ff_users` WHERE ID = %s LIMIT 1")
cursor.execute(get_user, (row[4],))
user = cursor.fetchone() #0 - ID, 5 - user_url, 8 - user_status, 9 - display_name
#separate function that returns data to insert into mysql
udata = verify(row) #0 - nickname, 1 - fb_av, 2 - fb_bg, 3 - fb_info, 4 - owner
ustat = row[1].split("/authors/")
if udata['owned'] or user[8] == ustat[1]:
update_user = ("UPDATE `ff_users` SET user_status = %s, display_name = %s, user_url = %s WHERE ID = %s LIMIT 1")
cursor.execute(update_user, (ustat[1], udata['nickname'], row[1], user[0]))
status = 2 #success
get = ("SELECT `meta_value` FROM `ff_usermeta` WHERE `user_id`= %s AND `meta_key`='ff_capabilities' LIMIT 1")
cursor.execute(get, (row[4],))
rights = cursor.fetchone()
if rights == 'a:1:{s:10:"subscriber";b:1;}':
promote = ("UPDATE `ff_usermeta` SET `meta_value` = 'a:1:{s:6:\"author\";b:1;}' "
"WHERE `user_id` = %s AND `meta_key`='ff_capabilities' LIMIT 1")
cursor.execute(promote, (row[0],))
#list of meta_key values in same order as returned data
ff = ['nickname', 'fb_av', 'fb_bg', 'fb_info']
for x in range(0,3): #goes through each one of the above list
if udata[ff[x]]: #yes this actually works, who would've thought?..
#current meta_key added directly into the string
get = ("SELECT `meta_value` FROM `ff_usermeta` WHERE `user_id`= %s AND `meta_key`='" + ff[x] + "' LIMIT 1")
cursor.execute(get, (row[4],))
meta = cursor.fetchone()
if(meta): #update if it exists, otherwise insert new row
qq = ("UPDATE `ff_usermeta` SET `meta_value` = %s "
"WHERE `user_id` = %s AND `meta_key`='" + ff[x] + "' LIMIT 1")
else:
qq = ("INSERT INTO `ff_usermeta`(`meta_value`, `meta_key`, `user_id`) "
"VALUES ('%s','" + ff[x] + "','%s'")
cursor.execute(qq, (udata[ff[x]], row[0])) #same execute works for both
else:
status = 3 #verification failed
#update queue to reflect its status
update = ("UPDATE `ff_qq` SET status = %s WHERE id = %s LIMIT 1")
cursor.execute(update, (status, row[0]))
cnx.commit()
cursor.close()
now = datetime.now()
print(now.strftime("%d.%m.%Y %H:%M:%S"))
time.sleep(180) #sleep until it's time to re-check the queue
cnx.close()

Pypyodbc TabError: inconsistent use of tabs and spaces in indentation

This is a common problem it seems on here but in my case I cant find an answer. Why is it saying inconsistent use of tabs and indentation here
def exectute_SQL(): #This function executes SQL to pull counts from a table where it wasnt possible to get an excel
con = pypyodbc.connect(conn_str)
cur = con.cursor()
sql = "SELECT * FROM Elig_Own.DST_Report_Validation_Test" #WHERE ysn_active = '1'"
cur.execute(sql)
rows = cur.fetchall()
for row in rows:
strFnd = 0
strReportName = row[1]
strSrcName = row[2]
strDestName = row[3]
strFileName = row[4]
try:
for report in strReportName:
if report == 'STR_DB Load to SQL':
cur.execute("$result = SELECT TOP 1 COUNT(*) FROM Elig_Own.STR_DB GROUP BY LAST_UPDATED ORDER BY LAST_UPDATED DESC;")
cur.execute("INSERT INTO Elig_Own.DST_Report_Status_Test(TDate, Report, Records, Status) VALUES(CAST(GetDate() AS Date), 'STR_DB Load to SQL', ?, 'Passed')",(result))
con.commit()
except:
print("Couldnt execute script")
And This is the error message
C:\Users\cn192406\Documents\Programs>python File_Check_Dart_Functions.py
File "File_Check_Dart_Functions.py", line 73
cur.execute("$result = SELECT TOP 1 COUNT(*) FROM Elig_Own.STR_DB GROUP BY LAST_UPDATED ORDER BY LAST_UPDATED DESC;")
TabError: inconsistent use of tabs and spaces in indentation

Try this:
def exectute_SQL(): # This function executes SQL to pull counts from a table where it wasnt possible to get an excel
con = pypyodbc.connect(conn_str)
cur = con.cursor()
sql = "SELECT * FROM Elig_Own.DST_Report_Validation_Test" # WHERE ysn_active = '1'"
cur.execute(sql)
rows = cur.fetchall()
for row in rows:
strFnd = 0
strReportName = row[1]
strSrcName = row[2]
strDestName = row[3]
strFileName = row[4]
try:
for report in strReportName:
if report == "STR_DB Load to SQL":
cur.execute(
"$result = SELECT TOP 1 COUNT(*) FROM Elig_Own.STR_DB GROUP BY LAST_UPDATED ORDER BY LAST_UPDATED DESC;"
)
cur.execute(
"INSERT INTO Elig_Own.DST_Report_Status_Test(TDate, Report, Records, Status) VALUES(CAST(GetDate() AS Date), 'STR_DB Load to SQL', ?, 'Passed')",
(result),
)
con.commit()
except Exception as e:
pass

Python code not creating tables on the database but able to query the results postgres

My usecase is to write create a temp table in the postgres database and fetch records from it and insert into a different table.
The code i used is:
import psycopg2
import sys
import pprint
from __future__ import print_function
from os.path import join,dirname,abspath
import xlrd
import os.path
newlist = []
itemidlist = []
def main():
conn_string = "host='prod-dump.cvv9i14mrv4k.us-east-1.rds.amazonaws.com' dbname='ebdb' user='ebroot' password='*********'"
# print the connection string we will use to connect
# print "Connecting to database" % (conn_string)
# get a connection, if a connect cannot be made an exception will be raised here
conn = psycopg2.connect(conn_string)
# conn.cursor will return a cursor object, you can use this cursor to perform queries
cursor = conn.cursor()
dealer_id = input("Please enter dealer_id: ")
group_id = input("Please enter group_id: ")
scriptpath = os.path.dirname('__file__')
filename = os.path.join(scriptpath, 'Winco - Gusti.xlsx')
xl_workbook = xlrd.open_workbook(filename, "rb")
xl_sheet = xl_workbook.sheet_by_index(0)
print('Sheet Name: %s' % xl_sheet.name)
row=xl_sheet.row(0)
from xlrd.sheet import ctype_text
print('(Column #) type:value')
for idx, cell_obj in enumerate(row):
cell_type_str = ctype_text.get(cell_obj.ctype, 'unknown type')
#print('(%s) %s %s' % (idx, cell_type_str, cell_obj.value))
num_cols = xl_sheet.ncols
for row_idx in range(0, xl_sheet.nrows): # Iterate through rows
num_cols = xl_sheet.ncols
id_obj = xl_sheet.cell(row_idx, 1) # Get cell object by row, col
itemid = id_obj.value
#if itemid not in itemidlist:
itemidlist.append(itemid)
# execute our Query
'''
cursor.execute("""
if not exists(SELECT 1 FROM model_enable AS c WHERE c.name = %s);
BEGIN;
INSERT INTO model_enable (name) VALUES (%s)
END;
""" %(itemid,itemid))
'''
cursor.execute("drop table temp_mbp1")
try:
cursor.execute("SELECT p.model_no, pc.id as PCid, g.id AS GROUPid into public.temp_mbp1 FROM products p, \
model_enable me, products_clients pc, groups g WHERE p.model_no = me.name \
and p.id = pc.product_id and pc.client_id = %s and pc.client_id = g.client_id and g.id = %s"\
% (dealer_id,group_id)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
cursor.execute("select count(*) from public.temp_mbp1")
# retrieve the records from the database
records = cursor.fetchall()
# print out the records using pretty print
# note that the NAMES of the columns are not shown, instead just indexes.
# for most people this isn't very useful so we'll show you how to return
# columns as a dictionary (hash) in the next example.
pprint.pprint(records)
if __name__ == "__main__":
main()
The try except block in between the program is not throwing any error but the table is not getting created in the postgres database as i see in the data admin.
The output shown is:
Please enter dealer_id: 90
Please enter group_id: 13
Sheet Name: Winco Full 8_15_17
(Column #) type:value
[(3263,)]
Thanks,
Santosh

You didn't commit the changes, so they aren't saved in the database. Add to the bottom, just below the pprint statement:
conn.commit()

mariadb for update doesn't work correctly

I was testing mariadb for update with python mysqldb and it doesn't works correctly for me. The same rows are getting twice when running two python scripts symultaneously as cronjob. When I testing on two mysql consoles everything going fine. Using innodb. My code:
cursor.execute("start transaction")
cursor.execute("SELECT id FROM t1 where tw1=2 AND tw2=1 AND tw3=3 order by id limit 1000 FOR UPDATE")
rows = [r for r in cursor.fetchall()]
ids = []
for row in rows:
ids.append(str(row['id']))
for i in range(2):
cursor.execute("SELECT id FROM t2 where tw1="sth" AND tw2 =0 AND tw3=0 LIMIT 1 FOR UPDATE")
d = cursor.fetchone()
if not d:
#here: insert to t2 and get last id as d_id
cursor.execute("update t2 SET tu1=1,tu2=500,tu3=0 WHERE id = %s" % d_id)
for row in rows:
#job with rows
cursor.execute( "Update t1 SET tw1=tw1+1 WHERE id IN (" + ','.join(ids) + ")")
cursor.execute("commit")
tw1 from t1 should be max 3 but it is very often 4. When I put ids to file I see that there are the same ids for 2 diffrent program runs. What I'm doing wrong?

Categories

Resources