Develop Reference

Python - Normalize data with Regex

I am trying to use Regex cleaning steps in Python to test to see if a pattern matches and if so, clean it to the specified carrier.
For instance, if re.match("\bA\.?X\.?A\.?\b", Carrier): Carrier = CarrierMatch
I've tried this by running a for loop on the number of raw carrier fields followed by another for loop on all of the match descriptions (just printing for now) and it takes FOREVER to run. Hoping someone out there has a better method.
Ideally I would like to see if it's possible to compile all match descriptions for Carrier I have in SQL (~2,000) and pull out the regex match pattern(s) to then use to append the carrier field.
For reference the SQL data fields are [raw_pattern], [Carrier]
import sys
import re
import pyodbc
import sys
import os
import pandas as pd
from datetime import datetime
import time
regexlist = list()
carrierlist = list()
rpt_id = 1234
#rpt_id = sys.argv[1]
plan_typs = list()
try:
conn = pyodbc.connect('Driver={SQL Server};'
'Server=xxxxxxxxx;'
'Database=xxxxxxxxx;'
'Trusted_Connection=xxxxx;')
except:
print('Connection Failed')
sys.exit()
cursor = conn.cursor()
sql = "delete from [dbo].[python_test1] where rpt_id = '""" + str(rpt_id) + """'"""
cursor.execute(sql)
conn.commit()
cursor = conn.cursor()
sql = "insert into [dbo].[python_test1](rpt_id, raw_carr_nm) select distinct rpt_id, raw_carr_nm from [dbo].[wrk_data] where rpt_id = '""" + str(rpt_id) + """'"""
cursor.execute(sql)
conn.commit()
sql = "SELECT [raw_pattern], [Carrier] FROM [dbo].[ref_regex_t]"
regex1 = pd.read_sql(sql, conn)
sql = "select * from [dbo].[python_test1] where rpt_id = '""" + str(rpt_id) + """'"""
carriers = pd.read_sql(sql, conn)
for index, row in regex1.iterrows():
regexlist.append(row['raw_pattern'])
for index, row in carriers.iterrows():
carrierlist.append(row['Carrier'])
for i in carrierlist:
print('"' + i + '"')
for i in regexlist:
print('"' + i + '"')

python sqlite3 - cursor vs connections - executing UPDATE statements

I am having trouble with executing UPDATE statements.
In the code below when I write con.execute() it writes to the database, but if I write c.execute() it doesn't write to the database.
Long comment is where the code in question lies.
Most Relevant snippet
def write_data(con, c):
geolocator = Nominatim()
table = c.execute('SELECT * FROM playerdata WHERE birth_place_long is null AND birth_place is not null').fetchall()
for i in table:
try:
location = getgeo(i[4], geolocator)
lat, long = location.latitude, location.longitude
except:
print(i[0])
''' changing to c.execute doesnt work '''
con.execute('UPDATE playerdata SET birth_place_lat = ?, birth_place_long = ? WHERE id = ?', (1, 1, i[0]))
print("failed" + i[1] + " " + i[2])
con.commit()
else:
c.execute('UPDATE playerdata SET birth_place_lat = ?, birth_place_long = ? WHERE id = ?', (lat, long, i[0]))
print(i[1] + " " + i[2])
con.commit()
con.close()
Full code below:
import csv
from geopy.geocoders import Nominatim
import sqlite3
import socket
import geopy.exc
import pandas as pd
def connect_db():
con = sqlite3.connect('c:/s_db2.sqlite3')
c = con.cursor()
return con, c
def getgeo(place, geolocator):
try:
location = geolocator.geocode(place)
return location
except (socket.timeout):
return getgeo(place, geolocator)
except geopy.exc.GeocoderTimedOut:
return 0
def write_data(con, c):
geolocator = Nominatim()
table = c.execute('SELECT * FROM playerdata WHERE birth_place_long is null AND birth_place is not null').fetchall()
for i in table:
try:
location = getgeo(i[4], geolocator)
lat, long = location.latitude, location.longitude
except:
print(i[0])
''' changing to c.execute doesnt work '''
con.execute('UPDATE playerdata SET birth_place_lat = ?, birth_place_long = ? WHERE id = ?', (1, 1, i[0]))
print("failed" + i[1] + " " + i[2])
con.commit()
else:
c.execute('UPDATE playerdata SET birth_place_lat = ?, birth_place_long = ? WHERE id = ?', (lat, long, i[0]))
print(i[1] + " " + i[2])
con.commit()
con.close()
while True:
write_data(connect_db()[0], connect_db()[1])
print("************* re-running ****************")

This line in your loop:
write_data(connect_db()[0], connect_db()[1])
connect_db() is being called twice in the same call of write_data().
This is similar to:
con_1, c_1 = connect_db()
con_2, c_2 = connect_db()
write_data(con_1, c_2)
You are mixing instances of the connections with cursors of other instances.
instead you may want:
con_1, c_1 = connect_db()
write_data(con_1, c_1)

The conn is using a connection instance which has its own setup, i.e autocommit. But, a curso, you need to define the autocommit set, or rather to persist the commit after executing the sql.
i.e use con.commit() after with con: c.execute()
NB, if this logic is executing after the try-catch block (i.e exception) you probably need to rollback before executing this one.

Python code not creating tables on the database but able to query the results postgres

My usecase is to write create a temp table in the postgres database and fetch records from it and insert into a different table.
The code i used is:
import psycopg2
import sys
import pprint
from __future__ import print_function
from os.path import join,dirname,abspath
import xlrd
import os.path
newlist = []
itemidlist = []
def main():
conn_string = "host='prod-dump.cvv9i14mrv4k.us-east-1.rds.amazonaws.com' dbname='ebdb' user='ebroot' password='*********'"
# print the connection string we will use to connect
# print "Connecting to database" % (conn_string)
# get a connection, if a connect cannot be made an exception will be raised here
conn = psycopg2.connect(conn_string)
# conn.cursor will return a cursor object, you can use this cursor to perform queries
cursor = conn.cursor()
dealer_id = input("Please enter dealer_id: ")
group_id = input("Please enter group_id: ")
scriptpath = os.path.dirname('__file__')
filename = os.path.join(scriptpath, 'Winco - Gusti.xlsx')
xl_workbook = xlrd.open_workbook(filename, "rb")
xl_sheet = xl_workbook.sheet_by_index(0)
print('Sheet Name: %s' % xl_sheet.name)
row=xl_sheet.row(0)
from xlrd.sheet import ctype_text
print('(Column #) type:value')
for idx, cell_obj in enumerate(row):
cell_type_str = ctype_text.get(cell_obj.ctype, 'unknown type')
#print('(%s) %s %s' % (idx, cell_type_str, cell_obj.value))
num_cols = xl_sheet.ncols
for row_idx in range(0, xl_sheet.nrows): # Iterate through rows
num_cols = xl_sheet.ncols
id_obj = xl_sheet.cell(row_idx, 1) # Get cell object by row, col
itemid = id_obj.value
#if itemid not in itemidlist:
itemidlist.append(itemid)
# execute our Query
'''
cursor.execute("""
if not exists(SELECT 1 FROM model_enable AS c WHERE c.name = %s);
BEGIN;
INSERT INTO model_enable (name) VALUES (%s)
END;
""" %(itemid,itemid))
'''
cursor.execute("drop table temp_mbp1")
try:
cursor.execute("SELECT p.model_no, pc.id as PCid, g.id AS GROUPid into public.temp_mbp1 FROM products p, \
model_enable me, products_clients pc, groups g WHERE p.model_no = me.name \
and p.id = pc.product_id and pc.client_id = %s and pc.client_id = g.client_id and g.id = %s"\
% (dealer_id,group_id)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
cursor.execute("select count(*) from public.temp_mbp1")
# retrieve the records from the database
records = cursor.fetchall()
# print out the records using pretty print
# note that the NAMES of the columns are not shown, instead just indexes.
# for most people this isn't very useful so we'll show you how to return
# columns as a dictionary (hash) in the next example.
pprint.pprint(records)
if __name__ == "__main__":
main()
The try except block in between the program is not throwing any error but the table is not getting created in the postgres database as i see in the data admin.
The output shown is:
Please enter dealer_id: 90
Please enter group_id: 13
Sheet Name: Winco Full 8_15_17
(Column #) type:value
[(3263,)]
Thanks,
Santosh

You didn't commit the changes, so they aren't saved in the database. Add to the bottom, just below the pprint statement:
conn.commit()

Python cursor is returning number of rows instead of rows

Writing a script to clean up some data. Super unoptimized but this cursor is
returning the number of results in the like query rather than the rows what am I doing wrong.
#!/usr/bin/python
import re
import MySQLdb
import collections
db = MySQLdb.connect(host="localhost", # your host, usually localhost
user="admin", # your username
passwd="", # your password
db="test") # name of the data base
# you must create a Cursor object. It will let
# you execute all the query you need
cur = db.cursor()
# Use all the SQL you like
cur.execute("SELECT * FROM vendor")
seen = []
# print all the first cell of all the rows
for row in cur.fetchall() :
for word in row[1].split(' '):
seen.append(word)
_digits = re.compile('\d')
def contains_digits(d):
return bool(_digits.search(d))
count_word = collections.Counter(seen)
found_multi = [i for i in count_word if count_word[i] > 1 and not contains_digits(i) and len(i) > 1]
unique_multiples = list(found_multi)
groups = dict()
for word in unique_multiples:
like_str = '%' + word + '%'
res = cur.execute("""SELECT * FROM vendor where name like %s""", like_str)

You are storing the result of cur.execute(), which is the number of rows. You are never actually fetching any of the results.
Use .fetchall() to get all result rows or iterate over the cursor after executing:
for word in unique_multiples:
like_str = '%' + word + '%'
cur.execute("""SELECT * FROM vendor where name like %s""", like_str)
for row in cur:
print row

is it possible to change ms access table name with python

I have several ms access databases that each have a table named PlotStatus-name-3/13/12.
I need to import each of these tables into a .csv table. If I manually change the name of the tables to PlotStatus_name_3_13_12, this code works. Does anyone know how to change the table namees using python?
#connect to access database
for filename in os.listdir(prog_rep_local):
if filename[-6:] == ".accdb":
DBtable = os.path.join(prog_rep_local, filename)
conn = pyodbc.connect(r'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=' + DBtable)
cursor = conn.cursor()
ct = cursor.tables
for row in ct():
rtn = row.table_name
if rtn[:10] == "PlotStatus":
#this does not work:
#Oldpath = os.path.join(prog_rep_local, filename, rtn)
#print Oldpath
#fpr = Oldpath.replace('-', '_')#.replace("/","_")
#print fpr
#newname = os.rename(Oldpath, fpr) this does not work
#print newname
#spqaccdb = "SELECT * FROM " + newname
#this workds if I manually change the table names in advance
sqlaccdb = "SELECT * FROM " + rtn
print sqlaccdb
cursor.execute(sqlaccdb)
rows = cursor.fetchall()

An easier solution would be to just add brackets around the table name so that the /s don't throw off the SQL command interpreter.
sqlaccdb = "SELECT * FROM [" + rtn + "]"