I use SQLite to merge two dataframes I have parameters in. Some work, others don't.
This works:
s = path_autre_qdv+'\\' + fichier_in
DATA = p.read_csv(s)
con = sql.connect('test_db.sqlite')
con.create_function('POWER', 2, sqlite_power)
c = con.cursor()
COM_DETAIL.to_sql('COM_DETAIL_SQL', con, if_exists='replace', index = False)
DATA.to_sql('DATA_SQL', con, if_exists='replace', index = False)
sq = """
SELECT
c.COM_CODE
,c.COM_NAME
,c.COM_LONG
,c.COM_LAT
,d.*
FROM
COM_DETAIL_SQL c
INNER JOIN
DATA_SQL d
ON POWER((c.COM_LONG - d.longitude)/ ? ,2) + POWER((c.COM_LAT - d.latitude)/ ? ,2) <= 1;
"""
par = (0.5, 0.5,)
RES = read_query(con,sq,par)
This doesn't:
s = path_autre_qdv+'\\' + fichier_in
DATA = p.read_csv(s)
con = sql.connect('test_db.sqlite')
con.create_function('POWER', 2, sqlite_power)
c = con.cursor()
COM_DETAIL.to_sql('COM_DETAIL_SQL', con, if_exists='replace', index = False)
DATA.to_sql('DATA_SQL', con, if_exists='replace', index = False)
sq = """
SELECT
c.COM_CODE
,c.COM_NAME
,c.COM_LONG
,c.COM_LAT
,d.*
FROM
COM_DETAIL_SQL c
INNER JOIN
DATA_SQL d
ON POWER((c.COM_LONG - d.longitude)/ ? ,2) + POWER((c.COM_LAT - d.latitude)/ ? ,2) <= 1
AND d.? IS NOT NULL AND d.? IS NOT NULL AND d.? IS NOT NULL;
"""
par = (0.5, 0.5,cinqv,meanv,nonentecinqv,)
RES = read_query(con,sq,par)
Difference is last filter, with three new parameters: cinqv, meanv and nonentecinqv given in input of the function, and are columns of DATA dataframe.
The error :
OperationalError: near "?": syntax error
Related
def SelectData(self):
query = "Select * from python_training.rules"
curr = self.con.cursor()
curr.execute(query)
for row in curr:
result = row
print(result)
The above function returns :
(1, 'CU_GFC_ID', '11A')
(2, 'GFCID', '10')
(1, 'GFCID', '11')
How to make each row as a individual list object?
Check below code convert result to list:
import pandas as pd
import sqlite3
df = pd.DataFrame({'col1':[1,2], 'col2':['a','b']})
conn = sqlite3.connect(':memory:')
df.to_sql(name='df', con=conn, index=False)
print('{:9} | {:^9}'.format('tuple', 'list'))
def SelectData():
query = "Select * from df"
curr = conn.cursor()
curr.execute(query)
for row in curr:
result = list(row)
print('{:9} | {:9}'.format(str(row), str(result)))
SelectData()
Output:
I am trying to insert data into a Postgres table. I have used the below code. Can anyone help me with why I am getting this error and how it could be solved? I did refer to a few solutions and nothing worked. I have used two functions here. One for data preparation and one for data loading. The df contains 700000 rows. I had written a df with 1000 rows using the same approach and it had worked, but when I am doing it for a larger df I am getting the error: current transaction is aborted, commands ignored until end of transaction block
db_engine = create_engine(
'postgresql+psycopg2://{0}:{1}#{2}:{3}/{4}'.format(
'predictive_maintenance_rw',
'',
'dev-da-aurora.cluster-xxxxxxxxxxx.eu-west-1.rds.amazonaws.com',
5432,
'predictive_maintenance'
)
)
conn = psycopg2.connect(
dbname = "predictive_maintenance",
user = "user_rw",
password = "xxxxxx",
host = "dev-da-aurora.cluster-xxxxxx.eu-west-1.rds.amazonaws.com",
port = 5432
)
def data_prep_circularity(path,engine, conn,schema,table):
for x in os.listdir(path):
for y in os.listdir(path+"\\" + x):
if "FP" in y :
for z in os.listdir(path+"\\" + x + "\\" + y):
final_data = []
if "Circularity" in z:
cols_cir = ["axis", "machinenumber", "time", "timeoffset", "reference_value_pos_axis_1","actual_value_pos_axis_1",
"actual_value_pos_axis_2","reference_value_pos_axis_2","plane"]
rows_cir = []
for xml_files in os.listdir (path+"\\" + x + "\\" + y +"\\" + z):
fullname_cir = (path+"\\" + x + "\\" + y +"\\" + z+"\\"+xml_files)
xmlparse_cir = Xet.parse(fullname_cir)
# Parsing the XML file
root_cir = xmlparse_cir.getroot()
for i in root_cir:
# finding axis
if "XY" in fullname_cir:
axis = 'XY'
else :
axis = 'XZ'
# Finding the machine number
machinenumber = x.split("_")[1]
time = fullname_cir.split("-",2)[2].split("\\",1)[0]
timeoffset = i.find("timeoffset").text
reference_value_pos_axis_1 = i.find("f3").text
actual_value_pos_axis_1 = i.find("f1").text
actual_value_pos_axis_2 = i.find("f2").text
reference_value_pos_axis_2 = i.find("f4").text
# finding plane
if "10" in fullname_cir:
plane = '10'
else :
plane = '150'
rows_cir.append({"axis": axis,
"machinenumber": machinenumber,
"time": time ,
"timeoffset": timeoffset,
"reference_value_pos_axis_1" :reference_value_pos_axis_1,
"actual_value_pos_axis_1": actual_value_pos_axis_1,
"actual_value_pos_axis_2": actual_value_pos_axis_2,
"reference_value_pos_axis_2" :reference_value_pos_axis_2,
"plane": plane })
df = pd.DataFrame(rows_cir, columns=cols_cir)
# Writing dataframe to csv
df.reset_index(drop=True, inplace=True)
table_name='dev_fingerprint_circularitytest'
data_load(df, engine, conn, schema, table_name)
def data_load(chunk, engine, conn, schema, table_name):
sio = StringIO()
sio.write(chunk.replace(['\r','\n','\t'], ' ', regex=True).to_csv(index=None, header=None, sep='\t', na_rep = 'NULL')) # Write the Pandas DataFrame as a csv to the buffer
#sio.write(chunk.to_csv(index=None, header=None, sep='\t', na_rep = 'NULL')) # Write the Pandas DataFrame as a csv to the buffer
sio.seek(0) # Be sure to reset the position to the start of the stream
with conn.cursor() as c:
c.copy_from(sio, schema +"." + table_name, columns=chunk.columns, sep='\t', null='NULL')
conn.commit()
data_prep_circularity(path,db_engine, conn, 'public','dev_fingerprint_circularitytest')
I have two lists for two tables with their key columns
keycol1 = [col1,col2]
keycol2 = [col3,col4]
I want to frame a sql query with these two colums as join condition
for column1,column2 in zip(keycol1,keycol2):
join = "table1."+ column1 + " = " + "table2." + column2 and
qry = "select * from table1 join table2 on " + join
But this gives me an extra and at the end. How to avoid it?
Expected query:
select * from table1 join table2 on table1.col1 = table2.col3 and table1.col2 = table2.col4
Here is one way.
keycol1 = ['col1', 'col2']
keycol2 = ['col3', 'col4']
join = ' and '.join(['table1.{0} = table2.{1}'.format(c1, c2) \
for c1, c2 in zip(keycol1, keycol2)])
qry = "select * from table1 join table2 on " + join
# 'select * from table1 join table2 on table1.col1 = table2.col3 and table1.col2 = table2.col4'
Explanation
Use a list comprehension with str.format to form each condition.
Combine your conditions using ' and '.join().
You can also do it using the map function along with join:
keycol1 = ['col1', 'col2']
keycol2 = ['col3', 'col4']
for column1,column2 in zip(keycol1,keycol2):
joined = " and ".join(map(lambda c: "table1."+ c[0] + " = " + "table2." + c[1], zip(keycol1, keycol2)))
qry = "select * from table1 join table2 on " + joined
print(qry)
Output:
select * from table1 join table2 on table1.col1 = table2.col3 and table1.col2 = table2.col4
I am trying to bulk insert locations on wordpress. I have defined functions to check and adding terms and taxonomy
def checkTerm(term,con):
cur = con.cursor()
query = "SELECT term_id FROM wp_terms as t WHERE t.name = '%s'" % term
print query
cur.execute(query)
rows = cur.fetchall()
if rows: return rows[0][0]
else : return None
def addTerm(term,slug,con):
cur = con.cursor()
try:
query = "INSERT INTO `wp_terms` (`name`,`slug`,`term_group`) VALUES ('%s','%s',0)" % (term,slug)
print query
cur.execute(query)
con.commit()
rows = checkTerm(term,con)
if rows: return rows[0][0]
else : return None
except:
return None
def checkTaxonomy(term_id,con):
cur = con.cursor()
query = "SELECT tt.term_taxonomy_id,tt.parent FROM wp_term_taxonomy AS tt INNER JOIN wp_terms AS t ON tt.term_id = t.term_id WHERE tt.taxonomy = 'project_location' AND t.term_id = '%s'" % term_id
print query
cur.execute(query)
rows = cur.fetchall()
if rows: return rows
else : return None
def addTaxonomy(term_id,taxonomy,description,parent,count,con):
cur = con.cursor()
query = "INSERT INTO `wp_term_taxonomy` (`term_id`,`taxonomy`,`description`,`parent`,`count`) VALUES ('%s','%s','%s','%s','%s')" % (term_id,taxonomy,description,parent,count)
print query
cur.execute(query)
con.commit()
rows = checkTaxonomy(term_id,con)
if rows: return rows
else: return None
I store cities in dictionary of dicionaries
df = pd.read_table('./Argentina.csv',sep='\t',header=None,engine='python')
for line in xrange(len(df)):
stringa = str(df[17][line])
location = str(df[1][line])
population = int(df[14][line])
if population < limit_pop: continue
string_state = stringa.split("/")
country = string_state[1]
state = string_state[2]
if not country in states:
states[country] = {}
if not state in states[country]:
states[country][state] = [location]
else :
states[country][state].append(location)
Then I try to insert terms and taxonomies in the wordpress db
con = mdb.connect('localhost', 'root', 'mypassword, 'Wordpress')
for country in states:
country_id = checkTerm(country.replace("_"," "),con)
if not country_id:
country_id = addTerm(country.replace("_"," "),country,con)
taxonomy = checkTaxonomy(country_id,con)
if not taxonomy:
taxonomy = addTaxonomy(country_id,'project_location','','0','0',con)
parent = dict((y, x) for x, y in taxonomy)
if not 0 in parent:
taxonomy = addTaxonomy(country_id,'project_location','','0','0',con)
for state in states[country]:
state_id = checkTerm(state.replace("_"," "),con)
if not state_id:
state_id = addTerm(state.replace("_"," "),state,con)
taxonomy = checkTaxonomy(state_id,con)
if not taxonomy:
taxonomy = addTaxonomy(state_id,'project_location','',country_id,'0',con)
parent = dict((y, x) for x, y in taxonomy)
if not country_id in parent:
taxonomy = addTaxonomy(state_id,'project_location','',country_id,'0',con)
for location in states[country][state]:
location_id=checkTerm(location.replace("_"," "),con)
if not location_id:
location_id = addTerm(location.replace("_"," "),location,con)
taxonomy = checkTaxonomy(location_id,con)
if not taxonomy:
taxonomy = addTaxonomy(location_id,'project_location','',state_id,'0',con)
parent = dict((y, x) for x, y in taxonomy)
if not state_id in parent:
taxonomy = addTaxonomy(location_id,'project_location','',state_id,'0',con)
When I try to execute the script I found this behaviour
SELECT term_id FROM wp_terms as t WHERE t.name = 'Argentina'
INSERT INTO `wp_terms` (`name`,`slug`,`term_group`) VALUES ('Argentina','Argentina',0)
SELECT term_id FROM wp_terms as t WHERE t.name = 'Argentina'
SELECT tt.term_taxonomy_id,tt.parent FROM wp_term_taxonomy AS tt INNER JOIN wp_terms AS t ON tt.term_id = t.term_id WHERE tt.taxonomy = 'project_location' AND t.term_id = 'None'
INSERT INTO `wp_term_taxonomy` (`term_id`,`taxonomy`,`description`,`parent`,`count`) VALUES ('None','project_location','','0','0')
SELECT tt.term_taxonomy_id,tt.parent FROM wp_term_taxonomy AS tt INNER JOIN wp_terms AS t ON tt.term_id = t.term_id WHERE tt.taxonomy = 'project_location' AND t.term_id = 'None'
And the script stop with the following error
./import.py:59: Warning: Truncated incorrect DOUBLE value: 'None'
cur.execute(query)
./import.py:69: Warning: Incorrect integer value: 'None' for column 'term_id' at row 1
cur.execute(query)
Traceback (most recent call last):
File "./import.py", line 115, in <module>
parent = dict((y, x) for x, y in taxonomy)
TypeError: 'NoneType' object is not iterable
This means that the insert statements are not executed. I don't understand. I con.commit() the query but it is still not executed. Where is the problem?
Solution:
I changed
import MySQLdb as mdb
to
import mysql.connector
and
con = mdb.connect(host='localhost',user='root', password='passowrd',database= 'Wordpress');
to
con = mysql.connector.connect(host='localhost',user='root', password='password',database= 'Wordpress',autocommit=False,buffered=False);
i use a function calculate servers prices so i made a function which retrieve a defalut prices of a server components and calculate server price for each server exsit in my DB but i try to run this function, i get this error:
function.py
import MySQLdb
def calculations_metric (param) :
db = MySQLdb.connect("localhost", "root", "aqw", "PFE_Project")
cursor = db.cursor()
sql = "SELECT * FROM examples_calculationsmetric"
cursor.execute(sql)
results = cursor.fetchall()
for row in results:
RAM_prices = int(row[1])
Core_prices = int(row[2])
HHD_SATA_prices =int(row[3])
HHD_SSD_prices =int(row[4])
CPU_priority = int(row[5])
Avaibility = int(row[6])
db.close()
db1 = MySQLdb.connect("localhost", "root", "aqw", "PFE_Project")
cursor1 = db1.cursor()
sql1 = "SELECT * FROM examples_servercomponents WHERE id ='%d'" %(param)
cursor1.execute(sql1)
results1 = cursor1.fetchall()
for row in results1:
if row[6] == 'SATA':
Core_price = int(row[2]) * Core_prices # the error is here
Priority_price = int(row[3]) * CPU_priority
RAM_price = int(row[4]) * RAM_prices
HDD_price = int(row[5]) * HHD_SATA_prices
Availibility_price = int(row[7])*Avaibility
elif row[6] == 'SSD':
Core_price = int(row[2]) * Core_prices
Priority_price = int(row[3]) * CPU_priority
RAM_price = int(row[4]) * RAM_prices
HDD_price = int(row[5]) * HHD_SSD_prices
Availibility_price = int(row[7])*Avaibility
price = Core_price + Priority_price + RAM_price + HDD_price + Availibility_price
db1.close()
return price
i don't get what is the error so if can anyone help i will be so greatful
When your SELECT * FROM examples_calculationsmetric doesn't return any results, Core_prices is never set (nor are the other variables in that loop).
Python names do not exist until assigned to, so if results is an empty list, the names inside the for loop never get assigned to and thus do not exist by the time you loop over results1 later on.
You could set default values for those names as a work-around:
RAM_prices = 0
Core_prices = 0
HHD_SATA_prices = 0
HHD_SSD_prices = 0
CPU_priority = 0
Avaibility = 0
to at least ensure that they are defined.