Another way to say this is "find all substrings within a string"; while there's a bunch of these posts on stackoverflow already, none I've found quite get the job done.
#query variable
query = "select * from table1 a join table2 b on a.id = b.id left join table3 c on b.id = c.id where x = 5;"
query_list = query.split()
#find the adjacent word in a list
def find_adjacents(some_value, some_list):
i = some_list.index(some_value)
return some_list[i:i+2]
print('from tables:', find_adjacents("from", query_list)[1])
print('join tables:', find_adjacents("join", query_list)[1])
>>>from tables: table1
>>>join tables: table2
There should be one more join tables: table3. How do I get this to print All instances of the adjacent substring "join"?
Is this what you're after?
# Query variable
query = "select * from table1 a join table2 b on a.id = b.id left join table3 c on b.id = c.id where x = 5;"
query_list = query.split()
def find_adjacents(some_value, some_list):
val = [index for index, value in enumerate(some_list) if value == some_value]
allVal = []
for x in val:
allVal.append(some_list[x+1])
return allVal
>>> print('from tables:', find_adjacents("from", query_list))
from tables: ['table1']
>>> print('join tables:', find_adjacents("join", query_list))
join tables: ['table2', 'table3']
Related
I use SQLite to merge two dataframes I have parameters in. Some work, others don't.
This works:
s = path_autre_qdv+'\\' + fichier_in
DATA = p.read_csv(s)
con = sql.connect('test_db.sqlite')
con.create_function('POWER', 2, sqlite_power)
c = con.cursor()
COM_DETAIL.to_sql('COM_DETAIL_SQL', con, if_exists='replace', index = False)
DATA.to_sql('DATA_SQL', con, if_exists='replace', index = False)
sq = """
SELECT
c.COM_CODE
,c.COM_NAME
,c.COM_LONG
,c.COM_LAT
,d.*
FROM
COM_DETAIL_SQL c
INNER JOIN
DATA_SQL d
ON POWER((c.COM_LONG - d.longitude)/ ? ,2) + POWER((c.COM_LAT - d.latitude)/ ? ,2) <= 1;
"""
par = (0.5, 0.5,)
RES = read_query(con,sq,par)
This doesn't:
s = path_autre_qdv+'\\' + fichier_in
DATA = p.read_csv(s)
con = sql.connect('test_db.sqlite')
con.create_function('POWER', 2, sqlite_power)
c = con.cursor()
COM_DETAIL.to_sql('COM_DETAIL_SQL', con, if_exists='replace', index = False)
DATA.to_sql('DATA_SQL', con, if_exists='replace', index = False)
sq = """
SELECT
c.COM_CODE
,c.COM_NAME
,c.COM_LONG
,c.COM_LAT
,d.*
FROM
COM_DETAIL_SQL c
INNER JOIN
DATA_SQL d
ON POWER((c.COM_LONG - d.longitude)/ ? ,2) + POWER((c.COM_LAT - d.latitude)/ ? ,2) <= 1
AND d.? IS NOT NULL AND d.? IS NOT NULL AND d.? IS NOT NULL;
"""
par = (0.5, 0.5,cinqv,meanv,nonentecinqv,)
RES = read_query(con,sq,par)
Difference is last filter, with three new parameters: cinqv, meanv and nonentecinqv given in input of the function, and are columns of DATA dataframe.
The error :
OperationalError: near "?": syntax error
def SelectData(self):
query = "Select * from python_training.rules"
curr = self.con.cursor()
curr.execute(query)
for row in curr:
result = row
print(result)
The above function returns :
(1, 'CU_GFC_ID', '11A')
(2, 'GFCID', '10')
(1, 'GFCID', '11')
How to make each row as a individual list object?
Check below code convert result to list:
import pandas as pd
import sqlite3
df = pd.DataFrame({'col1':[1,2], 'col2':['a','b']})
conn = sqlite3.connect(':memory:')
df.to_sql(name='df', con=conn, index=False)
print('{:9} | {:^9}'.format('tuple', 'list'))
def SelectData():
query = "Select * from df"
curr = conn.cursor()
curr.execute(query)
for row in curr:
result = list(row)
print('{:9} | {:9}'.format(str(row), str(result)))
SelectData()
Output:
I need to create a dictionary where I am able to split all the select attributes, from table, where condition into different list, but it should have just 1 key. So far I have tried the below-
import pandas
import re
import sqlparse
query = "(select t1.id, t1.feed_id, t2.eff_dt, name, type from table1 t1 INNER JOIN table2 t2 ON ( t1.eff_dt = t2.eff_dt WHERE t2.eff_dt = CAST ( ('2020/12/20') AS DATE) and (t2.name = 'abc' or t2.name='pqr'))"
query2= "SELECT DISTINCT id, feed_id, eff_date FROM"
newInput= input('enter the eff_dt:')
new_where = '("'+newInput+'")'
query1= re.sub("(?<=WHERE t2.eff_dt = CAST \( )(.*)(?= AS DATE)", new_where , query, flags=re.IGNORECASE)
formatted_query = sqlparse.format(query1, reindent=True, keyword_case='lower')
formatted_query1 = sqlparse.format(query2, reindent=True, keyword_case='lower')
a = formatted_query.strip()
a1 = formatted_query1.strip()
newformat = a1+a
newformat.lstrip()
dict_list = newformat.split('\n')
dictOfWords = { i : dict_list[i] for i in range(0, len(dict_list) ) }
newDict = dict(zip(dictOfWords.keys(), [v.strip() if isinstance(v,str) else v for v in dictOfWords.values()]))
newDict
This takes the input from user for date and gives the output as below-
enter the eff_dt:2022/01/20
{0: 'select distinct id,',
1: 'feed_id,',
2: 'eff_date',
3: 'from(',
4: 'select t1.id,',
5: 't1.feed_id,',
6: 't2.eff_dt,',
7: 'name,',
8: 'type',
9: 'from table1 t1',
10: 'inner join table2 t2 on (t1.eff_dt = t2.eff_dt',
11: 'where t2.eff_dt = cast (("2022/01/20") as DATE)',
12: "and (t2.name = 'abc'",
13: "or t2.name='pqr'))"}
But I need the output as below-
{0: 'select distinct id,','feed_id,','eff_date','from(','select t1.id,','t1.feed_id,','t2.eff_dt,','name,','type','from table1 t1','inner join table2 t2 on (t1.eff_dt = t2.eff_dt','where t2.eff_dt = cast (("2022/01/20") as DATE)',"and (t2.name = 'abc'","or t2.name='pqr'))"}
Is there a way to do it? Remove all the keys except 1, but keep all the values in the dictionary. I would really appreciate some help.
Let's make a new dictionary.
This keeps the first key of the original dictionary as the only key and adds all the values as a list to that key.
# let d be the original dictionary
k,v = list(d.keys()), list(d.values())
new = {k[0]:v}
print(new)
I have updated your code directly, let me know if you still face errors.
import pandas
import re
import sqlparse
query = "(select t1.id, t1.feed_id, t2.eff_dt, name, type from table1 t1 INNER JOIN table2 t2 ON ( t1.eff_dt = t2.eff_dt WHERE t2.eff_dt = CAST ( ('2020/12/20') AS DATE) and (t2.name = 'abc' or t2.name='pqr'))"
query2= "SELECT DISTINCT id, feed_id, eff_date FROM"
newInput= input('enter the eff_dt:')
new_where = '("'+newInput+'")'
query1= re.sub("(?<=WHERE t2.eff_dt = CAST \( )(.*)(?= AS DATE)", new_where , query, flags=re.IGNORECASE)
formatted_query = sqlparse.format(query1, reindent=True, keyword_case='lower')
formatted_query1 = sqlparse.format(query2, reindent=True, keyword_case='lower')
a = formatted_query.strip()
a1 = formatted_query1.strip()
newformat = a1+a
newformat.lstrip()
dict_list = newformat.split('\n')
dictOfWords = { i : dict_list[i] for i in range(0, len(dict_list) ) }
newDict = dict(zip(dictOfWords.keys(), [v.strip() if isinstance(v,str) else v for v in dictOfWords.values()]))
k,v = list(newDict.keys()), list(newDict.values())
new = {k[0]:v}
new
{0: ['select distinct id,',
'feed_id,',
'eff_date',
'from(',
'select t1.id,',
't1.feed_id,',
't2.eff_dt,',
'name,',
'type',
'from table1 t1',
'inner join table2 t2 on (t1.eff_dt = t2.eff_dt',
'where t2.eff_dt = cast (("2022/01/20") as DATE)',
"and (t2.name = 'abc'",
"or t2.name='pqr'))"]}
a = {0:'zero', 1: 'one', 2: 'two'}
res = {0: list(a.values())}
print(res) # --> {0: ['zero', 'one', 'two']}
I have two lists for two tables with their key columns
keycol1 = [col1,col2]
keycol2 = [col3,col4]
I want to frame a sql query with these two colums as join condition
for column1,column2 in zip(keycol1,keycol2):
join = "table1."+ column1 + " = " + "table2." + column2 and
qry = "select * from table1 join table2 on " + join
But this gives me an extra and at the end. How to avoid it?
Expected query:
select * from table1 join table2 on table1.col1 = table2.col3 and table1.col2 = table2.col4
Here is one way.
keycol1 = ['col1', 'col2']
keycol2 = ['col3', 'col4']
join = ' and '.join(['table1.{0} = table2.{1}'.format(c1, c2) \
for c1, c2 in zip(keycol1, keycol2)])
qry = "select * from table1 join table2 on " + join
# 'select * from table1 join table2 on table1.col1 = table2.col3 and table1.col2 = table2.col4'
Explanation
Use a list comprehension with str.format to form each condition.
Combine your conditions using ' and '.join().
You can also do it using the map function along with join:
keycol1 = ['col1', 'col2']
keycol2 = ['col3', 'col4']
for column1,column2 in zip(keycol1,keycol2):
joined = " and ".join(map(lambda c: "table1."+ c[0] + " = " + "table2." + c[1], zip(keycol1, keycol2)))
qry = "select * from table1 join table2 on " + joined
print(qry)
Output:
select * from table1 join table2 on table1.col1 = table2.col3 and table1.col2 = table2.col4
I am trying to bulk insert locations on wordpress. I have defined functions to check and adding terms and taxonomy
def checkTerm(term,con):
cur = con.cursor()
query = "SELECT term_id FROM wp_terms as t WHERE t.name = '%s'" % term
print query
cur.execute(query)
rows = cur.fetchall()
if rows: return rows[0][0]
else : return None
def addTerm(term,slug,con):
cur = con.cursor()
try:
query = "INSERT INTO `wp_terms` (`name`,`slug`,`term_group`) VALUES ('%s','%s',0)" % (term,slug)
print query
cur.execute(query)
con.commit()
rows = checkTerm(term,con)
if rows: return rows[0][0]
else : return None
except:
return None
def checkTaxonomy(term_id,con):
cur = con.cursor()
query = "SELECT tt.term_taxonomy_id,tt.parent FROM wp_term_taxonomy AS tt INNER JOIN wp_terms AS t ON tt.term_id = t.term_id WHERE tt.taxonomy = 'project_location' AND t.term_id = '%s'" % term_id
print query
cur.execute(query)
rows = cur.fetchall()
if rows: return rows
else : return None
def addTaxonomy(term_id,taxonomy,description,parent,count,con):
cur = con.cursor()
query = "INSERT INTO `wp_term_taxonomy` (`term_id`,`taxonomy`,`description`,`parent`,`count`) VALUES ('%s','%s','%s','%s','%s')" % (term_id,taxonomy,description,parent,count)
print query
cur.execute(query)
con.commit()
rows = checkTaxonomy(term_id,con)
if rows: return rows
else: return None
I store cities in dictionary of dicionaries
df = pd.read_table('./Argentina.csv',sep='\t',header=None,engine='python')
for line in xrange(len(df)):
stringa = str(df[17][line])
location = str(df[1][line])
population = int(df[14][line])
if population < limit_pop: continue
string_state = stringa.split("/")
country = string_state[1]
state = string_state[2]
if not country in states:
states[country] = {}
if not state in states[country]:
states[country][state] = [location]
else :
states[country][state].append(location)
Then I try to insert terms and taxonomies in the wordpress db
con = mdb.connect('localhost', 'root', 'mypassword, 'Wordpress')
for country in states:
country_id = checkTerm(country.replace("_"," "),con)
if not country_id:
country_id = addTerm(country.replace("_"," "),country,con)
taxonomy = checkTaxonomy(country_id,con)
if not taxonomy:
taxonomy = addTaxonomy(country_id,'project_location','','0','0',con)
parent = dict((y, x) for x, y in taxonomy)
if not 0 in parent:
taxonomy = addTaxonomy(country_id,'project_location','','0','0',con)
for state in states[country]:
state_id = checkTerm(state.replace("_"," "),con)
if not state_id:
state_id = addTerm(state.replace("_"," "),state,con)
taxonomy = checkTaxonomy(state_id,con)
if not taxonomy:
taxonomy = addTaxonomy(state_id,'project_location','',country_id,'0',con)
parent = dict((y, x) for x, y in taxonomy)
if not country_id in parent:
taxonomy = addTaxonomy(state_id,'project_location','',country_id,'0',con)
for location in states[country][state]:
location_id=checkTerm(location.replace("_"," "),con)
if not location_id:
location_id = addTerm(location.replace("_"," "),location,con)
taxonomy = checkTaxonomy(location_id,con)
if not taxonomy:
taxonomy = addTaxonomy(location_id,'project_location','',state_id,'0',con)
parent = dict((y, x) for x, y in taxonomy)
if not state_id in parent:
taxonomy = addTaxonomy(location_id,'project_location','',state_id,'0',con)
When I try to execute the script I found this behaviour
SELECT term_id FROM wp_terms as t WHERE t.name = 'Argentina'
INSERT INTO `wp_terms` (`name`,`slug`,`term_group`) VALUES ('Argentina','Argentina',0)
SELECT term_id FROM wp_terms as t WHERE t.name = 'Argentina'
SELECT tt.term_taxonomy_id,tt.parent FROM wp_term_taxonomy AS tt INNER JOIN wp_terms AS t ON tt.term_id = t.term_id WHERE tt.taxonomy = 'project_location' AND t.term_id = 'None'
INSERT INTO `wp_term_taxonomy` (`term_id`,`taxonomy`,`description`,`parent`,`count`) VALUES ('None','project_location','','0','0')
SELECT tt.term_taxonomy_id,tt.parent FROM wp_term_taxonomy AS tt INNER JOIN wp_terms AS t ON tt.term_id = t.term_id WHERE tt.taxonomy = 'project_location' AND t.term_id = 'None'
And the script stop with the following error
./import.py:59: Warning: Truncated incorrect DOUBLE value: 'None'
cur.execute(query)
./import.py:69: Warning: Incorrect integer value: 'None' for column 'term_id' at row 1
cur.execute(query)
Traceback (most recent call last):
File "./import.py", line 115, in <module>
parent = dict((y, x) for x, y in taxonomy)
TypeError: 'NoneType' object is not iterable
This means that the insert statements are not executed. I don't understand. I con.commit() the query but it is still not executed. Where is the problem?
Solution:
I changed
import MySQLdb as mdb
to
import mysql.connector
and
con = mdb.connect(host='localhost',user='root', password='passowrd',database= 'Wordpress');
to
con = mysql.connector.connect(host='localhost',user='root', password='password',database= 'Wordpress',autocommit=False,buffered=False);