I'm connecting to Redshift with Psycopg with the following piece of code
import psycopg2
con = psycopg2.connect(dbname='events', host=myhost,
port=my_post, user=my_user, password=my_pwd)
Then, check if a table called "event" exists and I do that like so
def table_exists(con, table_str):
#
exists = False
try:
cur = con.cursor()
cur.execute("select exists(select relname from pg_class where relname='" + table_str + "')")
exists = cur.fetchone()[0]
cur.close()
except psycopg2.Error as e:
print(e)
return exists
table_exists(con, "event")
Which returns True. However, when I try to read the table (for instance, below I try to return the column names) I get an error message saying that the "relation doesn't exists"
def get_table_col_names(con, table_str):
#
col_names = []
try:
cur = con.cursor()
cur.execute("select * from " + table_str + " LIMIT 0")
for desc in cur.description:
col_names.append(desc[0])
cur.close()
except psycopg2.Error as e:
print(e)
#
return col_names
get_table_col_names(con, "event")
Can you point me in the direction of what's happening?
for me its working fine
import psycopg2
rs_conn = psycopg2.connect(host='MYHOST',
user='NAME',
port='PORT NUM',
password='PASS',
dbname='DBNAME')
def get_table_col_names(con, table_str):
col_names = []
try:
cur = con.cursor()
cur.execute("select * from " + table_str + " LIMIT 0")
for desc in cur.description:
col_names.append(desc[0])
cur.close()
except psycopg2.Error as e:
print(e)
return col_names
columnnames = get_table_col_names(rs_conn, "TABLE NAME")
print(columnnames)
output:
[column names]
Related
I am trying to get rows quantity from query using python to mysql. I've been using rowcount but returned value is always 0.
def getLastData(md5TwitterDate):
conectar = connection()
try:
cursor = conectar.cursor()
query = "SELECT fecha,numeroreporte,internoid FROM jsywe_ncnsismos_sismosigp WHERE internoid = '%s' and published=1"
cursor.execute(query, md5TwitterDate)
lastEvent = cursor.fetchall()
rowsa = cursor.rowcount
except Error as ex:
print("Error to get data: ", ex)
finally:
if conectar.is_connected():
conectar.close()
#return False if rowsa> 0 else True
return rowsa
Also tried this way setting a variable to cursor.execute but in this case always get none
def getLastData(md5TwitterDate):
conectar = connection()
try:
cursor = conectar.cursor()
query = "SELECT fecha,numeroreporte,internoid FROM jsywe_ncnsismos_sismosigp WHERE internoid = '%s' and published=1"
rowsa = cursor.execute(query, md5TwitterDate)
lastEvent = cursor.fetchall()
except Error as ex:
print("Error to get data: ", ex)
finally:
if conectar.is_connected():
conectar.close()
#return False if filas > 0 else True
return rowsa
Tested query on database and it works, it returns 1 row
I have a database class which abstracts some basic crud logic.
The issue lies in the fetch_single method:
The sql_insecure query works fine, and returns the expected results.
The sql_prepared query doesn't return any errors, but also doesn't return any results which match the parameters, when they clearly do exist within the db.
sql_prepared follows the same approach to prepared statements that the insert_single method implements, and this method also returns the expected results.
My question is; why is the sql_prepared query not returning any results?
import sqlite3
class Database:
def __init__(self, db: str):
try:
self.conn = sqlite3.connect(db)
self.cursor = self.conn.cursor()
except sqlite3.Error as e:
print(e)
self.__del__
def fetch_all(self, table: str):
try:
query = self.cursor.execute("SELECT * FROM ?", table)
rows = self.cursor.fetchall()
return rows
except sqlite3.Error as e:
print(e)
return False
def fetch_single(self, table: str, column_name: str, column_value):
sql_formatted_value = "'{value}'".format(value=column_value)
placeholder = ":{column_name}".format(column_name=column_name)
sql_insecrue = "SELECT * FROM %s WHERE %s=%s Limit 1" % (
table, column_name, sql_formatted_value)
sql_prepared = "SELECT * FROM %s WHERE %s=%s LIMIT 1" % (
table, column_name, placeholder)
# try:
# self.cursor.execute(sql_insecrue)
# rows = self.cursor.fetchall()
# return rows
# except sqlite3.Error as e:
# print(e)
# return False
try:
self.cursor.execute(sql_prepared, [sql_formatted_value, ])
rows = self.cursor.fetchall()
return rows
except sqlite3.Error as e:
print(e)
return False
def insert_single(self, table: str, data: list):
columns = ""
placeholders = ""
values = []
data_length = len(data)
for index, (key, value) in enumerate(data):
# we need to dynamically build some strings based on the data
# let's generate some placeholders to execute prepared statements
columns += "{column_name}".format(column_name=key)
placeholders += ":{column_name}".format(column_name=key)
# let's fill the insert values into a list to use with execute
values.append(value)
# only add a comma if there is another item to assess
if index < (data_length - 1):
columns += ', '
placeholders += ', '
sql = "INSERT INTO %s (%s) VALUES (%s)" % (
table, columns, placeholders)
try:
self.cursor.execute(sql, values)
self.conn.commit()
except sqlite3.Error as e:
print(e)
You cannot substitute table name using ? in prepared statements because it is not considered a query parameter.
I recommend doing something like this:
self.cursor.execute(f"DELETE FROM {table} WHERE id=?", [id])
In other words, use standard python format statements to specify your table name, but use prepared statement anchors like ? for any query parameters.
okay, i found the problem.
its was my sloppy sql syntax.
using backticks around the table and column name solved the issue.
def fetch_single(self, table: str, column_name: str, column_value):
sql_formatted_value = "'{value}'".format(value=column_value)
placeholder = ":{column_name}".format(column_name=column_name)
sql_insecure = "SELECT * FROM %s WHERE %s=%s" % (
table, column_name, sql_formatted_value)
sql_prepared = "SELECT * FROM `%s` WHERE `%s`=%s" % (
table, column_name, placeholder)
print(sql_insecure)
print(sql_prepared)
# try:
# self.cursor.execute(sql_insecure)
# row = self.cursor.fetchall()
# print(row)
# return row
# except sqlite3.Error as e:
# print(e)
# return False
try:
self.cursor.execute(sql_prepared,
[column_value, ])
row = self.cursor.fetchone()
return row
except sqlite3.Error as e:
print(e)
return False
I am using postgres in Python with the library psycopg2. After connecting to a database, I am trying to check if a table with a given name exists. In postgres, I am doing this with the following lines:
\connect myDB
select exists(select * from pg_tables where schemaname='public' AND tablename='mytable';)
This works if the table exists but also if it doesn't.
In python I am doing this with the following lines:
import psycopg2 as pg
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT;
from psycopg2 import sql;
conn = pg.connect(user='postgres', host='localhost', password="pwd");
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT);
conn.autocommit = True
sql_table_check = sql.SQL("select exists(select * from pg_tables where schemaname='public' AND tablename={});")\
.format(sql.Identifier("mytable"));
cur = conn.cursor()
but this is returning the error
psycopg2.errors.UndefinedColumn: column "mytable" does not exist
LINE 1: ...m pg_tables where schemaname='public' AND tablename="mytable");
because such table has not been created yet.
What is the correct way to check if a column exists in psycopg2?
EDIT
Note that I would like to check the existence of the table in the database I am connected to, I don't mind if it exists in another database.
My comment as an answer:
import psycopg2
con = psycopg2.connect("dbname=production host=localhost user=postgres")
tbl_sql = "SELECT count(*) FROM pg_tables WHERE schemaname='public' AND tablename= %s"
cur = con.cursor()
cur.execute(tbl_sql, ('cell_per',))
cur.fetchone()
(1,)
cur.execute(tbl_sql, ('cell_p',))
cur.fetchone()
(0,)
so you can go over error by using try and except to continue
sql_table_check = sql.SQL("select exists(select * from pg_tables where schemaname='public' AND tablename={});")\
.format(sql.Identifier("mytable"));
try:
cur = conn.cursor()
print('Table exists')
except:
print('Table does not exist')
Edited based on comment
you can also catch the error to check it later by
sql_table_check = sql.SQL("select exists(select * from pg_tables where schemaname='public' AND tablename={});")\
.format(sql.Identifier("mytable"));
try:
cur = conn.cursor()
print('Table exists')
except Exception as e:
print(e)
print('Table does not exist')
for example, and simply, if we try:
try:
a = 5 / 0
except Exception as e:
print(e)
We will get output
division by zero
you can get the exact format string through debugging e content in the exception part.
so we can then, use this to identify the error, to be used again like:
try:
a = a / 0
except Exception as e:
print(e)
if e.args[0] == 'division by zero':
print('it is division by zero Error')
else:
raise(e)
so if the error is not the intended one the other error will raise.
you may get the error exception from psycopg2 documentation as that for python in https://docs.python.org/3/library/exceptions.html
as in the follwoing code:
try:
a = 5 / 0
except ZeroDivisionError:
print('it is division by zero Error')
so we get:
it is division by zero Error
but when we have another error like:
try:
a = 5 / 0
except ZeroDivisionError:
print('it is division by zero Error')
we get the other error
NameError: name 'b' is not defined
I am having trouble with executing UPDATE statements.
In the code below when I write con.execute() it writes to the database, but if I write c.execute() it doesn't write to the database.
Long comment is where the code in question lies.
Most Relevant snippet
def write_data(con, c):
geolocator = Nominatim()
table = c.execute('SELECT * FROM playerdata WHERE birth_place_long is null AND birth_place is not null').fetchall()
for i in table:
try:
location = getgeo(i[4], geolocator)
lat, long = location.latitude, location.longitude
except:
print(i[0])
''' changing to c.execute doesnt work '''
con.execute('UPDATE playerdata SET birth_place_lat = ?, birth_place_long = ? WHERE id = ?', (1, 1, i[0]))
print("failed" + i[1] + " " + i[2])
con.commit()
else:
c.execute('UPDATE playerdata SET birth_place_lat = ?, birth_place_long = ? WHERE id = ?', (lat, long, i[0]))
print(i[1] + " " + i[2])
con.commit()
con.close()
Full code below:
import csv
from geopy.geocoders import Nominatim
import sqlite3
import socket
import geopy.exc
import pandas as pd
def connect_db():
con = sqlite3.connect('c:/s_db2.sqlite3')
c = con.cursor()
return con, c
def getgeo(place, geolocator):
try:
location = geolocator.geocode(place)
return location
except (socket.timeout):
return getgeo(place, geolocator)
except geopy.exc.GeocoderTimedOut:
return 0
def write_data(con, c):
geolocator = Nominatim()
table = c.execute('SELECT * FROM playerdata WHERE birth_place_long is null AND birth_place is not null').fetchall()
for i in table:
try:
location = getgeo(i[4], geolocator)
lat, long = location.latitude, location.longitude
except:
print(i[0])
''' changing to c.execute doesnt work '''
con.execute('UPDATE playerdata SET birth_place_lat = ?, birth_place_long = ? WHERE id = ?', (1, 1, i[0]))
print("failed" + i[1] + " " + i[2])
con.commit()
else:
c.execute('UPDATE playerdata SET birth_place_lat = ?, birth_place_long = ? WHERE id = ?', (lat, long, i[0]))
print(i[1] + " " + i[2])
con.commit()
con.close()
while True:
write_data(connect_db()[0], connect_db()[1])
print("************* re-running ****************")
This line in your loop:
write_data(connect_db()[0], connect_db()[1])
connect_db() is being called twice in the same call of write_data().
This is similar to:
con_1, c_1 = connect_db()
con_2, c_2 = connect_db()
write_data(con_1, c_2)
You are mixing instances of the connections with cursors of other instances.
instead you may want:
con_1, c_1 = connect_db()
write_data(con_1, c_1)
The conn is using a connection instance which has its own setup, i.e autocommit. But, a curso, you need to define the autocommit set, or rather to persist the commit after executing the sql.
i.e use con.commit() after with con: c.execute()
NB, if this logic is executing after the try-catch block (i.e exception) you probably need to rollback before executing this one.
Hello
I have a question about SQLite functions, maybe.
So, question:
How to check if name I set in Python is in certain column?
Example:
name = 'John'
Table name = my_table
Column name = users
Code details:
C = conn.cursor()
Please
Use parameter in the query as required. See the attached example for better understanding.
Sample SQLite code for searching value in tables
import sqlite3 as sqlite
import sys
conn = sqlite.connect("test.db")
def insert_single_row(name, age):
try:
age = str(age)
with conn:
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS USER_TABLE(NAME TEXT, AGE INTEGER);")
cursor.execute("INSERT INTO USER_TABLE(NAME, AGE) VALUES ('"+name+"',"+age+")")
return cursor.lastrowid
except:
raise ValueError('Error occurred in insert_single_row(name, age)')
def get_parameterized_row(name):
try:
with conn:
cursor = conn.cursor()
cursor.execute("SELECT * FROM USER_TABLE WHERE NAME = :NAME",
{"NAME":name})
conn.commit()
return cursor.fetchall()
except:
raise ValueError('Error occurred in get_parameterized_row(name)')
if __name__ == '__main__':
try:
return_id = insert_single_row("Shovon", 24)
return_id = insert_single_row("Shovon", 23)
return_id = insert_single_row("Sho", 24)
all_row = get_parameterized_row("Shovon")
for row in all_row:
print(row)
except Exception as e:
print(str(e))
Output:
('Shovon', 24)
('Shovon', 23)
Here I have created a table called USER_TABLE with two attributes: NAME and AGE. Then I inserted several values in the table and searched for a specific NAME. Hope it gives a way to start using SQLite in the project.