I currently have a function which takes a dictionary as an input and returns all columns in database table as a dict:
import sqlite3
def get_person_dict_from_dict(self):
database = "/Users/Mary/Documents/Database/{DB}.db".format(DB=self['db'])
conn = sqlite3.connect(database)
conn.row_factory = sqlite3.Row
c = conn.cursor()
sql_command = "SELECT * FROM {dbTableIn} WHERE Identifier = '{nameIn}' AND Day = {dateIn};".format(
dbTableIn=self['my_table'],
dateIn=self['date'],
nameIn=self['names'])
c.execute(sql_command)
r = c.fetchall()
result = [dict(row) for row in r]
dict_out = result[0]
return dict_out
inputDict = {"date" : '19891229',"names" : 'Mary', "db" :'MyDatabase', "my_table" :'Measurements'}
outputDict = get_person_dict_from_dict(inputDict)
This works fine. However, how do I refine it such that:
1) I can include an additional argument, as a list/tuple/dict of variable length, such that I can pull only the attributes that I am interested in rather than all of the available metrics. For example:
attributesWanted = ['Age', 'Height']
inputDict = {attributesWanted, "date" : '19891229',"names" : 'Mary', "db" :'MyDatabase', "my_table" :'Measurements'}
yet, for example, be able to flexibly use attributesWanted = ['Age', 'Height', 'ShoeSize'] if needed.
2) Be able to do this for multiple individuals. E.g namesWanted = ['Mary', 'Joe']
Preferably would be able to use a single dictionary as an input to the function. I have tried including lists and tuples but run up against errors such as
TypeError: 'tuple' object is not a mapping
I would recommend to use external function which parse the clause to the relevant column and pass the args as
column_type so you will be able to defer between the value types
another thing is using kwargs (keyword args) in the given function
here is a short example which parse the args and concat them.
import sqlite3
def format_arg(key, value):
## remove the suffix
key_without_suffix = "_".join(key.split("_")[:-1])
if key.endswith("str"):
return "{key} = '{value}'".format(key=key_without_suffix, value=value)
if key.endswith("date"):
return "{key} = {value}".format(key=key_without_suffix, value=value)
if key.endswith("list"):
return "{key} IN ({value})".format(key=key_without_suffix, value=(",".join(value)))
def get_person_dict_from_dict(db, table, **kwargs):
database = "/Users/Mary/Documents/Database/{DB}.db".format(DB=db)
conn = sqlite3.connect(database)
conn.row_factory = sqlite3.Row
c = conn.cursor()
where_clause = " AND ".join([format_arg(k,v) for k,v in kwargs.items()])
sql_command = "SELECT * FROM {table} WHERE {where};".format(
table=table, where=where_clause)
c.execute(sql_command)
r = c.fetchall()
result = [dict(row) for row in r]
dict_out = result[0]
return dict_out
input_dict = {"Date_date": "19891229", "names_list": ["Marry", "Anne"]}
get_person_dict_from_dict(db="MyDatabase", table="Measurements", **input_dict)
Related
I need to get several rows by their id from myTable,
the required row id's are in a list mylist,
I'm getting:
"'int' object does not support indexing"
What am I doing wrong? (I'm using psycopg2 if that is relevant)
sql = "SELECT myTable.* where id = %s"
mylist = list()
mylist.append(492)
mylist.append(450)
all_meta = list()
cur = engine.cursor()
cur.executemany(sql, mylist)
all_elements_from_db = cur.fetchmany()
for ob in all_elements_from_db:
meta = dict()
meta[ID] = ob[0]
meta[SEC_ID] = ob[1]
all_meta.append(meta)
cur.close()
I'm trying to pass multiple values to my INSERT function using psycopg2. According to the documentation, you must past a list or tuple, which I am doing, but it's still breaking with the following error:
"File "c:\action.py", line 42, in putTicket
cur.execute(SQL, ins)
TypeError: not all arguments converted during string formatting"
Code:
data = response.json() # get JSON response
record = data[0] # header row
col = record.keys() # column names
a = []
def putTicket(conn):
cur = conn.cursor()
for record in data: # data rows
a = []
y = record.values() # values on this row in an array
for col in y:
a.append(col)
ins = tuple(a)
SQL = "INSERT INTO fd_tickets VALUES (%s)"
cur.execute(SQL, ins)
print("Using psycopg2...")
myConnection = psycopg2.connect(host=hostname, user=username, password=password, dbname=database, port=port)
putTicket(myConnection)
myConnection.close()
You have multiple values, but only one placeholder. You need as many placeholders as there are values in your tuple.
placeholders = ','.join(['%s'] * len(a))
SQL = "INSERT INTO fd_tickets VALUES ({})".format(placeholders)
I have a folder called 'testfolder' that includes two files -- 'Sigurdlogfile' and '2004ADlogfile'. Each file has a list of strings called entries. I need to run my code on both of them and am using glob to do this. My code creates a dictionary for each file and stores data extracted using regex where the dictionary keys are stored in commonterms below. Then it inserts each dictionary into a mysql table. It does all of this successfully, but my second sql statement is not inserting how it should (per file).
import glob
import re
files = glob.glob('/home/user/testfolder/*logfile*')
commonterms = (["freq", "\s?(\d+e?\d*)\s?"],
["tx", "#txpattern"],
["rx", "#rxpattern"], ...)
terms = [commonterms[i][0] for i in range(len(commonterms))]
patterns = [commonterms[i][1] for i in range(len(commonterms))]
def getTerms(entry):
for i in range(len(terms)):
term = re.search(patterns[i], entry)
if term:
term = term.groups()[0] if term.groups()[0] is not None else term.groups()[1]
else:
term = 'NULL'
d[terms[i]] += [term]
return d
for filename in files:
#code to create 'entries'
objkey = re.match(r'/home/user/testfolder/(.+?)logfile', filename).group(1)
d = {t: [] for t in terms}
for entry in entries:
d = getTerms(entry)
import MySQLdb
db = MySQLdb.connect(host='', user='', passwd='', db='')
cursor = db.cursor()
cols = d.keys()
vals = d.values()
for i in range(len(entries)):
lst = [item[i] for item in vals]
csv = "'{}'".format("','".join(lst))
sql1 = "INSERT INTO table (%s) VALUES (%s);" % (','.join(cols), csv.replace("'NULL'", "NULL"))
cursor.execute(sql1)
#now in my 2nd sql statement I need to update the table with data from an old table, which is where I have the problem...
sql2 = "UPDATE table, oldtable SET table.key1 = oldtable.key1,
table.key2 = oldtable.key2 WHERE oldtable.obj = %s;" % repr(objkey)
cursor.execute(sql2)
db.commit()
db.close()
The problem is that in the second sql statement, it ends up inserting that data into all columns of the table from only one of the objkeys, but I need it to insert different data depending on which file the code is currently running on. I can't figure out why this is, since I've defined objkey inside my for filename in files loop. How can I fix this?
Instead of doing separate INSERT and UPDATE, do them together to incorporate the fields from the old table.
for i in range(len(entries)):
lst = [item[i] for item in vals]
csv = "'{}'".format("','".join(lst))
sql1 = """INSERT INTO table (key1, key2, %s)
SELECT o.key1, o.key2, a.*
FROM (SELECT %s) AS a
LEFT JOIN oldtable AS o ON o.obj = %s""" % (','.join(cols), csv.replace("'NULL'", "NULL"), repr(objkey))
cursor.execute(sql1)
I am trying to select from a specific row and then column in SQL.
I want to find a specific user_name row and then select the access_id from the row.
Here is all of my code.
import sys, ConfigParser, numpy
import MySQLdb as mdb
from plaid.utils import json
class SQLConnection:
"""Used to connect to a SQL database and send queries to it"""
config_file = 'db.cfg'
section_name = 'Database Details'
_db_name = ''
_hostname = ''
_ip_address = ''
_username = ''
_password = ''
def __init__(self):
config = ConfigParser.RawConfigParser()
config.read(self.config_file)
print "making"
try:
_db_name = config.get(self.section_name, 'db_name')
_hostname = config.get(self.section_name, 'hostname')
_ip_address = config.get(self.section_name, 'ip_address')
_user = config.get(self.section_name, 'user')
_password = config.get(self.section_name, 'password')
except ConfigParser.NoOptionError as e:
print ('one of the options in the config file has no value\n{0}: ' +
'{1}').format(e.errno, e.strerror)
sys.exit()
self.con = mdb.connect(_hostname, _user, _password, _db_name)
self.con.autocommit(False)
self.con.ping(True)
self.cur = self.con.cursor(mdb.cursors.DictCursor)
def query(self, sql_query, values=None):
"""
take in 1 or more query strings and perform a transaction
#param sql_query: either a single string or an array of strings
representing individual queries
#param values: either a single json object or an array of json objects
representing quoted values to insert into the relative query
(values and sql_query indexes must line up)
"""
# TODO check sql_query and values to see if they are lists
# if sql_query is a string
if isinstance(sql_query, basestring):
self.cur.execute(sql_query, values)
self.con.commit()
# otherwise sql_query should be a list of strings
else:
# execute each query with relative values
for query, sub_values in zip(sql_query, values):
self.cur.execute(query, sub_values)
# commit all these queries
self.con.commit
return self.cur.fetchall
def get_plaid_token(self,username):
result= self.query("SELECT access_id FROM `users` WHERE `user_name` LIKE %s",[username])
print type (result)
return result
print SQLConnection().get_plaid_token("test")
I would like the get the transaction ID but for some reason "result" returns
> <bound method DictCursor.fetchall of <MySQLdb.cursors.DictCursor
> object at 0x000000000396F278>>
result is also of type "instancemethod"
try changing this line:
return self.cur.fetchall
to
return self.cur.fetchall()
Without the parentheses after the method name, you are returning a reference to that method itself, not running the method.
I would like to create a "generic" function that contains SQL SELECT, but I want this function to be able to accept some parameters that are not defined.
I have this code:
def request(self, identifier= "%", sender="%", receiver="%", context="%", state="%"):
result = None
myConnection = MemoryManagement().getConnection()
myCursor = myConnection.cursor()
myCursor.execute("SELECT * from MemorySpace WHERE id = ? and sender = ? and receiver = ? and context = ? and state = ?", (identifier, sender, receiver, context, state))
result = myCursor.fetchall()
myCursor.close()
myConnection.close()
return result
The thing is that the "%" does not work as a wildcard...I would like to know if there is a solution that I could program myself for this problem ? (i.e. I do not want to use a framework or any kind of layer to access the sql database :) )
Thank you in advance for your help
How about something like this :)
Also adds some reusability
BASE_QUERY = '''
SELECT *
FROM MemorySpace
'''
def get_connection(self):
return MemoryManagement().getConnection()
def get_result(self, query, args):
cursor = self.get_connection()
cursor.execute(query, args)
result = cursor.fetchall()
cursor.close()
return result
def request(self, **kwargs):
args = []
filters = []
for k, v in kwargs.iteritems():
filters.append('%s = ?' % k)
args.append(v)
query = BASE_QUERY
if filters:
query += ' AND '.join(filters)
return self.get_result(query, args)