INPUT FILE:
$ cat inputs.csv
'18-01-2019', 296.0
'18-01-2019', 296.0
'18-01-2019', 296.0
CODE:
import csv
import sqlite3
import pprint
conn = sqlite3.connect('metrics.db')
c = conn.cursor()
def read_file(filename):
with open(filename, 'r') as f:
yield from f
for row in read_file('inputs.csv'):
data = row.split(',')
values = '({}, {})'.format(data[0], data[1].strip())
print('Values are: {}'.format(values))
try:
query = '\'INSERT INTO metric_db VALUES (?, ?)\', {}'.format(values)
print('Query is: {}'.format(query))
c.execute(query)
except sqlite3.IntegrityError as e:
pass
conn.commit()
conn.close()
OUTPUT ERROR:
Values are: ('18-01-2019', 296.0)
Query is: 'INSERT INTO metric_db VALUES (?, ?)', ('18-01-2019', 296.0)
Traceback (most recent call last):
File "write_to_db.py", line 21, in <module>
c.execute(query)
sqlite3.OperationalError: near "'INSERT INTO metric_db VALUES (?, ?)'": syntax error
I thought this was easier. Quite a few SO threads on this error. But I am still not there yet :(
Change your query statement:
query = 'INSERT INTO metric_db VALUES {}'.format(values)
EDIT
To Avoid SQL injection and use correct date format:
import csv
import sqlite3
import pprint
from datetime import datetime
conn = sqlite3.connect('metrics.db')
c = conn.cursor()
def read_file(filename):
with open(filename, 'r') as f:
yield from f
for row in read_file('inputs.csv'):
data = row.split(',')
values = '({}, {})'.format(data[0], data[1].strip())
print('Values are: {}'.format(values))
date_readed = datetime.strptime(data[0], '%d-%m-%Y').strftime('%Y-%m-%d')
try:
query = 'INSERT INTO metric_db VALUES (?, ?)'
c.execute(query,(date_readed,data[1],))
except sqlite3.IntegrityError as e:
pass
conn.commit()
conn.close()
Check your schema.sql file. just try to copy-paste this file from tutorial.
Related
I am new to Python and started off with sqlite.
I have two csv transaction.csv and users.csv from where I am reading the data and writing to the sqlite database.Below is the snippet
import csv
import sqlite3 as db
def readCSV_users():
with open('users.csv',mode='r') as data:
dr = csv.DictReader(data, delimiter=',')
users_data = [(i['user_id'], i['is_active']) for i in dr if i['is_active']=='True']
#---------------------
return users_data
def readCSV_transactions():
with open('transactions.csv',mode='r') as d:
dr = csv.DictReader(d, delimiter=',')
trans_data = [(i['user_id'], i['is_blocked'],i['transaction_amount'],i['transaction_category_id']) for i in dr if i['is_blocked']=='False']
#---------------------
return trans_data
def SQLite_connection(database):
try:
# connect to the database
conn = db.connect(database)
print("Database connection is established successfully!")
conn = db.connect(':memory:')
print("Established database connection to a database\
that resides in the memory!")
cur = conn.cursor()
return cur,conn
except exception as Err:
print(Err)
def dbQuery(users_data,trans_data,cur,conn):
try:
cur.executescript(""" CREATE TABLE if not exists users(user_id text,is_active text);
CREATE TABLE if not exists transactions(user_id text,is_blocked text,transaction_amount text,transaction_category_id text);
INSERT INTO users VALUES (?,?),users_data;
INSERT INTO transactions VALUES (?,?,?,?),trans_data""")
conn.commit()
a=[]
rows = curr.execute("SELECT * FROM users").fetchall()
for r in rows:
a.append(r)
return a
except Err:
print(Err)
finally:
conn.close()
if __name__ == "__main__":
database='uit'
users_data=readCSV_users()
trans_data=readCSV_transactions()
curr,conn=SQLite_connection(database)
print(dbQuery(users_data,trans_data,curr,conn))
But I am facing below error.I believe the ? is throwing the error in executescript
cur.executescript(""" CREATE TABLE if not exists users(user_id text,is_active text);
sqlite3.OperationalError: near "users_data": syntax error
Any pointers to resolve this?
Putting users_data directly in query is wrong. It treats it as normal string.
But it seems executescript can't use arguments.
You would have to put values directly in place of ?.
Or you have to use execute()
cur.execute("INSERT INTO users VALUES (?,?);", users_data)
cur.execute("INSERT INTO transactions VALUES (?,?,?,?)", trans_data)
I know there are some other posts out there, but I was not able to find the specific question I had in mind.
I'm using US_baby_names csv file. and want to import this csv file line by line into sqlite3 as a table.
I'm able to create the table called storage.
I'm then trying to read lines in the csv file and put it into that table, but I must be doing something wrong.
import sqlite3 as sql
from sqlite3 import Error
import csv
def CreateConnection ( dbFileName ):
try:
conn = sql.connect(dbFileName)
return conn
except Error as e:
print(e)
return None
def CreateNew( dbConnection, new):
sql = """INSERT INTO storage (dat, Id, Name, Year, group, subgroup, Count)
VALUES (?,?,?,?,?,?,?)"""
try:
cursor = dbConnection.cursor()
cursor.execute(sql, new)
return cursor.lastrowid
except Error as e:
print(e)
def Main():
database = "storage.db"
dbConnection = CreateConnection(database)
with open('storage.csv', 'rb') as fin:
dr = csv.DictReader(fin)
to_db = [(i['dat'], i['Id'], i['Name'], i['Year'], i['group'], i['subgroup'], i['Count']) \
for i in dr]
cursor.executemany(CreateNew(sql, to_db))
dbConnection.close()
if __name__ == "__main__":
Main()
I believe my cursor.executemany is wrong, but I'm not able to figure out what else to do..
Thanks
You are almost right with much of your code, but:
in cursor.execute(sql, new) you are passing an iterable, new, to sqlite3.execute() (which requires a simple SQL statement), instead of sqlite3.executemany().
Moreover, the result of CreateNew() is an integer, lastrowid, and you pass that result to executemany().
You must use Connection.commit() to save the changes to the database, and Connection.rollback() to discard them.
You must open the file for the csv.DictReader class as a text file, in r or rt mode.
Finally, remember that sqlite3.Connection is a context manager, so you can use it in a with statement.
This should be your desired outcome:
import sqlite3 as sql
from sqlite3 import Error
import csv
def create_table(conn):
sql = "CREATE TABLE IF NOT EXISTS baby_names("\
"dat TEXT,"\
"Id INTEGER PRIMARY KEY,"\
"Name TEXT NOT NULL,"\
"Year INTEGER NOT NULL,"\
"Gender TEXT NOT NULL,"\
"State TEXT NOT NULL,"\
"Count INTEGER)"
conn.execute(sql)
conn.execute("DELETE FROM baby_names")
def select_all(conn):
for r in conn.execute("SELECT * FROM baby_names").fetchall():
print(r)
def execute_sql_statement(conn, data):
sql = "INSERT INTO baby_names "\
"(dat, Id, Name, Year, Gender, State, Count) "\
"VALUES (?,?,?,?,?,?,?)"
try:
cursor = conn.executemany(sql, data)
except Error as e:
print(e)
conn.rollback()
return None
else:
conn.commit()
return cursor.lastrowid
def main():
with sql.connect('baby_names.db') as conn, open('US_Baby_Names_right.csv', 'r') as fin:
create_table(conn)
dr = csv.DictReader(fin)
data = [(i['dat'], i['Id'], i['Name'], i['Year'], i['Gender'], i['State'], i['Count']) for i in dr ]
lastrowid = execute_sql_statement(conn, data)
select_all(conn)
main()
I added a create_table() function just to test my code. I also made up a sample test file as follows:
dat,Id,Name,Year,Gender,State,Count
1,1,John,1998,M,Washington,2
2,2,Luke,2000,M,Arkansas,10
3,3,Carrie,1999,F,Texas,3
The output of the select_all() function is:
('1',1,'John',1998,'M','Washington',2)
('2',2,'Luke',2000,'M','Arkansas',10)
('3',3,'Carrie',1999,'F','Texas',3)
I am trying to add the contents of variables into a SQLite DB but I am getting an error of
sqlite3.InterfaceError: Error binding parameter 0 - probably unsupported type.
My code is:-
import requests
import json
import eventlet
import os
import sqlite3
#Get the currect vuln_sets
response = requests.get('https://vulners.com/api/v3/search/stats/')
vuln_set = json.loads(response.text)
vuln_type = vuln_set['data']['type_results']
for k in vuln_type:
vuln_bulletinfamily = vuln_set['data']['type_results'][k]['bulletinFamily']
vuln_name = vuln_set['data']['type_results'][k]['displayName']
vuln_count = vuln_set['data']['type_results'][k]['count']
con = sqlite3.connect('vuln_sets.db')
with con:
cur = con.cursor()
con.row_factory = sqlite3.Row
cur.execute("REPLACE INTO vuln_sets (vulntype, displayname, bulletinfamily, count) values (?, ?, ?, ?)", (vuln_type, vuln_bulletinfamily, vuln_name, vuln_count))
con.commit()
The variables contain the JSON key pairs as I need to insert some of them into the DB for processing but a different project.
The stacktrace is:
Traceback (most recent call last):
File "test.py", line 24, in <module>
cur.execute("REPLACE INTO vuln_sets (vulntype, displayname, bulletinfamily, count) values (?, ?, ?, ?);", (vuln_type, vuln_bulletinfamily, vuln_name, vuln_count))
sqlite3.InterfaceError: Error binding parameter 0 - probably unsupported type.
#roganjosh's comment fixed it! I needed to incude the DB transactions in the for loop as below:
import requests
import json
import eventlet
import os
import sqlite3
#Get the currect vuln_sets
response = requests.get('https://vulners.com/api/v3/search/stats/')
vuln_set = json.loads(response.text)
vuln_type = vuln_set['data']['type_results']
for k in vuln_type:
vuln_bulletinfamily = vuln_set['data']['type_results'][k]['bulletinFamily']
vuln_name = vuln_set['data']['type_results'][k]['displayName']
vuln_count = vuln_set['data']['type_results'][k]['count']
con = sqlite3.connect('vuln_sets.db')
with con:
cur = con.cursor()
con.row_factory = sqlite3.Row
cur.execute("REPLACE INTO vuln_sets (vulntype, displayname, bulletinfamily, count) values (?, ?, ?, ?)", (k, vuln_name, vuln_bulletinfamily, vuln_count))
con.commit()
I'm trying to parse a Russian web-site (in Cyrillic) and insert data to a mySQL DB. The parsing is fine, but I can't save the data in the DB because of the Cyrillic letters. Python give me this error:
Traceback (most recent call last):
File "/Users/kr/PycharmProjects/education_py/vape_map.py", line 40, in <module>
print parse_shop_meta()
File "/Users/kr/PycharmProjects/education_py/vape_map.py", line 35, in parse_shop_meta
VALUES (%s, %s, %s, %s)""",(shop_title, shop_address, shop_phone, shop_site, shop_desc))
File "/Library/Python/2.7/site-packages/MySQLdb/cursors.py", line 210, in execute
query = query % args
TypeError: not all arguments converted during string formatting
My code:
# -- coding: utf-8 --
import requests
from lxml.html import fromstring
import csv
import MySQLdb
db = MySQLdb.connect(host="localhost", user="root", passwd="***", db="vape_map", charset='utf8')
def get_shop_urls():
i = 1
all_shop_urls = []
while i < 2:
url = requests.get("http://vapemap.ru/shop/?city=%D0%9C%D0%BE%D1%81%D0%BA%D0%B2%D0%B0&page={}".format(i))
page_html = fromstring(url.content)
shop_urls = page_html.xpath('//h3[#class="title"]/a/#href')
all_shop_urls += shop_urls
i +=1
return all_shop_urls
def parse_shop_meta():
shops_meta = []
csvfile = open('vape_shops.csv', 'wb')
writer = csv.writer(csvfile, quotechar='|', quoting=csv.QUOTE_ALL)
cursor = db.cursor()
for shop in get_shop_urls():
url = requests.get("http://vapemap.ru{}".format(shop), 'utf-8')
page_html = fromstring(url.content)
shop_title = page_html.xpath('//h1[#class="title"]/text()')
shop_address = page_html.xpath('//div[#class="address"]/text()')
shop_phone = page_html.xpath('//div[#class="phone"]/a/text()')
shop_site = page_html.xpath('//div[#class="site"]/a/text()')
shop_desc = page_html.xpath('//div[#class="shop-desc"]/text()')
sql = """INSERT INTO vape_shops(title, address, phone, site, description)
VALUES (%s, %s, %s, %s)""",(shop_title, shop_address, shop_phone, shop_site, shop_desc)
cursor.execute(sql, (shop_title[0], shop_address[0], shop_phone[0], shop_site[0], shop_desc[0]))
db.commit()
db.close()
return shops_meta
print parse_shop_meta()
%D0%9C%D0%BE%D1%81%D0%BA%D0%B2%D0%B0 is the encoding for Москва, so that looks OK. But you also need to establish that utf8 will be used in the connection to MySQL. And specify that the target column is CHARACTER SET utf8.
More details and Python-specifics
i have the following problem:
i want to insert the temperature of my RPI using SQLite3 and Python.
The python script that i want to use:
import subprocess
import os
import sqlite3 as lite
import datetime
import sys
import time
def get_temperature():
"Returns the temperature in degrees C"
try:
s = subprocess.check_output(["cat","/sys/class/thermal/thermal_zone0/temp"])
return s[:-1]
except:
return 0
try:
con = lite.connect('/www/auslastung.s3db')
cur = con.cursor()
temp = int(get_temperature())
zeit = time.strftime('%Y-%m-%d %H:%M:%S')
cur.execute("INSERT INTO 'temperatur' ('Wert', 'Zeit') VALUES (%s, %s)", (temp, zeit))
con.commit()
except lite.Error, e:
if con:
con.rollback()
print "Error %s" % e.args[0]
sys.exit(1)
finally:
if con:
con.close()
Every time i want to run this, i just get the error:
Error near "%": syntax error
What should i do to solve this?
Replace
cur.execute("INSERT INTO 'temperatur' ('Wert', 'Zeit') VALUES (%s, %s)", (temp, zeit))
with
cur.execute("INSERT INTO 'temperatur' ('Wert', 'Zeit') VALUES (?, ?)", (temp, zeit))
There is also a problem with your finally clause. It will fail with the error NameError: name 'con' is not defined if con is never assigned to in the first place (e.g., if the directory /www/ does not exist, so con = lite.connect('/www/auslastung.s3db') fails). You could do the following to avoid this issue:
con = None
try:
# ...
except lite.Error, e:
if con is not None:
con.rollback()
# ...
finally:
if con is not None:
con.close()
You can also replace:
cur.execute("INSERT INTO 'temperatur' ('Wert', 'Zeit') VALUES (%s, %s)", (temp, zeit))
with
cur.execute("INSERT INTO 'temperatur' ('Wert', 'Zeit') VALUES (%s, %s)" % (temp, zeit))
Either #nwk or my answer should work depending on preference.