I'm trying to implement a few simple SQL insert statements with python-mariadb-connector but cannot figure out what I'm doing wrong.
The database looks like this:
SET FOREIGN_KEY_CHECKS = false;
CREATE OR REPLACE TABLE `forums` (
`id` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE OR REPLACE TABLE `accounts` (
`id` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE OR REPLACE TABLE `posts` (
`id` int(10) unsigned NOT NULL,
`forum_id` int(10) unsigned NOT NULL,
`account_id` int(10) unsigned NOT NULL,
PRIMARY KEY (`id`),
KEY `posts_forum_fk` (`forum_id`),
KEY `posts_account_fk` (`account_id`),
CONSTRAINT `posts_account_fk` FOREIGN KEY (`account_id`) REFERENCES `accounts` (`id`),
CONSTRAINT `posts_forum_fk` FOREIGN KEY (`forum_id`) REFERENCES `forums` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE OR REPLACE TABLE `comments` (
`id` int(10) unsigned NOT NULL,
`post_id` int(10) unsigned NOT NULL,
`account_id` int(10) unsigned NOT NULL,
`parent_id` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `comments_post_fk` (`post_id`),
KEY `comments_account_fk` (`account_id`),
-- KEY `comments_comments_fk` (`parent_id`),
-- CONSTRAINT `comments_comments_fk` FOREIGN KEY (`parent_id`) REFERENCES `comments` (`id`),
CONSTRAINT `comments_account_fk` FOREIGN KEY (`account_id`) REFERENCES `accounts` (`id`),
CONSTRAINT `comments_post_fk` FOREIGN KEY (`post_id`) REFERENCES `posts` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
SET FOREIGN_KEY_CHECKS = true;
The code for inserting data looks like this:
import mariadb
config = {
"user": "db_user_name",
"password": "db_passwd",
"host": "db_host",
"port": 3306,
"database": "db_name"
}
if __name__ == '__main__':
with mariadb.connect(**config) as conn:
cur = conn.cursor()
cur.executemany(
"INSERT INTO `forums` (`id`) VALUES (?)",
[(1,), (2,), (3,)]
)
cur.executemany(
"INSERT INTO `accounts` (`id`) VALUES (?)",
[(1,), (2,), (3,), (4,)]
)
cur.executemany(
"INSERT INTO `posts` (`id`, `forum_id`, `account_id`) VALUES (?, ?, ?)",
[(6, 3, 1)]
)
cur.executemany(
"INSERT INTO `comments` (`id`, `post_id`, `account_id`, `parent_id`) VALUES (?, ?, ?, ?)",
[(1, 6, 1, None), (2, 6, 2, 1)]
) # exception happens here
When executing this, I get the following error:
Traceback (most recent call last):
File ".../db_test.py", line 28, in <module>
cur.executemany(
mariadb.DatabaseError.DataError: Invalid parameter type at row 2, column 4
Im not sure how executemany is implemented but I think it should do something similar to the following SQL-query:
INSERT INTO `forums` (`id`) VALUES (1), (2), (3);
INSERT INTO `accounts` (`id`) VALUES (1), (2), (3), (4);
INSERT INTO `posts` (`id`, `forum_id`, `account_id`) VALUES (6, 3, 1);
INSERT INTO `comments` (`id`, `post_id`, `account_id`, `parent_id`)
VALUES (1, 6, 1, NULL), (2, 6, 2, 1);
which works just fine for me...
Is it a bug or am I doing something wrong here?
It took me a while
cur.executemany(
"INSERT INTO `comments` (`id`, `post_id`, `account_id`, `parent_id`) VALUES (?, ?, ?, ?)",
[(1, 6, 1, None), (2, 6, 2, 1)]
)
But in your comment table, you have this constraint
CONSTRAINT `comments_comments_fk` FOREIGN KEY (`parent_id`)
REFERENCES `comments` (`id`),
Before you can enter (2, 6, 2, 1) the tuple (1, 6, 1, None) has to already commit in the database and in a bulk insert the commit is made after all inserts are in the database, but the first tuple isn't at the time not there
so if you make this, both rows will appear in the database(i also committed all other tables after bulk insert):
MariaDB
mycursor.execute(
"INSERT INTO `comments` (`id`, `post_id`, `account_id`,
`parent_id`) VALUES (?, ?, ?, ?)",
(1, 6, 1,None ))
mydb.commit()
mycursor.executemany(
"INSERT INTO `comments` (`id`, `post_id`, `account_id`, `parent_id`) VALUES (?, ?, ?, ?)",
[ (2, 6, 2, 1)])
mydb.commit()
MySQL
sql = """INSERT INTO forums (id) VALUES (%s)"""
val = [("1",), ("2",), ("3",)]
mycursor.executemany(sql,val)
mydb.commit()
mycursor.executemany(
"INSERT INTO accounts (id) VALUES (%s)",
[(1,), (2,), (3,), (4,)]
)
mydb.commit()
mycursor.executemany(
"INSERT INTO posts (id, forum_id, account_id) VALUES (%s, %s, %s)",
[(6, 3, 1)]
)
mydb.commit()
mycursor.execute(
"INSERT INTO `comments` (`id`, `post_id`, `account_id`, `parent_id`) VALUES (%s, %s, %s, %s)",
(1, 6, 1,None ))
mydb.commit()
mycursor.executemany(
"INSERT INTO `comments` (`id`, `post_id`, `account_id`, `parent_id`) VALUES (%s, %s, %s, %s)",
[ (2, 6, 2, 1)])
mydb.commit()
Related
My backend using Python and Flask splits JSON data into various endpoints, to retrieve in my client Swift app rather than having to download the full data client-side.
JSON file :
{
"pilots": [
{
"cid": 1234567,
"name": "John Smith",
"callsign": "TIA1",
"server": "USA-WEST",
"pilot_rating": 3,
"latitude": 18.42663,
"longitude": 116.15007,
"altitude": 41038,
"groundspeed": 435,
"transponder": "2200",
"heading": 154,
"qnh_i_hg": 29.96,
"qnh_mb": 1015,
"flight_plan": {
"flight_rules": "I",
"aircraft": "B737/M-VGDW/C",
"aircraft_faa": "B737/L",
"aircraft_short": "B737",
"departure": "LTBA",
"arrival": "WAMM",
"alternate": "",
"cruise_tas": "437",
"altitude": "33000",
"deptime": "1230",
"enroute_time": "1415",
"fuel_time": "1542",
"remarks": "PBN/B1D1O1S1 DOF/221107 REG/VPCTY EET/LTAA0020 UDDD0137 UBBA0151 UTAK0222 UTAA0247 UTAV0309 UTSD0322 UTTR0345 UAII0352 UTTR0354 UCFO0412 UCFM0434 ZWUQ0451 ZLHW0606 ZPKM0741 ZGZU0856 VHHK0946 RPHI1020 WAAF1251 SEL/EJKS CODE/ADF5D2 OPR/TEXAS AIR LLC ORGN/KCHIUALE PER/C RMK/CALLSIGN \"TEXAS\" /V/",
"route": "ASMAP UL333 SIV UA4 ERZ UB374 INDUR N449 DUKAN A480 KRS B701 TUGTA A909 BABUM A477 POGON L143 TISIB L141 KAMUD W186 SADAN Y1 OMBON B330 AVPAM A599 POU B330 CH A583 ZAM A461 BONDA",
"revision_id": 4,
"assigned_transponder": "0363"
},
"logon_time": "2022-11-06T07:07:42.1130925Z",
"last_updated": "2022-11-07T22:36:19.1087668Z"
}
}
I import JSON into various SQLite tables. JSON data is updated every 60 seconds, so I need to update my copy accordingly. My current solution is to delete data in database then reinsert, but that's most likely not the right way. I'm not sure how I should go about diffing records in database against latest JSON, I could retrieve all records then compare old and new line by line, but this could be even less efficient. What's a robust way of doing this in Python?
Code to insert pilots and associated flight plans :
def _store_pilots(pilots):
"""Removes all records from the db, then stores pilots and associated flight plans, checking for duplicate CIDs."""
pilots_list = []
cid_list = []
fp_list = []
for pilot in pilots:
cid = int(pilot['cid'])
if cid in cid_list:
continue
cid_list.append(cid)
pilot_tuple = (
pilot['cid'], pilot['name'],
pilot['callsign'], pilot['server'],
pilot['pilot_rating'],
pilot['latitude'], pilot['longitude'],
pilot['altitude'], pilot['groundspeed'],
pilot['transponder'], pilot['heading'],
pilot['qnh_i_hg'], pilot['qnh_mb'],
pilot['logon_time'], pilot['last_updated']
)
pilots_list.append(pilot_tuple)
if pilot['flight_plan']:
fp = pilot['flight_plan']
fp_tuple = (
pilot['cid'], fp['flight_rules'],
fp['aircraft'], fp['aircraft_faa'], fp['aircraft_short'],
fp['departure'], fp['arrival'], fp['alternate'],
fp['cruise_tas'], fp['altitude'],
fp['deptime'], fp['enroute_time'], fp['fuel_time'],
fp['remarks'], fp['route'], fp['revision_id']
)
fp_list.append(fp_tuple)
with sqlite3.connect(DATABASE_PATH) as connection:
connection.execute("PRAGMA foreign_keys = 1")
c = connection.cursor()
c.execute("DELETE FROM pilots")
c.executemany("""
INSERT INTO pilots(cid, name, callsign, server, pilot_rating, latitude, longitude, altitude, groundspeed, transponder, heading, qnh_i_hg, qnh_mb, logon_time, last_updated)
values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", pilots_list)
c.executemany("""
INSERT INTO flight_plans VALUES (?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?
)
""", fp_list)
You can create a table with columns cid, raw_json_value, json_hash and check hash value before update / insert. Here is an example:
data = [{'cid': 1, ...}, {'cid': 2, ...}, {'cid': 3, ...}]
for item in data: # type: dict
cid = item['cid']
json_hash = hash(json.dumps(data))
# Record - let's say a model from db
record = get_record_by_cid(cid)
if not record:
save_record(cid, item, json_hash)
continue
if record.json_hash != json_hash:
update_record(cid, item, json_hash)
Python: 2.7.12
pyodbc: 4.0.24
OS: Ubuntu 16.4
DB: MySQL 8
driver: MySQL 8
Expected behaviour: resultset should have numbers in columns with datatype int
Actual Behaviour: All of the columns with int data type have 0's (If parameterised query is used)
Here's the queries -
1.
cursor.execute("SELECT * FROM TABLE where id =7")
Result set:
[(7, 1, None, 1, u'An', u'Zed', None, u'Ms', datetime.datetime(2016, 12, 20, 0, 0), u'F', u'Not To Be Disclosed', None, None, u'SPRING', None, u'4000', datetime.datetime(2009, 5, 20, 18, 55), datetime.datetime(2019, 1, 4, 14, 25, 58, 763000), 0, None, None, None, bytearray(b'\x00\x00\x00\x00\x01(n\xba'))]
2.
cursor.execute("SELECT * FROM patients where patient_id=?", [7])`
or
cursor.execute("SELECT * FROM patients where patient_id=?", ['7'])
or
cursor.execute("SELECT * FROM patients where patient_id IN ", [7])
Result set:
[(0, 0, None, 0, u'An', u'Zed', None, u'Ms', datetime.datetime(2016, 12, 20, 0, 0), u'F', u'Not To Be Disclosed', None, None, u'SPRING', None, u'4000', datetime.datetime(2009, 5, 20, 18, 55), datetime.datetime(2019, 1, 4, 14, 25, 58, 763000), 0, None, None, None, bytearray(b'\x00\x00\x00\x00\x01(n\xba'))]
Rest of the result set is okay except for the columns with int data type that all have 0's if paramterized query is used.
It seems like it should have worked without issues. Can I get some help here.
Edit : Here's the schema of the table:
CREATE TABLE `patient
`lastname` varchar(30) DEFAULT NULL,
`known_as` varchar(30) DEFAULT NULL,
`title` varchar(50) DEFAULT NULL,
`dob` datetime DEFAULT NULL,
`sex` char(1) DEFAULT NULL,
`address1` varchar(30) DEFAULT NULL,
`address2` varchar(30) DEFAULT NULL,
`address3` varchar(30) DEFAULT NULL,
`city` varchar(30) DEFAULT NULL,
`state` varchar(16) DEFAULT NULL,
`postcode` char(4) DEFAULT NULL,
`datecreated` datetime NOT NULL,
`dateupdated` datetime(6) DEFAULT NULL,
`isrep` tinyint(1) DEFAULT NULL,
`photo` longblob,
`foreign_images_imported` tinyint(1) DEFAULT NULL,
`ismerged` tinyint(1) DEFAULT NULL,
`rowversion` varbinary(8) DEFAULT NULL,
PRIMARY KEY (`patient_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
You have encountered this bug in MySQL Connector/ODBC.
EDIT: The bug has now been fixed.
The following (Python 3) test code verifies that MySQL Connector/ODBC returns zero (incorrect), while mysqlclient returns the correct value:
import MySQLdb # from mysqlclient
import pyodbc
host = 'localhost'
user = 'root'
passwd = 'whatever'
db = 'mydb'
port = 3307
charset = 'utf8mb4'
use_odbc = False # or True
print(f'{"" if use_odbc else "not "}using ODBC ...')
if use_odbc:
connection_string = (
f'DRIVER=MySQL ODBC 8.0 ANSI Driver;'
f'SERVER={host};UID={user};PWD={passwd};DATABASE={db};PORT={port};'
f'charset={charset};'
)
cnxn = pyodbc.connect(connection_string)
print(f'{cnxn.getinfo(pyodbc.SQL_DRIVER_NAME)}, version {cnxn.getinfo(pyodbc.SQL_DRIVER_VER)}')
else:
cnxn = MySQLdb.connect(
host=host, user=user, passwd=passwd, db=db, port=port, charset=charset
)
int_value = 123
crsr = cnxn.cursor()
crsr.execute("CREATE TEMPORARY TABLE foo (id varchar(10) PRIMARY KEY, intcol int, othercol longblob)")
crsr.execute(f"INSERT INTO foo (id, intcol) VALUES ('Alfa', {int_value})")
sql = f"SELECT intcol, othercol FROM foo WHERE id = {'?' if use_odbc else '%s'}"
crsr.execute(sql, ('Alfa',))
result = crsr.fetchone()[0]
print(f'{"pass" if result == int_value else "FAIL"} -- expected: {repr(int_value)} ; actual: {repr(result)}')
Console output with use_odbc = True:
using ODBC ...
myodbc8a.dll, version 08.00.0018
FAIL -- expected: 123 ; actual: 0
Console output with use_odbc = False:
not using ODBC ...
pass -- expected: 123 ; actual: 123
FWIW i just posted a question where i was seeing this in version 3.1.14 of the ODBC connector but NOT in version 3.1.10.
When using django shell: python manage.py shell
import datetime
from django.utils import timezone
DE = datetime.datetime(1970, 1, 1, tzinfo=timezone.utc)
cursor.execute("insert into monitor_regionserver (cluster_id, task_id, name, last_attempt_time, load, numberOfRegions, numberOfRequests, memStoreSizeMB, storefileSizeMB, readRequestsCount, writeRequestsCount, readRequestsCountPerSec, writeRequestsCountPerSec, replication_last_attempt_time, replicationMetrics) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [30L, 484L, '', DE, 0, 0, 0, 0, 0, 0, 0, 0, 0, DE, ''])
The cursor command looks like this (text-wrapped):
cursor.execute("insert into monitor_regionserver (cluster_id, task_id, name, last_attempt_time, load, numberOfRegions, numberOfRequests, memStoreSizeMB, storefileSizeMB, readRequestsCount, writeRequestsCount, readRequestsCountPerSec, writeRequestsCountPerSec, replication_last_attempt_time, replicationMetrics) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [30L, 484L, '', DE, 0, 0, 0, 0, 0, 0, 0, 0, 0, DE, ''])
I always get the error:
ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'load, numberOfRegions, numberOfRequests, memStoreSizeMB, storefileSizeMB, readRe' at line 1")
I'm confused with this problem, any idea to share?
load is a keyword in mysql see here
try to rename the column
Here is a query I have that runs fine in MySQL workbench with the included sample values and works fine if I manually plug in the values in the code, but fails when I use the values as parameters. Any ideas?
Python Code:
print player
cur.execute("""
INSERT INTO scoredata
(gameid, playerid, starter, pos, min, fgm, fga, tpm, tpa, ftm, fta, oreb, reb, ast, stl, blk, tos, pf, pts)
VALUES
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
"""), (player[0],
int(player[20]),
int(player[19]),
player[3],
int(player[4]),
int(player[5]),
int(player[6]),
int(player[7]),
int(player[8]),
int(player[9]),
int(player[10]),
int(player[11]),
int(player[12]),
int(player[13]),
int(player[14]),
int(player[15]),
int(player[16]),
int(player[17]),
int(player[18]) )
db.commit()
Error message:
['330060130', 103, 'Roy Devyn Marble', 'G-F', '28', '4', '9', '3', '6', '3', '3', '0', '2', '1', '0', '0', '0', '1', '14', 1, 1391]
Traceback (most recent call last):
File "C:\Users\jcaine\workspace\BasketballStats\src\BasketballStats\basketballstats.py", line 350, in <module>
insert_game_data('20130106', '20130106')
File "C:\Users\jcaine\workspace\BasketballStats\src\BasketballStats\basketballstats.py", line 284, in insert_game_data
"""), (player[0], int(player[20]), int(player[19]), player[3], int(player[4]), int(player[5]), int(player[6]), int(player[7]), int(player[8]), int(player[9]), int(player[10]), int(player[11]), int(player[12]), int(player[13]), int(player[14]), int(player[15]), int(player[16]), int(player[17]), int(player[18]) )
File "c:\users\jcaine\appdata\local\temp\easy_install-7_fysp\MySQL_python-1.2.3-py2.7-win32.egg.tmp\MySQLdb\cursors.py", line 174, in execute
File "c:\users\jcaine\appdata\local\temp\easy_install-7_fysp\MySQL_python-1.2.3-py2.7-win32.egg.tmp\MySQLdb\connections.py", line 36, in defaulterrorhandler
_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)' at line 4")
MySQL scoredata Table Columns:
gameid varchar
playerid int
starter int
pos varchar
min int
fgm int
fga int
tpm int
tpa int
ftm int
fta int
oreb int
reb int
ast int
stl int
blk int
tos int
pf int
pts int
MySQL Code that runs fine in Workbench:
INSERT INTO scoredata (gameid, playerid, starter, pos, min, fgm, fga, tpm,
tpa, ftm, fta, oreb, reb, ast, stl, blk, tos, pf, pts)
VALUES ('3300601300', 1391, 1, 'G-F', 28, 4, 9, 3, 6, 3, 3, 0, 2, 1, 0, 0, 0, 1, 14)
You're not passing data to the execute call. Note the closing brace in your example.
cur.execute("""
INSERT INTO scoredata
(gameid, playerid, starter, pos, min, fgm, fga, tpm, tpa, ftm, fta, oreb, reb, ast, stl, blk, tos, pf, pts)
VALUES
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
""")//*Remove me*
, (player[0],
int(player[20]),
int(player[19]),
player[3],
int(player[4]),
int(player[5]),
int(player[6]),
int(player[7]),
int(player[8]),
int(player[9]),
int(player[10]),
int(player[11]),
int(player[12]),
int(player[13]),
int(player[14]),
int(player[15]),
int(player[16]),
int(player[17]),
int(player[18]) )
db.commit()
I have executed this code to insert a dictionary into my table in database,
d = {'err': '0', 'tst': '0', 'Type o': 'FTP', 'recip': 'ADMIN', 'id': '101', 'origin': 'REPORT', 'Type recip': 'SMTP', 'date': '2010-01-10 18:47:52'}
db = MySQLdb.connect("localhost","admin","password","database")
cursor = db.cursor()
cursor.execute("""INSERT INTO mytable(ID, ERR, TST, DATE, ORIGIN, TYPE_O, RECIP, TYPE_RECIP) VALUES (%(id)s, %(err)s, %(tst)s, %(date)s, %(origin)s, %(Type o)s, %(recip)s, %(Type recip)s)""", d)
db.commit()
db.close()
Create statement of my table:
CREATE TABLE mytable (
`ID` tinyint unsigned NOT NULL,
`ERR` tinyint NOT NULL,
`TST` tinyint unsigned NOT NULL,
`DATE` datetime NOT NULL,
`ORIGIN` varchar(30) NOT NULL,
`TYPE_O` varchar(10) NOT NULL,
`RECIP` varchar(30) NOT NULL,
`TYPE_RECIP` varchar(10) NOT NULL,
PRIMARY KEY (`ID`,`DATE`)
) ENGINE = InnoDB;
But i have an error, it says:
1064, "you have an error in your SQL syntax; check the manual that
corresponds to you MySQL server version... )
Be aware of SQL injections and use the second argument to execute for inserting your query parameters:
cursor.execute("""
INSERT INTO
table
(name, age, origin, date)
VALUES
(%(name)s, %(age)s, %(origin)s, %(date)s)
""", d)