Develop Reference

Python: Post json to MySql Database

I'm a bit at a loss for how to just push my json data to MySql. I have the cursor. Do I need to create a table first? Is there a method to simply push my json data straight in?
with open("trades.json") as f:
trades_data = json.load(f)
trades_data = reconfigure_data(trades_data)
db = mysql.connector.connect(
host="localhost",
user=os.environ.get('MYSQL_DB_USER'),
password=os.environ.get('MYSQL_DB_USER_PASS',
database='trades')
)
cursor = db.cursor()

I found the syntax that works:
cursor = db.cursor()
# create table on database
sql = "CREATE TABLE IF NOT EXISTS `test` (`id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, `objectid` VARCHAR(100), `type` VARCHAR(5), `ownerid` VARCHAR(100), `color` VARCHAR(15), `shape` VARCHAR(15), `size` VARCHAR(15), PRIMARY KEY (`id`) );"
print(sql)
cursor.execute(sql)
with open("trades.json") as f:
trades_data = json.load(f)
for trade in trades_data:
for right in trade['rights']:
sql = f"INSERT INTO test (objectid, type, ownerid, color, shape, size) VALUES ('{trade['objectid']}', '{trade['type']}', '{trade['ownerid']}', '{right['color']}', '{right['shape']}', '{right['size']}' )"
cursor.execute(sql)
db.commit()

How to get columns name in mysqldb with a Python 2.7?

If I am using select * from query it is working well, but when I am trying to query the columns name too, it isnt working (maybe because I have got a column called "FROM" but that's why i used 'FROM!?)
Here my code:
connection = MySQLdb.connect(host='localhost',
user='admin',
passwd='',
db='database1',
use_unicode=True,
charset="utf8")
cursor = connection.cursor()
query = """ select ACTUAL_TIME, 'FROM, ID
union all
select ACTUAL_TIME, FROM , ID
from TEST
into outfile '/tmp/test.csv'
fields terminated by ';'
enclosed by '"'
lines terminated by '\n';
"""
cursor.execute(query)
connection.commit()
cursor.close()
I get this error message:
raise errorvalue
_mysql_exceptions.OperationalError: (1054, "Unknown column 'ACTUAL_TIME' in 'field list'")
EDIT: SHOW CREATE TABLE TEST;
| TEST | CREATE TABLE `TEST` (
`ACTUAL_TIME` varchar(100) DEFAULT NULL,
`FROM` varchar(100) DEFAULT NULL,
`STATUS` varchar(100) DEFAULT NULL,
`ID` int(10) NOT NULL AUTO_INCREMENT,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=76287 DEFAULT CHARSET=utf8 |

try this :
connection = MySQLdb.connect(host='localhost',
user='admin',
passwd='',
db='database1',
use_unicode=True,
charset="utf8")
cursor = connection.cursor()
query = """ select 'ACTUAL_TIME', 'FROM', 'ID' -- add single quotes
union all
select `ACTUAL_TIME`, `FROM`, `ID` -- add here backtick in column names
from TEST
into outfile '/tmp/test.csv'
fields terminated by ';'
enclosed by '"'
lines terminated by '\n';
"""
cursor.execute(query)
connection.commit()
cursor.close()
or else you can use this to get column names "SHOW columns"
or :
cursor.execute("SELECT * FROM table_name LIMIT 0")
print cursor.description

columns = cursor.description
result = [{columns[index][0]:column for index, column in enumerate(value)} for value in cursor.fetchall()]
print(result)

No data inserted after successful SQLite statement within Python

Extension from previous question
Attempting to insert SQL values into database after pulling from XML file, but none seem to be appearing in database after insert statement embedded in Python code. Without the SQL section included, the entries are printed as expected. I am not getting an error in my Python environment (Anaconda Navigator), so totally lost on how the queries were processed, but nothing was entered! I tried a basic select statement to display the table, but get an empty table back.
Select Query
%sql SELECT * FROM publication;
Main Python code
import sqlite3
con = sqlite3.connect("publications.db")
cur = con.cursor()
from xml.dom import minidom
xmldoc = minidom.parse("test.xml")
#loop through <pub> tags to find number of pubs to grab
root = xmldoc.getElementsByTagName("root")[0]
pubs = [a.firstChild.data for a in root.getElementsByTagName("pub")]
num_pubs = len(pubs)
count = 0
while(count < num_pubs):
#get data from each <pub> tag
temp_pub = root.getElementsByTagName("pub")[count]
temp_ID = temp_pub.getElementsByTagName("ID")[0].firstChild.data
temp_title = temp_pub.getElementsByTagName("title")[0].firstChild.data
temp_year = temp_pub.getElementsByTagName("year")[0].firstChild.data
temp_booktitle = temp_pub.getElementsByTagName("booktitle")[0].firstChild.data
temp_pages = temp_pub.getElementsByTagName("pages")[0].firstChild.data
temp_authors = temp_pub.getElementsByTagName("authors")[0]
temp_author_array = [a.firstChild.data for a in temp_authors.getElementsByTagName("author")]
num_authors = len(temp_author_array)
count = count + 1
#process results into sqlite
pub_params = (temp_ID, temp_title)
cur.execute("INSERT INTO publication (id, ptitle) VALUES (?, ?)", pub_params)
journal_params = (temp_booktitle, temp_pages, temp_year)
cur.execute("INSERT INTO journal (jtitle, pages, year) VALUES (?, ?, ?)", journal_params)
x = 0
while(x < num_authors):
cur.execute("INSERT OR IGNORE INTO authors (name) VALUES (?)", (temp_author_array[x],))
x = x + 1
#display results
print("\nEntry processed: ", count)
print("------------------\nPublication ID: ", temp_ID)
print("Publication Title: ", temp_title)
print("Year: ", temp_year)
print("Journal title: ", temp_booktitle)
print("Pages: ", temp_pages)
i = 0
print("Authors: ")
while(i < num_authors):
print("-",temp_author_array[i])
i = i + 1
print("\nNumber of entries processed: ", count)
SQL queries
%%sql
DROP TABLE IF EXISTS publication;
CREATE TABLE publication(
id INT PRIMARY KEY NOT NULL,
ptitle VARCHAR NOT NULL
);
/* Author Entity set and writes_for relationship */
DROP TABLE IF EXISTS authors;
CREATE TABLE authors(
name VARCHAR(200) PRIMARY KEY NOT NULL,
pub_id INT,
pub_title VARCHAR(200),
FOREIGN KEY(pub_id, pub_title) REFERENCES publication(id, ptitle)
);
/* Journal Entity set and apart_of relationship */
DROP TABLE IF EXISTS journal;
CREATE TABLE journal(
jtitle VARCHAR(200) PRIMARY KEY NOT NULL,
pages INT,
year INT(4),
pub_id INT,
pub_title VARCHAR(200),
FOREIGN KEY(pub_id, pub_title) REFERENCES publication(id, ptitle)
);
/* Wrote relationship b/w journal & authors */
DROP TABLE IF EXISTS wrote;
CREATE TABLE wrote(
name VARCHAR(100) NOT NULL,
jtitle VARCHAR(50) NOT NULL,
PRIMARY KEY(name, jtitle),
FOREIGN KEY(name) REFERENCES authors(name),
FOREIGN KEY(jtitle) REFERENCES journal(jtitle)
);

You need to call con.commit() in order to commit the data to the database. If you use the connection as a context manager (with con:), the connection will commit any changes you make (or roll them back if there is an error).
Explicitly closing the connection is also a good practice.

It looks like you are forgetting to commit and close the connection. You need to call these two functions in order to properly close the connection and to save the work you have done to the database.
conn.commit()
conn.close()

Python request from MySQL JSON column returns a string instead of a float

My MySQL table:
CREATE TABLE ref_data (
id BIGINT AUTO_INCREMENT NOT NULL,
symbol VARCHAR(64) NOT NULL,
metadata JSON NOT NULL,
PRIMARY KEY (id)
);
INSERT INTO ref_data (symbol, metadata)
VALUES ('XYZ', '{"currency": "USD", "tick_size": 0.01}');
My Python script:
import mysql.connector
con = mysql.connector.connect(**config)
cur = con.cursor()
cur.execute(
"""
SELECT JSON_EXTRACT(metadata, '$.tick_size')
FROM ref_data
WHERE symbol = 'CL';
""")
And the result is a unicode:
cur.fetchall()[0][0]
>> u'0.01'
When I run the same query within MySQL Workbench I get a double. I know I could convert the string to a float, but the point of using a JSON was flexibility and not having to specify what each column is, etc.
Thanks!

Python: How to fill/append data from python loop to SQL table?

I created a python script in Pycharm that works like a charm... But these days I'm sensing that I could have a problem with size of monthly .csv file and also I would like to analyze data using SQL over Python so I could automatize process and make charts, pies from that queries.
So instead of exporting to csv I would like to append to SQL table.
Here is a part of code that exports data to .csv:
for content in driver.find_elements_by_class_name('companychatroom'):
people = content.find_element_by_xpath('.//div[#class="theid"]').text
if people != "Monthy" and people != "Python" :
pass
mybook = open(r'C:\Users\Director\PycharmProjects\MyProjects\Employees\archive' + datetime.now().strftime("%m_%y") + '.csv', 'a')
mybook.write('%20s,%20s\n'%(people, datetime.now().strftime("%d/%m/%y %H:%M")))
mybook.close()
================== EDIT: ==============
I tried over sqlite3 and these is what I could manage to write by now and it works... But how to append data into SQL table it always overwrite previous with INSERT TO and it shouldn't?
import sqlite3
from datetime import datetime
sqlite_file = (r"C:\Users\Director\PycharmProjects\MyProjects\Employees\MyDatabase.db")
conn = sqlite3.connect(sqlite_file)
cursor = conn.cursor()
table_name = 'Archive' + datetime.now().strftime("%m_%y")
sql = 'CREATE TABLE IF NOT EXISTS ' + table_name + '("first_name" varchar NOT NULL, "Currdat"date NOT NULL)'
cursor.execute(sql)
sql = 'INSERT INTO ' + table_name + '(first_name,Currdat) VALUES ("value1",CURRENT_TIMESTAMP);'
cursor.execute(sql)
sql = 'SELECT * FROM Archive06_16 '
for row in cursor.execute(sql):
print(row)
cursor.close()
Thanks in advance,