Develop Reference

Python: Post json to MySql Database

I'm a bit at a loss for how to just push my json data to MySql. I have the cursor. Do I need to create a table first? Is there a method to simply push my json data straight in?
with open("trades.json") as f:
trades_data = json.load(f)
trades_data = reconfigure_data(trades_data)
db = mysql.connector.connect(
host="localhost",
user=os.environ.get('MYSQL_DB_USER'),
password=os.environ.get('MYSQL_DB_USER_PASS',
database='trades')
)
cursor = db.cursor()

I found the syntax that works:
cursor = db.cursor()
# create table on database
sql = "CREATE TABLE IF NOT EXISTS `test` (`id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, `objectid` VARCHAR(100), `type` VARCHAR(5), `ownerid` VARCHAR(100), `color` VARCHAR(15), `shape` VARCHAR(15), `size` VARCHAR(15), PRIMARY KEY (`id`) );"
print(sql)
cursor.execute(sql)
with open("trades.json") as f:
trades_data = json.load(f)
for trade in trades_data:
for right in trade['rights']:
sql = f"INSERT INTO test (objectid, type, ownerid, color, shape, size) VALUES ('{trade['objectid']}', '{trade['type']}', '{trade['ownerid']}', '{right['color']}', '{right['shape']}', '{right['size']}' )"
cursor.execute(sql)
db.commit()

A script doesn't execute the command in cur.execute()

I'm new to PostgreSQL and psycopg2, and I face a problem.
import psycopg2
def create_tables(whichone):
in_str_station = "CREATE TABLE station (id SMALLINT NOT NULL PRIMARY KEY, name VARCHAR(40) NOT NULL," \
" country VARCHAR(3) NOT NULL, latitude VARCHAR(10), " \
"longitude VARCHAR(10),height SMALLINT);"
in_str_dailyData = "CREATE TABLE daily_data (id BIGSERIAL NOT NULL PRIMARY KEY," \
"s_id SMALLINT NOT NULL REFERENCES station(id)," \
"d_date DATE NOT NULL, d_mean NUMERIC(6, 1), quality SMALLINT);"
int_str_monthlyMean = "CREATE TABLE monthly_mean (id BIGSERIAL NOT NULL PRIMARY KEY,"\
"s_id SMALLINT NOT NULL REFERENCES station(id),"\
"m_date DATE NOT NULL, m_mean NUMERIC(9, 3),"\
"var NUMERIC(9, 3), std NUMERIC(9, 3));"
in_str_yearlymean = "CREATE TABLE yearly_mean (id BIGSERIAL NOT NULL PRIMARY KEY, " \
"s_id SMALLINT NOT NULL REFERENCES station(id)," \
"y_date DATE NOT NULL, y_mean NUMERIC(9, 3),var NUMERIC(9, 3)," \
"std NUMERIC(9, 3), var_m NUMERIC(9, 3), std_m NUMERIC(9, 3));"
database_list = {'station': in_str_station, 'monthly_mean': int_str_monthlyMean,
'daily_data': in_str_dailyData, 'yearly_mean': in_str_yearlymean}
try:
conn = psycopg2.connect(
host="localhost",
database="climate",
user="postgres",
password="1")
cur = conn.cursor()
in_str = database_list.get(whichone)
cur.execute(in_str)
output_ = cur.fetchall()
print(output_)
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
After I run the script, no matter which one of the in_str_ I choose, the table is not created. I have checked and when I copy the content of in_str that I executed in cur.execute and use it in the PostgreSQL shell, everything works.
Where did I make the mistake?

Call conn.commit() after cur.execute(), but before conn.close(). Transactions are not implicitly committed with psycopg2:
If the connection is closed (using the close() method) or destroyed (using del or by letting it fall out of scope) while a transaction is in progress, the server will discard the transaction.

I don't know if you just did that here but it's indented wrong. You need to indent code after the function.
def foo(a):
pass

Error Inserting records into mariadb table using Python : 1366 (22007): Incorrect integer value: '%s' for column

I am inserting records in maria db table from a file using python. Population column in the file is empty. I want it to go as empty value in table as well. Population column in table is set as integer and can accept null value. I am trying the below code -
Table Definition -
CREATE TABLE local_db.table_x (
Unique_code varchar(50) NOT NULL,
city varchar(200) DEFAULT NULL,
state varchar(50) DEFAULT NULL,
population bigint(20) DEFAULT NULL,
Govt varchar(50) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
input_file = "input_file"
csv_data = csv.reader(open(input_file))
try:
connection = mysql.connector.connect(host='localhost',
database='local_db',
user='root',
password='root',
port = '3306')
cursor = connection.cursor()
for row in csv_data:
cursor.execute("""
INSERT INTO table_x(Unique_code,city,state,population,Govt)
VALUES("%s", "%s", "%s","%s", "%s")
ON DUPLICATE KEY UPDATE city = VALUES(city),state = VALUES(state), \
population = VALUES(population),Govt = VALUES(Govt)""")
connection.commit()
print(cursor.rowcount, "Record inserted successfully into table_x")
cursor.close()
except mysql.connector.Error as error:
print("Failed to insert record into table_x table {}".format(error))
finally:
if (connection.is_connected()):
connection.close()
print("MySQL connection is closed")
But I am getting below error -
Failed to insert record into table_x table 1366 (22007): Incorrect integer value: '%s' for column local_db.table_x.population at row 1
MySQL connection is closed
In other thread it was suggested to change
SET sql_mode = ""
But its not an option for me since I would be running it on organization server which I can not change for this only. Please suggest what code changes I can do here to handle this situation.

The population column is a bigint(20) DEFAULT NULL & hence you cannot provide the value "%s" format it should be a integer which is being passed.
Also on values front.
It can have NULL
Any integer value.
So incase you want it to be null & not empty string("") you can skip to insert value to population column all together by removing it from the insert command
INSERT INTO table_x(Unique_code,city,state,Govt)
VALUES("%s", "%s", "%s","%s")
ON DUPLICATE KEY UPDATE city = VALUES(city),state = VALUES(state), Govt = VALUES(Govt)""")

Assuming that each row of your CSV has 5 values, corresponding to the code, city, state, population, and government, in that order, you should be using this syntax for the insert query:
for row in csv_data:
params = row.split(",")
sql = """
INSERT INTO table_x (Unique_code, city, state, population, Govt)
VALUES(?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE city = VALUES(city), state = VALUES(state),
population = VALUES(population),Govt = VALUES(Govt)"""
cursor.execute(sql, params)
The first parameter to execute() should be the SQL prepared statement, and the second parameter should be a tuple. In this case, it should be a tuple with 5 values, corresponding to the five ? placeholders in the VALUES clause of your query. An example of a valid tuple here might be:
params = ('ABC123', 'New York', 'New York', 15000000, 'USA',)
Then call via:
cursor.execute(sql, params)

How to get columns name in mysqldb with a Python 2.7?

If I am using select * from query it is working well, but when I am trying to query the columns name too, it isnt working (maybe because I have got a column called "FROM" but that's why i used 'FROM!?)
Here my code:
connection = MySQLdb.connect(host='localhost',
user='admin',
passwd='',
db='database1',
use_unicode=True,
charset="utf8")
cursor = connection.cursor()
query = """ select ACTUAL_TIME, 'FROM, ID
union all
select ACTUAL_TIME, FROM , ID
from TEST
into outfile '/tmp/test.csv'
fields terminated by ';'
enclosed by '"'
lines terminated by '\n';
"""
cursor.execute(query)
connection.commit()
cursor.close()
I get this error message:
raise errorvalue
_mysql_exceptions.OperationalError: (1054, "Unknown column 'ACTUAL_TIME' in 'field list'")
EDIT: SHOW CREATE TABLE TEST;
| TEST | CREATE TABLE `TEST` (
`ACTUAL_TIME` varchar(100) DEFAULT NULL,
`FROM` varchar(100) DEFAULT NULL,
`STATUS` varchar(100) DEFAULT NULL,
`ID` int(10) NOT NULL AUTO_INCREMENT,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=76287 DEFAULT CHARSET=utf8 |

try this :
connection = MySQLdb.connect(host='localhost',
user='admin',
passwd='',
db='database1',
use_unicode=True,
charset="utf8")
cursor = connection.cursor()
query = """ select 'ACTUAL_TIME', 'FROM', 'ID' -- add single quotes
union all
select `ACTUAL_TIME`, `FROM`, `ID` -- add here backtick in column names
from TEST
into outfile '/tmp/test.csv'
fields terminated by ';'
enclosed by '"'
lines terminated by '\n';
"""
cursor.execute(query)
connection.commit()
cursor.close()
or else you can use this to get column names "SHOW columns"
or :
cursor.execute("SELECT * FROM table_name LIMIT 0")
print cursor.description

columns = cursor.description
result = [{columns[index][0]:column for index, column in enumerate(value)} for value in cursor.fetchall()]
print(result)

Mysql Python: Not all parameters were used in MySQL statement

So I am trying to input data into my table as variables but I seem to keep on getting the error mysql.connector.errors.ProgrammingError: Not all parameters were used in the SQL statement, I have used the %s instead of the row names but I seem to keep on getting the same error. I suspect it is my syntax but I cannot seem to figure it out btw this is my first time using python and MySQL together
import mysql.connector
Sensor_ID = "test"
Location = "room"
Sensor_IP = "192.168.1.1"
Sensor_1 = "10"
Sensor_1_Unit = "*C"
Sensor_2 =""
Sensor_2_Unit = ""
Sensor_3 = ""
Sensor_3_Unit = ""
conn = mysql.connector.connect(user='******', password='********', host='******', database='*****') #blanked my user n pass
mycursor = conn.cursor()
mycursor.execute('SHOW TABLES')
print(mycursor.fetchall())
print ""
mycursor.execute("SHOW VARIABLES LIKE '%version%'")
print "Version:",(mycursor.fetchall())
#works up to here
mycursor.execute("INSERT INTO iot_sensors VALUES (ID, Sensor_ID, Location, Sensor_IP, Sensor_1, Sensor_1_Unit, Sensor_2,Sensor_2_Unit, Sensor_3, Sensor_3_Unit)",(Sensor_ID,Location,Sensor_IP,Sensor_1,Sensor_1_Unit,Sensor_2,Sensor_2_Unit,Sens or_3,Sensor_3_Unit))
conn.commit()
# Sensor_ID,Location,Sensor_IP,Sensor_1,Sensor_1_Unit,Sensor_2,Sensor_2_Unit,Sensor_3,Sensor_3_Unit
CREATE TABLE IoT_Sensors(
ID INT NOT NULL AUTO_INCREMENT,
Sensor_ID VARCHAR (15) NOT NULL,
Location VARCHAR (20) NOT NULL,
Sensor_IP VARCHAR (15) NOT NULL,
Sensor_1 VARCHAR (15) NOT NULL,
Sensor_1_Unit VARCHAR (15) NOT NULL,
Sensor_2 VARCHAR (15),
Sensor_2_Unit VARCHAR (15),
Sensor_3 VARCHAR (15),
Sensor_3_Unit VARCHAR (15),
Time_Stamp TIMESTAMP NOT NULL,
PRIMARY KEY (ID));

You need to put %s into your SQL statement.
mycursor.execute("INSERT INTO iot_sensors VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",(Sensor_ID,Location,Sensor_IP,Sensor_1,Sensor_1_Unit,Sensor_2,Sensor_2_Unit,Sens or_3,Sensor_3_Unit))
See the example in the docs.

Looks like you are missing formatting your actual variables from the insert statement. Try formatting them using one of the known methods, %s or .format method. You are also not using the timestamp, last column, value in your table when inserting. If you just reference the table you will have a column mismatch. You will have to be explicit about what columns you are populating. You could use CURRENT_TIMESTAMP for that since it says NOT NULL for column description.
query = ("""Insert into iot_sensors (Sensor_ID, Location, Sensor_IP, Sensor_1, Sensor_1_Unit, Sensor_2, Sensor_2_Unit, Sensor_3, Sensor_3_Unit) values
('{0}','{1}','{2}','{3}','{4}','{5}','{6}','{7}','{8}');""".format(Sensor_ID,Location,Sensor_IP,Sensor_1,Sensor_1_Unit,Sensor_2,Sensor_2_Unit,Sensor_3,Sensor_3_Unit))
mycursor.execute(query)
or
query = ("""Insert into iot_sensors (Sensor_ID, Location, Sensor_IP, Sensor_1, Sensor_1_Unit, Sensor_2, Sensor_2_Unit, Sensor_3, Sensor_3_Unit) values
(%s,%s,%s,%s,%s,%s,%s,%s,%s);""" % (Sensor_ID,Location,Sensor_IP,Sensor_1,Sensor_1_Unit,Sensor_2,Sensor_2_Unit,Sensor_3,Sensor_3_Unit))
mycursor.execute(query)