Develop Reference

Insert json file to mysql with flask

I'm trying to insert the data from a json file called output.json and i have this code:
I'm getting this error:
MySQLdb._exceptions.ProgrammingError: not all arguments converted during bytes formatting
In mysql i insert like insert into t1 values ({JSONFILE})
from flask import Flask
from flask_mysqldb import MySQL
app = Flask(__name__)
app.config['MYSQL_HOST'] = 'localhost'
app.config['MYSQL_USER'] = 'root'
app.config['MYSQL_PASSWORD'] = '*****'
app.config['MYSQL_DB'] = '*****'
mysql = MySQL(app)
#app.route("/sendf",methods=['POST'])
def sendfilet():
cursor = mysql.connection.cursor()
file = open('output.json', 'r')
file_content = file.read()
file.close()
sql = "INSERT INTO t1 (tablename) VALUES (%s)"
val = (json.dumps(file_content))
cursor.execute(sql, val)
db.commit()
db.close()
return 200
if __name__ == '__main__':
app.run(debug=True,port=5050)
my json is like this:
{"_id":{"$oid":"60f458945d77cb5ec7872b61"},"insertionDate":{"$date":"2021-07-18T16:36:36.193Z"},"sessionData":{"time":["1364","1374","1384"],"yaw":["0.15","0.3","0.45"],"pitch":["0.36","0.76","1.08"],"roll":["-0.13","-0.25","-0.35"],"ax":["-0.42","-0.41","-0.41"],"ay":["-0.15","-0.13","-0.1"],"az":["0.9","0.91","1"],"gx":["0","0","0"],"gy":["-0.01","0","-0.01"],"gz":["0.02","0.02","0.02"],"mx":["0.26","0.26","0.26"],"my":["0.01","0.01","0.01"],"mz":["-0.04","-0.04","-0.07"]},"metaData":{"userId":123456,"gender":"M","ageGroup":"SENIOR","weightKg":70,"heightCm":175,"poolSizeM":50}}

In order to store your JSON data into MySQL in python, you can to create a MySQLUtil so that you can insert your JSON data in MySQL.
You can do mysql = MySQLUtil()
After this, you need to make a connection to the database and then store the JSON data into a variable, and simply use MySQLstrong text.execSql() to insert the data.
(your json variable name) = pymysql.escape_string(// your json data)
Mysql = "insert data (index name) value"('" + json_data + "') "
mysql.execSql(sql)
you can read more here Python JSON encoder and decoder

For test Python code, first you try it:
from pathlib import Path
from . import mysql # because I use blueprint, so i import it from __init__.py, you can pass it.
#app.route("/sendf", methods=['GET'])
def sendfilet():
try:
print("File Path:", Path(__file__).absolute())
print("Directory Path:", Path().absolute()) # Directory of current working directory, not
#file_path = str(Path().absolute())+'/output.json'
parent_path = Path(__file__).parent.absolute()
file_path = f"{str(parent_path)}/output.json"
print(file_path, file_path)
file = open(file_path, 'r')
file_content = file.read()
file.close()
#print(file_content)
try:
cursor = mysql.connection.cursor()
cursor.execute("SELECT COUNT(*) FROM test_json")
property_count = cursor.fetchone()[0] # to get current ID,
sql = """INSERT INTO test_json (id, json_content)
VALUES (%s, %s)"""
#sql = "INSERT INTO test (json_content) VALUES (%s)"
row_id = property_count+1 # make new id
val = (row_id, file_content)
cursor.execute(sql, val)
mysql.connection.commit()
except Exception as e:
print(e)
except Exception as e:
print(e)
return "OK", 200
if you dont have the column ID with primary key in table, error will come.
My test table with name "test_json"
I think you should use SQLAlchemy, it is better than flask_mysqldb, easy to autoincrement id.
id = db.Column(db.BigInteger, autoincrement=True, primary_key=True)

How to load the JSON file too the PSQL database

I have a JSON file. Now I need to load the JSON data to my PSQL database.
So far I tried this one
import psycopg2
import json
with open('new.json') as f:
data = f.read()
dd = json.loads(data)
conn = psycopg2.connect(database="newdb", user = "postgres", password = "postgres",host = "127.0.0.1", port = "5432")
print "Opened database successfully"
cur = conn.cursor()
cur.execute(''' CREATE TABLE jsontable(SUM INT NOT NULL,
APP CHAR[30] NOT NULL,
ID INT NOT NULL,
DOMAINNAME TEXT NOT NULL,
DOMAINID INT NOT NULL);''')
print "Table Created successfully"
cur.execute('''INSERT INTO jsontable(data)
VALUES(%s)
''',
(data, str(dd['sum'],str(dd['app'],str(dd['id'],str(dd['Domain_name'],str(dd['Domain_Id'])))
print ("Data Entered successfully")
conn.commit()
conn.close()
Please provide some examples, how to pass the JSON file data to the database

Personally I like asyncpg as it's fully async especially if you're using Python 3.x and essentially all you need to do is put await in front of the sync commands.
import asyncpg
import json
with open('new.json') as f:
data = f.read()
dd = json.loads(data)
conn = await asyncpg.connect(database="newdb", user = "postgres", password = "postgres",host = "127.0.0.1", port = "5432")
print "Opened database successfully"
await con.execute(''' CREATE TABLE jsontable(SUM INT NOT NULL,
APP CHAR[30] NOT NULL,
ID INT NOT NULL,
DOMAINNAME TEXT NOT NULL,
DOMAINID INT NOT NULL);''')
print "Table Created successfully"
await con.execute('''INSERT INTO jsontable(SUM, APP, ID, DOMAINNAME, DOMAINID)
VALUES($1, $2, $3, $4, $5)
''',(str(dd['sum'],str(dd['app'],str(dd['id'],str(dd['Domain_name'],str(dd['Domain_Id'])))
print ("Data Entered successfully")
await conn.commit()
await conn.close()

Get list of MySQL databases with python

I'm trying to get the names of all databases associated with my MySQL server via python (2.7), but am instead just getting the number of databases.
I looked around, and it seems like the only answer may be to use sys for a command line call, get the name of the databases, and proceed from there, but I can't believe that's the only way.
Current Code:
import MySQLdb
serv = MySQLdb.connect(host = "localhost", user = "root", passwd = "abcdefg")
c = serv.cursor()
print c.execute("SHOW DATABASES")
Output:
4
Thanks in advance for the help.

I am not sure if mysql connector is same as your library, but using msyql connector your answer would be something like this:
import mysql.connector
conn = mysql.connector.connect (user='user', password='password',
host='server_address',buffered=True)
cursor = conn.cursor()
databases = ("show databases")
cursor.execute(databases)
for (databases) in cursor:
print databases[0]

Just taking previous comments and reorganicing:
Current Code:
import MySQLdb
serv = MySQLdb.connect(host = "localhost", user = "root", passwd = "abcdefg")
c = serv.cursor()
print c.execute("SHOW DATABASES")
With suggested modifications:
import MySQLdb
serv = MySQLdb.connect(host = "localhost", user = "root", passwd = "abcdefg")
c = serv.cursor()
c.execute("SHOW DATABASES")
l = c.fetchall()
print l
l = [ i[0] for i in l ]
print l
In my case the output looks like (which is the desired output):
(('information_schema',), ('data',), ('mysql',), ('performance_schema',), ('sys',))
['information_schema', 'data', 'mysql', 'performance_schema', 'sys']
Please note the following common error to prevent mistakes: if you try c.execute("SHOW DATABASES").fetchall() you will have an error: AttributeError: 'int' object has no attribute 'fetchall'. This is way you should write firstly c.execute("SHOW DATABASES") and secondly c.fetchall().

Getting 2013 lost connection error when trying to connect to mysql with Python

I am trying to connect to mysql database to retrieve some id for some users and use those id to retrieve another set of data from another table. It should be easy but I am getting mysql errors. Here's a snippet of what I am trying to do.
import MySQLdb
from langdetect import detect
my_db = MySQLdb.connect(
host="localhost",
port = 3306,
user="user",
passwd="password",
db="mydb",
charset = 'utf8'
)
sql1 = """SELECT id, comment FROM users WHERE usertype = 5 LIMIT 100"""
users = []
db_cursor = my_db.cursor()
db_cursor.execute(sql1)
users = db_cursor.fetchall()
sql2 = """SELECT first_name, last_name, email FROM user_contact WHERE id = %s"""
user_contact =[]
for user in users:
comment = user[1]
if detect(comment) == 'en':
id = user[0]
db_cursor = my_db.cursor()
db_cursor.execute(sql2, (id))
temp = db_cursor.fetchall()
user_contact . append (temp)
print (user_contact)
This is the error message I get when I try to run this query.
_mysql_exceptions.OperationalError: (2013, 'Lost connection to MySQL server during query')
The first part of the query will normally go through but it usually fails when it tries to connect to mysql again for the second part. I tested with just 100 records just to check if it's an issue with the query running too long but it's still the same even with 10 records.

For your second part, you might not execute sql;)
Try to change
for user in users:
comment = user[1]
if detect(comment) == 'en':
id = user[0]
db_cursor = my_db.cursor()
temp = db_cursor.fetchall()
user_contact . append (temp)
to
for user in users:
comment = user[1]
if detect(comment) == 'en':
id = user[0]
db_cursor = my_db.cursor()
db_cursor.execute(sql1, (id))
temp = db_cursor.fetchall()
user_contact . append (temp)

(see the screenshot..edited now)could not insert data into MySQLdb using python-flask

I created a db table from terminal, and now i want to insert data to it using following code,sql_insert_reg statement which is used as sql insert command is same as that i use in terminal insert operations but using in python file does not insert data .I am learning use of mysql in flask,here's my code.This code does not give error but does nothing as i expect it to!
mysql = MySQL()
app.config['MYSQL_DATABASE_USER'] = 'root'
app.config['MYSQL_DATABASE_PASSWORD'] = 'root'
app.config['MYSQL_DATABASE_DB'] = 'EmpData'
app.config['MYSQL_DATABASE_HOST'] = 'localhost'
mysql.init_app(app)
class RegistrationForm(Form):
username = TextField('Username', [validators.Length(min=4, max=25)])
password = PasswordField('New Password', [
validators.Required(),
validators.EqualTo('confirm', message='Passwords must match')])
confirm = PasswordField('Repeat Password')
accept_tos = BooleanField('I accept the TOS', [validators.Required()])
#app.route('/register',methods=['GET','POST'])
def register():
form = RegistrationForm(request.form)
flash('login details')
session['tmp'] = 43
if request.method == 'POST' and form.validate():
username = form.username.data
password = form.password.data
sql_insert_reg = "INSERT INTO User(userName,password) VALUES(%s,%s)"
#flash(sql_insert_reg)
conn = mysql.connect()
cursor = mysql.get_db().cursor()
cursor.execute(sql_insert_reg,(username,password))
conn.commit()
return render_template('register.html',form=form)
this is the screenshot i uploaded below..please see the entries useId 2 then goes directly to 6 ..and i got to see this by altering the answer as suggested to me!!can anyone lookout the problem behind the scene!
Please help me!

This is what a typical INSERT statement looks like:
INSERT INTO table (column1,column2,column3) VALUES (value1,value2,value3);
Note that if your first column is auto-incremental (e.g. some sort of index), you can ommit that from the statement and just write it as follows:
INSERT INTO User (user_column, pass_column) VALUES ('foo', 'bar');

So... don't do this:
sql = "INSERT INTO Table VALUES(NULL,'%s','%s')"%(username,password)+";"
Do this instead:
sql = "INSERT INTO Table (col1, col2) VALUES (%s, %s)"
cursor.execute(sql, (value1, value2))
Why? Because that will sanitize your input and you don't end up registering Bobby Drop Table as a user.
If doing it that way doesn't do what you expect, please provide more detail on what is happening, what you're expecting and how you know that you don't have what you expect to see.

this could be your problem
ursor.execute(sql_insert_reg,(username,password))
looks like it should be
cursor.execute(sql_insert_reg,(username,password))
and if thats not it, i would just use sqlalchemy to generate the sql for you
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker,scoped_session
import sqlalchemy as sa
Model = declarative_base()
engine = sa.create_engine('mysql://DB_USERNAME:DB_PASSWORD#DB_HOST:DB_PORT/DB_NAME')
Session = scoped_session(sessionmaker(bind=engine))
db_session = Session()
class User(Model):
__tablename__ = 'users'
id = sq.Column(sq.Integer,primary_key=True)
username = sq.Column(sq.String(255),nullable=False,unique=True)
password = sq.Column(sq.Text,nullable=False)
then if you want to manipulate the data you can change this from above
username = form.username.data
password = form.password.data
sql_insert_reg = "INSERT INTO User(userName,password) VALUES(%s,%s)"
#flash(sql_insert_reg)
conn = mysql.connect()
cursor = mysql.get_db().cursor()
ursor.execute(sql_insert_reg,(username,password))
conn.commit()
to this
user = User(username=form.username.data,password=form.password.data)
db_session.add(user)
db_session.commit()

I changed some code and found that instead of using the above lines of code in mysql i got this magic!
import MySQLdb
# Open database connection
db = MySQLdb.connect("localhost","testuser","test123","TESTDB" )
# prepare a cursor object using cursor() method
cursor = db.cursor()
sql = """INSERT INTO EMPLOYEE(FIRST_NAME,
LAST_NAME, AGE, SEX, INCOME)
VALUES ('Mac', 'Mohan', 20, 'M', 2000)"""
try:
# Execute the SQL command
cursor.execute(sql)
# Commit your changes in the database
db.commit()
except:
# Rollback in case there is any error
db.rollback()
# disconnect from server
db.close()