import mysql.connector
import pymongo
import json
from pymongo import MongoClient
mysql_host="**"
mysql_database="**"
mysql_schema = "**"
mysql_user="**"
mysql_password="**"
mongodb_host = "**"
mongodb_dbname = "**"
mysqldb = mysql.connector.connect(
host="**",
database="**",
user="**",
password="***"
)
if (mysqldb.is_connected()):
print("successfully Connected")
mycursor = mysqldb.cursor(dictionary=True)
mycursor.execute("SELECT Name from MYSQLTABLE;")
Ndata = mycursor.fetchall()
print(Ndata)
mycursor.execute("SELECT location from MYSQLTABLE;")
Ldata = mycursor.fetchall()
print(Ldata)
myclient = pymongo.MongoClient(mongodb_host)
mydb = myclient[mongodb_dbname]
mycol = mydb["CASES"]
if len(Ndata) > 0:
if len(Ldata) > 0:
if len(iddata) > 0:
x = mycol.insert_many([{ "Name" : Ndata, "location" : Ldata}])
print(len(x.inserted_ids))
I have Written the above code, the data migrated through this process is an array,
i want to migrate data as a string
you can also suggest any other easy way to migrate data from mysql to mongodb using python through proper mapping (selected columns of mysql to selected fields of mongodb)
Related
I'm trying to perform bulk extracts/loads from Oracle to MySQL using cx_Oracle and SQL Alchemy.
I found this example online and it works well for most data types, but fails from Blob data types:
https://vbaoverall.com/transfer-data-from-oracle-to-mysql-using-sqlalchemy-python/
I have about 43 tables and about 12 of them have BLOB data types.
import cx_Oracle
import pandas as pd
from sqlalchemy import create_engine
import pymysql
import warnings
warnings.filterwarnings('ignore')
# list out all 43 tables:
table_list = [
"FILE",
"ATTACHMENT",
"DOCUMENTS",
"USERS",
"INFO",
"ONE",
"TWO",
"THREE",
"FOUR",
"...."
]
# Set Oralce Connection
dsn_tns = cx_Oracle.makedsn('source.example.com', '1530', service_name='test')
oracle_connection = cx_Oracle.connect(user='root', password='toot', dsn=dsn_tns)
# Open Oracle cursor
cursor = oracle_connection.cursor()
# set mysql connection with foreign key checks
mysql_engine = create_engine("mysql+pymysql://root:toot#target.example.com:3306/target")
mysql_engine.execute("SET FOREIGN_KEY_CHECKS=0")
# loop thru tables:
for table in table_list:
# select from oracle
sql = "SELECT * FROM " + table
# read into pandas df
data=pd.read_sql(sql, oracle_connection)
# insert into mysql
mysql_engine.execute("TRUNCATE TABLE "+table)
data.to_sql(table, con=mysql_engine, if_exists='append', index=False, chunksize=10000)
print("{}: sucessfully inserted {} rows.".format(table, data.shape[0]))
# update foreign key checks
mysql_engine.execute("SET FOREIGN_KEY_CHECKS=1")
#close connection
oracle_connection.close()
mysql_engine.dispose()
Here's the error I'm getting:
return "'%s'" % escape_string(str(value), mapping)
TypeError: __str__ returned non-string (type bytes)
Thanks to #Gord Thompson, I found out I just needed to specify dtype=
import cx_Oracle
import pandas as pd
from sqlalchemy import create_engine
import sqlalchemy
import pymysql
import warnings
warnings.filterwarnings('ignore')
table_list = [
"FILE",
"ATTACHMENT",
"DOCUMENTS",
"USERS",
"INFO",
"ONE",
"TWO",
"THREE",
"FOUR",
"...."
]
# Set Oralce Connection
dsn_tns = cx_Oracle.makedsn('source.example.com', '1530', service_name='test')
oracle_connection = cx_Oracle.connect(user='root', password='toot', dsn=dsn_tns)
# Open Oracle cursor
cursor = oracle_connection.cursor()
# set mysql connection with foreign key checks
mysql_engine = create_engine("mysql+pymysql://root:toot#target.example.com:3306/target")
mysql_engine.execute("SET FOREIGN_KEY_CHECKS=0")
for table in table_list:
# select from oracle
sql = "SELECT * FROM " + table
# read into pandas df
data=pd.read_sql(sql, oracle_connection)
dtype = {}
if table == "ATTACHMENT":
dtype['FILE_CONTENT'] = sqlalchemy.types.PickleType
# insert into mysql
mysql_engine.execute("TRUNCATE TABLE "+table)
data.to_sql(table, con=mysql_engine, if_exists='append', index = False, chunksize =10000, dtype=dtype)
print("{}: sucessfully inserted {} rows.".format(table, data.shape[0]))
# update foreign key checks
mysql_engine.execute("SET FOREIGN_KEY_CHECKS=1")
#close connection
oracle_connection.close()
mysql_engine.dispose()
I am using python to interact with mysql and when I access columns from mysql i get an output like this:
[('some', 't5vd._kZ'), ('something', 'anything')]
i want it to be:
some, t5vd._kZ
something, anything
my code:
import mysql.connector
mydb = mysql.connector.connect(
host = 'localhost',
user = '####',
password = '######',
database = 'database'
)
cursor = mydb.cursor()
cursor.execute('SELECT * FROM passwords')
result = str(cursor.fetchall())
passwd = ''
print(result)
You can do
for tup in result:
print(*tup)
Ok, I have tried several kinds of solutions recommended by others on this site and other sited. However, I can't get it work as I would like it to do.
I get a XML-response which I normalize and then save to a CSV. This first part works fine.
Instead of saving it to CSV I would like to save it into an existing table in an access database. The second part below:
Would like to use an existing table instead of creating a new one
The result is not separated with ";" into different columns. Everything ends up in the same column not separated, see image below
response = requests.get(u,headers=h).json()
dp = pd.json_normalize(response,'Units')
response_list.append(dp)
export = pd.concat(response_list)
export.to_csv(r'C:\Users\username\Documents\Python Scripts\Test\Test2_'+str(now)+'.csv', index=False, sep=';',encoding='utf-8')
access_path = r"C:\Users\username\Documents\Python Scripts\Test\Test_db.accdb"
conn = pyodbc.connect("DRIVER={{Microsoft Access Driver (*.mdb, *.accdb)}};DBQ={};" \
.format(access_path))
strSQL = "SELECT * INTO projects2 FROM [text;HDR=Yes;FMT=sep(;);" + \
"Database=C:\\Users\\username\\Documents\\Python Scripts\\Test].Testdata.csv;"
cur = conn.cursor()
cur.execute(strSQL)
conn.commit()
conn.close()
If you already have the data in a well-formed pandas DataFrame then you don't really need to dump it to a CSV file; you can use the sqlalchemy-access dialect to push the data directly into an Access table using pandas' to_sql() method:
from pprint import pprint
import urllib
import pandas as pd
import sqlalchemy as sa
connection_string = (
r"DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};"
r"DBQ=C:\Users\Public\Database1.accdb;"
r"ExtendedAnsiSQL=1;"
)
connection_uri = f"access+pyodbc:///?odbc_connect={urllib.parse.quote_plus(connection_string)}"
engine = sa.create_engine(connection_uri)
with engine.begin() as conn:
# existing data in table
pprint(
conn.execute(sa.text("SELECT * FROM user_table")).fetchall(), width=30
)
"""
[('gord', 'gord#example.com'),
('jennifer', 'jennifer#example.com')]
"""
# DataFrame to insert
df = pd.DataFrame(
[
("newdev", "newdev#example.com"),
("newerdev", "newerdev#example.com"),
],
columns=["username", "email"],
)
df.to_sql("user_table", engine, index=False, if_exists="append")
with engine.begin() as conn:
# updated table
pprint(
conn.execute(sa.text("SELECT * FROM user_table")).fetchall(), width=30
)
"""
[('gord', 'gord#example.com'),
('jennifer', 'jennifer#example.com'),
('newdev', 'newdev#example.com'),
('newerdev', 'newerdev#example.com')]
"""
(Disclosure: I am currently the maintainer of the sqlalchemy-access dialect.)
Solved with the following code
SE_export_Tuple = list(zip(SE_export.Name,SE_export.URL,SE_export.ImageUrl,......,SE_export.ID))
print(SE_export_Tuple)
access_path = r"C:\Users\username\Documents\Python Scripts\Test\Test_db.accdb"
conn = pyodbc.connect("DRIVER={{Microsoft Access Driver (*.mdb, *.accdb)}};DBQ={};" \
.format(access_path))
cursor = conn.cursor()
mySql_insert_query="INSERT INTO Temp_table (UnitName,URL,ImageUrl,.......,ID) VALUES (?,?,?,......,?)"
cursor.executemany(mySql_insert_query,SE_export_Tuple)
conn.commit()
conn.close()
However, when I add many fields I get an error at "executemany", saying:
cursor.executemany(mySql_insert_query,SE_export_Tuple)
Error: ('HY004', '[HY004] [Microsoft][ODBC Microsoft Access Driver]Invalid SQL data type (67) (SQLBindParameter)')
I am trying to store a pickled nested dictionary in Postgresql (I am aware that this is a quick & dirty method and won't be able to access dictionary contents from Postgresql - usually bad practice)
# boilerplate, preamble and upstream work.
import psycopg2
''' Inputs: nd = dictionary to be pickled '''
pickled = pickle.dumps(nd)
connection = psycopg2.connect(user = "-----",
password = "----",
host = "----",
port = "----",
database = "----")
name = 'database1'
print('Connected...')
cursor = connection.cursor()
print(connection.get_dsn_parameters(),"\n")
cursor.execute("CREATE TABLE thetable (name TEXT, ablob BYTEA)")
print('Created Table...')
cursor.execute("INSERT INTO thetable VALUES(%s)",(psycopg2.Binary(pickled),))
connection.commit()
print('Added Data...')
cursor.close()
connection.close()
print('Connection closed...')
When I come to data data retrieval, I am having many issues importing the data from Postgres - essentially the data is to be opened, unpickled back to the dictionary and visualised. I have tried:
import psycopg2
from io import BytesIO
connection = psycopg2.connect(user = "----",
password = "----",
host = "----",
port = "----",
database = "----")
cursor = connection.cursor()
cursor.execute("SELECT ablob FROM thetable")
result, = cursor.fetchone()
cursor.close()
connection.rollback()
result = BytesIO(result)
print(pickle.load(result))
As per this link: https://www.oreilly.com/library/view/python-cookbook/0596001673/ch08s08.html, and consulted: Insert an image in postgresql database and: saving python object in postgres table with pickle, however have been unable to return the pickled dictionary.
Any advice in achieving this is greatly appreciated!
When your CREATE TABLE lists two fields, you have to list in INSERT which ones you want to fill, unless you fill them all.
import psycopg2
import pickle
dict = {
"foo": "bar"
}
p = pickle.dumps(dict)
connection = psycopg2.connect(database = "test")
cursor = connection.cursor()
cursor.execute("CREATE TABLE thetable (name TEXT, ablob BYTEA)")
cursor.execute("INSERT INTO thetable VALUES(%s,%s)",('test',p))
connection.commit()
cursor.close()
connection.close()
and reading
import psycopg2
import pickle
connection = psycopg2.connect(database = "test")
cursor = connection.cursor()
cursor.execute("SELECT ablob FROM thetable WHERE name='test';")
result = cursor.fetchone()
print pickle.loads(result[0])
cursor.close()
connection.close()
I'm using a mysql lib on python, and when I try to do this query:
SELECT product FROM database1.contacts WHERE contact="%s" % (contact)
I get this:
(u'example',)
But I expect this:
example
Here is my code:
import mysql.connector
db_user = "root"
db_passwd = ""
db_host = "localhost"
db_name = "database1"
connector = mysql.connector.connect(user=db_user, password=db_passwd, host=db_host, database=db_name,
buffered=True)
cursor = connector.cursor()
contact = "943832628"
get_product_sql = 'SELECT product FROM database1.contacts WHERE contact="%s"' % (contact)
cursor.execute(get_product_sql)
for product in cursor:
print product
You are printing the whole row; print just the first column:
for product in cursor:
print product[0]
or use tuple unpacking in the loop:
for product, in cursor:
print product