I'm using MySqldb with Python 2.7 to allow Python to make connections to another MySQL server
import MySQLdb
db = MySQLdb.connect(host="sql.domain.com",
user="dev",
passwd="*******",
db="appdb")
Instead of connecting normally like this, how can the connection be made through a SSH tunnel using SSH key pairs?
The SSH tunnel should ideally be opened by Python. The SSH tunnel host and the MySQL server are the same machine.
Only this worked for me
import pymysql
import paramiko
import pandas as pd
from paramiko import SSHClient
from sshtunnel import SSHTunnelForwarder
from os.path import expanduser
home = expanduser('~')
mypkey = paramiko.RSAKey.from_private_key_file(home + pkeyfilepath)
# if you want to use ssh password use - ssh_password='your ssh password', bellow
sql_hostname = 'sql_hostname'
sql_username = 'sql_username'
sql_password = 'sql_password'
sql_main_database = 'db_name'
sql_port = 3306
ssh_host = 'ssh_hostname'
ssh_user = 'ssh_username'
ssh_port = 22
sql_ip = '1.1.1.1.1'
with SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
remote_bind_address=(sql_hostname, sql_port)) as tunnel:
conn = pymysql.connect(host='127.0.0.1', user=sql_username,
passwd=sql_password, db=sql_main_database,
port=tunnel.local_bind_port)
query = '''SELECT VERSION();'''
data = pd.read_sql_query(query, conn)
conn.close()
I'm guessing you'll need port forwarding. I recommend sshtunnel.SSHTunnelForwarder
import mysql.connector
import sshtunnel
with sshtunnel.SSHTunnelForwarder(
(_host, _ssh_port),
ssh_username=_username,
ssh_password=_password,
remote_bind_address=(_remote_bind_address, _remote_mysql_port),
local_bind_address=(_local_bind_address, _local_mysql_port)
) as tunnel:
connection = mysql.connector.connect(
user=_db_user,
password=_db_password,
host=_local_bind_address,
database=_db_name,
port=_local_mysql_port)
...
from sshtunnel import SSHTunnelForwarder
import pymysql
import pandas as pd
tunnel = SSHTunnelForwarder(('SSH_HOST', 22), ssh_password=SSH_PASS, ssh_username=SSH_UNAME,
remote_bind_address=(DB_HOST, 3306))
tunnel.start()
conn = pymysql.connect(host='127.0.0.1', user=DB_UNAME, passwd=DB_PASS, port=tunnel.local_bind_port)
data = pd.read_sql_query("SHOW DATABASES;", conn)
credits to https://www.reddit.com/r/learnpython/comments/53wph1/connecting_to_a_mysql_database_in_a_python_script/
If your private key file is encrypted, this is what worked for me:
mypkey = paramiko.RSAKey.from_private_key_file(<<file location>>, password='password')
sql_hostname = 'sql_hostname'
sql_username = 'sql_username'
sql_password = 'sql_password'
sql_main_database = 'sql_main_database'
sql_port = 3306
ssh_host = 'ssh_host'
ssh_user = 'ssh_user'
ssh_port = 22
with SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
ssh_password='ssh_password',
remote_bind_address=(sql_hostname, sql_port)) as tunnel:
conn = pymysql.connect(host='localhost', user=sql_username,
passwd=sql_password, db=sql_main_database,
port=tunnel.local_bind_port)
query = '''SELECT VERSION();'''
data = pd.read_sql_query(query, conn)
print(data)
conn.close()
You may only write the path to the private key file: ssh_pkey='/home/userName/.ssh/id_ed25519' (documentation is here: https://sshtunnel.readthedocs.io/en/latest/).
If you use mysql.connector from Oracle you must use a construction
cnx = mysql.connector.MySQLConnection(...
Important: a construction
cnx = mysql.connector.connect(...
does not work via an SSh! It is a bug.
(The documentation is here: https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html).
Also, your SQL statement must be ideal. In case of an error on SQL server side, you do not receive an error message from SQL-server.
import sshtunnel
import numpy as np
with sshtunnel.SSHTunnelForwarder(ssh_address_or_host='ssh_host',
ssh_username="ssh_username",
ssh_pkey='/home/userName/.ssh/id_ed25519',
remote_bind_address=('localhost', 3306),
) as tunnel:
cnx = mysql.connector.MySQLConnection(user='sql_username',
password='sql_password',
host='127.0.0.1',
database='db_name',
port=tunnel.local_bind_port)
cursor = cnx.cursor()
cursor.execute('SELECT * FROM db_name.tableName;')
arr = np.array(cursor.fetchall())
cursor.close()
cnx.close()
This works for me:
import mysql.connector
import sshtunnel
with sshtunnel.SSHTunnelForwarder(
('ip-of-ssh-server', 'port-in-number-format'),
ssh_username = 'ssh-username',
ssh_password = 'ssh-password',
remote_bind_address = ('127.0.0.1', 3306)
) as tunnel:
connection = mysql.connector.connect(
user = 'database-username',
password = 'database-password',
host = '127.0.0.1',
port = tunnel.local_bind_port,
database = 'databasename',
)
mycursor = connection.cursor()
query = "SELECT * FROM datos"
mycursor.execute(query)
Someone said this in another comment. If you use the python mysql.connector from Oracle then you must use a construction cnx = mysql.connector.MySQLConnection(....
Important: a construction cnx = mysql.connector.connect(... does not work via an SSH! This bug cost me a whole day trying to understand why connections were being dropped by the remote server:
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.MySQLConnection(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password)
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection.close()
If you are using python, and all the username, password, host and port are correct then there is just one thing left, that is using the argument (use_pure=True). This argument uses python to parse the details and password. You can see the doc of mysql.connector.connect() arguments.
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.MySQLConnection(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password,
use_pure='True')
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection. Close()
Paramiko is the best python module to do ssh tunneling. Check out the code here:
https://github.com/paramiko/paramiko/blob/master/demos/forward.py
As said in comments this one works perfect.
SSH Tunnel for Python MySQLdb connection
Best practice is to parameterize the connection variables.
Here is how I have implemented. Works like charm!
import mysql.connector
import sshtunnel
import pandas as pd
import configparser
config = configparser.ConfigParser()
config.read('c:/work/tmf/data_model/tools/config.ini')
ssh_host = config['db_qa01']['SSH_HOST']
ssh_port = int(config['db_qa01']['SSH_PORT'])
ssh_username = config['db_qa01']['SSH_USER']
ssh_pkey = config['db_qa01']['SSH_PKEY']
sql_host = config['db_qa01']['HOST']
sql_port = int(config['db_qa01']['PORT'])
sql_username = config['db_qa01']['USER']
sql_password = config['db_qa01']['PASSWORD']
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.connect(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password)
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection.close()
Related
I'm trying to create a table in mySQL server running on pythonAnywhere from my local machine.
I followed the getting started guide, https://help.pythonanywhere.com/pages/AccessingMySQLFromOutsidePythonAnywhere, but I'm running into a OperationalError: (2013, 'Lost connection to MySQL server during query').
Here is my code:
import MySQLdb
import sshtunnel
sshtunnel.SSH_TIMEOUT = 10
sshtunnel.TUNNEL_TIMEOUT = 10
with sshtunnel.SSHTunnelForwarder(
('ssh.pythonanywhere.com'),
ssh_username='MyUSERNAME', ssh_password='***',
remote_bind_address=('MyUSERNAME.mysql.pythonanywhere-services.com', 3306)
) as tunnel:
connection = MySQLdb.connect(
user='MyUSERNAME',
passwd='***',
host='127.0.0.1', port=tunnel.local_bind_port,
db='MyUSERNAME$liveSports',
)
cur = connection.cursor()
with connection:
cur.execute("CREATE TABLE table_one (date TEXT, start_time TEXT)")
I'm not sure why I'm getting this error, or how to resolve it.
Similar errors, Lost connection to MySQL server during query , suggest that either I'm sending an incorrect query to my server, but as far as I know this is a valid query, or that my packet is too large, which I don't believe an empty table would be.
I'm new to SQL, but I can't seem to find an answer to this question.
You must leave the tunnel open. This is the easy way:
import MySQLdb
import sshtunnel
sshtunnel.SSH_TIMEOUT = 10
sshtunnel.TUNNEL_TIMEOUT = 10
tunnel = sshtunnel.SSHTunnelForwarder(
('ssh.pythonanywhere.com'),
ssh_username='MyUSERNAME', ssh_password='***',
remote_bind_address=('MyUSERNAME.mysql.pythonanywhere-services.com', 3306)
)
connection = MySQLdb.connect(
user='MyUSERNAME',
passwd='***',
host='127.0.0.1', port=tunnel.local_bind_port,
db='MyUSERNAME$liveSports',
)
cur = connection.cursor()
cur.execute("CREATE TABLE table_one (date TEXT, start_time TEXT)")
You could put all of your database stuff in a function and use
with sshtunnel.SSHTunnelForwarder(
('ssh.pythonanywhere.com'),
ssh_username='MyUSERNAME', ssh_password='***',
remote_bind_address=('MyUSERNAME.mysql.pythonanywhere-services.com', 3306)
) as tunnel:
do_all_processing()
def do_all_processing():
connection = MySQLdb.connect(
user='MyUSERNAME',
passwd='***',
host='127.0.0.1', port=tunnel.local_bind_port,
db='MyUSERNAME$liveSports',
)
...etc...
I have check these answers in StackOverflow but they don't work for me: A1, A2 and A3.
I write my code follows as these answers and got'(1045, "Access denied for user 'root'#'localhost' (using password: YES)")'. Why is that?
my code here:
first try:
from sshtunnel import SSHTunnelForwarder
with SSHTunnelForwarder(
('****.ucd.ie', 22),
ssh_password="***",
ssh_username="s***",
remote_bind_address=('127.0.0.1', 3306)) as server:
con = pymysql.connect(user='root',passwd='***',db='dublinbus',host='127.0.0.1',port=3306)
second try:
from sshtunnel import SSHTunnelForwarder
import pymysql
import pandas as pd
tunnel = SSHTunnelForwarder(('****.ucd.ie', 22), ssh_password='***', ssh_username='s***t',
remote_bind_address=('127.0.0.1', 3306))
tunnel.start()
conn = pymysql.connect(host='127.0.0.1', user='root', passwd='****', port=3306)
data = pd.read_sql_query("SHOW DATABASES;", conn)
third try:
from sshtunnel import SSHTunnelForwarder
import pymysql
with SSHTunnelForwarder(
('****.ucd.ie',22),
ssh_password='****',
ssh_username='s****t',
remote_bind_address=('127.0.0.1', 3306)) as server:
db_connect = pymysql.connect(host='127.0.0.1',
port=3306,
user='root',
passwd='****',
db='dublinbus')
cur = db_connect.cursor()
cur.execute('SELECT stop_num FROM dublinbus.stops limit 10;')
data=cur.fetchone()
print(data[0])
All of them give me: (1045, "Access denied for user 'root'#'localhost' (using password: YES)")
So how could I connect to my remote Mysql via SSH?
Moreover, I use the same 'ssh hostname' and 'mysql hostname' 'mysql server port' in my local mysql workbench. I donot know whether it will have some impact or not.
#I use *** to replace some private information.
this could work:
from sshtunnel import SSHTunnelForwarder
import pymysql
import pandas as pd
server = SSHTunnelForwarder(
ssh_address=('****', 22),
ssh_username="****",
ssh_password="****",
remote_bind_address=("127.0.0.1", 3306))
server.start()
con = pymysql.connect(user='root',passwd='***',db='****',host='127.0.0.1',port=server.local_bind_port)
I'm downloading the tables from my SQL server on PythonAnywhere.com using a ssh tunnel following their description. Using the following code everything works fine in terms of downloading the tables, but the code then hangs at tunnel.close(). Any suggestions on how to stop it from hanging?
from __future__ import print_function
from mysql.connector import connect as sql_connect
import sshtunnel
from sshtunnel import SSHTunnelForwarder
from copy import deepcopy
import cPickle as pickle
import os
import datetime
sshtunnel.SSH_TIMEOUT = 5.0
sshtunnel.TUNNEL_TIMEOUT = 5.0
remote_bind_address = ('{}.mysql.pythonanywhere-services.com'.format(SSH_USERNAME), 3306)
tunnel = SSHTunnelForwarder(('ssh.pythonanywhere.com'),
ssh_username=SSH_USERNAME, ssh_password=SSH_PASSWORD,
remote_bind_address=remote_bind_address)
tunnel.start()
connection = sql_connect(user=SSH_USERNAME, password=DATABASE_PASSWORD,
host='127.0.0.1', port=tunnel.local_bind_port,
database=DATABASE_NAME)
print("Connection successful!")
cursor = connection.cursor() # get the cursor
cursor.execute("USE {}".format(DATABASE_NAME)) # select the database
# fetch all tables
cursor.execute("SHOW TABLES")
tables = deepcopy(cursor.fetchall()) # return data from last query
for (table_name,) in tables:
if 'contribute' in table_name:
print(table_name)
# may hang
connection.close()
tunnel.close()
I want create proxy/socks service from my ssh linux server. I already created port forwarder with sshtunnel for mysql, example code:
from sshtunnel import SSHTunnelForwarder
from sqlalchemy import create_engine
from sqlalchemy.engine.url import URL
with SSHTunnelForwarder(('mydomain.com', 22), ssh_username='root', ssh_password='password', local_bind_address=('127.0.0.1', 3306), remote_bind_address=('127.0.0.1', 3306)) as server:
myDB = URL(drivername='mysql+pymysql', host='127.0.0.1', database='test_db', username='root', password='THawr_tapH3f', port=3306)
engine = create_engine(name_or_url=myDB)
connection = engine.connect()
connection.close()
I want to transform my ssh linux machine in proxy/socks4/5 service with python; how can I do this?
I want to use the python MySQLdb to access a remote MySQL server with --local-infile flag in order to be able to load data from a local file. (as mentioned in this question Importing CSV to MySQL table returns an error #1148)
I use
db = MySQLdb.connect(host="127.0.0.1", port=3307, user="someuser", passwd="password", db="sql_db")
to create a database connection. How do I mimic mysql -u username -p dbname --local-infile using MySQLdb
I know this is late for the initial question, but if someone comes here looking for the same you can do this:
import mysql.connector
mydb = mysql.connector.connect(
host="localhost",
user="root",
passwd="",
allow_local_infile=True,
)
Here you can check the docs for aditional flags: https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html
If you want to set it as a DB config rather than in the connection, you can do it like this:
mydb = mysql.connector.connect(
host="localhost",
user="root",
passwd="",
)
mycursor = mydb.cursor()
try:
command = "SET GLOBAL local_infile = 'ON';"
mycursor.execute(command)
except mysql.connector.errors.DatabaseError:
pass
You can put your DB configurations in to a local file, and then read it when using.
config.ini
[MySQL]
host=192.168.20.28
user=root
password=123456
db_name=ovp_global
charset=utf8
py code:
import MySQLdb
import ConfigParser
config = ConfigParser.ConfigParser()
config.readfp(open("config.ini", "r"))
def get_connection():
host = config.get('MySQL', 'host')
user = config.get('MySQL', 'user')
passwd = config.get('MySQL', 'password')
db = config.get('MySQL', 'db_name')
charset = config.get('MySQL', 'charset')
return MySQLdb.connect(host=host, user=user, passwd=passwd, db=db, charset=charset)