I'm using MySqldb with Python 2.7 to allow Python to make connections to another MySQL server
import MySQLdb
db = MySQLdb.connect(host="sql.domain.com",
user="dev",
passwd="*******",
db="appdb")
Instead of connecting normally like this, how can the connection be made through a SSH tunnel using SSH key pairs?
The SSH tunnel should ideally be opened by Python. The SSH tunnel host and the MySQL server are the same machine.
Only this worked for me
import pymysql
import paramiko
import pandas as pd
from paramiko import SSHClient
from sshtunnel import SSHTunnelForwarder
from os.path import expanduser
home = expanduser('~')
mypkey = paramiko.RSAKey.from_private_key_file(home + pkeyfilepath)
# if you want to use ssh password use - ssh_password='your ssh password', bellow
sql_hostname = 'sql_hostname'
sql_username = 'sql_username'
sql_password = 'sql_password'
sql_main_database = 'db_name'
sql_port = 3306
ssh_host = 'ssh_hostname'
ssh_user = 'ssh_username'
ssh_port = 22
sql_ip = '1.1.1.1.1'
with SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
remote_bind_address=(sql_hostname, sql_port)) as tunnel:
conn = pymysql.connect(host='127.0.0.1', user=sql_username,
passwd=sql_password, db=sql_main_database,
port=tunnel.local_bind_port)
query = '''SELECT VERSION();'''
data = pd.read_sql_query(query, conn)
conn.close()
I'm guessing you'll need port forwarding. I recommend sshtunnel.SSHTunnelForwarder
import mysql.connector
import sshtunnel
with sshtunnel.SSHTunnelForwarder(
(_host, _ssh_port),
ssh_username=_username,
ssh_password=_password,
remote_bind_address=(_remote_bind_address, _remote_mysql_port),
local_bind_address=(_local_bind_address, _local_mysql_port)
) as tunnel:
connection = mysql.connector.connect(
user=_db_user,
password=_db_password,
host=_local_bind_address,
database=_db_name,
port=_local_mysql_port)
...
from sshtunnel import SSHTunnelForwarder
import pymysql
import pandas as pd
tunnel = SSHTunnelForwarder(('SSH_HOST', 22), ssh_password=SSH_PASS, ssh_username=SSH_UNAME,
remote_bind_address=(DB_HOST, 3306))
tunnel.start()
conn = pymysql.connect(host='127.0.0.1', user=DB_UNAME, passwd=DB_PASS, port=tunnel.local_bind_port)
data = pd.read_sql_query("SHOW DATABASES;", conn)
credits to https://www.reddit.com/r/learnpython/comments/53wph1/connecting_to_a_mysql_database_in_a_python_script/
If your private key file is encrypted, this is what worked for me:
mypkey = paramiko.RSAKey.from_private_key_file(<<file location>>, password='password')
sql_hostname = 'sql_hostname'
sql_username = 'sql_username'
sql_password = 'sql_password'
sql_main_database = 'sql_main_database'
sql_port = 3306
ssh_host = 'ssh_host'
ssh_user = 'ssh_user'
ssh_port = 22
with SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
ssh_password='ssh_password',
remote_bind_address=(sql_hostname, sql_port)) as tunnel:
conn = pymysql.connect(host='localhost', user=sql_username,
passwd=sql_password, db=sql_main_database,
port=tunnel.local_bind_port)
query = '''SELECT VERSION();'''
data = pd.read_sql_query(query, conn)
print(data)
conn.close()
You may only write the path to the private key file: ssh_pkey='/home/userName/.ssh/id_ed25519' (documentation is here: https://sshtunnel.readthedocs.io/en/latest/).
If you use mysql.connector from Oracle you must use a construction
cnx = mysql.connector.MySQLConnection(...
Important: a construction
cnx = mysql.connector.connect(...
does not work via an SSh! It is a bug.
(The documentation is here: https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html).
Also, your SQL statement must be ideal. In case of an error on SQL server side, you do not receive an error message from SQL-server.
import sshtunnel
import numpy as np
with sshtunnel.SSHTunnelForwarder(ssh_address_or_host='ssh_host',
ssh_username="ssh_username",
ssh_pkey='/home/userName/.ssh/id_ed25519',
remote_bind_address=('localhost', 3306),
) as tunnel:
cnx = mysql.connector.MySQLConnection(user='sql_username',
password='sql_password',
host='127.0.0.1',
database='db_name',
port=tunnel.local_bind_port)
cursor = cnx.cursor()
cursor.execute('SELECT * FROM db_name.tableName;')
arr = np.array(cursor.fetchall())
cursor.close()
cnx.close()
This works for me:
import mysql.connector
import sshtunnel
with sshtunnel.SSHTunnelForwarder(
('ip-of-ssh-server', 'port-in-number-format'),
ssh_username = 'ssh-username',
ssh_password = 'ssh-password',
remote_bind_address = ('127.0.0.1', 3306)
) as tunnel:
connection = mysql.connector.connect(
user = 'database-username',
password = 'database-password',
host = '127.0.0.1',
port = tunnel.local_bind_port,
database = 'databasename',
)
mycursor = connection.cursor()
query = "SELECT * FROM datos"
mycursor.execute(query)
Someone said this in another comment. If you use the python mysql.connector from Oracle then you must use a construction cnx = mysql.connector.MySQLConnection(....
Important: a construction cnx = mysql.connector.connect(... does not work via an SSH! This bug cost me a whole day trying to understand why connections were being dropped by the remote server:
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.MySQLConnection(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password)
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection.close()
If you are using python, and all the username, password, host and port are correct then there is just one thing left, that is using the argument (use_pure=True). This argument uses python to parse the details and password. You can see the doc of mysql.connector.connect() arguments.
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.MySQLConnection(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password,
use_pure='True')
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection. Close()
Paramiko is the best python module to do ssh tunneling. Check out the code here:
https://github.com/paramiko/paramiko/blob/master/demos/forward.py
As said in comments this one works perfect.
SSH Tunnel for Python MySQLdb connection
Best practice is to parameterize the connection variables.
Here is how I have implemented. Works like charm!
import mysql.connector
import sshtunnel
import pandas as pd
import configparser
config = configparser.ConfigParser()
config.read('c:/work/tmf/data_model/tools/config.ini')
ssh_host = config['db_qa01']['SSH_HOST']
ssh_port = int(config['db_qa01']['SSH_PORT'])
ssh_username = config['db_qa01']['SSH_USER']
ssh_pkey = config['db_qa01']['SSH_PKEY']
sql_host = config['db_qa01']['HOST']
sql_port = int(config['db_qa01']['PORT'])
sql_username = config['db_qa01']['USER']
sql_password = config['db_qa01']['PASSWORD']
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.connect(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password)
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection.close()
Related
When I try to use mysql.connector in python, nothing happens, the terminal freezes and nothing happens infinitely
import mysql.connector
hostname = '********.mysql.database.azure.com'
username = '***************.mysql.database.azure.com'
password = '*******'
database = '*********'
port = 3306
print("Hello World");
try:
# conn = mariadb.connect(
conn = mysql.connector.connect(
user=username,
password=password,
host=hostname,
port=port,
ssl_ca="BaltimoreCyberTrustRoot.crt.pem"
)
print("Connection established")
except:
print("Error")
C:\Users\Nathan Almeida\Documents\workspace\research-api>python connection.py
Hello World
Imagem com o codigo
Problem solved
I took a look at the mysql documentation and found mysql.connector.MySQLConnection.
I changed mysql.connector.connect to mysql.connector.MySQLConnection and it worked.
But I still don't understand why mysql.connector.connect doesn't work and freezes the terminal.
Is there a way to do something like the following in GCF?
import pymysql
import paramiko
from sshtunnel import SSHTunnelForwarder
def hello_world(request, data=''):
with SSHTunnelForwarder(ssh_address_or_host='56.48.28.32', ssh_username='ubuntu', ssh_pkey='id_rsa', remote_bind_address=('aval.myixal.net', 3306)) as tunnel:
conn = pymysql.connect(host='127.0.0.1', user='xian', passwd='bwQi', db='avails', port=tunnel.local_bind_port)
cursor = conn.cursor()
cursor.execute('SELECT * FROM global_event LIMIT 10')
data = str(cursor.fetchone())
conn.close()
return data
I have included the id_rsa key and the requirements. This will work locally, but I'm guessing the whole 'port-forwarding' thing might be a non-starter on GCF. Is there a way to do something like the above?
Totally possible. Quick test i did:
import io
import pymysql
import paramiko
from sshtunnel import SSHTunnelForwarder
def hello_world(request):
k = '''-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAB
...
zLFzhIqql0Av4K4XKQAAABVqYWJic29uQHRlc3QtaW5zdGFuY2UBAgME
-----END OPENSSH PRIVATE KEY-----'''
key = paramiko.RSAKey.from_private_key(io.StringIO(k))
with SSHTunnelForwarder(ssh_address_or_host='<ssh-ip-address>', ssh_username='<ssh-user-name>', ssh_pkey=key, remote_bind_address=('127.0.0.1', 3306)) as tunnel:
conn = pymysql.connect(host='127.0.0.1', user='<mysql-user>', passwd='<mysql-pass>', port=tunnel.local_bind_port)
cursor = conn.cursor()
cursor.execute('SELECT NOW();')
data = str(cursor.fetchone())
conn.close()
return data
result:
gcurl https://us-central1-my-project.cloudfunctions.net/function-1
(datetime.datetime(2021, 5, 13, 5, 4, 16),)
make sure not to hardcode your values like I did, instead use env variables.
I've been given a postgres DB in my uni project in which I have to SSH into the network from which I can then access the DB. I've set up the connection in DBeaver using the SSH tab and it works perfectly fine. However, using Python, I can connect to the SSH just fine, but cannot connect to the DB itself. I've checked with another DB that doesn't require SSH and that works just fine. Here is my code. Note: I've already tried using SSHTunnel, too, to no avail. Also, ignore my quick hack to anonymize my SSH login data, as I didn't find how to use a proper config file with paramiko late at night yesterday...
import os
from psycopg2 import connect, Error
from paramiko import SSHClient
from config import config
with open("ssh_config.txt", "r") as f:
lines = f.readlines()
hostname = lines[0].strip()
username = lines[1].strip()
password = lines[2].strip()
ssh = SSHClient()
ssh.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
ssh.connect(hostname=hostname, username=username, password=password)
print("SSH connected.")
try:
params = config()
conn = connect(**params)
cursor = conn.cursor()
print("DB connected.")
# Print PostgreSQL connection properties.
print(conn.get_dsn_parameters(), "\n")
# Print PostgreSQL version.
cursor.execute("SELECT version();")
record = cursor.fetchone()
print("You are connected to - ", record, "\n")
except (Exception, Error) as error:
print("Error while connecting to PostgreSQL", error)
Any help would be appreciated. Thanks!
I've figured it out myself. Here is the updated code. Basically, I had to forward the remote address to localhost and then connect to localhost instead of the DB address.
from psycopg2 import connect, Error
from sshtunnel import SSHTunnelForwarder
from config import config
with open("ssh_config.txt", "r") as f:
lines = f.readlines()
hostname = lines[0].strip()
username = lines[1].strip()
password = lines[2].strip()
remote_bind_address = lines[3].strip()
try:
with SSHTunnelForwarder(
(hostname, 22),
ssh_username=username,
ssh_password=password,
remote_bind_address=(remote_bind_address, 5432),
local_bind_address=("localhost", 8080)) \
as tunnel:
tunnel.start()
print("SSH connected.")
params = config()
conn = connect(**params)
cursor = conn.cursor()
print("DB connected.")
# Print PostgreSQL connection properties.
print(conn.get_dsn_parameters(), "\n")
# Print PostgreSQL version.
cursor.execute("SELECT version();")
record = cursor.fetchone()
print("You are connected to - ", record, "\n")
cursor.close()
conn.close()
tunnel.close()
print("DB disconnected.")
except (Exception, Error) as error:
print("Error while connecting to DB", error)
I want to access a remote database through SSH tunnel.
server = SSHTunnelForwarder(
('172.17.9.125', 22),
ssh_password="123456",
ssh_username="root",
remote_bind_address=('127.0.0.1', 3306))
server.start()
database = pymysql.connect(host='127.0.0.1',
port=3306,
user='root',
passwd='root')
try:
dbsql = "CREATE DATABASE TestDB" # Create database
except Exception as e:
print("Error: ", e)
database.cursor().execute(dbsql)
global db, cursor
db = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='root', db='TestDB')
cursor = db.cursor()
print("Connected to MySQL database")
f = open("testdb.sql") # Execute .sql file, creating data tables
full_sql = f.read()
sql_commands = full_sql.split(';')[:-1]
try:
for sql_command in sql_commands:
if sql_command is not None:
cursor.execute(sql_command)
else:
print("Null")
print("Created database")
except Exception as e:
print("Error: ", e)
I want to create a new database named as "TESTDB" on this remote server, 172.17.9.125.
But I get this database created in localhost. What am I doing wrong here?
Since you already have a database running, your 3306 port is used in that so won't be able to bind.
Solution : bind it to some other port and try to connect to that.
You can bind to another address using local_bind_address=('0.0.0.0', 1234) (this will be your target local address/port to which it will be bound) in your arguments to SSHTunnelForwarder.
So your connection TunnelForwarder should be something like this
server = SSHTunnelForwarder(
('172.17.9.125', 22),
ssh_password="123456",
ssh_username="root",
local_bind_address=('0.0.0.0', 1234),
remote_bind_address=('127.0.0.1', 3306))
And now connection will be made to port 1234
database = pymysql.connect(host='127.0.0.1',
port=1234,
user='root',
passwd='root')
db = pymysql.connect(host='127.0.0.1', port=1234, user='root', passwd='root', db='TestDB')
Previously I've always used something like...
def getConn():
cnx = mysql.connector.connect(user='user', password='pass',
host='hostIP',
database='database')
cur = cnx.cursor(buffered=True)
return [cnx, cur]
To return cnx and cur objects for use. That works fine.
I now need to use an SSH connection to a DB.
The example below executes the query within the function, but won't return the cnx or cur objects for use afterwards, so I get a print of the result, followed by the error
mysql.connector.errors.InterfaceError: 2013: Lost connection to MySQL server during query
I appear to be having the same issue (although a different error returned) to Why won't Python return my mysql-connector cursor from a function?
That question deals with why - I'd like to know if there is a solution.
def returnConnect():
mypkey = paramiko.RSAKey.from_private_key_file('/Users/me/.ssh/id_rsa')
sql_hostname = '127.0.0.1'
with SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
remote_bind_address=(sql_hostname, sql_port)) as tunnel:
cnx = mysql.connector.connect(host='127.0.0.1', user=sql_username,
passwd=sql_password, db=sql_main_database,
port=tunnel.local_bind_port)
cur = cnx.cursor(buffered=True)
sql = "select * from table"
cur.execute(sql)
result = cur.fetchall()
print result
return [cnx, cur]
conn = returnConnect()
cnx = conn[0]
cur = conn[1]
sql = "select * from table"
cur.execute(sql)
result = cur.fetchall()
cnx.close()
print result
Python calls special method __exit__ after with..as execution, so your connection closes after returning from the context of with. Assign the tunnel forvarder to a variable like follow and you`ll be able to use the connection outside the function scope
tunnel = SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
remote_bind_address=(sql_hostname, sql_port))
Read more about compound with statement in the docs.