I am new in using python and I want to try first connecting to mongodb database using python
This is my code but is has error
import pymongo
DATABASE_NAME = "testdb"
DATABASE_HOST = "mongodb://10.10.100.100:22222"
DATABASE_USERNAME = "testuserid"
DATABASE_PASSWORD = "testpass"
try:
myclient = pymongo.MongoClient(DATABASE_HOST)
myclient.test.authenticate( DATABASE_USERNAME , DATABASE_PASSWORD )
mydb = myclient[DATABASE_NAME]
print("[+] Database connected!")
except Exception as e:
print("[+] Database connection error!")
raise e
10.10.100.100 is just my sample address and also 22222 is sample port
DATABASE_URI = f"mongodb://{DATABASE_USERNAME}:{DATABASE_PASSWORD}#{DATABASE_HOST}"
myclient = pymongo.MongoClient(DATABASE_URI)
I'm using MySqldb with Python 2.7 to allow Python to make connections to another MySQL server
import MySQLdb
db = MySQLdb.connect(host="sql.domain.com",
user="dev",
passwd="*******",
db="appdb")
Instead of connecting normally like this, how can the connection be made through a SSH tunnel using SSH key pairs?
The SSH tunnel should ideally be opened by Python. The SSH tunnel host and the MySQL server are the same machine.
Only this worked for me
import pymysql
import paramiko
import pandas as pd
from paramiko import SSHClient
from sshtunnel import SSHTunnelForwarder
from os.path import expanduser
home = expanduser('~')
mypkey = paramiko.RSAKey.from_private_key_file(home + pkeyfilepath)
# if you want to use ssh password use - ssh_password='your ssh password', bellow
sql_hostname = 'sql_hostname'
sql_username = 'sql_username'
sql_password = 'sql_password'
sql_main_database = 'db_name'
sql_port = 3306
ssh_host = 'ssh_hostname'
ssh_user = 'ssh_username'
ssh_port = 22
sql_ip = '1.1.1.1.1'
with SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
remote_bind_address=(sql_hostname, sql_port)) as tunnel:
conn = pymysql.connect(host='127.0.0.1', user=sql_username,
passwd=sql_password, db=sql_main_database,
port=tunnel.local_bind_port)
query = '''SELECT VERSION();'''
data = pd.read_sql_query(query, conn)
conn.close()
I'm guessing you'll need port forwarding. I recommend sshtunnel.SSHTunnelForwarder
import mysql.connector
import sshtunnel
with sshtunnel.SSHTunnelForwarder(
(_host, _ssh_port),
ssh_username=_username,
ssh_password=_password,
remote_bind_address=(_remote_bind_address, _remote_mysql_port),
local_bind_address=(_local_bind_address, _local_mysql_port)
) as tunnel:
connection = mysql.connector.connect(
user=_db_user,
password=_db_password,
host=_local_bind_address,
database=_db_name,
port=_local_mysql_port)
...
from sshtunnel import SSHTunnelForwarder
import pymysql
import pandas as pd
tunnel = SSHTunnelForwarder(('SSH_HOST', 22), ssh_password=SSH_PASS, ssh_username=SSH_UNAME,
remote_bind_address=(DB_HOST, 3306))
tunnel.start()
conn = pymysql.connect(host='127.0.0.1', user=DB_UNAME, passwd=DB_PASS, port=tunnel.local_bind_port)
data = pd.read_sql_query("SHOW DATABASES;", conn)
credits to https://www.reddit.com/r/learnpython/comments/53wph1/connecting_to_a_mysql_database_in_a_python_script/
If your private key file is encrypted, this is what worked for me:
mypkey = paramiko.RSAKey.from_private_key_file(<<file location>>, password='password')
sql_hostname = 'sql_hostname'
sql_username = 'sql_username'
sql_password = 'sql_password'
sql_main_database = 'sql_main_database'
sql_port = 3306
ssh_host = 'ssh_host'
ssh_user = 'ssh_user'
ssh_port = 22
with SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
ssh_password='ssh_password',
remote_bind_address=(sql_hostname, sql_port)) as tunnel:
conn = pymysql.connect(host='localhost', user=sql_username,
passwd=sql_password, db=sql_main_database,
port=tunnel.local_bind_port)
query = '''SELECT VERSION();'''
data = pd.read_sql_query(query, conn)
print(data)
conn.close()
You may only write the path to the private key file: ssh_pkey='/home/userName/.ssh/id_ed25519' (documentation is here: https://sshtunnel.readthedocs.io/en/latest/).
If you use mysql.connector from Oracle you must use a construction
cnx = mysql.connector.MySQLConnection(...
Important: a construction
cnx = mysql.connector.connect(...
does not work via an SSh! It is a bug.
(The documentation is here: https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html).
Also, your SQL statement must be ideal. In case of an error on SQL server side, you do not receive an error message from SQL-server.
import sshtunnel
import numpy as np
with sshtunnel.SSHTunnelForwarder(ssh_address_or_host='ssh_host',
ssh_username="ssh_username",
ssh_pkey='/home/userName/.ssh/id_ed25519',
remote_bind_address=('localhost', 3306),
) as tunnel:
cnx = mysql.connector.MySQLConnection(user='sql_username',
password='sql_password',
host='127.0.0.1',
database='db_name',
port=tunnel.local_bind_port)
cursor = cnx.cursor()
cursor.execute('SELECT * FROM db_name.tableName;')
arr = np.array(cursor.fetchall())
cursor.close()
cnx.close()
This works for me:
import mysql.connector
import sshtunnel
with sshtunnel.SSHTunnelForwarder(
('ip-of-ssh-server', 'port-in-number-format'),
ssh_username = 'ssh-username',
ssh_password = 'ssh-password',
remote_bind_address = ('127.0.0.1', 3306)
) as tunnel:
connection = mysql.connector.connect(
user = 'database-username',
password = 'database-password',
host = '127.0.0.1',
port = tunnel.local_bind_port,
database = 'databasename',
)
mycursor = connection.cursor()
query = "SELECT * FROM datos"
mycursor.execute(query)
Someone said this in another comment. If you use the python mysql.connector from Oracle then you must use a construction cnx = mysql.connector.MySQLConnection(....
Important: a construction cnx = mysql.connector.connect(... does not work via an SSH! This bug cost me a whole day trying to understand why connections were being dropped by the remote server:
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.MySQLConnection(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password)
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection.close()
If you are using python, and all the username, password, host and port are correct then there is just one thing left, that is using the argument (use_pure=True). This argument uses python to parse the details and password. You can see the doc of mysql.connector.connect() arguments.
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.MySQLConnection(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password,
use_pure='True')
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection. Close()
Paramiko is the best python module to do ssh tunneling. Check out the code here:
https://github.com/paramiko/paramiko/blob/master/demos/forward.py
As said in comments this one works perfect.
SSH Tunnel for Python MySQLdb connection
Best practice is to parameterize the connection variables.
Here is how I have implemented. Works like charm!
import mysql.connector
import sshtunnel
import pandas as pd
import configparser
config = configparser.ConfigParser()
config.read('c:/work/tmf/data_model/tools/config.ini')
ssh_host = config['db_qa01']['SSH_HOST']
ssh_port = int(config['db_qa01']['SSH_PORT'])
ssh_username = config['db_qa01']['SSH_USER']
ssh_pkey = config['db_qa01']['SSH_PKEY']
sql_host = config['db_qa01']['HOST']
sql_port = int(config['db_qa01']['PORT'])
sql_username = config['db_qa01']['USER']
sql_password = config['db_qa01']['PASSWORD']
with sshtunnel.SSHTunnelForwarder(
(ssh_host,ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(sql_host, sql_port)) as tunnel:
connection = mysql.connector.connect(
host='127.0.0.1',
port=tunnel.local_bind_port,
user=sql_username,
password=sql_password)
query = 'select version();'
data = pd.read_sql_query(query, connection)
print(data)
connection.close()
I've been given a postgres DB in my uni project in which I have to SSH into the network from which I can then access the DB. I've set up the connection in DBeaver using the SSH tab and it works perfectly fine. However, using Python, I can connect to the SSH just fine, but cannot connect to the DB itself. I've checked with another DB that doesn't require SSH and that works just fine. Here is my code. Note: I've already tried using SSHTunnel, too, to no avail. Also, ignore my quick hack to anonymize my SSH login data, as I didn't find how to use a proper config file with paramiko late at night yesterday...
import os
from psycopg2 import connect, Error
from paramiko import SSHClient
from config import config
with open("ssh_config.txt", "r") as f:
lines = f.readlines()
hostname = lines[0].strip()
username = lines[1].strip()
password = lines[2].strip()
ssh = SSHClient()
ssh.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
ssh.connect(hostname=hostname, username=username, password=password)
print("SSH connected.")
try:
params = config()
conn = connect(**params)
cursor = conn.cursor()
print("DB connected.")
# Print PostgreSQL connection properties.
print(conn.get_dsn_parameters(), "\n")
# Print PostgreSQL version.
cursor.execute("SELECT version();")
record = cursor.fetchone()
print("You are connected to - ", record, "\n")
except (Exception, Error) as error:
print("Error while connecting to PostgreSQL", error)
Any help would be appreciated. Thanks!
I've figured it out myself. Here is the updated code. Basically, I had to forward the remote address to localhost and then connect to localhost instead of the DB address.
from psycopg2 import connect, Error
from sshtunnel import SSHTunnelForwarder
from config import config
with open("ssh_config.txt", "r") as f:
lines = f.readlines()
hostname = lines[0].strip()
username = lines[1].strip()
password = lines[2].strip()
remote_bind_address = lines[3].strip()
try:
with SSHTunnelForwarder(
(hostname, 22),
ssh_username=username,
ssh_password=password,
remote_bind_address=(remote_bind_address, 5432),
local_bind_address=("localhost", 8080)) \
as tunnel:
tunnel.start()
print("SSH connected.")
params = config()
conn = connect(**params)
cursor = conn.cursor()
print("DB connected.")
# Print PostgreSQL connection properties.
print(conn.get_dsn_parameters(), "\n")
# Print PostgreSQL version.
cursor.execute("SELECT version();")
record = cursor.fetchone()
print("You are connected to - ", record, "\n")
cursor.close()
conn.close()
tunnel.close()
print("DB disconnected.")
except (Exception, Error) as error:
print("Error while connecting to DB", error)
I have a problem with my python mysql connection which I need help with.
My setup is two Pi's running servers on each one. One Pi (SolartPi) has Mysql database collecting data. The other pi (OfficePi) is connecting to the solarPi database to retrieve and update data over a network connection.
My main script works all ok until I have to reboot the SolarPi for a maintenance or power problem and the connection to the OfficePi is lost. The python script on the officePi then goes into a fault loop "2006, MYSQL Server has gone away" Below is a sample of this script.
import MySQLdb
connSolar = MySQLdb.connect("192.xxx.x.x", "external", "xxxxx", "xxxxx")
#eternal connection to solar pi database
cursSolar = connSolar.cursor()
while 1:
try:
cursSolar.execute("SELECT * FROM dashboard")
connSolar.commit()
for reading in cursSolar.fetchall():
heatingDemand = reading[2] #get heating demand from dB
print heatingDemand
except (MySQLdb.Error, MySQLdb.Warning) as e:
print (e)
connSolar.close()
So I tried rewriting this with the script from stackoverflow and a web site as shown below, but this now terminates the program when SolarPi is rebooted with the following error
_mysql_exceptions.OperationalError: (2003, 'Can\'t connect to MySQL server on \'192.xxx.x.x' (111 "Connection refused")')
import MySQLdb
class DB:
con = None
def connect(self):
self.conn = MySQLdb.connect("192.xxx.x.x", "xxxxx", "xxxxxx", "house") #eternal connection to solar pi database
def query(self, sql):
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except (AttributeError, MySQLdb.OperationalError):
self.connect()
cursor = self.conn.cursor()
cursor.execute(sql)
return cursor
while 1:
db = DB()
sql = "SELECT * FROM dashboard"
cur = db.query(sql)
for reading in cur.fetchall():
heatingDemand = reading[2] #get heating demand from dB
print heatingDemand
Is there a way for the OfficePi to keep trying to connect to SolarPi mysql database when it has shut down.
Change your code to check a valid connection each loop otherwise pass:
import MySQLdb
class DB:
def connect(self):
try:
self.conn = MySQLdb.connect("192.xxx.x.x", "xxxxx", "xxxxxx", "house")
except (MySQLdb.Error, MySQLdb.Warning) as e:
print (e)
self.conn = None
return self.conn
def query(self, sql):
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except (AttributeError, MySQLdb.OperationalError):
self.connect()
cursor = self.conn.cursor()
cursor.execute(sql)
return cursor
while 1:
db = DB()
conn = db.connect()
if conn:
sql = "SELECT * FROM dashboard"
cur = db.query(sql)
for reading in cur.fetchall():
heatingDemand = reading[2] #get heating demand from dB
print heatingDemand
Previously I've always used something like...
def getConn():
cnx = mysql.connector.connect(user='user', password='pass',
host='hostIP',
database='database')
cur = cnx.cursor(buffered=True)
return [cnx, cur]
To return cnx and cur objects for use. That works fine.
I now need to use an SSH connection to a DB.
The example below executes the query within the function, but won't return the cnx or cur objects for use afterwards, so I get a print of the result, followed by the error
mysql.connector.errors.InterfaceError: 2013: Lost connection to MySQL server during query
I appear to be having the same issue (although a different error returned) to Why won't Python return my mysql-connector cursor from a function?
That question deals with why - I'd like to know if there is a solution.
def returnConnect():
mypkey = paramiko.RSAKey.from_private_key_file('/Users/me/.ssh/id_rsa')
sql_hostname = '127.0.0.1'
with SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
remote_bind_address=(sql_hostname, sql_port)) as tunnel:
cnx = mysql.connector.connect(host='127.0.0.1', user=sql_username,
passwd=sql_password, db=sql_main_database,
port=tunnel.local_bind_port)
cur = cnx.cursor(buffered=True)
sql = "select * from table"
cur.execute(sql)
result = cur.fetchall()
print result
return [cnx, cur]
conn = returnConnect()
cnx = conn[0]
cur = conn[1]
sql = "select * from table"
cur.execute(sql)
result = cur.fetchall()
cnx.close()
print result
Python calls special method __exit__ after with..as execution, so your connection closes after returning from the context of with. Assign the tunnel forvarder to a variable like follow and you`ll be able to use the connection outside the function scope
tunnel = SSHTunnelForwarder(
(ssh_host, ssh_port),
ssh_username=ssh_user,
ssh_pkey=mypkey,
remote_bind_address=(sql_hostname, sql_port))
Read more about compound with statement in the docs.