This is a very straight forward question regarding how to insert or select data from/to a database ? Since i'm trying to keep my code as clean as possible, this is how i'm actually performing queries and inserts/updates:
import sys
import MySQLdb
from ConfigParser import SafeConfigParser
#------------------------------------------------------------
# Select and insert
# this func should be called like:
# db_call('c:\dbconf.cfg','select * from something')
# or insert / update statement.
#------------------------------------------------------------
def db_call(cfgFile, sql):
parser = SafeConfigParser()
parser.read(cfgFile)
dbType = parser.get('database', 'db_type')
db_host = parser.get('database', 'db_host')
db_name = parser.get('database', 'db_name')
db_user = parser.get('database', 'db_login')
db_pass = parser.get('database', 'db_pass')
con = MySQLdb.connect(host=db_host, db=db_name,
user=db_user, passwd=db_pass
)
cur = con.cursor()
try:
try:
cur.execute(sql)
if re.match(r'INSERT|insert|UPDATE|update|DELETE|delete', sql):
con.commit()
else:
data = cur.fetchall()
resultList = []
for data_out in data:
resultList.append(data_out)
return resultList
except MySQLdb.Error, e:
con.rollback()
print "Error "
print e.args
sys.exit(1)
else:
con.commit()
finally:
con.close()
But, using this method i have to keep all the queries inside my main class, where that can be a problem if any change happens into the table structure,
But, going for sp call, i can have the code more clean, passing only the sp name and fields. But sometimes this could lead me to have one python function for more specific cases, ( as an example, sp that receives 2,3 or 4 inputs must have diferent python functions for each )
import sys
import MySQLdb
from ConfigParser import SafeConfigParser
#------------------------------------------------------------
# Select only!!!!!!
# this func should be called like:
# db_call('fn_your_function','field_a','field_b')
# or insert / update statement.
#------------------------------------------------------------
def db_call(self, cfgFile, query):
parser = SafeConfigParser()
parser.read(cfgFile)
dbType = parser.get('database', 'db_type')
db_host = parser.get('database', 'db_host')
db_name = parser.get('database', 'db_name')
db_user = parser.get('database', 'db_login')
db_pass = parser.get('database', 'db_pass')
con = MySQLdb.connect(host=db_host, db=db_name,
user=db_user, passwd=db_pass
)
cur = con.cursor()
try:
cur.callproc(query[0], (query[1],query[2]))
data = cur.fetchall()
resultList = []
for data_out in data:
resultList.append(data_out)
return resultList
con.close()
except MySQLdb.Error, e:
con.rollback()
print "Error "
print e.args
sys.exit(1)
Im not sure if here is the right place to ask this, but before voting to close it (if is the case ) please reply with the information where i could ask this kind of question :)
Thanks in advance.
If your goal is to abstract away the schema of your DB from your objects' implementations, you should probably be looking at ORMs/persistence frameworks. There are a number of them in Python. As examples, SQLAlchemy is popular and Django, a popular web framework, has one built in.
Related
This is not something complicated but not sure why is it not working
import mysql.connector
def get_connection(host, user, password, db_name):
connection = None
try:
connection = mysql.connector.connect(
host=host,
user=user,
use_unicode=True,
password=password,
database=db_name
)
connection.set_charset_collation('utf8')
print('Connected')
except Exception as ex:
print(str(ex))
finally:
return connection
with connection.cursor() as cursor:
sql = 'UPDATE {} set underlying_price=9'.format(table_name)
cursor.execute(sql)
connection.commit()
print('No of Rows Updated ...', cursor.rowcount)
It always returns 0 no matter what. The same query shows correct count on TablePlus
MysQL API provides this method but I do not know how to call it as calling against connection variable gives error
I am not sure why your code does not work. But i am using pymysql, and it works
import os
import pandas as pd
from types import SimpleNamespace
from sqlalchemy import create_engine
import pymysql
PARAM = SimpleNamespace()
PARAM.DB_user='yourname'
PARAM.DB_password='yourpassword'
PARAM.DB_name ='world'
PARAM.DB_ip = 'localhost'
def get_DB_engine_con(PARAM):
DB_name = PARAM.DB_name
DB_ip = PARAM.DB_ip
DB_user = PARAM.DB_user
DB_password = PARAM.DB_password
## engine = create_engine("mysql+pymysql://{user}:{pw}#{ip}/{db}".format(user=DB_user,pw=DB_password,db=DB_name,ip=DB_ip))
conn = pymysql.connect(host=DB_ip, user=DB_user,passwd=DB_password,db=DB_name)
cur = conn.cursor()
return cur, conn ## , engine
cur, conn = get_DB_engine_con(PARAM)
and my data
if i run the code
table_name='ct2'
sql = "UPDATE {} set CountryCode='NL' ".format(table_name)
cur.execute(sql)
conn.commit()
print('No of Rows Updated ...', cur.rowcount)
the result No of Rows Updated ... 10 is printed. and the NLD is changed to NL
If using mysql.connector
import mysql.connector
connection = mysql.connector.connect(
host=PARAM.DB_ip,
user=PARAM.DB_user,
use_unicode=True,
password=PARAM.DB_password,
database=PARAM.DB_name
)
cur = connection.cursor()
table_name='ct2'
sql = "UPDATE {} set CountryCode='NL2' ".format(table_name)
cur.execute(sql)
print('No of Rows Updated ...', cur.rowcount)
connection.commit()
it still works
and the country code is updated to NL2 and No of Rows Updated ... 10 is printed. The second time i run then No of Rows Updated ... 0 is printed.
Not sure why it does not work on your machine.
I'm learning python since last few weeks. For better learning, I decided to work on some project. So here is my Class for MySQL connection and demo example as well. Can you please tell me. What other improvement can be possible for following code?
Structure?
What else I can do to optimize code?
And Please forgive. If I'm doing some silly mistakes in code. (I'm learning)
#!/usr/bin/python
import pymysql
# select (table, parameter)
# insert (table, data)
# update (table, id, data)
# delete (table, id)
class MySQL:
def __init__(self):
self.sort_by = ""
self.order = ""
# initiate database connection.
self.connection = pymysql.connect(host='localhost',
user='root',
password='',
db='sherlock',
charset='utf8mb4')
self.cursor = self.connection.cursor(pymysql.cursors.DictCursor)
# this function is for selecting any feild on any table.(feilds veriable is optinal)
def select(self, table, *feilds):
flds = "" #differnt name for feilds veriable.
if not feilds:
flds = '*'
else:
for f in feilds:
if not flds:
flds = f
else:
flds += ",`%s`" % f
sql = "SELECT %s FROM `%s` " % (flds, table)
if self.sort_by:
sql = sql +"order by "+ str(self.sort_by) +" "+ str(self.order)
print sql
self.cursor.execute(sql)
result = self.cursor.fetchall()
return result
# This function is for data sorting for Mysql; but optinal.
# example : SELECT * FROM `users` order by id asc
def order_by(self, sort_by="", order="", *args, **kwargs):
self.sort_by = sort_by
self.order = order
# this function is for closing Mysql connection
def close(self):
self.connection.close()
########### END OF MySQL CLASS #############
sql = MySQL()
# sql.order_by function should be called before the sql.select() function.
sql.order_by("email")
# this will select all the feilds from `users` table.
# you can specify whichever feilds you want to return. like : sql.select("users", "id, email")
result = sql.select("users", "password")
for email in result:
print email["password"]
sql.close()
import MySQLdb
import time
try:
db = MySQLdb.connect(host="", #your host, usually localhost
user="", #your username
passwd="", #your password
db="") #name of the data base
cur = db.cursor()
except mysql.connector.Error as err:
print("Something went wrong: {}".format(err))
SQL = "INSERT INTO TBL_PYTest (Time) VALUES (%s)"
Count = 0
while Count < 5:
UTime = int(time.time())
print UTime
cur.execute(SQL, (UTime))
time.sleep(5)
Count = Count + 1
print Count
Why isn't this working? its printing correctly but the database stays empty.
Ive checked the DB and it seems fine
All the details are correct
You would need to commit your transaction , or set autocommit as True.
Issue: I can't figure out how to run a query in the correct way so that it returns a mapped dictionary. The query will use counts from multiple tables.
I am using psycopg2 for a postgresql database, and I will be using the results to create a report on day to day deltas on these counts.
Given that, can someone provide an example on how to execute multiple queries and return a dictionary that I can use for comparison purposes? Thanks! I image in a for loop is needed somewhere in here.
tables = ['table1', 'table2']
def db_query():
query = "select count(*) from (a_table) where error_string != '';"
conn = psycopg2.connect(database=db, user=user, password=password, host=host)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute(query, tables)
output = cur.fetchall()
conn.close()
return output
I haven't used postgresql, so you might want to also check this out as a reference: How to store count values in python.
That being said, rearrange your code into something like this. Be sure to make conn global so you don't have to make more than one connection, and make sure you're also closing cur:
conn = None
def driverFunc():
global conn
try:
conn = psycopg2.connect(database=db, user=user, password=password, host=host)
tables = ['table1', 'table2']
countDict = {}
for thisTable in tables:
db_query(thisTable, countDict)
finally:
if not conn == None:
conn.close()
def db_query(tableName, countDict):
# Beware of SQL injection with the following line:
query = "select count(*) from " + tableName + " where error_string != '';"
cur = None
try:
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute(query)
countDict[tableName] = int(cur.fetchone())
finally:
if not cur == None:
cur.close()
Using the content of this xml example saved on a local file called test.xml , I'm trying to parse the contents and insert into my database using the following code:
import cx_Oracle
import re
import os
import xml.etree.ElementTree as ET
from ConfigParser import SafeConfigParser
def db_callmany(cfgFile, sql,params):
parser = SafeConfigParser()
parser.read(cfgFile)
dsn = parser.get('odbc', 'dsn')
uid = parser.get('odbc', 'user')
pwd = parser.get('odbc', 'pass')
try:
con = None
con = cx_Oracle.connect(uid , pwd, dsn)
cur = con.cursor()
cur.executemany(sql,params)
con.commit()
except cx_Oracle.DatabaseError, e:
print 'Error %s' % e
sys.exit(1)
finally:
if con:
con.close()
if __name__ == '__main__':
try:
cfgFile='c:\\tests\\dbInfo.cfg'
tree = ET.parse('test.xml')
root = tree.getroot()
mdfList = []
for book in root.findall('book'):
author = book.find('genre').text
title = book.find('price').text
the = str((author,title))
mdfList.append(the)
sql = "INSERT INTO book_table ( GENRE, PRICE )"
db_callmany(cfgFile,sql,mdfList)
except KeyboardInterrupt:
sys.stdout('\nInterrupted.\n')
But executing this code I receive the following error:
Error ORA-01036: illegal variable name/number
Exit code: 1
Not sure what I can be missing here in order to get this code working.
CX_Oracle requires placeholders and then a sequence of dictionaries for executemany rather than a sequence of sequences - so something like:
mdfList = list()
for book in root.findall('book'):
Values = dict()
Values['GENRE'] = book.find('genre').text
Values['PRICE'] = book.find('price').text
mdfList.append(Values)
sql = "INSERT INTO book_table (GENRE, PRICE) VALUES (:GENRE, :PRICE)"
db_callmany(cfgFile, sql, mdfList)