We Keep Coding

psycopg2: update multiple rows with one query

I tried to update multiple rows (approx. 350000) with a single query by implementing the following function:
def update_items(rows_to_update):
sql_query = """UPDATE contact as t SET
name = e.name
FROM (VALUES %s) AS e(id, name)
WHERE e.id = t.id;"""
conn = get_db_connection()
cur = conn.cursor()
psycopg2.extras.execute_values (
cur, sql_query, rows_to_update, template=None, page_size=100
)
While trying to run the function above, only 31 records were updated. Then, I tried to update row by row with the following function:
def update_items_row_by_row(rows_to_update):
sql_query = """UPDATE contact SET name = %s WHERE id = %s"""
conn = get_db_connection()
with tqdm(total=len(rows_to_update)) as pbar:
for id, name in rows_to_update:
cur = conn.cursor()
# execute the UPDATE statement
cur.execute(sql_query, (name, id))
# get the number of updated rows
# Commit the changes to the database
conn.commit()
cur.close()
pbar.update(1)
The latter has updated all the records so far but is very slow (estimated to end in 9 hours).
Does anyone know what is the efficient way to update multiple records?

By splitting the list into chunks of size equal to page_size, it worked well:
def update_items(rows_to_update):
sql_query = """UPDATE contact as t SET
name = data.name
FROM (VALUES %s) AS data (id, name)
WHERE t.id = data.id"""
conn = get_db_connection()
cur = conn.cursor()
n = 100
with tqdm(total=len(rows_to_update)) as pbar:
for i in range(0, len(rows_to_update), n):
psycopg2.extras.execute_values (
cur, sql_query, rows_to_update[i:i + n], template=None, page_size=n
)
conn.commit()
pbar.update(cur.rowcount)
cur.close()
conn.close()

The problem with your original function appears to be that you forgot to apply commit. When you execute an insert/update query with psycopg2 a transaction is opened but not finalized until commit is called. See my edits in your function (towards the bottom).
def update_items(rows_to_update):
sql_query = """UPDATE contact as t SET
name = e.name
FROM (VALUES %s) AS e(id, name)
WHERE e.id = t.id;"""
conn = get_db_connection()
cur = conn.cursor()
psycopg2.extras.execute_values(cur, sql_query, rows_to_update)
## solution below ##
conn.commit() # <- We MUST commit to reflect the inserted data
cur.close()
conn.close()
return "success :)"
If you don't want to call conn.commit() each time you create a new cursor, you can use autocommit such as
conn = get_db_connection()
conn.set_session(autocommit=True)

Sqlite 3: Error opening the databaseIncorrect number of bindings supplied. The current statement uses 1, and there are 4 supplied

I've already tried adding in a comma after Name and the question mark in "VALUES" and was getting a syntax error for my parthenthesis.
#app.route("/Disease/new", methods = ["POST"])
def addDisease():
newDisease = {}
conn = None
try:
jsonPostData = request.get_json()
Name = jsonPostData["Name"]
conn = sqlite3.connect("./dbs/ContactTracer.db")
conn.row_factory = sqlite3.Row
sql = """
INSERT INTO Disease(Name) VALUES(?)
"""
cursor = conn.cursor()
cursor.execute(sql, (Name))
conn.commit()
sql = """
SELECT Disease.ID, Disease.Name
From Disease
Where Disease.ID = ?
"""
cursor.execute(sql,(cursor.lastrowid,))
row = cursor.fetchone()
newDisease["ID"] = row["ID"]
newDisease["Name"] = row["Name"]
except Error as e:
print(f"Error opening the database{e}")
abort(500)
finally:
if conn:
conn.close()
return newDisease

Remove the () and check if INSERT succeeded
cursor.execute(sql, Name)
...
if cursor.lastrowid:
cursor.execute(sql, cursor.lastrowid)

how to mock python postgres database

I try to mock my db but when I test it the result is None.
try:
con = psycopg2.connect(
host="yhvh",
database="python_db",
user="postgres",
password="pass",
)
except:
print("Unable to connect database")
# Open a cursor to perform database operation
cur = con.cursor()
def read(con):
"""
Read data in Database
"""
print("Read")
# execute the query
data ="SELECT id, name FROM employees"
cur.execute(
data
)
# fetchall - returns all entries
rows = cur.fetchall()
for r in rows:
print(f"id {r[0]} name {r[1]}")
this is the code for my testing
def test_read(self):
expected = [9, 'aaa']
with patch('psycopg2.connect') as mock_connect:
mock_con_cm = mock_connect.return_value
mock_con = mock_con_cm.__enter__.return_value
mock_cur = mock_con.cursor.return_value
mock_cur.fetchall.return_value = expected
result = db.read(mock_connect)
self.assertEqual(expected, result)
I get an assertionError: [9, 'aaa'] != None
How the result to have a value that would result is equal to expected ?

First you need to return the rows which contains the list of data from read function if not it will return None.
Then use assertListEqual(expected, result) to check the elements in the list.
Your final code will look like this.
def read(con):
"""
Read data in Database
"""
print("Read")
# execute the query
data ="SELECT id, name FROM employees"
cur.execute(
data
)
# fetchall - returns all entries
rows = cur.fetchall()
for r in rows:
print(f"id {r[0]} name {r[1]}")
return rows
And assertion should be,
self.assertListEqual(expected, result)

python Sqlite3 parameter subs

I have a small problem with this class which handle my DB. It still saying:
cursor.execute(sql)
ValueError: operation parameter must be str
I tried lots of things but nothing work as i want. I looked over https://docs.python.org/3.4/library/sqlite3.html and i'm sure i do the same things.
import sqlite3
class Database():
def __init__(self):
try:
self.db = sqlite3.connect('../database.sqlite')
self.cur = self.db.cursor()
self.cur.execute('pragma foreign_keys="1"')
except sqlite3.Error as e:
raise e
def select(self,sql):
cursor = self.db.cursor()
cursor.execute(sql)
records = cursor.fetchall()
cursor.close()
return records
def insert(self,sql):
cursor = self.db.cursor()
cursor.execute(sql)
newID = cursor.lastrowid
self.db.commit()
cursor.close()
return newID
def execute(self,sql):
""" execute any SQL statement but no return value given """
cursor = self.db.cursor()
cursor.execute(sql)
self.db.commit()
cursor.close()
if __name__ == '__main__':
db = Database()
#sql = "SELECT skuref, titre_prod FROM product"
t = ("888888",)
sql= "UPDATE product SET created = 1 WHERE skuref = ?", t
db.execute(sql)
If someone can help me it would be grateful.Later i wanted to pass something like this in the main program inside a for loop
lastpost = record[0]
if created = True
sql = "UPDATE product SET created = 1 WHERE skuref = ?",(lastpost,)
db.execute(sql)

sql is a tuple containing SQL statement and the parameters.
Change as following, so that sql and parameters are passed separately, instead of being passed as a tuple:
def execute(self, sql):
""" execute any SQL statement but no return value given """
cursor = self.db.cursor()
cursor.execute(*sql) # <------
self.db.commit()
cursor.close()

With your statement
sql = "UPDATE product SET created = 1 WHERE skuref = ?",(lastpost,)
you have created a tupel like
("UPDATE product SET created = 1 WHERE skuref = ?", (lastpost,))
You have to give the arguments as parameters to the execute() function.
Also your if statement is bad: no :, = instead of == and the whole check for True is no nesesary.
Try this:
lastpost = record[0]
if created:
sql = "UPDATE product SET created = 1 WHERE skuref = ?"
db.execute(sql, lastpost)

query from postgresql using python as dictionary

I'm using Python 2.7 and postgresql 9.1.
Trying to get dictionary from query, I've tried the code as described here:
http://wiki.postgresql.org/wiki/Using_psycopg2_with_PostgreSQL
import psycopg2
import psycopg2.extras
conn = psycopg2.connect("dbname=mydb host=localhost user=user password=password")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute ("select * from port")
type(cur.fetchall())
It is printing the next answer:
<type 'list'>
printing the item itself, show me that it is list.
The excepted answer was dictionary.
Edit:
Trying the next:
ans = cur.fetchall()[0]
print ans
print type(ans)
returns
[288, 'T', 51, 1, 1, '192.168.39.188']
<type 'list'>

Tnx a lot Andrey Shokhin ,
full answer is:
#!/var/bin/python
import psycopg2
import psycopg2.extras
conn = psycopg2.connect("dbname=uniart4_pr host=localhost user=user password=password")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute ("select * from port")
ans =cur.fetchall()
ans1 = []
for row in ans:
ans1.append(dict(row))
print ans1 #actually it's return

It's normal: when you call .fetchall() method returns list of tuples. But if you write
type(cur.fetchone())
it will return only one tuple with type:
<class 'psycopg2.extras.DictRow'>
After this you can use it as list or like dictionary:
cur.execute('SELECT id, msg FROM table;')
rec = cur.fetchone()
print rec[0], rec['msg']
You can also use a simple cursor iterator:
res = [json.dumps(dict(record)) for record in cursor] # it calls .fetchone() in loop

Perhaps to optimize it further we can have
#!/var/bin/python
import psycopg2
import psycopg2.extras
def get_dict_resultset(sql):
conn = psycopg2.connect("dbname=pem host=localhost user=postgres password=Drupal#1008")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute (sql)
ans =cur.fetchall()
dict_result = []
for row in ans:
dict_result.append(dict(row))
return dict_result
sql = """select * from tablename"""
return get_dict_resultset(sql)

If you don't want to use a psycopg2.extras.DictCursor you can create a list of dictionaries for the results using cursor.description:
# connect
connection = psycopg2.connect()
cursor = connection.cursor()
# query
cursor.execute("SELECT * FROM myTable")
# transform result
columns = list(cursor.description)
result = cursor.fetchall()
# make dict
results = []
for row in result:
row_dict = {}
for i, col in enumerate(columns):
row_dict[col.name] = row[i]
results.append(row_dict)
# display
print(result)
I use the following function fairly regularly:
def select_query_dict(connection, query, data=[]):
"""
Run generic select query on db, returns a list of dictionaries
"""
logger.debug('Running query: {}'.format(query))
# Open a cursor to perform database operations
cursor = connection.cursor()
logging.debug('Db connection succesful')
# execute the query
try:
logger.info('Running query.')
if len(data):
cursor.execute(query, data)
else:
cursor.execute(query)
columns = list(cursor.description)
result = cursor.fetchall()
logging.debug('Query executed succesfully')
except (Exception, psycopg2.DatabaseError) as e:
logging.error(e)
cursor.close()
exit(1)
cursor.close()
# make dict
results = []
for row in result:
row_dict = {}
for i, col in enumerate(columns):
row_dict[col.name] = row[i]
results.append(row_dict)
return results

In addition to just return only the query results as a list of dictionaries, I would suggest returning key-value pairs (column-name:row-value). Here my suggestion:
import psycopg2
import psycopg2.extras
conn = None
try:
conn = psycopg2.connect("dbname=uniart4_pr host=localhost user=user password=password")
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor:
cursor.execute("SELECT * FROM table")
column_names = [desc[0] for desc in cursor.description]
res = cursor.fetchall()
cursor.close()
return map(lambda x: dict(zip(column_names, x)), res))
except (Exception, psycopg2.DatabaseError) as e:
logger.error(e)
finally:
if conn is not None:
conn.close()

There is a built in solution to get your result as a collection of dictionary:
from psycopg2.extras import RealDictCursor
cur = conn.cursor(cursor_factory=RealDictCursor)
Modified from: https://www.peterbe.com/plog/from-postgres-to-json-strings, copyright 2013 Peter Bengtsson

For me when I convert the row to dictionary failed (solutions mentioned by others)and also could not use cursor factory.
I am using PostgreSQL 9.6.10, Below code worked for me but I am not sure if its the right way to do it.
def convert_to_dict(columns, results):
"""
This method converts the resultset from postgres to dictionary
interates the data and maps the columns to the values in result set and converts to dictionary
:param columns: List - column names return when query is executed
:param results: List / Tupple - result set from when query is executed
:return: list of dictionary- mapped with table column name and to its values
"""
allResults = []
columns = [col.name for col in columns]
if type(results) is list:
for value in results:
allResults.append(dict(zip(columns, value)))
return allResults
elif type(results) is tuple:
allResults.append(dict(zip(columns, results)))
return allResults
Way to use it:
conn = psycopg2.connect("dbname=pem host=localhost user=postgres,password=Drupal#1008")
cur = conn.cursor()
cur.execute("select * from tableNAme")
resultset = cursor.fetchall()
result = convert_to_dict(cursor.description, resultset)
print(result)
resultset = cursor.fetchone()
result = convert_to_dict(cursor.description, resultset)
print(result)

Contents of './config.py'
#!/usr/bin/python
PGCONF = {
"user": "postgres",
"password": "postgres",
"host": "localhost",
"database": "database_name"
}
contents of './main.py'
#!/usr/bin/python
from config import PGCONF
import psycopg2
import psycopg2.extras
# open connection
conn = psycopg2.connect(**PGCONF)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
# declare lambda function
fetch_all_as_dict = lambda cursor: [dict(row) for row in cursor]
# execute any query of your choice
cur.execute("""select * from table_name limit 1""")
# get all rows as list of dicts
print(fetch_all_as_dict(cur))
# close cursor and connection
cur.close()
conn.close()