I'm using Python 2.7 and postgresql 9.1.
Trying to get dictionary from query, I've tried the code as described here:
http://wiki.postgresql.org/wiki/Using_psycopg2_with_PostgreSQL
import psycopg2
import psycopg2.extras
conn = psycopg2.connect("dbname=mydb host=localhost user=user password=password")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute ("select * from port")
type(cur.fetchall())
It is printing the next answer:
<type 'list'>
printing the item itself, show me that it is list.
The excepted answer was dictionary.
Edit:
Trying the next:
ans = cur.fetchall()[0]
print ans
print type(ans)
returns
[288, 'T', 51, 1, 1, '192.168.39.188']
<type 'list'>
Tnx a lot Andrey Shokhin ,
full answer is:
#!/var/bin/python
import psycopg2
import psycopg2.extras
conn = psycopg2.connect("dbname=uniart4_pr host=localhost user=user password=password")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute ("select * from port")
ans =cur.fetchall()
ans1 = []
for row in ans:
ans1.append(dict(row))
print ans1 #actually it's return
It's normal: when you call .fetchall() method returns list of tuples. But if you write
type(cur.fetchone())
it will return only one tuple with type:
<class 'psycopg2.extras.DictRow'>
After this you can use it as list or like dictionary:
cur.execute('SELECT id, msg FROM table;')
rec = cur.fetchone()
print rec[0], rec['msg']
You can also use a simple cursor iterator:
res = [json.dumps(dict(record)) for record in cursor] # it calls .fetchone() in loop
Perhaps to optimize it further we can have
#!/var/bin/python
import psycopg2
import psycopg2.extras
def get_dict_resultset(sql):
conn = psycopg2.connect("dbname=pem host=localhost user=postgres password=Drupal#1008")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute (sql)
ans =cur.fetchall()
dict_result = []
for row in ans:
dict_result.append(dict(row))
return dict_result
sql = """select * from tablename"""
return get_dict_resultset(sql)
If you don't want to use a psycopg2.extras.DictCursor you can create a list of dictionaries for the results using cursor.description:
# connect
connection = psycopg2.connect()
cursor = connection.cursor()
# query
cursor.execute("SELECT * FROM myTable")
# transform result
columns = list(cursor.description)
result = cursor.fetchall()
# make dict
results = []
for row in result:
row_dict = {}
for i, col in enumerate(columns):
row_dict[col.name] = row[i]
results.append(row_dict)
# display
print(result)
I use the following function fairly regularly:
def select_query_dict(connection, query, data=[]):
"""
Run generic select query on db, returns a list of dictionaries
"""
logger.debug('Running query: {}'.format(query))
# Open a cursor to perform database operations
cursor = connection.cursor()
logging.debug('Db connection succesful')
# execute the query
try:
logger.info('Running query.')
if len(data):
cursor.execute(query, data)
else:
cursor.execute(query)
columns = list(cursor.description)
result = cursor.fetchall()
logging.debug('Query executed succesfully')
except (Exception, psycopg2.DatabaseError) as e:
logging.error(e)
cursor.close()
exit(1)
cursor.close()
# make dict
results = []
for row in result:
row_dict = {}
for i, col in enumerate(columns):
row_dict[col.name] = row[i]
results.append(row_dict)
return results
In addition to just return only the query results as a list of dictionaries, I would suggest returning key-value pairs (column-name:row-value). Here my suggestion:
import psycopg2
import psycopg2.extras
conn = None
try:
conn = psycopg2.connect("dbname=uniart4_pr host=localhost user=user password=password")
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor:
cursor.execute("SELECT * FROM table")
column_names = [desc[0] for desc in cursor.description]
res = cursor.fetchall()
cursor.close()
return map(lambda x: dict(zip(column_names, x)), res))
except (Exception, psycopg2.DatabaseError) as e:
logger.error(e)
finally:
if conn is not None:
conn.close()
There is a built in solution to get your result as a collection of dictionary:
from psycopg2.extras import RealDictCursor
cur = conn.cursor(cursor_factory=RealDictCursor)
Modified from: https://www.peterbe.com/plog/from-postgres-to-json-strings, copyright 2013 Peter Bengtsson
For me when I convert the row to dictionary failed (solutions mentioned by others)and also could not use cursor factory.
I am using PostgreSQL 9.6.10, Below code worked for me but I am not sure if its the right way to do it.
def convert_to_dict(columns, results):
"""
This method converts the resultset from postgres to dictionary
interates the data and maps the columns to the values in result set and converts to dictionary
:param columns: List - column names return when query is executed
:param results: List / Tupple - result set from when query is executed
:return: list of dictionary- mapped with table column name and to its values
"""
allResults = []
columns = [col.name for col in columns]
if type(results) is list:
for value in results:
allResults.append(dict(zip(columns, value)))
return allResults
elif type(results) is tuple:
allResults.append(dict(zip(columns, results)))
return allResults
Way to use it:
conn = psycopg2.connect("dbname=pem host=localhost user=postgres,password=Drupal#1008")
cur = conn.cursor()
cur.execute("select * from tableNAme")
resultset = cursor.fetchall()
result = convert_to_dict(cursor.description, resultset)
print(result)
resultset = cursor.fetchone()
result = convert_to_dict(cursor.description, resultset)
print(result)
Contents of './config.py'
#!/usr/bin/python
PGCONF = {
"user": "postgres",
"password": "postgres",
"host": "localhost",
"database": "database_name"
}
contents of './main.py'
#!/usr/bin/python
from config import PGCONF
import psycopg2
import psycopg2.extras
# open connection
conn = psycopg2.connect(**PGCONF)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
# declare lambda function
fetch_all_as_dict = lambda cursor: [dict(row) for row in cursor]
# execute any query of your choice
cur.execute("""select * from table_name limit 1""")
# get all rows as list of dicts
print(fetch_all_as_dict(cur))
# close cursor and connection
cur.close()
conn.close()
Related
I have a SQL-file (SQLite format 3) that I can query with the DB Browser for SQLite (Windows). Whenever I use Python to access the db I get a Null result.
import sqlite3
conn = sqlite3.connect('C:/tmp/test.sql')
cursor = conn.cursor()
conn.execute('select count(*) from Player')
print("result is:", cursor.fetchone()) # result is: None
Every Select statement leads to "result is: None".
Any ideas?
Bart.
import sqlite3
connection = sqlite3.connect(database_name)
cursor = connection.cursor()
cursor.execute("select val from table_name where x = 'something';")
result = cursor.fetchone()
# directly returning result also gives null
if result:
return result[0] # tuple returned in result
cursor.close()
connection.close()
I have testtable() function that works to create the table if necessary and list all the PDF file names in a column. However, when I execute my view() function, it prints an empty list. Am I missing something or just going about this in the wrong way?
import os, sys
import sqlite3
import csv
testdb = 'pdftestdir.db'
def testtable():
conn = sqlite3.connect(testdb)
cur = conn.cursor()
cur.execute('CREATE TABLE IF NOT EXISTS test (name TEXT)')
path = os.listdir('/root/Desktop/PDF')
conn = sqlite3.connect(testdb)
cur = conn.cursor()
cur.execute('SELECT * FROM test')
exists = cur.fetchall()
for name in path:
if name.endswith('.pdf'):
if not exists:
cur.execute('INSERT INTO test VALUES (?)', (name,))
else:
pass
conn.commit()
conn.close()
def view():
conn = sqlite3.connect(testdb)
cur = conn.cursor()
cur.execute('SELECT * FROM test')
cur.fetchall()
rows = cur.fetchall()
conn.close()
print(rows)
You unnecessarily call cur.fetchall() without storing the returning value to a variable, and the cursor has already reached the end of the rows returned with that call, so the second time you call cur.fetchall() it no longer has any more rows to return.
You can fix this by simply removing the redundant call.
Change:
cur.fetchall()
rows = cur.fetchall()
to:
rows = cur.fetchall()
I try to mock my db but when I test it the result is None.
try:
con = psycopg2.connect(
host="yhvh",
database="python_db",
user="postgres",
password="pass",
)
except:
print("Unable to connect database")
# Open a cursor to perform database operation
cur = con.cursor()
def read(con):
"""
Read data in Database
"""
print("Read")
# execute the query
data ="SELECT id, name FROM employees"
cur.execute(
data
)
# fetchall - returns all entries
rows = cur.fetchall()
for r in rows:
print(f"id {r[0]} name {r[1]}")
this is the code for my testing
def test_read(self):
expected = [9, 'aaa']
with patch('psycopg2.connect') as mock_connect:
mock_con_cm = mock_connect.return_value
mock_con = mock_con_cm.__enter__.return_value
mock_cur = mock_con.cursor.return_value
mock_cur.fetchall.return_value = expected
result = db.read(mock_connect)
self.assertEqual(expected, result)
I get an assertionError: [9, 'aaa'] != None
How the result to have a value that would result is equal to expected ?
First you need to return the rows which contains the list of data from read function if not it will return None.
Then use assertListEqual(expected, result) to check the elements in the list.
Your final code will look like this.
def read(con):
"""
Read data in Database
"""
print("Read")
# execute the query
data ="SELECT id, name FROM employees"
cur.execute(
data
)
# fetchall - returns all entries
rows = cur.fetchall()
for r in rows:
print(f"id {r[0]} name {r[1]}")
return rows
And assertion should be,
self.assertListEqual(expected, result)
I have a problem with creating SQL query for Oracle database using Python.
I want to bind string variable and it does not work, could you tell me what am I doing wrong?
This is my code:
import cx_Oracle
dokList = []
def LoadDatabase():
conn = None
cursor = None
try:
conn = cx_Oracle.connect("login", "password", "localhost")
cursor = conn.cursor()
query = "SELECT * FROM DOCUMENT WHERE DOC = :param"
for doknumber in dokList:
cursor.execute(query, {'doknr':doknumber})
print(cursor.rowcount)
except cx_Oracle.DatabaseError as err:
print(err)
finally:
if cursor:
cursor.close()
if conn:
conn.close()
def CheckData():
with open('changedNamed.txt') as f:
lines = f.readlines()
for line in lines:
dokList.append(line)
CheckData()
LoadDatabase()
The output of cursor.rowcount is 0 but it should be number greater than 0.
You're using a dictionary ({'doknr' : doknumber}) for your parameter, so it's a named parameter - the :param needs to match the key name. Try this:
query = "SELECT * FROM DOCUMENT WHERE DOC = :doknr"
for doknumber in dokList:
cursor.execute(query, {'doknr':doknumber})
print(cursor.rowcount)
For future troubleshooting, to check whether your parameter is getting passed properly, you can also try changing your query to "select :param from dual".
In one of my django views I query database using plain sql (not orm) and return results.
sql = "select * from foo_bar"
cursor = connection.cursor()
cursor.execute(sql)
rows = cursor.fetchall()
I am getting the data fine, but not the column names. How can I get the field names of the result set that is returned?
On the Django docs, there's a pretty simple method provided (which does indeed use cursor.description, as Ignacio answered).
def dictfetchall(cursor):
"Return all rows from a cursor as a dict"
columns = [col[0] for col in cursor.description]
return [
dict(zip(columns, row))
for row in cursor.fetchall()
]
According to PEP 249, you can try using cursor.description, but this is not entirely reliable.
I have found a nice solution in Doug Hellmann's blog:
http://doughellmann.com/2007/12/30/using-raw-sql-in-django.html
from itertools import *
from django.db import connection
def query_to_dicts(query_string, *query_args):
"""Run a simple query and produce a generator
that returns the results as a bunch of dictionaries
with keys for the column values selected.
"""
cursor = connection.cursor()
cursor.execute(query_string, query_args)
col_names = [desc[0] for desc in cursor.description]
while True:
row = cursor.fetchone()
if row is None:
break
row_dict = dict(izip(col_names, row))
yield row_dict
return
Example usage:
row_dicts = query_to_dicts("""select * from table""")
try the following code :
def read_data(db_name,tbl_name):
details = sfconfig_1.dbdetails
connect_string = 'DRIVER=ODBC Driver 17 for SQL Server;SERVER={server}; DATABASE={database};UID={username}\
;PWD={password};Encrypt=YES;TrustServerCertificate=YES'.format(**details)
connection = pyodbc.connect(connect_string)#connecting to the server
print("connencted to db")
# query syntax
query = 'select top 100 * from '+'[{}].[dbo].[{}]'.format(db_name,tbl_name) + ' t where t.chargeid ='+ "'622102*3'"+';'
#print(query,"\n")
df = pd.read_sql_query(query,con=connection)
print(df.iloc[0])
return "connected to db...................."