I have a database named products in sql and i wish to get all the rows as a dictionary or json. I've seen an example here but how do i pass username, password and host?
This is the example:
import json
import psycopg2
def db(database_name='products'):
return psycopg2.connect(database=database_name)
def query_db(query, args=(), one=False):
cur = db().cursor()
cur.execute(query, args)
r = [dict((cur.description[i][0], value) for i, value in enumerate(row)) for row in cur.fetchall()]
cur.connection.close()
return (r[0] if r else None) if one else r
my_query = query_db("SELECT * FROM main_prod WHERE id = 1")
print(my_query)
json_output = json.dumps(my_query)
print(json_output)
When i use it like this i'm getting this error:
File "/home/alex/Documents/Proiecte/Python/bapp/venv/lib/python3.5/site-packages/psycopg2/__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: fe_sendauth: no password supplied
When i'm doing like this
import json
import psycopg2
def db(database_name='products', password='...', host='123.123.123.13', user='alex'):
return psycopg2.connect(database=database_name, password=password, host=host, user=user)
def query_db(query, args=(), one=False):
cur = db().cursor()
cur.execute(query, args)
r = [dict((cur.description[i][0], value) for i, value in enumerate(row)) for row in cur.fetchall()]
cur.connection.close()
return (r[0] if r else None) if one else r
my_query = query_db("SELECT * FROM main_prod WHERE id = 1")
print(my_query)
json_output = json.dumps(my_query)
print(json_output)
It won't print anything, it just remains like in sleep.
How can i do it?
Try this:
import psycopg2
import json
def main():
conn_string = "database_name='products', password='...', host='123.123.123.13', user='alex'"
# print the connection string we will use to connect
print "Connecting to database\n ->%s" % (conn_string)
# get a connection, if a connect cannot be made an exception will be raised here
conn = psycopg2.connect(conn_string)
# conn.cursor will return a cursor object, you can use this cursor to perform queries
cursor = conn.cursor()
# execute our Query
cursor.execute("SELECT * FROM main_prod WHERE id = 1")
# retrieve the records from the database
records = cursor.fetchall()
objects = [
{
'id': row.id,
} for row in records
] # there you tell what data you want to return
json_output = json.dumps(objects)
print(json_output)
if __name__ == "__main__":
main()
Related
I've already tried adding in a comma after Name and the question mark in "VALUES" and was getting a syntax error for my parthenthesis.
#app.route("/Disease/new", methods = ["POST"])
def addDisease():
newDisease = {}
conn = None
try:
jsonPostData = request.get_json()
Name = jsonPostData["Name"]
conn = sqlite3.connect("./dbs/ContactTracer.db")
conn.row_factory = sqlite3.Row
sql = """
INSERT INTO Disease(Name) VALUES(?)
"""
cursor = conn.cursor()
cursor.execute(sql, (Name))
conn.commit()
sql = """
SELECT Disease.ID, Disease.Name
From Disease
Where Disease.ID = ?
"""
cursor.execute(sql,(cursor.lastrowid,))
row = cursor.fetchone()
newDisease["ID"] = row["ID"]
newDisease["Name"] = row["Name"]
except Error as e:
print(f"Error opening the database{e}")
abort(500)
finally:
if conn:
conn.close()
return newDisease
Remove the () and check if INSERT succeeded
cursor.execute(sql, Name)
...
if cursor.lastrowid:
cursor.execute(sql, cursor.lastrowid)
I built a to-do list API with Flask and SQlite, and now I'm trying to use AUTOINCREMENT for incrementing the id's for the tasks. However, I am getting an error ("Error: NOT NULL constraint failed: incomplete.id") when I try to add something to the list. I'm not sure why, I looked at the sqlite documentation, and I seem to be following. I even tried reformatting the create table statements. I'm not sure what else to do, i'd really appreciate some guidance/advice/help. Thanks!
Here is my helper.py
import helper
from flask import Flask, request, jsonify, Response
import json
app = Flask(__name__)
#app.route('/')
def hello_world():
return 'Hello World!'
#app.route('/tasks/new', methods=['PUT'])
def add_task():
# global idCount
# idCount = idCount + 1
# get item from the POST body, request module used to parse request and get HTTP body data. response is used to return response to the client, of type JSON
req_data = request.get_json()
task = req_data['task']
# add task to the list
res_data = helper.add_to_incomplete(task)
# return error if task cant be added
if res_data is None:
response = Response("{'error': 'Task not added - " + task + "'}", mimetype='application/json')
return response;
response = Response(json.dumps(res_data), mimetype='application/json')
return response
#app.route('/tasks/all', methods = ["GET"])
def get_all_items():
res_data = helper.get_all_completes(), helper.get_all_incompletes()
response = Response(json.dumps(res_data), mimetype='application/json')
return response
#app.route('/tasks/complete', methods = ["POST"])
def complete_task():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.add_to_complete(inputId)
# find matching task to input id
return "completed task" + inputId
#app.route('/tasks/incomplete', methods = ["PATCH"])
def uncomplete_task():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.uncomplete(inputId)
# find matching task to input id
return "un-completed task" + inputId
#app.route('/tasks/remove', methods = ["DELETE"])
def delete():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.delete_task(inputId)
if res_data is None:
response = Response("{'error': 'Error deleting task - '" + task + "}", status=400 , mimetype='application/json')
return "deleted task id" + " " + inputId
#app.route('/tasks/empty', methods = ["EMPTY"])
def delete_all():
helper.empty()
return "you deleted everything"
Here is my helper.py:
import sqlite3
import random
#for id's because users dont set them
DB_PATH = './todo.db'
# connect to database
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute("CREATE TABLE IF NOT EXISTS complete (id INTEGER PRIMARY KEY, task TEXT NOT NULL);")
# save the change
c.execute("CREATE TABLE IF NOT EXISTS incomplete (id INTEGER PRIMARY KEY, task TEXT NOT NULL);")
conn.commit()
def add_to_incomplete(task):
try:
# id = str(random.randrange(100,999))
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('insert into incomplete(task) values(?)', (task,))
conn.commit()
return {"id": id}
except Exception as e:
print('Error: ', e)
return None
def add_to_complete(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select task from incomplete where id=?', (inputId,))
tasks = c.fetchone()[0]
c.execute('insert into complete values(?,?)', (inputId,tasks))
delete_task(inputId)
conn.commit()
return {"id": id}
except Exception as e:
print('Error: ', e)
return None
def get_all_completes():
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select * from complete')
rows = c.fetchall()
conn.commit()
return { "complete": rows }
except Exception as e:
print('Error: ', e)
return None
def get_all_incompletes():
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select * from incomplete')
rows = c.fetchall()
conn.commit()
return { "incomplete": rows }
except Exception as e:
print('Error: ', e)
return None
def uncomplete(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select task from complete where id=?', (inputId,))
tasks = c.fetchone()[0]
c.execute('insert into incomplete values(?,?)', (inputId,tasks))
delete_task(inputId)
conn.commit()
return {"id": id}
except Exception as e:
print('Error: ', e)
return None
def delete_task(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('delete from complete where id=?', (inputId,))
c.execute('delete from incomplete where id=?', (inputId,))
conn.commit()
return {"id":id}
except Exception as e:
print('Error: ', e)
return None
def empty():
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('delete from complete')
c.execute('delete from incomplete')
conn.commit()
return "you deleted everything mwahaha"
except Exception as e:
print('Error: ', e)
return None
I would suggest changing your sql table creation code to:
create table if not exists complete
(
id int auto_increment,
constraint complete_pk
primary key (id)
);
However a better option is to use SQLAlchemy
In my example i need to store a list of ints and a list of strings int a database. I'm currently just converting the entire list of ints and list of strings into a single int. I was wondering if this was an ideal workflow or if anyone had some alternative recommendations on how i could handle this. My concern with storing it as a string is how would i late then retrieve that information properly as a pythonic list of ints and strings?
import sqlite3
import hashlib
database = 'test.db'
def create_table():
connection = sqlite3.connect(database)
cursor = connection.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS assets(url BLOB UNIQUE, colors BLOB, tags BLOB)")
connection.commit()
cursor.close()
connection.close()
def kill_table():
connection = sqlite3.connect(database)
cursor = connection.cursor()
cursor.execute('''DROP TABLE IF EXISTS assets''')
connection.commit()
def read_from_db():
connection = sqlite3.connect(database)
cursor = connection.cursor()
cursor.execute('SELECT * FROM assets')
data = cursor.fetchall()
print(len(data))
for row in data:
print(row)
cursor.close()
connection.close()
def get_data_entry(url=''):
connection = sqlite3.connect(database)
cursor = connection.cursor()
url = hashlib.md5(url).hexdigest()
cursor.execute('SELECT * FROM assets WHERE url=?', (url,))
data = cursor.fetchall()
if len(data) == 1:
return data[0]
else:
print 'Found multiple entry instances'
return False
def append_data_entries(url, colors, tags):
'''
Args:
url (str): name of image item
colors (list): list of dominant image colors
tags (list): list of tags
'''
if not url or not colors or not tags:
return False
url = hashlib.md5(url).hexdigest()
colors = str(colors)
tags = str(tags)
# updates or inserts
cursor.execute("REPLACE INTO assets(url, colors, tags) VALUES (?, ?, ?)",
(url, colors, tags))
return True
if __name__ == '__main__':
'Example'
kill_table()
create_table()
# add test data to database
connection = sqlite3.connect(database)
cursor = connection.cursor()
for i in range(10):
url = '{num:08d}'.format(num=i)
append_data_entries(url, '[[0,0,0],[10,10,10],[50,50,50]]','["red","green","blue","orange"]')
connection.commit()
cursor.close()
connection.close()
read_from_db()
print 'ITEM:', get_data_entry('00000006')
When retrieving the data, it returns a string tuple as expected, then you must convert each element if necessary in a suitable data type, for this particular case the ast.literal_eval function should work:
def convert(in_data):
def cvt(data):
try:
return ast.literal_eval(data)
except Exception:
return str(data)
return tuple(map(cvt, in_data))
Example code:
import sqlite3
import hashlib
import ast
database = 'test.db'
def create_table():
connection = sqlite3.connect(database)
cursor = connection.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS assets(url BLOB UNIQUE, colors BLOB, tags BLOB)")
connection.commit()
cursor.close()
connection.close()
def kill_table():
connection = sqlite3.connect(database)
cursor = connection.cursor()
cursor.execute('''DROP TABLE IF EXISTS assets''')
connection.commit()
def convert(in_data):
def cvt(data):
try:
return ast.literal_eval(data)
except Exception:
return str(data)
return tuple(map(cvt, in_data))
def read_from_db():
connection = sqlite3.connect(database)
cursor = connection.cursor()
cursor.execute('SELECT * FROM assets')
data = cursor.fetchall()
print(len(data))
for row in data:
print(convert(row))
cursor.close()
connection.close()
def get_data_entry(url=''):
connection = sqlite3.connect(database)
cursor = connection.cursor()
url = hashlib.md5(url).hexdigest()
cursor.execute('SELECT * FROM assets WHERE url=?', (url,))
data = cursor.fetchall()
if len(data) == 1:
return convert(data[0])
else:
print('Found multiple entry instances')
return False
def append_data_entries(url, colors, tags):
'''
Args:
url (str): name of image item
colors (list): list of dominant image colors
tags (list): list of tags
'''
if not url or not colors or not tags:
return False
url = hashlib.md5(url).hexdigest()
colors = str(colors)
tags = str(tags)
# updates or inserts
cursor.execute("REPLACE INTO assets(url, colors, tags) VALUES (?, ?, ?)",
(url, colors, tags))
return True
if __name__ == '__main__':
'Example'
kill_table()
create_table()
# add test data to database
connection = sqlite3.connect(database)
cursor = connection.cursor()
for i in range(10):
url = '{num:08d}'.format(num=i)
append_data_entries(url, '[[0,0,0],[10,10,10],[50,50,50]]','["red","green","blue","orange"]')
connection.commit()
cursor.close()
connection.close()
read_from_db()
print('ITEM:', get_data_entry('00000006'))
I'm a new to python and I need some help with one loop. Here is the code:
def show_result(self):
listed = self.listed
res = {}
for key in listed:
query_result = 0
is_listed = 0
rbl = key
if not listed[key].get('ERROR'):
query_result = "success"
if listed[key]['LISTED']:
is_listed = 1
else:
is_listed = 0
else:
query_result = "error"
pass
res[key] = [ ['rbl', rbl], ['host', listed['SEARCH_HOST']], ['lookup', query_result], ['is_listed', is_listed] ]
return res
try:
con = db.connect(db_server, db_user, db_password, db_name)
cur = con.cursor()
#hosts = json array with Hosts and IP's
for host in hosts:
searcher = checkHost(host, rbls, report)
result = searcher.show_result()
#
# Need to loop through result to get rbl, host, lookup and is_listed variables
# to be able to execute the query and commit it later when loop is finished
#
cur.execute("INSERT INTO checks_reports_df8 (`rbl`, `host`, `lookup`, `is_listed`) VALUES(%s, %s, '%s')", (rbl, host, lookup, is_listed))
con.commit()
except db.Error, e:
if con:
con.rollback()
finally:
if con:
con.close()
If I pprint result from searcher.show_result() here is what I've get:
{u'0spam-killlist.fusionzero.com': [['rbl', u'0spam-killlist.fusionzero.com'],
['host', u'127.0.0.2'],
['lookup', 'success'],
['is_listed', 0]],
u'zen.spamhaus.org': [['rbl', u'zen.spamhaus.org'],
['host', u'127.0.0.2'],
['lookup', 'success'],
['is_listed', 1]]}
My problem is that I don't know how to loop through the result from searcher.show_result(). If my approach is wrong, the result returned from searcher.show_result() can be changed.
I'm using Python 2.7 and postgresql 9.1.
Trying to get dictionary from query, I've tried the code as described here:
http://wiki.postgresql.org/wiki/Using_psycopg2_with_PostgreSQL
import psycopg2
import psycopg2.extras
conn = psycopg2.connect("dbname=mydb host=localhost user=user password=password")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute ("select * from port")
type(cur.fetchall())
It is printing the next answer:
<type 'list'>
printing the item itself, show me that it is list.
The excepted answer was dictionary.
Edit:
Trying the next:
ans = cur.fetchall()[0]
print ans
print type(ans)
returns
[288, 'T', 51, 1, 1, '192.168.39.188']
<type 'list'>
Tnx a lot Andrey Shokhin ,
full answer is:
#!/var/bin/python
import psycopg2
import psycopg2.extras
conn = psycopg2.connect("dbname=uniart4_pr host=localhost user=user password=password")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute ("select * from port")
ans =cur.fetchall()
ans1 = []
for row in ans:
ans1.append(dict(row))
print ans1 #actually it's return
It's normal: when you call .fetchall() method returns list of tuples. But if you write
type(cur.fetchone())
it will return only one tuple with type:
<class 'psycopg2.extras.DictRow'>
After this you can use it as list or like dictionary:
cur.execute('SELECT id, msg FROM table;')
rec = cur.fetchone()
print rec[0], rec['msg']
You can also use a simple cursor iterator:
res = [json.dumps(dict(record)) for record in cursor] # it calls .fetchone() in loop
Perhaps to optimize it further we can have
#!/var/bin/python
import psycopg2
import psycopg2.extras
def get_dict_resultset(sql):
conn = psycopg2.connect("dbname=pem host=localhost user=postgres password=Drupal#1008")
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute (sql)
ans =cur.fetchall()
dict_result = []
for row in ans:
dict_result.append(dict(row))
return dict_result
sql = """select * from tablename"""
return get_dict_resultset(sql)
If you don't want to use a psycopg2.extras.DictCursor you can create a list of dictionaries for the results using cursor.description:
# connect
connection = psycopg2.connect()
cursor = connection.cursor()
# query
cursor.execute("SELECT * FROM myTable")
# transform result
columns = list(cursor.description)
result = cursor.fetchall()
# make dict
results = []
for row in result:
row_dict = {}
for i, col in enumerate(columns):
row_dict[col.name] = row[i]
results.append(row_dict)
# display
print(result)
I use the following function fairly regularly:
def select_query_dict(connection, query, data=[]):
"""
Run generic select query on db, returns a list of dictionaries
"""
logger.debug('Running query: {}'.format(query))
# Open a cursor to perform database operations
cursor = connection.cursor()
logging.debug('Db connection succesful')
# execute the query
try:
logger.info('Running query.')
if len(data):
cursor.execute(query, data)
else:
cursor.execute(query)
columns = list(cursor.description)
result = cursor.fetchall()
logging.debug('Query executed succesfully')
except (Exception, psycopg2.DatabaseError) as e:
logging.error(e)
cursor.close()
exit(1)
cursor.close()
# make dict
results = []
for row in result:
row_dict = {}
for i, col in enumerate(columns):
row_dict[col.name] = row[i]
results.append(row_dict)
return results
In addition to just return only the query results as a list of dictionaries, I would suggest returning key-value pairs (column-name:row-value). Here my suggestion:
import psycopg2
import psycopg2.extras
conn = None
try:
conn = psycopg2.connect("dbname=uniart4_pr host=localhost user=user password=password")
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor:
cursor.execute("SELECT * FROM table")
column_names = [desc[0] for desc in cursor.description]
res = cursor.fetchall()
cursor.close()
return map(lambda x: dict(zip(column_names, x)), res))
except (Exception, psycopg2.DatabaseError) as e:
logger.error(e)
finally:
if conn is not None:
conn.close()
There is a built in solution to get your result as a collection of dictionary:
from psycopg2.extras import RealDictCursor
cur = conn.cursor(cursor_factory=RealDictCursor)
Modified from: https://www.peterbe.com/plog/from-postgres-to-json-strings, copyright 2013 Peter Bengtsson
For me when I convert the row to dictionary failed (solutions mentioned by others)and also could not use cursor factory.
I am using PostgreSQL 9.6.10, Below code worked for me but I am not sure if its the right way to do it.
def convert_to_dict(columns, results):
"""
This method converts the resultset from postgres to dictionary
interates the data and maps the columns to the values in result set and converts to dictionary
:param columns: List - column names return when query is executed
:param results: List / Tupple - result set from when query is executed
:return: list of dictionary- mapped with table column name and to its values
"""
allResults = []
columns = [col.name for col in columns]
if type(results) is list:
for value in results:
allResults.append(dict(zip(columns, value)))
return allResults
elif type(results) is tuple:
allResults.append(dict(zip(columns, results)))
return allResults
Way to use it:
conn = psycopg2.connect("dbname=pem host=localhost user=postgres,password=Drupal#1008")
cur = conn.cursor()
cur.execute("select * from tableNAme")
resultset = cursor.fetchall()
result = convert_to_dict(cursor.description, resultset)
print(result)
resultset = cursor.fetchone()
result = convert_to_dict(cursor.description, resultset)
print(result)
Contents of './config.py'
#!/usr/bin/python
PGCONF = {
"user": "postgres",
"password": "postgres",
"host": "localhost",
"database": "database_name"
}
contents of './main.py'
#!/usr/bin/python
from config import PGCONF
import psycopg2
import psycopg2.extras
# open connection
conn = psycopg2.connect(**PGCONF)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
# declare lambda function
fetch_all_as_dict = lambda cursor: [dict(row) for row in cursor]
# execute any query of your choice
cur.execute("""select * from table_name limit 1""")
# get all rows as list of dicts
print(fetch_all_as_dict(cur))
# close cursor and connection
cur.close()
conn.close()