I have a file temperature.txt with columns:
city
avghigh
avglow
coldmonth
coldavghigh
coldavglow
warmmonth
warmavghigh
warmavglow
I need to return the names of the cities which have the same average low temperature.
I also have this function:
def run_query(db, q, args=None):
conn = sqlite3.connect(db)
cur = conn.cursor()
if args is None:
cur.execute(q)
else:
cur.execute(q, args)
results = cur.fetchall()
cur.close()
conn.commit()
conn.close()
return results
all I got thus far (If it's correct is)
return run_query(noname.db, ('Select Cities, AvgLow from Table')
In SQL, this can be easily done using a self join on the table to get matching AvgLow for different cities like this:
Select
t.Cities,
t.AvgLow
from Table1 t
INNER JOIN Table1 t1 ON t.AvgLow = t1.AvgLow
and t.Cities <> t1.Cities
ORDER BY t.AvgLow;
SQLite Demo
Related
I write to see how I can get only one data to show by a print when I make a query in python, when I do the query it should only give me a number but I cannot show or access it.
def run_query(self, query, parameters = ()):
with sqlite3.connect(self.db_name) as conn:
cursor = conn.cursor()
result = cursor.execute(query, parameters)
conn.commit()
return result
def get_horarios(self):
query = 'SELECT hora FROM horarios where horario=1'
db_rows = self.run_query(query)
print(db_rows)
To show the first row in the results of the query:
print(db_rows.fetchone())
To show all of the results of the query:
print(db_rows.fetchall())
or
for row in db_rows.fetchall():
print(row)
The query always return a list. To access the first item, you can do:
print(db_rows[0])
I tried to update multiple rows (approx. 350000) with a single query by implementing the following function:
def update_items(rows_to_update):
sql_query = """UPDATE contact as t SET
name = e.name
FROM (VALUES %s) AS e(id, name)
WHERE e.id = t.id;"""
conn = get_db_connection()
cur = conn.cursor()
psycopg2.extras.execute_values (
cur, sql_query, rows_to_update, template=None, page_size=100
)
While trying to run the function above, only 31 records were updated. Then, I tried to update row by row with the following function:
def update_items_row_by_row(rows_to_update):
sql_query = """UPDATE contact SET name = %s WHERE id = %s"""
conn = get_db_connection()
with tqdm(total=len(rows_to_update)) as pbar:
for id, name in rows_to_update:
cur = conn.cursor()
# execute the UPDATE statement
cur.execute(sql_query, (name, id))
# get the number of updated rows
# Commit the changes to the database
conn.commit()
cur.close()
pbar.update(1)
The latter has updated all the records so far but is very slow (estimated to end in 9 hours).
Does anyone know what is the efficient way to update multiple records?
By splitting the list into chunks of size equal to page_size, it worked well:
def update_items(rows_to_update):
sql_query = """UPDATE contact as t SET
name = data.name
FROM (VALUES %s) AS data (id, name)
WHERE t.id = data.id"""
conn = get_db_connection()
cur = conn.cursor()
n = 100
with tqdm(total=len(rows_to_update)) as pbar:
for i in range(0, len(rows_to_update), n):
psycopg2.extras.execute_values (
cur, sql_query, rows_to_update[i:i + n], template=None, page_size=n
)
conn.commit()
pbar.update(cur.rowcount)
cur.close()
conn.close()
The problem with your original function appears to be that you forgot to apply commit. When you execute an insert/update query with psycopg2 a transaction is opened but not finalized until commit is called. See my edits in your function (towards the bottom).
def update_items(rows_to_update):
sql_query = """UPDATE contact as t SET
name = e.name
FROM (VALUES %s) AS e(id, name)
WHERE e.id = t.id;"""
conn = get_db_connection()
cur = conn.cursor()
psycopg2.extras.execute_values(cur, sql_query, rows_to_update)
## solution below ##
conn.commit() # <- We MUST commit to reflect the inserted data
cur.close()
conn.close()
return "success :)"
If you don't want to call conn.commit() each time you create a new cursor, you can use autocommit such as
conn = get_db_connection()
conn.set_session(autocommit=True)
I try to mock my db but when I test it the result is None.
try:
con = psycopg2.connect(
host="yhvh",
database="python_db",
user="postgres",
password="pass",
)
except:
print("Unable to connect database")
# Open a cursor to perform database operation
cur = con.cursor()
def read(con):
"""
Read data in Database
"""
print("Read")
# execute the query
data ="SELECT id, name FROM employees"
cur.execute(
data
)
# fetchall - returns all entries
rows = cur.fetchall()
for r in rows:
print(f"id {r[0]} name {r[1]}")
this is the code for my testing
def test_read(self):
expected = [9, 'aaa']
with patch('psycopg2.connect') as mock_connect:
mock_con_cm = mock_connect.return_value
mock_con = mock_con_cm.__enter__.return_value
mock_cur = mock_con.cursor.return_value
mock_cur.fetchall.return_value = expected
result = db.read(mock_connect)
self.assertEqual(expected, result)
I get an assertionError: [9, 'aaa'] != None
How the result to have a value that would result is equal to expected ?
First you need to return the rows which contains the list of data from read function if not it will return None.
Then use assertListEqual(expected, result) to check the elements in the list.
Your final code will look like this.
def read(con):
"""
Read data in Database
"""
print("Read")
# execute the query
data ="SELECT id, name FROM employees"
cur.execute(
data
)
# fetchall - returns all entries
rows = cur.fetchall()
for r in rows:
print(f"id {r[0]} name {r[1]}")
return rows
And assertion should be,
self.assertListEqual(expected, result)
I have an sqlite db called clients.db with a table called prices. Within the table I have columns ['date', 'xyz', 'abc', 'sta, 'vert']. I am accessing the database from python 3.
I can get a specific number easily enough using:
conn = sqlite3.connect('clients.db')
c = conn.cursor()
c.execute('''SELECT "xyz" FROM prices WHERE date=?''', ('2019-01-07', ))
conn.close()
print(c.fetchone()[0])
This returns 1902 as expected.
However when I try the below, instead of the expected number I get xyz.
conn = sqlite3.connect('clients.db')
c = conn.cursor()
c.execute('''SELECT ? FROM prices WHERE date=?''', ('xyz', '2019-01-07', ))
conn.close()
print(c.fetchone()[0])
and when I add a =? I get sqlite3.OperationalError: near "=": syntax error:
conn = sqlite3.connect('clients.db')
c = conn.cursor()
c.execute('''SELECT =? FROM prices WHERE date=?''', ('xyz', '2019-01-07', ))
conn.close()
print(c.fetchone()[0])
From Python documentation:
Instead, use the DB-API’s parameter substitution. Put ? as a
placeholder wherever you want to use a value, and then provide a tuple
of values as the second argument to the cursor’s execute() method.
You need to use ? placeholder for values but for column names you can use string formatting.
I have created a class, inserted some dummy rows and run a select query which is mentioned in the question.
import sqlite3
class Database(object):
def __init__(self):
self.conn = sqlite3.connect('clients.db')
self.c = self.conn.cursor()
def create_table(self):
try:
self.c.execute('''CREATE TABLE prices (date text, xyz text, abc text, sta text, vert text)''')
except:
pass
def insert_dummy_rows(self):
values = [('2019-01-07', 'xyz1', 'abc1', 'sta1', 'vert1'),
('2019-01-07', 'xyz2', 'abc2', 'sta2', 'vert2'),
('2019-01-08', 'xyz3', 'abc3', 'sta3', 'vert3'),
]
self.c.executemany('INSERT INTO prices VALUES (?,?,?,?,?)', values)
self.conn.commit()
def close_connection(self):
self.conn.close()
def get_single_row(self):
t = ('2019-01-07',)
query = "SELECT {} FROM prices WHERE date=?".format('xyz')
self.c.execute(query, t)
return self.c.fetchone()[0]
if __name__ == '__main__':
db = Database()
db.create_table()
db.insert_dummy_rows()
print(db.get_single_row())
Output:
xyz1
Issue: I can't figure out how to run a query in the correct way so that it returns a mapped dictionary. The query will use counts from multiple tables.
I am using psycopg2 for a postgresql database, and I will be using the results to create a report on day to day deltas on these counts.
Given that, can someone provide an example on how to execute multiple queries and return a dictionary that I can use for comparison purposes? Thanks! I image in a for loop is needed somewhere in here.
tables = ['table1', 'table2']
def db_query():
query = "select count(*) from (a_table) where error_string != '';"
conn = psycopg2.connect(database=db, user=user, password=password, host=host)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute(query, tables)
output = cur.fetchall()
conn.close()
return output
I haven't used postgresql, so you might want to also check this out as a reference: How to store count values in python.
That being said, rearrange your code into something like this. Be sure to make conn global so you don't have to make more than one connection, and make sure you're also closing cur:
conn = None
def driverFunc():
global conn
try:
conn = psycopg2.connect(database=db, user=user, password=password, host=host)
tables = ['table1', 'table2']
countDict = {}
for thisTable in tables:
db_query(thisTable, countDict)
finally:
if not conn == None:
conn.close()
def db_query(tableName, countDict):
# Beware of SQL injection with the following line:
query = "select count(*) from " + tableName + " where error_string != '';"
cur = None
try:
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute(query)
countDict[tableName] = int(cur.fetchone())
finally:
if not cur == None:
cur.close()