I just don't know what to do to reuse aiomysql connection pool by reading the aiohttp examples or by google.
Here is my code
import aiomysql
import asyncio
async def select(loop, sql):
pool = await aiomysql.create_pool(host='127.0.0.1', port=3306,
user='root', password='123456',
db='test', loop=loop)
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
r = await cur.fetchone()
print(r)
async def insert(loop, sql):
pool = await aiomysql.create_pool(host='127.0.0.1', port=3306,
user='root', password='123456',
db='test', loop=loop)
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def main(loop):
c1 = select(loop=loop, sql='select * from minifw')
c2 = insert(loop=loop, sql="insert into minifw (name) values ('hello')")
tasks = [
asyncio.ensure_future(c1),
asyncio.ensure_future(c2)
]
return await asyncio.gather(*tasks)
if __name__ == '__main__':
cur_loop = asyncio.get_event_loop()
cur_loop.run_until_complete(main(cur_loop))
If i run this code, the create_pool will be executed twice.So I want to know how to change this code to reuse aiomysql connecton pool.
Thanks!
You can define pool in main func, like this:
import aiomysql
import asyncio
async def select(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
r = await cur.fetchone()
print(r)
async def insert(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def main(loop):
pool = await aiomysql.create_pool(
host='127.0.0.1',
port=3306,
user='root',
password='123456',
db='test',
loop=loop)
c1 = select(loop=loop, sql='select * from minifw limit 1', pool=pool)
c2 = insert(loop=loop, sql="insert into minifw (name) values ('hello')", pool=pool)
tasks = [asyncio.ensure_future(c1), asyncio.ensure_future(c2)]
return await asyncio.gather(*tasks)
if __name__ == '__main__':
cur_loop = asyncio.get_event_loop()
cur_loop.run_until_complete(main(cur_loop))
Related
I am trying to create an async connection using psycopg3. I was using psycopg2 without async and need to move to async database functions. The docs do not give much information.
So this is what I was using with psycopg2. It worked good.
con = psycopg2.connect(host="HOSTNAME", port="PORT", database=("DATABASE", user="USER", password="PASSWORD")
cursor = con.cursor()
Then when I needed to run a query I would just use
cursor.execute(query, params)
cursor.fetchall() # or con.commit() depending on insert or select statement.
Now that I am moving to async functions, I have tried this
con = await psycopg.AsyncConnection.connect(host="HOSTNAME", port="PORT", database="DATABASE", user="USER", password="PASSWORD")
cursor = await con.cursor()
But I get the error that I cannot use await outside of a function.
The docs tell me to do this
async with await psycopg.AsyncConnection.connect() as aconn:
async with aconn.cursor() as cur:
await cur.execute(...)
So do I need to write this in every function that I want to either read or write records with?
Couple examples in my code using psycopg2 currently
async def check_guild(guild_id):
cursor.execute("SELECT guild_id, guild_name, su_id FROM guild WHERE guild_id = %s", [guild_id])
guild = cursor.fetchone()
return guild
async def config_raffle(guild_id, channel_id, channel_name, channel_cat_id, token, token_id, default_address, su_id, fee):
try:
cursor.execute("""INSERT INTO raffle_config (guild_id, channel_id, channel_name, channel_cat_id, token, default_token, default_address, su_id, fee) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (guild_id) DO UPDATE SET channel_id = EXCLUDED.channel_id, channel_name = EXCLUDED.channel_name, channel_cat_id = EXCLUDED.channel_cat_id, token = EXCLUDED.token,
default_token = EXCLUDED.default_token, default_address = EXCLUDED.default_address, su_id = EXCLUDED.su_id, fee = EXCLUDED.fee""",
(guild_id, channel_id, channel_name, channel_cat_id, token, token_id, default_address, su_id, fee))
con.commit()
except:
logging.exception("Exception", exc_info=True)
con.rollback()
print("Error: 25")
return True
So I am thinking maybe my better option is to use the AsyncConnectionPool. I have a db.py file setup like this:
import psycopg_pool
import os
import dotenv
dotenv.load_dotenv()
conninfo = f'host={os.getenv("HOSTNAME")} port={os.getenv("PORT")} dbname={os.getenv("DATABASE")} user={os.getenv("USER")} password={os.getenv("PASSWORD")}'
pool = psycopg_pool.AsyncConnectionPool(conninfo=conninfo, open=False)
async def open_pool():
await pool.open()
I open the pool when my program runs the on_ready function.
I created new tables this way just fine, but when I try to retrieve records I get this error.
discord.ext.commands.errors.CommandInvokeError: Command raised an exception: AttributeError: 'AsyncConnection' object has no attribute 'fetchone'
Ended up sorting this out this way:
import psycopg_pool
import os
import dotenv
dotenv.load_dotenv()
conninfo = f'host={os.getenv("HOSTNAME")} port={os.getenv("PORT")} dbname={os.getenv("DATABASE")} user={os.getenv("USER")} password={os.getenv("PASSWORD")}'
pool = psycopg_pool.AsyncConnectionPool(conninfo=conninfo, open=False)
async def open_pool():
await pool.open()
await pool.wait()
print("Connection Pool Opened")
async def select_fetchall(query, args):
async with pool.connection() as conn:
async with conn.cursor() as cursor:
await cursor.execute(query, args)
results = await cursor.fetchall()
return results
async def write(query, args):
async with pool.connection() as conn:
async with conn.cursor() as cursor:
await cursor.execute(query, args)
if 'RETURNING' in query:
results = await cursor.fetchone()
return results
else:
return
Then I just call the functions when I need to read or write to the database and pass the query and args.
I want to do some commands on very high requests like 10000 and above, but even when I do for like 100 commands, it blocks some of them and if I insert 100 values, 99 are added. When I add 1000 values 15 are missing and it gives the below error
import aiomysql
import asyncio
import logging
import random
import time
logging.basicConfig(level=logging.DEBUG)
print(f'Logging Enabled: Functions being defiled')
async def select(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
# r = await cur.fetchone()
# print(r)
async def insert(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def update(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def main(loop):
start = time.time()
tasks = []
print(f'Def Main Looped Called: Mysql being tried to Login ')
pool = await aiomysql.create_pool(
host='localhost',
user='root',
password='',
db='mydatabase',
loop=loop,
minsize=50,
maxsize=500)
for x in range(100):
list = random.sample(range(0, 9), 7)
digit1 = random.sample(range(1, 6), 1)
digit2 = random.sample(range(0, 9), 1)
netno = random.sample(range(1, 4), 1)
netno2 = random.sample(range(1, 4), 1)
substatus = random.sample(range(1, 3), 1)
output = "".join(str(i) for i in list)
output1 = "".join(str(i) for i in digit1)
output2 = "".join(str(i) for i in digit2)
netoutput = "".join(str(i) for i in netno)
netoutput2 = "".join(str(i) for i in substatus)
netoutput3 = "".join(str(i) for i in netno2)
final = ('923' + output1 + output2 + output)
c1 = select(loop=cur_loop, sql="SELECT DISTINCT id, msisdn, network_type, network_name, subscription_status, package_id \
FROM testdata JOIN networkstatus ON testdata.network_type=networkstatus.network_id \
WHERE network_type=" +netoutput+ " AND subscription_status= " +netoutput2+ " \
AND package_id=" +netoutput3+ ";", pool=pool)
endtimes=time.time()
c2 = insert(loop=cur_loop, sql="INSERT INTO testdata (msisdn,network_type,subscription_status,package_id) \
VALUES (" +final+ "," +netoutput+ "," +netoutput2+ "," +netoutput+ ")", pool=pool)
endtimei=time.time()
c3 = update(loop=cur_loop, sql="UPDATE testdata SET network_type = " +netoutput+ ", subscription_status= " +netoutput2+ ",\
package_id = " +netoutput+ " WHERE network_type=" +netoutput+ " AND subscription_status= " +netoutput2+ " \
AND package_id=" +netoutput+ ";", pool=pool)
endtimeu=time.time()
print(f' Sql Connected Main Function Ended')
print('Time taken by select:', endtimes-start, 'Time taken by insert:', endtimei-start, 'Time taken by update:', endtimeu-start)
# await c1,c2,c3
tasks = [asyncio.ensure_future(c1), asyncio.ensure_future(c2), asyncio.ensure_future(c3)]
print(f'Task Line Executed')
end = time.time()
print('Total time of execution:', end-start)
return await asyncio.gather(*tasks)
if __name__ == '__main__':
print(f'Code Started: Async Event Loop')
cur_loop = asyncio.get_event_loop()
asyncio.set_event_loop(cur_loop)
print(f'Before Loop Run')
cur_loop.run_until_complete(main(cur_loop))
it select update and insert values but not complete, some values are missing as it gives the error
I am using aiomysql and MariaDB. I can create a table or select data, but I can't insert data into the table. If you SELECT data with fetchall(), then it will show what you just inserted, but immediately delete from the database.
async def test_example(loop):
pool = await aiomysql.create_pool(host='127.0.0.1', port=3306,
user='root', password='',
db='test', loop=loop)
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute("INSERT INTO `tbl`(`id`, `val`) VALUES (37, 'z');")
print(cur.fetchall())
pool.close()
await pool.wait_closed()
loop = asyncio.get_event_loop()
loop.run_until_complete(test_example(loop))
Why?
From the PEP-249 specification:
.fetchall()
Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples).
Since sql INSERT statement doesn't produce a result set you should try a SELECT statement before trying to obtain information from the database server.
Remove the quotes from the table and column names.
import aiomysql
import asyncio
async def select(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
r = await cur.fetchone()
print(r)
async def insert(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def main(loop):
pool = await aiomysql.create_pool(host='127.0.0.1', port=3306,
user='root', password='',
db='test', loop=loop)
c1 = select(loop=loop, sql='select * from tbl', pool=pool)
c2 = insert(loop=loop, sql="INSERT INTO tbl(id, val) VALUES (37, 'z');", pool=pool)
tasks = [asyncio.ensure_future(c1), asyncio.ensure_future(c2)]
return await asyncio.gather(*tasks)
if __name__ == '__main__':
cur_loop = asyncio.get_event_loop()
cur_loop.run_until_complete(main(cur_loop))
I'm trying to make some connections to the MySQL database asynchronously, where there is a dispute and printing on the screen or result that comes first ... but I'm not able to execute ...
import asyncio
import aiomysql
import pymysql.cursors
from datetime import datetime
loop = asyncio.get_event_loop()
sql1 = 'SELECT * FROM aluguel'
sql2 = 'SELECT * FROM filme'
sql3 = 'SELECT * FROM ator'
sql4 = 'SELECT * FROM pagamento'
sql5 = 'SELECT * FROM cliente'
def getHora():
data = datetime.now()
hora = data.hour
minu = data.minute
seg = data.second
mseg = data.microsecond
str_hora = str(hora) + ':' + str(minu) + ':' + str(seg) + ':' + str(mseg)
return str_hora
async def test_example():
conn1 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
conn2 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
conn3 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
conn4 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
conn5 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
try:
print(getHora())
cur1 = await conn1.cursor()
cur2 = await conn2.cursor()
cur3 = await conn3.cursor()
cur4 = await conn4.cursor()
cur5 = await conn5.cursor()
print('req 1',await cur1.execute(sql1))
print('req 2',await cur2.execute(sql2))
print('req 3',await cur3.execute(sql3))
print('req 4',await cur4.execute(sql4))
print('req 5',await cur5.execute(sql5))
await cur1.close()
await cur2.close()
await cur3.close()
await cur4.close()
await cur5.close()
finally:
conn1.close()
conn2.close()
conn3.close()
conn4.close()
conn5.close()
print(getHora())
loop.run_until_complete(test_example())
this was the last code I got, trying to make five connections to the bank and 5 queries but the code above always comes first .. does anyone have any idea how I can make them compete?
To compete, you need to run the coroutines in parallel using asyncio.create_task or asyncio.gather:
async def test(reqid, sql):
conn = await aiomysql.connect(
host='', port=3306, user='administrator', password='', db='sakila')
try:
print(getHora())
cur = await conn.cursor()
print(reqid, await cur.execute(sql))
await cur.close()
finally:
conn.close()
print(getHora())
async def compete():
await asyncio.gather(
test('req 1', sql1),
test('req 2', sql2),
test('req 3', sql3),
test('req 4', sql4),
test('req 5', sql5),
)
asyncio.run(compete())
I have the following blocking code:
from flask_sqlalchemy import SQLAlchemy
def find_places():
query = "Select ... From ... Where ..."
result = db.session.execute(query)
return json.dumps([dict(r) for r in result)
#app.route('/')
def videos():
return find_places()
if __name__ == '__main__':
db = SQLAlchemy(app)
app.run()
How can I make this code asynchronous?
Take a look at aiopg, it's the best (and possibly the only) asynchronous Postgres library for Python.
They also have optional SQLAlchemy integration. I'll just copy from their README:
import asyncio
from aiopg.sa import create_engine
import sqlalchemy as sa
metadata = sa.MetaData()
tbl = sa.Table('tbl', metadata,
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('val', sa.String(255)))
async def create_table(engine):
async with engine.acquire() as conn:
await conn.execute('DROP TABLE IF EXISTS tbl')
await conn.execute('''CREATE TABLE tbl (
id serial PRIMARY KEY,
val varchar(255))''')
async def go():
async with create_engine(user='aiopg',
database='aiopg',
host='127.0.0.1',
password='passwd') as engine:
async with engine.acquire() as conn:
await conn.execute(tbl.insert().values(val='abc'))
async for row in conn.execute(tbl.select()):
print(row.id, row.val)
loop = asyncio.get_event_loop()
loop.run_until_complete(go())