I am using aiomysql and MariaDB. I can create a table or select data, but I can't insert data into the table. If you SELECT data with fetchall(), then it will show what you just inserted, but immediately delete from the database.
async def test_example(loop):
pool = await aiomysql.create_pool(host='127.0.0.1', port=3306,
user='root', password='',
db='test', loop=loop)
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute("INSERT INTO `tbl`(`id`, `val`) VALUES (37, 'z');")
print(cur.fetchall())
pool.close()
await pool.wait_closed()
loop = asyncio.get_event_loop()
loop.run_until_complete(test_example(loop))
Why?
From the PEP-249 specification:
.fetchall()
Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples).
Since sql INSERT statement doesn't produce a result set you should try a SELECT statement before trying to obtain information from the database server.
Remove the quotes from the table and column names.
import aiomysql
import asyncio
async def select(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
r = await cur.fetchone()
print(r)
async def insert(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def main(loop):
pool = await aiomysql.create_pool(host='127.0.0.1', port=3306,
user='root', password='',
db='test', loop=loop)
c1 = select(loop=loop, sql='select * from tbl', pool=pool)
c2 = insert(loop=loop, sql="INSERT INTO tbl(id, val) VALUES (37, 'z');", pool=pool)
tasks = [asyncio.ensure_future(c1), asyncio.ensure_future(c2)]
return await asyncio.gather(*tasks)
if __name__ == '__main__':
cur_loop = asyncio.get_event_loop()
cur_loop.run_until_complete(main(cur_loop))
Related
I am trying to create an async connection using psycopg3. I was using psycopg2 without async and need to move to async database functions. The docs do not give much information.
So this is what I was using with psycopg2. It worked good.
con = psycopg2.connect(host="HOSTNAME", port="PORT", database=("DATABASE", user="USER", password="PASSWORD")
cursor = con.cursor()
Then when I needed to run a query I would just use
cursor.execute(query, params)
cursor.fetchall() # or con.commit() depending on insert or select statement.
Now that I am moving to async functions, I have tried this
con = await psycopg.AsyncConnection.connect(host="HOSTNAME", port="PORT", database="DATABASE", user="USER", password="PASSWORD")
cursor = await con.cursor()
But I get the error that I cannot use await outside of a function.
The docs tell me to do this
async with await psycopg.AsyncConnection.connect() as aconn:
async with aconn.cursor() as cur:
await cur.execute(...)
So do I need to write this in every function that I want to either read or write records with?
Couple examples in my code using psycopg2 currently
async def check_guild(guild_id):
cursor.execute("SELECT guild_id, guild_name, su_id FROM guild WHERE guild_id = %s", [guild_id])
guild = cursor.fetchone()
return guild
async def config_raffle(guild_id, channel_id, channel_name, channel_cat_id, token, token_id, default_address, su_id, fee):
try:
cursor.execute("""INSERT INTO raffle_config (guild_id, channel_id, channel_name, channel_cat_id, token, default_token, default_address, su_id, fee) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (guild_id) DO UPDATE SET channel_id = EXCLUDED.channel_id, channel_name = EXCLUDED.channel_name, channel_cat_id = EXCLUDED.channel_cat_id, token = EXCLUDED.token,
default_token = EXCLUDED.default_token, default_address = EXCLUDED.default_address, su_id = EXCLUDED.su_id, fee = EXCLUDED.fee""",
(guild_id, channel_id, channel_name, channel_cat_id, token, token_id, default_address, su_id, fee))
con.commit()
except:
logging.exception("Exception", exc_info=True)
con.rollback()
print("Error: 25")
return True
So I am thinking maybe my better option is to use the AsyncConnectionPool. I have a db.py file setup like this:
import psycopg_pool
import os
import dotenv
dotenv.load_dotenv()
conninfo = f'host={os.getenv("HOSTNAME")} port={os.getenv("PORT")} dbname={os.getenv("DATABASE")} user={os.getenv("USER")} password={os.getenv("PASSWORD")}'
pool = psycopg_pool.AsyncConnectionPool(conninfo=conninfo, open=False)
async def open_pool():
await pool.open()
I open the pool when my program runs the on_ready function.
I created new tables this way just fine, but when I try to retrieve records I get this error.
discord.ext.commands.errors.CommandInvokeError: Command raised an exception: AttributeError: 'AsyncConnection' object has no attribute 'fetchone'
Ended up sorting this out this way:
import psycopg_pool
import os
import dotenv
dotenv.load_dotenv()
conninfo = f'host={os.getenv("HOSTNAME")} port={os.getenv("PORT")} dbname={os.getenv("DATABASE")} user={os.getenv("USER")} password={os.getenv("PASSWORD")}'
pool = psycopg_pool.AsyncConnectionPool(conninfo=conninfo, open=False)
async def open_pool():
await pool.open()
await pool.wait()
print("Connection Pool Opened")
async def select_fetchall(query, args):
async with pool.connection() as conn:
async with conn.cursor() as cursor:
await cursor.execute(query, args)
results = await cursor.fetchall()
return results
async def write(query, args):
async with pool.connection() as conn:
async with conn.cursor() as cursor:
await cursor.execute(query, args)
if 'RETURNING' in query:
results = await cursor.fetchone()
return results
else:
return
Then I just call the functions when I need to read or write to the database and pass the query and args.
Hi I created the following event. It takes input one time after that it stores the guild id , channel id and role id in sqlite db . After that when someone in that particular guild mentions minimum 3 users in a particular channel , the bot gives them new role.
class ScrimsCog(commands.Cog, name='Scrims-Commands') :
def __init__(self,bot):
self.bot = bot
#commands.Cog.listener()
async def on_message(self, message):
if message.guild:
db = sqlite3.connect('main.sqlite')
cursor = db.cursor()
cursor.execute(
f"SELECT * FROM main WHERE guild_id = ?", (message.guild.id, ))
result = cursor.fetchone()
if result:
channel = self.bot.get_channel(result[2])
role = message.guild.get_role(result[1])
if role:
if message.channel == channel:
if len(message.mentions) >= 3:
await message.add_reaction(emoji="<a:tick:748476262640779276>")
user = message.author
await user.add_roles(role)
await self.bot.process_commands(message)
#commands.group(invoke_without_command=True)
async def scrimsmod(self,ctx):
await ctx.send('Available Setup Commands: \nscrimsmod channel <#channel>\nscrimsmod role <message>')
#scrimsmod.command()
async def channel(self, ctx, channel:discord.TextChannel):
if ctx.message.author.guild_permissions.manage_messages:
db = sqlite3.connect('main.sqlite')
cursor = db.cursor()
cursor.execute(f"SELECT channel_id FROM main WHERE guild_id = {ctx.guild.id}")
result = cursor.fetchone()
if result is None:
sql = ("INSERT INTO main(guild_id, channel_id) VALUES(?,?)")
val = (ctx.guild.id, channel.id)
await ctx.send(f" Default Registration Channel has been set to {channel.mention}")
elif result is not None:
sql = ("UPDATE main SET channel_id = ? WHERE guild_id = ?")
val = (channel.id, ctx.guild.id)
await ctx.send(f"Default Registration Channel has been updated to {channel.mention}")
cursor.execute(sql, val)
db.commit()
cursor.close()
db.close()
#scrimsmod.command()
async def role(self, ctx,role: discord.Role):
if ctx.message.author.guild_permissions.manage_messages:
db = sqlite3.connect('main.sqlite')
cursor = db.cursor()
cursor.execute(f"SELECT role FROM main WHERE guild_id = {ctx.guild.id}")
result = cursor.fetchone()
if result is None:
sql = ("INSERT INTO main(guild_id, role) VALUES(?,?)")
val = (ctx.guild.id, role.id)
await ctx.send(f"Default role to give on correct registration have been set to `{role}`")
elif result is not None:
sql = ("UPDATE main SET role = ? WHERE guild_id = ?")
val = (role.id, ctx.guild.id)
await ctx.send(f"Default role to give on correct registration have been updated to `{role}`")
cursor.execute(sql, val)
db.commit()
cursor.close()
db.close()
Well, I believe the code is fine it doesn't throw any error . The issue is that it perfectly take input without any error but doesn't store it into db.
I believe the problem is with my db,
code -
db = sqlite3.connect('main.sqlite',timeout=10)
cursor = db.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS main(
guild_id INTEGER,
role INTEGER,
channel_id INTEGER
)
''')
I believe the problem is here , I should take TEXT in place INTEGER but I am not sure. Answer me with What do you I am doing wrong and how should I fix that.
I answered your last question with assuming that role and channel is TEXT type in the table and that answer should've solve your problem. If you change them, it will work fine and also it will be more efficient if you define the database variable and cursor on top of the on_message event just for once. So you don't have do connect to the database everytime.
I'm trying to make some connections to the MySQL database asynchronously, where there is a dispute and printing on the screen or result that comes first ... but I'm not able to execute ...
import asyncio
import aiomysql
import pymysql.cursors
from datetime import datetime
loop = asyncio.get_event_loop()
sql1 = 'SELECT * FROM aluguel'
sql2 = 'SELECT * FROM filme'
sql3 = 'SELECT * FROM ator'
sql4 = 'SELECT * FROM pagamento'
sql5 = 'SELECT * FROM cliente'
def getHora():
data = datetime.now()
hora = data.hour
minu = data.minute
seg = data.second
mseg = data.microsecond
str_hora = str(hora) + ':' + str(minu) + ':' + str(seg) + ':' + str(mseg)
return str_hora
async def test_example():
conn1 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
conn2 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
conn3 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
conn4 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
conn5 = await aiomysql.connect(host='',
port=3306,
user='administrator',
password='',
db='sakila',
loop=None)
try:
print(getHora())
cur1 = await conn1.cursor()
cur2 = await conn2.cursor()
cur3 = await conn3.cursor()
cur4 = await conn4.cursor()
cur5 = await conn5.cursor()
print('req 1',await cur1.execute(sql1))
print('req 2',await cur2.execute(sql2))
print('req 3',await cur3.execute(sql3))
print('req 4',await cur4.execute(sql4))
print('req 5',await cur5.execute(sql5))
await cur1.close()
await cur2.close()
await cur3.close()
await cur4.close()
await cur5.close()
finally:
conn1.close()
conn2.close()
conn3.close()
conn4.close()
conn5.close()
print(getHora())
loop.run_until_complete(test_example())
this was the last code I got, trying to make five connections to the bank and 5 queries but the code above always comes first .. does anyone have any idea how I can make them compete?
To compete, you need to run the coroutines in parallel using asyncio.create_task or asyncio.gather:
async def test(reqid, sql):
conn = await aiomysql.connect(
host='', port=3306, user='administrator', password='', db='sakila')
try:
print(getHora())
cur = await conn.cursor()
print(reqid, await cur.execute(sql))
await cur.close()
finally:
conn.close()
print(getHora())
async def compete():
await asyncio.gather(
test('req 1', sql1),
test('req 2', sql2),
test('req 3', sql3),
test('req 4', sql4),
test('req 5', sql5),
)
asyncio.run(compete())
I have the following working code:
async def loadnames(self, ctx):
"""
Load the current alias of every member in the database.
Useful after joining a new guild or after a long downtime.
"""
async with ctx.message.channel.typing():
message = await ctx.send(f'Adding members to the database... {ctx.author.mention}')
db = await aiomysql.connect(host=globals.mysql_host,user=globals.mysql_user,password=globals.mysql_passwd,db=globals.mysql_db)
cur = await db.cursor()
#await cur.execute("SELECT discord,name FROM aliases WHERE discord=%s AND name=%s", (member.id,str(member)))
#result = await cur.fetchall()
count = 0
for member in ctx.guild.members:
try:
result = await cur.execute("SELECT * FROM aliases WHERE discord=%s AND name=%s", (member.id,str(member)))
if result == 0:
count = count + 1
await cur.execute("INSERT INTO aliases (discord,name) VALUES(%s,%s)", (member.id,str(member)))
await db.commit()
except:
await ctx.send(f'Error adding `{member}`! {ctx.author.mention}')
await cur.close()
db.close()
await message.edit(content=f'{count} members added to the database! {ctx.author.mention}')
Now, this works perfectly fine, the only issue I see with this is that I'm executing a new SELECT query for every iteration of my loop. So I'd like to put the SELECT statement outside of the loop, using cur.fetchall() to put it in a list (see commented lines in the code above), but I don't know after that how to check if a pair of (member.id,str(member)) is in my results.
Consider a single insert-select query in loop using the well know duplicate avoidance in SQL: NOT IN vs. NOT EXISTS vs. LEFT JOIN / IS NULL. Below runs the LEFT JOIN / IS NULL approach:
for member in ctx.guild.members:
try:
sql = """INSERT INTO aliases (discord, name)
SELECT a1.discord, a1.name
FROM aliases a1
LEFT JOIN aliases a2
ON a1.discord = a2.discord
AND a1.name = a2.name
AND a1.discord=%s AND a2.name=%s
WHERE a2.name IS NULL AND a2.discord IS NULL
"""
result = await cur.execute(sql, (member.id, str(member)))
count = cur.rowcount
await db.commit()
except:
await ctx.send(f'Error adding `{member}`! {ctx.author.mention}')
Even better, populate member IDs in a temp table and join to above query for only one query for all member IDs. SQL's set-based processing beats out application layer looping!
# CLEAN AND POPULATE TEMP TABLE
await cur.execute("DELETE FROM mytempTable")
members_list = [(member.id, str(member)) for member in ctx.guild.members]
await cur.executemany("INSERT INTO mytempTable (discord, name) VALUES (%s, %s)",
members_list)
await db.commit()
# ONLY ONE INSERT QUERY
sql = """INSERT INTO aliases (discord, name)
SELECT a1.discord, a1.name
FROM aliases a1
INNER JOIN mytempTable t
ON a1.discord = t.discord
AND a1.name = t.name
LEFT JOIN aliases a2
ON a1.discord = a2.discord
AND a1.name = a2.name
WHERE a2.name IS NULL AND a2.discord IS NULL
"""
result = await cur.execute(sql, (member.id, str(member)))
count = cur.rowcount
await db.commit()
Derived from Parfait's answer, I ended up with this:
async def loadnames(self, ctx):
"""
Load the current alias of every member in the database.
Useful after joining a new guild or after a long downtime.
"""
async with ctx.message.channel.typing():
count = 0
message = await ctx.send(f'Adding members to the database... {ctx.author.mention}')
db = await aiomysql.connect(host=globals.mysql_host,user=globals.mysql_user,password=globals.mysql_passwd,db=globals.mysql_db)
cur = await db.cursor()
await cur.execute("SELECT discord, name FROM aliases")
result = await cur.fetchall()
new_member_list = [(member.id, str(member)) for member in ctx.guild.members]
member_list = [(member[0],member[1]) for member in result]
diff_list = [member for member in new_member_list if member not in member_list]
count = await cur.executemany("INSERT INTO aliases (discord, name) VALUES (%s, %s)", diff_list)
await db.commit()
await cur.close()
db.close()
if count is None: count = 0
await message.edit(content=f'{count} members added to the database! {ctx.author.mention}')
Only one SELECT query at the beginning, then we play with the different lists to get the executemany INSERT statement which will do only the required number of insertions.
I just don't know what to do to reuse aiomysql connection pool by reading the aiohttp examples or by google.
Here is my code
import aiomysql
import asyncio
async def select(loop, sql):
pool = await aiomysql.create_pool(host='127.0.0.1', port=3306,
user='root', password='123456',
db='test', loop=loop)
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
r = await cur.fetchone()
print(r)
async def insert(loop, sql):
pool = await aiomysql.create_pool(host='127.0.0.1', port=3306,
user='root', password='123456',
db='test', loop=loop)
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def main(loop):
c1 = select(loop=loop, sql='select * from minifw')
c2 = insert(loop=loop, sql="insert into minifw (name) values ('hello')")
tasks = [
asyncio.ensure_future(c1),
asyncio.ensure_future(c2)
]
return await asyncio.gather(*tasks)
if __name__ == '__main__':
cur_loop = asyncio.get_event_loop()
cur_loop.run_until_complete(main(cur_loop))
If i run this code, the create_pool will be executed twice.So I want to know how to change this code to reuse aiomysql connecton pool.
Thanks!
You can define pool in main func, like this:
import aiomysql
import asyncio
async def select(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
r = await cur.fetchone()
print(r)
async def insert(loop, sql, pool):
async with pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def main(loop):
pool = await aiomysql.create_pool(
host='127.0.0.1',
port=3306,
user='root',
password='123456',
db='test',
loop=loop)
c1 = select(loop=loop, sql='select * from minifw limit 1', pool=pool)
c2 = insert(loop=loop, sql="insert into minifw (name) values ('hello')", pool=pool)
tasks = [asyncio.ensure_future(c1), asyncio.ensure_future(c2)]
return await asyncio.gather(*tasks)
if __name__ == '__main__':
cur_loop = asyncio.get_event_loop()
cur_loop.run_until_complete(main(cur_loop))