I need a sustainable method to handle files wrapped in media group.
My function handle_files is waiting for media files. When user uploads media file it goes through the series of different checks. If it passes all tests (size restriction, formats restriction) the media file is downloaded and processed.
It looks like this:
async def handle_files(message: types.Message, state: FSMContext):
user_data = await state.get_data()
locale = user_data['locale']
list_of_files = user_data['list_of_files']
try:
file = message.document
file_name = file['file_name']
except Exception as e:
await message.answer('Error while downloading file')
return None
file_name = unidecode(file_name)
file_size = file['file_size']
if file_size >= 20971520:
await message.answer('File is too big')
return None
invalid_format, formats = check_invalid_format(file_name, function)
if invalid_format:
await message.answer(file_name + 'has unsupported format. Supported formats: ' + ', '.join(formats))
return None
output_folder = os.path.join('temp', str(message.from_user.id))
if not os.path.exists(output_folder):
os.makedirs(output_folder, exist_ok=True)
file_path = os.path.join(output_folder, file_name)
await file.download(destination_file=file_path)
list_of_files.append(file_path)
await state.update_data(list_of_files=list_of_files)
await message.answer('Added files: '.format('\n'.join(list_of_files)))
Working with separate files looks fine. After downloading files user gets a list of added files one by one.
But the problem is that when user uploads files in media group one file overrides another. It looks like that.
So, only one file is appending to the list list_of_files that prevents me from processing both files.
I tried to solve the problem by initializing user_data dictionary one more time:
...
await file.download(destination_file=file_path)
user_data = await state.get_data()
list_of_files = user_data['list_of_files']
list_of_files.append(file_path)
await state.update_data(list_of_files=list_of_files)
...
It solved one part of my problem but this solution is not elegant and, supposedly, not quite sustainable. The message is duplicated.
I need that after uploading media group user gets one message containing the list of all files from this media group.
I suppose that the problem here is linked with asyncio. But I've spent already a lot of time but the solution haven't been found. Looking for you help.
check this link.
Here you can find the middleware, which will return list of messages from media group, and then you could use for loop to handle them.
import asyncio
from typing import List, Union
from aiogram import Bot, Dispatcher, executor, types
from aiogram.dispatcher.handler import CancelHandler
from aiogram.dispatcher.middlewares import BaseMiddleware
bot = Bot(token="TOKEN_HERE") # Place your token here
dp = Dispatcher(bot)
class AlbumMiddleware(BaseMiddleware):
"""This middleware is for capturing media groups."""
album_data: dict = {}
def __init__(self, latency: Union[int, float] = 0.01):
"""
You can provide custom latency to make sure
albums are handled properly in highload.
"""
self.latency = latency
super().__init__()
async def on_process_message(self, message: types.Message, data: dict):
if not message.media_group_id:
return
try:
self.album_data[message.media_group_id].append(message)
raise CancelHandler() # Tell aiogram to cancel handler for this group element
except KeyError:
self.album_data[message.media_group_id] = [message]
await asyncio.sleep(self.latency)
message.conf["is_last"] = True
data["album"] = self.album_data[message.media_group_id]
async def on_post_process_message(self, message: types.Message, result: dict, data: dict):
"""Clean up after handling our album."""
if message.media_group_id and message.conf.get("is_last"):
del self.album_data[message.media_group_id]
#dp.message_handler(is_media_group=True, content_types=types.ContentType.ANY)
async def handle_albums(message: types.Message, album: List[types.Message]):
"""This handler will receive a complete album of any type."""
media_group = types.MediaGroup()
for obj in album:
if obj.photo:
file_id = obj.photo[-1].file_id
else:
file_id = obj[obj.content_type].file_id
try:
# We can also add a caption to each file by specifying `"caption": "text"`
media_group.attach({"media": file_id, "type": obj.content_type})
except ValueError:
return await message.answer("This type of album is not supported by aiogram.")
await message.answer_media_group(media_group)
if __name__ == "__main__":
dp.middleware.setup(AlbumMiddleware())
executor.start_polling(dp, skip_updates=True)
Related
I am uploading file using flask rest-api and flask. As the file size is large I am using celery to upload the file on server. Below is the code.
Flask Rest API
#app.route('/upload',methods=['GET','POST'])
def upload():
file = request.files.get("file")
if not file:
return "some_error_msg"
elif file.filename == "":
return "some_error_msg"
if file:
filename = secure_filename(file.filename)
result = task_upload.apply_async(args=(filename, ABC, queue="upload")
return "some_task_id"
Celery task
#celery_app.task(bind=True)
def task_upload(self, filename: str, contents: Any) -> bool:
status = False
try:
status = save_file(filename, contents)
except exception as e:
print(f"Exception: {e}")
return status
Save method
def save_file(filename: str, contents: Any) -> bool:
file: Path = MEDIA_DIRPATH / filename
status: bool = False
# method-1 This code is using flask fileStorage, contents= is filestorage object
if contents:
contents.save(file)
status = True
# method-2 This code is using request.stream, contents= is IOBytes object
with open(file, "ab") as fp:
chunk = 4091
while True:
some code.
f.write(chunk)
status = True
return status
I am getting error while trying both methods
For Method-1, where I tried passing file variable(fileStorage type object) and getting error as
exc_info=(<class 'kombu.exceptions.EncodeError'>, EncodeError(TypeError('Object of type FileStorage is not JSON serializable'))
For Method-2, where I tried passing request.stream and getting error as
<gunicorn.http.body.Body object at some number>
TypeError: Object of type Body is not JSON serializable
How can I pass file(ABC) to celery task?
I am preferring method-1 but any will do. Please suggest.
You can use any way from the following like gevent, multiprocessing, multithreading, nginx upload module.
For my use-case thread was better fit. This is pseudo code structure.
class UploadWorker(Thread):
"""Create a upload worker background thread."""
def __init__(self, name: str, daemon: bool, filename: str, contents: str, read_length: int) -> None:
"""Initialize the defaults."""
self.filename: str = filename
self.contents: str = contents
self.read_length: int = read_length
self._kill: Event = Event()
super().__init__(name=name, daemon=daemon)
def run(self) -> None:
"""Run the target function."""
print(f"Thread is_set: {self._kill.is_set()=}")
while not self._kill.is_set():
save_file(self.filename, self.contents, self.read_length)
# Better copy the function code directly here instead of calling the function
def kill(self):
"""Revoke or abort the running thread."""
self._kill.set()
Then create object of background worker
upload_worker = UploadWorker("uploader", True, filename, contents, read_length)
To kill or cancel use
upload_worker.kill()
Reference Link Is there any way to kill a Thread?
I am trying to attach a pdf file in a MS Teams bot.
I get the following error " [on_turn_error] unhandled error: (BadArgument) Unknown attachment type". Would anyone know why it might not work?
The following is a portion of my code that concerns the error... unfortunately since it is a chatbot it is not appropriate to put the full code here.
Thank you for your advice.
class MyBot(ActivityHandler):
async def on_message_activity(self, turn_context: TurnContext):
elif str(turn_context.activity.text).upper() in {'PDF'}:
reply = Activity(type=ActivityTypes.message)
reply.text = "This is the pdf file."
reply.attachments = [self._get_inline_attachment()]
await turn_context.send_activity(reply)
#truncated#
def _get_inline_attachment(self) -> Attachment:
file_path = os.path.join(os.getcwd(), "TEST.pdf")
with open(file_path, "rb") as pdf_file:
dencoded_string = base64.b64encode(pdf_file.read()).decode()
return Attachment(
name="TEST.pdf",
content_type="application/pdf",
content_url=f"data:application/pdf;base64,{dencoded_string}",
)
#Nivedipa-MSFT indicated that the sample code for file sharing on MS Teams is avail here: https://github.com/microsoft/BotBuilder-Samples/blob/22fcff680a3e11006eb09b81ac5ed4de345933e2/archive/samples/python/56.teams-file-upload/bots/teams_file_bot.py
If I may add, "supportsFiles" has to be enabled in the manifest for this to work: https://learn.microsoft.com/en-us/microsoftteams/platform/resources/schema/manifest-schema#bots and https://learn.microsoft.com/en-us/microsoftteams/platform/resources/bot-v3/bots-files#configure-your-bot-to-support-files
I'm trying to use the catch_up() function to get all file updates on boot, however, everytime I run my code, only half of the file is downloaded, sometimes the file is completely empt.
However, when I try to run with "iter_messages" I manage to download everything perfectly.
HELP!?
#client.on(events.NewMessage)
async def new_messages(event):
if hasattr(event.message.peer_id, "channel_id"):
print("Um dos canais");
else:
if hasattr(event.message.peer_id, 'chat_id'):
print("Tipo: ","chat");
dialog = int(event.message.peer_id.chat_id);
else:
print("Tipo: ","conversa");
dialog = int(event.message.peer_id.user_id)
'''getting the files'''
path = ""
if hasattr(event.media, "document"):
print("================\n", event.message.id, "\n================");
path = await client.download_media(event.media, file="arquivos_chimera/");
print(event)
if hasattr(event.media, "photo"):
print("================\n", event.message.id, "\n================");
path = await client.download_media(event.media, file="imagens_chimera/")
print(event)
'''getting the Telegram date'''
data = str(event.message.date);
'''text of the message'''
temp_message = await async_ajuste_SQL(event.message.message);
if path != "":
temp_message = path + " - " + temp_message;
'''Quem enviou a mensagem'''
if event.message.from_id==None:
from_ = event.message.peer_id.user_id;
else:
from_ = event.message.from_id.user_id
cur.execute(f"insert into tabela_de_mensagens values ({event.message.id}, {dialog}, {from_}, '{data}', '{temp_message}', 0);");
con.commit();
async def main():
await client.catch_up();
NOTE: the problem only ocurrs to images, delete, edit and new message updates come perfectly
So, after some testing, I realized that the problem was the fact that I was using an event handler without using a keep alive function, i.e., the event handler only works while the main function works, so, if you try to run the event handler with catch_up alone, it will only get the first updates, but will stop shortly after that (hence, why my image files were created, but not completed).
To get a solution, you can look at the following links:
https://github.com/LonamiWebs/Telethon/issues/1534
https://github.com/LonamiWebs/Telethon/issues/3146
https://docs.python.org/3.8/library/asyncio-task.html#asyncio.wait
I am building a bot that sends a txt file based on a command.
Problem is that when two people send the same command at the same time, the bot only sends one of the two twice, which is something I should avoid at all costs.
Do you have any solutions?
I tried to come up with a solution but it didn't work: I tried to delete the file right after it has been sent, but as I said, it didn't work.
NOTE: the bot successfully sends two embeds with the correct information, but then sends two identical files (always the one generated by the second request)
Here's the code of the command:
#bot.command(name='sendfile', help='Sends a file', pass_context=True)
async def quick(ctx, *args):
try:
arg1 = str(args[0])
arg2 = args[1].capitalize()
arg3 = args[2].upper()
quantity = args[3]
generate_file(order_id, arg2, quantity)
except:
embed = Embed(title="FILE GENERATION", description="An error has occurred. Please retry", color=16711702)
fields = [("Error ID", "1", False)]
for name, value, inline in fields:
embed.add_field(name=name, value=value, inline=inline)
print(f"{bcolors.FAIL} [{datetime.now()}] - ERROR {bcolors.ENDC}")
raise TypeError
embed = Embed(title="FILE GENERATION", description="FILE DETAILS", color=15859711)
fields = [("ARG 1", f"{arg1}", False), ("Arg2", f"{arg2}", False), ("arg3", f'{arg3}',False), ("Quantity", f"{quantity}", False)]
for name, value, inline in fields:
embed.add_field(name=name, value=value, inline=inline)
await ctx.send(embed=embed)
print(f"{bcolors.OKCYAN} [{datetime.now()}] - Command successfully executed! {bcolors.ENDC}")
with open("generated_file.txt", "rb") as file:
await ctx.send(file=discord.File(file, "generated_file.txt"))
Thanks for your help :)
The solution I came up with it solves the problem but it's not the best, but it works.
The idea is basically that I generate an alphanumeric string and include that in the file name.
e.g.: 'file_fu56d.txt'
After creation, I send it over to discord and then delete it.
Here's some code:
list= []
random_string = get_random_string(5)
for p in range(0, int(quantity)):
to_append = f'{arg1}-something-{arg2}-soimething_else-{arg2}\n'
list.append(to_append)
# writes the txt file assigning each element of the list to each line
with open(f'file_{random_string}.txt', 'w') as file:
file.writelines(list)
print(f'[{datetime.now()}] - {quantity} lines successfully generated.')
# opens the file and sends it to discord
with open(f"file_{random_string}.txt", "rb") as file:
await ctx.send(file=discord.File(file, f"file_{random_string}.txt"))
print(f"[{datetime.now()}] - File successfully sent! ")
# deletes the file
os.remove(f'file_{random_string}.txt')
print(f"[{datetime.now()}] - 'file_{random_string}.txt' successfully deleted!")
I recently made a command that saves the information into a JSON file. So basically, I have 2 commands, first command sets the global variable, and the second command uses the variables provided to add into the JSON file. And once I tested it, it saves the text as a global variable, and then it saved into the JSON file as {'test'}. I don't want the {''}, so is there a way to don't have {''}, only the text test?
Script:
#global variables
namereg = None
cbreg = None #more
bdreg = None
descreg = None
libreg = None
invreg = None
btreg = None
ssreg = None
slugreg = None
#client.command(pass_context=True)
async def namereg(ctx, *, arg):
global namereg
namereg = {arg}
embed = discord.Embed(title='Registed Name.',description=f'Set the name as {arg}',colour=discord.Color.dark_green())
print(f'{arg}')
await ctx.send(embed = embed)
#client.command(pass_context=True)
async def add(ctx):
role_names = [role.name for role in ctx.message.author.roles]
if "Server Moderator" in role_names:
def write_json(data, filename='bots.json'):
with open (filename, "w") as f:
json.dump(data, f, indent=4)
with open ('bots.json') as json_file:
data = json.load(json_file)
temp = data["bots"]
y = {"name": f"{namereg}"}
temp.append(y)
write_json(data)
embed = discord.Embed(title='Added!',description='Successfully added with the following!',timestamp=ctx.message.created_at,colour=discord.Color.dark_green())
await ctx.send(embed = embed)
If there is a way to not have {''}, please reply to this thread! Thank you.
If you're writing it to a JSON file, the quotes will be added every time as part of JSON syntax. If you just need to write a dictionary to a file (which is also readable), you can write it to a normal .txt file.
Issues:
namereg = None
#client.command(pass_context=True)
async def namereg(ctx, *, arg):
global namereg
This is broken. Functions at the top level of your code are global variables, and are in the same namespace. Give it a different name from the storage variable.
namereg = {arg}
This takes the string that came from the user's input, and creates a set with a single element. That is not what you want. You wanted the input string to be the registered name, so just assign it directly.
y = {"name": f"{namereg}"}
I assume you did this fancy formatting because you were getting an error before (because the json will not serialize sets by default, because the JSON data format does not have a direct way to represent them). You should have listened to this error message more closely, by questioning why you had data of the invalid type in the first place. The {} and '' in your output come from the string representation of the set that you stringify using the string formatting. The plain string that you want to use does not require any formatting to convert to string, because it is already a string.